foss.global/codefeed
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add organization allowlist and denylist filters, enhance changelog loading, and improve fetch functions

Browse Source 
- Introduced orgAllowlist and orgDenylist properties to filter organizations during fetching.
- Enhanced loadChangelogFromRepo to check multiple potential changelog file names.
- Updated fetchTags to return a map of tag names associated with their SHAs.
- Improved pagination logic in fetchAllOrganizations and fetchRepositoriesForOrg to handle larger datasets.
- Added retry logic in fetchFunction to handle rate limiting and server errors more gracefully.
- Modified ITag interface to include an optional name property for better tag handling.
This commit is contained in:
Jürgen Kunz
2025-09-12 22:06:02 +00:00
parent d0a00aedea
commit 98f5c466a6
4 changed files with 2951 additions and 3079 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
package.json
View File

@@ -16,17 +16,17 @@
"buildDocs": "(tsdoc)"
},
"devDependencies": {
"@git.zone/tsbuild": "^2.3.2",
"@git.zone/tsbundle": "^2.2.5",
"@git.zone/tsbuild": "^2.6.8",
"@git.zone/tsbundle": "^2.5.1",
"@git.zone/tsrun": "^1.2.46",
"@git.zone/tstest": "^1.0.96",
"@push.rocks/tapbundle": "^5.6.3",
"@git.zone/tstest": "^2.3.8",
"@push.rocks/tapbundle": "^6.0.3",
"@types/node": "^22.15.2"
},
"dependencies": {
"@push.rocks/lik": "^6.2.2",
"@push.rocks/qenv": "^6.1.0",
"@push.rocks/smartnpm": "^2.0.4",
"@push.rocks/qenv": "^6.1.3",
"@push.rocks/smartnpm": "^2.0.6",
"@push.rocks/smarttime": "^4.1.1",
"@push.rocks/smartxml": "^1.1.1"
},

5808
pnpm-lock.yaml generated
View File

File diff suppressed because it is too large Load Diff

207
ts/index.ts
View File

@@ -4,6 +4,7 @@ export class CodeFeed {
private baseUrl: string;
private token?: string;
private lastRunTimestamp: string;
private pageLimit = 50;
// Raw changelog content for the current repository
private changelogContent: string = '';
// npm registry helper for published-on-npm checks
@@ -16,6 +17,9 @@ export class CodeFeed {
private enableNpmCheck: boolean = true;
// return only tagged commits (false by default)
private enableTaggedOnly: boolean = false;
// allow/deny filters
private orgAllowlist?: string[];
private orgDenylist?: string[];
constructor(
baseUrl: string,
@@ -26,6 +30,8 @@ export class CodeFeed {
cacheWindowMs?: number;
enableNpmCheck?: boolean;
taggedOnly?: boolean;
orgAllowlist?: string[];
orgDenylist?: string[];
}
) {
this.baseUrl = baseUrl;
@@ -37,6 +43,8 @@ export class CodeFeed {
this.cacheWindowMs = options?.cacheWindowMs;
this.enableNpmCheck = options?.enableNpmCheck ?? true;
this.enableTaggedOnly = options?.taggedOnly ?? false;
this.orgAllowlist = options?.orgAllowlist;
this.orgDenylist = options?.orgDenylist;
this.cache = [];
// npm registry instance for version lookups
this.npmRegistry = new plugins.smartnpm.NpmRegistry();
@@ -61,7 +69,14 @@ export class CodeFeed {
}
// 1) get all organizations
const orgs = await this.fetchAllOrganizations();
let orgs = await this.fetchAllOrganizations();
// apply allow/deny filters
if (this.orgAllowlist && this.orgAllowlist.length > 0) {
orgs = orgs.filter((o) => this.orgAllowlist!.includes(o));
}
if (this.orgDenylist && this.orgDenylist.length > 0) {
orgs = orgs.filter((o) => !this.orgDenylist!.includes(o));
}
// 2) fetch repos per org in parallel
const repoLists = await Promise.all(
@@ -122,11 +137,15 @@ export class CodeFeed {
await this.loadChangelogFromRepo(owner, name);
// fetch tags for this repo
let taggedShas: Set<string>;
let tagNameBySha: Map<string, string>;
try {
taggedShas = await this.fetchTags(owner, name);
const tagInfo = await this.fetchTags(owner, name);
taggedShas = tagInfo.shas;
tagNameBySha = tagInfo.map;
} catch (e: any) {
console.error(`Failed to fetch tags for ${owner}/${name}:`, e.message);
taggedShas = new Set<string>();
tagNameBySha = new Map<string, string>();
}
// fetch npm package info only if any new commits correspond to a tag
const hasTaggedCommit = commits.some((c) => taggedShas.has(c.sha));
@@ -141,14 +160,23 @@ export class CodeFeed {
}
// build commit entries
for (const c of commits) {
const versionCandidate = c.commit.message.replace(/\n/g, '').trim();
const isTagged = taggedShas.has(c.sha);
const publishedOnNpm = isTagged && pkgInfo
? pkgInfo.allVersions.some((v) => v.version === versionCandidate)
// derive version from tag name if present (strip leading 'v')
let versionFromTag: string | undefined;
if (isTagged) {
const tagName = tagNameBySha.get(c.sha);
if (tagName) {
versionFromTag = tagName.startsWith('v') ? tagName.substring(1) : tagName;
}
}
const publishedOnNpm = isTagged && pkgInfo && versionFromTag
? pkgInfo.allVersions.some((v) => v.version === versionFromTag)
: false;
let changelogEntry: string | undefined;
if (this.changelogContent) {
changelogEntry = this.getChangelogForVersion(versionCandidate);
if (versionFromTag) {
changelogEntry = this.getChangelogForVersion(versionFromTag);
}
}
newResults.push({
baseUrl: this.baseUrl,
@@ -187,40 +215,49 @@ export class CodeFeed {
return this.cache;
}
// no caching: apply tagged-only filter if requested
// sort and dedupe
const seen = new Set<string>();
const unique = newResults.filter((c) => {
if (seen.has(c.hash)) return false;
seen.add(c.hash);
return true;
});
unique.sort((a, b) => b.timestamp.localeCompare(a.timestamp));
if (this.enableTaggedOnly) {
return newResults.filter((c) => c.tagged === true);
return unique.filter((c) => c.tagged === true);
}
return newResults;
return unique;
}
/**
* Load the changelog directly from the Gitea repository.
*/
private async loadChangelogFromRepo(owner: string, repo: string): Promise<void> {
const url = `/api/v1/repos/${owner}/${repo}/contents/changelog.md`;
const headers: Record<string, string> = {};
if (this.token) {
headers['Authorization'] = `token ${this.token}`;
if (this.token) headers['Authorization'] = `token ${this.token}`;
const candidates = [
'CHANGELOG.md',
'changelog.md',
'Changelog.md',
'docs/CHANGELOG.md',
];
for (const path of candidates) {
const url = `/api/v1/repos/${owner}/${repo}/contents/${encodeURIComponent(path)}`;
const response = await this.fetchFunction(url, { headers });
if (!response.ok) {
continue;
}
try {
const data = await response.json();
if (data && data.content) {
this.changelogContent = Buffer.from(data.content, 'base64').toString('utf8');
return;
}
} catch {
// continue trying others
}
}
const response = await this.fetchFunction(url, { headers });
if (!response.ok) {
console.error(
`Could not fetch CHANGELOG.md from ${owner}/${repo}: ${response.status} ${response.statusText}`
);
this.changelogContent = '';
return;
}
const data = await response.json();
if (!data.content) {
console.warn(`No content field found in response for ${owner}/${repo}/changelog.md`);
this.changelogContent = '';
return;
}
// decode base64 content
this.changelogContent = Buffer.from(data.content, 'base64').toString('utf8');
this.changelogContent = '';
}
/**
@@ -257,49 +294,68 @@ export class CodeFeed {
/**
* Fetch all tags for a given repo and return the set of tagged commit SHAs
*/
private async fetchTags(owner: string, repo: string): Promise<Set<string>> {
private async fetchTags(owner: string, repo: string): Promise<{ shas: Set<string>; map: Map<string, string> }> {
const taggedShas = new Set<string>();
const tagNameBySha = new Map<string, string>();
let page = 1;
while (true) {
const url = `/api/v1/repos/${owner}/${repo}/tags?limit=50&page=${page}`;
const url = `/api/v1/repos/${owner}/${repo}/tags?limit=${this.pageLimit}&page=${page}`;
const resp = await this.fetchFunction(url, {
headers: this.token ? { Authorization: `token ${this.token}` } : {},
});
if (!resp.ok) {
console.error(`Failed to fetch tags for ${owner}/${repo}: ${resp.status} ${resp.statusText}`);
return taggedShas;
return { shas: taggedShas, map: tagNameBySha };
}
const data: plugins.interfaces.ITag[] = await resp.json();
if (data.length === 0) break;
for (const t of data) {
if (t.commit?.sha) taggedShas.add(t.commit.sha);
const sha = t.commit?.sha;
if (sha) {
taggedShas.add(sha);
if (t.name) tagNameBySha.set(sha, t.name);
}
}
if (data.length < 50) break;
if (data.length < this.pageLimit) break;
page++;
}
return taggedShas;
return { shas: taggedShas, map: tagNameBySha };
}
private async fetchAllOrganizations(): Promise<string[]> {
const resp = await this.fetchFunction('/api/v1/orgs', {
headers: this.token ? { Authorization: `token ${this.token}` } : {},
});
if (!resp.ok) {
throw new Error(`Failed to fetch organizations: ${resp.statusText}`);
const headers = this.token ? { Authorization: `token ${this.token}` } : {};
let page = 1;
const orgs: string[] = [];
while (true) {
const resp = await this.fetchFunction(`/api/v1/orgs?limit=${this.pageLimit}&page=${page}`, { headers });
if (!resp.ok) {
throw new Error(`Failed to fetch organizations: ${resp.status} ${resp.statusText}`);
}
const data: { username: string }[] = await resp.json();
if (data.length === 0) break;
orgs.push(...data.map((o) => o.username));
if (data.length < this.pageLimit) break;
page++;
}
const data: { username: string }[] = await resp.json();
return data.map((o) => o.username);
return orgs;
}
private async fetchRepositoriesForOrg(org: string): Promise<plugins.interfaces.IRepository[]> {
const resp = await this.fetchFunction(`/api/v1/orgs/${org}/repos?limit=50`, {
headers: this.token ? { Authorization: `token ${this.token}` } : {},
});
if (!resp.ok) {
throw new Error(`Failed to fetch repositories for ${org}: ${resp.statusText}`);
const headers = this.token ? { Authorization: `token ${this.token}` } : {};
let page = 1;
const repos: plugins.interfaces.IRepository[] = [];
while (true) {
const resp = await this.fetchFunction(`/api/v1/orgs/${org}/repos?limit=${this.pageLimit}&page=${page}`, { headers });
if (!resp.ok) {
throw new Error(`Failed to fetch repositories for ${org}: ${resp.status} ${resp.statusText}`);
}
const data: plugins.interfaces.IRepository[] = await resp.json();
if (data.length === 0) break;
repos.push(...data);
if (data.length < this.pageLimit) break;
page++;
}
const data: plugins.interfaces.IRepository[] = await resp.json();
return data;
return repos;
}
private async fetchRecentCommitsForRepo(
@@ -308,23 +364,52 @@ export class CodeFeed {
sinceTimestamp?: string
): Promise<plugins.interfaces.ICommit[]> {
const since = sinceTimestamp ?? this.lastRunTimestamp;
const resp = await this.fetchFunction(
`/api/v1/repos/${owner}/${repo}/commits?since=${encodeURIComponent(
since
)}&limit=50`,
{ headers: this.token ? { Authorization: `token ${this.token}` } : {} }
);
if (!resp.ok) {
throw new Error(`Failed to fetch commits for ${owner}/${repo}: ${resp.statusText}`);
const headers = this.token ? { Authorization: `token ${this.token}` } : {};
let page = 1;
const commits: plugins.interfaces.ICommit[] = [];
while (true) {
const url = `/api/v1/repos/${owner}/${repo}/commits?since=${encodeURIComponent(since)}&limit=${this.pageLimit}&page=${page}`;
const resp = await this.fetchFunction(url, { headers });
if (!resp.ok) {
throw new Error(`Failed to fetch commits for ${owner}/${repo}: ${resp.status} ${resp.statusText}`);
}
const data: plugins.interfaces.ICommit[] = await resp.json();
if (data.length === 0) break;
commits.push(...data);
if (data.length < this.pageLimit) break;
page++;
}
const data: plugins.interfaces.ICommit[] = await resp.json();
return data;
return commits;
}
public async fetchFunction(
urlArg: string,
optionsArg: RequestInit = {}
): Promise<Response> {
return fetch(`${this.baseUrl}${urlArg}`, optionsArg);
const maxAttempts = 4;
let attempt = 0;
let lastError: any;
while (attempt < maxAttempts) {
try {
const resp = await fetch(`${this.baseUrl}${urlArg}`, optionsArg);
// retry on 429 and 5xx
if (resp.status === 429 || resp.status >= 500) {
const retryAfter = Number(resp.headers.get('retry-after'));
const backoffMs = retryAfter
? retryAfter * 1000
: Math.min(32000, 1000 * Math.pow(2, attempt)) + Math.floor(Math.random() * 250);
await new Promise((r) => setTimeout(r, backoffMs));
attempt++;
continue;
}
return resp;
} catch (e: any) {
lastError = e;
const backoffMs = Math.min(32000, 1000 * Math.pow(2, attempt)) + Math.floor(Math.random() * 250);
await new Promise((r) => setTimeout(r, backoffMs));
attempt++;
}
}
throw new Error(`fetchFunction failed after retries for ${urlArg}: ${lastError?.message ?? 'unknown error'}`);
}
}

1
ts/interfaces/index.ts
View File

@@ -22,6 +22,7 @@ export interface ICommit {
}
export interface ITag {
name?: string;
commit?: {
sha?: string;
};