feat: add organization allowlist and denylist filters, enhance changelog loading, and improve fetch functions

- Introduced orgAllowlist and orgDenylist properties to filter organizations during fetching.
- Enhanced loadChangelogFromRepo to check multiple potential changelog file names.
- Updated fetchTags to return a map of tag names associated with their SHAs.
- Improved pagination logic in fetchAllOrganizations and fetchRepositoriesForOrg to handle larger datasets.
- Added retry logic in fetchFunction to handle rate limiting and server errors more gracefully.
- Modified ITag interface to include an optional name property for better tag handling.

package.json

@@ -16,17 +16,17 @@
     "buildDocs": "(tsdoc)"
   },
   "devDependencies": {
-    "@git.zone/tsbuild": "^2.3.2",
+    "@git.zone/tsbuild": "^2.6.8",
-    "@git.zone/tsbundle": "^2.2.5",
+    "@git.zone/tsbundle": "^2.5.1",
     "@git.zone/tsrun": "^1.2.46",
-    "@git.zone/tstest": "^1.0.96",
+    "@git.zone/tstest": "^2.3.8",
-    "@push.rocks/tapbundle": "^5.6.3",
+    "@push.rocks/tapbundle": "^6.0.3",
     "@types/node": "^22.15.2"
   },
   "dependencies": {
     "@push.rocks/lik": "^6.2.2",
-    "@push.rocks/qenv": "^6.1.0",
+    "@push.rocks/qenv": "^6.1.3",
-    "@push.rocks/smartnpm": "^2.0.4",
+    "@push.rocks/smartnpm": "^2.0.6",
     "@push.rocks/smarttime": "^4.1.1",
     "@push.rocks/smartxml": "^1.1.1"
   },

ts/index.ts

@@ -4,6 +4,7 @@ export class CodeFeed {
   private baseUrl: string;
   private token?: string;
   private lastRunTimestamp: string;
+  private pageLimit = 50;
   // Raw changelog content for the current repository
   private changelogContent: string = '';
   // npm registry helper for published-on-npm checks
@@ -16,6 +17,9 @@ export class CodeFeed {
   private enableNpmCheck: boolean = true;
   // return only tagged commits (false by default)
   private enableTaggedOnly: boolean = false;
+  // allow/deny filters
+  private orgAllowlist?: string[];
+  private orgDenylist?: string[];
 
   constructor(
     baseUrl: string,
@@ -26,6 +30,8 @@ export class CodeFeed {
       cacheWindowMs?: number;
       enableNpmCheck?: boolean;
       taggedOnly?: boolean;
+      orgAllowlist?: string[];
+      orgDenylist?: string[];
     }
   ) {
     this.baseUrl = baseUrl;
@@ -37,6 +43,8 @@ export class CodeFeed {
     this.cacheWindowMs = options?.cacheWindowMs;
     this.enableNpmCheck = options?.enableNpmCheck ?? true;
     this.enableTaggedOnly = options?.taggedOnly ?? false;
+    this.orgAllowlist = options?.orgAllowlist;
+    this.orgDenylist = options?.orgDenylist;
     this.cache = [];
     // npm registry instance for version lookups
     this.npmRegistry = new plugins.smartnpm.NpmRegistry();
@@ -61,7 +69,14 @@ export class CodeFeed {
     }
 
     // 1) get all organizations
-    const orgs = await this.fetchAllOrganizations();
+    let orgs = await this.fetchAllOrganizations();
+    // apply allow/deny filters
+    if (this.orgAllowlist && this.orgAllowlist.length > 0) {
+      orgs = orgs.filter((o) => this.orgAllowlist!.includes(o));
+    }
+    if (this.orgDenylist && this.orgDenylist.length > 0) {
+      orgs = orgs.filter((o) => !this.orgDenylist!.includes(o));
+    }
 
     // 2) fetch repos per org in parallel
     const repoLists = await Promise.all(
@@ -122,11 +137,15 @@ export class CodeFeed {
       await this.loadChangelogFromRepo(owner, name);
       // fetch tags for this repo
       let taggedShas: Set<string>;
+      let tagNameBySha: Map<string, string>;
       try {
-        taggedShas = await this.fetchTags(owner, name);
+        const tagInfo = await this.fetchTags(owner, name);
+        taggedShas = tagInfo.shas;
+        tagNameBySha = tagInfo.map;
       } catch (e: any) {
         console.error(`Failed to fetch tags for ${owner}/${name}:`, e.message);
         taggedShas = new Set<string>();
+        tagNameBySha = new Map<string, string>();
       }
       // fetch npm package info only if any new commits correspond to a tag
       const hasTaggedCommit = commits.some((c) => taggedShas.has(c.sha));
@@ -141,14 +160,23 @@ export class CodeFeed {
       }
       // build commit entries
       for (const c of commits) {
-        const versionCandidate = c.commit.message.replace(/\n/g, '').trim();
         const isTagged = taggedShas.has(c.sha);
-        const publishedOnNpm = isTagged && pkgInfo
+        // derive version from tag name if present (strip leading 'v')
-          ? pkgInfo.allVersions.some((v) => v.version === versionCandidate)
+        let versionFromTag: string | undefined;
+        if (isTagged) {
+          const tagName = tagNameBySha.get(c.sha);
+          if (tagName) {
+            versionFromTag = tagName.startsWith('v') ? tagName.substring(1) : tagName;
+          }
+        }
+        const publishedOnNpm = isTagged && pkgInfo && versionFromTag
+          ? pkgInfo.allVersions.some((v) => v.version === versionFromTag)
           : false;
         let changelogEntry: string | undefined;
         if (this.changelogContent) {
-          changelogEntry = this.getChangelogForVersion(versionCandidate);
+          if (versionFromTag) {
+            changelogEntry = this.getChangelogForVersion(versionFromTag);
+          }
         }
         newResults.push({
           baseUrl: this.baseUrl,
@@ -187,40 +215,49 @@ export class CodeFeed {
       return this.cache;
     }
     // no caching: apply tagged-only filter if requested
+    // sort and dedupe
+    const seen = new Set<string>();
+    const unique = newResults.filter((c) => {
+      if (seen.has(c.hash)) return false;
+      seen.add(c.hash);
+      return true;
+    });
+    unique.sort((a, b) => b.timestamp.localeCompare(a.timestamp));
     if (this.enableTaggedOnly) {
-      return newResults.filter((c) => c.tagged === true);
+      return unique.filter((c) => c.tagged === true);
     }
-    return newResults;
+    return unique;
   }
 
   /**
    * Load the changelog directly from the Gitea repository.
    */
   private async loadChangelogFromRepo(owner: string, repo: string): Promise<void> {
-    const url = `/api/v1/repos/${owner}/${repo}/contents/changelog.md`;
     const headers: Record<string, string> = {};
-    if (this.token) {
+    if (this.token) headers['Authorization'] = `token ${this.token}`;
-      headers['Authorization'] = `token ${this.token}`;
+    const candidates = [
+      'CHANGELOG.md',
+      'changelog.md',
+      'Changelog.md',
+      'docs/CHANGELOG.md',
+    ];
+    for (const path of candidates) {
+      const url = `/api/v1/repos/${owner}/${repo}/contents/${encodeURIComponent(path)}`;
+      const response = await this.fetchFunction(url, { headers });
+      if (!response.ok) {
+        continue;
+      }
+      try {
+        const data = await response.json();
+        if (data && data.content) {
+          this.changelogContent = Buffer.from(data.content, 'base64').toString('utf8');
+          return;
+        }
+      } catch {
+        // continue trying others
+      }
     }
-
+    this.changelogContent = '';
-    const response = await this.fetchFunction(url, { headers });
-    if (!response.ok) {
-      console.error(
-        `Could not fetch CHANGELOG.md from ${owner}/${repo}: ${response.status} ${response.statusText}`
-      );
-      this.changelogContent = '';
-      return;
-    }
-
-    const data = await response.json();
-    if (!data.content) {
-      console.warn(`No content field found in response for ${owner}/${repo}/changelog.md`);
-      this.changelogContent = '';
-      return;
-    }
-
-    // decode base64 content
-    this.changelogContent = Buffer.from(data.content, 'base64').toString('utf8');
   }
 
   /**
@@ -257,49 +294,68 @@ export class CodeFeed {
   /**
    * Fetch all tags for a given repo and return the set of tagged commit SHAs
    */
-  private async fetchTags(owner: string, repo: string): Promise<Set<string>> {
+  private async fetchTags(owner: string, repo: string): Promise<{ shas: Set<string>; map: Map<string, string> }> {
     const taggedShas = new Set<string>();
+    const tagNameBySha = new Map<string, string>();
     let page = 1;
     while (true) {
-      const url = `/api/v1/repos/${owner}/${repo}/tags?limit=50&page=${page}`;
+      const url = `/api/v1/repos/${owner}/${repo}/tags?limit=${this.pageLimit}&page=${page}`;
       const resp = await this.fetchFunction(url, {
         headers: this.token ? { Authorization: `token ${this.token}` } : {},
       });
       if (!resp.ok) {
         console.error(`Failed to fetch tags for ${owner}/${repo}: ${resp.status} ${resp.statusText}`);
-        return taggedShas;
+        return { shas: taggedShas, map: tagNameBySha };
       }
       const data: plugins.interfaces.ITag[] = await resp.json();
       if (data.length === 0) break;
       for (const t of data) {
-        if (t.commit?.sha) taggedShas.add(t.commit.sha);
+        const sha = t.commit?.sha;
+        if (sha) {
+          taggedShas.add(sha);
+          if (t.name) tagNameBySha.set(sha, t.name);
+        }
       }
-      if (data.length < 50) break;
+      if (data.length < this.pageLimit) break;
       page++;
     }
-    return taggedShas;
+    return { shas: taggedShas, map: tagNameBySha };
   }
 
   private async fetchAllOrganizations(): Promise<string[]> {
-    const resp = await this.fetchFunction('/api/v1/orgs', {
+    const headers = this.token ? { Authorization: `token ${this.token}` } : {};
-      headers: this.token ? { Authorization: `token ${this.token}` } : {},
+    let page = 1;
-    });
+    const orgs: string[] = [];
-    if (!resp.ok) {
+    while (true) {
-      throw new Error(`Failed to fetch organizations: ${resp.statusText}`);
+      const resp = await this.fetchFunction(`/api/v1/orgs?limit=${this.pageLimit}&page=${page}`, { headers });
+      if (!resp.ok) {
+        throw new Error(`Failed to fetch organizations: ${resp.status} ${resp.statusText}`);
+      }
+      const data: { username: string }[] = await resp.json();
+      if (data.length === 0) break;
+      orgs.push(...data.map((o) => o.username));
+      if (data.length < this.pageLimit) break;
+      page++;
     }
-    const data: { username: string }[] = await resp.json();
+    return orgs;
-    return data.map((o) => o.username);
   }
 
   private async fetchRepositoriesForOrg(org: string): Promise<plugins.interfaces.IRepository[]> {
-    const resp = await this.fetchFunction(`/api/v1/orgs/${org}/repos?limit=50`, {
+    const headers = this.token ? { Authorization: `token ${this.token}` } : {};
-      headers: this.token ? { Authorization: `token ${this.token}` } : {},
+    let page = 1;
-    });
+    const repos: plugins.interfaces.IRepository[] = [];
-    if (!resp.ok) {
+    while (true) {
-      throw new Error(`Failed to fetch repositories for ${org}: ${resp.statusText}`);
+      const resp = await this.fetchFunction(`/api/v1/orgs/${org}/repos?limit=${this.pageLimit}&page=${page}`, { headers });
+      if (!resp.ok) {
+        throw new Error(`Failed to fetch repositories for ${org}: ${resp.status} ${resp.statusText}`);
+      }
+      const data: plugins.interfaces.IRepository[] = await resp.json();
+      if (data.length === 0) break;
+      repos.push(...data);
+      if (data.length < this.pageLimit) break;
+      page++;
     }
-    const data: plugins.interfaces.IRepository[] = await resp.json();
+    return repos;
-    return data;
   }
 
   private async fetchRecentCommitsForRepo(
@@ -308,23 +364,52 @@ export class CodeFeed {
     sinceTimestamp?: string
   ): Promise<plugins.interfaces.ICommit[]> {
     const since = sinceTimestamp ?? this.lastRunTimestamp;
-    const resp = await this.fetchFunction(
+    const headers = this.token ? { Authorization: `token ${this.token}` } : {};
-      `/api/v1/repos/${owner}/${repo}/commits?since=${encodeURIComponent(
+    let page = 1;
-        since
+    const commits: plugins.interfaces.ICommit[] = [];
-      )}&limit=50`,
+    while (true) {
-      { headers: this.token ? { Authorization: `token ${this.token}` } : {} }
+      const url = `/api/v1/repos/${owner}/${repo}/commits?since=${encodeURIComponent(since)}&limit=${this.pageLimit}&page=${page}`;
-    );
+      const resp = await this.fetchFunction(url, { headers });
-    if (!resp.ok) {
+      if (!resp.ok) {
-      throw new Error(`Failed to fetch commits for ${owner}/${repo}: ${resp.statusText}`);
+        throw new Error(`Failed to fetch commits for ${owner}/${repo}: ${resp.status} ${resp.statusText}`);
+      }
+      const data: plugins.interfaces.ICommit[] = await resp.json();
+      if (data.length === 0) break;
+      commits.push(...data);
+      if (data.length < this.pageLimit) break;
+      page++;
     }
-    const data: plugins.interfaces.ICommit[] = await resp.json();
+    return commits;
-    return data;
   }
 
   public async fetchFunction(
     urlArg: string,
     optionsArg: RequestInit = {}
   ): Promise<Response> {
-    return fetch(`${this.baseUrl}${urlArg}`, optionsArg);
+    const maxAttempts = 4;
+    let attempt = 0;
+    let lastError: any;
+    while (attempt < maxAttempts) {
+      try {
+        const resp = await fetch(`${this.baseUrl}${urlArg}`, optionsArg);
+        // retry on 429 and 5xx
+        if (resp.status === 429 || resp.status >= 500) {
+          const retryAfter = Number(resp.headers.get('retry-after'));
+          const backoffMs = retryAfter
+            ? retryAfter * 1000
+            : Math.min(32000, 1000 * Math.pow(2, attempt)) + Math.floor(Math.random() * 250);
+          await new Promise((r) => setTimeout(r, backoffMs));
+          attempt++;
+          continue;
+        }
+        return resp;
+      } catch (e: any) {
+        lastError = e;
+        const backoffMs = Math.min(32000, 1000 * Math.pow(2, attempt)) + Math.floor(Math.random() * 250);
+        await new Promise((r) => setTimeout(r, backoffMs));
+        attempt++;
+      }
+    }
+    throw new Error(`fetchFunction failed after retries for ${urlArg}: ${lastError?.message ?? 'unknown error'}`);
   }
-}
+}

ts/interfaces/index.ts

@@ -22,6 +22,7 @@ export interface ICommit {
 }
 
 export interface ITag {
+  name?: string;
   commit?: {
     sha?: string;
   };