feat(core): Add organization-level activity fetching and RSS parsing

changelog.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## 2024-12-13 - 1.2.0 - feat(core)
+Add organization-level activity fetching and RSS parsing
+
+- Integrated smartxml package for XML parsing.
+- Implemented fetching of all organizations within a Gitea instance.
+- Added functionality to check new activities in organization RSS feeds.
+- Enhanced fetching logic to include repository commits and tags.
+
 ## 2024-12-13 - 1.1.0 - feat(core)
 Add tracking of commits published on npm
 

package.json

@@ -23,7 +23,8 @@
   },
   "dependencies": {
     "@push.rocks/qenv": "^6.1.0",
-    "@push.rocks/smartnpm": "^2.0.4"
+    "@push.rocks/smartnpm": "^2.0.4",
+    "@push.rocks/smartxml": "^1.0.8"
   },
   "repository": {
     "type": "git",

pnpm-lock.yaml

@@ -14,6 +14,9 @@ importers:
       '@push.rocks/smartnpm':
         specifier: ^2.0.4
         version: 2.0.4
+      '@push.rocks/smartxml':
+        specifier: ^1.0.8
+        version: 1.0.8
     devDependencies:
       '@git.zone/tsbuild':
         specifier: ^2.1.25

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@foss.global/codefeed',
-  version: '1.1.0',
+  version: '1.2.0',
   description: 'a module for creating feeds for code development'
 }

ts/codefeed.plugins.ts

@@ -1,8 +1,10 @@
 // @push.rocks
-import * as qenv from '@push.rocks/qenv'
+import * as qenv from '@push.rocks/qenv';
-import * as smartnpm from '@push.rocks/smartnpm'
+import * as smartnpm from '@push.rocks/smartnpm';
+import * as smartxml from '@push.rocks/smartxml';
 
 export {
   qenv,
   smartnpm,
+  smartxml,
 }

ts/index.ts

@@ -48,13 +48,80 @@ export class CodeFeed {
   private baseUrl: string;
   private token?: string;
   private npmRegistry = new plugins.smartnpm.NpmRegistry();
+  private smartxmlInstance = new plugins.smartxml.SmartXml();
+  private lastRunTimestamp: string;
 
-  constructor(baseUrl: string, token?: string) {
+  constructor(baseUrl: string, token?: string, lastRunTimestamp?: string) {
     this.baseUrl = baseUrl;
     this.token = token;
-    console.log('CodeFeed initialized');
+    this.lastRunTimestamp = lastRunTimestamp || new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString();
+    console.log('CodeFeed initialized with last run timestamp:', this.lastRunTimestamp);
   }
 
+  /**
+   * Fetch all organizations from the Gitea instance.
+   */
+  private async fetchAllOrganizations(): Promise<string[]> {
+    const url = `${this.baseUrl}/api/v1/orgs`;
+    const response = await fetch(url, {
+      headers: this.token ? { Authorization: `token ${this.token}` } : {},
+    });
+
+    if (!response.ok) {
+      throw new Error(`Failed to fetch organizations: ${response.statusText}`);
+    }
+
+    const data: { username: string }[] = await response.json();
+    return data.map((org) => org.username);
+  }
+
+  /**
+   * Fetch organization-level activity RSS feed.
+   */
+  private async fetchOrgRssFeed(optionsArg: {
+    orgName: string,
+    repoName?: string,
+  }): Promise<any[]> {
+    let rssUrl: string
+    if (optionsArg.orgName && !optionsArg.repoName) {
+      rssUrl = `${this.baseUrl}/${optionsArg.orgName}.atom`;
+    } else if (optionsArg.orgName && optionsArg.repoName) {
+      rssUrl = `${this.baseUrl}/${optionsArg.orgName}/${optionsArg.repoName}.atom`;
+    }
+
+    const response = await fetch(rssUrl);
+    if (!response.ok) {
+      throw new Error(`Failed to fetch RSS feed for organization ${optionsArg.orgName}/${optionsArg.repoName}: ${response.statusText}`);
+    }
+
+    const rssText = await response.text();
+
+    // Parse the Atom feed using fast-xml-parser
+    const rssData = this.smartxmlInstance.parseXmlToObject(rssText);
+
+    // Return the <entry> elements from the feed
+    return rssData.feed.entry || [];
+  }
+
+  /**
+   * Check if the organization's RSS feed has any new activities since the last run.
+   */
+  private async hasNewActivity(optionsArg: {
+    orgName: string,
+    repoName?: string,
+  }): Promise<boolean> {
+    const entries = await this.fetchOrgRssFeed(optionsArg);
+
+    // Filter entries to find new activities since the last run
+    return entries.some((entry: any) => {
+      const updated = new Date(entry.updated);
+      return updated > new Date(this.lastRunTimestamp);
+    });
+  }
+
+  /**
+   * Fetch all repositories accessible to the token/user.
+   */
   private async fetchAllRepositories(): Promise<Repository[]> {
     let page = 1;
     const allRepos: Repository[] = [];
@@ -65,7 +132,7 @@ export class CodeFeed {
       url.searchParams.set('page', page.toString());
 
       const resp = await fetch(url.href, {
-        headers: this.token ? { 'Authorization': `token ${this.token}` } : {}
+        headers: this.token ? { 'Authorization': `token ${this.token}` } : {},
       });
 
       if (!resp.ok) {
@@ -84,6 +151,9 @@ export class CodeFeed {
     return allRepos;
   }
 
+  /**
+   * Fetch all tags for a given repository.
+   */
   private async fetchTags(owner: string, repo: string): Promise<Set<string>> {
     let page = 1;
     const tags: Tag[] = [];
@@ -94,7 +164,7 @@ export class CodeFeed {
       url.searchParams.set('page', page.toString());
 
       const resp = await fetch(url.href, {
-        headers: this.token ? { 'Authorization': `token ${this.token}` } : {}
+        headers: this.token ? { 'Authorization': `token ${this.token}` } : {},
       });
 
       if (!resp.ok) {
@@ -121,6 +191,9 @@ export class CodeFeed {
     return taggedCommitShas;
   }
 
+  /**
+   * Fetch commits from the last 24 hours for a repository.
+   */
   private async fetchRecentCommitsForRepo(owner: string, repo: string): Promise<Commit[]> {
     const twentyFourHoursAgo = new Date(Date.now() - 24 * 60 * 60 * 1000);
     let page = 1;
@@ -128,12 +201,13 @@ export class CodeFeed {
 
     while (true) {
       const url = new URL(`${this.baseUrl}/api/v1/repos/${owner}/${repo}/commits`);
-      url.searchParams.set('limit', '1');
+      url.searchParams.set('limit', '50');
       url.searchParams.set('page', page.toString());
 
       const resp = await fetch(url.href, {
-        headers: this.token ? { 'Authorization': `token ${this.token}` } : {}
+        headers: this.token ? { 'Authorization': `token ${this.token}` } : {},
       });
+
       if (!resp.ok) {
         console.error(`Failed to fetch commits for ${owner}/${repo}: ${resp.status} ${resp.statusText} at ${url.href}`);
         throw new Error(`Failed to fetch commits for ${owner}/${repo}: ${resp.statusText}`);
@@ -149,7 +223,6 @@ export class CodeFeed {
         if (commitDate > twentyFourHoursAgo) {
           recentCommits.push(commit);
         } else {
-          // If we encounter a commit older than 24 hours, we can stop fetching more pages
           return recentCommits;
         }
       }
@@ -160,13 +233,47 @@ export class CodeFeed {
     return recentCommits;
   }
 
+  /**
+   * Fetch all commits by querying all organizations.
+   */
   public async fetchAllCommitsFromInstance(): Promise<CommitResult[]> {
-    const repos = await this.fetchAllRepositories();
+    const orgs = await this.fetchAllOrganizations();
-    const skippedRepos: string[] = [];
+    console.log(`Found ${orgs.length} organizations`);
-    console.log(`Found ${repos.length} repositories`);
     let allCommits: CommitResult[] = [];
 
-    for (const r of repos) {
+    for (const orgName of orgs) {
+      console.log(`Checking activity for organization: ${orgName}`);
+
+      try {
+        const hasActivity = await this.hasNewActivity({
+          orgName,
+        });
+        if (!hasActivity) {
+          console.log(`No new activity for organization: ${orgName}`);
+          continue;
+        }
+      } catch (error) {
+        console.error(`Error fetching activity for organization ${orgName}:`, error.message);
+        continue;
+      }
+
+      console.log(`New activity detected for organization: ${orgName}. Processing repositories...`);
+
+      const repos = await this.fetchAllRepositories();
+      for (const r of repos.filter((repo) => repo.owner.login === orgName)) {
+        try {
+          const hasActivity = await this.hasNewActivity({
+            orgName,
+            repoName: r.name,
+          });
+          if (!hasActivity) {
+            console.log(`No new activity for repository: ${orgName}/${r.name}`);
+            continue;
+          }
+        } catch (error) {
+          console.error(`Error fetching activity for repository ${orgName}/${r.name}:`, error.message);
+          continue;
+        }
         const org = r.owner.login;
         const repo = r.name;
         console.log(`Processing repository ${org}/${repo}`);
@@ -174,10 +281,8 @@ export class CodeFeed {
         try {
           const taggedCommitShas = await this.fetchTags(org, repo);
           const commits = await this.fetchRecentCommitsForRepo(org, repo);
-        console.log(`${org}/${repo} -> Found ${commits.length} commits`);
+
-        const commitResults: CommitResult[] = [];
+          const commitResults = commits.map((c) => ({
-        for (const c of commits) {
-          const commit: CommitResult = {
             baseUrl: this.baseUrl,
             org,
             repo,
@@ -186,40 +291,32 @@ export class CodeFeed {
             commitMessage: c.commit.message,
             tagged: taggedCommitShas.has(c.sha),
             publishedOnNpm: false,
-          }
+          }));
-          commitResults.push(commit);
-        }
 
           if (commitResults.length > 0) {
             try {
               const packageInfo = await this.npmRegistry.getPackageInfo(`@${org}/${repo}`);
-            for (const commit of commitResults.filter(c => c.tagged)) {
+              for (const commit of commitResults.filter((c) => c.tagged)) {
-              const correspondingVersion = packageInfo.allVersions.find(versionArg => {
+                const correspondingVersion = packageInfo.allVersions.find((versionArg) => {
                   return versionArg.version === commit.commitMessage.replace('\n', '');
                 });
                 if (correspondingVersion) {
                   commit.publishedOnNpm = true;
                 }
               }
-          } catch (error: any) {
+            } catch (error) {
               console.error(`Failed to fetch package info for ${org}/${repo}:`, error.message);
-            continue;
             }
           }
 
           allCommits.push(...commitResults);
-      } catch (error: any) {
+        } catch (error) {
-        skippedRepos.push(`${org}/${repo}`);
           console.error(`Skipping repository ${org}/${repo} due to error:`, error.message);
-        continue;
+        }
       }
     }
 
-    console.log(`Found ${allCommits.length} relevant commits`);
+    console.log(`Processed ${allCommits.length} commits in total.`);
-    console.log(`Skipped ${skippedRepos.length} repositories due to errors`);
-    for (const s of skippedRepos) {
-      console.log(`Skipped ${s}`);
-    }
     for (const c of allCommits) {
       console.log(`______________________________________________________
       Commit ${c.hash} by ${c.org}/${c.repo} at ${c.timestamp}
@@ -227,7 +324,6 @@ export class CodeFeed {
       Published on npm: ${c.publishedOnNpm}
     `);  
     }
-
     return allCommits;
   }
 }