1.2.1

changelog.md

@@ -1,5 +1,23 @@
 # Changelog
 
+## 2024-12-13 - 1.2.1 - fix(core)
+No changes detected
+
+
+## 2024-12-13 - 1.2.0 - feat(core)
+Add organization-level activity fetching and RSS parsing
+
+- Integrated smartxml package for XML parsing.
+- Implemented fetching of all organizations within a Gitea instance.
+- Added functionality to check new activities in organization RSS feeds.
+- Enhanced fetching logic to include repository commits and tags.
+
+## 2024-12-13 - 1.1.0 - feat(core)
+Add tracking of commits published on npm
+
+- Introduced a check for published commits on npm using smartnpm.
+- Enhanced fetchAllCommitsFromInstance to include 'publishedOnNpm' status in results.
+
 ## 2024-12-13 - 1.0.2 - fix(core)
 Improve error handling in fetchRecentCommitsForRepo method
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@foss.global/codefeed",
-  "version": "1.0.2",
+  "version": "1.2.1",
   "private": false,
   "description": "a module for creating feeds for code development",
   "main": "dist_ts/index.js",
@@ -22,7 +22,9 @@
     "@types/node": "^20.8.7"
   },
   "dependencies": {
-    "@push.rocks/qenv": "^6.1.0"
+    "@push.rocks/qenv": "^6.1.0",
+    "@push.rocks/smartnpm": "^2.0.4",
+    "@push.rocks/smartxml": "^1.0.8"
   },
   "repository": {
     "type": "git",

pnpm-lock.yaml

@@ -11,6 +11,12 @@ importers:
       '@push.rocks/qenv':
         specifier: ^6.1.0
         version: 6.1.0
+      '@push.rocks/smartnpm':
+        specifier: ^2.0.4
+        version: 2.0.4
+      '@push.rocks/smartxml':
+        specifier: ^1.0.8
+        version: 1.0.8
     devDependencies:
       '@git.zone/tsbuild':
         specifier: ^2.1.25

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@foss.global/codefeed',
-  version: '1.0.2',
+  version: '1.2.1',
   description: 'a module for creating feeds for code development'
 }

ts/codefeed.plugins.ts

@@ -1,6 +1,10 @@
 // @push.rocks
-import * as qenv from '@push.rocks/qenv'
+import * as qenv from '@push.rocks/qenv';
+import * as smartnpm from '@push.rocks/smartnpm';
+import * as smartxml from '@push.rocks/smartxml';
 
 export {
   qenv,
+  smartnpm,
+  smartxml,
 }

ts/index.ts

@@ -41,18 +41,87 @@ interface CommitResult {
   hash: string;
   commitMessage: string;
   tagged: boolean;
+  publishedOnNpm: boolean;
 }
 
 export class CodeFeed {
   private baseUrl: string;
   private token?: string;
+  private npmRegistry = new plugins.smartnpm.NpmRegistry();
+  private smartxmlInstance = new plugins.smartxml.SmartXml();
+  private lastRunTimestamp: string;
 
-  constructor(baseUrl: string, token?: string) {
+  constructor(baseUrl: string, token?: string, lastRunTimestamp?: string) {
     this.baseUrl = baseUrl;
     this.token = token;
-    console.log('CodeFeed initialized');
+    this.lastRunTimestamp = lastRunTimestamp || new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString();
+    console.log('CodeFeed initialized with last run timestamp:', this.lastRunTimestamp);
   }
 
+  /**
+   * Fetch all organizations from the Gitea instance.
+   */
+  private async fetchAllOrganizations(): Promise<string[]> {
+    const url = `${this.baseUrl}/api/v1/orgs`;
+    const response = await fetch(url, {
+      headers: this.token ? { Authorization: `token ${this.token}` } : {},
+    });
+
+    if (!response.ok) {
+      throw new Error(`Failed to fetch organizations: ${response.statusText}`);
+    }
+
+    const data: { username: string }[] = await response.json();
+    return data.map((org) => org.username);
+  }
+
+  /**
+   * Fetch organization-level activity RSS feed.
+   */
+  private async fetchOrgRssFeed(optionsArg: {
+    orgName: string,
+    repoName?: string,
+  }): Promise<any[]> {
+    let rssUrl: string
+    if (optionsArg.orgName && !optionsArg.repoName) {
+      rssUrl = `${this.baseUrl}/${optionsArg.orgName}.atom`;
+    } else if (optionsArg.orgName && optionsArg.repoName) {
+      rssUrl = `${this.baseUrl}/${optionsArg.orgName}/${optionsArg.repoName}.atom`;
+    }
+
+    const response = await fetch(rssUrl);
+    if (!response.ok) {
+      throw new Error(`Failed to fetch RSS feed for organization ${optionsArg.orgName}/${optionsArg.repoName}: ${response.statusText}`);
+    }
+
+    const rssText = await response.text();
+
+    // Parse the Atom feed using fast-xml-parser
+    const rssData = this.smartxmlInstance.parseXmlToObject(rssText);
+
+    // Return the <entry> elements from the feed
+    return rssData.feed.entry || [];
+  }
+
+  /**
+   * Check if the organization's RSS feed has any new activities since the last run.
+   */
+  private async hasNewActivity(optionsArg: {
+    orgName: string,
+    repoName?: string,
+  }): Promise<boolean> {
+    const entries = await this.fetchOrgRssFeed(optionsArg);
+
+    // Filter entries to find new activities since the last run
+    return entries.some((entry: any) => {
+      const updated = new Date(entry.updated);
+      return updated > new Date(this.lastRunTimestamp);
+    });
+  }
+
+  /**
+   * Fetch all repositories accessible to the token/user.
+   */
   private async fetchAllRepositories(): Promise<Repository[]> {
     let page = 1;
     const allRepos: Repository[] = [];
@@ -63,7 +132,7 @@ export class CodeFeed {
       url.searchParams.set('page', page.toString());
 
       const resp = await fetch(url.href, {
-        headers: this.token ? { 'Authorization': `token ${this.token}` } : {}
+        headers: this.token ? { 'Authorization': `token ${this.token}` } : {},
       });
 
       if (!resp.ok) {
@@ -82,6 +151,9 @@ export class CodeFeed {
     return allRepos;
   }
 
+  /**
+   * Fetch all tags for a given repository.
+   */
   private async fetchTags(owner: string, repo: string): Promise<Set<string>> {
     let page = 1;
     const tags: Tag[] = [];
@@ -92,9 +164,9 @@ export class CodeFeed {
       url.searchParams.set('page', page.toString());
 
       const resp = await fetch(url.href, {
-        headers: this.token ? { 'Authorization': `token ${this.token}` } : {}
+        headers: this.token ? { 'Authorization': `token ${this.token}` } : {},
       });
-      
+
       if (!resp.ok) {
         console.error(`Failed to fetch tags for ${owner}/${repo}: ${resp.status} ${resp.statusText} at ${url.href}`);
         throw new Error(`Failed to fetch tags for ${owner}/${repo}: ${resp.statusText}`);
@@ -119,6 +191,9 @@ export class CodeFeed {
     return taggedCommitShas;
   }
 
+  /**
+   * Fetch commits from the last 24 hours for a repository.
+   */
   private async fetchRecentCommitsForRepo(owner: string, repo: string): Promise<Commit[]> {
     const twentyFourHoursAgo = new Date(Date.now() - 24 * 60 * 60 * 1000);
     let page = 1;
@@ -126,12 +201,13 @@ export class CodeFeed {
 
     while (true) {
       const url = new URL(`${this.baseUrl}/api/v1/repos/${owner}/${repo}/commits`);
-      url.searchParams.set('limit', '1');
+      url.searchParams.set('limit', '50');
       url.searchParams.set('page', page.toString());
 
       const resp = await fetch(url.href, {
-        headers: this.token ? { 'Authorization': `token ${this.token}` } : {}
+        headers: this.token ? { 'Authorization': `token ${this.token}` } : {},
       });
+
       if (!resp.ok) {
         console.error(`Failed to fetch commits for ${owner}/${repo}: ${resp.status} ${resp.statusText} at ${url.href}`);
         throw new Error(`Failed to fetch commits for ${owner}/${repo}: ${resp.statusText}`);
@@ -147,7 +223,6 @@ export class CodeFeed {
         if (commitDate > twentyFourHoursAgo) {
           recentCommits.push(commit);
         } else {
-          // If we encounter a commit older than 24 hours, we can stop fetching more pages
           return recentCommits;
         }
       }
@@ -158,37 +233,97 @@ export class CodeFeed {
     return recentCommits;
   }
 
+  /**
+   * Fetch all commits by querying all organizations.
+   */
   public async fetchAllCommitsFromInstance(): Promise<CommitResult[]> {
-    const repos = await this.fetchAllRepositories();
+    const orgs = await this.fetchAllOrganizations();
+    console.log(`Found ${orgs.length} organizations`);
     let allCommits: CommitResult[] = [];
 
-    for (const r of repos) {
-      const org = r.owner.login;
-      const repo = r.name;
-      console.log(`Processing repository ${org}/${repo}`);
+    for (const orgName of orgs) {
+      console.log(`Checking activity for organization: ${orgName}`);
 
       try {
-        const taggedCommitShas = await this.fetchTags(org, repo);
-        const commits = await this.fetchRecentCommitsForRepo(org, repo);
-        console.log(`${org}/${repo} -> Found ${commits.length} commits`);
-
-        const formatted = commits.map((c): CommitResult => ({
-          baseUrl: this.baseUrl,
-          org,
-          repo,
-          timestamp: c.commit.author.date,
-          hash: c.sha,
-          commitMessage: c.commit.message,
-          tagged: taggedCommitShas.has(c.sha)
-        }));
-
-        allCommits.push(...formatted);
-      } catch (error: any) {
-        console.error(`Skipping repository ${org}/${repo} due to error:`, error.message);
+        const hasActivity = await this.hasNewActivity({
+          orgName,
+        });
+        if (!hasActivity) {
+          console.log(`No new activity for organization: ${orgName}`);
+          continue;
+        }
+      } catch (error) {
+        console.error(`Error fetching activity for organization ${orgName}:`, error.message);
         continue;
       }
+
+      console.log(`New activity detected for organization: ${orgName}. Processing repositories...`);
+
+      const repos = await this.fetchAllRepositories();
+      for (const r of repos.filter((repo) => repo.owner.login === orgName)) {
+        try {
+          const hasActivity = await this.hasNewActivity({
+            orgName,
+            repoName: r.name,
+          });
+          if (!hasActivity) {
+            console.log(`No new activity for repository: ${orgName}/${r.name}`);
+            continue;
+          }
+        } catch (error) {
+          console.error(`Error fetching activity for repository ${orgName}/${r.name}:`, error.message);
+          continue;
+        }
+        const org = r.owner.login;
+        const repo = r.name;
+        console.log(`Processing repository ${org}/${repo}`);
+
+        try {
+          const taggedCommitShas = await this.fetchTags(org, repo);
+          const commits = await this.fetchRecentCommitsForRepo(org, repo);
+
+          const commitResults = commits.map((c) => ({
+            baseUrl: this.baseUrl,
+            org,
+            repo,
+            timestamp: c.commit.author.date,
+            hash: c.sha,
+            commitMessage: c.commit.message,
+            tagged: taggedCommitShas.has(c.sha),
+            publishedOnNpm: false,
+          }));
+
+          if (commitResults.length > 0) {
+            try {
+              const packageInfo = await this.npmRegistry.getPackageInfo(`@${org}/${repo}`);
+              for (const commit of commitResults.filter((c) => c.tagged)) {
+                const correspondingVersion = packageInfo.allVersions.find((versionArg) => {
+                  return versionArg.version === commit.commitMessage.replace('\n', '');
+                });
+                if (correspondingVersion) {
+                  commit.publishedOnNpm = true;
+                }
+              }
+            } catch (error) {
+              console.error(`Failed to fetch package info for ${org}/${repo}:`, error.message);
+            }
+          }
+
+          allCommits.push(...commitResults);
+        } catch (error) {
+          console.error(`Skipping repository ${org}/${repo} due to error:`, error.message);
+        }
+      }
     }
 
+    console.log(`Processed ${allCommits.length} commits in total.`);
+    for (const c of allCommits) {
+      console.log(`______________________________________________________
+      Commit ${c.hash} by ${c.org}/${c.repo} at ${c.timestamp}
+      ${c.commitMessage}
+      Published on npm: ${c.publishedOnNpm}
+    `);  
+    }
     return allCommits;
   }
 }