feat(core): Add changelog fetching and parsing functionality

changelog.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## 2024-12-14 - 1.6.0 - feat(core)
+Add changelog fetching and parsing functionality
+
+- Implemented loadChangelogFromRepo to directly load the changelog from a Gitea repository.
+- Introduced parsing functionality to extract specific version details from the loaded changelog.
+- Updated CodeFeed class to utilize the changelog for version verification and commit processing.
+
 ## 2024-12-14 - 1.5.3 - fix(core)
 Fix filtering logic for returning only tagged commits
 

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@foss.global/codefeed',
-  version: '1.5.3',
+  version: '1.6.0',
   description: 'a module for creating feeds for code development'
 }

ts/index.ts

@@ -1,12 +1,12 @@
 import * as plugins from './codefeed.plugins.js';
 
-
 export class CodeFeed {
   private baseUrl: string;
   private token?: string;
   private npmRegistry = new plugins.smartnpm.NpmRegistry();
   private smartxmlInstance = new plugins.smartxml.SmartXml();
   private lastRunTimestamp: string;
+  private changelogContent: string;
 
   constructor(baseUrl: string, token?: string, lastRunTimestamp?: string) {
     this.baseUrl = baseUrl;
@@ -16,8 +16,66 @@ export class CodeFeed {
   }
 
   /**
-   * Fetch all organizations from the Gitea instance.
+   * Load the changelog directly from the Gitea repository.
    */
+  private async loadChangelogFromRepo(owner: string, repo: string): Promise<void> {
+    const url = `${this.baseUrl}/api/v1/repos/${owner}/${repo}/contents/changelog.md`;
+    const headers: Record<string, string> = {};
+    if (this.token) {
+      headers['Authorization'] = `token ${this.token}`;
+    }
+
+    const response = await fetch(url, { headers });
+
+    if (!response.ok) {
+      console.error(`Could not fetch CHANGELOG.md from ${owner}/${repo}: ${response.status} ${response.statusText}`);
+      this.changelogContent = '';
+      return;
+    }
+
+    const data = await response.json();
+    if (!data.content) {
+      console.warn(`No content field found in response for ${owner}/${repo}/changelog.md`);
+      this.changelogContent = '';
+      return;
+    }
+
+    const decodedContent = Buffer.from(data.content, 'base64').toString('utf8');
+    this.changelogContent = decodedContent;
+  }
+
+  /**
+   * Parse the changelog to find the entry for a given version.
+   * The changelog format is assumed as:
+   *
+   * # Changelog
+   *
+   * ## <date> - <version> - <description>
+   * <changes...>
+   */
+  private getChangelogForVersion(version: string): string | undefined {
+    if (!this.changelogContent) {
+      return undefined;
+    }
+    const lines = this.changelogContent.split('\n');
+    const versionHeaderIndex = lines.findIndex((line) => line.includes(`- ${version} -`));
+    if (versionHeaderIndex === -1) {
+      return undefined;
+    }
+
+    const changelogLines: string[] = [];
+    for (let i = versionHeaderIndex + 1; i < lines.length; i++) {
+      const line = lines[i];
+      // The next version header starts with `## `
+      if (line.startsWith('## ')) {
+        break;
+      }
+      changelogLines.push(line);
+    }
+
+    return changelogLines.join('\n').trim();
+  }
+
   private async fetchAllOrganizations(): Promise<string[]> {
     const url = `${this.baseUrl}/api/v1/orgs`;
     const response = await fetch(url, {
@@ -32,18 +90,17 @@ export class CodeFeed {
     return data.map((org) => org.username);
   }
 
-  /**
-   * Fetch organization-level activity RSS feed.
-   */
   private async fetchOrgRssFeed(optionsArg: {
     orgName: string,
     repoName?: string,
   }): Promise<any[]> {
-    let rssUrl: string
+    let rssUrl: string;
     if (optionsArg.orgName && !optionsArg.repoName) {
       rssUrl = `${this.baseUrl}/${optionsArg.orgName}.atom`;
     } else if (optionsArg.orgName && optionsArg.repoName) {
       rssUrl = `${this.baseUrl}/${optionsArg.orgName}/${optionsArg.repoName}.atom`;
+    } else {
+      throw new Error('Invalid arguments provided to fetchOrgRssFeed.');
     }
 
     const response = await fetch(rssUrl);
@@ -52,36 +109,25 @@ export class CodeFeed {
     }
 
     const rssText = await response.text();
-
-    // Parse the Atom feed using fast-xml-parser
     const rssData = this.smartxmlInstance.parseXmlToObject(rssText);
-
-    // Return the <entry> elements from the feed
     return rssData.feed.entry || [];
   }
 
-  /**
-   * Check if the organization's RSS feed has any new activities since the last run.
-   */
   private async hasNewActivity(optionsArg: {
     orgName: string,
     repoName?: string,
   }): Promise<boolean> {
     const entries = await this.fetchOrgRssFeed(optionsArg);
 
-    // Filter entries to find new activities since the last run
     return entries.some((entry: any) => {
       const updated = new Date(entry.updated);
       return updated > new Date(this.lastRunTimestamp);
     });
   }
 
-  /**
+  private async fetchAllRepositories(): Promise<plugins.interfaces.IRepository[]> {
-   * Fetch all repositories accessible to the token/user.
-   */
-  private async fetchAllRepositories(): Promise<plugins.interfaces.Repository[]> {
     let page = 1;
-    const allRepos: plugins.interfaces.Repository[] = [];
+    const allRepos: plugins.interfaces.IRepository[] = [];
 
     while (true) {
       const url = new URL(`${this.baseUrl}/api/v1/repos/search`);
@@ -96,7 +142,7 @@ export class CodeFeed {
         throw new Error(`Failed to fetch repositories: ${resp.statusText}`);
       }
 
-      const data: plugins.interfaces.RepoSearchResponse = await resp.json();
+      const data: plugins.interfaces.IRepoSearchResponse = await resp.json();
       allRepos.push(...data.data);
 
       if (data.data.length < 50) {
@@ -108,12 +154,9 @@ export class CodeFeed {
     return allRepos;
   }
 
-  /**
-   * Fetch all tags for a given repository.
-   */
   private async fetchTags(owner: string, repo: string): Promise<Set<string>> {
     let page = 1;
-    const tags: plugins.interfaces.Tag[] = [];
+    const tags: plugins.interfaces.ITag[] = [];
 
     while (true) {
       const url = new URL(`${this.baseUrl}/api/v1/repos/${owner}/${repo}/tags`);
@@ -129,7 +172,7 @@ export class CodeFeed {
         throw new Error(`Failed to fetch tags for ${owner}/${repo}: ${resp.statusText}`);
       }
 
-      const data: plugins.interfaces.Tag[] = await resp.json();
+      const data: plugins.interfaces.ITag[] = await resp.json();
       tags.push(...data);
 
       if (data.length < 50) {
@@ -148,13 +191,10 @@ export class CodeFeed {
     return taggedCommitShas;
   }
 
-  /**
+  private async fetchRecentCommitsForRepo(owner: string, repo: string): Promise<plugins.interfaces.ICommit[]> {
-   * Fetch commits from the last 24 hours for a repository.
-   */
-  private async fetchRecentCommitsForRepo(owner: string, repo: string): Promise<plugins.interfaces.Commit[]> {
     const twentyFourHoursAgo = new Date(Date.now() - 24 * 60 * 60 * 1000);
     let page = 1;
-    const recentCommits: plugins.interfaces.Commit[] = [];
+    const recentCommits: plugins.interfaces.ICommit[] = [];
 
     while (true) {
       const url = new URL(`${this.baseUrl}/api/v1/repos/${owner}/${repo}/commits`);
@@ -170,7 +210,7 @@ export class CodeFeed {
         throw new Error(`Failed to fetch commits for ${owner}/${repo}: ${resp.statusText}`);
       }
 
-      const data: plugins.interfaces.Commit[] = await resp.json();
+      const data: plugins.interfaces.ICommit[] = await resp.json();
       if (data.length === 0) {
         break;
       }
@@ -190,13 +230,10 @@ export class CodeFeed {
     return recentCommits;
   }
 
-  /**
+  public async fetchAllCommitsFromInstance(): Promise<plugins.interfaces.ICommitResult[]> {
-   * Fetch all commits by querying all organizations.
-   */
-  public async fetchAllCommitsFromInstance(): Promise<plugins.interfaces.CommitResult[]> {
     const orgs = await this.fetchAllOrganizations();
     console.log(`Found ${orgs.length} organizations`);
-    let allCommits: plugins.interfaces.CommitResult[] = [];
+    let allCommits: plugins.interfaces.ICommitResult[] = [];
 
     for (const orgName of orgs) {
       console.log(`Checking activity for organization: ${orgName}`);
@@ -209,7 +246,7 @@ export class CodeFeed {
           console.log(`No new activity for organization: ${orgName}`);
           continue;
         }
-      } catch (error) {
+      } catch (error: any) {
         console.error(`Error fetching activity for organization ${orgName}:`, error.message);
         continue;
       }
@@ -227,10 +264,11 @@ export class CodeFeed {
             console.log(`No new activity for repository: ${orgName}/${r.name}`);
             continue;
           }
-        } catch (error) {
+        } catch (error: any) {
           console.error(`Error fetching activity for repository ${orgName}/${r.name}:`, error.message);
           continue;
         }
+
         const org = r.owner.login;
         const repo = r.name;
         console.log(`Processing repository ${org}/${repo}`);
@@ -239,8 +277,11 @@ export class CodeFeed {
           const taggedCommitShas = await this.fetchTags(org, repo);
           const commits = await this.fetchRecentCommitsForRepo(org, repo);
 
+          // Load the changelog from this repo.
+          await this.loadChangelogFromRepo(org, repo);
+
           const commitResults = commits.map((c) => {
-            const commit: plugins.interfaces.CommitResult = {
+            const commit: plugins.interfaces.ICommitResult = {
               baseUrl: this.baseUrl,
               org,
               repo,
@@ -250,45 +291,55 @@ export class CodeFeed {
               commitMessage: c.commit.message,
               tagged: taggedCommitShas.has(c.sha),
               publishedOnNpm: false,
-            }
+              changelog: undefined
+            };
             return commit;
           });
 
           if (commitResults.length > 0) {
             try {
               const packageInfo = await this.npmRegistry.getPackageInfo(`@${org}/${repo}`);
-              for (const commit of commitResults.filter((c) => c.tagged)) {
+              for (const commitResult of commitResults.filter((c) => c.tagged)) {
+                const versionCandidate = commitResult.commitMessage.replace('\n', '').trim();
                 const correspondingVersion = packageInfo.allVersions.find((versionArg) => {
-                  return versionArg.version === commit.commitMessage.replace('\n', '');
+                  return versionArg.version === versionCandidate;
                 });
                 if (correspondingVersion) {
-                  commit.publishedOnNpm = true;
+                  commitResult.publishedOnNpm = true;
+                  const changelogEntry = this.getChangelogForVersion(versionCandidate);
+                  if (changelogEntry) {
+                    commitResult.changelog = changelogEntry;
+                  }
                 }
               }
-            } catch (error) {
+            } catch (error: any) {
               console.error(`Failed to fetch package info for ${org}/${repo}:`, error.message);
             }
           }
 
           allCommits.push(...commitResults);
-        } catch (error) {
+        } catch (error: any) {
           console.error(`Skipping repository ${org}/${repo} due to error:`, error.message);
         }
       }
     }
 
     console.log(`Processed ${allCommits.length} commits in total.`);
+
+    allCommits = allCommits.filter(commitArg => commitArg.tagged);
+
+    console.log(`Filtered to ${allCommits.length} commits with tagged statuses.`);
+
     for (const c of allCommits) {
       console.log(`      ==========================================================================
       ${c.prettyAgoTime} ago:
       ${c.org}/${c.repo}
       ${c.commitMessage}
       Published on npm: ${c.publishedOnNpm}
+      ${c.changelog ? `Changelog:\n${c.changelog}\n` : ''}
     `);  
     }
 
-    allCommits = allCommits.filter(commitArg => commitArg.tagged);
-
     return allCommits;
   }
 }

ts/interfaces/index.ts

@@ -1,37 +1,37 @@
-export interface RepositoryOwner {
+export interface IRepositoryOwner {
   login: string;
 }
 
-export interface Repository {
+export interface IRepository {
-  owner: RepositoryOwner;
+  owner: IRepositoryOwner;
   name: string;
 }
 
-export interface CommitAuthor {
+export interface ICommitAuthor {
   date: string;
 }
 
-export interface CommitDetail {
+export interface ICommitDetail {
   message: string;
-  author: CommitAuthor;
+  author: ICommitAuthor;
 }
 
-export interface Commit {
+export interface ICommit {
   sha: string;
-  commit: CommitDetail;
+  commit: ICommitDetail;
 }
 
-export interface Tag {
+export interface ITag {
   commit?: {
     sha?: string;
   };
 }
 
-export interface RepoSearchResponse {
+export interface IRepoSearchResponse {
-  data: Repository[];
+  data: IRepository[];
 }
 
-export interface CommitResult {
+export interface ICommitResult {
   baseUrl: string;
   org: string;
   repo: string;
@@ -41,4 +41,5 @@ export interface CommitResult {
   tagged: boolean;
   publishedOnNpm: boolean;
   prettyAgoTime: string;
+  changelog: string | undefined;
 }