1.6.1

changelog.md

@@ -1,5 +1,32 @@
 # Changelog
 
+## 2024-12-14 - 1.6.1 - fix(docs)
+Updated project metadata and expanded documentation for installation and usage.
+
+- Updated description and keywords in package.json and npmextra.json.
+- Significant expansion of the README.md with detailed installation, usage, and feature instructions.
+
+## 2024-12-14 - 1.6.0 - feat(core)
+Add changelog fetching and parsing functionality
+
+- Implemented loadChangelogFromRepo to directly load the changelog from a Gitea repository.
+- Introduced parsing functionality to extract specific version details from the loaded changelog.
+- Updated CodeFeed class to utilize the changelog for version verification and commit processing.
+
+## 2024-12-14 - 1.5.3 - fix(core)
+Fix filtering logic for returning only tagged commits
+
+- Ensure `allCommits` is filtered to only include commits with 'tagged' status before returning.
+
+## 2024-12-14 - 1.5.2 - fix(core)
+Ensure stability of core functionalities.
+
+
+## 2024-12-14 - 1.5.1 - fix(core)
+Refine logging format in CodeFeed class
+
+- Modified console log format in fetchAllCommitsFromInstance method for better readability.
+
 ## 2024-12-14 - 1.5.0 - feat(core)
 Refactor TypeScript interfaces and improve module exports
 

npmextra.json

@@ -5,10 +5,23 @@
       "githost": "code.foss.global",
       "gitscope": "foss.global",
       "gitrepo": "codefeed",
-      "description": "a module for creating feeds for code development",
+      "description": "The @foss.global/codefeed module is designed for generating feeds from Gitea repositories, enhancing development workflows by processing commit data and repository activities.",
       "npmPackagename": "@foss.global/codefeed",
       "license": "MIT",
-      "projectDomain": "foss.global"
+      "projectDomain": "foss.global",
+      "keywords": [
+        "codefeed",
+        "Gitea",
+        "commits",
+        "changelog",
+        "repository",
+        "development tools",
+        "npm",
+        "module",
+        "code analysis",
+        "activity feed",
+        "version control"
+      ]
     }
   },
   "npmci": {

package.json

@@ -1,8 +1,8 @@
 {
   "name": "@foss.global/codefeed",
-  "version": "1.5.0",
+  "version": "1.6.1",
   "private": false,
-  "description": "a module for creating feeds for code development",
+  "description": "The @foss.global/codefeed module is designed for generating feeds from Gitea repositories, enhancing development workflows by processing commit data and repository activities.",
   "exports": {
     ".": "./dist_ts/index.js",
     "./interfaces": "./dist_ts/interfaces/index.js"
@@ -48,5 +48,18 @@
     "cli.js",
     "npmextra.json",
     "readme.md"
+  ],
+  "keywords": [
+    "codefeed",
+    "Gitea",
+    "commits",
+    "changelog",
+    "repository",
+    "development tools",
+    "npm",
+    "module",
+    "code analysis",
+    "activity feed",
+    "version control"
   ]
 }

readme.md

@@ -1,7 +1,130 @@
+```markdown
 # @foss.global/codefeed
 
-a module for creating feeds for code development
+A module for creating feeds for code development.
 
-## How to create the docs
+## Install
 
-To create docs run gitzone aidoc.
+To install the `@foss.global/codefeed` package, you can run the following npm command in your project directory:
+
+```bash
+npm install @foss.global/codefeed
+```
+
+Ensure that you have a compatible version of Node.js installed and that your project is set up to support ECMAScript modules. The `@foss.global/codefeed` module uses ESM syntax.
+
+## Usage
+
+The `@foss.global/codefeed` package is designed to help developers generate feeds for code developments, specifically targeting Gitea repositories. It fetches and processes commit data, changelogs, and repository activities for further analysis or visualization. Here, we'll delve into how you can utilize the different features of the `CodeFeed` class.
+
+### Setting Up CodeFeed
+
+To get started, import the `CodeFeed` class from the module:
+
+```typescript
+import { CodeFeed } from '@foss.global/codefeed';
+```
+
+Then, create an instance of `CodeFeed`. You'll need the base URL of your Gitea instance and optionally an API token if your repositories require authentication:
+
+```typescript
+const codeFeed = new CodeFeed('https://your-gitea-instance-url.com', 'your-api-token');
+```
+
+The constructor can also accept a `lastRunTimestamp` which indicates the last time a sync was performed. If not provided, it defaults to 24 hours prior to the current time.
+
+### Fetching Commits
+
+One of the core functionalities of CodeFeed is fetching commits from a Gitea instance. By calling `fetchAllCommitsFromInstance`, you can retrieve commits across multiple repositories:
+
+```typescript
+(async () => {
+  try {
+    const commits = await codeFeed.fetchAllCommitsFromInstance();
+    console.log(commits);
+  } catch (error) {
+    console.error('An error occurred while fetching commits:', error);
+  }
+})();
+```
+
+This method scans all organizations and repositories, filters commits tagged within the last 24 hours, and enriches them with metadata like changelogs or npm publication status.
+
+Each commit object in the resulting array conforms to the `ICommitResult` interface, containing details such as:
+- `baseUrl`
+- `org`
+- `repo`
+- `timestamp`
+- `hash`
+- `commitMessage`
+- `tagged` (boolean)
+- `publishedOnNpm` (boolean)
+- `prettyAgoTime` (human-readable relative time)
+- `changelog` (text from the `changelog.md` associated with a commit)
+
+### Understanding the Data Fetch Process
+
+#### Fetching Organizations
+
+The `fetchAllOrganizations` method collects all organizations within the Gitea instance:
+
+```typescript
+const organizations = await codeFeed.fetchAllOrganizations();
+console.log('Organizations:', organizations);
+```
+
+This method interacts with the Gitea API to pull organization names, aiding further requests that require organization context.
+
+#### Fetching Repositories
+
+Repositories under these organizations can be retrieved using `fetchAllRepositories`:
+
+```typescript
+const repositories = await codeFeed.fetchAllRepositories();
+console.log('Repositories:', repositories);
+```
+
+Here, filtering by organization can help narrow down the scope further when dealing with large instances.
+
+#### Fetching Tags and Commits
+
+To handle repository-specific details, use:
+
+- `fetchTags(owner: string, repo: string)`: Appropriately handles paginated tag data within a repository.
+  
+- `fetchRecentCommitsForRepo(owner: string, repo: string)`: Gathers commit data specific to the past 24 hours for a given repository.
+
+```typescript
+const tags = await codeFeed.fetchTags('orgName', 'repoName');
+const recentCommits = await codeFeed.fetchRecentCommitsForRepo('orgName', 'repoName');
+
+console.log('Tags:', tags);
+console.log('Recent Commits:', recentCommits);
+```
+
+### Changelog Integration
+
+Loading changelog content from a repository is integrated into the flow with `loadChangelogFromRepo`. This can be accessed when processing specific commits:
+
+```typescript
+await codeFeed.loadChangelogFromRepo('org', 'repo');
+const changelog = codeFeed.getChangelogForVersion('1.0.0');
+console.log('Changelog for version 1.0.0:', changelog);
+```
+
+### Reacting to Repository Activity
+
+The method `hasNewActivity` checks for recent changes within an organization or a repository. This is particularly useful for setting up alerting systems or continuous integration triggers:
+
+```typescript
+const hasActivity = await codeFeed.hasNewActivity({ orgName: 'orgName', repoName: 'repoName' });
+console.log('New activity detected:', hasActivity);
+```
+
+### Conclusion
+
+The `@foss.global/codefeed` module provides robust capabilities for extracting and managing feed data related to code developments in Gitea environments. Through systematic setup and leveraging API-driven methods, it becomes a valuable tool for developers aiming to keep track of software progress and changes efficiently. The integration hooks like changelog and npm verification further enrich its utility, offering consolidated insights into each commit's journey from codebase to published package.
+
+Explore integrating these capabilities into your development workflows to enhance tracking, deployment pipelines, or analytics systems within your projects. Remember to always handle API tokens securely and adhere to best practices when managing access to repository resources. Stay updated on any changes or enhancements to this module for further feature exposures or bug fixes. Happy coding!
+```
+undefined

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@foss.global/codefeed',
-  version: '1.5.0',
-  description: 'a module for creating feeds for code development'
+  version: '1.6.1',
+  description: 'The @foss.global/codefeed module is designed for generating feeds from Gitea repositories, enhancing development workflows by processing commit data and repository activities.'
 }

ts/index.ts

@@ -1,12 +1,12 @@
 import * as plugins from './codefeed.plugins.js';
 
-
 export class CodeFeed {
   private baseUrl: string;
   private token?: string;
   private npmRegistry = new plugins.smartnpm.NpmRegistry();
   private smartxmlInstance = new plugins.smartxml.SmartXml();
   private lastRunTimestamp: string;
+  private changelogContent: string;
 
   constructor(baseUrl: string, token?: string, lastRunTimestamp?: string) {
     this.baseUrl = baseUrl;
@@ -16,8 +16,66 @@ export class CodeFeed {
   }
 
   /**
-   * Fetch all organizations from the Gitea instance.
+   * Load the changelog directly from the Gitea repository.
    */
+  private async loadChangelogFromRepo(owner: string, repo: string): Promise<void> {
+    const url = `${this.baseUrl}/api/v1/repos/${owner}/${repo}/contents/changelog.md`;
+    const headers: Record<string, string> = {};
+    if (this.token) {
+      headers['Authorization'] = `token ${this.token}`;
+    }
+
+    const response = await fetch(url, { headers });
+
+    if (!response.ok) {
+      console.error(`Could not fetch CHANGELOG.md from ${owner}/${repo}: ${response.status} ${response.statusText}`);
+      this.changelogContent = '';
+      return;
+    }
+
+    const data = await response.json();
+    if (!data.content) {
+      console.warn(`No content field found in response for ${owner}/${repo}/changelog.md`);
+      this.changelogContent = '';
+      return;
+    }
+
+    const decodedContent = Buffer.from(data.content, 'base64').toString('utf8');
+    this.changelogContent = decodedContent;
+  }
+
+  /**
+   * Parse the changelog to find the entry for a given version.
+   * The changelog format is assumed as:
+   *
+   * # Changelog
+   *
+   * ## <date> - <version> - <description>
+   * <changes...>
+   */
+  private getChangelogForVersion(version: string): string | undefined {
+    if (!this.changelogContent) {
+      return undefined;
+    }
+    const lines = this.changelogContent.split('\n');
+    const versionHeaderIndex = lines.findIndex((line) => line.includes(`- ${version} -`));
+    if (versionHeaderIndex === -1) {
+      return undefined;
+    }
+
+    const changelogLines: string[] = [];
+    for (let i = versionHeaderIndex + 1; i < lines.length; i++) {
+      const line = lines[i];
+      // The next version header starts with `## `
+      if (line.startsWith('## ')) {
+        break;
+      }
+      changelogLines.push(line);
+    }
+
+    return changelogLines.join('\n').trim();
+  }
+
   private async fetchAllOrganizations(): Promise<string[]> {
     const url = `${this.baseUrl}/api/v1/orgs`;
     const response = await fetch(url, {
@@ -32,18 +90,17 @@ export class CodeFeed {
     return data.map((org) => org.username);
   }
 
-  /**
-   * Fetch organization-level activity RSS feed.
-   */
   private async fetchOrgRssFeed(optionsArg: {
     orgName: string,
     repoName?: string,
   }): Promise<any[]> {
-    let rssUrl: string
+    let rssUrl: string;
     if (optionsArg.orgName && !optionsArg.repoName) {
       rssUrl = `${this.baseUrl}/${optionsArg.orgName}.atom`;
     } else if (optionsArg.orgName && optionsArg.repoName) {
       rssUrl = `${this.baseUrl}/${optionsArg.orgName}/${optionsArg.repoName}.atom`;
+    } else {
+      throw new Error('Invalid arguments provided to fetchOrgRssFeed.');
     }
 
     const response = await fetch(rssUrl);
@@ -52,36 +109,25 @@ export class CodeFeed {
     }
 
     const rssText = await response.text();
-
-    // Parse the Atom feed using fast-xml-parser
     const rssData = this.smartxmlInstance.parseXmlToObject(rssText);
-
-    // Return the <entry> elements from the feed
     return rssData.feed.entry || [];
   }
 
-  /**
-   * Check if the organization's RSS feed has any new activities since the last run.
-   */
   private async hasNewActivity(optionsArg: {
     orgName: string,
     repoName?: string,
   }): Promise<boolean> {
     const entries = await this.fetchOrgRssFeed(optionsArg);
 
-    // Filter entries to find new activities since the last run
     return entries.some((entry: any) => {
       const updated = new Date(entry.updated);
       return updated > new Date(this.lastRunTimestamp);
     });
   }
 
-  /**
-   * Fetch all repositories accessible to the token/user.
-   */
-  private async fetchAllRepositories(): Promise<plugins.interfaces.Repository[]> {
+  private async fetchAllRepositories(): Promise<plugins.interfaces.IRepository[]> {
     let page = 1;
-    const allRepos: plugins.interfaces.Repository[] = [];
+    const allRepos: plugins.interfaces.IRepository[] = [];
 
     while (true) {
       const url = new URL(`${this.baseUrl}/api/v1/repos/search`);
@@ -96,7 +142,7 @@ export class CodeFeed {
         throw new Error(`Failed to fetch repositories: ${resp.statusText}`);
       }
 
-      const data: plugins.interfaces.RepoSearchResponse = await resp.json();
+      const data: plugins.interfaces.IRepoSearchResponse = await resp.json();
       allRepos.push(...data.data);
 
       if (data.data.length < 50) {
@@ -108,12 +154,9 @@ export class CodeFeed {
     return allRepos;
   }
 
-  /**
-   * Fetch all tags for a given repository.
-   */
   private async fetchTags(owner: string, repo: string): Promise<Set<string>> {
     let page = 1;
-    const tags: plugins.interfaces.Tag[] = [];
+    const tags: plugins.interfaces.ITag[] = [];
 
     while (true) {
       const url = new URL(`${this.baseUrl}/api/v1/repos/${owner}/${repo}/tags`);
@@ -129,7 +172,7 @@ export class CodeFeed {
         throw new Error(`Failed to fetch tags for ${owner}/${repo}: ${resp.statusText}`);
       }
 
-      const data: plugins.interfaces.Tag[] = await resp.json();
+      const data: plugins.interfaces.ITag[] = await resp.json();
       tags.push(...data);
 
       if (data.length < 50) {
@@ -148,13 +191,10 @@ export class CodeFeed {
     return taggedCommitShas;
   }
 
-  /**
-   * Fetch commits from the last 24 hours for a repository.
-   */
-  private async fetchRecentCommitsForRepo(owner: string, repo: string): Promise<plugins.interfaces.Commit[]> {
+  private async fetchRecentCommitsForRepo(owner: string, repo: string): Promise<plugins.interfaces.ICommit[]> {
     const twentyFourHoursAgo = new Date(Date.now() - 24 * 60 * 60 * 1000);
     let page = 1;
-    const recentCommits: plugins.interfaces.Commit[] = [];
+    const recentCommits: plugins.interfaces.ICommit[] = [];
 
     while (true) {
       const url = new URL(`${this.baseUrl}/api/v1/repos/${owner}/${repo}/commits`);
@@ -170,7 +210,7 @@ export class CodeFeed {
         throw new Error(`Failed to fetch commits for ${owner}/${repo}: ${resp.statusText}`);
       }
 
-      const data: plugins.interfaces.Commit[] = await resp.json();
+      const data: plugins.interfaces.ICommit[] = await resp.json();
       if (data.length === 0) {
         break;
       }
@@ -190,13 +230,10 @@ export class CodeFeed {
     return recentCommits;
   }
 
-  /**
-   * Fetch all commits by querying all organizations.
-   */
-  public async fetchAllCommitsFromInstance(): Promise<plugins.interfaces.CommitResult[]> {
+  public async fetchAllCommitsFromInstance(): Promise<plugins.interfaces.ICommitResult[]> {
     const orgs = await this.fetchAllOrganizations();
     console.log(`Found ${orgs.length} organizations`);
-    let allCommits: plugins.interfaces.CommitResult[] = [];
+    let allCommits: plugins.interfaces.ICommitResult[] = [];
 
     for (const orgName of orgs) {
       console.log(`Checking activity for organization: ${orgName}`);
@@ -209,7 +246,7 @@ export class CodeFeed {
           console.log(`No new activity for organization: ${orgName}`);
           continue;
         }
-      } catch (error) {
+      } catch (error: any) {
         console.error(`Error fetching activity for organization ${orgName}:`, error.message);
         continue;
       }
@@ -227,10 +264,11 @@ export class CodeFeed {
             console.log(`No new activity for repository: ${orgName}/${r.name}`);
             continue;
           }
-        } catch (error) {
+        } catch (error: any) {
           console.error(`Error fetching activity for repository ${orgName}/${r.name}:`, error.message);
           continue;
         }
+
         const org = r.owner.login;
         const repo = r.name;
         console.log(`Processing repository ${org}/${repo}`);
@@ -239,8 +277,11 @@ export class CodeFeed {
           const taggedCommitShas = await this.fetchTags(org, repo);
           const commits = await this.fetchRecentCommitsForRepo(org, repo);
 
+          // Load the changelog from this repo.
+          await this.loadChangelogFromRepo(org, repo);
+
           const commitResults = commits.map((c) => {
-            const commit: plugins.interfaces.CommitResult = {
+            const commit: plugins.interfaces.ICommitResult = {
               baseUrl: this.baseUrl,
               org,
               repo,
@@ -250,42 +291,55 @@ export class CodeFeed {
               commitMessage: c.commit.message,
               tagged: taggedCommitShas.has(c.sha),
               publishedOnNpm: false,
-            }
+              changelog: undefined
+            };
             return commit;
           });
 
           if (commitResults.length > 0) {
             try {
               const packageInfo = await this.npmRegistry.getPackageInfo(`@${org}/${repo}`);
-              for (const commit of commitResults.filter((c) => c.tagged)) {
+              for (const commitResult of commitResults.filter((c) => c.tagged)) {
+                const versionCandidate = commitResult.commitMessage.replace('\n', '').trim();
                 const correspondingVersion = packageInfo.allVersions.find((versionArg) => {
-                  return versionArg.version === commit.commitMessage.replace('\n', '');
+                  return versionArg.version === versionCandidate;
                 });
                 if (correspondingVersion) {
-                  commit.publishedOnNpm = true;
+                  commitResult.publishedOnNpm = true;
+                  const changelogEntry = this.getChangelogForVersion(versionCandidate);
+                  if (changelogEntry) {
+                    commitResult.changelog = changelogEntry;
                   }
                 }
-            } catch (error) {
+              }
+            } catch (error: any) {
               console.error(`Failed to fetch package info for ${org}/${repo}:`, error.message);
             }
           }
 
           allCommits.push(...commitResults);
-        } catch (error) {
+        } catch (error: any) {
           console.error(`Skipping repository ${org}/${repo} due to error:`, error.message);
         }
       }
     }
 
     console.log(`Processed ${allCommits.length} commits in total.`);
+
+    allCommits = allCommits.filter(commitArg => commitArg.tagged);
+
+    console.log(`Filtered to ${allCommits.length} commits with tagged statuses.`);
+
     for (const c of allCommits) {
-      console.log(`______________________________________________________
+      console.log(`      ==========================================================================
       ${c.prettyAgoTime} ago:
-      Commit ${c.hash} by ${c.org}/${c.repo} at ${c.timestamp}
+      ${c.org}/${c.repo}
       ${c.commitMessage}
       Published on npm: ${c.publishedOnNpm}
+      ${c.changelog ? `Changelog:\n${c.changelog}\n` : ''}
     `);  
     }
+
     return allCommits;
   }
 }

ts/interfaces/index.ts

@@ -1,37 +1,37 @@
-export interface RepositoryOwner {
+export interface IRepositoryOwner {
   login: string;
 }
 
-export interface Repository {
-  owner: RepositoryOwner;
+export interface IRepository {
+  owner: IRepositoryOwner;
   name: string;
 }
 
-export interface CommitAuthor {
+export interface ICommitAuthor {
   date: string;
 }
 
-export interface CommitDetail {
+export interface ICommitDetail {
   message: string;
-  author: CommitAuthor;
+  author: ICommitAuthor;
 }
 
-export interface Commit {
+export interface ICommit {
   sha: string;
-  commit: CommitDetail;
+  commit: ICommitDetail;
 }
 
-export interface Tag {
+export interface ITag {
   commit?: {
     sha?: string;
   };
 }
 
-export interface RepoSearchResponse {
-  data: Repository[];
+export interface IRepoSearchResponse {
+  data: IRepository[];
 }
 
-export interface CommitResult {
+export interface ICommitResult {
   baseUrl: string;
   org: string;
   repo: string;
@@ -41,4 +41,5 @@ export interface CommitResult {
   tagged: boolean;
   publishedOnNpm: boolean;
   prettyAgoTime: string;
+  changelog: string | undefined;
 }