From bb248ed408189749adf1aa949c92a7f465f8cfba Mon Sep 17 00:00:00 2001 From: Philipp Kunz Date: Sat, 14 Dec 2024 00:54:38 +0100 Subject: [PATCH] feat(core): Add changelog fetching and parsing functionality --- changelog.md | 7 ++ ts/00_commitinfo_data.ts | 2 +- ts/index.ts | 145 ++++++++++++++++++++++++++------------- ts/interfaces/index.ts | 25 +++---- 4 files changed, 119 insertions(+), 60 deletions(-) diff --git a/changelog.md b/changelog.md index 34689a9..f2330ee 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,12 @@ # Changelog +## 2024-12-14 - 1.6.0 - feat(core) +Add changelog fetching and parsing functionality + +- Implemented loadChangelogFromRepo to directly load the changelog from a Gitea repository. +- Introduced parsing functionality to extract specific version details from the loaded changelog. +- Updated CodeFeed class to utilize the changelog for version verification and commit processing. + ## 2024-12-14 - 1.5.3 - fix(core) Fix filtering logic for returning only tagged commits diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index 2276768..aff4138 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@foss.global/codefeed', - version: '1.5.3', + version: '1.6.0', description: 'a module for creating feeds for code development' } diff --git a/ts/index.ts b/ts/index.ts index f476c46..31830c8 100644 --- a/ts/index.ts +++ b/ts/index.ts @@ -1,12 +1,12 @@ import * as plugins from './codefeed.plugins.js'; - export class CodeFeed { private baseUrl: string; private token?: string; private npmRegistry = new plugins.smartnpm.NpmRegistry(); private smartxmlInstance = new plugins.smartxml.SmartXml(); private lastRunTimestamp: string; + private changelogContent: string; constructor(baseUrl: string, token?: string, lastRunTimestamp?: string) { this.baseUrl = baseUrl; @@ -16,8 +16,66 @@ export class CodeFeed { } /** - * Fetch all organizations from the Gitea instance. + * Load the changelog directly from the Gitea repository. */ + private async loadChangelogFromRepo(owner: string, repo: string): Promise { + const url = `${this.baseUrl}/api/v1/repos/${owner}/${repo}/contents/changelog.md`; + const headers: Record = {}; + if (this.token) { + headers['Authorization'] = `token ${this.token}`; + } + + const response = await fetch(url, { headers }); + + if (!response.ok) { + console.error(`Could not fetch CHANGELOG.md from ${owner}/${repo}: ${response.status} ${response.statusText}`); + this.changelogContent = ''; + return; + } + + const data = await response.json(); + if (!data.content) { + console.warn(`No content field found in response for ${owner}/${repo}/changelog.md`); + this.changelogContent = ''; + return; + } + + const decodedContent = Buffer.from(data.content, 'base64').toString('utf8'); + this.changelogContent = decodedContent; + } + + /** + * Parse the changelog to find the entry for a given version. + * The changelog format is assumed as: + * + * # Changelog + * + * ## - - + * + */ + private getChangelogForVersion(version: string): string | undefined { + if (!this.changelogContent) { + return undefined; + } + const lines = this.changelogContent.split('\n'); + const versionHeaderIndex = lines.findIndex((line) => line.includes(`- ${version} -`)); + if (versionHeaderIndex === -1) { + return undefined; + } + + const changelogLines: string[] = []; + for (let i = versionHeaderIndex + 1; i < lines.length; i++) { + const line = lines[i]; + // The next version header starts with `## ` + if (line.startsWith('## ')) { + break; + } + changelogLines.push(line); + } + + return changelogLines.join('\n').trim(); + } + private async fetchAllOrganizations(): Promise { const url = `${this.baseUrl}/api/v1/orgs`; const response = await fetch(url, { @@ -32,18 +90,17 @@ export class CodeFeed { return data.map((org) => org.username); } - /** - * Fetch organization-level activity RSS feed. - */ private async fetchOrgRssFeed(optionsArg: { orgName: string, repoName?: string, }): Promise { - let rssUrl: string + let rssUrl: string; if (optionsArg.orgName && !optionsArg.repoName) { rssUrl = `${this.baseUrl}/${optionsArg.orgName}.atom`; } else if (optionsArg.orgName && optionsArg.repoName) { rssUrl = `${this.baseUrl}/${optionsArg.orgName}/${optionsArg.repoName}.atom`; + } else { + throw new Error('Invalid arguments provided to fetchOrgRssFeed.'); } const response = await fetch(rssUrl); @@ -52,36 +109,25 @@ export class CodeFeed { } const rssText = await response.text(); - - // Parse the Atom feed using fast-xml-parser const rssData = this.smartxmlInstance.parseXmlToObject(rssText); - - // Return the elements from the feed return rssData.feed.entry || []; } - /** - * Check if the organization's RSS feed has any new activities since the last run. - */ private async hasNewActivity(optionsArg: { orgName: string, repoName?: string, }): Promise { const entries = await this.fetchOrgRssFeed(optionsArg); - // Filter entries to find new activities since the last run return entries.some((entry: any) => { const updated = new Date(entry.updated); return updated > new Date(this.lastRunTimestamp); }); } - /** - * Fetch all repositories accessible to the token/user. - */ - private async fetchAllRepositories(): Promise { + private async fetchAllRepositories(): Promise { let page = 1; - const allRepos: plugins.interfaces.Repository[] = []; + const allRepos: plugins.interfaces.IRepository[] = []; while (true) { const url = new URL(`${this.baseUrl}/api/v1/repos/search`); @@ -96,7 +142,7 @@ export class CodeFeed { throw new Error(`Failed to fetch repositories: ${resp.statusText}`); } - const data: plugins.interfaces.RepoSearchResponse = await resp.json(); + const data: plugins.interfaces.IRepoSearchResponse = await resp.json(); allRepos.push(...data.data); if (data.data.length < 50) { @@ -108,12 +154,9 @@ export class CodeFeed { return allRepos; } - /** - * Fetch all tags for a given repository. - */ private async fetchTags(owner: string, repo: string): Promise> { let page = 1; - const tags: plugins.interfaces.Tag[] = []; + const tags: plugins.interfaces.ITag[] = []; while (true) { const url = new URL(`${this.baseUrl}/api/v1/repos/${owner}/${repo}/tags`); @@ -129,7 +172,7 @@ export class CodeFeed { throw new Error(`Failed to fetch tags for ${owner}/${repo}: ${resp.statusText}`); } - const data: plugins.interfaces.Tag[] = await resp.json(); + const data: plugins.interfaces.ITag[] = await resp.json(); tags.push(...data); if (data.length < 50) { @@ -148,13 +191,10 @@ export class CodeFeed { return taggedCommitShas; } - /** - * Fetch commits from the last 24 hours for a repository. - */ - private async fetchRecentCommitsForRepo(owner: string, repo: string): Promise { + private async fetchRecentCommitsForRepo(owner: string, repo: string): Promise { const twentyFourHoursAgo = new Date(Date.now() - 24 * 60 * 60 * 1000); let page = 1; - const recentCommits: plugins.interfaces.Commit[] = []; + const recentCommits: plugins.interfaces.ICommit[] = []; while (true) { const url = new URL(`${this.baseUrl}/api/v1/repos/${owner}/${repo}/commits`); @@ -170,7 +210,7 @@ export class CodeFeed { throw new Error(`Failed to fetch commits for ${owner}/${repo}: ${resp.statusText}`); } - const data: plugins.interfaces.Commit[] = await resp.json(); + const data: plugins.interfaces.ICommit[] = await resp.json(); if (data.length === 0) { break; } @@ -190,13 +230,10 @@ export class CodeFeed { return recentCommits; } - /** - * Fetch all commits by querying all organizations. - */ - public async fetchAllCommitsFromInstance(): Promise { + public async fetchAllCommitsFromInstance(): Promise { const orgs = await this.fetchAllOrganizations(); console.log(`Found ${orgs.length} organizations`); - let allCommits: plugins.interfaces.CommitResult[] = []; + let allCommits: plugins.interfaces.ICommitResult[] = []; for (const orgName of orgs) { console.log(`Checking activity for organization: ${orgName}`); @@ -209,7 +246,7 @@ export class CodeFeed { console.log(`No new activity for organization: ${orgName}`); continue; } - } catch (error) { + } catch (error: any) { console.error(`Error fetching activity for organization ${orgName}:`, error.message); continue; } @@ -227,10 +264,11 @@ export class CodeFeed { console.log(`No new activity for repository: ${orgName}/${r.name}`); continue; } - } catch (error) { + } catch (error: any) { console.error(`Error fetching activity for repository ${orgName}/${r.name}:`, error.message); continue; } + const org = r.owner.login; const repo = r.name; console.log(`Processing repository ${org}/${repo}`); @@ -239,8 +277,11 @@ export class CodeFeed { const taggedCommitShas = await this.fetchTags(org, repo); const commits = await this.fetchRecentCommitsForRepo(org, repo); + // Load the changelog from this repo. + await this.loadChangelogFromRepo(org, repo); + const commitResults = commits.map((c) => { - const commit: plugins.interfaces.CommitResult = { + const commit: plugins.interfaces.ICommitResult = { baseUrl: this.baseUrl, org, repo, @@ -250,45 +291,55 @@ export class CodeFeed { commitMessage: c.commit.message, tagged: taggedCommitShas.has(c.sha), publishedOnNpm: false, - } + changelog: undefined + }; return commit; }); if (commitResults.length > 0) { try { const packageInfo = await this.npmRegistry.getPackageInfo(`@${org}/${repo}`); - for (const commit of commitResults.filter((c) => c.tagged)) { + for (const commitResult of commitResults.filter((c) => c.tagged)) { + const versionCandidate = commitResult.commitMessage.replace('\n', '').trim(); const correspondingVersion = packageInfo.allVersions.find((versionArg) => { - return versionArg.version === commit.commitMessage.replace('\n', ''); + return versionArg.version === versionCandidate; }); if (correspondingVersion) { - commit.publishedOnNpm = true; + commitResult.publishedOnNpm = true; + const changelogEntry = this.getChangelogForVersion(versionCandidate); + if (changelogEntry) { + commitResult.changelog = changelogEntry; + } } } - } catch (error) { + } catch (error: any) { console.error(`Failed to fetch package info for ${org}/${repo}:`, error.message); } } allCommits.push(...commitResults); - } catch (error) { + } catch (error: any) { console.error(`Skipping repository ${org}/${repo} due to error:`, error.message); } } } console.log(`Processed ${allCommits.length} commits in total.`); + + allCommits = allCommits.filter(commitArg => commitArg.tagged); + + console.log(`Filtered to ${allCommits.length} commits with tagged statuses.`); + for (const c of allCommits) { console.log(` ========================================================================== ${c.prettyAgoTime} ago: ${c.org}/${c.repo} ${c.commitMessage} Published on npm: ${c.publishedOnNpm} + ${c.changelog ? `Changelog:\n${c.changelog}\n` : ''} `); } - allCommits = allCommits.filter(commitArg => commitArg.tagged); - return allCommits; } } \ No newline at end of file diff --git a/ts/interfaces/index.ts b/ts/interfaces/index.ts index ff7b6ee..fd539ff 100644 --- a/ts/interfaces/index.ts +++ b/ts/interfaces/index.ts @@ -1,37 +1,37 @@ -export interface RepositoryOwner { +export interface IRepositoryOwner { login: string; } -export interface Repository { - owner: RepositoryOwner; +export interface IRepository { + owner: IRepositoryOwner; name: string; } -export interface CommitAuthor { +export interface ICommitAuthor { date: string; } -export interface CommitDetail { +export interface ICommitDetail { message: string; - author: CommitAuthor; + author: ICommitAuthor; } -export interface Commit { +export interface ICommit { sha: string; - commit: CommitDetail; + commit: ICommitDetail; } -export interface Tag { +export interface ITag { commit?: { sha?: string; }; } -export interface RepoSearchResponse { - data: Repository[]; +export interface IRepoSearchResponse { + data: IRepository[]; } -export interface CommitResult { +export interface ICommitResult { baseUrl: string; org: string; repo: string; @@ -41,4 +41,5 @@ export interface CommitResult { tagged: boolean; publishedOnNpm: boolean; prettyAgoTime: string; + changelog: string | undefined; }