foss.global/codefeed
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(core): Add changelog fetching and parsing functionality

Browse Source
This commit is contained in:
Philipp Kunz 2024-12-14 00:54:38 +01:00
parent e843197211
commit bb248ed408
4 changed files with 119 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
changelog.md
View File

@ -1,5 +1,12 @@
# Changelog
## 2024-12-14 - 1.6.0 - feat(core)
Add changelog fetching and parsing functionality
- Implemented loadChangelogFromRepo to directly load the changelog from a Gitea repository.
- Introduced parsing functionality to extract specific version details from the loaded changelog.
- Updated CodeFeed class to utilize the changelog for version verification and commit processing.
## 2024-12-14 - 1.5.3 - fix(core)
Fix filtering logic for returning only tagged commits

2
ts/00_commitinfo_data.ts
View File

@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@foss.global/codefeed',
version: '1.5.3',
version: '1.6.0',
description: 'a module for creating feeds for code development'
}

145
ts/index.ts
View File

@ -1,12 +1,12 @@
import * as plugins from './codefeed.plugins.js';
export class CodeFeed {
private baseUrl: string;
private token?: string;
private npmRegistry = new plugins.smartnpm.NpmRegistry();
private smartxmlInstance = new plugins.smartxml.SmartXml();
private lastRunTimestamp: string;
private changelogContent: string;
constructor(baseUrl: string, token?: string, lastRunTimestamp?: string) {
this.baseUrl = baseUrl;
@ -16,8 +16,66 @@ export class CodeFeed {
}
/**
* Fetch all organizations from the Gitea instance.
* Load the changelog directly from the Gitea repository.
*/
private async loadChangelogFromRepo(owner: string, repo: string): Promise<void> {
const url = `${this.baseUrl}/api/v1/repos/${owner}/${repo}/contents/changelog.md`;
const headers: Record<string, string> = {};
if (this.token) {
headers['Authorization'] = `token ${this.token}`;
}
const response = await fetch(url, { headers });
if (!response.ok) {
console.error(`Could not fetch CHANGELOG.md from ${owner}/${repo}: ${response.status} ${response.statusText}`);
this.changelogContent = '';
return;
}
const data = await response.json();
if (!data.content) {
console.warn(`No content field found in response for ${owner}/${repo}/changelog.md`);
this.changelogContent = '';
return;
}
const decodedContent = Buffer.from(data.content, 'base64').toString('utf8');
this.changelogContent = decodedContent;
}
/**
* Parse the changelog to find the entry for a given version.
* The changelog format is assumed as:
*
* # Changelog
*
* ## <date> - <version> - <description>
* <changes...>
*/
private getChangelogForVersion(version: string): string | undefined {
if (!this.changelogContent) {
return undefined;
}
const lines = this.changelogContent.split('\n');
const versionHeaderIndex = lines.findIndex((line) => line.includes(`- ${version} -`));
if (versionHeaderIndex === -1) {
return undefined;
}
const changelogLines: string[] = [];
for (let i = versionHeaderIndex + 1; i < lines.length; i++) {
const line = lines[i];
// The next version header starts with `## `
if (line.startsWith('## ')) {
break;
}
changelogLines.push(line);
}
return changelogLines.join('\n').trim();
}
private async fetchAllOrganizations(): Promise<string[]> {
const url = `${this.baseUrl}/api/v1/orgs`;
const response = await fetch(url, {
@ -32,18 +90,17 @@ export class CodeFeed {
return data.map((org) => org.username);
}
/**
* Fetch organization-level activity RSS feed.
*/
private async fetchOrgRssFeed(optionsArg: {
orgName: string,
repoName?: string,
}): Promise<any[]> {
let rssUrl: string
let rssUrl: string;
if (optionsArg.orgName && !optionsArg.repoName) {
rssUrl = `${this.baseUrl}/${optionsArg.orgName}.atom`;
} else if (optionsArg.orgName && optionsArg.repoName) {
rssUrl = `${this.baseUrl}/${optionsArg.orgName}/${optionsArg.repoName}.atom`;
} else {
throw new Error('Invalid arguments provided to fetchOrgRssFeed.');
}
const response = await fetch(rssUrl);
@ -52,36 +109,25 @@ export class CodeFeed {
}
const rssText = await response.text();
// Parse the Atom feed using fast-xml-parser
const rssData = this.smartxmlInstance.parseXmlToObject(rssText);
// Return the <entry> elements from the feed
return rssData.feed.entry || [];
}
/**
* Check if the organization's RSS feed has any new activities since the last run.
*/
private async hasNewActivity(optionsArg: {
orgName: string,
repoName?: string,
}): Promise<boolean> {
const entries = await this.fetchOrgRssFeed(optionsArg);
// Filter entries to find new activities since the last run
return entries.some((entry: any) => {
const updated = new Date(entry.updated);
return updated > new Date(this.lastRunTimestamp);
});
}
/**
* Fetch all repositories accessible to the token/user.
*/
private async fetchAllRepositories(): Promise<plugins.interfaces.Repository[]> {
private async fetchAllRepositories(): Promise<plugins.interfaces.IRepository[]> {
let page = 1;
const allRepos: plugins.interfaces.Repository[] = [];
const allRepos: plugins.interfaces.IRepository[] = [];
while (true) {
const url = new URL(`${this.baseUrl}/api/v1/repos/search`);
@ -96,7 +142,7 @@ export class CodeFeed {
throw new Error(`Failed to fetch repositories: ${resp.statusText}`);
}
const data: plugins.interfaces.RepoSearchResponse = await resp.json();
const data: plugins.interfaces.IRepoSearchResponse = await resp.json();
allRepos.push(...data.data);
if (data.data.length < 50) {
@ -108,12 +154,9 @@ export class CodeFeed {
return allRepos;
}
/**
* Fetch all tags for a given repository.
*/
private async fetchTags(owner: string, repo: string): Promise<Set<string>> {
let page = 1;
const tags: plugins.interfaces.Tag[] = [];
const tags: plugins.interfaces.ITag[] = [];
while (true) {
const url = new URL(`${this.baseUrl}/api/v1/repos/${owner}/${repo}/tags`);
@ -129,7 +172,7 @@ export class CodeFeed {
throw new Error(`Failed to fetch tags for ${owner}/${repo}: ${resp.statusText}`);
}
const data: plugins.interfaces.Tag[] = await resp.json();
const data: plugins.interfaces.ITag[] = await resp.json();
tags.push(...data);
if (data.length < 50) {
@ -148,13 +191,10 @@ export class CodeFeed {
return taggedCommitShas;
}
/**
* Fetch commits from the last 24 hours for a repository.
*/
private async fetchRecentCommitsForRepo(owner: string, repo: string): Promise<plugins.interfaces.Commit[]> {
private async fetchRecentCommitsForRepo(owner: string, repo: string): Promise<plugins.interfaces.ICommit[]> {
const twentyFourHoursAgo = new Date(Date.now() - 24 * 60 * 60 * 1000);
let page = 1;
const recentCommits: plugins.interfaces.Commit[] = [];
const recentCommits: plugins.interfaces.ICommit[] = [];
while (true) {
const url = new URL(`${this.baseUrl}/api/v1/repos/${owner}/${repo}/commits`);
@ -170,7 +210,7 @@ export class CodeFeed {
throw new Error(`Failed to fetch commits for ${owner}/${repo}: ${resp.statusText}`);
}
const data: plugins.interfaces.Commit[] = await resp.json();
const data: plugins.interfaces.ICommit[] = await resp.json();
if (data.length === 0) {
break;
}
@ -190,13 +230,10 @@ export class CodeFeed {
return recentCommits;
}
/**
* Fetch all commits by querying all organizations.
*/
public async fetchAllCommitsFromInstance(): Promise<plugins.interfaces.CommitResult[]> {
public async fetchAllCommitsFromInstance(): Promise<plugins.interfaces.ICommitResult[]> {
const orgs = await this.fetchAllOrganizations();
console.log(`Found ${orgs.length} organizations`);
let allCommits: plugins.interfaces.CommitResult[] = [];
let allCommits: plugins.interfaces.ICommitResult[] = [];
for (const orgName of orgs) {
console.log(`Checking activity for organization: ${orgName}`);
@ -209,7 +246,7 @@ export class CodeFeed {
console.log(`No new activity for organization: ${orgName}`);
continue;
}
} catch (error) {
} catch (error: any) {
console.error(`Error fetching activity for organization ${orgName}:`, error.message);
continue;
}
@ -227,10 +264,11 @@ export class CodeFeed {
console.log(`No new activity for repository: ${orgName}/${r.name}`);
continue;
}
} catch (error) {
} catch (error: any) {
console.error(`Error fetching activity for repository ${orgName}/${r.name}:`, error.message);
continue;
}
const org = r.owner.login;
const repo = r.name;
console.log(`Processing repository ${org}/${repo}`);
@ -239,8 +277,11 @@ export class CodeFeed {
const taggedCommitShas = await this.fetchTags(org, repo);
const commits = await this.fetchRecentCommitsForRepo(org, repo);
// Load the changelog from this repo.
await this.loadChangelogFromRepo(org, repo);
const commitResults = commits.map((c) => {
const commit: plugins.interfaces.CommitResult = {
const commit: plugins.interfaces.ICommitResult = {
baseUrl: this.baseUrl,
org,
repo,
@ -250,45 +291,55 @@ export class CodeFeed {
commitMessage: c.commit.message,
tagged: taggedCommitShas.has(c.sha),
publishedOnNpm: false,
}
changelog: undefined
};
return commit;
});
if (commitResults.length > 0) {
try {
const packageInfo = await this.npmRegistry.getPackageInfo(`@${org}/${repo}`);
for (const commit of commitResults.filter((c) => c.tagged)) {
for (const commitResult of commitResults.filter((c) => c.tagged)) {
const versionCandidate = commitResult.commitMessage.replace('\n', '').trim();
const correspondingVersion = packageInfo.allVersions.find((versionArg) => {
return versionArg.version === commit.commitMessage.replace('\n', '');
return versionArg.version === versionCandidate;
});
if (correspondingVersion) {
commit.publishedOnNpm = true;
commitResult.publishedOnNpm = true;
const changelogEntry = this.getChangelogForVersion(versionCandidate);
if (changelogEntry) {
commitResult.changelog = changelogEntry;
}
}
}
} catch (error) {
} catch (error: any) {
console.error(`Failed to fetch package info for ${org}/${repo}:`, error.message);
}
}
allCommits.push(...commitResults);
} catch (error) {
} catch (error: any) {
console.error(`Skipping repository ${org}/${repo} due to error:`, error.message);
}
}
}
console.log(`Processed ${allCommits.length} commits in total.`);
allCommits = allCommits.filter(commitArg => commitArg.tagged);
console.log(`Filtered to ${allCommits.length} commits with tagged statuses.`);
for (const c of allCommits) {
console.log(` ==========================================================================
${c.prettyAgoTime} ago:
${c.org}/${c.repo}
${c.commitMessage}
Published on npm: ${c.publishedOnNpm}
${c.changelog ? `Changelog:\n${c.changelog}\n` : ''}
`);
}
allCommits = allCommits.filter(commitArg => commitArg.tagged);
return allCommits;
}
}

25
ts/interfaces/index.ts
View File

@ -1,37 +1,37 @@
export interface RepositoryOwner {
export interface IRepositoryOwner {
login: string;
}
export interface Repository {
owner: RepositoryOwner;
export interface IRepository {
owner: IRepositoryOwner;
name: string;
}
export interface CommitAuthor {
export interface ICommitAuthor {
date: string;
}
export interface CommitDetail {
export interface ICommitDetail {
message: string;
author: CommitAuthor;
author: ICommitAuthor;
}
export interface Commit {
export interface ICommit {
sha: string;
commit: CommitDetail;
commit: ICommitDetail;
}
export interface Tag {
export interface ITag {
commit?: {
sha?: string;
};
}
export interface RepoSearchResponse {
data: Repository[];
export interface IRepoSearchResponse {
data: IRepository[];
}
export interface CommitResult {
export interface ICommitResult {
baseUrl: string;
org: string;
repo: string;
@ -41,4 +41,5 @@ export interface CommitResult {
tagged: boolean;
publishedOnNpm: boolean;
prettyAgoTime: string;
changelog: string | undefined;
}