import * as plugins from './codefeed.plugins.js'; export class CodeFeed { private baseUrl: string; private token?: string; private npmRegistry = new plugins.smartnpm.NpmRegistry(); private smartxmlInstance = new plugins.smartxml.SmartXml(); private lastRunTimestamp: string; private changelogContent: string; constructor(baseUrl: string, token?: string, lastRunTimestamp?: string) { this.baseUrl = baseUrl; this.token = token; this.lastRunTimestamp = lastRunTimestamp || new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString(); console.log('CodeFeed initialized with last run timestamp:', this.lastRunTimestamp); } /** * Load the changelog directly from the Gitea repository. */ private async loadChangelogFromRepo(owner: string, repo: string): Promise { const url = `${this.baseUrl}/api/v1/repos/${owner}/${repo}/contents/changelog.md`; const headers: Record = {}; if (this.token) { headers['Authorization'] = `token ${this.token}`; } const response = await fetch(url, { headers }); if (!response.ok) { console.error(`Could not fetch CHANGELOG.md from ${owner}/${repo}: ${response.status} ${response.statusText}`); this.changelogContent = ''; return; } const data = await response.json(); if (!data.content) { console.warn(`No content field found in response for ${owner}/${repo}/changelog.md`); this.changelogContent = ''; return; } const decodedContent = Buffer.from(data.content, 'base64').toString('utf8'); this.changelogContent = decodedContent; } /** * Parse the changelog to find the entry for a given version. * The changelog format is assumed as: * * # Changelog * * ## - - * */ private getChangelogForVersion(version: string): string | undefined { if (!this.changelogContent) { return undefined; } const lines = this.changelogContent.split('\n'); const versionHeaderIndex = lines.findIndex((line) => line.includes(`- ${version} -`)); if (versionHeaderIndex === -1) { return undefined; } const changelogLines: string[] = []; for (let i = versionHeaderIndex + 1; i < lines.length; i++) { const line = lines[i]; // The next version header starts with `## ` if (line.startsWith('## ')) { break; } changelogLines.push(line); } return changelogLines.join('\n').trim(); } private async fetchAllOrganizations(): Promise { const url = `${this.baseUrl}/api/v1/orgs`; const response = await fetch(url, { headers: this.token ? { Authorization: `token ${this.token}` } : {}, }); if (!response.ok) { throw new Error(`Failed to fetch organizations: ${response.statusText}`); } const data: { username: string }[] = await response.json(); return data.map((org) => org.username); } private async fetchOrgRssFeed(optionsArg: { orgName: string, repoName?: string, }): Promise { let rssUrl: string; if (optionsArg.orgName && !optionsArg.repoName) { rssUrl = `${this.baseUrl}/${optionsArg.orgName}.atom`; } else if (optionsArg.orgName && optionsArg.repoName) { rssUrl = `${this.baseUrl}/${optionsArg.orgName}/${optionsArg.repoName}.atom`; } else { throw new Error('Invalid arguments provided to fetchOrgRssFeed.'); } const response = await fetch(rssUrl); if (!response.ok) { throw new Error(`Failed to fetch RSS feed for organization ${optionsArg.orgName}/${optionsArg.repoName}: ${response.statusText}`); } const rssText = await response.text(); const rssData = this.smartxmlInstance.parseXmlToObject(rssText); return rssData.feed.entry || []; } private async hasNewActivity(optionsArg: { orgName: string, repoName?: string, }): Promise { const entries = await this.fetchOrgRssFeed(optionsArg); return entries.some((entry: any) => { const updated = new Date(entry.updated); return updated > new Date(this.lastRunTimestamp); }); } private async fetchAllRepositories(): Promise { let page = 1; const allRepos: plugins.interfaces.IRepository[] = []; while (true) { const url = new URL(`${this.baseUrl}/api/v1/repos/search`); url.searchParams.set('limit', '50'); url.searchParams.set('page', page.toString()); const resp = await fetch(url.href, { headers: this.token ? { 'Authorization': `token ${this.token}` } : {}, }); if (!resp.ok) { throw new Error(`Failed to fetch repositories: ${resp.statusText}`); } const data: plugins.interfaces.IRepoSearchResponse = await resp.json(); allRepos.push(...data.data); if (data.data.length < 50) { break; } page++; } return allRepos; } private async fetchTags(owner: string, repo: string): Promise> { let page = 1; const tags: plugins.interfaces.ITag[] = []; while (true) { const url = new URL(`${this.baseUrl}/api/v1/repos/${owner}/${repo}/tags`); url.searchParams.set('limit', '50'); url.searchParams.set('page', page.toString()); const resp = await fetch(url.href, { headers: this.token ? { 'Authorization': `token ${this.token}` } : {}, }); if (!resp.ok) { console.error(`Failed to fetch tags for ${owner}/${repo}: ${resp.status} ${resp.statusText} at ${url.href}`); throw new Error(`Failed to fetch tags for ${owner}/${repo}: ${resp.statusText}`); } const data: plugins.interfaces.ITag[] = await resp.json(); tags.push(...data); if (data.length < 50) { break; } page++; } const taggedCommitShas = new Set(); for (const t of tags) { if (t.commit?.sha) { taggedCommitShas.add(t.commit.sha); } } return taggedCommitShas; } private async fetchRecentCommitsForRepo(owner: string, repo: string): Promise { const twentyFourHoursAgo = new Date(Date.now() - 24 * 60 * 60 * 1000); let page = 1; const recentCommits: plugins.interfaces.ICommit[] = []; while (true) { const url = new URL(`${this.baseUrl}/api/v1/repos/${owner}/${repo}/commits`); url.searchParams.set('limit', '50'); url.searchParams.set('page', page.toString()); const resp = await fetch(url.href, { headers: this.token ? { 'Authorization': `token ${this.token}` } : {}, }); if (!resp.ok) { console.error(`Failed to fetch commits for ${owner}/${repo}: ${resp.status} ${resp.statusText} at ${url.href}`); throw new Error(`Failed to fetch commits for ${owner}/${repo}: ${resp.statusText}`); } const data: plugins.interfaces.ICommit[] = await resp.json(); if (data.length === 0) { break; } for (const commit of data) { const commitDate = new Date(commit.commit.author.date); if (commitDate > twentyFourHoursAgo) { recentCommits.push(commit); } else { return recentCommits; } } page++; } return recentCommits; } public async fetchAllCommitsFromInstance(): Promise { const orgs = await this.fetchAllOrganizations(); console.log(`Found ${orgs.length} organizations`); let allCommits: plugins.interfaces.ICommitResult[] = []; for (const orgName of orgs) { console.log(`Checking activity for organization: ${orgName}`); try { const hasActivity = await this.hasNewActivity({ orgName, }); if (!hasActivity) { console.log(`No new activity for organization: ${orgName}`); continue; } } catch (error: any) { console.error(`Error fetching activity for organization ${orgName}:`, error.message); continue; } console.log(`New activity detected for organization: ${orgName}. Processing repositories...`); const repos = await this.fetchAllRepositories(); for (const r of repos.filter((repo) => repo.owner.login === orgName)) { try { const hasActivity = await this.hasNewActivity({ orgName, repoName: r.name, }); if (!hasActivity) { console.log(`No new activity for repository: ${orgName}/${r.name}`); continue; } } catch (error: any) { console.error(`Error fetching activity for repository ${orgName}/${r.name}:`, error.message); continue; } const org = r.owner.login; const repo = r.name; console.log(`Processing repository ${org}/${repo}`); try { const taggedCommitShas = await this.fetchTags(org, repo); const commits = await this.fetchRecentCommitsForRepo(org, repo); // Load the changelog from this repo. await this.loadChangelogFromRepo(org, repo); const commitResults = commits.map((c) => { const commit: plugins.interfaces.ICommitResult = { baseUrl: this.baseUrl, org, repo, timestamp: c.commit.author.date, prettyAgoTime: plugins.smarttime.getMilliSecondsAsHumanReadableAgoTime(new Date(c.commit.author.date).getTime()), hash: c.sha, commitMessage: c.commit.message, tagged: taggedCommitShas.has(c.sha), publishedOnNpm: false, changelog: undefined }; return commit; }); if (commitResults.length > 0) { try { const packageInfo = await this.npmRegistry.getPackageInfo(`@${org}/${repo}`); for (const commitResult of commitResults.filter((c) => c.tagged)) { const versionCandidate = commitResult.commitMessage.replace('\n', '').trim(); const correspondingVersion = packageInfo.allVersions.find((versionArg) => { return versionArg.version === versionCandidate; }); if (correspondingVersion) { commitResult.publishedOnNpm = true; const changelogEntry = this.getChangelogForVersion(versionCandidate); if (changelogEntry) { commitResult.changelog = changelogEntry; } } } } catch (error: any) { console.error(`Failed to fetch package info for ${org}/${repo}:`, error.message); } } allCommits.push(...commitResults); } catch (error: any) { console.error(`Skipping repository ${org}/${repo} due to error:`, error.message); } } } console.log(`Processed ${allCommits.length} commits in total.`); allCommits = allCommits.filter(commitArg => commitArg.tagged); console.log(`Filtered to ${allCommits.length} commits with tagged statuses.`); for (const c of allCommits) { console.log(` ========================================================================== ${c.prettyAgoTime} ago: ${c.org}/${c.repo} ${c.commitMessage} Published on npm: ${c.publishedOnNpm} ${c.changelog ? `Changelog:\n${c.changelog}\n` : ''} `); } return allCommits; } }