import * as plugins from './codefeed.plugins.js'; export class CodeFeed { private baseUrl: string; private token?: string; private npmRegistry = new plugins.smartnpm.NpmRegistry(); private smartxmlInstance = new plugins.smartxml.SmartXml(); private lastRunTimestamp: string; constructor(baseUrl: string, token?: string, lastRunTimestamp?: string) { this.baseUrl = baseUrl; this.token = token; this.lastRunTimestamp = lastRunTimestamp || new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString(); console.log('CodeFeed initialized with last run timestamp:', this.lastRunTimestamp); } /** * Fetch all organizations from the Gitea instance. */ private async fetchAllOrganizations(): Promise { const url = `${this.baseUrl}/api/v1/orgs`; const response = await fetch(url, { headers: this.token ? { Authorization: `token ${this.token}` } : {}, }); if (!response.ok) { throw new Error(`Failed to fetch organizations: ${response.statusText}`); } const data: { username: string }[] = await response.json(); return data.map((org) => org.username); } /** * Fetch organization-level activity RSS feed. */ private async fetchOrgRssFeed(optionsArg: { orgName: string, repoName?: string, }): Promise { let rssUrl: string if (optionsArg.orgName && !optionsArg.repoName) { rssUrl = `${this.baseUrl}/${optionsArg.orgName}.atom`; } else if (optionsArg.orgName && optionsArg.repoName) { rssUrl = `${this.baseUrl}/${optionsArg.orgName}/${optionsArg.repoName}.atom`; } const response = await fetch(rssUrl); if (!response.ok) { throw new Error(`Failed to fetch RSS feed for organization ${optionsArg.orgName}/${optionsArg.repoName}: ${response.statusText}`); } const rssText = await response.text(); // Parse the Atom feed using fast-xml-parser const rssData = this.smartxmlInstance.parseXmlToObject(rssText); // Return the elements from the feed return rssData.feed.entry || []; } /** * Check if the organization's RSS feed has any new activities since the last run. */ private async hasNewActivity(optionsArg: { orgName: string, repoName?: string, }): Promise { const entries = await this.fetchOrgRssFeed(optionsArg); // Filter entries to find new activities since the last run return entries.some((entry: any) => { const updated = new Date(entry.updated); return updated > new Date(this.lastRunTimestamp); }); } /** * Fetch all repositories accessible to the token/user. */ private async fetchAllRepositories(): Promise { let page = 1; const allRepos: plugins.interfaces.Repository[] = []; while (true) { const url = new URL(`${this.baseUrl}/api/v1/repos/search`); url.searchParams.set('limit', '50'); url.searchParams.set('page', page.toString()); const resp = await fetch(url.href, { headers: this.token ? { 'Authorization': `token ${this.token}` } : {}, }); if (!resp.ok) { throw new Error(`Failed to fetch repositories: ${resp.statusText}`); } const data: plugins.interfaces.RepoSearchResponse = await resp.json(); allRepos.push(...data.data); if (data.data.length < 50) { break; } page++; } return allRepos; } /** * Fetch all tags for a given repository. */ private async fetchTags(owner: string, repo: string): Promise> { let page = 1; const tags: plugins.interfaces.Tag[] = []; while (true) { const url = new URL(`${this.baseUrl}/api/v1/repos/${owner}/${repo}/tags`); url.searchParams.set('limit', '50'); url.searchParams.set('page', page.toString()); const resp = await fetch(url.href, { headers: this.token ? { 'Authorization': `token ${this.token}` } : {}, }); if (!resp.ok) { console.error(`Failed to fetch tags for ${owner}/${repo}: ${resp.status} ${resp.statusText} at ${url.href}`); throw new Error(`Failed to fetch tags for ${owner}/${repo}: ${resp.statusText}`); } const data: plugins.interfaces.Tag[] = await resp.json(); tags.push(...data); if (data.length < 50) { break; } page++; } const taggedCommitShas = new Set(); for (const t of tags) { if (t.commit?.sha) { taggedCommitShas.add(t.commit.sha); } } return taggedCommitShas; } /** * Fetch commits from the last 24 hours for a repository. */ private async fetchRecentCommitsForRepo(owner: string, repo: string): Promise { const twentyFourHoursAgo = new Date(Date.now() - 24 * 60 * 60 * 1000); let page = 1; const recentCommits: plugins.interfaces.Commit[] = []; while (true) { const url = new URL(`${this.baseUrl}/api/v1/repos/${owner}/${repo}/commits`); url.searchParams.set('limit', '50'); url.searchParams.set('page', page.toString()); const resp = await fetch(url.href, { headers: this.token ? { 'Authorization': `token ${this.token}` } : {}, }); if (!resp.ok) { console.error(`Failed to fetch commits for ${owner}/${repo}: ${resp.status} ${resp.statusText} at ${url.href}`); throw new Error(`Failed to fetch commits for ${owner}/${repo}: ${resp.statusText}`); } const data: plugins.interfaces.Commit[] = await resp.json(); if (data.length === 0) { break; } for (const commit of data) { const commitDate = new Date(commit.commit.author.date); if (commitDate > twentyFourHoursAgo) { recentCommits.push(commit); } else { return recentCommits; } } page++; } return recentCommits; } /** * Fetch all commits by querying all organizations. */ public async fetchAllCommitsFromInstance(): Promise { const orgs = await this.fetchAllOrganizations(); console.log(`Found ${orgs.length} organizations`); let allCommits: plugins.interfaces.CommitResult[] = []; for (const orgName of orgs) { console.log(`Checking activity for organization: ${orgName}`); try { const hasActivity = await this.hasNewActivity({ orgName, }); if (!hasActivity) { console.log(`No new activity for organization: ${orgName}`); continue; } } catch (error) { console.error(`Error fetching activity for organization ${orgName}:`, error.message); continue; } console.log(`New activity detected for organization: ${orgName}. Processing repositories...`); const repos = await this.fetchAllRepositories(); for (const r of repos.filter((repo) => repo.owner.login === orgName)) { try { const hasActivity = await this.hasNewActivity({ orgName, repoName: r.name, }); if (!hasActivity) { console.log(`No new activity for repository: ${orgName}/${r.name}`); continue; } } catch (error) { console.error(`Error fetching activity for repository ${orgName}/${r.name}:`, error.message); continue; } const org = r.owner.login; const repo = r.name; console.log(`Processing repository ${org}/${repo}`); try { const taggedCommitShas = await this.fetchTags(org, repo); const commits = await this.fetchRecentCommitsForRepo(org, repo); const commitResults = commits.map((c) => { const commit: plugins.interfaces.CommitResult = { baseUrl: this.baseUrl, org, repo, timestamp: c.commit.author.date, prettyAgoTime: plugins.smarttime.getMilliSecondsAsHumanReadableAgoTime(new Date(c.commit.author.date).getTime()), hash: c.sha, commitMessage: c.commit.message, tagged: taggedCommitShas.has(c.sha), publishedOnNpm: false, } return commit; }); if (commitResults.length > 0) { try { const packageInfo = await this.npmRegistry.getPackageInfo(`@${org}/${repo}`); for (const commit of commitResults.filter((c) => c.tagged)) { const correspondingVersion = packageInfo.allVersions.find((versionArg) => { return versionArg.version === commit.commitMessage.replace('\n', ''); }); if (correspondingVersion) { commit.publishedOnNpm = true; } } } catch (error) { console.error(`Failed to fetch package info for ${org}/${repo}:`, error.message); } } allCommits.push(...commitResults); } catch (error) { console.error(`Skipping repository ${org}/${repo} due to error:`, error.message); } } } console.log(`Processed ${allCommits.length} commits in total.`); for (const c of allCommits) { console.log(`______________________________________________________ ${c.prettyAgoTime} ago: Commit ${c.hash} by ${c.org}/${c.repo} at ${c.timestamp} ${c.commitMessage} Published on npm: ${c.publishedOnNpm} `); } return allCommits; } }