diff --git a/changelog.md b/changelog.md index 6dd0f70..8f6286b 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,13 @@ # Changelog +## 2024-12-13 - 1.2.0 - feat(core) +Add organization-level activity fetching and RSS parsing + +- Integrated smartxml package for XML parsing. +- Implemented fetching of all organizations within a Gitea instance. +- Added functionality to check new activities in organization RSS feeds. +- Enhanced fetching logic to include repository commits and tags. + ## 2024-12-13 - 1.1.0 - feat(core) Add tracking of commits published on npm diff --git a/package.json b/package.json index 3f2bb83..125c689 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,8 @@ }, "dependencies": { "@push.rocks/qenv": "^6.1.0", - "@push.rocks/smartnpm": "^2.0.4" + "@push.rocks/smartnpm": "^2.0.4", + "@push.rocks/smartxml": "^1.0.8" }, "repository": { "type": "git", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 50b9415..3cd4622 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -14,6 +14,9 @@ importers: '@push.rocks/smartnpm': specifier: ^2.0.4 version: 2.0.4 + '@push.rocks/smartxml': + specifier: ^1.0.8 + version: 1.0.8 devDependencies: '@git.zone/tsbuild': specifier: ^2.1.25 diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index 06dcb4b..b8c3566 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@foss.global/codefeed', - version: '1.1.0', + version: '1.2.0', description: 'a module for creating feeds for code development' } diff --git a/ts/codefeed.plugins.ts b/ts/codefeed.plugins.ts index 1d25ce4..bdb6cc3 100644 --- a/ts/codefeed.plugins.ts +++ b/ts/codefeed.plugins.ts @@ -1,8 +1,10 @@ // @push.rocks -import * as qenv from '@push.rocks/qenv' -import * as smartnpm from '@push.rocks/smartnpm' +import * as qenv from '@push.rocks/qenv'; +import * as smartnpm from '@push.rocks/smartnpm'; +import * as smartxml from '@push.rocks/smartxml'; export { qenv, smartnpm, + smartxml, } \ No newline at end of file diff --git a/ts/index.ts b/ts/index.ts index e03d605..81b1689 100644 --- a/ts/index.ts +++ b/ts/index.ts @@ -47,14 +47,81 @@ interface CommitResult { export class CodeFeed { private baseUrl: string; private token?: string; - private npmRegistry = new plugins.smartnpm.NpmRegistry(); + private npmRegistry = new plugins.smartnpm.NpmRegistry(); + private smartxmlInstance = new plugins.smartxml.SmartXml(); + private lastRunTimestamp: string; - constructor(baseUrl: string, token?: string) { + constructor(baseUrl: string, token?: string, lastRunTimestamp?: string) { this.baseUrl = baseUrl; this.token = token; - console.log('CodeFeed initialized'); + this.lastRunTimestamp = lastRunTimestamp || new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString(); + console.log('CodeFeed initialized with last run timestamp:', this.lastRunTimestamp); } + /** + * Fetch all organizations from the Gitea instance. + */ + private async fetchAllOrganizations(): Promise { + const url = `${this.baseUrl}/api/v1/orgs`; + const response = await fetch(url, { + headers: this.token ? { Authorization: `token ${this.token}` } : {}, + }); + + if (!response.ok) { + throw new Error(`Failed to fetch organizations: ${response.statusText}`); + } + + const data: { username: string }[] = await response.json(); + return data.map((org) => org.username); + } + + /** + * Fetch organization-level activity RSS feed. + */ + private async fetchOrgRssFeed(optionsArg: { + orgName: string, + repoName?: string, + }): Promise { + let rssUrl: string + if (optionsArg.orgName && !optionsArg.repoName) { + rssUrl = `${this.baseUrl}/${optionsArg.orgName}.atom`; + } else if (optionsArg.orgName && optionsArg.repoName) { + rssUrl = `${this.baseUrl}/${optionsArg.orgName}/${optionsArg.repoName}.atom`; + } + + const response = await fetch(rssUrl); + if (!response.ok) { + throw new Error(`Failed to fetch RSS feed for organization ${optionsArg.orgName}/${optionsArg.repoName}: ${response.statusText}`); + } + + const rssText = await response.text(); + + // Parse the Atom feed using fast-xml-parser + const rssData = this.smartxmlInstance.parseXmlToObject(rssText); + + // Return the elements from the feed + return rssData.feed.entry || []; + } + + /** + * Check if the organization's RSS feed has any new activities since the last run. + */ + private async hasNewActivity(optionsArg: { + orgName: string, + repoName?: string, + }): Promise { + const entries = await this.fetchOrgRssFeed(optionsArg); + + // Filter entries to find new activities since the last run + return entries.some((entry: any) => { + const updated = new Date(entry.updated); + return updated > new Date(this.lastRunTimestamp); + }); + } + + /** + * Fetch all repositories accessible to the token/user. + */ private async fetchAllRepositories(): Promise { let page = 1; const allRepos: Repository[] = []; @@ -65,7 +132,7 @@ export class CodeFeed { url.searchParams.set('page', page.toString()); const resp = await fetch(url.href, { - headers: this.token ? { 'Authorization': `token ${this.token}` } : {} + headers: this.token ? { 'Authorization': `token ${this.token}` } : {}, }); if (!resp.ok) { @@ -84,6 +151,9 @@ export class CodeFeed { return allRepos; } + /** + * Fetch all tags for a given repository. + */ private async fetchTags(owner: string, repo: string): Promise> { let page = 1; const tags: Tag[] = []; @@ -94,9 +164,9 @@ export class CodeFeed { url.searchParams.set('page', page.toString()); const resp = await fetch(url.href, { - headers: this.token ? { 'Authorization': `token ${this.token}` } : {} + headers: this.token ? { 'Authorization': `token ${this.token}` } : {}, }); - + if (!resp.ok) { console.error(`Failed to fetch tags for ${owner}/${repo}: ${resp.status} ${resp.statusText} at ${url.href}`); throw new Error(`Failed to fetch tags for ${owner}/${repo}: ${resp.statusText}`); @@ -121,6 +191,9 @@ export class CodeFeed { return taggedCommitShas; } + /** + * Fetch commits from the last 24 hours for a repository. + */ private async fetchRecentCommitsForRepo(owner: string, repo: string): Promise { const twentyFourHoursAgo = new Date(Date.now() - 24 * 60 * 60 * 1000); let page = 1; @@ -128,12 +201,13 @@ export class CodeFeed { while (true) { const url = new URL(`${this.baseUrl}/api/v1/repos/${owner}/${repo}/commits`); - url.searchParams.set('limit', '1'); + url.searchParams.set('limit', '50'); url.searchParams.set('page', page.toString()); const resp = await fetch(url.href, { - headers: this.token ? { 'Authorization': `token ${this.token}` } : {} + headers: this.token ? { 'Authorization': `token ${this.token}` } : {}, }); + if (!resp.ok) { console.error(`Failed to fetch commits for ${owner}/${repo}: ${resp.status} ${resp.statusText} at ${url.href}`); throw new Error(`Failed to fetch commits for ${owner}/${repo}: ${resp.statusText}`); @@ -149,7 +223,6 @@ export class CodeFeed { if (commitDate > twentyFourHoursAgo) { recentCommits.push(commit); } else { - // If we encounter a commit older than 24 hours, we can stop fetching more pages return recentCommits; } } @@ -160,24 +233,56 @@ export class CodeFeed { return recentCommits; } + /** + * Fetch all commits by querying all organizations. + */ public async fetchAllCommitsFromInstance(): Promise { - const repos = await this.fetchAllRepositories(); - const skippedRepos: string[] = []; - console.log(`Found ${repos.length} repositories`); + const orgs = await this.fetchAllOrganizations(); + console.log(`Found ${orgs.length} organizations`); let allCommits: CommitResult[] = []; - for (const r of repos) { - const org = r.owner.login; - const repo = r.name; - console.log(`Processing repository ${org}/${repo}`); + for (const orgName of orgs) { + console.log(`Checking activity for organization: ${orgName}`); try { - const taggedCommitShas = await this.fetchTags(org, repo); - const commits = await this.fetchRecentCommitsForRepo(org, repo); - console.log(`${org}/${repo} -> Found ${commits.length} commits`); - const commitResults: CommitResult[] = []; - for (const c of commits) { - const commit: CommitResult = { + const hasActivity = await this.hasNewActivity({ + orgName, + }); + if (!hasActivity) { + console.log(`No new activity for organization: ${orgName}`); + continue; + } + } catch (error) { + console.error(`Error fetching activity for organization ${orgName}:`, error.message); + continue; + } + + console.log(`New activity detected for organization: ${orgName}. Processing repositories...`); + + const repos = await this.fetchAllRepositories(); + for (const r of repos.filter((repo) => repo.owner.login === orgName)) { + try { + const hasActivity = await this.hasNewActivity({ + orgName, + repoName: r.name, + }); + if (!hasActivity) { + console.log(`No new activity for repository: ${orgName}/${r.name}`); + continue; + } + } catch (error) { + console.error(`Error fetching activity for repository ${orgName}/${r.name}:`, error.message); + continue; + } + const org = r.owner.login; + const repo = r.name; + console.log(`Processing repository ${org}/${repo}`); + + try { + const taggedCommitShas = await this.fetchTags(org, repo); + const commits = await this.fetchRecentCommitsForRepo(org, repo); + + const commitResults = commits.map((c) => ({ baseUrl: this.baseUrl, org, repo, @@ -186,40 +291,32 @@ export class CodeFeed { commitMessage: c.commit.message, tagged: taggedCommitShas.has(c.sha), publishedOnNpm: false, - } - commitResults.push(commit); - } + })); - if (commitResults.length > 0) { - try { - const packageInfo = await this.npmRegistry.getPackageInfo(`@${org}/${repo}`); - for (const commit of commitResults.filter(c => c.tagged)) { - const correspondingVersion = packageInfo.allVersions.find(versionArg => { - return versionArg.version === commit.commitMessage.replace('\n', ''); - }); - if (correspondingVersion) { - commit.publishedOnNpm = true; + if (commitResults.length > 0) { + try { + const packageInfo = await this.npmRegistry.getPackageInfo(`@${org}/${repo}`); + for (const commit of commitResults.filter((c) => c.tagged)) { + const correspondingVersion = packageInfo.allVersions.find((versionArg) => { + return versionArg.version === commit.commitMessage.replace('\n', ''); + }); + if (correspondingVersion) { + commit.publishedOnNpm = true; + } } + } catch (error) { + console.error(`Failed to fetch package info for ${org}/${repo}:`, error.message); } - } catch (error: any) { - console.error(`Failed to fetch package info for ${org}/${repo}:`, error.message); - continue; } - } - allCommits.push(...commitResults); - } catch (error: any) { - skippedRepos.push(`${org}/${repo}`); - console.error(`Skipping repository ${org}/${repo} due to error:`, error.message); - continue; + allCommits.push(...commitResults); + } catch (error) { + console.error(`Skipping repository ${org}/${repo} due to error:`, error.message); + } } } - console.log(`Found ${allCommits.length} relevant commits`); - console.log(`Skipped ${skippedRepos.length} repositories due to errors`); - for (const s of skippedRepos) { - console.log(`Skipped ${s}`); - } + console.log(`Processed ${allCommits.length} commits in total.`); for (const c of allCommits) { console.log(`______________________________________________________ Commit ${c.hash} by ${c.org}/${c.repo} at ${c.timestamp} @@ -227,7 +324,6 @@ export class CodeFeed { Published on npm: ${c.publishedOnNpm} `); } - return allCommits; } } \ No newline at end of file