feat(core): Add organization-level activity fetching and RSS parsing
This commit is contained in:
parent
c1e15ab47c
commit
76c662356e
@ -1,5 +1,13 @@
|
||||
# Changelog
|
||||
|
||||
## 2024-12-13 - 1.2.0 - feat(core)
|
||||
Add organization-level activity fetching and RSS parsing
|
||||
|
||||
- Integrated smartxml package for XML parsing.
|
||||
- Implemented fetching of all organizations within a Gitea instance.
|
||||
- Added functionality to check new activities in organization RSS feeds.
|
||||
- Enhanced fetching logic to include repository commits and tags.
|
||||
|
||||
## 2024-12-13 - 1.1.0 - feat(core)
|
||||
Add tracking of commits published on npm
|
||||
|
||||
|
||||
@ -23,7 +23,8 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@push.rocks/qenv": "^6.1.0",
|
||||
"@push.rocks/smartnpm": "^2.0.4"
|
||||
"@push.rocks/smartnpm": "^2.0.4",
|
||||
"@push.rocks/smartxml": "^1.0.8"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
|
||||
3
pnpm-lock.yaml
generated
3
pnpm-lock.yaml
generated
@ -14,6 +14,9 @@ importers:
|
||||
'@push.rocks/smartnpm':
|
||||
specifier: ^2.0.4
|
||||
version: 2.0.4
|
||||
'@push.rocks/smartxml':
|
||||
specifier: ^1.0.8
|
||||
version: 1.0.8
|
||||
devDependencies:
|
||||
'@git.zone/tsbuild':
|
||||
specifier: ^2.1.25
|
||||
|
||||
@ -3,6 +3,6 @@
|
||||
*/
|
||||
export const commitinfo = {
|
||||
name: '@foss.global/codefeed',
|
||||
version: '1.1.0',
|
||||
version: '1.2.0',
|
||||
description: 'a module for creating feeds for code development'
|
||||
}
|
||||
|
||||
@ -1,8 +1,10 @@
|
||||
// @push.rocks
|
||||
import * as qenv from '@push.rocks/qenv'
|
||||
import * as smartnpm from '@push.rocks/smartnpm'
|
||||
import * as qenv from '@push.rocks/qenv';
|
||||
import * as smartnpm from '@push.rocks/smartnpm';
|
||||
import * as smartxml from '@push.rocks/smartxml';
|
||||
|
||||
export {
|
||||
qenv,
|
||||
smartnpm,
|
||||
smartxml,
|
||||
}
|
||||
194
ts/index.ts
194
ts/index.ts
@ -47,14 +47,81 @@ interface CommitResult {
|
||||
export class CodeFeed {
|
||||
private baseUrl: string;
|
||||
private token?: string;
|
||||
private npmRegistry = new plugins.smartnpm.NpmRegistry();
|
||||
private npmRegistry = new plugins.smartnpm.NpmRegistry();
|
||||
private smartxmlInstance = new plugins.smartxml.SmartXml();
|
||||
private lastRunTimestamp: string;
|
||||
|
||||
constructor(baseUrl: string, token?: string) {
|
||||
constructor(baseUrl: string, token?: string, lastRunTimestamp?: string) {
|
||||
this.baseUrl = baseUrl;
|
||||
this.token = token;
|
||||
console.log('CodeFeed initialized');
|
||||
this.lastRunTimestamp = lastRunTimestamp || new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString();
|
||||
console.log('CodeFeed initialized with last run timestamp:', this.lastRunTimestamp);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch all organizations from the Gitea instance.
|
||||
*/
|
||||
private async fetchAllOrganizations(): Promise<string[]> {
|
||||
const url = `${this.baseUrl}/api/v1/orgs`;
|
||||
const response = await fetch(url, {
|
||||
headers: this.token ? { Authorization: `token ${this.token}` } : {},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch organizations: ${response.statusText}`);
|
||||
}
|
||||
|
||||
const data: { username: string }[] = await response.json();
|
||||
return data.map((org) => org.username);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch organization-level activity RSS feed.
|
||||
*/
|
||||
private async fetchOrgRssFeed(optionsArg: {
|
||||
orgName: string,
|
||||
repoName?: string,
|
||||
}): Promise<any[]> {
|
||||
let rssUrl: string
|
||||
if (optionsArg.orgName && !optionsArg.repoName) {
|
||||
rssUrl = `${this.baseUrl}/${optionsArg.orgName}.atom`;
|
||||
} else if (optionsArg.orgName && optionsArg.repoName) {
|
||||
rssUrl = `${this.baseUrl}/${optionsArg.orgName}/${optionsArg.repoName}.atom`;
|
||||
}
|
||||
|
||||
const response = await fetch(rssUrl);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch RSS feed for organization ${optionsArg.orgName}/${optionsArg.repoName}: ${response.statusText}`);
|
||||
}
|
||||
|
||||
const rssText = await response.text();
|
||||
|
||||
// Parse the Atom feed using fast-xml-parser
|
||||
const rssData = this.smartxmlInstance.parseXmlToObject(rssText);
|
||||
|
||||
// Return the <entry> elements from the feed
|
||||
return rssData.feed.entry || [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the organization's RSS feed has any new activities since the last run.
|
||||
*/
|
||||
private async hasNewActivity(optionsArg: {
|
||||
orgName: string,
|
||||
repoName?: string,
|
||||
}): Promise<boolean> {
|
||||
const entries = await this.fetchOrgRssFeed(optionsArg);
|
||||
|
||||
// Filter entries to find new activities since the last run
|
||||
return entries.some((entry: any) => {
|
||||
const updated = new Date(entry.updated);
|
||||
return updated > new Date(this.lastRunTimestamp);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch all repositories accessible to the token/user.
|
||||
*/
|
||||
private async fetchAllRepositories(): Promise<Repository[]> {
|
||||
let page = 1;
|
||||
const allRepos: Repository[] = [];
|
||||
@ -65,7 +132,7 @@ export class CodeFeed {
|
||||
url.searchParams.set('page', page.toString());
|
||||
|
||||
const resp = await fetch(url.href, {
|
||||
headers: this.token ? { 'Authorization': `token ${this.token}` } : {}
|
||||
headers: this.token ? { 'Authorization': `token ${this.token}` } : {},
|
||||
});
|
||||
|
||||
if (!resp.ok) {
|
||||
@ -84,6 +151,9 @@ export class CodeFeed {
|
||||
return allRepos;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch all tags for a given repository.
|
||||
*/
|
||||
private async fetchTags(owner: string, repo: string): Promise<Set<string>> {
|
||||
let page = 1;
|
||||
const tags: Tag[] = [];
|
||||
@ -94,9 +164,9 @@ export class CodeFeed {
|
||||
url.searchParams.set('page', page.toString());
|
||||
|
||||
const resp = await fetch(url.href, {
|
||||
headers: this.token ? { 'Authorization': `token ${this.token}` } : {}
|
||||
headers: this.token ? { 'Authorization': `token ${this.token}` } : {},
|
||||
});
|
||||
|
||||
|
||||
if (!resp.ok) {
|
||||
console.error(`Failed to fetch tags for ${owner}/${repo}: ${resp.status} ${resp.statusText} at ${url.href}`);
|
||||
throw new Error(`Failed to fetch tags for ${owner}/${repo}: ${resp.statusText}`);
|
||||
@ -121,6 +191,9 @@ export class CodeFeed {
|
||||
return taggedCommitShas;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch commits from the last 24 hours for a repository.
|
||||
*/
|
||||
private async fetchRecentCommitsForRepo(owner: string, repo: string): Promise<Commit[]> {
|
||||
const twentyFourHoursAgo = new Date(Date.now() - 24 * 60 * 60 * 1000);
|
||||
let page = 1;
|
||||
@ -128,12 +201,13 @@ export class CodeFeed {
|
||||
|
||||
while (true) {
|
||||
const url = new URL(`${this.baseUrl}/api/v1/repos/${owner}/${repo}/commits`);
|
||||
url.searchParams.set('limit', '1');
|
||||
url.searchParams.set('limit', '50');
|
||||
url.searchParams.set('page', page.toString());
|
||||
|
||||
const resp = await fetch(url.href, {
|
||||
headers: this.token ? { 'Authorization': `token ${this.token}` } : {}
|
||||
headers: this.token ? { 'Authorization': `token ${this.token}` } : {},
|
||||
});
|
||||
|
||||
if (!resp.ok) {
|
||||
console.error(`Failed to fetch commits for ${owner}/${repo}: ${resp.status} ${resp.statusText} at ${url.href}`);
|
||||
throw new Error(`Failed to fetch commits for ${owner}/${repo}: ${resp.statusText}`);
|
||||
@ -149,7 +223,6 @@ export class CodeFeed {
|
||||
if (commitDate > twentyFourHoursAgo) {
|
||||
recentCommits.push(commit);
|
||||
} else {
|
||||
// If we encounter a commit older than 24 hours, we can stop fetching more pages
|
||||
return recentCommits;
|
||||
}
|
||||
}
|
||||
@ -160,24 +233,56 @@ export class CodeFeed {
|
||||
return recentCommits;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch all commits by querying all organizations.
|
||||
*/
|
||||
public async fetchAllCommitsFromInstance(): Promise<CommitResult[]> {
|
||||
const repos = await this.fetchAllRepositories();
|
||||
const skippedRepos: string[] = [];
|
||||
console.log(`Found ${repos.length} repositories`);
|
||||
const orgs = await this.fetchAllOrganizations();
|
||||
console.log(`Found ${orgs.length} organizations`);
|
||||
let allCommits: CommitResult[] = [];
|
||||
|
||||
for (const r of repos) {
|
||||
const org = r.owner.login;
|
||||
const repo = r.name;
|
||||
console.log(`Processing repository ${org}/${repo}`);
|
||||
for (const orgName of orgs) {
|
||||
console.log(`Checking activity for organization: ${orgName}`);
|
||||
|
||||
try {
|
||||
const taggedCommitShas = await this.fetchTags(org, repo);
|
||||
const commits = await this.fetchRecentCommitsForRepo(org, repo);
|
||||
console.log(`${org}/${repo} -> Found ${commits.length} commits`);
|
||||
const commitResults: CommitResult[] = [];
|
||||
for (const c of commits) {
|
||||
const commit: CommitResult = {
|
||||
const hasActivity = await this.hasNewActivity({
|
||||
orgName,
|
||||
});
|
||||
if (!hasActivity) {
|
||||
console.log(`No new activity for organization: ${orgName}`);
|
||||
continue;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error fetching activity for organization ${orgName}:`, error.message);
|
||||
continue;
|
||||
}
|
||||
|
||||
console.log(`New activity detected for organization: ${orgName}. Processing repositories...`);
|
||||
|
||||
const repos = await this.fetchAllRepositories();
|
||||
for (const r of repos.filter((repo) => repo.owner.login === orgName)) {
|
||||
try {
|
||||
const hasActivity = await this.hasNewActivity({
|
||||
orgName,
|
||||
repoName: r.name,
|
||||
});
|
||||
if (!hasActivity) {
|
||||
console.log(`No new activity for repository: ${orgName}/${r.name}`);
|
||||
continue;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error fetching activity for repository ${orgName}/${r.name}:`, error.message);
|
||||
continue;
|
||||
}
|
||||
const org = r.owner.login;
|
||||
const repo = r.name;
|
||||
console.log(`Processing repository ${org}/${repo}`);
|
||||
|
||||
try {
|
||||
const taggedCommitShas = await this.fetchTags(org, repo);
|
||||
const commits = await this.fetchRecentCommitsForRepo(org, repo);
|
||||
|
||||
const commitResults = commits.map((c) => ({
|
||||
baseUrl: this.baseUrl,
|
||||
org,
|
||||
repo,
|
||||
@ -186,40 +291,32 @@ export class CodeFeed {
|
||||
commitMessage: c.commit.message,
|
||||
tagged: taggedCommitShas.has(c.sha),
|
||||
publishedOnNpm: false,
|
||||
}
|
||||
commitResults.push(commit);
|
||||
}
|
||||
}));
|
||||
|
||||
if (commitResults.length > 0) {
|
||||
try {
|
||||
const packageInfo = await this.npmRegistry.getPackageInfo(`@${org}/${repo}`);
|
||||
for (const commit of commitResults.filter(c => c.tagged)) {
|
||||
const correspondingVersion = packageInfo.allVersions.find(versionArg => {
|
||||
return versionArg.version === commit.commitMessage.replace('\n', '');
|
||||
});
|
||||
if (correspondingVersion) {
|
||||
commit.publishedOnNpm = true;
|
||||
if (commitResults.length > 0) {
|
||||
try {
|
||||
const packageInfo = await this.npmRegistry.getPackageInfo(`@${org}/${repo}`);
|
||||
for (const commit of commitResults.filter((c) => c.tagged)) {
|
||||
const correspondingVersion = packageInfo.allVersions.find((versionArg) => {
|
||||
return versionArg.version === commit.commitMessage.replace('\n', '');
|
||||
});
|
||||
if (correspondingVersion) {
|
||||
commit.publishedOnNpm = true;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Failed to fetch package info for ${org}/${repo}:`, error.message);
|
||||
}
|
||||
} catch (error: any) {
|
||||
console.error(`Failed to fetch package info for ${org}/${repo}:`, error.message);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
allCommits.push(...commitResults);
|
||||
} catch (error: any) {
|
||||
skippedRepos.push(`${org}/${repo}`);
|
||||
console.error(`Skipping repository ${org}/${repo} due to error:`, error.message);
|
||||
continue;
|
||||
allCommits.push(...commitResults);
|
||||
} catch (error) {
|
||||
console.error(`Skipping repository ${org}/${repo} due to error:`, error.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Found ${allCommits.length} relevant commits`);
|
||||
console.log(`Skipped ${skippedRepos.length} repositories due to errors`);
|
||||
for (const s of skippedRepos) {
|
||||
console.log(`Skipped ${s}`);
|
||||
}
|
||||
console.log(`Processed ${allCommits.length} commits in total.`);
|
||||
for (const c of allCommits) {
|
||||
console.log(`______________________________________________________
|
||||
Commit ${c.hash} by ${c.org}/${c.repo} at ${c.timestamp}
|
||||
@ -227,7 +324,6 @@ export class CodeFeed {
|
||||
Published on npm: ${c.publishedOnNpm}
|
||||
`);
|
||||
}
|
||||
|
||||
return allCommits;
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user