291 lines
9.3 KiB
TypeScript
291 lines
9.3 KiB
TypeScript
import * as plugins from './codefeed.plugins.js';
|
|
|
|
|
|
export class CodeFeed {
|
|
private baseUrl: string;
|
|
private token?: string;
|
|
private npmRegistry = new plugins.smartnpm.NpmRegistry();
|
|
private smartxmlInstance = new plugins.smartxml.SmartXml();
|
|
private lastRunTimestamp: string;
|
|
|
|
constructor(baseUrl: string, token?: string, lastRunTimestamp?: string) {
|
|
this.baseUrl = baseUrl;
|
|
this.token = token;
|
|
this.lastRunTimestamp = lastRunTimestamp || new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString();
|
|
console.log('CodeFeed initialized with last run timestamp:', this.lastRunTimestamp);
|
|
}
|
|
|
|
/**
|
|
* Fetch all organizations from the Gitea instance.
|
|
*/
|
|
private async fetchAllOrganizations(): Promise<string[]> {
|
|
const url = `${this.baseUrl}/api/v1/orgs`;
|
|
const response = await fetch(url, {
|
|
headers: this.token ? { Authorization: `token ${this.token}` } : {},
|
|
});
|
|
|
|
if (!response.ok) {
|
|
throw new Error(`Failed to fetch organizations: ${response.statusText}`);
|
|
}
|
|
|
|
const data: { username: string }[] = await response.json();
|
|
return data.map((org) => org.username);
|
|
}
|
|
|
|
/**
|
|
* Fetch organization-level activity RSS feed.
|
|
*/
|
|
private async fetchOrgRssFeed(optionsArg: {
|
|
orgName: string,
|
|
repoName?: string,
|
|
}): Promise<any[]> {
|
|
let rssUrl: string
|
|
if (optionsArg.orgName && !optionsArg.repoName) {
|
|
rssUrl = `${this.baseUrl}/${optionsArg.orgName}.atom`;
|
|
} else if (optionsArg.orgName && optionsArg.repoName) {
|
|
rssUrl = `${this.baseUrl}/${optionsArg.orgName}/${optionsArg.repoName}.atom`;
|
|
}
|
|
|
|
const response = await fetch(rssUrl);
|
|
if (!response.ok) {
|
|
throw new Error(`Failed to fetch RSS feed for organization ${optionsArg.orgName}/${optionsArg.repoName}: ${response.statusText}`);
|
|
}
|
|
|
|
const rssText = await response.text();
|
|
|
|
// Parse the Atom feed using fast-xml-parser
|
|
const rssData = this.smartxmlInstance.parseXmlToObject(rssText);
|
|
|
|
// Return the <entry> elements from the feed
|
|
return rssData.feed.entry || [];
|
|
}
|
|
|
|
/**
|
|
* Check if the organization's RSS feed has any new activities since the last run.
|
|
*/
|
|
private async hasNewActivity(optionsArg: {
|
|
orgName: string,
|
|
repoName?: string,
|
|
}): Promise<boolean> {
|
|
const entries = await this.fetchOrgRssFeed(optionsArg);
|
|
|
|
// Filter entries to find new activities since the last run
|
|
return entries.some((entry: any) => {
|
|
const updated = new Date(entry.updated);
|
|
return updated > new Date(this.lastRunTimestamp);
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Fetch all repositories accessible to the token/user.
|
|
*/
|
|
private async fetchAllRepositories(): Promise<plugins.interfaces.Repository[]> {
|
|
let page = 1;
|
|
const allRepos: plugins.interfaces.Repository[] = [];
|
|
|
|
while (true) {
|
|
const url = new URL(`${this.baseUrl}/api/v1/repos/search`);
|
|
url.searchParams.set('limit', '50');
|
|
url.searchParams.set('page', page.toString());
|
|
|
|
const resp = await fetch(url.href, {
|
|
headers: this.token ? { 'Authorization': `token ${this.token}` } : {},
|
|
});
|
|
|
|
if (!resp.ok) {
|
|
throw new Error(`Failed to fetch repositories: ${resp.statusText}`);
|
|
}
|
|
|
|
const data: plugins.interfaces.RepoSearchResponse = await resp.json();
|
|
allRepos.push(...data.data);
|
|
|
|
if (data.data.length < 50) {
|
|
break;
|
|
}
|
|
page++;
|
|
}
|
|
|
|
return allRepos;
|
|
}
|
|
|
|
/**
|
|
* Fetch all tags for a given repository.
|
|
*/
|
|
private async fetchTags(owner: string, repo: string): Promise<Set<string>> {
|
|
let page = 1;
|
|
const tags: plugins.interfaces.Tag[] = [];
|
|
|
|
while (true) {
|
|
const url = new URL(`${this.baseUrl}/api/v1/repos/${owner}/${repo}/tags`);
|
|
url.searchParams.set('limit', '50');
|
|
url.searchParams.set('page', page.toString());
|
|
|
|
const resp = await fetch(url.href, {
|
|
headers: this.token ? { 'Authorization': `token ${this.token}` } : {},
|
|
});
|
|
|
|
if (!resp.ok) {
|
|
console.error(`Failed to fetch tags for ${owner}/${repo}: ${resp.status} ${resp.statusText} at ${url.href}`);
|
|
throw new Error(`Failed to fetch tags for ${owner}/${repo}: ${resp.statusText}`);
|
|
}
|
|
|
|
const data: plugins.interfaces.Tag[] = await resp.json();
|
|
tags.push(...data);
|
|
|
|
if (data.length < 50) {
|
|
break;
|
|
}
|
|
page++;
|
|
}
|
|
|
|
const taggedCommitShas = new Set<string>();
|
|
for (const t of tags) {
|
|
if (t.commit?.sha) {
|
|
taggedCommitShas.add(t.commit.sha);
|
|
}
|
|
}
|
|
|
|
return taggedCommitShas;
|
|
}
|
|
|
|
/**
|
|
* Fetch commits from the last 24 hours for a repository.
|
|
*/
|
|
private async fetchRecentCommitsForRepo(owner: string, repo: string): Promise<plugins.interfaces.Commit[]> {
|
|
const twentyFourHoursAgo = new Date(Date.now() - 24 * 60 * 60 * 1000);
|
|
let page = 1;
|
|
const recentCommits: plugins.interfaces.Commit[] = [];
|
|
|
|
while (true) {
|
|
const url = new URL(`${this.baseUrl}/api/v1/repos/${owner}/${repo}/commits`);
|
|
url.searchParams.set('limit', '50');
|
|
url.searchParams.set('page', page.toString());
|
|
|
|
const resp = await fetch(url.href, {
|
|
headers: this.token ? { 'Authorization': `token ${this.token}` } : {},
|
|
});
|
|
|
|
if (!resp.ok) {
|
|
console.error(`Failed to fetch commits for ${owner}/${repo}: ${resp.status} ${resp.statusText} at ${url.href}`);
|
|
throw new Error(`Failed to fetch commits for ${owner}/${repo}: ${resp.statusText}`);
|
|
}
|
|
|
|
const data: plugins.interfaces.Commit[] = await resp.json();
|
|
if (data.length === 0) {
|
|
break;
|
|
}
|
|
|
|
for (const commit of data) {
|
|
const commitDate = new Date(commit.commit.author.date);
|
|
if (commitDate > twentyFourHoursAgo) {
|
|
recentCommits.push(commit);
|
|
} else {
|
|
return recentCommits;
|
|
}
|
|
}
|
|
|
|
page++;
|
|
}
|
|
|
|
return recentCommits;
|
|
}
|
|
|
|
/**
|
|
* Fetch all commits by querying all organizations.
|
|
*/
|
|
public async fetchAllCommitsFromInstance(): Promise<plugins.interfaces.CommitResult[]> {
|
|
const orgs = await this.fetchAllOrganizations();
|
|
console.log(`Found ${orgs.length} organizations`);
|
|
let allCommits: plugins.interfaces.CommitResult[] = [];
|
|
|
|
for (const orgName of orgs) {
|
|
console.log(`Checking activity for organization: ${orgName}`);
|
|
|
|
try {
|
|
const hasActivity = await this.hasNewActivity({
|
|
orgName,
|
|
});
|
|
if (!hasActivity) {
|
|
console.log(`No new activity for organization: ${orgName}`);
|
|
continue;
|
|
}
|
|
} catch (error) {
|
|
console.error(`Error fetching activity for organization ${orgName}:`, error.message);
|
|
continue;
|
|
}
|
|
|
|
console.log(`New activity detected for organization: ${orgName}. Processing repositories...`);
|
|
|
|
const repos = await this.fetchAllRepositories();
|
|
for (const r of repos.filter((repo) => repo.owner.login === orgName)) {
|
|
try {
|
|
const hasActivity = await this.hasNewActivity({
|
|
orgName,
|
|
repoName: r.name,
|
|
});
|
|
if (!hasActivity) {
|
|
console.log(`No new activity for repository: ${orgName}/${r.name}`);
|
|
continue;
|
|
}
|
|
} catch (error) {
|
|
console.error(`Error fetching activity for repository ${orgName}/${r.name}:`, error.message);
|
|
continue;
|
|
}
|
|
const org = r.owner.login;
|
|
const repo = r.name;
|
|
console.log(`Processing repository ${org}/${repo}`);
|
|
|
|
try {
|
|
const taggedCommitShas = await this.fetchTags(org, repo);
|
|
const commits = await this.fetchRecentCommitsForRepo(org, repo);
|
|
|
|
const commitResults = commits.map((c) => {
|
|
const commit: plugins.interfaces.CommitResult = {
|
|
baseUrl: this.baseUrl,
|
|
org,
|
|
repo,
|
|
timestamp: c.commit.author.date,
|
|
prettyAgoTime: plugins.smarttime.getMilliSecondsAsHumanReadableAgoTime(new Date(c.commit.author.date).getTime()),
|
|
hash: c.sha,
|
|
commitMessage: c.commit.message,
|
|
tagged: taggedCommitShas.has(c.sha),
|
|
publishedOnNpm: false,
|
|
}
|
|
return commit;
|
|
});
|
|
|
|
if (commitResults.length > 0) {
|
|
try {
|
|
const packageInfo = await this.npmRegistry.getPackageInfo(`@${org}/${repo}`);
|
|
for (const commit of commitResults.filter((c) => c.tagged)) {
|
|
const correspondingVersion = packageInfo.allVersions.find((versionArg) => {
|
|
return versionArg.version === commit.commitMessage.replace('\n', '');
|
|
});
|
|
if (correspondingVersion) {
|
|
commit.publishedOnNpm = true;
|
|
}
|
|
}
|
|
} catch (error) {
|
|
console.error(`Failed to fetch package info for ${org}/${repo}:`, error.message);
|
|
}
|
|
}
|
|
|
|
allCommits.push(...commitResults);
|
|
} catch (error) {
|
|
console.error(`Skipping repository ${org}/${repo} due to error:`, error.message);
|
|
}
|
|
}
|
|
}
|
|
|
|
console.log(`Processed ${allCommits.length} commits in total.`);
|
|
for (const c of allCommits) {
|
|
console.log(` ==========================================================================
|
|
${c.prettyAgoTime} ago:
|
|
${c.org}/${c.repo}
|
|
${c.commitMessage}
|
|
Published on npm: ${c.publishedOnNpm}
|
|
`);
|
|
}
|
|
return allCommits;
|
|
}
|
|
} |