345 lines
11 KiB
TypeScript
345 lines
11 KiB
TypeScript
import * as plugins from './codefeed.plugins.js';
|
|
|
|
export class CodeFeed {
|
|
private baseUrl: string;
|
|
private token?: string;
|
|
private npmRegistry = new plugins.smartnpm.NpmRegistry();
|
|
private smartxmlInstance = new plugins.smartxml.SmartXml();
|
|
private lastRunTimestamp: string;
|
|
private changelogContent: string;
|
|
|
|
constructor(baseUrl: string, token?: string, lastRunTimestamp?: string) {
|
|
this.baseUrl = baseUrl;
|
|
this.token = token;
|
|
this.lastRunTimestamp = lastRunTimestamp || new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString();
|
|
console.log('CodeFeed initialized with last run timestamp:', this.lastRunTimestamp);
|
|
}
|
|
|
|
/**
|
|
* Load the changelog directly from the Gitea repository.
|
|
*/
|
|
private async loadChangelogFromRepo(owner: string, repo: string): Promise<void> {
|
|
const url = `${this.baseUrl}/api/v1/repos/${owner}/${repo}/contents/changelog.md`;
|
|
const headers: Record<string, string> = {};
|
|
if (this.token) {
|
|
headers['Authorization'] = `token ${this.token}`;
|
|
}
|
|
|
|
const response = await fetch(url, { headers });
|
|
|
|
if (!response.ok) {
|
|
console.error(`Could not fetch CHANGELOG.md from ${owner}/${repo}: ${response.status} ${response.statusText}`);
|
|
this.changelogContent = '';
|
|
return;
|
|
}
|
|
|
|
const data = await response.json();
|
|
if (!data.content) {
|
|
console.warn(`No content field found in response for ${owner}/${repo}/changelog.md`);
|
|
this.changelogContent = '';
|
|
return;
|
|
}
|
|
|
|
const decodedContent = Buffer.from(data.content, 'base64').toString('utf8');
|
|
this.changelogContent = decodedContent;
|
|
}
|
|
|
|
/**
|
|
* Parse the changelog to find the entry for a given version.
|
|
* The changelog format is assumed as:
|
|
*
|
|
* # Changelog
|
|
*
|
|
* ## <date> - <version> - <description>
|
|
* <changes...>
|
|
*/
|
|
private getChangelogForVersion(version: string): string | undefined {
|
|
if (!this.changelogContent) {
|
|
return undefined;
|
|
}
|
|
const lines = this.changelogContent.split('\n');
|
|
const versionHeaderIndex = lines.findIndex((line) => line.includes(`- ${version} -`));
|
|
if (versionHeaderIndex === -1) {
|
|
return undefined;
|
|
}
|
|
|
|
const changelogLines: string[] = [];
|
|
for (let i = versionHeaderIndex + 1; i < lines.length; i++) {
|
|
const line = lines[i];
|
|
// The next version header starts with `## `
|
|
if (line.startsWith('## ')) {
|
|
break;
|
|
}
|
|
changelogLines.push(line);
|
|
}
|
|
|
|
return changelogLines.join('\n').trim();
|
|
}
|
|
|
|
private async fetchAllOrganizations(): Promise<string[]> {
|
|
const url = `${this.baseUrl}/api/v1/orgs`;
|
|
const response = await fetch(url, {
|
|
headers: this.token ? { Authorization: `token ${this.token}` } : {},
|
|
});
|
|
|
|
if (!response.ok) {
|
|
throw new Error(`Failed to fetch organizations: ${response.statusText}`);
|
|
}
|
|
|
|
const data: { username: string }[] = await response.json();
|
|
return data.map((org) => org.username);
|
|
}
|
|
|
|
private async fetchOrgRssFeed(optionsArg: {
|
|
orgName: string,
|
|
repoName?: string,
|
|
}): Promise<any[]> {
|
|
let rssUrl: string;
|
|
if (optionsArg.orgName && !optionsArg.repoName) {
|
|
rssUrl = `${this.baseUrl}/${optionsArg.orgName}.atom`;
|
|
} else if (optionsArg.orgName && optionsArg.repoName) {
|
|
rssUrl = `${this.baseUrl}/${optionsArg.orgName}/${optionsArg.repoName}.atom`;
|
|
} else {
|
|
throw new Error('Invalid arguments provided to fetchOrgRssFeed.');
|
|
}
|
|
|
|
const response = await fetch(rssUrl);
|
|
if (!response.ok) {
|
|
throw new Error(`Failed to fetch RSS feed for organization ${optionsArg.orgName}/${optionsArg.repoName}: ${response.statusText}`);
|
|
}
|
|
|
|
const rssText = await response.text();
|
|
const rssData = this.smartxmlInstance.parseXmlToObject(rssText);
|
|
return rssData.feed.entry || [];
|
|
}
|
|
|
|
private async hasNewActivity(optionsArg: {
|
|
orgName: string,
|
|
repoName?: string,
|
|
}): Promise<boolean> {
|
|
const entries = await this.fetchOrgRssFeed(optionsArg);
|
|
|
|
return entries.some((entry: any) => {
|
|
const updated = new Date(entry.updated);
|
|
return updated > new Date(this.lastRunTimestamp);
|
|
});
|
|
}
|
|
|
|
private async fetchAllRepositories(): Promise<plugins.interfaces.IRepository[]> {
|
|
let page = 1;
|
|
const allRepos: plugins.interfaces.IRepository[] = [];
|
|
|
|
while (true) {
|
|
const url = new URL(`${this.baseUrl}/api/v1/repos/search`);
|
|
url.searchParams.set('limit', '50');
|
|
url.searchParams.set('page', page.toString());
|
|
|
|
const resp = await fetch(url.href, {
|
|
headers: this.token ? { 'Authorization': `token ${this.token}` } : {},
|
|
});
|
|
|
|
if (!resp.ok) {
|
|
throw new Error(`Failed to fetch repositories: ${resp.statusText}`);
|
|
}
|
|
|
|
const data: plugins.interfaces.IRepoSearchResponse = await resp.json();
|
|
allRepos.push(...data.data);
|
|
|
|
if (data.data.length < 50) {
|
|
break;
|
|
}
|
|
page++;
|
|
}
|
|
|
|
return allRepos;
|
|
}
|
|
|
|
private async fetchTags(owner: string, repo: string): Promise<Set<string>> {
|
|
let page = 1;
|
|
const tags: plugins.interfaces.ITag[] = [];
|
|
|
|
while (true) {
|
|
const url = new URL(`${this.baseUrl}/api/v1/repos/${owner}/${repo}/tags`);
|
|
url.searchParams.set('limit', '50');
|
|
url.searchParams.set('page', page.toString());
|
|
|
|
const resp = await fetch(url.href, {
|
|
headers: this.token ? { 'Authorization': `token ${this.token}` } : {},
|
|
});
|
|
|
|
if (!resp.ok) {
|
|
console.error(`Failed to fetch tags for ${owner}/${repo}: ${resp.status} ${resp.statusText} at ${url.href}`);
|
|
throw new Error(`Failed to fetch tags for ${owner}/${repo}: ${resp.statusText}`);
|
|
}
|
|
|
|
const data: plugins.interfaces.ITag[] = await resp.json();
|
|
tags.push(...data);
|
|
|
|
if (data.length < 50) {
|
|
break;
|
|
}
|
|
page++;
|
|
}
|
|
|
|
const taggedCommitShas = new Set<string>();
|
|
for (const t of tags) {
|
|
if (t.commit?.sha) {
|
|
taggedCommitShas.add(t.commit.sha);
|
|
}
|
|
}
|
|
|
|
return taggedCommitShas;
|
|
}
|
|
|
|
private async fetchRecentCommitsForRepo(owner: string, repo: string): Promise<plugins.interfaces.ICommit[]> {
|
|
const twentyFourHoursAgo = new Date(Date.now() - 24 * 60 * 60 * 1000);
|
|
let page = 1;
|
|
const recentCommits: plugins.interfaces.ICommit[] = [];
|
|
|
|
while (true) {
|
|
const url = new URL(`${this.baseUrl}/api/v1/repos/${owner}/${repo}/commits`);
|
|
url.searchParams.set('limit', '50');
|
|
url.searchParams.set('page', page.toString());
|
|
|
|
const resp = await fetch(url.href, {
|
|
headers: this.token ? { 'Authorization': `token ${this.token}` } : {},
|
|
});
|
|
|
|
if (!resp.ok) {
|
|
console.error(`Failed to fetch commits for ${owner}/${repo}: ${resp.status} ${resp.statusText} at ${url.href}`);
|
|
throw new Error(`Failed to fetch commits for ${owner}/${repo}: ${resp.statusText}`);
|
|
}
|
|
|
|
const data: plugins.interfaces.ICommit[] = await resp.json();
|
|
if (data.length === 0) {
|
|
break;
|
|
}
|
|
|
|
for (const commit of data) {
|
|
const commitDate = new Date(commit.commit.author.date);
|
|
if (commitDate > twentyFourHoursAgo) {
|
|
recentCommits.push(commit);
|
|
} else {
|
|
return recentCommits;
|
|
}
|
|
}
|
|
|
|
page++;
|
|
}
|
|
|
|
return recentCommits;
|
|
}
|
|
|
|
public async fetchAllCommitsFromInstance(): Promise<plugins.interfaces.ICommitResult[]> {
|
|
const orgs = await this.fetchAllOrganizations();
|
|
console.log(`Found ${orgs.length} organizations`);
|
|
let allCommits: plugins.interfaces.ICommitResult[] = [];
|
|
|
|
for (const orgName of orgs) {
|
|
console.log(`Checking activity for organization: ${orgName}`);
|
|
|
|
try {
|
|
const hasActivity = await this.hasNewActivity({
|
|
orgName,
|
|
});
|
|
if (!hasActivity) {
|
|
console.log(`No new activity for organization: ${orgName}`);
|
|
continue;
|
|
}
|
|
} catch (error: any) {
|
|
console.error(`Error fetching activity for organization ${orgName}:`, error.message);
|
|
continue;
|
|
}
|
|
|
|
console.log(`New activity detected for organization: ${orgName}. Processing repositories...`);
|
|
|
|
const repos = await this.fetchAllRepositories();
|
|
for (const r of repos.filter((repo) => repo.owner.login === orgName)) {
|
|
try {
|
|
const hasActivity = await this.hasNewActivity({
|
|
orgName,
|
|
repoName: r.name,
|
|
});
|
|
if (!hasActivity) {
|
|
console.log(`No new activity for repository: ${orgName}/${r.name}`);
|
|
continue;
|
|
}
|
|
} catch (error: any) {
|
|
console.error(`Error fetching activity for repository ${orgName}/${r.name}:`, error.message);
|
|
continue;
|
|
}
|
|
|
|
const org = r.owner.login;
|
|
const repo = r.name;
|
|
console.log(`Processing repository ${org}/${repo}`);
|
|
|
|
try {
|
|
const taggedCommitShas = await this.fetchTags(org, repo);
|
|
const commits = await this.fetchRecentCommitsForRepo(org, repo);
|
|
|
|
// Load the changelog from this repo.
|
|
await this.loadChangelogFromRepo(org, repo);
|
|
|
|
const commitResults = commits.map((c) => {
|
|
const commit: plugins.interfaces.ICommitResult = {
|
|
baseUrl: this.baseUrl,
|
|
org,
|
|
repo,
|
|
timestamp: c.commit.author.date,
|
|
prettyAgoTime: plugins.smarttime.getMilliSecondsAsHumanReadableAgoTime(new Date(c.commit.author.date).getTime()),
|
|
hash: c.sha,
|
|
commitMessage: c.commit.message,
|
|
tagged: taggedCommitShas.has(c.sha),
|
|
publishedOnNpm: false,
|
|
changelog: undefined
|
|
};
|
|
return commit;
|
|
});
|
|
|
|
if (commitResults.length > 0) {
|
|
try {
|
|
const packageInfo = await this.npmRegistry.getPackageInfo(`@${org}/${repo}`);
|
|
for (const commitResult of commitResults.filter((c) => c.tagged)) {
|
|
const versionCandidate = commitResult.commitMessage.replace('\n', '').trim();
|
|
const correspondingVersion = packageInfo.allVersions.find((versionArg) => {
|
|
return versionArg.version === versionCandidate;
|
|
});
|
|
if (correspondingVersion) {
|
|
commitResult.publishedOnNpm = true;
|
|
const changelogEntry = this.getChangelogForVersion(versionCandidate);
|
|
if (changelogEntry) {
|
|
commitResult.changelog = changelogEntry;
|
|
}
|
|
}
|
|
}
|
|
} catch (error: any) {
|
|
console.error(`Failed to fetch package info for ${org}/${repo}:`, error.message);
|
|
}
|
|
}
|
|
|
|
allCommits.push(...commitResults);
|
|
} catch (error: any) {
|
|
console.error(`Skipping repository ${org}/${repo} due to error:`, error.message);
|
|
}
|
|
}
|
|
}
|
|
|
|
console.log(`Processed ${allCommits.length} commits in total.`);
|
|
|
|
allCommits = allCommits.filter(commitArg => commitArg.tagged);
|
|
|
|
console.log(`Filtered to ${allCommits.length} commits with tagged statuses.`);
|
|
|
|
for (const c of allCommits) {
|
|
console.log(` ==========================================================================
|
|
${c.prettyAgoTime} ago:
|
|
${c.org}/${c.repo}
|
|
${c.commitMessage}
|
|
Published on npm: ${c.publishedOnNpm}
|
|
${c.changelog ? `Changelog:\n${c.changelog}\n` : ''}
|
|
`);
|
|
}
|
|
|
|
return allCommits;
|
|
}
|
|
} |