feat(core): Enhance commit fetching with caching, concurrency improvements, and dependency upgrades
This commit is contained in:
parent
6032867a13
commit
e40e008429
@ -1,5 +1,13 @@
|
|||||||
# Changelog
|
# Changelog
|
||||||
|
|
||||||
|
## 2025-04-25 - 1.7.0 - feat(core)
|
||||||
|
Enhance commit fetching with caching, concurrency improvements, and dependency upgrades
|
||||||
|
|
||||||
|
- Updated development dependencies (@git.zone/tsbuild, @git.zone/tsbundle, @git.zone/tstest, @push.rocks/tapbundle, @types/node) and dependency versions
|
||||||
|
- Introduced optional caching options (enableCache, cacheWindowMs, enableNpmCheck) in the CodeFeed constructor to optimize commit retrieval
|
||||||
|
- Refactored commit fetching to use AsyncExecutionStack for controlled concurrency and improved performance
|
||||||
|
- Removed deprecated ts/codefeed.plugins.ts in favor of a consolidated plugins.ts module
|
||||||
|
|
||||||
## 2024-12-16 - 1.6.5 - fix(CodeFeed)
|
## 2024-12-16 - 1.6.5 - fix(CodeFeed)
|
||||||
Fixed timestamp initialization and commit fetching timeframe
|
Fixed timestamp initialization and commit fetching timeframe
|
||||||
|
|
||||||
|
16
package.json
16
package.json
@ -16,18 +16,19 @@
|
|||||||
"buildDocs": "(tsdoc)"
|
"buildDocs": "(tsdoc)"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@git.zone/tsbuild": "^2.1.25",
|
"@git.zone/tsbuild": "^2.3.2",
|
||||||
"@git.zone/tsbundle": "^2.0.5",
|
"@git.zone/tsbundle": "^2.2.5",
|
||||||
"@git.zone/tsrun": "^1.2.46",
|
"@git.zone/tsrun": "^1.2.46",
|
||||||
"@git.zone/tstest": "^1.0.44",
|
"@git.zone/tstest": "^1.0.96",
|
||||||
"@push.rocks/tapbundle": "^5.0.15",
|
"@push.rocks/tapbundle": "^5.6.3",
|
||||||
"@types/node": "^22.10.2"
|
"@types/node": "^22.15.2"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
|
"@push.rocks/lik": "^6.2.2",
|
||||||
"@push.rocks/qenv": "^6.1.0",
|
"@push.rocks/qenv": "^6.1.0",
|
||||||
"@push.rocks/smartnpm": "^2.0.4",
|
"@push.rocks/smartnpm": "^2.0.4",
|
||||||
"@push.rocks/smarttime": "^4.1.1",
|
"@push.rocks/smarttime": "^4.1.1",
|
||||||
"@push.rocks/smartxml": "^1.0.8"
|
"@push.rocks/smartxml": "^1.1.1"
|
||||||
},
|
},
|
||||||
"repository": {
|
"repository": {
|
||||||
"type": "git",
|
"type": "git",
|
||||||
@ -61,5 +62,6 @@
|
|||||||
"code analysis",
|
"code analysis",
|
||||||
"activity feed",
|
"activity feed",
|
||||||
"version control"
|
"version control"
|
||||||
]
|
],
|
||||||
|
"packageManager": "pnpm@10.7.0+sha512.6b865ad4b62a1d9842b61d674a393903b871d9244954f652b8842c2b553c72176b278f64c463e52d40fff8aba385c235c8c9ecf5cc7de4fd78b8bb6d49633ab6"
|
||||||
}
|
}
|
||||||
|
3368
pnpm-lock.yaml
generated
3368
pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
12
test/test.ts
12
test/test.ts
@ -9,12 +9,22 @@ let testCodeFeed: codefeed.CodeFeed;
|
|||||||
tap.test('first test', async () => {
|
tap.test('first test', async () => {
|
||||||
const token = await testQenv.getEnvVarOnDemand('GITEA_TOKEN');
|
const token = await testQenv.getEnvVarOnDemand('GITEA_TOKEN');
|
||||||
// console.log('token', token);
|
// console.log('token', token);
|
||||||
testCodeFeed = new codefeed.CodeFeed('https://code.foss.global', token);
|
// seed lastRunTimestamp to 1 year ago and enable in-memory caching for 1 year
|
||||||
|
const oneYearMs = 365 * 24 * 60 * 60 * 1000;
|
||||||
|
const oneYearAgo = new Date(Date.now() - oneYearMs).toISOString();
|
||||||
|
testCodeFeed = new codefeed.CodeFeed(
|
||||||
|
'https://code.foss.global',
|
||||||
|
token,
|
||||||
|
oneYearAgo,
|
||||||
|
{ enableCache: true, cacheWindowMs: oneYearMs, enableNpmCheck: false }
|
||||||
|
);
|
||||||
expect(testCodeFeed).toBeInstanceOf(codefeed.CodeFeed);
|
expect(testCodeFeed).toBeInstanceOf(codefeed.CodeFeed);
|
||||||
});
|
});
|
||||||
|
|
||||||
tap.test('fetchAllCommitsFromInstance', async () => {
|
tap.test('fetchAllCommitsFromInstance', async () => {
|
||||||
const commits = await testCodeFeed.fetchAllCommitsFromInstance();
|
const commits = await testCodeFeed.fetchAllCommitsFromInstance();
|
||||||
|
// log the actual results so we can inspect them
|
||||||
|
console.log('Fetched commits:', JSON.stringify(commits, null, 2));
|
||||||
expect(commits).toBeArray();
|
expect(commits).toBeArray();
|
||||||
expect(commits.length).toBeGreaterThan(0);
|
expect(commits.length).toBeGreaterThan(0);
|
||||||
// expect(commits[0]).toBeTypeofObject();
|
// expect(commits[0]).toBeTypeofObject();
|
||||||
|
@ -3,6 +3,6 @@
|
|||||||
*/
|
*/
|
||||||
export const commitinfo = {
|
export const commitinfo = {
|
||||||
name: '@foss.global/codefeed',
|
name: '@foss.global/codefeed',
|
||||||
version: '1.6.5',
|
version: '1.7.0',
|
||||||
description: 'The @foss.global/codefeed module is designed for generating feeds from Gitea repositories, enhancing development workflows by processing commit data and repository activities.'
|
description: 'The @foss.global/codefeed module is designed for generating feeds from Gitea repositories, enhancing development workflows by processing commit data and repository activities.'
|
||||||
}
|
}
|
||||||
|
512
ts/index.ts
512
ts/index.ts
@ -1,21 +1,187 @@
|
|||||||
import * as plugins from './codefeed.plugins.js';
|
import * as plugins from './plugins.js';
|
||||||
|
|
||||||
export class CodeFeed {
|
export class CodeFeed {
|
||||||
private baseUrl: string;
|
private baseUrl: string;
|
||||||
private token?: string;
|
private token?: string;
|
||||||
private npmRegistry = new plugins.smartnpm.NpmRegistry();
|
|
||||||
private smartxmlInstance = new plugins.smartxml.SmartXml();
|
|
||||||
private lastRunTimestamp: string;
|
private lastRunTimestamp: string;
|
||||||
private changelogContent: string;
|
// Raw changelog content for the current repository
|
||||||
|
private changelogContent: string = '';
|
||||||
|
// npm registry helper for published-on-npm checks
|
||||||
|
private npmRegistry: plugins.smartnpm.NpmRegistry;
|
||||||
|
// In-memory stateful cache of commits
|
||||||
|
private enableCache: boolean = false;
|
||||||
|
private cacheWindowMs?: number;
|
||||||
|
private cache: plugins.interfaces.ICommitResult[] = [];
|
||||||
|
// enable or disable npm publishedOnNpm checks (true by default)
|
||||||
|
private enableNpmCheck: boolean = true;
|
||||||
|
|
||||||
constructor(baseUrl: string, token?: string, lastRunTimestamp?: string) {
|
constructor(
|
||||||
|
baseUrl: string,
|
||||||
|
token?: string,
|
||||||
|
lastRunTimestamp?: string,
|
||||||
|
options?: {
|
||||||
|
enableCache?: boolean;
|
||||||
|
cacheWindowMs?: number;
|
||||||
|
enableNpmCheck?: boolean;
|
||||||
|
}
|
||||||
|
) {
|
||||||
this.baseUrl = baseUrl;
|
this.baseUrl = baseUrl;
|
||||||
this.token = token;
|
this.token = token;
|
||||||
this.lastRunTimestamp =
|
this.lastRunTimestamp =
|
||||||
lastRunTimestamp || new Date(Date.now() - 7 * 24 * 60 * 60 * 1000).toISOString();
|
lastRunTimestamp ?? new Date(Date.now() - 7 * 24 * 60 * 60 * 1000).toISOString();
|
||||||
|
// configure stateful caching
|
||||||
|
this.enableCache = options?.enableCache ?? false;
|
||||||
|
this.cacheWindowMs = options?.cacheWindowMs;
|
||||||
|
this.enableNpmCheck = options?.enableNpmCheck ?? true;
|
||||||
|
this.cache = [];
|
||||||
|
// npm registry instance for version lookups
|
||||||
|
this.npmRegistry = new plugins.smartnpm.NpmRegistry();
|
||||||
console.log('CodeFeed initialized with last run timestamp:', this.lastRunTimestamp);
|
console.log('CodeFeed initialized with last run timestamp:', this.lastRunTimestamp);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fetch all new commits (since lastRunTimestamp) across all orgs and repos.
|
||||||
|
*/
|
||||||
|
public async fetchAllCommitsFromInstance(): Promise<plugins.interfaces.ICommitResult[]> {
|
||||||
|
// Controlled concurrency with AsyncExecutionStack
|
||||||
|
const stack = new plugins.lik.AsyncExecutionStack();
|
||||||
|
stack.setNonExclusiveMaxConcurrency(5);
|
||||||
|
// determine since timestamp for this run (stateful caching)
|
||||||
|
let effectiveSince = this.lastRunTimestamp;
|
||||||
|
if (this.enableCache && this.cache.length > 0) {
|
||||||
|
// use newest timestamp in cache to fetch only tail
|
||||||
|
effectiveSince = this.cache.reduce(
|
||||||
|
(max, c) => (c.timestamp > max ? c.timestamp : max),
|
||||||
|
effectiveSince
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1) get all organizations
|
||||||
|
const orgs = await this.fetchAllOrganizations();
|
||||||
|
|
||||||
|
// 2) fetch repos per org in parallel
|
||||||
|
const repoLists = await Promise.all(
|
||||||
|
orgs.map((org) =>
|
||||||
|
stack.getNonExclusiveExecutionSlot(() => this.fetchRepositoriesForOrg(org))
|
||||||
|
)
|
||||||
|
);
|
||||||
|
// flatten to [{ owner, name }]
|
||||||
|
const allRepos = orgs.flatMap((org, i) =>
|
||||||
|
repoLists[i].map((r) => ({ owner: org, name: r.name }))
|
||||||
|
);
|
||||||
|
|
||||||
|
// 3) probe latest commit per repo and fetch full list only if new commits exist
|
||||||
|
const commitJobs = allRepos.map(({ owner, name }) =>
|
||||||
|
stack.getNonExclusiveExecutionSlot(async () => {
|
||||||
|
try {
|
||||||
|
// 3a) Probe the most recent commit (limit=1)
|
||||||
|
const probeResp = await this.fetchFunction(
|
||||||
|
`/api/v1/repos/${owner}/${name}/commits?limit=1`,
|
||||||
|
{ headers: this.token ? { Authorization: `token ${this.token}` } : {} }
|
||||||
|
);
|
||||||
|
if (!probeResp.ok) {
|
||||||
|
throw new Error(`Probe failed for ${owner}/${name}: ${probeResp.statusText}`);
|
||||||
|
}
|
||||||
|
const probeData: plugins.interfaces.ICommit[] = await probeResp.json();
|
||||||
|
// If no commits or no new commits since last run, skip
|
||||||
|
if (
|
||||||
|
probeData.length === 0 ||
|
||||||
|
new Date(probeData[0].commit.author.date).getTime() <=
|
||||||
|
new Date(effectiveSince).getTime()
|
||||||
|
) {
|
||||||
|
return { owner, name, commits: [] };
|
||||||
|
}
|
||||||
|
// 3b) Fetch commits since last run
|
||||||
|
const commits = await this.fetchRecentCommitsForRepo(
|
||||||
|
owner,
|
||||||
|
name,
|
||||||
|
effectiveSince
|
||||||
|
);
|
||||||
|
return { owner, name, commits };
|
||||||
|
} catch (e: any) {
|
||||||
|
console.error(`Failed to fetch commits for ${owner}/${name}:`, e.message);
|
||||||
|
return { owner, name, commits: [] };
|
||||||
|
}
|
||||||
|
})
|
||||||
|
);
|
||||||
|
const commitResults = await Promise.all(commitJobs);
|
||||||
|
|
||||||
|
// 4) build new commit entries with tagging, npm and changelog support
|
||||||
|
const newResults: plugins.interfaces.ICommitResult[] = [];
|
||||||
|
for (const { owner, name, commits } of commitResults) {
|
||||||
|
// skip repos with no new commits
|
||||||
|
if (commits.length === 0) {
|
||||||
|
this.changelogContent = '';
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// load changelog for this repo
|
||||||
|
await this.loadChangelogFromRepo(owner, name);
|
||||||
|
// fetch tags for this repo
|
||||||
|
let taggedShas: Set<string>;
|
||||||
|
try {
|
||||||
|
taggedShas = await this.fetchTags(owner, name);
|
||||||
|
} catch (e: any) {
|
||||||
|
console.error(`Failed to fetch tags for ${owner}/${name}:`, e.message);
|
||||||
|
taggedShas = new Set<string>();
|
||||||
|
}
|
||||||
|
// fetch npm package info only if any new commits correspond to a tag
|
||||||
|
const hasTaggedCommit = commits.some((c) => taggedShas.has(c.sha));
|
||||||
|
let pkgInfo: { allVersions: Array<{ version: string }> } | null = null;
|
||||||
|
if (hasTaggedCommit && this.enableNpmCheck) {
|
||||||
|
try {
|
||||||
|
pkgInfo = await this.npmRegistry.getPackageInfo(`@${owner}/${name}`);
|
||||||
|
} catch (e: any) {
|
||||||
|
console.error(`Failed to fetch package info for ${owner}/${name}:`, e.message);
|
||||||
|
pkgInfo = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// build commit entries
|
||||||
|
for (const c of commits) {
|
||||||
|
const versionCandidate = c.commit.message.replace(/\n/g, '').trim();
|
||||||
|
const isTagged = taggedShas.has(c.sha);
|
||||||
|
const publishedOnNpm = isTagged && pkgInfo
|
||||||
|
? pkgInfo.allVersions.some((v) => v.version === versionCandidate)
|
||||||
|
: false;
|
||||||
|
let changelogEntry: string | undefined;
|
||||||
|
if (this.changelogContent) {
|
||||||
|
changelogEntry = this.getChangelogForVersion(versionCandidate);
|
||||||
|
}
|
||||||
|
newResults.push({
|
||||||
|
baseUrl: this.baseUrl,
|
||||||
|
org: owner,
|
||||||
|
repo: name,
|
||||||
|
timestamp: c.commit.author.date,
|
||||||
|
prettyAgoTime: plugins.smarttime.getMilliSecondsAsHumanReadableAgoTime(
|
||||||
|
new Date(c.commit.author.date).getTime()
|
||||||
|
),
|
||||||
|
hash: c.sha,
|
||||||
|
commitMessage: c.commit.message,
|
||||||
|
tagged: isTagged,
|
||||||
|
publishedOnNpm,
|
||||||
|
changelog: changelogEntry,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// if caching is enabled, merge into in-memory cache and return full cache
|
||||||
|
if (this.enableCache) {
|
||||||
|
const existingHashes = new Set(this.cache.map((c) => c.hash));
|
||||||
|
const uniqueNew = newResults.filter((c) => !existingHashes.has(c.hash));
|
||||||
|
this.cache.push(...uniqueNew);
|
||||||
|
// trim commits older than window
|
||||||
|
if (this.cacheWindowMs !== undefined) {
|
||||||
|
const cutoff = Date.now() - this.cacheWindowMs;
|
||||||
|
this.cache = this.cache.filter((c) => new Date(c.timestamp).getTime() >= cutoff);
|
||||||
|
}
|
||||||
|
// advance lastRunTimestamp to now
|
||||||
|
this.lastRunTimestamp = new Date().toISOString();
|
||||||
|
// sort descending by timestamp
|
||||||
|
this.cache.sort((a, b) => b.timestamp.localeCompare(a.timestamp));
|
||||||
|
return this.cache;
|
||||||
|
}
|
||||||
|
// otherwise, return only newly fetched commits
|
||||||
|
return newResults;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Load the changelog directly from the Gitea repository.
|
* Load the changelog directly from the Gitea repository.
|
||||||
*/
|
*/
|
||||||
@ -27,7 +193,6 @@ export class CodeFeed {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const response = await this.fetchFunction(url, { headers });
|
const response = await this.fetchFunction(url, { headers });
|
||||||
|
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
console.error(
|
console.error(
|
||||||
`Could not fetch CHANGELOG.md from ${owner}/${repo}: ${response.status} ${response.statusText}`
|
`Could not fetch CHANGELOG.md from ${owner}/${repo}: ${response.status} ${response.statusText}`
|
||||||
@ -43,8 +208,8 @@ export class CodeFeed {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const decodedContent = Buffer.from(data.content, 'base64').toString('utf8');
|
// decode base64 content
|
||||||
this.changelogContent = decodedContent;
|
this.changelogContent = Buffer.from(data.content, 'base64').toString('utf8');
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -78,294 +243,77 @@ export class CodeFeed {
|
|||||||
|
|
||||||
return changelogLines.join('\n').trim();
|
return changelogLines.join('\n').trim();
|
||||||
}
|
}
|
||||||
|
/**
|
||||||
|
* Fetch all tags for a given repo and return the set of tagged commit SHAs
|
||||||
|
*/
|
||||||
|
private async fetchTags(owner: string, repo: string): Promise<Set<string>> {
|
||||||
|
const taggedShas = new Set<string>();
|
||||||
|
let page = 1;
|
||||||
|
while (true) {
|
||||||
|
const url = `/api/v1/repos/${owner}/${repo}/tags?limit=50&page=${page}`;
|
||||||
|
const resp = await this.fetchFunction(url, {
|
||||||
|
headers: this.token ? { Authorization: `token ${this.token}` } : {},
|
||||||
|
});
|
||||||
|
if (!resp.ok) {
|
||||||
|
console.error(`Failed to fetch tags for ${owner}/${repo}: ${resp.status} ${resp.statusText}`);
|
||||||
|
return taggedShas;
|
||||||
|
}
|
||||||
|
const data: plugins.interfaces.ITag[] = await resp.json();
|
||||||
|
if (data.length === 0) break;
|
||||||
|
for (const t of data) {
|
||||||
|
if (t.commit?.sha) taggedShas.add(t.commit.sha);
|
||||||
|
}
|
||||||
|
if (data.length < 50) break;
|
||||||
|
page++;
|
||||||
|
}
|
||||||
|
return taggedShas;
|
||||||
|
}
|
||||||
|
|
||||||
private async fetchAllOrganizations(): Promise<string[]> {
|
private async fetchAllOrganizations(): Promise<string[]> {
|
||||||
const url = `/api/v1/orgs`;
|
const resp = await this.fetchFunction('/api/v1/orgs', {
|
||||||
const response = await this.fetchFunction(url, {
|
|
||||||
headers: this.token ? { Authorization: `token ${this.token}` } : {},
|
headers: this.token ? { Authorization: `token ${this.token}` } : {},
|
||||||
});
|
});
|
||||||
|
if (!resp.ok) {
|
||||||
if (!response.ok) {
|
throw new Error(`Failed to fetch organizations: ${resp.statusText}`);
|
||||||
throw new Error(`Failed to fetch organizations: ${response.statusText}`);
|
|
||||||
}
|
}
|
||||||
|
const data: { username: string }[] = await resp.json();
|
||||||
const data: { username: string }[] = await response.json();
|
return data.map((o) => o.username);
|
||||||
return data.map((org) => org.username);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private async fetchOrgRssFeed(optionsArg: {
|
private async fetchRepositoriesForOrg(org: string): Promise<plugins.interfaces.IRepository[]> {
|
||||||
orgName: string;
|
const resp = await this.fetchFunction(`/api/v1/orgs/${org}/repos?limit=50`, {
|
||||||
repoName?: string;
|
headers: this.token ? { Authorization: `token ${this.token}` } : {},
|
||||||
}): Promise<any[]> {
|
|
||||||
let rssUrl: string;
|
|
||||||
if (optionsArg.orgName && !optionsArg.repoName) {
|
|
||||||
rssUrl = `/${optionsArg.orgName}.atom`;
|
|
||||||
} else if (optionsArg.orgName && optionsArg.repoName) {
|
|
||||||
rssUrl = `/${optionsArg.orgName}/${optionsArg.repoName}.atom`;
|
|
||||||
} else {
|
|
||||||
throw new Error('Invalid arguments provided to fetchOrgRssFeed.');
|
|
||||||
}
|
|
||||||
|
|
||||||
const response = await this.fetchFunction(rssUrl, {});
|
|
||||||
if (!response.ok) {
|
|
||||||
throw new Error(
|
|
||||||
`Failed to fetch RSS feed for organization ${optionsArg.orgName}/${optionsArg.repoName}: ${response.statusText}`
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const rssText = await response.text();
|
|
||||||
const rssData = this.smartxmlInstance.parseXmlToObject(rssText);
|
|
||||||
return rssData.feed.entry || [];
|
|
||||||
}
|
|
||||||
|
|
||||||
private async hasNewActivity(optionsArg: {
|
|
||||||
orgName: string;
|
|
||||||
repoName?: string;
|
|
||||||
}): Promise<boolean> {
|
|
||||||
const entries = await this.fetchOrgRssFeed(optionsArg);
|
|
||||||
|
|
||||||
return entries.some((entry: any) => {
|
|
||||||
const updated = new Date(entry.updated);
|
|
||||||
return updated > new Date(this.lastRunTimestamp);
|
|
||||||
});
|
});
|
||||||
}
|
if (!resp.ok) {
|
||||||
|
throw new Error(`Failed to fetch repositories for ${org}: ${resp.statusText}`);
|
||||||
private async fetchAllRepositories(): Promise<plugins.interfaces.IRepository[]> {
|
|
||||||
let page = 1;
|
|
||||||
const allRepos: plugins.interfaces.IRepository[] = [];
|
|
||||||
|
|
||||||
while (true) {
|
|
||||||
const url = `/api/v1/repos/search?limit=50&page=${page.toString()}`;
|
|
||||||
|
|
||||||
const resp = await this.fetchFunction(url, {
|
|
||||||
headers: this.token ? { Authorization: `token ${this.token}` } : {},
|
|
||||||
});
|
|
||||||
|
|
||||||
if (!resp.ok) {
|
|
||||||
throw new Error(`Failed to fetch repositories: ${resp.statusText}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const data: plugins.interfaces.IRepoSearchResponse = await resp.json();
|
|
||||||
allRepos.push(...data.data);
|
|
||||||
|
|
||||||
if (data.data.length < 50) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
page++;
|
|
||||||
}
|
}
|
||||||
|
const data: plugins.interfaces.IRepository[] = await resp.json();
|
||||||
return allRepos;
|
return data;
|
||||||
}
|
|
||||||
|
|
||||||
private async fetchTags(owner: string, repo: string): Promise<Set<string>> {
|
|
||||||
let page = 1;
|
|
||||||
const tags: plugins.interfaces.ITag[] = [];
|
|
||||||
|
|
||||||
while (true) {
|
|
||||||
const url = `/api/v1/repos/${owner}/${repo}/tags?limit=50&page=${page.toString()}`;
|
|
||||||
|
|
||||||
const resp = await this.fetchFunction(url, {
|
|
||||||
headers: this.token ? { Authorization: `token ${this.token}` } : {},
|
|
||||||
});
|
|
||||||
|
|
||||||
if (!resp.ok) {
|
|
||||||
console.error(
|
|
||||||
`Failed to fetch tags for ${owner}/${repo}: ${resp.status} ${resp.statusText} at ${url}`
|
|
||||||
);
|
|
||||||
throw new Error(`Failed to fetch tags for ${owner}/${repo}: ${resp.statusText}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const data: plugins.interfaces.ITag[] = await resp.json();
|
|
||||||
tags.push(...data);
|
|
||||||
|
|
||||||
if (data.length < 50) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
page++;
|
|
||||||
}
|
|
||||||
|
|
||||||
const taggedCommitShas = new Set<string>();
|
|
||||||
for (const t of tags) {
|
|
||||||
if (t.commit?.sha) {
|
|
||||||
taggedCommitShas.add(t.commit.sha);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return taggedCommitShas;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private async fetchRecentCommitsForRepo(
|
private async fetchRecentCommitsForRepo(
|
||||||
owner: string,
|
owner: string,
|
||||||
repo: string
|
repo: string,
|
||||||
|
sinceTimestamp?: string
|
||||||
): Promise<plugins.interfaces.ICommit[]> {
|
): Promise<plugins.interfaces.ICommit[]> {
|
||||||
const commitTimeframe = new Date(Date.now() - (7 * 24 * 60 * 60 * 1000));
|
const since = sinceTimestamp ?? this.lastRunTimestamp;
|
||||||
let page = 1;
|
const resp = await this.fetchFunction(
|
||||||
const recentCommits: plugins.interfaces.ICommit[] = [];
|
`/api/v1/repos/${owner}/${repo}/commits?since=${encodeURIComponent(
|
||||||
|
since
|
||||||
while (true) {
|
)}&limit=50`,
|
||||||
const url = `/api/v1/repos/${owner}/${repo}/commits?limit=50&page=${page.toString()}`;
|
{ headers: this.token ? { Authorization: `token ${this.token}` } : {} }
|
||||||
|
);
|
||||||
const resp = await this.fetchFunction(url, {
|
if (!resp.ok) {
|
||||||
headers: this.token ? { Authorization: `token ${this.token}` } : {},
|
throw new Error(`Failed to fetch commits for ${owner}/${repo}: ${resp.statusText}`);
|
||||||
});
|
|
||||||
|
|
||||||
if (!resp.ok) {
|
|
||||||
console.error(
|
|
||||||
`Failed to fetch commits for ${owner}/${repo}: ${resp.status} ${resp.statusText} at ${url}`
|
|
||||||
);
|
|
||||||
throw new Error(`Failed to fetch commits for ${owner}/${repo}: ${resp.statusText}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const data: plugins.interfaces.ICommit[] = await resp.json();
|
|
||||||
if (data.length === 0) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const commit of data) {
|
|
||||||
const commitDate = new Date(commit.commit.author.date);
|
|
||||||
if (commitDate > commitTimeframe) {
|
|
||||||
recentCommits.push(commit);
|
|
||||||
} else {
|
|
||||||
return recentCommits;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
page++;
|
|
||||||
}
|
}
|
||||||
|
const data: plugins.interfaces.ICommit[] = await resp.json();
|
||||||
return recentCommits;
|
return data;
|
||||||
}
|
}
|
||||||
|
|
||||||
public async fetchAllCommitsFromInstance(): Promise<plugins.interfaces.ICommitResult[]> {
|
public async fetchFunction(
|
||||||
const orgs = await this.fetchAllOrganizations();
|
urlArg: string,
|
||||||
console.log(`Found ${orgs.length} organizations`);
|
optionsArg: RequestInit = {}
|
||||||
let allCommits: plugins.interfaces.ICommitResult[] = [];
|
): Promise<Response> {
|
||||||
|
return fetch(`${this.baseUrl}${urlArg}`, optionsArg);
|
||||||
for (const orgName of orgs) {
|
|
||||||
console.log(`Checking activity for organization: ${orgName}`);
|
|
||||||
|
|
||||||
try {
|
|
||||||
const hasActivity = await this.hasNewActivity({
|
|
||||||
orgName,
|
|
||||||
});
|
|
||||||
if (!hasActivity) {
|
|
||||||
console.log(`No new activity for organization: ${orgName}`);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
} catch (error: any) {
|
|
||||||
console.error(`Error fetching activity for organization ${orgName}:`, error.message);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(`New activity detected for organization: ${orgName}. Processing repositories...`);
|
|
||||||
|
|
||||||
const repos = await this.fetchAllRepositories();
|
|
||||||
for (const r of repos.filter((repo) => repo.owner.login === orgName)) {
|
|
||||||
try {
|
|
||||||
const hasActivity = await this.hasNewActivity({
|
|
||||||
orgName,
|
|
||||||
repoName: r.name,
|
|
||||||
});
|
|
||||||
if (!hasActivity) {
|
|
||||||
console.log(`No new activity for repository: ${orgName}/${r.name}`);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
} catch (error: any) {
|
|
||||||
console.error(
|
|
||||||
`Error fetching activity for repository ${orgName}/${r.name}:`,
|
|
||||||
error.message
|
|
||||||
);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
const org = r.owner.login;
|
|
||||||
const repo = r.name;
|
|
||||||
console.log(`Processing repository ${org}/${repo}`);
|
|
||||||
|
|
||||||
try {
|
|
||||||
const taggedCommitShas = await this.fetchTags(org, repo);
|
|
||||||
const commits = await this.fetchRecentCommitsForRepo(org, repo);
|
|
||||||
|
|
||||||
// Load the changelog from this repo.
|
|
||||||
await this.loadChangelogFromRepo(org, repo);
|
|
||||||
|
|
||||||
const commitResults = commits.map((c) => {
|
|
||||||
const commit: plugins.interfaces.ICommitResult = {
|
|
||||||
baseUrl: this.baseUrl,
|
|
||||||
org,
|
|
||||||
repo,
|
|
||||||
timestamp: c.commit.author.date,
|
|
||||||
prettyAgoTime: plugins.smarttime.getMilliSecondsAsHumanReadableAgoTime(
|
|
||||||
new Date(c.commit.author.date).getTime()
|
|
||||||
),
|
|
||||||
hash: c.sha,
|
|
||||||
commitMessage: c.commit.message,
|
|
||||||
tagged: taggedCommitShas.has(c.sha),
|
|
||||||
publishedOnNpm: false,
|
|
||||||
changelog: undefined,
|
|
||||||
};
|
|
||||||
return commit;
|
|
||||||
});
|
|
||||||
|
|
||||||
if (commitResults.length > 0) {
|
|
||||||
try {
|
|
||||||
const packageInfo = await this.npmRegistry.getPackageInfo(`@${org}/${repo}`);
|
|
||||||
for (const commitResult of commitResults.filter((c) => c.tagged)) {
|
|
||||||
const versionCandidate = commitResult.commitMessage.replace('\n', '').trim();
|
|
||||||
const correspondingVersion = packageInfo.allVersions.find((versionArg) => {
|
|
||||||
return versionArg.version === versionCandidate;
|
|
||||||
});
|
|
||||||
if (correspondingVersion) {
|
|
||||||
commitResult.publishedOnNpm = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (error: any) {
|
|
||||||
console.error(`Failed to fetch package info for ${org}/${repo}:`, error.message);
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
for (const commitResult of commitResults.filter((c) => c.tagged)) {
|
|
||||||
const versionCandidate = commitResult.commitMessage.replace('\n', '').trim();
|
|
||||||
const changelogEntry = this.getChangelogForVersion(versionCandidate);
|
|
||||||
if (changelogEntry) {
|
|
||||||
commitResult.changelog = changelogEntry;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (error: any) {
|
|
||||||
console.error(`Failed to fetch changelog info for ${org}/${repo}:`, error.message);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
allCommits.push(...commitResults);
|
|
||||||
} catch (error: any) {
|
|
||||||
console.error(`Skipping repository ${org}/${repo} due to error:`, error.message);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(`Processed ${allCommits.length} commits in total.`);
|
|
||||||
|
|
||||||
allCommits = allCommits
|
|
||||||
.filter((commitArg) => commitArg.tagged)
|
|
||||||
.sort((a, b) => b.timestamp.localeCompare(a.timestamp));
|
|
||||||
|
|
||||||
console.log(`Filtered to ${allCommits.length} commits with tagged statuses.`);
|
|
||||||
|
|
||||||
for (const c of allCommits) {
|
|
||||||
console.log(` ==========================================================================
|
|
||||||
${c.prettyAgoTime} ago:
|
|
||||||
${c.org}/${c.repo}
|
|
||||||
${c.commitMessage}
|
|
||||||
Published on npm: ${c.publishedOnNpm}
|
|
||||||
${c.changelog ? `Changelog:\n${c.changelog}\n` : ''}
|
|
||||||
`);
|
|
||||||
}
|
|
||||||
|
|
||||||
return allCommits;
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
public async fetchFunction(urlArg: string, optionsArg: RequestInit): Promise<Response> {
|
|
||||||
const response = await fetch(`${this.baseUrl}${urlArg}`, optionsArg);
|
|
||||||
return response;
|
|
||||||
}
|
|
||||||
}
|
|
@ -10,10 +10,12 @@ import * as qenv from '@push.rocks/qenv';
|
|||||||
import * as smartnpm from '@push.rocks/smartnpm';
|
import * as smartnpm from '@push.rocks/smartnpm';
|
||||||
import * as smartxml from '@push.rocks/smartxml';
|
import * as smartxml from '@push.rocks/smartxml';
|
||||||
import * as smarttime from '@push.rocks/smarttime';
|
import * as smarttime from '@push.rocks/smarttime';
|
||||||
|
import * as lik from '@push.rocks/lik';
|
||||||
|
|
||||||
export {
|
export {
|
||||||
qenv,
|
qenv,
|
||||||
smartnpm,
|
smartnpm,
|
||||||
smartxml,
|
smartxml,
|
||||||
smarttime,
|
smarttime,
|
||||||
|
lik,
|
||||||
}
|
}
|
Loading…
x
Reference in New Issue
Block a user