codefeed/ts/index.ts

import * as plugins from './codefeed.plugins.js';

export class CodeFeed {
  private baseUrl: string;
  private token?: string;
  private npmRegistry = new plugins.smartnpm.NpmRegistry();
  private smartxmlInstance = new plugins.smartxml.SmartXml();
  private lastRunTimestamp: string;
  private changelogContent: string;

  constructor(baseUrl: string, token?: string, lastRunTimestamp?: string) {
    this.baseUrl = baseUrl;
    this.token = token;
    this.lastRunTimestamp = lastRunTimestamp || new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString();
    console.log('CodeFeed initialized with last run timestamp:', this.lastRunTimestamp);
  }

  /**
   * Load the changelog directly from the Gitea repository.
   */
  private async loadChangelogFromRepo(owner: string, repo: string): Promise<void> {
    const url = `${this.baseUrl}/api/v1/repos/${owner}/${repo}/contents/changelog.md`;
    const headers: Record<string, string> = {};
    if (this.token) {
      headers['Authorization'] = `token ${this.token}`;
    }

    const response = await fetch(url, { headers });

    if (!response.ok) {
      console.error(`Could not fetch CHANGELOG.md from ${owner}/${repo}: ${response.status} ${response.statusText}`);
      this.changelogContent = '';
      return;
    }

    const data = await response.json();
    if (!data.content) {
      console.warn(`No content field found in response for ${owner}/${repo}/changelog.md`);
      this.changelogContent = '';
      return;
    }

    const decodedContent = Buffer.from(data.content, 'base64').toString('utf8');
    this.changelogContent = decodedContent;
  }

  /**
   * Parse the changelog to find the entry for a given version.
   * The changelog format is assumed as:
   *
   * # Changelog
   *
   * ## <date> - <version> - <description>
   * <changes...>
   */
  private getChangelogForVersion(version: string): string | undefined {
    if (!this.changelogContent) {
      return undefined;
    }
    const lines = this.changelogContent.split('\n');
    const versionHeaderIndex = lines.findIndex((line) => line.includes(`- ${version} -`));
    if (versionHeaderIndex === -1) {
      return undefined;
    }

    const changelogLines: string[] = [];
    for (let i = versionHeaderIndex + 1; i < lines.length; i++) {
      const line = lines[i];
      // The next version header starts with `## `
      if (line.startsWith('## ')) {
        break;
      }
      changelogLines.push(line);
    }

    return changelogLines.join('\n').trim();
  }

  private async fetchAllOrganizations(): Promise<string[]> {
    const url = `${this.baseUrl}/api/v1/orgs`;
    const response = await fetch(url, {
      headers: this.token ? { Authorization: `token ${this.token}` } : {},
    });

    if (!response.ok) {
      throw new Error(`Failed to fetch organizations: ${response.statusText}`);
    }

    const data: { username: string }[] = await response.json();
    return data.map((org) => org.username);
  }

  private async fetchOrgRssFeed(optionsArg: {
    orgName: string,
    repoName?: string,
  }): Promise<any[]> {
    let rssUrl: string;
    if (optionsArg.orgName && !optionsArg.repoName) {
      rssUrl = `${this.baseUrl}/${optionsArg.orgName}.atom`;
    } else if (optionsArg.orgName && optionsArg.repoName) {
      rssUrl = `${this.baseUrl}/${optionsArg.orgName}/${optionsArg.repoName}.atom`;
    } else {
      throw new Error('Invalid arguments provided to fetchOrgRssFeed.');
    }

    const response = await fetch(rssUrl);
    if (!response.ok) {
      throw new Error(`Failed to fetch RSS feed for organization ${optionsArg.orgName}/${optionsArg.repoName}: ${response.statusText}`);
    }

    const rssText = await response.text();
    const rssData = this.smartxmlInstance.parseXmlToObject(rssText);
    return rssData.feed.entry || [];
  }

  private async hasNewActivity(optionsArg: {
    orgName: string,
    repoName?: string,
  }): Promise<boolean> {
    const entries = await this.fetchOrgRssFeed(optionsArg);

    return entries.some((entry: any) => {
      const updated = new Date(entry.updated);
      return updated > new Date(this.lastRunTimestamp);
    });
  }

  private async fetchAllRepositories(): Promise<plugins.interfaces.IRepository[]> {
    let page = 1;
    const allRepos: plugins.interfaces.IRepository[] = [];

    while (true) {
      const url = new URL(`${this.baseUrl}/api/v1/repos/search`);
      url.searchParams.set('limit', '50');
      url.searchParams.set('page', page.toString());

      const resp = await fetch(url.href, {
        headers: this.token ? { 'Authorization': `token ${this.token}` } : {},
      });

      if (!resp.ok) {
        throw new Error(`Failed to fetch repositories: ${resp.statusText}`);
      }

      const data: plugins.interfaces.IRepoSearchResponse = await resp.json();
      allRepos.push(...data.data);

      if (data.data.length < 50) {
        break;
      }
      page++;
    }

    return allRepos;
  }

  private async fetchTags(owner: string, repo: string): Promise<Set<string>> {
    let page = 1;
    const tags: plugins.interfaces.ITag[] = [];

    while (true) {
      const url = new URL(`${this.baseUrl}/api/v1/repos/${owner}/${repo}/tags`);
      url.searchParams.set('limit', '50');
      url.searchParams.set('page', page.toString());

      const resp = await fetch(url.href, {
        headers: this.token ? { 'Authorization': `token ${this.token}` } : {},
      });

      if (!resp.ok) {
        console.error(`Failed to fetch tags for ${owner}/${repo}: ${resp.status} ${resp.statusText} at ${url.href}`);
        throw new Error(`Failed to fetch tags for ${owner}/${repo}: ${resp.statusText}`);
      }

      const data: plugins.interfaces.ITag[] = await resp.json();
      tags.push(...data);

      if (data.length < 50) {
        break;
      }
      page++;
    }

    const taggedCommitShas = new Set<string>();
    for (const t of tags) {
      if (t.commit?.sha) {
        taggedCommitShas.add(t.commit.sha);
      }
    }

    return taggedCommitShas;
  }

  private async fetchRecentCommitsForRepo(owner: string, repo: string): Promise<plugins.interfaces.ICommit[]> {
    const twentyFourHoursAgo = new Date(Date.now() - 24 * 60 * 60 * 1000);
    let page = 1;
    const recentCommits: plugins.interfaces.ICommit[] = [];

    while (true) {
      const url = new URL(`${this.baseUrl}/api/v1/repos/${owner}/${repo}/commits`);
      url.searchParams.set('limit', '50');
      url.searchParams.set('page', page.toString());

      const resp = await fetch(url.href, {
        headers: this.token ? { 'Authorization': `token ${this.token}` } : {},
      });

      if (!resp.ok) {
        console.error(`Failed to fetch commits for ${owner}/${repo}: ${resp.status} ${resp.statusText} at ${url.href}`);
        throw new Error(`Failed to fetch commits for ${owner}/${repo}: ${resp.statusText}`);
      }

      const data: plugins.interfaces.ICommit[] = await resp.json();
      if (data.length === 0) {
        break;
      }

      for (const commit of data) {
        const commitDate = new Date(commit.commit.author.date);
        if (commitDate > twentyFourHoursAgo) {
          recentCommits.push(commit);
        } else {
          return recentCommits;
        }
      }

      page++;
    }

    return recentCommits;
  }

  public async fetchAllCommitsFromInstance(): Promise<plugins.interfaces.ICommitResult[]> {
    const orgs = await this.fetchAllOrganizations();
    console.log(`Found ${orgs.length} organizations`);
    let allCommits: plugins.interfaces.ICommitResult[] = [];

    for (const orgName of orgs) {
      console.log(`Checking activity for organization: ${orgName}`);

      try {
        const hasActivity = await this.hasNewActivity({
          orgName,
        });
        if (!hasActivity) {
          console.log(`No new activity for organization: ${orgName}`);
          continue;
        }
      } catch (error: any) {
        console.error(`Error fetching activity for organization ${orgName}:`, error.message);
        continue;
      }

      console.log(`New activity detected for organization: ${orgName}. Processing repositories...`);

      const repos = await this.fetchAllRepositories();
      for (const r of repos.filter((repo) => repo.owner.login === orgName)) {
        try {
          const hasActivity = await this.hasNewActivity({
            orgName,
            repoName: r.name,
          });
          if (!hasActivity) {
            console.log(`No new activity for repository: ${orgName}/${r.name}`);
            continue;
          }
        } catch (error: any) {
          console.error(`Error fetching activity for repository ${orgName}/${r.name}:`, error.message);
          continue;
        }

        const org = r.owner.login;
        const repo = r.name;
        console.log(`Processing repository ${org}/${repo}`);

        try {
          const taggedCommitShas = await this.fetchTags(org, repo);
          const commits = await this.fetchRecentCommitsForRepo(org, repo);

          // Load the changelog from this repo.
          await this.loadChangelogFromRepo(org, repo);

          const commitResults = commits.map((c) => {
            const commit: plugins.interfaces.ICommitResult = {
              baseUrl: this.baseUrl,
              org,
              repo,
              timestamp: c.commit.author.date,
              prettyAgoTime: plugins.smarttime.getMilliSecondsAsHumanReadableAgoTime(new Date(c.commit.author.date).getTime()),
              hash: c.sha,
              commitMessage: c.commit.message,
              tagged: taggedCommitShas.has(c.sha),
              publishedOnNpm: false,
              changelog: undefined
            };
            return commit;
          });

          if (commitResults.length > 0) {
            try {
              const packageInfo = await this.npmRegistry.getPackageInfo(`@${org}/${repo}`);
              for (const commitResult of commitResults.filter((c) => c.tagged)) {
                const versionCandidate = commitResult.commitMessage.replace('\n', '').trim();
                const correspondingVersion = packageInfo.allVersions.find((versionArg) => {
                  return versionArg.version === versionCandidate;
                });
                if (correspondingVersion) {
                  commitResult.publishedOnNpm = true;
                  const changelogEntry = this.getChangelogForVersion(versionCandidate);
                  if (changelogEntry) {
                    commitResult.changelog = changelogEntry;
                  }
                }
              }
            } catch (error: any) {
              console.error(`Failed to fetch package info for ${org}/${repo}:`, error.message);
            }
          }

          allCommits.push(...commitResults);
        } catch (error: any) {
          console.error(`Skipping repository ${org}/${repo} due to error:`, error.message);
        }
      }
    }

    console.log(`Processed ${allCommits.length} commits in total.`);

    allCommits = allCommits.filter(commitArg => commitArg.tagged);

    console.log(`Filtered to ${allCommits.length} commits with tagged statuses.`);

    for (const c of allCommits) {
      console.log(`      ==========================================================================
      ${c.prettyAgoTime} ago:
      ${c.org}/${c.repo}
      ${c.commitMessage}
      Published on npm: ${c.publishedOnNpm}
      ${c.changelog ? `Changelog:\n${c.changelog}\n` : ''}
    `);  
    }

    return allCommits;
  }
}