codefeed/ts/index.ts

import * as plugins from './codefeed.plugins.js';


export class CodeFeed {
  private baseUrl: string;
  private token?: string;
  private npmRegistry = new plugins.smartnpm.NpmRegistry();
  private smartxmlInstance = new plugins.smartxml.SmartXml();
  private lastRunTimestamp: string;

  constructor(baseUrl: string, token?: string, lastRunTimestamp?: string) {
    this.baseUrl = baseUrl;
    this.token = token;
    this.lastRunTimestamp = lastRunTimestamp || new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString();
    console.log('CodeFeed initialized with last run timestamp:', this.lastRunTimestamp);
  }

  /**
   * Fetch all organizations from the Gitea instance.
   */
  private async fetchAllOrganizations(): Promise<string[]> {
    const url = `${this.baseUrl}/api/v1/orgs`;
    const response = await fetch(url, {
      headers: this.token ? { Authorization: `token ${this.token}` } : {},
    });

    if (!response.ok) {
      throw new Error(`Failed to fetch organizations: ${response.statusText}`);
    }

    const data: { username: string }[] = await response.json();
    return data.map((org) => org.username);
  }

  /**
   * Fetch organization-level activity RSS feed.
   */
  private async fetchOrgRssFeed(optionsArg: {
    orgName: string,
    repoName?: string,
  }): Promise<any[]> {
    let rssUrl: string
    if (optionsArg.orgName && !optionsArg.repoName) {
      rssUrl = `${this.baseUrl}/${optionsArg.orgName}.atom`;
    } else if (optionsArg.orgName && optionsArg.repoName) {
      rssUrl = `${this.baseUrl}/${optionsArg.orgName}/${optionsArg.repoName}.atom`;
    }

    const response = await fetch(rssUrl);
    if (!response.ok) {
      throw new Error(`Failed to fetch RSS feed for organization ${optionsArg.orgName}/${optionsArg.repoName}: ${response.statusText}`);
    }

    const rssText = await response.text();

    // Parse the Atom feed using fast-xml-parser
    const rssData = this.smartxmlInstance.parseXmlToObject(rssText);

    // Return the <entry> elements from the feed
    return rssData.feed.entry || [];
  }

  /**
   * Check if the organization's RSS feed has any new activities since the last run.
   */
  private async hasNewActivity(optionsArg: {
    orgName: string,
    repoName?: string,
  }): Promise<boolean> {
    const entries = await this.fetchOrgRssFeed(optionsArg);

    // Filter entries to find new activities since the last run
    return entries.some((entry: any) => {
      const updated = new Date(entry.updated);
      return updated > new Date(this.lastRunTimestamp);
    });
  }

  /**
   * Fetch all repositories accessible to the token/user.
   */
  private async fetchAllRepositories(): Promise<plugins.interfaces.Repository[]> {
    let page = 1;
    const allRepos: plugins.interfaces.Repository[] = [];

    while (true) {
      const url = new URL(`${this.baseUrl}/api/v1/repos/search`);
      url.searchParams.set('limit', '50');
      url.searchParams.set('page', page.toString());

      const resp = await fetch(url.href, {
        headers: this.token ? { 'Authorization': `token ${this.token}` } : {},
      });

      if (!resp.ok) {
        throw new Error(`Failed to fetch repositories: ${resp.statusText}`);
      }

      const data: plugins.interfaces.RepoSearchResponse = await resp.json();
      allRepos.push(...data.data);

      if (data.data.length < 50) {
        break;
      }
      page++;
    }

    return allRepos;
  }

  /**
   * Fetch all tags for a given repository.
   */
  private async fetchTags(owner: string, repo: string): Promise<Set<string>> {
    let page = 1;
    const tags: plugins.interfaces.Tag[] = [];

    while (true) {
      const url = new URL(`${this.baseUrl}/api/v1/repos/${owner}/${repo}/tags`);
      url.searchParams.set('limit', '50');
      url.searchParams.set('page', page.toString());

      const resp = await fetch(url.href, {
        headers: this.token ? { 'Authorization': `token ${this.token}` } : {},
      });

      if (!resp.ok) {
        console.error(`Failed to fetch tags for ${owner}/${repo}: ${resp.status} ${resp.statusText} at ${url.href}`);
        throw new Error(`Failed to fetch tags for ${owner}/${repo}: ${resp.statusText}`);
      }

      const data: plugins.interfaces.Tag[] = await resp.json();
      tags.push(...data);

      if (data.length < 50) {
        break;
      }
      page++;
    }

    const taggedCommitShas = new Set<string>();
    for (const t of tags) {
      if (t.commit?.sha) {
        taggedCommitShas.add(t.commit.sha);
      }
    }

    return taggedCommitShas;
  }

  /**
   * Fetch commits from the last 24 hours for a repository.
   */
  private async fetchRecentCommitsForRepo(owner: string, repo: string): Promise<plugins.interfaces.Commit[]> {
    const twentyFourHoursAgo = new Date(Date.now() - 24 * 60 * 60 * 1000);
    let page = 1;
    const recentCommits: plugins.interfaces.Commit[] = [];

    while (true) {
      const url = new URL(`${this.baseUrl}/api/v1/repos/${owner}/${repo}/commits`);
      url.searchParams.set('limit', '50');
      url.searchParams.set('page', page.toString());

      const resp = await fetch(url.href, {
        headers: this.token ? { 'Authorization': `token ${this.token}` } : {},
      });

      if (!resp.ok) {
        console.error(`Failed to fetch commits for ${owner}/${repo}: ${resp.status} ${resp.statusText} at ${url.href}`);
        throw new Error(`Failed to fetch commits for ${owner}/${repo}: ${resp.statusText}`);
      }

      const data: plugins.interfaces.Commit[] = await resp.json();
      if (data.length === 0) {
        break;
      }

      for (const commit of data) {
        const commitDate = new Date(commit.commit.author.date);
        if (commitDate > twentyFourHoursAgo) {
          recentCommits.push(commit);
        } else {
          return recentCommits;
        }
      }

      page++;
    }

    return recentCommits;
  }

  /**
   * Fetch all commits by querying all organizations.
   */
  public async fetchAllCommitsFromInstance(): Promise<plugins.interfaces.CommitResult[]> {
    const orgs = await this.fetchAllOrganizations();
    console.log(`Found ${orgs.length} organizations`);
    let allCommits: plugins.interfaces.CommitResult[] = [];

    for (const orgName of orgs) {
      console.log(`Checking activity for organization: ${orgName}`);

      try {
        const hasActivity = await this.hasNewActivity({
          orgName,
        });
        if (!hasActivity) {
          console.log(`No new activity for organization: ${orgName}`);
          continue;
        }
      } catch (error) {
        console.error(`Error fetching activity for organization ${orgName}:`, error.message);
        continue;
      }

      console.log(`New activity detected for organization: ${orgName}. Processing repositories...`);

      const repos = await this.fetchAllRepositories();
      for (const r of repos.filter((repo) => repo.owner.login === orgName)) {
        try {
          const hasActivity = await this.hasNewActivity({
            orgName,
            repoName: r.name,
          });
          if (!hasActivity) {
            console.log(`No new activity for repository: ${orgName}/${r.name}`);
            continue;
          }
        } catch (error) {
          console.error(`Error fetching activity for repository ${orgName}/${r.name}:`, error.message);
          continue;
        }
        const org = r.owner.login;
        const repo = r.name;
        console.log(`Processing repository ${org}/${repo}`);

        try {
          const taggedCommitShas = await this.fetchTags(org, repo);
          const commits = await this.fetchRecentCommitsForRepo(org, repo);

          const commitResults = commits.map((c) => {
            const commit: plugins.interfaces.CommitResult = {
              baseUrl: this.baseUrl,
              org,
              repo,
              timestamp: c.commit.author.date,
              prettyAgoTime: plugins.smarttime.getMilliSecondsAsHumanReadableAgoTime(new Date(c.commit.author.date).getTime()),
              hash: c.sha,
              commitMessage: c.commit.message,
              tagged: taggedCommitShas.has(c.sha),
              publishedOnNpm: false,
            }
            return commit;
          });

          if (commitResults.length > 0) {
            try {
              const packageInfo = await this.npmRegistry.getPackageInfo(`@${org}/${repo}`);
              for (const commit of commitResults.filter((c) => c.tagged)) {
                const correspondingVersion = packageInfo.allVersions.find((versionArg) => {
                  return versionArg.version === commit.commitMessage.replace('\n', '');
                });
                if (correspondingVersion) {
                  commit.publishedOnNpm = true;
                }
              }
            } catch (error) {
              console.error(`Failed to fetch package info for ${org}/${repo}:`, error.message);
            }
          }

          allCommits.push(...commitResults);
        } catch (error) {
          console.error(`Skipping repository ${org}/${repo} due to error:`, error.message);
        }
      }
    }

    console.log(`Processed ${allCommits.length} commits in total.`);
    for (const c of allCommits) {
      console.log(`      ==========================================================================
      ${c.prettyAgoTime} ago:
      ${c.org}/${c.repo}
      ${c.commitMessage}
      Published on npm: ${c.publishedOnNpm}
    `);  
    }

    allCommits = allCommits.filter(commitArg => commitArg.tagged);

    return allCommits;
  }
}