feat(upstream): Add upstream proxy/cache subsystem and integrate per-protocol upstreams

ts/rubygems/classes.rubygemsupstream.ts

@@ -0,0 +1,230 @@
+import * as plugins from '../plugins.js';
+import { BaseUpstream } from '../upstream/classes.baseupstream.js';
+import type {
+  IProtocolUpstreamConfig,
+  IUpstreamFetchContext,
+  IUpstreamRegistryConfig,
+} from '../upstream/interfaces.upstream.js';
+
+/**
+ * RubyGems-specific upstream implementation.
+ *
+ * Handles:
+ * - Compact Index format (/versions, /info/{gem}, /names)
+ * - Gem file (.gem) downloading
+ * - Gem spec fetching
+ * - HTTP Range requests for incremental updates
+ */
+export class RubygemsUpstream extends BaseUpstream {
+  protected readonly protocolName = 'rubygems';
+
+  constructor(
+    config: IProtocolUpstreamConfig,
+    logger?: plugins.smartlog.Smartlog,
+  ) {
+    super(config, logger);
+  }
+
+  /**
+   * Fetch the /versions file (master list of all gems).
+   */
+  public async fetchVersions(etag?: string): Promise<{ data: string; etag?: string } | null> {
+    const headers: Record<string, string> = {
+      'accept': 'text/plain',
+    };
+
+    if (etag) {
+      headers['if-none-match'] = etag;
+    }
+
+    const context: IUpstreamFetchContext = {
+      protocol: 'rubygems',
+      resource: '*',
+      resourceType: 'versions',
+      path: '/versions',
+      method: 'GET',
+      headers,
+      query: {},
+    };
+
+    const result = await this.fetch(context);
+
+    if (!result || !result.success) {
+      return null;
+    }
+
+    let data: string;
+    if (Buffer.isBuffer(result.body)) {
+      data = result.body.toString('utf8');
+    } else if (typeof result.body === 'string') {
+      data = result.body;
+    } else {
+      return null;
+    }
+
+    return {
+      data,
+      etag: result.headers['etag'],
+    };
+  }
+
+  /**
+   * Fetch gem info file (/info/{gemname}).
+   */
+  public async fetchInfo(gemName: string): Promise<string | null> {
+    const context: IUpstreamFetchContext = {
+      protocol: 'rubygems',
+      resource: gemName,
+      resourceType: 'info',
+      path: `/info/${gemName}`,
+      method: 'GET',
+      headers: {
+        'accept': 'text/plain',
+      },
+      query: {},
+    };
+
+    const result = await this.fetch(context);
+
+    if (!result || !result.success) {
+      return null;
+    }
+
+    if (Buffer.isBuffer(result.body)) {
+      return result.body.toString('utf8');
+    }
+
+    return typeof result.body === 'string' ? result.body : null;
+  }
+
+  /**
+   * Fetch the /names file (list of all gem names).
+   */
+  public async fetchNames(): Promise<string | null> {
+    const context: IUpstreamFetchContext = {
+      protocol: 'rubygems',
+      resource: '*',
+      resourceType: 'names',
+      path: '/names',
+      method: 'GET',
+      headers: {
+        'accept': 'text/plain',
+      },
+      query: {},
+    };
+
+    const result = await this.fetch(context);
+
+    if (!result || !result.success) {
+      return null;
+    }
+
+    if (Buffer.isBuffer(result.body)) {
+      return result.body.toString('utf8');
+    }
+
+    return typeof result.body === 'string' ? result.body : null;
+  }
+
+  /**
+   * Fetch a gem file.
+   */
+  public async fetchGem(gemName: string, version: string): Promise<Buffer | null> {
+    const path = `/gems/${gemName}-${version}.gem`;
+
+    const context: IUpstreamFetchContext = {
+      protocol: 'rubygems',
+      resource: gemName,
+      resourceType: 'gem',
+      path,
+      method: 'GET',
+      headers: {
+        'accept': 'application/octet-stream',
+      },
+      query: {},
+    };
+
+    const result = await this.fetch(context);
+
+    if (!result || !result.success) {
+      return null;
+    }
+
+    return Buffer.isBuffer(result.body) ? result.body : Buffer.from(result.body);
+  }
+
+  /**
+   * Fetch gem spec (quick spec).
+   */
+  public async fetchQuickSpec(gemName: string, version: string): Promise<Buffer | null> {
+    const path = `/quick/Marshal.4.8/${gemName}-${version}.gemspec.rz`;
+
+    const context: IUpstreamFetchContext = {
+      protocol: 'rubygems',
+      resource: gemName,
+      resourceType: 'spec',
+      path,
+      method: 'GET',
+      headers: {
+        'accept': 'application/octet-stream',
+      },
+      query: {},
+    };
+
+    const result = await this.fetch(context);
+
+    if (!result || !result.success) {
+      return null;
+    }
+
+    return Buffer.isBuffer(result.body) ? result.body : Buffer.from(result.body);
+  }
+
+  /**
+   * Fetch gem versions JSON from API.
+   */
+  public async fetchVersionsJson(gemName: string): Promise<any[] | null> {
+    const path = `/api/v1/versions/${gemName}.json`;
+
+    const context: IUpstreamFetchContext = {
+      protocol: 'rubygems',
+      resource: gemName,
+      resourceType: 'versions-json',
+      path,
+      method: 'GET',
+      headers: {
+        'accept': 'application/json',
+      },
+      query: {},
+    };
+
+    const result = await this.fetch(context);
+
+    if (!result || !result.success) {
+      return null;
+    }
+
+    if (Buffer.isBuffer(result.body)) {
+      return JSON.parse(result.body.toString('utf8'));
+    }
+
+    return Array.isArray(result.body) ? result.body : null;
+  }
+
+  /**
+   * Override URL building for RubyGems-specific handling.
+   */
+  protected buildUpstreamUrl(
+    upstream: IUpstreamRegistryConfig,
+    context: IUpstreamFetchContext,
+  ): string {
+    let baseUrl = upstream.url;
+
+    // Remove trailing slash
+    if (baseUrl.endsWith('/')) {
+      baseUrl = baseUrl.slice(0, -1);
+    }
+
+    return `${baseUrl}${context.path}`;
+  }
+}