feat(upstream): Add upstream proxy/cache subsystem and integrate per-protocol upstreams

ts/cargo/classes.cargoregistry.ts

@@ -3,6 +3,7 @@ import { BaseRegistry } from '../core/classes.baseregistry.js';
 import { RegistryStorage } from '../core/classes.registrystorage.js';
 import { AuthManager } from '../core/classes.authmanager.js';
 import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js';
+import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js';
 import type {
   ICargoIndexEntry,
   ICargoPublishMetadata,
@@ -13,6 +14,7 @@ import type {
   ICargoSearchResponse,
   ICargoSearchResult,
 } from './interfaces.cargo.js';
+import { CargoUpstream } from './classes.cargoupstream.js';
 
 /**
  * Cargo/crates.io registry implementation
@@ -25,12 +27,14 @@ export class CargoRegistry extends BaseRegistry {
   private basePath: string = '/cargo';
   private registryUrl: string;
   private logger: Smartlog;
+  private upstream: CargoUpstream | null = null;
 
   constructor(
     storage: RegistryStorage,
     authManager: AuthManager,
     basePath: string = '/cargo',
-    registryUrl: string = 'http://localhost:5000/cargo'
+    registryUrl: string = 'http://localhost:5000/cargo',
+    upstreamConfig?: IProtocolUpstreamConfig
   ) {
     super();
     this.storage = storage;
@@ -50,6 +54,20 @@ export class CargoRegistry extends BaseRegistry {
       }
     });
     this.logger.enableConsole();
+
+    // Initialize upstream if configured
+    if (upstreamConfig?.enabled) {
+      this.upstream = new CargoUpstream(upstreamConfig, undefined, this.logger);
+    }
+  }
+
+  /**
+   * Clean up resources (timers, connections, etc.)
+   */
+  public destroy(): void {
+    if (this.upstream) {
+      this.upstream.stop();
+    }
   }
 
   public async init(): Promise<void> {
@@ -207,7 +225,25 @@ export class CargoRegistry extends BaseRegistry {
    * Serve index file for a crate
    */
   private async handleIndexFile(crateName: string): Promise<IResponse> {
-    const index = await this.storage.getCargoIndex(crateName);
+    let index = await this.storage.getCargoIndex(crateName);
+
+    // Try upstream if not found locally
+    if ((!index || index.length === 0) && this.upstream) {
+      const upstreamIndex = await this.upstream.fetchCrateIndex(crateName);
+      if (upstreamIndex) {
+        // Parse the newline-delimited JSON
+        const parsedIndex: ICargoIndexEntry[] = upstreamIndex
+          .split('\n')
+          .filter(line => line.trim())
+          .map(line => JSON.parse(line));
+
+        if (parsedIndex.length > 0) {
+          // Cache locally
+          await this.storage.putCargoIndex(crateName, parsedIndex);
+          index = parsedIndex;
+        }
+      }
+    }
 
     if (!index || index.length === 0) {
       return {
@@ -399,7 +435,16 @@ export class CargoRegistry extends BaseRegistry {
   ): Promise<IResponse> {
     this.logger.log('debug', 'handleDownload', { crate: crateName, version });
 
-    const crateFile = await this.storage.getCargoCrate(crateName, version);
+    let crateFile = await this.storage.getCargoCrate(crateName, version);
+
+    // Try upstream if not found locally
+    if (!crateFile && this.upstream) {
+      crateFile = await this.upstream.fetchCrate(crateName, version);
+      if (crateFile) {
+        // Cache locally
+        await this.storage.putCargoCrate(crateName, version, crateFile);
+      }
+    }
 
     if (!crateFile) {
       return {

ts/cargo/classes.cargoupstream.ts

@@ -0,0 +1,159 @@
+import * as plugins from '../plugins.js';
+import { BaseUpstream } from '../upstream/classes.baseupstream.js';
+import type {
+  IProtocolUpstreamConfig,
+  IUpstreamFetchContext,
+  IUpstreamRegistryConfig,
+} from '../upstream/interfaces.upstream.js';
+
+/**
+ * Cargo-specific upstream implementation.
+ *
+ * Handles:
+ * - Crate metadata (index) fetching
+ * - Crate file (.crate) downloading
+ * - Sparse index protocol support
+ * - Content-addressable caching for .crate files
+ */
+export class CargoUpstream extends BaseUpstream {
+  protected readonly protocolName = 'cargo';
+
+  /** Base URL for crate downloads (may differ from index URL) */
+  private readonly downloadUrl: string;
+
+  constructor(
+    config: IProtocolUpstreamConfig,
+    downloadUrl?: string,
+    logger?: plugins.smartlog.Smartlog,
+  ) {
+    super(config, logger);
+    // Default to crates.io download URL if not specified
+    this.downloadUrl = downloadUrl || 'https://static.crates.io/crates';
+  }
+
+  /**
+   * Fetch crate metadata from the sparse index.
+   */
+  public async fetchCrateIndex(crateName: string): Promise<string | null> {
+    const path = this.buildIndexPath(crateName);
+
+    const context: IUpstreamFetchContext = {
+      protocol: 'cargo',
+      resource: crateName,
+      resourceType: 'index',
+      path,
+      method: 'GET',
+      headers: {
+        'accept': 'text/plain',
+      },
+      query: {},
+    };
+
+    const result = await this.fetch(context);
+
+    if (!result || !result.success) {
+      return null;
+    }
+
+    if (Buffer.isBuffer(result.body)) {
+      return result.body.toString('utf8');
+    }
+
+    return typeof result.body === 'string' ? result.body : null;
+  }
+
+  /**
+   * Fetch a crate file from upstream.
+   */
+  public async fetchCrate(crateName: string, version: string): Promise<Buffer | null> {
+    // Crate downloads typically go to a different URL than the index
+    const path = `/${crateName}/${crateName}-${version}.crate`;
+
+    const context: IUpstreamFetchContext = {
+      protocol: 'cargo',
+      resource: crateName,
+      resourceType: 'crate',
+      path,
+      method: 'GET',
+      headers: {
+        'accept': 'application/octet-stream',
+      },
+      query: {},
+    };
+
+    // Use special handling for crate downloads
+    const result = await this.fetchCrateFile(crateName, version);
+    return result;
+  }
+
+  /**
+   * Fetch crate file directly from the download URL.
+   */
+  private async fetchCrateFile(crateName: string, version: string): Promise<Buffer | null> {
+    const context: IUpstreamFetchContext = {
+      protocol: 'cargo',
+      resource: crateName,
+      resourceType: 'crate',
+      path: `/${crateName}/${crateName}-${version}.crate`,
+      method: 'GET',
+      headers: {
+        'accept': 'application/octet-stream',
+      },
+      query: {},
+    };
+
+    const result = await this.fetch(context);
+
+    if (!result || !result.success) {
+      return null;
+    }
+
+    return Buffer.isBuffer(result.body) ? result.body : Buffer.from(result.body);
+  }
+
+  /**
+   * Build the sparse index path for a crate.
+   *
+   * Path structure:
+   * - 1 char: /1/{name}
+   * - 2 chars: /2/{name}
+   * - 3 chars: /3/{first char}/{name}
+   * - 4+ chars: /{first 2}/{next 2}/{name}
+   */
+  private buildIndexPath(crateName: string): string {
+    const lowerName = crateName.toLowerCase();
+    const len = lowerName.length;
+
+    if (len === 1) {
+      return `/1/${lowerName}`;
+    } else if (len === 2) {
+      return `/2/${lowerName}`;
+    } else if (len === 3) {
+      return `/3/${lowerName[0]}/${lowerName}`;
+    } else {
+      return `/${lowerName.slice(0, 2)}/${lowerName.slice(2, 4)}/${lowerName}`;
+    }
+  }
+
+  /**
+   * Override URL building for Cargo-specific handling.
+   */
+  protected buildUpstreamUrl(
+    upstream: IUpstreamRegistryConfig,
+    context: IUpstreamFetchContext,
+  ): string {
+    let baseUrl = upstream.url;
+
+    // For crate downloads, use the download URL
+    if (context.resourceType === 'crate') {
+      baseUrl = this.downloadUrl;
+    }
+
+    // Remove trailing slash
+    if (baseUrl.endsWith('/')) {
+      baseUrl = baseUrl.slice(0, -1);
+    }
+
+    return `${baseUrl}${context.path}`;
+  }
+}

ts/cargo/index.ts

@@ -3,4 +3,5 @@
  */
 
 export { CargoRegistry } from './classes.cargoregistry.js';
+export { CargoUpstream } from './classes.cargoupstream.js';
 export * from './interfaces.cargo.js';