feat(upstream): Add upstream proxy/cache subsystem and integrate per-protocol upstreams

ts/pypi/classes.pypiregistry.ts

@@ -3,6 +3,7 @@ import { BaseRegistry } from '../core/classes.baseregistry.js';
 import { RegistryStorage } from '../core/classes.registrystorage.js';
 import { AuthManager } from '../core/classes.authmanager.js';
 import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js';
+import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js';
 import { isBinaryData, toBuffer } from '../core/helpers.buffer.js';
 import type {
   IPypiPackageMetadata,
@@ -11,6 +12,7 @@ import type {
   IPypiUploadResponse,
 } from './interfaces.pypi.js';
 import * as helpers from './helpers.pypi.js';
+import { PypiUpstream } from './classes.pypiupstream.js';
 
 /**
  * PyPI registry implementation
@@ -22,12 +24,14 @@ export class PypiRegistry extends BaseRegistry {
   private basePath: string = '/pypi';
   private registryUrl: string;
   private logger: Smartlog;
+  private upstream: PypiUpstream | null = null;
 
   constructor(
     storage: RegistryStorage,
     authManager: AuthManager,
     basePath: string = '/pypi',
-    registryUrl: string = 'http://localhost:5000'
+    registryUrl: string = 'http://localhost:5000',
+    upstreamConfig?: IProtocolUpstreamConfig
   ) {
     super();
     this.storage = storage;
@@ -47,6 +51,20 @@ export class PypiRegistry extends BaseRegistry {
       }
     });
     this.logger.enableConsole();
+
+    // Initialize upstream if configured
+    if (upstreamConfig?.enabled) {
+      this.upstream = new PypiUpstream(upstreamConfig, registryUrl, this.logger);
+    }
+  }
+
+  /**
+   * Clean up resources (timers, connections, etc.)
+   */
+  public destroy(): void {
+    if (this.upstream) {
+      this.upstream.stop();
+    }
   }
 
   public async init(): Promise<void> {
@@ -214,7 +232,45 @@ export class PypiRegistry extends BaseRegistry {
     const normalized = helpers.normalizePypiPackageName(packageName);
 
     // Get package metadata
-    const metadata = await this.storage.getPypiPackageMetadata(normalized);
+    let metadata = await this.storage.getPypiPackageMetadata(normalized);
+
+    // Try upstream if not found locally
+    if (!metadata && this.upstream) {
+      const upstreamHtml = await this.upstream.fetchSimplePackage(normalized);
+      if (upstreamHtml) {
+        // Parse the HTML to extract file information and cache it
+        // For now, just return the upstream HTML directly (caching can be improved later)
+        const acceptHeader = context.headers['accept'] || context.headers['Accept'] || '';
+        const preferJson = acceptHeader.includes('application/vnd.pypi.simple') &&
+                           acceptHeader.includes('json');
+
+        if (preferJson) {
+          // Try to get JSON format from upstream
+          const upstreamJson = await this.upstream.fetchPackageJson(normalized);
+          if (upstreamJson) {
+            return {
+              status: 200,
+              headers: {
+                'Content-Type': 'application/vnd.pypi.simple.v1+json',
+                'Cache-Control': 'public, max-age=300'
+              },
+              body: upstreamJson,
+            };
+          }
+        }
+
+        // Return HTML format
+        return {
+          status: 200,
+          headers: {
+            'Content-Type': 'text/html; charset=utf-8',
+            'Cache-Control': 'public, max-age=300'
+          },
+          body: upstreamHtml,
+        };
+      }
+    }
+
     if (!metadata) {
       return this.errorResponse(404, 'Package not found');
     }
@@ -449,7 +505,16 @@ export class PypiRegistry extends BaseRegistry {
    */
   private async handleDownload(packageName: string, filename: string): Promise<IResponse> {
     const normalized = helpers.normalizePypiPackageName(packageName);
-    const fileData = await this.storage.getPypiPackageFile(normalized, filename);
+    let fileData = await this.storage.getPypiPackageFile(normalized, filename);
+
+    // Try upstream if not found locally
+    if (!fileData && this.upstream) {
+      fileData = await this.upstream.fetchPackageFile(normalized, filename);
+      if (fileData) {
+        // Cache locally
+        await this.storage.putPypiPackageFile(normalized, filename, fileData);
+      }
+    }
 
     if (!fileData) {
       return {