feat(upstream): Add upstream proxy/cache subsystem and integrate per-protocol upstreams

ts/pypi/classes.pypiregistry.ts

@@ -3,6 +3,7 @@ import { BaseRegistry } from '../core/classes.baseregistry.js';
 import { RegistryStorage } from '../core/classes.registrystorage.js';
 import { AuthManager } from '../core/classes.authmanager.js';
 import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js';
+import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js';
 import { isBinaryData, toBuffer } from '../core/helpers.buffer.js';
 import type {
   IPypiPackageMetadata,
@@ -11,6 +12,7 @@ import type {
   IPypiUploadResponse,
 } from './interfaces.pypi.js';
 import * as helpers from './helpers.pypi.js';
+import { PypiUpstream } from './classes.pypiupstream.js';
 
 /**
  * PyPI registry implementation
@@ -22,12 +24,14 @@ export class PypiRegistry extends BaseRegistry {
   private basePath: string = '/pypi';
   private registryUrl: string;
   private logger: Smartlog;
+  private upstream: PypiUpstream | null = null;
 
   constructor(
     storage: RegistryStorage,
     authManager: AuthManager,
     basePath: string = '/pypi',
-    registryUrl: string = 'http://localhost:5000'
+    registryUrl: string = 'http://localhost:5000',
+    upstreamConfig?: IProtocolUpstreamConfig
   ) {
     super();
     this.storage = storage;
@@ -47,6 +51,20 @@ export class PypiRegistry extends BaseRegistry {
       }
     });
     this.logger.enableConsole();
+
+    // Initialize upstream if configured
+    if (upstreamConfig?.enabled) {
+      this.upstream = new PypiUpstream(upstreamConfig, registryUrl, this.logger);
+    }
+  }
+
+  /**
+   * Clean up resources (timers, connections, etc.)
+   */
+  public destroy(): void {
+    if (this.upstream) {
+      this.upstream.stop();
+    }
   }
 
   public async init(): Promise<void> {
@@ -214,7 +232,45 @@ export class PypiRegistry extends BaseRegistry {
     const normalized = helpers.normalizePypiPackageName(packageName);
 
     // Get package metadata
-    const metadata = await this.storage.getPypiPackageMetadata(normalized);
+    let metadata = await this.storage.getPypiPackageMetadata(normalized);
+
+    // Try upstream if not found locally
+    if (!metadata && this.upstream) {
+      const upstreamHtml = await this.upstream.fetchSimplePackage(normalized);
+      if (upstreamHtml) {
+        // Parse the HTML to extract file information and cache it
+        // For now, just return the upstream HTML directly (caching can be improved later)
+        const acceptHeader = context.headers['accept'] || context.headers['Accept'] || '';
+        const preferJson = acceptHeader.includes('application/vnd.pypi.simple') &&
+                           acceptHeader.includes('json');
+
+        if (preferJson) {
+          // Try to get JSON format from upstream
+          const upstreamJson = await this.upstream.fetchPackageJson(normalized);
+          if (upstreamJson) {
+            return {
+              status: 200,
+              headers: {
+                'Content-Type': 'application/vnd.pypi.simple.v1+json',
+                'Cache-Control': 'public, max-age=300'
+              },
+              body: upstreamJson,
+            };
+          }
+        }
+
+        // Return HTML format
+        return {
+          status: 200,
+          headers: {
+            'Content-Type': 'text/html; charset=utf-8',
+            'Cache-Control': 'public, max-age=300'
+          },
+          body: upstreamHtml,
+        };
+      }
+    }
+
     if (!metadata) {
       return this.errorResponse(404, 'Package not found');
     }
@@ -449,7 +505,16 @@ export class PypiRegistry extends BaseRegistry {
    */
   private async handleDownload(packageName: string, filename: string): Promise<IResponse> {
     const normalized = helpers.normalizePypiPackageName(packageName);
-    const fileData = await this.storage.getPypiPackageFile(normalized, filename);
+    let fileData = await this.storage.getPypiPackageFile(normalized, filename);
+
+    // Try upstream if not found locally
+    if (!fileData && this.upstream) {
+      fileData = await this.upstream.fetchPackageFile(normalized, filename);
+      if (fileData) {
+        // Cache locally
+        await this.storage.putPypiPackageFile(normalized, filename, fileData);
+      }
+    }
 
     if (!fileData) {
       return {

ts/pypi/classes.pypiupstream.ts

@@ -0,0 +1,211 @@
+import * as plugins from '../plugins.js';
+import { BaseUpstream } from '../upstream/classes.baseupstream.js';
+import type {
+  IProtocolUpstreamConfig,
+  IUpstreamFetchContext,
+  IUpstreamRegistryConfig,
+} from '../upstream/interfaces.upstream.js';
+
+/**
+ * PyPI-specific upstream implementation.
+ *
+ * Handles:
+ * - Simple API (HTML) - PEP 503
+ * - JSON API - PEP 691
+ * - Package file downloads (wheels, sdists)
+ * - Package name normalization
+ */
+export class PypiUpstream extends BaseUpstream {
+  protected readonly protocolName = 'pypi';
+
+  /** Local registry URL for rewriting download URLs */
+  private readonly localRegistryUrl: string;
+
+  constructor(
+    config: IProtocolUpstreamConfig,
+    localRegistryUrl: string,
+    logger?: plugins.smartlog.Smartlog,
+  ) {
+    super(config, logger);
+    this.localRegistryUrl = localRegistryUrl;
+  }
+
+  /**
+   * Fetch Simple API index (list of all packages) in HTML format.
+   */
+  public async fetchSimpleIndex(): Promise<string | null> {
+    const context: IUpstreamFetchContext = {
+      protocol: 'pypi',
+      resource: '*',
+      resourceType: 'index',
+      path: '/simple/',
+      method: 'GET',
+      headers: {
+        'accept': 'text/html',
+      },
+      query: {},
+    };
+
+    const result = await this.fetch(context);
+
+    if (!result || !result.success) {
+      return null;
+    }
+
+    if (Buffer.isBuffer(result.body)) {
+      return result.body.toString('utf8');
+    }
+
+    return typeof result.body === 'string' ? result.body : null;
+  }
+
+  /**
+   * Fetch Simple API package page (list of files) in HTML format.
+   */
+  public async fetchSimplePackage(packageName: string): Promise<string | null> {
+    const normalizedName = this.normalizePackageName(packageName);
+    const path = `/simple/${normalizedName}/`;
+
+    const context: IUpstreamFetchContext = {
+      protocol: 'pypi',
+      resource: packageName,
+      resourceType: 'simple',
+      path,
+      method: 'GET',
+      headers: {
+        'accept': 'text/html',
+      },
+      query: {},
+    };
+
+    const result = await this.fetch(context);
+
+    if (!result || !result.success) {
+      return null;
+    }
+
+    if (Buffer.isBuffer(result.body)) {
+      return result.body.toString('utf8');
+    }
+
+    return typeof result.body === 'string' ? result.body : null;
+  }
+
+  /**
+   * Fetch package metadata using JSON API (PEP 691).
+   */
+  public async fetchPackageJson(packageName: string): Promise<any | null> {
+    const normalizedName = this.normalizePackageName(packageName);
+    const path = `/simple/${normalizedName}/`;
+
+    const context: IUpstreamFetchContext = {
+      protocol: 'pypi',
+      resource: packageName,
+      resourceType: 'metadata',
+      path,
+      method: 'GET',
+      headers: {
+        'accept': 'application/vnd.pypi.simple.v1+json',
+      },
+      query: {},
+    };
+
+    const result = await this.fetch(context);
+
+    if (!result || !result.success) {
+      return null;
+    }
+
+    if (Buffer.isBuffer(result.body)) {
+      return JSON.parse(result.body.toString('utf8'));
+    }
+
+    return result.body;
+  }
+
+  /**
+   * Fetch full package info from PyPI JSON API (/pypi/{package}/json).
+   */
+  public async fetchPypiJson(packageName: string): Promise<any | null> {
+    const normalizedName = this.normalizePackageName(packageName);
+    const path = `/pypi/${normalizedName}/json`;
+
+    const context: IUpstreamFetchContext = {
+      protocol: 'pypi',
+      resource: packageName,
+      resourceType: 'pypi-json',
+      path,
+      method: 'GET',
+      headers: {
+        'accept': 'application/json',
+      },
+      query: {},
+    };
+
+    const result = await this.fetch(context);
+
+    if (!result || !result.success) {
+      return null;
+    }
+
+    if (Buffer.isBuffer(result.body)) {
+      return JSON.parse(result.body.toString('utf8'));
+    }
+
+    return result.body;
+  }
+
+  /**
+   * Fetch a package file (wheel or sdist) from upstream.
+   */
+  public async fetchPackageFile(packageName: string, filename: string): Promise<Buffer | null> {
+    const normalizedName = this.normalizePackageName(packageName);
+    const path = `/packages/${normalizedName}/${filename}`;
+
+    const context: IUpstreamFetchContext = {
+      protocol: 'pypi',
+      resource: packageName,
+      resourceType: 'package',
+      path,
+      method: 'GET',
+      headers: {
+        'accept': 'application/octet-stream',
+      },
+      query: {},
+    };
+
+    const result = await this.fetch(context);
+
+    if (!result || !result.success) {
+      return null;
+    }
+
+    return Buffer.isBuffer(result.body) ? result.body : Buffer.from(result.body);
+  }
+
+  /**
+   * Normalize a PyPI package name according to PEP 503.
+   * - Lowercase all characters
+   * - Replace runs of ., -, _ with single -
+   */
+  private normalizePackageName(name: string): string {
+    return name.toLowerCase().replace(/[-_.]+/g, '-');
+  }
+
+  /**
+   * Override URL building for PyPI-specific handling.
+   */
+  protected buildUpstreamUrl(
+    upstream: IUpstreamRegistryConfig,
+    context: IUpstreamFetchContext,
+  ): string {
+    let baseUrl = upstream.url;
+
+    // Remove trailing slash
+    if (baseUrl.endsWith('/')) {
+      baseUrl = baseUrl.slice(0, -1);
+    }
+
+    return `${baseUrl}${context.path}`;
+  }
+}

ts/pypi/index.ts

@@ -5,4 +5,5 @@
 
 export * from './interfaces.pypi.js';
 export * from './classes.pypiregistry.js';
+export { PypiUpstream } from './classes.pypiupstream.js';
 export * as pypiHelpers from './helpers.pypi.js';