feat(upstream): Add upstream proxy/cache subsystem and integrate per-protocol upstreams

2025-11-27 14:20:01 +00:00
parent cfadc89b5a
commit 0610077eec
34 changed files with 3450 additions and 46 deletions
--- a/ts/npm/classes.npmregistry.ts
+++ b/ts/npm/classes.npmregistry.ts
@@ -3,6 +3,8 @@ import { BaseRegistry } from '../core/classes.baseregistry.js';
 import { RegistryStorage } from '../core/classes.registrystorage.js';
 import { AuthManager } from '../core/classes.authmanager.js';
 import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js';
+import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js';
+import { NpmUpstream } from './classes.npmupstream.js';
 import type {
  IPackument,
  INpmVersion,
@@ -25,12 +27,14 @@ export class NpmRegistry extends BaseRegistry {
  private basePath: string = '/npm';
  private registryUrl: string;
  private logger: Smartlog;
+  private upstream: NpmUpstream | null = null;

  constructor(
    storage: RegistryStorage,
    authManager: AuthManager,
    basePath: string = '/npm',
-    registryUrl: string = 'http://localhost:5000/npm'
+    registryUrl: string = 'http://localhost:5000/npm',
+    upstreamConfig?: IProtocolUpstreamConfig
  ) {
    super();
    this.storage = storage;
@@ -50,6 +54,14 @@ export class NpmRegistry extends BaseRegistry {
      }
    });
    this.logger.enableConsole();
+
+    // Initialize upstream if configured
+    if (upstreamConfig?.enabled) {
+      this.upstream = new NpmUpstream(upstreamConfig, registryUrl, this.logger);
+      this.logger.log('info', 'NPM upstream initialized', {
+        upstreams: upstreamConfig.upstreams.map(u => u.name),
+      });
+    }
  }

  public async init(): Promise<void> {
@@ -209,13 +221,28 @@ export class NpmRegistry extends BaseRegistry {
    token: IAuthToken | null,
    query: Record<string, string>
  ): Promise<IResponse> {
-    const packument = await this.storage.getNpmPackument(packageName);
+    let packument = await this.storage.getNpmPackument(packageName);
    this.logger.log('debug', `getPackument: ${packageName}`, {
      packageName,
      found: !!packument,
      versions: packument ? Object.keys(packument.versions).length : 0
    });

+    // If not found locally, try upstream
+    if (!packument && this.upstream) {
+      this.logger.log('debug', `getPackument: fetching from upstream`, { packageName });
+      const upstreamPackument = await this.upstream.fetchPackument(packageName);
+      if (upstreamPackument) {
+        this.logger.log('debug', `getPackument: found in upstream`, {
+          packageName,
+          versions: Object.keys(upstreamPackument.versions || {}).length
+        });
+        packument = upstreamPackument;
+        // Optionally cache the packument locally (without tarballs)
+        // We don't store tarballs here - they'll be fetched on demand
+      }
+    }
+
    if (!packument) {
      return {
        status: 404,
@@ -255,11 +282,21 @@ export class NpmRegistry extends BaseRegistry {
    token: IAuthToken | null
  ): Promise<IResponse> {
    this.logger.log('debug', 'handlePackageVersion', { packageName, version });
-    const packument = await this.storage.getNpmPackument(packageName);
+    let packument = await this.storage.getNpmPackument(packageName);
    this.logger.log('debug', 'handlePackageVersion packument', { found: !!packument });
    if (packument) {
      this.logger.log('debug', 'handlePackageVersion versions', { versions: Object.keys(packument.versions || {}) });
    }
+
+    // If not found locally, try upstream
+    if (!packument && this.upstream) {
+      this.logger.log('debug', 'handlePackageVersion: fetching from upstream', { packageName });
+      const upstreamPackument = await this.upstream.fetchPackument(packageName);
+      if (upstreamPackument) {
+        packument = upstreamPackument;
+      }
+    }
+
    if (!packument) {
      return {
        status: 404,
@@ -529,7 +566,7 @@ export class NpmRegistry extends BaseRegistry {
    token: IAuthToken | null
  ): Promise<IResponse> {
    // Extract version from filename: package-name-1.0.0.tgz
-    const versionMatch = filename.match(/-([\d.]+(?:-[a-z0-9.]+)?)\.tgz$/);
+    const versionMatch = filename.match(/-([\d.]+(?:-[a-z0-9.]+)?)\.tgz$/i);
    if (!versionMatch) {
      return {
        status: 400,
@@ -539,7 +576,26 @@ export class NpmRegistry extends BaseRegistry {
    }

    const version = versionMatch[1];
-    const tarball = await this.storage.getNpmTarball(packageName, version);
+    let tarball = await this.storage.getNpmTarball(packageName, version);
+
+    // If not found locally, try upstream
+    if (!tarball && this.upstream) {
+      this.logger.log('debug', 'handleTarballDownload: fetching from upstream', {
+        packageName,
+        version,
+      });
+      const upstreamTarball = await this.upstream.fetchTarball(packageName, version);
+      if (upstreamTarball) {
+        tarball = upstreamTarball;
+        // Cache the tarball locally for future requests
+        await this.storage.putNpmTarball(packageName, version, tarball);
+        this.logger.log('debug', 'handleTarballDownload: cached tarball locally', {
+          packageName,
+          version,
+          size: tarball.length,
+        });
+      }
+    }

    if (!tarball) {
      return {
--- a/ts/npm/classes.npmupstream.ts
+++ b/ts/npm/classes.npmupstream.ts
@@ -0,0 +1,260 @@
+import * as plugins from '../plugins.js';
+import { BaseUpstream } from '../upstream/classes.baseupstream.js';
+import type {
+  IProtocolUpstreamConfig,
+  IUpstreamFetchContext,
+  IUpstreamResult,
+  IUpstreamRegistryConfig,
+} from '../upstream/interfaces.upstream.js';
+import type { IPackument, INpmVersion } from './interfaces.npm.js';
+
+/**
+ * NPM-specific upstream implementation.
+ *
+ * Handles:
+ * - Package metadata (packument) fetching
+ * - Tarball proxying
+ * - Scoped package routing (@scope/* patterns)
+ * - NPM-specific URL rewriting
+ */
+export class NpmUpstream extends BaseUpstream {
+  protected readonly protocolName = 'npm';
+
+  /** Local registry URL for rewriting tarball URLs */
+  private readonly localRegistryUrl: string;
+
+  constructor(
+    config: IProtocolUpstreamConfig,
+    localRegistryUrl: string,
+    logger?: plugins.smartlog.Smartlog,
+  ) {
+    super(config, logger);
+    this.localRegistryUrl = localRegistryUrl;
+  }
+
+  /**
+   * Fetch a packument from upstream registries.
+   */
+  public async fetchPackument(packageName: string): Promise<IPackument | null> {
+    const context: IUpstreamFetchContext = {
+      protocol: 'npm',
+      resource: packageName,
+      resourceType: 'packument',
+      path: `/${encodeURIComponent(packageName).replace('%40', '@')}`,
+      method: 'GET',
+      headers: {
+        'accept': 'application/json',
+      },
+      query: {},
+    };
+
+    const result = await this.fetch(context);
+
+    if (!result || !result.success) {
+      return null;
+    }
+
+    // Parse and process packument
+    let packument: IPackument;
+    if (Buffer.isBuffer(result.body)) {
+      packument = JSON.parse(result.body.toString('utf8'));
+    } else {
+      packument = result.body;
+    }
+
+    // Rewrite tarball URLs to point to local registry
+    packument = this.rewriteTarballUrls(packument);
+
+    return packument;
+  }
+
+  /**
+   * Fetch a specific version from upstream registries.
+   */
+  public async fetchVersion(packageName: string, version: string): Promise<INpmVersion | null> {
+    const context: IUpstreamFetchContext = {
+      protocol: 'npm',
+      resource: packageName,
+      resourceType: 'version',
+      path: `/${encodeURIComponent(packageName).replace('%40', '@')}/${version}`,
+      method: 'GET',
+      headers: {
+        'accept': 'application/json',
+      },
+      query: {},
+    };
+
+    const result = await this.fetch(context);
+
+    if (!result || !result.success) {
+      return null;
+    }
+
+    let versionData: INpmVersion;
+    if (Buffer.isBuffer(result.body)) {
+      versionData = JSON.parse(result.body.toString('utf8'));
+    } else {
+      versionData = result.body;
+    }
+
+    // Rewrite tarball URL
+    if (versionData.dist?.tarball) {
+      versionData.dist.tarball = this.rewriteSingleTarballUrl(
+        packageName,
+        versionData.version,
+        versionData.dist.tarball,
+      );
+    }
+
+    return versionData;
+  }
+
+  /**
+   * Fetch a tarball from upstream registries.
+   */
+  public async fetchTarball(packageName: string, version: string): Promise<Buffer | null> {
+    // First, try to get the tarball URL from packument
+    const packument = await this.fetchPackument(packageName);
+    let tarballPath: string;
+
+    if (packument?.versions?.[version]?.dist?.tarball) {
+      // Extract path from original (upstream) tarball URL
+      const tarballUrl = packument.versions[version].dist.tarball;
+      try {
+        const url = new URL(tarballUrl);
+        tarballPath = url.pathname;
+      } catch {
+        // Fallback to standard NPM tarball path
+        tarballPath = this.buildTarballPath(packageName, version);
+      }
+    } else {
+      tarballPath = this.buildTarballPath(packageName, version);
+    }
+
+    const context: IUpstreamFetchContext = {
+      protocol: 'npm',
+      resource: packageName,
+      resourceType: 'tarball',
+      path: tarballPath,
+      method: 'GET',
+      headers: {
+        'accept': 'application/octet-stream',
+      },
+      query: {},
+    };
+
+    const result = await this.fetch(context);
+
+    if (!result || !result.success) {
+      return null;
+    }
+
+    return Buffer.isBuffer(result.body) ? result.body : Buffer.from(result.body);
+  }
+
+  /**
+   * Search packages in upstream registries.
+   */
+  public async search(text: string, size: number = 20, from: number = 0): Promise<any | null> {
+    const context: IUpstreamFetchContext = {
+      protocol: 'npm',
+      resource: '*',
+      resourceType: 'search',
+      path: '/-/v1/search',
+      method: 'GET',
+      headers: {
+        'accept': 'application/json',
+      },
+      query: {
+        text,
+        size: size.toString(),
+        from: from.toString(),
+      },
+    };
+
+    const result = await this.fetch(context);
+
+    if (!result || !result.success) {
+      return null;
+    }
+
+    if (Buffer.isBuffer(result.body)) {
+      return JSON.parse(result.body.toString('utf8'));
+    }
+
+    return result.body;
+  }
+
+  /**
+   * Build the standard NPM tarball path.
+   */
+  private buildTarballPath(packageName: string, version: string): string {
+    // NPM uses: /{package}/-/{package-name}-{version}.tgz
+    // For scoped packages: /@scope/name/-/name-version.tgz
+    if (packageName.startsWith('@')) {
+      const [scope, name] = packageName.split('/');
+      return `/${scope}/${name}/-/${name}-${version}.tgz`;
+    } else {
+      return `/${packageName}/-/${packageName}-${version}.tgz`;
+    }
+  }
+
+  /**
+   * Rewrite all tarball URLs in a packument to point to local registry.
+   */
+  private rewriteTarballUrls(packument: IPackument): IPackument {
+    if (!packument.versions) {
+      return packument;
+    }
+
+    const rewritten = { ...packument };
+    rewritten.versions = {};
+
+    for (const [version, versionData] of Object.entries(packument.versions)) {
+      const newVersionData = { ...versionData };
+      if (newVersionData.dist?.tarball) {
+        newVersionData.dist = {
+          ...newVersionData.dist,
+          tarball: this.rewriteSingleTarballUrl(
+            packument.name,
+            version,
+            newVersionData.dist.tarball,
+          ),
+        };
+      }
+      rewritten.versions[version] = newVersionData;
+    }
+
+    return rewritten;
+  }
+
+  /**
+   * Rewrite a single tarball URL to point to local registry.
+   */
+  private rewriteSingleTarballUrl(
+    packageName: string,
+    version: string,
+    _originalUrl: string,
+  ): string {
+    // Generate local tarball URL
+    // Format: {localRegistryUrl}/{package}/-/{package-name}-{version}.tgz
+    const safeName = packageName.replace('@', '').replace('/', '-');
+    return `${this.localRegistryUrl}/${packageName}/-/${safeName}-${version}.tgz`;
+  }
+
+  /**
+   * Override URL building for NPM-specific handling.
+   */
+  protected buildUpstreamUrl(
+    upstream: IUpstreamRegistryConfig,
+    context: IUpstreamFetchContext,
+  ): string {
+    // NPM registries often don't have trailing slashes
+    let baseUrl = upstream.url;
+    if (baseUrl.endsWith('/')) {
+      baseUrl = baseUrl.slice(0, -1);
+    }
+
+    return `${baseUrl}${context.path}`;
+  }
+}
--- a/ts/npm/index.ts
+++ b/ts/npm/index.ts
@@ -3,4 +3,5 @@
 */

 export { NpmRegistry } from './classes.npmregistry.js';
+export { NpmUpstream } from './classes.npmupstream.js';
 export * from './interfaces.npm.js';