From e9af3f8328e1f9b6074ec9e71cd880d2c891ac1d Mon Sep 17 00:00:00 2001 From: Juergen Kunz Date: Wed, 3 Dec 2025 22:16:40 +0000 Subject: [PATCH] feat(upstream): Add dynamic per-request upstream provider and integrate into registries --- changelog.md | 12 + test/helpers/registry.ts | 85 ++++++ test/test.upstream.provider.ts | 343 ++++++++++++++++++++++++ ts/00_commitinfo_data.ts | 2 +- ts/cargo/classes.cargoregistry.ts | 107 +++++--- ts/classes.smartregistry.ts | 14 +- ts/composer/classes.composerregistry.ts | 94 +++++-- ts/core/interfaces.core.ts | 11 +- ts/maven/classes.mavenregistry.ts | 121 ++++++--- ts/npm/classes.npmregistry.ts | 150 +++++++---- ts/oci/classes.ociregistry.ts | 156 +++++++---- ts/pypi/classes.pypiregistry.ts | 143 ++++++---- ts/rubygems/classes.rubygemsregistry.ts | 85 ++++-- ts/upstream/interfaces.upstream.ts | 81 +++++- 14 files changed, 1117 insertions(+), 287 deletions(-) create mode 100644 test/test.upstream.provider.ts diff --git a/changelog.md b/changelog.md index 259e71a..ff4e1ee 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,17 @@ # Changelog +## 2025-12-03 - 2.7.0 - feat(upstream) +Add dynamic per-request upstream provider and integrate into registries + +- Introduce IUpstreamProvider and IUpstreamResolutionContext to resolve upstream configs per request. +- Add StaticUpstreamProvider implementation for simple static upstream configurations. +- Propagate dynamic upstream provider through SmartRegistry and wire into protocol handlers (npm, oci, maven, cargo, composer, pypi, rubygems). +- Replace persistent per-protocol upstream instances with per-request resolution: registries now call provider.resolveUpstreamConfig(...) and instantiate protocol-specific Upstream when needed. +- Add IRequestActor to core interfaces and pass actor context (userId, ip, userAgent, etc.) to upstream resolution and storage/auth hooks. +- Update many protocol registries to accept an upstreamProvider instead of IProtocolUpstreamConfig and to attempt upstream fetches only when provider returns enabled config. +- Add utilities and tests: test helpers to create registries with upstream provider, a tracking upstream provider helper, StaticUpstreamProvider tests and extensive upstream/provider integration tests. +- Improve upstream interfaces and cache/fetch contexts (IUpstreamFetchContext includes actor) and add StaticUpstreamProvider class to upstream module. + ## 2025-11-27 - 2.6.0 - feat(core) Add core registry infrastructure: storage, auth, upstream cache, and protocol handlers diff --git a/test/helpers/registry.ts b/test/helpers/registry.ts index a39b636..5cd8d58 100644 --- a/test/helpers/registry.ts +++ b/test/helpers/registry.ts @@ -6,6 +6,8 @@ import { SmartRegistry } from '../../ts/classes.smartregistry.js'; import type { IRegistryConfig, IAuthToken, TRegistryProtocol } from '../../ts/core/interfaces.core.js'; import type { IAuthProvider, ITokenOptions } from '../../ts/core/interfaces.auth.js'; import type { IStorageHooks, IStorageHookContext, IBeforePutResult, IBeforeDeleteResult } from '../../ts/core/interfaces.storage.js'; +import { StaticUpstreamProvider } from '../../ts/upstream/interfaces.upstream.js'; +import type { IUpstreamProvider, IUpstreamResolutionContext, IProtocolUpstreamConfig } from '../../ts/upstream/interfaces.upstream.js'; const testQenv = new qenv.Qenv('./', './.nogit'); @@ -134,6 +136,89 @@ export async function createTestRegistry(): Promise { return registry; } +/** + * Create a test SmartRegistry instance with upstream provider configured + */ +export async function createTestRegistryWithUpstream( + upstreamProvider?: IUpstreamProvider +): Promise { + // Read S3 config from env.json + const s3AccessKey = await testQenv.getEnvVarOnDemand('S3_ACCESSKEY'); + const s3SecretKey = await testQenv.getEnvVarOnDemand('S3_SECRETKEY'); + const s3Endpoint = await testQenv.getEnvVarOnDemand('S3_ENDPOINT'); + const s3Port = await testQenv.getEnvVarOnDemand('S3_PORT'); + + // Default to StaticUpstreamProvider with npm.js configured + const defaultProvider = new StaticUpstreamProvider({ + npm: { + enabled: true, + upstreams: [{ id: 'npmjs', url: 'https://registry.npmjs.org', priority: 1, enabled: true }], + }, + oci: { + enabled: true, + upstreams: [{ id: 'dockerhub', url: 'https://registry-1.docker.io', priority: 1, enabled: true }], + }, + }); + + const config: IRegistryConfig = { + storage: { + accessKey: s3AccessKey || 'minioadmin', + accessSecret: s3SecretKey || 'minioadmin', + endpoint: s3Endpoint || 'localhost', + port: parseInt(s3Port || '9000', 10), + useSsl: false, + region: 'us-east-1', + bucketName: 'test-registry', + }, + auth: { + jwtSecret: 'test-secret-key', + tokenStore: 'memory', + npmTokens: { enabled: true }, + ociTokens: { + enabled: true, + realm: 'https://auth.example.com/token', + service: 'test-registry', + }, + pypiTokens: { enabled: true }, + rubygemsTokens: { enabled: true }, + }, + upstreamProvider: upstreamProvider || defaultProvider, + oci: { enabled: true, basePath: '/oci' }, + npm: { enabled: true, basePath: '/npm' }, + maven: { enabled: true, basePath: '/maven' }, + composer: { enabled: true, basePath: '/composer' }, + cargo: { enabled: true, basePath: '/cargo' }, + pypi: { enabled: true, basePath: '/pypi' }, + rubygems: { enabled: true, basePath: '/rubygems' }, + }; + + const registry = new SmartRegistry(config); + await registry.init(); + + return registry; +} + +/** + * Create a mock upstream provider that tracks all calls for testing + */ +export function createTrackingUpstreamProvider( + baseConfig?: Partial> +): { + provider: IUpstreamProvider; + calls: IUpstreamResolutionContext[]; +} { + const calls: IUpstreamResolutionContext[] = []; + + const provider: IUpstreamProvider = { + async resolveUpstreamConfig(context: IUpstreamResolutionContext) { + calls.push({ ...context }); + return baseConfig?.[context.protocol] ?? null; + }, + }; + + return { provider, calls }; +} + /** * Helper to create test authentication tokens */ diff --git a/test/test.upstream.provider.ts b/test/test.upstream.provider.ts new file mode 100644 index 0000000..0d818ff --- /dev/null +++ b/test/test.upstream.provider.ts @@ -0,0 +1,343 @@ +import { expect, tap } from '@git.zone/tstest/tapbundle'; +import { SmartRegistry } from '../ts/index.js'; +import { + createTestRegistryWithUpstream, + createTrackingUpstreamProvider, +} from './helpers/registry.js'; +import { StaticUpstreamProvider } from '../ts/upstream/interfaces.upstream.js'; +import type { + IUpstreamProvider, + IUpstreamResolutionContext, + IProtocolUpstreamConfig, +} from '../ts/upstream/interfaces.upstream.js'; +import type { TRegistryProtocol } from '../ts/core/interfaces.core.js'; + +// ============================================================================= +// StaticUpstreamProvider Tests +// ============================================================================= + +tap.test('StaticUpstreamProvider: should return config for configured protocol', async () => { + const npmConfig: IProtocolUpstreamConfig = { + enabled: true, + upstreams: [{ id: 'npmjs', url: 'https://registry.npmjs.org', priority: 1, enabled: true }], + }; + + const provider = new StaticUpstreamProvider({ + npm: npmConfig, + }); + + const result = await provider.resolveUpstreamConfig({ + protocol: 'npm', + resource: 'lodash', + scope: null, + method: 'GET', + resourceType: 'packument', + }); + + expect(result).toBeDefined(); + expect(result?.enabled).toEqual(true); + expect(result?.upstreams[0].id).toEqual('npmjs'); +}); + +tap.test('StaticUpstreamProvider: should return null for unconfigured protocol', async () => { + const provider = new StaticUpstreamProvider({ + npm: { + enabled: true, + upstreams: [{ id: 'npmjs', url: 'https://registry.npmjs.org', priority: 1, enabled: true }], + }, + }); + + const result = await provider.resolveUpstreamConfig({ + protocol: 'maven', + resource: 'com.example:lib', + scope: 'com.example', + method: 'GET', + resourceType: 'pom', + }); + + expect(result).toBeNull(); +}); + +tap.test('StaticUpstreamProvider: should support multiple protocols', async () => { + const provider = new StaticUpstreamProvider({ + npm: { + enabled: true, + upstreams: [{ id: 'npmjs', url: 'https://registry.npmjs.org', priority: 1, enabled: true }], + }, + oci: { + enabled: true, + upstreams: [{ id: 'dockerhub', url: 'https://registry-1.docker.io', priority: 1, enabled: true }], + }, + maven: { + enabled: true, + upstreams: [{ id: 'central', url: 'https://repo1.maven.org/maven2', priority: 1, enabled: true }], + }, + }); + + const npmResult = await provider.resolveUpstreamConfig({ + protocol: 'npm', + resource: 'lodash', + scope: null, + method: 'GET', + resourceType: 'packument', + }); + expect(npmResult?.upstreams[0].id).toEqual('npmjs'); + + const ociResult = await provider.resolveUpstreamConfig({ + protocol: 'oci', + resource: 'library/nginx', + scope: 'library', + method: 'GET', + resourceType: 'manifest', + }); + expect(ociResult?.upstreams[0].id).toEqual('dockerhub'); + + const mavenResult = await provider.resolveUpstreamConfig({ + protocol: 'maven', + resource: 'com.example:lib', + scope: 'com.example', + method: 'GET', + resourceType: 'pom', + }); + expect(mavenResult?.upstreams[0].id).toEqual('central'); +}); + +// ============================================================================= +// Registry with Provider Integration Tests +// ============================================================================= + +let registry: SmartRegistry; +let trackingProvider: ReturnType; + +tap.test('Provider Integration: should create registry with upstream provider', async () => { + trackingProvider = createTrackingUpstreamProvider({ + npm: { + enabled: true, + upstreams: [{ id: 'test-npm', url: 'https://registry.npmjs.org', priority: 1, enabled: true }], + }, + }); + + registry = await createTestRegistryWithUpstream(trackingProvider.provider); + + expect(registry).toBeInstanceOf(SmartRegistry); + expect(registry.isInitialized()).toEqual(true); +}); + +tap.test('Provider Integration: should call provider when fetching unknown npm package', async () => { + // Clear previous calls + trackingProvider.calls.length = 0; + + // Request a package that doesn't exist locally - should trigger upstream lookup + const response = await registry.handleRequest({ + method: 'GET', + path: '/npm/@test-scope/nonexistent-package', + headers: {}, + query: {}, + }); + + // Provider should have been called for the packument lookup + const npmCalls = trackingProvider.calls.filter(c => c.protocol === 'npm'); + + // The package doesn't exist locally, so upstream should be consulted + // Note: actual upstream fetch may fail since the package doesn't exist + expect(response.status).toBeOneOf([404, 200, 502]); // 404 if not found, 502 if upstream error +}); + +tap.test('Provider Integration: provider receives correct context for scoped npm package', async () => { + trackingProvider.calls.length = 0; + + // Use URL-encoded path for scoped packages as npm client does + await registry.handleRequest({ + method: 'GET', + path: '/npm/@myorg%2fmy-package', + headers: {}, + query: {}, + }); + + // Find any npm call - the exact resource type depends on routing + const npmCalls = trackingProvider.calls.filter(c => c.protocol === 'npm'); + + // Provider should be called for upstream lookup + if (npmCalls.length > 0) { + const call = npmCalls[0]; + expect(call.protocol).toEqual('npm'); + // The resource should include the scoped name + expect(call.resource).toInclude('myorg'); + expect(call.method).toEqual('GET'); + } +}); + +tap.test('Provider Integration: provider receives correct context for unscoped npm package', async () => { + trackingProvider.calls.length = 0; + + await registry.handleRequest({ + method: 'GET', + path: '/npm/lodash', + headers: {}, + query: {}, + }); + + const packumentCall = trackingProvider.calls.find( + c => c.protocol === 'npm' && c.resourceType === 'packument' + ); + + if (packumentCall) { + expect(packumentCall.protocol).toEqual('npm'); + expect(packumentCall.resource).toEqual('lodash'); + expect(packumentCall.scope).toBeNull(); // No scope for unscoped package + } +}); + +// ============================================================================= +// Custom Provider Implementation Tests +// ============================================================================= + +tap.test('Custom Provider: should support dynamic resolution based on context', async () => { + // Create a provider that returns different configs based on scope + const dynamicProvider: IUpstreamProvider = { + async resolveUpstreamConfig(context: IUpstreamResolutionContext) { + if (context.scope === 'internal') { + // Internal packages go to private registry + return { + enabled: true, + upstreams: [{ id: 'private', url: 'https://private.registry.com', priority: 1, enabled: true }], + }; + } + // Everything else goes to public registry + return { + enabled: true, + upstreams: [{ id: 'public', url: 'https://registry.npmjs.org', priority: 1, enabled: true }], + }; + }, + }; + + const internalResult = await dynamicProvider.resolveUpstreamConfig({ + protocol: 'npm', + resource: '@internal/utils', + scope: 'internal', + method: 'GET', + resourceType: 'packument', + }); + expect(internalResult?.upstreams[0].id).toEqual('private'); + + const publicResult = await dynamicProvider.resolveUpstreamConfig({ + protocol: 'npm', + resource: '@public/utils', + scope: 'public', + method: 'GET', + resourceType: 'packument', + }); + expect(publicResult?.upstreams[0].id).toEqual('public'); +}); + +tap.test('Custom Provider: should support actor-based resolution', async () => { + const actorAwareProvider: IUpstreamProvider = { + async resolveUpstreamConfig(context: IUpstreamResolutionContext) { + // Different upstreams based on user's organization + if (context.actor?.orgId === 'enterprise-org') { + return { + enabled: true, + upstreams: [{ id: 'enterprise', url: 'https://enterprise.registry.com', priority: 1, enabled: true }], + }; + } + return { + enabled: true, + upstreams: [{ id: 'default', url: 'https://registry.npmjs.org', priority: 1, enabled: true }], + }; + }, + }; + + const enterpriseResult = await actorAwareProvider.resolveUpstreamConfig({ + protocol: 'npm', + resource: 'lodash', + scope: null, + actor: { orgId: 'enterprise-org', userId: 'user1' }, + method: 'GET', + resourceType: 'packument', + }); + expect(enterpriseResult?.upstreams[0].id).toEqual('enterprise'); + + const defaultResult = await actorAwareProvider.resolveUpstreamConfig({ + protocol: 'npm', + resource: 'lodash', + scope: null, + actor: { orgId: 'free-org', userId: 'user2' }, + method: 'GET', + resourceType: 'packument', + }); + expect(defaultResult?.upstreams[0].id).toEqual('default'); +}); + +tap.test('Custom Provider: should support disabling upstream for specific resources', async () => { + const selectiveProvider: IUpstreamProvider = { + async resolveUpstreamConfig(context: IUpstreamResolutionContext) { + // Block upstream for internal packages + if (context.scope === 'internal') { + return null; // No upstream for internal packages + } + return { + enabled: true, + upstreams: [{ id: 'public', url: 'https://registry.npmjs.org', priority: 1, enabled: true }], + }; + }, + }; + + const internalResult = await selectiveProvider.resolveUpstreamConfig({ + protocol: 'npm', + resource: '@internal/secret', + scope: 'internal', + method: 'GET', + resourceType: 'packument', + }); + expect(internalResult).toBeNull(); + + const publicResult = await selectiveProvider.resolveUpstreamConfig({ + protocol: 'npm', + resource: 'lodash', + scope: null, + method: 'GET', + resourceType: 'packument', + }); + expect(publicResult).not.toBeNull(); +}); + +// ============================================================================= +// Registry without Provider Tests +// ============================================================================= + +tap.test('No Provider: registry should work without upstream provider', async () => { + const registryWithoutUpstream = await createTestRegistryWithUpstream( + // Pass a provider that always returns null + { + async resolveUpstreamConfig() { + return null; + }, + } + ); + + expect(registryWithoutUpstream).toBeInstanceOf(SmartRegistry); + + // Should return 404 for non-existent package (no upstream to check) + const response = await registryWithoutUpstream.handleRequest({ + method: 'GET', + path: '/npm/nonexistent-package-xyz', + headers: {}, + query: {}, + }); + + expect(response.status).toEqual(404); + + registryWithoutUpstream.destroy(); +}); + +// ============================================================================= +// Cleanup +// ============================================================================= + +tap.postTask('cleanup registry', async () => { + if (registry) { + registry.destroy(); + } +}); + +export default tap.start(); diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index 81ddc3d..55e7db9 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@push.rocks/smartregistry', - version: '2.6.0', + version: '2.7.0', description: 'A composable TypeScript library implementing OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems registries for building unified container and package registries' } diff --git a/ts/cargo/classes.cargoregistry.ts b/ts/cargo/classes.cargoregistry.ts index 4e865bf..3aba98e 100644 --- a/ts/cargo/classes.cargoregistry.ts +++ b/ts/cargo/classes.cargoregistry.ts @@ -2,8 +2,8 @@ import { Smartlog } from '@push.rocks/smartlog'; import { BaseRegistry } from '../core/classes.baseregistry.js'; import { RegistryStorage } from '../core/classes.registrystorage.js'; import { AuthManager } from '../core/classes.authmanager.js'; -import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js'; -import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js'; +import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from '../core/interfaces.core.js'; +import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js'; import type { ICargoIndexEntry, ICargoPublishMetadata, @@ -27,20 +27,21 @@ export class CargoRegistry extends BaseRegistry { private basePath: string = '/cargo'; private registryUrl: string; private logger: Smartlog; - private upstream: CargoUpstream | null = null; + private upstreamProvider: IUpstreamProvider | null = null; constructor( storage: RegistryStorage, authManager: AuthManager, basePath: string = '/cargo', registryUrl: string = 'http://localhost:5000/cargo', - upstreamConfig?: IProtocolUpstreamConfig + upstreamProvider?: IUpstreamProvider ) { super(); this.storage = storage; this.authManager = authManager; this.basePath = basePath; this.registryUrl = registryUrl; + this.upstreamProvider = upstreamProvider || null; // Initialize logger this.logger = new Smartlog({ @@ -54,20 +55,38 @@ export class CargoRegistry extends BaseRegistry { } }); this.logger.enableConsole(); + } - // Initialize upstream if configured - if (upstreamConfig?.enabled) { - this.upstream = new CargoUpstream(upstreamConfig, undefined, this.logger); - } + /** + * Get upstream for a specific request. + * Calls the provider to resolve upstream config dynamically. + */ + private async getUpstreamForRequest( + resource: string, + resourceType: string, + method: string, + actor?: IRequestActor + ): Promise { + if (!this.upstreamProvider) return null; + + const config = await this.upstreamProvider.resolveUpstreamConfig({ + protocol: 'cargo', + resource, + scope: resource, // For Cargo, crate name is the scope + actor, + method, + resourceType, + }); + + if (!config?.enabled) return null; + return new CargoUpstream(config, undefined, this.logger); } /** * Clean up resources (timers, connections, etc.) */ public destroy(): void { - if (this.upstream) { - this.upstream.stop(); - } + // No persistent upstream to clean up with dynamic provider } public async init(): Promise { @@ -94,6 +113,14 @@ export class CargoRegistry extends BaseRegistry { const authHeader = context.headers['authorization'] || context.headers['Authorization']; const token = authHeader ? await this.authManager.validateToken(authHeader, 'cargo') : null; + // Build actor from context and validated token + const actor: IRequestActor = { + ...context.actor, + userId: token?.userId, + ip: context.headers['x-forwarded-for'] || context.headers['X-Forwarded-For'], + userAgent: context.headers['user-agent'] || context.headers['User-Agent'], + }; + this.logger.log('debug', `handleRequest: ${context.method} ${path}`, { method: context.method, path, @@ -107,11 +134,11 @@ export class CargoRegistry extends BaseRegistry { // API endpoints if (path.startsWith('/api/v1/')) { - return this.handleApiRequest(path, context, token); + return this.handleApiRequest(path, context, token, actor); } // Index files (sparse protocol) - return this.handleIndexRequest(path); + return this.handleIndexRequest(path, actor); } /** @@ -132,7 +159,8 @@ export class CargoRegistry extends BaseRegistry { private async handleApiRequest( path: string, context: IRequestContext, - token: IAuthToken | null + token: IAuthToken | null, + actor?: IRequestActor ): Promise { // Publish: PUT /api/v1/crates/new if (path === '/api/v1/crates/new' && context.method === 'PUT') { @@ -142,7 +170,7 @@ export class CargoRegistry extends BaseRegistry { // Download: GET /api/v1/crates/{crate}/{version}/download const downloadMatch = path.match(/^\/api\/v1\/crates\/([^\/]+)\/([^\/]+)\/download$/); if (downloadMatch && context.method === 'GET') { - return this.handleDownload(downloadMatch[1], downloadMatch[2]); + return this.handleDownload(downloadMatch[1], downloadMatch[2], actor); } // Yank: DELETE /api/v1/crates/{crate}/{version}/yank @@ -175,7 +203,7 @@ export class CargoRegistry extends BaseRegistry { * Handle index file requests * Paths: /1/{name}, /2/{name}, /3/{c}/{name}, /{p1}/{p2}/{name} */ - private async handleIndexRequest(path: string): Promise { + private async handleIndexRequest(path: string, actor?: IRequestActor): Promise { // Parse index paths to extract crate name const pathParts = path.split('/').filter(p => p); let crateName: string | null = null; @@ -202,7 +230,7 @@ export class CargoRegistry extends BaseRegistry { }; } - return this.handleIndexFile(crateName); + return this.handleIndexFile(crateName, actor); } /** @@ -224,23 +252,26 @@ export class CargoRegistry extends BaseRegistry { /** * Serve index file for a crate */ - private async handleIndexFile(crateName: string): Promise { + private async handleIndexFile(crateName: string, actor?: IRequestActor): Promise { let index = await this.storage.getCargoIndex(crateName); // Try upstream if not found locally - if ((!index || index.length === 0) && this.upstream) { - const upstreamIndex = await this.upstream.fetchCrateIndex(crateName); - if (upstreamIndex) { - // Parse the newline-delimited JSON - const parsedIndex: ICargoIndexEntry[] = upstreamIndex - .split('\n') - .filter(line => line.trim()) - .map(line => JSON.parse(line)); + if (!index || index.length === 0) { + const upstream = await this.getUpstreamForRequest(crateName, 'index', 'GET', actor); + if (upstream) { + const upstreamIndex = await upstream.fetchCrateIndex(crateName); + if (upstreamIndex) { + // Parse the newline-delimited JSON + const parsedIndex: ICargoIndexEntry[] = upstreamIndex + .split('\n') + .filter(line => line.trim()) + .map(line => JSON.parse(line)); - if (parsedIndex.length > 0) { - // Cache locally - await this.storage.putCargoIndex(crateName, parsedIndex); - index = parsedIndex; + if (parsedIndex.length > 0) { + // Cache locally + await this.storage.putCargoIndex(crateName, parsedIndex); + index = parsedIndex; + } } } } @@ -431,18 +462,22 @@ export class CargoRegistry extends BaseRegistry { */ private async handleDownload( crateName: string, - version: string + version: string, + actor?: IRequestActor ): Promise { this.logger.log('debug', 'handleDownload', { crate: crateName, version }); let crateFile = await this.storage.getCargoCrate(crateName, version); // Try upstream if not found locally - if (!crateFile && this.upstream) { - crateFile = await this.upstream.fetchCrate(crateName, version); - if (crateFile) { - // Cache locally - await this.storage.putCargoCrate(crateName, version, crateFile); + if (!crateFile) { + const upstream = await this.getUpstreamForRequest(crateName, 'crate', 'GET', actor); + if (upstream) { + crateFile = await upstream.fetchCrate(crateName, version); + if (crateFile) { + // Cache locally + await this.storage.putCargoCrate(crateName, version, crateFile); + } } } diff --git a/ts/classes.smartregistry.ts b/ts/classes.smartregistry.ts index ca74d35..1ef8e2e 100644 --- a/ts/classes.smartregistry.ts +++ b/ts/classes.smartregistry.ts @@ -86,7 +86,7 @@ export class SmartRegistry { this.authManager, ociBasePath, ociTokens, - this.config.oci.upstream + this.config.upstreamProvider ); await ociRegistry.init(); this.registries.set('oci', ociRegistry); @@ -101,7 +101,7 @@ export class SmartRegistry { this.authManager, npmBasePath, registryUrl, - this.config.npm.upstream + this.config.upstreamProvider ); await npmRegistry.init(); this.registries.set('npm', npmRegistry); @@ -116,7 +116,7 @@ export class SmartRegistry { this.authManager, mavenBasePath, registryUrl, - this.config.maven.upstream + this.config.upstreamProvider ); await mavenRegistry.init(); this.registries.set('maven', mavenRegistry); @@ -131,7 +131,7 @@ export class SmartRegistry { this.authManager, cargoBasePath, registryUrl, - this.config.cargo.upstream + this.config.upstreamProvider ); await cargoRegistry.init(); this.registries.set('cargo', cargoRegistry); @@ -146,7 +146,7 @@ export class SmartRegistry { this.authManager, composerBasePath, registryUrl, - this.config.composer.upstream + this.config.upstreamProvider ); await composerRegistry.init(); this.registries.set('composer', composerRegistry); @@ -161,7 +161,7 @@ export class SmartRegistry { this.authManager, pypiBasePath, registryUrl, - this.config.pypi.upstream + this.config.upstreamProvider ); await pypiRegistry.init(); this.registries.set('pypi', pypiRegistry); @@ -176,7 +176,7 @@ export class SmartRegistry { this.authManager, rubygemsBasePath, registryUrl, - this.config.rubygems.upstream + this.config.upstreamProvider ); await rubygemsRegistry.init(); this.registries.set('rubygems', rubygemsRegistry); diff --git a/ts/composer/classes.composerregistry.ts b/ts/composer/classes.composerregistry.ts index 7b174ab..fb7c42d 100644 --- a/ts/composer/classes.composerregistry.ts +++ b/ts/composer/classes.composerregistry.ts @@ -6,8 +6,8 @@ import { BaseRegistry } from '../core/classes.baseregistry.js'; import type { RegistryStorage } from '../core/classes.registrystorage.js'; import type { AuthManager } from '../core/classes.authmanager.js'; -import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js'; -import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js'; +import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from '../core/interfaces.core.js'; +import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js'; import { isBinaryData, toBuffer } from '../core/helpers.buffer.js'; import type { IComposerPackage, @@ -30,34 +30,66 @@ export class ComposerRegistry extends BaseRegistry { private authManager: AuthManager; private basePath: string = '/composer'; private registryUrl: string; - private upstream: ComposerUpstream | null = null; + private upstreamProvider: IUpstreamProvider | null = null; constructor( storage: RegistryStorage, authManager: AuthManager, basePath: string = '/composer', registryUrl: string = 'http://localhost:5000/composer', - upstreamConfig?: IProtocolUpstreamConfig + upstreamProvider?: IUpstreamProvider ) { super(); this.storage = storage; this.authManager = authManager; this.basePath = basePath; this.registryUrl = registryUrl; + this.upstreamProvider = upstreamProvider || null; + } - // Initialize upstream if configured - if (upstreamConfig?.enabled) { - this.upstream = new ComposerUpstream(upstreamConfig); + /** + * Extract scope from Composer package name. + * For Composer, vendor is the scope. + * @example "symfony" from "symfony/console" + */ + private extractScope(vendorPackage: string): string | null { + const slashIndex = vendorPackage.indexOf('/'); + if (slashIndex > 0) { + return vendorPackage.substring(0, slashIndex); } + return null; + } + + /** + * Get upstream for a specific request. + * Calls the provider to resolve upstream config dynamically. + */ + private async getUpstreamForRequest( + resource: string, + resourceType: string, + method: string, + actor?: IRequestActor + ): Promise { + if (!this.upstreamProvider) return null; + + const config = await this.upstreamProvider.resolveUpstreamConfig({ + protocol: 'composer', + resource, + scope: this.extractScope(resource), + actor, + method, + resourceType, + }); + + if (!config?.enabled) return null; + return new ComposerUpstream(config); } /** * Clean up resources (timers, connections, etc.) */ public destroy(): void { - if (this.upstream) { - this.upstream.stop(); - } + // No persistent upstream to clean up with dynamic provider } public async init(): Promise { @@ -96,6 +128,14 @@ export class ComposerRegistry extends BaseRegistry { } } + // Build actor from context and validated token + const actor: IRequestActor = { + ...context.actor, + userId: token?.userId, + ip: context.headers['x-forwarded-for'] || context.headers['X-Forwarded-For'], + userAgent: context.headers['user-agent'] || context.headers['User-Agent'], + }; + // Root packages.json if (path === '/packages.json' || path === '' || path === '/') { return this.handlePackagesJson(); @@ -106,7 +146,7 @@ export class ComposerRegistry extends BaseRegistry { if (metadataMatch) { const [, vendorPackage, devSuffix] = metadataMatch; const includeDev = !!devSuffix; - return this.handlePackageMetadata(vendorPackage, includeDev, token); + return this.handlePackageMetadata(vendorPackage, includeDev, token, actor); } // Package list: /packages/list.json?filter=vendor/* @@ -176,26 +216,30 @@ export class ComposerRegistry extends BaseRegistry { private async handlePackageMetadata( vendorPackage: string, includeDev: boolean, - token: IAuthToken | null + token: IAuthToken | null, + actor?: IRequestActor ): Promise { // Read operations are public, no authentication required let metadata = await this.storage.getComposerPackageMetadata(vendorPackage); // Try upstream if not found locally - if (!metadata && this.upstream) { - const [vendor, packageName] = vendorPackage.split('/'); - if (vendor && packageName) { - const upstreamMetadata = includeDev - ? await this.upstream.fetchPackageDevMetadata(vendor, packageName) - : await this.upstream.fetchPackageMetadata(vendor, packageName); + if (!metadata) { + const upstream = await this.getUpstreamForRequest(vendorPackage, 'metadata', 'GET', actor); + if (upstream) { + const [vendor, packageName] = vendorPackage.split('/'); + if (vendor && packageName) { + const upstreamMetadata = includeDev + ? await upstream.fetchPackageDevMetadata(vendor, packageName) + : await upstream.fetchPackageMetadata(vendor, packageName); - if (upstreamMetadata && upstreamMetadata.packages) { - // Store upstream metadata locally - metadata = { - packages: upstreamMetadata.packages, - lastModified: new Date().toUTCString(), - }; - await this.storage.putComposerPackageMetadata(vendorPackage, metadata); + if (upstreamMetadata && upstreamMetadata.packages) { + // Store upstream metadata locally + metadata = { + packages: upstreamMetadata.packages, + lastModified: new Date().toUTCString(), + }; + await this.storage.putComposerPackageMetadata(vendorPackage, metadata); + } } } } diff --git a/ts/core/interfaces.core.ts b/ts/core/interfaces.core.ts index 8b32cb3..bbaf61e 100644 --- a/ts/core/interfaces.core.ts +++ b/ts/core/interfaces.core.ts @@ -3,7 +3,7 @@ */ import type * as plugins from '../plugins.js'; -import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js'; +import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js'; import type { IAuthProvider } from './interfaces.auth.js'; import type { IStorageHooks } from './interfaces.storage.js'; @@ -89,8 +89,6 @@ export interface IProtocolConfig { enabled: boolean; basePath: string; features?: Record; - /** Upstream registry configuration for proxying/caching */ - upstream?: IProtocolUpstreamConfig; } /** @@ -113,6 +111,13 @@ export interface IRegistryConfig { */ storageHooks?: IStorageHooks; + /** + * Dynamic upstream configuration provider. + * Called per-request to resolve which upstream registries to use. + * Use StaticUpstreamProvider for simple static configurations. + */ + upstreamProvider?: IUpstreamProvider; + oci?: IProtocolConfig; npm?: IProtocolConfig; maven?: IProtocolConfig; diff --git a/ts/maven/classes.mavenregistry.ts b/ts/maven/classes.mavenregistry.ts index f01b500..85c9e88 100644 --- a/ts/maven/classes.mavenregistry.ts +++ b/ts/maven/classes.mavenregistry.ts @@ -6,8 +6,8 @@ import { BaseRegistry } from '../core/classes.baseregistry.js'; import type { RegistryStorage } from '../core/classes.registrystorage.js'; import type { AuthManager } from '../core/classes.authmanager.js'; -import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js'; -import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js'; +import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from '../core/interfaces.core.js'; +import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js'; import { toBuffer } from '../core/helpers.buffer.js'; import type { IMavenCoordinate, IMavenMetadata, IChecksums } from './interfaces.maven.js'; import { @@ -33,34 +33,64 @@ export class MavenRegistry extends BaseRegistry { private authManager: AuthManager; private basePath: string = '/maven'; private registryUrl: string; - private upstream: MavenUpstream | null = null; + private upstreamProvider: IUpstreamProvider | null = null; constructor( storage: RegistryStorage, authManager: AuthManager, basePath: string, registryUrl: string, - upstreamConfig?: IProtocolUpstreamConfig + upstreamProvider?: IUpstreamProvider ) { super(); this.storage = storage; this.authManager = authManager; this.basePath = basePath; this.registryUrl = registryUrl; + this.upstreamProvider = upstreamProvider || null; + } - // Initialize upstream if configured - if (upstreamConfig?.enabled) { - this.upstream = new MavenUpstream(upstreamConfig); - } + /** + * Extract scope from Maven coordinates. + * For Maven, the groupId is the scope. + * @example "com.example" from "com.example:my-lib" + */ + private extractScope(groupId: string): string | null { + return groupId || null; + } + + /** + * Get upstream for a specific request. + * Calls the provider to resolve upstream config dynamically. + */ + private async getUpstreamForRequest( + resource: string, + resourceType: string, + method: string, + actor?: IRequestActor + ): Promise { + if (!this.upstreamProvider) return null; + + // For Maven, resource is "groupId:artifactId" + const [groupId] = resource.split(':'); + const config = await this.upstreamProvider.resolveUpstreamConfig({ + protocol: 'maven', + resource, + scope: this.extractScope(groupId), + actor, + method, + resourceType, + }); + + if (!config?.enabled) return null; + return new MavenUpstream(config); } /** * Clean up resources (timers, connections, etc.) */ public destroy(): void { - if (this.upstream) { - this.upstream.stop(); - } + // No persistent upstream to clean up with dynamic provider } public async init(): Promise { @@ -85,13 +115,21 @@ export class MavenRegistry extends BaseRegistry { token = await this.authManager.validateToken(tokenString, 'maven'); } + // Build actor from context and validated token + const actor: IRequestActor = { + ...context.actor, + userId: token?.userId, + ip: context.headers['x-forwarded-for'] || context.headers['X-Forwarded-For'], + userAgent: context.headers['user-agent'] || context.headers['User-Agent'], + }; + // Parse path to determine request type const coordinate = pathToGAV(path); if (!coordinate) { // Not a valid artifact path, could be metadata or root if (path.endsWith('/maven-metadata.xml')) { - return this.handleMetadataRequest(context.method, path, token); + return this.handleMetadataRequest(context.method, path, token, actor); } return { @@ -108,7 +146,7 @@ export class MavenRegistry extends BaseRegistry { } // Handle artifact requests (JAR, POM, WAR, etc.) - return this.handleArtifactRequest(context.method, coordinate, token, context.body); + return this.handleArtifactRequest(context.method, coordinate, token, context.body, actor); } protected async checkPermission( @@ -128,7 +166,8 @@ export class MavenRegistry extends BaseRegistry { method: string, coordinate: IMavenCoordinate, token: IAuthToken | null, - body?: Buffer | any + body?: Buffer | any, + actor?: IRequestActor ): Promise { const { groupId, artifactId, version } = coordinate; const filename = buildFilename(coordinate); @@ -139,7 +178,7 @@ export class MavenRegistry extends BaseRegistry { case 'HEAD': // Maven repositories typically allow anonymous reads return method === 'GET' - ? this.getArtifact(groupId, artifactId, version, filename) + ? this.getArtifact(groupId, artifactId, version, filename, actor) : this.headArtifact(groupId, artifactId, version, filename); case 'PUT': @@ -211,7 +250,8 @@ export class MavenRegistry extends BaseRegistry { private async handleMetadataRequest( method: string, path: string, - token: IAuthToken | null + token: IAuthToken | null, + actor?: IRequestActor ): Promise { // Parse path to extract groupId and artifactId // Path format: /com/example/my-lib/maven-metadata.xml @@ -232,7 +272,7 @@ export class MavenRegistry extends BaseRegistry { if (method === 'GET') { // Metadata is usually public (read permission optional) // Some registries allow anonymous metadata access - return this.getMetadata(groupId, artifactId); + return this.getMetadata(groupId, artifactId, actor); } return { @@ -250,22 +290,27 @@ export class MavenRegistry extends BaseRegistry { groupId: string, artifactId: string, version: string, - filename: string + filename: string, + actor?: IRequestActor ): Promise { let data = await this.storage.getMavenArtifact(groupId, artifactId, version, filename); // Try upstream if not found locally - if (!data && this.upstream) { - // Parse the filename to extract extension and classifier - const { extension, classifier } = this.parseFilename(filename, artifactId, version); - if (extension) { - data = await this.upstream.fetchArtifact(groupId, artifactId, version, extension, classifier); - if (data) { - // Cache the artifact locally - await this.storage.putMavenArtifact(groupId, artifactId, version, filename, data); - // Generate and store checksums - const checksums = await calculateChecksums(data); - await this.storeChecksums(groupId, artifactId, version, filename, checksums); + if (!data) { + const resource = `${groupId}:${artifactId}`; + const upstream = await this.getUpstreamForRequest(resource, 'artifact', 'GET', actor); + if (upstream) { + // Parse the filename to extract extension and classifier + const { extension, classifier } = this.parseFilename(filename, artifactId, version); + if (extension) { + data = await upstream.fetchArtifact(groupId, artifactId, version, extension, classifier); + if (data) { + // Cache the artifact locally + await this.storage.putMavenArtifact(groupId, artifactId, version, filename, data); + // Generate and store checksums + const checksums = await calculateChecksums(data); + await this.storeChecksums(groupId, artifactId, version, filename, checksums); + } } } } @@ -495,16 +540,20 @@ export class MavenRegistry extends BaseRegistry { // METADATA OPERATIONS // ======================================================================== - private async getMetadata(groupId: string, artifactId: string): Promise { + private async getMetadata(groupId: string, artifactId: string, actor?: IRequestActor): Promise { let metadataBuffer = await this.storage.getMavenMetadata(groupId, artifactId); // Try upstream if not found locally - if (!metadataBuffer && this.upstream) { - const upstreamMetadata = await this.upstream.fetchMetadata(groupId, artifactId); - if (upstreamMetadata) { - metadataBuffer = Buffer.from(upstreamMetadata, 'utf-8'); - // Cache the metadata locally - await this.storage.putMavenMetadata(groupId, artifactId, metadataBuffer); + if (!metadataBuffer) { + const resource = `${groupId}:${artifactId}`; + const upstream = await this.getUpstreamForRequest(resource, 'metadata', 'GET', actor); + if (upstream) { + const upstreamMetadata = await upstream.fetchMetadata(groupId, artifactId); + if (upstreamMetadata) { + metadataBuffer = Buffer.from(upstreamMetadata, 'utf-8'); + // Cache the metadata locally + await this.storage.putMavenMetadata(groupId, artifactId, metadataBuffer); + } } } diff --git a/ts/npm/classes.npmregistry.ts b/ts/npm/classes.npmregistry.ts index 090e28f..0a39668 100644 --- a/ts/npm/classes.npmregistry.ts +++ b/ts/npm/classes.npmregistry.ts @@ -2,8 +2,8 @@ import { Smartlog } from '@push.rocks/smartlog'; import { BaseRegistry } from '../core/classes.baseregistry.js'; import { RegistryStorage } from '../core/classes.registrystorage.js'; import { AuthManager } from '../core/classes.authmanager.js'; -import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js'; -import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js'; +import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from '../core/interfaces.core.js'; +import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js'; import { NpmUpstream } from './classes.npmupstream.js'; import type { IPackument, @@ -27,20 +27,21 @@ export class NpmRegistry extends BaseRegistry { private basePath: string = '/npm'; private registryUrl: string; private logger: Smartlog; - private upstream: NpmUpstream | null = null; + private upstreamProvider: IUpstreamProvider | null = null; constructor( storage: RegistryStorage, authManager: AuthManager, basePath: string = '/npm', registryUrl: string = 'http://localhost:5000/npm', - upstreamConfig?: IProtocolUpstreamConfig + upstreamProvider?: IUpstreamProvider ) { super(); this.storage = storage; this.authManager = authManager; this.basePath = basePath; this.registryUrl = registryUrl; + this.upstreamProvider = upstreamProvider || null; // Initialize logger this.logger = new Smartlog({ @@ -55,15 +56,51 @@ export class NpmRegistry extends BaseRegistry { }); this.logger.enableConsole(); - // Initialize upstream if configured - if (upstreamConfig?.enabled) { - this.upstream = new NpmUpstream(upstreamConfig, registryUrl, this.logger); - this.logger.log('info', 'NPM upstream initialized', { - upstreams: upstreamConfig.upstreams.map(u => u.name), - }); + if (upstreamProvider) { + this.logger.log('info', 'NPM upstream provider configured'); } } + /** + * Extract scope from npm package name. + * @example "@company/utils" -> "company" + * @example "lodash" -> null + */ + private extractScope(packageName: string): string | null { + if (packageName.startsWith('@')) { + const slashIndex = packageName.indexOf('/'); + if (slashIndex > 1) { + return packageName.substring(1, slashIndex); + } + } + return null; + } + + /** + * Get upstream for a specific request. + * Calls the provider to resolve upstream config dynamically. + */ + private async getUpstreamForRequest( + resource: string, + resourceType: string, + method: string, + actor?: IRequestActor + ): Promise { + if (!this.upstreamProvider) return null; + + const config = await this.upstreamProvider.resolveUpstreamConfig({ + protocol: 'npm', + resource, + scope: this.extractScope(resource), + actor, + method, + resourceType, + }); + + if (!config?.enabled) return null; + return new NpmUpstream(config, this.registryUrl, this.logger); + } + public async init(): Promise { // NPM registry initialization } @@ -80,6 +117,14 @@ export class NpmRegistry extends BaseRegistry { const tokenString = authHeader?.replace(/^Bearer\s+/i, ''); const token = tokenString ? await this.authManager.validateToken(tokenString, 'npm') : null; + // Build actor context for upstream resolution + const actor: IRequestActor = { + userId: token?.userId, + ip: context.headers['x-forwarded-for'] || context.headers['x-real-ip'], + userAgent: context.headers['user-agent'], + ...context.actor, // Include any pre-populated actor info + }; + this.logger.log('debug', `handleRequest: ${context.method} ${path}`, { method: context.method, path, @@ -118,7 +163,7 @@ export class NpmRegistry extends BaseRegistry { const tarballMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)\/-\/(.+\.tgz)$/); if (tarballMatch) { const [, packageName, filename] = tarballMatch; - return this.handleTarballDownload(packageName, filename, token); + return this.handleTarballDownload(packageName, filename, token, actor); } // Unpublish specific version: DELETE /{package}/-/{version} @@ -142,7 +187,7 @@ export class NpmRegistry extends BaseRegistry { if (versionMatch) { const [, packageName, version] = versionMatch; this.logger.log('debug', 'versionMatch', { packageName, version }); - return this.handlePackageVersion(packageName, version, token); + return this.handlePackageVersion(packageName, version, token, actor); } // Package operations: /{package} @@ -150,7 +195,7 @@ export class NpmRegistry extends BaseRegistry { if (packageMatch) { const packageName = packageMatch[1]; this.logger.log('debug', 'packageMatch', { packageName }); - return this.handlePackage(context.method, packageName, context.body, context.query, token); + return this.handlePackage(context.method, packageName, context.body, context.query, token, actor); } return { @@ -198,11 +243,12 @@ export class NpmRegistry extends BaseRegistry { packageName: string, body: any, query: Record, - token: IAuthToken | null + token: IAuthToken | null, + actor?: IRequestActor ): Promise { switch (method) { case 'GET': - return this.getPackument(packageName, token, query); + return this.getPackument(packageName, token, query, actor); case 'PUT': return this.publishPackage(packageName, body, token); case 'DELETE': @@ -219,7 +265,8 @@ export class NpmRegistry extends BaseRegistry { private async getPackument( packageName: string, token: IAuthToken | null, - query: Record + query: Record, + actor?: IRequestActor ): Promise { let packument = await this.storage.getNpmPackument(packageName); this.logger.log('debug', `getPackument: ${packageName}`, { @@ -229,17 +276,20 @@ export class NpmRegistry extends BaseRegistry { }); // If not found locally, try upstream - if (!packument && this.upstream) { - this.logger.log('debug', `getPackument: fetching from upstream`, { packageName }); - const upstreamPackument = await this.upstream.fetchPackument(packageName); - if (upstreamPackument) { - this.logger.log('debug', `getPackument: found in upstream`, { - packageName, - versions: Object.keys(upstreamPackument.versions || {}).length - }); - packument = upstreamPackument; - // Optionally cache the packument locally (without tarballs) - // We don't store tarballs here - they'll be fetched on demand + if (!packument) { + const upstream = await this.getUpstreamForRequest(packageName, 'packument', 'GET', actor); + if (upstream) { + this.logger.log('debug', `getPackument: fetching from upstream`, { packageName }); + const upstreamPackument = await upstream.fetchPackument(packageName); + if (upstreamPackument) { + this.logger.log('debug', `getPackument: found in upstream`, { + packageName, + versions: Object.keys(upstreamPackument.versions || {}).length + }); + packument = upstreamPackument; + // Optionally cache the packument locally (without tarballs) + // We don't store tarballs here - they'll be fetched on demand + } } } @@ -279,7 +329,8 @@ export class NpmRegistry extends BaseRegistry { private async handlePackageVersion( packageName: string, version: string, - token: IAuthToken | null + token: IAuthToken | null, + actor?: IRequestActor ): Promise { this.logger.log('debug', 'handlePackageVersion', { packageName, version }); let packument = await this.storage.getNpmPackument(packageName); @@ -289,11 +340,14 @@ export class NpmRegistry extends BaseRegistry { } // If not found locally, try upstream - if (!packument && this.upstream) { - this.logger.log('debug', 'handlePackageVersion: fetching from upstream', { packageName }); - const upstreamPackument = await this.upstream.fetchPackument(packageName); - if (upstreamPackument) { - packument = upstreamPackument; + if (!packument) { + const upstream = await this.getUpstreamForRequest(packageName, 'packument', 'GET', actor); + if (upstream) { + this.logger.log('debug', 'handlePackageVersion: fetching from upstream', { packageName }); + const upstreamPackument = await upstream.fetchPackument(packageName); + if (upstreamPackument) { + packument = upstreamPackument; + } } } @@ -563,7 +617,8 @@ export class NpmRegistry extends BaseRegistry { private async handleTarballDownload( packageName: string, filename: string, - token: IAuthToken | null + token: IAuthToken | null, + actor?: IRequestActor ): Promise { // Extract version from filename: package-name-1.0.0.tgz const versionMatch = filename.match(/-([\d.]+(?:-[a-z0-9.]+)?)\.tgz$/i); @@ -579,21 +634,24 @@ export class NpmRegistry extends BaseRegistry { let tarball = await this.storage.getNpmTarball(packageName, version); // If not found locally, try upstream - if (!tarball && this.upstream) { - this.logger.log('debug', 'handleTarballDownload: fetching from upstream', { - packageName, - version, - }); - const upstreamTarball = await this.upstream.fetchTarball(packageName, version); - if (upstreamTarball) { - tarball = upstreamTarball; - // Cache the tarball locally for future requests - await this.storage.putNpmTarball(packageName, version, tarball); - this.logger.log('debug', 'handleTarballDownload: cached tarball locally', { + if (!tarball) { + const upstream = await this.getUpstreamForRequest(packageName, 'tarball', 'GET', actor); + if (upstream) { + this.logger.log('debug', 'handleTarballDownload: fetching from upstream', { packageName, version, - size: tarball.length, }); + const upstreamTarball = await upstream.fetchTarball(packageName, version); + if (upstreamTarball) { + tarball = upstreamTarball; + // Cache the tarball locally for future requests + await this.storage.putNpmTarball(packageName, version, tarball); + this.logger.log('debug', 'handleTarballDownload: cached tarball locally', { + packageName, + version, + size: tarball.length, + }); + } } } diff --git a/ts/oci/classes.ociregistry.ts b/ts/oci/classes.ociregistry.ts index bc9c54e..d019df4 100644 --- a/ts/oci/classes.ociregistry.ts +++ b/ts/oci/classes.ociregistry.ts @@ -2,8 +2,8 @@ import { Smartlog } from '@push.rocks/smartlog'; import { BaseRegistry } from '../core/classes.baseregistry.js'; import { RegistryStorage } from '../core/classes.registrystorage.js'; import { AuthManager } from '../core/classes.authmanager.js'; -import type { IRequestContext, IResponse, IAuthToken, IRegistryError } from '../core/interfaces.core.js'; -import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js'; +import type { IRequestContext, IResponse, IAuthToken, IRegistryError, IRequestActor } from '../core/interfaces.core.js'; +import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js'; import { OciUpstream } from './classes.ociupstream.js'; import type { IUploadSession, @@ -24,7 +24,7 @@ export class OciRegistry extends BaseRegistry { private basePath: string = '/oci'; private cleanupInterval?: NodeJS.Timeout; private ociTokens?: { realm: string; service: string }; - private upstream: OciUpstream | null = null; + private upstreamProvider: IUpstreamProvider | null = null; private logger: Smartlog; constructor( @@ -32,13 +32,14 @@ export class OciRegistry extends BaseRegistry { authManager: AuthManager, basePath: string = '/oci', ociTokens?: { realm: string; service: string }, - upstreamConfig?: IProtocolUpstreamConfig + upstreamProvider?: IUpstreamProvider ) { super(); this.storage = storage; this.authManager = authManager; this.basePath = basePath; this.ociTokens = ociTokens; + this.upstreamProvider = upstreamProvider || null; // Initialize logger this.logger = new Smartlog({ @@ -53,15 +54,50 @@ export class OciRegistry extends BaseRegistry { }); this.logger.enableConsole(); - // Initialize upstream if configured - if (upstreamConfig?.enabled) { - this.upstream = new OciUpstream(upstreamConfig, basePath, this.logger); - this.logger.log('info', 'OCI upstream initialized', { - upstreams: upstreamConfig.upstreams.map(u => u.name), - }); + if (upstreamProvider) { + this.logger.log('info', 'OCI upstream provider configured'); } } + /** + * Extract scope from OCI repository name. + * @example "myorg/myimage" -> "myorg" + * @example "library/nginx" -> "library" + * @example "nginx" -> null + */ + private extractScope(repository: string): string | null { + const slashIndex = repository.indexOf('/'); + if (slashIndex > 0) { + return repository.substring(0, slashIndex); + } + return null; + } + + /** + * Get upstream for a specific request. + * Calls the provider to resolve upstream config dynamically. + */ + private async getUpstreamForRequest( + resource: string, + resourceType: string, + method: string, + actor?: IRequestActor + ): Promise { + if (!this.upstreamProvider) return null; + + const config = await this.upstreamProvider.resolveUpstreamConfig({ + protocol: 'oci', + resource, + scope: this.extractScope(resource), + actor, + method, + resourceType, + }); + + if (!config?.enabled) return null; + return new OciUpstream(config, this.basePath, this.logger); + } + public async init(): Promise { // Start cleanup of stale upload sessions this.startUploadSessionCleanup(); @@ -80,6 +116,14 @@ export class OciRegistry extends BaseRegistry { const tokenString = authHeader?.replace(/^Bearer\s+/i, ''); const token = tokenString ? await this.authManager.validateToken(tokenString, 'oci') : null; + // Build actor from context and validated token + const actor: IRequestActor = { + ...context.actor, + userId: token?.userId, + ip: context.headers['x-forwarded-for'] || context.headers['X-Forwarded-For'], + userAgent: context.headers['user-agent'] || context.headers['User-Agent'], + }; + // Route to appropriate handler if (path === '/v2/' || path === '/v2') { return this.handleVersionCheck(); @@ -91,14 +135,14 @@ export class OciRegistry extends BaseRegistry { const [, name, reference] = manifestMatch; // Prefer rawBody for content-addressable operations to preserve exact bytes const bodyData = context.rawBody || context.body; - return this.handleManifestRequest(context.method, name, reference, token, bodyData, context.headers); + return this.handleManifestRequest(context.method, name, reference, token, bodyData, context.headers, actor); } // Blob operations: /v2/{name}/blobs/{digest} const blobMatch = path.match(/^\/v2\/([^\/]+(?:\/[^\/]+)*)\/blobs\/(sha256:[a-f0-9]{64})$/); if (blobMatch) { const [, name, digest] = blobMatch; - return this.handleBlobRequest(context.method, name, digest, token, context.headers); + return this.handleBlobRequest(context.method, name, digest, token, context.headers, actor); } // Blob upload operations: /v2/{name}/blobs/uploads/ @@ -168,11 +212,12 @@ export class OciRegistry extends BaseRegistry { reference: string, token: IAuthToken | null, body?: Buffer | any, - headers?: Record + headers?: Record, + actor?: IRequestActor ): Promise { switch (method) { case 'GET': - return this.getManifest(repository, reference, token, headers); + return this.getManifest(repository, reference, token, headers, actor); case 'HEAD': return this.headManifest(repository, reference, token); case 'PUT': @@ -193,11 +238,12 @@ export class OciRegistry extends BaseRegistry { repository: string, digest: string, token: IAuthToken | null, - headers: Record + headers: Record, + actor?: IRequestActor ): Promise { switch (method) { case 'GET': - return this.getBlob(repository, digest, token, headers['range'] || headers['Range']); + return this.getBlob(repository, digest, token, headers['range'] || headers['Range'], actor); case 'HEAD': return this.headBlob(repository, digest, token); case 'DELETE': @@ -318,7 +364,8 @@ export class OciRegistry extends BaseRegistry { repository: string, reference: string, token: IAuthToken | null, - headers?: Record + headers?: Record, + actor?: IRequestActor ): Promise { if (!await this.checkPermission(token, repository, 'pull')) { return this.createUnauthorizedResponse(repository, 'pull'); @@ -346,30 +393,33 @@ export class OciRegistry extends BaseRegistry { } // If not found locally, try upstream - if (!manifestData && this.upstream) { - this.logger.log('debug', 'getManifest: fetching from upstream', { repository, reference }); - const upstreamResult = await this.upstream.fetchManifest(repository, reference); - if (upstreamResult) { - manifestData = Buffer.from(JSON.stringify(upstreamResult.manifest), 'utf8'); - contentType = upstreamResult.contentType; - digest = upstreamResult.digest; + if (!manifestData) { + const upstream = await this.getUpstreamForRequest(repository, 'manifest', 'GET', actor); + if (upstream) { + this.logger.log('debug', 'getManifest: fetching from upstream', { repository, reference }); + const upstreamResult = await upstream.fetchManifest(repository, reference); + if (upstreamResult) { + manifestData = Buffer.from(JSON.stringify(upstreamResult.manifest), 'utf8'); + contentType = upstreamResult.contentType; + digest = upstreamResult.digest; - // Cache the manifest locally - await this.storage.putOciManifest(repository, digest, manifestData, contentType); + // Cache the manifest locally + await this.storage.putOciManifest(repository, digest, manifestData, contentType); - // If reference is a tag, update tags mapping - if (!reference.startsWith('sha256:')) { - const tags = await this.getTagsData(repository); - tags[reference] = digest; - const tagsPath = `oci/tags/${repository}/tags.json`; - await this.storage.putObject(tagsPath, Buffer.from(JSON.stringify(tags), 'utf-8')); + // If reference is a tag, update tags mapping + if (!reference.startsWith('sha256:')) { + const tags = await this.getTagsData(repository); + tags[reference] = digest; + const tagsPath = `oci/tags/${repository}/tags.json`; + await this.storage.putObject(tagsPath, Buffer.from(JSON.stringify(tags), 'utf-8')); + } + + this.logger.log('debug', 'getManifest: cached manifest locally', { + repository, + reference, + digest, + }); } - - this.logger.log('debug', 'getManifest: cached manifest locally', { - repository, - reference, - digest, - }); } } @@ -514,7 +564,8 @@ export class OciRegistry extends BaseRegistry { repository: string, digest: string, token: IAuthToken | null, - range?: string + range?: string, + actor?: IRequestActor ): Promise { if (!await this.checkPermission(token, repository, 'pull')) { return this.createUnauthorizedResponse(repository, 'pull'); @@ -524,18 +575,21 @@ export class OciRegistry extends BaseRegistry { let data = await this.storage.getOciBlob(digest); // If not found locally, try upstream - if (!data && this.upstream) { - this.logger.log('debug', 'getBlob: fetching from upstream', { repository, digest }); - const upstreamBlob = await this.upstream.fetchBlob(repository, digest); - if (upstreamBlob) { - data = upstreamBlob; - // Cache the blob locally (blobs are content-addressable and immutable) - await this.storage.putOciBlob(digest, data); - this.logger.log('debug', 'getBlob: cached blob locally', { - repository, - digest, - size: data.length, - }); + if (!data) { + const upstream = await this.getUpstreamForRequest(repository, 'blob', 'GET', actor); + if (upstream) { + this.logger.log('debug', 'getBlob: fetching from upstream', { repository, digest }); + const upstreamBlob = await upstream.fetchBlob(repository, digest); + if (upstreamBlob) { + data = upstreamBlob; + // Cache the blob locally (blobs are content-addressable and immutable) + await this.storage.putOciBlob(digest, data); + this.logger.log('debug', 'getBlob: cached blob locally', { + repository, + digest, + size: data.length, + }); + } } } diff --git a/ts/pypi/classes.pypiregistry.ts b/ts/pypi/classes.pypiregistry.ts index 13d5f90..c43be4c 100644 --- a/ts/pypi/classes.pypiregistry.ts +++ b/ts/pypi/classes.pypiregistry.ts @@ -2,8 +2,8 @@ import { Smartlog } from '@push.rocks/smartlog'; import { BaseRegistry } from '../core/classes.baseregistry.js'; import { RegistryStorage } from '../core/classes.registrystorage.js'; import { AuthManager } from '../core/classes.authmanager.js'; -import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js'; -import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js'; +import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from '../core/interfaces.core.js'; +import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js'; import { isBinaryData, toBuffer } from '../core/helpers.buffer.js'; import type { IPypiPackageMetadata, @@ -24,20 +24,21 @@ export class PypiRegistry extends BaseRegistry { private basePath: string = '/pypi'; private registryUrl: string; private logger: Smartlog; - private upstream: PypiUpstream | null = null; + private upstreamProvider: IUpstreamProvider | null = null; constructor( storage: RegistryStorage, authManager: AuthManager, basePath: string = '/pypi', registryUrl: string = 'http://localhost:5000', - upstreamConfig?: IProtocolUpstreamConfig + upstreamProvider?: IUpstreamProvider ) { super(); this.storage = storage; this.authManager = authManager; this.basePath = basePath; this.registryUrl = registryUrl; + this.upstreamProvider = upstreamProvider || null; // Initialize logger this.logger = new Smartlog({ @@ -51,20 +52,38 @@ export class PypiRegistry extends BaseRegistry { } }); this.logger.enableConsole(); + } - // Initialize upstream if configured - if (upstreamConfig?.enabled) { - this.upstream = new PypiUpstream(upstreamConfig, registryUrl, this.logger); - } + /** + * Get upstream for a specific request. + * Calls the provider to resolve upstream config dynamically. + */ + private async getUpstreamForRequest( + resource: string, + resourceType: string, + method: string, + actor?: IRequestActor + ): Promise { + if (!this.upstreamProvider) return null; + + const config = await this.upstreamProvider.resolveUpstreamConfig({ + protocol: 'pypi', + resource, + scope: resource, // For PyPI, package name is the scope + actor, + method, + resourceType, + }); + + if (!config?.enabled) return null; + return new PypiUpstream(config, this.registryUrl, this.logger); } /** * Clean up resources (timers, connections, etc.) */ public destroy(): void { - if (this.upstream) { - this.upstream.stop(); - } + // No persistent upstream to clean up with dynamic provider } public async init(): Promise { @@ -84,15 +103,23 @@ export class PypiRegistry extends BaseRegistry { public async handleRequest(context: IRequestContext): Promise { let path = context.path.replace(this.basePath, ''); + // Extract token (Basic Auth or Bearer) + const token = await this.extractToken(context); + + // Build actor from context and validated token + const actor: IRequestActor = { + ...context.actor, + userId: token?.userId, + ip: context.headers['x-forwarded-for'] || context.headers['X-Forwarded-For'], + userAgent: context.headers['user-agent'] || context.headers['User-Agent'], + }; + // Also handle /simple path prefix if (path.startsWith('/simple')) { path = path.replace('/simple', ''); - return this.handleSimpleRequest(path, context); + return this.handleSimpleRequest(path, context, actor); } - // Extract token (Basic Auth or Bearer) - const token = await this.extractToken(context); - this.logger.log('debug', `handleRequest: ${context.method} ${path}`, { method: context.method, path, @@ -119,7 +146,7 @@ export class PypiRegistry extends BaseRegistry { // Package file download: GET /packages/{package}/{filename} const downloadMatch = path.match(/^\/packages\/([^\/]+)\/(.+)$/); if (downloadMatch && context.method === 'GET') { - return this.handleDownload(downloadMatch[1], downloadMatch[2]); + return this.handleDownload(downloadMatch[1], downloadMatch[2], actor); } // Delete package: DELETE /packages/{package} @@ -156,7 +183,7 @@ export class PypiRegistry extends BaseRegistry { /** * Handle Simple API requests (PEP 503 HTML or PEP 691 JSON) */ - private async handleSimpleRequest(path: string, context: IRequestContext): Promise { + private async handleSimpleRequest(path: string, context: IRequestContext, actor?: IRequestActor): Promise { // Ensure path ends with / (PEP 503 requirement) if (!path.endsWith('/') && !path.includes('.')) { return { @@ -174,7 +201,7 @@ export class PypiRegistry extends BaseRegistry { // Package index: /simple/{package}/ const packageMatch = path.match(/^\/([^\/]+)\/$/); if (packageMatch) { - return this.handleSimplePackage(packageMatch[1], context); + return this.handleSimplePackage(packageMatch[1], context, actor); } return { @@ -228,46 +255,49 @@ export class PypiRegistry extends BaseRegistry { * Handle Simple API package index * Returns HTML (PEP 503) or JSON (PEP 691) based on Accept header */ - private async handleSimplePackage(packageName: string, context: IRequestContext): Promise { + private async handleSimplePackage(packageName: string, context: IRequestContext, actor?: IRequestActor): Promise { const normalized = helpers.normalizePypiPackageName(packageName); // Get package metadata let metadata = await this.storage.getPypiPackageMetadata(normalized); // Try upstream if not found locally - if (!metadata && this.upstream) { - const upstreamHtml = await this.upstream.fetchSimplePackage(normalized); - if (upstreamHtml) { - // Parse the HTML to extract file information and cache it - // For now, just return the upstream HTML directly (caching can be improved later) - const acceptHeader = context.headers['accept'] || context.headers['Accept'] || ''; - const preferJson = acceptHeader.includes('application/vnd.pypi.simple') && - acceptHeader.includes('json'); + if (!metadata) { + const upstream = await this.getUpstreamForRequest(normalized, 'simple', 'GET', actor); + if (upstream) { + const upstreamHtml = await upstream.fetchSimplePackage(normalized); + if (upstreamHtml) { + // Parse the HTML to extract file information and cache it + // For now, just return the upstream HTML directly (caching can be improved later) + const acceptHeader = context.headers['accept'] || context.headers['Accept'] || ''; + const preferJson = acceptHeader.includes('application/vnd.pypi.simple') && + acceptHeader.includes('json'); - if (preferJson) { - // Try to get JSON format from upstream - const upstreamJson = await this.upstream.fetchPackageJson(normalized); - if (upstreamJson) { - return { - status: 200, - headers: { - 'Content-Type': 'application/vnd.pypi.simple.v1+json', - 'Cache-Control': 'public, max-age=300' - }, - body: upstreamJson, - }; + if (preferJson) { + // Try to get JSON format from upstream + const upstreamJson = await upstream.fetchPackageJson(normalized); + if (upstreamJson) { + return { + status: 200, + headers: { + 'Content-Type': 'application/vnd.pypi.simple.v1+json', + 'Cache-Control': 'public, max-age=300' + }, + body: upstreamJson, + }; + } } - } - // Return HTML format - return { - status: 200, - headers: { - 'Content-Type': 'text/html; charset=utf-8', - 'Cache-Control': 'public, max-age=300' - }, - body: upstreamHtml, - }; + // Return HTML format + return { + status: 200, + headers: { + 'Content-Type': 'text/html; charset=utf-8', + 'Cache-Control': 'public, max-age=300' + }, + body: upstreamHtml, + }; + } } } @@ -503,16 +533,19 @@ export class PypiRegistry extends BaseRegistry { /** * Handle package download */ - private async handleDownload(packageName: string, filename: string): Promise { + private async handleDownload(packageName: string, filename: string, actor?: IRequestActor): Promise { const normalized = helpers.normalizePypiPackageName(packageName); let fileData = await this.storage.getPypiPackageFile(normalized, filename); // Try upstream if not found locally - if (!fileData && this.upstream) { - fileData = await this.upstream.fetchPackageFile(normalized, filename); - if (fileData) { - // Cache locally - await this.storage.putPypiPackageFile(normalized, filename, fileData); + if (!fileData) { + const upstream = await this.getUpstreamForRequest(normalized, 'file', 'GET', actor); + if (upstream) { + fileData = await upstream.fetchPackageFile(normalized, filename); + if (fileData) { + // Cache locally + await this.storage.putPypiPackageFile(normalized, filename, fileData); + } } } diff --git a/ts/rubygems/classes.rubygemsregistry.ts b/ts/rubygems/classes.rubygemsregistry.ts index 8e8dc9c..0aac4bd 100644 --- a/ts/rubygems/classes.rubygemsregistry.ts +++ b/ts/rubygems/classes.rubygemsregistry.ts @@ -2,8 +2,8 @@ import { Smartlog } from '@push.rocks/smartlog'; import { BaseRegistry } from '../core/classes.baseregistry.js'; import { RegistryStorage } from '../core/classes.registrystorage.js'; import { AuthManager } from '../core/classes.authmanager.js'; -import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js'; -import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js'; +import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from '../core/interfaces.core.js'; +import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js'; import type { IRubyGemsMetadata, IRubyGemsVersionMetadata, @@ -25,20 +25,21 @@ export class RubyGemsRegistry extends BaseRegistry { private basePath: string = '/rubygems'; private registryUrl: string; private logger: Smartlog; - private upstream: RubygemsUpstream | null = null; + private upstreamProvider: IUpstreamProvider | null = null; constructor( storage: RegistryStorage, authManager: AuthManager, basePath: string = '/rubygems', registryUrl: string = 'http://localhost:5000/rubygems', - upstreamConfig?: IProtocolUpstreamConfig + upstreamProvider?: IUpstreamProvider ) { super(); this.storage = storage; this.authManager = authManager; this.basePath = basePath; this.registryUrl = registryUrl; + this.upstreamProvider = upstreamProvider || null; // Initialize logger this.logger = new Smartlog({ @@ -52,20 +53,38 @@ export class RubyGemsRegistry extends BaseRegistry { } }); this.logger.enableConsole(); + } - // Initialize upstream if configured - if (upstreamConfig?.enabled) { - this.upstream = new RubygemsUpstream(upstreamConfig, this.logger); - } + /** + * Get upstream for a specific request. + * Calls the provider to resolve upstream config dynamically. + */ + private async getUpstreamForRequest( + resource: string, + resourceType: string, + method: string, + actor?: IRequestActor + ): Promise { + if (!this.upstreamProvider) return null; + + const config = await this.upstreamProvider.resolveUpstreamConfig({ + protocol: 'rubygems', + resource, + scope: resource, // gem name is the scope + actor, + method, + resourceType, + }); + + if (!config?.enabled) return null; + return new RubygemsUpstream(config, this.logger); } /** * Clean up resources (timers, connections, etc.) */ public destroy(): void { - if (this.upstream) { - this.upstream.stop(); - } + // No persistent upstream to clean up with dynamic provider } public async init(): Promise { @@ -95,6 +114,14 @@ export class RubyGemsRegistry extends BaseRegistry { // Extract token (Authorization header) const token = await this.extractToken(context); + // Build actor from context and validated token + const actor: IRequestActor = { + ...context.actor, + userId: token?.userId, + ip: context.headers['x-forwarded-for'] || context.headers['X-Forwarded-For'], + userAgent: context.headers['user-agent'] || context.headers['User-Agent'], + }; + this.logger.log('debug', `handleRequest: ${context.method} ${path}`, { method: context.method, path, @@ -113,13 +140,13 @@ export class RubyGemsRegistry extends BaseRegistry { // Info file: GET /info/{gem} const infoMatch = path.match(/^\/info\/([^\/]+)$/); if (infoMatch && context.method === 'GET') { - return this.handleInfoFile(infoMatch[1]); + return this.handleInfoFile(infoMatch[1], actor); } // Gem download: GET /gems/{gem}-{version}[-{platform}].gem const downloadMatch = path.match(/^\/gems\/(.+\.gem)$/); if (downloadMatch && context.method === 'GET') { - return this.handleDownload(downloadMatch[1]); + return this.handleDownload(downloadMatch[1], actor); } // Legacy specs endpoints (Marshal format) @@ -232,16 +259,19 @@ export class RubyGemsRegistry extends BaseRegistry { /** * Handle /info/{gem} endpoint (Compact Index) */ - private async handleInfoFile(gemName: string): Promise { + private async handleInfoFile(gemName: string, actor?: IRequestActor): Promise { let content = await this.storage.getRubyGemsInfo(gemName); // Try upstream if not found locally - if (!content && this.upstream) { - const upstreamInfo = await this.upstream.fetchInfo(gemName); - if (upstreamInfo) { - // Cache locally - await this.storage.putRubyGemsInfo(gemName, upstreamInfo); - content = upstreamInfo; + if (!content) { + const upstream = await this.getUpstreamForRequest(gemName, 'info', 'GET', actor); + if (upstream) { + const upstreamInfo = await upstream.fetchInfo(gemName); + if (upstreamInfo) { + // Cache locally + await this.storage.putRubyGemsInfo(gemName, upstreamInfo); + content = upstreamInfo; + } } } @@ -267,7 +297,7 @@ export class RubyGemsRegistry extends BaseRegistry { /** * Handle gem file download */ - private async handleDownload(filename: string): Promise { + private async handleDownload(filename: string, actor?: IRequestActor): Promise { const parsed = helpers.parseGemFilename(filename); if (!parsed) { return this.errorResponse(400, 'Invalid gem filename'); @@ -280,11 +310,14 @@ export class RubyGemsRegistry extends BaseRegistry { ); // Try upstream if not found locally - if (!gemData && this.upstream) { - gemData = await this.upstream.fetchGem(parsed.name, parsed.version); - if (gemData) { - // Cache locally - await this.storage.putRubyGemsGem(parsed.name, parsed.version, gemData, parsed.platform); + if (!gemData) { + const upstream = await this.getUpstreamForRequest(parsed.name, 'gem', 'GET', actor); + if (upstream) { + gemData = await upstream.fetchGem(parsed.name, parsed.version); + if (gemData) { + // Cache locally + await this.storage.putRubyGemsGem(parsed.name, parsed.version, gemData, parsed.platform); + } } } diff --git a/ts/upstream/interfaces.upstream.ts b/ts/upstream/interfaces.upstream.ts index 99f6e82..1bfe342 100644 --- a/ts/upstream/interfaces.upstream.ts +++ b/ts/upstream/interfaces.upstream.ts @@ -1,4 +1,4 @@ -import type { TRegistryProtocol } from '../core/interfaces.core.js'; +import type { TRegistryProtocol, IRequestActor } from '../core/interfaces.core.js'; /** * Scope rule for routing requests to specific upstreams. @@ -146,6 +146,8 @@ export interface IUpstreamFetchContext { headers: Record; /** Query parameters */ query: Record; + /** Actor performing the request (for cache key isolation) */ + actor?: IRequestActor; } /** @@ -193,3 +195,80 @@ export const DEFAULT_RESILIENCE_CONFIG: IUpstreamResilienceConfig = { circuitBreakerThreshold: 5, circuitBreakerResetMs: 30000, }; + +// ============================================================================ +// Upstream Provider Interfaces +// ============================================================================ + +/** + * Context for resolving upstream configuration. + * Passed to IUpstreamProvider per-request to enable dynamic upstream routing. + */ +export interface IUpstreamResolutionContext { + /** Protocol being accessed */ + protocol: TRegistryProtocol; + /** Resource identifier (package name, repository, coordinates, etc.) */ + resource: string; + /** Extracted scope (e.g., "company" from "@company/pkg", "myorg" from "myorg/image") */ + scope: string | null; + /** Actor performing the request */ + actor?: IRequestActor; + /** HTTP method */ + method: string; + /** Resource type (packument, tarball, manifest, blob, etc.) */ + resourceType: string; +} + +/** + * Dynamic upstream configuration provider. + * Implement this interface to provide per-request upstream routing + * based on actor context (user, organization, etc.) + * + * @example + * ```typescript + * class OrgUpstreamProvider implements IUpstreamProvider { + * constructor(private db: Database) {} + * + * async resolveUpstreamConfig(ctx: IUpstreamResolutionContext) { + * if (ctx.actor?.orgId) { + * const orgConfig = await this.db.getOrgUpstream(ctx.actor.orgId, ctx.protocol); + * if (orgConfig) return orgConfig; + * } + * return this.db.getDefaultUpstream(ctx.protocol); + * } + * } + * ``` + */ +export interface IUpstreamProvider { + /** Optional initialization */ + init?(): Promise; + + /** + * Resolve upstream configuration for a request. + * @param context - Information about the current request + * @returns Upstream config to use, or null to skip upstream lookup + */ + resolveUpstreamConfig(context: IUpstreamResolutionContext): Promise; +} + +/** + * Static upstream provider for simple configurations. + * Use this when you have fixed upstream registries that don't change per-request. + * + * @example + * ```typescript + * const provider = new StaticUpstreamProvider({ + * npm: { + * enabled: true, + * upstreams: [{ id: 'npmjs', url: 'https://registry.npmjs.org', priority: 1, enabled: true, auth: { type: 'none' } }], + * }, + * }); + * ``` + */ +export class StaticUpstreamProvider implements IUpstreamProvider { + constructor(private configs: Partial>) {} + + async resolveUpstreamConfig(ctx: IUpstreamResolutionContext): Promise { + return this.configs[ctx.protocol] ?? null; + } +}