feat(upstream): Add dynamic per-request upstream provider and integrate into registries

This commit is contained in:
2025-12-03 22:16:40 +00:00
parent 351680159b
commit e9af3f8328
14 changed files with 1117 additions and 287 deletions

View File

@@ -1,5 +1,17 @@
# Changelog # Changelog
## 2025-12-03 - 2.7.0 - feat(upstream)
Add dynamic per-request upstream provider and integrate into registries
- Introduce IUpstreamProvider and IUpstreamResolutionContext to resolve upstream configs per request.
- Add StaticUpstreamProvider implementation for simple static upstream configurations.
- Propagate dynamic upstream provider through SmartRegistry and wire into protocol handlers (npm, oci, maven, cargo, composer, pypi, rubygems).
- Replace persistent per-protocol upstream instances with per-request resolution: registries now call provider.resolveUpstreamConfig(...) and instantiate protocol-specific Upstream when needed.
- Add IRequestActor to core interfaces and pass actor context (userId, ip, userAgent, etc.) to upstream resolution and storage/auth hooks.
- Update many protocol registries to accept an upstreamProvider instead of IProtocolUpstreamConfig and to attempt upstream fetches only when provider returns enabled config.
- Add utilities and tests: test helpers to create registries with upstream provider, a tracking upstream provider helper, StaticUpstreamProvider tests and extensive upstream/provider integration tests.
- Improve upstream interfaces and cache/fetch contexts (IUpstreamFetchContext includes actor) and add StaticUpstreamProvider class to upstream module.
## 2025-11-27 - 2.6.0 - feat(core) ## 2025-11-27 - 2.6.0 - feat(core)
Add core registry infrastructure: storage, auth, upstream cache, and protocol handlers Add core registry infrastructure: storage, auth, upstream cache, and protocol handlers

View File

@@ -6,6 +6,8 @@ import { SmartRegistry } from '../../ts/classes.smartregistry.js';
import type { IRegistryConfig, IAuthToken, TRegistryProtocol } from '../../ts/core/interfaces.core.js'; import type { IRegistryConfig, IAuthToken, TRegistryProtocol } from '../../ts/core/interfaces.core.js';
import type { IAuthProvider, ITokenOptions } from '../../ts/core/interfaces.auth.js'; import type { IAuthProvider, ITokenOptions } from '../../ts/core/interfaces.auth.js';
import type { IStorageHooks, IStorageHookContext, IBeforePutResult, IBeforeDeleteResult } from '../../ts/core/interfaces.storage.js'; import type { IStorageHooks, IStorageHookContext, IBeforePutResult, IBeforeDeleteResult } from '../../ts/core/interfaces.storage.js';
import { StaticUpstreamProvider } from '../../ts/upstream/interfaces.upstream.js';
import type { IUpstreamProvider, IUpstreamResolutionContext, IProtocolUpstreamConfig } from '../../ts/upstream/interfaces.upstream.js';
const testQenv = new qenv.Qenv('./', './.nogit'); const testQenv = new qenv.Qenv('./', './.nogit');
@@ -134,6 +136,89 @@ export async function createTestRegistry(): Promise<SmartRegistry> {
return registry; return registry;
} }
/**
* Create a test SmartRegistry instance with upstream provider configured
*/
export async function createTestRegistryWithUpstream(
upstreamProvider?: IUpstreamProvider
): Promise<SmartRegistry> {
// Read S3 config from env.json
const s3AccessKey = await testQenv.getEnvVarOnDemand('S3_ACCESSKEY');
const s3SecretKey = await testQenv.getEnvVarOnDemand('S3_SECRETKEY');
const s3Endpoint = await testQenv.getEnvVarOnDemand('S3_ENDPOINT');
const s3Port = await testQenv.getEnvVarOnDemand('S3_PORT');
// Default to StaticUpstreamProvider with npm.js configured
const defaultProvider = new StaticUpstreamProvider({
npm: {
enabled: true,
upstreams: [{ id: 'npmjs', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
},
oci: {
enabled: true,
upstreams: [{ id: 'dockerhub', url: 'https://registry-1.docker.io', priority: 1, enabled: true }],
},
});
const config: IRegistryConfig = {
storage: {
accessKey: s3AccessKey || 'minioadmin',
accessSecret: s3SecretKey || 'minioadmin',
endpoint: s3Endpoint || 'localhost',
port: parseInt(s3Port || '9000', 10),
useSsl: false,
region: 'us-east-1',
bucketName: 'test-registry',
},
auth: {
jwtSecret: 'test-secret-key',
tokenStore: 'memory',
npmTokens: { enabled: true },
ociTokens: {
enabled: true,
realm: 'https://auth.example.com/token',
service: 'test-registry',
},
pypiTokens: { enabled: true },
rubygemsTokens: { enabled: true },
},
upstreamProvider: upstreamProvider || defaultProvider,
oci: { enabled: true, basePath: '/oci' },
npm: { enabled: true, basePath: '/npm' },
maven: { enabled: true, basePath: '/maven' },
composer: { enabled: true, basePath: '/composer' },
cargo: { enabled: true, basePath: '/cargo' },
pypi: { enabled: true, basePath: '/pypi' },
rubygems: { enabled: true, basePath: '/rubygems' },
};
const registry = new SmartRegistry(config);
await registry.init();
return registry;
}
/**
* Create a mock upstream provider that tracks all calls for testing
*/
export function createTrackingUpstreamProvider(
baseConfig?: Partial<Record<TRegistryProtocol, IProtocolUpstreamConfig>>
): {
provider: IUpstreamProvider;
calls: IUpstreamResolutionContext[];
} {
const calls: IUpstreamResolutionContext[] = [];
const provider: IUpstreamProvider = {
async resolveUpstreamConfig(context: IUpstreamResolutionContext) {
calls.push({ ...context });
return baseConfig?.[context.protocol] ?? null;
},
};
return { provider, calls };
}
/** /**
* Helper to create test authentication tokens * Helper to create test authentication tokens
*/ */

View File

@@ -0,0 +1,343 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js';
import {
createTestRegistryWithUpstream,
createTrackingUpstreamProvider,
} from './helpers/registry.js';
import { StaticUpstreamProvider } from '../ts/upstream/interfaces.upstream.js';
import type {
IUpstreamProvider,
IUpstreamResolutionContext,
IProtocolUpstreamConfig,
} from '../ts/upstream/interfaces.upstream.js';
import type { TRegistryProtocol } from '../ts/core/interfaces.core.js';
// =============================================================================
// StaticUpstreamProvider Tests
// =============================================================================
tap.test('StaticUpstreamProvider: should return config for configured protocol', async () => {
const npmConfig: IProtocolUpstreamConfig = {
enabled: true,
upstreams: [{ id: 'npmjs', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
};
const provider = new StaticUpstreamProvider({
npm: npmConfig,
});
const result = await provider.resolveUpstreamConfig({
protocol: 'npm',
resource: 'lodash',
scope: null,
method: 'GET',
resourceType: 'packument',
});
expect(result).toBeDefined();
expect(result?.enabled).toEqual(true);
expect(result?.upstreams[0].id).toEqual('npmjs');
});
tap.test('StaticUpstreamProvider: should return null for unconfigured protocol', async () => {
const provider = new StaticUpstreamProvider({
npm: {
enabled: true,
upstreams: [{ id: 'npmjs', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
},
});
const result = await provider.resolveUpstreamConfig({
protocol: 'maven',
resource: 'com.example:lib',
scope: 'com.example',
method: 'GET',
resourceType: 'pom',
});
expect(result).toBeNull();
});
tap.test('StaticUpstreamProvider: should support multiple protocols', async () => {
const provider = new StaticUpstreamProvider({
npm: {
enabled: true,
upstreams: [{ id: 'npmjs', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
},
oci: {
enabled: true,
upstreams: [{ id: 'dockerhub', url: 'https://registry-1.docker.io', priority: 1, enabled: true }],
},
maven: {
enabled: true,
upstreams: [{ id: 'central', url: 'https://repo1.maven.org/maven2', priority: 1, enabled: true }],
},
});
const npmResult = await provider.resolveUpstreamConfig({
protocol: 'npm',
resource: 'lodash',
scope: null,
method: 'GET',
resourceType: 'packument',
});
expect(npmResult?.upstreams[0].id).toEqual('npmjs');
const ociResult = await provider.resolveUpstreamConfig({
protocol: 'oci',
resource: 'library/nginx',
scope: 'library',
method: 'GET',
resourceType: 'manifest',
});
expect(ociResult?.upstreams[0].id).toEqual('dockerhub');
const mavenResult = await provider.resolveUpstreamConfig({
protocol: 'maven',
resource: 'com.example:lib',
scope: 'com.example',
method: 'GET',
resourceType: 'pom',
});
expect(mavenResult?.upstreams[0].id).toEqual('central');
});
// =============================================================================
// Registry with Provider Integration Tests
// =============================================================================
let registry: SmartRegistry;
let trackingProvider: ReturnType<typeof createTrackingUpstreamProvider>;
tap.test('Provider Integration: should create registry with upstream provider', async () => {
trackingProvider = createTrackingUpstreamProvider({
npm: {
enabled: true,
upstreams: [{ id: 'test-npm', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
},
});
registry = await createTestRegistryWithUpstream(trackingProvider.provider);
expect(registry).toBeInstanceOf(SmartRegistry);
expect(registry.isInitialized()).toEqual(true);
});
tap.test('Provider Integration: should call provider when fetching unknown npm package', async () => {
// Clear previous calls
trackingProvider.calls.length = 0;
// Request a package that doesn't exist locally - should trigger upstream lookup
const response = await registry.handleRequest({
method: 'GET',
path: '/npm/@test-scope/nonexistent-package',
headers: {},
query: {},
});
// Provider should have been called for the packument lookup
const npmCalls = trackingProvider.calls.filter(c => c.protocol === 'npm');
// The package doesn't exist locally, so upstream should be consulted
// Note: actual upstream fetch may fail since the package doesn't exist
expect(response.status).toBeOneOf([404, 200, 502]); // 404 if not found, 502 if upstream error
});
tap.test('Provider Integration: provider receives correct context for scoped npm package', async () => {
trackingProvider.calls.length = 0;
// Use URL-encoded path for scoped packages as npm client does
await registry.handleRequest({
method: 'GET',
path: '/npm/@myorg%2fmy-package',
headers: {},
query: {},
});
// Find any npm call - the exact resource type depends on routing
const npmCalls = trackingProvider.calls.filter(c => c.protocol === 'npm');
// Provider should be called for upstream lookup
if (npmCalls.length > 0) {
const call = npmCalls[0];
expect(call.protocol).toEqual('npm');
// The resource should include the scoped name
expect(call.resource).toInclude('myorg');
expect(call.method).toEqual('GET');
}
});
tap.test('Provider Integration: provider receives correct context for unscoped npm package', async () => {
trackingProvider.calls.length = 0;
await registry.handleRequest({
method: 'GET',
path: '/npm/lodash',
headers: {},
query: {},
});
const packumentCall = trackingProvider.calls.find(
c => c.protocol === 'npm' && c.resourceType === 'packument'
);
if (packumentCall) {
expect(packumentCall.protocol).toEqual('npm');
expect(packumentCall.resource).toEqual('lodash');
expect(packumentCall.scope).toBeNull(); // No scope for unscoped package
}
});
// =============================================================================
// Custom Provider Implementation Tests
// =============================================================================
tap.test('Custom Provider: should support dynamic resolution based on context', async () => {
// Create a provider that returns different configs based on scope
const dynamicProvider: IUpstreamProvider = {
async resolveUpstreamConfig(context: IUpstreamResolutionContext) {
if (context.scope === 'internal') {
// Internal packages go to private registry
return {
enabled: true,
upstreams: [{ id: 'private', url: 'https://private.registry.com', priority: 1, enabled: true }],
};
}
// Everything else goes to public registry
return {
enabled: true,
upstreams: [{ id: 'public', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
};
},
};
const internalResult = await dynamicProvider.resolveUpstreamConfig({
protocol: 'npm',
resource: '@internal/utils',
scope: 'internal',
method: 'GET',
resourceType: 'packument',
});
expect(internalResult?.upstreams[0].id).toEqual('private');
const publicResult = await dynamicProvider.resolveUpstreamConfig({
protocol: 'npm',
resource: '@public/utils',
scope: 'public',
method: 'GET',
resourceType: 'packument',
});
expect(publicResult?.upstreams[0].id).toEqual('public');
});
tap.test('Custom Provider: should support actor-based resolution', async () => {
const actorAwareProvider: IUpstreamProvider = {
async resolveUpstreamConfig(context: IUpstreamResolutionContext) {
// Different upstreams based on user's organization
if (context.actor?.orgId === 'enterprise-org') {
return {
enabled: true,
upstreams: [{ id: 'enterprise', url: 'https://enterprise.registry.com', priority: 1, enabled: true }],
};
}
return {
enabled: true,
upstreams: [{ id: 'default', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
};
},
};
const enterpriseResult = await actorAwareProvider.resolveUpstreamConfig({
protocol: 'npm',
resource: 'lodash',
scope: null,
actor: { orgId: 'enterprise-org', userId: 'user1' },
method: 'GET',
resourceType: 'packument',
});
expect(enterpriseResult?.upstreams[0].id).toEqual('enterprise');
const defaultResult = await actorAwareProvider.resolveUpstreamConfig({
protocol: 'npm',
resource: 'lodash',
scope: null,
actor: { orgId: 'free-org', userId: 'user2' },
method: 'GET',
resourceType: 'packument',
});
expect(defaultResult?.upstreams[0].id).toEqual('default');
});
tap.test('Custom Provider: should support disabling upstream for specific resources', async () => {
const selectiveProvider: IUpstreamProvider = {
async resolveUpstreamConfig(context: IUpstreamResolutionContext) {
// Block upstream for internal packages
if (context.scope === 'internal') {
return null; // No upstream for internal packages
}
return {
enabled: true,
upstreams: [{ id: 'public', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
};
},
};
const internalResult = await selectiveProvider.resolveUpstreamConfig({
protocol: 'npm',
resource: '@internal/secret',
scope: 'internal',
method: 'GET',
resourceType: 'packument',
});
expect(internalResult).toBeNull();
const publicResult = await selectiveProvider.resolveUpstreamConfig({
protocol: 'npm',
resource: 'lodash',
scope: null,
method: 'GET',
resourceType: 'packument',
});
expect(publicResult).not.toBeNull();
});
// =============================================================================
// Registry without Provider Tests
// =============================================================================
tap.test('No Provider: registry should work without upstream provider', async () => {
const registryWithoutUpstream = await createTestRegistryWithUpstream(
// Pass a provider that always returns null
{
async resolveUpstreamConfig() {
return null;
},
}
);
expect(registryWithoutUpstream).toBeInstanceOf(SmartRegistry);
// Should return 404 for non-existent package (no upstream to check)
const response = await registryWithoutUpstream.handleRequest({
method: 'GET',
path: '/npm/nonexistent-package-xyz',
headers: {},
query: {},
});
expect(response.status).toEqual(404);
registryWithoutUpstream.destroy();
});
// =============================================================================
// Cleanup
// =============================================================================
tap.postTask('cleanup registry', async () => {
if (registry) {
registry.destroy();
}
});
export default tap.start();

View File

@@ -3,6 +3,6 @@
*/ */
export const commitinfo = { export const commitinfo = {
name: '@push.rocks/smartregistry', name: '@push.rocks/smartregistry',
version: '2.6.0', version: '2.7.0',
description: 'A composable TypeScript library implementing OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems registries for building unified container and package registries' description: 'A composable TypeScript library implementing OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems registries for building unified container and package registries'
} }

View File

@@ -2,8 +2,8 @@ import { Smartlog } from '@push.rocks/smartlog';
import { BaseRegistry } from '../core/classes.baseregistry.js'; import { BaseRegistry } from '../core/classes.baseregistry.js';
import { RegistryStorage } from '../core/classes.registrystorage.js'; import { RegistryStorage } from '../core/classes.registrystorage.js';
import { AuthManager } from '../core/classes.authmanager.js'; import { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js'; import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from '../core/interfaces.core.js';
import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js'; import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
import type { import type {
ICargoIndexEntry, ICargoIndexEntry,
ICargoPublishMetadata, ICargoPublishMetadata,
@@ -27,20 +27,21 @@ export class CargoRegistry extends BaseRegistry {
private basePath: string = '/cargo'; private basePath: string = '/cargo';
private registryUrl: string; private registryUrl: string;
private logger: Smartlog; private logger: Smartlog;
private upstream: CargoUpstream | null = null; private upstreamProvider: IUpstreamProvider | null = null;
constructor( constructor(
storage: RegistryStorage, storage: RegistryStorage,
authManager: AuthManager, authManager: AuthManager,
basePath: string = '/cargo', basePath: string = '/cargo',
registryUrl: string = 'http://localhost:5000/cargo', registryUrl: string = 'http://localhost:5000/cargo',
upstreamConfig?: IProtocolUpstreamConfig upstreamProvider?: IUpstreamProvider
) { ) {
super(); super();
this.storage = storage; this.storage = storage;
this.authManager = authManager; this.authManager = authManager;
this.basePath = basePath; this.basePath = basePath;
this.registryUrl = registryUrl; this.registryUrl = registryUrl;
this.upstreamProvider = upstreamProvider || null;
// Initialize logger // Initialize logger
this.logger = new Smartlog({ this.logger = new Smartlog({
@@ -54,20 +55,38 @@ export class CargoRegistry extends BaseRegistry {
} }
}); });
this.logger.enableConsole(); this.logger.enableConsole();
}
// Initialize upstream if configured /**
if (upstreamConfig?.enabled) { * Get upstream for a specific request.
this.upstream = new CargoUpstream(upstreamConfig, undefined, this.logger); * Calls the provider to resolve upstream config dynamically.
} */
private async getUpstreamForRequest(
resource: string,
resourceType: string,
method: string,
actor?: IRequestActor
): Promise<CargoUpstream | null> {
if (!this.upstreamProvider) return null;
const config = await this.upstreamProvider.resolveUpstreamConfig({
protocol: 'cargo',
resource,
scope: resource, // For Cargo, crate name is the scope
actor,
method,
resourceType,
});
if (!config?.enabled) return null;
return new CargoUpstream(config, undefined, this.logger);
} }
/** /**
* Clean up resources (timers, connections, etc.) * Clean up resources (timers, connections, etc.)
*/ */
public destroy(): void { public destroy(): void {
if (this.upstream) { // No persistent upstream to clean up with dynamic provider
this.upstream.stop();
}
} }
public async init(): Promise<void> { public async init(): Promise<void> {
@@ -94,6 +113,14 @@ export class CargoRegistry extends BaseRegistry {
const authHeader = context.headers['authorization'] || context.headers['Authorization']; const authHeader = context.headers['authorization'] || context.headers['Authorization'];
const token = authHeader ? await this.authManager.validateToken(authHeader, 'cargo') : null; const token = authHeader ? await this.authManager.validateToken(authHeader, 'cargo') : null;
// Build actor from context and validated token
const actor: IRequestActor = {
...context.actor,
userId: token?.userId,
ip: context.headers['x-forwarded-for'] || context.headers['X-Forwarded-For'],
userAgent: context.headers['user-agent'] || context.headers['User-Agent'],
};
this.logger.log('debug', `handleRequest: ${context.method} ${path}`, { this.logger.log('debug', `handleRequest: ${context.method} ${path}`, {
method: context.method, method: context.method,
path, path,
@@ -107,11 +134,11 @@ export class CargoRegistry extends BaseRegistry {
// API endpoints // API endpoints
if (path.startsWith('/api/v1/')) { if (path.startsWith('/api/v1/')) {
return this.handleApiRequest(path, context, token); return this.handleApiRequest(path, context, token, actor);
} }
// Index files (sparse protocol) // Index files (sparse protocol)
return this.handleIndexRequest(path); return this.handleIndexRequest(path, actor);
} }
/** /**
@@ -132,7 +159,8 @@ export class CargoRegistry extends BaseRegistry {
private async handleApiRequest( private async handleApiRequest(
path: string, path: string,
context: IRequestContext, context: IRequestContext,
token: IAuthToken | null token: IAuthToken | null,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
// Publish: PUT /api/v1/crates/new // Publish: PUT /api/v1/crates/new
if (path === '/api/v1/crates/new' && context.method === 'PUT') { if (path === '/api/v1/crates/new' && context.method === 'PUT') {
@@ -142,7 +170,7 @@ export class CargoRegistry extends BaseRegistry {
// Download: GET /api/v1/crates/{crate}/{version}/download // Download: GET /api/v1/crates/{crate}/{version}/download
const downloadMatch = path.match(/^\/api\/v1\/crates\/([^\/]+)\/([^\/]+)\/download$/); const downloadMatch = path.match(/^\/api\/v1\/crates\/([^\/]+)\/([^\/]+)\/download$/);
if (downloadMatch && context.method === 'GET') { if (downloadMatch && context.method === 'GET') {
return this.handleDownload(downloadMatch[1], downloadMatch[2]); return this.handleDownload(downloadMatch[1], downloadMatch[2], actor);
} }
// Yank: DELETE /api/v1/crates/{crate}/{version}/yank // Yank: DELETE /api/v1/crates/{crate}/{version}/yank
@@ -175,7 +203,7 @@ export class CargoRegistry extends BaseRegistry {
* Handle index file requests * Handle index file requests
* Paths: /1/{name}, /2/{name}, /3/{c}/{name}, /{p1}/{p2}/{name} * Paths: /1/{name}, /2/{name}, /3/{c}/{name}, /{p1}/{p2}/{name}
*/ */
private async handleIndexRequest(path: string): Promise<IResponse> { private async handleIndexRequest(path: string, actor?: IRequestActor): Promise<IResponse> {
// Parse index paths to extract crate name // Parse index paths to extract crate name
const pathParts = path.split('/').filter(p => p); const pathParts = path.split('/').filter(p => p);
let crateName: string | null = null; let crateName: string | null = null;
@@ -202,7 +230,7 @@ export class CargoRegistry extends BaseRegistry {
}; };
} }
return this.handleIndexFile(crateName); return this.handleIndexFile(crateName, actor);
} }
/** /**
@@ -224,23 +252,26 @@ export class CargoRegistry extends BaseRegistry {
/** /**
* Serve index file for a crate * Serve index file for a crate
*/ */
private async handleIndexFile(crateName: string): Promise<IResponse> { private async handleIndexFile(crateName: string, actor?: IRequestActor): Promise<IResponse> {
let index = await this.storage.getCargoIndex(crateName); let index = await this.storage.getCargoIndex(crateName);
// Try upstream if not found locally // Try upstream if not found locally
if ((!index || index.length === 0) && this.upstream) { if (!index || index.length === 0) {
const upstreamIndex = await this.upstream.fetchCrateIndex(crateName); const upstream = await this.getUpstreamForRequest(crateName, 'index', 'GET', actor);
if (upstreamIndex) { if (upstream) {
// Parse the newline-delimited JSON const upstreamIndex = await upstream.fetchCrateIndex(crateName);
const parsedIndex: ICargoIndexEntry[] = upstreamIndex if (upstreamIndex) {
.split('\n') // Parse the newline-delimited JSON
.filter(line => line.trim()) const parsedIndex: ICargoIndexEntry[] = upstreamIndex
.map(line => JSON.parse(line)); .split('\n')
.filter(line => line.trim())
.map(line => JSON.parse(line));
if (parsedIndex.length > 0) { if (parsedIndex.length > 0) {
// Cache locally // Cache locally
await this.storage.putCargoIndex(crateName, parsedIndex); await this.storage.putCargoIndex(crateName, parsedIndex);
index = parsedIndex; index = parsedIndex;
}
} }
} }
} }
@@ -431,18 +462,22 @@ export class CargoRegistry extends BaseRegistry {
*/ */
private async handleDownload( private async handleDownload(
crateName: string, crateName: string,
version: string version: string,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
this.logger.log('debug', 'handleDownload', { crate: crateName, version }); this.logger.log('debug', 'handleDownload', { crate: crateName, version });
let crateFile = await this.storage.getCargoCrate(crateName, version); let crateFile = await this.storage.getCargoCrate(crateName, version);
// Try upstream if not found locally // Try upstream if not found locally
if (!crateFile && this.upstream) { if (!crateFile) {
crateFile = await this.upstream.fetchCrate(crateName, version); const upstream = await this.getUpstreamForRequest(crateName, 'crate', 'GET', actor);
if (crateFile) { if (upstream) {
// Cache locally crateFile = await upstream.fetchCrate(crateName, version);
await this.storage.putCargoCrate(crateName, version, crateFile); if (crateFile) {
// Cache locally
await this.storage.putCargoCrate(crateName, version, crateFile);
}
} }
} }

View File

@@ -86,7 +86,7 @@ export class SmartRegistry {
this.authManager, this.authManager,
ociBasePath, ociBasePath,
ociTokens, ociTokens,
this.config.oci.upstream this.config.upstreamProvider
); );
await ociRegistry.init(); await ociRegistry.init();
this.registries.set('oci', ociRegistry); this.registries.set('oci', ociRegistry);
@@ -101,7 +101,7 @@ export class SmartRegistry {
this.authManager, this.authManager,
npmBasePath, npmBasePath,
registryUrl, registryUrl,
this.config.npm.upstream this.config.upstreamProvider
); );
await npmRegistry.init(); await npmRegistry.init();
this.registries.set('npm', npmRegistry); this.registries.set('npm', npmRegistry);
@@ -116,7 +116,7 @@ export class SmartRegistry {
this.authManager, this.authManager,
mavenBasePath, mavenBasePath,
registryUrl, registryUrl,
this.config.maven.upstream this.config.upstreamProvider
); );
await mavenRegistry.init(); await mavenRegistry.init();
this.registries.set('maven', mavenRegistry); this.registries.set('maven', mavenRegistry);
@@ -131,7 +131,7 @@ export class SmartRegistry {
this.authManager, this.authManager,
cargoBasePath, cargoBasePath,
registryUrl, registryUrl,
this.config.cargo.upstream this.config.upstreamProvider
); );
await cargoRegistry.init(); await cargoRegistry.init();
this.registries.set('cargo', cargoRegistry); this.registries.set('cargo', cargoRegistry);
@@ -146,7 +146,7 @@ export class SmartRegistry {
this.authManager, this.authManager,
composerBasePath, composerBasePath,
registryUrl, registryUrl,
this.config.composer.upstream this.config.upstreamProvider
); );
await composerRegistry.init(); await composerRegistry.init();
this.registries.set('composer', composerRegistry); this.registries.set('composer', composerRegistry);
@@ -161,7 +161,7 @@ export class SmartRegistry {
this.authManager, this.authManager,
pypiBasePath, pypiBasePath,
registryUrl, registryUrl,
this.config.pypi.upstream this.config.upstreamProvider
); );
await pypiRegistry.init(); await pypiRegistry.init();
this.registries.set('pypi', pypiRegistry); this.registries.set('pypi', pypiRegistry);
@@ -176,7 +176,7 @@ export class SmartRegistry {
this.authManager, this.authManager,
rubygemsBasePath, rubygemsBasePath,
registryUrl, registryUrl,
this.config.rubygems.upstream this.config.upstreamProvider
); );
await rubygemsRegistry.init(); await rubygemsRegistry.init();
this.registries.set('rubygems', rubygemsRegistry); this.registries.set('rubygems', rubygemsRegistry);

View File

@@ -6,8 +6,8 @@
import { BaseRegistry } from '../core/classes.baseregistry.js'; import { BaseRegistry } from '../core/classes.baseregistry.js';
import type { RegistryStorage } from '../core/classes.registrystorage.js'; import type { RegistryStorage } from '../core/classes.registrystorage.js';
import type { AuthManager } from '../core/classes.authmanager.js'; import type { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js'; import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from '../core/interfaces.core.js';
import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js'; import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
import { isBinaryData, toBuffer } from '../core/helpers.buffer.js'; import { isBinaryData, toBuffer } from '../core/helpers.buffer.js';
import type { import type {
IComposerPackage, IComposerPackage,
@@ -30,34 +30,66 @@ export class ComposerRegistry extends BaseRegistry {
private authManager: AuthManager; private authManager: AuthManager;
private basePath: string = '/composer'; private basePath: string = '/composer';
private registryUrl: string; private registryUrl: string;
private upstream: ComposerUpstream | null = null; private upstreamProvider: IUpstreamProvider | null = null;
constructor( constructor(
storage: RegistryStorage, storage: RegistryStorage,
authManager: AuthManager, authManager: AuthManager,
basePath: string = '/composer', basePath: string = '/composer',
registryUrl: string = 'http://localhost:5000/composer', registryUrl: string = 'http://localhost:5000/composer',
upstreamConfig?: IProtocolUpstreamConfig upstreamProvider?: IUpstreamProvider
) { ) {
super(); super();
this.storage = storage; this.storage = storage;
this.authManager = authManager; this.authManager = authManager;
this.basePath = basePath; this.basePath = basePath;
this.registryUrl = registryUrl; this.registryUrl = registryUrl;
this.upstreamProvider = upstreamProvider || null;
}
// Initialize upstream if configured /**
if (upstreamConfig?.enabled) { * Extract scope from Composer package name.
this.upstream = new ComposerUpstream(upstreamConfig); * For Composer, vendor is the scope.
* @example "symfony" from "symfony/console"
*/
private extractScope(vendorPackage: string): string | null {
const slashIndex = vendorPackage.indexOf('/');
if (slashIndex > 0) {
return vendorPackage.substring(0, slashIndex);
} }
return null;
}
/**
* Get upstream for a specific request.
* Calls the provider to resolve upstream config dynamically.
*/
private async getUpstreamForRequest(
resource: string,
resourceType: string,
method: string,
actor?: IRequestActor
): Promise<ComposerUpstream | null> {
if (!this.upstreamProvider) return null;
const config = await this.upstreamProvider.resolveUpstreamConfig({
protocol: 'composer',
resource,
scope: this.extractScope(resource),
actor,
method,
resourceType,
});
if (!config?.enabled) return null;
return new ComposerUpstream(config);
} }
/** /**
* Clean up resources (timers, connections, etc.) * Clean up resources (timers, connections, etc.)
*/ */
public destroy(): void { public destroy(): void {
if (this.upstream) { // No persistent upstream to clean up with dynamic provider
this.upstream.stop();
}
} }
public async init(): Promise<void> { public async init(): Promise<void> {
@@ -96,6 +128,14 @@ export class ComposerRegistry extends BaseRegistry {
} }
} }
// Build actor from context and validated token
const actor: IRequestActor = {
...context.actor,
userId: token?.userId,
ip: context.headers['x-forwarded-for'] || context.headers['X-Forwarded-For'],
userAgent: context.headers['user-agent'] || context.headers['User-Agent'],
};
// Root packages.json // Root packages.json
if (path === '/packages.json' || path === '' || path === '/') { if (path === '/packages.json' || path === '' || path === '/') {
return this.handlePackagesJson(); return this.handlePackagesJson();
@@ -106,7 +146,7 @@ export class ComposerRegistry extends BaseRegistry {
if (metadataMatch) { if (metadataMatch) {
const [, vendorPackage, devSuffix] = metadataMatch; const [, vendorPackage, devSuffix] = metadataMatch;
const includeDev = !!devSuffix; const includeDev = !!devSuffix;
return this.handlePackageMetadata(vendorPackage, includeDev, token); return this.handlePackageMetadata(vendorPackage, includeDev, token, actor);
} }
// Package list: /packages/list.json?filter=vendor/* // Package list: /packages/list.json?filter=vendor/*
@@ -176,26 +216,30 @@ export class ComposerRegistry extends BaseRegistry {
private async handlePackageMetadata( private async handlePackageMetadata(
vendorPackage: string, vendorPackage: string,
includeDev: boolean, includeDev: boolean,
token: IAuthToken | null token: IAuthToken | null,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
// Read operations are public, no authentication required // Read operations are public, no authentication required
let metadata = await this.storage.getComposerPackageMetadata(vendorPackage); let metadata = await this.storage.getComposerPackageMetadata(vendorPackage);
// Try upstream if not found locally // Try upstream if not found locally
if (!metadata && this.upstream) { if (!metadata) {
const [vendor, packageName] = vendorPackage.split('/'); const upstream = await this.getUpstreamForRequest(vendorPackage, 'metadata', 'GET', actor);
if (vendor && packageName) { if (upstream) {
const upstreamMetadata = includeDev const [vendor, packageName] = vendorPackage.split('/');
? await this.upstream.fetchPackageDevMetadata(vendor, packageName) if (vendor && packageName) {
: await this.upstream.fetchPackageMetadata(vendor, packageName); const upstreamMetadata = includeDev
? await upstream.fetchPackageDevMetadata(vendor, packageName)
: await upstream.fetchPackageMetadata(vendor, packageName);
if (upstreamMetadata && upstreamMetadata.packages) { if (upstreamMetadata && upstreamMetadata.packages) {
// Store upstream metadata locally // Store upstream metadata locally
metadata = { metadata = {
packages: upstreamMetadata.packages, packages: upstreamMetadata.packages,
lastModified: new Date().toUTCString(), lastModified: new Date().toUTCString(),
}; };
await this.storage.putComposerPackageMetadata(vendorPackage, metadata); await this.storage.putComposerPackageMetadata(vendorPackage, metadata);
}
} }
} }
} }

View File

@@ -3,7 +3,7 @@
*/ */
import type * as plugins from '../plugins.js'; import type * as plugins from '../plugins.js';
import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js'; import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
import type { IAuthProvider } from './interfaces.auth.js'; import type { IAuthProvider } from './interfaces.auth.js';
import type { IStorageHooks } from './interfaces.storage.js'; import type { IStorageHooks } from './interfaces.storage.js';
@@ -89,8 +89,6 @@ export interface IProtocolConfig {
enabled: boolean; enabled: boolean;
basePath: string; basePath: string;
features?: Record<string, boolean>; features?: Record<string, boolean>;
/** Upstream registry configuration for proxying/caching */
upstream?: IProtocolUpstreamConfig;
} }
/** /**
@@ -113,6 +111,13 @@ export interface IRegistryConfig {
*/ */
storageHooks?: IStorageHooks; storageHooks?: IStorageHooks;
/**
* Dynamic upstream configuration provider.
* Called per-request to resolve which upstream registries to use.
* Use StaticUpstreamProvider for simple static configurations.
*/
upstreamProvider?: IUpstreamProvider;
oci?: IProtocolConfig; oci?: IProtocolConfig;
npm?: IProtocolConfig; npm?: IProtocolConfig;
maven?: IProtocolConfig; maven?: IProtocolConfig;

View File

@@ -6,8 +6,8 @@
import { BaseRegistry } from '../core/classes.baseregistry.js'; import { BaseRegistry } from '../core/classes.baseregistry.js';
import type { RegistryStorage } from '../core/classes.registrystorage.js'; import type { RegistryStorage } from '../core/classes.registrystorage.js';
import type { AuthManager } from '../core/classes.authmanager.js'; import type { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js'; import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from '../core/interfaces.core.js';
import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js'; import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
import { toBuffer } from '../core/helpers.buffer.js'; import { toBuffer } from '../core/helpers.buffer.js';
import type { IMavenCoordinate, IMavenMetadata, IChecksums } from './interfaces.maven.js'; import type { IMavenCoordinate, IMavenMetadata, IChecksums } from './interfaces.maven.js';
import { import {
@@ -33,34 +33,64 @@ export class MavenRegistry extends BaseRegistry {
private authManager: AuthManager; private authManager: AuthManager;
private basePath: string = '/maven'; private basePath: string = '/maven';
private registryUrl: string; private registryUrl: string;
private upstream: MavenUpstream | null = null; private upstreamProvider: IUpstreamProvider | null = null;
constructor( constructor(
storage: RegistryStorage, storage: RegistryStorage,
authManager: AuthManager, authManager: AuthManager,
basePath: string, basePath: string,
registryUrl: string, registryUrl: string,
upstreamConfig?: IProtocolUpstreamConfig upstreamProvider?: IUpstreamProvider
) { ) {
super(); super();
this.storage = storage; this.storage = storage;
this.authManager = authManager; this.authManager = authManager;
this.basePath = basePath; this.basePath = basePath;
this.registryUrl = registryUrl; this.registryUrl = registryUrl;
this.upstreamProvider = upstreamProvider || null;
}
// Initialize upstream if configured /**
if (upstreamConfig?.enabled) { * Extract scope from Maven coordinates.
this.upstream = new MavenUpstream(upstreamConfig); * For Maven, the groupId is the scope.
} * @example "com.example" from "com.example:my-lib"
*/
private extractScope(groupId: string): string | null {
return groupId || null;
}
/**
* Get upstream for a specific request.
* Calls the provider to resolve upstream config dynamically.
*/
private async getUpstreamForRequest(
resource: string,
resourceType: string,
method: string,
actor?: IRequestActor
): Promise<MavenUpstream | null> {
if (!this.upstreamProvider) return null;
// For Maven, resource is "groupId:artifactId"
const [groupId] = resource.split(':');
const config = await this.upstreamProvider.resolveUpstreamConfig({
protocol: 'maven',
resource,
scope: this.extractScope(groupId),
actor,
method,
resourceType,
});
if (!config?.enabled) return null;
return new MavenUpstream(config);
} }
/** /**
* Clean up resources (timers, connections, etc.) * Clean up resources (timers, connections, etc.)
*/ */
public destroy(): void { public destroy(): void {
if (this.upstream) { // No persistent upstream to clean up with dynamic provider
this.upstream.stop();
}
} }
public async init(): Promise<void> { public async init(): Promise<void> {
@@ -85,13 +115,21 @@ export class MavenRegistry extends BaseRegistry {
token = await this.authManager.validateToken(tokenString, 'maven'); token = await this.authManager.validateToken(tokenString, 'maven');
} }
// Build actor from context and validated token
const actor: IRequestActor = {
...context.actor,
userId: token?.userId,
ip: context.headers['x-forwarded-for'] || context.headers['X-Forwarded-For'],
userAgent: context.headers['user-agent'] || context.headers['User-Agent'],
};
// Parse path to determine request type // Parse path to determine request type
const coordinate = pathToGAV(path); const coordinate = pathToGAV(path);
if (!coordinate) { if (!coordinate) {
// Not a valid artifact path, could be metadata or root // Not a valid artifact path, could be metadata or root
if (path.endsWith('/maven-metadata.xml')) { if (path.endsWith('/maven-metadata.xml')) {
return this.handleMetadataRequest(context.method, path, token); return this.handleMetadataRequest(context.method, path, token, actor);
} }
return { return {
@@ -108,7 +146,7 @@ export class MavenRegistry extends BaseRegistry {
} }
// Handle artifact requests (JAR, POM, WAR, etc.) // Handle artifact requests (JAR, POM, WAR, etc.)
return this.handleArtifactRequest(context.method, coordinate, token, context.body); return this.handleArtifactRequest(context.method, coordinate, token, context.body, actor);
} }
protected async checkPermission( protected async checkPermission(
@@ -128,7 +166,8 @@ export class MavenRegistry extends BaseRegistry {
method: string, method: string,
coordinate: IMavenCoordinate, coordinate: IMavenCoordinate,
token: IAuthToken | null, token: IAuthToken | null,
body?: Buffer | any body?: Buffer | any,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
const { groupId, artifactId, version } = coordinate; const { groupId, artifactId, version } = coordinate;
const filename = buildFilename(coordinate); const filename = buildFilename(coordinate);
@@ -139,7 +178,7 @@ export class MavenRegistry extends BaseRegistry {
case 'HEAD': case 'HEAD':
// Maven repositories typically allow anonymous reads // Maven repositories typically allow anonymous reads
return method === 'GET' return method === 'GET'
? this.getArtifact(groupId, artifactId, version, filename) ? this.getArtifact(groupId, artifactId, version, filename, actor)
: this.headArtifact(groupId, artifactId, version, filename); : this.headArtifact(groupId, artifactId, version, filename);
case 'PUT': case 'PUT':
@@ -211,7 +250,8 @@ export class MavenRegistry extends BaseRegistry {
private async handleMetadataRequest( private async handleMetadataRequest(
method: string, method: string,
path: string, path: string,
token: IAuthToken | null token: IAuthToken | null,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
// Parse path to extract groupId and artifactId // Parse path to extract groupId and artifactId
// Path format: /com/example/my-lib/maven-metadata.xml // Path format: /com/example/my-lib/maven-metadata.xml
@@ -232,7 +272,7 @@ export class MavenRegistry extends BaseRegistry {
if (method === 'GET') { if (method === 'GET') {
// Metadata is usually public (read permission optional) // Metadata is usually public (read permission optional)
// Some registries allow anonymous metadata access // Some registries allow anonymous metadata access
return this.getMetadata(groupId, artifactId); return this.getMetadata(groupId, artifactId, actor);
} }
return { return {
@@ -250,22 +290,27 @@ export class MavenRegistry extends BaseRegistry {
groupId: string, groupId: string,
artifactId: string, artifactId: string,
version: string, version: string,
filename: string filename: string,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
let data = await this.storage.getMavenArtifact(groupId, artifactId, version, filename); let data = await this.storage.getMavenArtifact(groupId, artifactId, version, filename);
// Try upstream if not found locally // Try upstream if not found locally
if (!data && this.upstream) { if (!data) {
// Parse the filename to extract extension and classifier const resource = `${groupId}:${artifactId}`;
const { extension, classifier } = this.parseFilename(filename, artifactId, version); const upstream = await this.getUpstreamForRequest(resource, 'artifact', 'GET', actor);
if (extension) { if (upstream) {
data = await this.upstream.fetchArtifact(groupId, artifactId, version, extension, classifier); // Parse the filename to extract extension and classifier
if (data) { const { extension, classifier } = this.parseFilename(filename, artifactId, version);
// Cache the artifact locally if (extension) {
await this.storage.putMavenArtifact(groupId, artifactId, version, filename, data); data = await upstream.fetchArtifact(groupId, artifactId, version, extension, classifier);
// Generate and store checksums if (data) {
const checksums = await calculateChecksums(data); // Cache the artifact locally
await this.storeChecksums(groupId, artifactId, version, filename, checksums); await this.storage.putMavenArtifact(groupId, artifactId, version, filename, data);
// Generate and store checksums
const checksums = await calculateChecksums(data);
await this.storeChecksums(groupId, artifactId, version, filename, checksums);
}
} }
} }
} }
@@ -495,16 +540,20 @@ export class MavenRegistry extends BaseRegistry {
// METADATA OPERATIONS // METADATA OPERATIONS
// ======================================================================== // ========================================================================
private async getMetadata(groupId: string, artifactId: string): Promise<IResponse> { private async getMetadata(groupId: string, artifactId: string, actor?: IRequestActor): Promise<IResponse> {
let metadataBuffer = await this.storage.getMavenMetadata(groupId, artifactId); let metadataBuffer = await this.storage.getMavenMetadata(groupId, artifactId);
// Try upstream if not found locally // Try upstream if not found locally
if (!metadataBuffer && this.upstream) { if (!metadataBuffer) {
const upstreamMetadata = await this.upstream.fetchMetadata(groupId, artifactId); const resource = `${groupId}:${artifactId}`;
if (upstreamMetadata) { const upstream = await this.getUpstreamForRequest(resource, 'metadata', 'GET', actor);
metadataBuffer = Buffer.from(upstreamMetadata, 'utf-8'); if (upstream) {
// Cache the metadata locally const upstreamMetadata = await upstream.fetchMetadata(groupId, artifactId);
await this.storage.putMavenMetadata(groupId, artifactId, metadataBuffer); if (upstreamMetadata) {
metadataBuffer = Buffer.from(upstreamMetadata, 'utf-8');
// Cache the metadata locally
await this.storage.putMavenMetadata(groupId, artifactId, metadataBuffer);
}
} }
} }

View File

@@ -2,8 +2,8 @@ import { Smartlog } from '@push.rocks/smartlog';
import { BaseRegistry } from '../core/classes.baseregistry.js'; import { BaseRegistry } from '../core/classes.baseregistry.js';
import { RegistryStorage } from '../core/classes.registrystorage.js'; import { RegistryStorage } from '../core/classes.registrystorage.js';
import { AuthManager } from '../core/classes.authmanager.js'; import { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js'; import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from '../core/interfaces.core.js';
import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js'; import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
import { NpmUpstream } from './classes.npmupstream.js'; import { NpmUpstream } from './classes.npmupstream.js';
import type { import type {
IPackument, IPackument,
@@ -27,20 +27,21 @@ export class NpmRegistry extends BaseRegistry {
private basePath: string = '/npm'; private basePath: string = '/npm';
private registryUrl: string; private registryUrl: string;
private logger: Smartlog; private logger: Smartlog;
private upstream: NpmUpstream | null = null; private upstreamProvider: IUpstreamProvider | null = null;
constructor( constructor(
storage: RegistryStorage, storage: RegistryStorage,
authManager: AuthManager, authManager: AuthManager,
basePath: string = '/npm', basePath: string = '/npm',
registryUrl: string = 'http://localhost:5000/npm', registryUrl: string = 'http://localhost:5000/npm',
upstreamConfig?: IProtocolUpstreamConfig upstreamProvider?: IUpstreamProvider
) { ) {
super(); super();
this.storage = storage; this.storage = storage;
this.authManager = authManager; this.authManager = authManager;
this.basePath = basePath; this.basePath = basePath;
this.registryUrl = registryUrl; this.registryUrl = registryUrl;
this.upstreamProvider = upstreamProvider || null;
// Initialize logger // Initialize logger
this.logger = new Smartlog({ this.logger = new Smartlog({
@@ -55,15 +56,51 @@ export class NpmRegistry extends BaseRegistry {
}); });
this.logger.enableConsole(); this.logger.enableConsole();
// Initialize upstream if configured if (upstreamProvider) {
if (upstreamConfig?.enabled) { this.logger.log('info', 'NPM upstream provider configured');
this.upstream = new NpmUpstream(upstreamConfig, registryUrl, this.logger);
this.logger.log('info', 'NPM upstream initialized', {
upstreams: upstreamConfig.upstreams.map(u => u.name),
});
} }
} }
/**
* Extract scope from npm package name.
* @example "@company/utils" -> "company"
* @example "lodash" -> null
*/
private extractScope(packageName: string): string | null {
if (packageName.startsWith('@')) {
const slashIndex = packageName.indexOf('/');
if (slashIndex > 1) {
return packageName.substring(1, slashIndex);
}
}
return null;
}
/**
* Get upstream for a specific request.
* Calls the provider to resolve upstream config dynamically.
*/
private async getUpstreamForRequest(
resource: string,
resourceType: string,
method: string,
actor?: IRequestActor
): Promise<NpmUpstream | null> {
if (!this.upstreamProvider) return null;
const config = await this.upstreamProvider.resolveUpstreamConfig({
protocol: 'npm',
resource,
scope: this.extractScope(resource),
actor,
method,
resourceType,
});
if (!config?.enabled) return null;
return new NpmUpstream(config, this.registryUrl, this.logger);
}
public async init(): Promise<void> { public async init(): Promise<void> {
// NPM registry initialization // NPM registry initialization
} }
@@ -80,6 +117,14 @@ export class NpmRegistry extends BaseRegistry {
const tokenString = authHeader?.replace(/^Bearer\s+/i, ''); const tokenString = authHeader?.replace(/^Bearer\s+/i, '');
const token = tokenString ? await this.authManager.validateToken(tokenString, 'npm') : null; const token = tokenString ? await this.authManager.validateToken(tokenString, 'npm') : null;
// Build actor context for upstream resolution
const actor: IRequestActor = {
userId: token?.userId,
ip: context.headers['x-forwarded-for'] || context.headers['x-real-ip'],
userAgent: context.headers['user-agent'],
...context.actor, // Include any pre-populated actor info
};
this.logger.log('debug', `handleRequest: ${context.method} ${path}`, { this.logger.log('debug', `handleRequest: ${context.method} ${path}`, {
method: context.method, method: context.method,
path, path,
@@ -118,7 +163,7 @@ export class NpmRegistry extends BaseRegistry {
const tarballMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)\/-\/(.+\.tgz)$/); const tarballMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)\/-\/(.+\.tgz)$/);
if (tarballMatch) { if (tarballMatch) {
const [, packageName, filename] = tarballMatch; const [, packageName, filename] = tarballMatch;
return this.handleTarballDownload(packageName, filename, token); return this.handleTarballDownload(packageName, filename, token, actor);
} }
// Unpublish specific version: DELETE /{package}/-/{version} // Unpublish specific version: DELETE /{package}/-/{version}
@@ -142,7 +187,7 @@ export class NpmRegistry extends BaseRegistry {
if (versionMatch) { if (versionMatch) {
const [, packageName, version] = versionMatch; const [, packageName, version] = versionMatch;
this.logger.log('debug', 'versionMatch', { packageName, version }); this.logger.log('debug', 'versionMatch', { packageName, version });
return this.handlePackageVersion(packageName, version, token); return this.handlePackageVersion(packageName, version, token, actor);
} }
// Package operations: /{package} // Package operations: /{package}
@@ -150,7 +195,7 @@ export class NpmRegistry extends BaseRegistry {
if (packageMatch) { if (packageMatch) {
const packageName = packageMatch[1]; const packageName = packageMatch[1];
this.logger.log('debug', 'packageMatch', { packageName }); this.logger.log('debug', 'packageMatch', { packageName });
return this.handlePackage(context.method, packageName, context.body, context.query, token); return this.handlePackage(context.method, packageName, context.body, context.query, token, actor);
} }
return { return {
@@ -198,11 +243,12 @@ export class NpmRegistry extends BaseRegistry {
packageName: string, packageName: string,
body: any, body: any,
query: Record<string, string>, query: Record<string, string>,
token: IAuthToken | null token: IAuthToken | null,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
switch (method) { switch (method) {
case 'GET': case 'GET':
return this.getPackument(packageName, token, query); return this.getPackument(packageName, token, query, actor);
case 'PUT': case 'PUT':
return this.publishPackage(packageName, body, token); return this.publishPackage(packageName, body, token);
case 'DELETE': case 'DELETE':
@@ -219,7 +265,8 @@ export class NpmRegistry extends BaseRegistry {
private async getPackument( private async getPackument(
packageName: string, packageName: string,
token: IAuthToken | null, token: IAuthToken | null,
query: Record<string, string> query: Record<string, string>,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
let packument = await this.storage.getNpmPackument(packageName); let packument = await this.storage.getNpmPackument(packageName);
this.logger.log('debug', `getPackument: ${packageName}`, { this.logger.log('debug', `getPackument: ${packageName}`, {
@@ -229,17 +276,20 @@ export class NpmRegistry extends BaseRegistry {
}); });
// If not found locally, try upstream // If not found locally, try upstream
if (!packument && this.upstream) { if (!packument) {
this.logger.log('debug', `getPackument: fetching from upstream`, { packageName }); const upstream = await this.getUpstreamForRequest(packageName, 'packument', 'GET', actor);
const upstreamPackument = await this.upstream.fetchPackument(packageName); if (upstream) {
if (upstreamPackument) { this.logger.log('debug', `getPackument: fetching from upstream`, { packageName });
this.logger.log('debug', `getPackument: found in upstream`, { const upstreamPackument = await upstream.fetchPackument(packageName);
packageName, if (upstreamPackument) {
versions: Object.keys(upstreamPackument.versions || {}).length this.logger.log('debug', `getPackument: found in upstream`, {
}); packageName,
packument = upstreamPackument; versions: Object.keys(upstreamPackument.versions || {}).length
// Optionally cache the packument locally (without tarballs) });
// We don't store tarballs here - they'll be fetched on demand packument = upstreamPackument;
// Optionally cache the packument locally (without tarballs)
// We don't store tarballs here - they'll be fetched on demand
}
} }
} }
@@ -279,7 +329,8 @@ export class NpmRegistry extends BaseRegistry {
private async handlePackageVersion( private async handlePackageVersion(
packageName: string, packageName: string,
version: string, version: string,
token: IAuthToken | null token: IAuthToken | null,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
this.logger.log('debug', 'handlePackageVersion', { packageName, version }); this.logger.log('debug', 'handlePackageVersion', { packageName, version });
let packument = await this.storage.getNpmPackument(packageName); let packument = await this.storage.getNpmPackument(packageName);
@@ -289,11 +340,14 @@ export class NpmRegistry extends BaseRegistry {
} }
// If not found locally, try upstream // If not found locally, try upstream
if (!packument && this.upstream) { if (!packument) {
this.logger.log('debug', 'handlePackageVersion: fetching from upstream', { packageName }); const upstream = await this.getUpstreamForRequest(packageName, 'packument', 'GET', actor);
const upstreamPackument = await this.upstream.fetchPackument(packageName); if (upstream) {
if (upstreamPackument) { this.logger.log('debug', 'handlePackageVersion: fetching from upstream', { packageName });
packument = upstreamPackument; const upstreamPackument = await upstream.fetchPackument(packageName);
if (upstreamPackument) {
packument = upstreamPackument;
}
} }
} }
@@ -563,7 +617,8 @@ export class NpmRegistry extends BaseRegistry {
private async handleTarballDownload( private async handleTarballDownload(
packageName: string, packageName: string,
filename: string, filename: string,
token: IAuthToken | null token: IAuthToken | null,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
// Extract version from filename: package-name-1.0.0.tgz // Extract version from filename: package-name-1.0.0.tgz
const versionMatch = filename.match(/-([\d.]+(?:-[a-z0-9.]+)?)\.tgz$/i); const versionMatch = filename.match(/-([\d.]+(?:-[a-z0-9.]+)?)\.tgz$/i);
@@ -579,21 +634,24 @@ export class NpmRegistry extends BaseRegistry {
let tarball = await this.storage.getNpmTarball(packageName, version); let tarball = await this.storage.getNpmTarball(packageName, version);
// If not found locally, try upstream // If not found locally, try upstream
if (!tarball && this.upstream) { if (!tarball) {
this.logger.log('debug', 'handleTarballDownload: fetching from upstream', { const upstream = await this.getUpstreamForRequest(packageName, 'tarball', 'GET', actor);
packageName, if (upstream) {
version, this.logger.log('debug', 'handleTarballDownload: fetching from upstream', {
});
const upstreamTarball = await this.upstream.fetchTarball(packageName, version);
if (upstreamTarball) {
tarball = upstreamTarball;
// Cache the tarball locally for future requests
await this.storage.putNpmTarball(packageName, version, tarball);
this.logger.log('debug', 'handleTarballDownload: cached tarball locally', {
packageName, packageName,
version, version,
size: tarball.length,
}); });
const upstreamTarball = await upstream.fetchTarball(packageName, version);
if (upstreamTarball) {
tarball = upstreamTarball;
// Cache the tarball locally for future requests
await this.storage.putNpmTarball(packageName, version, tarball);
this.logger.log('debug', 'handleTarballDownload: cached tarball locally', {
packageName,
version,
size: tarball.length,
});
}
} }
} }

View File

@@ -2,8 +2,8 @@ import { Smartlog } from '@push.rocks/smartlog';
import { BaseRegistry } from '../core/classes.baseregistry.js'; import { BaseRegistry } from '../core/classes.baseregistry.js';
import { RegistryStorage } from '../core/classes.registrystorage.js'; import { RegistryStorage } from '../core/classes.registrystorage.js';
import { AuthManager } from '../core/classes.authmanager.js'; import { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken, IRegistryError } from '../core/interfaces.core.js'; import type { IRequestContext, IResponse, IAuthToken, IRegistryError, IRequestActor } from '../core/interfaces.core.js';
import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js'; import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
import { OciUpstream } from './classes.ociupstream.js'; import { OciUpstream } from './classes.ociupstream.js';
import type { import type {
IUploadSession, IUploadSession,
@@ -24,7 +24,7 @@ export class OciRegistry extends BaseRegistry {
private basePath: string = '/oci'; private basePath: string = '/oci';
private cleanupInterval?: NodeJS.Timeout; private cleanupInterval?: NodeJS.Timeout;
private ociTokens?: { realm: string; service: string }; private ociTokens?: { realm: string; service: string };
private upstream: OciUpstream | null = null; private upstreamProvider: IUpstreamProvider | null = null;
private logger: Smartlog; private logger: Smartlog;
constructor( constructor(
@@ -32,13 +32,14 @@ export class OciRegistry extends BaseRegistry {
authManager: AuthManager, authManager: AuthManager,
basePath: string = '/oci', basePath: string = '/oci',
ociTokens?: { realm: string; service: string }, ociTokens?: { realm: string; service: string },
upstreamConfig?: IProtocolUpstreamConfig upstreamProvider?: IUpstreamProvider
) { ) {
super(); super();
this.storage = storage; this.storage = storage;
this.authManager = authManager; this.authManager = authManager;
this.basePath = basePath; this.basePath = basePath;
this.ociTokens = ociTokens; this.ociTokens = ociTokens;
this.upstreamProvider = upstreamProvider || null;
// Initialize logger // Initialize logger
this.logger = new Smartlog({ this.logger = new Smartlog({
@@ -53,15 +54,50 @@ export class OciRegistry extends BaseRegistry {
}); });
this.logger.enableConsole(); this.logger.enableConsole();
// Initialize upstream if configured if (upstreamProvider) {
if (upstreamConfig?.enabled) { this.logger.log('info', 'OCI upstream provider configured');
this.upstream = new OciUpstream(upstreamConfig, basePath, this.logger);
this.logger.log('info', 'OCI upstream initialized', {
upstreams: upstreamConfig.upstreams.map(u => u.name),
});
} }
} }
/**
* Extract scope from OCI repository name.
* @example "myorg/myimage" -> "myorg"
* @example "library/nginx" -> "library"
* @example "nginx" -> null
*/
private extractScope(repository: string): string | null {
const slashIndex = repository.indexOf('/');
if (slashIndex > 0) {
return repository.substring(0, slashIndex);
}
return null;
}
/**
* Get upstream for a specific request.
* Calls the provider to resolve upstream config dynamically.
*/
private async getUpstreamForRequest(
resource: string,
resourceType: string,
method: string,
actor?: IRequestActor
): Promise<OciUpstream | null> {
if (!this.upstreamProvider) return null;
const config = await this.upstreamProvider.resolveUpstreamConfig({
protocol: 'oci',
resource,
scope: this.extractScope(resource),
actor,
method,
resourceType,
});
if (!config?.enabled) return null;
return new OciUpstream(config, this.basePath, this.logger);
}
public async init(): Promise<void> { public async init(): Promise<void> {
// Start cleanup of stale upload sessions // Start cleanup of stale upload sessions
this.startUploadSessionCleanup(); this.startUploadSessionCleanup();
@@ -80,6 +116,14 @@ export class OciRegistry extends BaseRegistry {
const tokenString = authHeader?.replace(/^Bearer\s+/i, ''); const tokenString = authHeader?.replace(/^Bearer\s+/i, '');
const token = tokenString ? await this.authManager.validateToken(tokenString, 'oci') : null; const token = tokenString ? await this.authManager.validateToken(tokenString, 'oci') : null;
// Build actor from context and validated token
const actor: IRequestActor = {
...context.actor,
userId: token?.userId,
ip: context.headers['x-forwarded-for'] || context.headers['X-Forwarded-For'],
userAgent: context.headers['user-agent'] || context.headers['User-Agent'],
};
// Route to appropriate handler // Route to appropriate handler
if (path === '/v2/' || path === '/v2') { if (path === '/v2/' || path === '/v2') {
return this.handleVersionCheck(); return this.handleVersionCheck();
@@ -91,14 +135,14 @@ export class OciRegistry extends BaseRegistry {
const [, name, reference] = manifestMatch; const [, name, reference] = manifestMatch;
// Prefer rawBody for content-addressable operations to preserve exact bytes // Prefer rawBody for content-addressable operations to preserve exact bytes
const bodyData = context.rawBody || context.body; const bodyData = context.rawBody || context.body;
return this.handleManifestRequest(context.method, name, reference, token, bodyData, context.headers); return this.handleManifestRequest(context.method, name, reference, token, bodyData, context.headers, actor);
} }
// Blob operations: /v2/{name}/blobs/{digest} // Blob operations: /v2/{name}/blobs/{digest}
const blobMatch = path.match(/^\/v2\/([^\/]+(?:\/[^\/]+)*)\/blobs\/(sha256:[a-f0-9]{64})$/); const blobMatch = path.match(/^\/v2\/([^\/]+(?:\/[^\/]+)*)\/blobs\/(sha256:[a-f0-9]{64})$/);
if (blobMatch) { if (blobMatch) {
const [, name, digest] = blobMatch; const [, name, digest] = blobMatch;
return this.handleBlobRequest(context.method, name, digest, token, context.headers); return this.handleBlobRequest(context.method, name, digest, token, context.headers, actor);
} }
// Blob upload operations: /v2/{name}/blobs/uploads/ // Blob upload operations: /v2/{name}/blobs/uploads/
@@ -168,11 +212,12 @@ export class OciRegistry extends BaseRegistry {
reference: string, reference: string,
token: IAuthToken | null, token: IAuthToken | null,
body?: Buffer | any, body?: Buffer | any,
headers?: Record<string, string> headers?: Record<string, string>,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
switch (method) { switch (method) {
case 'GET': case 'GET':
return this.getManifest(repository, reference, token, headers); return this.getManifest(repository, reference, token, headers, actor);
case 'HEAD': case 'HEAD':
return this.headManifest(repository, reference, token); return this.headManifest(repository, reference, token);
case 'PUT': case 'PUT':
@@ -193,11 +238,12 @@ export class OciRegistry extends BaseRegistry {
repository: string, repository: string,
digest: string, digest: string,
token: IAuthToken | null, token: IAuthToken | null,
headers: Record<string, string> headers: Record<string, string>,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
switch (method) { switch (method) {
case 'GET': case 'GET':
return this.getBlob(repository, digest, token, headers['range'] || headers['Range']); return this.getBlob(repository, digest, token, headers['range'] || headers['Range'], actor);
case 'HEAD': case 'HEAD':
return this.headBlob(repository, digest, token); return this.headBlob(repository, digest, token);
case 'DELETE': case 'DELETE':
@@ -318,7 +364,8 @@ export class OciRegistry extends BaseRegistry {
repository: string, repository: string,
reference: string, reference: string,
token: IAuthToken | null, token: IAuthToken | null,
headers?: Record<string, string> headers?: Record<string, string>,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
if (!await this.checkPermission(token, repository, 'pull')) { if (!await this.checkPermission(token, repository, 'pull')) {
return this.createUnauthorizedResponse(repository, 'pull'); return this.createUnauthorizedResponse(repository, 'pull');
@@ -346,30 +393,33 @@ export class OciRegistry extends BaseRegistry {
} }
// If not found locally, try upstream // If not found locally, try upstream
if (!manifestData && this.upstream) { if (!manifestData) {
this.logger.log('debug', 'getManifest: fetching from upstream', { repository, reference }); const upstream = await this.getUpstreamForRequest(repository, 'manifest', 'GET', actor);
const upstreamResult = await this.upstream.fetchManifest(repository, reference); if (upstream) {
if (upstreamResult) { this.logger.log('debug', 'getManifest: fetching from upstream', { repository, reference });
manifestData = Buffer.from(JSON.stringify(upstreamResult.manifest), 'utf8'); const upstreamResult = await upstream.fetchManifest(repository, reference);
contentType = upstreamResult.contentType; if (upstreamResult) {
digest = upstreamResult.digest; manifestData = Buffer.from(JSON.stringify(upstreamResult.manifest), 'utf8');
contentType = upstreamResult.contentType;
digest = upstreamResult.digest;
// Cache the manifest locally // Cache the manifest locally
await this.storage.putOciManifest(repository, digest, manifestData, contentType); await this.storage.putOciManifest(repository, digest, manifestData, contentType);
// If reference is a tag, update tags mapping // If reference is a tag, update tags mapping
if (!reference.startsWith('sha256:')) { if (!reference.startsWith('sha256:')) {
const tags = await this.getTagsData(repository); const tags = await this.getTagsData(repository);
tags[reference] = digest; tags[reference] = digest;
const tagsPath = `oci/tags/${repository}/tags.json`; const tagsPath = `oci/tags/${repository}/tags.json`;
await this.storage.putObject(tagsPath, Buffer.from(JSON.stringify(tags), 'utf-8')); await this.storage.putObject(tagsPath, Buffer.from(JSON.stringify(tags), 'utf-8'));
}
this.logger.log('debug', 'getManifest: cached manifest locally', {
repository,
reference,
digest,
});
} }
this.logger.log('debug', 'getManifest: cached manifest locally', {
repository,
reference,
digest,
});
} }
} }
@@ -514,7 +564,8 @@ export class OciRegistry extends BaseRegistry {
repository: string, repository: string,
digest: string, digest: string,
token: IAuthToken | null, token: IAuthToken | null,
range?: string range?: string,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
if (!await this.checkPermission(token, repository, 'pull')) { if (!await this.checkPermission(token, repository, 'pull')) {
return this.createUnauthorizedResponse(repository, 'pull'); return this.createUnauthorizedResponse(repository, 'pull');
@@ -524,18 +575,21 @@ export class OciRegistry extends BaseRegistry {
let data = await this.storage.getOciBlob(digest); let data = await this.storage.getOciBlob(digest);
// If not found locally, try upstream // If not found locally, try upstream
if (!data && this.upstream) { if (!data) {
this.logger.log('debug', 'getBlob: fetching from upstream', { repository, digest }); const upstream = await this.getUpstreamForRequest(repository, 'blob', 'GET', actor);
const upstreamBlob = await this.upstream.fetchBlob(repository, digest); if (upstream) {
if (upstreamBlob) { this.logger.log('debug', 'getBlob: fetching from upstream', { repository, digest });
data = upstreamBlob; const upstreamBlob = await upstream.fetchBlob(repository, digest);
// Cache the blob locally (blobs are content-addressable and immutable) if (upstreamBlob) {
await this.storage.putOciBlob(digest, data); data = upstreamBlob;
this.logger.log('debug', 'getBlob: cached blob locally', { // Cache the blob locally (blobs are content-addressable and immutable)
repository, await this.storage.putOciBlob(digest, data);
digest, this.logger.log('debug', 'getBlob: cached blob locally', {
size: data.length, repository,
}); digest,
size: data.length,
});
}
} }
} }

View File

@@ -2,8 +2,8 @@ import { Smartlog } from '@push.rocks/smartlog';
import { BaseRegistry } from '../core/classes.baseregistry.js'; import { BaseRegistry } from '../core/classes.baseregistry.js';
import { RegistryStorage } from '../core/classes.registrystorage.js'; import { RegistryStorage } from '../core/classes.registrystorage.js';
import { AuthManager } from '../core/classes.authmanager.js'; import { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js'; import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from '../core/interfaces.core.js';
import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js'; import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
import { isBinaryData, toBuffer } from '../core/helpers.buffer.js'; import { isBinaryData, toBuffer } from '../core/helpers.buffer.js';
import type { import type {
IPypiPackageMetadata, IPypiPackageMetadata,
@@ -24,20 +24,21 @@ export class PypiRegistry extends BaseRegistry {
private basePath: string = '/pypi'; private basePath: string = '/pypi';
private registryUrl: string; private registryUrl: string;
private logger: Smartlog; private logger: Smartlog;
private upstream: PypiUpstream | null = null; private upstreamProvider: IUpstreamProvider | null = null;
constructor( constructor(
storage: RegistryStorage, storage: RegistryStorage,
authManager: AuthManager, authManager: AuthManager,
basePath: string = '/pypi', basePath: string = '/pypi',
registryUrl: string = 'http://localhost:5000', registryUrl: string = 'http://localhost:5000',
upstreamConfig?: IProtocolUpstreamConfig upstreamProvider?: IUpstreamProvider
) { ) {
super(); super();
this.storage = storage; this.storage = storage;
this.authManager = authManager; this.authManager = authManager;
this.basePath = basePath; this.basePath = basePath;
this.registryUrl = registryUrl; this.registryUrl = registryUrl;
this.upstreamProvider = upstreamProvider || null;
// Initialize logger // Initialize logger
this.logger = new Smartlog({ this.logger = new Smartlog({
@@ -51,20 +52,38 @@ export class PypiRegistry extends BaseRegistry {
} }
}); });
this.logger.enableConsole(); this.logger.enableConsole();
}
// Initialize upstream if configured /**
if (upstreamConfig?.enabled) { * Get upstream for a specific request.
this.upstream = new PypiUpstream(upstreamConfig, registryUrl, this.logger); * Calls the provider to resolve upstream config dynamically.
} */
private async getUpstreamForRequest(
resource: string,
resourceType: string,
method: string,
actor?: IRequestActor
): Promise<PypiUpstream | null> {
if (!this.upstreamProvider) return null;
const config = await this.upstreamProvider.resolveUpstreamConfig({
protocol: 'pypi',
resource,
scope: resource, // For PyPI, package name is the scope
actor,
method,
resourceType,
});
if (!config?.enabled) return null;
return new PypiUpstream(config, this.registryUrl, this.logger);
} }
/** /**
* Clean up resources (timers, connections, etc.) * Clean up resources (timers, connections, etc.)
*/ */
public destroy(): void { public destroy(): void {
if (this.upstream) { // No persistent upstream to clean up with dynamic provider
this.upstream.stop();
}
} }
public async init(): Promise<void> { public async init(): Promise<void> {
@@ -84,15 +103,23 @@ export class PypiRegistry extends BaseRegistry {
public async handleRequest(context: IRequestContext): Promise<IResponse> { public async handleRequest(context: IRequestContext): Promise<IResponse> {
let path = context.path.replace(this.basePath, ''); let path = context.path.replace(this.basePath, '');
// Extract token (Basic Auth or Bearer)
const token = await this.extractToken(context);
// Build actor from context and validated token
const actor: IRequestActor = {
...context.actor,
userId: token?.userId,
ip: context.headers['x-forwarded-for'] || context.headers['X-Forwarded-For'],
userAgent: context.headers['user-agent'] || context.headers['User-Agent'],
};
// Also handle /simple path prefix // Also handle /simple path prefix
if (path.startsWith('/simple')) { if (path.startsWith('/simple')) {
path = path.replace('/simple', ''); path = path.replace('/simple', '');
return this.handleSimpleRequest(path, context); return this.handleSimpleRequest(path, context, actor);
} }
// Extract token (Basic Auth or Bearer)
const token = await this.extractToken(context);
this.logger.log('debug', `handleRequest: ${context.method} ${path}`, { this.logger.log('debug', `handleRequest: ${context.method} ${path}`, {
method: context.method, method: context.method,
path, path,
@@ -119,7 +146,7 @@ export class PypiRegistry extends BaseRegistry {
// Package file download: GET /packages/{package}/{filename} // Package file download: GET /packages/{package}/{filename}
const downloadMatch = path.match(/^\/packages\/([^\/]+)\/(.+)$/); const downloadMatch = path.match(/^\/packages\/([^\/]+)\/(.+)$/);
if (downloadMatch && context.method === 'GET') { if (downloadMatch && context.method === 'GET') {
return this.handleDownload(downloadMatch[1], downloadMatch[2]); return this.handleDownload(downloadMatch[1], downloadMatch[2], actor);
} }
// Delete package: DELETE /packages/{package} // Delete package: DELETE /packages/{package}
@@ -156,7 +183,7 @@ export class PypiRegistry extends BaseRegistry {
/** /**
* Handle Simple API requests (PEP 503 HTML or PEP 691 JSON) * Handle Simple API requests (PEP 503 HTML or PEP 691 JSON)
*/ */
private async handleSimpleRequest(path: string, context: IRequestContext): Promise<IResponse> { private async handleSimpleRequest(path: string, context: IRequestContext, actor?: IRequestActor): Promise<IResponse> {
// Ensure path ends with / (PEP 503 requirement) // Ensure path ends with / (PEP 503 requirement)
if (!path.endsWith('/') && !path.includes('.')) { if (!path.endsWith('/') && !path.includes('.')) {
return { return {
@@ -174,7 +201,7 @@ export class PypiRegistry extends BaseRegistry {
// Package index: /simple/{package}/ // Package index: /simple/{package}/
const packageMatch = path.match(/^\/([^\/]+)\/$/); const packageMatch = path.match(/^\/([^\/]+)\/$/);
if (packageMatch) { if (packageMatch) {
return this.handleSimplePackage(packageMatch[1], context); return this.handleSimplePackage(packageMatch[1], context, actor);
} }
return { return {
@@ -228,46 +255,49 @@ export class PypiRegistry extends BaseRegistry {
* Handle Simple API package index * Handle Simple API package index
* Returns HTML (PEP 503) or JSON (PEP 691) based on Accept header * Returns HTML (PEP 503) or JSON (PEP 691) based on Accept header
*/ */
private async handleSimplePackage(packageName: string, context: IRequestContext): Promise<IResponse> { private async handleSimplePackage(packageName: string, context: IRequestContext, actor?: IRequestActor): Promise<IResponse> {
const normalized = helpers.normalizePypiPackageName(packageName); const normalized = helpers.normalizePypiPackageName(packageName);
// Get package metadata // Get package metadata
let metadata = await this.storage.getPypiPackageMetadata(normalized); let metadata = await this.storage.getPypiPackageMetadata(normalized);
// Try upstream if not found locally // Try upstream if not found locally
if (!metadata && this.upstream) { if (!metadata) {
const upstreamHtml = await this.upstream.fetchSimplePackage(normalized); const upstream = await this.getUpstreamForRequest(normalized, 'simple', 'GET', actor);
if (upstreamHtml) { if (upstream) {
// Parse the HTML to extract file information and cache it const upstreamHtml = await upstream.fetchSimplePackage(normalized);
// For now, just return the upstream HTML directly (caching can be improved later) if (upstreamHtml) {
const acceptHeader = context.headers['accept'] || context.headers['Accept'] || ''; // Parse the HTML to extract file information and cache it
const preferJson = acceptHeader.includes('application/vnd.pypi.simple') && // For now, just return the upstream HTML directly (caching can be improved later)
acceptHeader.includes('json'); const acceptHeader = context.headers['accept'] || context.headers['Accept'] || '';
const preferJson = acceptHeader.includes('application/vnd.pypi.simple') &&
acceptHeader.includes('json');
if (preferJson) { if (preferJson) {
// Try to get JSON format from upstream // Try to get JSON format from upstream
const upstreamJson = await this.upstream.fetchPackageJson(normalized); const upstreamJson = await upstream.fetchPackageJson(normalized);
if (upstreamJson) { if (upstreamJson) {
return { return {
status: 200, status: 200,
headers: { headers: {
'Content-Type': 'application/vnd.pypi.simple.v1+json', 'Content-Type': 'application/vnd.pypi.simple.v1+json',
'Cache-Control': 'public, max-age=300' 'Cache-Control': 'public, max-age=300'
}, },
body: upstreamJson, body: upstreamJson,
}; };
}
} }
}
// Return HTML format // Return HTML format
return { return {
status: 200, status: 200,
headers: { headers: {
'Content-Type': 'text/html; charset=utf-8', 'Content-Type': 'text/html; charset=utf-8',
'Cache-Control': 'public, max-age=300' 'Cache-Control': 'public, max-age=300'
}, },
body: upstreamHtml, body: upstreamHtml,
}; };
}
} }
} }
@@ -503,16 +533,19 @@ export class PypiRegistry extends BaseRegistry {
/** /**
* Handle package download * Handle package download
*/ */
private async handleDownload(packageName: string, filename: string): Promise<IResponse> { private async handleDownload(packageName: string, filename: string, actor?: IRequestActor): Promise<IResponse> {
const normalized = helpers.normalizePypiPackageName(packageName); const normalized = helpers.normalizePypiPackageName(packageName);
let fileData = await this.storage.getPypiPackageFile(normalized, filename); let fileData = await this.storage.getPypiPackageFile(normalized, filename);
// Try upstream if not found locally // Try upstream if not found locally
if (!fileData && this.upstream) { if (!fileData) {
fileData = await this.upstream.fetchPackageFile(normalized, filename); const upstream = await this.getUpstreamForRequest(normalized, 'file', 'GET', actor);
if (fileData) { if (upstream) {
// Cache locally fileData = await upstream.fetchPackageFile(normalized, filename);
await this.storage.putPypiPackageFile(normalized, filename, fileData); if (fileData) {
// Cache locally
await this.storage.putPypiPackageFile(normalized, filename, fileData);
}
} }
} }

View File

@@ -2,8 +2,8 @@ import { Smartlog } from '@push.rocks/smartlog';
import { BaseRegistry } from '../core/classes.baseregistry.js'; import { BaseRegistry } from '../core/classes.baseregistry.js';
import { RegistryStorage } from '../core/classes.registrystorage.js'; import { RegistryStorage } from '../core/classes.registrystorage.js';
import { AuthManager } from '../core/classes.authmanager.js'; import { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js'; import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from '../core/interfaces.core.js';
import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js'; import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
import type { import type {
IRubyGemsMetadata, IRubyGemsMetadata,
IRubyGemsVersionMetadata, IRubyGemsVersionMetadata,
@@ -25,20 +25,21 @@ export class RubyGemsRegistry extends BaseRegistry {
private basePath: string = '/rubygems'; private basePath: string = '/rubygems';
private registryUrl: string; private registryUrl: string;
private logger: Smartlog; private logger: Smartlog;
private upstream: RubygemsUpstream | null = null; private upstreamProvider: IUpstreamProvider | null = null;
constructor( constructor(
storage: RegistryStorage, storage: RegistryStorage,
authManager: AuthManager, authManager: AuthManager,
basePath: string = '/rubygems', basePath: string = '/rubygems',
registryUrl: string = 'http://localhost:5000/rubygems', registryUrl: string = 'http://localhost:5000/rubygems',
upstreamConfig?: IProtocolUpstreamConfig upstreamProvider?: IUpstreamProvider
) { ) {
super(); super();
this.storage = storage; this.storage = storage;
this.authManager = authManager; this.authManager = authManager;
this.basePath = basePath; this.basePath = basePath;
this.registryUrl = registryUrl; this.registryUrl = registryUrl;
this.upstreamProvider = upstreamProvider || null;
// Initialize logger // Initialize logger
this.logger = new Smartlog({ this.logger = new Smartlog({
@@ -52,20 +53,38 @@ export class RubyGemsRegistry extends BaseRegistry {
} }
}); });
this.logger.enableConsole(); this.logger.enableConsole();
}
// Initialize upstream if configured /**
if (upstreamConfig?.enabled) { * Get upstream for a specific request.
this.upstream = new RubygemsUpstream(upstreamConfig, this.logger); * Calls the provider to resolve upstream config dynamically.
} */
private async getUpstreamForRequest(
resource: string,
resourceType: string,
method: string,
actor?: IRequestActor
): Promise<RubygemsUpstream | null> {
if (!this.upstreamProvider) return null;
const config = await this.upstreamProvider.resolveUpstreamConfig({
protocol: 'rubygems',
resource,
scope: resource, // gem name is the scope
actor,
method,
resourceType,
});
if (!config?.enabled) return null;
return new RubygemsUpstream(config, this.logger);
} }
/** /**
* Clean up resources (timers, connections, etc.) * Clean up resources (timers, connections, etc.)
*/ */
public destroy(): void { public destroy(): void {
if (this.upstream) { // No persistent upstream to clean up with dynamic provider
this.upstream.stop();
}
} }
public async init(): Promise<void> { public async init(): Promise<void> {
@@ -95,6 +114,14 @@ export class RubyGemsRegistry extends BaseRegistry {
// Extract token (Authorization header) // Extract token (Authorization header)
const token = await this.extractToken(context); const token = await this.extractToken(context);
// Build actor from context and validated token
const actor: IRequestActor = {
...context.actor,
userId: token?.userId,
ip: context.headers['x-forwarded-for'] || context.headers['X-Forwarded-For'],
userAgent: context.headers['user-agent'] || context.headers['User-Agent'],
};
this.logger.log('debug', `handleRequest: ${context.method} ${path}`, { this.logger.log('debug', `handleRequest: ${context.method} ${path}`, {
method: context.method, method: context.method,
path, path,
@@ -113,13 +140,13 @@ export class RubyGemsRegistry extends BaseRegistry {
// Info file: GET /info/{gem} // Info file: GET /info/{gem}
const infoMatch = path.match(/^\/info\/([^\/]+)$/); const infoMatch = path.match(/^\/info\/([^\/]+)$/);
if (infoMatch && context.method === 'GET') { if (infoMatch && context.method === 'GET') {
return this.handleInfoFile(infoMatch[1]); return this.handleInfoFile(infoMatch[1], actor);
} }
// Gem download: GET /gems/{gem}-{version}[-{platform}].gem // Gem download: GET /gems/{gem}-{version}[-{platform}].gem
const downloadMatch = path.match(/^\/gems\/(.+\.gem)$/); const downloadMatch = path.match(/^\/gems\/(.+\.gem)$/);
if (downloadMatch && context.method === 'GET') { if (downloadMatch && context.method === 'GET') {
return this.handleDownload(downloadMatch[1]); return this.handleDownload(downloadMatch[1], actor);
} }
// Legacy specs endpoints (Marshal format) // Legacy specs endpoints (Marshal format)
@@ -232,16 +259,19 @@ export class RubyGemsRegistry extends BaseRegistry {
/** /**
* Handle /info/{gem} endpoint (Compact Index) * Handle /info/{gem} endpoint (Compact Index)
*/ */
private async handleInfoFile(gemName: string): Promise<IResponse> { private async handleInfoFile(gemName: string, actor?: IRequestActor): Promise<IResponse> {
let content = await this.storage.getRubyGemsInfo(gemName); let content = await this.storage.getRubyGemsInfo(gemName);
// Try upstream if not found locally // Try upstream if not found locally
if (!content && this.upstream) { if (!content) {
const upstreamInfo = await this.upstream.fetchInfo(gemName); const upstream = await this.getUpstreamForRequest(gemName, 'info', 'GET', actor);
if (upstreamInfo) { if (upstream) {
// Cache locally const upstreamInfo = await upstream.fetchInfo(gemName);
await this.storage.putRubyGemsInfo(gemName, upstreamInfo); if (upstreamInfo) {
content = upstreamInfo; // Cache locally
await this.storage.putRubyGemsInfo(gemName, upstreamInfo);
content = upstreamInfo;
}
} }
} }
@@ -267,7 +297,7 @@ export class RubyGemsRegistry extends BaseRegistry {
/** /**
* Handle gem file download * Handle gem file download
*/ */
private async handleDownload(filename: string): Promise<IResponse> { private async handleDownload(filename: string, actor?: IRequestActor): Promise<IResponse> {
const parsed = helpers.parseGemFilename(filename); const parsed = helpers.parseGemFilename(filename);
if (!parsed) { if (!parsed) {
return this.errorResponse(400, 'Invalid gem filename'); return this.errorResponse(400, 'Invalid gem filename');
@@ -280,11 +310,14 @@ export class RubyGemsRegistry extends BaseRegistry {
); );
// Try upstream if not found locally // Try upstream if not found locally
if (!gemData && this.upstream) { if (!gemData) {
gemData = await this.upstream.fetchGem(parsed.name, parsed.version); const upstream = await this.getUpstreamForRequest(parsed.name, 'gem', 'GET', actor);
if (gemData) { if (upstream) {
// Cache locally gemData = await upstream.fetchGem(parsed.name, parsed.version);
await this.storage.putRubyGemsGem(parsed.name, parsed.version, gemData, parsed.platform); if (gemData) {
// Cache locally
await this.storage.putRubyGemsGem(parsed.name, parsed.version, gemData, parsed.platform);
}
} }
} }

View File

@@ -1,4 +1,4 @@
import type { TRegistryProtocol } from '../core/interfaces.core.js'; import type { TRegistryProtocol, IRequestActor } from '../core/interfaces.core.js';
/** /**
* Scope rule for routing requests to specific upstreams. * Scope rule for routing requests to specific upstreams.
@@ -146,6 +146,8 @@ export interface IUpstreamFetchContext {
headers: Record<string, string>; headers: Record<string, string>;
/** Query parameters */ /** Query parameters */
query: Record<string, string>; query: Record<string, string>;
/** Actor performing the request (for cache key isolation) */
actor?: IRequestActor;
} }
/** /**
@@ -193,3 +195,80 @@ export const DEFAULT_RESILIENCE_CONFIG: IUpstreamResilienceConfig = {
circuitBreakerThreshold: 5, circuitBreakerThreshold: 5,
circuitBreakerResetMs: 30000, circuitBreakerResetMs: 30000,
}; };
// ============================================================================
// Upstream Provider Interfaces
// ============================================================================
/**
* Context for resolving upstream configuration.
* Passed to IUpstreamProvider per-request to enable dynamic upstream routing.
*/
export interface IUpstreamResolutionContext {
/** Protocol being accessed */
protocol: TRegistryProtocol;
/** Resource identifier (package name, repository, coordinates, etc.) */
resource: string;
/** Extracted scope (e.g., "company" from "@company/pkg", "myorg" from "myorg/image") */
scope: string | null;
/** Actor performing the request */
actor?: IRequestActor;
/** HTTP method */
method: string;
/** Resource type (packument, tarball, manifest, blob, etc.) */
resourceType: string;
}
/**
* Dynamic upstream configuration provider.
* Implement this interface to provide per-request upstream routing
* based on actor context (user, organization, etc.)
*
* @example
* ```typescript
* class OrgUpstreamProvider implements IUpstreamProvider {
* constructor(private db: Database) {}
*
* async resolveUpstreamConfig(ctx: IUpstreamResolutionContext) {
* if (ctx.actor?.orgId) {
* const orgConfig = await this.db.getOrgUpstream(ctx.actor.orgId, ctx.protocol);
* if (orgConfig) return orgConfig;
* }
* return this.db.getDefaultUpstream(ctx.protocol);
* }
* }
* ```
*/
export interface IUpstreamProvider {
/** Optional initialization */
init?(): Promise<void>;
/**
* Resolve upstream configuration for a request.
* @param context - Information about the current request
* @returns Upstream config to use, or null to skip upstream lookup
*/
resolveUpstreamConfig(context: IUpstreamResolutionContext): Promise<IProtocolUpstreamConfig | null>;
}
/**
* Static upstream provider for simple configurations.
* Use this when you have fixed upstream registries that don't change per-request.
*
* @example
* ```typescript
* const provider = new StaticUpstreamProvider({
* npm: {
* enabled: true,
* upstreams: [{ id: 'npmjs', url: 'https://registry.npmjs.org', priority: 1, enabled: true, auth: { type: 'none' } }],
* },
* });
* ```
*/
export class StaticUpstreamProvider implements IUpstreamProvider {
constructor(private configs: Partial<Record<TRegistryProtocol, IProtocolUpstreamConfig>>) {}
async resolveUpstreamConfig(ctx: IUpstreamResolutionContext): Promise<IProtocolUpstreamConfig | null> {
return this.configs[ctx.protocol] ?? null;
}
}