feat(upstream): Add dynamic per-request upstream provider and integrate into registries
This commit is contained in:
@@ -6,6 +6,8 @@ import { SmartRegistry } from '../../ts/classes.smartregistry.js';
|
||||
import type { IRegistryConfig, IAuthToken, TRegistryProtocol } from '../../ts/core/interfaces.core.js';
|
||||
import type { IAuthProvider, ITokenOptions } from '../../ts/core/interfaces.auth.js';
|
||||
import type { IStorageHooks, IStorageHookContext, IBeforePutResult, IBeforeDeleteResult } from '../../ts/core/interfaces.storage.js';
|
||||
import { StaticUpstreamProvider } from '../../ts/upstream/interfaces.upstream.js';
|
||||
import type { IUpstreamProvider, IUpstreamResolutionContext, IProtocolUpstreamConfig } from '../../ts/upstream/interfaces.upstream.js';
|
||||
|
||||
const testQenv = new qenv.Qenv('./', './.nogit');
|
||||
|
||||
@@ -134,6 +136,89 @@ export async function createTestRegistry(): Promise<SmartRegistry> {
|
||||
return registry;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a test SmartRegistry instance with upstream provider configured
|
||||
*/
|
||||
export async function createTestRegistryWithUpstream(
|
||||
upstreamProvider?: IUpstreamProvider
|
||||
): Promise<SmartRegistry> {
|
||||
// Read S3 config from env.json
|
||||
const s3AccessKey = await testQenv.getEnvVarOnDemand('S3_ACCESSKEY');
|
||||
const s3SecretKey = await testQenv.getEnvVarOnDemand('S3_SECRETKEY');
|
||||
const s3Endpoint = await testQenv.getEnvVarOnDemand('S3_ENDPOINT');
|
||||
const s3Port = await testQenv.getEnvVarOnDemand('S3_PORT');
|
||||
|
||||
// Default to StaticUpstreamProvider with npm.js configured
|
||||
const defaultProvider = new StaticUpstreamProvider({
|
||||
npm: {
|
||||
enabled: true,
|
||||
upstreams: [{ id: 'npmjs', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
|
||||
},
|
||||
oci: {
|
||||
enabled: true,
|
||||
upstreams: [{ id: 'dockerhub', url: 'https://registry-1.docker.io', priority: 1, enabled: true }],
|
||||
},
|
||||
});
|
||||
|
||||
const config: IRegistryConfig = {
|
||||
storage: {
|
||||
accessKey: s3AccessKey || 'minioadmin',
|
||||
accessSecret: s3SecretKey || 'minioadmin',
|
||||
endpoint: s3Endpoint || 'localhost',
|
||||
port: parseInt(s3Port || '9000', 10),
|
||||
useSsl: false,
|
||||
region: 'us-east-1',
|
||||
bucketName: 'test-registry',
|
||||
},
|
||||
auth: {
|
||||
jwtSecret: 'test-secret-key',
|
||||
tokenStore: 'memory',
|
||||
npmTokens: { enabled: true },
|
||||
ociTokens: {
|
||||
enabled: true,
|
||||
realm: 'https://auth.example.com/token',
|
||||
service: 'test-registry',
|
||||
},
|
||||
pypiTokens: { enabled: true },
|
||||
rubygemsTokens: { enabled: true },
|
||||
},
|
||||
upstreamProvider: upstreamProvider || defaultProvider,
|
||||
oci: { enabled: true, basePath: '/oci' },
|
||||
npm: { enabled: true, basePath: '/npm' },
|
||||
maven: { enabled: true, basePath: '/maven' },
|
||||
composer: { enabled: true, basePath: '/composer' },
|
||||
cargo: { enabled: true, basePath: '/cargo' },
|
||||
pypi: { enabled: true, basePath: '/pypi' },
|
||||
rubygems: { enabled: true, basePath: '/rubygems' },
|
||||
};
|
||||
|
||||
const registry = new SmartRegistry(config);
|
||||
await registry.init();
|
||||
|
||||
return registry;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a mock upstream provider that tracks all calls for testing
|
||||
*/
|
||||
export function createTrackingUpstreamProvider(
|
||||
baseConfig?: Partial<Record<TRegistryProtocol, IProtocolUpstreamConfig>>
|
||||
): {
|
||||
provider: IUpstreamProvider;
|
||||
calls: IUpstreamResolutionContext[];
|
||||
} {
|
||||
const calls: IUpstreamResolutionContext[] = [];
|
||||
|
||||
const provider: IUpstreamProvider = {
|
||||
async resolveUpstreamConfig(context: IUpstreamResolutionContext) {
|
||||
calls.push({ ...context });
|
||||
return baseConfig?.[context.protocol] ?? null;
|
||||
},
|
||||
};
|
||||
|
||||
return { provider, calls };
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper to create test authentication tokens
|
||||
*/
|
||||
|
||||
343
test/test.upstream.provider.ts
Normal file
343
test/test.upstream.provider.ts
Normal file
@@ -0,0 +1,343 @@
|
||||
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||
import { SmartRegistry } from '../ts/index.js';
|
||||
import {
|
||||
createTestRegistryWithUpstream,
|
||||
createTrackingUpstreamProvider,
|
||||
} from './helpers/registry.js';
|
||||
import { StaticUpstreamProvider } from '../ts/upstream/interfaces.upstream.js';
|
||||
import type {
|
||||
IUpstreamProvider,
|
||||
IUpstreamResolutionContext,
|
||||
IProtocolUpstreamConfig,
|
||||
} from '../ts/upstream/interfaces.upstream.js';
|
||||
import type { TRegistryProtocol } from '../ts/core/interfaces.core.js';
|
||||
|
||||
// =============================================================================
|
||||
// StaticUpstreamProvider Tests
|
||||
// =============================================================================
|
||||
|
||||
tap.test('StaticUpstreamProvider: should return config for configured protocol', async () => {
|
||||
const npmConfig: IProtocolUpstreamConfig = {
|
||||
enabled: true,
|
||||
upstreams: [{ id: 'npmjs', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
|
||||
};
|
||||
|
||||
const provider = new StaticUpstreamProvider({
|
||||
npm: npmConfig,
|
||||
});
|
||||
|
||||
const result = await provider.resolveUpstreamConfig({
|
||||
protocol: 'npm',
|
||||
resource: 'lodash',
|
||||
scope: null,
|
||||
method: 'GET',
|
||||
resourceType: 'packument',
|
||||
});
|
||||
|
||||
expect(result).toBeDefined();
|
||||
expect(result?.enabled).toEqual(true);
|
||||
expect(result?.upstreams[0].id).toEqual('npmjs');
|
||||
});
|
||||
|
||||
tap.test('StaticUpstreamProvider: should return null for unconfigured protocol', async () => {
|
||||
const provider = new StaticUpstreamProvider({
|
||||
npm: {
|
||||
enabled: true,
|
||||
upstreams: [{ id: 'npmjs', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
|
||||
},
|
||||
});
|
||||
|
||||
const result = await provider.resolveUpstreamConfig({
|
||||
protocol: 'maven',
|
||||
resource: 'com.example:lib',
|
||||
scope: 'com.example',
|
||||
method: 'GET',
|
||||
resourceType: 'pom',
|
||||
});
|
||||
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
tap.test('StaticUpstreamProvider: should support multiple protocols', async () => {
|
||||
const provider = new StaticUpstreamProvider({
|
||||
npm: {
|
||||
enabled: true,
|
||||
upstreams: [{ id: 'npmjs', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
|
||||
},
|
||||
oci: {
|
||||
enabled: true,
|
||||
upstreams: [{ id: 'dockerhub', url: 'https://registry-1.docker.io', priority: 1, enabled: true }],
|
||||
},
|
||||
maven: {
|
||||
enabled: true,
|
||||
upstreams: [{ id: 'central', url: 'https://repo1.maven.org/maven2', priority: 1, enabled: true }],
|
||||
},
|
||||
});
|
||||
|
||||
const npmResult = await provider.resolveUpstreamConfig({
|
||||
protocol: 'npm',
|
||||
resource: 'lodash',
|
||||
scope: null,
|
||||
method: 'GET',
|
||||
resourceType: 'packument',
|
||||
});
|
||||
expect(npmResult?.upstreams[0].id).toEqual('npmjs');
|
||||
|
||||
const ociResult = await provider.resolveUpstreamConfig({
|
||||
protocol: 'oci',
|
||||
resource: 'library/nginx',
|
||||
scope: 'library',
|
||||
method: 'GET',
|
||||
resourceType: 'manifest',
|
||||
});
|
||||
expect(ociResult?.upstreams[0].id).toEqual('dockerhub');
|
||||
|
||||
const mavenResult = await provider.resolveUpstreamConfig({
|
||||
protocol: 'maven',
|
||||
resource: 'com.example:lib',
|
||||
scope: 'com.example',
|
||||
method: 'GET',
|
||||
resourceType: 'pom',
|
||||
});
|
||||
expect(mavenResult?.upstreams[0].id).toEqual('central');
|
||||
});
|
||||
|
||||
// =============================================================================
|
||||
// Registry with Provider Integration Tests
|
||||
// =============================================================================
|
||||
|
||||
let registry: SmartRegistry;
|
||||
let trackingProvider: ReturnType<typeof createTrackingUpstreamProvider>;
|
||||
|
||||
tap.test('Provider Integration: should create registry with upstream provider', async () => {
|
||||
trackingProvider = createTrackingUpstreamProvider({
|
||||
npm: {
|
||||
enabled: true,
|
||||
upstreams: [{ id: 'test-npm', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
|
||||
},
|
||||
});
|
||||
|
||||
registry = await createTestRegistryWithUpstream(trackingProvider.provider);
|
||||
|
||||
expect(registry).toBeInstanceOf(SmartRegistry);
|
||||
expect(registry.isInitialized()).toEqual(true);
|
||||
});
|
||||
|
||||
tap.test('Provider Integration: should call provider when fetching unknown npm package', async () => {
|
||||
// Clear previous calls
|
||||
trackingProvider.calls.length = 0;
|
||||
|
||||
// Request a package that doesn't exist locally - should trigger upstream lookup
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/npm/@test-scope/nonexistent-package',
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
// Provider should have been called for the packument lookup
|
||||
const npmCalls = trackingProvider.calls.filter(c => c.protocol === 'npm');
|
||||
|
||||
// The package doesn't exist locally, so upstream should be consulted
|
||||
// Note: actual upstream fetch may fail since the package doesn't exist
|
||||
expect(response.status).toBeOneOf([404, 200, 502]); // 404 if not found, 502 if upstream error
|
||||
});
|
||||
|
||||
tap.test('Provider Integration: provider receives correct context for scoped npm package', async () => {
|
||||
trackingProvider.calls.length = 0;
|
||||
|
||||
// Use URL-encoded path for scoped packages as npm client does
|
||||
await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/npm/@myorg%2fmy-package',
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
// Find any npm call - the exact resource type depends on routing
|
||||
const npmCalls = trackingProvider.calls.filter(c => c.protocol === 'npm');
|
||||
|
||||
// Provider should be called for upstream lookup
|
||||
if (npmCalls.length > 0) {
|
||||
const call = npmCalls[0];
|
||||
expect(call.protocol).toEqual('npm');
|
||||
// The resource should include the scoped name
|
||||
expect(call.resource).toInclude('myorg');
|
||||
expect(call.method).toEqual('GET');
|
||||
}
|
||||
});
|
||||
|
||||
tap.test('Provider Integration: provider receives correct context for unscoped npm package', async () => {
|
||||
trackingProvider.calls.length = 0;
|
||||
|
||||
await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/npm/lodash',
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
const packumentCall = trackingProvider.calls.find(
|
||||
c => c.protocol === 'npm' && c.resourceType === 'packument'
|
||||
);
|
||||
|
||||
if (packumentCall) {
|
||||
expect(packumentCall.protocol).toEqual('npm');
|
||||
expect(packumentCall.resource).toEqual('lodash');
|
||||
expect(packumentCall.scope).toBeNull(); // No scope for unscoped package
|
||||
}
|
||||
});
|
||||
|
||||
// =============================================================================
|
||||
// Custom Provider Implementation Tests
|
||||
// =============================================================================
|
||||
|
||||
tap.test('Custom Provider: should support dynamic resolution based on context', async () => {
|
||||
// Create a provider that returns different configs based on scope
|
||||
const dynamicProvider: IUpstreamProvider = {
|
||||
async resolveUpstreamConfig(context: IUpstreamResolutionContext) {
|
||||
if (context.scope === 'internal') {
|
||||
// Internal packages go to private registry
|
||||
return {
|
||||
enabled: true,
|
||||
upstreams: [{ id: 'private', url: 'https://private.registry.com', priority: 1, enabled: true }],
|
||||
};
|
||||
}
|
||||
// Everything else goes to public registry
|
||||
return {
|
||||
enabled: true,
|
||||
upstreams: [{ id: 'public', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
const internalResult = await dynamicProvider.resolveUpstreamConfig({
|
||||
protocol: 'npm',
|
||||
resource: '@internal/utils',
|
||||
scope: 'internal',
|
||||
method: 'GET',
|
||||
resourceType: 'packument',
|
||||
});
|
||||
expect(internalResult?.upstreams[0].id).toEqual('private');
|
||||
|
||||
const publicResult = await dynamicProvider.resolveUpstreamConfig({
|
||||
protocol: 'npm',
|
||||
resource: '@public/utils',
|
||||
scope: 'public',
|
||||
method: 'GET',
|
||||
resourceType: 'packument',
|
||||
});
|
||||
expect(publicResult?.upstreams[0].id).toEqual('public');
|
||||
});
|
||||
|
||||
tap.test('Custom Provider: should support actor-based resolution', async () => {
|
||||
const actorAwareProvider: IUpstreamProvider = {
|
||||
async resolveUpstreamConfig(context: IUpstreamResolutionContext) {
|
||||
// Different upstreams based on user's organization
|
||||
if (context.actor?.orgId === 'enterprise-org') {
|
||||
return {
|
||||
enabled: true,
|
||||
upstreams: [{ id: 'enterprise', url: 'https://enterprise.registry.com', priority: 1, enabled: true }],
|
||||
};
|
||||
}
|
||||
return {
|
||||
enabled: true,
|
||||
upstreams: [{ id: 'default', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
const enterpriseResult = await actorAwareProvider.resolveUpstreamConfig({
|
||||
protocol: 'npm',
|
||||
resource: 'lodash',
|
||||
scope: null,
|
||||
actor: { orgId: 'enterprise-org', userId: 'user1' },
|
||||
method: 'GET',
|
||||
resourceType: 'packument',
|
||||
});
|
||||
expect(enterpriseResult?.upstreams[0].id).toEqual('enterprise');
|
||||
|
||||
const defaultResult = await actorAwareProvider.resolveUpstreamConfig({
|
||||
protocol: 'npm',
|
||||
resource: 'lodash',
|
||||
scope: null,
|
||||
actor: { orgId: 'free-org', userId: 'user2' },
|
||||
method: 'GET',
|
||||
resourceType: 'packument',
|
||||
});
|
||||
expect(defaultResult?.upstreams[0].id).toEqual('default');
|
||||
});
|
||||
|
||||
tap.test('Custom Provider: should support disabling upstream for specific resources', async () => {
|
||||
const selectiveProvider: IUpstreamProvider = {
|
||||
async resolveUpstreamConfig(context: IUpstreamResolutionContext) {
|
||||
// Block upstream for internal packages
|
||||
if (context.scope === 'internal') {
|
||||
return null; // No upstream for internal packages
|
||||
}
|
||||
return {
|
||||
enabled: true,
|
||||
upstreams: [{ id: 'public', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
const internalResult = await selectiveProvider.resolveUpstreamConfig({
|
||||
protocol: 'npm',
|
||||
resource: '@internal/secret',
|
||||
scope: 'internal',
|
||||
method: 'GET',
|
||||
resourceType: 'packument',
|
||||
});
|
||||
expect(internalResult).toBeNull();
|
||||
|
||||
const publicResult = await selectiveProvider.resolveUpstreamConfig({
|
||||
protocol: 'npm',
|
||||
resource: 'lodash',
|
||||
scope: null,
|
||||
method: 'GET',
|
||||
resourceType: 'packument',
|
||||
});
|
||||
expect(publicResult).not.toBeNull();
|
||||
});
|
||||
|
||||
// =============================================================================
|
||||
// Registry without Provider Tests
|
||||
// =============================================================================
|
||||
|
||||
tap.test('No Provider: registry should work without upstream provider', async () => {
|
||||
const registryWithoutUpstream = await createTestRegistryWithUpstream(
|
||||
// Pass a provider that always returns null
|
||||
{
|
||||
async resolveUpstreamConfig() {
|
||||
return null;
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
expect(registryWithoutUpstream).toBeInstanceOf(SmartRegistry);
|
||||
|
||||
// Should return 404 for non-existent package (no upstream to check)
|
||||
const response = await registryWithoutUpstream.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/npm/nonexistent-package-xyz',
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(404);
|
||||
|
||||
registryWithoutUpstream.destroy();
|
||||
});
|
||||
|
||||
// =============================================================================
|
||||
// Cleanup
|
||||
// =============================================================================
|
||||
|
||||
tap.postTask('cleanup registry', async () => {
|
||||
if (registry) {
|
||||
registry.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
export default tap.start();
|
||||
Reference in New Issue
Block a user