12 Commits

Author SHA1 Message Date
abf7605e14 v2.8.0
Some checks failed
Default (tags) / security (push) Failing after 0s
Default (tags) / test (push) Failing after 0s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-03-24 22:59:37 +00:00
7da1a35efe feat(core,storage,oci,registry-config): add streaming response support and configurable registry URLs across protocols 2026-03-24 22:59:37 +00:00
1f0acf2825 fix(oci): remove /v2/ from internal route patterns and make upstream apiPrefix configurable
The OCI handler had /v2/ baked into all regex patterns and Location headers.
When basePath was set to /v2 (as in stack.gallery), stripping it removed the
prefix that patterns expected, causing all OCI endpoints to 404.

Now patterns match on bare paths after basePath stripping, working correctly
regardless of the basePath value.

Also adds configurable apiPrefix to OCI upstream class (default /v2) for
registries behind reverse proxies with custom path prefixes.
2026-03-21 16:17:52 +00:00
37e4c5be4a fix(npm): decode URL-encoded package names after regex extraction
Scoped npm packages use %2f encoding for the slash in URLs (e.g. @scope%2fpackage).
Previously, the encoded name was used as-is for storage and packument metadata,
causing npm install to fail with EINVALIDPACKAGENAME. Now each regex extraction
point decodes the package name via decodeURIComponent while keeping the path
encoded for correct regex matching.
2026-03-21 11:59:52 +00:00
9bbc3da484 v2.7.0
Some checks failed
Default (tags) / security (push) Successful in 43s
Default (tags) / test (push) Failing after 41s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-12-03 22:16:40 +00:00
e9af3f8328 feat(upstream): Add dynamic per-request upstream provider and integrate into registries 2025-12-03 22:16:40 +00:00
351680159b v2.6.0
Some checks failed
Default (tags) / security (push) Successful in 44s
Default (tags) / test (push) Failing after 43s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-27 22:12:52 +00:00
0cabf284ed feat(core): Add core registry infrastructure: storage, auth, upstream cache, and protocol handlers 2025-11-27 22:12:52 +00:00
dbc8566aad v2.5.0
Some checks failed
Default (tags) / security (push) Successful in 43s
Default (tags) / test (push) Failing after 44s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-27 21:11:04 +00:00
bd64a7b140 feat(pypi,rubygems): Add PyPI and RubyGems protocol implementations, upstream caching, and auth/storage improvements 2025-11-27 21:11:04 +00:00
ae8dec9142 v2.4.0
Some checks failed
Default (tags) / security (push) Successful in 45s
Default (tags) / test (push) Failing after 45s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-27 20:59:49 +00:00
19da87a9df feat(core): Add pluggable auth providers, storage hooks, multi-upstream cache awareness, and PyPI/RubyGems protocol implementations 2025-11-27 20:59:49 +00:00
53 changed files with 8397 additions and 6015 deletions

24
.smartconfig.json Normal file
View File

@@ -0,0 +1,24 @@
{
"@git.zone/cli": {
"projectType": "npm",
"module": {
"githost": "code.foss.global",
"gitscope": "push.rocks",
"gitrepo": "smartregistry",
"description": "A composable TypeScript library implementing OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems registries for building unified container and package registries",
"npmPackagename": "@push.rocks/smartregistry",
"license": "MIT",
"projectDomain": "push.rocks"
},
"release": {
"registries": [
"https://verdaccio.lossless.digital",
"https://registry.npmjs.org"
],
"accessLevel": "public"
}
},
"@ship.zone/szci": {
"npmGlobalTools": []
}
}

View File

@@ -1,7 +1,7 @@
{ {
"json.schemas": [ "json.schemas": [
{ {
"fileMatch": ["/npmextra.json"], "fileMatch": ["/.smartconfig.json"],
"schema": { "schema": {
"type": "object", "type": "object",
"properties": { "properties": {

View File

@@ -1,5 +1,61 @@
# Changelog # Changelog
## 2026-03-24 - 2.8.0 - feat(core,storage,oci,registry-config)
add streaming response support and configurable registry URLs across protocols
- Normalize SmartRegistry responses to ReadableStream bodies at the public API boundary and add stream helper utilities for buffers, JSON, and hashing
- Add streaming storage accessors for OCI, npm, Maven, Cargo, Composer, PyPI, and RubyGems downloads to reduce in-memory buffering
- Make per-protocol registryUrl configurable so CLI and integration tests can use correct host and port values
- Refactor OCI blob uploads to persist chunks in storage during upload and clean up temporary chunk objects after completion or expiry
- Update tests and storage integration to use the new stream-based response model and smartstorage backend
## 2025-12-03 - 2.7.0 - feat(upstream)
Add dynamic per-request upstream provider and integrate into registries
- Introduce IUpstreamProvider and IUpstreamResolutionContext to resolve upstream configs per request.
- Add StaticUpstreamProvider implementation for simple static upstream configurations.
- Propagate dynamic upstream provider through SmartRegistry and wire into protocol handlers (npm, oci, maven, cargo, composer, pypi, rubygems).
- Replace persistent per-protocol upstream instances with per-request resolution: registries now call provider.resolveUpstreamConfig(...) and instantiate protocol-specific Upstream when needed.
- Add IRequestActor to core interfaces and pass actor context (userId, ip, userAgent, etc.) to upstream resolution and storage/auth hooks.
- Update many protocol registries to accept an upstreamProvider instead of IProtocolUpstreamConfig and to attempt upstream fetches only when provider returns enabled config.
- Add utilities and tests: test helpers to create registries with upstream provider, a tracking upstream provider helper, StaticUpstreamProvider tests and extensive upstream/provider integration tests.
- Improve upstream interfaces and cache/fetch contexts (IUpstreamFetchContext includes actor) and add StaticUpstreamProvider class to upstream module.
## 2025-11-27 - 2.6.0 - feat(core)
Add core registry infrastructure: storage, auth, upstream cache, and protocol handlers
- Introduce RegistryStorage: unified storage abstraction with hook support (before/after put/delete/get) and helpers for OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems paths and operations
- Add DefaultAuthProvider and AuthManager: in-memory token store, UUID tokens for package protocols, OCI JWT creation/validation, token lifecycle (create/validate/revoke) and authorization checking
- Add SmartRegistry orchestrator to initialize and route requests to protocol handlers (OCI, NPM, Maven, Cargo, Composer, PyPI, RubyGems)
- Implement upstream subsystem: UpstreamCache (in-memory + optional S3 persistence), BaseUpstream with multi-upstream routing, scope rules, retries, TTLs, stale-while-revalidate and negative caching
- Add circuit breaker implementation for upstream resilience with exponential backoff and per-upstream breakers
- Add protocol implementations and helpers: NpmRegistry/NpmUpstream (packument/tarball handling and tarball URL rewriting), PypiRegistry (PEP 503/691 support, uploads, metadata), MavenRegistry (artifact/metadata handling and checksum generation), CargoRegistry (sparse index, publish/download/yank)
- Utility exports and helpers: buffer helpers, plugins aggregator, path helpers, and various protocol-specific helper modules
## 2025-11-27 - 2.5.0 - feat(pypi,rubygems)
Add PyPI and RubyGems protocol implementations, upstream caching, and auth/storage improvements
- Implemented full PyPI support (PEP 503 Simple API HTML, PEP 691 JSON API, legacy upload handling, name normalization, hash verification, content negotiation, package/file storage and metadata management).
- Implemented RubyGems support (compact index, /versions, /info, /names endpoints, gem upload, yank/unyank, platform handling and file storage).
- Expanded RegistryStorage with protocol-specific helpers for OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems (get/put/delete/list helpers, metadata handling, context-aware hooks).
- Added AuthManager and DefaultAuthProvider improvements: unified token creation/validation for multiple protocols (npm, oci, maven, composer, cargo, pypi, rubygems) and OCI JWT support.
- Added upstream infrastructure: BaseUpstream, UpstreamCache (S3-backed optional, stale-while-revalidate, negative caching), circuit breaker with retries/backoff and resilience defaults.
- Added various protocol registries (NPM, Maven, Cargo, OCI, PyPI) with request routing, permission checks, and optional upstream proxying/caching.
## 2025-11-27 - 2.4.0 - feat(core)
Add pluggable auth providers, storage hooks, multi-upstream cache awareness, and PyPI/RubyGems protocol implementations
- Introduce pluggable authentication: IAuthProvider interface and DefaultAuthProvider (in-memory) with OCI JWT support and UUID tokens.
- AuthManager now accepts a custom provider and delegates all auth operations (authenticate, validateToken, create/revoke tokens, authorize, listUserTokens).
- Add storage hooks (IStorageHooks) and hook contexts: beforePut/afterPut/afterGet/beforeDelete/afterDelete. RegistryStorage now supports hooks, context management (setContext/withContext) and invokes hooks around operations.
- RegistryStorage expanded with many protocol-specific helper methods (OCI, NPM, Maven, Cargo, Composer, PyPI, RubyGems) and improved S3/SmartBucket integration.
- Upstream improvements: BaseUpstream and UpstreamCache became multi-upstream aware (cache keys now include upstream URL), cache operations are async and support negative caching, stale-while-revalidate, ETag/metadata persistence, and S3-backed storage layer.
- Circuit breaker, retry, resilience and scope-rule routing enhancements for upstreams; upstream fetch logic updated to prefer primary upstream for cache keys and background revalidation behavior.
- SmartRegistry API extended to accept custom authProvider and storageHooks, and now wires RegistryStorage and AuthManager with those options. Core exports updated to expose auth and storage interfaces and DefaultAuthProvider.
- Add full PyPI (PEP 503/691, upload API) and RubyGems (Compact Index, API v1, uploads/yank/unyank, specs endpoints) registry implementations with parsing, upload/download, metadata management and upstream proxying.
- Add utility helpers: binary buffer helpers (toBuffer/isBinaryData), pypi and rubygems helper modules, and numerous protocol-specific helpers and tests referenced in readme.hints.
- These changes are additive and designed to be backward compatible; bumping minor version.
## 2025-11-27 - 2.3.0 - feat(upstream) ## 2025-11-27 - 2.3.0 - feat(upstream)
Add upstream proxy/cache subsystem and integrate per-protocol upstreams Add upstream proxy/cache subsystem and integrate per-protocol upstreams

View File

@@ -1,18 +0,0 @@
{
"gitzone": {
"projectType": "npm",
"module": {
"githost": "code.foss.global",
"gitscope": "push.rocks",
"gitrepo": "smartregistry",
"description": "a registry for npm modules and oci images",
"npmPackagename": "@push.rocks/smartregistry",
"license": "MIT",
"projectDomain": "push.rocks"
}
},
"npmci": {
"npmGlobalTools": [],
"npmAccessLevel": "public"
}
}

View File

@@ -1,6 +1,6 @@
{ {
"name": "@push.rocks/smartregistry", "name": "@push.rocks/smartregistry",
"version": "2.3.0", "version": "2.8.0",
"private": false, "private": false,
"description": "A composable TypeScript library implementing OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems registries for building unified container and package registries", "description": "A composable TypeScript library implementing OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems registries for building unified container and package registries",
"main": "dist_ts/index.js", "main": "dist_ts/index.js",
@@ -10,17 +10,17 @@
"license": "MIT", "license": "MIT",
"scripts": { "scripts": {
"test": "(tstest test/ --verbose --logfile --timeout 240)", "test": "(tstest test/ --verbose --logfile --timeout 240)",
"build": "(tsbuild --web --allowimplicitany)", "build": "(tsbuild --allowimplicitany)",
"buildDocs": "(tsdoc)" "buildDocs": "(tsdoc)"
}, },
"devDependencies": { "devDependencies": {
"@git.zone/tsbuild": "^3.1.0", "@git.zone/tsbuild": "^4.4.0",
"@git.zone/tsbundle": "^2.0.5", "@git.zone/tsbundle": "^2.10.0",
"@git.zone/tsrun": "^2.0.0", "@git.zone/tsrun": "^2.0.2",
"@git.zone/tstest": "^3.1.0", "@git.zone/tstest": "^3.6.0",
"@push.rocks/smartarchive": "^5.0.1", "@push.rocks/smartarchive": "^5.2.1",
"@push.rocks/smarts3": "^5.1.0", "@push.rocks/smartstorage": "^6.3.2",
"@types/node": "^24.10.1" "@types/node": "^25.5.0"
}, },
"repository": { "repository": {
"type": "git", "type": "git",
@@ -39,7 +39,7 @@
"dist_ts_web/**/*", "dist_ts_web/**/*",
"assets/**/*", "assets/**/*",
"cli.js", "cli.js",
"npmextra.json", ".smartconfig.json",
"readme.md" "readme.md"
], ],
"pnpm": { "pnpm": {
@@ -47,13 +47,13 @@
}, },
"dependencies": { "dependencies": {
"@push.rocks/qenv": "^6.1.3", "@push.rocks/qenv": "^6.1.3",
"@push.rocks/smartbucket": "^4.3.0", "@push.rocks/smartbucket": "^4.5.1",
"@push.rocks/smartlog": "^3.1.10", "@push.rocks/smartlog": "^3.2.1",
"@push.rocks/smartpath": "^6.0.0", "@push.rocks/smartpath": "^6.0.0",
"@push.rocks/smartrequest": "^5.0.1", "@push.rocks/smartrequest": "^5.0.1",
"@tsclass/tsclass": "^9.3.0", "@tsclass/tsclass": "^9.5.0",
"adm-zip": "^0.5.10", "adm-zip": "^0.5.16",
"minimatch": "^10.1.1" "minimatch": "^10.2.4"
}, },
"packageManager": "pnpm@10.18.1+sha512.77a884a165cbba2d8d1c19e3b4880eee6d2fcabd0d879121e282196b80042351d5eb3ca0935fa599da1dc51265cc68816ad2bddd2a2de5ea9fdf92adbec7cd34" "packageManager": "pnpm@10.18.1+sha512.77a884a165cbba2d8d1c19e3b4880eee6d2fcabd0d879121e282196b80042351d5eb3ca0935fa599da1dc51265cc68816ad2bddd2a2de5ea9fdf92adbec7cd34"
} }

7100
pnpm-lock.yaml generated

File diff suppressed because it is too large Load Diff

1136
readme.md

File diff suppressed because it is too large Load Diff

View File

@@ -2,6 +2,7 @@ import { tap, expect } from '@git.zone/tstest';
import { RegistryStorage } from '../ts/core/classes.registrystorage.js'; import { RegistryStorage } from '../ts/core/classes.registrystorage.js';
import { CargoRegistry } from '../ts/cargo/classes.cargoregistry.js'; import { CargoRegistry } from '../ts/cargo/classes.cargoregistry.js';
import { AuthManager } from '../ts/core/classes.authmanager.js'; import { AuthManager } from '../ts/core/classes.authmanager.js';
import { streamToJson } from '../ts/core/helpers.stream.js';
// Test index path calculation // Test index path calculation
tap.test('should calculate correct index paths for different crate names', async () => { tap.test('should calculate correct index paths for different crate names', async () => {
@@ -123,9 +124,10 @@ tap.test('should return valid config.json', async () => {
expect(response.status).to.equal(200); expect(response.status).to.equal(200);
expect(response.headers['Content-Type']).to.equal('application/json'); expect(response.headers['Content-Type']).to.equal('application/json');
expect(response.body).to.be.an('object'); const body = await streamToJson(response.body);
expect(response.body.dl).to.include('/api/v1/crates/{crate}/{version}/download'); expect(body).to.be.an('object');
expect(response.body.api).to.equal('http://localhost:5000/cargo'); expect(body.dl).to.include('/api/v1/crates/{crate}/{version}/download');
expect(body.api).to.equal('http://localhost:5000/cargo');
}); });
export default tap.start(); export default tap.start();

View File

@@ -3,7 +3,11 @@ import * as crypto from 'crypto';
import * as smartarchive from '@push.rocks/smartarchive'; import * as smartarchive from '@push.rocks/smartarchive';
import * as smartbucket from '@push.rocks/smartbucket'; import * as smartbucket from '@push.rocks/smartbucket';
import { SmartRegistry } from '../../ts/classes.smartregistry.js'; import { SmartRegistry } from '../../ts/classes.smartregistry.js';
import type { IRegistryConfig } from '../../ts/core/interfaces.core.js'; import type { IRegistryConfig, IAuthToken, TRegistryProtocol } from '../../ts/core/interfaces.core.js';
import type { IAuthProvider, ITokenOptions } from '../../ts/core/interfaces.auth.js';
import type { IStorageHooks, IStorageHookContext, IBeforePutResult, IBeforeDeleteResult } from '../../ts/core/interfaces.storage.js';
import { StaticUpstreamProvider } from '../../ts/upstream/interfaces.upstream.js';
import type { IUpstreamProvider, IUpstreamResolutionContext, IProtocolUpstreamConfig } from '../../ts/upstream/interfaces.upstream.js';
const testQenv = new qenv.Qenv('./', './.nogit'); const testQenv = new qenv.Qenv('./', './.nogit');
@@ -61,7 +65,9 @@ export async function cleanupS3Bucket(prefix?: string): Promise<void> {
/** /**
* Create a test SmartRegistry instance with all protocols enabled * Create a test SmartRegistry instance with all protocols enabled
*/ */
export async function createTestRegistry(): Promise<SmartRegistry> { export async function createTestRegistry(options?: {
registryUrl?: string;
}): Promise<SmartRegistry> {
// Read S3 config from env.json // Read S3 config from env.json
const s3AccessKey = await testQenv.getEnvVarOnDemand('S3_ACCESSKEY'); const s3AccessKey = await testQenv.getEnvVarOnDemand('S3_ACCESSKEY');
const s3SecretKey = await testQenv.getEnvVarOnDemand('S3_SECRETKEY'); const s3SecretKey = await testQenv.getEnvVarOnDemand('S3_SECRETKEY');
@@ -99,30 +105,37 @@ export async function createTestRegistry(): Promise<SmartRegistry> {
oci: { oci: {
enabled: true, enabled: true,
basePath: '/oci', basePath: '/oci',
...(options?.registryUrl ? { registryUrl: `${options.registryUrl}/oci` } : {}),
}, },
npm: { npm: {
enabled: true, enabled: true,
basePath: '/npm', basePath: '/npm',
...(options?.registryUrl ? { registryUrl: `${options.registryUrl}/npm` } : {}),
}, },
maven: { maven: {
enabled: true, enabled: true,
basePath: '/maven', basePath: '/maven',
...(options?.registryUrl ? { registryUrl: `${options.registryUrl}/maven` } : {}),
}, },
composer: { composer: {
enabled: true, enabled: true,
basePath: '/composer', basePath: '/composer',
...(options?.registryUrl ? { registryUrl: `${options.registryUrl}/composer` } : {}),
}, },
cargo: { cargo: {
enabled: true, enabled: true,
basePath: '/cargo', basePath: '/cargo',
...(options?.registryUrl ? { registryUrl: `${options.registryUrl}/cargo` } : {}),
}, },
pypi: { pypi: {
enabled: true, enabled: true,
basePath: '/pypi', basePath: '/pypi',
...(options?.registryUrl ? { registryUrl: options.registryUrl } : {}),
}, },
rubygems: { rubygems: {
enabled: true, enabled: true,
basePath: '/rubygems', basePath: '/rubygems',
...(options?.registryUrl ? { registryUrl: `${options.registryUrl}/rubygems` } : {}),
}, },
}; };
@@ -132,6 +145,89 @@ export async function createTestRegistry(): Promise<SmartRegistry> {
return registry; return registry;
} }
/**
* Create a test SmartRegistry instance with upstream provider configured
*/
export async function createTestRegistryWithUpstream(
upstreamProvider?: IUpstreamProvider
): Promise<SmartRegistry> {
// Read S3 config from env.json
const s3AccessKey = await testQenv.getEnvVarOnDemand('S3_ACCESSKEY');
const s3SecretKey = await testQenv.getEnvVarOnDemand('S3_SECRETKEY');
const s3Endpoint = await testQenv.getEnvVarOnDemand('S3_ENDPOINT');
const s3Port = await testQenv.getEnvVarOnDemand('S3_PORT');
// Default to StaticUpstreamProvider with npm.js configured
const defaultProvider = new StaticUpstreamProvider({
npm: {
enabled: true,
upstreams: [{ id: 'npmjs', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
},
oci: {
enabled: true,
upstreams: [{ id: 'dockerhub', url: 'https://registry-1.docker.io', priority: 1, enabled: true }],
},
});
const config: IRegistryConfig = {
storage: {
accessKey: s3AccessKey || 'minioadmin',
accessSecret: s3SecretKey || 'minioadmin',
endpoint: s3Endpoint || 'localhost',
port: parseInt(s3Port || '9000', 10),
useSsl: false,
region: 'us-east-1',
bucketName: 'test-registry',
},
auth: {
jwtSecret: 'test-secret-key',
tokenStore: 'memory',
npmTokens: { enabled: true },
ociTokens: {
enabled: true,
realm: 'https://auth.example.com/token',
service: 'test-registry',
},
pypiTokens: { enabled: true },
rubygemsTokens: { enabled: true },
},
upstreamProvider: upstreamProvider || defaultProvider,
oci: { enabled: true, basePath: '/oci' },
npm: { enabled: true, basePath: '/npm' },
maven: { enabled: true, basePath: '/maven' },
composer: { enabled: true, basePath: '/composer' },
cargo: { enabled: true, basePath: '/cargo' },
pypi: { enabled: true, basePath: '/pypi' },
rubygems: { enabled: true, basePath: '/rubygems' },
};
const registry = new SmartRegistry(config);
await registry.init();
return registry;
}
/**
* Create a mock upstream provider that tracks all calls for testing
*/
export function createTrackingUpstreamProvider(
baseConfig?: Partial<Record<TRegistryProtocol, IProtocolUpstreamConfig>>
): {
provider: IUpstreamProvider;
calls: IUpstreamResolutionContext[];
} {
const calls: IUpstreamResolutionContext[] = [];
const provider: IUpstreamProvider = {
async resolveUpstreamConfig(context: IUpstreamResolutionContext) {
calls.push({ ...context });
return baseConfig?.[context.protocol] ?? null;
},
};
return { provider, calls };
}
/** /**
* Helper to create test authentication tokens * Helper to create test authentication tokens
*/ */
@@ -354,7 +450,7 @@ class TestClass
}, },
]; ];
return zipTools.createZip(entries); return Buffer.from(await zipTools.createZip(entries));
} }
/** /**
@@ -428,7 +524,7 @@ def hello():
}, },
]; ];
return zipTools.createZip(entries); return Buffer.from(await zipTools.createZip(entries));
} }
/** /**
@@ -489,7 +585,7 @@ def hello():
}, },
]; ];
return tarTools.packFilesToTarGz(entries); return Buffer.from(await tarTools.packFilesToTarGz(entries));
} }
/** /**
@@ -560,7 +656,7 @@ summary: Test gem for SmartRegistry
test_files: [] test_files: []
`; `;
const metadataGz = await gzipTools.compress(Buffer.from(metadataYaml, 'utf-8')); const metadataGz = Buffer.from(await gzipTools.compress(Buffer.from(metadataYaml, 'utf-8')));
// Create data.tar.gz content // Create data.tar.gz content
const libContent = `# ${gemName} const libContent = `# ${gemName}
@@ -581,7 +677,7 @@ end
}, },
]; ];
const dataTarGz = await tarTools.packFilesToTarGz(dataEntries); const dataTarGz = Buffer.from(await tarTools.packFilesToTarGz(dataEntries));
// Create the outer gem (tar.gz containing metadata.gz and data.tar.gz) // Create the outer gem (tar.gz containing metadata.gz and data.tar.gz)
const gemEntries: smartarchive.IArchiveEntry[] = [ const gemEntries: smartarchive.IArchiveEntry[] = [
@@ -596,7 +692,7 @@ end
]; ];
// RubyGems .gem files are plain tar archives (NOT gzipped), containing metadata.gz and data.tar.gz // RubyGems .gem files are plain tar archives (NOT gzipped), containing metadata.gz and data.tar.gz
return tarTools.packFiles(gemEntries); return Buffer.from(await tarTools.packFiles(gemEntries));
} }
/** /**
@@ -608,3 +704,228 @@ export function calculateRubyGemsChecksums(data: Buffer) {
sha256: crypto.createHash('sha256').update(data).digest('hex'), sha256: crypto.createHash('sha256').update(data).digest('hex'),
}; };
} }
// ============================================================================
// Enterprise Extensibility Test Helpers
// ============================================================================
/**
* Create a mock auth provider for testing pluggable authentication.
* Allows customizing behavior for different test scenarios.
*/
export function createMockAuthProvider(overrides?: Partial<IAuthProvider>): IAuthProvider {
const tokens = new Map<string, IAuthToken>();
return {
init: async () => {},
authenticate: async (credentials) => {
// Default: always authenticate successfully
return credentials.username;
},
validateToken: async (token, protocol) => {
const stored = tokens.get(token);
if (stored && (!protocol || stored.type === protocol)) {
return stored;
}
// Mock token for tests
if (token === 'valid-mock-token') {
return {
type: 'npm' as TRegistryProtocol,
userId: 'mock-user',
scopes: ['npm:*:*:*'],
};
}
return null;
},
createToken: async (userId, protocol, options) => {
const tokenId = `mock-${protocol}-${Date.now()}`;
const authToken: IAuthToken = {
type: protocol,
userId,
scopes: options?.scopes || [`${protocol}:*:*:*`],
readonly: options?.readonly,
expiresAt: options?.expiresIn ? new Date(Date.now() + options.expiresIn * 1000) : undefined,
};
tokens.set(tokenId, authToken);
return tokenId;
},
revokeToken: async (token) => {
tokens.delete(token);
},
authorize: async (token, resource, action) => {
if (!token) return false;
if (token.readonly && ['write', 'push', 'delete'].includes(action)) {
return false;
}
return true;
},
listUserTokens: async (userId) => {
const result: Array<{ key: string; readonly: boolean; created: string; protocol?: TRegistryProtocol }> = [];
for (const [key, token] of tokens.entries()) {
if (token.userId === userId) {
result.push({
key: `hash-${key.substring(0, 8)}`,
readonly: token.readonly || false,
created: new Date().toISOString(),
protocol: token.type,
});
}
}
return result;
},
...overrides,
};
}
/**
* Create test storage hooks that track all calls.
* Useful for verifying hook invocation order and parameters.
*/
export function createTrackingHooks(options?: {
beforePutAllowed?: boolean;
beforeDeleteAllowed?: boolean;
throwOnAfterPut?: boolean;
throwOnAfterGet?: boolean;
}): {
hooks: IStorageHooks;
calls: Array<{ method: string; context: IStorageHookContext; timestamp: number }>;
} {
const calls: Array<{ method: string; context: IStorageHookContext; timestamp: number }> = [];
return {
calls,
hooks: {
beforePut: async (ctx) => {
calls.push({ method: 'beforePut', context: ctx, timestamp: Date.now() });
return {
allowed: options?.beforePutAllowed !== false,
reason: options?.beforePutAllowed === false ? 'Blocked by test' : undefined,
};
},
afterPut: async (ctx) => {
calls.push({ method: 'afterPut', context: ctx, timestamp: Date.now() });
if (options?.throwOnAfterPut) {
throw new Error('Test error in afterPut');
}
},
beforeDelete: async (ctx) => {
calls.push({ method: 'beforeDelete', context: ctx, timestamp: Date.now() });
return {
allowed: options?.beforeDeleteAllowed !== false,
reason: options?.beforeDeleteAllowed === false ? 'Blocked by test' : undefined,
};
},
afterDelete: async (ctx) => {
calls.push({ method: 'afterDelete', context: ctx, timestamp: Date.now() });
},
afterGet: async (ctx) => {
calls.push({ method: 'afterGet', context: ctx, timestamp: Date.now() });
if (options?.throwOnAfterGet) {
throw new Error('Test error in afterGet');
}
},
},
};
}
/**
* Create a blocking storage hooks implementation for quota testing.
*/
export function createQuotaHooks(maxSizeBytes: number): {
hooks: IStorageHooks;
currentUsage: { bytes: number };
} {
const currentUsage = { bytes: 0 };
return {
currentUsage,
hooks: {
beforePut: async (ctx) => {
const size = ctx.metadata?.size || 0;
if (currentUsage.bytes + size > maxSizeBytes) {
return { allowed: false, reason: `Quota exceeded: ${currentUsage.bytes + size} > ${maxSizeBytes}` };
}
return { allowed: true };
},
afterPut: async (ctx) => {
currentUsage.bytes += ctx.metadata?.size || 0;
},
afterDelete: async (ctx) => {
currentUsage.bytes -= ctx.metadata?.size || 0;
if (currentUsage.bytes < 0) currentUsage.bytes = 0;
},
},
};
}
/**
* Create a SmartBucket storage backend for upstream cache testing.
*/
export async function createTestStorageBackend(): Promise<{
storage: {
getObject: (key: string) => Promise<Buffer | null>;
putObject: (key: string, data: Buffer) => Promise<void>;
deleteObject: (key: string) => Promise<void>;
listObjects: (prefix: string) => Promise<string[]>;
};
bucket: smartbucket.Bucket;
cleanup: () => Promise<void>;
}> {
const s3AccessKey = await testQenv.getEnvVarOnDemand('S3_ACCESSKEY');
const s3SecretKey = await testQenv.getEnvVarOnDemand('S3_SECRETKEY');
const s3Endpoint = await testQenv.getEnvVarOnDemand('S3_ENDPOINT');
const s3Port = await testQenv.getEnvVarOnDemand('S3_PORT');
const s3 = new smartbucket.SmartBucket({
accessKey: s3AccessKey || 'minioadmin',
accessSecret: s3SecretKey || 'minioadmin',
endpoint: s3Endpoint || 'localhost',
port: parseInt(s3Port || '9000', 10),
useSsl: false,
});
const testRunId = generateTestRunId();
const bucketName = 'test-cache-' + testRunId.substring(0, 8);
const bucket = await s3.createBucket(bucketName);
const storage = {
getObject: async (key: string): Promise<Buffer | null> => {
try {
const file = await bucket.fastGet({ path: key });
if (!file) return null;
const stream = await file.createReadStream();
const chunks: Buffer[] = [];
for await (const chunk of stream) {
chunks.push(Buffer.from(chunk));
}
return Buffer.concat(chunks);
} catch {
return null;
}
},
putObject: async (key: string, data: Buffer): Promise<void> => {
await bucket.fastPut({ path: key, contents: data, overwrite: true });
},
deleteObject: async (key: string): Promise<void> => {
await bucket.fastRemove({ path: key });
},
listObjects: async (prefix: string): Promise<string[]> => {
const files = await bucket.fastList({ prefix });
return files.map(f => f.name);
},
};
const cleanup = async () => {
try {
const files = await bucket.fastList({});
for (const file of files) {
await bucket.fastRemove({ path: file.name });
}
await s3.removeBucket(bucketName);
} catch {
// Ignore cleanup errors
}
};
return { storage, bucket, cleanup };
}

412
test/test.auth.provider.ts Normal file
View File

@@ -0,0 +1,412 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { DefaultAuthProvider } from '../ts/core/classes.defaultauthprovider.js';
import { AuthManager } from '../ts/core/classes.authmanager.js';
import type { IAuthProvider } from '../ts/core/interfaces.auth.js';
import type { IAuthConfig, IAuthToken } from '../ts/core/interfaces.core.js';
import { createMockAuthProvider } from './helpers/registry.js';
// ============================================================================
// Test State
// ============================================================================
let provider: DefaultAuthProvider;
let authConfig: IAuthConfig;
// ============================================================================
// Setup
// ============================================================================
tap.test('setup: should create DefaultAuthProvider', async () => {
authConfig = {
jwtSecret: 'test-secret-key-for-jwt-signing',
tokenStore: 'memory',
npmTokens: { enabled: true },
ociTokens: {
enabled: true,
realm: 'https://auth.example.com/token',
service: 'test-registry',
},
mavenTokens: { enabled: true },
cargoTokens: { enabled: true },
composerTokens: { enabled: true },
pypiTokens: { enabled: true },
rubygemsTokens: { enabled: true },
};
provider = new DefaultAuthProvider(authConfig);
await provider.init();
expect(provider).toBeInstanceOf(DefaultAuthProvider);
});
// ============================================================================
// Authentication Tests
// ============================================================================
tap.test('authenticate: should authenticate new user (auto-registration)', async () => {
const userId = await provider.authenticate({
username: 'newuser',
password: 'newpassword',
});
expect(userId).toEqual('newuser');
});
tap.test('authenticate: should authenticate existing user with correct password', async () => {
// First registration
await provider.authenticate({
username: 'existinguser',
password: 'correctpass',
});
// Second authentication with same credentials
const userId = await provider.authenticate({
username: 'existinguser',
password: 'correctpass',
});
expect(userId).toEqual('existinguser');
});
tap.test('authenticate: should reject authentication with wrong password', async () => {
// First registration
await provider.authenticate({
username: 'passworduser',
password: 'originalpass',
});
// Attempt with wrong password
const userId = await provider.authenticate({
username: 'passworduser',
password: 'wrongpass',
});
expect(userId).toBeNull();
});
// ============================================================================
// Token Creation Tests
// ============================================================================
tap.test('createToken: should create NPM token with correct scopes', async () => {
const token = await provider.createToken('testuser', 'npm', {
scopes: ['npm:package:*:*'],
});
expect(token).toBeTruthy();
expect(typeof token).toEqual('string');
// Validate the token
const validated = await provider.validateToken(token, 'npm');
expect(validated).toBeTruthy();
expect(validated!.type).toEqual('npm');
expect(validated!.userId).toEqual('testuser');
expect(validated!.scopes).toContain('npm:package:*:*');
});
tap.test('createToken: should create Maven token', async () => {
const token = await provider.createToken('mavenuser', 'maven', {
readonly: true,
});
expect(token).toBeTruthy();
const validated = await provider.validateToken(token, 'maven');
expect(validated).toBeTruthy();
expect(validated!.type).toEqual('maven');
expect(validated!.readonly).toBeTrue();
});
tap.test('createToken: should create OCI JWT token with correct claims', async () => {
const token = await provider.createToken('ociuser', 'oci', {
scopes: ['oci:repository:myrepo:push', 'oci:repository:myrepo:pull'],
expiresIn: 3600,
});
expect(token).toBeTruthy();
// OCI tokens are JWTs (contain dots)
expect(token.split('.').length).toEqual(3);
const validated = await provider.validateToken(token, 'oci');
expect(validated).toBeTruthy();
expect(validated!.type).toEqual('oci');
expect(validated!.userId).toEqual('ociuser');
expect(validated!.scopes.length).toBeGreaterThan(0);
});
tap.test('createToken: should create token with expiration', async () => {
const token = await provider.createToken('expiryuser', 'npm', {
expiresIn: 60, // 60 seconds
});
const validated = await provider.validateToken(token, 'npm');
expect(validated).toBeTruthy();
expect(validated!.expiresAt).toBeTruthy();
expect(validated!.expiresAt!.getTime()).toBeGreaterThan(Date.now());
});
// ============================================================================
// Token Validation Tests
// ============================================================================
tap.test('validateToken: should validate UUID token (NPM, Maven, etc.)', async () => {
const npmToken = await provider.createToken('validateuser', 'npm');
const validated = await provider.validateToken(npmToken);
expect(validated).toBeTruthy();
expect(validated!.type).toEqual('npm');
expect(validated!.userId).toEqual('validateuser');
});
tap.test('validateToken: should validate OCI JWT token', async () => {
const ociToken = await provider.createToken('ocivalidate', 'oci', {
scopes: ['oci:repository:*:*'],
});
const validated = await provider.validateToken(ociToken, 'oci');
expect(validated).toBeTruthy();
expect(validated!.type).toEqual('oci');
expect(validated!.userId).toEqual('ocivalidate');
});
tap.test('validateToken: should reject expired tokens', async () => {
const token = await provider.createToken('expireduser', 'npm', {
expiresIn: -1, // Already expired (in the past)
});
// The token should be created but will fail validation due to expiry
const validated = await provider.validateToken(token, 'npm');
// Token should be rejected because it's expired
expect(validated).toBeNull();
});
tap.test('validateToken: should reject invalid token', async () => {
const validated = await provider.validateToken('invalid-random-token');
expect(validated).toBeNull();
});
tap.test('validateToken: should reject token with wrong protocol', async () => {
const npmToken = await provider.createToken('protocoluser', 'npm');
// Try to validate as Maven token
const validated = await provider.validateToken(npmToken, 'maven');
expect(validated).toBeNull();
});
// ============================================================================
// Token Revocation Tests
// ============================================================================
tap.test('revokeToken: should revoke tokens', async () => {
const token = await provider.createToken('revokeuser', 'npm');
// Verify token works before revocation
let validated = await provider.validateToken(token);
expect(validated).toBeTruthy();
// Revoke the token
await provider.revokeToken(token);
// Token should no longer be valid
validated = await provider.validateToken(token);
expect(validated).toBeNull();
});
// ============================================================================
// Authorization Tests
// ============================================================================
tap.test('authorize: should authorize read actions for readonly tokens', async () => {
const token = await provider.createToken('readonlyuser', 'npm', {
readonly: true,
scopes: ['npm:package:*:read'],
});
const validated = await provider.validateToken(token);
const canRead = await provider.authorize(validated, 'npm:package:lodash', 'read');
expect(canRead).toBeTrue();
const canPull = await provider.authorize(validated, 'npm:package:lodash', 'pull');
expect(canPull).toBeTrue();
});
tap.test('authorize: should deny write actions for readonly tokens', async () => {
const token = await provider.createToken('readonlyuser2', 'npm', {
readonly: true,
scopes: ['npm:package:*:*'],
});
const validated = await provider.validateToken(token);
const canWrite = await provider.authorize(validated, 'npm:package:lodash', 'write');
expect(canWrite).toBeFalse();
const canPush = await provider.authorize(validated, 'npm:package:lodash', 'push');
expect(canPush).toBeFalse();
const canDelete = await provider.authorize(validated, 'npm:package:lodash', 'delete');
expect(canDelete).toBeFalse();
});
tap.test('authorize: should match scopes with wildcards', async () => {
// The scope system uses literal * as wildcard, not glob patterns
// npm:*:*:* means "all types, all names, all actions under npm"
const token = await provider.createToken('wildcarduser', 'npm', {
scopes: ['npm:*:*:*'],
});
const validated = await provider.validateToken(token);
// Should match any npm resource with full wildcard scope
const canAccessAnyPackage = await provider.authorize(validated, 'npm:package:lodash', 'read');
expect(canAccessAnyPackage).toBeTrue();
const canAccessScopedPackage = await provider.authorize(validated, 'npm:package:@myorg/foo', 'write');
expect(canAccessScopedPackage).toBeTrue();
});
tap.test('authorize: should deny access with null token', async () => {
const canAccess = await provider.authorize(null, 'npm:package:lodash', 'read');
expect(canAccess).toBeFalse();
});
// ============================================================================
// List Tokens Tests
// ============================================================================
tap.test('listUserTokens: should list user tokens', async () => {
// Create multiple tokens for the same user
const userId = 'listtokenuser';
await provider.createToken(userId, 'npm');
await provider.createToken(userId, 'maven', { readonly: true });
await provider.createToken(userId, 'cargo');
const tokens = await provider.listUserTokens!(userId);
expect(tokens.length).toBeGreaterThanOrEqual(3);
// Check that tokens have expected properties
for (const token of tokens) {
expect(token.key).toBeTruthy();
expect(typeof token.readonly).toEqual('boolean');
expect(token.created).toBeTruthy();
}
// Verify we have different protocols
const protocols = tokens.map(t => t.protocol);
expect(protocols).toContain('npm');
expect(protocols).toContain('maven');
expect(protocols).toContain('cargo');
});
// ============================================================================
// AuthManager Integration Tests
// ============================================================================
tap.test('AuthManager: should accept custom IAuthProvider', async () => {
const mockProvider = createMockAuthProvider({
authenticate: async (credentials) => {
if (credentials.username === 'custom' && credentials.password === 'pass') {
return 'custom-user-id';
}
return null;
},
});
const manager = new AuthManager(authConfig, mockProvider);
await manager.init();
// Use the custom provider
const userId = await manager.authenticate({
username: 'custom',
password: 'pass',
});
expect(userId).toEqual('custom-user-id');
// Wrong credentials should fail
const failed = await manager.authenticate({
username: 'custom',
password: 'wrong',
});
expect(failed).toBeNull();
});
tap.test('AuthManager: should use default provider when none specified', async () => {
const manager = new AuthManager(authConfig);
await manager.init();
// Should use DefaultAuthProvider internally
const userId = await manager.authenticate({
username: 'defaultuser',
password: 'defaultpass',
});
expect(userId).toEqual('defaultuser');
});
tap.test('AuthManager: should delegate token creation to provider', async () => {
let tokenCreated = false;
const mockProvider = createMockAuthProvider({
createToken: async (userId, protocol, options) => {
tokenCreated = true;
return `mock-token-${protocol}-${userId}`;
},
});
const manager = new AuthManager(authConfig, mockProvider);
await manager.init();
const token = await manager.createNpmToken('delegateuser', false);
expect(tokenCreated).toBeTrue();
expect(token).toContain('mock-token-npm');
});
// ============================================================================
// Edge Cases
// ============================================================================
tap.test('edge: should handle concurrent token operations', async () => {
const promises: Promise<string>[] = [];
// Create 10 tokens concurrently
for (let i = 0; i < 10; i++) {
promises.push(provider.createToken(`concurrent-user-${i}`, 'npm'));
}
const tokens = await Promise.all(promises);
// All tokens should be unique
const uniqueTokens = new Set(tokens);
expect(uniqueTokens.size).toEqual(10);
// All tokens should be valid
for (const token of tokens) {
const validated = await provider.validateToken(token);
expect(validated).toBeTruthy();
}
});
tap.test('edge: should handle empty scopes', async () => {
const token = await provider.createToken('emptyuser', 'npm', {
scopes: [],
});
const validated = await provider.validateToken(token);
expect(validated).toBeTruthy();
// Even with empty scopes, token should be valid
});
// ============================================================================
// Cleanup
// ============================================================================
tap.test('cleanup', async () => {
// No cleanup needed for in-memory provider
});
export default tap.start();

View File

@@ -79,16 +79,10 @@ async function createHttpServer(
res.setHeader(key, value); res.setHeader(key, value);
} }
// Send body // Send body (response.body is always ReadableStream<Uint8Array> or undefined)
if (response.body) { if (response.body) {
if (Buffer.isBuffer(response.body)) { const { Readable } = await import('stream');
res.end(response.body); Readable.fromWeb(response.body).pipe(res);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
} else { } else {
res.end(); res.end();
} }
@@ -251,8 +245,11 @@ function cleanupTestDir(dir: string): void {
// ======================================================================== // ========================================================================
tap.test('Cargo CLI: should setup registry and HTTP server', async () => { tap.test('Cargo CLI: should setup registry and HTTP server', async () => {
// Create registry // Use port 5000
registry = await createTestRegistry(); registryPort = 5000;
// Create registry with correct registryUrl for CLI tests
registry = await createTestRegistry({ registryUrl: `http://localhost:${registryPort}` });
const tokens = await createTestTokens(registry); const tokens = await createTestTokens(registry);
cargoToken = tokens.cargoToken; cargoToken = tokens.cargoToken;
@@ -266,10 +263,6 @@ tap.test('Cargo CLI: should setup registry and HTTP server', async () => {
} catch (error) { } catch (error) {
// Ignore error if operation fails // Ignore error if operation fails
} }
// Use port 5000 (hardcoded in CargoRegistry default config)
// TODO: Once registryUrl is configurable, use dynamic port like npm test (35001)
registryPort = 5000;
const serverSetup = await createHttpServer(registry, registryPort); const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server; server = serverSetup.server;
registryUrl = serverSetup.url; registryUrl = serverSetup.url;

View File

@@ -84,16 +84,10 @@ async function createHttpServer(
res.setHeader(key, value); res.setHeader(key, value);
} }
// Send body // Send body (response.body is always ReadableStream<Uint8Array> or undefined)
if (response.body) { if (response.body) {
if (Buffer.isBuffer(response.body)) { const { Readable } = await import('stream');
res.end(response.body); Readable.fromWeb(response.body).pipe(res);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
} else { } else {
res.end(); res.end();
} }
@@ -249,16 +243,16 @@ tap.test('Composer CLI: should verify composer is installed', async () => {
}); });
tap.test('Composer CLI: should setup registry and HTTP server', async () => { tap.test('Composer CLI: should setup registry and HTTP server', async () => {
// Create registry // Use port 38000 (avoids conflicts with other tests)
registry = await createTestRegistry(); registryPort = 38000;
// Create registry with correct registryUrl for CLI tests
registry = await createTestRegistry({ registryUrl: `http://localhost:${registryPort}` });
const tokens = await createTestTokens(registry); const tokens = await createTestTokens(registry);
composerToken = tokens.composerToken; composerToken = tokens.composerToken;
expect(registry).toBeInstanceOf(SmartRegistry); expect(registry).toBeInstanceOf(SmartRegistry);
expect(composerToken).toBeTypeOf('string'); expect(composerToken).toBeTypeOf('string');
// Use port 38000 (avoids conflicts with other tests)
registryPort = 38000;
const serverSetup = await createHttpServer(registry, registryPort); const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server; server = serverSetup.server;
registryUrl = serverSetup.url; registryUrl = serverSetup.url;

View File

@@ -1,5 +1,6 @@
import { expect, tap } from '@git.zone/tstest/tapbundle'; import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js'; import { SmartRegistry } from '../ts/index.js';
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
import { createTestRegistry, createTestTokens, createComposerZip } from './helpers/registry.js'; import { createTestRegistry, createTestTokens, createComposerZip } from './helpers/registry.js';
let registry: SmartRegistry; let registry: SmartRegistry;
@@ -41,9 +42,10 @@ tap.test('Composer: should return packages.json (GET /packages.json)', async ()
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('metadata-url'); const body = await streamToJson(response.body);
expect(response.body).toHaveProperty('available-packages'); expect(body).toHaveProperty('metadata-url');
expect(response.body['available-packages']).toBeInstanceOf(Array); expect(body).toHaveProperty('available-packages');
expect(body['available-packages']).toBeInstanceOf(Array);
}); });
tap.test('Composer: should upload a package (PUT /packages/{vendor/package})', async () => { tap.test('Composer: should upload a package (PUT /packages/{vendor/package})', async () => {
@@ -59,9 +61,10 @@ tap.test('Composer: should upload a package (PUT /packages/{vendor/package})', a
}); });
expect(response.status).toEqual(201); expect(response.status).toEqual(201);
expect(response.body.status).toEqual('success'); const body = await streamToJson(response.body);
expect(response.body.package).toEqual(testPackageName); expect(body.status).toEqual('success');
expect(response.body.version).toEqual(testVersion); expect(body.package).toEqual(testPackageName);
expect(body.version).toEqual(testVersion);
}); });
tap.test('Composer: should retrieve package metadata (GET /p2/{vendor/package}.json)', async () => { tap.test('Composer: should retrieve package metadata (GET /p2/{vendor/package}.json)', async () => {
@@ -73,11 +76,12 @@ tap.test('Composer: should retrieve package metadata (GET /p2/{vendor/package}.j
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('packages'); const body = await streamToJson(response.body);
expect(response.body.packages[testPackageName]).toBeInstanceOf(Array); expect(body).toHaveProperty('packages');
expect(response.body.packages[testPackageName].length).toEqual(1); expect(body.packages[testPackageName]).toBeInstanceOf(Array);
expect(body.packages[testPackageName].length).toEqual(1);
const packageData = response.body.packages[testPackageName][0]; const packageData = body.packages[testPackageName][0];
expect(packageData.name).toEqual(testPackageName); expect(packageData.name).toEqual(testPackageName);
expect(packageData.version).toEqual(testVersion); expect(packageData.version).toEqual(testVersion);
expect(packageData.version_normalized).toEqual('1.0.0.0'); expect(packageData.version_normalized).toEqual('1.0.0.0');
@@ -97,7 +101,8 @@ tap.test('Composer: should download package ZIP (GET /dists/{vendor/package}/{re
query: {}, query: {},
}); });
const reference = metadataResponse.body.packages[testPackageName][0].dist.reference; const metaBody = await streamToJson(metadataResponse.body);
const reference = metaBody.packages[testPackageName][0].dist.reference;
const response = await registry.handleRequest({ const response = await registry.handleRequest({
method: 'GET', method: 'GET',
@@ -107,7 +112,8 @@ tap.test('Composer: should download package ZIP (GET /dists/{vendor/package}/{re
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
expect(response.headers['Content-Type']).toEqual('application/zip'); expect(response.headers['Content-Type']).toEqual('application/zip');
expect(response.headers['Content-Disposition']).toContain('attachment'); expect(response.headers['Content-Disposition']).toContain('attachment');
}); });
@@ -121,9 +127,10 @@ tap.test('Composer: should list packages (GET /packages/list.json)', async () =>
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('packageNames'); const body = await streamToJson(response.body);
expect(response.body.packageNames).toBeInstanceOf(Array); expect(body).toHaveProperty('packageNames');
expect(response.body.packageNames).toContain(testPackageName); expect(body.packageNames).toBeInstanceOf(Array);
expect(body.packageNames).toContain(testPackageName);
}); });
tap.test('Composer: should filter package list (GET /packages/list.json?filter=vendor/*)', async () => { tap.test('Composer: should filter package list (GET /packages/list.json?filter=vendor/*)', async () => {
@@ -135,8 +142,9 @@ tap.test('Composer: should filter package list (GET /packages/list.json?filter=v
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body.packageNames).toBeInstanceOf(Array); const body = await streamToJson(response.body);
expect(response.body.packageNames).toContain(testPackageName); expect(body.packageNames).toBeInstanceOf(Array);
expect(body.packageNames).toContain(testPackageName);
}); });
tap.test('Composer: should prevent duplicate version upload', async () => { tap.test('Composer: should prevent duplicate version upload', async () => {
@@ -152,8 +160,9 @@ tap.test('Composer: should prevent duplicate version upload', async () => {
}); });
expect(response.status).toEqual(409); expect(response.status).toEqual(409);
expect(response.body.status).toEqual('error'); const body = await streamToJson(response.body);
expect(response.body.message).toContain('already exists'); expect(body.status).toEqual('error');
expect(body.message).toContain('already exists');
}); });
tap.test('Composer: should upload a second version', async () => { tap.test('Composer: should upload a second version', async () => {
@@ -172,8 +181,9 @@ tap.test('Composer: should upload a second version', async () => {
}); });
expect(response.status).toEqual(201); expect(response.status).toEqual(201);
expect(response.body.status).toEqual('success'); const body = await streamToJson(response.body);
expect(response.body.version).toEqual(testVersion2); expect(body.status).toEqual('success');
expect(body.version).toEqual(testVersion2);
}); });
tap.test('Composer: should return multiple versions in metadata', async () => { tap.test('Composer: should return multiple versions in metadata', async () => {
@@ -185,10 +195,11 @@ tap.test('Composer: should return multiple versions in metadata', async () => {
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body.packages[testPackageName]).toBeInstanceOf(Array); const body = await streamToJson(response.body);
expect(response.body.packages[testPackageName].length).toEqual(2); expect(body.packages[testPackageName]).toBeInstanceOf(Array);
expect(body.packages[testPackageName].length).toEqual(2);
const versions = response.body.packages[testPackageName].map((p: any) => p.version); const versions = body.packages[testPackageName].map((p: any) => p.version);
expect(versions).toContain('1.0.0'); expect(versions).toContain('1.0.0');
expect(versions).toContain('1.1.0'); expect(versions).toContain('1.1.0');
}); });
@@ -213,8 +224,9 @@ tap.test('Composer: should delete a specific version (DELETE /packages/{vendor/p
query: {}, query: {},
}); });
expect(metadataResponse.body.packages[testPackageName].length).toEqual(1); const metaBody = await streamToJson(metadataResponse.body);
expect(metadataResponse.body.packages[testPackageName][0].version).toEqual('1.1.0'); expect(metaBody.packages[testPackageName].length).toEqual(1);
expect(metaBody.packages[testPackageName][0].version).toEqual('1.1.0');
}); });
tap.test('Composer: should require auth for package upload', async () => { tap.test('Composer: should require auth for package upload', async () => {
@@ -231,7 +243,8 @@ tap.test('Composer: should require auth for package upload', async () => {
}); });
expect(response.status).toEqual(401); expect(response.status).toEqual(401);
expect(response.body.status).toEqual('error'); const body = await streamToJson(response.body);
expect(body.status).toEqual('error');
}); });
tap.test('Composer: should reject invalid ZIP (no composer.json)', async () => { tap.test('Composer: should reject invalid ZIP (no composer.json)', async () => {
@@ -249,8 +262,9 @@ tap.test('Composer: should reject invalid ZIP (no composer.json)', async () => {
}); });
expect(response.status).toEqual(400); expect(response.status).toEqual(400);
expect(response.body.status).toEqual('error'); const body = await streamToJson(response.body);
expect(response.body.message).toContain('composer.json'); expect(body.status).toEqual('error');
expect(body.message).toContain('composer.json');
}); });
tap.test('Composer: should delete entire package (DELETE /packages/{vendor/package})', async () => { tap.test('Composer: should delete entire package (DELETE /packages/{vendor/package})', async () => {

View File

@@ -1,5 +1,6 @@
import { expect, tap } from '@git.zone/tstest/tapbundle'; import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js'; import { SmartRegistry } from '../ts/index.js';
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
import { import {
createTestRegistry, createTestRegistry,
createTestTokens, createTestTokens,
@@ -79,7 +80,9 @@ tap.test('Integration: should handle /simple path for PyPI', async () => {
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toStartWith('text/html'); expect(response.headers['Content-Type']).toStartWith('text/html');
expect(response.body).toContain('integration-test-py'); const body = await streamToBuffer(response.body);
const text = body.toString('utf-8');
expect(text).toContain('integration-test-py');
}); });
tap.test('Integration: should reject PyPI token for RubyGems endpoint', async () => { tap.test('Integration: should reject PyPI token for RubyGems endpoint', async () => {
@@ -135,8 +138,9 @@ tap.test('Integration: should return 404 for unknown paths', async () => {
}); });
expect(response.status).toEqual(404); expect(response.status).toEqual(404);
expect(response.body).toHaveProperty('error'); const body = await streamToJson(response.body);
expect((response.body as any).error).toEqual('NOT_FOUND'); expect(body).toHaveProperty('error');
expect(body.error).toEqual('NOT_FOUND');
}); });
tap.test('Integration: should retrieve PyPI registry instance', async () => { tap.test('Integration: should retrieve PyPI registry instance', async () => {

View File

@@ -1,32 +1,34 @@
/** /**
* Integration test for smartregistry with smarts3 * Integration test for smartregistry with smartstorage
* Verifies that smartregistry works with a local S3-compatible server * Verifies that smartregistry works with a local S3-compatible server
*/ */
import { expect, tap } from '@git.zone/tstest/tapbundle'; import { expect, tap } from '@git.zone/tstest/tapbundle';
import * as smarts3Module from '@push.rocks/smarts3'; import * as smartstorageModule from '@push.rocks/smartstorage';
import { SmartRegistry } from '../ts/classes.smartregistry.js'; import { SmartRegistry } from '../ts/classes.smartregistry.js';
import type { IRegistryConfig } from '../ts/core/interfaces.core.js'; import type { IRegistryConfig } from '../ts/core/interfaces.core.js';
import { streamToJson } from '../ts/core/helpers.stream.js';
import * as crypto from 'crypto'; import * as crypto from 'crypto';
let s3Server: smarts3Module.Smarts3; let s3Server: smartstorageModule.SmartStorage;
let registry: SmartRegistry; let registry: SmartRegistry;
/** /**
* Setup: Start smarts3 server * Setup: Start smartstorage server
*/ */
tap.test('should start smarts3 server', async () => { tap.test('should start smartstorage server', async () => {
s3Server = await smarts3Module.Smarts3.createAndStart({ s3Server = await smartstorageModule.SmartStorage.createAndStart({
server: { server: {
port: 3456, // Use different port to avoid conflicts with other tests port: 3456,
host: '0.0.0.0', address: '0.0.0.0',
silent: true,
}, },
storage: { storage: {
cleanSlate: true, // Fresh storage for each test run cleanSlate: true,
bucketsDir: './.nogit/smarts3-test-buckets', directory: './.nogit/smartstorage-test-buckets',
}, },
logging: { logging: {
silent: true, // Reduce test output noise enabled: false,
}, },
}); });
@@ -34,20 +36,10 @@ tap.test('should start smarts3 server', async () => {
}); });
/** /**
* Setup: Create SmartRegistry with smarts3 configuration * Setup: Create SmartRegistry with smartstorage configuration
*/ */
tap.test('should create SmartRegistry instance with smarts3 IS3Descriptor', async () => { tap.test('should create SmartRegistry instance with smarts3 IS3Descriptor', async () => {
// Manually construct IS3Descriptor based on smarts3 configuration const s3Descriptor = await s3Server.getStorageDescriptor();
// Note: smarts3.getS3Descriptor() returns empty object as of v5.1.0
// This is a known limitation - smarts3 doesn't expose its config properly
const s3Descriptor = {
endpoint: 'localhost',
port: 3456,
accessKey: 'test', // smarts3 doesn't require real credentials
accessSecret: 'test',
useSsl: false,
region: 'us-east-1',
};
const config: IRegistryConfig = { const config: IRegistryConfig = {
storage: { storage: {
@@ -97,7 +89,7 @@ tap.test('should create SmartRegistry instance with smarts3 IS3Descriptor', asyn
}); });
/** /**
* Test NPM protocol with smarts3 * Test NPM protocol with smartstorage
*/ */
tap.test('NPM: should publish package to smarts3', async () => { tap.test('NPM: should publish package to smarts3', async () => {
const authManager = registry.getAuthManager(); const authManager = registry.getAuthManager();
@@ -139,7 +131,7 @@ tap.test('NPM: should publish package to smarts3', async () => {
body: packageData, body: packageData,
}); });
expect(response.status).toEqual(201); // 201 Created is correct for publishing expect(response.status).toEqual(201);
}); });
tap.test('NPM: should retrieve package from smarts3', async () => { tap.test('NPM: should retrieve package from smarts3', async () => {
@@ -151,12 +143,13 @@ tap.test('NPM: should retrieve package from smarts3', async () => {
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('name'); const body = await streamToJson(response.body);
expect(response.body.name).toEqual('test-package-smarts3'); expect(body).toHaveProperty('name');
expect(body.name).toEqual('test-package-smarts3');
}); });
/** /**
* Test OCI protocol with smarts3 * Test OCI protocol with smartstorage
*/ */
tap.test('OCI: should store blob in smarts3', async () => { tap.test('OCI: should store blob in smarts3', async () => {
const authManager = registry.getAuthManager(); const authManager = registry.getAuthManager();
@@ -173,7 +166,7 @@ tap.test('OCI: should store blob in smarts3', async () => {
// Initiate blob upload // Initiate blob upload
const initiateResponse = await registry.handleRequest({ const initiateResponse = await registry.handleRequest({
method: 'POST', method: 'POST',
path: '/oci/v2/test-image/blobs/uploads/', path: '/oci/test-image/blobs/uploads/',
headers: { headers: {
'Authorization': `Bearer ${token}`, 'Authorization': `Bearer ${token}`,
}, },
@@ -196,7 +189,7 @@ tap.test('OCI: should store blob in smarts3', async () => {
const uploadResponse = await registry.handleRequest({ const uploadResponse = await registry.handleRequest({
method: 'PUT', method: 'PUT',
path: `/oci/v2/test-image/blobs/uploads/${uploadId}`, path: `/oci/test-image/blobs/uploads/${uploadId}`,
headers: { headers: {
'Authorization': `Bearer ${token}`, 'Authorization': `Bearer ${token}`,
'Content-Type': 'application/octet-stream', 'Content-Type': 'application/octet-stream',
@@ -209,18 +202,9 @@ tap.test('OCI: should store blob in smarts3', async () => {
}); });
/** /**
* Test PyPI protocol with smarts3 * Test PyPI protocol with smartstorage
*/ */
tap.test('PyPI: should upload package to smarts3', async () => { tap.test('PyPI: should upload package to smarts3', async () => {
const authManager = registry.getAuthManager();
const userId = await authManager.authenticate({
username: 'testuser',
password: 'testpass',
});
const token = await authManager.createPypiToken(userId, false);
// Note: In a real test, this would be multipart/form-data
// For simplicity, we're testing the storage layer
const storage = registry.getStorage(); const storage = registry.getStorage();
// Store a test package file // Store a test package file
@@ -252,7 +236,7 @@ tap.test('PyPI: should upload package to smarts3', async () => {
}); });
/** /**
* Test Cargo protocol with smarts3 * Test Cargo protocol with smartstorage
*/ */
tap.test('Cargo: should store crate in smarts3', async () => { tap.test('Cargo: should store crate in smarts3', async () => {
const storage = registry.getStorage(); const storage = registry.getStorage();
@@ -281,11 +265,11 @@ tap.test('Cargo: should store crate in smarts3', async () => {
}); });
/** /**
* Cleanup: Stop smarts3 server * Cleanup: Stop smartstorage server
*/ */
tap.test('should stop smarts3 server', async () => { tap.test('should stop smartstorage server', async () => {
await s3Server.stop(); await s3Server.stop();
expect(true).toEqual(true); // Just verify it completes without error expect(true).toEqual(true);
}); });
export default tap.start(); export default tap.start();

View File

@@ -79,16 +79,10 @@ async function createHttpServer(
res.setHeader(key, value); res.setHeader(key, value);
} }
// Send body // Send body (response.body is always ReadableStream<Uint8Array> or undefined)
if (response.body) { if (response.body) {
if (Buffer.isBuffer(response.body)) { const { Readable } = await import('stream');
res.end(response.body); Readable.fromWeb(response.body).pipe(res);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
} else { } else {
res.end(); res.end();
} }
@@ -282,16 +276,16 @@ tap.test('Maven CLI: should verify mvn is installed', async () => {
}); });
tap.test('Maven CLI: should setup registry and HTTP server', async () => { tap.test('Maven CLI: should setup registry and HTTP server', async () => {
// Create registry // Use port 37000 (avoids conflicts with other tests)
registry = await createTestRegistry(); registryPort = 37000;
// Create registry with correct registryUrl for CLI tests
registry = await createTestRegistry({ registryUrl: `http://localhost:${registryPort}` });
const tokens = await createTestTokens(registry); const tokens = await createTestTokens(registry);
mavenToken = tokens.mavenToken; mavenToken = tokens.mavenToken;
expect(registry).toBeInstanceOf(SmartRegistry); expect(registry).toBeInstanceOf(SmartRegistry);
expect(mavenToken).toBeTypeOf('string'); expect(mavenToken).toBeTypeOf('string');
// Use port 37000 (avoids conflicts with other tests)
registryPort = 37000;
const serverSetup = await createHttpServer(registry, registryPort); const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server; server = serverSetup.server;
registryUrl = serverSetup.url; registryUrl = serverSetup.url;

View File

@@ -1,5 +1,6 @@
import { expect, tap } from '@git.zone/tstest/tapbundle'; import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js'; import { SmartRegistry } from '../ts/index.js';
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
import { import {
createTestRegistry, createTestRegistry,
createTestTokens, createTestTokens,
@@ -88,10 +89,11 @@ tap.test('Maven: should retrieve uploaded POM file (GET)', async () => {
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect((response.body as Buffer).toString('utf-8')).toContain(testGroupId); expect(body).toBeInstanceOf(Buffer);
expect((response.body as Buffer).toString('utf-8')).toContain(testArtifactId); expect(body.toString('utf-8')).toContain(testGroupId);
expect((response.body as Buffer).toString('utf-8')).toContain(testVersion); expect(body.toString('utf-8')).toContain(testArtifactId);
expect(body.toString('utf-8')).toContain(testVersion);
expect(response.headers['Content-Type']).toEqual('application/xml'); expect(response.headers['Content-Type']).toEqual('application/xml');
}); });
@@ -107,7 +109,8 @@ tap.test('Maven: should retrieve uploaded JAR file (GET)', async () => {
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
expect(response.headers['Content-Type']).toEqual('application/java-archive'); expect(response.headers['Content-Type']).toEqual('application/java-archive');
}); });
@@ -124,8 +127,9 @@ tap.test('Maven: should retrieve MD5 checksum for JAR (GET *.jar.md5)', async ()
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect((response.body as Buffer).toString('utf-8')).toEqual(checksums.md5); expect(body).toBeInstanceOf(Buffer);
expect(body.toString('utf-8')).toEqual(checksums.md5);
expect(response.headers['Content-Type']).toEqual('text/plain'); expect(response.headers['Content-Type']).toEqual('text/plain');
}); });
@@ -142,8 +146,9 @@ tap.test('Maven: should retrieve SHA1 checksum for JAR (GET *.jar.sha1)', async
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect((response.body as Buffer).toString('utf-8')).toEqual(checksums.sha1); expect(body).toBeInstanceOf(Buffer);
expect(body.toString('utf-8')).toEqual(checksums.sha1);
expect(response.headers['Content-Type']).toEqual('text/plain'); expect(response.headers['Content-Type']).toEqual('text/plain');
}); });
@@ -160,8 +165,9 @@ tap.test('Maven: should retrieve SHA256 checksum for JAR (GET *.jar.sha256)', as
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect((response.body as Buffer).toString('utf-8')).toEqual(checksums.sha256); expect(body).toBeInstanceOf(Buffer);
expect(body.toString('utf-8')).toEqual(checksums.sha256);
expect(response.headers['Content-Type']).toEqual('text/plain'); expect(response.headers['Content-Type']).toEqual('text/plain');
}); });
@@ -178,8 +184,9 @@ tap.test('Maven: should retrieve SHA512 checksum for JAR (GET *.jar.sha512)', as
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect((response.body as Buffer).toString('utf-8')).toEqual(checksums.sha512); expect(body).toBeInstanceOf(Buffer);
expect(body.toString('utf-8')).toEqual(checksums.sha512);
expect(response.headers['Content-Type']).toEqual('text/plain'); expect(response.headers['Content-Type']).toEqual('text/plain');
}); });
@@ -194,8 +201,9 @@ tap.test('Maven: should retrieve maven-metadata.xml (GET)', async () => {
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
const xml = (response.body as Buffer).toString('utf-8'); expect(body).toBeInstanceOf(Buffer);
const xml = body.toString('utf-8');
expect(xml).toContain('<groupId>'); expect(xml).toContain('<groupId>');
expect(xml).toContain('<artifactId>'); expect(xml).toContain('<artifactId>');
expect(xml).toContain('<version>1.0.0</version>'); expect(xml).toContain('<version>1.0.0</version>');
@@ -247,7 +255,8 @@ tap.test('Maven: should upload a second version and update metadata', async () =
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
const xml = (response.body as Buffer).toString('utf-8'); const metaBody = await streamToBuffer(response.body);
const xml = metaBody.toString('utf-8');
expect(xml).toContain('<version>1.0.0</version>'); expect(xml).toContain('<version>1.0.0</version>');
expect(xml).toContain('<version>2.0.0</version>'); expect(xml).toContain('<version>2.0.0</version>');
expect(xml).toContain('<latest>2.0.0</latest>'); expect(xml).toContain('<latest>2.0.0</latest>');
@@ -285,7 +294,8 @@ tap.test('Maven: should return 404 for non-existent artifact', async () => {
}); });
expect(response.status).toEqual(404); expect(response.status).toEqual(404);
expect(response.body).toHaveProperty('error'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
}); });
tap.test('Maven: should return 401 for unauthorized upload', async () => { tap.test('Maven: should return 401 for unauthorized upload', async () => {
@@ -304,7 +314,8 @@ tap.test('Maven: should return 401 for unauthorized upload', async () => {
}); });
expect(response.status).toEqual(401); expect(response.status).toEqual(401);
expect(response.body).toHaveProperty('error'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
}); });
tap.test('Maven: should reject POM upload with mismatched GAV', async () => { tap.test('Maven: should reject POM upload with mismatched GAV', async () => {
@@ -328,7 +339,8 @@ tap.test('Maven: should reject POM upload with mismatched GAV', async () => {
}); });
expect(response.status).toEqual(400); expect(response.status).toEqual(400);
expect(response.body).toHaveProperty('error'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
}); });
tap.test('Maven: should delete an artifact (DELETE)', async () => { tap.test('Maven: should delete an artifact (DELETE)', async () => {

View File

@@ -79,16 +79,10 @@ async function createHttpServer(
res.setHeader(key, value); res.setHeader(key, value);
} }
// Send body // Send body (response.body is always ReadableStream<Uint8Array> or undefined)
if (response.body) { if (response.body) {
if (Buffer.isBuffer(response.body)) { const { Readable } = await import('stream');
res.end(response.body); Readable.fromWeb(response.body).pipe(res);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
} else { } else {
res.end(); res.end();
} }
@@ -224,16 +218,16 @@ function cleanupTestDir(dir: string): void {
// ======================================================================== // ========================================================================
tap.test('NPM CLI: should setup registry and HTTP server', async () => { tap.test('NPM CLI: should setup registry and HTTP server', async () => {
// Create registry // Find available port
registry = await createTestRegistry(); registryPort = 35000;
// Create registry with correct registryUrl for CLI tests
registry = await createTestRegistry({ registryUrl: `http://localhost:${registryPort}` });
const tokens = await createTestTokens(registry); const tokens = await createTestTokens(registry);
npmToken = tokens.npmToken; npmToken = tokens.npmToken;
expect(registry).toBeInstanceOf(SmartRegistry); expect(registry).toBeInstanceOf(SmartRegistry);
expect(npmToken).toBeTypeOf('string'); expect(npmToken).toBeTypeOf('string');
// Find available port
registryPort = 35000;
const serverSetup = await createHttpServer(registry, registryPort); const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server; server = serverSetup.server;
registryUrl = serverSetup.url; registryUrl = serverSetup.url;

View File

@@ -1,5 +1,6 @@
import { expect, tap } from '@git.zone/tstest/tapbundle'; import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js'; import { SmartRegistry } from '../ts/index.js';
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
import { createTestRegistry, createTestTokens, createTestPackument } from './helpers/registry.js'; import { createTestRegistry, createTestTokens, createTestPackument } from './helpers/registry.js';
let registry: SmartRegistry; let registry: SmartRegistry;
@@ -34,8 +35,9 @@ tap.test('NPM: should handle user authentication (PUT /-/user/org.couchdb.user:{
}); });
expect(response.status).toEqual(201); expect(response.status).toEqual(201);
expect(response.body).toHaveProperty('token'); const body = await streamToJson(response.body);
expect((response.body as any).token).toBeTypeOf('string'); expect(body).toHaveProperty('token');
expect(body.token).toBeTypeOf('string');
}); });
tap.test('NPM: should publish a package (PUT /{package})', async () => { tap.test('NPM: should publish a package (PUT /{package})', async () => {
@@ -53,8 +55,9 @@ tap.test('NPM: should publish a package (PUT /{package})', async () => {
}); });
expect(response.status).toEqual(201); expect(response.status).toEqual(201);
expect(response.body).toHaveProperty('ok'); const body = await streamToJson(response.body);
expect((response.body as any).ok).toEqual(true); expect(body).toHaveProperty('ok');
expect(body.ok).toEqual(true);
}); });
tap.test('NPM: should retrieve package metadata (GET /{package})', async () => { tap.test('NPM: should retrieve package metadata (GET /{package})', async () => {
@@ -66,10 +69,11 @@ tap.test('NPM: should retrieve package metadata (GET /{package})', async () => {
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('name'); const body = await streamToJson(response.body);
expect((response.body as any).name).toEqual(testPackageName); expect(body).toHaveProperty('name');
expect((response.body as any).versions).toHaveProperty(testVersion); expect(body.name).toEqual(testPackageName);
expect((response.body as any)['dist-tags'].latest).toEqual(testVersion); expect(body.versions).toHaveProperty(testVersion);
expect(body['dist-tags'].latest).toEqual(testVersion);
}); });
tap.test('NPM: should retrieve specific version metadata (GET /{package}/{version})', async () => { tap.test('NPM: should retrieve specific version metadata (GET /{package}/{version})', async () => {
@@ -81,9 +85,10 @@ tap.test('NPM: should retrieve specific version metadata (GET /{package}/{versio
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('version'); const body = await streamToJson(response.body);
expect((response.body as any).version).toEqual(testVersion); expect(body).toHaveProperty('version');
expect((response.body as any).name).toEqual(testPackageName); expect(body.version).toEqual(testVersion);
expect(body.name).toEqual(testPackageName);
}); });
tap.test('NPM: should download tarball (GET /{package}/-/{tarball})', async () => { tap.test('NPM: should download tarball (GET /{package}/-/{tarball})', async () => {
@@ -95,8 +100,9 @@ tap.test('NPM: should download tarball (GET /{package}/-/{tarball})', async () =
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect((response.body as Buffer).toString('utf-8')).toEqual('fake tarball content'); expect(body).toBeInstanceOf(Buffer);
expect(body.toString('utf-8')).toEqual('fake tarball content');
expect(response.headers['Content-Type']).toEqual('application/octet-stream'); expect(response.headers['Content-Type']).toEqual('application/octet-stream');
}); });
@@ -127,7 +133,8 @@ tap.test('NPM: should publish a new version of the package', async () => {
}); });
expect(getResponse.status).toEqual(200); expect(getResponse.status).toEqual(200);
expect((getResponse.body as any).versions).toHaveProperty(newVersion); const getBody = await streamToJson(getResponse.body);
expect(getBody.versions).toHaveProperty(newVersion);
}); });
tap.test('NPM: should get dist-tags (GET /-/package/{pkg}/dist-tags)', async () => { tap.test('NPM: should get dist-tags (GET /-/package/{pkg}/dist-tags)', async () => {
@@ -139,8 +146,9 @@ tap.test('NPM: should get dist-tags (GET /-/package/{pkg}/dist-tags)', async ()
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('latest'); const body = await streamToJson(response.body);
expect((response.body as any).latest).toBeTypeOf('string'); expect(body).toHaveProperty('latest');
expect(body.latest).toBeTypeOf('string');
}); });
tap.test('NPM: should update dist-tag (PUT /-/package/{pkg}/dist-tags/{tag})', async () => { tap.test('NPM: should update dist-tag (PUT /-/package/{pkg}/dist-tags/{tag})', async () => {
@@ -165,7 +173,8 @@ tap.test('NPM: should update dist-tag (PUT /-/package/{pkg}/dist-tags/{tag})', a
query: {}, query: {},
}); });
expect((getResponse.body as any)['dist-tags'].beta).toEqual('1.1.0'); const getBody2 = await streamToJson(getResponse.body);
expect(getBody2['dist-tags'].beta).toEqual('1.1.0');
}); });
tap.test('NPM: should delete dist-tag (DELETE /-/package/{pkg}/dist-tags/{tag})', async () => { tap.test('NPM: should delete dist-tag (DELETE /-/package/{pkg}/dist-tags/{tag})', async () => {
@@ -188,7 +197,8 @@ tap.test('NPM: should delete dist-tag (DELETE /-/package/{pkg}/dist-tags/{tag})'
query: {}, query: {},
}); });
expect((getResponse.body as any)['dist-tags']).not.toHaveProperty('beta'); const getBody3 = await streamToJson(getResponse.body);
expect(getBody3['dist-tags']).not.toHaveProperty('beta');
}); });
tap.test('NPM: should create a new token (POST /-/npm/v1/tokens)', async () => { tap.test('NPM: should create a new token (POST /-/npm/v1/tokens)', async () => {
@@ -208,8 +218,9 @@ tap.test('NPM: should create a new token (POST /-/npm/v1/tokens)', async () => {
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('token'); const body = await streamToJson(response.body);
expect((response.body as any).readonly).toEqual(true); expect(body).toHaveProperty('token');
expect(body.readonly).toEqual(true);
}); });
tap.test('NPM: should list tokens (GET /-/npm/v1/tokens)', async () => { tap.test('NPM: should list tokens (GET /-/npm/v1/tokens)', async () => {
@@ -223,9 +234,10 @@ tap.test('NPM: should list tokens (GET /-/npm/v1/tokens)', async () => {
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('objects'); const body = await streamToJson(response.body);
expect((response.body as any).objects).toBeInstanceOf(Array); expect(body).toHaveProperty('objects');
expect((response.body as any).objects.length).toBeGreaterThan(0); expect(body.objects).toBeInstanceOf(Array);
expect(body.objects.length).toBeGreaterThan(0);
}); });
tap.test('NPM: should search packages (GET /-/v1/search)', async () => { tap.test('NPM: should search packages (GET /-/v1/search)', async () => {
@@ -240,9 +252,10 @@ tap.test('NPM: should search packages (GET /-/v1/search)', async () => {
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('objects'); const body = await streamToJson(response.body);
expect((response.body as any).objects).toBeInstanceOf(Array); expect(body).toHaveProperty('objects');
expect((response.body as any).total).toBeGreaterThan(0); expect(body.objects).toBeInstanceOf(Array);
expect(body.total).toBeGreaterThan(0);
}); });
tap.test('NPM: should search packages with specific query', async () => { tap.test('NPM: should search packages with specific query', async () => {
@@ -256,7 +269,8 @@ tap.test('NPM: should search packages with specific query', async () => {
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
const results = (response.body as any).objects; const body = await streamToJson(response.body);
const results = body.objects;
expect(results.length).toBeGreaterThan(0); expect(results.length).toBeGreaterThan(0);
expect(results[0].package.name).toEqual(testPackageName); expect(results[0].package.name).toEqual(testPackageName);
}); });
@@ -281,7 +295,8 @@ tap.test('NPM: should unpublish a specific version (DELETE /{package}/-/{version
query: {}, query: {},
}); });
expect((getResponse.body as any).versions).not.toHaveProperty(testVersion); const getBody4 = await streamToJson(getResponse.body);
expect(getBody4.versions).not.toHaveProperty(testVersion);
}); });
tap.test('NPM: should unpublish entire package (DELETE /{package}/-rev/{rev})', async () => { tap.test('NPM: should unpublish entire package (DELETE /{package}/-rev/{rev})', async () => {
@@ -316,7 +331,8 @@ tap.test('NPM: should return 404 for non-existent package', async () => {
}); });
expect(response.status).toEqual(404); expect(response.status).toEqual(404);
expect(response.body).toHaveProperty('error'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
}); });
tap.test('NPM: should return 401 for unauthorized publish', async () => { tap.test('NPM: should return 401 for unauthorized publish', async () => {
@@ -334,7 +350,8 @@ tap.test('NPM: should return 401 for unauthorized publish', async () => {
}); });
expect(response.status).toEqual(401); expect(response.status).toEqual(401);
expect(response.body).toHaveProperty('error'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
}); });
tap.test('NPM: should reject readonly token for write operations', async () => { tap.test('NPM: should reject readonly token for write operations', async () => {

View File

@@ -48,7 +48,7 @@ async function createDockerTestRegistry(port: number): Promise<SmartRegistry> {
}, },
oci: { oci: {
enabled: true, enabled: true,
basePath: '/oci', basePath: '/v2',
}, },
}; };
@@ -95,8 +95,7 @@ let testImageName: string;
* Create HTTP server wrapper around SmartRegistry * Create HTTP server wrapper around SmartRegistry
* CRITICAL: Always passes rawBody for content-addressable operations (OCI manifests/blobs) * CRITICAL: Always passes rawBody for content-addressable operations (OCI manifests/blobs)
* *
* Docker expects registry at /v2/ but SmartRegistry serves at /oci/v2/ * SmartRegistry OCI is configured with basePath '/v2' matching Docker's native /v2/ prefix.
* This wrapper rewrites paths for Docker compatibility
* *
* Also implements a simple /v2/token endpoint for Docker Bearer auth flow * Also implements a simple /v2/token endpoint for Docker Bearer auth flow
*/ */
@@ -130,10 +129,7 @@ async function createHttpServer(
// Log all requests for debugging // Log all requests for debugging
console.log(`[Registry] ${req.method} ${pathname}`); console.log(`[Registry] ${req.method} ${pathname}`);
// Docker expects /v2/ but SmartRegistry serves at /oci/v2/ // basePath is /v2 which matches Docker's native /v2/ prefix — no rewrite needed
if (pathname.startsWith('/v2')) {
pathname = '/oci' + pathname;
}
// Read raw body - ALWAYS preserve exact bytes for OCI // Read raw body - ALWAYS preserve exact bytes for OCI
const chunks: Buffer[] = []; const chunks: Buffer[] = [];
@@ -179,16 +175,10 @@ async function createHttpServer(
res.setHeader(key, value); res.setHeader(key, value);
} }
// Send body // Send body (response.body is always ReadableStream<Uint8Array> or undefined)
if (response.body) { if (response.body) {
if (Buffer.isBuffer(response.body)) { const { Readable } = await import('stream');
res.end(response.body); Readable.fromWeb(response.body).pipe(res);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
} else { } else {
res.end(); res.end();
} }
@@ -313,7 +303,7 @@ tap.test('Docker CLI: should verify server is responding', async () => {
// Give the server a moment to fully initialize // Give the server a moment to fully initialize
await new Promise(resolve => setTimeout(resolve, 500)); await new Promise(resolve => setTimeout(resolve, 500));
const response = await fetch(`${registryUrl}/oci/v2/`); const response = await fetch(`${registryUrl}/v2/`);
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
console.log('OCI v2 response:', await response.json()); console.log('OCI v2 response:', await response.json());
}); });
@@ -352,7 +342,7 @@ tap.test('Docker CLI: should push image to registry', async () => {
}); });
tap.test('Docker CLI: should verify manifest in registry via API', async () => { tap.test('Docker CLI: should verify manifest in registry via API', async () => {
const response = await fetch(`${registryUrl}/oci/v2/test-image/tags/list`, { const response = await fetch(`${registryUrl}/v2/test-image/tags/list`, {
headers: { Authorization: `Bearer ${ociToken}` }, headers: { Authorization: `Bearer ${ociToken}` },
}); });

View File

@@ -1,5 +1,6 @@
import { expect, tap } from '@git.zone/tstest/tapbundle'; import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js'; import { SmartRegistry } from '../ts/index.js';
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
import { createTestRegistry, createTestTokens, calculateDigest, createTestManifest } from './helpers/registry.js'; import { createTestRegistry, createTestTokens, calculateDigest, createTestManifest } from './helpers/registry.js';
let registry: SmartRegistry; let registry: SmartRegistry;
@@ -24,7 +25,7 @@ tap.test('OCI: should create registry instance', async () => {
tap.test('OCI: should handle version check (GET /v2/)', async () => { tap.test('OCI: should handle version check (GET /v2/)', async () => {
const response = await registry.handleRequest({ const response = await registry.handleRequest({
method: 'GET', method: 'GET',
path: '/oci/v2/', path: '/oci/',
headers: {}, headers: {},
query: {}, query: {},
}); });
@@ -36,7 +37,7 @@ tap.test('OCI: should handle version check (GET /v2/)', async () => {
tap.test('OCI: should initiate blob upload (POST /v2/{name}/blobs/uploads/)', async () => { tap.test('OCI: should initiate blob upload (POST /v2/{name}/blobs/uploads/)', async () => {
const response = await registry.handleRequest({ const response = await registry.handleRequest({
method: 'POST', method: 'POST',
path: '/oci/v2/test-repo/blobs/uploads/', path: '/oci/test-repo/blobs/uploads/',
headers: { headers: {
Authorization: `Bearer ${ociToken}`, Authorization: `Bearer ${ociToken}`,
}, },
@@ -53,7 +54,7 @@ tap.test('OCI: should upload blob in single PUT', async () => {
const response = await registry.handleRequest({ const response = await registry.handleRequest({
method: 'POST', method: 'POST',
path: '/oci/v2/test-repo/blobs/uploads/', path: '/oci/test-repo/blobs/uploads/',
headers: { headers: {
Authorization: `Bearer ${ociToken}`, Authorization: `Bearer ${ociToken}`,
}, },
@@ -73,7 +74,7 @@ tap.test('OCI: should upload config blob', async () => {
const response = await registry.handleRequest({ const response = await registry.handleRequest({
method: 'POST', method: 'POST',
path: '/oci/v2/test-repo/blobs/uploads/', path: '/oci/test-repo/blobs/uploads/',
headers: { headers: {
Authorization: `Bearer ${ociToken}`, Authorization: `Bearer ${ociToken}`,
}, },
@@ -90,7 +91,7 @@ tap.test('OCI: should upload config blob', async () => {
tap.test('OCI: should check if blob exists (HEAD /v2/{name}/blobs/{digest})', async () => { tap.test('OCI: should check if blob exists (HEAD /v2/{name}/blobs/{digest})', async () => {
const response = await registry.handleRequest({ const response = await registry.handleRequest({
method: 'HEAD', method: 'HEAD',
path: `/oci/v2/test-repo/blobs/${testBlobDigest}`, path: `/oci/test-repo/blobs/${testBlobDigest}`,
headers: { headers: {
Authorization: `Bearer ${ociToken}`, Authorization: `Bearer ${ociToken}`,
}, },
@@ -105,7 +106,7 @@ tap.test('OCI: should check if blob exists (HEAD /v2/{name}/blobs/{digest})', as
tap.test('OCI: should retrieve blob (GET /v2/{name}/blobs/{digest})', async () => { tap.test('OCI: should retrieve blob (GET /v2/{name}/blobs/{digest})', async () => {
const response = await registry.handleRequest({ const response = await registry.handleRequest({
method: 'GET', method: 'GET',
path: `/oci/v2/test-repo/blobs/${testBlobDigest}`, path: `/oci/test-repo/blobs/${testBlobDigest}`,
headers: { headers: {
Authorization: `Bearer ${ociToken}`, Authorization: `Bearer ${ociToken}`,
}, },
@@ -113,8 +114,9 @@ tap.test('OCI: should retrieve blob (GET /v2/{name}/blobs/{digest})', async () =
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect((response.body as Buffer).toString('utf-8')).toEqual('Hello from OCI test blob!'); expect(body).toBeInstanceOf(Buffer);
expect(body.toString('utf-8')).toEqual('Hello from OCI test blob!');
expect(response.headers['Docker-Content-Digest']).toEqual(testBlobDigest); expect(response.headers['Docker-Content-Digest']).toEqual(testBlobDigest);
}); });
@@ -126,7 +128,7 @@ tap.test('OCI: should upload manifest (PUT /v2/{name}/manifests/{reference})', a
const response = await registry.handleRequest({ const response = await registry.handleRequest({
method: 'PUT', method: 'PUT',
path: '/oci/v2/test-repo/manifests/v1.0.0', path: '/oci/test-repo/manifests/v1.0.0',
headers: { headers: {
Authorization: `Bearer ${ociToken}`, Authorization: `Bearer ${ociToken}`,
'Content-Type': 'application/vnd.oci.image.manifest.v1+json', 'Content-Type': 'application/vnd.oci.image.manifest.v1+json',
@@ -143,7 +145,7 @@ tap.test('OCI: should upload manifest (PUT /v2/{name}/manifests/{reference})', a
tap.test('OCI: should retrieve manifest by tag (GET /v2/{name}/manifests/{reference})', async () => { tap.test('OCI: should retrieve manifest by tag (GET /v2/{name}/manifests/{reference})', async () => {
const response = await registry.handleRequest({ const response = await registry.handleRequest({
method: 'GET', method: 'GET',
path: '/oci/v2/test-repo/manifests/v1.0.0', path: '/oci/test-repo/manifests/v1.0.0',
headers: { headers: {
Authorization: `Bearer ${ociToken}`, Authorization: `Bearer ${ociToken}`,
Accept: 'application/vnd.oci.image.manifest.v1+json', Accept: 'application/vnd.oci.image.manifest.v1+json',
@@ -152,9 +154,10 @@ tap.test('OCI: should retrieve manifest by tag (GET /v2/{name}/manifests/{refere
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
const manifest = JSON.parse((response.body as Buffer).toString('utf-8')); const manifest = JSON.parse(body.toString('utf-8'));
expect(manifest.schemaVersion).toEqual(2); expect(manifest.schemaVersion).toEqual(2);
expect(manifest.config.digest).toEqual(testConfigDigest); expect(manifest.config.digest).toEqual(testConfigDigest);
expect(manifest.layers[0].digest).toEqual(testBlobDigest); expect(manifest.layers[0].digest).toEqual(testBlobDigest);
@@ -163,7 +166,7 @@ tap.test('OCI: should retrieve manifest by tag (GET /v2/{name}/manifests/{refere
tap.test('OCI: should retrieve manifest by digest (GET /v2/{name}/manifests/{digest})', async () => { tap.test('OCI: should retrieve manifest by digest (GET /v2/{name}/manifests/{digest})', async () => {
const response = await registry.handleRequest({ const response = await registry.handleRequest({
method: 'GET', method: 'GET',
path: `/oci/v2/test-repo/manifests/${testManifestDigest}`, path: `/oci/test-repo/manifests/${testManifestDigest}`,
headers: { headers: {
Authorization: `Bearer ${ociToken}`, Authorization: `Bearer ${ociToken}`,
Accept: 'application/vnd.oci.image.manifest.v1+json', Accept: 'application/vnd.oci.image.manifest.v1+json',
@@ -178,7 +181,7 @@ tap.test('OCI: should retrieve manifest by digest (GET /v2/{name}/manifests/{dig
tap.test('OCI: should check if manifest exists (HEAD /v2/{name}/manifests/{reference})', async () => { tap.test('OCI: should check if manifest exists (HEAD /v2/{name}/manifests/{reference})', async () => {
const response = await registry.handleRequest({ const response = await registry.handleRequest({
method: 'HEAD', method: 'HEAD',
path: '/oci/v2/test-repo/manifests/v1.0.0', path: '/oci/test-repo/manifests/v1.0.0',
headers: { headers: {
Authorization: `Bearer ${ociToken}`, Authorization: `Bearer ${ociToken}`,
Accept: 'application/vnd.oci.image.manifest.v1+json', Accept: 'application/vnd.oci.image.manifest.v1+json',
@@ -193,7 +196,7 @@ tap.test('OCI: should check if manifest exists (HEAD /v2/{name}/manifests/{refer
tap.test('OCI: should list tags (GET /v2/{name}/tags/list)', async () => { tap.test('OCI: should list tags (GET /v2/{name}/tags/list)', async () => {
const response = await registry.handleRequest({ const response = await registry.handleRequest({
method: 'GET', method: 'GET',
path: '/oci/v2/test-repo/tags/list', path: '/oci/test-repo/tags/list',
headers: { headers: {
Authorization: `Bearer ${ociToken}`, Authorization: `Bearer ${ociToken}`,
}, },
@@ -201,9 +204,9 @@ tap.test('OCI: should list tags (GET /v2/{name}/tags/list)', async () => {
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('tags'); const tagList = await streamToJson(response.body);
expect(tagList).toHaveProperty('tags');
const tagList = response.body as any;
expect(tagList.name).toEqual('test-repo'); expect(tagList.name).toEqual('test-repo');
expect(tagList.tags).toBeInstanceOf(Array); expect(tagList.tags).toBeInstanceOf(Array);
expect(tagList.tags).toContain('v1.0.0'); expect(tagList.tags).toContain('v1.0.0');
@@ -212,7 +215,7 @@ tap.test('OCI: should list tags (GET /v2/{name}/tags/list)', async () => {
tap.test('OCI: should handle pagination for tag list', async () => { tap.test('OCI: should handle pagination for tag list', async () => {
const response = await registry.handleRequest({ const response = await registry.handleRequest({
method: 'GET', method: 'GET',
path: '/oci/v2/test-repo/tags/list', path: '/oci/test-repo/tags/list',
headers: { headers: {
Authorization: `Bearer ${ociToken}`, Authorization: `Bearer ${ociToken}`,
}, },
@@ -222,13 +225,14 @@ tap.test('OCI: should handle pagination for tag list', async () => {
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('tags'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('tags');
}); });
tap.test('OCI: should return 404 for non-existent blob', async () => { tap.test('OCI: should return 404 for non-existent blob', async () => {
const response = await registry.handleRequest({ const response = await registry.handleRequest({
method: 'GET', method: 'GET',
path: '/oci/v2/test-repo/blobs/sha256:0000000000000000000000000000000000000000000000000000000000000000', path: '/oci/test-repo/blobs/sha256:0000000000000000000000000000000000000000000000000000000000000000',
headers: { headers: {
Authorization: `Bearer ${ociToken}`, Authorization: `Bearer ${ociToken}`,
}, },
@@ -236,13 +240,14 @@ tap.test('OCI: should return 404 for non-existent blob', async () => {
}); });
expect(response.status).toEqual(404); expect(response.status).toEqual(404);
expect(response.body).toHaveProperty('errors'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('errors');
}); });
tap.test('OCI: should return 404 for non-existent manifest', async () => { tap.test('OCI: should return 404 for non-existent manifest', async () => {
const response = await registry.handleRequest({ const response = await registry.handleRequest({
method: 'GET', method: 'GET',
path: '/oci/v2/test-repo/manifests/non-existent-tag', path: '/oci/test-repo/manifests/non-existent-tag',
headers: { headers: {
Authorization: `Bearer ${ociToken}`, Authorization: `Bearer ${ociToken}`,
Accept: 'application/vnd.oci.image.manifest.v1+json', Accept: 'application/vnd.oci.image.manifest.v1+json',
@@ -251,13 +256,14 @@ tap.test('OCI: should return 404 for non-existent manifest', async () => {
}); });
expect(response.status).toEqual(404); expect(response.status).toEqual(404);
expect(response.body).toHaveProperty('errors'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('errors');
}); });
tap.test('OCI: should delete manifest (DELETE /v2/{name}/manifests/{digest})', async () => { tap.test('OCI: should delete manifest (DELETE /v2/{name}/manifests/{digest})', async () => {
const response = await registry.handleRequest({ const response = await registry.handleRequest({
method: 'DELETE', method: 'DELETE',
path: `/oci/v2/test-repo/manifests/${testManifestDigest}`, path: `/oci/test-repo/manifests/${testManifestDigest}`,
headers: { headers: {
Authorization: `Bearer ${ociToken}`, Authorization: `Bearer ${ociToken}`,
}, },
@@ -270,7 +276,7 @@ tap.test('OCI: should delete manifest (DELETE /v2/{name}/manifests/{digest})', a
tap.test('OCI: should delete blob (DELETE /v2/{name}/blobs/{digest})', async () => { tap.test('OCI: should delete blob (DELETE /v2/{name}/blobs/{digest})', async () => {
const response = await registry.handleRequest({ const response = await registry.handleRequest({
method: 'DELETE', method: 'DELETE',
path: `/oci/v2/test-repo/blobs/${testBlobDigest}`, path: `/oci/test-repo/blobs/${testBlobDigest}`,
headers: { headers: {
Authorization: `Bearer ${ociToken}`, Authorization: `Bearer ${ociToken}`,
}, },
@@ -283,7 +289,7 @@ tap.test('OCI: should delete blob (DELETE /v2/{name}/blobs/{digest})', async ()
tap.test('OCI: should handle unauthorized requests', async () => { tap.test('OCI: should handle unauthorized requests', async () => {
const response = await registry.handleRequest({ const response = await registry.handleRequest({
method: 'GET', method: 'GET',
path: '/oci/v2/test-repo/manifests/v1.0.0', path: '/oci/test-repo/manifests/v1.0.0',
headers: { headers: {
// No authorization header // No authorization header
}, },

View File

@@ -89,16 +89,10 @@ async function createHttpServer(
res.setHeader(key, value); res.setHeader(key, value);
} }
// Send body // Send body (response.body is always ReadableStream<Uint8Array> or undefined)
if (response.body) { if (response.body) {
if (Buffer.isBuffer(response.body)) { const { Readable } = await import('stream');
res.end(response.body); Readable.fromWeb(response.body).pipe(res);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
} else { } else {
res.end(); res.end();
} }
@@ -303,16 +297,16 @@ tap.test('PyPI CLI: should verify twine is installed', async () => {
}); });
tap.test('PyPI CLI: should setup registry and HTTP server', async () => { tap.test('PyPI CLI: should setup registry and HTTP server', async () => {
// Create registry // Use port 39000 (avoids conflicts with other tests)
registry = await createTestRegistry(); registryPort = 39000;
// Create registry with correct registryUrl for CLI tests
registry = await createTestRegistry({ registryUrl: `http://localhost:${registryPort}` });
const tokens = await createTestTokens(registry); const tokens = await createTestTokens(registry);
pypiToken = tokens.pypiToken; pypiToken = tokens.pypiToken;
expect(registry).toBeInstanceOf(SmartRegistry); expect(registry).toBeInstanceOf(SmartRegistry);
expect(pypiToken).toBeTypeOf('string'); expect(pypiToken).toBeTypeOf('string');
// Use port 39000 (avoids conflicts with other tests)
registryPort = 39000;
const serverSetup = await createHttpServer(registry, registryPort); const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server; server = serverSetup.server;
registryUrl = serverSetup.url; registryUrl = serverSetup.url;

View File

@@ -1,5 +1,6 @@
import { expect, tap } from '@git.zone/tstest/tapbundle'; import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js'; import { SmartRegistry } from '../ts/index.js';
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
import { import {
createTestRegistry, createTestRegistry,
createTestTokens, createTestTokens,
@@ -101,9 +102,10 @@ tap.test('PyPI: should retrieve Simple API root index HTML (GET /simple/)', asyn
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toStartWith('text/html'); expect(response.headers['Content-Type']).toStartWith('text/html');
expect(response.body).toBeTypeOf('string'); const body = await streamToBuffer(response.body);
const html = body.toString('utf-8');
expect(html).toBeTypeOf('string');
const html = response.body as string;
expect(html).toContain('<!DOCTYPE html>'); expect(html).toContain('<!DOCTYPE html>');
expect(html).toContain('<title>Simple Index</title>'); expect(html).toContain('<title>Simple Index</title>');
expect(html).toContain(normalizedPackageName); expect(html).toContain(normalizedPackageName);
@@ -121,9 +123,9 @@ tap.test('PyPI: should retrieve Simple API root index JSON (GET /simple/ with Ac
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/vnd.pypi.simple.v1+json'); expect(response.headers['Content-Type']).toEqual('application/vnd.pypi.simple.v1+json');
expect(response.body).toBeTypeOf('object'); const json = await streamToJson(response.body);
expect(json).toBeTypeOf('object');
const json = response.body as any;
expect(json).toHaveProperty('meta'); expect(json).toHaveProperty('meta');
expect(json).toHaveProperty('projects'); expect(json).toHaveProperty('projects');
expect(json.projects).toBeInstanceOf(Array); expect(json.projects).toBeInstanceOf(Array);
@@ -144,9 +146,10 @@ tap.test('PyPI: should retrieve Simple API package HTML (GET /simple/{package}/)
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toStartWith('text/html'); expect(response.headers['Content-Type']).toStartWith('text/html');
expect(response.body).toBeTypeOf('string'); const body = await streamToBuffer(response.body);
const html = body.toString('utf-8');
expect(html).toBeTypeOf('string');
const html = response.body as string;
expect(html).toContain('<!DOCTYPE html>'); expect(html).toContain('<!DOCTYPE html>');
expect(html).toContain(`<title>Links for ${normalizedPackageName}</title>`); expect(html).toContain(`<title>Links for ${normalizedPackageName}</title>`);
expect(html).toContain('.whl'); expect(html).toContain('.whl');
@@ -165,9 +168,9 @@ tap.test('PyPI: should retrieve Simple API package JSON (GET /simple/{package}/
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/vnd.pypi.simple.v1+json'); expect(response.headers['Content-Type']).toEqual('application/vnd.pypi.simple.v1+json');
expect(response.body).toBeTypeOf('object'); const json = await streamToJson(response.body);
expect(json).toBeTypeOf('object');
const json = response.body as any;
expect(json).toHaveProperty('meta'); expect(json).toHaveProperty('meta');
expect(json).toHaveProperty('name'); expect(json).toHaveProperty('name');
expect(json.name).toEqual(normalizedPackageName); expect(json.name).toEqual(normalizedPackageName);
@@ -187,8 +190,9 @@ tap.test('PyPI: should download wheel file (GET /pypi/packages/{package}/{filena
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect((response.body as Buffer).length).toEqual(testWheelData.length); expect(body).toBeInstanceOf(Buffer);
expect(body.length).toEqual(testWheelData.length);
expect(response.headers['Content-Type']).toEqual('application/octet-stream'); expect(response.headers['Content-Type']).toEqual('application/octet-stream');
}); });
@@ -234,7 +238,7 @@ tap.test('PyPI: should list both wheel and sdist in Simple API', async () => {
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
const json = response.body as any; const json = await streamToJson(response.body);
// PEP 691: files is an array of file objects // PEP 691: files is an array of file objects
expect(json.files.length).toEqual(2); expect(json.files.length).toEqual(2);
@@ -289,7 +293,7 @@ tap.test('PyPI: should list multiple versions in Simple API', async () => {
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
const json = response.body as any; const json = await streamToJson(response.body);
// PEP 691: files is an array of file objects // PEP 691: files is an array of file objects
expect(json.files.length).toBeGreaterThan(2); expect(json.files.length).toBeGreaterThan(2);
@@ -323,7 +327,8 @@ tap.test('PyPI: should return 404 for non-existent package', async () => {
}); });
expect(response.status).toEqual(404); expect(response.status).toEqual(404);
expect(response.body).toHaveProperty('error'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
}); });
tap.test('PyPI: should return 401 for unauthorized upload', async () => { tap.test('PyPI: should return 401 for unauthorized upload', async () => {
@@ -353,7 +358,8 @@ tap.test('PyPI: should return 401 for unauthorized upload', async () => {
}); });
expect(response.status).toEqual(401); expect(response.status).toEqual(401);
expect(response.body).toHaveProperty('error'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
}); });
tap.test('PyPI: should reject upload with mismatched hash', async () => { tap.test('PyPI: should reject upload with mismatched hash', async () => {
@@ -382,7 +388,8 @@ tap.test('PyPI: should reject upload with mismatched hash', async () => {
}); });
expect(response.status).toEqual(400); expect(response.status).toEqual(400);
expect(response.body).toHaveProperty('error'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
}); });
tap.test('PyPI: should handle package with requires-python metadata', async () => { tap.test('PyPI: should handle package with requires-python metadata', async () => {
@@ -425,7 +432,8 @@ tap.test('PyPI: should handle package with requires-python metadata', async () =
query: {}, query: {},
}); });
const html = getResponse.body as string; const getBody = await streamToBuffer(getResponse.body);
const html = getBody.toString('utf-8');
expect(html).toContain('data-requires-python'); expect(html).toContain('data-requires-python');
// Note: >= gets HTML-escaped to &gt;= in attribute values // Note: >= gets HTML-escaped to &gt;= in attribute values
expect(html).toContain('&gt;=3.8'); expect(html).toContain('&gt;=3.8');
@@ -441,9 +449,9 @@ tap.test('PyPI: should support JSON API for package metadata', async () => {
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/json'); expect(response.headers['Content-Type']).toEqual('application/json');
expect(response.body).toBeTypeOf('object'); const json = await streamToJson(response.body);
expect(json).toBeTypeOf('object');
const json = response.body as any;
expect(json).toHaveProperty('info'); expect(json).toHaveProperty('info');
expect(json.info).toHaveProperty('name'); expect(json.info).toHaveProperty('name');
expect(json.info.name).toEqual(normalizedPackageName); expect(json.info.name).toEqual(normalizedPackageName);
@@ -460,9 +468,9 @@ tap.test('PyPI: should support JSON API for specific version', async () => {
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/json'); expect(response.headers['Content-Type']).toEqual('application/json');
expect(response.body).toBeTypeOf('object'); const json = await streamToJson(response.body);
expect(json).toBeTypeOf('object');
const json = response.body as any;
expect(json).toHaveProperty('info'); expect(json).toHaveProperty('info');
expect(json.info.version).toEqual(testVersion); expect(json.info.version).toEqual(testVersion);
expect(json).toHaveProperty('urls'); expect(json).toHaveProperty('urls');

View File

@@ -79,16 +79,10 @@ async function createHttpServer(
res.setHeader(key, value); res.setHeader(key, value);
} }
// Send body // Send body (response.body is always ReadableStream<Uint8Array> or undefined)
if (response.body) { if (response.body) {
if (Buffer.isBuffer(response.body)) { const { Readable } = await import('stream');
res.end(response.body); Readable.fromWeb(response.body).pipe(res);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
} else { } else {
res.end(); res.end();
} }
@@ -194,16 +188,16 @@ function cleanupTestDir(dir: string): void {
// ======================================================================== // ========================================================================
tap.test('RubyGems CLI: should setup registry and HTTP server', async () => { tap.test('RubyGems CLI: should setup registry and HTTP server', async () => {
// Create registry // Use port 36000 (avoids npm:35000, cargo:5000 conflicts)
registry = await createTestRegistry(); registryPort = 36000;
// Create registry with correct registryUrl for CLI tests
registry = await createTestRegistry({ registryUrl: `http://localhost:${registryPort}` });
const tokens = await createTestTokens(registry); const tokens = await createTestTokens(registry);
rubygemsToken = tokens.rubygemsToken; rubygemsToken = tokens.rubygemsToken;
expect(registry).toBeInstanceOf(SmartRegistry); expect(registry).toBeInstanceOf(SmartRegistry);
expect(rubygemsToken).toBeTypeOf('string'); expect(rubygemsToken).toBeTypeOf('string');
// Use port 36000 (avoids npm:35000, cargo:5000 conflicts)
registryPort = 36000;
const serverSetup = await createHttpServer(registry, registryPort); const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server; server = serverSetup.server;
registryUrl = serverSetup.url; registryUrl = serverSetup.url;

View File

@@ -1,5 +1,6 @@
import { expect, tap } from '@git.zone/tstest/tapbundle'; import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js'; import { SmartRegistry } from '../ts/index.js';
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
import { import {
createTestRegistry, createTestRegistry,
createTestTokens, createTestTokens,
@@ -54,7 +55,8 @@ tap.test('RubyGems: should upload gem file (POST /rubygems/api/v1/gems)', async
}); });
expect(response.status).toEqual(201); expect(response.status).toEqual(201);
expect(response.body).toHaveProperty('message'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('message');
}); });
tap.test('RubyGems: should retrieve Compact Index versions file (GET /rubygems/versions)', async () => { tap.test('RubyGems: should retrieve Compact Index versions file (GET /rubygems/versions)', async () => {
@@ -67,9 +69,10 @@ tap.test('RubyGems: should retrieve Compact Index versions file (GET /rubygems/v
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('text/plain; charset=utf-8'); expect(response.headers['Content-Type']).toEqual('text/plain; charset=utf-8');
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
const content = (response.body as Buffer).toString('utf-8'); const content = body.toString('utf-8');
expect(content).toContain('created_at:'); expect(content).toContain('created_at:');
expect(content).toContain('---'); expect(content).toContain('---');
expect(content).toContain(testGemName); expect(content).toContain(testGemName);
@@ -86,9 +89,10 @@ tap.test('RubyGems: should retrieve Compact Index info file (GET /rubygems/info/
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('text/plain; charset=utf-8'); expect(response.headers['Content-Type']).toEqual('text/plain; charset=utf-8');
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
const content = (response.body as Buffer).toString('utf-8'); const content = body.toString('utf-8');
expect(content).toContain('---'); expect(content).toContain('---');
expect(content).toContain(testVersion); expect(content).toContain(testVersion);
expect(content).toContain('checksum:'); expect(content).toContain('checksum:');
@@ -104,9 +108,10 @@ tap.test('RubyGems: should retrieve Compact Index names file (GET /rubygems/name
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('text/plain; charset=utf-8'); expect(response.headers['Content-Type']).toEqual('text/plain; charset=utf-8');
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
const content = (response.body as Buffer).toString('utf-8'); const content = body.toString('utf-8');
expect(content).toContain('---'); expect(content).toContain('---');
expect(content).toContain(testGemName); expect(content).toContain(testGemName);
}); });
@@ -120,8 +125,9 @@ tap.test('RubyGems: should download gem file (GET /rubygems/gems/{gem}-{version}
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect((response.body as Buffer).length).toEqual(testGemData.length); expect(body).toBeInstanceOf(Buffer);
expect(body.length).toEqual(testGemData.length);
expect(response.headers['Content-Type']).toEqual('application/octet-stream'); expect(response.headers['Content-Type']).toEqual('application/octet-stream');
}); });
@@ -153,7 +159,8 @@ tap.test('RubyGems: should list multiple versions in Compact Index', async () =>
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
const content = (response.body as Buffer).toString('utf-8'); const body = await streamToBuffer(response.body);
const content = body.toString('utf-8');
const lines = content.split('\n'); const lines = content.split('\n');
const gemLine = lines.find(l => l.startsWith(`${testGemName} `)); const gemLine = lines.find(l => l.startsWith(`${testGemName} `));
@@ -172,7 +179,8 @@ tap.test('RubyGems: should list multiple versions in info file', async () => {
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
const content = (response.body as Buffer).toString('utf-8'); const body = await streamToBuffer(response.body);
const content = body.toString('utf-8');
expect(content).toContain('1.0.0'); expect(content).toContain('1.0.0');
expect(content).toContain('2.0.0'); expect(content).toContain('2.0.0');
}); });
@@ -203,7 +211,8 @@ tap.test('RubyGems: should support platform-specific gems', async () => {
query: {}, query: {},
}); });
const content = (versionsResponse.body as Buffer).toString('utf-8'); const versionsBody = await streamToBuffer(versionsResponse.body);
const content = versionsBody.toString('utf-8');
const lines = content.split('\n'); const lines = content.split('\n');
const gemLine = lines.find(l => l.startsWith(`${testGemName} `)); const gemLine = lines.find(l => l.startsWith(`${testGemName} `));
@@ -224,8 +233,9 @@ tap.test('RubyGems: should yank a gem version (DELETE /rubygems/api/v1/gems/yank
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('message'); const body = await streamToJson(response.body);
expect((response.body as any).message).toContain('yanked'); expect(body).toHaveProperty('message');
expect(body.message).toContain('yanked');
}); });
tap.test('RubyGems: should mark yanked version in Compact Index', async () => { tap.test('RubyGems: should mark yanked version in Compact Index', async () => {
@@ -238,7 +248,8 @@ tap.test('RubyGems: should mark yanked version in Compact Index', async () => {
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
const content = (response.body as Buffer).toString('utf-8'); const body = await streamToBuffer(response.body);
const content = body.toString('utf-8');
const lines = content.split('\n'); const lines = content.split('\n');
const gemLine = lines.find(l => l.startsWith(`${testGemName} `)); const gemLine = lines.find(l => l.startsWith(`${testGemName} `));
@@ -256,7 +267,8 @@ tap.test('RubyGems: should still allow downloading yanked gem', async () => {
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
}); });
tap.test('RubyGems: should unyank a gem version (PUT /rubygems/api/v1/gems/unyank)', async () => { tap.test('RubyGems: should unyank a gem version (PUT /rubygems/api/v1/gems/unyank)', async () => {
@@ -273,8 +285,9 @@ tap.test('RubyGems: should unyank a gem version (PUT /rubygems/api/v1/gems/unyan
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('message'); const body = await streamToJson(response.body);
expect((response.body as any).message).toContain('unyanked'); expect(body).toHaveProperty('message');
expect(body.message).toContain('unyanked');
}); });
tap.test('RubyGems: should remove yank marker after unyank', async () => { tap.test('RubyGems: should remove yank marker after unyank', async () => {
@@ -287,7 +300,8 @@ tap.test('RubyGems: should remove yank marker after unyank', async () => {
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
const content = (response.body as Buffer).toString('utf-8'); const body = await streamToBuffer(response.body);
const content = body.toString('utf-8');
const lines = content.split('\n'); const lines = content.split('\n');
const gemLine = lines.find(l => l.startsWith(`${testGemName} `)); const gemLine = lines.find(l => l.startsWith(`${testGemName} `));
@@ -309,9 +323,9 @@ tap.test('RubyGems: should retrieve versions JSON (GET /rubygems/api/v1/versions
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/json'); expect(response.headers['Content-Type']).toEqual('application/json');
expect(response.body).toBeTypeOf('object'); const json = await streamToJson(response.body);
expect(json).toBeTypeOf('object');
const json = response.body as any;
expect(json).toHaveProperty('name'); expect(json).toHaveProperty('name');
expect(json.name).toEqual(testGemName); expect(json.name).toEqual(testGemName);
expect(json).toHaveProperty('versions'); expect(json).toHaveProperty('versions');
@@ -331,9 +345,9 @@ tap.test('RubyGems: should retrieve dependencies JSON (GET /rubygems/api/v1/depe
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/json'); expect(response.headers['Content-Type']).toEqual('application/json');
expect(response.body).toBeTypeOf('object'); const json = await streamToJson(response.body);
expect(json).toBeTypeOf('object');
const json = response.body as any;
expect(Array.isArray(json)).toEqual(true); expect(Array.isArray(json)).toEqual(true);
}); });
@@ -346,7 +360,8 @@ tap.test('RubyGems: should retrieve gem spec (GET /rubygems/quick/Marshal.4.8/{g
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
}); });
tap.test('RubyGems: should support latest specs endpoint (GET /rubygems/latest_specs.4.8.gz)', async () => { tap.test('RubyGems: should support latest specs endpoint (GET /rubygems/latest_specs.4.8.gz)', async () => {
@@ -359,7 +374,8 @@ tap.test('RubyGems: should support latest specs endpoint (GET /rubygems/latest_s
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/octet-stream'); expect(response.headers['Content-Type']).toEqual('application/octet-stream');
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
}); });
tap.test('RubyGems: should support specs endpoint (GET /rubygems/specs.4.8.gz)', async () => { tap.test('RubyGems: should support specs endpoint (GET /rubygems/specs.4.8.gz)', async () => {
@@ -372,7 +388,8 @@ tap.test('RubyGems: should support specs endpoint (GET /rubygems/specs.4.8.gz)',
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/octet-stream'); expect(response.headers['Content-Type']).toEqual('application/octet-stream');
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
}); });
tap.test('RubyGems: should return 404 for non-existent gem', async () => { tap.test('RubyGems: should return 404 for non-existent gem', async () => {
@@ -384,7 +401,8 @@ tap.test('RubyGems: should return 404 for non-existent gem', async () => {
}); });
expect(response.status).toEqual(404); expect(response.status).toEqual(404);
expect(response.body).toHaveProperty('error'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
}); });
tap.test('RubyGems: should return 401 for unauthorized upload', async () => { tap.test('RubyGems: should return 401 for unauthorized upload', async () => {
@@ -402,7 +420,8 @@ tap.test('RubyGems: should return 401 for unauthorized upload', async () => {
}); });
expect(response.status).toEqual(401); expect(response.status).toEqual(401);
expect(response.body).toHaveProperty('error'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
}); });
tap.test('RubyGems: should return 401 for unauthorized yank', async () => { tap.test('RubyGems: should return 401 for unauthorized yank', async () => {
@@ -419,7 +438,8 @@ tap.test('RubyGems: should return 401 for unauthorized yank', async () => {
}); });
expect(response.status).toEqual(401); expect(response.status).toEqual(401);
expect(response.body).toHaveProperty('error'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
}); });
tap.test('RubyGems: should handle gem with dependencies', async () => { tap.test('RubyGems: should handle gem with dependencies', async () => {
@@ -450,7 +470,8 @@ tap.test('RubyGems: should handle gem with dependencies', async () => {
expect(infoResponse.status).toEqual(200); expect(infoResponse.status).toEqual(200);
const content = (infoResponse.body as Buffer).toString('utf-8'); const infoBody = await streamToBuffer(infoResponse.body);
const content = infoBody.toString('utf-8');
expect(content).toContain('checksum:'); expect(content).toContain('checksum:');
}); });

506
test/test.storage.hooks.ts Normal file
View File

@@ -0,0 +1,506 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import * as qenv from '@push.rocks/qenv';
import { RegistryStorage } from '../ts/core/classes.registrystorage.js';
import type { IStorageConfig } from '../ts/core/interfaces.core.js';
import type { IStorageHooks, IStorageHookContext } from '../ts/core/interfaces.storage.js';
import { createTrackingHooks, createQuotaHooks, generateTestRunId } from './helpers/registry.js';
const testQenv = new qenv.Qenv('./', './.nogit');
// ============================================================================
// Test State
// ============================================================================
let storage: RegistryStorage;
let storageConfig: IStorageConfig;
let testRunId: string;
// ============================================================================
// Setup
// ============================================================================
tap.test('setup: should create storage config', async () => {
testRunId = generateTestRunId();
const s3AccessKey = await testQenv.getEnvVarOnDemand('S3_ACCESSKEY');
const s3SecretKey = await testQenv.getEnvVarOnDemand('S3_SECRETKEY');
const s3Endpoint = await testQenv.getEnvVarOnDemand('S3_ENDPOINT');
const s3Port = await testQenv.getEnvVarOnDemand('S3_PORT');
storageConfig = {
accessKey: s3AccessKey || 'minioadmin',
accessSecret: s3SecretKey || 'minioadmin',
endpoint: s3Endpoint || 'localhost',
port: parseInt(s3Port || '9000', 10),
useSsl: false,
region: 'us-east-1',
bucketName: `test-hooks-${testRunId}`,
};
expect(storageConfig.bucketName).toBeTruthy();
});
// ============================================================================
// beforePut Hook Tests
// ============================================================================
tap.test('beforePut: should be called before storage', async () => {
const tracker = createTrackingHooks();
storage = new RegistryStorage(storageConfig, tracker.hooks);
await storage.init();
// Set context and put object
storage.setContext({
protocol: 'npm',
actor: { userId: 'testuser' },
metadata: { packageName: 'test-package' },
});
await storage.putObject('test/beforeput-called.txt', Buffer.from('test data'));
storage.clearContext();
// Verify beforePut was called
const beforePutCalls = tracker.calls.filter(c => c.method === 'beforePut');
expect(beforePutCalls.length).toEqual(1);
expect(beforePutCalls[0].context.operation).toEqual('put');
expect(beforePutCalls[0].context.key).toEqual('test/beforeput-called.txt');
expect(beforePutCalls[0].context.protocol).toEqual('npm');
});
tap.test('beforePut: returning {allowed: false} should block storage', async () => {
const tracker = createTrackingHooks({ beforePutAllowed: false });
const blockingStorage = new RegistryStorage(storageConfig, tracker.hooks);
await blockingStorage.init();
blockingStorage.setContext({
protocol: 'npm',
actor: { userId: 'testuser' },
});
let errorThrown = false;
try {
await blockingStorage.putObject('test/should-not-exist.txt', Buffer.from('blocked data'));
} catch (error) {
errorThrown = true;
expect((error as Error).message).toContain('Blocked by test');
}
blockingStorage.clearContext();
expect(errorThrown).toBeTrue();
// Verify object was NOT stored
const result = await blockingStorage.getObject('test/should-not-exist.txt');
expect(result).toBeNull();
});
// ============================================================================
// afterPut Hook Tests
// ============================================================================
tap.test('afterPut: should be called after successful storage', async () => {
const tracker = createTrackingHooks();
const trackedStorage = new RegistryStorage(storageConfig, tracker.hooks);
await trackedStorage.init();
trackedStorage.setContext({
protocol: 'maven',
actor: { userId: 'maven-user' },
});
await trackedStorage.putObject('test/afterput-test.txt', Buffer.from('after put test'));
trackedStorage.clearContext();
// Give async hook time to complete
await new Promise(resolve => setTimeout(resolve, 100));
const afterPutCalls = tracker.calls.filter(c => c.method === 'afterPut');
expect(afterPutCalls.length).toEqual(1);
expect(afterPutCalls[0].context.operation).toEqual('put');
});
tap.test('afterPut: should receive correct metadata (size, key, protocol)', async () => {
const tracker = createTrackingHooks();
const metadataStorage = new RegistryStorage(storageConfig, tracker.hooks);
await metadataStorage.init();
const testData = Buffer.from('metadata test data - some content here');
metadataStorage.setContext({
protocol: 'cargo',
actor: { userId: 'cargo-user', ip: '192.168.1.100' },
metadata: { packageName: 'my-crate', version: '1.0.0' },
});
await metadataStorage.putObject('test/metadata-test.txt', testData);
metadataStorage.clearContext();
await new Promise(resolve => setTimeout(resolve, 100));
const afterPutCalls = tracker.calls.filter(c => c.method === 'afterPut');
expect(afterPutCalls.length).toBeGreaterThanOrEqual(1);
const call = afterPutCalls[afterPutCalls.length - 1];
expect(call.context.metadata?.size).toEqual(testData.length);
expect(call.context.key).toEqual('test/metadata-test.txt');
expect(call.context.protocol).toEqual('cargo');
expect(call.context.actor?.userId).toEqual('cargo-user');
expect(call.context.actor?.ip).toEqual('192.168.1.100');
});
// ============================================================================
// beforeDelete Hook Tests
// ============================================================================
tap.test('beforeDelete: should be called before deletion', async () => {
const tracker = createTrackingHooks();
const deleteStorage = new RegistryStorage(storageConfig, tracker.hooks);
await deleteStorage.init();
// First, store an object
deleteStorage.setContext({ protocol: 'npm' });
await deleteStorage.putObject('test/to-delete.txt', Buffer.from('delete me'));
// Now delete it
await deleteStorage.deleteObject('test/to-delete.txt');
deleteStorage.clearContext();
const beforeDeleteCalls = tracker.calls.filter(c => c.method === 'beforeDelete');
expect(beforeDeleteCalls.length).toEqual(1);
expect(beforeDeleteCalls[0].context.operation).toEqual('delete');
expect(beforeDeleteCalls[0].context.key).toEqual('test/to-delete.txt');
});
tap.test('beforeDelete: returning {allowed: false} should block deletion', async () => {
const tracker = createTrackingHooks({ beforeDeleteAllowed: false });
const protectedStorage = new RegistryStorage(storageConfig, tracker.hooks);
await protectedStorage.init();
// First store an object
protectedStorage.setContext({ protocol: 'npm' });
await protectedStorage.putObject('test/protected.txt', Buffer.from('protected data'));
// Try to delete - should be blocked
let errorThrown = false;
try {
await protectedStorage.deleteObject('test/protected.txt');
} catch (error) {
errorThrown = true;
expect((error as Error).message).toContain('Blocked by test');
}
protectedStorage.clearContext();
expect(errorThrown).toBeTrue();
// Verify object still exists
const result = await protectedStorage.getObject('test/protected.txt');
expect(result).toBeTruthy();
});
// ============================================================================
// afterDelete Hook Tests
// ============================================================================
tap.test('afterDelete: should be called after successful deletion', async () => {
const tracker = createTrackingHooks();
const afterDeleteStorage = new RegistryStorage(storageConfig, tracker.hooks);
await afterDeleteStorage.init();
afterDeleteStorage.setContext({ protocol: 'pypi' });
await afterDeleteStorage.putObject('test/delete-tracked.txt', Buffer.from('to be deleted'));
await afterDeleteStorage.deleteObject('test/delete-tracked.txt');
afterDeleteStorage.clearContext();
await new Promise(resolve => setTimeout(resolve, 100));
const afterDeleteCalls = tracker.calls.filter(c => c.method === 'afterDelete');
expect(afterDeleteCalls.length).toEqual(1);
expect(afterDeleteCalls[0].context.operation).toEqual('delete');
});
// ============================================================================
// afterGet Hook Tests
// ============================================================================
tap.test('afterGet: should be called after reading object', async () => {
const tracker = createTrackingHooks();
const getStorage = new RegistryStorage(storageConfig, tracker.hooks);
await getStorage.init();
// Store an object first
getStorage.setContext({ protocol: 'rubygems' });
await getStorage.putObject('test/read-test.txt', Buffer.from('read me'));
// Clear calls to focus on the get
tracker.calls.length = 0;
// Read the object
const data = await getStorage.getObject('test/read-test.txt');
getStorage.clearContext();
await new Promise(resolve => setTimeout(resolve, 100));
expect(data).toBeTruthy();
expect(data!.toString()).toEqual('read me');
const afterGetCalls = tracker.calls.filter(c => c.method === 'afterGet');
expect(afterGetCalls.length).toEqual(1);
expect(afterGetCalls[0].context.operation).toEqual('get');
});
// ============================================================================
// Context Tests
// ============================================================================
tap.test('context: hooks should receive actor information', async () => {
const tracker = createTrackingHooks();
const actorStorage = new RegistryStorage(storageConfig, tracker.hooks);
await actorStorage.init();
actorStorage.setContext({
protocol: 'composer',
actor: {
userId: 'user-123',
tokenId: 'token-abc',
ip: '10.0.0.1',
userAgent: 'composer/2.0',
orgId: 'org-456',
sessionId: 'session-xyz',
},
});
await actorStorage.putObject('test/actor-test.txt', Buffer.from('actor test'));
actorStorage.clearContext();
const beforePutCall = tracker.calls.find(c => c.method === 'beforePut');
expect(beforePutCall).toBeTruthy();
expect(beforePutCall!.context.actor?.userId).toEqual('user-123');
expect(beforePutCall!.context.actor?.tokenId).toEqual('token-abc');
expect(beforePutCall!.context.actor?.ip).toEqual('10.0.0.1');
expect(beforePutCall!.context.actor?.userAgent).toEqual('composer/2.0');
expect(beforePutCall!.context.actor?.orgId).toEqual('org-456');
expect(beforePutCall!.context.actor?.sessionId).toEqual('session-xyz');
});
tap.test('withContext: should set and clear context correctly', async () => {
const tracker = createTrackingHooks();
const contextStorage = new RegistryStorage(storageConfig, tracker.hooks);
await contextStorage.init();
// Use withContext to ensure automatic cleanup
await contextStorage.withContext(
{
protocol: 'oci',
actor: { userId: 'oci-user' },
},
async () => {
await contextStorage.putObject('test/with-context.txt', Buffer.from('context managed'));
}
);
const call = tracker.calls.find(c => c.method === 'beforePut');
expect(call).toBeTruthy();
expect(call!.context.protocol).toEqual('oci');
expect(call!.context.actor?.userId).toEqual('oci-user');
});
tap.test('withContext: should clear context even on error', async () => {
const tracker = createTrackingHooks({ beforePutAllowed: false });
const errorStorage = new RegistryStorage(storageConfig, tracker.hooks);
await errorStorage.init();
let errorThrown = false;
try {
await errorStorage.withContext(
{
protocol: 'npm',
actor: { userId: 'error-user' },
},
async () => {
await errorStorage.putObject('test/error-context.txt', Buffer.from('will fail'));
}
);
} catch {
errorThrown = true;
}
expect(errorThrown).toBeTrue();
// Verify context was cleared - next operation without context should work
// (hooks won't be called without context)
tracker.hooks.beforePut = async () => ({ allowed: true });
await errorStorage.putObject('test/after-error.txt', Buffer.from('ok'));
});
// ============================================================================
// Graceful Degradation Tests
// ============================================================================
tap.test('graceful: hooks should not fail the operation if afterPut throws', async () => {
const tracker = createTrackingHooks({ throwOnAfterPut: true });
const gracefulStorage = new RegistryStorage(storageConfig, tracker.hooks);
await gracefulStorage.init();
gracefulStorage.setContext({ protocol: 'npm' });
// This should NOT throw even though afterPut throws
let errorThrown = false;
try {
await gracefulStorage.putObject('test/graceful-afterput.txt', Buffer.from('should succeed'));
} catch {
errorThrown = true;
}
gracefulStorage.clearContext();
expect(errorThrown).toBeFalse();
// Verify object was stored
const data = await gracefulStorage.getObject('test/graceful-afterput.txt');
expect(data).toBeTruthy();
});
tap.test('graceful: hooks should not fail the operation if afterGet throws', async () => {
const tracker = createTrackingHooks({ throwOnAfterGet: true });
const gracefulGetStorage = new RegistryStorage(storageConfig, tracker.hooks);
await gracefulGetStorage.init();
// Store first
gracefulGetStorage.setContext({ protocol: 'maven' });
await gracefulGetStorage.putObject('test/graceful-afterget.txt', Buffer.from('read me gracefully'));
// Read should succeed even though afterGet throws
let errorThrown = false;
try {
const data = await gracefulGetStorage.getObject('test/graceful-afterget.txt');
expect(data).toBeTruthy();
} catch {
errorThrown = true;
}
gracefulGetStorage.clearContext();
expect(errorThrown).toBeFalse();
});
// ============================================================================
// Quota Hooks Tests
// ============================================================================
tap.test('quota: should block storage when quota exceeded', async () => {
const maxSize = 100; // 100 bytes max
const quotaTracker = createQuotaHooks(maxSize);
const quotaStorage = new RegistryStorage(storageConfig, quotaTracker.hooks);
await quotaStorage.init();
quotaStorage.setContext({
protocol: 'npm',
actor: { userId: 'quota-user' },
});
// Store 50 bytes - should succeed
await quotaStorage.putObject('test/quota-1.txt', Buffer.from('x'.repeat(50)));
expect(quotaTracker.currentUsage.bytes).toEqual(50);
// Try to store 60 more bytes - should fail (total would be 110)
let errorThrown = false;
try {
await quotaStorage.putObject('test/quota-2.txt', Buffer.from('x'.repeat(60)));
} catch (error) {
errorThrown = true;
expect((error as Error).message).toContain('Quota exceeded');
}
quotaStorage.clearContext();
expect(errorThrown).toBeTrue();
expect(quotaTracker.currentUsage.bytes).toEqual(50); // Still 50, not 110
});
tap.test('quota: should update usage after delete', async () => {
const maxSize = 200;
const quotaTracker = createQuotaHooks(maxSize);
const quotaDelStorage = new RegistryStorage(storageConfig, quotaTracker.hooks);
await quotaDelStorage.init();
quotaDelStorage.setContext({
protocol: 'npm',
metadata: { size: 75 },
});
// Store and track
await quotaDelStorage.putObject('test/quota-del.txt', Buffer.from('x'.repeat(75)));
expect(quotaTracker.currentUsage.bytes).toEqual(75);
// Delete and verify usage decreases
await quotaDelStorage.deleteObject('test/quota-del.txt');
await new Promise(resolve => setTimeout(resolve, 100));
quotaDelStorage.clearContext();
// Usage should be reduced (though exact value depends on metadata)
expect(quotaTracker.currentUsage.bytes).toBeLessThanOrEqual(75);
});
// ============================================================================
// setHooks Tests
// ============================================================================
tap.test('setHooks: should allow setting hooks after construction', async () => {
const lateStorage = new RegistryStorage(storageConfig);
await lateStorage.init();
// Initially no hooks
await lateStorage.putObject('test/no-hooks.txt', Buffer.from('no hooks yet'));
// Add hooks later
const tracker = createTrackingHooks();
lateStorage.setHooks(tracker.hooks);
lateStorage.setContext({ protocol: 'npm' });
await lateStorage.putObject('test/with-late-hooks.txt', Buffer.from('now with hooks'));
lateStorage.clearContext();
const beforePutCalls = tracker.calls.filter(c => c.method === 'beforePut');
expect(beforePutCalls.length).toEqual(1);
});
// ============================================================================
// Cleanup
// ============================================================================
tap.test('cleanup: should clean up test bucket', async () => {
if (storage) {
// Clean up test objects
const prefixes = ['test/'];
for (const prefix of prefixes) {
try {
const objects = await storage.listObjects(prefix);
for (const obj of objects) {
await storage.deleteObject(obj);
}
} catch {
// Ignore cleanup errors
}
}
}
});
export default tap.start();

View File

@@ -1,5 +1,6 @@
import { expect, tap } from '@git.zone/tstest/tapbundle'; import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js'; import { SmartRegistry } from '../ts/index.js';
import { streamToJson } from '../ts/core/helpers.stream.js';
import { createTestRegistry, createTestTokens } from './helpers/registry.js'; import { createTestRegistry, createTestTokens } from './helpers/registry.js';
let registry: SmartRegistry; let registry: SmartRegistry;
@@ -54,8 +55,9 @@ tap.test('Integration: should return 404 for unknown paths', async () => {
}); });
expect(response.status).toEqual(404); expect(response.status).toEqual(404);
expect(response.body).toHaveProperty('error'); const body = await streamToJson(response.body);
expect((response.body as any).error).toEqual('NOT_FOUND'); expect(body).toHaveProperty('error');
expect(body.error).toEqual('NOT_FOUND');
}); });
tap.test('Integration: should create and validate tokens', async () => { tap.test('Integration: should create and validate tokens', async () => {

View File

@@ -0,0 +1,598 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import * as qenv from '@push.rocks/qenv';
import * as smartbucket from '@push.rocks/smartbucket';
import { UpstreamCache } from '../ts/upstream/classes.upstreamcache.js';
import type { IUpstreamFetchContext, IUpstreamCacheConfig } from '../ts/upstream/interfaces.upstream.js';
import type { IStorageBackend } from '../ts/core/interfaces.core.js';
import { generateTestRunId } from './helpers/registry.js';
const testQenv = new qenv.Qenv('./', './.nogit');
// ============================================================================
// Test State
// ============================================================================
let cache: UpstreamCache;
let storageBackend: IStorageBackend;
let s3Bucket: smartbucket.Bucket;
let smartBucket: smartbucket.SmartBucket;
let testRunId: string;
let bucketName: string;
// ============================================================================
// Helper Functions
// ============================================================================
function createFetchContext(overrides?: Partial<IUpstreamFetchContext>): IUpstreamFetchContext {
// Use resource name as path to ensure unique cache keys
const resource = overrides?.resource || 'lodash';
return {
protocol: 'npm',
resource,
resourceType: 'packument',
path: `/${resource}`,
method: 'GET',
headers: {},
query: {},
...overrides,
};
}
// ============================================================================
// Setup
// ============================================================================
tap.test('setup: should create S3 storage backend', async () => {
testRunId = generateTestRunId();
bucketName = `test-ucache-${testRunId.substring(0, 8)}`;
const s3AccessKey = await testQenv.getEnvVarOnDemand('S3_ACCESSKEY');
const s3SecretKey = await testQenv.getEnvVarOnDemand('S3_SECRETKEY');
const s3Endpoint = await testQenv.getEnvVarOnDemand('S3_ENDPOINT');
const s3Port = await testQenv.getEnvVarOnDemand('S3_PORT');
smartBucket = new smartbucket.SmartBucket({
accessKey: s3AccessKey || 'minioadmin',
accessSecret: s3SecretKey || 'minioadmin',
endpoint: s3Endpoint || 'localhost',
port: parseInt(s3Port || '9000', 10),
useSsl: false,
});
s3Bucket = await smartBucket.createBucket(bucketName);
// Create storage backend adapter
storageBackend = {
getObject: async (key: string): Promise<Buffer | null> => {
try {
// fastGet returns Buffer directly (or undefined if not found)
const data = await s3Bucket.fastGet({ path: key });
if (!data) {
return null;
}
return data;
} catch (error) {
// fastGet throws if object doesn't exist
return null;
}
},
putObject: async (key: string, data: Buffer): Promise<void> => {
await s3Bucket.fastPut({ path: key, contents: data, overwrite: true });
},
deleteObject: async (key: string): Promise<void> => {
await s3Bucket.fastRemove({ path: key });
},
listObjects: async (prefix: string): Promise<string[]> => {
const paths: string[] = [];
for await (const path of s3Bucket.listAllObjects(prefix)) {
paths.push(path);
}
return paths;
},
};
expect(storageBackend).toBeTruthy();
});
tap.test('setup: should create UpstreamCache with S3 storage', async () => {
cache = new UpstreamCache(
{ enabled: true, defaultTtlSeconds: 300 },
10000,
storageBackend
);
expect(cache.isEnabled()).toBeTrue();
expect(cache.hasStorage()).toBeTrue();
});
// ============================================================================
// Basic Cache Operations
// ============================================================================
tap.test('cache: should store cache entry in S3', async () => {
const context = createFetchContext({ resource: 'store-test' });
const testData = Buffer.from(JSON.stringify({ name: 'store-test', version: '1.0.0' }));
const upstreamUrl = 'https://registry.npmjs.org';
await cache.set(
context,
testData,
'application/json',
{ 'etag': '"abc123"' },
'npmjs',
upstreamUrl
);
// Verify in S3
const stats = cache.getStats();
expect(stats.totalEntries).toBeGreaterThanOrEqual(1);
});
tap.test('cache: should retrieve cache entry from S3', async () => {
const context = createFetchContext({ resource: 'retrieve-test' });
const testData = Buffer.from('retrieve test data');
const upstreamUrl = 'https://registry.npmjs.org';
await cache.set(
context,
testData,
'application/octet-stream',
{},
'npmjs',
upstreamUrl
);
const entry = await cache.get(context, upstreamUrl);
expect(entry).toBeTruthy();
expect(entry!.data.toString()).toEqual('retrieve test data');
expect(entry!.contentType).toEqual('application/octet-stream');
expect(entry!.upstreamId).toEqual('npmjs');
});
// ============================================================================
// Multi-Upstream URL Tests
// ============================================================================
tap.test('cache: should include upstream URL in cache path', async () => {
const context = createFetchContext({ resource: 'url-path-test' });
const testData = Buffer.from('url path test');
await cache.set(
context,
testData,
'text/plain',
{},
'npmjs',
'https://registry.npmjs.org'
);
// The cache key should include the escaped URL
const entry = await cache.get(context, 'https://registry.npmjs.org');
expect(entry).toBeTruthy();
expect(entry!.data.toString()).toEqual('url path test');
});
tap.test('cache: should handle multiple upstreams with different URLs', async () => {
const context = createFetchContext({ resource: '@company/private-pkg' });
// Store from private upstream
const privateData = Buffer.from('private package data');
await cache.set(
context,
privateData,
'application/json',
{},
'private-npm',
'https://npm.company.com'
);
// Store from public upstream (same resource name, different upstream)
const publicData = Buffer.from('public package data');
await cache.set(
context,
publicData,
'application/json',
{},
'public-npm',
'https://registry.npmjs.org'
);
// Retrieve from private - should get private data
const privateEntry = await cache.get(context, 'https://npm.company.com');
expect(privateEntry).toBeTruthy();
expect(privateEntry!.data.toString()).toEqual('private package data');
expect(privateEntry!.upstreamId).toEqual('private-npm');
// Retrieve from public - should get public data
const publicEntry = await cache.get(context, 'https://registry.npmjs.org');
expect(publicEntry).toBeTruthy();
expect(publicEntry!.data.toString()).toEqual('public package data');
expect(publicEntry!.upstreamId).toEqual('public-npm');
});
// ============================================================================
// TTL and Expiration Tests
// ============================================================================
tap.test('cache: should respect TTL expiration', async () => {
// Create cache with very short TTL
const shortTtlCache = new UpstreamCache(
{
enabled: true,
defaultTtlSeconds: 1, // 1 second TTL
staleWhileRevalidate: false,
staleMaxAgeSeconds: 0,
immutableTtlSeconds: 1,
negativeCacheTtlSeconds: 1,
},
1000,
storageBackend
);
const context = createFetchContext({ resource: 'ttl-test' });
const testData = Buffer.from('expires soon');
await shortTtlCache.set(
context,
testData,
'text/plain',
{},
'test-upstream',
'https://test.example.com'
);
// Should exist immediately
let entry = await shortTtlCache.get(context, 'https://test.example.com');
expect(entry).toBeTruthy();
// Wait for expiration
await new Promise(resolve => setTimeout(resolve, 1500));
// Should be expired now
entry = await shortTtlCache.get(context, 'https://test.example.com');
expect(entry).toBeNull();
shortTtlCache.stop();
});
tap.test('cache: should serve stale content during stale-while-revalidate window', async () => {
const staleCache = new UpstreamCache(
{
enabled: true,
defaultTtlSeconds: 1, // 1 second fresh
staleWhileRevalidate: true,
staleMaxAgeSeconds: 60, // 60 seconds stale window
immutableTtlSeconds: 1,
negativeCacheTtlSeconds: 1,
},
1000,
storageBackend
);
const context = createFetchContext({ resource: 'stale-test' });
const testData = Buffer.from('stale but usable');
await staleCache.set(
context,
testData,
'text/plain',
{},
'stale-upstream',
'https://stale.example.com'
);
// Wait for fresh period to expire
await new Promise(resolve => setTimeout(resolve, 1500));
// Should still be available but marked as stale
const entry = await staleCache.get(context, 'https://stale.example.com');
expect(entry).toBeTruthy();
expect(entry!.stale).toBeTrue();
expect(entry!.data.toString()).toEqual('stale but usable');
staleCache.stop();
});
tap.test('cache: should reject content past stale deadline', async () => {
const veryShortCache = new UpstreamCache(
{
enabled: true,
defaultTtlSeconds: 1,
staleWhileRevalidate: true,
staleMaxAgeSeconds: 1, // Only 1 second stale window
immutableTtlSeconds: 1,
negativeCacheTtlSeconds: 1,
},
1000,
storageBackend
);
const context = createFetchContext({ resource: 'very-stale-test' });
await veryShortCache.set(
context,
Buffer.from('will expire completely'),
'text/plain',
{},
'short-upstream',
'https://short.example.com'
);
// Wait for both fresh AND stale periods to expire
await new Promise(resolve => setTimeout(resolve, 2500));
const entry = await veryShortCache.get(context, 'https://short.example.com');
expect(entry).toBeNull();
veryShortCache.stop();
});
// ============================================================================
// Negative Cache Tests
// ============================================================================
tap.test('cache: should store negative cache entries (404)', async () => {
const context = createFetchContext({ resource: 'not-found-pkg' });
const upstreamUrl = 'https://registry.npmjs.org';
await cache.setNegative(context, 'npmjs', upstreamUrl);
const hasNegative = await cache.hasNegative(context, upstreamUrl);
expect(hasNegative).toBeTrue();
});
tap.test('cache: should retrieve negative cache entries', async () => {
const context = createFetchContext({ resource: 'negative-retrieve-test' });
const upstreamUrl = 'https://registry.npmjs.org';
await cache.setNegative(context, 'npmjs', upstreamUrl);
const entry = await cache.get(context, upstreamUrl);
expect(entry).toBeTruthy();
expect(entry!.data.length).toEqual(0); // Empty buffer indicates 404
});
// ============================================================================
// Eviction Tests
// ============================================================================
tap.test('cache: should evict oldest entries when memory limit reached', async () => {
// Create cache with very small limit
const smallCache = new UpstreamCache(
{ enabled: true, defaultTtlSeconds: 300 },
5, // Only 5 entries
storageBackend
);
// Add 10 entries
for (let i = 0; i < 10; i++) {
const context = createFetchContext({ resource: `evict-test-${i}` });
await smallCache.set(
context,
Buffer.from(`data ${i}`),
'text/plain',
{},
'evict-upstream',
'https://evict.example.com'
);
}
const stats = smallCache.getStats();
// Should have evicted some entries
expect(stats.totalEntries).toBeLessThanOrEqual(5);
smallCache.stop();
});
// ============================================================================
// Query Parameter Tests
// ============================================================================
tap.test('cache: cache key should include query parameters', async () => {
const context1 = createFetchContext({
resource: 'query-test',
query: { version: '1.0.0' },
});
const context2 = createFetchContext({
resource: 'query-test',
query: { version: '2.0.0' },
});
const upstreamUrl = 'https://registry.npmjs.org';
// Store with v1 query
await cache.set(
context1,
Buffer.from('version 1 data'),
'text/plain',
{},
'query-upstream',
upstreamUrl
);
// Store with v2 query
await cache.set(
context2,
Buffer.from('version 2 data'),
'text/plain',
{},
'query-upstream',
upstreamUrl
);
// Retrieve v1 - should get v1 data
const entry1 = await cache.get(context1, upstreamUrl);
expect(entry1).toBeTruthy();
expect(entry1!.data.toString()).toEqual('version 1 data');
// Retrieve v2 - should get v2 data
const entry2 = await cache.get(context2, upstreamUrl);
expect(entry2).toBeTruthy();
expect(entry2!.data.toString()).toEqual('version 2 data');
});
// ============================================================================
// S3 Persistence Tests
// ============================================================================
tap.test('cache: should load from S3 on memory cache miss', async () => {
// Use a unique resource name for this test
const uniqueResource = `persist-test-${Date.now()}`;
const persistContext = createFetchContext({ resource: uniqueResource });
const upstreamUrl = 'https://persist.example.com';
// Store in first cache instance
await cache.set(
persistContext,
Buffer.from('persisted data'),
'text/plain',
{},
'persist-upstream',
upstreamUrl
);
// Wait for S3 write to complete
await new Promise(resolve => setTimeout(resolve, 200));
// Verify the entry is in the original cache's memory
const originalEntry = await cache.get(persistContext, upstreamUrl);
expect(originalEntry).toBeTruthy();
// Create a new cache instance (simulates restart) with SAME storage backend
const freshCache = new UpstreamCache(
{ enabled: true, defaultTtlSeconds: 300 },
10000,
storageBackend
);
// Fresh cache has empty memory, should load from S3
const entry = await freshCache.get(persistContext, upstreamUrl);
expect(entry).toBeTruthy();
expect(entry!.data.toString()).toEqual('persisted data');
freshCache.stop();
});
// ============================================================================
// Cache Stats Tests
// ============================================================================
tap.test('cache: should return accurate stats', async () => {
const statsCache = new UpstreamCache(
{ enabled: true, defaultTtlSeconds: 300 },
1000,
storageBackend
);
// Add some entries
for (let i = 0; i < 3; i++) {
const context = createFetchContext({ resource: `stats-test-${i}` });
await statsCache.set(
context,
Buffer.from(`stats data ${i}`),
'text/plain',
{},
'stats-upstream',
'https://stats.example.com'
);
}
// Add a negative entry
const negContext = createFetchContext({ resource: 'stats-negative' });
await statsCache.setNegative(negContext, 'stats-upstream', 'https://stats.example.com');
const stats = statsCache.getStats();
expect(stats.totalEntries).toBeGreaterThanOrEqual(4);
expect(stats.enabled).toBeTrue();
expect(stats.hasStorage).toBeTrue();
expect(stats.maxEntries).toEqual(1000);
statsCache.stop();
});
// ============================================================================
// Invalidation Tests
// ============================================================================
tap.test('cache: should invalidate specific cache entry', async () => {
const invalidateContext = createFetchContext({ resource: 'invalidate-test' });
const upstreamUrl = 'https://invalidate.example.com';
await cache.set(
invalidateContext,
Buffer.from('to be invalidated'),
'text/plain',
{},
'inv-upstream',
upstreamUrl
);
// Verify it exists
let entry = await cache.get(invalidateContext, upstreamUrl);
expect(entry).toBeTruthy();
// Invalidate
const deleted = await cache.invalidate(invalidateContext, upstreamUrl);
expect(deleted).toBeTrue();
// Should be gone
entry = await cache.get(invalidateContext, upstreamUrl);
expect(entry).toBeNull();
});
tap.test('cache: should invalidate entries matching pattern', async () => {
const upstreamUrl = 'https://pattern.example.com';
// Add multiple entries
for (const name of ['pattern-a', 'pattern-b', 'other-c']) {
const context = createFetchContext({ resource: name });
await cache.set(
context,
Buffer.from(`data for ${name}`),
'text/plain',
{},
'pattern-upstream',
upstreamUrl
);
}
// Invalidate entries matching 'pattern-*'
const count = await cache.invalidatePattern(/pattern-/);
expect(count).toBeGreaterThanOrEqual(2);
// pattern-a should be gone
const entryA = await cache.get(createFetchContext({ resource: 'pattern-a' }), upstreamUrl);
expect(entryA).toBeNull();
// other-c should still exist
const entryC = await cache.get(createFetchContext({ resource: 'other-c' }), upstreamUrl);
expect(entryC).toBeTruthy();
});
// ============================================================================
// Cleanup
// ============================================================================
tap.test('cleanup: should stop cache and clean up bucket', async () => {
if (cache) {
cache.stop();
}
// Clean up test bucket
if (s3Bucket) {
try {
const files = await s3Bucket.fastList({});
for (const file of files) {
await s3Bucket.fastRemove({ path: file.name });
}
await smartBucket.removeBucket(bucketName);
} catch {
// Ignore cleanup errors
}
}
});
export default tap.start();

View File

@@ -0,0 +1,343 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js';
import {
createTestRegistryWithUpstream,
createTrackingUpstreamProvider,
} from './helpers/registry.js';
import { StaticUpstreamProvider } from '../ts/upstream/interfaces.upstream.js';
import type {
IUpstreamProvider,
IUpstreamResolutionContext,
IProtocolUpstreamConfig,
} from '../ts/upstream/interfaces.upstream.js';
import type { TRegistryProtocol } from '../ts/core/interfaces.core.js';
// =============================================================================
// StaticUpstreamProvider Tests
// =============================================================================
tap.test('StaticUpstreamProvider: should return config for configured protocol', async () => {
const npmConfig: IProtocolUpstreamConfig = {
enabled: true,
upstreams: [{ id: 'npmjs', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
};
const provider = new StaticUpstreamProvider({
npm: npmConfig,
});
const result = await provider.resolveUpstreamConfig({
protocol: 'npm',
resource: 'lodash',
scope: null,
method: 'GET',
resourceType: 'packument',
});
expect(result).toBeDefined();
expect(result?.enabled).toEqual(true);
expect(result?.upstreams[0].id).toEqual('npmjs');
});
tap.test('StaticUpstreamProvider: should return null for unconfigured protocol', async () => {
const provider = new StaticUpstreamProvider({
npm: {
enabled: true,
upstreams: [{ id: 'npmjs', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
},
});
const result = await provider.resolveUpstreamConfig({
protocol: 'maven',
resource: 'com.example:lib',
scope: 'com.example',
method: 'GET',
resourceType: 'pom',
});
expect(result).toBeNull();
});
tap.test('StaticUpstreamProvider: should support multiple protocols', async () => {
const provider = new StaticUpstreamProvider({
npm: {
enabled: true,
upstreams: [{ id: 'npmjs', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
},
oci: {
enabled: true,
upstreams: [{ id: 'dockerhub', url: 'https://registry-1.docker.io', priority: 1, enabled: true }],
},
maven: {
enabled: true,
upstreams: [{ id: 'central', url: 'https://repo1.maven.org/maven2', priority: 1, enabled: true }],
},
});
const npmResult = await provider.resolveUpstreamConfig({
protocol: 'npm',
resource: 'lodash',
scope: null,
method: 'GET',
resourceType: 'packument',
});
expect(npmResult?.upstreams[0].id).toEqual('npmjs');
const ociResult = await provider.resolveUpstreamConfig({
protocol: 'oci',
resource: 'library/nginx',
scope: 'library',
method: 'GET',
resourceType: 'manifest',
});
expect(ociResult?.upstreams[0].id).toEqual('dockerhub');
const mavenResult = await provider.resolveUpstreamConfig({
protocol: 'maven',
resource: 'com.example:lib',
scope: 'com.example',
method: 'GET',
resourceType: 'pom',
});
expect(mavenResult?.upstreams[0].id).toEqual('central');
});
// =============================================================================
// Registry with Provider Integration Tests
// =============================================================================
let registry: SmartRegistry;
let trackingProvider: ReturnType<typeof createTrackingUpstreamProvider>;
tap.test('Provider Integration: should create registry with upstream provider', async () => {
trackingProvider = createTrackingUpstreamProvider({
npm: {
enabled: true,
upstreams: [{ id: 'test-npm', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
},
});
registry = await createTestRegistryWithUpstream(trackingProvider.provider);
expect(registry).toBeInstanceOf(SmartRegistry);
expect(registry.isInitialized()).toEqual(true);
});
tap.test('Provider Integration: should call provider when fetching unknown npm package', async () => {
// Clear previous calls
trackingProvider.calls.length = 0;
// Request a package that doesn't exist locally - should trigger upstream lookup
const response = await registry.handleRequest({
method: 'GET',
path: '/npm/@test-scope/nonexistent-package',
headers: {},
query: {},
});
// Provider should have been called for the packument lookup
const npmCalls = trackingProvider.calls.filter(c => c.protocol === 'npm');
// The package doesn't exist locally, so upstream should be consulted
// Note: actual upstream fetch may fail since the package doesn't exist
expect(response.status).toBeOneOf([404, 200, 502]); // 404 if not found, 502 if upstream error
});
tap.test('Provider Integration: provider receives correct context for scoped npm package', async () => {
trackingProvider.calls.length = 0;
// Use URL-encoded path for scoped packages as npm client does
await registry.handleRequest({
method: 'GET',
path: '/npm/@myorg%2fmy-package',
headers: {},
query: {},
});
// Find any npm call - the exact resource type depends on routing
const npmCalls = trackingProvider.calls.filter(c => c.protocol === 'npm');
// Provider should be called for upstream lookup
if (npmCalls.length > 0) {
const call = npmCalls[0];
expect(call.protocol).toEqual('npm');
// The resource should include the scoped name
expect(call.resource).toInclude('myorg');
expect(call.method).toEqual('GET');
}
});
tap.test('Provider Integration: provider receives correct context for unscoped npm package', async () => {
trackingProvider.calls.length = 0;
await registry.handleRequest({
method: 'GET',
path: '/npm/lodash',
headers: {},
query: {},
});
const packumentCall = trackingProvider.calls.find(
c => c.protocol === 'npm' && c.resourceType === 'packument'
);
if (packumentCall) {
expect(packumentCall.protocol).toEqual('npm');
expect(packumentCall.resource).toEqual('lodash');
expect(packumentCall.scope).toBeNull(); // No scope for unscoped package
}
});
// =============================================================================
// Custom Provider Implementation Tests
// =============================================================================
tap.test('Custom Provider: should support dynamic resolution based on context', async () => {
// Create a provider that returns different configs based on scope
const dynamicProvider: IUpstreamProvider = {
async resolveUpstreamConfig(context: IUpstreamResolutionContext) {
if (context.scope === 'internal') {
// Internal packages go to private registry
return {
enabled: true,
upstreams: [{ id: 'private', url: 'https://private.registry.com', priority: 1, enabled: true }],
};
}
// Everything else goes to public registry
return {
enabled: true,
upstreams: [{ id: 'public', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
};
},
};
const internalResult = await dynamicProvider.resolveUpstreamConfig({
protocol: 'npm',
resource: '@internal/utils',
scope: 'internal',
method: 'GET',
resourceType: 'packument',
});
expect(internalResult?.upstreams[0].id).toEqual('private');
const publicResult = await dynamicProvider.resolveUpstreamConfig({
protocol: 'npm',
resource: '@public/utils',
scope: 'public',
method: 'GET',
resourceType: 'packument',
});
expect(publicResult?.upstreams[0].id).toEqual('public');
});
tap.test('Custom Provider: should support actor-based resolution', async () => {
const actorAwareProvider: IUpstreamProvider = {
async resolveUpstreamConfig(context: IUpstreamResolutionContext) {
// Different upstreams based on user's organization
if (context.actor?.orgId === 'enterprise-org') {
return {
enabled: true,
upstreams: [{ id: 'enterprise', url: 'https://enterprise.registry.com', priority: 1, enabled: true }],
};
}
return {
enabled: true,
upstreams: [{ id: 'default', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
};
},
};
const enterpriseResult = await actorAwareProvider.resolveUpstreamConfig({
protocol: 'npm',
resource: 'lodash',
scope: null,
actor: { orgId: 'enterprise-org', userId: 'user1' },
method: 'GET',
resourceType: 'packument',
});
expect(enterpriseResult?.upstreams[0].id).toEqual('enterprise');
const defaultResult = await actorAwareProvider.resolveUpstreamConfig({
protocol: 'npm',
resource: 'lodash',
scope: null,
actor: { orgId: 'free-org', userId: 'user2' },
method: 'GET',
resourceType: 'packument',
});
expect(defaultResult?.upstreams[0].id).toEqual('default');
});
tap.test('Custom Provider: should support disabling upstream for specific resources', async () => {
const selectiveProvider: IUpstreamProvider = {
async resolveUpstreamConfig(context: IUpstreamResolutionContext) {
// Block upstream for internal packages
if (context.scope === 'internal') {
return null; // No upstream for internal packages
}
return {
enabled: true,
upstreams: [{ id: 'public', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
};
},
};
const internalResult = await selectiveProvider.resolveUpstreamConfig({
protocol: 'npm',
resource: '@internal/secret',
scope: 'internal',
method: 'GET',
resourceType: 'packument',
});
expect(internalResult).toBeNull();
const publicResult = await selectiveProvider.resolveUpstreamConfig({
protocol: 'npm',
resource: 'lodash',
scope: null,
method: 'GET',
resourceType: 'packument',
});
expect(publicResult).not.toBeNull();
});
// =============================================================================
// Registry without Provider Tests
// =============================================================================
tap.test('No Provider: registry should work without upstream provider', async () => {
const registryWithoutUpstream = await createTestRegistryWithUpstream(
// Pass a provider that always returns null
{
async resolveUpstreamConfig() {
return null;
},
}
);
expect(registryWithoutUpstream).toBeInstanceOf(SmartRegistry);
// Should return 404 for non-existent package (no upstream to check)
const response = await registryWithoutUpstream.handleRequest({
method: 'GET',
path: '/npm/nonexistent-package-xyz',
headers: {},
query: {},
});
expect(response.status).toEqual(404);
registryWithoutUpstream.destroy();
});
// =============================================================================
// Cleanup
// =============================================================================
tap.postTask('cleanup registry', async () => {
if (registry) {
registry.destroy();
}
});
export default tap.start();

View File

@@ -3,6 +3,6 @@
*/ */
export const commitinfo = { export const commitinfo = {
name: '@push.rocks/smartregistry', name: '@push.rocks/smartregistry',
version: '2.3.0', version: '2.8.0',
description: 'A composable TypeScript library implementing OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems registries for building unified container and package registries' description: 'A composable TypeScript library implementing OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems registries for building unified container and package registries'
} }

View File

@@ -2,8 +2,8 @@ import { Smartlog } from '@push.rocks/smartlog';
import { BaseRegistry } from '../core/classes.baseregistry.js'; import { BaseRegistry } from '../core/classes.baseregistry.js';
import { RegistryStorage } from '../core/classes.registrystorage.js'; import { RegistryStorage } from '../core/classes.registrystorage.js';
import { AuthManager } from '../core/classes.authmanager.js'; import { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js'; import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from '../core/interfaces.core.js';
import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js'; import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
import type { import type {
ICargoIndexEntry, ICargoIndexEntry,
ICargoPublishMetadata, ICargoPublishMetadata,
@@ -27,20 +27,21 @@ export class CargoRegistry extends BaseRegistry {
private basePath: string = '/cargo'; private basePath: string = '/cargo';
private registryUrl: string; private registryUrl: string;
private logger: Smartlog; private logger: Smartlog;
private upstream: CargoUpstream | null = null; private upstreamProvider: IUpstreamProvider | null = null;
constructor( constructor(
storage: RegistryStorage, storage: RegistryStorage,
authManager: AuthManager, authManager: AuthManager,
basePath: string = '/cargo', basePath: string = '/cargo',
registryUrl: string = 'http://localhost:5000/cargo', registryUrl: string = 'http://localhost:5000/cargo',
upstreamConfig?: IProtocolUpstreamConfig upstreamProvider?: IUpstreamProvider
) { ) {
super(); super();
this.storage = storage; this.storage = storage;
this.authManager = authManager; this.authManager = authManager;
this.basePath = basePath; this.basePath = basePath;
this.registryUrl = registryUrl; this.registryUrl = registryUrl;
this.upstreamProvider = upstreamProvider || null;
// Initialize logger // Initialize logger
this.logger = new Smartlog({ this.logger = new Smartlog({
@@ -54,20 +55,38 @@ export class CargoRegistry extends BaseRegistry {
} }
}); });
this.logger.enableConsole(); this.logger.enableConsole();
}
// Initialize upstream if configured /**
if (upstreamConfig?.enabled) { * Get upstream for a specific request.
this.upstream = new CargoUpstream(upstreamConfig, undefined, this.logger); * Calls the provider to resolve upstream config dynamically.
} */
private async getUpstreamForRequest(
resource: string,
resourceType: string,
method: string,
actor?: IRequestActor
): Promise<CargoUpstream | null> {
if (!this.upstreamProvider) return null;
const config = await this.upstreamProvider.resolveUpstreamConfig({
protocol: 'cargo',
resource,
scope: resource, // For Cargo, crate name is the scope
actor,
method,
resourceType,
});
if (!config?.enabled) return null;
return new CargoUpstream(config, undefined, this.logger);
} }
/** /**
* Clean up resources (timers, connections, etc.) * Clean up resources (timers, connections, etc.)
*/ */
public destroy(): void { public destroy(): void {
if (this.upstream) { // No persistent upstream to clean up with dynamic provider
this.upstream.stop();
}
} }
public async init(): Promise<void> { public async init(): Promise<void> {
@@ -94,6 +113,14 @@ export class CargoRegistry extends BaseRegistry {
const authHeader = context.headers['authorization'] || context.headers['Authorization']; const authHeader = context.headers['authorization'] || context.headers['Authorization'];
const token = authHeader ? await this.authManager.validateToken(authHeader, 'cargo') : null; const token = authHeader ? await this.authManager.validateToken(authHeader, 'cargo') : null;
// Build actor from context and validated token
const actor: IRequestActor = {
...context.actor,
userId: token?.userId,
ip: context.headers['x-forwarded-for'] || context.headers['X-Forwarded-For'],
userAgent: context.headers['user-agent'] || context.headers['User-Agent'],
};
this.logger.log('debug', `handleRequest: ${context.method} ${path}`, { this.logger.log('debug', `handleRequest: ${context.method} ${path}`, {
method: context.method, method: context.method,
path, path,
@@ -107,11 +134,11 @@ export class CargoRegistry extends BaseRegistry {
// API endpoints // API endpoints
if (path.startsWith('/api/v1/')) { if (path.startsWith('/api/v1/')) {
return this.handleApiRequest(path, context, token); return this.handleApiRequest(path, context, token, actor);
} }
// Index files (sparse protocol) // Index files (sparse protocol)
return this.handleIndexRequest(path); return this.handleIndexRequest(path, actor);
} }
/** /**
@@ -132,7 +159,8 @@ export class CargoRegistry extends BaseRegistry {
private async handleApiRequest( private async handleApiRequest(
path: string, path: string,
context: IRequestContext, context: IRequestContext,
token: IAuthToken | null token: IAuthToken | null,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
// Publish: PUT /api/v1/crates/new // Publish: PUT /api/v1/crates/new
if (path === '/api/v1/crates/new' && context.method === 'PUT') { if (path === '/api/v1/crates/new' && context.method === 'PUT') {
@@ -142,7 +170,7 @@ export class CargoRegistry extends BaseRegistry {
// Download: GET /api/v1/crates/{crate}/{version}/download // Download: GET /api/v1/crates/{crate}/{version}/download
const downloadMatch = path.match(/^\/api\/v1\/crates\/([^\/]+)\/([^\/]+)\/download$/); const downloadMatch = path.match(/^\/api\/v1\/crates\/([^\/]+)\/([^\/]+)\/download$/);
if (downloadMatch && context.method === 'GET') { if (downloadMatch && context.method === 'GET') {
return this.handleDownload(downloadMatch[1], downloadMatch[2]); return this.handleDownload(downloadMatch[1], downloadMatch[2], actor);
} }
// Yank: DELETE /api/v1/crates/{crate}/{version}/yank // Yank: DELETE /api/v1/crates/{crate}/{version}/yank
@@ -175,7 +203,7 @@ export class CargoRegistry extends BaseRegistry {
* Handle index file requests * Handle index file requests
* Paths: /1/{name}, /2/{name}, /3/{c}/{name}, /{p1}/{p2}/{name} * Paths: /1/{name}, /2/{name}, /3/{c}/{name}, /{p1}/{p2}/{name}
*/ */
private async handleIndexRequest(path: string): Promise<IResponse> { private async handleIndexRequest(path: string, actor?: IRequestActor): Promise<IResponse> {
// Parse index paths to extract crate name // Parse index paths to extract crate name
const pathParts = path.split('/').filter(p => p); const pathParts = path.split('/').filter(p => p);
let crateName: string | null = null; let crateName: string | null = null;
@@ -202,7 +230,7 @@ export class CargoRegistry extends BaseRegistry {
}; };
} }
return this.handleIndexFile(crateName); return this.handleIndexFile(crateName, actor);
} }
/** /**
@@ -224,23 +252,26 @@ export class CargoRegistry extends BaseRegistry {
/** /**
* Serve index file for a crate * Serve index file for a crate
*/ */
private async handleIndexFile(crateName: string): Promise<IResponse> { private async handleIndexFile(crateName: string, actor?: IRequestActor): Promise<IResponse> {
let index = await this.storage.getCargoIndex(crateName); let index = await this.storage.getCargoIndex(crateName);
// Try upstream if not found locally // Try upstream if not found locally
if ((!index || index.length === 0) && this.upstream) { if (!index || index.length === 0) {
const upstreamIndex = await this.upstream.fetchCrateIndex(crateName); const upstream = await this.getUpstreamForRequest(crateName, 'index', 'GET', actor);
if (upstreamIndex) { if (upstream) {
// Parse the newline-delimited JSON const upstreamIndex = await upstream.fetchCrateIndex(crateName);
const parsedIndex: ICargoIndexEntry[] = upstreamIndex if (upstreamIndex) {
.split('\n') // Parse the newline-delimited JSON
.filter(line => line.trim()) const parsedIndex: ICargoIndexEntry[] = upstreamIndex
.map(line => JSON.parse(line)); .split('\n')
.filter(line => line.trim())
.map(line => JSON.parse(line));
if (parsedIndex.length > 0) { if (parsedIndex.length > 0) {
// Cache locally // Cache locally
await this.storage.putCargoIndex(crateName, parsedIndex); await this.storage.putCargoIndex(crateName, parsedIndex);
index = parsedIndex; index = parsedIndex;
}
} }
} }
} }
@@ -339,7 +370,7 @@ export class CargoRegistry extends BaseRegistry {
const parsed = this.parsePublishRequest(body); const parsed = this.parsePublishRequest(body);
metadata = parsed.metadata; metadata = parsed.metadata;
crateFile = parsed.crateFile; crateFile = parsed.crateFile;
} catch (error) { } catch (error: any) {
this.logger.log('error', 'handlePublish: parse error', { error: error.message }); this.logger.log('error', 'handlePublish: parse error', { error: error.message });
return { return {
status: 400, status: 400,
@@ -431,15 +462,31 @@ export class CargoRegistry extends BaseRegistry {
*/ */
private async handleDownload( private async handleDownload(
crateName: string, crateName: string,
version: string version: string,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
this.logger.log('debug', 'handleDownload', { crate: crateName, version }); this.logger.log('debug', 'handleDownload', { crate: crateName, version });
let crateFile = await this.storage.getCargoCrate(crateName, version); // Try streaming from local storage first
const streamResult = await this.storage.getCargoCrateStream(crateName, version);
if (streamResult) {
return {
status: 200,
headers: {
'Content-Type': 'application/gzip',
'Content-Length': streamResult.size.toString(),
'Content-Disposition': `attachment; filename="${crateName}-${version}.crate"`,
},
body: streamResult.stream,
};
}
// Try upstream if not found locally // Try upstream if not found locally
if (!crateFile && this.upstream) { let crateFile: Buffer | null = null;
crateFile = await this.upstream.fetchCrate(crateName, version); const upstream = await this.getUpstreamForRequest(crateName, 'crate', 'GET', actor);
if (upstream) {
crateFile = await upstream.fetchCrate(crateName, version);
if (crateFile) { if (crateFile) {
// Cache locally // Cache locally
await this.storage.putCargoCrate(crateName, version, crateFile); await this.storage.putCargoCrate(crateName, version, crateFile);
@@ -612,7 +659,7 @@ export class CargoRegistry extends BaseRegistry {
} }
} }
} }
} catch (error) { } catch (error: any) {
this.logger.log('error', 'handleSearch: error', { error: error.message }); this.logger.log('error', 'handleSearch: error', { error: error.message });
} }

View File

@@ -2,6 +2,7 @@ import { RegistryStorage } from './core/classes.registrystorage.js';
import { AuthManager } from './core/classes.authmanager.js'; import { AuthManager } from './core/classes.authmanager.js';
import { BaseRegistry } from './core/classes.baseregistry.js'; import { BaseRegistry } from './core/classes.baseregistry.js';
import type { IRegistryConfig, IRequestContext, IResponse } from './core/interfaces.core.js'; import type { IRegistryConfig, IRequestContext, IResponse } from './core/interfaces.core.js';
import { toReadableStream } from './core/helpers.stream.js';
import { OciRegistry } from './oci/classes.ociregistry.js'; import { OciRegistry } from './oci/classes.ociregistry.js';
import { NpmRegistry } from './npm/classes.npmregistry.js'; import { NpmRegistry } from './npm/classes.npmregistry.js';
import { MavenRegistry } from './maven/classes.mavenregistry.js'; import { MavenRegistry } from './maven/classes.mavenregistry.js';
@@ -11,8 +12,39 @@ import { PypiRegistry } from './pypi/classes.pypiregistry.js';
import { RubyGemsRegistry } from './rubygems/classes.rubygemsregistry.js'; import { RubyGemsRegistry } from './rubygems/classes.rubygemsregistry.js';
/** /**
* Main registry orchestrator * Main registry orchestrator.
* Routes requests to appropriate protocol handlers (OCI, NPM, Maven, Cargo, Composer, PyPI, or RubyGems) * Routes requests to appropriate protocol handlers (OCI, NPM, Maven, Cargo, Composer, PyPI, or RubyGems).
*
* Supports pluggable authentication and storage hooks:
*
* @example
* ```typescript
* // Basic usage with default in-memory auth
* const registry = new SmartRegistry(config);
*
* // With custom auth provider (LDAP, OAuth, etc.)
* const registry = new SmartRegistry({
* ...config,
* authProvider: new LdapAuthProvider(ldapClient),
* });
*
* // With storage hooks for quota tracking
* const registry = new SmartRegistry({
* ...config,
* storageHooks: {
* beforePut: async (ctx) => {
* const quota = await getQuota(ctx.actor?.orgId);
* if (ctx.metadata?.size > quota) {
* return { allowed: false, reason: 'Quota exceeded' };
* }
* return { allowed: true };
* },
* afterPut: async (ctx) => {
* await auditLog('storage.put', ctx);
* }
* }
* });
* ```
*/ */
export class SmartRegistry { export class SmartRegistry {
private storage: RegistryStorage; private storage: RegistryStorage;
@@ -23,8 +55,12 @@ export class SmartRegistry {
constructor(config: IRegistryConfig) { constructor(config: IRegistryConfig) {
this.config = config; this.config = config;
this.storage = new RegistryStorage(config.storage);
this.authManager = new AuthManager(config.auth); // Create storage with optional hooks
this.storage = new RegistryStorage(config.storage, config.storageHooks);
// Create auth manager with optional custom provider
this.authManager = new AuthManager(config.auth, config.authProvider);
} }
/** /**
@@ -51,7 +87,7 @@ export class SmartRegistry {
this.authManager, this.authManager,
ociBasePath, ociBasePath,
ociTokens, ociTokens,
this.config.oci.upstream this.config.upstreamProvider
); );
await ociRegistry.init(); await ociRegistry.init();
this.registries.set('oci', ociRegistry); this.registries.set('oci', ociRegistry);
@@ -60,13 +96,13 @@ export class SmartRegistry {
// Initialize NPM registry if enabled // Initialize NPM registry if enabled
if (this.config.npm?.enabled) { if (this.config.npm?.enabled) {
const npmBasePath = this.config.npm.basePath ?? '/npm'; const npmBasePath = this.config.npm.basePath ?? '/npm';
const registryUrl = `http://localhost:5000${npmBasePath}`; // TODO: Make configurable const registryUrl = this.config.npm.registryUrl ?? `http://localhost:5000${npmBasePath}`;
const npmRegistry = new NpmRegistry( const npmRegistry = new NpmRegistry(
this.storage, this.storage,
this.authManager, this.authManager,
npmBasePath, npmBasePath,
registryUrl, registryUrl,
this.config.npm.upstream this.config.upstreamProvider
); );
await npmRegistry.init(); await npmRegistry.init();
this.registries.set('npm', npmRegistry); this.registries.set('npm', npmRegistry);
@@ -75,13 +111,13 @@ export class SmartRegistry {
// Initialize Maven registry if enabled // Initialize Maven registry if enabled
if (this.config.maven?.enabled) { if (this.config.maven?.enabled) {
const mavenBasePath = this.config.maven.basePath ?? '/maven'; const mavenBasePath = this.config.maven.basePath ?? '/maven';
const registryUrl = `http://localhost:5000${mavenBasePath}`; // TODO: Make configurable const registryUrl = this.config.maven.registryUrl ?? `http://localhost:5000${mavenBasePath}`;
const mavenRegistry = new MavenRegistry( const mavenRegistry = new MavenRegistry(
this.storage, this.storage,
this.authManager, this.authManager,
mavenBasePath, mavenBasePath,
registryUrl, registryUrl,
this.config.maven.upstream this.config.upstreamProvider
); );
await mavenRegistry.init(); await mavenRegistry.init();
this.registries.set('maven', mavenRegistry); this.registries.set('maven', mavenRegistry);
@@ -90,13 +126,13 @@ export class SmartRegistry {
// Initialize Cargo registry if enabled // Initialize Cargo registry if enabled
if (this.config.cargo?.enabled) { if (this.config.cargo?.enabled) {
const cargoBasePath = this.config.cargo.basePath ?? '/cargo'; const cargoBasePath = this.config.cargo.basePath ?? '/cargo';
const registryUrl = `http://localhost:5000${cargoBasePath}`; // TODO: Make configurable const registryUrl = this.config.cargo.registryUrl ?? `http://localhost:5000${cargoBasePath}`;
const cargoRegistry = new CargoRegistry( const cargoRegistry = new CargoRegistry(
this.storage, this.storage,
this.authManager, this.authManager,
cargoBasePath, cargoBasePath,
registryUrl, registryUrl,
this.config.cargo.upstream this.config.upstreamProvider
); );
await cargoRegistry.init(); await cargoRegistry.init();
this.registries.set('cargo', cargoRegistry); this.registries.set('cargo', cargoRegistry);
@@ -105,13 +141,13 @@ export class SmartRegistry {
// Initialize Composer registry if enabled // Initialize Composer registry if enabled
if (this.config.composer?.enabled) { if (this.config.composer?.enabled) {
const composerBasePath = this.config.composer.basePath ?? '/composer'; const composerBasePath = this.config.composer.basePath ?? '/composer';
const registryUrl = `http://localhost:5000${composerBasePath}`; // TODO: Make configurable const registryUrl = this.config.composer.registryUrl ?? `http://localhost:5000${composerBasePath}`;
const composerRegistry = new ComposerRegistry( const composerRegistry = new ComposerRegistry(
this.storage, this.storage,
this.authManager, this.authManager,
composerBasePath, composerBasePath,
registryUrl, registryUrl,
this.config.composer.upstream this.config.upstreamProvider
); );
await composerRegistry.init(); await composerRegistry.init();
this.registries.set('composer', composerRegistry); this.registries.set('composer', composerRegistry);
@@ -120,13 +156,13 @@ export class SmartRegistry {
// Initialize PyPI registry if enabled // Initialize PyPI registry if enabled
if (this.config.pypi?.enabled) { if (this.config.pypi?.enabled) {
const pypiBasePath = this.config.pypi.basePath ?? '/pypi'; const pypiBasePath = this.config.pypi.basePath ?? '/pypi';
const registryUrl = `http://localhost:5000`; // TODO: Make configurable const registryUrl = this.config.pypi.registryUrl ?? `http://localhost:5000`;
const pypiRegistry = new PypiRegistry( const pypiRegistry = new PypiRegistry(
this.storage, this.storage,
this.authManager, this.authManager,
pypiBasePath, pypiBasePath,
registryUrl, registryUrl,
this.config.pypi.upstream this.config.upstreamProvider
); );
await pypiRegistry.init(); await pypiRegistry.init();
this.registries.set('pypi', pypiRegistry); this.registries.set('pypi', pypiRegistry);
@@ -135,13 +171,13 @@ export class SmartRegistry {
// Initialize RubyGems registry if enabled // Initialize RubyGems registry if enabled
if (this.config.rubygems?.enabled) { if (this.config.rubygems?.enabled) {
const rubygemsBasePath = this.config.rubygems.basePath ?? '/rubygems'; const rubygemsBasePath = this.config.rubygems.basePath ?? '/rubygems';
const registryUrl = `http://localhost:5000${rubygemsBasePath}`; // TODO: Make configurable const registryUrl = this.config.rubygems.registryUrl ?? `http://localhost:5000${rubygemsBasePath}`;
const rubygemsRegistry = new RubyGemsRegistry( const rubygemsRegistry = new RubyGemsRegistry(
this.storage, this.storage,
this.authManager, this.authManager,
rubygemsBasePath, rubygemsBasePath,
registryUrl, registryUrl,
this.config.rubygems.upstream this.config.upstreamProvider
); );
await rubygemsRegistry.init(); await rubygemsRegistry.init();
this.registries.set('rubygems', rubygemsRegistry); this.registries.set('rubygems', rubygemsRegistry);
@@ -156,75 +192,88 @@ export class SmartRegistry {
*/ */
public async handleRequest(context: IRequestContext): Promise<IResponse> { public async handleRequest(context: IRequestContext): Promise<IResponse> {
const path = context.path; const path = context.path;
let response: IResponse | undefined;
// Route to OCI registry // Route to OCI registry
if (this.config.oci?.enabled && path.startsWith(this.config.oci.basePath)) { if (!response && this.config.oci?.enabled && path.startsWith(this.config.oci.basePath)) {
const ociRegistry = this.registries.get('oci'); const ociRegistry = this.registries.get('oci');
if (ociRegistry) { if (ociRegistry) {
return ociRegistry.handleRequest(context); response = await ociRegistry.handleRequest(context);
} }
} }
// Route to NPM registry // Route to NPM registry
if (this.config.npm?.enabled && path.startsWith(this.config.npm.basePath)) { if (!response && this.config.npm?.enabled && path.startsWith(this.config.npm.basePath)) {
const npmRegistry = this.registries.get('npm'); const npmRegistry = this.registries.get('npm');
if (npmRegistry) { if (npmRegistry) {
return npmRegistry.handleRequest(context); response = await npmRegistry.handleRequest(context);
} }
} }
// Route to Maven registry // Route to Maven registry
if (this.config.maven?.enabled && path.startsWith(this.config.maven.basePath)) { if (!response && this.config.maven?.enabled && path.startsWith(this.config.maven.basePath)) {
const mavenRegistry = this.registries.get('maven'); const mavenRegistry = this.registries.get('maven');
if (mavenRegistry) { if (mavenRegistry) {
return mavenRegistry.handleRequest(context); response = await mavenRegistry.handleRequest(context);
} }
} }
// Route to Cargo registry // Route to Cargo registry
if (this.config.cargo?.enabled && path.startsWith(this.config.cargo.basePath)) { if (!response && this.config.cargo?.enabled && path.startsWith(this.config.cargo.basePath)) {
const cargoRegistry = this.registries.get('cargo'); const cargoRegistry = this.registries.get('cargo');
if (cargoRegistry) { if (cargoRegistry) {
return cargoRegistry.handleRequest(context); response = await cargoRegistry.handleRequest(context);
} }
} }
// Route to Composer registry // Route to Composer registry
if (this.config.composer?.enabled && path.startsWith(this.config.composer.basePath)) { if (!response && this.config.composer?.enabled && path.startsWith(this.config.composer.basePath)) {
const composerRegistry = this.registries.get('composer'); const composerRegistry = this.registries.get('composer');
if (composerRegistry) { if (composerRegistry) {
return composerRegistry.handleRequest(context); response = await composerRegistry.handleRequest(context);
} }
} }
// Route to PyPI registry (also handles /simple prefix) // Route to PyPI registry (also handles /simple prefix)
if (this.config.pypi?.enabled) { if (!response && this.config.pypi?.enabled) {
const pypiBasePath = this.config.pypi.basePath ?? '/pypi'; const pypiBasePath = this.config.pypi.basePath ?? '/pypi';
if (path.startsWith(pypiBasePath) || path.startsWith('/simple')) { if (path.startsWith(pypiBasePath) || path.startsWith('/simple')) {
const pypiRegistry = this.registries.get('pypi'); const pypiRegistry = this.registries.get('pypi');
if (pypiRegistry) { if (pypiRegistry) {
return pypiRegistry.handleRequest(context); response = await pypiRegistry.handleRequest(context);
} }
} }
} }
// Route to RubyGems registry // Route to RubyGems registry
if (this.config.rubygems?.enabled && path.startsWith(this.config.rubygems.basePath)) { if (!response && this.config.rubygems?.enabled && path.startsWith(this.config.rubygems.basePath)) {
const rubygemsRegistry = this.registries.get('rubygems'); const rubygemsRegistry = this.registries.get('rubygems');
if (rubygemsRegistry) { if (rubygemsRegistry) {
return rubygemsRegistry.handleRequest(context); response = await rubygemsRegistry.handleRequest(context);
} }
} }
// No matching registry // No matching registry
return { if (!response) {
status: 404, response = {
headers: { 'Content-Type': 'application/json' }, status: 404,
body: { headers: { 'Content-Type': 'application/json' },
error: 'NOT_FOUND', body: {
message: 'No registry handler for this path', error: 'NOT_FOUND',
}, message: 'No registry handler for this path',
}; },
};
}
// Normalize body to ReadableStream<Uint8Array> at the API boundary
if (response.body != null && !(response.body instanceof ReadableStream)) {
if (!Buffer.isBuffer(response.body) && typeof response.body === 'object' && !(response.body instanceof Uint8Array)) {
response.headers['Content-Type'] ??= 'application/json';
}
response.body = toReadableStream(response.body);
}
return response;
} }
/** /**

View File

@@ -6,8 +6,8 @@
import { BaseRegistry } from '../core/classes.baseregistry.js'; import { BaseRegistry } from '../core/classes.baseregistry.js';
import type { RegistryStorage } from '../core/classes.registrystorage.js'; import type { RegistryStorage } from '../core/classes.registrystorage.js';
import type { AuthManager } from '../core/classes.authmanager.js'; import type { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js'; import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from '../core/interfaces.core.js';
import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js'; import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
import { isBinaryData, toBuffer } from '../core/helpers.buffer.js'; import { isBinaryData, toBuffer } from '../core/helpers.buffer.js';
import type { import type {
IComposerPackage, IComposerPackage,
@@ -30,34 +30,66 @@ export class ComposerRegistry extends BaseRegistry {
private authManager: AuthManager; private authManager: AuthManager;
private basePath: string = '/composer'; private basePath: string = '/composer';
private registryUrl: string; private registryUrl: string;
private upstream: ComposerUpstream | null = null; private upstreamProvider: IUpstreamProvider | null = null;
constructor( constructor(
storage: RegistryStorage, storage: RegistryStorage,
authManager: AuthManager, authManager: AuthManager,
basePath: string = '/composer', basePath: string = '/composer',
registryUrl: string = 'http://localhost:5000/composer', registryUrl: string = 'http://localhost:5000/composer',
upstreamConfig?: IProtocolUpstreamConfig upstreamProvider?: IUpstreamProvider
) { ) {
super(); super();
this.storage = storage; this.storage = storage;
this.authManager = authManager; this.authManager = authManager;
this.basePath = basePath; this.basePath = basePath;
this.registryUrl = registryUrl; this.registryUrl = registryUrl;
this.upstreamProvider = upstreamProvider || null;
}
// Initialize upstream if configured /**
if (upstreamConfig?.enabled) { * Extract scope from Composer package name.
this.upstream = new ComposerUpstream(upstreamConfig); * For Composer, vendor is the scope.
* @example "symfony" from "symfony/console"
*/
private extractScope(vendorPackage: string): string | null {
const slashIndex = vendorPackage.indexOf('/');
if (slashIndex > 0) {
return vendorPackage.substring(0, slashIndex);
} }
return null;
}
/**
* Get upstream for a specific request.
* Calls the provider to resolve upstream config dynamically.
*/
private async getUpstreamForRequest(
resource: string,
resourceType: string,
method: string,
actor?: IRequestActor
): Promise<ComposerUpstream | null> {
if (!this.upstreamProvider) return null;
const config = await this.upstreamProvider.resolveUpstreamConfig({
protocol: 'composer',
resource,
scope: this.extractScope(resource),
actor,
method,
resourceType,
});
if (!config?.enabled) return null;
return new ComposerUpstream(config);
} }
/** /**
* Clean up resources (timers, connections, etc.) * Clean up resources (timers, connections, etc.)
*/ */
public destroy(): void { public destroy(): void {
if (this.upstream) { // No persistent upstream to clean up with dynamic provider
this.upstream.stop();
}
} }
public async init(): Promise<void> { public async init(): Promise<void> {
@@ -96,6 +128,14 @@ export class ComposerRegistry extends BaseRegistry {
} }
} }
// Build actor from context and validated token
const actor: IRequestActor = {
...context.actor,
userId: token?.userId,
ip: context.headers['x-forwarded-for'] || context.headers['X-Forwarded-For'],
userAgent: context.headers['user-agent'] || context.headers['User-Agent'],
};
// Root packages.json // Root packages.json
if (path === '/packages.json' || path === '' || path === '/') { if (path === '/packages.json' || path === '' || path === '/') {
return this.handlePackagesJson(); return this.handlePackagesJson();
@@ -106,7 +146,7 @@ export class ComposerRegistry extends BaseRegistry {
if (metadataMatch) { if (metadataMatch) {
const [, vendorPackage, devSuffix] = metadataMatch; const [, vendorPackage, devSuffix] = metadataMatch;
const includeDev = !!devSuffix; const includeDev = !!devSuffix;
return this.handlePackageMetadata(vendorPackage, includeDev, token); return this.handlePackageMetadata(vendorPackage, includeDev, token, actor);
} }
// Package list: /packages/list.json?filter=vendor/* // Package list: /packages/list.json?filter=vendor/*
@@ -176,26 +216,30 @@ export class ComposerRegistry extends BaseRegistry {
private async handlePackageMetadata( private async handlePackageMetadata(
vendorPackage: string, vendorPackage: string,
includeDev: boolean, includeDev: boolean,
token: IAuthToken | null token: IAuthToken | null,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
// Read operations are public, no authentication required // Read operations are public, no authentication required
let metadata = await this.storage.getComposerPackageMetadata(vendorPackage); let metadata = await this.storage.getComposerPackageMetadata(vendorPackage);
// Try upstream if not found locally // Try upstream if not found locally
if (!metadata && this.upstream) { if (!metadata) {
const [vendor, packageName] = vendorPackage.split('/'); const upstream = await this.getUpstreamForRequest(vendorPackage, 'metadata', 'GET', actor);
if (vendor && packageName) { if (upstream) {
const upstreamMetadata = includeDev const [vendor, packageName] = vendorPackage.split('/');
? await this.upstream.fetchPackageDevMetadata(vendor, packageName) if (vendor && packageName) {
: await this.upstream.fetchPackageMetadata(vendor, packageName); const upstreamMetadata = includeDev
? await upstream.fetchPackageDevMetadata(vendor, packageName)
: await upstream.fetchPackageMetadata(vendor, packageName);
if (upstreamMetadata && upstreamMetadata.packages) { if (upstreamMetadata && upstreamMetadata.packages) {
// Store upstream metadata locally // Store upstream metadata locally
metadata = { metadata = {
packages: upstreamMetadata.packages, packages: upstreamMetadata.packages,
lastModified: new Date().toUTCString(), lastModified: new Date().toUTCString(),
}; };
await this.storage.putComposerPackageMetadata(vendorPackage, metadata); await this.storage.putComposerPackageMetadata(vendorPackage, metadata);
}
} }
} }
} }
@@ -258,9 +302,9 @@ export class ComposerRegistry extends BaseRegistry {
token: IAuthToken | null token: IAuthToken | null
): Promise<IResponse> { ): Promise<IResponse> {
// Read operations are public, no authentication required // Read operations are public, no authentication required
const zipData = await this.storage.getComposerPackageZip(vendorPackage, reference); const streamResult = await this.storage.getComposerPackageZipStream(vendorPackage, reference);
if (!zipData) { if (!streamResult) {
return { return {
status: 404, status: 404,
headers: {}, headers: {},
@@ -272,10 +316,10 @@ export class ComposerRegistry extends BaseRegistry {
status: 200, status: 200,
headers: { headers: {
'Content-Type': 'application/zip', 'Content-Type': 'application/zip',
'Content-Length': zipData.length.toString(), 'Content-Length': streamResult.size.toString(),
'Content-Disposition': `attachment; filename="${reference}.zip"`, 'Content-Disposition': `attachment; filename="${reference}.zip"`,
}, },
body: zipData, body: streamResult.stream,
}; };
} }

View File

@@ -1,109 +1,79 @@
import type { IAuthConfig, IAuthToken, ICredentials, TRegistryProtocol } from './interfaces.core.js'; import type { IAuthConfig, IAuthToken, ICredentials, TRegistryProtocol } from './interfaces.core.js';
import * as crypto from 'crypto'; import type { IAuthProvider, ITokenOptions } from './interfaces.auth.js';
import { DefaultAuthProvider } from './classes.defaultauthprovider.js';
/** /**
* Unified authentication manager for all registry protocols * Unified authentication manager for all registry protocols.
* Handles both NPM UUID tokens and OCI JWT tokens * Delegates to a pluggable IAuthProvider for actual auth operations.
*
* @example
* ```typescript
* // Use default in-memory provider
* const auth = new AuthManager(config);
*
* // Use custom provider (LDAP, OAuth, etc.)
* const auth = new AuthManager(config, new LdapAuthProvider(ldapClient));
* ```
*/ */
export class AuthManager { export class AuthManager {
private tokenStore: Map<string, IAuthToken> = new Map(); private provider: IAuthProvider;
private userCredentials: Map<string, string> = new Map(); // username -> password hash (mock)
constructor(private config: IAuthConfig) {} constructor(
private config: IAuthConfig,
provider?: IAuthProvider
) {
// Use provided provider or default in-memory implementation
this.provider = provider || new DefaultAuthProvider(config);
}
/** /**
* Initialize the auth manager * Initialize the auth manager
*/ */
public async init(): Promise<void> { public async init(): Promise<void> {
// Initialize token store (in-memory for now) if (this.provider.init) {
// In production, this could be Redis or a database await this.provider.init();
}
} }
// ======================================================================== // ========================================================================
// UUID TOKEN CREATION (Base method for NPM, Maven, etc.) // UNIFIED AUTHENTICATION (Delegated to Provider)
// ======================================================================== // ========================================================================
/** /**
* Create a UUID-based token with custom scopes (base method) * Authenticate user credentials
* @param userId - User ID * @param credentials - Username and password
* @param protocol - Protocol type * @returns User ID or null
* @param scopes - Permission scopes
* @param readonly - Whether the token is readonly
* @returns UUID token string
*/ */
private async createUuidToken( public async authenticate(credentials: ICredentials): Promise<string | null> {
userId: string, return this.provider.authenticate(credentials);
protocol: TRegistryProtocol,
scopes: string[],
readonly: boolean = false
): Promise<string> {
const token = this.generateUuid();
const authToken: IAuthToken = {
type: protocol,
userId,
scopes,
readonly,
metadata: {
created: new Date().toISOString(),
},
};
this.tokenStore.set(token, authToken);
return token;
} }
/** /**
* Generic protocol token creation (internal helper) * Validate any token (NPM, Maven, OCI, PyPI, RubyGems, Composer, Cargo)
* @param userId - User ID * @param tokenString - Token string (UUID or JWT)
* @param protocol - Protocol type (npm, maven, composer, etc.) * @param protocol - Expected protocol type (optional, improves performance)
* @param readonly - Whether the token is readonly
* @returns UUID token string
*/
private async createProtocolToken(
userId: string,
protocol: TRegistryProtocol,
readonly: boolean
): Promise<string> {
const scopes = readonly
? [`${protocol}:*:*:read`]
: [`${protocol}:*:*:*`];
return this.createUuidToken(userId, protocol, scopes, readonly);
}
/**
* Generic protocol token validation (internal helper)
* @param token - UUID token string
* @param protocol - Expected protocol type
* @returns Auth token object or null * @returns Auth token object or null
*/ */
private async validateProtocolToken( public async validateToken(
token: string, tokenString: string,
protocol: TRegistryProtocol protocol?: TRegistryProtocol
): Promise<IAuthToken | null> { ): Promise<IAuthToken | null> {
if (!this.isValidUuid(token)) { return this.provider.validateToken(tokenString, protocol);
return null;
}
const authToken = this.tokenStore.get(token);
if (!authToken || authToken.type !== protocol) {
return null;
}
// Check expiration if set
if (authToken.expiresAt && authToken.expiresAt < new Date()) {
this.tokenStore.delete(token);
return null;
}
return authToken;
} }
/** /**
* Generic protocol token revocation (internal helper) * Check if token has permission for an action
* @param token - UUID token string * @param token - Auth token (or null for anonymous)
* @param resource - Resource being accessed (e.g., "npm:package:foo")
* @param action - Action being performed (read, write, push, pull, delete)
* @returns true if authorized
*/ */
private async revokeProtocolToken(token: string): Promise<void> { public async authorize(
this.tokenStore.delete(token); token: IAuthToken | null,
resource: string,
action: string
): Promise<boolean> {
return this.provider.authorize(token, resource, action);
} }
// ======================================================================== // ========================================================================
@@ -120,7 +90,7 @@ export class AuthManager {
if (!this.config.npmTokens.enabled) { if (!this.config.npmTokens.enabled) {
throw new Error('NPM tokens are not enabled'); throw new Error('NPM tokens are not enabled');
} }
return this.createProtocolToken(userId, 'npm', readonly); return this.provider.createToken(userId, 'npm', { readonly });
} }
/** /**
@@ -129,7 +99,7 @@ export class AuthManager {
* @returns Auth token object or null * @returns Auth token object or null
*/ */
public async validateNpmToken(token: string): Promise<IAuthToken | null> { public async validateNpmToken(token: string): Promise<IAuthToken | null> {
return this.validateProtocolToken(token, 'npm'); return this.provider.validateToken(token, 'npm');
} }
/** /**
@@ -137,7 +107,7 @@ export class AuthManager {
* @param token - NPM UUID token * @param token - NPM UUID token
*/ */
public async revokeNpmToken(token: string): Promise<void> { public async revokeNpmToken(token: string): Promise<void> {
return this.revokeProtocolToken(token); return this.provider.revokeToken(token);
} }
/** /**
@@ -149,20 +119,12 @@ export class AuthManager {
key: string; key: string;
readonly: boolean; readonly: boolean;
created: string; created: string;
protocol?: TRegistryProtocol;
}>> { }>> {
const tokens: Array<{key: string; readonly: boolean; created: string}> = []; if (this.provider.listUserTokens) {
return this.provider.listUserTokens(userId);
for (const [token, authToken] of this.tokenStore.entries()) {
if (authToken.userId === userId) {
tokens.push({
key: this.hashToken(token),
readonly: authToken.readonly || false,
created: authToken.metadata?.created || 'unknown',
});
}
} }
return [];
return tokens;
} }
// ======================================================================== // ========================================================================
@@ -174,39 +136,17 @@ export class AuthManager {
* @param userId - User ID * @param userId - User ID
* @param scopes - Permission scopes * @param scopes - Permission scopes
* @param expiresIn - Expiration time in seconds * @param expiresIn - Expiration time in seconds
* @returns JWT token string (HMAC-SHA256 signed) * @returns JWT token string
*/ */
public async createOciToken( public async createOciToken(
userId: string, userId: string,
scopes: string[], scopes: string[],
expiresIn: number = 3600 expiresIn: number = 3600
): Promise<string> { ): Promise<string> {
if (!this.config.ociTokens.enabled) { if (!this.config.ociTokens?.enabled) {
throw new Error('OCI tokens are not enabled'); throw new Error('OCI tokens are not enabled');
} }
return this.provider.createToken(userId, 'oci', { scopes, expiresIn });
const now = Math.floor(Date.now() / 1000);
const payload = {
iss: this.config.ociTokens.realm,
sub: userId,
aud: this.config.ociTokens.service,
exp: now + expiresIn,
nbf: now,
iat: now,
access: this.scopesToOciAccess(scopes),
};
// Create JWT with HMAC-SHA256 signature
const header = { alg: 'HS256', typ: 'JWT' };
const headerB64 = Buffer.from(JSON.stringify(header)).toString('base64url');
const payloadB64 = Buffer.from(JSON.stringify(payload)).toString('base64url');
const signature = crypto
.createHmac('sha256', this.config.jwtSecret)
.update(`${headerB64}.${payloadB64}`)
.digest('base64url');
return `${headerB64}.${payloadB64}.${signature}`;
} }
/** /**
@@ -215,80 +155,7 @@ export class AuthManager {
* @returns Auth token object or null * @returns Auth token object or null
*/ */
public async validateOciToken(jwt: string): Promise<IAuthToken | null> { public async validateOciToken(jwt: string): Promise<IAuthToken | null> {
try { return this.provider.validateToken(jwt, 'oci');
const parts = jwt.split('.');
if (parts.length !== 3) {
return null;
}
const [headerB64, payloadB64, signatureB64] = parts;
// Verify signature
const expectedSignature = crypto
.createHmac('sha256', this.config.jwtSecret)
.update(`${headerB64}.${payloadB64}`)
.digest('base64url');
if (signatureB64 !== expectedSignature) {
return null;
}
// Decode and parse payload
const payload = JSON.parse(Buffer.from(payloadB64, 'base64url').toString('utf-8'));
// Check expiration
const now = Math.floor(Date.now() / 1000);
if (payload.exp && payload.exp < now) {
return null;
}
// Check not-before time
if (payload.nbf && payload.nbf > now) {
return null;
}
// Convert to unified token format
const scopes = this.ociAccessToScopes(payload.access || []);
return {
type: 'oci',
userId: payload.sub,
scopes,
expiresAt: payload.exp ? new Date(payload.exp * 1000) : undefined,
metadata: {
iss: payload.iss,
aud: payload.aud,
},
};
} catch (error) {
return null;
}
}
// ========================================================================
// UNIFIED AUTHENTICATION
// ========================================================================
/**
* Authenticate user credentials
* @param credentials - Username and password
* @returns User ID or null
*/
public async authenticate(credentials: ICredentials): Promise<string | null> {
// Mock authentication - in production, verify against database
const storedPassword = this.userCredentials.get(credentials.username);
if (!storedPassword) {
// Auto-register for testing (remove in production)
this.userCredentials.set(credentials.username, credentials.password);
return credentials.username;
}
if (storedPassword === credentials.password) {
return credentials.username;
}
return null;
} }
// ======================================================================== // ========================================================================
@@ -302,7 +169,7 @@ export class AuthManager {
* @returns Maven UUID token * @returns Maven UUID token
*/ */
public async createMavenToken(userId: string, readonly: boolean = false): Promise<string> { public async createMavenToken(userId: string, readonly: boolean = false): Promise<string> {
return this.createProtocolToken(userId, 'maven', readonly); return this.provider.createToken(userId, 'maven', { readonly });
} }
/** /**
@@ -311,7 +178,7 @@ export class AuthManager {
* @returns Auth token object or null * @returns Auth token object or null
*/ */
public async validateMavenToken(token: string): Promise<IAuthToken | null> { public async validateMavenToken(token: string): Promise<IAuthToken | null> {
return this.validateProtocolToken(token, 'maven'); return this.provider.validateToken(token, 'maven');
} }
/** /**
@@ -319,7 +186,7 @@ export class AuthManager {
* @param token - Maven UUID token * @param token - Maven UUID token
*/ */
public async revokeMavenToken(token: string): Promise<void> { public async revokeMavenToken(token: string): Promise<void> {
return this.revokeProtocolToken(token); return this.provider.revokeToken(token);
} }
// ======================================================================== // ========================================================================
@@ -333,7 +200,7 @@ export class AuthManager {
* @returns Composer UUID token * @returns Composer UUID token
*/ */
public async createComposerToken(userId: string, readonly: boolean = false): Promise<string> { public async createComposerToken(userId: string, readonly: boolean = false): Promise<string> {
return this.createProtocolToken(userId, 'composer', readonly); return this.provider.createToken(userId, 'composer', { readonly });
} }
/** /**
@@ -342,7 +209,7 @@ export class AuthManager {
* @returns Auth token object or null * @returns Auth token object or null
*/ */
public async validateComposerToken(token: string): Promise<IAuthToken | null> { public async validateComposerToken(token: string): Promise<IAuthToken | null> {
return this.validateProtocolToken(token, 'composer'); return this.provider.validateToken(token, 'composer');
} }
/** /**
@@ -350,7 +217,7 @@ export class AuthManager {
* @param token - Composer UUID token * @param token - Composer UUID token
*/ */
public async revokeComposerToken(token: string): Promise<void> { public async revokeComposerToken(token: string): Promise<void> {
return this.revokeProtocolToken(token); return this.provider.revokeToken(token);
} }
// ======================================================================== // ========================================================================
@@ -364,7 +231,7 @@ export class AuthManager {
* @returns Cargo UUID token * @returns Cargo UUID token
*/ */
public async createCargoToken(userId: string, readonly: boolean = false): Promise<string> { public async createCargoToken(userId: string, readonly: boolean = false): Promise<string> {
return this.createProtocolToken(userId, 'cargo', readonly); return this.provider.createToken(userId, 'cargo', { readonly });
} }
/** /**
@@ -373,7 +240,7 @@ export class AuthManager {
* @returns Auth token object or null * @returns Auth token object or null
*/ */
public async validateCargoToken(token: string): Promise<IAuthToken | null> { public async validateCargoToken(token: string): Promise<IAuthToken | null> {
return this.validateProtocolToken(token, 'cargo'); return this.provider.validateToken(token, 'cargo');
} }
/** /**
@@ -381,7 +248,7 @@ export class AuthManager {
* @param token - Cargo UUID token * @param token - Cargo UUID token
*/ */
public async revokeCargoToken(token: string): Promise<void> { public async revokeCargoToken(token: string): Promise<void> {
return this.revokeProtocolToken(token); return this.provider.revokeToken(token);
} }
// ======================================================================== // ========================================================================
@@ -395,7 +262,7 @@ export class AuthManager {
* @returns PyPI UUID token * @returns PyPI UUID token
*/ */
public async createPypiToken(userId: string, readonly: boolean = false): Promise<string> { public async createPypiToken(userId: string, readonly: boolean = false): Promise<string> {
return this.createProtocolToken(userId, 'pypi', readonly); return this.provider.createToken(userId, 'pypi', { readonly });
} }
/** /**
@@ -404,7 +271,7 @@ export class AuthManager {
* @returns Auth token object or null * @returns Auth token object or null
*/ */
public async validatePypiToken(token: string): Promise<IAuthToken | null> { public async validatePypiToken(token: string): Promise<IAuthToken | null> {
return this.validateProtocolToken(token, 'pypi'); return this.provider.validateToken(token, 'pypi');
} }
/** /**
@@ -412,7 +279,7 @@ export class AuthManager {
* @param token - PyPI UUID token * @param token - PyPI UUID token
*/ */
public async revokePypiToken(token: string): Promise<void> { public async revokePypiToken(token: string): Promise<void> {
return this.revokeProtocolToken(token); return this.provider.revokeToken(token);
} }
// ======================================================================== // ========================================================================
@@ -426,7 +293,7 @@ export class AuthManager {
* @returns RubyGems UUID token * @returns RubyGems UUID token
*/ */
public async createRubyGemsToken(userId: string, readonly: boolean = false): Promise<string> { public async createRubyGemsToken(userId: string, readonly: boolean = false): Promise<string> {
return this.createProtocolToken(userId, 'rubygems', readonly); return this.provider.createToken(userId, 'rubygems', { readonly });
} }
/** /**
@@ -435,7 +302,7 @@ export class AuthManager {
* @returns Auth token object or null * @returns Auth token object or null
*/ */
public async validateRubyGemsToken(token: string): Promise<IAuthToken | null> { public async validateRubyGemsToken(token: string): Promise<IAuthToken | null> {
return this.validateProtocolToken(token, 'rubygems'); return this.provider.validateToken(token, 'rubygems');
} }
/** /**
@@ -443,211 +310,6 @@ export class AuthManager {
* @param token - RubyGems UUID token * @param token - RubyGems UUID token
*/ */
public async revokeRubyGemsToken(token: string): Promise<void> { public async revokeRubyGemsToken(token: string): Promise<void> {
return this.revokeProtocolToken(token); return this.provider.revokeToken(token);
}
// ========================================================================
// UNIFIED AUTHENTICATION
// ========================================================================
/**
* Validate any token (NPM, Maven, OCI, PyPI, RubyGems, Composer, Cargo)
* Optimized: O(1) lookup when protocol hint provided
* @param tokenString - Token string (UUID or JWT)
* @param protocol - Expected protocol type (optional, improves performance)
* @returns Auth token object or null
*/
public async validateToken(
tokenString: string,
protocol?: TRegistryProtocol
): Promise<IAuthToken | null> {
// OCI uses JWT (contains dots), not UUID - check first if OCI is expected
if (protocol === 'oci' || tokenString.includes('.')) {
const ociToken = await this.validateOciToken(tokenString);
if (ociToken && (!protocol || protocol === 'oci')) {
return ociToken;
}
// If protocol was explicitly OCI but validation failed, return null
if (protocol === 'oci') {
return null;
}
}
// UUID-based tokens: single O(1) Map lookup
if (this.isValidUuid(tokenString)) {
const authToken = this.tokenStore.get(tokenString);
if (authToken) {
// If protocol specified, verify it matches
if (protocol && authToken.type !== protocol) {
return null;
}
// Check expiration
if (authToken.expiresAt && authToken.expiresAt < new Date()) {
this.tokenStore.delete(tokenString);
return null;
}
return authToken;
}
}
return null;
}
/**
* Check if token has permission for an action
* @param token - Auth token
* @param resource - Resource being accessed (e.g., "package:foo" or "repository:bar")
* @param action - Action being performed (read, write, push, pull, delete)
* @returns true if authorized
*/
public async authorize(
token: IAuthToken | null,
resource: string,
action: string
): Promise<boolean> {
if (!token) {
return false;
}
// Check readonly flag
if (token.readonly && ['write', 'push', 'delete'].includes(action)) {
return false;
}
// Check scopes
for (const scope of token.scopes) {
if (this.matchesScope(scope, resource, action)) {
return true;
}
}
return false;
}
// ========================================================================
// HELPER METHODS
// ========================================================================
/**
* Check if a scope matches a resource and action
* Scope format: "{protocol}:{type}:{name}:{action}"
* Examples:
* - "npm:*:*" - All NPM access
* - "npm:package:foo:*" - All actions on package foo
* - "npm:package:foo:read" - Read-only on package foo
* - "oci:repository:*:pull" - Pull from any OCI repo
*/
private matchesScope(scope: string, resource: string, action: string): boolean {
const scopeParts = scope.split(':');
const resourceParts = resource.split(':');
// Scope must have at least protocol:type:name:action
if (scopeParts.length < 4) {
return false;
}
const [scopeProtocol, scopeType, scopeName, scopeAction] = scopeParts;
const [resourceProtocol, resourceType, resourceName] = resourceParts;
// Check protocol
if (scopeProtocol !== '*' && scopeProtocol !== resourceProtocol) {
return false;
}
// Check type
if (scopeType !== '*' && scopeType !== resourceType) {
return false;
}
// Check name
if (scopeName !== '*' && scopeName !== resourceName) {
return false;
}
// Check action
if (scopeAction !== '*' && scopeAction !== action) {
// Map action aliases
const actionAliases: Record<string, string[]> = {
read: ['pull', 'get'],
write: ['push', 'put', 'post'],
};
const aliases = actionAliases[scopeAction] || [];
if (!aliases.includes(action)) {
return false;
}
}
return true;
}
/**
* Convert unified scopes to OCI access array
*/
private scopesToOciAccess(scopes: string[]): Array<{
type: string;
name: string;
actions: string[];
}> {
const access: Array<{type: string; name: string; actions: string[]}> = [];
for (const scope of scopes) {
const parts = scope.split(':');
if (parts.length >= 4 && parts[0] === 'oci') {
access.push({
type: parts[1],
name: parts[2],
actions: [parts[3]],
});
}
}
return access;
}
/**
* Convert OCI access array to unified scopes
*/
private ociAccessToScopes(access: Array<{
type: string;
name: string;
actions: string[];
}>): string[] {
const scopes: string[] = [];
for (const item of access) {
for (const action of item.actions) {
scopes.push(`oci:${item.type}:${item.name}:${action}`);
}
}
return scopes;
}
/**
* Generate UUID for NPM tokens
*/
private generateUuid(): string {
return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => {
const r = (Math.random() * 16) | 0;
const v = c === 'x' ? r : (r & 0x3) | 0x8;
return v.toString(16);
});
}
/**
* Check if string is a valid UUID
*/
private isValidUuid(str: string): boolean {
const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
return uuidRegex.test(str);
}
/**
* Hash a token for identification (SHA-512 mock)
*/
private hashToken(token: string): string {
// In production, use actual SHA-512
return `sha512-${token.substring(0, 16)}...`;
} }
} }

View File

@@ -0,0 +1,393 @@
import * as crypto from 'crypto';
import type { IAuthProvider, ITokenOptions } from './interfaces.auth.js';
import type { IAuthConfig, IAuthToken, ICredentials, TRegistryProtocol } from './interfaces.core.js';
/**
* Default in-memory authentication provider.
* This is the reference implementation that stores tokens in memory.
* For production use, implement IAuthProvider with Redis, database, or external auth.
*/
export class DefaultAuthProvider implements IAuthProvider {
private tokenStore: Map<string, IAuthToken> = new Map();
private userCredentials: Map<string, string> = new Map(); // username -> password hash (mock)
constructor(private config: IAuthConfig) {}
/**
* Initialize the auth provider
*/
public async init(): Promise<void> {
// Initialize token store (in-memory for now)
// In production, this could be Redis or a database
}
// ========================================================================
// IAuthProvider Implementation
// ========================================================================
/**
* Authenticate user credentials
*/
public async authenticate(credentials: ICredentials): Promise<string | null> {
// Mock authentication - in production, verify against database/LDAP
const storedPassword = this.userCredentials.get(credentials.username);
if (!storedPassword) {
// Auto-register for testing (remove in production)
this.userCredentials.set(credentials.username, credentials.password);
return credentials.username;
}
if (storedPassword === credentials.password) {
return credentials.username;
}
return null;
}
/**
* Validate any token (NPM, Maven, OCI, PyPI, RubyGems, Composer, Cargo)
*/
public async validateToken(
tokenString: string,
protocol?: TRegistryProtocol
): Promise<IAuthToken | null> {
// OCI uses JWT (contains dots), not UUID - check first if OCI is expected
if (protocol === 'oci' || tokenString.includes('.')) {
const ociToken = await this.validateOciToken(tokenString);
if (ociToken && (!protocol || protocol === 'oci')) {
return ociToken;
}
// If protocol was explicitly OCI but validation failed, return null
if (protocol === 'oci') {
return null;
}
}
// UUID-based tokens: single O(1) Map lookup
if (this.isValidUuid(tokenString)) {
const authToken = this.tokenStore.get(tokenString);
if (authToken) {
// If protocol specified, verify it matches
if (protocol && authToken.type !== protocol) {
return null;
}
// Check expiration
if (authToken.expiresAt && authToken.expiresAt < new Date()) {
this.tokenStore.delete(tokenString);
return null;
}
return authToken;
}
}
return null;
}
/**
* Create a new token for a user
*/
public async createToken(
userId: string,
protocol: TRegistryProtocol,
options?: ITokenOptions
): Promise<string> {
// OCI tokens use JWT
if (protocol === 'oci') {
return this.createOciToken(userId, options?.scopes || ['oci:*:*:*'], options?.expiresIn || 3600);
}
// All other protocols use UUID tokens
const token = this.generateUuid();
const scopes = options?.scopes || (options?.readonly
? [`${protocol}:*:*:read`]
: [`${protocol}:*:*:*`]);
const authToken: IAuthToken = {
type: protocol,
userId,
scopes,
readonly: options?.readonly,
expiresAt: options?.expiresIn ? new Date(Date.now() + options.expiresIn * 1000) : undefined,
metadata: {
created: new Date().toISOString(),
},
};
this.tokenStore.set(token, authToken);
return token;
}
/**
* Revoke a token
*/
public async revokeToken(token: string): Promise<void> {
this.tokenStore.delete(token);
}
/**
* Check if token has permission for an action
*/
public async authorize(
token: IAuthToken | null,
resource: string,
action: string
): Promise<boolean> {
if (!token) {
return false;
}
// Check readonly flag
if (token.readonly && ['write', 'push', 'delete'].includes(action)) {
return false;
}
// Check scopes
for (const scope of token.scopes) {
if (this.matchesScope(scope, resource, action)) {
return true;
}
}
return false;
}
/**
* List all tokens for a user
*/
public async listUserTokens(userId: string): Promise<Array<{
key: string;
readonly: boolean;
created: string;
protocol?: TRegistryProtocol;
}>> {
const tokens: Array<{key: string; readonly: boolean; created: string; protocol?: TRegistryProtocol}> = [];
for (const [token, authToken] of this.tokenStore.entries()) {
if (authToken.userId === userId) {
tokens.push({
key: this.hashToken(token),
readonly: authToken.readonly || false,
created: authToken.metadata?.created || 'unknown',
protocol: authToken.type,
});
}
}
return tokens;
}
// ========================================================================
// OCI JWT Token Methods
// ========================================================================
/**
* Create an OCI JWT token
*/
private async createOciToken(
userId: string,
scopes: string[],
expiresIn: number = 3600
): Promise<string> {
if (!this.config.ociTokens?.enabled) {
throw new Error('OCI tokens are not enabled');
}
const now = Math.floor(Date.now() / 1000);
const payload = {
iss: this.config.ociTokens.realm,
sub: userId,
aud: this.config.ociTokens.service,
exp: now + expiresIn,
nbf: now,
iat: now,
access: this.scopesToOciAccess(scopes),
};
// Create JWT with HMAC-SHA256 signature
const header = { alg: 'HS256', typ: 'JWT' };
const headerB64 = Buffer.from(JSON.stringify(header)).toString('base64url');
const payloadB64 = Buffer.from(JSON.stringify(payload)).toString('base64url');
const signature = crypto
.createHmac('sha256', this.config.jwtSecret)
.update(`${headerB64}.${payloadB64}`)
.digest('base64url');
return `${headerB64}.${payloadB64}.${signature}`;
}
/**
* Validate an OCI JWT token
*/
private async validateOciToken(jwt: string): Promise<IAuthToken | null> {
try {
const parts = jwt.split('.');
if (parts.length !== 3) {
return null;
}
const [headerB64, payloadB64, signatureB64] = parts;
// Verify signature
const expectedSignature = crypto
.createHmac('sha256', this.config.jwtSecret)
.update(`${headerB64}.${payloadB64}`)
.digest('base64url');
if (signatureB64 !== expectedSignature) {
return null;
}
// Decode and parse payload
const payload = JSON.parse(Buffer.from(payloadB64, 'base64url').toString('utf-8'));
// Check expiration
const now = Math.floor(Date.now() / 1000);
if (payload.exp && payload.exp < now) {
return null;
}
// Check not-before time
if (payload.nbf && payload.nbf > now) {
return null;
}
// Convert to unified token format
const scopes = this.ociAccessToScopes(payload.access || []);
return {
type: 'oci',
userId: payload.sub,
scopes,
expiresAt: payload.exp ? new Date(payload.exp * 1000) : undefined,
metadata: {
iss: payload.iss,
aud: payload.aud,
},
};
} catch (error) {
return null;
}
}
// ========================================================================
// Helper Methods
// ========================================================================
/**
* Check if a scope matches a resource and action
*/
private matchesScope(scope: string, resource: string, action: string): boolean {
const scopeParts = scope.split(':');
const resourceParts = resource.split(':');
// Scope must have at least protocol:type:name:action
if (scopeParts.length < 4) {
return false;
}
const [scopeProtocol, scopeType, scopeName, scopeAction] = scopeParts;
const [resourceProtocol, resourceType, resourceName] = resourceParts;
// Check protocol
if (scopeProtocol !== '*' && scopeProtocol !== resourceProtocol) {
return false;
}
// Check type
if (scopeType !== '*' && scopeType !== resourceType) {
return false;
}
// Check name
if (scopeName !== '*' && scopeName !== resourceName) {
return false;
}
// Check action
if (scopeAction !== '*' && scopeAction !== action) {
// Map action aliases
const actionAliases: Record<string, string[]> = {
read: ['pull', 'get'],
write: ['push', 'put', 'post'],
};
const aliases = actionAliases[scopeAction] || [];
if (!aliases.includes(action)) {
return false;
}
}
return true;
}
/**
* Convert unified scopes to OCI access array
*/
private scopesToOciAccess(scopes: string[]): Array<{
type: string;
name: string;
actions: string[];
}> {
const access: Array<{type: string; name: string; actions: string[]}> = [];
for (const scope of scopes) {
const parts = scope.split(':');
if (parts.length >= 4 && parts[0] === 'oci') {
access.push({
type: parts[1],
name: parts[2],
actions: [parts[3]],
});
}
}
return access;
}
/**
* Convert OCI access array to unified scopes
*/
private ociAccessToScopes(access: Array<{
type: string;
name: string;
actions: string[];
}>): string[] {
const scopes: string[] = [];
for (const item of access) {
for (const action of item.actions) {
scopes.push(`oci:${item.type}:${item.name}:${action}`);
}
}
return scopes;
}
/**
* Generate UUID for tokens
*/
private generateUuid(): string {
return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => {
const r = (Math.random() * 16) | 0;
const v = c === 'x' ? r : (r & 0x3) | 0x8;
return v.toString(16);
});
}
/**
* Check if string is a valid UUID
*/
private isValidUuid(str: string): boolean {
const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
return uuidRegex.test(str);
}
/**
* Hash a token for identification
*/
private hashToken(token: string): string {
return `sha512-${token.substring(0, 16)}...`;
}
}

View File

@@ -1,17 +1,54 @@
import * as plugins from '../plugins.js'; import * as plugins from '../plugins.js';
import type { IStorageConfig, IStorageBackend } from './interfaces.core.js'; import type { IStorageConfig, IStorageBackend, TRegistryProtocol } from './interfaces.core.js';
import type {
IStorageHooks,
IStorageHookContext,
IStorageActor,
IStorageMetadata,
} from './interfaces.storage.js';
/** /**
* Storage abstraction layer for registry * Storage abstraction layer for registry.
* Provides a unified interface over SmartBucket * Provides a unified interface over SmartBucket with optional hooks
* for quota tracking, audit logging, cache invalidation, etc.
*
* @example
* ```typescript
* // Basic usage
* const storage = new RegistryStorage(config);
*
* // With hooks for quota tracking
* const storage = new RegistryStorage(config, {
* beforePut: async (ctx) => {
* const quota = await getQuota(ctx.actor?.orgId);
* const usage = await getUsage(ctx.actor?.orgId);
* if (usage + (ctx.metadata?.size || 0) > quota) {
* return { allowed: false, reason: 'Quota exceeded' };
* }
* return { allowed: true };
* },
* afterPut: async (ctx) => {
* await updateUsage(ctx.actor?.orgId, ctx.metadata?.size || 0);
* }
* });
* ```
*/ */
export class RegistryStorage implements IStorageBackend { export class RegistryStorage implements IStorageBackend {
private smartBucket: plugins.smartbucket.SmartBucket; private smartBucket!: plugins.smartbucket.SmartBucket;
private bucket: plugins.smartbucket.Bucket; private bucket!: plugins.smartbucket.Bucket;
private bucketName: string; private bucketName: string;
private hooks?: IStorageHooks;
constructor(private config: IStorageConfig) { constructor(private config: IStorageConfig, hooks?: IStorageHooks) {
this.bucketName = config.bucketName; this.bucketName = config.bucketName;
this.hooks = hooks;
}
/**
* Set storage hooks (can be called after construction)
*/
public setHooks(hooks: IStorageHooks): void {
this.hooks = hooks;
} }
/** /**
@@ -34,7 +71,24 @@ export class RegistryStorage implements IStorageBackend {
*/ */
public async getObject(key: string): Promise<Buffer | null> { public async getObject(key: string): Promise<Buffer | null> {
try { try {
return await this.bucket.fastGet({ path: key }); const data = await this.bucket.fastGet({ path: key });
// Call afterGet hook (non-blocking)
if (this.hooks?.afterGet && data) {
const context = this.currentContext;
if (context) {
this.hooks.afterGet({
operation: 'get',
key,
protocol: context.protocol,
actor: context.actor,
metadata: context.metadata,
timestamp: new Date(),
}).catch(() => {}); // Don't fail on hook errors
}
}
return data;
} catch (error) { } catch (error) {
return null; return null;
} }
@@ -48,19 +102,159 @@ export class RegistryStorage implements IStorageBackend {
data: Buffer, data: Buffer,
metadata?: Record<string, string> metadata?: Record<string, string>
): Promise<void> { ): Promise<void> {
// Call beforePut hook if available
if (this.hooks?.beforePut) {
const context = this.currentContext;
if (context) {
const hookContext: IStorageHookContext = {
operation: 'put',
key,
protocol: context.protocol,
actor: context.actor,
metadata: {
...context.metadata,
size: data.length,
},
timestamp: new Date(),
};
const result = await this.hooks.beforePut(hookContext);
if (!result.allowed) {
throw new Error(result.reason || 'Storage operation denied by hook');
}
}
}
// Note: SmartBucket doesn't support metadata yet // Note: SmartBucket doesn't support metadata yet
await this.bucket.fastPut({ await this.bucket.fastPut({
path: key, path: key,
contents: data, contents: data,
overwrite: true, // Always overwrite existing objects overwrite: true, // Always overwrite existing objects
}); });
// Call afterPut hook (non-blocking)
if (this.hooks?.afterPut) {
const context = this.currentContext;
if (context) {
this.hooks.afterPut({
operation: 'put',
key,
protocol: context.protocol,
actor: context.actor,
metadata: {
...context.metadata,
size: data.length,
},
timestamp: new Date(),
}).catch(() => {}); // Don't fail on hook errors
}
}
} }
/** /**
* Delete an object * Delete an object
*/ */
public async deleteObject(key: string): Promise<void> { public async deleteObject(key: string): Promise<void> {
// Call beforeDelete hook if available
if (this.hooks?.beforeDelete) {
const context = this.currentContext;
if (context) {
const hookContext: IStorageHookContext = {
operation: 'delete',
key,
protocol: context.protocol,
actor: context.actor,
metadata: context.metadata,
timestamp: new Date(),
};
const result = await this.hooks.beforeDelete(hookContext);
if (!result.allowed) {
throw new Error(result.reason || 'Delete operation denied by hook');
}
}
}
await this.bucket.fastRemove({ path: key }); await this.bucket.fastRemove({ path: key });
// Call afterDelete hook (non-blocking)
if (this.hooks?.afterDelete) {
const context = this.currentContext;
if (context) {
this.hooks.afterDelete({
operation: 'delete',
key,
protocol: context.protocol,
actor: context.actor,
metadata: context.metadata,
timestamp: new Date(),
}).catch(() => {}); // Don't fail on hook errors
}
}
}
// ========================================================================
// CONTEXT FOR HOOKS
// ========================================================================
/**
* Current operation context for hooks.
* Set this before performing storage operations to enable hooks.
*/
private currentContext?: {
protocol: TRegistryProtocol;
actor?: IStorageActor;
metadata?: IStorageMetadata;
};
/**
* Set the current operation context for hooks.
* Call this before performing storage operations.
*
* @example
* ```typescript
* storage.setContext({
* protocol: 'npm',
* actor: { userId: 'user123', ip: '192.168.1.1' },
* metadata: { packageName: 'lodash', version: '4.17.21' }
* });
* await storage.putNpmTarball('lodash', '4.17.21', tarball);
* storage.clearContext();
* ```
*/
public setContext(context: {
protocol: TRegistryProtocol;
actor?: IStorageActor;
metadata?: IStorageMetadata;
}): void {
this.currentContext = context;
}
/**
* Clear the current operation context.
*/
public clearContext(): void {
this.currentContext = undefined;
}
/**
* Execute a function with a temporary context.
* Context is automatically cleared after execution.
*/
public async withContext<T>(
context: {
protocol: TRegistryProtocol;
actor?: IStorageActor;
metadata?: IStorageMetadata;
},
fn: () => Promise<T>
): Promise<T> {
this.setContext(context);
try {
return await fn();
} finally {
this.clearContext();
}
} }
/** /**
@@ -1072,4 +1266,135 @@ export class RegistryStorage implements IStorageBackend {
private getRubyGemsMetadataPath(gemName: string): string { private getRubyGemsMetadataPath(gemName: string): string {
return `rubygems/metadata/${gemName}/metadata.json`; return `rubygems/metadata/${gemName}/metadata.json`;
} }
// ========================================================================
// STREAMING METHODS (Web Streams API)
// ========================================================================
/**
* Get an object as a ReadableStream. Returns null if not found.
*/
public async getObjectStream(key: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null> {
try {
const stat = await this.bucket.fastStat({ path: key });
const size = stat.ContentLength ?? 0;
const stream = await this.bucket.fastGetStream({ path: key }, 'webstream');
// Call afterGet hook (non-blocking)
if (this.hooks?.afterGet) {
const context = this.currentContext;
if (context) {
this.hooks.afterGet({
operation: 'get',
key,
protocol: context.protocol,
actor: context.actor,
metadata: context.metadata,
timestamp: new Date(),
}).catch(() => {});
}
}
return { stream: stream as ReadableStream<Uint8Array>, size };
} catch {
return null;
}
}
/**
* Store an object from a ReadableStream.
*/
public async putObjectStream(key: string, stream: ReadableStream<Uint8Array>): Promise<void> {
if (this.hooks?.beforePut) {
const context = this.currentContext;
if (context) {
const hookContext: IStorageHookContext = {
operation: 'put',
key,
protocol: context.protocol,
actor: context.actor,
metadata: context.metadata,
timestamp: new Date(),
};
const result = await this.hooks.beforePut(hookContext);
if (!result.allowed) {
throw new Error(result.reason || 'Storage operation denied by hook');
}
}
}
// Convert WebStream to Node Readable at the S3 SDK boundary
// AWS SDK v3 PutObjectCommand requires a Node.js Readable (not WebStream)
const { Readable } = await import('stream');
const nodeStream = Readable.fromWeb(stream as any);
await this.bucket.fastPutStream({
path: key,
readableStream: nodeStream,
overwrite: true,
});
if (this.hooks?.afterPut) {
const context = this.currentContext;
if (context) {
this.hooks.afterPut({
operation: 'put',
key,
protocol: context.protocol,
actor: context.actor,
metadata: context.metadata,
timestamp: new Date(),
}).catch(() => {});
}
}
}
/**
* Get object size without reading data (S3 HEAD request).
*/
public async getObjectSize(key: string): Promise<number | null> {
try {
const stat = await this.bucket.fastStat({ path: key });
return stat.ContentLength ?? null;
} catch {
return null;
}
}
// ---- Protocol-specific streaming wrappers ----
public async getOciBlobStream(digest: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null> {
return this.getObjectStream(this.getOciBlobPath(digest));
}
public async putOciBlobStream(digest: string, stream: ReadableStream<Uint8Array>): Promise<void> {
return this.putObjectStream(this.getOciBlobPath(digest), stream);
}
public async getOciBlobSize(digest: string): Promise<number | null> {
return this.getObjectSize(this.getOciBlobPath(digest));
}
public async getNpmTarballStream(packageName: string, version: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null> {
return this.getObjectStream(this.getNpmTarballPath(packageName, version));
}
public async getMavenArtifactStream(groupId: string, artifactId: string, version: string, filename: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null> {
return this.getObjectStream(this.getMavenArtifactPath(groupId, artifactId, version, filename));
}
public async getCargoCrateStream(crateName: string, version: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null> {
return this.getObjectStream(this.getCargoCratePath(crateName, version));
}
public async getComposerPackageZipStream(vendorPackage: string, reference: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null> {
return this.getObjectStream(this.getComposerZipPath(vendorPackage, reference));
}
public async getPypiPackageFileStream(packageName: string, filename: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null> {
return this.getObjectStream(this.getPypiPackageFilePath(packageName, filename));
}
public async getRubyGemsGemStream(gemName: string, version: string, platform?: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null> {
return this.getObjectStream(this.getRubyGemsGemPath(gemName, version, platform));
}
} }

63
ts/core/helpers.stream.ts Normal file
View File

@@ -0,0 +1,63 @@
import * as crypto from 'crypto';
/**
* Convert Buffer, Uint8Array, string, or JSON object to a ReadableStream<Uint8Array>.
*/
export function toReadableStream(data: Buffer | Uint8Array | string | object): ReadableStream<Uint8Array> {
const buf = Buffer.isBuffer(data)
? data
: data instanceof Uint8Array
? Buffer.from(data)
: typeof data === 'string'
? Buffer.from(data, 'utf-8')
: Buffer.from(JSON.stringify(data), 'utf-8');
return new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(new Uint8Array(buf));
controller.close();
},
});
}
/**
* Consume a ReadableStream into a Buffer.
*/
export async function streamToBuffer(stream: ReadableStream<Uint8Array>): Promise<Buffer> {
const reader = stream.getReader();
const chunks: Uint8Array[] = [];
while (true) {
const { done, value } = await reader.read();
if (done) break;
if (value) chunks.push(value);
}
return Buffer.concat(chunks);
}
/**
* Consume a ReadableStream into a parsed JSON object.
*/
export async function streamToJson<T = any>(stream: ReadableStream<Uint8Array>): Promise<T> {
const buf = await streamToBuffer(stream);
return JSON.parse(buf.toString('utf-8'));
}
/**
* Create a TransformStream that incrementally hashes data passing through.
* Data flows through unchanged; the digest is available after the stream completes.
*/
export function createHashTransform(algorithm: string = 'sha256'): {
transform: TransformStream<Uint8Array, Uint8Array>;
getDigest: () => string;
} {
const hash = crypto.createHash(algorithm);
const transform = new TransformStream<Uint8Array, Uint8Array>({
transform(chunk, controller) {
hash.update(chunk);
controller.enqueue(chunk);
},
});
return {
transform,
getDigest: () => hash.digest('hex'),
};
}

View File

@@ -2,9 +2,19 @@
* Core registry infrastructure exports * Core registry infrastructure exports
*/ */
// Interfaces // Core interfaces
export * from './interfaces.core.js'; export * from './interfaces.core.js';
// Auth interfaces and provider
export * from './interfaces.auth.js';
export { DefaultAuthProvider } from './classes.defaultauthprovider.js';
// Storage interfaces and hooks
export * from './interfaces.storage.js';
// Stream helpers
export { toReadableStream, streamToBuffer, streamToJson, createHashTransform } from './helpers.stream.js';
// Classes // Classes
export { BaseRegistry } from './classes.baseregistry.js'; export { BaseRegistry } from './classes.baseregistry.js';
export { RegistryStorage } from './classes.registrystorage.js'; export { RegistryStorage } from './classes.registrystorage.js';

View File

@@ -0,0 +1,91 @@
import type { IAuthToken, ICredentials, TRegistryProtocol } from './interfaces.core.js';
/**
* Options for creating a token
*/
export interface ITokenOptions {
/** Whether the token is readonly */
readonly?: boolean;
/** Permission scopes */
scopes?: string[];
/** Expiration time in seconds */
expiresIn?: number;
}
/**
* Pluggable authentication provider interface.
* Implement this to integrate external auth systems (LDAP, OAuth, SSO, OIDC).
*
* @example
* ```typescript
* class LdapAuthProvider implements IAuthProvider {
* constructor(private ldap: LdapClient, private redis: RedisClient) {}
*
* async authenticate(credentials: ICredentials): Promise<string | null> {
* return await this.ldap.bind(credentials.username, credentials.password);
* }
*
* async validateToken(token: string): Promise<IAuthToken | null> {
* return await this.redis.get(`token:${token}`);
* }
* // ...
* }
* ```
*/
export interface IAuthProvider {
/**
* Initialize the auth provider (optional)
*/
init?(): Promise<void>;
/**
* Authenticate user credentials (login flow)
* @param credentials - Username and password
* @returns User ID on success, null on failure
*/
authenticate(credentials: ICredentials): Promise<string | null>;
/**
* Validate an existing token
* @param token - Token string (UUID or JWT)
* @param protocol - Optional protocol hint for optimization
* @returns Auth token info or null if invalid
*/
validateToken(token: string, protocol?: TRegistryProtocol): Promise<IAuthToken | null>;
/**
* Create a new token for a user
* @param userId - User ID
* @param protocol - Protocol type (npm, oci, maven, etc.)
* @param options - Token options (readonly, scopes, expiration)
* @returns Token string
*/
createToken(userId: string, protocol: TRegistryProtocol, options?: ITokenOptions): Promise<string>;
/**
* Revoke a token
* @param token - Token string to revoke
*/
revokeToken(token: string): Promise<void>;
/**
* Check if user has permission for an action
* @param token - Auth token (or null for anonymous)
* @param resource - Resource being accessed (e.g., "npm:package:lodash")
* @param action - Action being performed (read, write, push, pull, delete)
* @returns true if authorized
*/
authorize(token: IAuthToken | null, resource: string, action: string): Promise<boolean>;
/**
* List all tokens for a user (optional)
* @param userId - User ID
* @returns List of token info
*/
listUserTokens?(userId: string): Promise<Array<{
key: string;
readonly: boolean;
created: string;
protocol?: TRegistryProtocol;
}>>;
}

View File

@@ -3,7 +3,9 @@
*/ */
import type * as plugins from '../plugins.js'; import type * as plugins from '../plugins.js';
import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js'; import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
import type { IAuthProvider } from './interfaces.auth.js';
import type { IStorageHooks } from './interfaces.storage.js';
/** /**
* Registry protocol types * Registry protocol types
@@ -86,9 +88,8 @@ export interface IAuthConfig {
export interface IProtocolConfig { export interface IProtocolConfig {
enabled: boolean; enabled: boolean;
basePath: string; basePath: string;
registryUrl?: string;
features?: Record<string, boolean>; features?: Record<string, boolean>;
/** Upstream registry configuration for proxying/caching */
upstream?: IProtocolUpstreamConfig;
} }
/** /**
@@ -97,6 +98,27 @@ export interface IProtocolConfig {
export interface IRegistryConfig { export interface IRegistryConfig {
storage: IStorageConfig; storage: IStorageConfig;
auth: IAuthConfig; auth: IAuthConfig;
/**
* Custom authentication provider.
* If not provided, uses the default in-memory auth provider.
* Implement IAuthProvider to integrate LDAP, OAuth, SSO, etc.
*/
authProvider?: IAuthProvider;
/**
* Storage event hooks for quota tracking, audit logging, etc.
* Called before/after storage operations.
*/
storageHooks?: IStorageHooks;
/**
* Dynamic upstream configuration provider.
* Called per-request to resolve which upstream registries to use.
* Use StaticUpstreamProvider for simple static configurations.
*/
upstreamProvider?: IUpstreamProvider;
oci?: IProtocolConfig; oci?: IProtocolConfig;
npm?: IProtocolConfig; npm?: IProtocolConfig;
maven?: IProtocolConfig; maven?: IProtocolConfig;
@@ -139,6 +161,21 @@ export interface IStorageBackend {
* Get object metadata * Get object metadata
*/ */
getMetadata(key: string): Promise<Record<string, string> | null>; getMetadata(key: string): Promise<Record<string, string> | null>;
/**
* Get an object as a ReadableStream. Returns null if not found.
*/
getObjectStream?(key: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null>;
/**
* Store an object from a ReadableStream.
*/
putObjectStream?(key: string, stream: ReadableStream<Uint8Array>): Promise<void>;
/**
* Get object size without reading data (S3 HEAD request).
*/
getObjectSize?(key: string): Promise<number | null>;
} }
/** /**
@@ -152,6 +189,24 @@ export interface IRegistryError {
}>; }>;
} }
/**
* Actor information - identifies who is performing the request
*/
export interface IRequestActor {
/** User ID (from validated token) */
userId?: string;
/** Token ID/hash for audit purposes */
tokenId?: string;
/** Client IP address */
ip?: string;
/** Client User-Agent */
userAgent?: string;
/** Organization ID (for multi-tenant setups) */
orgId?: string;
/** Session ID */
sessionId?: string;
}
/** /**
* Base request context * Base request context
*/ */
@@ -168,13 +223,21 @@ export interface IRequestContext {
*/ */
rawBody?: Buffer; rawBody?: Buffer;
token?: string; token?: string;
/**
* Actor information - identifies who is performing the request.
* Populated after authentication for audit logging, quota enforcement, etc.
*/
actor?: IRequestActor;
} }
/** /**
* Base response structure * Base response structure.
* `body` is always a `ReadableStream<Uint8Array>` at the public API boundary.
* Internal handlers may return Buffer/string/object — the SmartRegistry orchestrator
* auto-wraps them via `toReadableStream()` before returning to the caller.
*/ */
export interface IResponse { export interface IResponse {
status: number; status: number;
headers: Record<string, string>; headers: Record<string, string>;
body?: any; body?: ReadableStream<Uint8Array> | any;
} }

View File

@@ -0,0 +1,130 @@
import type { TRegistryProtocol } from './interfaces.core.js';
/**
* Actor information from request context
*/
export interface IStorageActor {
userId?: string;
tokenId?: string;
ip?: string;
userAgent?: string;
orgId?: string;
sessionId?: string;
}
/**
* Metadata about the storage operation
*/
export interface IStorageMetadata {
/** Content type of the object */
contentType?: string;
/** Size in bytes */
size?: number;
/** Content digest (e.g., sha256:abc123) */
digest?: string;
/** Package/artifact name */
packageName?: string;
/** Version */
version?: string;
}
/**
* Context passed to storage hooks
*/
export interface IStorageHookContext {
/** Type of operation */
operation: 'put' | 'delete' | 'get';
/** Storage key/path */
key: string;
/** Protocol that triggered this operation */
protocol: TRegistryProtocol;
/** Actor who performed the operation (if known) */
actor?: IStorageActor;
/** Metadata about the object */
metadata?: IStorageMetadata;
/** Timestamp of the operation */
timestamp: Date;
}
/**
* Result from a beforePut hook that can modify the operation
*/
export interface IBeforePutResult {
/** Whether to allow the operation */
allowed: boolean;
/** Optional reason for rejection */
reason?: string;
/** Optional modified metadata */
metadata?: IStorageMetadata;
}
/**
* Result from a beforeDelete hook
*/
export interface IBeforeDeleteResult {
/** Whether to allow the operation */
allowed: boolean;
/** Optional reason for rejection */
reason?: string;
}
/**
* Storage event hooks for quota tracking, audit logging, cache invalidation, etc.
*
* @example
* ```typescript
* const quotaHooks: IStorageHooks = {
* async beforePut(context) {
* const quota = await getQuota(context.actor?.orgId);
* const currentUsage = await getUsage(context.actor?.orgId);
* if (currentUsage + (context.metadata?.size || 0) > quota) {
* return { allowed: false, reason: 'Quota exceeded' };
* }
* return { allowed: true };
* },
*
* async afterPut(context) {
* await updateUsage(context.actor?.orgId, context.metadata?.size || 0);
* await auditLog('storage.put', context);
* },
*
* async afterDelete(context) {
* await invalidateCache(context.key);
* }
* };
* ```
*/
export interface IStorageHooks {
/**
* Called before storing an object.
* Return { allowed: false } to reject the operation.
* Use for quota checks, virus scanning, validation, etc.
*/
beforePut?(context: IStorageHookContext): Promise<IBeforePutResult>;
/**
* Called after successfully storing an object.
* Use for quota tracking, audit logging, notifications, etc.
*/
afterPut?(context: IStorageHookContext): Promise<void>;
/**
* Called before deleting an object.
* Return { allowed: false } to reject the operation.
* Use for preventing deletion of protected resources.
*/
beforeDelete?(context: IStorageHookContext): Promise<IBeforeDeleteResult>;
/**
* Called after successfully deleting an object.
* Use for quota updates, audit logging, cache invalidation, etc.
*/
afterDelete?(context: IStorageHookContext): Promise<void>;
/**
* Called after reading an object.
* Use for access logging, analytics, etc.
* Note: This is called even for cache hits.
*/
afterGet?(context: IStorageHookContext): Promise<void>;
}

View File

@@ -6,8 +6,8 @@
import { BaseRegistry } from '../core/classes.baseregistry.js'; import { BaseRegistry } from '../core/classes.baseregistry.js';
import type { RegistryStorage } from '../core/classes.registrystorage.js'; import type { RegistryStorage } from '../core/classes.registrystorage.js';
import type { AuthManager } from '../core/classes.authmanager.js'; import type { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js'; import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from '../core/interfaces.core.js';
import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js'; import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
import { toBuffer } from '../core/helpers.buffer.js'; import { toBuffer } from '../core/helpers.buffer.js';
import type { IMavenCoordinate, IMavenMetadata, IChecksums } from './interfaces.maven.js'; import type { IMavenCoordinate, IMavenMetadata, IChecksums } from './interfaces.maven.js';
import { import {
@@ -33,34 +33,64 @@ export class MavenRegistry extends BaseRegistry {
private authManager: AuthManager; private authManager: AuthManager;
private basePath: string = '/maven'; private basePath: string = '/maven';
private registryUrl: string; private registryUrl: string;
private upstream: MavenUpstream | null = null; private upstreamProvider: IUpstreamProvider | null = null;
constructor( constructor(
storage: RegistryStorage, storage: RegistryStorage,
authManager: AuthManager, authManager: AuthManager,
basePath: string, basePath: string,
registryUrl: string, registryUrl: string,
upstreamConfig?: IProtocolUpstreamConfig upstreamProvider?: IUpstreamProvider
) { ) {
super(); super();
this.storage = storage; this.storage = storage;
this.authManager = authManager; this.authManager = authManager;
this.basePath = basePath; this.basePath = basePath;
this.registryUrl = registryUrl; this.registryUrl = registryUrl;
this.upstreamProvider = upstreamProvider || null;
}
// Initialize upstream if configured /**
if (upstreamConfig?.enabled) { * Extract scope from Maven coordinates.
this.upstream = new MavenUpstream(upstreamConfig); * For Maven, the groupId is the scope.
} * @example "com.example" from "com.example:my-lib"
*/
private extractScope(groupId: string): string | null {
return groupId || null;
}
/**
* Get upstream for a specific request.
* Calls the provider to resolve upstream config dynamically.
*/
private async getUpstreamForRequest(
resource: string,
resourceType: string,
method: string,
actor?: IRequestActor
): Promise<MavenUpstream | null> {
if (!this.upstreamProvider) return null;
// For Maven, resource is "groupId:artifactId"
const [groupId] = resource.split(':');
const config = await this.upstreamProvider.resolveUpstreamConfig({
protocol: 'maven',
resource,
scope: this.extractScope(groupId),
actor,
method,
resourceType,
});
if (!config?.enabled) return null;
return new MavenUpstream(config);
} }
/** /**
* Clean up resources (timers, connections, etc.) * Clean up resources (timers, connections, etc.)
*/ */
public destroy(): void { public destroy(): void {
if (this.upstream) { // No persistent upstream to clean up with dynamic provider
this.upstream.stop();
}
} }
public async init(): Promise<void> { public async init(): Promise<void> {
@@ -85,13 +115,21 @@ export class MavenRegistry extends BaseRegistry {
token = await this.authManager.validateToken(tokenString, 'maven'); token = await this.authManager.validateToken(tokenString, 'maven');
} }
// Build actor from context and validated token
const actor: IRequestActor = {
...context.actor,
userId: token?.userId,
ip: context.headers['x-forwarded-for'] || context.headers['X-Forwarded-For'],
userAgent: context.headers['user-agent'] || context.headers['User-Agent'],
};
// Parse path to determine request type // Parse path to determine request type
const coordinate = pathToGAV(path); const coordinate = pathToGAV(path);
if (!coordinate) { if (!coordinate) {
// Not a valid artifact path, could be metadata or root // Not a valid artifact path, could be metadata or root
if (path.endsWith('/maven-metadata.xml')) { if (path.endsWith('/maven-metadata.xml')) {
return this.handleMetadataRequest(context.method, path, token); return this.handleMetadataRequest(context.method, path, token, actor);
} }
return { return {
@@ -108,7 +146,7 @@ export class MavenRegistry extends BaseRegistry {
} }
// Handle artifact requests (JAR, POM, WAR, etc.) // Handle artifact requests (JAR, POM, WAR, etc.)
return this.handleArtifactRequest(context.method, coordinate, token, context.body); return this.handleArtifactRequest(context.method, coordinate, token, context.body, actor);
} }
protected async checkPermission( protected async checkPermission(
@@ -128,7 +166,8 @@ export class MavenRegistry extends BaseRegistry {
method: string, method: string,
coordinate: IMavenCoordinate, coordinate: IMavenCoordinate,
token: IAuthToken | null, token: IAuthToken | null,
body?: Buffer | any body?: Buffer | any,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
const { groupId, artifactId, version } = coordinate; const { groupId, artifactId, version } = coordinate;
const filename = buildFilename(coordinate); const filename = buildFilename(coordinate);
@@ -139,7 +178,7 @@ export class MavenRegistry extends BaseRegistry {
case 'HEAD': case 'HEAD':
// Maven repositories typically allow anonymous reads // Maven repositories typically allow anonymous reads
return method === 'GET' return method === 'GET'
? this.getArtifact(groupId, artifactId, version, filename) ? this.getArtifact(groupId, artifactId, version, filename, actor)
: this.headArtifact(groupId, artifactId, version, filename); : this.headArtifact(groupId, artifactId, version, filename);
case 'PUT': case 'PUT':
@@ -211,7 +250,8 @@ export class MavenRegistry extends BaseRegistry {
private async handleMetadataRequest( private async handleMetadataRequest(
method: string, method: string,
path: string, path: string,
token: IAuthToken | null token: IAuthToken | null,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
// Parse path to extract groupId and artifactId // Parse path to extract groupId and artifactId
// Path format: /com/example/my-lib/maven-metadata.xml // Path format: /com/example/my-lib/maven-metadata.xml
@@ -232,7 +272,7 @@ export class MavenRegistry extends BaseRegistry {
if (method === 'GET') { if (method === 'GET') {
// Metadata is usually public (read permission optional) // Metadata is usually public (read permission optional)
// Some registries allow anonymous metadata access // Some registries allow anonymous metadata access
return this.getMetadata(groupId, artifactId); return this.getMetadata(groupId, artifactId, actor);
} }
return { return {
@@ -250,16 +290,33 @@ export class MavenRegistry extends BaseRegistry {
groupId: string, groupId: string,
artifactId: string, artifactId: string,
version: string, version: string,
filename: string filename: string,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
let data = await this.storage.getMavenArtifact(groupId, artifactId, version, filename); // Try local storage first (streaming)
const streamResult = await this.storage.getMavenArtifactStream(groupId, artifactId, version, filename);
if (streamResult) {
const ext = filename.split('.').pop() || '';
const contentType = this.getContentType(ext);
return {
status: 200,
headers: {
'Content-Type': contentType,
'Content-Length': streamResult.size.toString(),
},
body: streamResult.stream,
};
}
// Try upstream if not found locally // Try upstream if not found locally
if (!data && this.upstream) { let data: Buffer | null = null;
const resource = `${groupId}:${artifactId}`;
const upstream = await this.getUpstreamForRequest(resource, 'artifact', 'GET', actor);
if (upstream) {
// Parse the filename to extract extension and classifier // Parse the filename to extract extension and classifier
const { extension, classifier } = this.parseFilename(filename, artifactId, version); const { extension, classifier } = this.parseFilename(filename, artifactId, version);
if (extension) { if (extension) {
data = await this.upstream.fetchArtifact(groupId, artifactId, version, extension, classifier); data = await upstream.fetchArtifact(groupId, artifactId, version, extension, classifier);
if (data) { if (data) {
// Cache the artifact locally // Cache the artifact locally
await this.storage.putMavenArtifact(groupId, artifactId, version, filename, data); await this.storage.putMavenArtifact(groupId, artifactId, version, filename, data);
@@ -495,16 +552,20 @@ export class MavenRegistry extends BaseRegistry {
// METADATA OPERATIONS // METADATA OPERATIONS
// ======================================================================== // ========================================================================
private async getMetadata(groupId: string, artifactId: string): Promise<IResponse> { private async getMetadata(groupId: string, artifactId: string, actor?: IRequestActor): Promise<IResponse> {
let metadataBuffer = await this.storage.getMavenMetadata(groupId, artifactId); let metadataBuffer = await this.storage.getMavenMetadata(groupId, artifactId);
// Try upstream if not found locally // Try upstream if not found locally
if (!metadataBuffer && this.upstream) { if (!metadataBuffer) {
const upstreamMetadata = await this.upstream.fetchMetadata(groupId, artifactId); const resource = `${groupId}:${artifactId}`;
if (upstreamMetadata) { const upstream = await this.getUpstreamForRequest(resource, 'metadata', 'GET', actor);
metadataBuffer = Buffer.from(upstreamMetadata, 'utf-8'); if (upstream) {
// Cache the metadata locally const upstreamMetadata = await upstream.fetchMetadata(groupId, artifactId);
await this.storage.putMavenMetadata(groupId, artifactId, metadataBuffer); if (upstreamMetadata) {
metadataBuffer = Buffer.from(upstreamMetadata, 'utf-8');
// Cache the metadata locally
await this.storage.putMavenMetadata(groupId, artifactId, metadataBuffer);
}
} }
} }

View File

@@ -2,8 +2,8 @@ import { Smartlog } from '@push.rocks/smartlog';
import { BaseRegistry } from '../core/classes.baseregistry.js'; import { BaseRegistry } from '../core/classes.baseregistry.js';
import { RegistryStorage } from '../core/classes.registrystorage.js'; import { RegistryStorage } from '../core/classes.registrystorage.js';
import { AuthManager } from '../core/classes.authmanager.js'; import { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js'; import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from '../core/interfaces.core.js';
import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js'; import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
import { NpmUpstream } from './classes.npmupstream.js'; import { NpmUpstream } from './classes.npmupstream.js';
import type { import type {
IPackument, IPackument,
@@ -27,20 +27,21 @@ export class NpmRegistry extends BaseRegistry {
private basePath: string = '/npm'; private basePath: string = '/npm';
private registryUrl: string; private registryUrl: string;
private logger: Smartlog; private logger: Smartlog;
private upstream: NpmUpstream | null = null; private upstreamProvider: IUpstreamProvider | null = null;
constructor( constructor(
storage: RegistryStorage, storage: RegistryStorage,
authManager: AuthManager, authManager: AuthManager,
basePath: string = '/npm', basePath: string = '/npm',
registryUrl: string = 'http://localhost:5000/npm', registryUrl: string = 'http://localhost:5000/npm',
upstreamConfig?: IProtocolUpstreamConfig upstreamProvider?: IUpstreamProvider
) { ) {
super(); super();
this.storage = storage; this.storage = storage;
this.authManager = authManager; this.authManager = authManager;
this.basePath = basePath; this.basePath = basePath;
this.registryUrl = registryUrl; this.registryUrl = registryUrl;
this.upstreamProvider = upstreamProvider || null;
// Initialize logger // Initialize logger
this.logger = new Smartlog({ this.logger = new Smartlog({
@@ -55,15 +56,51 @@ export class NpmRegistry extends BaseRegistry {
}); });
this.logger.enableConsole(); this.logger.enableConsole();
// Initialize upstream if configured if (upstreamProvider) {
if (upstreamConfig?.enabled) { this.logger.log('info', 'NPM upstream provider configured');
this.upstream = new NpmUpstream(upstreamConfig, registryUrl, this.logger);
this.logger.log('info', 'NPM upstream initialized', {
upstreams: upstreamConfig.upstreams.map(u => u.name),
});
} }
} }
/**
* Extract scope from npm package name.
* @example "@company/utils" -> "company"
* @example "lodash" -> null
*/
private extractScope(packageName: string): string | null {
if (packageName.startsWith('@')) {
const slashIndex = packageName.indexOf('/');
if (slashIndex > 1) {
return packageName.substring(1, slashIndex);
}
}
return null;
}
/**
* Get upstream for a specific request.
* Calls the provider to resolve upstream config dynamically.
*/
private async getUpstreamForRequest(
resource: string,
resourceType: string,
method: string,
actor?: IRequestActor
): Promise<NpmUpstream | null> {
if (!this.upstreamProvider) return null;
const config = await this.upstreamProvider.resolveUpstreamConfig({
protocol: 'npm',
resource,
scope: this.extractScope(resource),
actor,
method,
resourceType,
});
if (!config?.enabled) return null;
return new NpmUpstream(config, this.registryUrl, this.logger);
}
public async init(): Promise<void> { public async init(): Promise<void> {
// NPM registry initialization // NPM registry initialization
} }
@@ -80,6 +117,14 @@ export class NpmRegistry extends BaseRegistry {
const tokenString = authHeader?.replace(/^Bearer\s+/i, ''); const tokenString = authHeader?.replace(/^Bearer\s+/i, '');
const token = tokenString ? await this.authManager.validateToken(tokenString, 'npm') : null; const token = tokenString ? await this.authManager.validateToken(tokenString, 'npm') : null;
// Build actor context for upstream resolution
const actor: IRequestActor = {
userId: token?.userId,
ip: context.headers['x-forwarded-for'] || context.headers['x-real-ip'],
userAgent: context.headers['user-agent'],
...context.actor, // Include any pre-populated actor info
};
this.logger.log('debug', `handleRequest: ${context.method} ${path}`, { this.logger.log('debug', `handleRequest: ${context.method} ${path}`, {
method: context.method, method: context.method,
path, path,
@@ -110,47 +155,47 @@ export class NpmRegistry extends BaseRegistry {
// Dist-tags: /-/package/{package}/dist-tags // Dist-tags: /-/package/{package}/dist-tags
const distTagsMatch = path.match(/^\/-\/package\/(@?[^\/]+(?:\/[^\/]+)?)\/dist-tags(?:\/(.+))?$/); const distTagsMatch = path.match(/^\/-\/package\/(@?[^\/]+(?:\/[^\/]+)?)\/dist-tags(?:\/(.+))?$/);
if (distTagsMatch) { if (distTagsMatch) {
const [, packageName, tag] = distTagsMatch; const [, rawPkgName, tag] = distTagsMatch;
return this.handleDistTags(context.method, packageName, tag, context.body, token); return this.handleDistTags(context.method, decodeURIComponent(rawPkgName), tag, context.body, token);
} }
// Tarball download: /{package}/-/{filename}.tgz // Tarball download: /{package}/-/{filename}.tgz
const tarballMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)\/-\/(.+\.tgz)$/); const tarballMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)\/-\/(.+\.tgz)$/);
if (tarballMatch) { if (tarballMatch) {
const [, packageName, filename] = tarballMatch; const [, rawPkgName, filename] = tarballMatch;
return this.handleTarballDownload(packageName, filename, token); return this.handleTarballDownload(decodeURIComponent(rawPkgName), filename, token, actor);
} }
// Unpublish specific version: DELETE /{package}/-/{version} // Unpublish specific version: DELETE /{package}/-/{version}
const unpublishVersionMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)\/-\/([^\/]+)$/); const unpublishVersionMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)\/-\/([^\/]+)$/);
if (unpublishVersionMatch && context.method === 'DELETE') { if (unpublishVersionMatch && context.method === 'DELETE') {
const [, packageName, version] = unpublishVersionMatch; const [, rawPkgName, version] = unpublishVersionMatch;
this.logger.log('debug', 'unpublishVersionMatch', { packageName, version }); this.logger.log('debug', 'unpublishVersionMatch', { packageName: decodeURIComponent(rawPkgName), version });
return this.unpublishVersion(packageName, version, token); return this.unpublishVersion(decodeURIComponent(rawPkgName), version, token);
} }
// Unpublish entire package: DELETE /{package}/-rev/{rev} // Unpublish entire package: DELETE /{package}/-rev/{rev}
const unpublishPackageMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)\/-rev\/([^\/]+)$/); const unpublishPackageMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)\/-rev\/([^\/]+)$/);
if (unpublishPackageMatch && context.method === 'DELETE') { if (unpublishPackageMatch && context.method === 'DELETE') {
const [, packageName, rev] = unpublishPackageMatch; const [, rawPkgName, rev] = unpublishPackageMatch;
this.logger.log('debug', 'unpublishPackageMatch', { packageName, rev }); this.logger.log('debug', 'unpublishPackageMatch', { packageName: decodeURIComponent(rawPkgName), rev });
return this.unpublishPackage(packageName, token); return this.unpublishPackage(decodeURIComponent(rawPkgName), token);
} }
// Package version: /{package}/{version} // Package version: /{package}/{version}
const versionMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)\/([^\/]+)$/); const versionMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)\/([^\/]+)$/);
if (versionMatch) { if (versionMatch) {
const [, packageName, version] = versionMatch; const [, rawPkgName, version] = versionMatch;
this.logger.log('debug', 'versionMatch', { packageName, version }); this.logger.log('debug', 'versionMatch', { packageName: decodeURIComponent(rawPkgName), version });
return this.handlePackageVersion(packageName, version, token); return this.handlePackageVersion(decodeURIComponent(rawPkgName), version, token, actor);
} }
// Package operations: /{package} // Package operations: /{package}
const packageMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)$/); const packageMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)$/);
if (packageMatch) { if (packageMatch) {
const packageName = packageMatch[1]; const packageName = decodeURIComponent(packageMatch[1]);
this.logger.log('debug', 'packageMatch', { packageName }); this.logger.log('debug', 'packageMatch', { packageName });
return this.handlePackage(context.method, packageName, context.body, context.query, token); return this.handlePackage(context.method, packageName, context.body, context.query, token, actor);
} }
return { return {
@@ -198,11 +243,12 @@ export class NpmRegistry extends BaseRegistry {
packageName: string, packageName: string,
body: any, body: any,
query: Record<string, string>, query: Record<string, string>,
token: IAuthToken | null token: IAuthToken | null,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
switch (method) { switch (method) {
case 'GET': case 'GET':
return this.getPackument(packageName, token, query); return this.getPackument(packageName, token, query, actor);
case 'PUT': case 'PUT':
return this.publishPackage(packageName, body, token); return this.publishPackage(packageName, body, token);
case 'DELETE': case 'DELETE':
@@ -219,7 +265,8 @@ export class NpmRegistry extends BaseRegistry {
private async getPackument( private async getPackument(
packageName: string, packageName: string,
token: IAuthToken | null, token: IAuthToken | null,
query: Record<string, string> query: Record<string, string>,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
let packument = await this.storage.getNpmPackument(packageName); let packument = await this.storage.getNpmPackument(packageName);
this.logger.log('debug', `getPackument: ${packageName}`, { this.logger.log('debug', `getPackument: ${packageName}`, {
@@ -229,17 +276,20 @@ export class NpmRegistry extends BaseRegistry {
}); });
// If not found locally, try upstream // If not found locally, try upstream
if (!packument && this.upstream) { if (!packument) {
this.logger.log('debug', `getPackument: fetching from upstream`, { packageName }); const upstream = await this.getUpstreamForRequest(packageName, 'packument', 'GET', actor);
const upstreamPackument = await this.upstream.fetchPackument(packageName); if (upstream) {
if (upstreamPackument) { this.logger.log('debug', `getPackument: fetching from upstream`, { packageName });
this.logger.log('debug', `getPackument: found in upstream`, { const upstreamPackument = await upstream.fetchPackument(packageName);
packageName, if (upstreamPackument) {
versions: Object.keys(upstreamPackument.versions || {}).length this.logger.log('debug', `getPackument: found in upstream`, {
}); packageName,
packument = upstreamPackument; versions: Object.keys(upstreamPackument.versions || {}).length
// Optionally cache the packument locally (without tarballs) });
// We don't store tarballs here - they'll be fetched on demand packument = upstreamPackument;
// Optionally cache the packument locally (without tarballs)
// We don't store tarballs here - they'll be fetched on demand
}
} }
} }
@@ -279,7 +329,8 @@ export class NpmRegistry extends BaseRegistry {
private async handlePackageVersion( private async handlePackageVersion(
packageName: string, packageName: string,
version: string, version: string,
token: IAuthToken | null token: IAuthToken | null,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
this.logger.log('debug', 'handlePackageVersion', { packageName, version }); this.logger.log('debug', 'handlePackageVersion', { packageName, version });
let packument = await this.storage.getNpmPackument(packageName); let packument = await this.storage.getNpmPackument(packageName);
@@ -289,11 +340,14 @@ export class NpmRegistry extends BaseRegistry {
} }
// If not found locally, try upstream // If not found locally, try upstream
if (!packument && this.upstream) { if (!packument) {
this.logger.log('debug', 'handlePackageVersion: fetching from upstream', { packageName }); const upstream = await this.getUpstreamForRequest(packageName, 'packument', 'GET', actor);
const upstreamPackument = await this.upstream.fetchPackument(packageName); if (upstream) {
if (upstreamPackument) { this.logger.log('debug', 'handlePackageVersion: fetching from upstream', { packageName });
packument = upstreamPackument; const upstreamPackument = await upstream.fetchPackument(packageName);
if (upstreamPackument) {
packument = upstreamPackument;
}
} }
} }
@@ -563,7 +617,8 @@ export class NpmRegistry extends BaseRegistry {
private async handleTarballDownload( private async handleTarballDownload(
packageName: string, packageName: string,
filename: string, filename: string,
token: IAuthToken | null token: IAuthToken | null,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
// Extract version from filename: package-name-1.0.0.tgz // Extract version from filename: package-name-1.0.0.tgz
const versionMatch = filename.match(/-([\d.]+(?:-[a-z0-9.]+)?)\.tgz$/i); const versionMatch = filename.match(/-([\d.]+(?:-[a-z0-9.]+)?)\.tgz$/i);
@@ -576,15 +631,29 @@ export class NpmRegistry extends BaseRegistry {
} }
const version = versionMatch[1]; const version = versionMatch[1];
let tarball = await this.storage.getNpmTarball(packageName, version);
// Try local storage first (streaming)
const streamResult = await this.storage.getNpmTarballStream(packageName, version);
if (streamResult) {
return {
status: 200,
headers: {
'Content-Type': 'application/octet-stream',
'Content-Length': streamResult.size.toString(),
},
body: streamResult.stream,
};
}
// If not found locally, try upstream // If not found locally, try upstream
if (!tarball && this.upstream) { let tarball: Buffer | null = null;
const upstream = await this.getUpstreamForRequest(packageName, 'tarball', 'GET', actor);
if (upstream) {
this.logger.log('debug', 'handleTarballDownload: fetching from upstream', { this.logger.log('debug', 'handleTarballDownload: fetching from upstream', {
packageName, packageName,
version, version,
}); });
const upstreamTarball = await this.upstream.fetchTarball(packageName, version); const upstreamTarball = await upstream.fetchTarball(packageName, version);
if (upstreamTarball) { if (upstreamTarball) {
tarball = upstreamTarball; tarball = upstreamTarball;
// Cache the tarball locally for future requests // Cache the tarball locally for future requests

View File

@@ -2,8 +2,9 @@ import { Smartlog } from '@push.rocks/smartlog';
import { BaseRegistry } from '../core/classes.baseregistry.js'; import { BaseRegistry } from '../core/classes.baseregistry.js';
import { RegistryStorage } from '../core/classes.registrystorage.js'; import { RegistryStorage } from '../core/classes.registrystorage.js';
import { AuthManager } from '../core/classes.authmanager.js'; import { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken, IRegistryError } from '../core/interfaces.core.js'; import type { IRequestContext, IResponse, IAuthToken, IRegistryError, IRequestActor } from '../core/interfaces.core.js';
import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js'; import { createHashTransform, streamToBuffer } from '../core/helpers.stream.js';
import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
import { OciUpstream } from './classes.ociupstream.js'; import { OciUpstream } from './classes.ociupstream.js';
import type { import type {
IUploadSession, IUploadSession,
@@ -24,7 +25,7 @@ export class OciRegistry extends BaseRegistry {
private basePath: string = '/oci'; private basePath: string = '/oci';
private cleanupInterval?: NodeJS.Timeout; private cleanupInterval?: NodeJS.Timeout;
private ociTokens?: { realm: string; service: string }; private ociTokens?: { realm: string; service: string };
private upstream: OciUpstream | null = null; private upstreamProvider: IUpstreamProvider | null = null;
private logger: Smartlog; private logger: Smartlog;
constructor( constructor(
@@ -32,13 +33,14 @@ export class OciRegistry extends BaseRegistry {
authManager: AuthManager, authManager: AuthManager,
basePath: string = '/oci', basePath: string = '/oci',
ociTokens?: { realm: string; service: string }, ociTokens?: { realm: string; service: string },
upstreamConfig?: IProtocolUpstreamConfig upstreamProvider?: IUpstreamProvider
) { ) {
super(); super();
this.storage = storage; this.storage = storage;
this.authManager = authManager; this.authManager = authManager;
this.basePath = basePath; this.basePath = basePath;
this.ociTokens = ociTokens; this.ociTokens = ociTokens;
this.upstreamProvider = upstreamProvider || null;
// Initialize logger // Initialize logger
this.logger = new Smartlog({ this.logger = new Smartlog({
@@ -53,15 +55,50 @@ export class OciRegistry extends BaseRegistry {
}); });
this.logger.enableConsole(); this.logger.enableConsole();
// Initialize upstream if configured if (upstreamProvider) {
if (upstreamConfig?.enabled) { this.logger.log('info', 'OCI upstream provider configured');
this.upstream = new OciUpstream(upstreamConfig, basePath, this.logger);
this.logger.log('info', 'OCI upstream initialized', {
upstreams: upstreamConfig.upstreams.map(u => u.name),
});
} }
} }
/**
* Extract scope from OCI repository name.
* @example "myorg/myimage" -> "myorg"
* @example "library/nginx" -> "library"
* @example "nginx" -> null
*/
private extractScope(repository: string): string | null {
const slashIndex = repository.indexOf('/');
if (slashIndex > 0) {
return repository.substring(0, slashIndex);
}
return null;
}
/**
* Get upstream for a specific request.
* Calls the provider to resolve upstream config dynamically.
*/
private async getUpstreamForRequest(
resource: string,
resourceType: string,
method: string,
actor?: IRequestActor
): Promise<OciUpstream | null> {
if (!this.upstreamProvider) return null;
const config = await this.upstreamProvider.resolveUpstreamConfig({
protocol: 'oci',
resource,
scope: this.extractScope(resource),
actor,
method,
resourceType,
});
if (!config?.enabled) return null;
return new OciUpstream(config, this.basePath, this.logger);
}
public async init(): Promise<void> { public async init(): Promise<void> {
// Start cleanup of stale upload sessions // Start cleanup of stale upload sessions
this.startUploadSessionCleanup(); this.startUploadSessionCleanup();
@@ -80,29 +117,37 @@ export class OciRegistry extends BaseRegistry {
const tokenString = authHeader?.replace(/^Bearer\s+/i, ''); const tokenString = authHeader?.replace(/^Bearer\s+/i, '');
const token = tokenString ? await this.authManager.validateToken(tokenString, 'oci') : null; const token = tokenString ? await this.authManager.validateToken(tokenString, 'oci') : null;
// Build actor from context and validated token
const actor: IRequestActor = {
...context.actor,
userId: token?.userId,
ip: context.headers['x-forwarded-for'] || context.headers['X-Forwarded-For'],
userAgent: context.headers['user-agent'] || context.headers['User-Agent'],
};
// Route to appropriate handler // Route to appropriate handler
if (path === '/v2/' || path === '/v2') { if (path === '/' || path === '') {
return this.handleVersionCheck(); return this.handleVersionCheck();
} }
// Manifest operations: /v2/{name}/manifests/{reference} // Manifest operations: /{name}/manifests/{reference}
const manifestMatch = path.match(/^\/v2\/([^\/]+(?:\/[^\/]+)*)\/manifests\/([^\/]+)$/); const manifestMatch = path.match(/^\/([^\/]+(?:\/[^\/]+)*)\/manifests\/([^\/]+)$/);
if (manifestMatch) { if (manifestMatch) {
const [, name, reference] = manifestMatch; const [, name, reference] = manifestMatch;
// Prefer rawBody for content-addressable operations to preserve exact bytes // Prefer rawBody for content-addressable operations to preserve exact bytes
const bodyData = context.rawBody || context.body; const bodyData = context.rawBody || context.body;
return this.handleManifestRequest(context.method, name, reference, token, bodyData, context.headers); return this.handleManifestRequest(context.method, name, reference, token, bodyData, context.headers, actor);
} }
// Blob operations: /v2/{name}/blobs/{digest} // Blob operations: /{name}/blobs/{digest}
const blobMatch = path.match(/^\/v2\/([^\/]+(?:\/[^\/]+)*)\/blobs\/(sha256:[a-f0-9]{64})$/); const blobMatch = path.match(/^\/([^\/]+(?:\/[^\/]+)*)\/blobs\/(sha256:[a-f0-9]{64})$/);
if (blobMatch) { if (blobMatch) {
const [, name, digest] = blobMatch; const [, name, digest] = blobMatch;
return this.handleBlobRequest(context.method, name, digest, token, context.headers); return this.handleBlobRequest(context.method, name, digest, token, context.headers, actor);
} }
// Blob upload operations: /v2/{name}/blobs/uploads/ // Blob upload operations: /{name}/blobs/uploads/
const uploadInitMatch = path.match(/^\/v2\/([^\/]+(?:\/[^\/]+)*)\/blobs\/uploads\/?$/); const uploadInitMatch = path.match(/^\/([^\/]+(?:\/[^\/]+)*)\/blobs\/uploads\/?$/);
if (uploadInitMatch && context.method === 'POST') { if (uploadInitMatch && context.method === 'POST') {
const [, name] = uploadInitMatch; const [, name] = uploadInitMatch;
// Prefer rawBody for content-addressable operations to preserve exact bytes // Prefer rawBody for content-addressable operations to preserve exact bytes
@@ -110,22 +155,22 @@ export class OciRegistry extends BaseRegistry {
return this.handleUploadInit(name, token, context.query, bodyData); return this.handleUploadInit(name, token, context.query, bodyData);
} }
// Blob upload operations: /v2/{name}/blobs/uploads/{uuid} // Blob upload operations: /{name}/blobs/uploads/{uuid}
const uploadMatch = path.match(/^\/v2\/([^\/]+(?:\/[^\/]+)*)\/blobs\/uploads\/([^\/]+)$/); const uploadMatch = path.match(/^\/([^\/]+(?:\/[^\/]+)*)\/blobs\/uploads\/([^\/]+)$/);
if (uploadMatch) { if (uploadMatch) {
const [, name, uploadId] = uploadMatch; const [, name, uploadId] = uploadMatch;
return this.handleUploadSession(context.method, uploadId, token, context); return this.handleUploadSession(context.method, uploadId, token, context);
} }
// Tags list: /v2/{name}/tags/list // Tags list: /{name}/tags/list
const tagsMatch = path.match(/^\/v2\/([^\/]+(?:\/[^\/]+)*)\/tags\/list$/); const tagsMatch = path.match(/^\/([^\/]+(?:\/[^\/]+)*)\/tags\/list$/);
if (tagsMatch) { if (tagsMatch) {
const [, name] = tagsMatch; const [, name] = tagsMatch;
return this.handleTagsList(name, token, context.query); return this.handleTagsList(name, token, context.query);
} }
// Referrers: /v2/{name}/referrers/{digest} // Referrers: /{name}/referrers/{digest}
const referrersMatch = path.match(/^\/v2\/([^\/]+(?:\/[^\/]+)*)\/referrers\/(sha256:[a-f0-9]{64})$/); const referrersMatch = path.match(/^\/([^\/]+(?:\/[^\/]+)*)\/referrers\/(sha256:[a-f0-9]{64})$/);
if (referrersMatch) { if (referrersMatch) {
const [, name, digest] = referrersMatch; const [, name, digest] = referrersMatch;
return this.handleReferrers(name, digest, token, context.query); return this.handleReferrers(name, digest, token, context.query);
@@ -168,11 +213,12 @@ export class OciRegistry extends BaseRegistry {
reference: string, reference: string,
token: IAuthToken | null, token: IAuthToken | null,
body?: Buffer | any, body?: Buffer | any,
headers?: Record<string, string> headers?: Record<string, string>,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
switch (method) { switch (method) {
case 'GET': case 'GET':
return this.getManifest(repository, reference, token, headers); return this.getManifest(repository, reference, token, headers, actor);
case 'HEAD': case 'HEAD':
return this.headManifest(repository, reference, token); return this.headManifest(repository, reference, token);
case 'PUT': case 'PUT':
@@ -193,11 +239,12 @@ export class OciRegistry extends BaseRegistry {
repository: string, repository: string,
digest: string, digest: string,
token: IAuthToken | null, token: IAuthToken | null,
headers: Record<string, string> headers: Record<string, string>,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
switch (method) { switch (method) {
case 'GET': case 'GET':
return this.getBlob(repository, digest, token, headers['range'] || headers['Range']); return this.getBlob(repository, digest, token, headers['range'] || headers['Range'], actor);
case 'HEAD': case 'HEAD':
return this.headBlob(repository, digest, token); return this.headBlob(repository, digest, token);
case 'DELETE': case 'DELETE':
@@ -243,7 +290,7 @@ export class OciRegistry extends BaseRegistry {
return { return {
status: 201, status: 201,
headers: { headers: {
'Location': `${this.basePath}/v2/${repository}/blobs/${digest}`, 'Location': `${this.basePath}/${repository}/blobs/${digest}`,
'Docker-Content-Digest': digest, 'Docker-Content-Digest': digest,
}, },
body: null, body: null,
@@ -256,6 +303,8 @@ export class OciRegistry extends BaseRegistry {
uploadId, uploadId,
repository, repository,
chunks: [], chunks: [],
chunkPaths: [],
chunkIndex: 0,
totalSize: 0, totalSize: 0,
createdAt: new Date(), createdAt: new Date(),
lastActivity: new Date(), lastActivity: new Date(),
@@ -266,7 +315,7 @@ export class OciRegistry extends BaseRegistry {
return { return {
status: 202, status: 202,
headers: { headers: {
'Location': `${this.basePath}/v2/${repository}/blobs/uploads/${uploadId}`, 'Location': `${this.basePath}/${repository}/blobs/uploads/${uploadId}`,
'Docker-Upload-UUID': uploadId, 'Docker-Upload-UUID': uploadId,
}, },
body: null, body: null,
@@ -318,7 +367,8 @@ export class OciRegistry extends BaseRegistry {
repository: string, repository: string,
reference: string, reference: string,
token: IAuthToken | null, token: IAuthToken | null,
headers?: Record<string, string> headers?: Record<string, string>,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
if (!await this.checkPermission(token, repository, 'pull')) { if (!await this.checkPermission(token, repository, 'pull')) {
return this.createUnauthorizedResponse(repository, 'pull'); return this.createUnauthorizedResponse(repository, 'pull');
@@ -346,30 +396,33 @@ export class OciRegistry extends BaseRegistry {
} }
// If not found locally, try upstream // If not found locally, try upstream
if (!manifestData && this.upstream) { if (!manifestData) {
this.logger.log('debug', 'getManifest: fetching from upstream', { repository, reference }); const upstream = await this.getUpstreamForRequest(repository, 'manifest', 'GET', actor);
const upstreamResult = await this.upstream.fetchManifest(repository, reference); if (upstream) {
if (upstreamResult) { this.logger.log('debug', 'getManifest: fetching from upstream', { repository, reference });
manifestData = Buffer.from(JSON.stringify(upstreamResult.manifest), 'utf8'); const upstreamResult = await upstream.fetchManifest(repository, reference);
contentType = upstreamResult.contentType; if (upstreamResult) {
digest = upstreamResult.digest; manifestData = Buffer.from(JSON.stringify(upstreamResult.manifest), 'utf8');
contentType = upstreamResult.contentType;
digest = upstreamResult.digest;
// Cache the manifest locally // Cache the manifest locally
await this.storage.putOciManifest(repository, digest, manifestData, contentType); await this.storage.putOciManifest(repository, digest, manifestData, contentType);
// If reference is a tag, update tags mapping // If reference is a tag, update tags mapping
if (!reference.startsWith('sha256:')) { if (!reference.startsWith('sha256:')) {
const tags = await this.getTagsData(repository); const tags = await this.getTagsData(repository);
tags[reference] = digest; tags[reference] = digest;
const tagsPath = `oci/tags/${repository}/tags.json`; const tagsPath = `oci/tags/${repository}/tags.json`;
await this.storage.putObject(tagsPath, Buffer.from(JSON.stringify(tags), 'utf-8')); await this.storage.putObject(tagsPath, Buffer.from(JSON.stringify(tags), 'utf-8'));
}
this.logger.log('debug', 'getManifest: cached manifest locally', {
repository,
reference,
digest,
});
} }
this.logger.log('debug', 'getManifest: cached manifest locally', {
repository,
reference,
digest,
});
} }
} }
@@ -477,7 +530,7 @@ export class OciRegistry extends BaseRegistry {
return { return {
status: 201, status: 201,
headers: { headers: {
'Location': `${this.basePath}/v2/${repository}/manifests/${digest}`, 'Location': `${this.basePath}/${repository}/manifests/${digest}`,
'Docker-Content-Digest': digest, 'Docker-Content-Digest': digest,
}, },
body: null, body: null,
@@ -514,19 +567,33 @@ export class OciRegistry extends BaseRegistry {
repository: string, repository: string,
digest: string, digest: string,
token: IAuthToken | null, token: IAuthToken | null,
range?: string range?: string,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
if (!await this.checkPermission(token, repository, 'pull')) { if (!await this.checkPermission(token, repository, 'pull')) {
return this.createUnauthorizedResponse(repository, 'pull'); return this.createUnauthorizedResponse(repository, 'pull');
} }
// Try local storage first // Try local storage first (streaming)
let data = await this.storage.getOciBlob(digest); const streamResult = await this.storage.getOciBlobStream(digest);
if (streamResult) {
return {
status: 200,
headers: {
'Content-Type': 'application/octet-stream',
'Content-Length': streamResult.size.toString(),
'Docker-Content-Digest': digest,
},
body: streamResult.stream,
};
}
// If not found locally, try upstream // If not found locally, try upstream
if (!data && this.upstream) { let data: Buffer | null = null;
const upstream = await this.getUpstreamForRequest(repository, 'blob', 'GET', actor);
if (upstream) {
this.logger.log('debug', 'getBlob: fetching from upstream', { repository, digest }); this.logger.log('debug', 'getBlob: fetching from upstream', { repository, digest });
const upstreamBlob = await this.upstream.fetchBlob(repository, digest); const upstreamBlob = await upstream.fetchBlob(repository, digest);
if (upstreamBlob) { if (upstreamBlob) {
data = upstreamBlob; data = upstreamBlob;
// Cache the blob locally (blobs are content-addressable and immutable) // Cache the blob locally (blobs are content-addressable and immutable)
@@ -566,17 +633,15 @@ export class OciRegistry extends BaseRegistry {
return this.createUnauthorizedHeadResponse(repository, 'pull'); return this.createUnauthorizedHeadResponse(repository, 'pull');
} }
const exists = await this.storage.ociBlobExists(digest); const blobSize = await this.storage.getOciBlobSize(digest);
if (!exists) { if (blobSize === null) {
return { status: 404, headers: {}, body: null }; return { status: 404, headers: {}, body: null };
} }
const blob = await this.storage.getOciBlob(digest);
return { return {
status: 200, status: 200,
headers: { headers: {
'Content-Length': blob ? blob.length.toString() : '0', 'Content-Length': blobSize.toString(),
'Docker-Content-Digest': digest, 'Docker-Content-Digest': digest,
}, },
body: null, body: null,
@@ -616,14 +681,19 @@ export class OciRegistry extends BaseRegistry {
} }
const chunkData = this.toBuffer(data); const chunkData = this.toBuffer(data);
session.chunks.push(chunkData);
// Write chunk to temp S3 object instead of accumulating in memory
const chunkPath = `oci/uploads/${uploadId}/chunk-${session.chunkIndex}`;
await this.storage.putObject(chunkPath, chunkData);
session.chunkPaths.push(chunkPath);
session.chunkIndex++;
session.totalSize += chunkData.length; session.totalSize += chunkData.length;
session.lastActivity = new Date(); session.lastActivity = new Date();
return { return {
status: 202, status: 202,
headers: { headers: {
'Location': `${this.basePath}/v2/${session.repository}/blobs/uploads/${uploadId}`, 'Location': `${this.basePath}/${session.repository}/blobs/uploads/${uploadId}`,
'Range': `0-${session.totalSize - 1}`, 'Range': `0-${session.totalSize - 1}`,
'Docker-Upload-UUID': uploadId, 'Docker-Upload-UUID': uploadId,
}, },
@@ -645,13 +715,52 @@ export class OciRegistry extends BaseRegistry {
}; };
} }
const chunks = [...session.chunks]; // If there's final data in the PUT body, write it as the last chunk
if (finalData) chunks.push(this.toBuffer(finalData)); if (finalData) {
const blobData = Buffer.concat(chunks); const buf = this.toBuffer(finalData);
const chunkPath = `oci/uploads/${uploadId}/chunk-${session.chunkIndex}`;
await this.storage.putObject(chunkPath, buf);
session.chunkPaths.push(chunkPath);
session.chunkIndex++;
session.totalSize += buf.length;
}
// Verify digest // Create a ReadableStream that assembles all chunks from S3 sequentially
const calculatedDigest = await this.calculateDigest(blobData); const chunkPaths = [...session.chunkPaths];
const storage = this.storage;
let chunkIdx = 0;
const assembledStream = new ReadableStream<Uint8Array>({
async pull(controller) {
if (chunkIdx >= chunkPaths.length) {
controller.close();
return;
}
const result = await storage.getObjectStream(chunkPaths[chunkIdx++]);
if (result) {
const reader = result.stream.getReader();
while (true) {
const { done, value } = await reader.read();
if (done) break;
if (value) controller.enqueue(value);
}
}
},
});
// Pipe through hash transform for incremental digest verification
const { transform: hashTransform, getDigest } = createHashTransform('sha256');
const hashedStream = assembledStream.pipeThrough(hashTransform);
// Consume stream to buffer for S3 upload
// (AWS SDK PutObjectCommand requires known content-length for streams;
// the key win is chunks are NOT accumulated in memory during PATCH — they live in S3)
const blobData = await streamToBuffer(hashedStream);
// Verify digest before storing
const calculatedDigest = `sha256:${getDigest()}`;
if (calculatedDigest !== digest) { if (calculatedDigest !== digest) {
await this.cleanupUploadChunks(session);
this.uploadSessions.delete(uploadId);
return { return {
status: 400, status: 400,
headers: {}, headers: {},
@@ -659,19 +768,36 @@ export class OciRegistry extends BaseRegistry {
}; };
} }
// Store verified blob
await this.storage.putOciBlob(digest, blobData); await this.storage.putOciBlob(digest, blobData);
// Cleanup temp chunks and session
await this.cleanupUploadChunks(session);
this.uploadSessions.delete(uploadId); this.uploadSessions.delete(uploadId);
return { return {
status: 201, status: 201,
headers: { headers: {
'Location': `${this.basePath}/v2/${session.repository}/blobs/${digest}`, 'Location': `${this.basePath}/${session.repository}/blobs/${digest}`,
'Docker-Content-Digest': digest, 'Docker-Content-Digest': digest,
}, },
body: null, body: null,
}; };
} }
/**
* Delete all temp S3 chunk objects for an upload session.
*/
private async cleanupUploadChunks(session: IUploadSession): Promise<void> {
for (const chunkPath of session.chunkPaths) {
try {
await this.storage.deleteObject(chunkPath);
} catch {
// Best-effort cleanup
}
}
}
private async getUploadStatus(uploadId: string): Promise<IResponse> { private async getUploadStatus(uploadId: string): Promise<IResponse> {
const session = this.uploadSessions.get(uploadId); const session = this.uploadSessions.get(uploadId);
if (!session) { if (!session) {
@@ -685,7 +811,7 @@ export class OciRegistry extends BaseRegistry {
return { return {
status: 204, status: 204,
headers: { headers: {
'Location': `${this.basePath}/v2/${session.repository}/blobs/uploads/${uploadId}`, 'Location': `${this.basePath}/${session.repository}/blobs/uploads/${uploadId}`,
'Range': session.totalSize > 0 ? `0-${session.totalSize - 1}` : '0-0', 'Range': session.totalSize > 0 ? `0-${session.totalSize - 1}` : '0-0',
'Docker-Upload-UUID': uploadId, 'Docker-Upload-UUID': uploadId,
}, },
@@ -830,7 +956,7 @@ export class OciRegistry extends BaseRegistry {
* Per OCI Distribution Spec, 401 responses MUST include WWW-Authenticate header. * Per OCI Distribution Spec, 401 responses MUST include WWW-Authenticate header.
*/ */
private createUnauthorizedResponse(repository: string, action: string): IResponse { private createUnauthorizedResponse(repository: string, action: string): IResponse {
const realm = this.ociTokens?.realm || `${this.basePath}/v2/token`; const realm = this.ociTokens?.realm || `${this.basePath}/token`;
const service = this.ociTokens?.service || 'registry'; const service = this.ociTokens?.service || 'registry';
return { return {
status: 401, status: 401,
@@ -845,7 +971,7 @@ export class OciRegistry extends BaseRegistry {
* Create an unauthorized HEAD response (no body per HTTP spec). * Create an unauthorized HEAD response (no body per HTTP spec).
*/ */
private createUnauthorizedHeadResponse(repository: string, action: string): IResponse { private createUnauthorizedHeadResponse(repository: string, action: string): IResponse {
const realm = this.ociTokens?.realm || `${this.basePath}/v2/token`; const realm = this.ociTokens?.realm || `${this.basePath}/token`;
const service = this.ociTokens?.service || 'registry'; const service = this.ociTokens?.service || 'registry';
return { return {
status: 401, status: 401,
@@ -863,6 +989,8 @@ export class OciRegistry extends BaseRegistry {
for (const [uploadId, session] of this.uploadSessions.entries()) { for (const [uploadId, session] of this.uploadSessions.entries()) {
if (now.getTime() - session.lastActivity.getTime() > maxAge) { if (now.getTime() - session.lastActivity.getTime() > maxAge) {
// Clean up temp S3 chunks for stale sessions
this.cleanupUploadChunks(session).catch(() => {});
this.uploadSessions.delete(uploadId); this.uploadSessions.delete(uploadId);
} }
} }

View File

@@ -24,13 +24,18 @@ export class OciUpstream extends BaseUpstream {
/** Local registry base path for URL building */ /** Local registry base path for URL building */
private readonly localBasePath: string; private readonly localBasePath: string;
/** API prefix for outbound OCI requests (default: /v2) */
private readonly apiPrefix: string;
constructor( constructor(
config: IProtocolUpstreamConfig, config: IProtocolUpstreamConfig,
localBasePath: string = '/oci', localBasePath: string = '/oci',
logger?: plugins.smartlog.Smartlog, logger?: plugins.smartlog.Smartlog,
apiPrefix: string = '/v2',
) { ) {
super(config, logger); super(config, logger);
this.localBasePath = localBasePath; this.localBasePath = localBasePath;
this.apiPrefix = apiPrefix;
} }
/** /**
@@ -44,7 +49,7 @@ export class OciUpstream extends BaseUpstream {
protocol: 'oci', protocol: 'oci',
resource: repository, resource: repository,
resourceType: 'manifest', resourceType: 'manifest',
path: `/v2/${repository}/manifests/${reference}`, path: `${this.apiPrefix}/${repository}/manifests/${reference}`,
method: 'GET', method: 'GET',
headers: { headers: {
'accept': [ 'accept': [
@@ -88,7 +93,7 @@ export class OciUpstream extends BaseUpstream {
protocol: 'oci', protocol: 'oci',
resource: repository, resource: repository,
resourceType: 'manifest', resourceType: 'manifest',
path: `/v2/${repository}/manifests/${reference}`, path: `${this.apiPrefix}/${repository}/manifests/${reference}`,
method: 'HEAD', method: 'HEAD',
headers: { headers: {
'accept': [ 'accept': [
@@ -127,7 +132,7 @@ export class OciUpstream extends BaseUpstream {
protocol: 'oci', protocol: 'oci',
resource: repository, resource: repository,
resourceType: 'blob', resourceType: 'blob',
path: `/v2/${repository}/blobs/${digest}`, path: `${this.apiPrefix}/${repository}/blobs/${digest}`,
method: 'GET', method: 'GET',
headers: { headers: {
'accept': 'application/octet-stream', 'accept': 'application/octet-stream',
@@ -155,7 +160,7 @@ export class OciUpstream extends BaseUpstream {
protocol: 'oci', protocol: 'oci',
resource: repository, resource: repository,
resourceType: 'blob', resourceType: 'blob',
path: `/v2/${repository}/blobs/${digest}`, path: `${this.apiPrefix}/${repository}/blobs/${digest}`,
method: 'HEAD', method: 'HEAD',
headers: {}, headers: {},
query: {}, query: {},
@@ -189,7 +194,7 @@ export class OciUpstream extends BaseUpstream {
protocol: 'oci', protocol: 'oci',
resource: repository, resource: repository,
resourceType: 'tags', resourceType: 'tags',
path: `/v2/${repository}/tags/list`, path: `${this.apiPrefix}/${repository}/tags/list`,
method: 'GET', method: 'GET',
headers: { headers: {
'accept': 'application/json', 'accept': 'application/json',
@@ -215,7 +220,8 @@ export class OciUpstream extends BaseUpstream {
/** /**
* Override URL building for OCI-specific handling. * Override URL building for OCI-specific handling.
* OCI registries use /v2/ prefix and may require special handling for Docker Hub. * OCI registries use a configurable API prefix (default /v2/) and may require
* special handling for Docker Hub.
*/ */
protected buildUpstreamUrl( protected buildUpstreamUrl(
upstream: IUpstreamRegistryConfig, upstream: IUpstreamRegistryConfig,
@@ -228,16 +234,20 @@ export class OciUpstream extends BaseUpstream {
baseUrl = baseUrl.slice(0, -1); baseUrl = baseUrl.slice(0, -1);
} }
// Use per-upstream apiPrefix if configured, otherwise use the instance default
const prefix = upstream.apiPrefix || this.apiPrefix;
// Handle Docker Hub special case // Handle Docker Hub special case
// Docker Hub uses registry-1.docker.io but library images need special handling // Docker Hub uses registry-1.docker.io but library images need special handling
if (baseUrl.includes('docker.io') || baseUrl.includes('registry-1.docker.io')) { if (baseUrl.includes('docker.io') || baseUrl.includes('registry-1.docker.io')) {
// For library images (e.g., "nginx" -> "library/nginx") // For library images (e.g., "nginx" -> "library/nginx")
const pathParts = context.path.match(/^\/v2\/([^\/]+)\/(.+)$/); const escapedPrefix = prefix.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
const pathParts = context.path.match(new RegExp(`^${escapedPrefix}\\/([^\\/]+)\\/(.+)$`));
if (pathParts) { if (pathParts) {
const [, repository, rest] = pathParts; const [, repository, rest] = pathParts;
// If repository doesn't contain a slash, it's a library image // If repository doesn't contain a slash, it's a library image
if (!repository.includes('/')) { if (!repository.includes('/')) {
return `${baseUrl}/v2/library/${repository}/${rest}`; return `${baseUrl}${prefix}/library/${repository}/${rest}`;
} }
} }
} }

View File

@@ -62,6 +62,10 @@ export interface IUploadSession {
uploadId: string; uploadId: string;
repository: string; repository: string;
chunks: Buffer[]; chunks: Buffer[];
/** S3 paths to temp chunk objects (streaming mode) */
chunkPaths: string[];
/** Index counter for naming temp chunk objects */
chunkIndex: number;
totalSize: number; totalSize: number;
createdAt: Date; createdAt: Date;
lastActivity: Date; lastActivity: Date;

View File

@@ -2,8 +2,8 @@ import { Smartlog } from '@push.rocks/smartlog';
import { BaseRegistry } from '../core/classes.baseregistry.js'; import { BaseRegistry } from '../core/classes.baseregistry.js';
import { RegistryStorage } from '../core/classes.registrystorage.js'; import { RegistryStorage } from '../core/classes.registrystorage.js';
import { AuthManager } from '../core/classes.authmanager.js'; import { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js'; import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from '../core/interfaces.core.js';
import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js'; import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
import { isBinaryData, toBuffer } from '../core/helpers.buffer.js'; import { isBinaryData, toBuffer } from '../core/helpers.buffer.js';
import type { import type {
IPypiPackageMetadata, IPypiPackageMetadata,
@@ -24,20 +24,21 @@ export class PypiRegistry extends BaseRegistry {
private basePath: string = '/pypi'; private basePath: string = '/pypi';
private registryUrl: string; private registryUrl: string;
private logger: Smartlog; private logger: Smartlog;
private upstream: PypiUpstream | null = null; private upstreamProvider: IUpstreamProvider | null = null;
constructor( constructor(
storage: RegistryStorage, storage: RegistryStorage,
authManager: AuthManager, authManager: AuthManager,
basePath: string = '/pypi', basePath: string = '/pypi',
registryUrl: string = 'http://localhost:5000', registryUrl: string = 'http://localhost:5000',
upstreamConfig?: IProtocolUpstreamConfig upstreamProvider?: IUpstreamProvider
) { ) {
super(); super();
this.storage = storage; this.storage = storage;
this.authManager = authManager; this.authManager = authManager;
this.basePath = basePath; this.basePath = basePath;
this.registryUrl = registryUrl; this.registryUrl = registryUrl;
this.upstreamProvider = upstreamProvider || null;
// Initialize logger // Initialize logger
this.logger = new Smartlog({ this.logger = new Smartlog({
@@ -51,20 +52,38 @@ export class PypiRegistry extends BaseRegistry {
} }
}); });
this.logger.enableConsole(); this.logger.enableConsole();
}
// Initialize upstream if configured /**
if (upstreamConfig?.enabled) { * Get upstream for a specific request.
this.upstream = new PypiUpstream(upstreamConfig, registryUrl, this.logger); * Calls the provider to resolve upstream config dynamically.
} */
private async getUpstreamForRequest(
resource: string,
resourceType: string,
method: string,
actor?: IRequestActor
): Promise<PypiUpstream | null> {
if (!this.upstreamProvider) return null;
const config = await this.upstreamProvider.resolveUpstreamConfig({
protocol: 'pypi',
resource,
scope: resource, // For PyPI, package name is the scope
actor,
method,
resourceType,
});
if (!config?.enabled) return null;
return new PypiUpstream(config, this.registryUrl, this.logger);
} }
/** /**
* Clean up resources (timers, connections, etc.) * Clean up resources (timers, connections, etc.)
*/ */
public destroy(): void { public destroy(): void {
if (this.upstream) { // No persistent upstream to clean up with dynamic provider
this.upstream.stop();
}
} }
public async init(): Promise<void> { public async init(): Promise<void> {
@@ -84,15 +103,23 @@ export class PypiRegistry extends BaseRegistry {
public async handleRequest(context: IRequestContext): Promise<IResponse> { public async handleRequest(context: IRequestContext): Promise<IResponse> {
let path = context.path.replace(this.basePath, ''); let path = context.path.replace(this.basePath, '');
// Extract token (Basic Auth or Bearer)
const token = await this.extractToken(context);
// Build actor from context and validated token
const actor: IRequestActor = {
...context.actor,
userId: token?.userId,
ip: context.headers['x-forwarded-for'] || context.headers['X-Forwarded-For'],
userAgent: context.headers['user-agent'] || context.headers['User-Agent'],
};
// Also handle /simple path prefix // Also handle /simple path prefix
if (path.startsWith('/simple')) { if (path.startsWith('/simple')) {
path = path.replace('/simple', ''); path = path.replace('/simple', '');
return this.handleSimpleRequest(path, context); return this.handleSimpleRequest(path, context, actor);
} }
// Extract token (Basic Auth or Bearer)
const token = await this.extractToken(context);
this.logger.log('debug', `handleRequest: ${context.method} ${path}`, { this.logger.log('debug', `handleRequest: ${context.method} ${path}`, {
method: context.method, method: context.method,
path, path,
@@ -119,7 +146,7 @@ export class PypiRegistry extends BaseRegistry {
// Package file download: GET /packages/{package}/{filename} // Package file download: GET /packages/{package}/{filename}
const downloadMatch = path.match(/^\/packages\/([^\/]+)\/(.+)$/); const downloadMatch = path.match(/^\/packages\/([^\/]+)\/(.+)$/);
if (downloadMatch && context.method === 'GET') { if (downloadMatch && context.method === 'GET') {
return this.handleDownload(downloadMatch[1], downloadMatch[2]); return this.handleDownload(downloadMatch[1], downloadMatch[2], actor);
} }
// Delete package: DELETE /packages/{package} // Delete package: DELETE /packages/{package}
@@ -156,7 +183,7 @@ export class PypiRegistry extends BaseRegistry {
/** /**
* Handle Simple API requests (PEP 503 HTML or PEP 691 JSON) * Handle Simple API requests (PEP 503 HTML or PEP 691 JSON)
*/ */
private async handleSimpleRequest(path: string, context: IRequestContext): Promise<IResponse> { private async handleSimpleRequest(path: string, context: IRequestContext, actor?: IRequestActor): Promise<IResponse> {
// Ensure path ends with / (PEP 503 requirement) // Ensure path ends with / (PEP 503 requirement)
if (!path.endsWith('/') && !path.includes('.')) { if (!path.endsWith('/') && !path.includes('.')) {
return { return {
@@ -174,7 +201,7 @@ export class PypiRegistry extends BaseRegistry {
// Package index: /simple/{package}/ // Package index: /simple/{package}/
const packageMatch = path.match(/^\/([^\/]+)\/$/); const packageMatch = path.match(/^\/([^\/]+)\/$/);
if (packageMatch) { if (packageMatch) {
return this.handleSimplePackage(packageMatch[1], context); return this.handleSimplePackage(packageMatch[1], context, actor);
} }
return { return {
@@ -228,46 +255,49 @@ export class PypiRegistry extends BaseRegistry {
* Handle Simple API package index * Handle Simple API package index
* Returns HTML (PEP 503) or JSON (PEP 691) based on Accept header * Returns HTML (PEP 503) or JSON (PEP 691) based on Accept header
*/ */
private async handleSimplePackage(packageName: string, context: IRequestContext): Promise<IResponse> { private async handleSimplePackage(packageName: string, context: IRequestContext, actor?: IRequestActor): Promise<IResponse> {
const normalized = helpers.normalizePypiPackageName(packageName); const normalized = helpers.normalizePypiPackageName(packageName);
// Get package metadata // Get package metadata
let metadata = await this.storage.getPypiPackageMetadata(normalized); let metadata = await this.storage.getPypiPackageMetadata(normalized);
// Try upstream if not found locally // Try upstream if not found locally
if (!metadata && this.upstream) { if (!metadata) {
const upstreamHtml = await this.upstream.fetchSimplePackage(normalized); const upstream = await this.getUpstreamForRequest(normalized, 'simple', 'GET', actor);
if (upstreamHtml) { if (upstream) {
// Parse the HTML to extract file information and cache it const upstreamHtml = await upstream.fetchSimplePackage(normalized);
// For now, just return the upstream HTML directly (caching can be improved later) if (upstreamHtml) {
const acceptHeader = context.headers['accept'] || context.headers['Accept'] || ''; // Parse the HTML to extract file information and cache it
const preferJson = acceptHeader.includes('application/vnd.pypi.simple') && // For now, just return the upstream HTML directly (caching can be improved later)
acceptHeader.includes('json'); const acceptHeader = context.headers['accept'] || context.headers['Accept'] || '';
const preferJson = acceptHeader.includes('application/vnd.pypi.simple') &&
acceptHeader.includes('json');
if (preferJson) { if (preferJson) {
// Try to get JSON format from upstream // Try to get JSON format from upstream
const upstreamJson = await this.upstream.fetchPackageJson(normalized); const upstreamJson = await upstream.fetchPackageJson(normalized);
if (upstreamJson) { if (upstreamJson) {
return { return {
status: 200, status: 200,
headers: { headers: {
'Content-Type': 'application/vnd.pypi.simple.v1+json', 'Content-Type': 'application/vnd.pypi.simple.v1+json',
'Cache-Control': 'public, max-age=300' 'Cache-Control': 'public, max-age=300'
}, },
body: upstreamJson, body: upstreamJson,
}; };
}
} }
}
// Return HTML format // Return HTML format
return { return {
status: 200, status: 200,
headers: { headers: {
'Content-Type': 'text/html; charset=utf-8', 'Content-Type': 'text/html; charset=utf-8',
'Cache-Control': 'public, max-age=300' 'Cache-Control': 'public, max-age=300'
}, },
body: upstreamHtml, body: upstreamHtml,
}; };
}
} }
} }
@@ -503,13 +533,29 @@ export class PypiRegistry extends BaseRegistry {
/** /**
* Handle package download * Handle package download
*/ */
private async handleDownload(packageName: string, filename: string): Promise<IResponse> { private async handleDownload(packageName: string, filename: string, actor?: IRequestActor): Promise<IResponse> {
const normalized = helpers.normalizePypiPackageName(packageName); const normalized = helpers.normalizePypiPackageName(packageName);
let fileData = await this.storage.getPypiPackageFile(normalized, filename);
// Try streaming from local storage first
const streamResult = await this.storage.getPypiPackageFileStream(normalized, filename);
if (streamResult) {
return {
status: 200,
headers: {
'Content-Type': 'application/octet-stream',
'Content-Disposition': `attachment; filename="${filename}"`,
'Content-Length': streamResult.size.toString()
},
body: streamResult.stream,
};
}
// Try upstream if not found locally // Try upstream if not found locally
if (!fileData && this.upstream) { let fileData: Buffer | null = null;
fileData = await this.upstream.fetchPackageFile(normalized, filename); const upstream = await this.getUpstreamForRequest(normalized, 'file', 'GET', actor);
if (upstream) {
fileData = await upstream.fetchPackageFile(normalized, filename);
if (fileData) { if (fileData) {
// Cache locally // Cache locally
await this.storage.putPypiPackageFile(normalized, filename, fileData); await this.storage.putPypiPackageFile(normalized, filename, fileData);

View File

@@ -2,8 +2,8 @@ import { Smartlog } from '@push.rocks/smartlog';
import { BaseRegistry } from '../core/classes.baseregistry.js'; import { BaseRegistry } from '../core/classes.baseregistry.js';
import { RegistryStorage } from '../core/classes.registrystorage.js'; import { RegistryStorage } from '../core/classes.registrystorage.js';
import { AuthManager } from '../core/classes.authmanager.js'; import { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js'; import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from '../core/interfaces.core.js';
import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js'; import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
import type { import type {
IRubyGemsMetadata, IRubyGemsMetadata,
IRubyGemsVersionMetadata, IRubyGemsVersionMetadata,
@@ -25,20 +25,21 @@ export class RubyGemsRegistry extends BaseRegistry {
private basePath: string = '/rubygems'; private basePath: string = '/rubygems';
private registryUrl: string; private registryUrl: string;
private logger: Smartlog; private logger: Smartlog;
private upstream: RubygemsUpstream | null = null; private upstreamProvider: IUpstreamProvider | null = null;
constructor( constructor(
storage: RegistryStorage, storage: RegistryStorage,
authManager: AuthManager, authManager: AuthManager,
basePath: string = '/rubygems', basePath: string = '/rubygems',
registryUrl: string = 'http://localhost:5000/rubygems', registryUrl: string = 'http://localhost:5000/rubygems',
upstreamConfig?: IProtocolUpstreamConfig upstreamProvider?: IUpstreamProvider
) { ) {
super(); super();
this.storage = storage; this.storage = storage;
this.authManager = authManager; this.authManager = authManager;
this.basePath = basePath; this.basePath = basePath;
this.registryUrl = registryUrl; this.registryUrl = registryUrl;
this.upstreamProvider = upstreamProvider || null;
// Initialize logger // Initialize logger
this.logger = new Smartlog({ this.logger = new Smartlog({
@@ -52,20 +53,38 @@ export class RubyGemsRegistry extends BaseRegistry {
} }
}); });
this.logger.enableConsole(); this.logger.enableConsole();
}
// Initialize upstream if configured /**
if (upstreamConfig?.enabled) { * Get upstream for a specific request.
this.upstream = new RubygemsUpstream(upstreamConfig, this.logger); * Calls the provider to resolve upstream config dynamically.
} */
private async getUpstreamForRequest(
resource: string,
resourceType: string,
method: string,
actor?: IRequestActor
): Promise<RubygemsUpstream | null> {
if (!this.upstreamProvider) return null;
const config = await this.upstreamProvider.resolveUpstreamConfig({
protocol: 'rubygems',
resource,
scope: resource, // gem name is the scope
actor,
method,
resourceType,
});
if (!config?.enabled) return null;
return new RubygemsUpstream(config, this.logger);
} }
/** /**
* Clean up resources (timers, connections, etc.) * Clean up resources (timers, connections, etc.)
*/ */
public destroy(): void { public destroy(): void {
if (this.upstream) { // No persistent upstream to clean up with dynamic provider
this.upstream.stop();
}
} }
public async init(): Promise<void> { public async init(): Promise<void> {
@@ -95,6 +114,14 @@ export class RubyGemsRegistry extends BaseRegistry {
// Extract token (Authorization header) // Extract token (Authorization header)
const token = await this.extractToken(context); const token = await this.extractToken(context);
// Build actor from context and validated token
const actor: IRequestActor = {
...context.actor,
userId: token?.userId,
ip: context.headers['x-forwarded-for'] || context.headers['X-Forwarded-For'],
userAgent: context.headers['user-agent'] || context.headers['User-Agent'],
};
this.logger.log('debug', `handleRequest: ${context.method} ${path}`, { this.logger.log('debug', `handleRequest: ${context.method} ${path}`, {
method: context.method, method: context.method,
path, path,
@@ -113,13 +140,13 @@ export class RubyGemsRegistry extends BaseRegistry {
// Info file: GET /info/{gem} // Info file: GET /info/{gem}
const infoMatch = path.match(/^\/info\/([^\/]+)$/); const infoMatch = path.match(/^\/info\/([^\/]+)$/);
if (infoMatch && context.method === 'GET') { if (infoMatch && context.method === 'GET') {
return this.handleInfoFile(infoMatch[1]); return this.handleInfoFile(infoMatch[1], actor);
} }
// Gem download: GET /gems/{gem}-{version}[-{platform}].gem // Gem download: GET /gems/{gem}-{version}[-{platform}].gem
const downloadMatch = path.match(/^\/gems\/(.+\.gem)$/); const downloadMatch = path.match(/^\/gems\/(.+\.gem)$/);
if (downloadMatch && context.method === 'GET') { if (downloadMatch && context.method === 'GET') {
return this.handleDownload(downloadMatch[1]); return this.handleDownload(downloadMatch[1], actor);
} }
// Legacy specs endpoints (Marshal format) // Legacy specs endpoints (Marshal format)
@@ -232,16 +259,19 @@ export class RubyGemsRegistry extends BaseRegistry {
/** /**
* Handle /info/{gem} endpoint (Compact Index) * Handle /info/{gem} endpoint (Compact Index)
*/ */
private async handleInfoFile(gemName: string): Promise<IResponse> { private async handleInfoFile(gemName: string, actor?: IRequestActor): Promise<IResponse> {
let content = await this.storage.getRubyGemsInfo(gemName); let content = await this.storage.getRubyGemsInfo(gemName);
// Try upstream if not found locally // Try upstream if not found locally
if (!content && this.upstream) { if (!content) {
const upstreamInfo = await this.upstream.fetchInfo(gemName); const upstream = await this.getUpstreamForRequest(gemName, 'info', 'GET', actor);
if (upstreamInfo) { if (upstream) {
// Cache locally const upstreamInfo = await upstream.fetchInfo(gemName);
await this.storage.putRubyGemsInfo(gemName, upstreamInfo); if (upstreamInfo) {
content = upstreamInfo; // Cache locally
await this.storage.putRubyGemsInfo(gemName, upstreamInfo);
content = upstreamInfo;
}
} }
} }
@@ -267,21 +297,36 @@ export class RubyGemsRegistry extends BaseRegistry {
/** /**
* Handle gem file download * Handle gem file download
*/ */
private async handleDownload(filename: string): Promise<IResponse> { private async handleDownload(filename: string, actor?: IRequestActor): Promise<IResponse> {
const parsed = helpers.parseGemFilename(filename); const parsed = helpers.parseGemFilename(filename);
if (!parsed) { if (!parsed) {
return this.errorResponse(400, 'Invalid gem filename'); return this.errorResponse(400, 'Invalid gem filename');
} }
let gemData = await this.storage.getRubyGemsGem( // Try streaming from local storage first
const streamResult = await this.storage.getRubyGemsGemStream(
parsed.name, parsed.name,
parsed.version, parsed.version,
parsed.platform parsed.platform
); );
if (streamResult) {
return {
status: 200,
headers: {
'Content-Type': 'application/octet-stream',
'Content-Disposition': `attachment; filename="${filename}"`,
'Content-Length': streamResult.size.toString()
},
body: streamResult.stream,
};
}
// Try upstream if not found locally // Try upstream if not found locally
if (!gemData && this.upstream) { let gemData: Buffer | null = null;
gemData = await this.upstream.fetchGem(parsed.name, parsed.version); const upstream = await this.getUpstreamForRequest(parsed.name, 'gem', 'GET', actor);
if (upstream) {
gemData = await upstream.fetchGem(parsed.name, parsed.version);
if (gemData) { if (gemData) {
// Cache locally // Cache locally
await this.storage.putRubyGemsGem(parsed.name, parsed.version, gemData, parsed.platform); await this.storage.putRubyGemsGem(parsed.name, parsed.version, gemData, parsed.platform);

View File

@@ -427,7 +427,7 @@ export async function extractGemMetadata(gemData: Buffer): Promise<{
// Step 2: Decompress the gzipped metadata // Step 2: Decompress the gzipped metadata
const gzipTools = new plugins.smartarchive.GzipTools(); const gzipTools = new plugins.smartarchive.GzipTools();
const metadataYaml = await gzipTools.decompress(metadataFile.contentBuffer); const metadataYaml = await gzipTools.decompress(metadataFile.contentBuffer);
const yamlContent = metadataYaml.toString('utf-8'); const yamlContent = Buffer.from(metadataYaml).toString('utf-8');
// Step 3: Parse the YAML to extract name, version, platform // Step 3: Parse the YAML to extract name, version, platform
// Look for name: field in YAML // Look for name: field in YAML
@@ -503,7 +503,7 @@ export async function generateSpecsGz(specs: Array<[string, string, string]>): P
} }
const uncompressed = Buffer.concat(parts); const uncompressed = Buffer.concat(parts);
return gzipTools.compress(uncompressed); return Buffer.from(await gzipTools.compress(uncompressed));
} }
/** /**

View File

@@ -110,8 +110,18 @@ export abstract class BaseUpstream {
return null; return null;
} }
// Get applicable upstreams sorted by priority
const applicableUpstreams = this.getApplicableUpstreams(context.resource);
if (applicableUpstreams.length === 0) {
return null;
}
// Use the first applicable upstream's URL for cache key
const primaryUpstreamUrl = applicableUpstreams[0]?.url;
// Check cache first // Check cache first
const cached = this.cache.get(context); const cached = await this.cache.get(context, primaryUpstreamUrl);
if (cached && !cached.stale) { if (cached && !cached.stale) {
return { return {
success: true, success: true,
@@ -125,7 +135,7 @@ export abstract class BaseUpstream {
} }
// Check for negative cache (recent 404) // Check for negative cache (recent 404)
if (this.cache.hasNegative(context)) { if (await this.cache.hasNegative(context, primaryUpstreamUrl)) {
return { return {
success: false, success: false,
status: 404, status: 404,
@@ -136,13 +146,6 @@ export abstract class BaseUpstream {
}; };
} }
// Get applicable upstreams sorted by priority
const applicableUpstreams = this.getApplicableUpstreams(context.resource);
if (applicableUpstreams.length === 0) {
return null;
}
// If we have stale cache, return it immediately and revalidate in background // If we have stale cache, return it immediately and revalidate in background
if (cached?.stale && this.cacheConfig.staleWhileRevalidate) { if (cached?.stale && this.cacheConfig.staleWhileRevalidate) {
// Fire and forget revalidation // Fire and forget revalidation
@@ -173,18 +176,19 @@ export abstract class BaseUpstream {
// Cache successful responses // Cache successful responses
if (result.success && result.body) { if (result.success && result.body) {
this.cache.set( await this.cache.set(
context, context,
Buffer.isBuffer(result.body) ? result.body : Buffer.from(JSON.stringify(result.body)), Buffer.isBuffer(result.body) ? result.body : Buffer.from(JSON.stringify(result.body)),
result.headers['content-type'] || 'application/octet-stream', result.headers['content-type'] || 'application/octet-stream',
result.headers, result.headers,
upstream.id, upstream.id,
upstream.url,
); );
} }
// Cache 404 responses // Cache 404 responses
if (result.status === 404) { if (result.status === 404) {
this.cache.setNegative(context, upstream.id); await this.cache.setNegative(context, upstream.id, upstream.url);
} }
return result; return result;
@@ -210,15 +214,15 @@ export abstract class BaseUpstream {
/** /**
* Invalidate cache for a resource pattern. * Invalidate cache for a resource pattern.
*/ */
public invalidateCache(pattern: RegExp): number { public async invalidateCache(pattern: RegExp): Promise<number> {
return this.cache.invalidatePattern(pattern); return this.cache.invalidatePattern(pattern);
} }
/** /**
* Clear all cache entries. * Clear all cache entries.
*/ */
public clearCache(): void { public async clearCache(): Promise<void> {
this.cache.clear(); await this.cache.clear();
} }
/** /**
@@ -501,12 +505,13 @@ export abstract class BaseUpstream {
); );
if (result.success && result.body) { if (result.success && result.body) {
this.cache.set( await this.cache.set(
context, context,
Buffer.isBuffer(result.body) ? result.body : Buffer.from(JSON.stringify(result.body)), Buffer.isBuffer(result.body) ? result.body : Buffer.from(JSON.stringify(result.body)),
result.headers['content-type'] || 'application/octet-stream', result.headers['content-type'] || 'application/octet-stream',
result.headers, result.headers,
upstream.id, upstream.id,
upstream.url,
); );
return; // Successfully revalidated return; // Successfully revalidated
} }

View File

@@ -4,9 +4,23 @@ import type {
IUpstreamFetchContext, IUpstreamFetchContext,
} from './interfaces.upstream.js'; } from './interfaces.upstream.js';
import { DEFAULT_CACHE_CONFIG } from './interfaces.upstream.js'; import { DEFAULT_CACHE_CONFIG } from './interfaces.upstream.js';
import type { IStorageBackend } from '../core/interfaces.core.js';
/** /**
* In-memory cache for upstream responses. * Cache metadata stored alongside cache entries.
*/
interface ICacheMetadata {
contentType: string;
headers: Record<string, string>;
cachedAt: string;
expiresAt?: string;
etag?: string;
upstreamId: string;
upstreamUrl: string;
}
/**
* S3-backed upstream cache with in-memory hot layer.
* *
* Features: * Features:
* - TTL-based expiration * - TTL-based expiration
@@ -14,26 +28,45 @@ import { DEFAULT_CACHE_CONFIG } from './interfaces.upstream.js';
* - Negative caching (404s) * - Negative caching (404s)
* - Content-type aware caching * - Content-type aware caching
* - ETag support for conditional requests * - ETag support for conditional requests
* - Multi-upstream support via URL-based cache paths
* - Persistent S3 storage with in-memory hot layer
* *
* Note: This is an in-memory implementation. For production with persistence, * Cache paths are structured as:
* extend this class to use RegistryStorage for S3-backed caching. * cache/{escaped-upstream-url}/{protocol}:{method}:{path}
*
* @example
* ```typescript
* // In-memory only (default)
* const cache = new UpstreamCache(config);
*
* // With S3 persistence
* const cache = new UpstreamCache(config, 10000, storage);
* ```
*/ */
export class UpstreamCache { export class UpstreamCache {
/** Cache storage */ /** In-memory hot cache */
private readonly cache: Map<string, ICacheEntry> = new Map(); private readonly memoryCache: Map<string, ICacheEntry> = new Map();
/** Configuration */ /** Configuration */
private readonly config: IUpstreamCacheConfig; private readonly config: IUpstreamCacheConfig;
/** Maximum cache entries (prevents memory bloat) */ /** Maximum in-memory cache entries */
private readonly maxEntries: number; private readonly maxMemoryEntries: number;
/** S3 storage backend (optional) */
private readonly storage?: IStorageBackend;
/** Cleanup interval handle */ /** Cleanup interval handle */
private cleanupInterval: ReturnType<typeof setInterval> | null = null; private cleanupInterval: ReturnType<typeof setInterval> | null = null;
constructor(config?: Partial<IUpstreamCacheConfig>, maxEntries: number = 10000) { constructor(
config?: Partial<IUpstreamCacheConfig>,
maxMemoryEntries: number = 10000,
storage?: IStorageBackend
) {
this.config = { ...DEFAULT_CACHE_CONFIG, ...config }; this.config = { ...DEFAULT_CACHE_CONFIG, ...config };
this.maxEntries = maxEntries; this.maxMemoryEntries = maxMemoryEntries;
this.storage = storage;
// Start periodic cleanup if caching is enabled // Start periodic cleanup if caching is enabled
if (this.config.enabled) { if (this.config.enabled) {
@@ -48,17 +81,36 @@ export class UpstreamCache {
return this.config.enabled; return this.config.enabled;
} }
/**
* Check if S3 storage is configured.
*/
public hasStorage(): boolean {
return !!this.storage;
}
/** /**
* Get cached entry for a request context. * Get cached entry for a request context.
* Checks memory first, then falls back to S3.
* Returns null if not found or expired (unless stale-while-revalidate). * Returns null if not found or expired (unless stale-while-revalidate).
*/ */
public get(context: IUpstreamFetchContext): ICacheEntry | null { public async get(context: IUpstreamFetchContext, upstreamUrl?: string): Promise<ICacheEntry | null> {
if (!this.config.enabled) { if (!this.config.enabled) {
return null; return null;
} }
const key = this.buildCacheKey(context); const key = this.buildCacheKey(context, upstreamUrl);
const entry = this.cache.get(key);
// Check memory cache first
let entry = this.memoryCache.get(key);
// If not in memory and we have storage, check S3
if (!entry && this.storage) {
entry = (await this.loadFromStorage(key)) ?? undefined;
if (entry) {
// Promote to memory cache
this.memoryCache.set(key, entry);
}
}
if (!entry) { if (!entry) {
return null; return null;
@@ -78,7 +130,10 @@ export class UpstreamCache {
} }
} }
// Entry is too old, remove it // Entry is too old, remove it
this.cache.delete(key); this.memoryCache.delete(key);
if (this.storage) {
await this.deleteFromStorage(key).catch(() => {});
}
return null; return null;
} }
@@ -86,26 +141,27 @@ export class UpstreamCache {
} }
/** /**
* Store a response in the cache. * Store a response in the cache (memory and optionally S3).
*/ */
public set( public async set(
context: IUpstreamFetchContext, context: IUpstreamFetchContext,
data: Buffer, data: Buffer,
contentType: string, contentType: string,
headers: Record<string, string>, headers: Record<string, string>,
upstreamId: string, upstreamId: string,
upstreamUrl: string,
options?: ICacheSetOptions, options?: ICacheSetOptions,
): void { ): Promise<void> {
if (!this.config.enabled) { if (!this.config.enabled) {
return; return;
} }
// Enforce max entries limit // Enforce max memory entries limit
if (this.cache.size >= this.maxEntries) { if (this.memoryCache.size >= this.maxMemoryEntries) {
this.evictOldest(); this.evictOldest();
} }
const key = this.buildCacheKey(context); const key = this.buildCacheKey(context, upstreamUrl);
const now = new Date(); const now = new Date();
// Determine TTL based on content type // Determine TTL based on content type
@@ -122,18 +178,24 @@ export class UpstreamCache {
stale: false, stale: false,
}; };
this.cache.set(key, entry); // Store in memory
this.memoryCache.set(key, entry);
// Store in S3 if available
if (this.storage) {
await this.saveToStorage(key, entry, upstreamUrl).catch(() => {});
}
} }
/** /**
* Store a negative cache entry (404 response). * Store a negative cache entry (404 response).
*/ */
public setNegative(context: IUpstreamFetchContext, upstreamId: string): void { public async setNegative(context: IUpstreamFetchContext, upstreamId: string, upstreamUrl: string): Promise<void> {
if (!this.config.enabled || this.config.negativeCacheTtlSeconds <= 0) { if (!this.config.enabled || this.config.negativeCacheTtlSeconds <= 0) {
return; return;
} }
const key = this.buildCacheKey(context); const key = this.buildCacheKey(context, upstreamUrl);
const now = new Date(); const now = new Date();
const entry: ICacheEntry = { const entry: ICacheEntry = {
@@ -146,34 +208,47 @@ export class UpstreamCache {
stale: false, stale: false,
}; };
this.cache.set(key, entry); this.memoryCache.set(key, entry);
if (this.storage) {
await this.saveToStorage(key, entry, upstreamUrl).catch(() => {});
}
} }
/** /**
* Check if there's a negative cache entry for this context. * Check if there's a negative cache entry for this context.
*/ */
public hasNegative(context: IUpstreamFetchContext): boolean { public async hasNegative(context: IUpstreamFetchContext, upstreamUrl?: string): Promise<boolean> {
const entry = this.get(context); const entry = await this.get(context, upstreamUrl);
return entry !== null && entry.data.length === 0; return entry !== null && entry.data.length === 0;
} }
/** /**
* Invalidate a specific cache entry. * Invalidate a specific cache entry.
*/ */
public invalidate(context: IUpstreamFetchContext): boolean { public async invalidate(context: IUpstreamFetchContext, upstreamUrl?: string): Promise<boolean> {
const key = this.buildCacheKey(context); const key = this.buildCacheKey(context, upstreamUrl);
return this.cache.delete(key); const deleted = this.memoryCache.delete(key);
if (this.storage) {
await this.deleteFromStorage(key).catch(() => {});
}
return deleted;
} }
/** /**
* Invalidate all entries matching a pattern. * Invalidate all entries matching a pattern.
* Useful for invalidating all versions of a package. * Useful for invalidating all versions of a package.
*/ */
public invalidatePattern(pattern: RegExp): number { public async invalidatePattern(pattern: RegExp): Promise<number> {
let count = 0; let count = 0;
for (const key of this.cache.keys()) { for (const key of this.memoryCache.keys()) {
if (pattern.test(key)) { if (pattern.test(key)) {
this.cache.delete(key); this.memoryCache.delete(key);
if (this.storage) {
await this.deleteFromStorage(key).catch(() => {});
}
count++; count++;
} }
} }
@@ -183,11 +258,14 @@ export class UpstreamCache {
/** /**
* Invalidate all entries from a specific upstream. * Invalidate all entries from a specific upstream.
*/ */
public invalidateUpstream(upstreamId: string): number { public async invalidateUpstream(upstreamId: string): Promise<number> {
let count = 0; let count = 0;
for (const [key, entry] of this.cache.entries()) { for (const [key, entry] of this.memoryCache.entries()) {
if (entry.upstreamId === upstreamId) { if (entry.upstreamId === upstreamId) {
this.cache.delete(key); this.memoryCache.delete(key);
if (this.storage) {
await this.deleteFromStorage(key).catch(() => {});
}
count++; count++;
} }
} }
@@ -195,10 +273,13 @@ export class UpstreamCache {
} }
/** /**
* Clear all cache entries. * Clear all cache entries (memory and S3).
*/ */
public clear(): void { public async clear(): Promise<void> {
this.cache.clear(); this.memoryCache.clear();
// Note: S3 cleanup would require listing and deleting all cache/* objects
// This is left as a future enhancement for bulk cleanup
} }
/** /**
@@ -211,7 +292,7 @@ export class UpstreamCache {
let totalSize = 0; let totalSize = 0;
const now = new Date(); const now = new Date();
for (const entry of this.cache.values()) { for (const entry of this.memoryCache.values()) {
totalSize += entry.data.length; totalSize += entry.data.length;
if (entry.data.length === 0) { if (entry.data.length === 0) {
@@ -224,13 +305,14 @@ export class UpstreamCache {
} }
return { return {
totalEntries: this.cache.size, totalEntries: this.memoryCache.size,
freshEntries: freshCount, freshEntries: freshCount,
staleEntries: staleCount, staleEntries: staleCount,
negativeEntries: negativeCount, negativeEntries: negativeCount,
totalSizeBytes: totalSize, totalSizeBytes: totalSize,
maxEntries: this.maxEntries, maxEntries: this.maxMemoryEntries,
enabled: this.config.enabled, enabled: this.config.enabled,
hasStorage: !!this.storage,
}; };
} }
@@ -244,17 +326,136 @@ export class UpstreamCache {
} }
} }
// ========================================================================
// Storage Methods
// ========================================================================
/**
* Build storage path for a cache key.
* Escapes upstream URL for safe use in S3 paths.
*/
private buildStoragePath(key: string): string {
return `cache/${key}`;
}
/**
* Build storage path for cache metadata.
*/
private buildMetadataPath(key: string): string {
return `cache/${key}.meta`;
}
/**
* Load a cache entry from S3 storage.
*/
private async loadFromStorage(key: string): Promise<ICacheEntry | null> {
if (!this.storage) return null;
try {
const dataPath = this.buildStoragePath(key);
const metaPath = this.buildMetadataPath(key);
// Load data and metadata in parallel
const [data, metaBuffer] = await Promise.all([
this.storage.getObject(dataPath),
this.storage.getObject(metaPath),
]);
if (!data || !metaBuffer) {
return null;
}
const meta: ICacheMetadata = JSON.parse(metaBuffer.toString('utf-8'));
return {
data,
contentType: meta.contentType,
headers: meta.headers,
cachedAt: new Date(meta.cachedAt),
expiresAt: meta.expiresAt ? new Date(meta.expiresAt) : undefined,
etag: meta.etag,
upstreamId: meta.upstreamId,
stale: false,
};
} catch {
return null;
}
}
/**
* Save a cache entry to S3 storage.
*/
private async saveToStorage(key: string, entry: ICacheEntry, upstreamUrl: string): Promise<void> {
if (!this.storage) return;
const dataPath = this.buildStoragePath(key);
const metaPath = this.buildMetadataPath(key);
const meta: ICacheMetadata = {
contentType: entry.contentType,
headers: entry.headers,
cachedAt: entry.cachedAt.toISOString(),
expiresAt: entry.expiresAt?.toISOString(),
etag: entry.etag,
upstreamId: entry.upstreamId,
upstreamUrl,
};
// Save data and metadata in parallel
await Promise.all([
this.storage.putObject(dataPath, entry.data),
this.storage.putObject(metaPath, Buffer.from(JSON.stringify(meta), 'utf-8')),
]);
}
/**
* Delete a cache entry from S3 storage.
*/
private async deleteFromStorage(key: string): Promise<void> {
if (!this.storage) return;
const dataPath = this.buildStoragePath(key);
const metaPath = this.buildMetadataPath(key);
await Promise.all([
this.storage.deleteObject(dataPath).catch(() => {}),
this.storage.deleteObject(metaPath).catch(() => {}),
]);
}
// ========================================================================
// Helper Methods
// ========================================================================
/**
* Escape a URL for safe use in storage paths.
*/
private escapeUrl(url: string): string {
// Remove protocol prefix and escape special characters
return url
.replace(/^https?:\/\//, '')
.replace(/[\/\\:*?"<>|]/g, '_')
.replace(/__+/g, '_');
}
/** /**
* Build a unique cache key for a request context. * Build a unique cache key for a request context.
* Includes escaped upstream URL for multi-upstream support.
*/ */
private buildCacheKey(context: IUpstreamFetchContext): string { private buildCacheKey(context: IUpstreamFetchContext, upstreamUrl?: string): string {
// Include method, protocol, path, and sorted query params // Include method, protocol, path, and sorted query params
const queryString = Object.keys(context.query) const queryString = Object.keys(context.query)
.sort() .sort()
.map(k => `${k}=${context.query[k]}`) .map(k => `${k}=${context.query[k]}`)
.join('&'); .join('&');
return `${context.protocol}:${context.method}:${context.path}${queryString ? '?' + queryString : ''}`; const baseKey = `${context.protocol}:${context.method}:${context.path}${queryString ? '?' + queryString : ''}`;
if (upstreamUrl) {
return `${this.escapeUrl(upstreamUrl)}/${baseKey}`;
}
return baseKey;
} }
/** /**
@@ -333,27 +534,27 @@ export class UpstreamCache {
*/ */
private evictOldest(): void { private evictOldest(): void {
// Evict 10% of max entries // Evict 10% of max entries
const evictCount = Math.ceil(this.maxEntries * 0.1); const evictCount = Math.ceil(this.maxMemoryEntries * 0.1);
let evicted = 0; let evicted = 0;
// First, try to evict stale entries // First, try to evict stale entries
const now = new Date(); const now = new Date();
for (const [key, entry] of this.cache.entries()) { for (const [key, entry] of this.memoryCache.entries()) {
if (evicted >= evictCount) break; if (evicted >= evictCount) break;
if (entry.stale || (entry.expiresAt && entry.expiresAt < now)) { if (entry.stale || (entry.expiresAt && entry.expiresAt < now)) {
this.cache.delete(key); this.memoryCache.delete(key);
evicted++; evicted++;
} }
} }
// If not enough evicted, evict oldest by cachedAt // If not enough evicted, evict oldest by cachedAt
if (evicted < evictCount) { if (evicted < evictCount) {
const entries = Array.from(this.cache.entries()) const entries = Array.from(this.memoryCache.entries())
.sort((a, b) => a[1].cachedAt.getTime() - b[1].cachedAt.getTime()); .sort((a, b) => a[1].cachedAt.getTime() - b[1].cachedAt.getTime());
for (const [key] of entries) { for (const [key] of entries) {
if (evicted >= evictCount) break; if (evicted >= evictCount) break;
this.cache.delete(key); this.memoryCache.delete(key);
evicted++; evicted++;
} }
} }
@@ -375,17 +576,17 @@ export class UpstreamCache {
} }
/** /**
* Remove all expired entries. * Remove all expired entries from memory cache.
*/ */
private cleanup(): void { private cleanup(): void {
const now = new Date(); const now = new Date();
const staleDeadline = new Date(now.getTime() - this.config.staleMaxAgeSeconds * 1000); const staleDeadline = new Date(now.getTime() - this.config.staleMaxAgeSeconds * 1000);
for (const [key, entry] of this.cache.entries()) { for (const [key, entry] of this.memoryCache.entries()) {
if (entry.expiresAt) { if (entry.expiresAt) {
// Remove if past stale deadline // Remove if past stale deadline
if (entry.expiresAt < staleDeadline) { if (entry.expiresAt < staleDeadline) {
this.cache.delete(key); this.memoryCache.delete(key);
} }
} }
} }
@@ -406,7 +607,7 @@ export interface ICacheSetOptions {
* Cache statistics. * Cache statistics.
*/ */
export interface ICacheStats { export interface ICacheStats {
/** Total number of cached entries */ /** Total number of cached entries in memory */
totalEntries: number; totalEntries: number;
/** Number of fresh (non-expired) entries */ /** Number of fresh (non-expired) entries */
freshEntries: number; freshEntries: number;
@@ -414,10 +615,12 @@ export interface ICacheStats {
staleEntries: number; staleEntries: number;
/** Number of negative cache entries */ /** Number of negative cache entries */
negativeEntries: number; negativeEntries: number;
/** Total size of cached data in bytes */ /** Total size of cached data in bytes (memory only) */
totalSizeBytes: number; totalSizeBytes: number;
/** Maximum allowed entries */ /** Maximum allowed memory entries */
maxEntries: number; maxEntries: number;
/** Whether caching is enabled */ /** Whether caching is enabled */
enabled: boolean; enabled: boolean;
/** Whether S3 storage is configured */
hasStorage: boolean;
} }

View File

@@ -1,4 +1,4 @@
import type { TRegistryProtocol } from '../core/interfaces.core.js'; import type { TRegistryProtocol, IRequestActor } from '../core/interfaces.core.js';
/** /**
* Scope rule for routing requests to specific upstreams. * Scope rule for routing requests to specific upstreams.
@@ -86,6 +86,8 @@ export interface IUpstreamRegistryConfig {
cache?: Partial<IUpstreamCacheConfig>; cache?: Partial<IUpstreamCacheConfig>;
/** Resilience configuration overrides */ /** Resilience configuration overrides */
resilience?: Partial<IUpstreamResilienceConfig>; resilience?: Partial<IUpstreamResilienceConfig>;
/** API path prefix for OCI registries (default: /v2). Useful for registries behind reverse proxies. */
apiPrefix?: string;
} }
/** /**
@@ -146,6 +148,8 @@ export interface IUpstreamFetchContext {
headers: Record<string, string>; headers: Record<string, string>;
/** Query parameters */ /** Query parameters */
query: Record<string, string>; query: Record<string, string>;
/** Actor performing the request (for cache key isolation) */
actor?: IRequestActor;
} }
/** /**
@@ -193,3 +197,80 @@ export const DEFAULT_RESILIENCE_CONFIG: IUpstreamResilienceConfig = {
circuitBreakerThreshold: 5, circuitBreakerThreshold: 5,
circuitBreakerResetMs: 30000, circuitBreakerResetMs: 30000,
}; };
// ============================================================================
// Upstream Provider Interfaces
// ============================================================================
/**
* Context for resolving upstream configuration.
* Passed to IUpstreamProvider per-request to enable dynamic upstream routing.
*/
export interface IUpstreamResolutionContext {
/** Protocol being accessed */
protocol: TRegistryProtocol;
/** Resource identifier (package name, repository, coordinates, etc.) */
resource: string;
/** Extracted scope (e.g., "company" from "@company/pkg", "myorg" from "myorg/image") */
scope: string | null;
/** Actor performing the request */
actor?: IRequestActor;
/** HTTP method */
method: string;
/** Resource type (packument, tarball, manifest, blob, etc.) */
resourceType: string;
}
/**
* Dynamic upstream configuration provider.
* Implement this interface to provide per-request upstream routing
* based on actor context (user, organization, etc.)
*
* @example
* ```typescript
* class OrgUpstreamProvider implements IUpstreamProvider {
* constructor(private db: Database) {}
*
* async resolveUpstreamConfig(ctx: IUpstreamResolutionContext) {
* if (ctx.actor?.orgId) {
* const orgConfig = await this.db.getOrgUpstream(ctx.actor.orgId, ctx.protocol);
* if (orgConfig) return orgConfig;
* }
* return this.db.getDefaultUpstream(ctx.protocol);
* }
* }
* ```
*/
export interface IUpstreamProvider {
/** Optional initialization */
init?(): Promise<void>;
/**
* Resolve upstream configuration for a request.
* @param context - Information about the current request
* @returns Upstream config to use, or null to skip upstream lookup
*/
resolveUpstreamConfig(context: IUpstreamResolutionContext): Promise<IProtocolUpstreamConfig | null>;
}
/**
* Static upstream provider for simple configurations.
* Use this when you have fixed upstream registries that don't change per-request.
*
* @example
* ```typescript
* const provider = new StaticUpstreamProvider({
* npm: {
* enabled: true,
* upstreams: [{ id: 'npmjs', url: 'https://registry.npmjs.org', priority: 1, enabled: true, auth: { type: 'none' } }],
* },
* });
* ```
*/
export class StaticUpstreamProvider implements IUpstreamProvider {
constructor(private configs: Partial<Record<TRegistryProtocol, IProtocolUpstreamConfig>>) {}
async resolveUpstreamConfig(ctx: IUpstreamResolutionContext): Promise<IProtocolUpstreamConfig | null> {
return this.configs[ctx.protocol] ?? null;
}
}

View File

@@ -4,9 +4,7 @@
"module": "NodeNext", "module": "NodeNext",
"moduleResolution": "NodeNext", "moduleResolution": "NodeNext",
"esModuleInterop": true, "esModuleInterop": true,
"verbatimModuleSyntax": true, "verbatimModuleSyntax": true
"baseUrl": ".",
"paths": {}
}, },
"exclude": ["dist_*/**/*.d.ts"] "exclude": ["dist_*/**/*.d.ts"]
} }