feat(core,storage,oci,registry-config): add streaming response support and configurable registry URLs across protocols

This commit is contained in:
2026-03-24 22:59:37 +00:00
parent 1f0acf2825
commit 7da1a35efe
42 changed files with 4179 additions and 5396 deletions

24
.smartconfig.json Normal file
View File

@@ -0,0 +1,24 @@
{
"@git.zone/cli": {
"projectType": "npm",
"module": {
"githost": "code.foss.global",
"gitscope": "push.rocks",
"gitrepo": "smartregistry",
"description": "A composable TypeScript library implementing OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems registries for building unified container and package registries",
"npmPackagename": "@push.rocks/smartregistry",
"license": "MIT",
"projectDomain": "push.rocks"
},
"release": {
"registries": [
"https://verdaccio.lossless.digital",
"https://registry.npmjs.org"
],
"accessLevel": "public"
}
},
"@ship.zone/szci": {
"npmGlobalTools": []
}
}

View File

@@ -1,7 +1,7 @@
{ {
"json.schemas": [ "json.schemas": [
{ {
"fileMatch": ["/npmextra.json"], "fileMatch": ["/.smartconfig.json"],
"schema": { "schema": {
"type": "object", "type": "object",
"properties": { "properties": {

View File

@@ -1,5 +1,14 @@
# Changelog # Changelog
## 2026-03-24 - 2.8.0 - feat(core,storage,oci,registry-config)
add streaming response support and configurable registry URLs across protocols
- Normalize SmartRegistry responses to ReadableStream bodies at the public API boundary and add stream helper utilities for buffers, JSON, and hashing
- Add streaming storage accessors for OCI, npm, Maven, Cargo, Composer, PyPI, and RubyGems downloads to reduce in-memory buffering
- Make per-protocol registryUrl configurable so CLI and integration tests can use correct host and port values
- Refactor OCI blob uploads to persist chunks in storage during upload and clean up temporary chunk objects after completion or expiry
- Update tests and storage integration to use the new stream-based response model and smartstorage backend
## 2025-12-03 - 2.7.0 - feat(upstream) ## 2025-12-03 - 2.7.0 - feat(upstream)
Add dynamic per-request upstream provider and integrate into registries Add dynamic per-request upstream provider and integrate into registries

View File

@@ -1,18 +0,0 @@
{
"gitzone": {
"projectType": "npm",
"module": {
"githost": "code.foss.global",
"gitscope": "push.rocks",
"gitrepo": "smartregistry",
"description": "a registry for npm modules and oci images",
"npmPackagename": "@push.rocks/smartregistry",
"license": "MIT",
"projectDomain": "push.rocks"
}
},
"npmci": {
"npmGlobalTools": [],
"npmAccessLevel": "public"
}
}

View File

@@ -10,17 +10,17 @@
"license": "MIT", "license": "MIT",
"scripts": { "scripts": {
"test": "(tstest test/ --verbose --logfile --timeout 240)", "test": "(tstest test/ --verbose --logfile --timeout 240)",
"build": "(tsbuild --web --allowimplicitany)", "build": "(tsbuild --allowimplicitany)",
"buildDocs": "(tsdoc)" "buildDocs": "(tsdoc)"
}, },
"devDependencies": { "devDependencies": {
"@git.zone/tsbuild": "^3.1.0", "@git.zone/tsbuild": "^4.4.0",
"@git.zone/tsbundle": "^2.0.5", "@git.zone/tsbundle": "^2.10.0",
"@git.zone/tsrun": "^2.0.0", "@git.zone/tsrun": "^2.0.2",
"@git.zone/tstest": "^3.1.0", "@git.zone/tstest": "^3.6.0",
"@push.rocks/smartarchive": "^5.0.1", "@push.rocks/smartarchive": "^5.2.1",
"@push.rocks/smarts3": "^5.1.0", "@push.rocks/smartstorage": "^6.3.2",
"@types/node": "^24.10.1" "@types/node": "^25.5.0"
}, },
"repository": { "repository": {
"type": "git", "type": "git",
@@ -39,7 +39,7 @@
"dist_ts_web/**/*", "dist_ts_web/**/*",
"assets/**/*", "assets/**/*",
"cli.js", "cli.js",
"npmextra.json", ".smartconfig.json",
"readme.md" "readme.md"
], ],
"pnpm": { "pnpm": {
@@ -47,13 +47,13 @@
}, },
"dependencies": { "dependencies": {
"@push.rocks/qenv": "^6.1.3", "@push.rocks/qenv": "^6.1.3",
"@push.rocks/smartbucket": "^4.3.0", "@push.rocks/smartbucket": "^4.5.1",
"@push.rocks/smartlog": "^3.1.10", "@push.rocks/smartlog": "^3.2.1",
"@push.rocks/smartpath": "^6.0.0", "@push.rocks/smartpath": "^6.0.0",
"@push.rocks/smartrequest": "^5.0.1", "@push.rocks/smartrequest": "^5.0.1",
"@tsclass/tsclass": "^9.3.0", "@tsclass/tsclass": "^9.5.0",
"adm-zip": "^0.5.10", "adm-zip": "^0.5.16",
"minimatch": "^10.1.1" "minimatch": "^10.2.4"
}, },
"packageManager": "pnpm@10.18.1+sha512.77a884a165cbba2d8d1c19e3b4880eee6d2fcabd0d879121e282196b80042351d5eb3ca0935fa599da1dc51265cc68816ad2bddd2a2de5ea9fdf92adbec7cd34" "packageManager": "pnpm@10.18.1+sha512.77a884a165cbba2d8d1c19e3b4880eee6d2fcabd0d879121e282196b80042351d5eb3ca0935fa599da1dc51265cc68816ad2bddd2a2de5ea9fdf92adbec7cd34"
} }

7100
pnpm-lock.yaml generated

File diff suppressed because it is too large Load Diff

1110
readme.md

File diff suppressed because it is too large Load Diff

View File

@@ -2,6 +2,7 @@ import { tap, expect } from '@git.zone/tstest';
import { RegistryStorage } from '../ts/core/classes.registrystorage.js'; import { RegistryStorage } from '../ts/core/classes.registrystorage.js';
import { CargoRegistry } from '../ts/cargo/classes.cargoregistry.js'; import { CargoRegistry } from '../ts/cargo/classes.cargoregistry.js';
import { AuthManager } from '../ts/core/classes.authmanager.js'; import { AuthManager } from '../ts/core/classes.authmanager.js';
import { streamToJson } from '../ts/core/helpers.stream.js';
// Test index path calculation // Test index path calculation
tap.test('should calculate correct index paths for different crate names', async () => { tap.test('should calculate correct index paths for different crate names', async () => {
@@ -123,9 +124,10 @@ tap.test('should return valid config.json', async () => {
expect(response.status).to.equal(200); expect(response.status).to.equal(200);
expect(response.headers['Content-Type']).to.equal('application/json'); expect(response.headers['Content-Type']).to.equal('application/json');
expect(response.body).to.be.an('object'); const body = await streamToJson(response.body);
expect(response.body.dl).to.include('/api/v1/crates/{crate}/{version}/download'); expect(body).to.be.an('object');
expect(response.body.api).to.equal('http://localhost:5000/cargo'); expect(body.dl).to.include('/api/v1/crates/{crate}/{version}/download');
expect(body.api).to.equal('http://localhost:5000/cargo');
}); });
export default tap.start(); export default tap.start();

View File

@@ -65,7 +65,9 @@ export async function cleanupS3Bucket(prefix?: string): Promise<void> {
/** /**
* Create a test SmartRegistry instance with all protocols enabled * Create a test SmartRegistry instance with all protocols enabled
*/ */
export async function createTestRegistry(): Promise<SmartRegistry> { export async function createTestRegistry(options?: {
registryUrl?: string;
}): Promise<SmartRegistry> {
// Read S3 config from env.json // Read S3 config from env.json
const s3AccessKey = await testQenv.getEnvVarOnDemand('S3_ACCESSKEY'); const s3AccessKey = await testQenv.getEnvVarOnDemand('S3_ACCESSKEY');
const s3SecretKey = await testQenv.getEnvVarOnDemand('S3_SECRETKEY'); const s3SecretKey = await testQenv.getEnvVarOnDemand('S3_SECRETKEY');
@@ -103,30 +105,37 @@ export async function createTestRegistry(): Promise<SmartRegistry> {
oci: { oci: {
enabled: true, enabled: true,
basePath: '/oci', basePath: '/oci',
...(options?.registryUrl ? { registryUrl: `${options.registryUrl}/oci` } : {}),
}, },
npm: { npm: {
enabled: true, enabled: true,
basePath: '/npm', basePath: '/npm',
...(options?.registryUrl ? { registryUrl: `${options.registryUrl}/npm` } : {}),
}, },
maven: { maven: {
enabled: true, enabled: true,
basePath: '/maven', basePath: '/maven',
...(options?.registryUrl ? { registryUrl: `${options.registryUrl}/maven` } : {}),
}, },
composer: { composer: {
enabled: true, enabled: true,
basePath: '/composer', basePath: '/composer',
...(options?.registryUrl ? { registryUrl: `${options.registryUrl}/composer` } : {}),
}, },
cargo: { cargo: {
enabled: true, enabled: true,
basePath: '/cargo', basePath: '/cargo',
...(options?.registryUrl ? { registryUrl: `${options.registryUrl}/cargo` } : {}),
}, },
pypi: { pypi: {
enabled: true, enabled: true,
basePath: '/pypi', basePath: '/pypi',
...(options?.registryUrl ? { registryUrl: options.registryUrl } : {}),
}, },
rubygems: { rubygems: {
enabled: true, enabled: true,
basePath: '/rubygems', basePath: '/rubygems',
...(options?.registryUrl ? { registryUrl: `${options.registryUrl}/rubygems` } : {}),
}, },
}; };
@@ -441,7 +450,7 @@ class TestClass
}, },
]; ];
return zipTools.createZip(entries); return Buffer.from(await zipTools.createZip(entries));
} }
/** /**
@@ -515,7 +524,7 @@ def hello():
}, },
]; ];
return zipTools.createZip(entries); return Buffer.from(await zipTools.createZip(entries));
} }
/** /**
@@ -576,7 +585,7 @@ def hello():
}, },
]; ];
return tarTools.packFilesToTarGz(entries); return Buffer.from(await tarTools.packFilesToTarGz(entries));
} }
/** /**
@@ -647,7 +656,7 @@ summary: Test gem for SmartRegistry
test_files: [] test_files: []
`; `;
const metadataGz = await gzipTools.compress(Buffer.from(metadataYaml, 'utf-8')); const metadataGz = Buffer.from(await gzipTools.compress(Buffer.from(metadataYaml, 'utf-8')));
// Create data.tar.gz content // Create data.tar.gz content
const libContent = `# ${gemName} const libContent = `# ${gemName}
@@ -668,7 +677,7 @@ end
}, },
]; ];
const dataTarGz = await tarTools.packFilesToTarGz(dataEntries); const dataTarGz = Buffer.from(await tarTools.packFilesToTarGz(dataEntries));
// Create the outer gem (tar.gz containing metadata.gz and data.tar.gz) // Create the outer gem (tar.gz containing metadata.gz and data.tar.gz)
const gemEntries: smartarchive.IArchiveEntry[] = [ const gemEntries: smartarchive.IArchiveEntry[] = [
@@ -683,7 +692,7 @@ end
]; ];
// RubyGems .gem files are plain tar archives (NOT gzipped), containing metadata.gz and data.tar.gz // RubyGems .gem files are plain tar archives (NOT gzipped), containing metadata.gz and data.tar.gz
return tarTools.packFiles(gemEntries); return Buffer.from(await tarTools.packFiles(gemEntries));
} }
/** /**

View File

@@ -79,16 +79,10 @@ async function createHttpServer(
res.setHeader(key, value); res.setHeader(key, value);
} }
// Send body // Send body (response.body is always ReadableStream<Uint8Array> or undefined)
if (response.body) { if (response.body) {
if (Buffer.isBuffer(response.body)) { const { Readable } = await import('stream');
res.end(response.body); Readable.fromWeb(response.body).pipe(res);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
} else { } else {
res.end(); res.end();
} }
@@ -251,8 +245,11 @@ function cleanupTestDir(dir: string): void {
// ======================================================================== // ========================================================================
tap.test('Cargo CLI: should setup registry and HTTP server', async () => { tap.test('Cargo CLI: should setup registry and HTTP server', async () => {
// Create registry // Use port 5000
registry = await createTestRegistry(); registryPort = 5000;
// Create registry with correct registryUrl for CLI tests
registry = await createTestRegistry({ registryUrl: `http://localhost:${registryPort}` });
const tokens = await createTestTokens(registry); const tokens = await createTestTokens(registry);
cargoToken = tokens.cargoToken; cargoToken = tokens.cargoToken;
@@ -266,10 +263,6 @@ tap.test('Cargo CLI: should setup registry and HTTP server', async () => {
} catch (error) { } catch (error) {
// Ignore error if operation fails // Ignore error if operation fails
} }
// Use port 5000 (hardcoded in CargoRegistry default config)
// TODO: Once registryUrl is configurable, use dynamic port like npm test (35001)
registryPort = 5000;
const serverSetup = await createHttpServer(registry, registryPort); const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server; server = serverSetup.server;
registryUrl = serverSetup.url; registryUrl = serverSetup.url;

View File

@@ -84,16 +84,10 @@ async function createHttpServer(
res.setHeader(key, value); res.setHeader(key, value);
} }
// Send body // Send body (response.body is always ReadableStream<Uint8Array> or undefined)
if (response.body) { if (response.body) {
if (Buffer.isBuffer(response.body)) { const { Readable } = await import('stream');
res.end(response.body); Readable.fromWeb(response.body).pipe(res);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
} else { } else {
res.end(); res.end();
} }
@@ -249,16 +243,16 @@ tap.test('Composer CLI: should verify composer is installed', async () => {
}); });
tap.test('Composer CLI: should setup registry and HTTP server', async () => { tap.test('Composer CLI: should setup registry and HTTP server', async () => {
// Create registry // Use port 38000 (avoids conflicts with other tests)
registry = await createTestRegistry(); registryPort = 38000;
// Create registry with correct registryUrl for CLI tests
registry = await createTestRegistry({ registryUrl: `http://localhost:${registryPort}` });
const tokens = await createTestTokens(registry); const tokens = await createTestTokens(registry);
composerToken = tokens.composerToken; composerToken = tokens.composerToken;
expect(registry).toBeInstanceOf(SmartRegistry); expect(registry).toBeInstanceOf(SmartRegistry);
expect(composerToken).toBeTypeOf('string'); expect(composerToken).toBeTypeOf('string');
// Use port 38000 (avoids conflicts with other tests)
registryPort = 38000;
const serverSetup = await createHttpServer(registry, registryPort); const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server; server = serverSetup.server;
registryUrl = serverSetup.url; registryUrl = serverSetup.url;

View File

@@ -1,5 +1,6 @@
import { expect, tap } from '@git.zone/tstest/tapbundle'; import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js'; import { SmartRegistry } from '../ts/index.js';
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
import { createTestRegistry, createTestTokens, createComposerZip } from './helpers/registry.js'; import { createTestRegistry, createTestTokens, createComposerZip } from './helpers/registry.js';
let registry: SmartRegistry; let registry: SmartRegistry;
@@ -41,9 +42,10 @@ tap.test('Composer: should return packages.json (GET /packages.json)', async ()
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('metadata-url'); const body = await streamToJson(response.body);
expect(response.body).toHaveProperty('available-packages'); expect(body).toHaveProperty('metadata-url');
expect(response.body['available-packages']).toBeInstanceOf(Array); expect(body).toHaveProperty('available-packages');
expect(body['available-packages']).toBeInstanceOf(Array);
}); });
tap.test('Composer: should upload a package (PUT /packages/{vendor/package})', async () => { tap.test('Composer: should upload a package (PUT /packages/{vendor/package})', async () => {
@@ -59,9 +61,10 @@ tap.test('Composer: should upload a package (PUT /packages/{vendor/package})', a
}); });
expect(response.status).toEqual(201); expect(response.status).toEqual(201);
expect(response.body.status).toEqual('success'); const body = await streamToJson(response.body);
expect(response.body.package).toEqual(testPackageName); expect(body.status).toEqual('success');
expect(response.body.version).toEqual(testVersion); expect(body.package).toEqual(testPackageName);
expect(body.version).toEqual(testVersion);
}); });
tap.test('Composer: should retrieve package metadata (GET /p2/{vendor/package}.json)', async () => { tap.test('Composer: should retrieve package metadata (GET /p2/{vendor/package}.json)', async () => {
@@ -73,11 +76,12 @@ tap.test('Composer: should retrieve package metadata (GET /p2/{vendor/package}.j
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('packages'); const body = await streamToJson(response.body);
expect(response.body.packages[testPackageName]).toBeInstanceOf(Array); expect(body).toHaveProperty('packages');
expect(response.body.packages[testPackageName].length).toEqual(1); expect(body.packages[testPackageName]).toBeInstanceOf(Array);
expect(body.packages[testPackageName].length).toEqual(1);
const packageData = response.body.packages[testPackageName][0]; const packageData = body.packages[testPackageName][0];
expect(packageData.name).toEqual(testPackageName); expect(packageData.name).toEqual(testPackageName);
expect(packageData.version).toEqual(testVersion); expect(packageData.version).toEqual(testVersion);
expect(packageData.version_normalized).toEqual('1.0.0.0'); expect(packageData.version_normalized).toEqual('1.0.0.0');
@@ -97,7 +101,8 @@ tap.test('Composer: should download package ZIP (GET /dists/{vendor/package}/{re
query: {}, query: {},
}); });
const reference = metadataResponse.body.packages[testPackageName][0].dist.reference; const metaBody = await streamToJson(metadataResponse.body);
const reference = metaBody.packages[testPackageName][0].dist.reference;
const response = await registry.handleRequest({ const response = await registry.handleRequest({
method: 'GET', method: 'GET',
@@ -107,7 +112,8 @@ tap.test('Composer: should download package ZIP (GET /dists/{vendor/package}/{re
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
expect(response.headers['Content-Type']).toEqual('application/zip'); expect(response.headers['Content-Type']).toEqual('application/zip');
expect(response.headers['Content-Disposition']).toContain('attachment'); expect(response.headers['Content-Disposition']).toContain('attachment');
}); });
@@ -121,9 +127,10 @@ tap.test('Composer: should list packages (GET /packages/list.json)', async () =>
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('packageNames'); const body = await streamToJson(response.body);
expect(response.body.packageNames).toBeInstanceOf(Array); expect(body).toHaveProperty('packageNames');
expect(response.body.packageNames).toContain(testPackageName); expect(body.packageNames).toBeInstanceOf(Array);
expect(body.packageNames).toContain(testPackageName);
}); });
tap.test('Composer: should filter package list (GET /packages/list.json?filter=vendor/*)', async () => { tap.test('Composer: should filter package list (GET /packages/list.json?filter=vendor/*)', async () => {
@@ -135,8 +142,9 @@ tap.test('Composer: should filter package list (GET /packages/list.json?filter=v
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body.packageNames).toBeInstanceOf(Array); const body = await streamToJson(response.body);
expect(response.body.packageNames).toContain(testPackageName); expect(body.packageNames).toBeInstanceOf(Array);
expect(body.packageNames).toContain(testPackageName);
}); });
tap.test('Composer: should prevent duplicate version upload', async () => { tap.test('Composer: should prevent duplicate version upload', async () => {
@@ -152,8 +160,9 @@ tap.test('Composer: should prevent duplicate version upload', async () => {
}); });
expect(response.status).toEqual(409); expect(response.status).toEqual(409);
expect(response.body.status).toEqual('error'); const body = await streamToJson(response.body);
expect(response.body.message).toContain('already exists'); expect(body.status).toEqual('error');
expect(body.message).toContain('already exists');
}); });
tap.test('Composer: should upload a second version', async () => { tap.test('Composer: should upload a second version', async () => {
@@ -172,8 +181,9 @@ tap.test('Composer: should upload a second version', async () => {
}); });
expect(response.status).toEqual(201); expect(response.status).toEqual(201);
expect(response.body.status).toEqual('success'); const body = await streamToJson(response.body);
expect(response.body.version).toEqual(testVersion2); expect(body.status).toEqual('success');
expect(body.version).toEqual(testVersion2);
}); });
tap.test('Composer: should return multiple versions in metadata', async () => { tap.test('Composer: should return multiple versions in metadata', async () => {
@@ -185,10 +195,11 @@ tap.test('Composer: should return multiple versions in metadata', async () => {
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body.packages[testPackageName]).toBeInstanceOf(Array); const body = await streamToJson(response.body);
expect(response.body.packages[testPackageName].length).toEqual(2); expect(body.packages[testPackageName]).toBeInstanceOf(Array);
expect(body.packages[testPackageName].length).toEqual(2);
const versions = response.body.packages[testPackageName].map((p: any) => p.version); const versions = body.packages[testPackageName].map((p: any) => p.version);
expect(versions).toContain('1.0.0'); expect(versions).toContain('1.0.0');
expect(versions).toContain('1.1.0'); expect(versions).toContain('1.1.0');
}); });
@@ -213,8 +224,9 @@ tap.test('Composer: should delete a specific version (DELETE /packages/{vendor/p
query: {}, query: {},
}); });
expect(metadataResponse.body.packages[testPackageName].length).toEqual(1); const metaBody = await streamToJson(metadataResponse.body);
expect(metadataResponse.body.packages[testPackageName][0].version).toEqual('1.1.0'); expect(metaBody.packages[testPackageName].length).toEqual(1);
expect(metaBody.packages[testPackageName][0].version).toEqual('1.1.0');
}); });
tap.test('Composer: should require auth for package upload', async () => { tap.test('Composer: should require auth for package upload', async () => {
@@ -231,7 +243,8 @@ tap.test('Composer: should require auth for package upload', async () => {
}); });
expect(response.status).toEqual(401); expect(response.status).toEqual(401);
expect(response.body.status).toEqual('error'); const body = await streamToJson(response.body);
expect(body.status).toEqual('error');
}); });
tap.test('Composer: should reject invalid ZIP (no composer.json)', async () => { tap.test('Composer: should reject invalid ZIP (no composer.json)', async () => {
@@ -249,8 +262,9 @@ tap.test('Composer: should reject invalid ZIP (no composer.json)', async () => {
}); });
expect(response.status).toEqual(400); expect(response.status).toEqual(400);
expect(response.body.status).toEqual('error'); const body = await streamToJson(response.body);
expect(response.body.message).toContain('composer.json'); expect(body.status).toEqual('error');
expect(body.message).toContain('composer.json');
}); });
tap.test('Composer: should delete entire package (DELETE /packages/{vendor/package})', async () => { tap.test('Composer: should delete entire package (DELETE /packages/{vendor/package})', async () => {

View File

@@ -1,5 +1,6 @@
import { expect, tap } from '@git.zone/tstest/tapbundle'; import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js'; import { SmartRegistry } from '../ts/index.js';
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
import { import {
createTestRegistry, createTestRegistry,
createTestTokens, createTestTokens,
@@ -79,7 +80,9 @@ tap.test('Integration: should handle /simple path for PyPI', async () => {
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toStartWith('text/html'); expect(response.headers['Content-Type']).toStartWith('text/html');
expect(response.body).toContain('integration-test-py'); const body = await streamToBuffer(response.body);
const text = body.toString('utf-8');
expect(text).toContain('integration-test-py');
}); });
tap.test('Integration: should reject PyPI token for RubyGems endpoint', async () => { tap.test('Integration: should reject PyPI token for RubyGems endpoint', async () => {
@@ -135,8 +138,9 @@ tap.test('Integration: should return 404 for unknown paths', async () => {
}); });
expect(response.status).toEqual(404); expect(response.status).toEqual(404);
expect(response.body).toHaveProperty('error'); const body = await streamToJson(response.body);
expect((response.body as any).error).toEqual('NOT_FOUND'); expect(body).toHaveProperty('error');
expect(body.error).toEqual('NOT_FOUND');
}); });
tap.test('Integration: should retrieve PyPI registry instance', async () => { tap.test('Integration: should retrieve PyPI registry instance', async () => {

View File

@@ -1,32 +1,34 @@
/** /**
* Integration test for smartregistry with smarts3 * Integration test for smartregistry with smartstorage
* Verifies that smartregistry works with a local S3-compatible server * Verifies that smartregistry works with a local S3-compatible server
*/ */
import { expect, tap } from '@git.zone/tstest/tapbundle'; import { expect, tap } from '@git.zone/tstest/tapbundle';
import * as smarts3Module from '@push.rocks/smarts3'; import * as smartstorageModule from '@push.rocks/smartstorage';
import { SmartRegistry } from '../ts/classes.smartregistry.js'; import { SmartRegistry } from '../ts/classes.smartregistry.js';
import type { IRegistryConfig } from '../ts/core/interfaces.core.js'; import type { IRegistryConfig } from '../ts/core/interfaces.core.js';
import { streamToJson } from '../ts/core/helpers.stream.js';
import * as crypto from 'crypto'; import * as crypto from 'crypto';
let s3Server: smarts3Module.Smarts3; let s3Server: smartstorageModule.SmartStorage;
let registry: SmartRegistry; let registry: SmartRegistry;
/** /**
* Setup: Start smarts3 server * Setup: Start smartstorage server
*/ */
tap.test('should start smarts3 server', async () => { tap.test('should start smartstorage server', async () => {
s3Server = await smarts3Module.Smarts3.createAndStart({ s3Server = await smartstorageModule.SmartStorage.createAndStart({
server: { server: {
port: 3456, // Use different port to avoid conflicts with other tests port: 3456,
host: '0.0.0.0', address: '0.0.0.0',
silent: true,
}, },
storage: { storage: {
cleanSlate: true, // Fresh storage for each test run cleanSlate: true,
bucketsDir: './.nogit/smarts3-test-buckets', directory: './.nogit/smartstorage-test-buckets',
}, },
logging: { logging: {
silent: true, // Reduce test output noise enabled: false,
}, },
}); });
@@ -34,20 +36,10 @@ tap.test('should start smarts3 server', async () => {
}); });
/** /**
* Setup: Create SmartRegistry with smarts3 configuration * Setup: Create SmartRegistry with smartstorage configuration
*/ */
tap.test('should create SmartRegistry instance with smarts3 IS3Descriptor', async () => { tap.test('should create SmartRegistry instance with smarts3 IS3Descriptor', async () => {
// Manually construct IS3Descriptor based on smarts3 configuration const s3Descriptor = await s3Server.getStorageDescriptor();
// Note: smarts3.getS3Descriptor() returns empty object as of v5.1.0
// This is a known limitation - smarts3 doesn't expose its config properly
const s3Descriptor = {
endpoint: 'localhost',
port: 3456,
accessKey: 'test', // smarts3 doesn't require real credentials
accessSecret: 'test',
useSsl: false,
region: 'us-east-1',
};
const config: IRegistryConfig = { const config: IRegistryConfig = {
storage: { storage: {
@@ -97,7 +89,7 @@ tap.test('should create SmartRegistry instance with smarts3 IS3Descriptor', asyn
}); });
/** /**
* Test NPM protocol with smarts3 * Test NPM protocol with smartstorage
*/ */
tap.test('NPM: should publish package to smarts3', async () => { tap.test('NPM: should publish package to smarts3', async () => {
const authManager = registry.getAuthManager(); const authManager = registry.getAuthManager();
@@ -139,7 +131,7 @@ tap.test('NPM: should publish package to smarts3', async () => {
body: packageData, body: packageData,
}); });
expect(response.status).toEqual(201); // 201 Created is correct for publishing expect(response.status).toEqual(201);
}); });
tap.test('NPM: should retrieve package from smarts3', async () => { tap.test('NPM: should retrieve package from smarts3', async () => {
@@ -151,12 +143,13 @@ tap.test('NPM: should retrieve package from smarts3', async () => {
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('name'); const body = await streamToJson(response.body);
expect(response.body.name).toEqual('test-package-smarts3'); expect(body).toHaveProperty('name');
expect(body.name).toEqual('test-package-smarts3');
}); });
/** /**
* Test OCI protocol with smarts3 * Test OCI protocol with smartstorage
*/ */
tap.test('OCI: should store blob in smarts3', async () => { tap.test('OCI: should store blob in smarts3', async () => {
const authManager = registry.getAuthManager(); const authManager = registry.getAuthManager();
@@ -173,7 +166,7 @@ tap.test('OCI: should store blob in smarts3', async () => {
// Initiate blob upload // Initiate blob upload
const initiateResponse = await registry.handleRequest({ const initiateResponse = await registry.handleRequest({
method: 'POST', method: 'POST',
path: '/oci/v2/test-image/blobs/uploads/', path: '/oci/test-image/blobs/uploads/',
headers: { headers: {
'Authorization': `Bearer ${token}`, 'Authorization': `Bearer ${token}`,
}, },
@@ -196,7 +189,7 @@ tap.test('OCI: should store blob in smarts3', async () => {
const uploadResponse = await registry.handleRequest({ const uploadResponse = await registry.handleRequest({
method: 'PUT', method: 'PUT',
path: `/oci/v2/test-image/blobs/uploads/${uploadId}`, path: `/oci/test-image/blobs/uploads/${uploadId}`,
headers: { headers: {
'Authorization': `Bearer ${token}`, 'Authorization': `Bearer ${token}`,
'Content-Type': 'application/octet-stream', 'Content-Type': 'application/octet-stream',
@@ -209,18 +202,9 @@ tap.test('OCI: should store blob in smarts3', async () => {
}); });
/** /**
* Test PyPI protocol with smarts3 * Test PyPI protocol with smartstorage
*/ */
tap.test('PyPI: should upload package to smarts3', async () => { tap.test('PyPI: should upload package to smarts3', async () => {
const authManager = registry.getAuthManager();
const userId = await authManager.authenticate({
username: 'testuser',
password: 'testpass',
});
const token = await authManager.createPypiToken(userId, false);
// Note: In a real test, this would be multipart/form-data
// For simplicity, we're testing the storage layer
const storage = registry.getStorage(); const storage = registry.getStorage();
// Store a test package file // Store a test package file
@@ -252,7 +236,7 @@ tap.test('PyPI: should upload package to smarts3', async () => {
}); });
/** /**
* Test Cargo protocol with smarts3 * Test Cargo protocol with smartstorage
*/ */
tap.test('Cargo: should store crate in smarts3', async () => { tap.test('Cargo: should store crate in smarts3', async () => {
const storage = registry.getStorage(); const storage = registry.getStorage();
@@ -281,11 +265,11 @@ tap.test('Cargo: should store crate in smarts3', async () => {
}); });
/** /**
* Cleanup: Stop smarts3 server * Cleanup: Stop smartstorage server
*/ */
tap.test('should stop smarts3 server', async () => { tap.test('should stop smartstorage server', async () => {
await s3Server.stop(); await s3Server.stop();
expect(true).toEqual(true); // Just verify it completes without error expect(true).toEqual(true);
}); });
export default tap.start(); export default tap.start();

View File

@@ -79,16 +79,10 @@ async function createHttpServer(
res.setHeader(key, value); res.setHeader(key, value);
} }
// Send body // Send body (response.body is always ReadableStream<Uint8Array> or undefined)
if (response.body) { if (response.body) {
if (Buffer.isBuffer(response.body)) { const { Readable } = await import('stream');
res.end(response.body); Readable.fromWeb(response.body).pipe(res);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
} else { } else {
res.end(); res.end();
} }
@@ -282,16 +276,16 @@ tap.test('Maven CLI: should verify mvn is installed', async () => {
}); });
tap.test('Maven CLI: should setup registry and HTTP server', async () => { tap.test('Maven CLI: should setup registry and HTTP server', async () => {
// Create registry // Use port 37000 (avoids conflicts with other tests)
registry = await createTestRegistry(); registryPort = 37000;
// Create registry with correct registryUrl for CLI tests
registry = await createTestRegistry({ registryUrl: `http://localhost:${registryPort}` });
const tokens = await createTestTokens(registry); const tokens = await createTestTokens(registry);
mavenToken = tokens.mavenToken; mavenToken = tokens.mavenToken;
expect(registry).toBeInstanceOf(SmartRegistry); expect(registry).toBeInstanceOf(SmartRegistry);
expect(mavenToken).toBeTypeOf('string'); expect(mavenToken).toBeTypeOf('string');
// Use port 37000 (avoids conflicts with other tests)
registryPort = 37000;
const serverSetup = await createHttpServer(registry, registryPort); const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server; server = serverSetup.server;
registryUrl = serverSetup.url; registryUrl = serverSetup.url;

View File

@@ -1,5 +1,6 @@
import { expect, tap } from '@git.zone/tstest/tapbundle'; import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js'; import { SmartRegistry } from '../ts/index.js';
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
import { import {
createTestRegistry, createTestRegistry,
createTestTokens, createTestTokens,
@@ -88,10 +89,11 @@ tap.test('Maven: should retrieve uploaded POM file (GET)', async () => {
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect((response.body as Buffer).toString('utf-8')).toContain(testGroupId); expect(body).toBeInstanceOf(Buffer);
expect((response.body as Buffer).toString('utf-8')).toContain(testArtifactId); expect(body.toString('utf-8')).toContain(testGroupId);
expect((response.body as Buffer).toString('utf-8')).toContain(testVersion); expect(body.toString('utf-8')).toContain(testArtifactId);
expect(body.toString('utf-8')).toContain(testVersion);
expect(response.headers['Content-Type']).toEqual('application/xml'); expect(response.headers['Content-Type']).toEqual('application/xml');
}); });
@@ -107,7 +109,8 @@ tap.test('Maven: should retrieve uploaded JAR file (GET)', async () => {
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
expect(response.headers['Content-Type']).toEqual('application/java-archive'); expect(response.headers['Content-Type']).toEqual('application/java-archive');
}); });
@@ -124,8 +127,9 @@ tap.test('Maven: should retrieve MD5 checksum for JAR (GET *.jar.md5)', async ()
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect((response.body as Buffer).toString('utf-8')).toEqual(checksums.md5); expect(body).toBeInstanceOf(Buffer);
expect(body.toString('utf-8')).toEqual(checksums.md5);
expect(response.headers['Content-Type']).toEqual('text/plain'); expect(response.headers['Content-Type']).toEqual('text/plain');
}); });
@@ -142,8 +146,9 @@ tap.test('Maven: should retrieve SHA1 checksum for JAR (GET *.jar.sha1)', async
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect((response.body as Buffer).toString('utf-8')).toEqual(checksums.sha1); expect(body).toBeInstanceOf(Buffer);
expect(body.toString('utf-8')).toEqual(checksums.sha1);
expect(response.headers['Content-Type']).toEqual('text/plain'); expect(response.headers['Content-Type']).toEqual('text/plain');
}); });
@@ -160,8 +165,9 @@ tap.test('Maven: should retrieve SHA256 checksum for JAR (GET *.jar.sha256)', as
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect((response.body as Buffer).toString('utf-8')).toEqual(checksums.sha256); expect(body).toBeInstanceOf(Buffer);
expect(body.toString('utf-8')).toEqual(checksums.sha256);
expect(response.headers['Content-Type']).toEqual('text/plain'); expect(response.headers['Content-Type']).toEqual('text/plain');
}); });
@@ -178,8 +184,9 @@ tap.test('Maven: should retrieve SHA512 checksum for JAR (GET *.jar.sha512)', as
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect((response.body as Buffer).toString('utf-8')).toEqual(checksums.sha512); expect(body).toBeInstanceOf(Buffer);
expect(body.toString('utf-8')).toEqual(checksums.sha512);
expect(response.headers['Content-Type']).toEqual('text/plain'); expect(response.headers['Content-Type']).toEqual('text/plain');
}); });
@@ -194,8 +201,9 @@ tap.test('Maven: should retrieve maven-metadata.xml (GET)', async () => {
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
const xml = (response.body as Buffer).toString('utf-8'); expect(body).toBeInstanceOf(Buffer);
const xml = body.toString('utf-8');
expect(xml).toContain('<groupId>'); expect(xml).toContain('<groupId>');
expect(xml).toContain('<artifactId>'); expect(xml).toContain('<artifactId>');
expect(xml).toContain('<version>1.0.0</version>'); expect(xml).toContain('<version>1.0.0</version>');
@@ -247,7 +255,8 @@ tap.test('Maven: should upload a second version and update metadata', async () =
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
const xml = (response.body as Buffer).toString('utf-8'); const metaBody = await streamToBuffer(response.body);
const xml = metaBody.toString('utf-8');
expect(xml).toContain('<version>1.0.0</version>'); expect(xml).toContain('<version>1.0.0</version>');
expect(xml).toContain('<version>2.0.0</version>'); expect(xml).toContain('<version>2.0.0</version>');
expect(xml).toContain('<latest>2.0.0</latest>'); expect(xml).toContain('<latest>2.0.0</latest>');
@@ -285,7 +294,8 @@ tap.test('Maven: should return 404 for non-existent artifact', async () => {
}); });
expect(response.status).toEqual(404); expect(response.status).toEqual(404);
expect(response.body).toHaveProperty('error'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
}); });
tap.test('Maven: should return 401 for unauthorized upload', async () => { tap.test('Maven: should return 401 for unauthorized upload', async () => {
@@ -304,7 +314,8 @@ tap.test('Maven: should return 401 for unauthorized upload', async () => {
}); });
expect(response.status).toEqual(401); expect(response.status).toEqual(401);
expect(response.body).toHaveProperty('error'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
}); });
tap.test('Maven: should reject POM upload with mismatched GAV', async () => { tap.test('Maven: should reject POM upload with mismatched GAV', async () => {
@@ -328,7 +339,8 @@ tap.test('Maven: should reject POM upload with mismatched GAV', async () => {
}); });
expect(response.status).toEqual(400); expect(response.status).toEqual(400);
expect(response.body).toHaveProperty('error'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
}); });
tap.test('Maven: should delete an artifact (DELETE)', async () => { tap.test('Maven: should delete an artifact (DELETE)', async () => {

View File

@@ -79,16 +79,10 @@ async function createHttpServer(
res.setHeader(key, value); res.setHeader(key, value);
} }
// Send body // Send body (response.body is always ReadableStream<Uint8Array> or undefined)
if (response.body) { if (response.body) {
if (Buffer.isBuffer(response.body)) { const { Readable } = await import('stream');
res.end(response.body); Readable.fromWeb(response.body).pipe(res);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
} else { } else {
res.end(); res.end();
} }
@@ -224,16 +218,16 @@ function cleanupTestDir(dir: string): void {
// ======================================================================== // ========================================================================
tap.test('NPM CLI: should setup registry and HTTP server', async () => { tap.test('NPM CLI: should setup registry and HTTP server', async () => {
// Create registry // Find available port
registry = await createTestRegistry(); registryPort = 35000;
// Create registry with correct registryUrl for CLI tests
registry = await createTestRegistry({ registryUrl: `http://localhost:${registryPort}` });
const tokens = await createTestTokens(registry); const tokens = await createTestTokens(registry);
npmToken = tokens.npmToken; npmToken = tokens.npmToken;
expect(registry).toBeInstanceOf(SmartRegistry); expect(registry).toBeInstanceOf(SmartRegistry);
expect(npmToken).toBeTypeOf('string'); expect(npmToken).toBeTypeOf('string');
// Find available port
registryPort = 35000;
const serverSetup = await createHttpServer(registry, registryPort); const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server; server = serverSetup.server;
registryUrl = serverSetup.url; registryUrl = serverSetup.url;

View File

@@ -1,5 +1,6 @@
import { expect, tap } from '@git.zone/tstest/tapbundle'; import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js'; import { SmartRegistry } from '../ts/index.js';
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
import { createTestRegistry, createTestTokens, createTestPackument } from './helpers/registry.js'; import { createTestRegistry, createTestTokens, createTestPackument } from './helpers/registry.js';
let registry: SmartRegistry; let registry: SmartRegistry;
@@ -34,8 +35,9 @@ tap.test('NPM: should handle user authentication (PUT /-/user/org.couchdb.user:{
}); });
expect(response.status).toEqual(201); expect(response.status).toEqual(201);
expect(response.body).toHaveProperty('token'); const body = await streamToJson(response.body);
expect((response.body as any).token).toBeTypeOf('string'); expect(body).toHaveProperty('token');
expect(body.token).toBeTypeOf('string');
}); });
tap.test('NPM: should publish a package (PUT /{package})', async () => { tap.test('NPM: should publish a package (PUT /{package})', async () => {
@@ -53,8 +55,9 @@ tap.test('NPM: should publish a package (PUT /{package})', async () => {
}); });
expect(response.status).toEqual(201); expect(response.status).toEqual(201);
expect(response.body).toHaveProperty('ok'); const body = await streamToJson(response.body);
expect((response.body as any).ok).toEqual(true); expect(body).toHaveProperty('ok');
expect(body.ok).toEqual(true);
}); });
tap.test('NPM: should retrieve package metadata (GET /{package})', async () => { tap.test('NPM: should retrieve package metadata (GET /{package})', async () => {
@@ -66,10 +69,11 @@ tap.test('NPM: should retrieve package metadata (GET /{package})', async () => {
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('name'); const body = await streamToJson(response.body);
expect((response.body as any).name).toEqual(testPackageName); expect(body).toHaveProperty('name');
expect((response.body as any).versions).toHaveProperty(testVersion); expect(body.name).toEqual(testPackageName);
expect((response.body as any)['dist-tags'].latest).toEqual(testVersion); expect(body.versions).toHaveProperty(testVersion);
expect(body['dist-tags'].latest).toEqual(testVersion);
}); });
tap.test('NPM: should retrieve specific version metadata (GET /{package}/{version})', async () => { tap.test('NPM: should retrieve specific version metadata (GET /{package}/{version})', async () => {
@@ -81,9 +85,10 @@ tap.test('NPM: should retrieve specific version metadata (GET /{package}/{versio
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('version'); const body = await streamToJson(response.body);
expect((response.body as any).version).toEqual(testVersion); expect(body).toHaveProperty('version');
expect((response.body as any).name).toEqual(testPackageName); expect(body.version).toEqual(testVersion);
expect(body.name).toEqual(testPackageName);
}); });
tap.test('NPM: should download tarball (GET /{package}/-/{tarball})', async () => { tap.test('NPM: should download tarball (GET /{package}/-/{tarball})', async () => {
@@ -95,8 +100,9 @@ tap.test('NPM: should download tarball (GET /{package}/-/{tarball})', async () =
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect((response.body as Buffer).toString('utf-8')).toEqual('fake tarball content'); expect(body).toBeInstanceOf(Buffer);
expect(body.toString('utf-8')).toEqual('fake tarball content');
expect(response.headers['Content-Type']).toEqual('application/octet-stream'); expect(response.headers['Content-Type']).toEqual('application/octet-stream');
}); });
@@ -127,7 +133,8 @@ tap.test('NPM: should publish a new version of the package', async () => {
}); });
expect(getResponse.status).toEqual(200); expect(getResponse.status).toEqual(200);
expect((getResponse.body as any).versions).toHaveProperty(newVersion); const getBody = await streamToJson(getResponse.body);
expect(getBody.versions).toHaveProperty(newVersion);
}); });
tap.test('NPM: should get dist-tags (GET /-/package/{pkg}/dist-tags)', async () => { tap.test('NPM: should get dist-tags (GET /-/package/{pkg}/dist-tags)', async () => {
@@ -139,8 +146,9 @@ tap.test('NPM: should get dist-tags (GET /-/package/{pkg}/dist-tags)', async ()
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('latest'); const body = await streamToJson(response.body);
expect((response.body as any).latest).toBeTypeOf('string'); expect(body).toHaveProperty('latest');
expect(body.latest).toBeTypeOf('string');
}); });
tap.test('NPM: should update dist-tag (PUT /-/package/{pkg}/dist-tags/{tag})', async () => { tap.test('NPM: should update dist-tag (PUT /-/package/{pkg}/dist-tags/{tag})', async () => {
@@ -165,7 +173,8 @@ tap.test('NPM: should update dist-tag (PUT /-/package/{pkg}/dist-tags/{tag})', a
query: {}, query: {},
}); });
expect((getResponse.body as any)['dist-tags'].beta).toEqual('1.1.0'); const getBody2 = await streamToJson(getResponse.body);
expect(getBody2['dist-tags'].beta).toEqual('1.1.0');
}); });
tap.test('NPM: should delete dist-tag (DELETE /-/package/{pkg}/dist-tags/{tag})', async () => { tap.test('NPM: should delete dist-tag (DELETE /-/package/{pkg}/dist-tags/{tag})', async () => {
@@ -188,7 +197,8 @@ tap.test('NPM: should delete dist-tag (DELETE /-/package/{pkg}/dist-tags/{tag})'
query: {}, query: {},
}); });
expect((getResponse.body as any)['dist-tags']).not.toHaveProperty('beta'); const getBody3 = await streamToJson(getResponse.body);
expect(getBody3['dist-tags']).not.toHaveProperty('beta');
}); });
tap.test('NPM: should create a new token (POST /-/npm/v1/tokens)', async () => { tap.test('NPM: should create a new token (POST /-/npm/v1/tokens)', async () => {
@@ -208,8 +218,9 @@ tap.test('NPM: should create a new token (POST /-/npm/v1/tokens)', async () => {
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('token'); const body = await streamToJson(response.body);
expect((response.body as any).readonly).toEqual(true); expect(body).toHaveProperty('token');
expect(body.readonly).toEqual(true);
}); });
tap.test('NPM: should list tokens (GET /-/npm/v1/tokens)', async () => { tap.test('NPM: should list tokens (GET /-/npm/v1/tokens)', async () => {
@@ -223,9 +234,10 @@ tap.test('NPM: should list tokens (GET /-/npm/v1/tokens)', async () => {
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('objects'); const body = await streamToJson(response.body);
expect((response.body as any).objects).toBeInstanceOf(Array); expect(body).toHaveProperty('objects');
expect((response.body as any).objects.length).toBeGreaterThan(0); expect(body.objects).toBeInstanceOf(Array);
expect(body.objects.length).toBeGreaterThan(0);
}); });
tap.test('NPM: should search packages (GET /-/v1/search)', async () => { tap.test('NPM: should search packages (GET /-/v1/search)', async () => {
@@ -240,9 +252,10 @@ tap.test('NPM: should search packages (GET /-/v1/search)', async () => {
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('objects'); const body = await streamToJson(response.body);
expect((response.body as any).objects).toBeInstanceOf(Array); expect(body).toHaveProperty('objects');
expect((response.body as any).total).toBeGreaterThan(0); expect(body.objects).toBeInstanceOf(Array);
expect(body.total).toBeGreaterThan(0);
}); });
tap.test('NPM: should search packages with specific query', async () => { tap.test('NPM: should search packages with specific query', async () => {
@@ -256,7 +269,8 @@ tap.test('NPM: should search packages with specific query', async () => {
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
const results = (response.body as any).objects; const body = await streamToJson(response.body);
const results = body.objects;
expect(results.length).toBeGreaterThan(0); expect(results.length).toBeGreaterThan(0);
expect(results[0].package.name).toEqual(testPackageName); expect(results[0].package.name).toEqual(testPackageName);
}); });
@@ -281,7 +295,8 @@ tap.test('NPM: should unpublish a specific version (DELETE /{package}/-/{version
query: {}, query: {},
}); });
expect((getResponse.body as any).versions).not.toHaveProperty(testVersion); const getBody4 = await streamToJson(getResponse.body);
expect(getBody4.versions).not.toHaveProperty(testVersion);
}); });
tap.test('NPM: should unpublish entire package (DELETE /{package}/-rev/{rev})', async () => { tap.test('NPM: should unpublish entire package (DELETE /{package}/-rev/{rev})', async () => {
@@ -316,7 +331,8 @@ tap.test('NPM: should return 404 for non-existent package', async () => {
}); });
expect(response.status).toEqual(404); expect(response.status).toEqual(404);
expect(response.body).toHaveProperty('error'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
}); });
tap.test('NPM: should return 401 for unauthorized publish', async () => { tap.test('NPM: should return 401 for unauthorized publish', async () => {
@@ -334,7 +350,8 @@ tap.test('NPM: should return 401 for unauthorized publish', async () => {
}); });
expect(response.status).toEqual(401); expect(response.status).toEqual(401);
expect(response.body).toHaveProperty('error'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
}); });
tap.test('NPM: should reject readonly token for write operations', async () => { tap.test('NPM: should reject readonly token for write operations', async () => {

View File

@@ -175,16 +175,10 @@ async function createHttpServer(
res.setHeader(key, value); res.setHeader(key, value);
} }
// Send body // Send body (response.body is always ReadableStream<Uint8Array> or undefined)
if (response.body) { if (response.body) {
if (Buffer.isBuffer(response.body)) { const { Readable } = await import('stream');
res.end(response.body); Readable.fromWeb(response.body).pipe(res);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
} else { } else {
res.end(); res.end();
} }

View File

@@ -1,5 +1,6 @@
import { expect, tap } from '@git.zone/tstest/tapbundle'; import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js'; import { SmartRegistry } from '../ts/index.js';
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
import { createTestRegistry, createTestTokens, calculateDigest, createTestManifest } from './helpers/registry.js'; import { createTestRegistry, createTestTokens, calculateDigest, createTestManifest } from './helpers/registry.js';
let registry: SmartRegistry; let registry: SmartRegistry;
@@ -113,8 +114,9 @@ tap.test('OCI: should retrieve blob (GET /v2/{name}/blobs/{digest})', async () =
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect((response.body as Buffer).toString('utf-8')).toEqual('Hello from OCI test blob!'); expect(body).toBeInstanceOf(Buffer);
expect(body.toString('utf-8')).toEqual('Hello from OCI test blob!');
expect(response.headers['Docker-Content-Digest']).toEqual(testBlobDigest); expect(response.headers['Docker-Content-Digest']).toEqual(testBlobDigest);
}); });
@@ -152,9 +154,10 @@ tap.test('OCI: should retrieve manifest by tag (GET /v2/{name}/manifests/{refere
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
const manifest = JSON.parse((response.body as Buffer).toString('utf-8')); const manifest = JSON.parse(body.toString('utf-8'));
expect(manifest.schemaVersion).toEqual(2); expect(manifest.schemaVersion).toEqual(2);
expect(manifest.config.digest).toEqual(testConfigDigest); expect(manifest.config.digest).toEqual(testConfigDigest);
expect(manifest.layers[0].digest).toEqual(testBlobDigest); expect(manifest.layers[0].digest).toEqual(testBlobDigest);
@@ -201,9 +204,9 @@ tap.test('OCI: should list tags (GET /v2/{name}/tags/list)', async () => {
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('tags'); const tagList = await streamToJson(response.body);
expect(tagList).toHaveProperty('tags');
const tagList = response.body as any;
expect(tagList.name).toEqual('test-repo'); expect(tagList.name).toEqual('test-repo');
expect(tagList.tags).toBeInstanceOf(Array); expect(tagList.tags).toBeInstanceOf(Array);
expect(tagList.tags).toContain('v1.0.0'); expect(tagList.tags).toContain('v1.0.0');
@@ -222,7 +225,8 @@ tap.test('OCI: should handle pagination for tag list', async () => {
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('tags'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('tags');
}); });
tap.test('OCI: should return 404 for non-existent blob', async () => { tap.test('OCI: should return 404 for non-existent blob', async () => {
@@ -236,7 +240,8 @@ tap.test('OCI: should return 404 for non-existent blob', async () => {
}); });
expect(response.status).toEqual(404); expect(response.status).toEqual(404);
expect(response.body).toHaveProperty('errors'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('errors');
}); });
tap.test('OCI: should return 404 for non-existent manifest', async () => { tap.test('OCI: should return 404 for non-existent manifest', async () => {
@@ -251,7 +256,8 @@ tap.test('OCI: should return 404 for non-existent manifest', async () => {
}); });
expect(response.status).toEqual(404); expect(response.status).toEqual(404);
expect(response.body).toHaveProperty('errors'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('errors');
}); });
tap.test('OCI: should delete manifest (DELETE /v2/{name}/manifests/{digest})', async () => { tap.test('OCI: should delete manifest (DELETE /v2/{name}/manifests/{digest})', async () => {

View File

@@ -89,16 +89,10 @@ async function createHttpServer(
res.setHeader(key, value); res.setHeader(key, value);
} }
// Send body // Send body (response.body is always ReadableStream<Uint8Array> or undefined)
if (response.body) { if (response.body) {
if (Buffer.isBuffer(response.body)) { const { Readable } = await import('stream');
res.end(response.body); Readable.fromWeb(response.body).pipe(res);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
} else { } else {
res.end(); res.end();
} }
@@ -303,16 +297,16 @@ tap.test('PyPI CLI: should verify twine is installed', async () => {
}); });
tap.test('PyPI CLI: should setup registry and HTTP server', async () => { tap.test('PyPI CLI: should setup registry and HTTP server', async () => {
// Create registry // Use port 39000 (avoids conflicts with other tests)
registry = await createTestRegistry(); registryPort = 39000;
// Create registry with correct registryUrl for CLI tests
registry = await createTestRegistry({ registryUrl: `http://localhost:${registryPort}` });
const tokens = await createTestTokens(registry); const tokens = await createTestTokens(registry);
pypiToken = tokens.pypiToken; pypiToken = tokens.pypiToken;
expect(registry).toBeInstanceOf(SmartRegistry); expect(registry).toBeInstanceOf(SmartRegistry);
expect(pypiToken).toBeTypeOf('string'); expect(pypiToken).toBeTypeOf('string');
// Use port 39000 (avoids conflicts with other tests)
registryPort = 39000;
const serverSetup = await createHttpServer(registry, registryPort); const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server; server = serverSetup.server;
registryUrl = serverSetup.url; registryUrl = serverSetup.url;

View File

@@ -1,5 +1,6 @@
import { expect, tap } from '@git.zone/tstest/tapbundle'; import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js'; import { SmartRegistry } from '../ts/index.js';
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
import { import {
createTestRegistry, createTestRegistry,
createTestTokens, createTestTokens,
@@ -101,9 +102,10 @@ tap.test('PyPI: should retrieve Simple API root index HTML (GET /simple/)', asyn
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toStartWith('text/html'); expect(response.headers['Content-Type']).toStartWith('text/html');
expect(response.body).toBeTypeOf('string'); const body = await streamToBuffer(response.body);
const html = body.toString('utf-8');
expect(html).toBeTypeOf('string');
const html = response.body as string;
expect(html).toContain('<!DOCTYPE html>'); expect(html).toContain('<!DOCTYPE html>');
expect(html).toContain('<title>Simple Index</title>'); expect(html).toContain('<title>Simple Index</title>');
expect(html).toContain(normalizedPackageName); expect(html).toContain(normalizedPackageName);
@@ -121,9 +123,9 @@ tap.test('PyPI: should retrieve Simple API root index JSON (GET /simple/ with Ac
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/vnd.pypi.simple.v1+json'); expect(response.headers['Content-Type']).toEqual('application/vnd.pypi.simple.v1+json');
expect(response.body).toBeTypeOf('object'); const json = await streamToJson(response.body);
expect(json).toBeTypeOf('object');
const json = response.body as any;
expect(json).toHaveProperty('meta'); expect(json).toHaveProperty('meta');
expect(json).toHaveProperty('projects'); expect(json).toHaveProperty('projects');
expect(json.projects).toBeInstanceOf(Array); expect(json.projects).toBeInstanceOf(Array);
@@ -144,9 +146,10 @@ tap.test('PyPI: should retrieve Simple API package HTML (GET /simple/{package}/)
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toStartWith('text/html'); expect(response.headers['Content-Type']).toStartWith('text/html');
expect(response.body).toBeTypeOf('string'); const body = await streamToBuffer(response.body);
const html = body.toString('utf-8');
expect(html).toBeTypeOf('string');
const html = response.body as string;
expect(html).toContain('<!DOCTYPE html>'); expect(html).toContain('<!DOCTYPE html>');
expect(html).toContain(`<title>Links for ${normalizedPackageName}</title>`); expect(html).toContain(`<title>Links for ${normalizedPackageName}</title>`);
expect(html).toContain('.whl'); expect(html).toContain('.whl');
@@ -165,9 +168,9 @@ tap.test('PyPI: should retrieve Simple API package JSON (GET /simple/{package}/
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/vnd.pypi.simple.v1+json'); expect(response.headers['Content-Type']).toEqual('application/vnd.pypi.simple.v1+json');
expect(response.body).toBeTypeOf('object'); const json = await streamToJson(response.body);
expect(json).toBeTypeOf('object');
const json = response.body as any;
expect(json).toHaveProperty('meta'); expect(json).toHaveProperty('meta');
expect(json).toHaveProperty('name'); expect(json).toHaveProperty('name');
expect(json.name).toEqual(normalizedPackageName); expect(json.name).toEqual(normalizedPackageName);
@@ -187,8 +190,9 @@ tap.test('PyPI: should download wheel file (GET /pypi/packages/{package}/{filena
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect((response.body as Buffer).length).toEqual(testWheelData.length); expect(body).toBeInstanceOf(Buffer);
expect(body.length).toEqual(testWheelData.length);
expect(response.headers['Content-Type']).toEqual('application/octet-stream'); expect(response.headers['Content-Type']).toEqual('application/octet-stream');
}); });
@@ -234,7 +238,7 @@ tap.test('PyPI: should list both wheel and sdist in Simple API', async () => {
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
const json = response.body as any; const json = await streamToJson(response.body);
// PEP 691: files is an array of file objects // PEP 691: files is an array of file objects
expect(json.files.length).toEqual(2); expect(json.files.length).toEqual(2);
@@ -289,7 +293,7 @@ tap.test('PyPI: should list multiple versions in Simple API', async () => {
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
const json = response.body as any; const json = await streamToJson(response.body);
// PEP 691: files is an array of file objects // PEP 691: files is an array of file objects
expect(json.files.length).toBeGreaterThan(2); expect(json.files.length).toBeGreaterThan(2);
@@ -323,7 +327,8 @@ tap.test('PyPI: should return 404 for non-existent package', async () => {
}); });
expect(response.status).toEqual(404); expect(response.status).toEqual(404);
expect(response.body).toHaveProperty('error'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
}); });
tap.test('PyPI: should return 401 for unauthorized upload', async () => { tap.test('PyPI: should return 401 for unauthorized upload', async () => {
@@ -353,7 +358,8 @@ tap.test('PyPI: should return 401 for unauthorized upload', async () => {
}); });
expect(response.status).toEqual(401); expect(response.status).toEqual(401);
expect(response.body).toHaveProperty('error'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
}); });
tap.test('PyPI: should reject upload with mismatched hash', async () => { tap.test('PyPI: should reject upload with mismatched hash', async () => {
@@ -382,7 +388,8 @@ tap.test('PyPI: should reject upload with mismatched hash', async () => {
}); });
expect(response.status).toEqual(400); expect(response.status).toEqual(400);
expect(response.body).toHaveProperty('error'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
}); });
tap.test('PyPI: should handle package with requires-python metadata', async () => { tap.test('PyPI: should handle package with requires-python metadata', async () => {
@@ -425,7 +432,8 @@ tap.test('PyPI: should handle package with requires-python metadata', async () =
query: {}, query: {},
}); });
const html = getResponse.body as string; const getBody = await streamToBuffer(getResponse.body);
const html = getBody.toString('utf-8');
expect(html).toContain('data-requires-python'); expect(html).toContain('data-requires-python');
// Note: >= gets HTML-escaped to &gt;= in attribute values // Note: >= gets HTML-escaped to &gt;= in attribute values
expect(html).toContain('&gt;=3.8'); expect(html).toContain('&gt;=3.8');
@@ -441,9 +449,9 @@ tap.test('PyPI: should support JSON API for package metadata', async () => {
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/json'); expect(response.headers['Content-Type']).toEqual('application/json');
expect(response.body).toBeTypeOf('object'); const json = await streamToJson(response.body);
expect(json).toBeTypeOf('object');
const json = response.body as any;
expect(json).toHaveProperty('info'); expect(json).toHaveProperty('info');
expect(json.info).toHaveProperty('name'); expect(json.info).toHaveProperty('name');
expect(json.info.name).toEqual(normalizedPackageName); expect(json.info.name).toEqual(normalizedPackageName);
@@ -460,9 +468,9 @@ tap.test('PyPI: should support JSON API for specific version', async () => {
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/json'); expect(response.headers['Content-Type']).toEqual('application/json');
expect(response.body).toBeTypeOf('object'); const json = await streamToJson(response.body);
expect(json).toBeTypeOf('object');
const json = response.body as any;
expect(json).toHaveProperty('info'); expect(json).toHaveProperty('info');
expect(json.info.version).toEqual(testVersion); expect(json.info.version).toEqual(testVersion);
expect(json).toHaveProperty('urls'); expect(json).toHaveProperty('urls');

View File

@@ -79,16 +79,10 @@ async function createHttpServer(
res.setHeader(key, value); res.setHeader(key, value);
} }
// Send body // Send body (response.body is always ReadableStream<Uint8Array> or undefined)
if (response.body) { if (response.body) {
if (Buffer.isBuffer(response.body)) { const { Readable } = await import('stream');
res.end(response.body); Readable.fromWeb(response.body).pipe(res);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
} else { } else {
res.end(); res.end();
} }
@@ -194,16 +188,16 @@ function cleanupTestDir(dir: string): void {
// ======================================================================== // ========================================================================
tap.test('RubyGems CLI: should setup registry and HTTP server', async () => { tap.test('RubyGems CLI: should setup registry and HTTP server', async () => {
// Create registry // Use port 36000 (avoids npm:35000, cargo:5000 conflicts)
registry = await createTestRegistry(); registryPort = 36000;
// Create registry with correct registryUrl for CLI tests
registry = await createTestRegistry({ registryUrl: `http://localhost:${registryPort}` });
const tokens = await createTestTokens(registry); const tokens = await createTestTokens(registry);
rubygemsToken = tokens.rubygemsToken; rubygemsToken = tokens.rubygemsToken;
expect(registry).toBeInstanceOf(SmartRegistry); expect(registry).toBeInstanceOf(SmartRegistry);
expect(rubygemsToken).toBeTypeOf('string'); expect(rubygemsToken).toBeTypeOf('string');
// Use port 36000 (avoids npm:35000, cargo:5000 conflicts)
registryPort = 36000;
const serverSetup = await createHttpServer(registry, registryPort); const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server; server = serverSetup.server;
registryUrl = serverSetup.url; registryUrl = serverSetup.url;

View File

@@ -1,5 +1,6 @@
import { expect, tap } from '@git.zone/tstest/tapbundle'; import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js'; import { SmartRegistry } from '../ts/index.js';
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
import { import {
createTestRegistry, createTestRegistry,
createTestTokens, createTestTokens,
@@ -54,7 +55,8 @@ tap.test('RubyGems: should upload gem file (POST /rubygems/api/v1/gems)', async
}); });
expect(response.status).toEqual(201); expect(response.status).toEqual(201);
expect(response.body).toHaveProperty('message'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('message');
}); });
tap.test('RubyGems: should retrieve Compact Index versions file (GET /rubygems/versions)', async () => { tap.test('RubyGems: should retrieve Compact Index versions file (GET /rubygems/versions)', async () => {
@@ -67,9 +69,10 @@ tap.test('RubyGems: should retrieve Compact Index versions file (GET /rubygems/v
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('text/plain; charset=utf-8'); expect(response.headers['Content-Type']).toEqual('text/plain; charset=utf-8');
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
const content = (response.body as Buffer).toString('utf-8'); const content = body.toString('utf-8');
expect(content).toContain('created_at:'); expect(content).toContain('created_at:');
expect(content).toContain('---'); expect(content).toContain('---');
expect(content).toContain(testGemName); expect(content).toContain(testGemName);
@@ -86,9 +89,10 @@ tap.test('RubyGems: should retrieve Compact Index info file (GET /rubygems/info/
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('text/plain; charset=utf-8'); expect(response.headers['Content-Type']).toEqual('text/plain; charset=utf-8');
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
const content = (response.body as Buffer).toString('utf-8'); const content = body.toString('utf-8');
expect(content).toContain('---'); expect(content).toContain('---');
expect(content).toContain(testVersion); expect(content).toContain(testVersion);
expect(content).toContain('checksum:'); expect(content).toContain('checksum:');
@@ -104,9 +108,10 @@ tap.test('RubyGems: should retrieve Compact Index names file (GET /rubygems/name
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('text/plain; charset=utf-8'); expect(response.headers['Content-Type']).toEqual('text/plain; charset=utf-8');
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
const content = (response.body as Buffer).toString('utf-8'); const content = body.toString('utf-8');
expect(content).toContain('---'); expect(content).toContain('---');
expect(content).toContain(testGemName); expect(content).toContain(testGemName);
}); });
@@ -120,8 +125,9 @@ tap.test('RubyGems: should download gem file (GET /rubygems/gems/{gem}-{version}
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect((response.body as Buffer).length).toEqual(testGemData.length); expect(body).toBeInstanceOf(Buffer);
expect(body.length).toEqual(testGemData.length);
expect(response.headers['Content-Type']).toEqual('application/octet-stream'); expect(response.headers['Content-Type']).toEqual('application/octet-stream');
}); });
@@ -153,7 +159,8 @@ tap.test('RubyGems: should list multiple versions in Compact Index', async () =>
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
const content = (response.body as Buffer).toString('utf-8'); const body = await streamToBuffer(response.body);
const content = body.toString('utf-8');
const lines = content.split('\n'); const lines = content.split('\n');
const gemLine = lines.find(l => l.startsWith(`${testGemName} `)); const gemLine = lines.find(l => l.startsWith(`${testGemName} `));
@@ -172,7 +179,8 @@ tap.test('RubyGems: should list multiple versions in info file', async () => {
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
const content = (response.body as Buffer).toString('utf-8'); const body = await streamToBuffer(response.body);
const content = body.toString('utf-8');
expect(content).toContain('1.0.0'); expect(content).toContain('1.0.0');
expect(content).toContain('2.0.0'); expect(content).toContain('2.0.0');
}); });
@@ -203,7 +211,8 @@ tap.test('RubyGems: should support platform-specific gems', async () => {
query: {}, query: {},
}); });
const content = (versionsResponse.body as Buffer).toString('utf-8'); const versionsBody = await streamToBuffer(versionsResponse.body);
const content = versionsBody.toString('utf-8');
const lines = content.split('\n'); const lines = content.split('\n');
const gemLine = lines.find(l => l.startsWith(`${testGemName} `)); const gemLine = lines.find(l => l.startsWith(`${testGemName} `));
@@ -224,8 +233,9 @@ tap.test('RubyGems: should yank a gem version (DELETE /rubygems/api/v1/gems/yank
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('message'); const body = await streamToJson(response.body);
expect((response.body as any).message).toContain('yanked'); expect(body).toHaveProperty('message');
expect(body.message).toContain('yanked');
}); });
tap.test('RubyGems: should mark yanked version in Compact Index', async () => { tap.test('RubyGems: should mark yanked version in Compact Index', async () => {
@@ -238,7 +248,8 @@ tap.test('RubyGems: should mark yanked version in Compact Index', async () => {
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
const content = (response.body as Buffer).toString('utf-8'); const body = await streamToBuffer(response.body);
const content = body.toString('utf-8');
const lines = content.split('\n'); const lines = content.split('\n');
const gemLine = lines.find(l => l.startsWith(`${testGemName} `)); const gemLine = lines.find(l => l.startsWith(`${testGemName} `));
@@ -256,7 +267,8 @@ tap.test('RubyGems: should still allow downloading yanked gem', async () => {
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
}); });
tap.test('RubyGems: should unyank a gem version (PUT /rubygems/api/v1/gems/unyank)', async () => { tap.test('RubyGems: should unyank a gem version (PUT /rubygems/api/v1/gems/unyank)', async () => {
@@ -273,8 +285,9 @@ tap.test('RubyGems: should unyank a gem version (PUT /rubygems/api/v1/gems/unyan
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('message'); const body = await streamToJson(response.body);
expect((response.body as any).message).toContain('unyanked'); expect(body).toHaveProperty('message');
expect(body.message).toContain('unyanked');
}); });
tap.test('RubyGems: should remove yank marker after unyank', async () => { tap.test('RubyGems: should remove yank marker after unyank', async () => {
@@ -287,7 +300,8 @@ tap.test('RubyGems: should remove yank marker after unyank', async () => {
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
const content = (response.body as Buffer).toString('utf-8'); const body = await streamToBuffer(response.body);
const content = body.toString('utf-8');
const lines = content.split('\n'); const lines = content.split('\n');
const gemLine = lines.find(l => l.startsWith(`${testGemName} `)); const gemLine = lines.find(l => l.startsWith(`${testGemName} `));
@@ -309,9 +323,9 @@ tap.test('RubyGems: should retrieve versions JSON (GET /rubygems/api/v1/versions
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/json'); expect(response.headers['Content-Type']).toEqual('application/json');
expect(response.body).toBeTypeOf('object'); const json = await streamToJson(response.body);
expect(json).toBeTypeOf('object');
const json = response.body as any;
expect(json).toHaveProperty('name'); expect(json).toHaveProperty('name');
expect(json.name).toEqual(testGemName); expect(json.name).toEqual(testGemName);
expect(json).toHaveProperty('versions'); expect(json).toHaveProperty('versions');
@@ -331,9 +345,9 @@ tap.test('RubyGems: should retrieve dependencies JSON (GET /rubygems/api/v1/depe
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/json'); expect(response.headers['Content-Type']).toEqual('application/json');
expect(response.body).toBeTypeOf('object'); const json = await streamToJson(response.body);
expect(json).toBeTypeOf('object');
const json = response.body as any;
expect(Array.isArray(json)).toEqual(true); expect(Array.isArray(json)).toEqual(true);
}); });
@@ -346,7 +360,8 @@ tap.test('RubyGems: should retrieve gem spec (GET /rubygems/quick/Marshal.4.8/{g
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
}); });
tap.test('RubyGems: should support latest specs endpoint (GET /rubygems/latest_specs.4.8.gz)', async () => { tap.test('RubyGems: should support latest specs endpoint (GET /rubygems/latest_specs.4.8.gz)', async () => {
@@ -359,7 +374,8 @@ tap.test('RubyGems: should support latest specs endpoint (GET /rubygems/latest_s
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/octet-stream'); expect(response.headers['Content-Type']).toEqual('application/octet-stream');
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
}); });
tap.test('RubyGems: should support specs endpoint (GET /rubygems/specs.4.8.gz)', async () => { tap.test('RubyGems: should support specs endpoint (GET /rubygems/specs.4.8.gz)', async () => {
@@ -372,7 +388,8 @@ tap.test('RubyGems: should support specs endpoint (GET /rubygems/specs.4.8.gz)',
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/octet-stream'); expect(response.headers['Content-Type']).toEqual('application/octet-stream');
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
}); });
tap.test('RubyGems: should return 404 for non-existent gem', async () => { tap.test('RubyGems: should return 404 for non-existent gem', async () => {
@@ -384,7 +401,8 @@ tap.test('RubyGems: should return 404 for non-existent gem', async () => {
}); });
expect(response.status).toEqual(404); expect(response.status).toEqual(404);
expect(response.body).toHaveProperty('error'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
}); });
tap.test('RubyGems: should return 401 for unauthorized upload', async () => { tap.test('RubyGems: should return 401 for unauthorized upload', async () => {
@@ -402,7 +420,8 @@ tap.test('RubyGems: should return 401 for unauthorized upload', async () => {
}); });
expect(response.status).toEqual(401); expect(response.status).toEqual(401);
expect(response.body).toHaveProperty('error'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
}); });
tap.test('RubyGems: should return 401 for unauthorized yank', async () => { tap.test('RubyGems: should return 401 for unauthorized yank', async () => {
@@ -419,7 +438,8 @@ tap.test('RubyGems: should return 401 for unauthorized yank', async () => {
}); });
expect(response.status).toEqual(401); expect(response.status).toEqual(401);
expect(response.body).toHaveProperty('error'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
}); });
tap.test('RubyGems: should handle gem with dependencies', async () => { tap.test('RubyGems: should handle gem with dependencies', async () => {
@@ -450,7 +470,8 @@ tap.test('RubyGems: should handle gem with dependencies', async () => {
expect(infoResponse.status).toEqual(200); expect(infoResponse.status).toEqual(200);
const content = (infoResponse.body as Buffer).toString('utf-8'); const infoBody = await streamToBuffer(infoResponse.body);
const content = infoBody.toString('utf-8');
expect(content).toContain('checksum:'); expect(content).toContain('checksum:');
}); });

View File

@@ -1,5 +1,6 @@
import { expect, tap } from '@git.zone/tstest/tapbundle'; import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js'; import { SmartRegistry } from '../ts/index.js';
import { streamToJson } from '../ts/core/helpers.stream.js';
import { createTestRegistry, createTestTokens } from './helpers/registry.js'; import { createTestRegistry, createTestTokens } from './helpers/registry.js';
let registry: SmartRegistry; let registry: SmartRegistry;
@@ -54,8 +55,9 @@ tap.test('Integration: should return 404 for unknown paths', async () => {
}); });
expect(response.status).toEqual(404); expect(response.status).toEqual(404);
expect(response.body).toHaveProperty('error'); const body = await streamToJson(response.body);
expect((response.body as any).error).toEqual('NOT_FOUND'); expect(body).toHaveProperty('error');
expect(body.error).toEqual('NOT_FOUND');
}); });
tap.test('Integration: should create and validate tokens', async () => { tap.test('Integration: should create and validate tokens', async () => {

View File

@@ -3,6 +3,6 @@
*/ */
export const commitinfo = { export const commitinfo = {
name: '@push.rocks/smartregistry', name: '@push.rocks/smartregistry',
version: '2.7.0', version: '2.8.0',
description: 'A composable TypeScript library implementing OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems registries for building unified container and package registries' description: 'A composable TypeScript library implementing OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems registries for building unified container and package registries'
} }

View File

@@ -370,7 +370,7 @@ export class CargoRegistry extends BaseRegistry {
const parsed = this.parsePublishRequest(body); const parsed = this.parsePublishRequest(body);
metadata = parsed.metadata; metadata = parsed.metadata;
crateFile = parsed.crateFile; crateFile = parsed.crateFile;
} catch (error) { } catch (error: any) {
this.logger.log('error', 'handlePublish: parse error', { error: error.message }); this.logger.log('error', 'handlePublish: parse error', { error: error.message });
return { return {
status: 400, status: 400,
@@ -467,10 +467,23 @@ export class CargoRegistry extends BaseRegistry {
): Promise<IResponse> { ): Promise<IResponse> {
this.logger.log('debug', 'handleDownload', { crate: crateName, version }); this.logger.log('debug', 'handleDownload', { crate: crateName, version });
let crateFile = await this.storage.getCargoCrate(crateName, version); // Try streaming from local storage first
const streamResult = await this.storage.getCargoCrateStream(crateName, version);
if (streamResult) {
return {
status: 200,
headers: {
'Content-Type': 'application/gzip',
'Content-Length': streamResult.size.toString(),
'Content-Disposition': `attachment; filename="${crateName}-${version}.crate"`,
},
body: streamResult.stream,
};
}
// Try upstream if not found locally // Try upstream if not found locally
if (!crateFile) { let crateFile: Buffer | null = null;
const upstream = await this.getUpstreamForRequest(crateName, 'crate', 'GET', actor); const upstream = await this.getUpstreamForRequest(crateName, 'crate', 'GET', actor);
if (upstream) { if (upstream) {
crateFile = await upstream.fetchCrate(crateName, version); crateFile = await upstream.fetchCrate(crateName, version);
@@ -479,7 +492,6 @@ export class CargoRegistry extends BaseRegistry {
await this.storage.putCargoCrate(crateName, version, crateFile); await this.storage.putCargoCrate(crateName, version, crateFile);
} }
} }
}
if (!crateFile) { if (!crateFile) {
return { return {
@@ -647,7 +659,7 @@ export class CargoRegistry extends BaseRegistry {
} }
} }
} }
} catch (error) { } catch (error: any) {
this.logger.log('error', 'handleSearch: error', { error: error.message }); this.logger.log('error', 'handleSearch: error', { error: error.message });
} }

View File

@@ -2,6 +2,7 @@ import { RegistryStorage } from './core/classes.registrystorage.js';
import { AuthManager } from './core/classes.authmanager.js'; import { AuthManager } from './core/classes.authmanager.js';
import { BaseRegistry } from './core/classes.baseregistry.js'; import { BaseRegistry } from './core/classes.baseregistry.js';
import type { IRegistryConfig, IRequestContext, IResponse } from './core/interfaces.core.js'; import type { IRegistryConfig, IRequestContext, IResponse } from './core/interfaces.core.js';
import { toReadableStream } from './core/helpers.stream.js';
import { OciRegistry } from './oci/classes.ociregistry.js'; import { OciRegistry } from './oci/classes.ociregistry.js';
import { NpmRegistry } from './npm/classes.npmregistry.js'; import { NpmRegistry } from './npm/classes.npmregistry.js';
import { MavenRegistry } from './maven/classes.mavenregistry.js'; import { MavenRegistry } from './maven/classes.mavenregistry.js';
@@ -95,7 +96,7 @@ export class SmartRegistry {
// Initialize NPM registry if enabled // Initialize NPM registry if enabled
if (this.config.npm?.enabled) { if (this.config.npm?.enabled) {
const npmBasePath = this.config.npm.basePath ?? '/npm'; const npmBasePath = this.config.npm.basePath ?? '/npm';
const registryUrl = `http://localhost:5000${npmBasePath}`; // TODO: Make configurable const registryUrl = this.config.npm.registryUrl ?? `http://localhost:5000${npmBasePath}`;
const npmRegistry = new NpmRegistry( const npmRegistry = new NpmRegistry(
this.storage, this.storage,
this.authManager, this.authManager,
@@ -110,7 +111,7 @@ export class SmartRegistry {
// Initialize Maven registry if enabled // Initialize Maven registry if enabled
if (this.config.maven?.enabled) { if (this.config.maven?.enabled) {
const mavenBasePath = this.config.maven.basePath ?? '/maven'; const mavenBasePath = this.config.maven.basePath ?? '/maven';
const registryUrl = `http://localhost:5000${mavenBasePath}`; // TODO: Make configurable const registryUrl = this.config.maven.registryUrl ?? `http://localhost:5000${mavenBasePath}`;
const mavenRegistry = new MavenRegistry( const mavenRegistry = new MavenRegistry(
this.storage, this.storage,
this.authManager, this.authManager,
@@ -125,7 +126,7 @@ export class SmartRegistry {
// Initialize Cargo registry if enabled // Initialize Cargo registry if enabled
if (this.config.cargo?.enabled) { if (this.config.cargo?.enabled) {
const cargoBasePath = this.config.cargo.basePath ?? '/cargo'; const cargoBasePath = this.config.cargo.basePath ?? '/cargo';
const registryUrl = `http://localhost:5000${cargoBasePath}`; // TODO: Make configurable const registryUrl = this.config.cargo.registryUrl ?? `http://localhost:5000${cargoBasePath}`;
const cargoRegistry = new CargoRegistry( const cargoRegistry = new CargoRegistry(
this.storage, this.storage,
this.authManager, this.authManager,
@@ -140,7 +141,7 @@ export class SmartRegistry {
// Initialize Composer registry if enabled // Initialize Composer registry if enabled
if (this.config.composer?.enabled) { if (this.config.composer?.enabled) {
const composerBasePath = this.config.composer.basePath ?? '/composer'; const composerBasePath = this.config.composer.basePath ?? '/composer';
const registryUrl = `http://localhost:5000${composerBasePath}`; // TODO: Make configurable const registryUrl = this.config.composer.registryUrl ?? `http://localhost:5000${composerBasePath}`;
const composerRegistry = new ComposerRegistry( const composerRegistry = new ComposerRegistry(
this.storage, this.storage,
this.authManager, this.authManager,
@@ -155,7 +156,7 @@ export class SmartRegistry {
// Initialize PyPI registry if enabled // Initialize PyPI registry if enabled
if (this.config.pypi?.enabled) { if (this.config.pypi?.enabled) {
const pypiBasePath = this.config.pypi.basePath ?? '/pypi'; const pypiBasePath = this.config.pypi.basePath ?? '/pypi';
const registryUrl = `http://localhost:5000`; // TODO: Make configurable const registryUrl = this.config.pypi.registryUrl ?? `http://localhost:5000`;
const pypiRegistry = new PypiRegistry( const pypiRegistry = new PypiRegistry(
this.storage, this.storage,
this.authManager, this.authManager,
@@ -170,7 +171,7 @@ export class SmartRegistry {
// Initialize RubyGems registry if enabled // Initialize RubyGems registry if enabled
if (this.config.rubygems?.enabled) { if (this.config.rubygems?.enabled) {
const rubygemsBasePath = this.config.rubygems.basePath ?? '/rubygems'; const rubygemsBasePath = this.config.rubygems.basePath ?? '/rubygems';
const registryUrl = `http://localhost:5000${rubygemsBasePath}`; // TODO: Make configurable const registryUrl = this.config.rubygems.registryUrl ?? `http://localhost:5000${rubygemsBasePath}`;
const rubygemsRegistry = new RubyGemsRegistry( const rubygemsRegistry = new RubyGemsRegistry(
this.storage, this.storage,
this.authManager, this.authManager,
@@ -191,68 +192,70 @@ export class SmartRegistry {
*/ */
public async handleRequest(context: IRequestContext): Promise<IResponse> { public async handleRequest(context: IRequestContext): Promise<IResponse> {
const path = context.path; const path = context.path;
let response: IResponse | undefined;
// Route to OCI registry // Route to OCI registry
if (this.config.oci?.enabled && path.startsWith(this.config.oci.basePath)) { if (!response && this.config.oci?.enabled && path.startsWith(this.config.oci.basePath)) {
const ociRegistry = this.registries.get('oci'); const ociRegistry = this.registries.get('oci');
if (ociRegistry) { if (ociRegistry) {
return ociRegistry.handleRequest(context); response = await ociRegistry.handleRequest(context);
} }
} }
// Route to NPM registry // Route to NPM registry
if (this.config.npm?.enabled && path.startsWith(this.config.npm.basePath)) { if (!response && this.config.npm?.enabled && path.startsWith(this.config.npm.basePath)) {
const npmRegistry = this.registries.get('npm'); const npmRegistry = this.registries.get('npm');
if (npmRegistry) { if (npmRegistry) {
return npmRegistry.handleRequest(context); response = await npmRegistry.handleRequest(context);
} }
} }
// Route to Maven registry // Route to Maven registry
if (this.config.maven?.enabled && path.startsWith(this.config.maven.basePath)) { if (!response && this.config.maven?.enabled && path.startsWith(this.config.maven.basePath)) {
const mavenRegistry = this.registries.get('maven'); const mavenRegistry = this.registries.get('maven');
if (mavenRegistry) { if (mavenRegistry) {
return mavenRegistry.handleRequest(context); response = await mavenRegistry.handleRequest(context);
} }
} }
// Route to Cargo registry // Route to Cargo registry
if (this.config.cargo?.enabled && path.startsWith(this.config.cargo.basePath)) { if (!response && this.config.cargo?.enabled && path.startsWith(this.config.cargo.basePath)) {
const cargoRegistry = this.registries.get('cargo'); const cargoRegistry = this.registries.get('cargo');
if (cargoRegistry) { if (cargoRegistry) {
return cargoRegistry.handleRequest(context); response = await cargoRegistry.handleRequest(context);
} }
} }
// Route to Composer registry // Route to Composer registry
if (this.config.composer?.enabled && path.startsWith(this.config.composer.basePath)) { if (!response && this.config.composer?.enabled && path.startsWith(this.config.composer.basePath)) {
const composerRegistry = this.registries.get('composer'); const composerRegistry = this.registries.get('composer');
if (composerRegistry) { if (composerRegistry) {
return composerRegistry.handleRequest(context); response = await composerRegistry.handleRequest(context);
} }
} }
// Route to PyPI registry (also handles /simple prefix) // Route to PyPI registry (also handles /simple prefix)
if (this.config.pypi?.enabled) { if (!response && this.config.pypi?.enabled) {
const pypiBasePath = this.config.pypi.basePath ?? '/pypi'; const pypiBasePath = this.config.pypi.basePath ?? '/pypi';
if (path.startsWith(pypiBasePath) || path.startsWith('/simple')) { if (path.startsWith(pypiBasePath) || path.startsWith('/simple')) {
const pypiRegistry = this.registries.get('pypi'); const pypiRegistry = this.registries.get('pypi');
if (pypiRegistry) { if (pypiRegistry) {
return pypiRegistry.handleRequest(context); response = await pypiRegistry.handleRequest(context);
} }
} }
} }
// Route to RubyGems registry // Route to RubyGems registry
if (this.config.rubygems?.enabled && path.startsWith(this.config.rubygems.basePath)) { if (!response && this.config.rubygems?.enabled && path.startsWith(this.config.rubygems.basePath)) {
const rubygemsRegistry = this.registries.get('rubygems'); const rubygemsRegistry = this.registries.get('rubygems');
if (rubygemsRegistry) { if (rubygemsRegistry) {
return rubygemsRegistry.handleRequest(context); response = await rubygemsRegistry.handleRequest(context);
} }
} }
// No matching registry // No matching registry
return { if (!response) {
response = {
status: 404, status: 404,
headers: { 'Content-Type': 'application/json' }, headers: { 'Content-Type': 'application/json' },
body: { body: {
@@ -262,6 +265,17 @@ export class SmartRegistry {
}; };
} }
// Normalize body to ReadableStream<Uint8Array> at the API boundary
if (response.body != null && !(response.body instanceof ReadableStream)) {
if (!Buffer.isBuffer(response.body) && typeof response.body === 'object' && !(response.body instanceof Uint8Array)) {
response.headers['Content-Type'] ??= 'application/json';
}
response.body = toReadableStream(response.body);
}
return response;
}
/** /**
* Get the storage instance (for testing/advanced use) * Get the storage instance (for testing/advanced use)
*/ */

View File

@@ -302,9 +302,9 @@ export class ComposerRegistry extends BaseRegistry {
token: IAuthToken | null token: IAuthToken | null
): Promise<IResponse> { ): Promise<IResponse> {
// Read operations are public, no authentication required // Read operations are public, no authentication required
const zipData = await this.storage.getComposerPackageZip(vendorPackage, reference); const streamResult = await this.storage.getComposerPackageZipStream(vendorPackage, reference);
if (!zipData) { if (!streamResult) {
return { return {
status: 404, status: 404,
headers: {}, headers: {},
@@ -316,10 +316,10 @@ export class ComposerRegistry extends BaseRegistry {
status: 200, status: 200,
headers: { headers: {
'Content-Type': 'application/zip', 'Content-Type': 'application/zip',
'Content-Length': zipData.length.toString(), 'Content-Length': streamResult.size.toString(),
'Content-Disposition': `attachment; filename="${reference}.zip"`, 'Content-Disposition': `attachment; filename="${reference}.zip"`,
}, },
body: zipData, body: streamResult.stream,
}; };
} }

View File

@@ -34,8 +34,8 @@ import type {
* ``` * ```
*/ */
export class RegistryStorage implements IStorageBackend { export class RegistryStorage implements IStorageBackend {
private smartBucket: plugins.smartbucket.SmartBucket; private smartBucket!: plugins.smartbucket.SmartBucket;
private bucket: plugins.smartbucket.Bucket; private bucket!: plugins.smartbucket.Bucket;
private bucketName: string; private bucketName: string;
private hooks?: IStorageHooks; private hooks?: IStorageHooks;
@@ -1266,4 +1266,135 @@ export class RegistryStorage implements IStorageBackend {
private getRubyGemsMetadataPath(gemName: string): string { private getRubyGemsMetadataPath(gemName: string): string {
return `rubygems/metadata/${gemName}/metadata.json`; return `rubygems/metadata/${gemName}/metadata.json`;
} }
// ========================================================================
// STREAMING METHODS (Web Streams API)
// ========================================================================
/**
* Get an object as a ReadableStream. Returns null if not found.
*/
public async getObjectStream(key: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null> {
try {
const stat = await this.bucket.fastStat({ path: key });
const size = stat.ContentLength ?? 0;
const stream = await this.bucket.fastGetStream({ path: key }, 'webstream');
// Call afterGet hook (non-blocking)
if (this.hooks?.afterGet) {
const context = this.currentContext;
if (context) {
this.hooks.afterGet({
operation: 'get',
key,
protocol: context.protocol,
actor: context.actor,
metadata: context.metadata,
timestamp: new Date(),
}).catch(() => {});
}
}
return { stream: stream as ReadableStream<Uint8Array>, size };
} catch {
return null;
}
}
/**
* Store an object from a ReadableStream.
*/
public async putObjectStream(key: string, stream: ReadableStream<Uint8Array>): Promise<void> {
if (this.hooks?.beforePut) {
const context = this.currentContext;
if (context) {
const hookContext: IStorageHookContext = {
operation: 'put',
key,
protocol: context.protocol,
actor: context.actor,
metadata: context.metadata,
timestamp: new Date(),
};
const result = await this.hooks.beforePut(hookContext);
if (!result.allowed) {
throw new Error(result.reason || 'Storage operation denied by hook');
}
}
}
// Convert WebStream to Node Readable at the S3 SDK boundary
// AWS SDK v3 PutObjectCommand requires a Node.js Readable (not WebStream)
const { Readable } = await import('stream');
const nodeStream = Readable.fromWeb(stream as any);
await this.bucket.fastPutStream({
path: key,
readableStream: nodeStream,
overwrite: true,
});
if (this.hooks?.afterPut) {
const context = this.currentContext;
if (context) {
this.hooks.afterPut({
operation: 'put',
key,
protocol: context.protocol,
actor: context.actor,
metadata: context.metadata,
timestamp: new Date(),
}).catch(() => {});
}
}
}
/**
* Get object size without reading data (S3 HEAD request).
*/
public async getObjectSize(key: string): Promise<number | null> {
try {
const stat = await this.bucket.fastStat({ path: key });
return stat.ContentLength ?? null;
} catch {
return null;
}
}
// ---- Protocol-specific streaming wrappers ----
public async getOciBlobStream(digest: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null> {
return this.getObjectStream(this.getOciBlobPath(digest));
}
public async putOciBlobStream(digest: string, stream: ReadableStream<Uint8Array>): Promise<void> {
return this.putObjectStream(this.getOciBlobPath(digest), stream);
}
public async getOciBlobSize(digest: string): Promise<number | null> {
return this.getObjectSize(this.getOciBlobPath(digest));
}
public async getNpmTarballStream(packageName: string, version: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null> {
return this.getObjectStream(this.getNpmTarballPath(packageName, version));
}
public async getMavenArtifactStream(groupId: string, artifactId: string, version: string, filename: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null> {
return this.getObjectStream(this.getMavenArtifactPath(groupId, artifactId, version, filename));
}
public async getCargoCrateStream(crateName: string, version: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null> {
return this.getObjectStream(this.getCargoCratePath(crateName, version));
}
public async getComposerPackageZipStream(vendorPackage: string, reference: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null> {
return this.getObjectStream(this.getComposerZipPath(vendorPackage, reference));
}
public async getPypiPackageFileStream(packageName: string, filename: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null> {
return this.getObjectStream(this.getPypiPackageFilePath(packageName, filename));
}
public async getRubyGemsGemStream(gemName: string, version: string, platform?: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null> {
return this.getObjectStream(this.getRubyGemsGemPath(gemName, version, platform));
}
} }

63
ts/core/helpers.stream.ts Normal file
View File

@@ -0,0 +1,63 @@
import * as crypto from 'crypto';
/**
* Convert Buffer, Uint8Array, string, or JSON object to a ReadableStream<Uint8Array>.
*/
export function toReadableStream(data: Buffer | Uint8Array | string | object): ReadableStream<Uint8Array> {
const buf = Buffer.isBuffer(data)
? data
: data instanceof Uint8Array
? Buffer.from(data)
: typeof data === 'string'
? Buffer.from(data, 'utf-8')
: Buffer.from(JSON.stringify(data), 'utf-8');
return new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(new Uint8Array(buf));
controller.close();
},
});
}
/**
* Consume a ReadableStream into a Buffer.
*/
export async function streamToBuffer(stream: ReadableStream<Uint8Array>): Promise<Buffer> {
const reader = stream.getReader();
const chunks: Uint8Array[] = [];
while (true) {
const { done, value } = await reader.read();
if (done) break;
if (value) chunks.push(value);
}
return Buffer.concat(chunks);
}
/**
* Consume a ReadableStream into a parsed JSON object.
*/
export async function streamToJson<T = any>(stream: ReadableStream<Uint8Array>): Promise<T> {
const buf = await streamToBuffer(stream);
return JSON.parse(buf.toString('utf-8'));
}
/**
* Create a TransformStream that incrementally hashes data passing through.
* Data flows through unchanged; the digest is available after the stream completes.
*/
export function createHashTransform(algorithm: string = 'sha256'): {
transform: TransformStream<Uint8Array, Uint8Array>;
getDigest: () => string;
} {
const hash = crypto.createHash(algorithm);
const transform = new TransformStream<Uint8Array, Uint8Array>({
transform(chunk, controller) {
hash.update(chunk);
controller.enqueue(chunk);
},
});
return {
transform,
getDigest: () => hash.digest('hex'),
};
}

View File

@@ -12,6 +12,9 @@ export { DefaultAuthProvider } from './classes.defaultauthprovider.js';
// Storage interfaces and hooks // Storage interfaces and hooks
export * from './interfaces.storage.js'; export * from './interfaces.storage.js';
// Stream helpers
export { toReadableStream, streamToBuffer, streamToJson, createHashTransform } from './helpers.stream.js';
// Classes // Classes
export { BaseRegistry } from './classes.baseregistry.js'; export { BaseRegistry } from './classes.baseregistry.js';
export { RegistryStorage } from './classes.registrystorage.js'; export { RegistryStorage } from './classes.registrystorage.js';

View File

@@ -88,6 +88,7 @@ export interface IAuthConfig {
export interface IProtocolConfig { export interface IProtocolConfig {
enabled: boolean; enabled: boolean;
basePath: string; basePath: string;
registryUrl?: string;
features?: Record<string, boolean>; features?: Record<string, boolean>;
} }
@@ -160,6 +161,21 @@ export interface IStorageBackend {
* Get object metadata * Get object metadata
*/ */
getMetadata(key: string): Promise<Record<string, string> | null>; getMetadata(key: string): Promise<Record<string, string> | null>;
/**
* Get an object as a ReadableStream. Returns null if not found.
*/
getObjectStream?(key: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null>;
/**
* Store an object from a ReadableStream.
*/
putObjectStream?(key: string, stream: ReadableStream<Uint8Array>): Promise<void>;
/**
* Get object size without reading data (S3 HEAD request).
*/
getObjectSize?(key: string): Promise<number | null>;
} }
/** /**
@@ -215,10 +231,13 @@ export interface IRequestContext {
} }
/** /**
* Base response structure * Base response structure.
* `body` is always a `ReadableStream<Uint8Array>` at the public API boundary.
* Internal handlers may return Buffer/string/object — the SmartRegistry orchestrator
* auto-wraps them via `toReadableStream()` before returning to the caller.
*/ */
export interface IResponse { export interface IResponse {
status: number; status: number;
headers: Record<string, string>; headers: Record<string, string>;
body?: any; body?: ReadableStream<Uint8Array> | any;
} }

View File

@@ -293,10 +293,23 @@ export class MavenRegistry extends BaseRegistry {
filename: string, filename: string,
actor?: IRequestActor actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
let data = await this.storage.getMavenArtifact(groupId, artifactId, version, filename); // Try local storage first (streaming)
const streamResult = await this.storage.getMavenArtifactStream(groupId, artifactId, version, filename);
if (streamResult) {
const ext = filename.split('.').pop() || '';
const contentType = this.getContentType(ext);
return {
status: 200,
headers: {
'Content-Type': contentType,
'Content-Length': streamResult.size.toString(),
},
body: streamResult.stream,
};
}
// Try upstream if not found locally // Try upstream if not found locally
if (!data) { let data: Buffer | null = null;
const resource = `${groupId}:${artifactId}`; const resource = `${groupId}:${artifactId}`;
const upstream = await this.getUpstreamForRequest(resource, 'artifact', 'GET', actor); const upstream = await this.getUpstreamForRequest(resource, 'artifact', 'GET', actor);
if (upstream) { if (upstream) {
@@ -313,7 +326,6 @@ export class MavenRegistry extends BaseRegistry {
} }
} }
} }
}
if (!data) { if (!data) {
return { return {

View File

@@ -631,10 +631,22 @@ export class NpmRegistry extends BaseRegistry {
} }
const version = versionMatch[1]; const version = versionMatch[1];
let tarball = await this.storage.getNpmTarball(packageName, version);
// Try local storage first (streaming)
const streamResult = await this.storage.getNpmTarballStream(packageName, version);
if (streamResult) {
return {
status: 200,
headers: {
'Content-Type': 'application/octet-stream',
'Content-Length': streamResult.size.toString(),
},
body: streamResult.stream,
};
}
// If not found locally, try upstream // If not found locally, try upstream
if (!tarball) { let tarball: Buffer | null = null;
const upstream = await this.getUpstreamForRequest(packageName, 'tarball', 'GET', actor); const upstream = await this.getUpstreamForRequest(packageName, 'tarball', 'GET', actor);
if (upstream) { if (upstream) {
this.logger.log('debug', 'handleTarballDownload: fetching from upstream', { this.logger.log('debug', 'handleTarballDownload: fetching from upstream', {
@@ -653,7 +665,6 @@ export class NpmRegistry extends BaseRegistry {
}); });
} }
} }
}
if (!tarball) { if (!tarball) {
return { return {

View File

@@ -3,6 +3,7 @@ import { BaseRegistry } from '../core/classes.baseregistry.js';
import { RegistryStorage } from '../core/classes.registrystorage.js'; import { RegistryStorage } from '../core/classes.registrystorage.js';
import { AuthManager } from '../core/classes.authmanager.js'; import { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken, IRegistryError, IRequestActor } from '../core/interfaces.core.js'; import type { IRequestContext, IResponse, IAuthToken, IRegistryError, IRequestActor } from '../core/interfaces.core.js';
import { createHashTransform, streamToBuffer } from '../core/helpers.stream.js';
import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js'; import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
import { OciUpstream } from './classes.ociupstream.js'; import { OciUpstream } from './classes.ociupstream.js';
import type { import type {
@@ -302,6 +303,8 @@ export class OciRegistry extends BaseRegistry {
uploadId, uploadId,
repository, repository,
chunks: [], chunks: [],
chunkPaths: [],
chunkIndex: 0,
totalSize: 0, totalSize: 0,
createdAt: new Date(), createdAt: new Date(),
lastActivity: new Date(), lastActivity: new Date(),
@@ -571,11 +574,22 @@ export class OciRegistry extends BaseRegistry {
return this.createUnauthorizedResponse(repository, 'pull'); return this.createUnauthorizedResponse(repository, 'pull');
} }
// Try local storage first // Try local storage first (streaming)
let data = await this.storage.getOciBlob(digest); const streamResult = await this.storage.getOciBlobStream(digest);
if (streamResult) {
return {
status: 200,
headers: {
'Content-Type': 'application/octet-stream',
'Content-Length': streamResult.size.toString(),
'Docker-Content-Digest': digest,
},
body: streamResult.stream,
};
}
// If not found locally, try upstream // If not found locally, try upstream
if (!data) { let data: Buffer | null = null;
const upstream = await this.getUpstreamForRequest(repository, 'blob', 'GET', actor); const upstream = await this.getUpstreamForRequest(repository, 'blob', 'GET', actor);
if (upstream) { if (upstream) {
this.logger.log('debug', 'getBlob: fetching from upstream', { repository, digest }); this.logger.log('debug', 'getBlob: fetching from upstream', { repository, digest });
@@ -591,7 +605,6 @@ export class OciRegistry extends BaseRegistry {
}); });
} }
} }
}
if (!data) { if (!data) {
return { return {
@@ -620,17 +633,15 @@ export class OciRegistry extends BaseRegistry {
return this.createUnauthorizedHeadResponse(repository, 'pull'); return this.createUnauthorizedHeadResponse(repository, 'pull');
} }
const exists = await this.storage.ociBlobExists(digest); const blobSize = await this.storage.getOciBlobSize(digest);
if (!exists) { if (blobSize === null) {
return { status: 404, headers: {}, body: null }; return { status: 404, headers: {}, body: null };
} }
const blob = await this.storage.getOciBlob(digest);
return { return {
status: 200, status: 200,
headers: { headers: {
'Content-Length': blob ? blob.length.toString() : '0', 'Content-Length': blobSize.toString(),
'Docker-Content-Digest': digest, 'Docker-Content-Digest': digest,
}, },
body: null, body: null,
@@ -670,7 +681,12 @@ export class OciRegistry extends BaseRegistry {
} }
const chunkData = this.toBuffer(data); const chunkData = this.toBuffer(data);
session.chunks.push(chunkData);
// Write chunk to temp S3 object instead of accumulating in memory
const chunkPath = `oci/uploads/${uploadId}/chunk-${session.chunkIndex}`;
await this.storage.putObject(chunkPath, chunkData);
session.chunkPaths.push(chunkPath);
session.chunkIndex++;
session.totalSize += chunkData.length; session.totalSize += chunkData.length;
session.lastActivity = new Date(); session.lastActivity = new Date();
@@ -699,13 +715,52 @@ export class OciRegistry extends BaseRegistry {
}; };
} }
const chunks = [...session.chunks]; // If there's final data in the PUT body, write it as the last chunk
if (finalData) chunks.push(this.toBuffer(finalData)); if (finalData) {
const blobData = Buffer.concat(chunks); const buf = this.toBuffer(finalData);
const chunkPath = `oci/uploads/${uploadId}/chunk-${session.chunkIndex}`;
await this.storage.putObject(chunkPath, buf);
session.chunkPaths.push(chunkPath);
session.chunkIndex++;
session.totalSize += buf.length;
}
// Verify digest // Create a ReadableStream that assembles all chunks from S3 sequentially
const calculatedDigest = await this.calculateDigest(blobData); const chunkPaths = [...session.chunkPaths];
const storage = this.storage;
let chunkIdx = 0;
const assembledStream = new ReadableStream<Uint8Array>({
async pull(controller) {
if (chunkIdx >= chunkPaths.length) {
controller.close();
return;
}
const result = await storage.getObjectStream(chunkPaths[chunkIdx++]);
if (result) {
const reader = result.stream.getReader();
while (true) {
const { done, value } = await reader.read();
if (done) break;
if (value) controller.enqueue(value);
}
}
},
});
// Pipe through hash transform for incremental digest verification
const { transform: hashTransform, getDigest } = createHashTransform('sha256');
const hashedStream = assembledStream.pipeThrough(hashTransform);
// Consume stream to buffer for S3 upload
// (AWS SDK PutObjectCommand requires known content-length for streams;
// the key win is chunks are NOT accumulated in memory during PATCH — they live in S3)
const blobData = await streamToBuffer(hashedStream);
// Verify digest before storing
const calculatedDigest = `sha256:${getDigest()}`;
if (calculatedDigest !== digest) { if (calculatedDigest !== digest) {
await this.cleanupUploadChunks(session);
this.uploadSessions.delete(uploadId);
return { return {
status: 400, status: 400,
headers: {}, headers: {},
@@ -713,7 +768,11 @@ export class OciRegistry extends BaseRegistry {
}; };
} }
// Store verified blob
await this.storage.putOciBlob(digest, blobData); await this.storage.putOciBlob(digest, blobData);
// Cleanup temp chunks and session
await this.cleanupUploadChunks(session);
this.uploadSessions.delete(uploadId); this.uploadSessions.delete(uploadId);
return { return {
@@ -726,6 +785,19 @@ export class OciRegistry extends BaseRegistry {
}; };
} }
/**
* Delete all temp S3 chunk objects for an upload session.
*/
private async cleanupUploadChunks(session: IUploadSession): Promise<void> {
for (const chunkPath of session.chunkPaths) {
try {
await this.storage.deleteObject(chunkPath);
} catch {
// Best-effort cleanup
}
}
}
private async getUploadStatus(uploadId: string): Promise<IResponse> { private async getUploadStatus(uploadId: string): Promise<IResponse> {
const session = this.uploadSessions.get(uploadId); const session = this.uploadSessions.get(uploadId);
if (!session) { if (!session) {
@@ -917,6 +989,8 @@ export class OciRegistry extends BaseRegistry {
for (const [uploadId, session] of this.uploadSessions.entries()) { for (const [uploadId, session] of this.uploadSessions.entries()) {
if (now.getTime() - session.lastActivity.getTime() > maxAge) { if (now.getTime() - session.lastActivity.getTime() > maxAge) {
// Clean up temp S3 chunks for stale sessions
this.cleanupUploadChunks(session).catch(() => {});
this.uploadSessions.delete(uploadId); this.uploadSessions.delete(uploadId);
} }
} }

View File

@@ -62,6 +62,10 @@ export interface IUploadSession {
uploadId: string; uploadId: string;
repository: string; repository: string;
chunks: Buffer[]; chunks: Buffer[];
/** S3 paths to temp chunk objects (streaming mode) */
chunkPaths: string[];
/** Index counter for naming temp chunk objects */
chunkIndex: number;
totalSize: number; totalSize: number;
createdAt: Date; createdAt: Date;
lastActivity: Date; lastActivity: Date;

View File

@@ -535,10 +535,24 @@ export class PypiRegistry extends BaseRegistry {
*/ */
private async handleDownload(packageName: string, filename: string, actor?: IRequestActor): Promise<IResponse> { private async handleDownload(packageName: string, filename: string, actor?: IRequestActor): Promise<IResponse> {
const normalized = helpers.normalizePypiPackageName(packageName); const normalized = helpers.normalizePypiPackageName(packageName);
let fileData = await this.storage.getPypiPackageFile(normalized, filename);
// Try streaming from local storage first
const streamResult = await this.storage.getPypiPackageFileStream(normalized, filename);
if (streamResult) {
return {
status: 200,
headers: {
'Content-Type': 'application/octet-stream',
'Content-Disposition': `attachment; filename="${filename}"`,
'Content-Length': streamResult.size.toString()
},
body: streamResult.stream,
};
}
// Try upstream if not found locally // Try upstream if not found locally
if (!fileData) { let fileData: Buffer | null = null;
const upstream = await this.getUpstreamForRequest(normalized, 'file', 'GET', actor); const upstream = await this.getUpstreamForRequest(normalized, 'file', 'GET', actor);
if (upstream) { if (upstream) {
fileData = await upstream.fetchPackageFile(normalized, filename); fileData = await upstream.fetchPackageFile(normalized, filename);
@@ -547,7 +561,6 @@ export class PypiRegistry extends BaseRegistry {
await this.storage.putPypiPackageFile(normalized, filename, fileData); await this.storage.putPypiPackageFile(normalized, filename, fileData);
} }
} }
}
if (!fileData) { if (!fileData) {
return { return {

View File

@@ -303,14 +303,27 @@ export class RubyGemsRegistry extends BaseRegistry {
return this.errorResponse(400, 'Invalid gem filename'); return this.errorResponse(400, 'Invalid gem filename');
} }
let gemData = await this.storage.getRubyGemsGem( // Try streaming from local storage first
const streamResult = await this.storage.getRubyGemsGemStream(
parsed.name, parsed.name,
parsed.version, parsed.version,
parsed.platform parsed.platform
); );
if (streamResult) {
return {
status: 200,
headers: {
'Content-Type': 'application/octet-stream',
'Content-Disposition': `attachment; filename="${filename}"`,
'Content-Length': streamResult.size.toString()
},
body: streamResult.stream,
};
}
// Try upstream if not found locally // Try upstream if not found locally
if (!gemData) { let gemData: Buffer | null = null;
const upstream = await this.getUpstreamForRequest(parsed.name, 'gem', 'GET', actor); const upstream = await this.getUpstreamForRequest(parsed.name, 'gem', 'GET', actor);
if (upstream) { if (upstream) {
gemData = await upstream.fetchGem(parsed.name, parsed.version); gemData = await upstream.fetchGem(parsed.name, parsed.version);
@@ -319,7 +332,6 @@ export class RubyGemsRegistry extends BaseRegistry {
await this.storage.putRubyGemsGem(parsed.name, parsed.version, gemData, parsed.platform); await this.storage.putRubyGemsGem(parsed.name, parsed.version, gemData, parsed.platform);
} }
} }
}
if (!gemData) { if (!gemData) {
return this.errorResponse(404, 'Gem not found'); return this.errorResponse(404, 'Gem not found');

View File

@@ -427,7 +427,7 @@ export async function extractGemMetadata(gemData: Buffer): Promise<{
// Step 2: Decompress the gzipped metadata // Step 2: Decompress the gzipped metadata
const gzipTools = new plugins.smartarchive.GzipTools(); const gzipTools = new plugins.smartarchive.GzipTools();
const metadataYaml = await gzipTools.decompress(metadataFile.contentBuffer); const metadataYaml = await gzipTools.decompress(metadataFile.contentBuffer);
const yamlContent = metadataYaml.toString('utf-8'); const yamlContent = Buffer.from(metadataYaml).toString('utf-8');
// Step 3: Parse the YAML to extract name, version, platform // Step 3: Parse the YAML to extract name, version, platform
// Look for name: field in YAML // Look for name: field in YAML
@@ -503,7 +503,7 @@ export async function generateSpecsGz(specs: Array<[string, string, string]>): P
} }
const uncompressed = Buffer.concat(parts); const uncompressed = Buffer.concat(parts);
return gzipTools.compress(uncompressed); return Buffer.from(await gzipTools.compress(uncompressed));
} }
/** /**

View File

@@ -105,7 +105,7 @@ export class UpstreamCache {
// If not in memory and we have storage, check S3 // If not in memory and we have storage, check S3
if (!entry && this.storage) { if (!entry && this.storage) {
entry = await this.loadFromStorage(key); entry = (await this.loadFromStorage(key)) ?? undefined;
if (entry) { if (entry) {
// Promote to memory cache // Promote to memory cache
this.memoryCache.set(key, entry); this.memoryCache.set(key, entry);

View File

@@ -4,9 +4,7 @@
"module": "NodeNext", "module": "NodeNext",
"moduleResolution": "NodeNext", "moduleResolution": "NodeNext",
"esModuleInterop": true, "esModuleInterop": true,
"verbatimModuleSyntax": true, "verbatimModuleSyntax": true
"baseUrl": ".",
"paths": {}
}, },
"exclude": ["dist_*/**/*.d.ts"] "exclude": ["dist_*/**/*.d.ts"]
} }