8 Commits

Author SHA1 Message Date
09335d41f3 v2.8.2
Some checks failed
Default (tags) / security (push) Failing after 0s
Default (tags) / test (push) Failing after 0s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-03-27 17:37:24 +00:00
2221eef722 fix(maven,tests): handle Maven Basic auth and accept deploy-plugin metadata/checksum uploads while stabilizing npm CLI test cleanup 2026-03-27 17:37:24 +00:00
26ddf1a59f v2.8.1
Some checks failed
Default (tags) / security (push) Failing after 0s
Default (tags) / test (push) Failing after 0s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-03-24 23:23:03 +00:00
5acd1d6166 fix(registry): align OCI and RubyGems API behavior and improve npm search result ordering 2026-03-24 23:23:03 +00:00
abf7605e14 v2.8.0
Some checks failed
Default (tags) / security (push) Failing after 0s
Default (tags) / test (push) Failing after 0s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-03-24 22:59:37 +00:00
7da1a35efe feat(core,storage,oci,registry-config): add streaming response support and configurable registry URLs across protocols 2026-03-24 22:59:37 +00:00
1f0acf2825 fix(oci): remove /v2/ from internal route patterns and make upstream apiPrefix configurable
The OCI handler had /v2/ baked into all regex patterns and Location headers.
When basePath was set to /v2 (as in stack.gallery), stripping it removed the
prefix that patterns expected, causing all OCI endpoints to 404.

Now patterns match on bare paths after basePath stripping, working correctly
regardless of the basePath value.

Also adds configurable apiPrefix to OCI upstream class (default /v2) for
registries behind reverse proxies with custom path prefixes.
2026-03-21 16:17:52 +00:00
37e4c5be4a fix(npm): decode URL-encoded package names after regex extraction
Scoped npm packages use %2f encoding for the slash in URLs (e.g. @scope%2fpackage).
Previously, the encoded name was used as-is for storage and packument metadata,
causing npm install to fail with EINVALIDPACKAGENAME. Now each regex extraction
point decodes the package name via decodeURIComponent while keeping the path
encoded for correct regex matching.
2026-03-21 11:59:52 +00:00
44 changed files with 4430 additions and 5596 deletions

24
.smartconfig.json Normal file
View File

@@ -0,0 +1,24 @@
{
"@git.zone/cli": {
"projectType": "npm",
"module": {
"githost": "code.foss.global",
"gitscope": "push.rocks",
"gitrepo": "smartregistry",
"description": "A composable TypeScript library implementing OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems registries for building unified container and package registries",
"npmPackagename": "@push.rocks/smartregistry",
"license": "MIT",
"projectDomain": "push.rocks"
},
"release": {
"registries": [
"https://verdaccio.lossless.digital",
"https://registry.npmjs.org"
],
"accessLevel": "public"
}
},
"@ship.zone/szci": {
"npmGlobalTools": []
}
}

View File

@@ -1,7 +1,7 @@
{
"json.schemas": [
{
"fileMatch": ["/npmextra.json"],
"fileMatch": ["/.smartconfig.json"],
"schema": {
"type": "object",
"properties": {

View File

@@ -1,5 +1,30 @@
# Changelog
## 2026-03-27 - 2.8.2 - fix(maven,tests)
handle Maven Basic auth and accept deploy-plugin metadata/checksum uploads while stabilizing npm CLI test cleanup
- Validate Maven tokens from Basic auth credentials by extracting the password portion before token validation.
- Return successful responses for PUT requests to checksum and maven-metadata endpoints so Maven deploy uploads do not fail when files are auto-generated.
- Improve npm CLI integration test isolation and cleanup by using a temporary test directory, copying per-package .npmrc files, and cleaning stale published packages before test runs.
- Tighten test teardown by destroying the registry explicitly and simplifying package/install fixture generation.
## 2026-03-24 - 2.8.1 - fix(registry)
align OCI and RubyGems API behavior and improve npm search result ordering
- handle OCI version checks on /v2 and /v2/ endpoints
- return RubyGems versions JSON in the expected flat array format and update unyank coverage to use the HTTP endpoint
- prioritize exact and prefix matches in npm search results
- update documentation to reflect full upstream proxy support
## 2026-03-24 - 2.8.0 - feat(core,storage,oci,registry-config)
add streaming response support and configurable registry URLs across protocols
- Normalize SmartRegistry responses to ReadableStream bodies at the public API boundary and add stream helper utilities for buffers, JSON, and hashing
- Add streaming storage accessors for OCI, npm, Maven, Cargo, Composer, PyPI, and RubyGems downloads to reduce in-memory buffering
- Make per-protocol registryUrl configurable so CLI and integration tests can use correct host and port values
- Refactor OCI blob uploads to persist chunks in storage during upload and clean up temporary chunk objects after completion or expiry
- Update tests and storage integration to use the new stream-based response model and smartstorage backend
## 2025-12-03 - 2.7.0 - feat(upstream)
Add dynamic per-request upstream provider and integrate into registries

View File

@@ -1,18 +0,0 @@
{
"gitzone": {
"projectType": "npm",
"module": {
"githost": "code.foss.global",
"gitscope": "push.rocks",
"gitrepo": "smartregistry",
"description": "a registry for npm modules and oci images",
"npmPackagename": "@push.rocks/smartregistry",
"license": "MIT",
"projectDomain": "push.rocks"
}
},
"npmci": {
"npmGlobalTools": [],
"npmAccessLevel": "public"
}
}

View File

@@ -1,6 +1,6 @@
{
"name": "@push.rocks/smartregistry",
"version": "2.7.0",
"version": "2.8.2",
"private": false,
"description": "A composable TypeScript library implementing OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems registries for building unified container and package registries",
"main": "dist_ts/index.js",
@@ -10,17 +10,17 @@
"license": "MIT",
"scripts": {
"test": "(tstest test/ --verbose --logfile --timeout 240)",
"build": "(tsbuild --web --allowimplicitany)",
"build": "(tsbuild --allowimplicitany)",
"buildDocs": "(tsdoc)"
},
"devDependencies": {
"@git.zone/tsbuild": "^3.1.0",
"@git.zone/tsbundle": "^2.0.5",
"@git.zone/tsrun": "^2.0.0",
"@git.zone/tstest": "^3.1.0",
"@push.rocks/smartarchive": "^5.0.1",
"@push.rocks/smarts3": "^5.1.0",
"@types/node": "^24.10.1"
"@git.zone/tsbuild": "^4.4.0",
"@git.zone/tsbundle": "^2.10.0",
"@git.zone/tsrun": "^2.0.2",
"@git.zone/tstest": "^3.6.0",
"@push.rocks/smartarchive": "^5.2.1",
"@push.rocks/smartstorage": "^6.3.2",
"@types/node": "^25.5.0"
},
"repository": {
"type": "git",
@@ -39,7 +39,7 @@
"dist_ts_web/**/*",
"assets/**/*",
"cli.js",
"npmextra.json",
".smartconfig.json",
"readme.md"
],
"pnpm": {
@@ -47,13 +47,13 @@
},
"dependencies": {
"@push.rocks/qenv": "^6.1.3",
"@push.rocks/smartbucket": "^4.3.0",
"@push.rocks/smartlog": "^3.1.10",
"@push.rocks/smartbucket": "^4.5.1",
"@push.rocks/smartlog": "^3.2.1",
"@push.rocks/smartpath": "^6.0.0",
"@push.rocks/smartrequest": "^5.0.1",
"@tsclass/tsclass": "^9.3.0",
"adm-zip": "^0.5.10",
"minimatch": "^10.1.1"
"@tsclass/tsclass": "^9.5.0",
"adm-zip": "^0.5.16",
"minimatch": "^10.2.4"
},
"packageManager": "pnpm@10.18.1+sha512.77a884a165cbba2d8d1c19e3b4880eee6d2fcabd0d879121e282196b80042351d5eb3ca0935fa599da1dc51265cc68816ad2bddd2a2de5ea9fdf92adbec7cd34"
}

7100
pnpm-lock.yaml generated

File diff suppressed because it is too large Load Diff

1148
readme.md

File diff suppressed because it is too large Load Diff

View File

@@ -2,6 +2,7 @@ import { tap, expect } from '@git.zone/tstest';
import { RegistryStorage } from '../ts/core/classes.registrystorage.js';
import { CargoRegistry } from '../ts/cargo/classes.cargoregistry.js';
import { AuthManager } from '../ts/core/classes.authmanager.js';
import { streamToJson } from '../ts/core/helpers.stream.js';
// Test index path calculation
tap.test('should calculate correct index paths for different crate names', async () => {
@@ -123,9 +124,10 @@ tap.test('should return valid config.json', async () => {
expect(response.status).to.equal(200);
expect(response.headers['Content-Type']).to.equal('application/json');
expect(response.body).to.be.an('object');
expect(response.body.dl).to.include('/api/v1/crates/{crate}/{version}/download');
expect(response.body.api).to.equal('http://localhost:5000/cargo');
const body = await streamToJson(response.body);
expect(body).to.be.an('object');
expect(body.dl).to.include('/api/v1/crates/{crate}/{version}/download');
expect(body.api).to.equal('http://localhost:5000/cargo');
});
export default tap.start();

View File

@@ -65,7 +65,9 @@ export async function cleanupS3Bucket(prefix?: string): Promise<void> {
/**
* Create a test SmartRegistry instance with all protocols enabled
*/
export async function createTestRegistry(): Promise<SmartRegistry> {
export async function createTestRegistry(options?: {
registryUrl?: string;
}): Promise<SmartRegistry> {
// Read S3 config from env.json
const s3AccessKey = await testQenv.getEnvVarOnDemand('S3_ACCESSKEY');
const s3SecretKey = await testQenv.getEnvVarOnDemand('S3_SECRETKEY');
@@ -103,30 +105,37 @@ export async function createTestRegistry(): Promise<SmartRegistry> {
oci: {
enabled: true,
basePath: '/oci',
...(options?.registryUrl ? { registryUrl: `${options.registryUrl}/oci` } : {}),
},
npm: {
enabled: true,
basePath: '/npm',
...(options?.registryUrl ? { registryUrl: `${options.registryUrl}/npm` } : {}),
},
maven: {
enabled: true,
basePath: '/maven',
...(options?.registryUrl ? { registryUrl: `${options.registryUrl}/maven` } : {}),
},
composer: {
enabled: true,
basePath: '/composer',
...(options?.registryUrl ? { registryUrl: `${options.registryUrl}/composer` } : {}),
},
cargo: {
enabled: true,
basePath: '/cargo',
...(options?.registryUrl ? { registryUrl: `${options.registryUrl}/cargo` } : {}),
},
pypi: {
enabled: true,
basePath: '/pypi',
...(options?.registryUrl ? { registryUrl: options.registryUrl } : {}),
},
rubygems: {
enabled: true,
basePath: '/rubygems',
...(options?.registryUrl ? { registryUrl: `${options.registryUrl}/rubygems` } : {}),
},
};
@@ -441,7 +450,7 @@ class TestClass
},
];
return zipTools.createZip(entries);
return Buffer.from(await zipTools.createZip(entries));
}
/**
@@ -515,7 +524,7 @@ def hello():
},
];
return zipTools.createZip(entries);
return Buffer.from(await zipTools.createZip(entries));
}
/**
@@ -576,7 +585,7 @@ def hello():
},
];
return tarTools.packFilesToTarGz(entries);
return Buffer.from(await tarTools.packFilesToTarGz(entries));
}
/**
@@ -647,7 +656,7 @@ summary: Test gem for SmartRegistry
test_files: []
`;
const metadataGz = await gzipTools.compress(Buffer.from(metadataYaml, 'utf-8'));
const metadataGz = Buffer.from(await gzipTools.compress(Buffer.from(metadataYaml, 'utf-8')));
// Create data.tar.gz content
const libContent = `# ${gemName}
@@ -668,7 +677,7 @@ end
},
];
const dataTarGz = await tarTools.packFilesToTarGz(dataEntries);
const dataTarGz = Buffer.from(await tarTools.packFilesToTarGz(dataEntries));
// Create the outer gem (tar.gz containing metadata.gz and data.tar.gz)
const gemEntries: smartarchive.IArchiveEntry[] = [
@@ -683,7 +692,7 @@ end
];
// RubyGems .gem files are plain tar archives (NOT gzipped), containing metadata.gz and data.tar.gz
return tarTools.packFiles(gemEntries);
return Buffer.from(await tarTools.packFiles(gemEntries));
}
/**

View File

@@ -79,16 +79,10 @@ async function createHttpServer(
res.setHeader(key, value);
}
// Send body
// Send body (response.body is always ReadableStream<Uint8Array> or undefined)
if (response.body) {
if (Buffer.isBuffer(response.body)) {
res.end(response.body);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
const { Readable } = await import('stream');
Readable.fromWeb(response.body).pipe(res);
} else {
res.end();
}
@@ -251,8 +245,11 @@ function cleanupTestDir(dir: string): void {
// ========================================================================
tap.test('Cargo CLI: should setup registry and HTTP server', async () => {
// Create registry
registry = await createTestRegistry();
// Use port 5000
registryPort = 5000;
// Create registry with correct registryUrl for CLI tests
registry = await createTestRegistry({ registryUrl: `http://localhost:${registryPort}` });
const tokens = await createTestTokens(registry);
cargoToken = tokens.cargoToken;
@@ -266,10 +263,6 @@ tap.test('Cargo CLI: should setup registry and HTTP server', async () => {
} catch (error) {
// Ignore error if operation fails
}
// Use port 5000 (hardcoded in CargoRegistry default config)
// TODO: Once registryUrl is configurable, use dynamic port like npm test (35001)
registryPort = 5000;
const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server;
registryUrl = serverSetup.url;

View File

@@ -84,16 +84,10 @@ async function createHttpServer(
res.setHeader(key, value);
}
// Send body
// Send body (response.body is always ReadableStream<Uint8Array> or undefined)
if (response.body) {
if (Buffer.isBuffer(response.body)) {
res.end(response.body);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
const { Readable } = await import('stream');
Readable.fromWeb(response.body).pipe(res);
} else {
res.end();
}
@@ -249,16 +243,16 @@ tap.test('Composer CLI: should verify composer is installed', async () => {
});
tap.test('Composer CLI: should setup registry and HTTP server', async () => {
// Create registry
registry = await createTestRegistry();
// Use port 38000 (avoids conflicts with other tests)
registryPort = 38000;
// Create registry with correct registryUrl for CLI tests
registry = await createTestRegistry({ registryUrl: `http://localhost:${registryPort}` });
const tokens = await createTestTokens(registry);
composerToken = tokens.composerToken;
expect(registry).toBeInstanceOf(SmartRegistry);
expect(composerToken).toBeTypeOf('string');
// Use port 38000 (avoids conflicts with other tests)
registryPort = 38000;
const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server;
registryUrl = serverSetup.url;

View File

@@ -1,5 +1,6 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js';
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
import { createTestRegistry, createTestTokens, createComposerZip } from './helpers/registry.js';
let registry: SmartRegistry;
@@ -41,9 +42,10 @@ tap.test('Composer: should return packages.json (GET /packages.json)', async ()
});
expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('metadata-url');
expect(response.body).toHaveProperty('available-packages');
expect(response.body['available-packages']).toBeInstanceOf(Array);
const body = await streamToJson(response.body);
expect(body).toHaveProperty('metadata-url');
expect(body).toHaveProperty('available-packages');
expect(body['available-packages']).toBeInstanceOf(Array);
});
tap.test('Composer: should upload a package (PUT /packages/{vendor/package})', async () => {
@@ -59,9 +61,10 @@ tap.test('Composer: should upload a package (PUT /packages/{vendor/package})', a
});
expect(response.status).toEqual(201);
expect(response.body.status).toEqual('success');
expect(response.body.package).toEqual(testPackageName);
expect(response.body.version).toEqual(testVersion);
const body = await streamToJson(response.body);
expect(body.status).toEqual('success');
expect(body.package).toEqual(testPackageName);
expect(body.version).toEqual(testVersion);
});
tap.test('Composer: should retrieve package metadata (GET /p2/{vendor/package}.json)', async () => {
@@ -73,11 +76,12 @@ tap.test('Composer: should retrieve package metadata (GET /p2/{vendor/package}.j
});
expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('packages');
expect(response.body.packages[testPackageName]).toBeInstanceOf(Array);
expect(response.body.packages[testPackageName].length).toEqual(1);
const body = await streamToJson(response.body);
expect(body).toHaveProperty('packages');
expect(body.packages[testPackageName]).toBeInstanceOf(Array);
expect(body.packages[testPackageName].length).toEqual(1);
const packageData = response.body.packages[testPackageName][0];
const packageData = body.packages[testPackageName][0];
expect(packageData.name).toEqual(testPackageName);
expect(packageData.version).toEqual(testVersion);
expect(packageData.version_normalized).toEqual('1.0.0.0');
@@ -97,7 +101,8 @@ tap.test('Composer: should download package ZIP (GET /dists/{vendor/package}/{re
query: {},
});
const reference = metadataResponse.body.packages[testPackageName][0].dist.reference;
const metaBody = await streamToJson(metadataResponse.body);
const reference = metaBody.packages[testPackageName][0].dist.reference;
const response = await registry.handleRequest({
method: 'GET',
@@ -107,7 +112,8 @@ tap.test('Composer: should download package ZIP (GET /dists/{vendor/package}/{re
});
expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer);
const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
expect(response.headers['Content-Type']).toEqual('application/zip');
expect(response.headers['Content-Disposition']).toContain('attachment');
});
@@ -121,9 +127,10 @@ tap.test('Composer: should list packages (GET /packages/list.json)', async () =>
});
expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('packageNames');
expect(response.body.packageNames).toBeInstanceOf(Array);
expect(response.body.packageNames).toContain(testPackageName);
const body = await streamToJson(response.body);
expect(body).toHaveProperty('packageNames');
expect(body.packageNames).toBeInstanceOf(Array);
expect(body.packageNames).toContain(testPackageName);
});
tap.test('Composer: should filter package list (GET /packages/list.json?filter=vendor/*)', async () => {
@@ -135,8 +142,9 @@ tap.test('Composer: should filter package list (GET /packages/list.json?filter=v
});
expect(response.status).toEqual(200);
expect(response.body.packageNames).toBeInstanceOf(Array);
expect(response.body.packageNames).toContain(testPackageName);
const body = await streamToJson(response.body);
expect(body.packageNames).toBeInstanceOf(Array);
expect(body.packageNames).toContain(testPackageName);
});
tap.test('Composer: should prevent duplicate version upload', async () => {
@@ -152,8 +160,9 @@ tap.test('Composer: should prevent duplicate version upload', async () => {
});
expect(response.status).toEqual(409);
expect(response.body.status).toEqual('error');
expect(response.body.message).toContain('already exists');
const body = await streamToJson(response.body);
expect(body.status).toEqual('error');
expect(body.message).toContain('already exists');
});
tap.test('Composer: should upload a second version', async () => {
@@ -172,8 +181,9 @@ tap.test('Composer: should upload a second version', async () => {
});
expect(response.status).toEqual(201);
expect(response.body.status).toEqual('success');
expect(response.body.version).toEqual(testVersion2);
const body = await streamToJson(response.body);
expect(body.status).toEqual('success');
expect(body.version).toEqual(testVersion2);
});
tap.test('Composer: should return multiple versions in metadata', async () => {
@@ -185,10 +195,11 @@ tap.test('Composer: should return multiple versions in metadata', async () => {
});
expect(response.status).toEqual(200);
expect(response.body.packages[testPackageName]).toBeInstanceOf(Array);
expect(response.body.packages[testPackageName].length).toEqual(2);
const body = await streamToJson(response.body);
expect(body.packages[testPackageName]).toBeInstanceOf(Array);
expect(body.packages[testPackageName].length).toEqual(2);
const versions = response.body.packages[testPackageName].map((p: any) => p.version);
const versions = body.packages[testPackageName].map((p: any) => p.version);
expect(versions).toContain('1.0.0');
expect(versions).toContain('1.1.0');
});
@@ -213,8 +224,9 @@ tap.test('Composer: should delete a specific version (DELETE /packages/{vendor/p
query: {},
});
expect(metadataResponse.body.packages[testPackageName].length).toEqual(1);
expect(metadataResponse.body.packages[testPackageName][0].version).toEqual('1.1.0');
const metaBody = await streamToJson(metadataResponse.body);
expect(metaBody.packages[testPackageName].length).toEqual(1);
expect(metaBody.packages[testPackageName][0].version).toEqual('1.1.0');
});
tap.test('Composer: should require auth for package upload', async () => {
@@ -231,7 +243,8 @@ tap.test('Composer: should require auth for package upload', async () => {
});
expect(response.status).toEqual(401);
expect(response.body.status).toEqual('error');
const body = await streamToJson(response.body);
expect(body.status).toEqual('error');
});
tap.test('Composer: should reject invalid ZIP (no composer.json)', async () => {
@@ -249,8 +262,9 @@ tap.test('Composer: should reject invalid ZIP (no composer.json)', async () => {
});
expect(response.status).toEqual(400);
expect(response.body.status).toEqual('error');
expect(response.body.message).toContain('composer.json');
const body = await streamToJson(response.body);
expect(body.status).toEqual('error');
expect(body.message).toContain('composer.json');
});
tap.test('Composer: should delete entire package (DELETE /packages/{vendor/package})', async () => {

View File

@@ -1,5 +1,6 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js';
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
import {
createTestRegistry,
createTestTokens,
@@ -79,7 +80,9 @@ tap.test('Integration: should handle /simple path for PyPI', async () => {
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toStartWith('text/html');
expect(response.body).toContain('integration-test-py');
const body = await streamToBuffer(response.body);
const text = body.toString('utf-8');
expect(text).toContain('integration-test-py');
});
tap.test('Integration: should reject PyPI token for RubyGems endpoint', async () => {
@@ -135,8 +138,9 @@ tap.test('Integration: should return 404 for unknown paths', async () => {
});
expect(response.status).toEqual(404);
expect(response.body).toHaveProperty('error');
expect((response.body as any).error).toEqual('NOT_FOUND');
const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
expect(body.error).toEqual('NOT_FOUND');
});
tap.test('Integration: should retrieve PyPI registry instance', async () => {

View File

@@ -1,32 +1,34 @@
/**
* Integration test for smartregistry with smarts3
* Integration test for smartregistry with smartstorage
* Verifies that smartregistry works with a local S3-compatible server
*/
import { expect, tap } from '@git.zone/tstest/tapbundle';
import * as smarts3Module from '@push.rocks/smarts3';
import * as smartstorageModule from '@push.rocks/smartstorage';
import { SmartRegistry } from '../ts/classes.smartregistry.js';
import type { IRegistryConfig } from '../ts/core/interfaces.core.js';
import { streamToJson } from '../ts/core/helpers.stream.js';
import * as crypto from 'crypto';
let s3Server: smarts3Module.Smarts3;
let s3Server: smartstorageModule.SmartStorage;
let registry: SmartRegistry;
/**
* Setup: Start smarts3 server
* Setup: Start smartstorage server
*/
tap.test('should start smarts3 server', async () => {
s3Server = await smarts3Module.Smarts3.createAndStart({
tap.test('should start smartstorage server', async () => {
s3Server = await smartstorageModule.SmartStorage.createAndStart({
server: {
port: 3456, // Use different port to avoid conflicts with other tests
host: '0.0.0.0',
port: 3456,
address: '0.0.0.0',
silent: true,
},
storage: {
cleanSlate: true, // Fresh storage for each test run
bucketsDir: './.nogit/smarts3-test-buckets',
cleanSlate: true,
directory: './.nogit/smartstorage-test-buckets',
},
logging: {
silent: true, // Reduce test output noise
enabled: false,
},
});
@@ -34,20 +36,10 @@ tap.test('should start smarts3 server', async () => {
});
/**
* Setup: Create SmartRegistry with smarts3 configuration
* Setup: Create SmartRegistry with smartstorage configuration
*/
tap.test('should create SmartRegistry instance with smarts3 IS3Descriptor', async () => {
// Manually construct IS3Descriptor based on smarts3 configuration
// Note: smarts3.getS3Descriptor() returns empty object as of v5.1.0
// This is a known limitation - smarts3 doesn't expose its config properly
const s3Descriptor = {
endpoint: 'localhost',
port: 3456,
accessKey: 'test', // smarts3 doesn't require real credentials
accessSecret: 'test',
useSsl: false,
region: 'us-east-1',
};
const s3Descriptor = await s3Server.getStorageDescriptor();
const config: IRegistryConfig = {
storage: {
@@ -97,7 +89,7 @@ tap.test('should create SmartRegistry instance with smarts3 IS3Descriptor', asyn
});
/**
* Test NPM protocol with smarts3
* Test NPM protocol with smartstorage
*/
tap.test('NPM: should publish package to smarts3', async () => {
const authManager = registry.getAuthManager();
@@ -139,7 +131,7 @@ tap.test('NPM: should publish package to smarts3', async () => {
body: packageData,
});
expect(response.status).toEqual(201); // 201 Created is correct for publishing
expect(response.status).toEqual(201);
});
tap.test('NPM: should retrieve package from smarts3', async () => {
@@ -151,12 +143,13 @@ tap.test('NPM: should retrieve package from smarts3', async () => {
});
expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('name');
expect(response.body.name).toEqual('test-package-smarts3');
const body = await streamToJson(response.body);
expect(body).toHaveProperty('name');
expect(body.name).toEqual('test-package-smarts3');
});
/**
* Test OCI protocol with smarts3
* Test OCI protocol with smartstorage
*/
tap.test('OCI: should store blob in smarts3', async () => {
const authManager = registry.getAuthManager();
@@ -173,7 +166,7 @@ tap.test('OCI: should store blob in smarts3', async () => {
// Initiate blob upload
const initiateResponse = await registry.handleRequest({
method: 'POST',
path: '/oci/v2/test-image/blobs/uploads/',
path: '/oci/test-image/blobs/uploads/',
headers: {
'Authorization': `Bearer ${token}`,
},
@@ -196,7 +189,7 @@ tap.test('OCI: should store blob in smarts3', async () => {
const uploadResponse = await registry.handleRequest({
method: 'PUT',
path: `/oci/v2/test-image/blobs/uploads/${uploadId}`,
path: `/oci/test-image/blobs/uploads/${uploadId}`,
headers: {
'Authorization': `Bearer ${token}`,
'Content-Type': 'application/octet-stream',
@@ -209,18 +202,9 @@ tap.test('OCI: should store blob in smarts3', async () => {
});
/**
* Test PyPI protocol with smarts3
* Test PyPI protocol with smartstorage
*/
tap.test('PyPI: should upload package to smarts3', async () => {
const authManager = registry.getAuthManager();
const userId = await authManager.authenticate({
username: 'testuser',
password: 'testpass',
});
const token = await authManager.createPypiToken(userId, false);
// Note: In a real test, this would be multipart/form-data
// For simplicity, we're testing the storage layer
const storage = registry.getStorage();
// Store a test package file
@@ -252,7 +236,7 @@ tap.test('PyPI: should upload package to smarts3', async () => {
});
/**
* Test Cargo protocol with smarts3
* Test Cargo protocol with smartstorage
*/
tap.test('Cargo: should store crate in smarts3', async () => {
const storage = registry.getStorage();
@@ -281,11 +265,11 @@ tap.test('Cargo: should store crate in smarts3', async () => {
});
/**
* Cleanup: Stop smarts3 server
* Cleanup: Stop smartstorage server
*/
tap.test('should stop smarts3 server', async () => {
tap.test('should stop smartstorage server', async () => {
registry.destroy();
await s3Server.stop();
expect(true).toEqual(true); // Just verify it completes without error
});
export default tap.start();

View File

@@ -79,16 +79,10 @@ async function createHttpServer(
res.setHeader(key, value);
}
// Send body
// Send body (response.body is always ReadableStream<Uint8Array> or undefined)
if (response.body) {
if (Buffer.isBuffer(response.body)) {
res.end(response.body);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
const { Readable } = await import('stream');
Readable.fromWeb(response.body).pipe(res);
} else {
res.end();
}
@@ -282,16 +276,16 @@ tap.test('Maven CLI: should verify mvn is installed', async () => {
});
tap.test('Maven CLI: should setup registry and HTTP server', async () => {
// Create registry
registry = await createTestRegistry();
// Use port 37000 (avoids conflicts with other tests)
registryPort = 37000;
// Create registry with correct registryUrl for CLI tests
registry = await createTestRegistry({ registryUrl: `http://localhost:${registryPort}` });
const tokens = await createTestTokens(registry);
mavenToken = tokens.mavenToken;
expect(registry).toBeInstanceOf(SmartRegistry);
expect(mavenToken).toBeTypeOf('string');
// Use port 37000 (avoids conflicts with other tests)
registryPort = 37000;
const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server;
registryUrl = serverSetup.url;

View File

@@ -1,5 +1,6 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js';
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
import {
createTestRegistry,
createTestTokens,
@@ -88,10 +89,11 @@ tap.test('Maven: should retrieve uploaded POM file (GET)', async () => {
});
expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer);
expect((response.body as Buffer).toString('utf-8')).toContain(testGroupId);
expect((response.body as Buffer).toString('utf-8')).toContain(testArtifactId);
expect((response.body as Buffer).toString('utf-8')).toContain(testVersion);
const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
expect(body.toString('utf-8')).toContain(testGroupId);
expect(body.toString('utf-8')).toContain(testArtifactId);
expect(body.toString('utf-8')).toContain(testVersion);
expect(response.headers['Content-Type']).toEqual('application/xml');
});
@@ -107,7 +109,8 @@ tap.test('Maven: should retrieve uploaded JAR file (GET)', async () => {
});
expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer);
const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
expect(response.headers['Content-Type']).toEqual('application/java-archive');
});
@@ -124,8 +127,9 @@ tap.test('Maven: should retrieve MD5 checksum for JAR (GET *.jar.md5)', async ()
});
expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer);
expect((response.body as Buffer).toString('utf-8')).toEqual(checksums.md5);
const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
expect(body.toString('utf-8')).toEqual(checksums.md5);
expect(response.headers['Content-Type']).toEqual('text/plain');
});
@@ -142,8 +146,9 @@ tap.test('Maven: should retrieve SHA1 checksum for JAR (GET *.jar.sha1)', async
});
expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer);
expect((response.body as Buffer).toString('utf-8')).toEqual(checksums.sha1);
const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
expect(body.toString('utf-8')).toEqual(checksums.sha1);
expect(response.headers['Content-Type']).toEqual('text/plain');
});
@@ -160,8 +165,9 @@ tap.test('Maven: should retrieve SHA256 checksum for JAR (GET *.jar.sha256)', as
});
expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer);
expect((response.body as Buffer).toString('utf-8')).toEqual(checksums.sha256);
const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
expect(body.toString('utf-8')).toEqual(checksums.sha256);
expect(response.headers['Content-Type']).toEqual('text/plain');
});
@@ -178,8 +184,9 @@ tap.test('Maven: should retrieve SHA512 checksum for JAR (GET *.jar.sha512)', as
});
expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer);
expect((response.body as Buffer).toString('utf-8')).toEqual(checksums.sha512);
const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
expect(body.toString('utf-8')).toEqual(checksums.sha512);
expect(response.headers['Content-Type']).toEqual('text/plain');
});
@@ -194,8 +201,9 @@ tap.test('Maven: should retrieve maven-metadata.xml (GET)', async () => {
});
expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer);
const xml = (response.body as Buffer).toString('utf-8');
const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
const xml = body.toString('utf-8');
expect(xml).toContain('<groupId>');
expect(xml).toContain('<artifactId>');
expect(xml).toContain('<version>1.0.0</version>');
@@ -247,7 +255,8 @@ tap.test('Maven: should upload a second version and update metadata', async () =
});
expect(response.status).toEqual(200);
const xml = (response.body as Buffer).toString('utf-8');
const metaBody = await streamToBuffer(response.body);
const xml = metaBody.toString('utf-8');
expect(xml).toContain('<version>1.0.0</version>');
expect(xml).toContain('<version>2.0.0</version>');
expect(xml).toContain('<latest>2.0.0</latest>');
@@ -285,7 +294,8 @@ tap.test('Maven: should return 404 for non-existent artifact', async () => {
});
expect(response.status).toEqual(404);
expect(response.body).toHaveProperty('error');
const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
});
tap.test('Maven: should return 401 for unauthorized upload', async () => {
@@ -304,7 +314,8 @@ tap.test('Maven: should return 401 for unauthorized upload', async () => {
});
expect(response.status).toEqual(401);
expect(response.body).toHaveProperty('error');
const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
});
tap.test('Maven: should reject POM upload with mismatched GAV', async () => {
@@ -328,7 +339,8 @@ tap.test('Maven: should reject POM upload with mismatched GAV', async () => {
});
expect(response.status).toEqual(400);
expect(response.body).toHaveProperty('error');
const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
});
tap.test('Maven: should delete an artifact (DELETE)', async () => {

View File

@@ -6,14 +6,14 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { tapNodeTools } from '@git.zone/tstest/tapbundle_serverside';
import { SmartRegistry } from '../ts/index.js';
import { createTestRegistry, createTestTokens } from './helpers/registry.js';
import { createTestRegistry, createTestTokens, cleanupS3Bucket } from './helpers/registry.js';
import type { IRequestContext, IResponse } from '../ts/core/interfaces.core.js';
import * as http from 'http';
import * as url from 'url';
import * as fs from 'fs';
import * as path from 'path';
// Test context
// Test state
let registry: SmartRegistry;
let server: http.Server;
let registryUrl: string;
@@ -32,21 +32,22 @@ async function createHttpServer(
return new Promise((resolve, reject) => {
const httpServer = http.createServer(async (req, res) => {
try {
// Parse request
const parsedUrl = url.parse(req.url || '', true);
const pathname = parsedUrl.pathname || '/';
const query = parsedUrl.query;
const parsedUrl = new url.URL(req.url || '/', `http://localhost:${port}`);
const pathname = parsedUrl.pathname;
const query: Record<string, string> = {};
parsedUrl.searchParams.forEach((value, key) => {
query[key] = value;
});
// Read body
const chunks: Buffer[] = [];
for await (const chunk of req) {
chunks.push(chunk);
}
const bodyBuffer = Buffer.concat(chunks);
let body: any = undefined;
if (req.method === 'PUT' || req.method === 'POST' || req.method === 'PATCH') {
const chunks: Buffer[] = [];
for await (const chunk of req) {
chunks.push(Buffer.from(chunk));
}
const bodyBuffer = Buffer.concat(chunks);
// Parse body based on content type
let body: any;
if (bodyBuffer.length > 0) {
const contentType = req.headers['content-type'] || '';
if (contentType.includes('application/json')) {
try {
@@ -79,16 +80,10 @@ async function createHttpServer(
res.setHeader(key, value);
}
// Send body
// Send body (response.body is always ReadableStream<Uint8Array> or undefined)
if (response.body) {
if (Buffer.isBuffer(response.body)) {
res.end(response.body);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
const { Readable } = await import('stream');
Readable.fromWeb(response.body).pipe(res);
} else {
res.end();
}
@@ -130,7 +125,7 @@ function createTestPackage(
version: string,
targetDir: string
): string {
const packageDir = path.join(targetDir, packageName);
const packageDir = path.join(targetDir, packageName.replace(/\//g, '-'));
fs.mkdirSync(packageDir, { recursive: true });
// Create package.json
@@ -139,12 +134,7 @@ function createTestPackage(
version: version,
description: `Test package ${packageName}`,
main: 'index.js',
scripts: {
test: 'echo "Test passed"',
},
keywords: ['test'],
author: 'Test Author',
license: 'MIT',
scripts: {},
};
fs.writeFileSync(
@@ -153,25 +143,24 @@ function createTestPackage(
'utf-8'
);
// Create index.js
const indexJs = `module.exports = {
name: '${packageName}',
version: '${version}',
message: 'Hello from ${packageName}@${version}'
};
`;
fs.writeFileSync(path.join(packageDir, 'index.js'), indexJs, 'utf-8');
// Create a simple index.js
fs.writeFileSync(
path.join(packageDir, 'index.js'),
`module.exports = { name: '${packageName}', version: '${version}' };\n`,
'utf-8'
);
// Create README.md
const readme = `# ${packageName}
fs.writeFileSync(
path.join(packageDir, 'README.md'),
`# ${packageName}\n\nTest package version ${version}\n`,
'utf-8'
);
Test package for SmartRegistry.
Version: ${version}
`;
fs.writeFileSync(path.join(packageDir, 'README.md'), readme, 'utf-8');
// Copy .npmrc into the package directory
if (npmrcPath && fs.existsSync(npmrcPath)) {
fs.copyFileSync(npmrcPath, path.join(packageDir, '.npmrc'));
}
return packageDir;
}
@@ -183,31 +172,30 @@ async function runNpmCommand(
command: string,
cwd: string
): Promise<{ stdout: string; stderr: string; exitCode: number }> {
// Prepare environment variables
const envVars = [
`NPM_CONFIG_USERCONFIG="${npmrcPath}"`,
`NPM_CONFIG_CACHE="${path.join(testDir, '.npm-cache')}"`,
`NPM_CONFIG_PREFIX="${path.join(testDir, '.npm-global')}"`,
`NPM_CONFIG_REGISTRY="${registryUrl}/npm/"`,
].join(' ');
const { exec } = await import('child_process');
// Build command with cd to correct directory and environment variables
const fullCommand = `cd "${cwd}" && ${envVars} ${command}`;
try {
const result = await tapNodeTools.runCommand(fullCommand);
return {
stdout: result.stdout || '',
stderr: result.stderr || '',
exitCode: result.exitCode || 0,
};
} catch (error: any) {
return {
stdout: error.stdout || '',
stderr: error.stderr || String(error),
exitCode: error.exitCode || 1,
};
// Build isolated env that prevents npm from reading ~/.npmrc
const env: Record<string, string> = {};
// Copy only essential env vars (PATH, etc.) — exclude HOME to prevent ~/.npmrc reading
for (const key of ['PATH', 'NODE', 'NVM_DIR', 'NVM_BIN', 'LANG', 'TERM', 'SHELL']) {
if (process.env[key]) env[key] = process.env[key]!;
}
env.HOME = testDir;
env.NPM_CONFIG_USERCONFIG = npmrcPath;
env.NPM_CONFIG_GLOBALCONFIG = '/dev/null';
env.NPM_CONFIG_CACHE = path.join(testDir, '.npm-cache');
env.NPM_CONFIG_PREFIX = path.join(testDir, '.npm-global');
env.NPM_CONFIG_REGISTRY = `${registryUrl}/npm/`;
return new Promise((resolve) => {
exec(command, { cwd, env, timeout: 30000 }, (error, stdout, stderr) => {
resolve({
stdout: stdout || '',
stderr: stderr || '',
exitCode: error ? (error as any).code ?? 1 : 0,
});
});
});
}
/**
@@ -224,16 +212,26 @@ function cleanupTestDir(dir: string): void {
// ========================================================================
tap.test('NPM CLI: should setup registry and HTTP server', async () => {
// Create registry
registry = await createTestRegistry();
// Find available port
registryPort = 35000;
// Create registry with correct registryUrl for CLI tests
registry = await createTestRegistry({ registryUrl: `http://localhost:${registryPort}` });
const tokens = await createTestTokens(registry);
npmToken = tokens.npmToken;
// Clean up stale npm CLI test data via unpublish API
for (const pkg of ['test-package-cli', '@testscope%2fscoped-package']) {
await registry.handleRequest({
method: 'DELETE',
path: `/npm/${pkg}/-rev/cleanup`,
headers: { Authorization: `Bearer ${npmToken}` },
query: {},
});
}
expect(registry).toBeInstanceOf(SmartRegistry);
expect(npmToken).toBeTypeOf('string');
// Find available port
registryPort = 35000;
const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server;
registryUrl = serverSetup.url;
@@ -241,8 +239,8 @@ tap.test('NPM CLI: should setup registry and HTTP server', async () => {
expect(server).toBeDefined();
expect(registryUrl).toEqual(`http://localhost:${registryPort}`);
// Setup test directory
testDir = path.join(process.cwd(), '.nogit', 'test-npm-cli');
// Setup test directory — use /tmp to isolate from project tree
testDir = path.join('/tmp', 'smartregistry-test-npm-cli');
cleanupTestDir(testDir);
fs.mkdirSync(testDir, { recursive: true });
@@ -291,20 +289,16 @@ tap.test('NPM CLI: should install published package', async () => {
const installDir = path.join(testDir, 'install-test');
fs.mkdirSync(installDir, { recursive: true });
// Create package.json for installation
const packageJson = {
name: 'install-test',
version: '1.0.0',
dependencies: {
[packageName]: '1.0.0',
},
};
// Create a minimal package.json for install target
fs.writeFileSync(
path.join(installDir, 'package.json'),
JSON.stringify(packageJson, null, 2),
JSON.stringify({ name: 'install-test', version: '1.0.0', dependencies: { [packageName]: '1.0.0' } }),
'utf-8'
);
// Copy .npmrc
if (npmrcPath && fs.existsSync(npmrcPath)) {
fs.copyFileSync(npmrcPath, path.join(installDir, '.npmrc'));
}
const result = await runNpmCommand('npm install', installDir);
console.log('npm install output:', result.stdout);
@@ -313,17 +307,8 @@ tap.test('NPM CLI: should install published package', async () => {
expect(result.exitCode).toEqual(0);
// Verify package was installed
const nodeModulesPath = path.join(installDir, 'node_modules', packageName);
expect(fs.existsSync(nodeModulesPath)).toEqual(true);
expect(fs.existsSync(path.join(nodeModulesPath, 'package.json'))).toEqual(true);
expect(fs.existsSync(path.join(nodeModulesPath, 'index.js'))).toEqual(true);
// Verify package contents
const installedPackageJson = JSON.parse(
fs.readFileSync(path.join(nodeModulesPath, 'package.json'), 'utf-8')
);
expect(installedPackageJson.name).toEqual(packageName);
expect(installedPackageJson.version).toEqual('1.0.0');
const installed = fs.existsSync(path.join(installDir, 'node_modules', packageName, 'package.json'));
expect(installed).toEqual(true);
});
tap.test('NPM CLI: should publish second version', async () => {
@@ -375,17 +360,14 @@ tap.test('NPM CLI: should fail to publish without auth', async () => {
const version = '1.0.0';
const packageDir = createTestPackage(packageName, version, testDir);
// Temporarily remove .npmrc
const npmrcBackup = fs.readFileSync(npmrcPath, 'utf-8');
fs.writeFileSync(npmrcPath, 'registry=' + registryUrl + '/npm/\n', 'utf-8');
// Temporarily remove .npmrc (write one without auth)
const noAuthNpmrc = path.join(packageDir, '.npmrc');
fs.writeFileSync(noAuthNpmrc, `registry=${registryUrl}/npm/\n`, 'utf-8');
const result = await runNpmCommand('npm publish', packageDir);
console.log('npm publish unauth output:', result.stdout);
console.log('npm publish unauth stderr:', result.stderr);
// Restore .npmrc
fs.writeFileSync(npmrcPath, npmrcBackup, 'utf-8');
// Should fail with auth error
expect(result.exitCode).not.toEqual(0);
});
@@ -399,14 +381,7 @@ tap.postTask('cleanup npm cli tests', async () => {
}
// Cleanup test directory
if (testDir) {
cleanupTestDir(testDir);
}
// Destroy registry
if (registry) {
registry.destroy();
}
cleanupTestDir(testDir);
});
export default tap.start();

View File

@@ -1,5 +1,6 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js';
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
import { createTestRegistry, createTestTokens, createTestPackument } from './helpers/registry.js';
let registry: SmartRegistry;
@@ -34,8 +35,9 @@ tap.test('NPM: should handle user authentication (PUT /-/user/org.couchdb.user:{
});
expect(response.status).toEqual(201);
expect(response.body).toHaveProperty('token');
expect((response.body as any).token).toBeTypeOf('string');
const body = await streamToJson(response.body);
expect(body).toHaveProperty('token');
expect(body.token).toBeTypeOf('string');
});
tap.test('NPM: should publish a package (PUT /{package})', async () => {
@@ -53,8 +55,9 @@ tap.test('NPM: should publish a package (PUT /{package})', async () => {
});
expect(response.status).toEqual(201);
expect(response.body).toHaveProperty('ok');
expect((response.body as any).ok).toEqual(true);
const body = await streamToJson(response.body);
expect(body).toHaveProperty('ok');
expect(body.ok).toEqual(true);
});
tap.test('NPM: should retrieve package metadata (GET /{package})', async () => {
@@ -66,10 +69,11 @@ tap.test('NPM: should retrieve package metadata (GET /{package})', async () => {
});
expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('name');
expect((response.body as any).name).toEqual(testPackageName);
expect((response.body as any).versions).toHaveProperty(testVersion);
expect((response.body as any)['dist-tags'].latest).toEqual(testVersion);
const body = await streamToJson(response.body);
expect(body).toHaveProperty('name');
expect(body.name).toEqual(testPackageName);
expect(body.versions).toHaveProperty(testVersion);
expect(body['dist-tags'].latest).toEqual(testVersion);
});
tap.test('NPM: should retrieve specific version metadata (GET /{package}/{version})', async () => {
@@ -81,9 +85,10 @@ tap.test('NPM: should retrieve specific version metadata (GET /{package}/{versio
});
expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('version');
expect((response.body as any).version).toEqual(testVersion);
expect((response.body as any).name).toEqual(testPackageName);
const body = await streamToJson(response.body);
expect(body).toHaveProperty('version');
expect(body.version).toEqual(testVersion);
expect(body.name).toEqual(testPackageName);
});
tap.test('NPM: should download tarball (GET /{package}/-/{tarball})', async () => {
@@ -95,8 +100,9 @@ tap.test('NPM: should download tarball (GET /{package}/-/{tarball})', async () =
});
expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer);
expect((response.body as Buffer).toString('utf-8')).toEqual('fake tarball content');
const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
expect(body.toString('utf-8')).toEqual('fake tarball content');
expect(response.headers['Content-Type']).toEqual('application/octet-stream');
});
@@ -127,7 +133,8 @@ tap.test('NPM: should publish a new version of the package', async () => {
});
expect(getResponse.status).toEqual(200);
expect((getResponse.body as any).versions).toHaveProperty(newVersion);
const getBody = await streamToJson(getResponse.body);
expect(getBody.versions).toHaveProperty(newVersion);
});
tap.test('NPM: should get dist-tags (GET /-/package/{pkg}/dist-tags)', async () => {
@@ -139,8 +146,9 @@ tap.test('NPM: should get dist-tags (GET /-/package/{pkg}/dist-tags)', async ()
});
expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('latest');
expect((response.body as any).latest).toBeTypeOf('string');
const body = await streamToJson(response.body);
expect(body).toHaveProperty('latest');
expect(body.latest).toBeTypeOf('string');
});
tap.test('NPM: should update dist-tag (PUT /-/package/{pkg}/dist-tags/{tag})', async () => {
@@ -165,7 +173,8 @@ tap.test('NPM: should update dist-tag (PUT /-/package/{pkg}/dist-tags/{tag})', a
query: {},
});
expect((getResponse.body as any)['dist-tags'].beta).toEqual('1.1.0');
const getBody2 = await streamToJson(getResponse.body);
expect(getBody2['dist-tags'].beta).toEqual('1.1.0');
});
tap.test('NPM: should delete dist-tag (DELETE /-/package/{pkg}/dist-tags/{tag})', async () => {
@@ -188,7 +197,8 @@ tap.test('NPM: should delete dist-tag (DELETE /-/package/{pkg}/dist-tags/{tag})'
query: {},
});
expect((getResponse.body as any)['dist-tags']).not.toHaveProperty('beta');
const getBody3 = await streamToJson(getResponse.body);
expect(getBody3['dist-tags']).not.toHaveProperty('beta');
});
tap.test('NPM: should create a new token (POST /-/npm/v1/tokens)', async () => {
@@ -208,8 +218,9 @@ tap.test('NPM: should create a new token (POST /-/npm/v1/tokens)', async () => {
});
expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('token');
expect((response.body as any).readonly).toEqual(true);
const body = await streamToJson(response.body);
expect(body).toHaveProperty('token');
expect(body.readonly).toEqual(true);
});
tap.test('NPM: should list tokens (GET /-/npm/v1/tokens)', async () => {
@@ -223,9 +234,10 @@ tap.test('NPM: should list tokens (GET /-/npm/v1/tokens)', async () => {
});
expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('objects');
expect((response.body as any).objects).toBeInstanceOf(Array);
expect((response.body as any).objects.length).toBeGreaterThan(0);
const body = await streamToJson(response.body);
expect(body).toHaveProperty('objects');
expect(body.objects).toBeInstanceOf(Array);
expect(body.objects.length).toBeGreaterThan(0);
});
tap.test('NPM: should search packages (GET /-/v1/search)', async () => {
@@ -240,9 +252,10 @@ tap.test('NPM: should search packages (GET /-/v1/search)', async () => {
});
expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('objects');
expect((response.body as any).objects).toBeInstanceOf(Array);
expect((response.body as any).total).toBeGreaterThan(0);
const body = await streamToJson(response.body);
expect(body).toHaveProperty('objects');
expect(body.objects).toBeInstanceOf(Array);
expect(body.total).toBeGreaterThan(0);
});
tap.test('NPM: should search packages with specific query', async () => {
@@ -256,7 +269,8 @@ tap.test('NPM: should search packages with specific query', async () => {
});
expect(response.status).toEqual(200);
const results = (response.body as any).objects;
const body = await streamToJson(response.body);
const results = body.objects;
expect(results.length).toBeGreaterThan(0);
expect(results[0].package.name).toEqual(testPackageName);
});
@@ -281,7 +295,8 @@ tap.test('NPM: should unpublish a specific version (DELETE /{package}/-/{version
query: {},
});
expect((getResponse.body as any).versions).not.toHaveProperty(testVersion);
const getBody4 = await streamToJson(getResponse.body);
expect(getBody4.versions).not.toHaveProperty(testVersion);
});
tap.test('NPM: should unpublish entire package (DELETE /{package}/-rev/{rev})', async () => {
@@ -316,7 +331,8 @@ tap.test('NPM: should return 404 for non-existent package', async () => {
});
expect(response.status).toEqual(404);
expect(response.body).toHaveProperty('error');
const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
});
tap.test('NPM: should return 401 for unauthorized publish', async () => {
@@ -334,7 +350,8 @@ tap.test('NPM: should return 401 for unauthorized publish', async () => {
});
expect(response.status).toEqual(401);
expect(response.body).toHaveProperty('error');
const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
});
tap.test('NPM: should reject readonly token for write operations', async () => {

View File

@@ -48,7 +48,7 @@ async function createDockerTestRegistry(port: number): Promise<SmartRegistry> {
},
oci: {
enabled: true,
basePath: '/oci',
basePath: '/v2',
},
};
@@ -95,8 +95,7 @@ let testImageName: string;
* Create HTTP server wrapper around SmartRegistry
* CRITICAL: Always passes rawBody for content-addressable operations (OCI manifests/blobs)
*
* Docker expects registry at /v2/ but SmartRegistry serves at /oci/v2/
* This wrapper rewrites paths for Docker compatibility
* SmartRegistry OCI is configured with basePath '/v2' matching Docker's native /v2/ prefix.
*
* Also implements a simple /v2/token endpoint for Docker Bearer auth flow
*/
@@ -130,10 +129,7 @@ async function createHttpServer(
// Log all requests for debugging
console.log(`[Registry] ${req.method} ${pathname}`);
// Docker expects /v2/ but SmartRegistry serves at /oci/v2/
if (pathname.startsWith('/v2')) {
pathname = '/oci' + pathname;
}
// basePath is /v2 which matches Docker's native /v2/ prefix — no rewrite needed
// Read raw body - ALWAYS preserve exact bytes for OCI
const chunks: Buffer[] = [];
@@ -179,16 +175,10 @@ async function createHttpServer(
res.setHeader(key, value);
}
// Send body
// Send body (response.body is always ReadableStream<Uint8Array> or undefined)
if (response.body) {
if (Buffer.isBuffer(response.body)) {
res.end(response.body);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
const { Readable } = await import('stream');
Readable.fromWeb(response.body).pipe(res);
} else {
res.end();
}
@@ -313,7 +303,7 @@ tap.test('Docker CLI: should verify server is responding', async () => {
// Give the server a moment to fully initialize
await new Promise(resolve => setTimeout(resolve, 500));
const response = await fetch(`${registryUrl}/oci/v2/`);
const response = await fetch(`${registryUrl}/v2/`);
expect(response.status).toEqual(200);
console.log('OCI v2 response:', await response.json());
});
@@ -352,7 +342,7 @@ tap.test('Docker CLI: should push image to registry', async () => {
});
tap.test('Docker CLI: should verify manifest in registry via API', async () => {
const response = await fetch(`${registryUrl}/oci/v2/test-image/tags/list`, {
const response = await fetch(`${registryUrl}/v2/test-image/tags/list`, {
headers: { Authorization: `Bearer ${ociToken}` },
});

View File

@@ -1,5 +1,6 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js';
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
import { createTestRegistry, createTestTokens, calculateDigest, createTestManifest } from './helpers/registry.js';
let registry: SmartRegistry;
@@ -24,7 +25,7 @@ tap.test('OCI: should create registry instance', async () => {
tap.test('OCI: should handle version check (GET /v2/)', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/oci/v2/',
path: '/oci/',
headers: {},
query: {},
});
@@ -36,7 +37,7 @@ tap.test('OCI: should handle version check (GET /v2/)', async () => {
tap.test('OCI: should initiate blob upload (POST /v2/{name}/blobs/uploads/)', async () => {
const response = await registry.handleRequest({
method: 'POST',
path: '/oci/v2/test-repo/blobs/uploads/',
path: '/oci/test-repo/blobs/uploads/',
headers: {
Authorization: `Bearer ${ociToken}`,
},
@@ -53,7 +54,7 @@ tap.test('OCI: should upload blob in single PUT', async () => {
const response = await registry.handleRequest({
method: 'POST',
path: '/oci/v2/test-repo/blobs/uploads/',
path: '/oci/test-repo/blobs/uploads/',
headers: {
Authorization: `Bearer ${ociToken}`,
},
@@ -73,7 +74,7 @@ tap.test('OCI: should upload config blob', async () => {
const response = await registry.handleRequest({
method: 'POST',
path: '/oci/v2/test-repo/blobs/uploads/',
path: '/oci/test-repo/blobs/uploads/',
headers: {
Authorization: `Bearer ${ociToken}`,
},
@@ -90,7 +91,7 @@ tap.test('OCI: should upload config blob', async () => {
tap.test('OCI: should check if blob exists (HEAD /v2/{name}/blobs/{digest})', async () => {
const response = await registry.handleRequest({
method: 'HEAD',
path: `/oci/v2/test-repo/blobs/${testBlobDigest}`,
path: `/oci/test-repo/blobs/${testBlobDigest}`,
headers: {
Authorization: `Bearer ${ociToken}`,
},
@@ -105,7 +106,7 @@ tap.test('OCI: should check if blob exists (HEAD /v2/{name}/blobs/{digest})', as
tap.test('OCI: should retrieve blob (GET /v2/{name}/blobs/{digest})', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: `/oci/v2/test-repo/blobs/${testBlobDigest}`,
path: `/oci/test-repo/blobs/${testBlobDigest}`,
headers: {
Authorization: `Bearer ${ociToken}`,
},
@@ -113,8 +114,9 @@ tap.test('OCI: should retrieve blob (GET /v2/{name}/blobs/{digest})', async () =
});
expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer);
expect((response.body as Buffer).toString('utf-8')).toEqual('Hello from OCI test blob!');
const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
expect(body.toString('utf-8')).toEqual('Hello from OCI test blob!');
expect(response.headers['Docker-Content-Digest']).toEqual(testBlobDigest);
});
@@ -126,7 +128,7 @@ tap.test('OCI: should upload manifest (PUT /v2/{name}/manifests/{reference})', a
const response = await registry.handleRequest({
method: 'PUT',
path: '/oci/v2/test-repo/manifests/v1.0.0',
path: '/oci/test-repo/manifests/v1.0.0',
headers: {
Authorization: `Bearer ${ociToken}`,
'Content-Type': 'application/vnd.oci.image.manifest.v1+json',
@@ -143,7 +145,7 @@ tap.test('OCI: should upload manifest (PUT /v2/{name}/manifests/{reference})', a
tap.test('OCI: should retrieve manifest by tag (GET /v2/{name}/manifests/{reference})', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/oci/v2/test-repo/manifests/v1.0.0',
path: '/oci/test-repo/manifests/v1.0.0',
headers: {
Authorization: `Bearer ${ociToken}`,
Accept: 'application/vnd.oci.image.manifest.v1+json',
@@ -152,9 +154,10 @@ tap.test('OCI: should retrieve manifest by tag (GET /v2/{name}/manifests/{refere
});
expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer);
const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
const manifest = JSON.parse((response.body as Buffer).toString('utf-8'));
const manifest = JSON.parse(body.toString('utf-8'));
expect(manifest.schemaVersion).toEqual(2);
expect(manifest.config.digest).toEqual(testConfigDigest);
expect(manifest.layers[0].digest).toEqual(testBlobDigest);
@@ -163,7 +166,7 @@ tap.test('OCI: should retrieve manifest by tag (GET /v2/{name}/manifests/{refere
tap.test('OCI: should retrieve manifest by digest (GET /v2/{name}/manifests/{digest})', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: `/oci/v2/test-repo/manifests/${testManifestDigest}`,
path: `/oci/test-repo/manifests/${testManifestDigest}`,
headers: {
Authorization: `Bearer ${ociToken}`,
Accept: 'application/vnd.oci.image.manifest.v1+json',
@@ -178,7 +181,7 @@ tap.test('OCI: should retrieve manifest by digest (GET /v2/{name}/manifests/{dig
tap.test('OCI: should check if manifest exists (HEAD /v2/{name}/manifests/{reference})', async () => {
const response = await registry.handleRequest({
method: 'HEAD',
path: '/oci/v2/test-repo/manifests/v1.0.0',
path: '/oci/test-repo/manifests/v1.0.0',
headers: {
Authorization: `Bearer ${ociToken}`,
Accept: 'application/vnd.oci.image.manifest.v1+json',
@@ -193,7 +196,7 @@ tap.test('OCI: should check if manifest exists (HEAD /v2/{name}/manifests/{refer
tap.test('OCI: should list tags (GET /v2/{name}/tags/list)', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/oci/v2/test-repo/tags/list',
path: '/oci/test-repo/tags/list',
headers: {
Authorization: `Bearer ${ociToken}`,
},
@@ -201,9 +204,9 @@ tap.test('OCI: should list tags (GET /v2/{name}/tags/list)', async () => {
});
expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('tags');
const tagList = await streamToJson(response.body);
expect(tagList).toHaveProperty('tags');
const tagList = response.body as any;
expect(tagList.name).toEqual('test-repo');
expect(tagList.tags).toBeInstanceOf(Array);
expect(tagList.tags).toContain('v1.0.0');
@@ -212,7 +215,7 @@ tap.test('OCI: should list tags (GET /v2/{name}/tags/list)', async () => {
tap.test('OCI: should handle pagination for tag list', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/oci/v2/test-repo/tags/list',
path: '/oci/test-repo/tags/list',
headers: {
Authorization: `Bearer ${ociToken}`,
},
@@ -222,13 +225,14 @@ tap.test('OCI: should handle pagination for tag list', async () => {
});
expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('tags');
const body = await streamToJson(response.body);
expect(body).toHaveProperty('tags');
});
tap.test('OCI: should return 404 for non-existent blob', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/oci/v2/test-repo/blobs/sha256:0000000000000000000000000000000000000000000000000000000000000000',
path: '/oci/test-repo/blobs/sha256:0000000000000000000000000000000000000000000000000000000000000000',
headers: {
Authorization: `Bearer ${ociToken}`,
},
@@ -236,13 +240,14 @@ tap.test('OCI: should return 404 for non-existent blob', async () => {
});
expect(response.status).toEqual(404);
expect(response.body).toHaveProperty('errors');
const body = await streamToJson(response.body);
expect(body).toHaveProperty('errors');
});
tap.test('OCI: should return 404 for non-existent manifest', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/oci/v2/test-repo/manifests/non-existent-tag',
path: '/oci/test-repo/manifests/non-existent-tag',
headers: {
Authorization: `Bearer ${ociToken}`,
Accept: 'application/vnd.oci.image.manifest.v1+json',
@@ -251,13 +256,14 @@ tap.test('OCI: should return 404 for non-existent manifest', async () => {
});
expect(response.status).toEqual(404);
expect(response.body).toHaveProperty('errors');
const body = await streamToJson(response.body);
expect(body).toHaveProperty('errors');
});
tap.test('OCI: should delete manifest (DELETE /v2/{name}/manifests/{digest})', async () => {
const response = await registry.handleRequest({
method: 'DELETE',
path: `/oci/v2/test-repo/manifests/${testManifestDigest}`,
path: `/oci/test-repo/manifests/${testManifestDigest}`,
headers: {
Authorization: `Bearer ${ociToken}`,
},
@@ -270,7 +276,7 @@ tap.test('OCI: should delete manifest (DELETE /v2/{name}/manifests/{digest})', a
tap.test('OCI: should delete blob (DELETE /v2/{name}/blobs/{digest})', async () => {
const response = await registry.handleRequest({
method: 'DELETE',
path: `/oci/v2/test-repo/blobs/${testBlobDigest}`,
path: `/oci/test-repo/blobs/${testBlobDigest}`,
headers: {
Authorization: `Bearer ${ociToken}`,
},
@@ -283,7 +289,7 @@ tap.test('OCI: should delete blob (DELETE /v2/{name}/blobs/{digest})', async ()
tap.test('OCI: should handle unauthorized requests', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/oci/v2/test-repo/manifests/v1.0.0',
path: '/oci/test-repo/manifests/v1.0.0',
headers: {
// No authorization header
},

View File

@@ -89,16 +89,10 @@ async function createHttpServer(
res.setHeader(key, value);
}
// Send body
// Send body (response.body is always ReadableStream<Uint8Array> or undefined)
if (response.body) {
if (Buffer.isBuffer(response.body)) {
res.end(response.body);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
const { Readable } = await import('stream');
Readable.fromWeb(response.body).pipe(res);
} else {
res.end();
}
@@ -303,16 +297,16 @@ tap.test('PyPI CLI: should verify twine is installed', async () => {
});
tap.test('PyPI CLI: should setup registry and HTTP server', async () => {
// Create registry
registry = await createTestRegistry();
// Use port 39000 (avoids conflicts with other tests)
registryPort = 39000;
// Create registry with correct registryUrl for CLI tests
registry = await createTestRegistry({ registryUrl: `http://localhost:${registryPort}` });
const tokens = await createTestTokens(registry);
pypiToken = tokens.pypiToken;
expect(registry).toBeInstanceOf(SmartRegistry);
expect(pypiToken).toBeTypeOf('string');
// Use port 39000 (avoids conflicts with other tests)
registryPort = 39000;
const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server;
registryUrl = serverSetup.url;

View File

@@ -1,5 +1,6 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js';
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
import {
createTestRegistry,
createTestTokens,
@@ -101,9 +102,10 @@ tap.test('PyPI: should retrieve Simple API root index HTML (GET /simple/)', asyn
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toStartWith('text/html');
expect(response.body).toBeTypeOf('string');
const body = await streamToBuffer(response.body);
const html = body.toString('utf-8');
expect(html).toBeTypeOf('string');
const html = response.body as string;
expect(html).toContain('<!DOCTYPE html>');
expect(html).toContain('<title>Simple Index</title>');
expect(html).toContain(normalizedPackageName);
@@ -121,9 +123,9 @@ tap.test('PyPI: should retrieve Simple API root index JSON (GET /simple/ with Ac
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/vnd.pypi.simple.v1+json');
expect(response.body).toBeTypeOf('object');
const json = await streamToJson(response.body);
expect(json).toBeTypeOf('object');
const json = response.body as any;
expect(json).toHaveProperty('meta');
expect(json).toHaveProperty('projects');
expect(json.projects).toBeInstanceOf(Array);
@@ -144,9 +146,10 @@ tap.test('PyPI: should retrieve Simple API package HTML (GET /simple/{package}/)
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toStartWith('text/html');
expect(response.body).toBeTypeOf('string');
const body = await streamToBuffer(response.body);
const html = body.toString('utf-8');
expect(html).toBeTypeOf('string');
const html = response.body as string;
expect(html).toContain('<!DOCTYPE html>');
expect(html).toContain(`<title>Links for ${normalizedPackageName}</title>`);
expect(html).toContain('.whl');
@@ -165,9 +168,9 @@ tap.test('PyPI: should retrieve Simple API package JSON (GET /simple/{package}/
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/vnd.pypi.simple.v1+json');
expect(response.body).toBeTypeOf('object');
const json = await streamToJson(response.body);
expect(json).toBeTypeOf('object');
const json = response.body as any;
expect(json).toHaveProperty('meta');
expect(json).toHaveProperty('name');
expect(json.name).toEqual(normalizedPackageName);
@@ -187,8 +190,9 @@ tap.test('PyPI: should download wheel file (GET /pypi/packages/{package}/{filena
});
expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer);
expect((response.body as Buffer).length).toEqual(testWheelData.length);
const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
expect(body.length).toEqual(testWheelData.length);
expect(response.headers['Content-Type']).toEqual('application/octet-stream');
});
@@ -234,7 +238,7 @@ tap.test('PyPI: should list both wheel and sdist in Simple API', async () => {
expect(response.status).toEqual(200);
const json = response.body as any;
const json = await streamToJson(response.body);
// PEP 691: files is an array of file objects
expect(json.files.length).toEqual(2);
@@ -289,7 +293,7 @@ tap.test('PyPI: should list multiple versions in Simple API', async () => {
expect(response.status).toEqual(200);
const json = response.body as any;
const json = await streamToJson(response.body);
// PEP 691: files is an array of file objects
expect(json.files.length).toBeGreaterThan(2);
@@ -323,7 +327,8 @@ tap.test('PyPI: should return 404 for non-existent package', async () => {
});
expect(response.status).toEqual(404);
expect(response.body).toHaveProperty('error');
const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
});
tap.test('PyPI: should return 401 for unauthorized upload', async () => {
@@ -353,7 +358,8 @@ tap.test('PyPI: should return 401 for unauthorized upload', async () => {
});
expect(response.status).toEqual(401);
expect(response.body).toHaveProperty('error');
const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
});
tap.test('PyPI: should reject upload with mismatched hash', async () => {
@@ -382,7 +388,8 @@ tap.test('PyPI: should reject upload with mismatched hash', async () => {
});
expect(response.status).toEqual(400);
expect(response.body).toHaveProperty('error');
const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
});
tap.test('PyPI: should handle package with requires-python metadata', async () => {
@@ -425,7 +432,8 @@ tap.test('PyPI: should handle package with requires-python metadata', async () =
query: {},
});
const html = getResponse.body as string;
const getBody = await streamToBuffer(getResponse.body);
const html = getBody.toString('utf-8');
expect(html).toContain('data-requires-python');
// Note: >= gets HTML-escaped to &gt;= in attribute values
expect(html).toContain('&gt;=3.8');
@@ -441,9 +449,9 @@ tap.test('PyPI: should support JSON API for package metadata', async () => {
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/json');
expect(response.body).toBeTypeOf('object');
const json = await streamToJson(response.body);
expect(json).toBeTypeOf('object');
const json = response.body as any;
expect(json).toHaveProperty('info');
expect(json.info).toHaveProperty('name');
expect(json.info.name).toEqual(normalizedPackageName);
@@ -460,9 +468,9 @@ tap.test('PyPI: should support JSON API for specific version', async () => {
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/json');
expect(response.body).toBeTypeOf('object');
const json = await streamToJson(response.body);
expect(json).toBeTypeOf('object');
const json = response.body as any;
expect(json).toHaveProperty('info');
expect(json.info.version).toEqual(testVersion);
expect(json).toHaveProperty('urls');

View File

@@ -79,16 +79,10 @@ async function createHttpServer(
res.setHeader(key, value);
}
// Send body
// Send body (response.body is always ReadableStream<Uint8Array> or undefined)
if (response.body) {
if (Buffer.isBuffer(response.body)) {
res.end(response.body);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
const { Readable } = await import('stream');
Readable.fromWeb(response.body).pipe(res);
} else {
res.end();
}
@@ -154,11 +148,16 @@ async function runGemCommand(
cwd: string,
includeAuth: boolean = true
): Promise<{ stdout: string; stderr: string; exitCode: number }> {
// When not including auth, use a temp HOME without credentials
const effectiveHome = includeAuth ? gemHome : path.join(gemHome, 'noauth');
if (!includeAuth) {
fs.mkdirSync(effectiveHome, { recursive: true });
}
// Prepare environment variables
const envVars = [
`HOME="${gemHome}"`,
`HOME="${effectiveHome}"`,
`GEM_HOME="${gemHome}"`,
includeAuth ? '' : 'RUBYGEMS_API_KEY=""',
].filter(Boolean).join(' ');
// Build command with cd to correct directory and environment variables
@@ -194,16 +193,16 @@ function cleanupTestDir(dir: string): void {
// ========================================================================
tap.test('RubyGems CLI: should setup registry and HTTP server', async () => {
// Create registry
registry = await createTestRegistry();
// Use port 36000 (avoids npm:35000, cargo:5000 conflicts)
registryPort = 36000;
// Create registry with correct registryUrl for CLI tests
registry = await createTestRegistry({ registryUrl: `http://localhost:${registryPort}` });
const tokens = await createTestTokens(registry);
rubygemsToken = tokens.rubygemsToken;
expect(registry).toBeInstanceOf(SmartRegistry);
expect(rubygemsToken).toBeTypeOf('string');
// Use port 36000 (avoids npm:35000, cargo:5000 conflicts)
registryPort = 36000;
const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server;
registryUrl = serverSetup.url;
@@ -366,31 +365,33 @@ tap.test('RubyGems CLI: should unyank a version', async () => {
const gemName = 'test-gem-cli';
const version = '1.0.0';
const result = await runGemCommand(
`gem yank ${gemName} -v ${version} --undo --host ${registryUrl}/rubygems`,
testDir
// Use PUT /api/v1/gems/unyank via HTTP API (gem yank --undo removed in Ruby 4.0)
const response = await fetch(
`${registryUrl}/rubygems/api/v1/gems/unyank?gem_name=${gemName}&version=${version}`,
{
method: 'PUT',
headers: {
'Authorization': rubygemsToken,
},
}
);
console.log('gem unyank output:', result.stdout);
console.log('gem unyank stderr:', result.stderr);
console.log('gem unyank status:', response.status);
expect(result.exitCode).toEqual(0);
expect(response.status).toEqual(200);
// Verify version is not yanked in /versions file
const response = await fetch(`${registryUrl}/rubygems/versions`);
const versionsData = await response.text();
const versionsResponse = await fetch(`${registryUrl}/rubygems/versions`);
const versionsData = await versionsResponse.text();
console.log('Versions after unyank:', versionsData);
// Should not have '-' prefix anymore (or have both without prefix)
// Check that we have the version without yank marker
// Should not have '-' prefix anymore
const lines = versionsData.trim().split('\n');
const gemLine = lines.find(line => line.startsWith(gemName));
if (gemLine) {
// Parse format: "gemname version[,version...] md5"
const parts = gemLine.split(' ');
const versions = parts[1];
// Should have 1.0.0 without '-' prefix
expect(versions).toContain('1.0.0');
expect(versions).not.toContain('-1.0.0');
}

View File

@@ -1,5 +1,6 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js';
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
import {
createTestRegistry,
createTestTokens,
@@ -54,7 +55,8 @@ tap.test('RubyGems: should upload gem file (POST /rubygems/api/v1/gems)', async
});
expect(response.status).toEqual(201);
expect(response.body).toHaveProperty('message');
const body = await streamToJson(response.body);
expect(body).toHaveProperty('message');
});
tap.test('RubyGems: should retrieve Compact Index versions file (GET /rubygems/versions)', async () => {
@@ -67,9 +69,10 @@ tap.test('RubyGems: should retrieve Compact Index versions file (GET /rubygems/v
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('text/plain; charset=utf-8');
expect(response.body).toBeInstanceOf(Buffer);
const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
const content = (response.body as Buffer).toString('utf-8');
const content = body.toString('utf-8');
expect(content).toContain('created_at:');
expect(content).toContain('---');
expect(content).toContain(testGemName);
@@ -86,9 +89,10 @@ tap.test('RubyGems: should retrieve Compact Index info file (GET /rubygems/info/
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('text/plain; charset=utf-8');
expect(response.body).toBeInstanceOf(Buffer);
const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
const content = (response.body as Buffer).toString('utf-8');
const content = body.toString('utf-8');
expect(content).toContain('---');
expect(content).toContain(testVersion);
expect(content).toContain('checksum:');
@@ -104,9 +108,10 @@ tap.test('RubyGems: should retrieve Compact Index names file (GET /rubygems/name
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('text/plain; charset=utf-8');
expect(response.body).toBeInstanceOf(Buffer);
const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
const content = (response.body as Buffer).toString('utf-8');
const content = body.toString('utf-8');
expect(content).toContain('---');
expect(content).toContain(testGemName);
});
@@ -120,8 +125,9 @@ tap.test('RubyGems: should download gem file (GET /rubygems/gems/{gem}-{version}
});
expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer);
expect((response.body as Buffer).length).toEqual(testGemData.length);
const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
expect(body.length).toEqual(testGemData.length);
expect(response.headers['Content-Type']).toEqual('application/octet-stream');
});
@@ -153,7 +159,8 @@ tap.test('RubyGems: should list multiple versions in Compact Index', async () =>
expect(response.status).toEqual(200);
const content = (response.body as Buffer).toString('utf-8');
const body = await streamToBuffer(response.body);
const content = body.toString('utf-8');
const lines = content.split('\n');
const gemLine = lines.find(l => l.startsWith(`${testGemName} `));
@@ -172,7 +179,8 @@ tap.test('RubyGems: should list multiple versions in info file', async () => {
expect(response.status).toEqual(200);
const content = (response.body as Buffer).toString('utf-8');
const body = await streamToBuffer(response.body);
const content = body.toString('utf-8');
expect(content).toContain('1.0.0');
expect(content).toContain('2.0.0');
});
@@ -203,7 +211,8 @@ tap.test('RubyGems: should support platform-specific gems', async () => {
query: {},
});
const content = (versionsResponse.body as Buffer).toString('utf-8');
const versionsBody = await streamToBuffer(versionsResponse.body);
const content = versionsBody.toString('utf-8');
const lines = content.split('\n');
const gemLine = lines.find(l => l.startsWith(`${testGemName} `));
@@ -224,8 +233,9 @@ tap.test('RubyGems: should yank a gem version (DELETE /rubygems/api/v1/gems/yank
});
expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('message');
expect((response.body as any).message).toContain('yanked');
const body = await streamToJson(response.body);
expect(body).toHaveProperty('message');
expect(body.message).toContain('yanked');
});
tap.test('RubyGems: should mark yanked version in Compact Index', async () => {
@@ -238,7 +248,8 @@ tap.test('RubyGems: should mark yanked version in Compact Index', async () => {
expect(response.status).toEqual(200);
const content = (response.body as Buffer).toString('utf-8');
const body = await streamToBuffer(response.body);
const content = body.toString('utf-8');
const lines = content.split('\n');
const gemLine = lines.find(l => l.startsWith(`${testGemName} `));
@@ -256,7 +267,8 @@ tap.test('RubyGems: should still allow downloading yanked gem', async () => {
});
expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer);
const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
});
tap.test('RubyGems: should unyank a gem version (PUT /rubygems/api/v1/gems/unyank)', async () => {
@@ -273,8 +285,9 @@ tap.test('RubyGems: should unyank a gem version (PUT /rubygems/api/v1/gems/unyan
});
expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('message');
expect((response.body as any).message).toContain('unyanked');
const body = await streamToJson(response.body);
expect(body).toHaveProperty('message');
expect(body.message).toContain('unyanked');
});
tap.test('RubyGems: should remove yank marker after unyank', async () => {
@@ -287,7 +300,8 @@ tap.test('RubyGems: should remove yank marker after unyank', async () => {
expect(response.status).toEqual(200);
const content = (response.body as Buffer).toString('utf-8');
const body = await streamToBuffer(response.body);
const content = body.toString('utf-8');
const lines = content.split('\n');
const gemLine = lines.find(l => l.startsWith(`${testGemName} `));
@@ -309,14 +323,10 @@ tap.test('RubyGems: should retrieve versions JSON (GET /rubygems/api/v1/versions
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/json');
expect(response.body).toBeTypeOf('object');
const json = response.body as any;
expect(json).toHaveProperty('name');
expect(json.name).toEqual(testGemName);
expect(json).toHaveProperty('versions');
expect(json.versions).toBeTypeOf('object');
expect(json.versions.length).toBeGreaterThan(0);
const json = await streamToJson(response.body);
expect(json).toBeInstanceOf(Array);
expect(json.length).toBeGreaterThan(0);
expect(json[0]).toHaveProperty('number');
});
tap.test('RubyGems: should retrieve dependencies JSON (GET /rubygems/api/v1/dependencies)', async () => {
@@ -331,9 +341,9 @@ tap.test('RubyGems: should retrieve dependencies JSON (GET /rubygems/api/v1/depe
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/json');
expect(response.body).toBeTypeOf('object');
const json = await streamToJson(response.body);
expect(json).toBeTypeOf('object');
const json = response.body as any;
expect(Array.isArray(json)).toEqual(true);
});
@@ -346,7 +356,8 @@ tap.test('RubyGems: should retrieve gem spec (GET /rubygems/quick/Marshal.4.8/{g
});
expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer);
const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
});
tap.test('RubyGems: should support latest specs endpoint (GET /rubygems/latest_specs.4.8.gz)', async () => {
@@ -359,7 +370,8 @@ tap.test('RubyGems: should support latest specs endpoint (GET /rubygems/latest_s
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/octet-stream');
expect(response.body).toBeInstanceOf(Buffer);
const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
});
tap.test('RubyGems: should support specs endpoint (GET /rubygems/specs.4.8.gz)', async () => {
@@ -372,7 +384,8 @@ tap.test('RubyGems: should support specs endpoint (GET /rubygems/specs.4.8.gz)',
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/octet-stream');
expect(response.body).toBeInstanceOf(Buffer);
const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
});
tap.test('RubyGems: should return 404 for non-existent gem', async () => {
@@ -384,7 +397,8 @@ tap.test('RubyGems: should return 404 for non-existent gem', async () => {
});
expect(response.status).toEqual(404);
expect(response.body).toHaveProperty('error');
const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
});
tap.test('RubyGems: should return 401 for unauthorized upload', async () => {
@@ -402,7 +416,8 @@ tap.test('RubyGems: should return 401 for unauthorized upload', async () => {
});
expect(response.status).toEqual(401);
expect(response.body).toHaveProperty('error');
const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
});
tap.test('RubyGems: should return 401 for unauthorized yank', async () => {
@@ -419,7 +434,8 @@ tap.test('RubyGems: should return 401 for unauthorized yank', async () => {
});
expect(response.status).toEqual(401);
expect(response.body).toHaveProperty('error');
const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
});
tap.test('RubyGems: should handle gem with dependencies', async () => {
@@ -450,7 +466,8 @@ tap.test('RubyGems: should handle gem with dependencies', async () => {
expect(infoResponse.status).toEqual(200);
const content = (infoResponse.body as Buffer).toString('utf-8');
const infoBody = await streamToBuffer(infoResponse.body);
const content = infoBody.toString('utf-8');
expect(content).toContain('checksum:');
});

View File

@@ -1,5 +1,6 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js';
import { streamToJson } from '../ts/core/helpers.stream.js';
import { createTestRegistry, createTestTokens } from './helpers/registry.js';
let registry: SmartRegistry;
@@ -54,8 +55,9 @@ tap.test('Integration: should return 404 for unknown paths', async () => {
});
expect(response.status).toEqual(404);
expect(response.body).toHaveProperty('error');
expect((response.body as any).error).toEqual('NOT_FOUND');
const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
expect(body.error).toEqual('NOT_FOUND');
});
tap.test('Integration: should create and validate tokens', async () => {

View File

@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@push.rocks/smartregistry',
version: '2.7.0',
version: '2.8.2',
description: 'A composable TypeScript library implementing OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems registries for building unified container and package registries'
}

View File

@@ -370,7 +370,7 @@ export class CargoRegistry extends BaseRegistry {
const parsed = this.parsePublishRequest(body);
metadata = parsed.metadata;
crateFile = parsed.crateFile;
} catch (error) {
} catch (error: any) {
this.logger.log('error', 'handlePublish: parse error', { error: error.message });
return {
status: 400,
@@ -467,17 +467,29 @@ export class CargoRegistry extends BaseRegistry {
): Promise<IResponse> {
this.logger.log('debug', 'handleDownload', { crate: crateName, version });
let crateFile = await this.storage.getCargoCrate(crateName, version);
// Try streaming from local storage first
const streamResult = await this.storage.getCargoCrateStream(crateName, version);
if (streamResult) {
return {
status: 200,
headers: {
'Content-Type': 'application/gzip',
'Content-Length': streamResult.size.toString(),
'Content-Disposition': `attachment; filename="${crateName}-${version}.crate"`,
},
body: streamResult.stream,
};
}
// Try upstream if not found locally
if (!crateFile) {
const upstream = await this.getUpstreamForRequest(crateName, 'crate', 'GET', actor);
if (upstream) {
crateFile = await upstream.fetchCrate(crateName, version);
if (crateFile) {
// Cache locally
await this.storage.putCargoCrate(crateName, version, crateFile);
}
let crateFile: Buffer | null = null;
const upstream = await this.getUpstreamForRequest(crateName, 'crate', 'GET', actor);
if (upstream) {
crateFile = await upstream.fetchCrate(crateName, version);
if (crateFile) {
// Cache locally
await this.storage.putCargoCrate(crateName, version, crateFile);
}
}
@@ -647,7 +659,7 @@ export class CargoRegistry extends BaseRegistry {
}
}
}
} catch (error) {
} catch (error: any) {
this.logger.log('error', 'handleSearch: error', { error: error.message });
}

View File

@@ -2,6 +2,7 @@ import { RegistryStorage } from './core/classes.registrystorage.js';
import { AuthManager } from './core/classes.authmanager.js';
import { BaseRegistry } from './core/classes.baseregistry.js';
import type { IRegistryConfig, IRequestContext, IResponse } from './core/interfaces.core.js';
import { toReadableStream } from './core/helpers.stream.js';
import { OciRegistry } from './oci/classes.ociregistry.js';
import { NpmRegistry } from './npm/classes.npmregistry.js';
import { MavenRegistry } from './maven/classes.mavenregistry.js';
@@ -95,7 +96,7 @@ export class SmartRegistry {
// Initialize NPM registry if enabled
if (this.config.npm?.enabled) {
const npmBasePath = this.config.npm.basePath ?? '/npm';
const registryUrl = `http://localhost:5000${npmBasePath}`; // TODO: Make configurable
const registryUrl = this.config.npm.registryUrl ?? `http://localhost:5000${npmBasePath}`;
const npmRegistry = new NpmRegistry(
this.storage,
this.authManager,
@@ -110,7 +111,7 @@ export class SmartRegistry {
// Initialize Maven registry if enabled
if (this.config.maven?.enabled) {
const mavenBasePath = this.config.maven.basePath ?? '/maven';
const registryUrl = `http://localhost:5000${mavenBasePath}`; // TODO: Make configurable
const registryUrl = this.config.maven.registryUrl ?? `http://localhost:5000${mavenBasePath}`;
const mavenRegistry = new MavenRegistry(
this.storage,
this.authManager,
@@ -125,7 +126,7 @@ export class SmartRegistry {
// Initialize Cargo registry if enabled
if (this.config.cargo?.enabled) {
const cargoBasePath = this.config.cargo.basePath ?? '/cargo';
const registryUrl = `http://localhost:5000${cargoBasePath}`; // TODO: Make configurable
const registryUrl = this.config.cargo.registryUrl ?? `http://localhost:5000${cargoBasePath}`;
const cargoRegistry = new CargoRegistry(
this.storage,
this.authManager,
@@ -140,7 +141,7 @@ export class SmartRegistry {
// Initialize Composer registry if enabled
if (this.config.composer?.enabled) {
const composerBasePath = this.config.composer.basePath ?? '/composer';
const registryUrl = `http://localhost:5000${composerBasePath}`; // TODO: Make configurable
const registryUrl = this.config.composer.registryUrl ?? `http://localhost:5000${composerBasePath}`;
const composerRegistry = new ComposerRegistry(
this.storage,
this.authManager,
@@ -155,7 +156,7 @@ export class SmartRegistry {
// Initialize PyPI registry if enabled
if (this.config.pypi?.enabled) {
const pypiBasePath = this.config.pypi.basePath ?? '/pypi';
const registryUrl = `http://localhost:5000`; // TODO: Make configurable
const registryUrl = this.config.pypi.registryUrl ?? `http://localhost:5000`;
const pypiRegistry = new PypiRegistry(
this.storage,
this.authManager,
@@ -170,7 +171,7 @@ export class SmartRegistry {
// Initialize RubyGems registry if enabled
if (this.config.rubygems?.enabled) {
const rubygemsBasePath = this.config.rubygems.basePath ?? '/rubygems';
const registryUrl = `http://localhost:5000${rubygemsBasePath}`; // TODO: Make configurable
const registryUrl = this.config.rubygems.registryUrl ?? `http://localhost:5000${rubygemsBasePath}`;
const rubygemsRegistry = new RubyGemsRegistry(
this.storage,
this.authManager,
@@ -191,75 +192,88 @@ export class SmartRegistry {
*/
public async handleRequest(context: IRequestContext): Promise<IResponse> {
const path = context.path;
let response: IResponse | undefined;
// Route to OCI registry
if (this.config.oci?.enabled && path.startsWith(this.config.oci.basePath)) {
if (!response && this.config.oci?.enabled && path.startsWith(this.config.oci.basePath)) {
const ociRegistry = this.registries.get('oci');
if (ociRegistry) {
return ociRegistry.handleRequest(context);
response = await ociRegistry.handleRequest(context);
}
}
// Route to NPM registry
if (this.config.npm?.enabled && path.startsWith(this.config.npm.basePath)) {
if (!response && this.config.npm?.enabled && path.startsWith(this.config.npm.basePath)) {
const npmRegistry = this.registries.get('npm');
if (npmRegistry) {
return npmRegistry.handleRequest(context);
response = await npmRegistry.handleRequest(context);
}
}
// Route to Maven registry
if (this.config.maven?.enabled && path.startsWith(this.config.maven.basePath)) {
if (!response && this.config.maven?.enabled && path.startsWith(this.config.maven.basePath)) {
const mavenRegistry = this.registries.get('maven');
if (mavenRegistry) {
return mavenRegistry.handleRequest(context);
response = await mavenRegistry.handleRequest(context);
}
}
// Route to Cargo registry
if (this.config.cargo?.enabled && path.startsWith(this.config.cargo.basePath)) {
if (!response && this.config.cargo?.enabled && path.startsWith(this.config.cargo.basePath)) {
const cargoRegistry = this.registries.get('cargo');
if (cargoRegistry) {
return cargoRegistry.handleRequest(context);
response = await cargoRegistry.handleRequest(context);
}
}
// Route to Composer registry
if (this.config.composer?.enabled && path.startsWith(this.config.composer.basePath)) {
if (!response && this.config.composer?.enabled && path.startsWith(this.config.composer.basePath)) {
const composerRegistry = this.registries.get('composer');
if (composerRegistry) {
return composerRegistry.handleRequest(context);
response = await composerRegistry.handleRequest(context);
}
}
// Route to PyPI registry (also handles /simple prefix)
if (this.config.pypi?.enabled) {
if (!response && this.config.pypi?.enabled) {
const pypiBasePath = this.config.pypi.basePath ?? '/pypi';
if (path.startsWith(pypiBasePath) || path.startsWith('/simple')) {
const pypiRegistry = this.registries.get('pypi');
if (pypiRegistry) {
return pypiRegistry.handleRequest(context);
response = await pypiRegistry.handleRequest(context);
}
}
}
// Route to RubyGems registry
if (this.config.rubygems?.enabled && path.startsWith(this.config.rubygems.basePath)) {
if (!response && this.config.rubygems?.enabled && path.startsWith(this.config.rubygems.basePath)) {
const rubygemsRegistry = this.registries.get('rubygems');
if (rubygemsRegistry) {
return rubygemsRegistry.handleRequest(context);
response = await rubygemsRegistry.handleRequest(context);
}
}
// No matching registry
return {
status: 404,
headers: { 'Content-Type': 'application/json' },
body: {
error: 'NOT_FOUND',
message: 'No registry handler for this path',
},
};
if (!response) {
response = {
status: 404,
headers: { 'Content-Type': 'application/json' },
body: {
error: 'NOT_FOUND',
message: 'No registry handler for this path',
},
};
}
// Normalize body to ReadableStream<Uint8Array> at the API boundary
if (response.body != null && !(response.body instanceof ReadableStream)) {
if (!Buffer.isBuffer(response.body) && typeof response.body === 'object' && !(response.body instanceof Uint8Array)) {
response.headers['Content-Type'] ??= 'application/json';
}
response.body = toReadableStream(response.body);
}
return response;
}
/**

View File

@@ -302,9 +302,9 @@ export class ComposerRegistry extends BaseRegistry {
token: IAuthToken | null
): Promise<IResponse> {
// Read operations are public, no authentication required
const zipData = await this.storage.getComposerPackageZip(vendorPackage, reference);
const streamResult = await this.storage.getComposerPackageZipStream(vendorPackage, reference);
if (!zipData) {
if (!streamResult) {
return {
status: 404,
headers: {},
@@ -316,10 +316,10 @@ export class ComposerRegistry extends BaseRegistry {
status: 200,
headers: {
'Content-Type': 'application/zip',
'Content-Length': zipData.length.toString(),
'Content-Length': streamResult.size.toString(),
'Content-Disposition': `attachment; filename="${reference}.zip"`,
},
body: zipData,
body: streamResult.stream,
};
}

View File

@@ -34,8 +34,8 @@ import type {
* ```
*/
export class RegistryStorage implements IStorageBackend {
private smartBucket: plugins.smartbucket.SmartBucket;
private bucket: plugins.smartbucket.Bucket;
private smartBucket!: plugins.smartbucket.SmartBucket;
private bucket!: plugins.smartbucket.Bucket;
private bucketName: string;
private hooks?: IStorageHooks;
@@ -1266,4 +1266,135 @@ export class RegistryStorage implements IStorageBackend {
private getRubyGemsMetadataPath(gemName: string): string {
return `rubygems/metadata/${gemName}/metadata.json`;
}
// ========================================================================
// STREAMING METHODS (Web Streams API)
// ========================================================================
/**
* Get an object as a ReadableStream. Returns null if not found.
*/
public async getObjectStream(key: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null> {
try {
const stat = await this.bucket.fastStat({ path: key });
const size = stat.ContentLength ?? 0;
const stream = await this.bucket.fastGetStream({ path: key }, 'webstream');
// Call afterGet hook (non-blocking)
if (this.hooks?.afterGet) {
const context = this.currentContext;
if (context) {
this.hooks.afterGet({
operation: 'get',
key,
protocol: context.protocol,
actor: context.actor,
metadata: context.metadata,
timestamp: new Date(),
}).catch(() => {});
}
}
return { stream: stream as ReadableStream<Uint8Array>, size };
} catch {
return null;
}
}
/**
* Store an object from a ReadableStream.
*/
public async putObjectStream(key: string, stream: ReadableStream<Uint8Array>): Promise<void> {
if (this.hooks?.beforePut) {
const context = this.currentContext;
if (context) {
const hookContext: IStorageHookContext = {
operation: 'put',
key,
protocol: context.protocol,
actor: context.actor,
metadata: context.metadata,
timestamp: new Date(),
};
const result = await this.hooks.beforePut(hookContext);
if (!result.allowed) {
throw new Error(result.reason || 'Storage operation denied by hook');
}
}
}
// Convert WebStream to Node Readable at the S3 SDK boundary
// AWS SDK v3 PutObjectCommand requires a Node.js Readable (not WebStream)
const { Readable } = await import('stream');
const nodeStream = Readable.fromWeb(stream as any);
await this.bucket.fastPutStream({
path: key,
readableStream: nodeStream,
overwrite: true,
});
if (this.hooks?.afterPut) {
const context = this.currentContext;
if (context) {
this.hooks.afterPut({
operation: 'put',
key,
protocol: context.protocol,
actor: context.actor,
metadata: context.metadata,
timestamp: new Date(),
}).catch(() => {});
}
}
}
/**
* Get object size without reading data (S3 HEAD request).
*/
public async getObjectSize(key: string): Promise<number | null> {
try {
const stat = await this.bucket.fastStat({ path: key });
return stat.ContentLength ?? null;
} catch {
return null;
}
}
// ---- Protocol-specific streaming wrappers ----
public async getOciBlobStream(digest: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null> {
return this.getObjectStream(this.getOciBlobPath(digest));
}
public async putOciBlobStream(digest: string, stream: ReadableStream<Uint8Array>): Promise<void> {
return this.putObjectStream(this.getOciBlobPath(digest), stream);
}
public async getOciBlobSize(digest: string): Promise<number | null> {
return this.getObjectSize(this.getOciBlobPath(digest));
}
public async getNpmTarballStream(packageName: string, version: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null> {
return this.getObjectStream(this.getNpmTarballPath(packageName, version));
}
public async getMavenArtifactStream(groupId: string, artifactId: string, version: string, filename: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null> {
return this.getObjectStream(this.getMavenArtifactPath(groupId, artifactId, version, filename));
}
public async getCargoCrateStream(crateName: string, version: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null> {
return this.getObjectStream(this.getCargoCratePath(crateName, version));
}
public async getComposerPackageZipStream(vendorPackage: string, reference: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null> {
return this.getObjectStream(this.getComposerZipPath(vendorPackage, reference));
}
public async getPypiPackageFileStream(packageName: string, filename: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null> {
return this.getObjectStream(this.getPypiPackageFilePath(packageName, filename));
}
public async getRubyGemsGemStream(gemName: string, version: string, platform?: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null> {
return this.getObjectStream(this.getRubyGemsGemPath(gemName, version, platform));
}
}

63
ts/core/helpers.stream.ts Normal file
View File

@@ -0,0 +1,63 @@
import * as crypto from 'crypto';
/**
* Convert Buffer, Uint8Array, string, or JSON object to a ReadableStream<Uint8Array>.
*/
export function toReadableStream(data: Buffer | Uint8Array | string | object): ReadableStream<Uint8Array> {
const buf = Buffer.isBuffer(data)
? data
: data instanceof Uint8Array
? Buffer.from(data)
: typeof data === 'string'
? Buffer.from(data, 'utf-8')
: Buffer.from(JSON.stringify(data), 'utf-8');
return new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(new Uint8Array(buf));
controller.close();
},
});
}
/**
* Consume a ReadableStream into a Buffer.
*/
export async function streamToBuffer(stream: ReadableStream<Uint8Array>): Promise<Buffer> {
const reader = stream.getReader();
const chunks: Uint8Array[] = [];
while (true) {
const { done, value } = await reader.read();
if (done) break;
if (value) chunks.push(value);
}
return Buffer.concat(chunks);
}
/**
* Consume a ReadableStream into a parsed JSON object.
*/
export async function streamToJson<T = any>(stream: ReadableStream<Uint8Array>): Promise<T> {
const buf = await streamToBuffer(stream);
return JSON.parse(buf.toString('utf-8'));
}
/**
* Create a TransformStream that incrementally hashes data passing through.
* Data flows through unchanged; the digest is available after the stream completes.
*/
export function createHashTransform(algorithm: string = 'sha256'): {
transform: TransformStream<Uint8Array, Uint8Array>;
getDigest: () => string;
} {
const hash = crypto.createHash(algorithm);
const transform = new TransformStream<Uint8Array, Uint8Array>({
transform(chunk, controller) {
hash.update(chunk);
controller.enqueue(chunk);
},
});
return {
transform,
getDigest: () => hash.digest('hex'),
};
}

View File

@@ -12,6 +12,9 @@ export { DefaultAuthProvider } from './classes.defaultauthprovider.js';
// Storage interfaces and hooks
export * from './interfaces.storage.js';
// Stream helpers
export { toReadableStream, streamToBuffer, streamToJson, createHashTransform } from './helpers.stream.js';
// Classes
export { BaseRegistry } from './classes.baseregistry.js';
export { RegistryStorage } from './classes.registrystorage.js';

View File

@@ -88,6 +88,7 @@ export interface IAuthConfig {
export interface IProtocolConfig {
enabled: boolean;
basePath: string;
registryUrl?: string;
features?: Record<string, boolean>;
}
@@ -160,6 +161,21 @@ export interface IStorageBackend {
* Get object metadata
*/
getMetadata(key: string): Promise<Record<string, string> | null>;
/**
* Get an object as a ReadableStream. Returns null if not found.
*/
getObjectStream?(key: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null>;
/**
* Store an object from a ReadableStream.
*/
putObjectStream?(key: string, stream: ReadableStream<Uint8Array>): Promise<void>;
/**
* Get object size without reading data (S3 HEAD request).
*/
getObjectSize?(key: string): Promise<number | null>;
}
/**
@@ -215,10 +231,13 @@ export interface IRequestContext {
}
/**
* Base response structure
* Base response structure.
* `body` is always a `ReadableStream<Uint8Array>` at the public API boundary.
* Internal handlers may return Buffer/string/object — the SmartRegistry orchestrator
* auto-wraps them via `toReadableStream()` before returning to the caller.
*/
export interface IResponse {
status: number;
headers: Record<string, string>;
body?: any;
body?: ReadableStream<Uint8Array> | any;
}

View File

@@ -110,9 +110,17 @@ export class MavenRegistry extends BaseRegistry {
let token: IAuthToken | null = null;
if (authHeader) {
const tokenString = authHeader.replace(/^(Bearer|Basic)\s+/i, '');
// For now, try to validate as Maven token (reuse npm token type)
token = await this.authManager.validateToken(tokenString, 'maven');
if (/^Basic\s+/i.test(authHeader)) {
// Maven sends Basic Auth: base64(username:password) — extract the password as token
const base64 = authHeader.replace(/^Basic\s+/i, '');
const decoded = Buffer.from(base64, 'base64').toString('utf-8');
const colonIndex = decoded.indexOf(':');
const password = colonIndex >= 0 ? decoded.substring(colonIndex + 1) : decoded;
token = await this.authManager.validateToken(password, 'maven');
} else {
const tokenString = authHeader.replace(/^Bearer\s+/i, '');
token = await this.authManager.validateToken(tokenString, 'maven');
}
}
// Build actor from context and validated token
@@ -240,9 +248,19 @@ export class MavenRegistry extends BaseRegistry {
return this.getChecksum(groupId, artifactId, version, coordinate, path);
}
// Accept PUT silently — Maven deploy-plugin uploads checksums alongside artifacts,
// but our registry auto-generates them, so we just acknowledge the upload
if (method === 'PUT') {
return {
status: 200,
headers: {},
body: { status: 'ok' },
};
}
return {
status: 405,
headers: { 'Allow': 'GET, HEAD' },
headers: { 'Allow': 'GET, HEAD, PUT' },
body: { error: 'METHOD_NOT_ALLOWED', message: 'Checksums are auto-generated' },
};
}
@@ -275,9 +293,19 @@ export class MavenRegistry extends BaseRegistry {
return this.getMetadata(groupId, artifactId, actor);
}
// Accept PUT silently — Maven deploy-plugin uploads maven-metadata.xml,
// but our registry auto-generates it, so we just acknowledge the upload
if (method === 'PUT') {
return {
status: 200,
headers: {},
body: { status: 'ok' },
};
}
return {
status: 405,
headers: { 'Allow': 'GET' },
headers: { 'Allow': 'GET, PUT' },
body: { error: 'METHOD_NOT_ALLOWED', message: 'Metadata is auto-generated' },
};
}
@@ -293,24 +321,36 @@ export class MavenRegistry extends BaseRegistry {
filename: string,
actor?: IRequestActor
): Promise<IResponse> {
let data = await this.storage.getMavenArtifact(groupId, artifactId, version, filename);
// Try local storage first (streaming)
const streamResult = await this.storage.getMavenArtifactStream(groupId, artifactId, version, filename);
if (streamResult) {
const ext = filename.split('.').pop() || '';
const contentType = this.getContentType(ext);
return {
status: 200,
headers: {
'Content-Type': contentType,
'Content-Length': streamResult.size.toString(),
},
body: streamResult.stream,
};
}
// Try upstream if not found locally
if (!data) {
const resource = `${groupId}:${artifactId}`;
const upstream = await this.getUpstreamForRequest(resource, 'artifact', 'GET', actor);
if (upstream) {
// Parse the filename to extract extension and classifier
const { extension, classifier } = this.parseFilename(filename, artifactId, version);
if (extension) {
data = await upstream.fetchArtifact(groupId, artifactId, version, extension, classifier);
if (data) {
// Cache the artifact locally
await this.storage.putMavenArtifact(groupId, artifactId, version, filename, data);
// Generate and store checksums
const checksums = await calculateChecksums(data);
await this.storeChecksums(groupId, artifactId, version, filename, checksums);
}
let data: Buffer | null = null;
const resource = `${groupId}:${artifactId}`;
const upstream = await this.getUpstreamForRequest(resource, 'artifact', 'GET', actor);
if (upstream) {
// Parse the filename to extract extension and classifier
const { extension, classifier } = this.parseFilename(filename, artifactId, version);
if (extension) {
data = await upstream.fetchArtifact(groupId, artifactId, version, extension, classifier);
if (data) {
// Cache the artifact locally
await this.storage.putMavenArtifact(groupId, artifactId, version, filename, data);
// Generate and store checksums
const checksums = await calculateChecksums(data);
await this.storeChecksums(groupId, artifactId, version, filename, checksums);
}
}
}

View File

@@ -155,45 +155,45 @@ export class NpmRegistry extends BaseRegistry {
// Dist-tags: /-/package/{package}/dist-tags
const distTagsMatch = path.match(/^\/-\/package\/(@?[^\/]+(?:\/[^\/]+)?)\/dist-tags(?:\/(.+))?$/);
if (distTagsMatch) {
const [, packageName, tag] = distTagsMatch;
return this.handleDistTags(context.method, packageName, tag, context.body, token);
const [, rawPkgName, tag] = distTagsMatch;
return this.handleDistTags(context.method, decodeURIComponent(rawPkgName), tag, context.body, token);
}
// Tarball download: /{package}/-/{filename}.tgz
const tarballMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)\/-\/(.+\.tgz)$/);
if (tarballMatch) {
const [, packageName, filename] = tarballMatch;
return this.handleTarballDownload(packageName, filename, token, actor);
const [, rawPkgName, filename] = tarballMatch;
return this.handleTarballDownload(decodeURIComponent(rawPkgName), filename, token, actor);
}
// Unpublish specific version: DELETE /{package}/-/{version}
const unpublishVersionMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)\/-\/([^\/]+)$/);
if (unpublishVersionMatch && context.method === 'DELETE') {
const [, packageName, version] = unpublishVersionMatch;
this.logger.log('debug', 'unpublishVersionMatch', { packageName, version });
return this.unpublishVersion(packageName, version, token);
const [, rawPkgName, version] = unpublishVersionMatch;
this.logger.log('debug', 'unpublishVersionMatch', { packageName: decodeURIComponent(rawPkgName), version });
return this.unpublishVersion(decodeURIComponent(rawPkgName), version, token);
}
// Unpublish entire package: DELETE /{package}/-rev/{rev}
const unpublishPackageMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)\/-rev\/([^\/]+)$/);
if (unpublishPackageMatch && context.method === 'DELETE') {
const [, packageName, rev] = unpublishPackageMatch;
this.logger.log('debug', 'unpublishPackageMatch', { packageName, rev });
return this.unpublishPackage(packageName, token);
const [, rawPkgName, rev] = unpublishPackageMatch;
this.logger.log('debug', 'unpublishPackageMatch', { packageName: decodeURIComponent(rawPkgName), rev });
return this.unpublishPackage(decodeURIComponent(rawPkgName), token);
}
// Package version: /{package}/{version}
const versionMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)\/([^\/]+)$/);
if (versionMatch) {
const [, packageName, version] = versionMatch;
this.logger.log('debug', 'versionMatch', { packageName, version });
return this.handlePackageVersion(packageName, version, token, actor);
const [, rawPkgName, version] = versionMatch;
this.logger.log('debug', 'versionMatch', { packageName: decodeURIComponent(rawPkgName), version });
return this.handlePackageVersion(decodeURIComponent(rawPkgName), version, token, actor);
}
// Package operations: /{package}
const packageMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)$/);
if (packageMatch) {
const packageName = packageMatch[1];
const packageName = decodeURIComponent(packageMatch[1]);
this.logger.log('debug', 'packageMatch', { packageName });
return this.handlePackage(context.method, packageName, context.body, context.query, token, actor);
}
@@ -631,27 +631,38 @@ export class NpmRegistry extends BaseRegistry {
}
const version = versionMatch[1];
let tarball = await this.storage.getNpmTarball(packageName, version);
// Try local storage first (streaming)
const streamResult = await this.storage.getNpmTarballStream(packageName, version);
if (streamResult) {
return {
status: 200,
headers: {
'Content-Type': 'application/octet-stream',
'Content-Length': streamResult.size.toString(),
},
body: streamResult.stream,
};
}
// If not found locally, try upstream
if (!tarball) {
const upstream = await this.getUpstreamForRequest(packageName, 'tarball', 'GET', actor);
if (upstream) {
this.logger.log('debug', 'handleTarballDownload: fetching from upstream', {
let tarball: Buffer | null = null;
const upstream = await this.getUpstreamForRequest(packageName, 'tarball', 'GET', actor);
if (upstream) {
this.logger.log('debug', 'handleTarballDownload: fetching from upstream', {
packageName,
version,
});
const upstreamTarball = await upstream.fetchTarball(packageName, version);
if (upstreamTarball) {
tarball = upstreamTarball;
// Cache the tarball locally for future requests
await this.storage.putNpmTarball(packageName, version, tarball);
this.logger.log('debug', 'handleTarballDownload: cached tarball locally', {
packageName,
version,
size: tarball.length,
});
const upstreamTarball = await upstream.fetchTarball(packageName, version);
if (upstreamTarball) {
tarball = upstreamTarball;
// Cache the tarball locally for future requests
await this.storage.putNpmTarball(packageName, version, tarball);
this.logger.log('debug', 'handleTarballDownload: cached tarball locally', {
packageName,
version,
size: tarball.length,
});
}
}
}
@@ -738,6 +749,22 @@ export class NpmRegistry extends BaseRegistry {
this.logger.log('error', 'handleSearch failed', { error: (error as Error).message });
}
// Sort results by relevance: exact match first, then prefix match, then substring match
if (text) {
const lowerText = text.toLowerCase();
results.sort((a, b) => {
const aName = a.package.name.toLowerCase();
const bName = b.package.name.toLowerCase();
const aExact = aName === lowerText ? 0 : 1;
const bExact = bName === lowerText ? 0 : 1;
if (aExact !== bExact) return aExact - bExact;
const aPrefix = aName.startsWith(lowerText) ? 0 : 1;
const bPrefix = bName.startsWith(lowerText) ? 0 : 1;
if (aPrefix !== bPrefix) return aPrefix - bPrefix;
return aName.localeCompare(bName);
});
}
// Apply pagination
const paginatedResults = results.slice(from, from + size);

View File

@@ -3,6 +3,7 @@ import { BaseRegistry } from '../core/classes.baseregistry.js';
import { RegistryStorage } from '../core/classes.registrystorage.js';
import { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken, IRegistryError, IRequestActor } from '../core/interfaces.core.js';
import { createHashTransform, streamToBuffer } from '../core/helpers.stream.js';
import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
import { OciUpstream } from './classes.ociupstream.js';
import type {
@@ -125,12 +126,13 @@ export class OciRegistry extends BaseRegistry {
};
// Route to appropriate handler
if (path === '/v2/' || path === '/v2') {
// OCI spec: GET /v2/ is the version check endpoint
if (path === '/' || path === '' || path === '/v2/' || path === '/v2') {
return this.handleVersionCheck();
}
// Manifest operations: /v2/{name}/manifests/{reference}
const manifestMatch = path.match(/^\/v2\/([^\/]+(?:\/[^\/]+)*)\/manifests\/([^\/]+)$/);
// Manifest operations: /{name}/manifests/{reference}
const manifestMatch = path.match(/^\/([^\/]+(?:\/[^\/]+)*)\/manifests\/([^\/]+)$/);
if (manifestMatch) {
const [, name, reference] = manifestMatch;
// Prefer rawBody for content-addressable operations to preserve exact bytes
@@ -138,15 +140,15 @@ export class OciRegistry extends BaseRegistry {
return this.handleManifestRequest(context.method, name, reference, token, bodyData, context.headers, actor);
}
// Blob operations: /v2/{name}/blobs/{digest}
const blobMatch = path.match(/^\/v2\/([^\/]+(?:\/[^\/]+)*)\/blobs\/(sha256:[a-f0-9]{64})$/);
// Blob operations: /{name}/blobs/{digest}
const blobMatch = path.match(/^\/([^\/]+(?:\/[^\/]+)*)\/blobs\/(sha256:[a-f0-9]{64})$/);
if (blobMatch) {
const [, name, digest] = blobMatch;
return this.handleBlobRequest(context.method, name, digest, token, context.headers, actor);
}
// Blob upload operations: /v2/{name}/blobs/uploads/
const uploadInitMatch = path.match(/^\/v2\/([^\/]+(?:\/[^\/]+)*)\/blobs\/uploads\/?$/);
// Blob upload operations: /{name}/blobs/uploads/
const uploadInitMatch = path.match(/^\/([^\/]+(?:\/[^\/]+)*)\/blobs\/uploads\/?$/);
if (uploadInitMatch && context.method === 'POST') {
const [, name] = uploadInitMatch;
// Prefer rawBody for content-addressable operations to preserve exact bytes
@@ -154,22 +156,22 @@ export class OciRegistry extends BaseRegistry {
return this.handleUploadInit(name, token, context.query, bodyData);
}
// Blob upload operations: /v2/{name}/blobs/uploads/{uuid}
const uploadMatch = path.match(/^\/v2\/([^\/]+(?:\/[^\/]+)*)\/blobs\/uploads\/([^\/]+)$/);
// Blob upload operations: /{name}/blobs/uploads/{uuid}
const uploadMatch = path.match(/^\/([^\/]+(?:\/[^\/]+)*)\/blobs\/uploads\/([^\/]+)$/);
if (uploadMatch) {
const [, name, uploadId] = uploadMatch;
return this.handleUploadSession(context.method, uploadId, token, context);
}
// Tags list: /v2/{name}/tags/list
const tagsMatch = path.match(/^\/v2\/([^\/]+(?:\/[^\/]+)*)\/tags\/list$/);
// Tags list: /{name}/tags/list
const tagsMatch = path.match(/^\/([^\/]+(?:\/[^\/]+)*)\/tags\/list$/);
if (tagsMatch) {
const [, name] = tagsMatch;
return this.handleTagsList(name, token, context.query);
}
// Referrers: /v2/{name}/referrers/{digest}
const referrersMatch = path.match(/^\/v2\/([^\/]+(?:\/[^\/]+)*)\/referrers\/(sha256:[a-f0-9]{64})$/);
// Referrers: /{name}/referrers/{digest}
const referrersMatch = path.match(/^\/([^\/]+(?:\/[^\/]+)*)\/referrers\/(sha256:[a-f0-9]{64})$/);
if (referrersMatch) {
const [, name, digest] = referrersMatch;
return this.handleReferrers(name, digest, token, context.query);
@@ -289,7 +291,7 @@ export class OciRegistry extends BaseRegistry {
return {
status: 201,
headers: {
'Location': `${this.basePath}/v2/${repository}/blobs/${digest}`,
'Location': `${this.basePath}/${repository}/blobs/${digest}`,
'Docker-Content-Digest': digest,
},
body: null,
@@ -302,6 +304,8 @@ export class OciRegistry extends BaseRegistry {
uploadId,
repository,
chunks: [],
chunkPaths: [],
chunkIndex: 0,
totalSize: 0,
createdAt: new Date(),
lastActivity: new Date(),
@@ -312,7 +316,7 @@ export class OciRegistry extends BaseRegistry {
return {
status: 202,
headers: {
'Location': `${this.basePath}/v2/${repository}/blobs/uploads/${uploadId}`,
'Location': `${this.basePath}/${repository}/blobs/uploads/${uploadId}`,
'Docker-Upload-UUID': uploadId,
},
body: null,
@@ -527,7 +531,7 @@ export class OciRegistry extends BaseRegistry {
return {
status: 201,
headers: {
'Location': `${this.basePath}/v2/${repository}/manifests/${digest}`,
'Location': `${this.basePath}/${repository}/manifests/${digest}`,
'Docker-Content-Digest': digest,
},
body: null,
@@ -571,25 +575,35 @@ export class OciRegistry extends BaseRegistry {
return this.createUnauthorizedResponse(repository, 'pull');
}
// Try local storage first
let data = await this.storage.getOciBlob(digest);
// Try local storage first (streaming)
const streamResult = await this.storage.getOciBlobStream(digest);
if (streamResult) {
return {
status: 200,
headers: {
'Content-Type': 'application/octet-stream',
'Content-Length': streamResult.size.toString(),
'Docker-Content-Digest': digest,
},
body: streamResult.stream,
};
}
// If not found locally, try upstream
if (!data) {
const upstream = await this.getUpstreamForRequest(repository, 'blob', 'GET', actor);
if (upstream) {
this.logger.log('debug', 'getBlob: fetching from upstream', { repository, digest });
const upstreamBlob = await upstream.fetchBlob(repository, digest);
if (upstreamBlob) {
data = upstreamBlob;
// Cache the blob locally (blobs are content-addressable and immutable)
await this.storage.putOciBlob(digest, data);
this.logger.log('debug', 'getBlob: cached blob locally', {
repository,
digest,
size: data.length,
});
}
let data: Buffer | null = null;
const upstream = await this.getUpstreamForRequest(repository, 'blob', 'GET', actor);
if (upstream) {
this.logger.log('debug', 'getBlob: fetching from upstream', { repository, digest });
const upstreamBlob = await upstream.fetchBlob(repository, digest);
if (upstreamBlob) {
data = upstreamBlob;
// Cache the blob locally (blobs are content-addressable and immutable)
await this.storage.putOciBlob(digest, data);
this.logger.log('debug', 'getBlob: cached blob locally', {
repository,
digest,
size: data.length,
});
}
}
@@ -620,17 +634,15 @@ export class OciRegistry extends BaseRegistry {
return this.createUnauthorizedHeadResponse(repository, 'pull');
}
const exists = await this.storage.ociBlobExists(digest);
if (!exists) {
const blobSize = await this.storage.getOciBlobSize(digest);
if (blobSize === null) {
return { status: 404, headers: {}, body: null };
}
const blob = await this.storage.getOciBlob(digest);
return {
status: 200,
headers: {
'Content-Length': blob ? blob.length.toString() : '0',
'Content-Length': blobSize.toString(),
'Docker-Content-Digest': digest,
},
body: null,
@@ -670,14 +682,19 @@ export class OciRegistry extends BaseRegistry {
}
const chunkData = this.toBuffer(data);
session.chunks.push(chunkData);
// Write chunk to temp S3 object instead of accumulating in memory
const chunkPath = `oci/uploads/${uploadId}/chunk-${session.chunkIndex}`;
await this.storage.putObject(chunkPath, chunkData);
session.chunkPaths.push(chunkPath);
session.chunkIndex++;
session.totalSize += chunkData.length;
session.lastActivity = new Date();
return {
status: 202,
headers: {
'Location': `${this.basePath}/v2/${session.repository}/blobs/uploads/${uploadId}`,
'Location': `${this.basePath}/${session.repository}/blobs/uploads/${uploadId}`,
'Range': `0-${session.totalSize - 1}`,
'Docker-Upload-UUID': uploadId,
},
@@ -699,13 +716,52 @@ export class OciRegistry extends BaseRegistry {
};
}
const chunks = [...session.chunks];
if (finalData) chunks.push(this.toBuffer(finalData));
const blobData = Buffer.concat(chunks);
// If there's final data in the PUT body, write it as the last chunk
if (finalData) {
const buf = this.toBuffer(finalData);
const chunkPath = `oci/uploads/${uploadId}/chunk-${session.chunkIndex}`;
await this.storage.putObject(chunkPath, buf);
session.chunkPaths.push(chunkPath);
session.chunkIndex++;
session.totalSize += buf.length;
}
// Verify digest
const calculatedDigest = await this.calculateDigest(blobData);
// Create a ReadableStream that assembles all chunks from S3 sequentially
const chunkPaths = [...session.chunkPaths];
const storage = this.storage;
let chunkIdx = 0;
const assembledStream = new ReadableStream<Uint8Array>({
async pull(controller) {
if (chunkIdx >= chunkPaths.length) {
controller.close();
return;
}
const result = await storage.getObjectStream(chunkPaths[chunkIdx++]);
if (result) {
const reader = result.stream.getReader();
while (true) {
const { done, value } = await reader.read();
if (done) break;
if (value) controller.enqueue(value);
}
}
},
});
// Pipe through hash transform for incremental digest verification
const { transform: hashTransform, getDigest } = createHashTransform('sha256');
const hashedStream = assembledStream.pipeThrough(hashTransform);
// Consume stream to buffer for S3 upload
// (AWS SDK PutObjectCommand requires known content-length for streams;
// the key win is chunks are NOT accumulated in memory during PATCH — they live in S3)
const blobData = await streamToBuffer(hashedStream);
// Verify digest before storing
const calculatedDigest = `sha256:${getDigest()}`;
if (calculatedDigest !== digest) {
await this.cleanupUploadChunks(session);
this.uploadSessions.delete(uploadId);
return {
status: 400,
headers: {},
@@ -713,19 +769,36 @@ export class OciRegistry extends BaseRegistry {
};
}
// Store verified blob
await this.storage.putOciBlob(digest, blobData);
// Cleanup temp chunks and session
await this.cleanupUploadChunks(session);
this.uploadSessions.delete(uploadId);
return {
status: 201,
headers: {
'Location': `${this.basePath}/v2/${session.repository}/blobs/${digest}`,
'Location': `${this.basePath}/${session.repository}/blobs/${digest}`,
'Docker-Content-Digest': digest,
},
body: null,
};
}
/**
* Delete all temp S3 chunk objects for an upload session.
*/
private async cleanupUploadChunks(session: IUploadSession): Promise<void> {
for (const chunkPath of session.chunkPaths) {
try {
await this.storage.deleteObject(chunkPath);
} catch {
// Best-effort cleanup
}
}
}
private async getUploadStatus(uploadId: string): Promise<IResponse> {
const session = this.uploadSessions.get(uploadId);
if (!session) {
@@ -739,7 +812,7 @@ export class OciRegistry extends BaseRegistry {
return {
status: 204,
headers: {
'Location': `${this.basePath}/v2/${session.repository}/blobs/uploads/${uploadId}`,
'Location': `${this.basePath}/${session.repository}/blobs/uploads/${uploadId}`,
'Range': session.totalSize > 0 ? `0-${session.totalSize - 1}` : '0-0',
'Docker-Upload-UUID': uploadId,
},
@@ -884,7 +957,7 @@ export class OciRegistry extends BaseRegistry {
* Per OCI Distribution Spec, 401 responses MUST include WWW-Authenticate header.
*/
private createUnauthorizedResponse(repository: string, action: string): IResponse {
const realm = this.ociTokens?.realm || `${this.basePath}/v2/token`;
const realm = this.ociTokens?.realm || `${this.basePath}/token`;
const service = this.ociTokens?.service || 'registry';
return {
status: 401,
@@ -899,7 +972,7 @@ export class OciRegistry extends BaseRegistry {
* Create an unauthorized HEAD response (no body per HTTP spec).
*/
private createUnauthorizedHeadResponse(repository: string, action: string): IResponse {
const realm = this.ociTokens?.realm || `${this.basePath}/v2/token`;
const realm = this.ociTokens?.realm || `${this.basePath}/token`;
const service = this.ociTokens?.service || 'registry';
return {
status: 401,
@@ -917,6 +990,8 @@ export class OciRegistry extends BaseRegistry {
for (const [uploadId, session] of this.uploadSessions.entries()) {
if (now.getTime() - session.lastActivity.getTime() > maxAge) {
// Clean up temp S3 chunks for stale sessions
this.cleanupUploadChunks(session).catch(() => {});
this.uploadSessions.delete(uploadId);
}
}

View File

@@ -24,13 +24,18 @@ export class OciUpstream extends BaseUpstream {
/** Local registry base path for URL building */
private readonly localBasePath: string;
/** API prefix for outbound OCI requests (default: /v2) */
private readonly apiPrefix: string;
constructor(
config: IProtocolUpstreamConfig,
localBasePath: string = '/oci',
logger?: plugins.smartlog.Smartlog,
apiPrefix: string = '/v2',
) {
super(config, logger);
this.localBasePath = localBasePath;
this.apiPrefix = apiPrefix;
}
/**
@@ -44,7 +49,7 @@ export class OciUpstream extends BaseUpstream {
protocol: 'oci',
resource: repository,
resourceType: 'manifest',
path: `/v2/${repository}/manifests/${reference}`,
path: `${this.apiPrefix}/${repository}/manifests/${reference}`,
method: 'GET',
headers: {
'accept': [
@@ -88,7 +93,7 @@ export class OciUpstream extends BaseUpstream {
protocol: 'oci',
resource: repository,
resourceType: 'manifest',
path: `/v2/${repository}/manifests/${reference}`,
path: `${this.apiPrefix}/${repository}/manifests/${reference}`,
method: 'HEAD',
headers: {
'accept': [
@@ -127,7 +132,7 @@ export class OciUpstream extends BaseUpstream {
protocol: 'oci',
resource: repository,
resourceType: 'blob',
path: `/v2/${repository}/blobs/${digest}`,
path: `${this.apiPrefix}/${repository}/blobs/${digest}`,
method: 'GET',
headers: {
'accept': 'application/octet-stream',
@@ -155,7 +160,7 @@ export class OciUpstream extends BaseUpstream {
protocol: 'oci',
resource: repository,
resourceType: 'blob',
path: `/v2/${repository}/blobs/${digest}`,
path: `${this.apiPrefix}/${repository}/blobs/${digest}`,
method: 'HEAD',
headers: {},
query: {},
@@ -189,7 +194,7 @@ export class OciUpstream extends BaseUpstream {
protocol: 'oci',
resource: repository,
resourceType: 'tags',
path: `/v2/${repository}/tags/list`,
path: `${this.apiPrefix}/${repository}/tags/list`,
method: 'GET',
headers: {
'accept': 'application/json',
@@ -215,7 +220,8 @@ export class OciUpstream extends BaseUpstream {
/**
* Override URL building for OCI-specific handling.
* OCI registries use /v2/ prefix and may require special handling for Docker Hub.
* OCI registries use a configurable API prefix (default /v2/) and may require
* special handling for Docker Hub.
*/
protected buildUpstreamUrl(
upstream: IUpstreamRegistryConfig,
@@ -228,16 +234,20 @@ export class OciUpstream extends BaseUpstream {
baseUrl = baseUrl.slice(0, -1);
}
// Use per-upstream apiPrefix if configured, otherwise use the instance default
const prefix = upstream.apiPrefix || this.apiPrefix;
// Handle Docker Hub special case
// Docker Hub uses registry-1.docker.io but library images need special handling
if (baseUrl.includes('docker.io') || baseUrl.includes('registry-1.docker.io')) {
// For library images (e.g., "nginx" -> "library/nginx")
const pathParts = context.path.match(/^\/v2\/([^\/]+)\/(.+)$/);
const escapedPrefix = prefix.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
const pathParts = context.path.match(new RegExp(`^${escapedPrefix}\\/([^\\/]+)\\/(.+)$`));
if (pathParts) {
const [, repository, rest] = pathParts;
// If repository doesn't contain a slash, it's a library image
if (!repository.includes('/')) {
return `${baseUrl}/v2/library/${repository}/${rest}`;
return `${baseUrl}${prefix}/library/${repository}/${rest}`;
}
}
}

View File

@@ -62,6 +62,10 @@ export interface IUploadSession {
uploadId: string;
repository: string;
chunks: Buffer[];
/** S3 paths to temp chunk objects (streaming mode) */
chunkPaths: string[];
/** Index counter for naming temp chunk objects */
chunkIndex: number;
totalSize: number;
createdAt: Date;
lastActivity: Date;

View File

@@ -535,17 +535,30 @@ export class PypiRegistry extends BaseRegistry {
*/
private async handleDownload(packageName: string, filename: string, actor?: IRequestActor): Promise<IResponse> {
const normalized = helpers.normalizePypiPackageName(packageName);
let fileData = await this.storage.getPypiPackageFile(normalized, filename);
// Try streaming from local storage first
const streamResult = await this.storage.getPypiPackageFileStream(normalized, filename);
if (streamResult) {
return {
status: 200,
headers: {
'Content-Type': 'application/octet-stream',
'Content-Disposition': `attachment; filename="${filename}"`,
'Content-Length': streamResult.size.toString()
},
body: streamResult.stream,
};
}
// Try upstream if not found locally
if (!fileData) {
const upstream = await this.getUpstreamForRequest(normalized, 'file', 'GET', actor);
if (upstream) {
fileData = await upstream.fetchPackageFile(normalized, filename);
if (fileData) {
// Cache locally
await this.storage.putPypiPackageFile(normalized, filename, fileData);
}
let fileData: Buffer | null = null;
const upstream = await this.getUpstreamForRequest(normalized, 'file', 'GET', actor);
if (upstream) {
fileData = await upstream.fetchPackageFile(normalized, filename);
if (fileData) {
// Cache locally
await this.storage.putPypiPackageFile(normalized, filename, fileData);
}
}

View File

@@ -303,21 +303,33 @@ export class RubyGemsRegistry extends BaseRegistry {
return this.errorResponse(400, 'Invalid gem filename');
}
let gemData = await this.storage.getRubyGemsGem(
// Try streaming from local storage first
const streamResult = await this.storage.getRubyGemsGemStream(
parsed.name,
parsed.version,
parsed.platform
);
if (streamResult) {
return {
status: 200,
headers: {
'Content-Type': 'application/octet-stream',
'Content-Disposition': `attachment; filename="${filename}"`,
'Content-Length': streamResult.size.toString()
},
body: streamResult.stream,
};
}
// Try upstream if not found locally
if (!gemData) {
const upstream = await this.getUpstreamForRequest(parsed.name, 'gem', 'GET', actor);
if (upstream) {
gemData = await upstream.fetchGem(parsed.name, parsed.version);
if (gemData) {
// Cache locally
await this.storage.putRubyGemsGem(parsed.name, parsed.version, gemData, parsed.platform);
}
let gemData: Buffer | null = null;
const upstream = await this.getUpstreamForRequest(parsed.name, 'gem', 'GET', actor);
if (upstream) {
gemData = await upstream.fetchGem(parsed.name, parsed.version);
if (gemData) {
// Cache locally
await this.storage.putRubyGemsGem(parsed.name, parsed.version, gemData, parsed.platform);
}
}

View File

@@ -254,14 +254,12 @@ export function generateVersionsJson(
uploadTime?: string;
}>
): any {
return {
name: gemName,
versions: versions.map(v => ({
number: v.version,
platform: v.platform || 'ruby',
built_at: v.uploadTime,
})),
};
// RubyGems.org API returns a flat array at /api/v1/versions/{gem}.json
return versions.map(v => ({
number: v.version,
platform: v.platform || 'ruby',
built_at: v.uploadTime,
}));
}
/**
@@ -427,7 +425,7 @@ export async function extractGemMetadata(gemData: Buffer): Promise<{
// Step 2: Decompress the gzipped metadata
const gzipTools = new plugins.smartarchive.GzipTools();
const metadataYaml = await gzipTools.decompress(metadataFile.contentBuffer);
const yamlContent = metadataYaml.toString('utf-8');
const yamlContent = Buffer.from(metadataYaml).toString('utf-8');
// Step 3: Parse the YAML to extract name, version, platform
// Look for name: field in YAML
@@ -503,7 +501,7 @@ export async function generateSpecsGz(specs: Array<[string, string, string]>): P
}
const uncompressed = Buffer.concat(parts);
return gzipTools.compress(uncompressed);
return Buffer.from(await gzipTools.compress(uncompressed));
}
/**

View File

@@ -105,7 +105,7 @@ export class UpstreamCache {
// If not in memory and we have storage, check S3
if (!entry && this.storage) {
entry = await this.loadFromStorage(key);
entry = (await this.loadFromStorage(key)) ?? undefined;
if (entry) {
// Promote to memory cache
this.memoryCache.set(key, entry);

View File

@@ -86,6 +86,8 @@ export interface IUpstreamRegistryConfig {
cache?: Partial<IUpstreamCacheConfig>;
/** Resilience configuration overrides */
resilience?: Partial<IUpstreamResilienceConfig>;
/** API path prefix for OCI registries (default: /v2). Useful for registries behind reverse proxies. */
apiPrefix?: string;
}
/**

View File

@@ -4,9 +4,7 @@
"module": "NodeNext",
"moduleResolution": "NodeNext",
"esModuleInterop": true,
"verbatimModuleSyntax": true,
"baseUrl": ".",
"paths": {}
"verbatimModuleSyntax": true
},
"exclude": ["dist_*/**/*.d.ts"]
}