Compare commits

...

12 Commits

Author SHA1 Message Date
jkunz 10190a39fc v2.9.0
Default (tags) / security (push) Failing after 0s
Default (tags) / test (push) Failing after 0s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-04-16 10:42:33 +00:00
jkunz 9643ef98b9 feat(registry): add declarative protocol routing and request-scoped storage hook context across registries 2026-04-16 10:42:33 +00:00
jkunz 09335d41f3 v2.8.2
Default (tags) / security (push) Failing after 0s
Default (tags) / test (push) Failing after 0s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-03-27 17:37:24 +00:00
jkunz 2221eef722 fix(maven,tests): handle Maven Basic auth and accept deploy-plugin metadata/checksum uploads while stabilizing npm CLI test cleanup 2026-03-27 17:37:24 +00:00
jkunz 26ddf1a59f v2.8.1
Default (tags) / security (push) Failing after 0s
Default (tags) / test (push) Failing after 0s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-03-24 23:23:03 +00:00
jkunz 5acd1d6166 fix(registry): align OCI and RubyGems API behavior and improve npm search result ordering 2026-03-24 23:23:03 +00:00
jkunz abf7605e14 v2.8.0
Default (tags) / security (push) Failing after 0s
Default (tags) / test (push) Failing after 0s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-03-24 22:59:37 +00:00
jkunz 7da1a35efe feat(core,storage,oci,registry-config): add streaming response support and configurable registry URLs across protocols 2026-03-24 22:59:37 +00:00
jkunz 1f0acf2825 fix(oci): remove /v2/ from internal route patterns and make upstream apiPrefix configurable
The OCI handler had /v2/ baked into all regex patterns and Location headers.
When basePath was set to /v2 (as in stack.gallery), stripping it removed the
prefix that patterns expected, causing all OCI endpoints to 404.

Now patterns match on bare paths after basePath stripping, working correctly
regardless of the basePath value.

Also adds configurable apiPrefix to OCI upstream class (default /v2) for
registries behind reverse proxies with custom path prefixes.
2026-03-21 16:17:52 +00:00
jkunz 37e4c5be4a fix(npm): decode URL-encoded package names after regex extraction
Scoped npm packages use %2f encoding for the slash in URLs (e.g. @scope%2fpackage).
Previously, the encoded name was used as-is for storage and packument metadata,
causing npm install to fail with EINVALIDPACKAGENAME. Now each regex extraction
point decodes the package name via decodeURIComponent while keeping the path
encoded for correct regex matching.
2026-03-21 11:59:52 +00:00
jkunz 9bbc3da484 v2.7.0
Default (tags) / security (push) Successful in 43s
Default (tags) / test (push) Failing after 41s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-12-03 22:16:40 +00:00
jkunz e9af3f8328 feat(upstream): Add dynamic per-request upstream provider and integrate into registries 2025-12-03 22:16:40 +00:00
58 changed files with 7444 additions and 7372 deletions
+24
View File
@@ -0,0 +1,24 @@
{
"@git.zone/cli": {
"projectType": "npm",
"module": {
"githost": "code.foss.global",
"gitscope": "push.rocks",
"gitrepo": "smartregistry",
"description": "A composable TypeScript library implementing OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems registries for building unified container and package registries",
"npmPackagename": "@push.rocks/smartregistry",
"license": "MIT",
"projectDomain": "push.rocks"
},
"release": {
"registries": [
"https://verdaccio.lossless.digital",
"https://registry.npmjs.org"
],
"accessLevel": "public"
}
},
"@ship.zone/szci": {
"npmGlobalTools": []
}
}
+1 -1
View File
@@ -1,7 +1,7 @@
{ {
"json.schemas": [ "json.schemas": [
{ {
"fileMatch": ["/npmextra.json"], "fileMatch": ["/.smartconfig.json"],
"schema": { "schema": {
"type": "object", "type": "object",
"properties": { "properties": {
+46
View File
@@ -1,5 +1,51 @@
# Changelog # Changelog
## 2026-04-16 - 2.9.0 - feat(registry)
add declarative protocol routing and request-scoped storage hook context across registries
- Refactors protocol registration and request dispatch in SmartRegistry around shared registry descriptors.
- Wraps protocol request handling in storage context so hooks receive protocol, actor, package, and version metadata without cross-request leakage.
- Adds shared base registry helpers for header parsing, bearer/basic auth extraction, actor construction, and protocol logger creation.
- Improves NPM route parsing and publish helpers, including support for unencoded scoped package metadata and publish paths.
- Introduces centralized registry storage path helpers and expands test helpers and coverage for concurrent context isolation and real request hook metadata.
## 2026-03-27 - 2.8.2 - fix(maven,tests)
handle Maven Basic auth and accept deploy-plugin metadata/checksum uploads while stabilizing npm CLI test cleanup
- Validate Maven tokens from Basic auth credentials by extracting the password portion before token validation.
- Return successful responses for PUT requests to checksum and maven-metadata endpoints so Maven deploy uploads do not fail when files are auto-generated.
- Improve npm CLI integration test isolation and cleanup by using a temporary test directory, copying per-package .npmrc files, and cleaning stale published packages before test runs.
- Tighten test teardown by destroying the registry explicitly and simplifying package/install fixture generation.
## 2026-03-24 - 2.8.1 - fix(registry)
align OCI and RubyGems API behavior and improve npm search result ordering
- handle OCI version checks on /v2 and /v2/ endpoints
- return RubyGems versions JSON in the expected flat array format and update unyank coverage to use the HTTP endpoint
- prioritize exact and prefix matches in npm search results
- update documentation to reflect full upstream proxy support
## 2026-03-24 - 2.8.0 - feat(core,storage,oci,registry-config)
add streaming response support and configurable registry URLs across protocols
- Normalize SmartRegistry responses to ReadableStream bodies at the public API boundary and add stream helper utilities for buffers, JSON, and hashing
- Add streaming storage accessors for OCI, npm, Maven, Cargo, Composer, PyPI, and RubyGems downloads to reduce in-memory buffering
- Make per-protocol registryUrl configurable so CLI and integration tests can use correct host and port values
- Refactor OCI blob uploads to persist chunks in storage during upload and clean up temporary chunk objects after completion or expiry
- Update tests and storage integration to use the new stream-based response model and smartstorage backend
## 2025-12-03 - 2.7.0 - feat(upstream)
Add dynamic per-request upstream provider and integrate into registries
- Introduce IUpstreamProvider and IUpstreamResolutionContext to resolve upstream configs per request.
- Add StaticUpstreamProvider implementation for simple static upstream configurations.
- Propagate dynamic upstream provider through SmartRegistry and wire into protocol handlers (npm, oci, maven, cargo, composer, pypi, rubygems).
- Replace persistent per-protocol upstream instances with per-request resolution: registries now call provider.resolveUpstreamConfig(...) and instantiate protocol-specific Upstream when needed.
- Add IRequestActor to core interfaces and pass actor context (userId, ip, userAgent, etc.) to upstream resolution and storage/auth hooks.
- Update many protocol registries to accept an upstreamProvider instead of IProtocolUpstreamConfig and to attempt upstream fetches only when provider returns enabled config.
- Add utilities and tests: test helpers to create registries with upstream provider, a tracking upstream provider helper, StaticUpstreamProvider tests and extensive upstream/provider integration tests.
- Improve upstream interfaces and cache/fetch contexts (IUpstreamFetchContext includes actor) and add StaticUpstreamProvider class to upstream module.
## 2025-11-27 - 2.6.0 - feat(core) ## 2025-11-27 - 2.6.0 - feat(core)
Add core registry infrastructure: storage, auth, upstream cache, and protocol handlers Add core registry infrastructure: storage, auth, upstream cache, and protocol handlers
-18
View File
@@ -1,18 +0,0 @@
{
"gitzone": {
"projectType": "npm",
"module": {
"githost": "code.foss.global",
"gitscope": "push.rocks",
"gitrepo": "smartregistry",
"description": "a registry for npm modules and oci images",
"npmPackagename": "@push.rocks/smartregistry",
"license": "MIT",
"projectDomain": "push.rocks"
}
},
"npmci": {
"npmGlobalTools": [],
"npmAccessLevel": "public"
}
}
+15 -15
View File
@@ -1,6 +1,6 @@
{ {
"name": "@push.rocks/smartregistry", "name": "@push.rocks/smartregistry",
"version": "2.6.0", "version": "2.9.0",
"private": false, "private": false,
"description": "A composable TypeScript library implementing OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems registries for building unified container and package registries", "description": "A composable TypeScript library implementing OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems registries for building unified container and package registries",
"main": "dist_ts/index.js", "main": "dist_ts/index.js",
@@ -10,17 +10,17 @@
"license": "MIT", "license": "MIT",
"scripts": { "scripts": {
"test": "(tstest test/ --verbose --logfile --timeout 240)", "test": "(tstest test/ --verbose --logfile --timeout 240)",
"build": "(tsbuild --web --allowimplicitany)", "build": "(tsbuild --allowimplicitany)",
"buildDocs": "(tsdoc)" "buildDocs": "(tsdoc)"
}, },
"devDependencies": { "devDependencies": {
"@git.zone/tsbuild": "^3.1.0", "@git.zone/tsbuild": "^4.4.0",
"@git.zone/tsbundle": "^2.0.5", "@git.zone/tsbundle": "^2.10.0",
"@git.zone/tsrun": "^2.0.0", "@git.zone/tsrun": "^2.0.2",
"@git.zone/tstest": "^3.1.0", "@git.zone/tstest": "^3.6.0",
"@push.rocks/smartarchive": "^5.0.1", "@push.rocks/smartarchive": "^5.2.1",
"@push.rocks/smarts3": "^5.1.0", "@push.rocks/smartstorage": "^6.3.2",
"@types/node": "^24.10.1" "@types/node": "^25.5.0"
}, },
"repository": { "repository": {
"type": "git", "type": "git",
@@ -39,7 +39,7 @@
"dist_ts_web/**/*", "dist_ts_web/**/*",
"assets/**/*", "assets/**/*",
"cli.js", "cli.js",
"npmextra.json", ".smartconfig.json",
"readme.md" "readme.md"
], ],
"pnpm": { "pnpm": {
@@ -47,13 +47,13 @@
}, },
"dependencies": { "dependencies": {
"@push.rocks/qenv": "^6.1.3", "@push.rocks/qenv": "^6.1.3",
"@push.rocks/smartbucket": "^4.3.0", "@push.rocks/smartbucket": "^4.5.1",
"@push.rocks/smartlog": "^3.1.10", "@push.rocks/smartlog": "^3.2.1",
"@push.rocks/smartpath": "^6.0.0", "@push.rocks/smartpath": "^6.0.0",
"@push.rocks/smartrequest": "^5.0.1", "@push.rocks/smartrequest": "^5.0.1",
"@tsclass/tsclass": "^9.3.0", "@tsclass/tsclass": "^9.5.0",
"adm-zip": "^0.5.10", "adm-zip": "^0.5.16",
"minimatch": "^10.1.1" "minimatch": "^10.2.4"
}, },
"packageManager": "pnpm@10.18.1+sha512.77a884a165cbba2d8d1c19e3b4880eee6d2fcabd0d879121e282196b80042351d5eb3ca0935fa599da1dc51265cc68816ad2bddd2a2de5ea9fdf92adbec7cd34" "packageManager": "pnpm@10.18.1+sha512.77a884a165cbba2d8d1c19e3b4880eee6d2fcabd0d879121e282196b80042351d5eb3ca0935fa599da1dc51265cc68816ad2bddd2a2de5ea9fdf92adbec7cd34"
} }
+2988 -4138
View File
File diff suppressed because it is too large Load Diff
+350 -803
View File
File diff suppressed because it is too large Load Diff
+5 -3
View File
@@ -2,6 +2,7 @@ import { tap, expect } from '@git.zone/tstest';
import { RegistryStorage } from '../ts/core/classes.registrystorage.js'; import { RegistryStorage } from '../ts/core/classes.registrystorage.js';
import { CargoRegistry } from '../ts/cargo/classes.cargoregistry.js'; import { CargoRegistry } from '../ts/cargo/classes.cargoregistry.js';
import { AuthManager } from '../ts/core/classes.authmanager.js'; import { AuthManager } from '../ts/core/classes.authmanager.js';
import { streamToJson } from '../ts/core/helpers.stream.js';
// Test index path calculation // Test index path calculation
tap.test('should calculate correct index paths for different crate names', async () => { tap.test('should calculate correct index paths for different crate names', async () => {
@@ -123,9 +124,10 @@ tap.test('should return valid config.json', async () => {
expect(response.status).to.equal(200); expect(response.status).to.equal(200);
expect(response.headers['Content-Type']).to.equal('application/json'); expect(response.headers['Content-Type']).to.equal('application/json');
expect(response.body).to.be.an('object'); const body = await streamToJson(response.body);
expect(response.body.dl).to.include('/api/v1/crates/{crate}/{version}/download'); expect(body).to.be.an('object');
expect(response.body.api).to.equal('http://localhost:5000/cargo'); expect(body.dl).to.include('/api/v1/crates/{crate}/{version}/download');
expect(body.api).to.equal('http://localhost:5000/cargo');
}); });
export default tap.start(); export default tap.start();
+420
View File
@@ -0,0 +1,420 @@
import * as crypto from 'crypto';
import * as smartarchive from '@push.rocks/smartarchive';
/**
* Helper to calculate SHA-256 digest in OCI format
*/
export function calculateDigest(data: Buffer): string {
const hash = crypto.createHash('sha256').update(data).digest('hex');
return `sha256:${hash}`;
}
/**
* Helper to create a minimal valid OCI manifest
*/
export function createTestManifest(configDigest: string, layerDigest: string) {
return {
schemaVersion: 2,
mediaType: 'application/vnd.oci.image.manifest.v1+json',
config: {
mediaType: 'application/vnd.oci.image.config.v1+json',
size: 123,
digest: configDigest,
},
layers: [
{
mediaType: 'application/vnd.oci.image.layer.v1.tar+gzip',
size: 456,
digest: layerDigest,
},
],
};
}
/**
* Helper to create a minimal valid NPM packument
*/
export function createTestPackument(packageName: string, version: string, tarballData: Buffer) {
const shasum = crypto.createHash('sha1').update(tarballData).digest('hex');
const integrity = `sha512-${crypto.createHash('sha512').update(tarballData).digest('base64')}`;
return {
name: packageName,
versions: {
[version]: {
name: packageName,
version: version,
description: 'Test package',
main: 'index.js',
scripts: {},
dist: {
shasum: shasum,
integrity: integrity,
tarball: `http://localhost:5000/npm/${packageName}/-/${packageName}-${version}.tgz`,
},
},
},
'dist-tags': {
latest: version,
},
_attachments: {
[`${packageName}-${version}.tgz`]: {
content_type: 'application/octet-stream',
data: tarballData.toString('base64'),
length: tarballData.length,
},
},
};
}
/**
* Helper to create a minimal valid Maven POM file
*/
export function createTestPom(
groupId: string,
artifactId: string,
version: string,
packaging: string = 'jar'
): string {
return `<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>${groupId}</groupId>
<artifactId>${artifactId}</artifactId>
<version>${version}</version>
<packaging>${packaging}</packaging>
<name>${artifactId}</name>
<description>Test Maven artifact</description>
</project>`;
}
/**
* Helper to create a test JAR file (minimal ZIP with manifest)
*/
export function createTestJar(): Buffer {
const manifestContent = `Manifest-Version: 1.0
Created-By: SmartRegistry Test
`;
return Buffer.from(manifestContent, 'utf-8');
}
/**
* Helper to calculate Maven checksums
*/
export function calculateMavenChecksums(data: Buffer) {
return {
md5: crypto.createHash('md5').update(data).digest('hex'),
sha1: crypto.createHash('sha1').update(data).digest('hex'),
sha256: crypto.createHash('sha256').update(data).digest('hex'),
sha512: crypto.createHash('sha512').update(data).digest('hex'),
};
}
/**
* Helper to create a Composer package ZIP using smartarchive
*/
export async function createComposerZip(
vendorPackage: string,
version: string,
options?: {
description?: string;
license?: string[];
authors?: Array<{ name: string; email?: string }>;
}
): Promise<Buffer> {
const zipTools = new smartarchive.ZipTools();
const composerJson = {
name: vendorPackage,
version: version,
type: 'library',
description: options?.description || 'Test Composer package',
license: options?.license || ['MIT'],
authors: options?.authors || [{ name: 'Test Author', email: 'test@example.com' }],
require: {
php: '>=7.4',
},
autoload: {
'psr-4': {
'Vendor\\TestPackage\\': 'src/',
},
},
};
const [vendor, pkg] = vendorPackage.split('/');
const namespace = `${vendor.charAt(0).toUpperCase() + vendor.slice(1)}\\${pkg.charAt(0).toUpperCase() + pkg.slice(1).replace(/-/g, '')}`;
const testPhpContent = `<?php
namespace ${namespace};
class TestClass
{
public function greet(): string
{
return "Hello from ${vendorPackage}!";
}
}
`;
const entries: smartarchive.IArchiveEntry[] = [
{
archivePath: 'composer.json',
content: Buffer.from(JSON.stringify(composerJson, null, 2), 'utf-8'),
},
{
archivePath: 'src/TestClass.php',
content: Buffer.from(testPhpContent, 'utf-8'),
},
{
archivePath: 'README.md',
content: Buffer.from(`# ${vendorPackage}\n\nTest package`, 'utf-8'),
},
];
return Buffer.from(await zipTools.createZip(entries));
}
/**
* Helper to create a test Python wheel file (minimal ZIP structure) using smartarchive
*/
export async function createPythonWheel(
packageName: string,
version: string,
pyVersion: string = 'py3'
): Promise<Buffer> {
const zipTools = new smartarchive.ZipTools();
const normalizedName = packageName.replace(/-/g, '_');
const distInfoDir = `${normalizedName}-${version}.dist-info`;
const metadata = `Metadata-Version: 2.1
Name: ${packageName}
Version: ${version}
Summary: Test Python package
Home-page: https://example.com
Author: Test Author
Author-email: test@example.com
License: MIT
Platform: UNKNOWN
Classifier: Programming Language :: Python :: 3
Requires-Python: >=3.7
Description-Content-Type: text/markdown
# ${packageName}
Test package for SmartRegistry
`;
const wheelContent = `Wheel-Version: 1.0
Generator: test 1.0.0
Root-Is-Purelib: true
Tag: ${pyVersion}-none-any
`;
const moduleContent = `"""${packageName} module"""
__version__ = "${version}"
def hello():
return "Hello from ${packageName}!"
`;
const entries: smartarchive.IArchiveEntry[] = [
{
archivePath: `${distInfoDir}/METADATA`,
content: Buffer.from(metadata, 'utf-8'),
},
{
archivePath: `${distInfoDir}/WHEEL`,
content: Buffer.from(wheelContent, 'utf-8'),
},
{
archivePath: `${distInfoDir}/RECORD`,
content: Buffer.from('', 'utf-8'),
},
{
archivePath: `${distInfoDir}/top_level.txt`,
content: Buffer.from(normalizedName, 'utf-8'),
},
{
archivePath: `${normalizedName}/__init__.py`,
content: Buffer.from(moduleContent, 'utf-8'),
},
];
return Buffer.from(await zipTools.createZip(entries));
}
/**
* Helper to create a test Python source distribution (sdist) using smartarchive
*/
export async function createPythonSdist(
packageName: string,
version: string
): Promise<Buffer> {
const tarTools = new smartarchive.TarTools();
const normalizedName = packageName.replace(/-/g, '_');
const dirPrefix = `${packageName}-${version}`;
const pkgInfo = `Metadata-Version: 2.1
Name: ${packageName}
Version: ${version}
Summary: Test Python package
Home-page: https://example.com
Author: Test Author
Author-email: test@example.com
License: MIT
`;
const setupPy = `from setuptools import setup, find_packages
setup(
name="${packageName}",
version="${version}",
packages=find_packages(),
python_requires=">=3.7",
)
`;
const moduleContent = `"""${packageName} module"""
__version__ = "${version}"
def hello():
return "Hello from ${packageName}!"
`;
const entries: smartarchive.IArchiveEntry[] = [
{
archivePath: `${dirPrefix}/PKG-INFO`,
content: Buffer.from(pkgInfo, 'utf-8'),
},
{
archivePath: `${dirPrefix}/setup.py`,
content: Buffer.from(setupPy, 'utf-8'),
},
{
archivePath: `${dirPrefix}/${normalizedName}/__init__.py`,
content: Buffer.from(moduleContent, 'utf-8'),
},
];
return Buffer.from(await tarTools.packFilesToTarGz(entries));
}
/**
* Helper to calculate PyPI file hashes
*/
export function calculatePypiHashes(data: Buffer) {
return {
md5: crypto.createHash('md5').update(data).digest('hex'),
sha256: crypto.createHash('sha256').update(data).digest('hex'),
blake2b: crypto.createHash('blake2b512').update(data).digest('hex'),
};
}
/**
* Helper to create a test RubyGem file (minimal tar.gz structure) using smartarchive
*/
export async function createRubyGem(
gemName: string,
version: string,
platform: string = 'ruby'
): Promise<Buffer> {
const tarTools = new smartarchive.TarTools();
const gzipTools = new smartarchive.GzipTools();
const metadataYaml = `--- !ruby/object:Gem::Specification
name: ${gemName}
version: !ruby/object:Gem::Version
version: ${version}
platform: ${platform}
authors:
- Test Author
autorequire:
bindir: bin
cert_chain: []
date: ${new Date().toISOString().split('T')[0]}
dependencies: []
description: Test RubyGem
email: test@example.com
executables: []
extensions: []
extra_rdoc_files: []
files:
- lib/${gemName}.rb
homepage: https://example.com
licenses:
- MIT
metadata: {}
post_install_message:
rdoc_options: []
require_paths:
- lib
required_ruby_version: !ruby/object:Gem::Requirement
requirements:
- - ">="
- !ruby/object:Gem::Version
version: '2.7'
required_rubygems_version: !ruby/object:Gem::Requirement
requirements:
- - ">="
- !ruby/object:Gem::Version
version: '0'
requirements: []
rubygems_version: 3.0.0
signing_key:
specification_version: 4
summary: Test gem for SmartRegistry
test_files: []
`;
const metadataGz = Buffer.from(await gzipTools.compress(Buffer.from(metadataYaml, 'utf-8')));
const libContent = `# ${gemName}
module ${gemName.charAt(0).toUpperCase() + gemName.slice(1).replace(/-/g, '')}
VERSION = "${version}"
def self.hello
"Hello from #{gemName}!"
end
end
`;
const dataEntries: smartarchive.IArchiveEntry[] = [
{
archivePath: `lib/${gemName}.rb`,
content: Buffer.from(libContent, 'utf-8'),
},
];
const dataTarGz = Buffer.from(await tarTools.packFilesToTarGz(dataEntries));
const gemEntries: smartarchive.IArchiveEntry[] = [
{
archivePath: 'metadata.gz',
content: metadataGz,
},
{
archivePath: 'data.tar.gz',
content: dataTarGz,
},
];
return Buffer.from(await tarTools.packFiles(gemEntries));
}
/**
* Helper to calculate RubyGems checksums
*/
export function calculateRubyGemsChecksums(data: Buffer) {
return {
md5: crypto.createHash('md5').update(data).digest('hex'),
sha256: crypto.createHash('sha256').update(data).digest('hex'),
};
}
+8
View File
@@ -0,0 +1,8 @@
/**
* Generate a unique test run ID for avoiding conflicts between test runs.
*/
export function generateTestRunId(): string {
const timestamp = Date.now().toString(36);
const random = Math.random().toString(36).substring(2, 6);
return `${timestamp}${random}`;
}
+125
View File
@@ -0,0 +1,125 @@
import type { IAuthToken, TRegistryProtocol } from '../../ts/core/interfaces.core.js';
import type { IAuthProvider } from '../../ts/core/interfaces.auth.js';
import type {
IUpstreamProvider,
IUpstreamRegistryConfig,
IUpstreamResolutionContext,
IProtocolUpstreamConfig,
} from '../../ts/upstream/interfaces.upstream.js';
type TTestUpstreamRegistryConfig = Omit<Partial<IUpstreamRegistryConfig>, 'id' | 'url' | 'priority' | 'enabled'> &
Pick<IUpstreamRegistryConfig, 'id' | 'url' | 'priority' | 'enabled'>;
type TTestProtocolUpstreamConfig = Omit<IProtocolUpstreamConfig, 'upstreams'> & {
upstreams: TTestUpstreamRegistryConfig[];
};
function normalizeUpstreamRegistryConfig(
upstream: TTestUpstreamRegistryConfig
): IUpstreamRegistryConfig {
return {
...upstream,
name: upstream.name ?? upstream.id,
auth: upstream.auth ?? { type: 'none' },
};
}
function normalizeProtocolUpstreamConfig(
config: TTestProtocolUpstreamConfig | undefined
): IProtocolUpstreamConfig | null {
if (!config) {
return null;
}
return {
...config,
upstreams: config.upstreams.map(normalizeUpstreamRegistryConfig),
};
}
/**
* Create a mock upstream provider that tracks all calls for testing
*/
export function createTrackingUpstreamProvider(
baseConfig?: Partial<Record<TRegistryProtocol, TTestProtocolUpstreamConfig>>
): {
provider: IUpstreamProvider;
calls: IUpstreamResolutionContext[];
} {
const calls: IUpstreamResolutionContext[] = [];
const provider: IUpstreamProvider = {
async resolveUpstreamConfig(context: IUpstreamResolutionContext) {
calls.push({ ...context });
return normalizeProtocolUpstreamConfig(baseConfig?.[context.protocol]);
},
};
return { provider, calls };
}
/**
* Create a mock auth provider for testing pluggable authentication.
* Allows customizing behavior for different test scenarios.
*/
export function createMockAuthProvider(overrides?: Partial<IAuthProvider>): IAuthProvider {
const tokens = new Map<string, IAuthToken>();
return {
init: async () => {},
authenticate: async (credentials) => {
return credentials.username;
},
validateToken: async (token, protocol) => {
const stored = tokens.get(token);
if (stored && (!protocol || stored.type === protocol)) {
return stored;
}
if (token === 'valid-mock-token') {
return {
type: 'npm' as TRegistryProtocol,
userId: 'mock-user',
scopes: ['npm:*:*:*'],
};
}
return null;
},
createToken: async (userId, protocol, options) => {
const tokenId = `mock-${protocol}-${Date.now()}`;
const authToken: IAuthToken = {
type: protocol,
userId,
scopes: options?.scopes || [`${protocol}:*:*:*`],
readonly: options?.readonly,
expiresAt: options?.expiresIn ? new Date(Date.now() + options.expiresIn * 1000) : undefined,
};
tokens.set(tokenId, authToken);
return tokenId;
},
revokeToken: async (token) => {
tokens.delete(token);
},
authorize: async (token, resource, action) => {
if (!token) return false;
if (token.readonly && ['write', 'push', 'delete'].includes(action)) {
return false;
}
return true;
},
listUserTokens: async (userId) => {
const result: Array<{ key: string; readonly: boolean; created: string; protocol?: TRegistryProtocol }> = [];
for (const [key, token] of tokens.entries()) {
if (token.userId === userId) {
result.push({
key: `hash-${key.substring(0, 8)}`,
readonly: token.readonly || false,
created: new Date().toISOString(),
protocol: token.type,
});
}
}
return result;
},
...overrides,
};
}
+44 -785
View File
@@ -1,28 +1,34 @@
import * as qenv from '@push.rocks/qenv'; import * as qenv from '@push.rocks/qenv';
import * as crypto from 'crypto';
import * as smartarchive from '@push.rocks/smartarchive';
import * as smartbucket from '@push.rocks/smartbucket'; import * as smartbucket from '@push.rocks/smartbucket';
import { SmartRegistry } from '../../ts/classes.smartregistry.js'; import { SmartRegistry } from '../../ts/classes.smartregistry.js';
import type { IRegistryConfig, IAuthToken, TRegistryProtocol } from '../../ts/core/interfaces.core.js'; import type { IStorageHooks } from '../../ts/core/interfaces.storage.js';
import type { IAuthProvider, ITokenOptions } from '../../ts/core/interfaces.auth.js'; import type { IUpstreamProvider } from '../../ts/upstream/interfaces.upstream.js';
import type { IStorageHooks, IStorageHookContext, IBeforePutResult, IBeforeDeleteResult } from '../../ts/core/interfaces.storage.js'; import { buildTestRegistryConfig, createDefaultTestUpstreamProvider } from './registryconfig.js';
import { generateTestRunId } from './ids.js';
export {
calculateDigest,
createTestManifest,
createTestPackument,
createTestPom,
createTestJar,
calculateMavenChecksums,
createComposerZip,
createPythonWheel,
createPythonSdist,
calculatePypiHashes,
createRubyGem,
calculateRubyGemsChecksums,
} from './fixtures.js';
export { createMockAuthProvider, createTrackingUpstreamProvider } from './providers.js';
export { buildTestRegistryConfig, createDefaultTestUpstreamProvider } from './registryconfig.js';
export { createTestStorageBackend } from './storagebackend.js';
export { generateTestRunId } from './ids.js';
export { createTestTokens } from './tokens.js';
export { createTrackingHooks, createQuotaHooks } from './storagehooks.js';
const testQenv = new qenv.Qenv('./', './.nogit'); const testQenv = new qenv.Qenv('./', './.nogit');
/**
* Clean up S3 bucket contents for a fresh test run
* @param prefix Optional prefix to delete (e.g., 'cargo/', 'npm/', 'composer/')
*/
/**
* Generate a unique test run ID for avoiding conflicts between test runs
* Uses timestamp + random suffix for uniqueness
*/
export function generateTestRunId(): string {
const timestamp = Date.now().toString(36);
const random = Math.random().toString(36).substring(2, 6);
return `${timestamp}${random}`;
}
export async function cleanupS3Bucket(prefix?: string): Promise<void> { export async function cleanupS3Bucket(prefix?: string): Promise<void> {
const s3AccessKey = await testQenv.getEnvVarOnDemand('S3_ACCESSKEY'); const s3AccessKey = await testQenv.getEnvVarOnDemand('S3_ACCESSKEY');
const s3SecretKey = await testQenv.getEnvVarOnDemand('S3_SECRETKEY'); const s3SecretKey = await testQenv.getEnvVarOnDemand('S3_SECRETKEY');
@@ -38,19 +44,17 @@ export async function cleanupS3Bucket(prefix?: string): Promise<void> {
}); });
try { try {
const bucket = await s3.getBucket('test-registry'); const bucket = await s3.getBucketByName('test-registry');
if (bucket) { if (bucket) {
if (prefix) { if (prefix) {
// Delete only objects with the given prefix // Delete only objects with the given prefix
const files = await bucket.fastList({ prefix }); for await (const path of bucket.listAllObjects(prefix)) {
for (const file of files) { await bucket.fastRemove({ path });
await bucket.fastRemove({ path: file.name });
} }
} else { } else {
// Delete all objects in the bucket // Delete all objects in the bucket
const files = await bucket.fastList({}); for await (const path of bucket.listAllObjects()) {
for (const file of files) { await bucket.fastRemove({ path });
await bucket.fastRemove({ path: file.name });
} }
} }
} }
@@ -63,70 +67,11 @@ export async function cleanupS3Bucket(prefix?: string): Promise<void> {
/** /**
* Create a test SmartRegistry instance with all protocols enabled * Create a test SmartRegistry instance with all protocols enabled
*/ */
export async function createTestRegistry(): Promise<SmartRegistry> { export async function createTestRegistry(options?: {
// Read S3 config from env.json registryUrl?: string;
const s3AccessKey = await testQenv.getEnvVarOnDemand('S3_ACCESSKEY'); storageHooks?: IStorageHooks;
const s3SecretKey = await testQenv.getEnvVarOnDemand('S3_SECRETKEY'); }): Promise<SmartRegistry> {
const s3Endpoint = await testQenv.getEnvVarOnDemand('S3_ENDPOINT'); const config = await buildTestRegistryConfig(options);
const s3Port = await testQenv.getEnvVarOnDemand('S3_PORT');
const config: IRegistryConfig = {
storage: {
accessKey: s3AccessKey || 'minioadmin',
accessSecret: s3SecretKey || 'minioadmin',
endpoint: s3Endpoint || 'localhost',
port: parseInt(s3Port || '9000', 10),
useSsl: false,
region: 'us-east-1',
bucketName: 'test-registry',
},
auth: {
jwtSecret: 'test-secret-key',
tokenStore: 'memory',
npmTokens: {
enabled: true,
},
ociTokens: {
enabled: true,
realm: 'https://auth.example.com/token',
service: 'test-registry',
},
pypiTokens: {
enabled: true,
},
rubygemsTokens: {
enabled: true,
},
},
oci: {
enabled: true,
basePath: '/oci',
},
npm: {
enabled: true,
basePath: '/npm',
},
maven: {
enabled: true,
basePath: '/maven',
},
composer: {
enabled: true,
basePath: '/composer',
},
cargo: {
enabled: true,
basePath: '/cargo',
},
pypi: {
enabled: true,
basePath: '/pypi',
},
rubygems: {
enabled: true,
basePath: '/rubygems',
},
};
const registry = new SmartRegistry(config); const registry = new SmartRegistry(config);
await registry.init(); await registry.init();
@@ -135,703 +80,17 @@ export async function createTestRegistry(): Promise<SmartRegistry> {
} }
/** /**
* Helper to create test authentication tokens * Create a test SmartRegistry instance with upstream provider configured
*/ */
export async function createTestTokens(registry: SmartRegistry) { export async function createTestRegistryWithUpstream(
const authManager = registry.getAuthManager(); upstreamProvider?: IUpstreamProvider
): Promise<SmartRegistry> {
// Authenticate and create tokens const config = await buildTestRegistryConfig({
const userId = await authManager.authenticate({ upstreamProvider: upstreamProvider || createDefaultTestUpstreamProvider(),
username: 'testuser',
password: 'testpass',
}); });
if (!userId) { const registry = new SmartRegistry(config);
throw new Error('Failed to authenticate test user'); await registry.init();
}
// Create NPM token return registry;
const npmToken = await authManager.createNpmToken(userId, false);
// Create OCI token with full access
const ociToken = await authManager.createOciToken(
userId,
['oci:repository:*:*'],
3600
);
// Create Maven token with full access
const mavenToken = await authManager.createMavenToken(userId, false);
// Create Composer token with full access
const composerToken = await authManager.createComposerToken(userId, false);
// Create Cargo token with full access
const cargoToken = await authManager.createCargoToken(userId, false);
// Create PyPI token with full access
const pypiToken = await authManager.createPypiToken(userId, false);
// Create RubyGems token with full access
const rubygemsToken = await authManager.createRubyGemsToken(userId, false);
return { npmToken, ociToken, mavenToken, composerToken, cargoToken, pypiToken, rubygemsToken, userId };
}
/**
* Helper to calculate SHA-256 digest in OCI format
*/
export function calculateDigest(data: Buffer): string {
const hash = crypto.createHash('sha256').update(data).digest('hex');
return `sha256:${hash}`;
}
/**
* Helper to create a minimal valid OCI manifest
*/
export function createTestManifest(configDigest: string, layerDigest: string) {
return {
schemaVersion: 2,
mediaType: 'application/vnd.oci.image.manifest.v1+json',
config: {
mediaType: 'application/vnd.oci.image.config.v1+json',
size: 123,
digest: configDigest,
},
layers: [
{
mediaType: 'application/vnd.oci.image.layer.v1.tar+gzip',
size: 456,
digest: layerDigest,
},
],
};
}
/**
* Helper to create a minimal valid NPM packument
*/
export function createTestPackument(packageName: string, version: string, tarballData: Buffer) {
const shasum = crypto.createHash('sha1').update(tarballData).digest('hex');
const integrity = `sha512-${crypto.createHash('sha512').update(tarballData).digest('base64')}`;
return {
name: packageName,
versions: {
[version]: {
name: packageName,
version: version,
description: 'Test package',
main: 'index.js',
scripts: {},
dist: {
shasum: shasum,
integrity: integrity,
tarball: `http://localhost:5000/npm/${packageName}/-/${packageName}-${version}.tgz`,
},
},
},
'dist-tags': {
latest: version,
},
_attachments: {
[`${packageName}-${version}.tgz`]: {
content_type: 'application/octet-stream',
data: tarballData.toString('base64'),
length: tarballData.length,
},
},
};
}
/**
* Helper to create a minimal valid Maven POM file
*/
export function createTestPom(
groupId: string,
artifactId: string,
version: string,
packaging: string = 'jar'
): string {
return `<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>${groupId}</groupId>
<artifactId>${artifactId}</artifactId>
<version>${version}</version>
<packaging>${packaging}</packaging>
<name>${artifactId}</name>
<description>Test Maven artifact</description>
</project>`;
}
/**
* Helper to create a test JAR file (minimal ZIP with manifest)
*/
export function createTestJar(): Buffer {
// Create a simple JAR structure (just a manifest)
// In practice, this is a ZIP file with at least META-INF/MANIFEST.MF
const manifestContent = `Manifest-Version: 1.0
Created-By: SmartRegistry Test
`;
// For testing, we'll just create a buffer with dummy content
// Real JAR would be a proper ZIP archive
return Buffer.from(manifestContent, 'utf-8');
}
/**
* Helper to calculate Maven checksums
*/
export function calculateMavenChecksums(data: Buffer) {
return {
md5: crypto.createHash('md5').update(data).digest('hex'),
sha1: crypto.createHash('sha1').update(data).digest('hex'),
sha256: crypto.createHash('sha256').update(data).digest('hex'),
sha512: crypto.createHash('sha512').update(data).digest('hex'),
};
}
/**
* Helper to create a Composer package ZIP using smartarchive
*/
export async function createComposerZip(
vendorPackage: string,
version: string,
options?: {
description?: string;
license?: string[];
authors?: Array<{ name: string; email?: string }>;
}
): Promise<Buffer> {
const zipTools = new smartarchive.ZipTools();
const composerJson = {
name: vendorPackage,
version: version,
type: 'library',
description: options?.description || 'Test Composer package',
license: options?.license || ['MIT'],
authors: options?.authors || [{ name: 'Test Author', email: 'test@example.com' }],
require: {
php: '>=7.4',
},
autoload: {
'psr-4': {
'Vendor\\TestPackage\\': 'src/',
},
},
};
// Add a test PHP file
const [vendor, pkg] = vendorPackage.split('/');
const namespace = `${vendor.charAt(0).toUpperCase() + vendor.slice(1)}\\${pkg.charAt(0).toUpperCase() + pkg.slice(1).replace(/-/g, '')}`;
const testPhpContent = `<?php
namespace ${namespace};
class TestClass
{
public function greet(): string
{
return "Hello from ${vendorPackage}!";
}
}
`;
const entries: smartarchive.IArchiveEntry[] = [
{
archivePath: 'composer.json',
content: Buffer.from(JSON.stringify(composerJson, null, 2), 'utf-8'),
},
{
archivePath: 'src/TestClass.php',
content: Buffer.from(testPhpContent, 'utf-8'),
},
{
archivePath: 'README.md',
content: Buffer.from(`# ${vendorPackage}\n\nTest package`, 'utf-8'),
},
];
return zipTools.createZip(entries);
}
/**
* Helper to create a test Python wheel file (minimal ZIP structure) using smartarchive
*/
export async function createPythonWheel(
packageName: string,
version: string,
pyVersion: string = 'py3'
): Promise<Buffer> {
const zipTools = new smartarchive.ZipTools();
const normalizedName = packageName.replace(/-/g, '_');
const distInfoDir = `${normalizedName}-${version}.dist-info`;
// Create METADATA file
const metadata = `Metadata-Version: 2.1
Name: ${packageName}
Version: ${version}
Summary: Test Python package
Home-page: https://example.com
Author: Test Author
Author-email: test@example.com
License: MIT
Platform: UNKNOWN
Classifier: Programming Language :: Python :: 3
Requires-Python: >=3.7
Description-Content-Type: text/markdown
# ${packageName}
Test package for SmartRegistry
`;
// Create WHEEL file
const wheelContent = `Wheel-Version: 1.0
Generator: test 1.0.0
Root-Is-Purelib: true
Tag: ${pyVersion}-none-any
`;
// Create a simple Python module
const moduleContent = `"""${packageName} module"""
__version__ = "${version}"
def hello():
return "Hello from ${packageName}!"
`;
const entries: smartarchive.IArchiveEntry[] = [
{
archivePath: `${distInfoDir}/METADATA`,
content: Buffer.from(metadata, 'utf-8'),
},
{
archivePath: `${distInfoDir}/WHEEL`,
content: Buffer.from(wheelContent, 'utf-8'),
},
{
archivePath: `${distInfoDir}/RECORD`,
content: Buffer.from('', 'utf-8'),
},
{
archivePath: `${distInfoDir}/top_level.txt`,
content: Buffer.from(normalizedName, 'utf-8'),
},
{
archivePath: `${normalizedName}/__init__.py`,
content: Buffer.from(moduleContent, 'utf-8'),
},
];
return zipTools.createZip(entries);
}
/**
* Helper to create a test Python source distribution (sdist) using smartarchive
*/
export async function createPythonSdist(
packageName: string,
version: string
): Promise<Buffer> {
const tarTools = new smartarchive.TarTools();
const normalizedName = packageName.replace(/-/g, '_');
const dirPrefix = `${packageName}-${version}`;
// PKG-INFO
const pkgInfo = `Metadata-Version: 2.1
Name: ${packageName}
Version: ${version}
Summary: Test Python package
Home-page: https://example.com
Author: Test Author
Author-email: test@example.com
License: MIT
`;
// setup.py
const setupPy = `from setuptools import setup, find_packages
setup(
name="${packageName}",
version="${version}",
packages=find_packages(),
python_requires=">=3.7",
)
`;
// Module file
const moduleContent = `"""${packageName} module"""
__version__ = "${version}"
def hello():
return "Hello from ${packageName}!"
`;
const entries: smartarchive.IArchiveEntry[] = [
{
archivePath: `${dirPrefix}/PKG-INFO`,
content: Buffer.from(pkgInfo, 'utf-8'),
},
{
archivePath: `${dirPrefix}/setup.py`,
content: Buffer.from(setupPy, 'utf-8'),
},
{
archivePath: `${dirPrefix}/${normalizedName}/__init__.py`,
content: Buffer.from(moduleContent, 'utf-8'),
},
];
return tarTools.packFilesToTarGz(entries);
}
/**
* Helper to calculate PyPI file hashes
*/
export function calculatePypiHashes(data: Buffer) {
return {
md5: crypto.createHash('md5').update(data).digest('hex'),
sha256: crypto.createHash('sha256').update(data).digest('hex'),
blake2b: crypto.createHash('blake2b512').update(data).digest('hex'),
};
}
/**
* Helper to create a test RubyGem file (minimal tar.gz structure) using smartarchive
*/
export async function createRubyGem(
gemName: string,
version: string,
platform: string = 'ruby'
): Promise<Buffer> {
const tarTools = new smartarchive.TarTools();
const gzipTools = new smartarchive.GzipTools();
// Create metadata.gz (simplified)
const metadataYaml = `--- !ruby/object:Gem::Specification
name: ${gemName}
version: !ruby/object:Gem::Version
version: ${version}
platform: ${platform}
authors:
- Test Author
autorequire:
bindir: bin
cert_chain: []
date: ${new Date().toISOString().split('T')[0]}
dependencies: []
description: Test RubyGem
email: test@example.com
executables: []
extensions: []
extra_rdoc_files: []
files:
- lib/${gemName}.rb
homepage: https://example.com
licenses:
- MIT
metadata: {}
post_install_message:
rdoc_options: []
require_paths:
- lib
required_ruby_version: !ruby/object:Gem::Requirement
requirements:
- - ">="
- !ruby/object:Gem::Version
version: '2.7'
required_rubygems_version: !ruby/object:Gem::Requirement
requirements:
- - ">="
- !ruby/object:Gem::Version
version: '0'
requirements: []
rubygems_version: 3.0.0
signing_key:
specification_version: 4
summary: Test gem for SmartRegistry
test_files: []
`;
const metadataGz = await gzipTools.compress(Buffer.from(metadataYaml, 'utf-8'));
// Create data.tar.gz content
const libContent = `# ${gemName}
module ${gemName.charAt(0).toUpperCase() + gemName.slice(1).replace(/-/g, '')}
VERSION = "${version}"
def self.hello
"Hello from #{gemName}!"
end
end
`;
const dataEntries: smartarchive.IArchiveEntry[] = [
{
archivePath: `lib/${gemName}.rb`,
content: Buffer.from(libContent, 'utf-8'),
},
];
const dataTarGz = await tarTools.packFilesToTarGz(dataEntries);
// Create the outer gem (tar.gz containing metadata.gz and data.tar.gz)
const gemEntries: smartarchive.IArchiveEntry[] = [
{
archivePath: 'metadata.gz',
content: metadataGz,
},
{
archivePath: 'data.tar.gz',
content: dataTarGz,
},
];
// RubyGems .gem files are plain tar archives (NOT gzipped), containing metadata.gz and data.tar.gz
return tarTools.packFiles(gemEntries);
}
/**
* Helper to calculate RubyGems checksums
*/
export function calculateRubyGemsChecksums(data: Buffer) {
return {
md5: crypto.createHash('md5').update(data).digest('hex'),
sha256: crypto.createHash('sha256').update(data).digest('hex'),
};
}
// ============================================================================
// Enterprise Extensibility Test Helpers
// ============================================================================
/**
* Create a mock auth provider for testing pluggable authentication.
* Allows customizing behavior for different test scenarios.
*/
export function createMockAuthProvider(overrides?: Partial<IAuthProvider>): IAuthProvider {
const tokens = new Map<string, IAuthToken>();
return {
init: async () => {},
authenticate: async (credentials) => {
// Default: always authenticate successfully
return credentials.username;
},
validateToken: async (token, protocol) => {
const stored = tokens.get(token);
if (stored && (!protocol || stored.type === protocol)) {
return stored;
}
// Mock token for tests
if (token === 'valid-mock-token') {
return {
type: 'npm' as TRegistryProtocol,
userId: 'mock-user',
scopes: ['npm:*:*:*'],
};
}
return null;
},
createToken: async (userId, protocol, options) => {
const tokenId = `mock-${protocol}-${Date.now()}`;
const authToken: IAuthToken = {
type: protocol,
userId,
scopes: options?.scopes || [`${protocol}:*:*:*`],
readonly: options?.readonly,
expiresAt: options?.expiresIn ? new Date(Date.now() + options.expiresIn * 1000) : undefined,
};
tokens.set(tokenId, authToken);
return tokenId;
},
revokeToken: async (token) => {
tokens.delete(token);
},
authorize: async (token, resource, action) => {
if (!token) return false;
if (token.readonly && ['write', 'push', 'delete'].includes(action)) {
return false;
}
return true;
},
listUserTokens: async (userId) => {
const result: Array<{ key: string; readonly: boolean; created: string; protocol?: TRegistryProtocol }> = [];
for (const [key, token] of tokens.entries()) {
if (token.userId === userId) {
result.push({
key: `hash-${key.substring(0, 8)}`,
readonly: token.readonly || false,
created: new Date().toISOString(),
protocol: token.type,
});
}
}
return result;
},
...overrides,
};
}
/**
* Create test storage hooks that track all calls.
* Useful for verifying hook invocation order and parameters.
*/
export function createTrackingHooks(options?: {
beforePutAllowed?: boolean;
beforeDeleteAllowed?: boolean;
throwOnAfterPut?: boolean;
throwOnAfterGet?: boolean;
}): {
hooks: IStorageHooks;
calls: Array<{ method: string; context: IStorageHookContext; timestamp: number }>;
} {
const calls: Array<{ method: string; context: IStorageHookContext; timestamp: number }> = [];
return {
calls,
hooks: {
beforePut: async (ctx) => {
calls.push({ method: 'beforePut', context: ctx, timestamp: Date.now() });
return {
allowed: options?.beforePutAllowed !== false,
reason: options?.beforePutAllowed === false ? 'Blocked by test' : undefined,
};
},
afterPut: async (ctx) => {
calls.push({ method: 'afterPut', context: ctx, timestamp: Date.now() });
if (options?.throwOnAfterPut) {
throw new Error('Test error in afterPut');
}
},
beforeDelete: async (ctx) => {
calls.push({ method: 'beforeDelete', context: ctx, timestamp: Date.now() });
return {
allowed: options?.beforeDeleteAllowed !== false,
reason: options?.beforeDeleteAllowed === false ? 'Blocked by test' : undefined,
};
},
afterDelete: async (ctx) => {
calls.push({ method: 'afterDelete', context: ctx, timestamp: Date.now() });
},
afterGet: async (ctx) => {
calls.push({ method: 'afterGet', context: ctx, timestamp: Date.now() });
if (options?.throwOnAfterGet) {
throw new Error('Test error in afterGet');
}
},
},
};
}
/**
* Create a blocking storage hooks implementation for quota testing.
*/
export function createQuotaHooks(maxSizeBytes: number): {
hooks: IStorageHooks;
currentUsage: { bytes: number };
} {
const currentUsage = { bytes: 0 };
return {
currentUsage,
hooks: {
beforePut: async (ctx) => {
const size = ctx.metadata?.size || 0;
if (currentUsage.bytes + size > maxSizeBytes) {
return { allowed: false, reason: `Quota exceeded: ${currentUsage.bytes + size} > ${maxSizeBytes}` };
}
return { allowed: true };
},
afterPut: async (ctx) => {
currentUsage.bytes += ctx.metadata?.size || 0;
},
afterDelete: async (ctx) => {
currentUsage.bytes -= ctx.metadata?.size || 0;
if (currentUsage.bytes < 0) currentUsage.bytes = 0;
},
},
};
}
/**
* Create a SmartBucket storage backend for upstream cache testing.
*/
export async function createTestStorageBackend(): Promise<{
storage: {
getObject: (key: string) => Promise<Buffer | null>;
putObject: (key: string, data: Buffer) => Promise<void>;
deleteObject: (key: string) => Promise<void>;
listObjects: (prefix: string) => Promise<string[]>;
};
bucket: smartbucket.Bucket;
cleanup: () => Promise<void>;
}> {
const s3AccessKey = await testQenv.getEnvVarOnDemand('S3_ACCESSKEY');
const s3SecretKey = await testQenv.getEnvVarOnDemand('S3_SECRETKEY');
const s3Endpoint = await testQenv.getEnvVarOnDemand('S3_ENDPOINT');
const s3Port = await testQenv.getEnvVarOnDemand('S3_PORT');
const s3 = new smartbucket.SmartBucket({
accessKey: s3AccessKey || 'minioadmin',
accessSecret: s3SecretKey || 'minioadmin',
endpoint: s3Endpoint || 'localhost',
port: parseInt(s3Port || '9000', 10),
useSsl: false,
});
const testRunId = generateTestRunId();
const bucketName = 'test-cache-' + testRunId.substring(0, 8);
const bucket = await s3.createBucket(bucketName);
const storage = {
getObject: async (key: string): Promise<Buffer | null> => {
try {
const file = await bucket.fastGet({ path: key });
if (!file) return null;
const stream = await file.createReadStream();
const chunks: Buffer[] = [];
for await (const chunk of stream) {
chunks.push(Buffer.from(chunk));
}
return Buffer.concat(chunks);
} catch {
return null;
}
},
putObject: async (key: string, data: Buffer): Promise<void> => {
await bucket.fastPut({ path: key, contents: data, overwrite: true });
},
deleteObject: async (key: string): Promise<void> => {
await bucket.fastRemove({ path: key });
},
listObjects: async (prefix: string): Promise<string[]> => {
const files = await bucket.fastList({ prefix });
return files.map(f => f.name);
},
};
const cleanup = async () => {
try {
const files = await bucket.fastList({});
for (const file of files) {
await bucket.fastRemove({ path: file.name });
}
await s3.removeBucket(bucketName);
} catch {
// Ignore cleanup errors
}
};
return { storage, bucket, cleanup };
} }
+122
View File
@@ -0,0 +1,122 @@
import * as qenv from '@push.rocks/qenv';
import type { IRegistryConfig } from '../../ts/core/interfaces.core.js';
import type { IStorageHooks } from '../../ts/core/interfaces.storage.js';
import { StaticUpstreamProvider } from '../../ts/upstream/interfaces.upstream.js';
import type { IUpstreamProvider } from '../../ts/upstream/interfaces.upstream.js';
const testQenv = new qenv.Qenv('./', './.nogit');
async function getTestStorageConfig(): Promise<IRegistryConfig['storage']> {
const s3AccessKey = await testQenv.getEnvVarOnDemand('S3_ACCESSKEY');
const s3SecretKey = await testQenv.getEnvVarOnDemand('S3_SECRETKEY');
const s3Endpoint = await testQenv.getEnvVarOnDemand('S3_ENDPOINT');
const s3Port = await testQenv.getEnvVarOnDemand('S3_PORT');
return {
accessKey: s3AccessKey || 'minioadmin',
accessSecret: s3SecretKey || 'minioadmin',
endpoint: s3Endpoint || 'localhost',
port: parseInt(s3Port || '9000', 10),
useSsl: false,
region: 'us-east-1',
bucketName: 'test-registry',
};
}
function getTestAuthConfig(): IRegistryConfig['auth'] {
return {
jwtSecret: 'test-secret-key',
tokenStore: 'memory',
npmTokens: {
enabled: true,
},
ociTokens: {
enabled: true,
realm: 'https://auth.example.com/token',
service: 'test-registry',
},
pypiTokens: {
enabled: true,
},
rubygemsTokens: {
enabled: true,
},
};
}
export function createDefaultTestUpstreamProvider(): IUpstreamProvider {
return new StaticUpstreamProvider({
npm: {
enabled: true,
upstreams: [{
id: 'npmjs',
name: 'npmjs',
url: 'https://registry.npmjs.org',
priority: 1,
enabled: true,
auth: { type: 'none' },
}],
},
oci: {
enabled: true,
upstreams: [{
id: 'dockerhub',
name: 'dockerhub',
url: 'https://registry-1.docker.io',
priority: 1,
enabled: true,
auth: { type: 'none' },
}],
},
});
}
export async function buildTestRegistryConfig(options?: {
registryUrl?: string;
storageHooks?: IStorageHooks;
upstreamProvider?: IUpstreamProvider;
}): Promise<IRegistryConfig> {
const config: IRegistryConfig = {
storage: await getTestStorageConfig(),
auth: getTestAuthConfig(),
...(options?.storageHooks ? { storageHooks: options.storageHooks } : {}),
...(options?.upstreamProvider ? { upstreamProvider: options.upstreamProvider } : {}),
oci: {
enabled: true,
basePath: '/oci',
...(options?.registryUrl ? { registryUrl: `${options.registryUrl}/oci` } : {}),
},
npm: {
enabled: true,
basePath: '/npm',
...(options?.registryUrl ? { registryUrl: `${options.registryUrl}/npm` } : {}),
},
maven: {
enabled: true,
basePath: '/maven',
...(options?.registryUrl ? { registryUrl: `${options.registryUrl}/maven` } : {}),
},
composer: {
enabled: true,
basePath: '/composer',
...(options?.registryUrl ? { registryUrl: `${options.registryUrl}/composer` } : {}),
},
cargo: {
enabled: true,
basePath: '/cargo',
...(options?.registryUrl ? { registryUrl: `${options.registryUrl}/cargo` } : {}),
},
pypi: {
enabled: true,
basePath: '/pypi',
...(options?.registryUrl ? { registryUrl: options.registryUrl } : {}),
},
rubygems: {
enabled: true,
basePath: '/rubygems',
...(options?.registryUrl ? { registryUrl: `${options.registryUrl}/rubygems` } : {}),
},
};
return config;
}
+72
View File
@@ -0,0 +1,72 @@
import * as qenv from '@push.rocks/qenv';
import * as smartbucket from '@push.rocks/smartbucket';
import { generateTestRunId } from './ids.js';
const testQenv = new qenv.Qenv('./', './.nogit');
/**
* Create a SmartBucket storage backend for upstream cache testing.
*/
export async function createTestStorageBackend(): Promise<{
storage: {
getObject: (key: string) => Promise<Buffer | null>;
putObject: (key: string, data: Buffer) => Promise<void>;
deleteObject: (key: string) => Promise<void>;
listObjects: (prefix: string) => Promise<string[]>;
};
bucket: smartbucket.Bucket;
cleanup: () => Promise<void>;
}> {
const s3AccessKey = await testQenv.getEnvVarOnDemand('S3_ACCESSKEY');
const s3SecretKey = await testQenv.getEnvVarOnDemand('S3_SECRETKEY');
const s3Endpoint = await testQenv.getEnvVarOnDemand('S3_ENDPOINT');
const s3Port = await testQenv.getEnvVarOnDemand('S3_PORT');
const s3 = new smartbucket.SmartBucket({
accessKey: s3AccessKey || 'minioadmin',
accessSecret: s3SecretKey || 'minioadmin',
endpoint: s3Endpoint || 'localhost',
port: parseInt(s3Port || '9000', 10),
useSsl: false,
});
const testRunId = generateTestRunId();
const bucketName = 'test-cache-' + testRunId.substring(0, 8);
const bucket = await s3.createBucket(bucketName);
const storage = {
getObject: async (key: string): Promise<Buffer | null> => {
try {
return await bucket.fastGet({ path: key });
} catch {
return null;
}
},
putObject: async (key: string, data: Buffer): Promise<void> => {
await bucket.fastPut({ path: key, contents: data, overwrite: true });
},
deleteObject: async (key: string): Promise<void> => {
await bucket.fastRemove({ path: key });
},
listObjects: async (prefix: string): Promise<string[]> => {
const paths: string[] = [];
for await (const path of bucket.listAllObjects(prefix)) {
paths.push(path);
}
return paths;
},
};
const cleanup = async () => {
try {
for await (const path of bucket.listAllObjects()) {
await bucket.fastRemove({ path });
}
await s3.removeBucket(bucketName);
} catch {
// Ignore cleanup errors
}
};
return { storage, bucket, cleanup };
}
+82
View File
@@ -0,0 +1,82 @@
import type { IStorageHooks, IStorageHookContext } from '../../ts/core/interfaces.storage.js';
/**
* Create test storage hooks that track all calls.
* Useful for verifying hook invocation order and parameters.
*/
export function createTrackingHooks(options?: {
beforePutAllowed?: boolean;
beforeDeleteAllowed?: boolean;
throwOnAfterPut?: boolean;
throwOnAfterGet?: boolean;
}): {
hooks: IStorageHooks;
calls: Array<{ method: string; context: IStorageHookContext; timestamp: number }>;
} {
const calls: Array<{ method: string; context: IStorageHookContext; timestamp: number }> = [];
return {
calls,
hooks: {
beforePut: async (ctx) => {
calls.push({ method: 'beforePut', context: ctx, timestamp: Date.now() });
return {
allowed: options?.beforePutAllowed !== false,
reason: options?.beforePutAllowed === false ? 'Blocked by test' : undefined,
};
},
afterPut: async (ctx) => {
calls.push({ method: 'afterPut', context: ctx, timestamp: Date.now() });
if (options?.throwOnAfterPut) {
throw new Error('Test error in afterPut');
}
},
beforeDelete: async (ctx) => {
calls.push({ method: 'beforeDelete', context: ctx, timestamp: Date.now() });
return {
allowed: options?.beforeDeleteAllowed !== false,
reason: options?.beforeDeleteAllowed === false ? 'Blocked by test' : undefined,
};
},
afterDelete: async (ctx) => {
calls.push({ method: 'afterDelete', context: ctx, timestamp: Date.now() });
},
afterGet: async (ctx) => {
calls.push({ method: 'afterGet', context: ctx, timestamp: Date.now() });
if (options?.throwOnAfterGet) {
throw new Error('Test error in afterGet');
}
},
},
};
}
/**
* Create a blocking storage hooks implementation for quota testing.
*/
export function createQuotaHooks(maxSizeBytes: number): {
hooks: IStorageHooks;
currentUsage: { bytes: number };
} {
const currentUsage = { bytes: 0 };
return {
currentUsage,
hooks: {
beforePut: async (ctx) => {
const size = ctx.metadata?.size || 0;
if (currentUsage.bytes + size > maxSizeBytes) {
return { allowed: false, reason: `Quota exceeded: ${currentUsage.bytes + size} > ${maxSizeBytes}` };
}
return { allowed: true };
},
afterPut: async (ctx) => {
currentUsage.bytes += ctx.metadata?.size || 0;
},
afterDelete: async (ctx) => {
currentUsage.bytes -= ctx.metadata?.size || 0;
if (currentUsage.bytes < 0) currentUsage.bytes = 0;
},
},
};
}
+27
View File
@@ -0,0 +1,27 @@
import type { SmartRegistry } from '../../ts/classes.smartregistry.js';
/**
* Helper to create test authentication tokens.
*/
export async function createTestTokens(registry: SmartRegistry) {
const authManager = registry.getAuthManager();
const userId = await authManager.authenticate({
username: 'testuser',
password: 'testpass',
});
if (!userId) {
throw new Error('Failed to authenticate test user');
}
const npmToken = await authManager.createNpmToken(userId, false);
const ociToken = await authManager.createOciToken(userId, ['oci:repository:*:*'], 3600);
const mavenToken = await authManager.createMavenToken(userId, false);
const composerToken = await authManager.createComposerToken(userId, false);
const cargoToken = await authManager.createCargoToken(userId, false);
const pypiToken = await authManager.createPypiToken(userId, false);
const rubygemsToken = await authManager.createRubyGemsToken(userId, false);
return { npmToken, ociToken, mavenToken, composerToken, cargoToken, pypiToken, rubygemsToken, userId };
}
+8 -15
View File
@@ -79,16 +79,10 @@ async function createHttpServer(
res.setHeader(key, value); res.setHeader(key, value);
} }
// Send body // Send body (response.body is always ReadableStream<Uint8Array> or undefined)
if (response.body) { if (response.body) {
if (Buffer.isBuffer(response.body)) { const { Readable } = await import('stream');
res.end(response.body); Readable.fromWeb(response.body).pipe(res);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
} else { } else {
res.end(); res.end();
} }
@@ -251,8 +245,11 @@ function cleanupTestDir(dir: string): void {
// ======================================================================== // ========================================================================
tap.test('Cargo CLI: should setup registry and HTTP server', async () => { tap.test('Cargo CLI: should setup registry and HTTP server', async () => {
// Create registry // Use port 5000
registry = await createTestRegistry(); registryPort = 5000;
// Create registry with correct registryUrl for CLI tests
registry = await createTestRegistry({ registryUrl: `http://localhost:${registryPort}` });
const tokens = await createTestTokens(registry); const tokens = await createTestTokens(registry);
cargoToken = tokens.cargoToken; cargoToken = tokens.cargoToken;
@@ -266,10 +263,6 @@ tap.test('Cargo CLI: should setup registry and HTTP server', async () => {
} catch (error) { } catch (error) {
// Ignore error if operation fails // Ignore error if operation fails
} }
// Use port 5000 (hardcoded in CargoRegistry default config)
// TODO: Once registryUrl is configurable, use dynamic port like npm test (35001)
registryPort = 5000;
const serverSetup = await createHttpServer(registry, registryPort); const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server; server = serverSetup.server;
registryUrl = serverSetup.url; registryUrl = serverSetup.url;
+8 -14
View File
@@ -84,16 +84,10 @@ async function createHttpServer(
res.setHeader(key, value); res.setHeader(key, value);
} }
// Send body // Send body (response.body is always ReadableStream<Uint8Array> or undefined)
if (response.body) { if (response.body) {
if (Buffer.isBuffer(response.body)) { const { Readable } = await import('stream');
res.end(response.body); Readable.fromWeb(response.body).pipe(res);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
} else { } else {
res.end(); res.end();
} }
@@ -249,16 +243,16 @@ tap.test('Composer CLI: should verify composer is installed', async () => {
}); });
tap.test('Composer CLI: should setup registry and HTTP server', async () => { tap.test('Composer CLI: should setup registry and HTTP server', async () => {
// Create registry // Use port 38000 (avoids conflicts with other tests)
registry = await createTestRegistry(); registryPort = 38000;
// Create registry with correct registryUrl for CLI tests
registry = await createTestRegistry({ registryUrl: `http://localhost:${registryPort}` });
const tokens = await createTestTokens(registry); const tokens = await createTestTokens(registry);
composerToken = tokens.composerToken; composerToken = tokens.composerToken;
expect(registry).toBeInstanceOf(SmartRegistry); expect(registry).toBeInstanceOf(SmartRegistry);
expect(composerToken).toBeTypeOf('string'); expect(composerToken).toBeTypeOf('string');
// Use port 38000 (avoids conflicts with other tests)
registryPort = 38000;
const serverSetup = await createHttpServer(registry, registryPort); const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server; server = serverSetup.server;
registryUrl = serverSetup.url; registryUrl = serverSetup.url;
+43 -29
View File
@@ -1,5 +1,6 @@
import { expect, tap } from '@git.zone/tstest/tapbundle'; import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js'; import { SmartRegistry } from '../ts/index.js';
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
import { createTestRegistry, createTestTokens, createComposerZip } from './helpers/registry.js'; import { createTestRegistry, createTestTokens, createComposerZip } from './helpers/registry.js';
let registry: SmartRegistry; let registry: SmartRegistry;
@@ -41,9 +42,10 @@ tap.test('Composer: should return packages.json (GET /packages.json)', async ()
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('metadata-url'); const body = await streamToJson(response.body);
expect(response.body).toHaveProperty('available-packages'); expect(body).toHaveProperty('metadata-url');
expect(response.body['available-packages']).toBeInstanceOf(Array); expect(body).toHaveProperty('available-packages');
expect(body['available-packages']).toBeInstanceOf(Array);
}); });
tap.test('Composer: should upload a package (PUT /packages/{vendor/package})', async () => { tap.test('Composer: should upload a package (PUT /packages/{vendor/package})', async () => {
@@ -59,9 +61,10 @@ tap.test('Composer: should upload a package (PUT /packages/{vendor/package})', a
}); });
expect(response.status).toEqual(201); expect(response.status).toEqual(201);
expect(response.body.status).toEqual('success'); const body = await streamToJson(response.body);
expect(response.body.package).toEqual(testPackageName); expect(body.status).toEqual('success');
expect(response.body.version).toEqual(testVersion); expect(body.package).toEqual(testPackageName);
expect(body.version).toEqual(testVersion);
}); });
tap.test('Composer: should retrieve package metadata (GET /p2/{vendor/package}.json)', async () => { tap.test('Composer: should retrieve package metadata (GET /p2/{vendor/package}.json)', async () => {
@@ -73,11 +76,12 @@ tap.test('Composer: should retrieve package metadata (GET /p2/{vendor/package}.j
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('packages'); const body = await streamToJson(response.body);
expect(response.body.packages[testPackageName]).toBeInstanceOf(Array); expect(body).toHaveProperty('packages');
expect(response.body.packages[testPackageName].length).toEqual(1); expect(body.packages[testPackageName]).toBeInstanceOf(Array);
expect(body.packages[testPackageName].length).toEqual(1);
const packageData = response.body.packages[testPackageName][0]; const packageData = body.packages[testPackageName][0];
expect(packageData.name).toEqual(testPackageName); expect(packageData.name).toEqual(testPackageName);
expect(packageData.version).toEqual(testVersion); expect(packageData.version).toEqual(testVersion);
expect(packageData.version_normalized).toEqual('1.0.0.0'); expect(packageData.version_normalized).toEqual('1.0.0.0');
@@ -97,7 +101,8 @@ tap.test('Composer: should download package ZIP (GET /dists/{vendor/package}/{re
query: {}, query: {},
}); });
const reference = metadataResponse.body.packages[testPackageName][0].dist.reference; const metaBody = await streamToJson(metadataResponse.body);
const reference = metaBody.packages[testPackageName][0].dist.reference;
const response = await registry.handleRequest({ const response = await registry.handleRequest({
method: 'GET', method: 'GET',
@@ -107,7 +112,8 @@ tap.test('Composer: should download package ZIP (GET /dists/{vendor/package}/{re
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
expect(response.headers['Content-Type']).toEqual('application/zip'); expect(response.headers['Content-Type']).toEqual('application/zip');
expect(response.headers['Content-Disposition']).toContain('attachment'); expect(response.headers['Content-Disposition']).toContain('attachment');
}); });
@@ -121,9 +127,10 @@ tap.test('Composer: should list packages (GET /packages/list.json)', async () =>
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('packageNames'); const body = await streamToJson(response.body);
expect(response.body.packageNames).toBeInstanceOf(Array); expect(body).toHaveProperty('packageNames');
expect(response.body.packageNames).toContain(testPackageName); expect(body.packageNames).toBeInstanceOf(Array);
expect(body.packageNames).toContain(testPackageName);
}); });
tap.test('Composer: should filter package list (GET /packages/list.json?filter=vendor/*)', async () => { tap.test('Composer: should filter package list (GET /packages/list.json?filter=vendor/*)', async () => {
@@ -135,8 +142,9 @@ tap.test('Composer: should filter package list (GET /packages/list.json?filter=v
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body.packageNames).toBeInstanceOf(Array); const body = await streamToJson(response.body);
expect(response.body.packageNames).toContain(testPackageName); expect(body.packageNames).toBeInstanceOf(Array);
expect(body.packageNames).toContain(testPackageName);
}); });
tap.test('Composer: should prevent duplicate version upload', async () => { tap.test('Composer: should prevent duplicate version upload', async () => {
@@ -152,8 +160,9 @@ tap.test('Composer: should prevent duplicate version upload', async () => {
}); });
expect(response.status).toEqual(409); expect(response.status).toEqual(409);
expect(response.body.status).toEqual('error'); const body = await streamToJson(response.body);
expect(response.body.message).toContain('already exists'); expect(body.status).toEqual('error');
expect(body.message).toContain('already exists');
}); });
tap.test('Composer: should upload a second version', async () => { tap.test('Composer: should upload a second version', async () => {
@@ -172,8 +181,9 @@ tap.test('Composer: should upload a second version', async () => {
}); });
expect(response.status).toEqual(201); expect(response.status).toEqual(201);
expect(response.body.status).toEqual('success'); const body = await streamToJson(response.body);
expect(response.body.version).toEqual(testVersion2); expect(body.status).toEqual('success');
expect(body.version).toEqual(testVersion2);
}); });
tap.test('Composer: should return multiple versions in metadata', async () => { tap.test('Composer: should return multiple versions in metadata', async () => {
@@ -185,10 +195,11 @@ tap.test('Composer: should return multiple versions in metadata', async () => {
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body.packages[testPackageName]).toBeInstanceOf(Array); const body = await streamToJson(response.body);
expect(response.body.packages[testPackageName].length).toEqual(2); expect(body.packages[testPackageName]).toBeInstanceOf(Array);
expect(body.packages[testPackageName].length).toEqual(2);
const versions = response.body.packages[testPackageName].map((p: any) => p.version); const versions = body.packages[testPackageName].map((p: any) => p.version);
expect(versions).toContain('1.0.0'); expect(versions).toContain('1.0.0');
expect(versions).toContain('1.1.0'); expect(versions).toContain('1.1.0');
}); });
@@ -213,8 +224,9 @@ tap.test('Composer: should delete a specific version (DELETE /packages/{vendor/p
query: {}, query: {},
}); });
expect(metadataResponse.body.packages[testPackageName].length).toEqual(1); const metaBody = await streamToJson(metadataResponse.body);
expect(metadataResponse.body.packages[testPackageName][0].version).toEqual('1.1.0'); expect(metaBody.packages[testPackageName].length).toEqual(1);
expect(metaBody.packages[testPackageName][0].version).toEqual('1.1.0');
}); });
tap.test('Composer: should require auth for package upload', async () => { tap.test('Composer: should require auth for package upload', async () => {
@@ -231,7 +243,8 @@ tap.test('Composer: should require auth for package upload', async () => {
}); });
expect(response.status).toEqual(401); expect(response.status).toEqual(401);
expect(response.body.status).toEqual('error'); const body = await streamToJson(response.body);
expect(body.status).toEqual('error');
}); });
tap.test('Composer: should reject invalid ZIP (no composer.json)', async () => { tap.test('Composer: should reject invalid ZIP (no composer.json)', async () => {
@@ -249,8 +262,9 @@ tap.test('Composer: should reject invalid ZIP (no composer.json)', async () => {
}); });
expect(response.status).toEqual(400); expect(response.status).toEqual(400);
expect(response.body.status).toEqual('error'); const body = await streamToJson(response.body);
expect(response.body.message).toContain('composer.json'); expect(body.status).toEqual('error');
expect(body.message).toContain('composer.json');
}); });
tap.test('Composer: should delete entire package (DELETE /packages/{vendor/package})', async () => { tap.test('Composer: should delete entire package (DELETE /packages/{vendor/package})', async () => {
+7 -3
View File
@@ -1,5 +1,6 @@
import { expect, tap } from '@git.zone/tstest/tapbundle'; import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js'; import { SmartRegistry } from '../ts/index.js';
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
import { import {
createTestRegistry, createTestRegistry,
createTestTokens, createTestTokens,
@@ -79,7 +80,9 @@ tap.test('Integration: should handle /simple path for PyPI', async () => {
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toStartWith('text/html'); expect(response.headers['Content-Type']).toStartWith('text/html');
expect(response.body).toContain('integration-test-py'); const body = await streamToBuffer(response.body);
const text = body.toString('utf-8');
expect(text).toContain('integration-test-py');
}); });
tap.test('Integration: should reject PyPI token for RubyGems endpoint', async () => { tap.test('Integration: should reject PyPI token for RubyGems endpoint', async () => {
@@ -135,8 +138,9 @@ tap.test('Integration: should return 404 for unknown paths', async () => {
}); });
expect(response.status).toEqual(404); expect(response.status).toEqual(404);
expect(response.body).toHaveProperty('error'); const body = await streamToJson(response.body);
expect((response.body as any).error).toEqual('NOT_FOUND'); expect(body).toHaveProperty('error');
expect(body.error).toEqual('NOT_FOUND');
}); });
tap.test('Integration: should retrieve PyPI registry instance', async () => { tap.test('Integration: should retrieve PyPI registry instance', async () => {
+28 -44
View File
@@ -1,32 +1,34 @@
/** /**
* Integration test for smartregistry with smarts3 * Integration test for smartregistry with smartstorage
* Verifies that smartregistry works with a local S3-compatible server * Verifies that smartregistry works with a local S3-compatible server
*/ */
import { expect, tap } from '@git.zone/tstest/tapbundle'; import { expect, tap } from '@git.zone/tstest/tapbundle';
import * as smarts3Module from '@push.rocks/smarts3'; import * as smartstorageModule from '@push.rocks/smartstorage';
import { SmartRegistry } from '../ts/classes.smartregistry.js'; import { SmartRegistry } from '../ts/classes.smartregistry.js';
import type { IRegistryConfig } from '../ts/core/interfaces.core.js'; import type { IRegistryConfig } from '../ts/core/interfaces.core.js';
import { streamToJson } from '../ts/core/helpers.stream.js';
import * as crypto from 'crypto'; import * as crypto from 'crypto';
let s3Server: smarts3Module.Smarts3; let s3Server: smartstorageModule.SmartStorage;
let registry: SmartRegistry; let registry: SmartRegistry;
/** /**
* Setup: Start smarts3 server * Setup: Start smartstorage server
*/ */
tap.test('should start smarts3 server', async () => { tap.test('should start smartstorage server', async () => {
s3Server = await smarts3Module.Smarts3.createAndStart({ s3Server = await smartstorageModule.SmartStorage.createAndStart({
server: { server: {
port: 3456, // Use different port to avoid conflicts with other tests port: 3456,
host: '0.0.0.0', address: '0.0.0.0',
silent: true,
}, },
storage: { storage: {
cleanSlate: true, // Fresh storage for each test run cleanSlate: true,
bucketsDir: './.nogit/smarts3-test-buckets', directory: './.nogit/smartstorage-test-buckets',
}, },
logging: { logging: {
silent: true, // Reduce test output noise enabled: false,
}, },
}); });
@@ -34,20 +36,10 @@ tap.test('should start smarts3 server', async () => {
}); });
/** /**
* Setup: Create SmartRegistry with smarts3 configuration * Setup: Create SmartRegistry with smartstorage configuration
*/ */
tap.test('should create SmartRegistry instance with smarts3 IS3Descriptor', async () => { tap.test('should create SmartRegistry instance with smarts3 IS3Descriptor', async () => {
// Manually construct IS3Descriptor based on smarts3 configuration const s3Descriptor = await s3Server.getStorageDescriptor();
// Note: smarts3.getS3Descriptor() returns empty object as of v5.1.0
// This is a known limitation - smarts3 doesn't expose its config properly
const s3Descriptor = {
endpoint: 'localhost',
port: 3456,
accessKey: 'test', // smarts3 doesn't require real credentials
accessSecret: 'test',
useSsl: false,
region: 'us-east-1',
};
const config: IRegistryConfig = { const config: IRegistryConfig = {
storage: { storage: {
@@ -97,7 +89,7 @@ tap.test('should create SmartRegistry instance with smarts3 IS3Descriptor', asyn
}); });
/** /**
* Test NPM protocol with smarts3 * Test NPM protocol with smartstorage
*/ */
tap.test('NPM: should publish package to smarts3', async () => { tap.test('NPM: should publish package to smarts3', async () => {
const authManager = registry.getAuthManager(); const authManager = registry.getAuthManager();
@@ -139,7 +131,7 @@ tap.test('NPM: should publish package to smarts3', async () => {
body: packageData, body: packageData,
}); });
expect(response.status).toEqual(201); // 201 Created is correct for publishing expect(response.status).toEqual(201);
}); });
tap.test('NPM: should retrieve package from smarts3', async () => { tap.test('NPM: should retrieve package from smarts3', async () => {
@@ -151,12 +143,13 @@ tap.test('NPM: should retrieve package from smarts3', async () => {
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('name'); const body = await streamToJson(response.body);
expect(response.body.name).toEqual('test-package-smarts3'); expect(body).toHaveProperty('name');
expect(body.name).toEqual('test-package-smarts3');
}); });
/** /**
* Test OCI protocol with smarts3 * Test OCI protocol with smartstorage
*/ */
tap.test('OCI: should store blob in smarts3', async () => { tap.test('OCI: should store blob in smarts3', async () => {
const authManager = registry.getAuthManager(); const authManager = registry.getAuthManager();
@@ -173,7 +166,7 @@ tap.test('OCI: should store blob in smarts3', async () => {
// Initiate blob upload // Initiate blob upload
const initiateResponse = await registry.handleRequest({ const initiateResponse = await registry.handleRequest({
method: 'POST', method: 'POST',
path: '/oci/v2/test-image/blobs/uploads/', path: '/oci/test-image/blobs/uploads/',
headers: { headers: {
'Authorization': `Bearer ${token}`, 'Authorization': `Bearer ${token}`,
}, },
@@ -196,7 +189,7 @@ tap.test('OCI: should store blob in smarts3', async () => {
const uploadResponse = await registry.handleRequest({ const uploadResponse = await registry.handleRequest({
method: 'PUT', method: 'PUT',
path: `/oci/v2/test-image/blobs/uploads/${uploadId}`, path: `/oci/test-image/blobs/uploads/${uploadId}`,
headers: { headers: {
'Authorization': `Bearer ${token}`, 'Authorization': `Bearer ${token}`,
'Content-Type': 'application/octet-stream', 'Content-Type': 'application/octet-stream',
@@ -209,18 +202,9 @@ tap.test('OCI: should store blob in smarts3', async () => {
}); });
/** /**
* Test PyPI protocol with smarts3 * Test PyPI protocol with smartstorage
*/ */
tap.test('PyPI: should upload package to smarts3', async () => { tap.test('PyPI: should upload package to smarts3', async () => {
const authManager = registry.getAuthManager();
const userId = await authManager.authenticate({
username: 'testuser',
password: 'testpass',
});
const token = await authManager.createPypiToken(userId, false);
// Note: In a real test, this would be multipart/form-data
// For simplicity, we're testing the storage layer
const storage = registry.getStorage(); const storage = registry.getStorage();
// Store a test package file // Store a test package file
@@ -252,7 +236,7 @@ tap.test('PyPI: should upload package to smarts3', async () => {
}); });
/** /**
* Test Cargo protocol with smarts3 * Test Cargo protocol with smartstorage
*/ */
tap.test('Cargo: should store crate in smarts3', async () => { tap.test('Cargo: should store crate in smarts3', async () => {
const storage = registry.getStorage(); const storage = registry.getStorage();
@@ -281,11 +265,11 @@ tap.test('Cargo: should store crate in smarts3', async () => {
}); });
/** /**
* Cleanup: Stop smarts3 server * Cleanup: Stop smartstorage server
*/ */
tap.test('should stop smarts3 server', async () => { tap.test('should stop smartstorage server', async () => {
registry.destroy();
await s3Server.stop(); await s3Server.stop();
expect(true).toEqual(true); // Just verify it completes without error
}); });
export default tap.start(); export default tap.start();
+8 -14
View File
@@ -79,16 +79,10 @@ async function createHttpServer(
res.setHeader(key, value); res.setHeader(key, value);
} }
// Send body // Send body (response.body is always ReadableStream<Uint8Array> or undefined)
if (response.body) { if (response.body) {
if (Buffer.isBuffer(response.body)) { const { Readable } = await import('stream');
res.end(response.body); Readable.fromWeb(response.body).pipe(res);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
} else { } else {
res.end(); res.end();
} }
@@ -282,16 +276,16 @@ tap.test('Maven CLI: should verify mvn is installed', async () => {
}); });
tap.test('Maven CLI: should setup registry and HTTP server', async () => { tap.test('Maven CLI: should setup registry and HTTP server', async () => {
// Create registry // Use port 37000 (avoids conflicts with other tests)
registry = await createTestRegistry(); registryPort = 37000;
// Create registry with correct registryUrl for CLI tests
registry = await createTestRegistry({ registryUrl: `http://localhost:${registryPort}` });
const tokens = await createTestTokens(registry); const tokens = await createTestTokens(registry);
mavenToken = tokens.mavenToken; mavenToken = tokens.mavenToken;
expect(registry).toBeInstanceOf(SmartRegistry); expect(registry).toBeInstanceOf(SmartRegistry);
expect(mavenToken).toBeTypeOf('string'); expect(mavenToken).toBeTypeOf('string');
// Use port 37000 (avoids conflicts with other tests)
registryPort = 37000;
const serverSetup = await createHttpServer(registry, registryPort); const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server; server = serverSetup.server;
registryUrl = serverSetup.url; registryUrl = serverSetup.url;
+31 -19
View File
@@ -1,5 +1,6 @@
import { expect, tap } from '@git.zone/tstest/tapbundle'; import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js'; import { SmartRegistry } from '../ts/index.js';
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
import { import {
createTestRegistry, createTestRegistry,
createTestTokens, createTestTokens,
@@ -88,10 +89,11 @@ tap.test('Maven: should retrieve uploaded POM file (GET)', async () => {
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect((response.body as Buffer).toString('utf-8')).toContain(testGroupId); expect(body).toBeInstanceOf(Buffer);
expect((response.body as Buffer).toString('utf-8')).toContain(testArtifactId); expect(body.toString('utf-8')).toContain(testGroupId);
expect((response.body as Buffer).toString('utf-8')).toContain(testVersion); expect(body.toString('utf-8')).toContain(testArtifactId);
expect(body.toString('utf-8')).toContain(testVersion);
expect(response.headers['Content-Type']).toEqual('application/xml'); expect(response.headers['Content-Type']).toEqual('application/xml');
}); });
@@ -107,7 +109,8 @@ tap.test('Maven: should retrieve uploaded JAR file (GET)', async () => {
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
expect(response.headers['Content-Type']).toEqual('application/java-archive'); expect(response.headers['Content-Type']).toEqual('application/java-archive');
}); });
@@ -124,8 +127,9 @@ tap.test('Maven: should retrieve MD5 checksum for JAR (GET *.jar.md5)', async ()
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect((response.body as Buffer).toString('utf-8')).toEqual(checksums.md5); expect(body).toBeInstanceOf(Buffer);
expect(body.toString('utf-8')).toEqual(checksums.md5);
expect(response.headers['Content-Type']).toEqual('text/plain'); expect(response.headers['Content-Type']).toEqual('text/plain');
}); });
@@ -142,8 +146,9 @@ tap.test('Maven: should retrieve SHA1 checksum for JAR (GET *.jar.sha1)', async
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect((response.body as Buffer).toString('utf-8')).toEqual(checksums.sha1); expect(body).toBeInstanceOf(Buffer);
expect(body.toString('utf-8')).toEqual(checksums.sha1);
expect(response.headers['Content-Type']).toEqual('text/plain'); expect(response.headers['Content-Type']).toEqual('text/plain');
}); });
@@ -160,8 +165,9 @@ tap.test('Maven: should retrieve SHA256 checksum for JAR (GET *.jar.sha256)', as
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect((response.body as Buffer).toString('utf-8')).toEqual(checksums.sha256); expect(body).toBeInstanceOf(Buffer);
expect(body.toString('utf-8')).toEqual(checksums.sha256);
expect(response.headers['Content-Type']).toEqual('text/plain'); expect(response.headers['Content-Type']).toEqual('text/plain');
}); });
@@ -178,8 +184,9 @@ tap.test('Maven: should retrieve SHA512 checksum for JAR (GET *.jar.sha512)', as
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect((response.body as Buffer).toString('utf-8')).toEqual(checksums.sha512); expect(body).toBeInstanceOf(Buffer);
expect(body.toString('utf-8')).toEqual(checksums.sha512);
expect(response.headers['Content-Type']).toEqual('text/plain'); expect(response.headers['Content-Type']).toEqual('text/plain');
}); });
@@ -194,8 +201,9 @@ tap.test('Maven: should retrieve maven-metadata.xml (GET)', async () => {
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
const xml = (response.body as Buffer).toString('utf-8'); expect(body).toBeInstanceOf(Buffer);
const xml = body.toString('utf-8');
expect(xml).toContain('<groupId>'); expect(xml).toContain('<groupId>');
expect(xml).toContain('<artifactId>'); expect(xml).toContain('<artifactId>');
expect(xml).toContain('<version>1.0.0</version>'); expect(xml).toContain('<version>1.0.0</version>');
@@ -247,7 +255,8 @@ tap.test('Maven: should upload a second version and update metadata', async () =
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
const xml = (response.body as Buffer).toString('utf-8'); const metaBody = await streamToBuffer(response.body);
const xml = metaBody.toString('utf-8');
expect(xml).toContain('<version>1.0.0</version>'); expect(xml).toContain('<version>1.0.0</version>');
expect(xml).toContain('<version>2.0.0</version>'); expect(xml).toContain('<version>2.0.0</version>');
expect(xml).toContain('<latest>2.0.0</latest>'); expect(xml).toContain('<latest>2.0.0</latest>');
@@ -285,7 +294,8 @@ tap.test('Maven: should return 404 for non-existent artifact', async () => {
}); });
expect(response.status).toEqual(404); expect(response.status).toEqual(404);
expect(response.body).toHaveProperty('error'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
}); });
tap.test('Maven: should return 401 for unauthorized upload', async () => { tap.test('Maven: should return 401 for unauthorized upload', async () => {
@@ -304,7 +314,8 @@ tap.test('Maven: should return 401 for unauthorized upload', async () => {
}); });
expect(response.status).toEqual(401); expect(response.status).toEqual(401);
expect(response.body).toHaveProperty('error'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
}); });
tap.test('Maven: should reject POM upload with mismatched GAV', async () => { tap.test('Maven: should reject POM upload with mismatched GAV', async () => {
@@ -328,7 +339,8 @@ tap.test('Maven: should reject POM upload with mismatched GAV', async () => {
}); });
expect(response.status).toEqual(400); expect(response.status).toEqual(400);
expect(response.body).toHaveProperty('error'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
}); });
tap.test('Maven: should delete an artifact (DELETE)', async () => { tap.test('Maven: should delete an artifact (DELETE)', async () => {
+86 -111
View File
@@ -6,14 +6,14 @@
import { expect, tap } from '@git.zone/tstest/tapbundle'; import { expect, tap } from '@git.zone/tstest/tapbundle';
import { tapNodeTools } from '@git.zone/tstest/tapbundle_serverside'; import { tapNodeTools } from '@git.zone/tstest/tapbundle_serverside';
import { SmartRegistry } from '../ts/index.js'; import { SmartRegistry } from '../ts/index.js';
import { createTestRegistry, createTestTokens } from './helpers/registry.js'; import { createTestRegistry, createTestTokens, cleanupS3Bucket } from './helpers/registry.js';
import type { IRequestContext, IResponse } from '../ts/core/interfaces.core.js'; import type { IRequestContext, IResponse } from '../ts/core/interfaces.core.js';
import * as http from 'http'; import * as http from 'http';
import * as url from 'url'; import * as url from 'url';
import * as fs from 'fs'; import * as fs from 'fs';
import * as path from 'path'; import * as path from 'path';
// Test context // Test state
let registry: SmartRegistry; let registry: SmartRegistry;
let server: http.Server; let server: http.Server;
let registryUrl: string; let registryUrl: string;
@@ -32,21 +32,22 @@ async function createHttpServer(
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const httpServer = http.createServer(async (req, res) => { const httpServer = http.createServer(async (req, res) => {
try { try {
// Parse request const parsedUrl = new url.URL(req.url || '/', `http://localhost:${port}`);
const parsedUrl = url.parse(req.url || '', true); const pathname = parsedUrl.pathname;
const pathname = parsedUrl.pathname || '/'; const query: Record<string, string> = {};
const query = parsedUrl.query; parsedUrl.searchParams.forEach((value, key) => {
query[key] = value;
});
// Read body // Read body
const chunks: Buffer[] = []; let body: any = undefined;
for await (const chunk of req) { if (req.method === 'PUT' || req.method === 'POST' || req.method === 'PATCH') {
chunks.push(chunk); const chunks: Buffer[] = [];
} for await (const chunk of req) {
const bodyBuffer = Buffer.concat(chunks); chunks.push(Buffer.from(chunk));
}
const bodyBuffer = Buffer.concat(chunks);
// Parse body based on content type
let body: any;
if (bodyBuffer.length > 0) {
const contentType = req.headers['content-type'] || ''; const contentType = req.headers['content-type'] || '';
if (contentType.includes('application/json')) { if (contentType.includes('application/json')) {
try { try {
@@ -79,16 +80,10 @@ async function createHttpServer(
res.setHeader(key, value); res.setHeader(key, value);
} }
// Send body // Send body (response.body is always ReadableStream<Uint8Array> or undefined)
if (response.body) { if (response.body) {
if (Buffer.isBuffer(response.body)) { const { Readable } = await import('stream');
res.end(response.body); Readable.fromWeb(response.body).pipe(res);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
} else { } else {
res.end(); res.end();
} }
@@ -130,7 +125,7 @@ function createTestPackage(
version: string, version: string,
targetDir: string targetDir: string
): string { ): string {
const packageDir = path.join(targetDir, packageName); const packageDir = path.join(targetDir, packageName.replace(/\//g, '-'));
fs.mkdirSync(packageDir, { recursive: true }); fs.mkdirSync(packageDir, { recursive: true });
// Create package.json // Create package.json
@@ -139,12 +134,7 @@ function createTestPackage(
version: version, version: version,
description: `Test package ${packageName}`, description: `Test package ${packageName}`,
main: 'index.js', main: 'index.js',
scripts: { scripts: {},
test: 'echo "Test passed"',
},
keywords: ['test'],
author: 'Test Author',
license: 'MIT',
}; };
fs.writeFileSync( fs.writeFileSync(
@@ -153,25 +143,24 @@ function createTestPackage(
'utf-8' 'utf-8'
); );
// Create index.js // Create a simple index.js
const indexJs = `module.exports = { fs.writeFileSync(
name: '${packageName}', path.join(packageDir, 'index.js'),
version: '${version}', `module.exports = { name: '${packageName}', version: '${version}' };\n`,
message: 'Hello from ${packageName}@${version}' 'utf-8'
}; );
`;
fs.writeFileSync(path.join(packageDir, 'index.js'), indexJs, 'utf-8');
// Create README.md // Create README.md
const readme = `# ${packageName} fs.writeFileSync(
path.join(packageDir, 'README.md'),
`# ${packageName}\n\nTest package version ${version}\n`,
'utf-8'
);
Test package for SmartRegistry. // Copy .npmrc into the package directory
if (npmrcPath && fs.existsSync(npmrcPath)) {
Version: ${version} fs.copyFileSync(npmrcPath, path.join(packageDir, '.npmrc'));
`; }
fs.writeFileSync(path.join(packageDir, 'README.md'), readme, 'utf-8');
return packageDir; return packageDir;
} }
@@ -183,31 +172,30 @@ async function runNpmCommand(
command: string, command: string,
cwd: string cwd: string
): Promise<{ stdout: string; stderr: string; exitCode: number }> { ): Promise<{ stdout: string; stderr: string; exitCode: number }> {
// Prepare environment variables const { exec } = await import('child_process');
const envVars = [
`NPM_CONFIG_USERCONFIG="${npmrcPath}"`,
`NPM_CONFIG_CACHE="${path.join(testDir, '.npm-cache')}"`,
`NPM_CONFIG_PREFIX="${path.join(testDir, '.npm-global')}"`,
`NPM_CONFIG_REGISTRY="${registryUrl}/npm/"`,
].join(' ');
// Build command with cd to correct directory and environment variables // Build isolated env that prevents npm from reading ~/.npmrc
const fullCommand = `cd "${cwd}" && ${envVars} ${command}`; const env: Record<string, string> = {};
// Copy only essential env vars (PATH, etc.) — exclude HOME to prevent ~/.npmrc reading
try { for (const key of ['PATH', 'NODE', 'NVM_DIR', 'NVM_BIN', 'LANG', 'TERM', 'SHELL']) {
const result = await tapNodeTools.runCommand(fullCommand); if (process.env[key]) env[key] = process.env[key]!;
return {
stdout: result.stdout || '',
stderr: result.stderr || '',
exitCode: result.exitCode || 0,
};
} catch (error: any) {
return {
stdout: error.stdout || '',
stderr: error.stderr || String(error),
exitCode: error.exitCode || 1,
};
} }
env.HOME = testDir;
env.NPM_CONFIG_USERCONFIG = npmrcPath;
env.NPM_CONFIG_GLOBALCONFIG = '/dev/null';
env.NPM_CONFIG_CACHE = path.join(testDir, '.npm-cache');
env.NPM_CONFIG_PREFIX = path.join(testDir, '.npm-global');
env.NPM_CONFIG_REGISTRY = `${registryUrl}/npm/`;
return new Promise((resolve) => {
exec(command, { cwd, env, timeout: 30000 }, (error, stdout, stderr) => {
resolve({
stdout: stdout || '',
stderr: stderr || '',
exitCode: error ? (error as any).code ?? 1 : 0,
});
});
});
} }
/** /**
@@ -224,16 +212,26 @@ function cleanupTestDir(dir: string): void {
// ======================================================================== // ========================================================================
tap.test('NPM CLI: should setup registry and HTTP server', async () => { tap.test('NPM CLI: should setup registry and HTTP server', async () => {
// Create registry // Find available port
registry = await createTestRegistry(); registryPort = 35000;
// Create registry with correct registryUrl for CLI tests
registry = await createTestRegistry({ registryUrl: `http://localhost:${registryPort}` });
const tokens = await createTestTokens(registry); const tokens = await createTestTokens(registry);
npmToken = tokens.npmToken; npmToken = tokens.npmToken;
// Clean up stale npm CLI test data via unpublish API
for (const pkg of ['test-package-cli', '@testscope%2fscoped-package']) {
await registry.handleRequest({
method: 'DELETE',
path: `/npm/${pkg}/-rev/cleanup`,
headers: { Authorization: `Bearer ${npmToken}` },
query: {},
});
}
expect(registry).toBeInstanceOf(SmartRegistry); expect(registry).toBeInstanceOf(SmartRegistry);
expect(npmToken).toBeTypeOf('string'); expect(npmToken).toBeTypeOf('string');
// Find available port
registryPort = 35000;
const serverSetup = await createHttpServer(registry, registryPort); const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server; server = serverSetup.server;
registryUrl = serverSetup.url; registryUrl = serverSetup.url;
@@ -241,8 +239,8 @@ tap.test('NPM CLI: should setup registry and HTTP server', async () => {
expect(server).toBeDefined(); expect(server).toBeDefined();
expect(registryUrl).toEqual(`http://localhost:${registryPort}`); expect(registryUrl).toEqual(`http://localhost:${registryPort}`);
// Setup test directory // Setup test directory — use /tmp to isolate from project tree
testDir = path.join(process.cwd(), '.nogit', 'test-npm-cli'); testDir = path.join('/tmp', 'smartregistry-test-npm-cli');
cleanupTestDir(testDir); cleanupTestDir(testDir);
fs.mkdirSync(testDir, { recursive: true }); fs.mkdirSync(testDir, { recursive: true });
@@ -291,20 +289,16 @@ tap.test('NPM CLI: should install published package', async () => {
const installDir = path.join(testDir, 'install-test'); const installDir = path.join(testDir, 'install-test');
fs.mkdirSync(installDir, { recursive: true }); fs.mkdirSync(installDir, { recursive: true });
// Create package.json for installation // Create a minimal package.json for install target
const packageJson = {
name: 'install-test',
version: '1.0.0',
dependencies: {
[packageName]: '1.0.0',
},
};
fs.writeFileSync( fs.writeFileSync(
path.join(installDir, 'package.json'), path.join(installDir, 'package.json'),
JSON.stringify(packageJson, null, 2), JSON.stringify({ name: 'install-test', version: '1.0.0', dependencies: { [packageName]: '1.0.0' } }),
'utf-8' 'utf-8'
); );
// Copy .npmrc
if (npmrcPath && fs.existsSync(npmrcPath)) {
fs.copyFileSync(npmrcPath, path.join(installDir, '.npmrc'));
}
const result = await runNpmCommand('npm install', installDir); const result = await runNpmCommand('npm install', installDir);
console.log('npm install output:', result.stdout); console.log('npm install output:', result.stdout);
@@ -313,17 +307,8 @@ tap.test('NPM CLI: should install published package', async () => {
expect(result.exitCode).toEqual(0); expect(result.exitCode).toEqual(0);
// Verify package was installed // Verify package was installed
const nodeModulesPath = path.join(installDir, 'node_modules', packageName); const installed = fs.existsSync(path.join(installDir, 'node_modules', packageName, 'package.json'));
expect(fs.existsSync(nodeModulesPath)).toEqual(true); expect(installed).toEqual(true);
expect(fs.existsSync(path.join(nodeModulesPath, 'package.json'))).toEqual(true);
expect(fs.existsSync(path.join(nodeModulesPath, 'index.js'))).toEqual(true);
// Verify package contents
const installedPackageJson = JSON.parse(
fs.readFileSync(path.join(nodeModulesPath, 'package.json'), 'utf-8')
);
expect(installedPackageJson.name).toEqual(packageName);
expect(installedPackageJson.version).toEqual('1.0.0');
}); });
tap.test('NPM CLI: should publish second version', async () => { tap.test('NPM CLI: should publish second version', async () => {
@@ -375,17 +360,14 @@ tap.test('NPM CLI: should fail to publish without auth', async () => {
const version = '1.0.0'; const version = '1.0.0';
const packageDir = createTestPackage(packageName, version, testDir); const packageDir = createTestPackage(packageName, version, testDir);
// Temporarily remove .npmrc // Temporarily remove .npmrc (write one without auth)
const npmrcBackup = fs.readFileSync(npmrcPath, 'utf-8'); const noAuthNpmrc = path.join(packageDir, '.npmrc');
fs.writeFileSync(npmrcPath, 'registry=' + registryUrl + '/npm/\n', 'utf-8'); fs.writeFileSync(noAuthNpmrc, `registry=${registryUrl}/npm/\n`, 'utf-8');
const result = await runNpmCommand('npm publish', packageDir); const result = await runNpmCommand('npm publish', packageDir);
console.log('npm publish unauth output:', result.stdout); console.log('npm publish unauth output:', result.stdout);
console.log('npm publish unauth stderr:', result.stderr); console.log('npm publish unauth stderr:', result.stderr);
// Restore .npmrc
fs.writeFileSync(npmrcPath, npmrcBackup, 'utf-8');
// Should fail with auth error // Should fail with auth error
expect(result.exitCode).not.toEqual(0); expect(result.exitCode).not.toEqual(0);
}); });
@@ -399,14 +381,7 @@ tap.postTask('cleanup npm cli tests', async () => {
} }
// Cleanup test directory // Cleanup test directory
if (testDir) { cleanupTestDir(testDir);
cleanupTestDir(testDir);
}
// Destroy registry
if (registry) {
registry.destroy();
}
}); });
export default tap.start(); export default tap.start();
+92 -31
View File
@@ -1,6 +1,7 @@
import { expect, tap } from '@git.zone/tstest/tapbundle'; import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js'; import { SmartRegistry } from '../ts/index.js';
import { createTestRegistry, createTestTokens, createTestPackument } from './helpers/registry.js'; import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
import { createTestRegistry, createTestTokens, createTestPackument, generateTestRunId } from './helpers/registry.js';
let registry: SmartRegistry; let registry: SmartRegistry;
let npmToken: string; let npmToken: string;
@@ -34,8 +35,9 @@ tap.test('NPM: should handle user authentication (PUT /-/user/org.couchdb.user:{
}); });
expect(response.status).toEqual(201); expect(response.status).toEqual(201);
expect(response.body).toHaveProperty('token'); const body = await streamToJson(response.body);
expect((response.body as any).token).toBeTypeOf('string'); expect(body).toHaveProperty('token');
expect(body.token).toBeTypeOf('string');
}); });
tap.test('NPM: should publish a package (PUT /{package})', async () => { tap.test('NPM: should publish a package (PUT /{package})', async () => {
@@ -53,8 +55,9 @@ tap.test('NPM: should publish a package (PUT /{package})', async () => {
}); });
expect(response.status).toEqual(201); expect(response.status).toEqual(201);
expect(response.body).toHaveProperty('ok'); const body = await streamToJson(response.body);
expect((response.body as any).ok).toEqual(true); expect(body).toHaveProperty('ok');
expect(body.ok).toEqual(true);
}); });
tap.test('NPM: should retrieve package metadata (GET /{package})', async () => { tap.test('NPM: should retrieve package metadata (GET /{package})', async () => {
@@ -66,10 +69,11 @@ tap.test('NPM: should retrieve package metadata (GET /{package})', async () => {
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('name'); const body = await streamToJson(response.body);
expect((response.body as any).name).toEqual(testPackageName); expect(body).toHaveProperty('name');
expect((response.body as any).versions).toHaveProperty(testVersion); expect(body.name).toEqual(testPackageName);
expect((response.body as any)['dist-tags'].latest).toEqual(testVersion); expect(body.versions).toHaveProperty(testVersion);
expect(body['dist-tags'].latest).toEqual(testVersion);
}); });
tap.test('NPM: should retrieve specific version metadata (GET /{package}/{version})', async () => { tap.test('NPM: should retrieve specific version metadata (GET /{package}/{version})', async () => {
@@ -81,9 +85,10 @@ tap.test('NPM: should retrieve specific version metadata (GET /{package}/{versio
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('version'); const body = await streamToJson(response.body);
expect((response.body as any).version).toEqual(testVersion); expect(body).toHaveProperty('version');
expect((response.body as any).name).toEqual(testPackageName); expect(body.version).toEqual(testVersion);
expect(body.name).toEqual(testPackageName);
}); });
tap.test('NPM: should download tarball (GET /{package}/-/{tarball})', async () => { tap.test('NPM: should download tarball (GET /{package}/-/{tarball})', async () => {
@@ -95,8 +100,9 @@ tap.test('NPM: should download tarball (GET /{package}/-/{tarball})', async () =
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect((response.body as Buffer).toString('utf-8')).toEqual('fake tarball content'); expect(body).toBeInstanceOf(Buffer);
expect(body.toString('utf-8')).toEqual('fake tarball content');
expect(response.headers['Content-Type']).toEqual('application/octet-stream'); expect(response.headers['Content-Type']).toEqual('application/octet-stream');
}); });
@@ -127,7 +133,52 @@ tap.test('NPM: should publish a new version of the package', async () => {
}); });
expect(getResponse.status).toEqual(200); expect(getResponse.status).toEqual(200);
expect((getResponse.body as any).versions).toHaveProperty(newVersion); const getBody = await streamToJson(getResponse.body);
expect(getBody.versions).toHaveProperty(newVersion);
});
tap.test('NPM: should support unencoded scoped package publish and metadata routes', async () => {
const scopedPackageName = `@scope/test-package-${generateTestRunId()}`;
const scopedVersion = '2.0.0';
const scopedTarballData = Buffer.from('scoped tarball content', 'utf-8');
const packument = createTestPackument(scopedPackageName, scopedVersion, scopedTarballData);
const publishResponse = await registry.handleRequest({
method: 'PUT',
path: `/npm/${scopedPackageName}`,
headers: {
Authorization: `Bearer ${npmToken}`,
'Content-Type': 'application/json',
},
query: {},
body: packument,
});
expect(publishResponse.status).toEqual(201);
const metadataResponse = await registry.handleRequest({
method: 'GET',
path: `/npm/${scopedPackageName}`,
headers: {},
query: {},
});
expect(metadataResponse.status).toEqual(200);
const metadataBody = await streamToJson(metadataResponse.body);
expect(metadataBody.name).toEqual(scopedPackageName);
expect(metadataBody.versions).toHaveProperty(scopedVersion);
const versionResponse = await registry.handleRequest({
method: 'GET',
path: `/npm/${scopedPackageName}/${scopedVersion}`,
headers: {},
query: {},
});
expect(versionResponse.status).toEqual(200);
const versionBody = await streamToJson(versionResponse.body);
expect(versionBody.name).toEqual(scopedPackageName);
expect(versionBody.version).toEqual(scopedVersion);
}); });
tap.test('NPM: should get dist-tags (GET /-/package/{pkg}/dist-tags)', async () => { tap.test('NPM: should get dist-tags (GET /-/package/{pkg}/dist-tags)', async () => {
@@ -139,8 +190,9 @@ tap.test('NPM: should get dist-tags (GET /-/package/{pkg}/dist-tags)', async ()
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('latest'); const body = await streamToJson(response.body);
expect((response.body as any).latest).toBeTypeOf('string'); expect(body).toHaveProperty('latest');
expect(body.latest).toBeTypeOf('string');
}); });
tap.test('NPM: should update dist-tag (PUT /-/package/{pkg}/dist-tags/{tag})', async () => { tap.test('NPM: should update dist-tag (PUT /-/package/{pkg}/dist-tags/{tag})', async () => {
@@ -165,7 +217,8 @@ tap.test('NPM: should update dist-tag (PUT /-/package/{pkg}/dist-tags/{tag})', a
query: {}, query: {},
}); });
expect((getResponse.body as any)['dist-tags'].beta).toEqual('1.1.0'); const getBody2 = await streamToJson(getResponse.body);
expect(getBody2['dist-tags'].beta).toEqual('1.1.0');
}); });
tap.test('NPM: should delete dist-tag (DELETE /-/package/{pkg}/dist-tags/{tag})', async () => { tap.test('NPM: should delete dist-tag (DELETE /-/package/{pkg}/dist-tags/{tag})', async () => {
@@ -188,7 +241,8 @@ tap.test('NPM: should delete dist-tag (DELETE /-/package/{pkg}/dist-tags/{tag})'
query: {}, query: {},
}); });
expect((getResponse.body as any)['dist-tags']).not.toHaveProperty('beta'); const getBody3 = await streamToJson(getResponse.body);
expect(getBody3['dist-tags']).not.toHaveProperty('beta');
}); });
tap.test('NPM: should create a new token (POST /-/npm/v1/tokens)', async () => { tap.test('NPM: should create a new token (POST /-/npm/v1/tokens)', async () => {
@@ -208,8 +262,9 @@ tap.test('NPM: should create a new token (POST /-/npm/v1/tokens)', async () => {
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('token'); const body = await streamToJson(response.body);
expect((response.body as any).readonly).toEqual(true); expect(body).toHaveProperty('token');
expect(body.readonly).toEqual(true);
}); });
tap.test('NPM: should list tokens (GET /-/npm/v1/tokens)', async () => { tap.test('NPM: should list tokens (GET /-/npm/v1/tokens)', async () => {
@@ -223,9 +278,10 @@ tap.test('NPM: should list tokens (GET /-/npm/v1/tokens)', async () => {
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('objects'); const body = await streamToJson(response.body);
expect((response.body as any).objects).toBeInstanceOf(Array); expect(body).toHaveProperty('objects');
expect((response.body as any).objects.length).toBeGreaterThan(0); expect(body.objects).toBeInstanceOf(Array);
expect(body.objects.length).toBeGreaterThan(0);
}); });
tap.test('NPM: should search packages (GET /-/v1/search)', async () => { tap.test('NPM: should search packages (GET /-/v1/search)', async () => {
@@ -240,9 +296,10 @@ tap.test('NPM: should search packages (GET /-/v1/search)', async () => {
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('objects'); const body = await streamToJson(response.body);
expect((response.body as any).objects).toBeInstanceOf(Array); expect(body).toHaveProperty('objects');
expect((response.body as any).total).toBeGreaterThan(0); expect(body.objects).toBeInstanceOf(Array);
expect(body.total).toBeGreaterThan(0);
}); });
tap.test('NPM: should search packages with specific query', async () => { tap.test('NPM: should search packages with specific query', async () => {
@@ -256,7 +313,8 @@ tap.test('NPM: should search packages with specific query', async () => {
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
const results = (response.body as any).objects; const body = await streamToJson(response.body);
const results = body.objects;
expect(results.length).toBeGreaterThan(0); expect(results.length).toBeGreaterThan(0);
expect(results[0].package.name).toEqual(testPackageName); expect(results[0].package.name).toEqual(testPackageName);
}); });
@@ -281,7 +339,8 @@ tap.test('NPM: should unpublish a specific version (DELETE /{package}/-/{version
query: {}, query: {},
}); });
expect((getResponse.body as any).versions).not.toHaveProperty(testVersion); const getBody4 = await streamToJson(getResponse.body);
expect(getBody4.versions).not.toHaveProperty(testVersion);
}); });
tap.test('NPM: should unpublish entire package (DELETE /{package}/-rev/{rev})', async () => { tap.test('NPM: should unpublish entire package (DELETE /{package}/-rev/{rev})', async () => {
@@ -316,7 +375,8 @@ tap.test('NPM: should return 404 for non-existent package', async () => {
}); });
expect(response.status).toEqual(404); expect(response.status).toEqual(404);
expect(response.body).toHaveProperty('error'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
}); });
tap.test('NPM: should return 401 for unauthorized publish', async () => { tap.test('NPM: should return 401 for unauthorized publish', async () => {
@@ -334,7 +394,8 @@ tap.test('NPM: should return 401 for unauthorized publish', async () => {
}); });
expect(response.status).toEqual(401); expect(response.status).toEqual(401);
expect(response.body).toHaveProperty('error'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
}); });
tap.test('NPM: should reject readonly token for write operations', async () => { tap.test('NPM: should reject readonly token for write operations', async () => {
+8 -18
View File
@@ -48,7 +48,7 @@ async function createDockerTestRegistry(port: number): Promise<SmartRegistry> {
}, },
oci: { oci: {
enabled: true, enabled: true,
basePath: '/oci', basePath: '/v2',
}, },
}; };
@@ -95,8 +95,7 @@ let testImageName: string;
* Create HTTP server wrapper around SmartRegistry * Create HTTP server wrapper around SmartRegistry
* CRITICAL: Always passes rawBody for content-addressable operations (OCI manifests/blobs) * CRITICAL: Always passes rawBody for content-addressable operations (OCI manifests/blobs)
* *
* Docker expects registry at /v2/ but SmartRegistry serves at /oci/v2/ * SmartRegistry OCI is configured with basePath '/v2' matching Docker's native /v2/ prefix.
* This wrapper rewrites paths for Docker compatibility
* *
* Also implements a simple /v2/token endpoint for Docker Bearer auth flow * Also implements a simple /v2/token endpoint for Docker Bearer auth flow
*/ */
@@ -130,10 +129,7 @@ async function createHttpServer(
// Log all requests for debugging // Log all requests for debugging
console.log(`[Registry] ${req.method} ${pathname}`); console.log(`[Registry] ${req.method} ${pathname}`);
// Docker expects /v2/ but SmartRegistry serves at /oci/v2/ // basePath is /v2 which matches Docker's native /v2/ prefix — no rewrite needed
if (pathname.startsWith('/v2')) {
pathname = '/oci' + pathname;
}
// Read raw body - ALWAYS preserve exact bytes for OCI // Read raw body - ALWAYS preserve exact bytes for OCI
const chunks: Buffer[] = []; const chunks: Buffer[] = [];
@@ -179,16 +175,10 @@ async function createHttpServer(
res.setHeader(key, value); res.setHeader(key, value);
} }
// Send body // Send body (response.body is always ReadableStream<Uint8Array> or undefined)
if (response.body) { if (response.body) {
if (Buffer.isBuffer(response.body)) { const { Readable } = await import('stream');
res.end(response.body); Readable.fromWeb(response.body).pipe(res);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
} else { } else {
res.end(); res.end();
} }
@@ -313,7 +303,7 @@ tap.test('Docker CLI: should verify server is responding', async () => {
// Give the server a moment to fully initialize // Give the server a moment to fully initialize
await new Promise(resolve => setTimeout(resolve, 500)); await new Promise(resolve => setTimeout(resolve, 500));
const response = await fetch(`${registryUrl}/oci/v2/`); const response = await fetch(`${registryUrl}/v2/`);
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
console.log('OCI v2 response:', await response.json()); console.log('OCI v2 response:', await response.json());
}); });
@@ -352,7 +342,7 @@ tap.test('Docker CLI: should push image to registry', async () => {
}); });
tap.test('Docker CLI: should verify manifest in registry via API', async () => { tap.test('Docker CLI: should verify manifest in registry via API', async () => {
const response = await fetch(`${registryUrl}/oci/v2/test-image/tags/list`, { const response = await fetch(`${registryUrl}/v2/test-image/tags/list`, {
headers: { Authorization: `Bearer ${ociToken}` }, headers: { Authorization: `Bearer ${ociToken}` },
}); });
+32 -26
View File
@@ -1,5 +1,6 @@
import { expect, tap } from '@git.zone/tstest/tapbundle'; import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js'; import { SmartRegistry } from '../ts/index.js';
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
import { createTestRegistry, createTestTokens, calculateDigest, createTestManifest } from './helpers/registry.js'; import { createTestRegistry, createTestTokens, calculateDigest, createTestManifest } from './helpers/registry.js';
let registry: SmartRegistry; let registry: SmartRegistry;
@@ -24,7 +25,7 @@ tap.test('OCI: should create registry instance', async () => {
tap.test('OCI: should handle version check (GET /v2/)', async () => { tap.test('OCI: should handle version check (GET /v2/)', async () => {
const response = await registry.handleRequest({ const response = await registry.handleRequest({
method: 'GET', method: 'GET',
path: '/oci/v2/', path: '/oci/',
headers: {}, headers: {},
query: {}, query: {},
}); });
@@ -36,7 +37,7 @@ tap.test('OCI: should handle version check (GET /v2/)', async () => {
tap.test('OCI: should initiate blob upload (POST /v2/{name}/blobs/uploads/)', async () => { tap.test('OCI: should initiate blob upload (POST /v2/{name}/blobs/uploads/)', async () => {
const response = await registry.handleRequest({ const response = await registry.handleRequest({
method: 'POST', method: 'POST',
path: '/oci/v2/test-repo/blobs/uploads/', path: '/oci/test-repo/blobs/uploads/',
headers: { headers: {
Authorization: `Bearer ${ociToken}`, Authorization: `Bearer ${ociToken}`,
}, },
@@ -53,7 +54,7 @@ tap.test('OCI: should upload blob in single PUT', async () => {
const response = await registry.handleRequest({ const response = await registry.handleRequest({
method: 'POST', method: 'POST',
path: '/oci/v2/test-repo/blobs/uploads/', path: '/oci/test-repo/blobs/uploads/',
headers: { headers: {
Authorization: `Bearer ${ociToken}`, Authorization: `Bearer ${ociToken}`,
}, },
@@ -73,7 +74,7 @@ tap.test('OCI: should upload config blob', async () => {
const response = await registry.handleRequest({ const response = await registry.handleRequest({
method: 'POST', method: 'POST',
path: '/oci/v2/test-repo/blobs/uploads/', path: '/oci/test-repo/blobs/uploads/',
headers: { headers: {
Authorization: `Bearer ${ociToken}`, Authorization: `Bearer ${ociToken}`,
}, },
@@ -90,7 +91,7 @@ tap.test('OCI: should upload config blob', async () => {
tap.test('OCI: should check if blob exists (HEAD /v2/{name}/blobs/{digest})', async () => { tap.test('OCI: should check if blob exists (HEAD /v2/{name}/blobs/{digest})', async () => {
const response = await registry.handleRequest({ const response = await registry.handleRequest({
method: 'HEAD', method: 'HEAD',
path: `/oci/v2/test-repo/blobs/${testBlobDigest}`, path: `/oci/test-repo/blobs/${testBlobDigest}`,
headers: { headers: {
Authorization: `Bearer ${ociToken}`, Authorization: `Bearer ${ociToken}`,
}, },
@@ -105,7 +106,7 @@ tap.test('OCI: should check if blob exists (HEAD /v2/{name}/blobs/{digest})', as
tap.test('OCI: should retrieve blob (GET /v2/{name}/blobs/{digest})', async () => { tap.test('OCI: should retrieve blob (GET /v2/{name}/blobs/{digest})', async () => {
const response = await registry.handleRequest({ const response = await registry.handleRequest({
method: 'GET', method: 'GET',
path: `/oci/v2/test-repo/blobs/${testBlobDigest}`, path: `/oci/test-repo/blobs/${testBlobDigest}`,
headers: { headers: {
Authorization: `Bearer ${ociToken}`, Authorization: `Bearer ${ociToken}`,
}, },
@@ -113,8 +114,9 @@ tap.test('OCI: should retrieve blob (GET /v2/{name}/blobs/{digest})', async () =
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect((response.body as Buffer).toString('utf-8')).toEqual('Hello from OCI test blob!'); expect(body).toBeInstanceOf(Buffer);
expect(body.toString('utf-8')).toEqual('Hello from OCI test blob!');
expect(response.headers['Docker-Content-Digest']).toEqual(testBlobDigest); expect(response.headers['Docker-Content-Digest']).toEqual(testBlobDigest);
}); });
@@ -126,7 +128,7 @@ tap.test('OCI: should upload manifest (PUT /v2/{name}/manifests/{reference})', a
const response = await registry.handleRequest({ const response = await registry.handleRequest({
method: 'PUT', method: 'PUT',
path: '/oci/v2/test-repo/manifests/v1.0.0', path: '/oci/test-repo/manifests/v1.0.0',
headers: { headers: {
Authorization: `Bearer ${ociToken}`, Authorization: `Bearer ${ociToken}`,
'Content-Type': 'application/vnd.oci.image.manifest.v1+json', 'Content-Type': 'application/vnd.oci.image.manifest.v1+json',
@@ -143,7 +145,7 @@ tap.test('OCI: should upload manifest (PUT /v2/{name}/manifests/{reference})', a
tap.test('OCI: should retrieve manifest by tag (GET /v2/{name}/manifests/{reference})', async () => { tap.test('OCI: should retrieve manifest by tag (GET /v2/{name}/manifests/{reference})', async () => {
const response = await registry.handleRequest({ const response = await registry.handleRequest({
method: 'GET', method: 'GET',
path: '/oci/v2/test-repo/manifests/v1.0.0', path: '/oci/test-repo/manifests/v1.0.0',
headers: { headers: {
Authorization: `Bearer ${ociToken}`, Authorization: `Bearer ${ociToken}`,
Accept: 'application/vnd.oci.image.manifest.v1+json', Accept: 'application/vnd.oci.image.manifest.v1+json',
@@ -152,9 +154,10 @@ tap.test('OCI: should retrieve manifest by tag (GET /v2/{name}/manifests/{refere
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
const manifest = JSON.parse((response.body as Buffer).toString('utf-8')); const manifest = JSON.parse(body.toString('utf-8'));
expect(manifest.schemaVersion).toEqual(2); expect(manifest.schemaVersion).toEqual(2);
expect(manifest.config.digest).toEqual(testConfigDigest); expect(manifest.config.digest).toEqual(testConfigDigest);
expect(manifest.layers[0].digest).toEqual(testBlobDigest); expect(manifest.layers[0].digest).toEqual(testBlobDigest);
@@ -163,7 +166,7 @@ tap.test('OCI: should retrieve manifest by tag (GET /v2/{name}/manifests/{refere
tap.test('OCI: should retrieve manifest by digest (GET /v2/{name}/manifests/{digest})', async () => { tap.test('OCI: should retrieve manifest by digest (GET /v2/{name}/manifests/{digest})', async () => {
const response = await registry.handleRequest({ const response = await registry.handleRequest({
method: 'GET', method: 'GET',
path: `/oci/v2/test-repo/manifests/${testManifestDigest}`, path: `/oci/test-repo/manifests/${testManifestDigest}`,
headers: { headers: {
Authorization: `Bearer ${ociToken}`, Authorization: `Bearer ${ociToken}`,
Accept: 'application/vnd.oci.image.manifest.v1+json', Accept: 'application/vnd.oci.image.manifest.v1+json',
@@ -178,7 +181,7 @@ tap.test('OCI: should retrieve manifest by digest (GET /v2/{name}/manifests/{dig
tap.test('OCI: should check if manifest exists (HEAD /v2/{name}/manifests/{reference})', async () => { tap.test('OCI: should check if manifest exists (HEAD /v2/{name}/manifests/{reference})', async () => {
const response = await registry.handleRequest({ const response = await registry.handleRequest({
method: 'HEAD', method: 'HEAD',
path: '/oci/v2/test-repo/manifests/v1.0.0', path: '/oci/test-repo/manifests/v1.0.0',
headers: { headers: {
Authorization: `Bearer ${ociToken}`, Authorization: `Bearer ${ociToken}`,
Accept: 'application/vnd.oci.image.manifest.v1+json', Accept: 'application/vnd.oci.image.manifest.v1+json',
@@ -193,7 +196,7 @@ tap.test('OCI: should check if manifest exists (HEAD /v2/{name}/manifests/{refer
tap.test('OCI: should list tags (GET /v2/{name}/tags/list)', async () => { tap.test('OCI: should list tags (GET /v2/{name}/tags/list)', async () => {
const response = await registry.handleRequest({ const response = await registry.handleRequest({
method: 'GET', method: 'GET',
path: '/oci/v2/test-repo/tags/list', path: '/oci/test-repo/tags/list',
headers: { headers: {
Authorization: `Bearer ${ociToken}`, Authorization: `Bearer ${ociToken}`,
}, },
@@ -201,9 +204,9 @@ tap.test('OCI: should list tags (GET /v2/{name}/tags/list)', async () => {
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('tags'); const tagList = await streamToJson(response.body);
expect(tagList).toHaveProperty('tags');
const tagList = response.body as any;
expect(tagList.name).toEqual('test-repo'); expect(tagList.name).toEqual('test-repo');
expect(tagList.tags).toBeInstanceOf(Array); expect(tagList.tags).toBeInstanceOf(Array);
expect(tagList.tags).toContain('v1.0.0'); expect(tagList.tags).toContain('v1.0.0');
@@ -212,7 +215,7 @@ tap.test('OCI: should list tags (GET /v2/{name}/tags/list)', async () => {
tap.test('OCI: should handle pagination for tag list', async () => { tap.test('OCI: should handle pagination for tag list', async () => {
const response = await registry.handleRequest({ const response = await registry.handleRequest({
method: 'GET', method: 'GET',
path: '/oci/v2/test-repo/tags/list', path: '/oci/test-repo/tags/list',
headers: { headers: {
Authorization: `Bearer ${ociToken}`, Authorization: `Bearer ${ociToken}`,
}, },
@@ -222,13 +225,14 @@ tap.test('OCI: should handle pagination for tag list', async () => {
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('tags'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('tags');
}); });
tap.test('OCI: should return 404 for non-existent blob', async () => { tap.test('OCI: should return 404 for non-existent blob', async () => {
const response = await registry.handleRequest({ const response = await registry.handleRequest({
method: 'GET', method: 'GET',
path: '/oci/v2/test-repo/blobs/sha256:0000000000000000000000000000000000000000000000000000000000000000', path: '/oci/test-repo/blobs/sha256:0000000000000000000000000000000000000000000000000000000000000000',
headers: { headers: {
Authorization: `Bearer ${ociToken}`, Authorization: `Bearer ${ociToken}`,
}, },
@@ -236,13 +240,14 @@ tap.test('OCI: should return 404 for non-existent blob', async () => {
}); });
expect(response.status).toEqual(404); expect(response.status).toEqual(404);
expect(response.body).toHaveProperty('errors'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('errors');
}); });
tap.test('OCI: should return 404 for non-existent manifest', async () => { tap.test('OCI: should return 404 for non-existent manifest', async () => {
const response = await registry.handleRequest({ const response = await registry.handleRequest({
method: 'GET', method: 'GET',
path: '/oci/v2/test-repo/manifests/non-existent-tag', path: '/oci/test-repo/manifests/non-existent-tag',
headers: { headers: {
Authorization: `Bearer ${ociToken}`, Authorization: `Bearer ${ociToken}`,
Accept: 'application/vnd.oci.image.manifest.v1+json', Accept: 'application/vnd.oci.image.manifest.v1+json',
@@ -251,13 +256,14 @@ tap.test('OCI: should return 404 for non-existent manifest', async () => {
}); });
expect(response.status).toEqual(404); expect(response.status).toEqual(404);
expect(response.body).toHaveProperty('errors'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('errors');
}); });
tap.test('OCI: should delete manifest (DELETE /v2/{name}/manifests/{digest})', async () => { tap.test('OCI: should delete manifest (DELETE /v2/{name}/manifests/{digest})', async () => {
const response = await registry.handleRequest({ const response = await registry.handleRequest({
method: 'DELETE', method: 'DELETE',
path: `/oci/v2/test-repo/manifests/${testManifestDigest}`, path: `/oci/test-repo/manifests/${testManifestDigest}`,
headers: { headers: {
Authorization: `Bearer ${ociToken}`, Authorization: `Bearer ${ociToken}`,
}, },
@@ -270,7 +276,7 @@ tap.test('OCI: should delete manifest (DELETE /v2/{name}/manifests/{digest})', a
tap.test('OCI: should delete blob (DELETE /v2/{name}/blobs/{digest})', async () => { tap.test('OCI: should delete blob (DELETE /v2/{name}/blobs/{digest})', async () => {
const response = await registry.handleRequest({ const response = await registry.handleRequest({
method: 'DELETE', method: 'DELETE',
path: `/oci/v2/test-repo/blobs/${testBlobDigest}`, path: `/oci/test-repo/blobs/${testBlobDigest}`,
headers: { headers: {
Authorization: `Bearer ${ociToken}`, Authorization: `Bearer ${ociToken}`,
}, },
@@ -283,7 +289,7 @@ tap.test('OCI: should delete blob (DELETE /v2/{name}/blobs/{digest})', async ()
tap.test('OCI: should handle unauthorized requests', async () => { tap.test('OCI: should handle unauthorized requests', async () => {
const response = await registry.handleRequest({ const response = await registry.handleRequest({
method: 'GET', method: 'GET',
path: '/oci/v2/test-repo/manifests/v1.0.0', path: '/oci/test-repo/manifests/v1.0.0',
headers: { headers: {
// No authorization header // No authorization header
}, },
+8 -14
View File
@@ -89,16 +89,10 @@ async function createHttpServer(
res.setHeader(key, value); res.setHeader(key, value);
} }
// Send body // Send body (response.body is always ReadableStream<Uint8Array> or undefined)
if (response.body) { if (response.body) {
if (Buffer.isBuffer(response.body)) { const { Readable } = await import('stream');
res.end(response.body); Readable.fromWeb(response.body).pipe(res);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
} else { } else {
res.end(); res.end();
} }
@@ -303,16 +297,16 @@ tap.test('PyPI CLI: should verify twine is installed', async () => {
}); });
tap.test('PyPI CLI: should setup registry and HTTP server', async () => { tap.test('PyPI CLI: should setup registry and HTTP server', async () => {
// Create registry // Use port 39000 (avoids conflicts with other tests)
registry = await createTestRegistry(); registryPort = 39000;
// Create registry with correct registryUrl for CLI tests
registry = await createTestRegistry({ registryUrl: `http://localhost:${registryPort}` });
const tokens = await createTestTokens(registry); const tokens = await createTestTokens(registry);
pypiToken = tokens.pypiToken; pypiToken = tokens.pypiToken;
expect(registry).toBeInstanceOf(SmartRegistry); expect(registry).toBeInstanceOf(SmartRegistry);
expect(pypiToken).toBeTypeOf('string'); expect(pypiToken).toBeTypeOf('string');
// Use port 39000 (avoids conflicts with other tests)
registryPort = 39000;
const serverSetup = await createHttpServer(registry, registryPort); const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server; server = serverSetup.server;
registryUrl = serverSetup.url; registryUrl = serverSetup.url;
+28 -20
View File
@@ -1,5 +1,6 @@
import { expect, tap } from '@git.zone/tstest/tapbundle'; import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js'; import { SmartRegistry } from '../ts/index.js';
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
import { import {
createTestRegistry, createTestRegistry,
createTestTokens, createTestTokens,
@@ -101,9 +102,10 @@ tap.test('PyPI: should retrieve Simple API root index HTML (GET /simple/)', asyn
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toStartWith('text/html'); expect(response.headers['Content-Type']).toStartWith('text/html');
expect(response.body).toBeTypeOf('string'); const body = await streamToBuffer(response.body);
const html = body.toString('utf-8');
expect(html).toBeTypeOf('string');
const html = response.body as string;
expect(html).toContain('<!DOCTYPE html>'); expect(html).toContain('<!DOCTYPE html>');
expect(html).toContain('<title>Simple Index</title>'); expect(html).toContain('<title>Simple Index</title>');
expect(html).toContain(normalizedPackageName); expect(html).toContain(normalizedPackageName);
@@ -121,9 +123,9 @@ tap.test('PyPI: should retrieve Simple API root index JSON (GET /simple/ with Ac
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/vnd.pypi.simple.v1+json'); expect(response.headers['Content-Type']).toEqual('application/vnd.pypi.simple.v1+json');
expect(response.body).toBeTypeOf('object'); const json = await streamToJson(response.body);
expect(json).toBeTypeOf('object');
const json = response.body as any;
expect(json).toHaveProperty('meta'); expect(json).toHaveProperty('meta');
expect(json).toHaveProperty('projects'); expect(json).toHaveProperty('projects');
expect(json.projects).toBeInstanceOf(Array); expect(json.projects).toBeInstanceOf(Array);
@@ -144,9 +146,10 @@ tap.test('PyPI: should retrieve Simple API package HTML (GET /simple/{package}/)
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toStartWith('text/html'); expect(response.headers['Content-Type']).toStartWith('text/html');
expect(response.body).toBeTypeOf('string'); const body = await streamToBuffer(response.body);
const html = body.toString('utf-8');
expect(html).toBeTypeOf('string');
const html = response.body as string;
expect(html).toContain('<!DOCTYPE html>'); expect(html).toContain('<!DOCTYPE html>');
expect(html).toContain(`<title>Links for ${normalizedPackageName}</title>`); expect(html).toContain(`<title>Links for ${normalizedPackageName}</title>`);
expect(html).toContain('.whl'); expect(html).toContain('.whl');
@@ -165,9 +168,9 @@ tap.test('PyPI: should retrieve Simple API package JSON (GET /simple/{package}/
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/vnd.pypi.simple.v1+json'); expect(response.headers['Content-Type']).toEqual('application/vnd.pypi.simple.v1+json');
expect(response.body).toBeTypeOf('object'); const json = await streamToJson(response.body);
expect(json).toBeTypeOf('object');
const json = response.body as any;
expect(json).toHaveProperty('meta'); expect(json).toHaveProperty('meta');
expect(json).toHaveProperty('name'); expect(json).toHaveProperty('name');
expect(json.name).toEqual(normalizedPackageName); expect(json.name).toEqual(normalizedPackageName);
@@ -187,8 +190,9 @@ tap.test('PyPI: should download wheel file (GET /pypi/packages/{package}/{filena
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect((response.body as Buffer).length).toEqual(testWheelData.length); expect(body).toBeInstanceOf(Buffer);
expect(body.length).toEqual(testWheelData.length);
expect(response.headers['Content-Type']).toEqual('application/octet-stream'); expect(response.headers['Content-Type']).toEqual('application/octet-stream');
}); });
@@ -234,7 +238,7 @@ tap.test('PyPI: should list both wheel and sdist in Simple API', async () => {
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
const json = response.body as any; const json = await streamToJson(response.body);
// PEP 691: files is an array of file objects // PEP 691: files is an array of file objects
expect(json.files.length).toEqual(2); expect(json.files.length).toEqual(2);
@@ -289,7 +293,7 @@ tap.test('PyPI: should list multiple versions in Simple API', async () => {
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
const json = response.body as any; const json = await streamToJson(response.body);
// PEP 691: files is an array of file objects // PEP 691: files is an array of file objects
expect(json.files.length).toBeGreaterThan(2); expect(json.files.length).toBeGreaterThan(2);
@@ -323,7 +327,8 @@ tap.test('PyPI: should return 404 for non-existent package', async () => {
}); });
expect(response.status).toEqual(404); expect(response.status).toEqual(404);
expect(response.body).toHaveProperty('error'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
}); });
tap.test('PyPI: should return 401 for unauthorized upload', async () => { tap.test('PyPI: should return 401 for unauthorized upload', async () => {
@@ -353,7 +358,8 @@ tap.test('PyPI: should return 401 for unauthorized upload', async () => {
}); });
expect(response.status).toEqual(401); expect(response.status).toEqual(401);
expect(response.body).toHaveProperty('error'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
}); });
tap.test('PyPI: should reject upload with mismatched hash', async () => { tap.test('PyPI: should reject upload with mismatched hash', async () => {
@@ -382,7 +388,8 @@ tap.test('PyPI: should reject upload with mismatched hash', async () => {
}); });
expect(response.status).toEqual(400); expect(response.status).toEqual(400);
expect(response.body).toHaveProperty('error'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
}); });
tap.test('PyPI: should handle package with requires-python metadata', async () => { tap.test('PyPI: should handle package with requires-python metadata', async () => {
@@ -425,7 +432,8 @@ tap.test('PyPI: should handle package with requires-python metadata', async () =
query: {}, query: {},
}); });
const html = getResponse.body as string; const getBody = await streamToBuffer(getResponse.body);
const html = getBody.toString('utf-8');
expect(html).toContain('data-requires-python'); expect(html).toContain('data-requires-python');
// Note: >= gets HTML-escaped to &gt;= in attribute values // Note: >= gets HTML-escaped to &gt;= in attribute values
expect(html).toContain('&gt;=3.8'); expect(html).toContain('&gt;=3.8');
@@ -441,9 +449,9 @@ tap.test('PyPI: should support JSON API for package metadata', async () => {
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/json'); expect(response.headers['Content-Type']).toEqual('application/json');
expect(response.body).toBeTypeOf('object'); const json = await streamToJson(response.body);
expect(json).toBeTypeOf('object');
const json = response.body as any;
expect(json).toHaveProperty('info'); expect(json).toHaveProperty('info');
expect(json.info).toHaveProperty('name'); expect(json.info).toHaveProperty('name');
expect(json.info.name).toEqual(normalizedPackageName); expect(json.info.name).toEqual(normalizedPackageName);
@@ -460,9 +468,9 @@ tap.test('PyPI: should support JSON API for specific version', async () => {
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/json'); expect(response.headers['Content-Type']).toEqual('application/json');
expect(response.body).toBeTypeOf('object'); const json = await streamToJson(response.body);
expect(json).toBeTypeOf('object');
const json = response.body as any;
expect(json).toHaveProperty('info'); expect(json).toHaveProperty('info');
expect(json.info.version).toEqual(testVersion); expect(json.info.version).toEqual(testVersion);
expect(json).toHaveProperty('urls'); expect(json).toHaveProperty('urls');
+29 -28
View File
@@ -79,16 +79,10 @@ async function createHttpServer(
res.setHeader(key, value); res.setHeader(key, value);
} }
// Send body // Send body (response.body is always ReadableStream<Uint8Array> or undefined)
if (response.body) { if (response.body) {
if (Buffer.isBuffer(response.body)) { const { Readable } = await import('stream');
res.end(response.body); Readable.fromWeb(response.body).pipe(res);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
} else { } else {
res.end(); res.end();
} }
@@ -154,11 +148,16 @@ async function runGemCommand(
cwd: string, cwd: string,
includeAuth: boolean = true includeAuth: boolean = true
): Promise<{ stdout: string; stderr: string; exitCode: number }> { ): Promise<{ stdout: string; stderr: string; exitCode: number }> {
// When not including auth, use a temp HOME without credentials
const effectiveHome = includeAuth ? gemHome : path.join(gemHome, 'noauth');
if (!includeAuth) {
fs.mkdirSync(effectiveHome, { recursive: true });
}
// Prepare environment variables // Prepare environment variables
const envVars = [ const envVars = [
`HOME="${gemHome}"`, `HOME="${effectiveHome}"`,
`GEM_HOME="${gemHome}"`, `GEM_HOME="${gemHome}"`,
includeAuth ? '' : 'RUBYGEMS_API_KEY=""',
].filter(Boolean).join(' '); ].filter(Boolean).join(' ');
// Build command with cd to correct directory and environment variables // Build command with cd to correct directory and environment variables
@@ -194,16 +193,16 @@ function cleanupTestDir(dir: string): void {
// ======================================================================== // ========================================================================
tap.test('RubyGems CLI: should setup registry and HTTP server', async () => { tap.test('RubyGems CLI: should setup registry and HTTP server', async () => {
// Create registry // Use port 36000 (avoids npm:35000, cargo:5000 conflicts)
registry = await createTestRegistry(); registryPort = 36000;
// Create registry with correct registryUrl for CLI tests
registry = await createTestRegistry({ registryUrl: `http://localhost:${registryPort}` });
const tokens = await createTestTokens(registry); const tokens = await createTestTokens(registry);
rubygemsToken = tokens.rubygemsToken; rubygemsToken = tokens.rubygemsToken;
expect(registry).toBeInstanceOf(SmartRegistry); expect(registry).toBeInstanceOf(SmartRegistry);
expect(rubygemsToken).toBeTypeOf('string'); expect(rubygemsToken).toBeTypeOf('string');
// Use port 36000 (avoids npm:35000, cargo:5000 conflicts)
registryPort = 36000;
const serverSetup = await createHttpServer(registry, registryPort); const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server; server = serverSetup.server;
registryUrl = serverSetup.url; registryUrl = serverSetup.url;
@@ -366,31 +365,33 @@ tap.test('RubyGems CLI: should unyank a version', async () => {
const gemName = 'test-gem-cli'; const gemName = 'test-gem-cli';
const version = '1.0.0'; const version = '1.0.0';
const result = await runGemCommand( // Use PUT /api/v1/gems/unyank via HTTP API (gem yank --undo removed in Ruby 4.0)
`gem yank ${gemName} -v ${version} --undo --host ${registryUrl}/rubygems`, const response = await fetch(
testDir `${registryUrl}/rubygems/api/v1/gems/unyank?gem_name=${gemName}&version=${version}`,
{
method: 'PUT',
headers: {
'Authorization': rubygemsToken,
},
}
); );
console.log('gem unyank output:', result.stdout); console.log('gem unyank status:', response.status);
console.log('gem unyank stderr:', result.stderr);
expect(result.exitCode).toEqual(0); expect(response.status).toEqual(200);
// Verify version is not yanked in /versions file // Verify version is not yanked in /versions file
const response = await fetch(`${registryUrl}/rubygems/versions`); const versionsResponse = await fetch(`${registryUrl}/rubygems/versions`);
const versionsData = await response.text(); const versionsData = await versionsResponse.text();
console.log('Versions after unyank:', versionsData); console.log('Versions after unyank:', versionsData);
// Should not have '-' prefix anymore (or have both without prefix) // Should not have '-' prefix anymore
// Check that we have the version without yank marker
const lines = versionsData.trim().split('\n'); const lines = versionsData.trim().split('\n');
const gemLine = lines.find(line => line.startsWith(gemName)); const gemLine = lines.find(line => line.startsWith(gemName));
if (gemLine) { if (gemLine) {
// Parse format: "gemname version[,version...] md5"
const parts = gemLine.split(' '); const parts = gemLine.split(' ');
const versions = parts[1]; const versions = parts[1];
// Should have 1.0.0 without '-' prefix
expect(versions).toContain('1.0.0'); expect(versions).toContain('1.0.0');
expect(versions).not.toContain('-1.0.0'); expect(versions).not.toContain('-1.0.0');
} }
+53 -36
View File
@@ -1,5 +1,6 @@
import { expect, tap } from '@git.zone/tstest/tapbundle'; import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js'; import { SmartRegistry } from '../ts/index.js';
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
import { import {
createTestRegistry, createTestRegistry,
createTestTokens, createTestTokens,
@@ -54,7 +55,8 @@ tap.test('RubyGems: should upload gem file (POST /rubygems/api/v1/gems)', async
}); });
expect(response.status).toEqual(201); expect(response.status).toEqual(201);
expect(response.body).toHaveProperty('message'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('message');
}); });
tap.test('RubyGems: should retrieve Compact Index versions file (GET /rubygems/versions)', async () => { tap.test('RubyGems: should retrieve Compact Index versions file (GET /rubygems/versions)', async () => {
@@ -67,9 +69,10 @@ tap.test('RubyGems: should retrieve Compact Index versions file (GET /rubygems/v
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('text/plain; charset=utf-8'); expect(response.headers['Content-Type']).toEqual('text/plain; charset=utf-8');
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
const content = (response.body as Buffer).toString('utf-8'); const content = body.toString('utf-8');
expect(content).toContain('created_at:'); expect(content).toContain('created_at:');
expect(content).toContain('---'); expect(content).toContain('---');
expect(content).toContain(testGemName); expect(content).toContain(testGemName);
@@ -86,9 +89,10 @@ tap.test('RubyGems: should retrieve Compact Index info file (GET /rubygems/info/
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('text/plain; charset=utf-8'); expect(response.headers['Content-Type']).toEqual('text/plain; charset=utf-8');
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
const content = (response.body as Buffer).toString('utf-8'); const content = body.toString('utf-8');
expect(content).toContain('---'); expect(content).toContain('---');
expect(content).toContain(testVersion); expect(content).toContain(testVersion);
expect(content).toContain('checksum:'); expect(content).toContain('checksum:');
@@ -104,9 +108,10 @@ tap.test('RubyGems: should retrieve Compact Index names file (GET /rubygems/name
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('text/plain; charset=utf-8'); expect(response.headers['Content-Type']).toEqual('text/plain; charset=utf-8');
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
const content = (response.body as Buffer).toString('utf-8'); const content = body.toString('utf-8');
expect(content).toContain('---'); expect(content).toContain('---');
expect(content).toContain(testGemName); expect(content).toContain(testGemName);
}); });
@@ -120,8 +125,9 @@ tap.test('RubyGems: should download gem file (GET /rubygems/gems/{gem}-{version}
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect((response.body as Buffer).length).toEqual(testGemData.length); expect(body).toBeInstanceOf(Buffer);
expect(body.length).toEqual(testGemData.length);
expect(response.headers['Content-Type']).toEqual('application/octet-stream'); expect(response.headers['Content-Type']).toEqual('application/octet-stream');
}); });
@@ -153,7 +159,8 @@ tap.test('RubyGems: should list multiple versions in Compact Index', async () =>
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
const content = (response.body as Buffer).toString('utf-8'); const body = await streamToBuffer(response.body);
const content = body.toString('utf-8');
const lines = content.split('\n'); const lines = content.split('\n');
const gemLine = lines.find(l => l.startsWith(`${testGemName} `)); const gemLine = lines.find(l => l.startsWith(`${testGemName} `));
@@ -172,7 +179,8 @@ tap.test('RubyGems: should list multiple versions in info file', async () => {
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
const content = (response.body as Buffer).toString('utf-8'); const body = await streamToBuffer(response.body);
const content = body.toString('utf-8');
expect(content).toContain('1.0.0'); expect(content).toContain('1.0.0');
expect(content).toContain('2.0.0'); expect(content).toContain('2.0.0');
}); });
@@ -203,7 +211,8 @@ tap.test('RubyGems: should support platform-specific gems', async () => {
query: {}, query: {},
}); });
const content = (versionsResponse.body as Buffer).toString('utf-8'); const versionsBody = await streamToBuffer(versionsResponse.body);
const content = versionsBody.toString('utf-8');
const lines = content.split('\n'); const lines = content.split('\n');
const gemLine = lines.find(l => l.startsWith(`${testGemName} `)); const gemLine = lines.find(l => l.startsWith(`${testGemName} `));
@@ -224,8 +233,9 @@ tap.test('RubyGems: should yank a gem version (DELETE /rubygems/api/v1/gems/yank
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('message'); const body = await streamToJson(response.body);
expect((response.body as any).message).toContain('yanked'); expect(body).toHaveProperty('message');
expect(body.message).toContain('yanked');
}); });
tap.test('RubyGems: should mark yanked version in Compact Index', async () => { tap.test('RubyGems: should mark yanked version in Compact Index', async () => {
@@ -238,7 +248,8 @@ tap.test('RubyGems: should mark yanked version in Compact Index', async () => {
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
const content = (response.body as Buffer).toString('utf-8'); const body = await streamToBuffer(response.body);
const content = body.toString('utf-8');
const lines = content.split('\n'); const lines = content.split('\n');
const gemLine = lines.find(l => l.startsWith(`${testGemName} `)); const gemLine = lines.find(l => l.startsWith(`${testGemName} `));
@@ -256,7 +267,8 @@ tap.test('RubyGems: should still allow downloading yanked gem', async () => {
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
}); });
tap.test('RubyGems: should unyank a gem version (PUT /rubygems/api/v1/gems/unyank)', async () => { tap.test('RubyGems: should unyank a gem version (PUT /rubygems/api/v1/gems/unyank)', async () => {
@@ -273,8 +285,9 @@ tap.test('RubyGems: should unyank a gem version (PUT /rubygems/api/v1/gems/unyan
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('message'); const body = await streamToJson(response.body);
expect((response.body as any).message).toContain('unyanked'); expect(body).toHaveProperty('message');
expect(body.message).toContain('unyanked');
}); });
tap.test('RubyGems: should remove yank marker after unyank', async () => { tap.test('RubyGems: should remove yank marker after unyank', async () => {
@@ -287,7 +300,8 @@ tap.test('RubyGems: should remove yank marker after unyank', async () => {
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
const content = (response.body as Buffer).toString('utf-8'); const body = await streamToBuffer(response.body);
const content = body.toString('utf-8');
const lines = content.split('\n'); const lines = content.split('\n');
const gemLine = lines.find(l => l.startsWith(`${testGemName} `)); const gemLine = lines.find(l => l.startsWith(`${testGemName} `));
@@ -309,14 +323,10 @@ tap.test('RubyGems: should retrieve versions JSON (GET /rubygems/api/v1/versions
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/json'); expect(response.headers['Content-Type']).toEqual('application/json');
expect(response.body).toBeTypeOf('object'); const json = await streamToJson(response.body);
expect(json).toBeInstanceOf(Array);
const json = response.body as any; expect(json.length).toBeGreaterThan(0);
expect(json).toHaveProperty('name'); expect(json[0]).toHaveProperty('number');
expect(json.name).toEqual(testGemName);
expect(json).toHaveProperty('versions');
expect(json.versions).toBeTypeOf('object');
expect(json.versions.length).toBeGreaterThan(0);
}); });
tap.test('RubyGems: should retrieve dependencies JSON (GET /rubygems/api/v1/dependencies)', async () => { tap.test('RubyGems: should retrieve dependencies JSON (GET /rubygems/api/v1/dependencies)', async () => {
@@ -331,9 +341,9 @@ tap.test('RubyGems: should retrieve dependencies JSON (GET /rubygems/api/v1/depe
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/json'); expect(response.headers['Content-Type']).toEqual('application/json');
expect(response.body).toBeTypeOf('object'); const json = await streamToJson(response.body);
expect(json).toBeTypeOf('object');
const json = response.body as any;
expect(Array.isArray(json)).toEqual(true); expect(Array.isArray(json)).toEqual(true);
}); });
@@ -346,7 +356,8 @@ tap.test('RubyGems: should retrieve gem spec (GET /rubygems/quick/Marshal.4.8/{g
}); });
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
}); });
tap.test('RubyGems: should support latest specs endpoint (GET /rubygems/latest_specs.4.8.gz)', async () => { tap.test('RubyGems: should support latest specs endpoint (GET /rubygems/latest_specs.4.8.gz)', async () => {
@@ -359,7 +370,8 @@ tap.test('RubyGems: should support latest specs endpoint (GET /rubygems/latest_s
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/octet-stream'); expect(response.headers['Content-Type']).toEqual('application/octet-stream');
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
}); });
tap.test('RubyGems: should support specs endpoint (GET /rubygems/specs.4.8.gz)', async () => { tap.test('RubyGems: should support specs endpoint (GET /rubygems/specs.4.8.gz)', async () => {
@@ -372,7 +384,8 @@ tap.test('RubyGems: should support specs endpoint (GET /rubygems/specs.4.8.gz)',
expect(response.status).toEqual(200); expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/octet-stream'); expect(response.headers['Content-Type']).toEqual('application/octet-stream');
expect(response.body).toBeInstanceOf(Buffer); const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
}); });
tap.test('RubyGems: should return 404 for non-existent gem', async () => { tap.test('RubyGems: should return 404 for non-existent gem', async () => {
@@ -384,7 +397,8 @@ tap.test('RubyGems: should return 404 for non-existent gem', async () => {
}); });
expect(response.status).toEqual(404); expect(response.status).toEqual(404);
expect(response.body).toHaveProperty('error'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
}); });
tap.test('RubyGems: should return 401 for unauthorized upload', async () => { tap.test('RubyGems: should return 401 for unauthorized upload', async () => {
@@ -402,7 +416,8 @@ tap.test('RubyGems: should return 401 for unauthorized upload', async () => {
}); });
expect(response.status).toEqual(401); expect(response.status).toEqual(401);
expect(response.body).toHaveProperty('error'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
}); });
tap.test('RubyGems: should return 401 for unauthorized yank', async () => { tap.test('RubyGems: should return 401 for unauthorized yank', async () => {
@@ -419,7 +434,8 @@ tap.test('RubyGems: should return 401 for unauthorized yank', async () => {
}); });
expect(response.status).toEqual(401); expect(response.status).toEqual(401);
expect(response.body).toHaveProperty('error'); const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
}); });
tap.test('RubyGems: should handle gem with dependencies', async () => { tap.test('RubyGems: should handle gem with dependencies', async () => {
@@ -450,7 +466,8 @@ tap.test('RubyGems: should handle gem with dependencies', async () => {
expect(infoResponse.status).toEqual(200); expect(infoResponse.status).toEqual(200);
const content = (infoResponse.body as Buffer).toString('utf-8'); const infoBody = await streamToBuffer(infoResponse.body);
const content = infoBody.toString('utf-8');
expect(content).toContain('checksum:'); expect(content).toContain('checksum:');
}); });
+142 -1
View File
@@ -3,7 +3,14 @@ import * as qenv from '@push.rocks/qenv';
import { RegistryStorage } from '../ts/core/classes.registrystorage.js'; import { RegistryStorage } from '../ts/core/classes.registrystorage.js';
import type { IStorageConfig } from '../ts/core/interfaces.core.js'; import type { IStorageConfig } from '../ts/core/interfaces.core.js';
import type { IStorageHooks, IStorageHookContext } from '../ts/core/interfaces.storage.js'; import type { IStorageHooks, IStorageHookContext } from '../ts/core/interfaces.storage.js';
import { createTrackingHooks, createQuotaHooks, generateTestRunId } from './helpers/registry.js'; import {
createQuotaHooks,
createTestPackument,
createTestRegistry,
createTestTokens,
createTrackingHooks,
generateTestRunId,
} from './helpers/registry.js';
const testQenv = new qenv.Qenv('./', './.nogit'); const testQenv = new qenv.Qenv('./', './.nogit');
@@ -344,6 +351,140 @@ tap.test('withContext: should clear context even on error', async () => {
await errorStorage.putObject('test/after-error.txt', Buffer.from('ok')); await errorStorage.putObject('test/after-error.txt', Buffer.from('ok'));
}); });
tap.test('withContext: should isolate concurrent async operations', async () => {
const tracker = createTrackingHooks();
const concurrentStorage = new RegistryStorage(storageConfig, tracker.hooks);
await concurrentStorage.init();
const bucket = (concurrentStorage as any).bucket;
const originalFastPut = bucket.fastPut.bind(bucket);
const pendingWrites: Array<() => void> = [];
let startedWrites = 0;
let waitingWrites = 0;
let startedResolve: () => void;
let waitingResolve: () => void;
const bothWritesStarted = new Promise<void>((resolve) => {
startedResolve = resolve;
});
const bothWritesWaiting = new Promise<void>((resolve) => {
waitingResolve = resolve;
});
bucket.fastPut = async (options: any) => {
startedWrites += 1;
if (startedWrites === 2) {
startedResolve();
}
await bothWritesStarted;
await new Promise<void>((resolve) => {
pendingWrites.push(resolve);
waitingWrites += 1;
if (waitingWrites === 2) {
waitingResolve();
}
});
return originalFastPut(options);
};
try {
const opA = concurrentStorage.withContext(
{
protocol: 'npm',
actor: { userId: 'user-a' },
metadata: { packageName: 'package-a' },
},
async () => {
await concurrentStorage.putObject('test/concurrent-a.txt', Buffer.from('a'));
}
);
const opB = concurrentStorage.withContext(
{
protocol: 'npm',
actor: { userId: 'user-b' },
metadata: { packageName: 'package-b' },
},
async () => {
await concurrentStorage.putObject('test/concurrent-b.txt', Buffer.from('b'));
}
);
await bothWritesWaiting;
pendingWrites[0]!();
pendingWrites[1]!();
await Promise.all([opA, opB]);
await new Promise(resolve => setTimeout(resolve, 100));
} finally {
bucket.fastPut = originalFastPut;
}
const afterPutCalls = tracker.calls.filter(
(call) => call.method === 'afterPut' && call.context.key.startsWith('test/concurrent-')
);
expect(afterPutCalls.length).toEqual(2);
const callByKey = new Map(afterPutCalls.map((call) => [call.context.key, call]));
expect(callByKey.get('test/concurrent-a.txt')?.context.actor?.userId).toEqual('user-a');
expect(callByKey.get('test/concurrent-a.txt')?.context.metadata?.packageName).toEqual('package-a');
expect(callByKey.get('test/concurrent-b.txt')?.context.actor?.userId).toEqual('user-b');
expect(callByKey.get('test/concurrent-b.txt')?.context.metadata?.packageName).toEqual('package-b');
});
tap.test('request hooks: should receive context during real npm publish requests', async () => {
const tracker = createTrackingHooks();
const registry = await createTestRegistry({ storageHooks: tracker.hooks });
try {
const tokens = await createTestTokens(registry);
const packageName = `hooked-package-${generateTestRunId()}`;
const version = '1.0.0';
const tarball = Buffer.from('hooked tarball data', 'utf-8');
const packument = createTestPackument(packageName, version, tarball);
const response = await registry.handleRequest({
method: 'PUT',
path: `/npm/${packageName}`,
headers: {
Authorization: `Bearer ${tokens.npmToken}`,
'Content-Type': 'application/json',
},
query: {},
body: packument,
});
expect(response.status).toEqual(201);
await new Promise(resolve => setTimeout(resolve, 100));
const npmWrites = tracker.calls.filter(
(call) => call.method === 'beforePut' && call.context.metadata?.packageName === packageName
);
expect(npmWrites.length).toBeGreaterThanOrEqual(2);
const packumentWrite = npmWrites.find(
(call) => call.context.key === `npm/packages/${packageName}/index.json`
);
expect(packumentWrite).toBeTruthy();
expect(packumentWrite!.context.protocol).toEqual('npm');
expect(packumentWrite!.context.actor?.userId).toEqual(tokens.userId);
expect(packumentWrite!.context.metadata?.packageName).toEqual(packageName);
const tarballWrite = npmWrites.find(
(call) => call.context.key.endsWith(`-${version}.tgz`)
);
expect(tarballWrite).toBeTruthy();
expect(tarballWrite!.context.metadata?.packageName).toEqual(packageName);
expect(tarballWrite!.context.metadata?.version).toEqual(version);
} finally {
registry.destroy();
}
});
// ============================================================================ // ============================================================================
// Graceful Degradation Tests // Graceful Degradation Tests
// ============================================================================ // ============================================================================
+4 -2
View File
@@ -1,5 +1,6 @@
import { expect, tap } from '@git.zone/tstest/tapbundle'; import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js'; import { SmartRegistry } from '../ts/index.js';
import { streamToJson } from '../ts/core/helpers.stream.js';
import { createTestRegistry, createTestTokens } from './helpers/registry.js'; import { createTestRegistry, createTestTokens } from './helpers/registry.js';
let registry: SmartRegistry; let registry: SmartRegistry;
@@ -54,8 +55,9 @@ tap.test('Integration: should return 404 for unknown paths', async () => {
}); });
expect(response.status).toEqual(404); expect(response.status).toEqual(404);
expect(response.body).toHaveProperty('error'); const body = await streamToJson(response.body);
expect((response.body as any).error).toEqual('NOT_FOUND'); expect(body).toHaveProperty('error');
expect(body.error).toEqual('NOT_FOUND');
}); });
tap.test('Integration: should create and validate tokens', async () => { tap.test('Integration: should create and validate tokens', async () => {
+343
View File
@@ -0,0 +1,343 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js';
import {
createTestRegistryWithUpstream,
createTrackingUpstreamProvider,
} from './helpers/registry.js';
import { StaticUpstreamProvider } from '../ts/upstream/interfaces.upstream.js';
import type {
IUpstreamProvider,
IUpstreamResolutionContext,
IProtocolUpstreamConfig,
} from '../ts/upstream/interfaces.upstream.js';
import type { TRegistryProtocol } from '../ts/core/interfaces.core.js';
// =============================================================================
// StaticUpstreamProvider Tests
// =============================================================================
tap.test('StaticUpstreamProvider: should return config for configured protocol', async () => {
const npmConfig: IProtocolUpstreamConfig = {
enabled: true,
upstreams: [{ id: 'npmjs', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
};
const provider = new StaticUpstreamProvider({
npm: npmConfig,
});
const result = await provider.resolveUpstreamConfig({
protocol: 'npm',
resource: 'lodash',
scope: null,
method: 'GET',
resourceType: 'packument',
});
expect(result).toBeDefined();
expect(result?.enabled).toEqual(true);
expect(result?.upstreams[0].id).toEqual('npmjs');
});
tap.test('StaticUpstreamProvider: should return null for unconfigured protocol', async () => {
const provider = new StaticUpstreamProvider({
npm: {
enabled: true,
upstreams: [{ id: 'npmjs', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
},
});
const result = await provider.resolveUpstreamConfig({
protocol: 'maven',
resource: 'com.example:lib',
scope: 'com.example',
method: 'GET',
resourceType: 'pom',
});
expect(result).toBeNull();
});
tap.test('StaticUpstreamProvider: should support multiple protocols', async () => {
const provider = new StaticUpstreamProvider({
npm: {
enabled: true,
upstreams: [{ id: 'npmjs', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
},
oci: {
enabled: true,
upstreams: [{ id: 'dockerhub', url: 'https://registry-1.docker.io', priority: 1, enabled: true }],
},
maven: {
enabled: true,
upstreams: [{ id: 'central', url: 'https://repo1.maven.org/maven2', priority: 1, enabled: true }],
},
});
const npmResult = await provider.resolveUpstreamConfig({
protocol: 'npm',
resource: 'lodash',
scope: null,
method: 'GET',
resourceType: 'packument',
});
expect(npmResult?.upstreams[0].id).toEqual('npmjs');
const ociResult = await provider.resolveUpstreamConfig({
protocol: 'oci',
resource: 'library/nginx',
scope: 'library',
method: 'GET',
resourceType: 'manifest',
});
expect(ociResult?.upstreams[0].id).toEqual('dockerhub');
const mavenResult = await provider.resolveUpstreamConfig({
protocol: 'maven',
resource: 'com.example:lib',
scope: 'com.example',
method: 'GET',
resourceType: 'pom',
});
expect(mavenResult?.upstreams[0].id).toEqual('central');
});
// =============================================================================
// Registry with Provider Integration Tests
// =============================================================================
let registry: SmartRegistry;
let trackingProvider: ReturnType<typeof createTrackingUpstreamProvider>;
tap.test('Provider Integration: should create registry with upstream provider', async () => {
trackingProvider = createTrackingUpstreamProvider({
npm: {
enabled: true,
upstreams: [{ id: 'test-npm', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
},
});
registry = await createTestRegistryWithUpstream(trackingProvider.provider);
expect(registry).toBeInstanceOf(SmartRegistry);
expect(registry.isInitialized()).toEqual(true);
});
tap.test('Provider Integration: should call provider when fetching unknown npm package', async () => {
// Clear previous calls
trackingProvider.calls.length = 0;
// Request a package that doesn't exist locally - should trigger upstream lookup
const response = await registry.handleRequest({
method: 'GET',
path: '/npm/@test-scope/nonexistent-package',
headers: {},
query: {},
});
// Provider should have been called for the packument lookup
const npmCalls = trackingProvider.calls.filter(c => c.protocol === 'npm');
// The package doesn't exist locally, so upstream should be consulted
// Note: actual upstream fetch may fail since the package doesn't exist
expect(response.status).toBeOneOf([404, 200, 502]); // 404 if not found, 502 if upstream error
});
tap.test('Provider Integration: provider receives correct context for scoped npm package', async () => {
trackingProvider.calls.length = 0;
// Use URL-encoded path for scoped packages as npm client does
await registry.handleRequest({
method: 'GET',
path: '/npm/@myorg%2fmy-package',
headers: {},
query: {},
});
// Find any npm call - the exact resource type depends on routing
const npmCalls = trackingProvider.calls.filter(c => c.protocol === 'npm');
// Provider should be called for upstream lookup
if (npmCalls.length > 0) {
const call = npmCalls[0];
expect(call.protocol).toEqual('npm');
// The resource should include the scoped name
expect(call.resource).toInclude('myorg');
expect(call.method).toEqual('GET');
}
});
tap.test('Provider Integration: provider receives correct context for unscoped npm package', async () => {
trackingProvider.calls.length = 0;
await registry.handleRequest({
method: 'GET',
path: '/npm/lodash',
headers: {},
query: {},
});
const packumentCall = trackingProvider.calls.find(
c => c.protocol === 'npm' && c.resourceType === 'packument'
);
if (packumentCall) {
expect(packumentCall.protocol).toEqual('npm');
expect(packumentCall.resource).toEqual('lodash');
expect(packumentCall.scope).toBeNull(); // No scope for unscoped package
}
});
// =============================================================================
// Custom Provider Implementation Tests
// =============================================================================
tap.test('Custom Provider: should support dynamic resolution based on context', async () => {
// Create a provider that returns different configs based on scope
const dynamicProvider: IUpstreamProvider = {
async resolveUpstreamConfig(context: IUpstreamResolutionContext) {
if (context.scope === 'internal') {
// Internal packages go to private registry
return {
enabled: true,
upstreams: [{ id: 'private', url: 'https://private.registry.com', priority: 1, enabled: true }],
};
}
// Everything else goes to public registry
return {
enabled: true,
upstreams: [{ id: 'public', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
};
},
};
const internalResult = await dynamicProvider.resolveUpstreamConfig({
protocol: 'npm',
resource: '@internal/utils',
scope: 'internal',
method: 'GET',
resourceType: 'packument',
});
expect(internalResult?.upstreams[0].id).toEqual('private');
const publicResult = await dynamicProvider.resolveUpstreamConfig({
protocol: 'npm',
resource: '@public/utils',
scope: 'public',
method: 'GET',
resourceType: 'packument',
});
expect(publicResult?.upstreams[0].id).toEqual('public');
});
tap.test('Custom Provider: should support actor-based resolution', async () => {
const actorAwareProvider: IUpstreamProvider = {
async resolveUpstreamConfig(context: IUpstreamResolutionContext) {
// Different upstreams based on user's organization
if (context.actor?.orgId === 'enterprise-org') {
return {
enabled: true,
upstreams: [{ id: 'enterprise', url: 'https://enterprise.registry.com', priority: 1, enabled: true }],
};
}
return {
enabled: true,
upstreams: [{ id: 'default', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
};
},
};
const enterpriseResult = await actorAwareProvider.resolveUpstreamConfig({
protocol: 'npm',
resource: 'lodash',
scope: null,
actor: { orgId: 'enterprise-org', userId: 'user1' },
method: 'GET',
resourceType: 'packument',
});
expect(enterpriseResult?.upstreams[0].id).toEqual('enterprise');
const defaultResult = await actorAwareProvider.resolveUpstreamConfig({
protocol: 'npm',
resource: 'lodash',
scope: null,
actor: { orgId: 'free-org', userId: 'user2' },
method: 'GET',
resourceType: 'packument',
});
expect(defaultResult?.upstreams[0].id).toEqual('default');
});
tap.test('Custom Provider: should support disabling upstream for specific resources', async () => {
const selectiveProvider: IUpstreamProvider = {
async resolveUpstreamConfig(context: IUpstreamResolutionContext) {
// Block upstream for internal packages
if (context.scope === 'internal') {
return null; // No upstream for internal packages
}
return {
enabled: true,
upstreams: [{ id: 'public', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
};
},
};
const internalResult = await selectiveProvider.resolveUpstreamConfig({
protocol: 'npm',
resource: '@internal/secret',
scope: 'internal',
method: 'GET',
resourceType: 'packument',
});
expect(internalResult).toBeNull();
const publicResult = await selectiveProvider.resolveUpstreamConfig({
protocol: 'npm',
resource: 'lodash',
scope: null,
method: 'GET',
resourceType: 'packument',
});
expect(publicResult).not.toBeNull();
});
// =============================================================================
// Registry without Provider Tests
// =============================================================================
tap.test('No Provider: registry should work without upstream provider', async () => {
const registryWithoutUpstream = await createTestRegistryWithUpstream(
// Pass a provider that always returns null
{
async resolveUpstreamConfig() {
return null;
},
}
);
expect(registryWithoutUpstream).toBeInstanceOf(SmartRegistry);
// Should return 404 for non-existent package (no upstream to check)
const response = await registryWithoutUpstream.handleRequest({
method: 'GET',
path: '/npm/nonexistent-package-xyz',
headers: {},
query: {},
});
expect(response.status).toEqual(404);
registryWithoutUpstream.destroy();
});
// =============================================================================
// Cleanup
// =============================================================================
tap.postTask('cleanup registry', async () => {
if (registry) {
registry.destroy();
}
});
export default tap.start();
+1 -1
View File
@@ -3,6 +3,6 @@
*/ */
export const commitinfo = { export const commitinfo = {
name: '@push.rocks/smartregistry', name: '@push.rocks/smartregistry',
version: '2.6.0', version: '2.9.0',
description: 'A composable TypeScript library implementing OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems registries for building unified container and package registries' description: 'A composable TypeScript library implementing OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems registries for building unified container and package registries'
} }
+88 -56
View File
@@ -2,8 +2,8 @@ import { Smartlog } from '@push.rocks/smartlog';
import { BaseRegistry } from '../core/classes.baseregistry.js'; import { BaseRegistry } from '../core/classes.baseregistry.js';
import { RegistryStorage } from '../core/classes.registrystorage.js'; import { RegistryStorage } from '../core/classes.registrystorage.js';
import { AuthManager } from '../core/classes.authmanager.js'; import { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js'; import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from '../core/interfaces.core.js';
import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js'; import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
import type { import type {
ICargoIndexEntry, ICargoIndexEntry,
ICargoPublishMetadata, ICargoPublishMetadata,
@@ -27,47 +27,55 @@ export class CargoRegistry extends BaseRegistry {
private basePath: string = '/cargo'; private basePath: string = '/cargo';
private registryUrl: string; private registryUrl: string;
private logger: Smartlog; private logger: Smartlog;
private upstream: CargoUpstream | null = null; private upstreamProvider: IUpstreamProvider | null = null;
constructor( constructor(
storage: RegistryStorage, storage: RegistryStorage,
authManager: AuthManager, authManager: AuthManager,
basePath: string = '/cargo', basePath: string = '/cargo',
registryUrl: string = 'http://localhost:5000/cargo', registryUrl: string = 'http://localhost:5000/cargo',
upstreamConfig?: IProtocolUpstreamConfig upstreamProvider?: IUpstreamProvider
) { ) {
super(); super();
this.storage = storage; this.storage = storage;
this.authManager = authManager; this.authManager = authManager;
this.basePath = basePath; this.basePath = basePath;
this.registryUrl = registryUrl; this.registryUrl = registryUrl;
this.upstreamProvider = upstreamProvider || null;
// Initialize logger this.logger = this.createProtocolLogger('cargo-registry', 'cargo');
this.logger = new Smartlog({ }
logContext: {
company: 'push.rocks', /**
companyunit: 'smartregistry', * Get upstream for a specific request.
containerName: 'cargo-registry', * Calls the provider to resolve upstream config dynamically.
environment: (process.env.NODE_ENV as any) || 'development', */
runtime: 'node', private async getUpstreamForRequest(
zone: 'cargo' resource: string,
} resourceType: string,
method: string,
actor?: IRequestActor
): Promise<CargoUpstream | null> {
if (!this.upstreamProvider) return null;
const config = await this.upstreamProvider.resolveUpstreamConfig({
protocol: 'cargo',
resource,
scope: resource, // For Cargo, crate name is the scope
actor,
method,
resourceType,
}); });
this.logger.enableConsole();
// Initialize upstream if configured if (!config?.enabled) return null;
if (upstreamConfig?.enabled) { return new CargoUpstream(config, undefined, this.logger);
this.upstream = new CargoUpstream(upstreamConfig, undefined, this.logger);
}
} }
/** /**
* Clean up resources (timers, connections, etc.) * Clean up resources (timers, connections, etc.)
*/ */
public destroy(): void { public destroy(): void {
if (this.upstream) { // No persistent upstream to clean up with dynamic provider
this.upstream.stop();
}
} }
public async init(): Promise<void> { public async init(): Promise<void> {
@@ -91,27 +99,31 @@ export class CargoRegistry extends BaseRegistry {
const path = context.path.replace(this.basePath, ''); const path = context.path.replace(this.basePath, '');
// Extract token (Cargo uses Authorization header WITHOUT "Bearer" prefix) // Extract token (Cargo uses Authorization header WITHOUT "Bearer" prefix)
const authHeader = context.headers['authorization'] || context.headers['Authorization']; const authHeader = this.getAuthorizationHeader(context);
const token = authHeader ? await this.authManager.validateToken(authHeader, 'cargo') : null; const token = authHeader ? await this.authManager.validateToken(authHeader, 'cargo') : null;
const actor: IRequestActor = this.buildRequestActor(context, token);
this.logger.log('debug', `handleRequest: ${context.method} ${path}`, { this.logger.log('debug', `handleRequest: ${context.method} ${path}`, {
method: context.method, method: context.method,
path, path,
hasAuth: !!token hasAuth: !!token
}); });
// Config endpoint (required for sparse protocol) return this.storage.withContext({ protocol: 'cargo', actor }, async () => {
if (path === '/config.json') { // Config endpoint (required for sparse protocol)
return this.handleConfigJson(); if (path === '/config.json') {
} return this.handleConfigJson();
}
// API endpoints // API endpoints
if (path.startsWith('/api/v1/')) { if (path.startsWith('/api/v1/')) {
return this.handleApiRequest(path, context, token); return this.handleApiRequest(path, context, token, actor);
} }
// Index files (sparse protocol) // Index files (sparse protocol)
return this.handleIndexRequest(path); return this.handleIndexRequest(path, actor);
});
} }
/** /**
@@ -132,7 +144,8 @@ export class CargoRegistry extends BaseRegistry {
private async handleApiRequest( private async handleApiRequest(
path: string, path: string,
context: IRequestContext, context: IRequestContext,
token: IAuthToken | null token: IAuthToken | null,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
// Publish: PUT /api/v1/crates/new // Publish: PUT /api/v1/crates/new
if (path === '/api/v1/crates/new' && context.method === 'PUT') { if (path === '/api/v1/crates/new' && context.method === 'PUT') {
@@ -142,7 +155,7 @@ export class CargoRegistry extends BaseRegistry {
// Download: GET /api/v1/crates/{crate}/{version}/download // Download: GET /api/v1/crates/{crate}/{version}/download
const downloadMatch = path.match(/^\/api\/v1\/crates\/([^\/]+)\/([^\/]+)\/download$/); const downloadMatch = path.match(/^\/api\/v1\/crates\/([^\/]+)\/([^\/]+)\/download$/);
if (downloadMatch && context.method === 'GET') { if (downloadMatch && context.method === 'GET') {
return this.handleDownload(downloadMatch[1], downloadMatch[2]); return this.handleDownload(downloadMatch[1], downloadMatch[2], actor);
} }
// Yank: DELETE /api/v1/crates/{crate}/{version}/yank // Yank: DELETE /api/v1/crates/{crate}/{version}/yank
@@ -175,7 +188,7 @@ export class CargoRegistry extends BaseRegistry {
* Handle index file requests * Handle index file requests
* Paths: /1/{name}, /2/{name}, /3/{c}/{name}, /{p1}/{p2}/{name} * Paths: /1/{name}, /2/{name}, /3/{c}/{name}, /{p1}/{p2}/{name}
*/ */
private async handleIndexRequest(path: string): Promise<IResponse> { private async handleIndexRequest(path: string, actor?: IRequestActor): Promise<IResponse> {
// Parse index paths to extract crate name // Parse index paths to extract crate name
const pathParts = path.split('/').filter(p => p); const pathParts = path.split('/').filter(p => p);
let crateName: string | null = null; let crateName: string | null = null;
@@ -202,7 +215,7 @@ export class CargoRegistry extends BaseRegistry {
}; };
} }
return this.handleIndexFile(crateName); return this.handleIndexFile(crateName, actor);
} }
/** /**
@@ -224,23 +237,26 @@ export class CargoRegistry extends BaseRegistry {
/** /**
* Serve index file for a crate * Serve index file for a crate
*/ */
private async handleIndexFile(crateName: string): Promise<IResponse> { private async handleIndexFile(crateName: string, actor?: IRequestActor): Promise<IResponse> {
let index = await this.storage.getCargoIndex(crateName); let index = await this.storage.getCargoIndex(crateName);
// Try upstream if not found locally // Try upstream if not found locally
if ((!index || index.length === 0) && this.upstream) { if (!index || index.length === 0) {
const upstreamIndex = await this.upstream.fetchCrateIndex(crateName); const upstream = await this.getUpstreamForRequest(crateName, 'index', 'GET', actor);
if (upstreamIndex) { if (upstream) {
// Parse the newline-delimited JSON const upstreamIndex = await upstream.fetchCrateIndex(crateName);
const parsedIndex: ICargoIndexEntry[] = upstreamIndex if (upstreamIndex) {
.split('\n') // Parse the newline-delimited JSON
.filter(line => line.trim()) const parsedIndex: ICargoIndexEntry[] = upstreamIndex
.map(line => JSON.parse(line)); .split('\n')
.filter(line => line.trim())
.map(line => JSON.parse(line));
if (parsedIndex.length > 0) { if (parsedIndex.length > 0) {
// Cache locally // Cache locally
await this.storage.putCargoIndex(crateName, parsedIndex); await this.storage.putCargoIndex(crateName, parsedIndex);
index = parsedIndex; index = parsedIndex;
}
} }
} }
} }
@@ -339,7 +355,7 @@ export class CargoRegistry extends BaseRegistry {
const parsed = this.parsePublishRequest(body); const parsed = this.parsePublishRequest(body);
metadata = parsed.metadata; metadata = parsed.metadata;
crateFile = parsed.crateFile; crateFile = parsed.crateFile;
} catch (error) { } catch (error: any) {
this.logger.log('error', 'handlePublish: parse error', { error: error.message }); this.logger.log('error', 'handlePublish: parse error', { error: error.message });
return { return {
status: 400, status: 400,
@@ -431,15 +447,31 @@ export class CargoRegistry extends BaseRegistry {
*/ */
private async handleDownload( private async handleDownload(
crateName: string, crateName: string,
version: string version: string,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
this.logger.log('debug', 'handleDownload', { crate: crateName, version }); this.logger.log('debug', 'handleDownload', { crate: crateName, version });
let crateFile = await this.storage.getCargoCrate(crateName, version); // Try streaming from local storage first
const streamResult = await this.storage.getCargoCrateStream(crateName, version);
if (streamResult) {
return {
status: 200,
headers: {
'Content-Type': 'application/gzip',
'Content-Length': streamResult.size.toString(),
'Content-Disposition': `attachment; filename="${crateName}-${version}.crate"`,
},
body: streamResult.stream,
};
}
// Try upstream if not found locally // Try upstream if not found locally
if (!crateFile && this.upstream) { let crateFile: Buffer | null = null;
crateFile = await this.upstream.fetchCrate(crateName, version); const upstream = await this.getUpstreamForRequest(crateName, 'crate', 'GET', actor);
if (upstream) {
crateFile = await upstream.fetchCrate(crateName, version);
if (crateFile) { if (crateFile) {
// Cache locally // Cache locally
await this.storage.putCargoCrate(crateName, version, crateFile); await this.storage.putCargoCrate(crateName, version, crateFile);
@@ -612,7 +644,7 @@ export class CargoRegistry extends BaseRegistry {
} }
} }
} }
} catch (error) { } catch (error: any) {
this.logger.log('error', 'handleSearch: error', { error: error.message }); this.logger.log('error', 'handleSearch: error', { error: error.message });
} }
+176 -170
View File
@@ -1,7 +1,14 @@
import { RegistryStorage } from './core/classes.registrystorage.js'; import { RegistryStorage } from './core/classes.registrystorage.js';
import { AuthManager } from './core/classes.authmanager.js'; import { AuthManager } from './core/classes.authmanager.js';
import { BaseRegistry } from './core/classes.baseregistry.js'; import { BaseRegistry } from './core/classes.baseregistry.js';
import type { IRegistryConfig, IRequestContext, IResponse } from './core/interfaces.core.js'; import type {
IProtocolConfig,
IRegistryConfig,
IRequestContext,
IResponse,
TRegistryProtocol,
} from './core/interfaces.core.js';
import { toReadableStream } from './core/helpers.stream.js';
import { OciRegistry } from './oci/classes.ociregistry.js'; import { OciRegistry } from './oci/classes.ociregistry.js';
import { NpmRegistry } from './npm/classes.npmregistry.js'; import { NpmRegistry } from './npm/classes.npmregistry.js';
import { MavenRegistry } from './maven/classes.mavenregistry.js'; import { MavenRegistry } from './maven/classes.mavenregistry.js';
@@ -10,6 +17,129 @@ import { ComposerRegistry } from './composer/classes.composerregistry.js';
import { PypiRegistry } from './pypi/classes.pypiregistry.js'; import { PypiRegistry } from './pypi/classes.pypiregistry.js';
import { RubyGemsRegistry } from './rubygems/classes.rubygemsregistry.js'; import { RubyGemsRegistry } from './rubygems/classes.rubygemsregistry.js';
type TRegistryDescriptor = {
protocol: TRegistryProtocol;
getConfig: (config: IRegistryConfig) => IProtocolConfig | undefined;
matchesPath: (config: IRegistryConfig, path: string) => boolean;
create: (args: {
storage: RegistryStorage;
authManager: AuthManager;
config: IRegistryConfig;
protocolConfig: IProtocolConfig;
}) => BaseRegistry;
};
const registryDescriptors: TRegistryDescriptor[] = [
{
protocol: 'oci',
getConfig: (config) => config.oci,
matchesPath: (config, path) => path.startsWith(config.oci?.basePath ?? '/oci'),
create: ({ storage, authManager, config, protocolConfig }) => {
const ociTokens = config.auth.ociTokens?.enabled ? {
realm: config.auth.ociTokens.realm,
service: config.auth.ociTokens.service,
} : undefined;
return new OciRegistry(
storage,
authManager,
protocolConfig.basePath ?? '/oci',
ociTokens,
config.upstreamProvider
);
},
},
{
protocol: 'npm',
getConfig: (config) => config.npm,
matchesPath: (config, path) => path.startsWith(config.npm?.basePath ?? '/npm'),
create: ({ storage, authManager, config, protocolConfig }) => {
const basePath = protocolConfig.basePath ?? '/npm';
return new NpmRegistry(
storage,
authManager,
basePath,
protocolConfig.registryUrl ?? `http://localhost:5000${basePath}`,
config.upstreamProvider
);
},
},
{
protocol: 'maven',
getConfig: (config) => config.maven,
matchesPath: (config, path) => path.startsWith(config.maven?.basePath ?? '/maven'),
create: ({ storage, authManager, config, protocolConfig }) => {
const basePath = protocolConfig.basePath ?? '/maven';
return new MavenRegistry(
storage,
authManager,
basePath,
protocolConfig.registryUrl ?? `http://localhost:5000${basePath}`,
config.upstreamProvider
);
},
},
{
protocol: 'cargo',
getConfig: (config) => config.cargo,
matchesPath: (config, path) => path.startsWith(config.cargo?.basePath ?? '/cargo'),
create: ({ storage, authManager, config, protocolConfig }) => {
const basePath = protocolConfig.basePath ?? '/cargo';
return new CargoRegistry(
storage,
authManager,
basePath,
protocolConfig.registryUrl ?? `http://localhost:5000${basePath}`,
config.upstreamProvider
);
},
},
{
protocol: 'composer',
getConfig: (config) => config.composer,
matchesPath: (config, path) => path.startsWith(config.composer?.basePath ?? '/composer'),
create: ({ storage, authManager, config, protocolConfig }) => {
const basePath = protocolConfig.basePath ?? '/composer';
return new ComposerRegistry(
storage,
authManager,
basePath,
protocolConfig.registryUrl ?? `http://localhost:5000${basePath}`,
config.upstreamProvider
);
},
},
{
protocol: 'pypi',
getConfig: (config) => config.pypi,
matchesPath: (config, path) => {
const basePath = config.pypi?.basePath ?? '/pypi';
return path.startsWith(basePath) || path.startsWith('/simple');
},
create: ({ storage, authManager, config, protocolConfig }) => new PypiRegistry(
storage,
authManager,
protocolConfig.basePath ?? '/pypi',
protocolConfig.registryUrl ?? 'http://localhost:5000',
config.upstreamProvider
),
},
{
protocol: 'rubygems',
getConfig: (config) => config.rubygems,
matchesPath: (config, path) => path.startsWith(config.rubygems?.basePath ?? '/rubygems'),
create: ({ storage, authManager, config, protocolConfig }) => {
const basePath = protocolConfig.basePath ?? '/rubygems';
return new RubyGemsRegistry(
storage,
authManager,
basePath,
protocolConfig.registryUrl ?? `http://localhost:5000${basePath}`,
config.upstreamProvider
);
},
},
];
/** /**
* Main registry orchestrator. * Main registry orchestrator.
* Routes requests to appropriate protocol handlers (OCI, NPM, Maven, Cargo, Composer, PyPI, or RubyGems). * Routes requests to appropriate protocol handlers (OCI, NPM, Maven, Cargo, Composer, PyPI, or RubyGems).
@@ -48,7 +178,7 @@ import { RubyGemsRegistry } from './rubygems/classes.rubygemsregistry.js';
export class SmartRegistry { export class SmartRegistry {
private storage: RegistryStorage; private storage: RegistryStorage;
private authManager: AuthManager; private authManager: AuthManager;
private registries: Map<string, BaseRegistry> = new Map(); private registries: Map<TRegistryProtocol, BaseRegistry> = new Map();
private config: IRegistryConfig; private config: IRegistryConfig;
private initialized: boolean = false; private initialized: boolean = false;
@@ -74,112 +204,20 @@ export class SmartRegistry {
// Initialize auth manager // Initialize auth manager
await this.authManager.init(); await this.authManager.init();
// Initialize OCI registry if enabled for (const descriptor of registryDescriptors) {
if (this.config.oci?.enabled) { const protocolConfig = descriptor.getConfig(this.config);
const ociBasePath = this.config.oci.basePath ?? '/oci'; if (!protocolConfig?.enabled) {
const ociTokens = this.config.auth.ociTokens?.enabled ? { continue;
realm: this.config.auth.ociTokens.realm, }
service: this.config.auth.ociTokens.service,
} : undefined;
const ociRegistry = new OciRegistry(
this.storage,
this.authManager,
ociBasePath,
ociTokens,
this.config.oci.upstream
);
await ociRegistry.init();
this.registries.set('oci', ociRegistry);
}
// Initialize NPM registry if enabled const registry = descriptor.create({
if (this.config.npm?.enabled) { storage: this.storage,
const npmBasePath = this.config.npm.basePath ?? '/npm'; authManager: this.authManager,
const registryUrl = `http://localhost:5000${npmBasePath}`; // TODO: Make configurable config: this.config,
const npmRegistry = new NpmRegistry( protocolConfig,
this.storage, });
this.authManager, await registry.init();
npmBasePath, this.registries.set(descriptor.protocol, registry);
registryUrl,
this.config.npm.upstream
);
await npmRegistry.init();
this.registries.set('npm', npmRegistry);
}
// Initialize Maven registry if enabled
if (this.config.maven?.enabled) {
const mavenBasePath = this.config.maven.basePath ?? '/maven';
const registryUrl = `http://localhost:5000${mavenBasePath}`; // TODO: Make configurable
const mavenRegistry = new MavenRegistry(
this.storage,
this.authManager,
mavenBasePath,
registryUrl,
this.config.maven.upstream
);
await mavenRegistry.init();
this.registries.set('maven', mavenRegistry);
}
// Initialize Cargo registry if enabled
if (this.config.cargo?.enabled) {
const cargoBasePath = this.config.cargo.basePath ?? '/cargo';
const registryUrl = `http://localhost:5000${cargoBasePath}`; // TODO: Make configurable
const cargoRegistry = new CargoRegistry(
this.storage,
this.authManager,
cargoBasePath,
registryUrl,
this.config.cargo.upstream
);
await cargoRegistry.init();
this.registries.set('cargo', cargoRegistry);
}
// Initialize Composer registry if enabled
if (this.config.composer?.enabled) {
const composerBasePath = this.config.composer.basePath ?? '/composer';
const registryUrl = `http://localhost:5000${composerBasePath}`; // TODO: Make configurable
const composerRegistry = new ComposerRegistry(
this.storage,
this.authManager,
composerBasePath,
registryUrl,
this.config.composer.upstream
);
await composerRegistry.init();
this.registries.set('composer', composerRegistry);
}
// Initialize PyPI registry if enabled
if (this.config.pypi?.enabled) {
const pypiBasePath = this.config.pypi.basePath ?? '/pypi';
const registryUrl = `http://localhost:5000`; // TODO: Make configurable
const pypiRegistry = new PypiRegistry(
this.storage,
this.authManager,
pypiBasePath,
registryUrl,
this.config.pypi.upstream
);
await pypiRegistry.init();
this.registries.set('pypi', pypiRegistry);
}
// Initialize RubyGems registry if enabled
if (this.config.rubygems?.enabled) {
const rubygemsBasePath = this.config.rubygems.basePath ?? '/rubygems';
const registryUrl = `http://localhost:5000${rubygemsBasePath}`; // TODO: Make configurable
const rubygemsRegistry = new RubyGemsRegistry(
this.storage,
this.authManager,
rubygemsBasePath,
registryUrl,
this.config.rubygems.upstream
);
await rubygemsRegistry.init();
this.registries.set('rubygems', rubygemsRegistry);
} }
this.initialized = true; this.initialized = true;
@@ -191,75 +229,45 @@ export class SmartRegistry {
*/ */
public async handleRequest(context: IRequestContext): Promise<IResponse> { public async handleRequest(context: IRequestContext): Promise<IResponse> {
const path = context.path; const path = context.path;
let response: IResponse | undefined;
// Route to OCI registry for (const descriptor of registryDescriptors) {
if (this.config.oci?.enabled && path.startsWith(this.config.oci.basePath)) { if (response) {
const ociRegistry = this.registries.get('oci'); break;
if (ociRegistry) {
return ociRegistry.handleRequest(context);
} }
}
// Route to NPM registry const protocolConfig = descriptor.getConfig(this.config);
if (this.config.npm?.enabled && path.startsWith(this.config.npm.basePath)) { if (!protocolConfig?.enabled || !descriptor.matchesPath(this.config, path)) {
const npmRegistry = this.registries.get('npm'); continue;
if (npmRegistry) {
return npmRegistry.handleRequest(context);
} }
}
// Route to Maven registry const registry = this.registries.get(descriptor.protocol);
if (this.config.maven?.enabled && path.startsWith(this.config.maven.basePath)) { if (registry) {
const mavenRegistry = this.registries.get('maven'); response = await registry.handleRequest(context);
if (mavenRegistry) {
return mavenRegistry.handleRequest(context);
}
}
// Route to Cargo registry
if (this.config.cargo?.enabled && path.startsWith(this.config.cargo.basePath)) {
const cargoRegistry = this.registries.get('cargo');
if (cargoRegistry) {
return cargoRegistry.handleRequest(context);
}
}
// Route to Composer registry
if (this.config.composer?.enabled && path.startsWith(this.config.composer.basePath)) {
const composerRegistry = this.registries.get('composer');
if (composerRegistry) {
return composerRegistry.handleRequest(context);
}
}
// Route to PyPI registry (also handles /simple prefix)
if (this.config.pypi?.enabled) {
const pypiBasePath = this.config.pypi.basePath ?? '/pypi';
if (path.startsWith(pypiBasePath) || path.startsWith('/simple')) {
const pypiRegistry = this.registries.get('pypi');
if (pypiRegistry) {
return pypiRegistry.handleRequest(context);
}
}
}
// Route to RubyGems registry
if (this.config.rubygems?.enabled && path.startsWith(this.config.rubygems.basePath)) {
const rubygemsRegistry = this.registries.get('rubygems');
if (rubygemsRegistry) {
return rubygemsRegistry.handleRequest(context);
} }
} }
// No matching registry // No matching registry
return { if (!response) {
status: 404, response = {
headers: { 'Content-Type': 'application/json' }, status: 404,
body: { headers: { 'Content-Type': 'application/json' },
error: 'NOT_FOUND', body: {
message: 'No registry handler for this path', error: 'NOT_FOUND',
}, message: 'No registry handler for this path',
}; },
};
}
// Normalize body to ReadableStream<Uint8Array> at the API boundary
if (response.body != null && !(response.body instanceof ReadableStream)) {
if (!Buffer.isBuffer(response.body) && typeof response.body === 'object' && !(response.body instanceof Uint8Array)) {
response.headers['Content-Type'] ??= 'application/json';
}
response.body = toReadableStream(response.body);
}
return response;
} }
/** /**
@@ -295,9 +303,7 @@ export class SmartRegistry {
*/ */
public destroy(): void { public destroy(): void {
for (const registry of this.registries.values()) { for (const registry of this.registries.values()) {
if (typeof (registry as any).destroy === 'function') { registry.destroy();
(registry as any).destroy();
}
} }
} }
} }
+128 -87
View File
@@ -6,8 +6,8 @@
import { BaseRegistry } from '../core/classes.baseregistry.js'; import { BaseRegistry } from '../core/classes.baseregistry.js';
import type { RegistryStorage } from '../core/classes.registrystorage.js'; import type { RegistryStorage } from '../core/classes.registrystorage.js';
import type { AuthManager } from '../core/classes.authmanager.js'; import type { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js'; import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from '../core/interfaces.core.js';
import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js'; import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
import { isBinaryData, toBuffer } from '../core/helpers.buffer.js'; import { isBinaryData, toBuffer } from '../core/helpers.buffer.js';
import type { import type {
IComposerPackage, IComposerPackage,
@@ -30,34 +30,66 @@ export class ComposerRegistry extends BaseRegistry {
private authManager: AuthManager; private authManager: AuthManager;
private basePath: string = '/composer'; private basePath: string = '/composer';
private registryUrl: string; private registryUrl: string;
private upstream: ComposerUpstream | null = null; private upstreamProvider: IUpstreamProvider | null = null;
constructor( constructor(
storage: RegistryStorage, storage: RegistryStorage,
authManager: AuthManager, authManager: AuthManager,
basePath: string = '/composer', basePath: string = '/composer',
registryUrl: string = 'http://localhost:5000/composer', registryUrl: string = 'http://localhost:5000/composer',
upstreamConfig?: IProtocolUpstreamConfig upstreamProvider?: IUpstreamProvider
) { ) {
super(); super();
this.storage = storage; this.storage = storage;
this.authManager = authManager; this.authManager = authManager;
this.basePath = basePath; this.basePath = basePath;
this.registryUrl = registryUrl; this.registryUrl = registryUrl;
this.upstreamProvider = upstreamProvider || null;
}
// Initialize upstream if configured /**
if (upstreamConfig?.enabled) { * Extract scope from Composer package name.
this.upstream = new ComposerUpstream(upstreamConfig); * For Composer, vendor is the scope.
* @example "symfony" from "symfony/console"
*/
private extractScope(vendorPackage: string): string | null {
const slashIndex = vendorPackage.indexOf('/');
if (slashIndex > 0) {
return vendorPackage.substring(0, slashIndex);
} }
return null;
}
/**
* Get upstream for a specific request.
* Calls the provider to resolve upstream config dynamically.
*/
private async getUpstreamForRequest(
resource: string,
resourceType: string,
method: string,
actor?: IRequestActor
): Promise<ComposerUpstream | null> {
if (!this.upstreamProvider) return null;
const config = await this.upstreamProvider.resolveUpstreamConfig({
protocol: 'composer',
resource,
scope: this.extractScope(resource),
actor,
method,
resourceType,
});
if (!config?.enabled) return null;
return new ComposerUpstream(config);
} }
/** /**
* Clean up resources (timers, connections, etc.) * Clean up resources (timers, connections, etc.)
*/ */
public destroy(): void { public destroy(): void {
if (this.upstream) { // No persistent upstream to clean up with dynamic provider
this.upstream.stop();
}
} }
public async init(): Promise<void> { public async init(): Promise<void> {
@@ -72,81 +104,86 @@ export class ComposerRegistry extends BaseRegistry {
const path = context.path.replace(this.basePath, ''); const path = context.path.replace(this.basePath, '');
// Extract token from Authorization header // Extract token from Authorization header
const authHeader = context.headers['authorization'] || context.headers['Authorization']; const authHeader = this.getAuthorizationHeader(context);
let token: IAuthToken | null = null; let token: IAuthToken | null = null;
if (authHeader) { if (authHeader) {
if (authHeader.startsWith('Bearer ')) { const tokenString = this.extractBearerToken(authHeader);
const tokenString = authHeader.replace(/^Bearer\s+/i, ''); if (tokenString) {
token = await this.authManager.validateToken(tokenString, 'composer'); token = await this.authManager.validateToken(tokenString, 'composer');
} else if (authHeader.startsWith('Basic ')) { } else {
// Handle HTTP Basic Auth // Handle HTTP Basic Auth
const credentials = Buffer.from(authHeader.replace(/^Basic\s+/i, ''), 'base64').toString('utf-8'); const basicCredentials = this.parseBasicAuthHeader(authHeader);
const [username, password] = credentials.split(':'); if (basicCredentials) {
const userId = await this.authManager.authenticate({ username, password }); const userId = await this.authManager.authenticate(basicCredentials);
if (userId) { if (userId) {
// Create temporary token for this request // Create temporary token for this request
token = { token = {
type: 'composer', type: 'composer',
userId, userId,
scopes: ['composer:*:*:read'], scopes: ['composer:*:*:read'],
readonly: true, readonly: true,
}; };
}
} }
} }
} }
// Root packages.json const actor: IRequestActor = this.buildRequestActor(context, token);
if (path === '/packages.json' || path === '' || path === '/') {
return this.handlePackagesJson();
}
// Package metadata: /p2/{vendor}/{package}.json or /p2/{vendor}/{package}~dev.json return this.storage.withContext({ protocol: 'composer', actor }, async () => {
const metadataMatch = path.match(/^\/p2\/([^\/]+\/[^\/]+?)(~dev)?\.json$/); // Root packages.json
if (metadataMatch) { if (path === '/packages.json' || path === '' || path === '/') {
const [, vendorPackage, devSuffix] = metadataMatch; return this.handlePackagesJson();
const includeDev = !!devSuffix; }
return this.handlePackageMetadata(vendorPackage, includeDev, token);
}
// Package list: /packages/list.json?filter=vendor/* // Package metadata: /p2/{vendor}/{package}.json or /p2/{vendor}/{package}~dev.json
if (path.startsWith('/packages/list.json')) { const metadataMatch = path.match(/^\/p2\/([^\/]+\/[^\/]+?)(~dev)?\.json$/);
const filter = context.query['filter']; if (metadataMatch) {
return this.handlePackageList(filter, token); const [, vendorPackage, devSuffix] = metadataMatch;
} const includeDev = !!devSuffix;
return this.handlePackageMetadata(vendorPackage, includeDev, token, actor);
}
// Package ZIP download: /dists/{vendor}/{package}/{reference}.zip // Package list: /packages/list.json?filter=vendor/*
const distMatch = path.match(/^\/dists\/([^\/]+\/[^\/]+)\/([^\/]+)\.zip$/); if (path.startsWith('/packages/list.json')) {
if (distMatch) { const filter = context.query['filter'];
const [, vendorPackage, reference] = distMatch; return this.handlePackageList(filter, token);
return this.handlePackageDownload(vendorPackage, reference, token); }
}
// Package upload: PUT /packages/{vendor}/{package} // Package ZIP download: /dists/{vendor}/{package}/{reference}.zip
const uploadMatch = path.match(/^\/packages\/([^\/]+\/[^\/]+)$/); const distMatch = path.match(/^\/dists\/([^\/]+\/[^\/]+)\/([^\/]+)\.zip$/);
if (uploadMatch && context.method === 'PUT') { if (distMatch) {
const vendorPackage = uploadMatch[1]; const [, vendorPackage, reference] = distMatch;
return this.handlePackageUpload(vendorPackage, context.body, token); return this.handlePackageDownload(vendorPackage, reference, token);
} }
// Package delete: DELETE /packages/{vendor}/{package} // Package upload: PUT /packages/{vendor}/{package}
if (uploadMatch && context.method === 'DELETE') { const uploadMatch = path.match(/^\/packages\/([^\/]+\/[^\/]+)$/);
const vendorPackage = uploadMatch[1]; if (uploadMatch && context.method === 'PUT') {
return this.handlePackageDelete(vendorPackage, token); const vendorPackage = uploadMatch[1];
} return this.handlePackageUpload(vendorPackage, context.body, token);
}
// Version delete: DELETE /packages/{vendor}/{package}/{version} // Package delete: DELETE /packages/{vendor}/{package}
const versionDeleteMatch = path.match(/^\/packages\/([^\/]+\/[^\/]+)\/(.+)$/); if (uploadMatch && context.method === 'DELETE') {
if (versionDeleteMatch && context.method === 'DELETE') { const vendorPackage = uploadMatch[1];
const [, vendorPackage, version] = versionDeleteMatch; return this.handlePackageDelete(vendorPackage, token);
return this.handleVersionDelete(vendorPackage, version, token); }
}
return { // Version delete: DELETE /packages/{vendor}/{package}/{version}
status: 404, const versionDeleteMatch = path.match(/^\/packages\/([^\/]+\/[^\/]+)\/(.+)$/);
headers: { 'Content-Type': 'application/json' }, if (versionDeleteMatch && context.method === 'DELETE') {
body: { status: 'error', message: 'Not found' }, const [, vendorPackage, version] = versionDeleteMatch;
}; return this.handleVersionDelete(vendorPackage, version, token);
}
return {
status: 404,
headers: { 'Content-Type': 'application/json' },
body: { status: 'error', message: 'Not found' },
};
});
} }
protected async checkPermission( protected async checkPermission(
@@ -176,26 +213,30 @@ export class ComposerRegistry extends BaseRegistry {
private async handlePackageMetadata( private async handlePackageMetadata(
vendorPackage: string, vendorPackage: string,
includeDev: boolean, includeDev: boolean,
token: IAuthToken | null token: IAuthToken | null,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
// Read operations are public, no authentication required // Read operations are public, no authentication required
let metadata = await this.storage.getComposerPackageMetadata(vendorPackage); let metadata = await this.storage.getComposerPackageMetadata(vendorPackage);
// Try upstream if not found locally // Try upstream if not found locally
if (!metadata && this.upstream) { if (!metadata) {
const [vendor, packageName] = vendorPackage.split('/'); const upstream = await this.getUpstreamForRequest(vendorPackage, 'metadata', 'GET', actor);
if (vendor && packageName) { if (upstream) {
const upstreamMetadata = includeDev const [vendor, packageName] = vendorPackage.split('/');
? await this.upstream.fetchPackageDevMetadata(vendor, packageName) if (vendor && packageName) {
: await this.upstream.fetchPackageMetadata(vendor, packageName); const upstreamMetadata = includeDev
? await upstream.fetchPackageDevMetadata(vendor, packageName)
: await upstream.fetchPackageMetadata(vendor, packageName);
if (upstreamMetadata && upstreamMetadata.packages) { if (upstreamMetadata && upstreamMetadata.packages) {
// Store upstream metadata locally // Store upstream metadata locally
metadata = { metadata = {
packages: upstreamMetadata.packages, packages: upstreamMetadata.packages,
lastModified: new Date().toUTCString(), lastModified: new Date().toUTCString(),
}; };
await this.storage.putComposerPackageMetadata(vendorPackage, metadata); await this.storage.putComposerPackageMetadata(vendorPackage, metadata);
}
} }
} }
} }
@@ -258,9 +299,9 @@ export class ComposerRegistry extends BaseRegistry {
token: IAuthToken | null token: IAuthToken | null
): Promise<IResponse> { ): Promise<IResponse> {
// Read operations are public, no authentication required // Read operations are public, no authentication required
const zipData = await this.storage.getComposerPackageZip(vendorPackage, reference); const streamResult = await this.storage.getComposerPackageZipStream(vendorPackage, reference);
if (!zipData) { if (!streamResult) {
return { return {
status: 404, status: 404,
headers: {}, headers: {},
@@ -272,10 +313,10 @@ export class ComposerRegistry extends BaseRegistry {
status: 200, status: 200,
headers: { headers: {
'Content-Type': 'application/zip', 'Content-Type': 'application/zip',
'Content-Length': zipData.length.toString(), 'Content-Length': streamResult.size.toString(),
'Content-Disposition': `attachment; filename="${reference}.zip"`, 'Content-Disposition': `attachment; filename="${reference}.zip"`,
}, },
body: zipData, body: streamResult.stream,
}; };
} }
+118 -1
View File
@@ -1,14 +1,131 @@
import type { IRequestContext, IResponse, IAuthToken } from './interfaces.core.js'; import * as plugins from '../plugins.js';
import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from './interfaces.core.js';
/** /**
* Abstract base class for all registry protocol implementations * Abstract base class for all registry protocol implementations
*/ */
export abstract class BaseRegistry { export abstract class BaseRegistry {
protected getHeader(contextOrHeaders: IRequestContext | Record<string, string>, name: string): string | undefined {
const headers = 'headers' in contextOrHeaders ? contextOrHeaders.headers : contextOrHeaders;
if (headers[name] !== undefined) {
return headers[name];
}
const lowerName = name.toLowerCase();
for (const [headerName, value] of Object.entries(headers)) {
if (headerName.toLowerCase() === lowerName) {
return value;
}
}
return undefined;
}
protected getAuthorizationHeader(context: IRequestContext): string | undefined {
return this.getHeader(context, 'authorization');
}
protected getClientIp(context: IRequestContext): string | undefined {
const forwardedFor = this.getHeader(context, 'x-forwarded-for');
if (forwardedFor) {
return forwardedFor.split(',')[0]?.trim();
}
return this.getHeader(context, 'x-real-ip');
}
protected getUserAgent(context: IRequestContext): string | undefined {
return this.getHeader(context, 'user-agent');
}
protected extractBearerToken(contextOrHeader: IRequestContext | string | undefined): string | null {
const authHeader = typeof contextOrHeader === 'string'
? contextOrHeader
: contextOrHeader
? this.getAuthorizationHeader(contextOrHeader)
: undefined;
if (!authHeader || !/^Bearer\s+/i.test(authHeader)) {
return null;
}
return authHeader.replace(/^Bearer\s+/i, '');
}
protected parseBasicAuthHeader(authHeader: string | undefined): { username: string; password: string } | null {
if (!authHeader || !/^Basic\s+/i.test(authHeader)) {
return null;
}
const base64 = authHeader.replace(/^Basic\s+/i, '');
const decoded = Buffer.from(base64, 'base64').toString('utf-8');
const separatorIndex = decoded.indexOf(':');
if (separatorIndex < 0) {
return {
username: decoded,
password: '',
};
}
return {
username: decoded.substring(0, separatorIndex),
password: decoded.substring(separatorIndex + 1),
};
}
protected buildRequestActor(context: IRequestContext, token: IAuthToken | null): IRequestActor {
const actor: IRequestActor = {
...(context.actor ?? {}),
};
if (token?.userId) {
actor.userId = token.userId;
}
const ip = this.getClientIp(context);
if (ip) {
actor.ip = ip;
}
const userAgent = this.getUserAgent(context);
if (userAgent) {
actor.userAgent = userAgent;
}
return actor;
}
protected createProtocolLogger(
containerName: string,
zone: string
): plugins.smartlog.Smartlog {
const logger = new plugins.smartlog.Smartlog({
logContext: {
company: 'push.rocks',
companyunit: 'smartregistry',
containerName,
environment: (process.env.NODE_ENV as any) || 'development',
runtime: 'node',
zone,
}
});
logger.enableConsole();
return logger;
}
/** /**
* Initialize the registry * Initialize the registry
*/ */
abstract init(): Promise<void>; abstract init(): Promise<void>;
/**
* Clean up timers, connections, and other registry resources.
*/
public destroy(): void {
// Default no-op for registries without persistent resources.
}
/** /**
* Handle an incoming HTTP request * Handle an incoming HTTP request
* @param context - Request context * @param context - Request context
File diff suppressed because it is too large Load Diff
+109
View File
@@ -0,0 +1,109 @@
function digestToHash(digest: string): string {
return digest.split(':')[1];
}
export function getOciBlobPath(digest: string): string {
return `oci/blobs/sha256/${digestToHash(digest)}`;
}
export function getOciManifestPath(repository: string, digest: string): string {
return `oci/manifests/${repository}/${digestToHash(digest)}`;
}
export function getNpmPackumentPath(packageName: string): string {
return `npm/packages/${packageName}/index.json`;
}
export function getNpmTarballPath(packageName: string, version: string): string {
const safeName = packageName.replace('@', '').replace('/', '-');
return `npm/packages/${packageName}/${safeName}-${version}.tgz`;
}
export function getMavenArtifactPath(
groupId: string,
artifactId: string,
version: string,
filename: string
): string {
const groupPath = groupId.replace(/\./g, '/');
return `maven/artifacts/${groupPath}/${artifactId}/${version}/${filename}`;
}
export function getMavenMetadataPath(groupId: string, artifactId: string): string {
const groupPath = groupId.replace(/\./g, '/');
return `maven/metadata/${groupPath}/${artifactId}/maven-metadata.xml`;
}
export function getCargoConfigPath(): string {
return 'cargo/config.json';
}
export function getCargoIndexPath(crateName: string): string {
const lower = crateName.toLowerCase();
const len = lower.length;
if (len === 1) {
return `cargo/index/1/${lower}`;
}
if (len === 2) {
return `cargo/index/2/${lower}`;
}
if (len === 3) {
return `cargo/index/3/${lower.charAt(0)}/${lower}`;
}
const prefix1 = lower.substring(0, 2);
const prefix2 = lower.substring(2, 4);
return `cargo/index/${prefix1}/${prefix2}/${lower}`;
}
export function getCargoCratePath(crateName: string, version: string): string {
return `cargo/crates/${crateName}/${crateName}-${version}.crate`;
}
export function getComposerMetadataPath(vendorPackage: string): string {
return `composer/packages/${vendorPackage}/metadata.json`;
}
export function getComposerZipPath(vendorPackage: string, reference: string): string {
return `composer/packages/${vendorPackage}/${reference}.zip`;
}
export function getPypiMetadataPath(packageName: string): string {
return `pypi/metadata/${packageName}/metadata.json`;
}
export function getPypiSimpleIndexPath(packageName: string): string {
return `pypi/simple/${packageName}/index.html`;
}
export function getPypiSimpleRootIndexPath(): string {
return 'pypi/simple/index.html';
}
export function getPypiPackageFilePath(packageName: string, filename: string): string {
return `pypi/packages/${packageName}/${filename}`;
}
export function getRubyGemsVersionsPath(): string {
return 'rubygems/versions';
}
export function getRubyGemsInfoPath(gemName: string): string {
return `rubygems/info/${gemName}`;
}
export function getRubyGemsNamesPath(): string {
return 'rubygems/names';
}
export function getRubyGemsGemPath(gemName: string, version: string, platform?: string): string {
const filename = platform ? `${gemName}-${version}-${platform}.gem` : `${gemName}-${version}.gem`;
return `rubygems/gems/${filename}`;
}
export function getRubyGemsMetadataPath(gemName: string): string {
return `rubygems/metadata/${gemName}/metadata.json`;
}
+63
View File
@@ -0,0 +1,63 @@
import * as crypto from 'crypto';
/**
* Convert Buffer, Uint8Array, string, or JSON object to a ReadableStream<Uint8Array>.
*/
export function toReadableStream(data: Buffer | Uint8Array | string | object): ReadableStream<Uint8Array> {
const buf = Buffer.isBuffer(data)
? data
: data instanceof Uint8Array
? Buffer.from(data)
: typeof data === 'string'
? Buffer.from(data, 'utf-8')
: Buffer.from(JSON.stringify(data), 'utf-8');
return new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(new Uint8Array(buf));
controller.close();
},
});
}
/**
* Consume a ReadableStream into a Buffer.
*/
export async function streamToBuffer(stream: ReadableStream<Uint8Array>): Promise<Buffer> {
const reader = stream.getReader();
const chunks: Uint8Array[] = [];
while (true) {
const { done, value } = await reader.read();
if (done) break;
if (value) chunks.push(value);
}
return Buffer.concat(chunks);
}
/**
* Consume a ReadableStream into a parsed JSON object.
*/
export async function streamToJson<T = any>(stream: ReadableStream<Uint8Array>): Promise<T> {
const buf = await streamToBuffer(stream);
return JSON.parse(buf.toString('utf-8'));
}
/**
* Create a TransformStream that incrementally hashes data passing through.
* Data flows through unchanged; the digest is available after the stream completes.
*/
export function createHashTransform(algorithm: string = 'sha256'): {
transform: TransformStream<Uint8Array, Uint8Array>;
getDigest: () => string;
} {
const hash = crypto.createHash(algorithm);
const transform = new TransformStream<Uint8Array, Uint8Array>({
transform(chunk, controller) {
hash.update(chunk);
controller.enqueue(chunk);
},
});
return {
transform,
getDigest: () => hash.digest('hex'),
};
}
+3
View File
@@ -12,6 +12,9 @@ export { DefaultAuthProvider } from './classes.defaultauthprovider.js';
// Storage interfaces and hooks // Storage interfaces and hooks
export * from './interfaces.storage.js'; export * from './interfaces.storage.js';
// Stream helpers
export { toReadableStream, streamToBuffer, streamToJson, createHashTransform } from './helpers.stream.js';
// Classes // Classes
export { BaseRegistry } from './classes.baseregistry.js'; export { BaseRegistry } from './classes.baseregistry.js';
export { RegistryStorage } from './classes.registrystorage.js'; export { RegistryStorage } from './classes.registrystorage.js';
+29 -5
View File
@@ -3,7 +3,7 @@
*/ */
import type * as plugins from '../plugins.js'; import type * as plugins from '../plugins.js';
import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js'; import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
import type { IAuthProvider } from './interfaces.auth.js'; import type { IAuthProvider } from './interfaces.auth.js';
import type { IStorageHooks } from './interfaces.storage.js'; import type { IStorageHooks } from './interfaces.storage.js';
@@ -88,9 +88,8 @@ export interface IAuthConfig {
export interface IProtocolConfig { export interface IProtocolConfig {
enabled: boolean; enabled: boolean;
basePath: string; basePath: string;
registryUrl?: string;
features?: Record<string, boolean>; features?: Record<string, boolean>;
/** Upstream registry configuration for proxying/caching */
upstream?: IProtocolUpstreamConfig;
} }
/** /**
@@ -113,6 +112,13 @@ export interface IRegistryConfig {
*/ */
storageHooks?: IStorageHooks; storageHooks?: IStorageHooks;
/**
* Dynamic upstream configuration provider.
* Called per-request to resolve which upstream registries to use.
* Use StaticUpstreamProvider for simple static configurations.
*/
upstreamProvider?: IUpstreamProvider;
oci?: IProtocolConfig; oci?: IProtocolConfig;
npm?: IProtocolConfig; npm?: IProtocolConfig;
maven?: IProtocolConfig; maven?: IProtocolConfig;
@@ -155,6 +161,21 @@ export interface IStorageBackend {
* Get object metadata * Get object metadata
*/ */
getMetadata(key: string): Promise<Record<string, string> | null>; getMetadata(key: string): Promise<Record<string, string> | null>;
/**
* Get an object as a ReadableStream. Returns null if not found.
*/
getObjectStream?(key: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null>;
/**
* Store an object from a ReadableStream.
*/
putObjectStream?(key: string, stream: ReadableStream<Uint8Array>): Promise<void>;
/**
* Get object size without reading data (S3 HEAD request).
*/
getObjectSize?(key: string): Promise<number | null>;
} }
/** /**
@@ -210,10 +231,13 @@ export interface IRequestContext {
} }
/** /**
* Base response structure * Base response structure.
* `body` is always a `ReadableStream<Uint8Array>` at the public API boundary.
* Internal handlers may return Buffer/string/object the SmartRegistry orchestrator
* auto-wraps them via `toReadableStream()` before returning to the caller.
*/ */
export interface IResponse { export interface IResponse {
status: number; status: number;
headers: Record<string, string>; headers: Record<string, string>;
body?: any; body?: ReadableStream<Uint8Array> | any;
} }
+134 -53
View File
@@ -6,8 +6,8 @@
import { BaseRegistry } from '../core/classes.baseregistry.js'; import { BaseRegistry } from '../core/classes.baseregistry.js';
import type { RegistryStorage } from '../core/classes.registrystorage.js'; import type { RegistryStorage } from '../core/classes.registrystorage.js';
import type { AuthManager } from '../core/classes.authmanager.js'; import type { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js'; import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from '../core/interfaces.core.js';
import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js'; import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
import { toBuffer } from '../core/helpers.buffer.js'; import { toBuffer } from '../core/helpers.buffer.js';
import type { IMavenCoordinate, IMavenMetadata, IChecksums } from './interfaces.maven.js'; import type { IMavenCoordinate, IMavenMetadata, IChecksums } from './interfaces.maven.js';
import { import {
@@ -33,34 +33,64 @@ export class MavenRegistry extends BaseRegistry {
private authManager: AuthManager; private authManager: AuthManager;
private basePath: string = '/maven'; private basePath: string = '/maven';
private registryUrl: string; private registryUrl: string;
private upstream: MavenUpstream | null = null; private upstreamProvider: IUpstreamProvider | null = null;
constructor( constructor(
storage: RegistryStorage, storage: RegistryStorage,
authManager: AuthManager, authManager: AuthManager,
basePath: string, basePath: string,
registryUrl: string, registryUrl: string,
upstreamConfig?: IProtocolUpstreamConfig upstreamProvider?: IUpstreamProvider
) { ) {
super(); super();
this.storage = storage; this.storage = storage;
this.authManager = authManager; this.authManager = authManager;
this.basePath = basePath; this.basePath = basePath;
this.registryUrl = registryUrl; this.registryUrl = registryUrl;
this.upstreamProvider = upstreamProvider || null;
}
// Initialize upstream if configured /**
if (upstreamConfig?.enabled) { * Extract scope from Maven coordinates.
this.upstream = new MavenUpstream(upstreamConfig); * For Maven, the groupId is the scope.
} * @example "com.example" from "com.example:my-lib"
*/
private extractScope(groupId: string): string | null {
return groupId || null;
}
/**
* Get upstream for a specific request.
* Calls the provider to resolve upstream config dynamically.
*/
private async getUpstreamForRequest(
resource: string,
resourceType: string,
method: string,
actor?: IRequestActor
): Promise<MavenUpstream | null> {
if (!this.upstreamProvider) return null;
// For Maven, resource is "groupId:artifactId"
const [groupId] = resource.split(':');
const config = await this.upstreamProvider.resolveUpstreamConfig({
protocol: 'maven',
resource,
scope: this.extractScope(groupId),
actor,
method,
resourceType,
});
if (!config?.enabled) return null;
return new MavenUpstream(config);
} }
/** /**
* Clean up resources (timers, connections, etc.) * Clean up resources (timers, connections, etc.)
*/ */
public destroy(): void { public destroy(): void {
if (this.upstream) { // No persistent upstream to clean up with dynamic provider
this.upstream.stop();
}
} }
public async init(): Promise<void> { public async init(): Promise<void> {
@@ -75,40 +105,48 @@ export class MavenRegistry extends BaseRegistry {
// Remove base path from URL // Remove base path from URL
const path = context.path.replace(this.basePath, ''); const path = context.path.replace(this.basePath, '');
// Extract token from Authorization header const authHeader = this.getAuthorizationHeader(context);
const authHeader = context.headers['authorization'] || context.headers['Authorization'];
let token: IAuthToken | null = null; let token: IAuthToken | null = null;
if (authHeader) { if (authHeader) {
const tokenString = authHeader.replace(/^(Bearer|Basic)\s+/i, ''); const basicCredentials = this.parseBasicAuthHeader(authHeader);
// For now, try to validate as Maven token (reuse npm token type) if (basicCredentials) {
token = await this.authManager.validateToken(tokenString, 'maven'); // Maven sends Basic Auth: base64(username:password) — extract the password as token
token = await this.authManager.validateToken(basicCredentials.password, 'maven');
} else {
const tokenString = this.extractBearerToken(authHeader);
token = tokenString ? await this.authManager.validateToken(tokenString, 'maven') : null;
}
} }
// Parse path to determine request type const actor: IRequestActor = this.buildRequestActor(context, token);
const coordinate = pathToGAV(path);
if (!coordinate) { return this.storage.withContext({ protocol: 'maven', actor }, async () => {
// Not a valid artifact path, could be metadata or root // Parse path to determine request type
if (path.endsWith('/maven-metadata.xml')) { const coordinate = pathToGAV(path);
return this.handleMetadataRequest(context.method, path, token);
if (!coordinate) {
// Not a valid artifact path, could be metadata or root
if (path.endsWith('/maven-metadata.xml')) {
return this.handleMetadataRequest(context.method, path, token, actor);
}
return {
status: 404,
headers: { 'Content-Type': 'application/json' },
body: { error: 'NOT_FOUND', message: 'Invalid Maven path' },
};
} }
return { // Check if it's a checksum file
status: 404, if (coordinate.extension === 'md5' || coordinate.extension === 'sha1' ||
headers: { 'Content-Type': 'application/json' }, coordinate.extension === 'sha256' || coordinate.extension === 'sha512') {
body: { error: 'NOT_FOUND', message: 'Invalid Maven path' }, return this.handleChecksumRequest(context.method, coordinate, token, path);
}; }
}
// Check if it's a checksum file // Handle artifact requests (JAR, POM, WAR, etc.)
if (coordinate.extension === 'md5' || coordinate.extension === 'sha1' || return this.handleArtifactRequest(context.method, coordinate, token, context.body, actor);
coordinate.extension === 'sha256' || coordinate.extension === 'sha512') { });
return this.handleChecksumRequest(context.method, coordinate, token, path);
}
// Handle artifact requests (JAR, POM, WAR, etc.)
return this.handleArtifactRequest(context.method, coordinate, token, context.body);
} }
protected async checkPermission( protected async checkPermission(
@@ -128,7 +166,8 @@ export class MavenRegistry extends BaseRegistry {
method: string, method: string,
coordinate: IMavenCoordinate, coordinate: IMavenCoordinate,
token: IAuthToken | null, token: IAuthToken | null,
body?: Buffer | any body?: Buffer | any,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
const { groupId, artifactId, version } = coordinate; const { groupId, artifactId, version } = coordinate;
const filename = buildFilename(coordinate); const filename = buildFilename(coordinate);
@@ -139,7 +178,7 @@ export class MavenRegistry extends BaseRegistry {
case 'HEAD': case 'HEAD':
// Maven repositories typically allow anonymous reads // Maven repositories typically allow anonymous reads
return method === 'GET' return method === 'GET'
? this.getArtifact(groupId, artifactId, version, filename) ? this.getArtifact(groupId, artifactId, version, filename, actor)
: this.headArtifact(groupId, artifactId, version, filename); : this.headArtifact(groupId, artifactId, version, filename);
case 'PUT': case 'PUT':
@@ -201,9 +240,19 @@ export class MavenRegistry extends BaseRegistry {
return this.getChecksum(groupId, artifactId, version, coordinate, path); return this.getChecksum(groupId, artifactId, version, coordinate, path);
} }
// Accept PUT silently — Maven deploy-plugin uploads checksums alongside artifacts,
// but our registry auto-generates them, so we just acknowledge the upload
if (method === 'PUT') {
return {
status: 200,
headers: {},
body: { status: 'ok' },
};
}
return { return {
status: 405, status: 405,
headers: { 'Allow': 'GET, HEAD' }, headers: { 'Allow': 'GET, HEAD, PUT' },
body: { error: 'METHOD_NOT_ALLOWED', message: 'Checksums are auto-generated' }, body: { error: 'METHOD_NOT_ALLOWED', message: 'Checksums are auto-generated' },
}; };
} }
@@ -211,7 +260,8 @@ export class MavenRegistry extends BaseRegistry {
private async handleMetadataRequest( private async handleMetadataRequest(
method: string, method: string,
path: string, path: string,
token: IAuthToken | null token: IAuthToken | null,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
// Parse path to extract groupId and artifactId // Parse path to extract groupId and artifactId
// Path format: /com/example/my-lib/maven-metadata.xml // Path format: /com/example/my-lib/maven-metadata.xml
@@ -232,12 +282,22 @@ export class MavenRegistry extends BaseRegistry {
if (method === 'GET') { if (method === 'GET') {
// Metadata is usually public (read permission optional) // Metadata is usually public (read permission optional)
// Some registries allow anonymous metadata access // Some registries allow anonymous metadata access
return this.getMetadata(groupId, artifactId); return this.getMetadata(groupId, artifactId, actor);
}
// Accept PUT silently — Maven deploy-plugin uploads maven-metadata.xml,
// but our registry auto-generates it, so we just acknowledge the upload
if (method === 'PUT') {
return {
status: 200,
headers: {},
body: { status: 'ok' },
};
} }
return { return {
status: 405, status: 405,
headers: { 'Allow': 'GET' }, headers: { 'Allow': 'GET, PUT' },
body: { error: 'METHOD_NOT_ALLOWED', message: 'Metadata is auto-generated' }, body: { error: 'METHOD_NOT_ALLOWED', message: 'Metadata is auto-generated' },
}; };
} }
@@ -250,16 +310,33 @@ export class MavenRegistry extends BaseRegistry {
groupId: string, groupId: string,
artifactId: string, artifactId: string,
version: string, version: string,
filename: string filename: string,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
let data = await this.storage.getMavenArtifact(groupId, artifactId, version, filename); // Try local storage first (streaming)
const streamResult = await this.storage.getMavenArtifactStream(groupId, artifactId, version, filename);
if (streamResult) {
const ext = filename.split('.').pop() || '';
const contentType = this.getContentType(ext);
return {
status: 200,
headers: {
'Content-Type': contentType,
'Content-Length': streamResult.size.toString(),
},
body: streamResult.stream,
};
}
// Try upstream if not found locally // Try upstream if not found locally
if (!data && this.upstream) { let data: Buffer | null = null;
const resource = `${groupId}:${artifactId}`;
const upstream = await this.getUpstreamForRequest(resource, 'artifact', 'GET', actor);
if (upstream) {
// Parse the filename to extract extension and classifier // Parse the filename to extract extension and classifier
const { extension, classifier } = this.parseFilename(filename, artifactId, version); const { extension, classifier } = this.parseFilename(filename, artifactId, version);
if (extension) { if (extension) {
data = await this.upstream.fetchArtifact(groupId, artifactId, version, extension, classifier); data = await upstream.fetchArtifact(groupId, artifactId, version, extension, classifier);
if (data) { if (data) {
// Cache the artifact locally // Cache the artifact locally
await this.storage.putMavenArtifact(groupId, artifactId, version, filename, data); await this.storage.putMavenArtifact(groupId, artifactId, version, filename, data);
@@ -495,16 +572,20 @@ export class MavenRegistry extends BaseRegistry {
// METADATA OPERATIONS // METADATA OPERATIONS
// ======================================================================== // ========================================================================
private async getMetadata(groupId: string, artifactId: string): Promise<IResponse> { private async getMetadata(groupId: string, artifactId: string, actor?: IRequestActor): Promise<IResponse> {
let metadataBuffer = await this.storage.getMavenMetadata(groupId, artifactId); let metadataBuffer = await this.storage.getMavenMetadata(groupId, artifactId);
// Try upstream if not found locally // Try upstream if not found locally
if (!metadataBuffer && this.upstream) { if (!metadataBuffer) {
const upstreamMetadata = await this.upstream.fetchMetadata(groupId, artifactId); const resource = `${groupId}:${artifactId}`;
if (upstreamMetadata) { const upstream = await this.getUpstreamForRequest(resource, 'metadata', 'GET', actor);
metadataBuffer = Buffer.from(upstreamMetadata, 'utf-8'); if (upstream) {
// Cache the metadata locally const upstreamMetadata = await upstream.fetchMetadata(groupId, artifactId);
await this.storage.putMavenMetadata(groupId, artifactId, metadataBuffer); if (upstreamMetadata) {
metadataBuffer = Buffer.from(upstreamMetadata, 'utf-8');
// Cache the metadata locally
await this.storage.putMavenMetadata(groupId, artifactId, metadataBuffer);
}
} }
} }
+278 -211
View File
@@ -2,12 +2,11 @@ import { Smartlog } from '@push.rocks/smartlog';
import { BaseRegistry } from '../core/classes.baseregistry.js'; import { BaseRegistry } from '../core/classes.baseregistry.js';
import { RegistryStorage } from '../core/classes.registrystorage.js'; import { RegistryStorage } from '../core/classes.registrystorage.js';
import { AuthManager } from '../core/classes.authmanager.js'; import { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js'; import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from '../core/interfaces.core.js';
import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js'; import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
import { NpmUpstream } from './classes.npmupstream.js'; import { NpmUpstream } from './classes.npmupstream.js';
import type { import type {
IPackument, IPackument,
INpmVersion,
IPublishRequest, IPublishRequest,
ISearchResponse, ISearchResponse,
ISearchResult, ISearchResult,
@@ -16,6 +15,13 @@ import type {
IUserAuthRequest, IUserAuthRequest,
INpmError, INpmError,
} from './interfaces.npm.js'; } from './interfaces.npm.js';
import {
createNewPackument,
getAttachmentForVersion,
preparePublishedVersion,
recordPublishedVersion,
} from './helpers.npmpublish.js';
import { parseNpmRequestRoute } from './helpers.npmroutes.js';
/** /**
* NPM Registry implementation * NPM Registry implementation
@@ -27,43 +33,69 @@ export class NpmRegistry extends BaseRegistry {
private basePath: string = '/npm'; private basePath: string = '/npm';
private registryUrl: string; private registryUrl: string;
private logger: Smartlog; private logger: Smartlog;
private upstream: NpmUpstream | null = null; private upstreamProvider: IUpstreamProvider | null = null;
constructor( constructor(
storage: RegistryStorage, storage: RegistryStorage,
authManager: AuthManager, authManager: AuthManager,
basePath: string = '/npm', basePath: string = '/npm',
registryUrl: string = 'http://localhost:5000/npm', registryUrl: string = 'http://localhost:5000/npm',
upstreamConfig?: IProtocolUpstreamConfig upstreamProvider?: IUpstreamProvider
) { ) {
super(); super();
this.storage = storage; this.storage = storage;
this.authManager = authManager; this.authManager = authManager;
this.basePath = basePath; this.basePath = basePath;
this.registryUrl = registryUrl; this.registryUrl = registryUrl;
this.upstreamProvider = upstreamProvider || null;
// Initialize logger this.logger = this.createProtocolLogger('npm-registry', 'npm');
this.logger = new Smartlog({
logContext: {
company: 'push.rocks',
companyunit: 'smartregistry',
containerName: 'npm-registry',
environment: (process.env.NODE_ENV as any) || 'development',
runtime: 'node',
zone: 'npm'
}
});
this.logger.enableConsole();
// Initialize upstream if configured if (upstreamProvider) {
if (upstreamConfig?.enabled) { this.logger.log('info', 'NPM upstream provider configured');
this.upstream = new NpmUpstream(upstreamConfig, registryUrl, this.logger);
this.logger.log('info', 'NPM upstream initialized', {
upstreams: upstreamConfig.upstreams.map(u => u.name),
});
} }
} }
/**
* Extract scope from npm package name.
* @example "@company/utils" -> "company"
* @example "lodash" -> null
*/
private extractScope(packageName: string): string | null {
if (packageName.startsWith('@')) {
const slashIndex = packageName.indexOf('/');
if (slashIndex > 1) {
return packageName.substring(1, slashIndex);
}
}
return null;
}
/**
* Get upstream for a specific request.
* Calls the provider to resolve upstream config dynamically.
*/
private async getUpstreamForRequest(
resource: string,
resourceType: string,
method: string,
actor?: IRequestActor
): Promise<NpmUpstream | null> {
if (!this.upstreamProvider) return null;
const config = await this.upstreamProvider.resolveUpstreamConfig({
protocol: 'npm',
resource,
scope: this.extractScope(resource),
actor,
method,
resourceType,
});
if (!config?.enabled) return null;
return new NpmUpstream(config, this.registryUrl, this.logger);
}
public async init(): Promise<void> { public async init(): Promise<void> {
// NPM registry initialization // NPM registry initialization
} }
@@ -75,89 +107,86 @@ export class NpmRegistry extends BaseRegistry {
public async handleRequest(context: IRequestContext): Promise<IResponse> { public async handleRequest(context: IRequestContext): Promise<IResponse> {
const path = context.path.replace(this.basePath, ''); const path = context.path.replace(this.basePath, '');
// Extract token from Authorization header const tokenString = this.extractBearerToken(context);
const authHeader = context.headers['authorization'] || context.headers['Authorization'];
const tokenString = authHeader?.replace(/^Bearer\s+/i, '');
const token = tokenString ? await this.authManager.validateToken(tokenString, 'npm') : null; const token = tokenString ? await this.authManager.validateToken(tokenString, 'npm') : null;
const actor: IRequestActor = this.buildRequestActor(context, token);
this.logger.log('debug', `handleRequest: ${context.method} ${path}`, { this.logger.log('debug', `handleRequest: ${context.method} ${path}`, {
method: context.method, method: context.method,
path, path,
hasAuth: !!token hasAuth: !!token
}); });
// Registry root return this.storage.withContext({ protocol: 'npm', actor }, async () => {
if (path === '/' || path === '') { const route = parseNpmRequestRoute(path, context.method);
return this.handleRegistryInfo(); if (!route) {
} return {
status: 404,
headers: { 'Content-Type': 'application/json' },
body: this.createError('E404', 'Not found'),
};
}
// Search: /-/v1/search switch (route.type) {
if (path.startsWith('/-/v1/search')) { case 'root':
return this.handleSearch(context.query); return this.handleRegistryInfo();
} case 'search':
return this.handleSearch(context.query);
case 'userAuth':
return this.handleUserAuth(context.method, route.username, context.body, token);
case 'tokens':
return this.handleTokens(context.method, route.path, context.body, token);
case 'distTags':
return this.withPackageContext(
route.packageName,
actor,
async () => this.handleDistTags(context.method, route.packageName, route.tag, context.body, token)
);
case 'tarball':
return this.handleTarballDownload(route.packageName, route.filename, token, actor);
case 'unpublishVersion':
this.logger.log('debug', 'unpublishVersionMatch', {
packageName: route.packageName,
version: route.version,
});
return this.withPackageVersionContext(
route.packageName,
route.version,
actor,
async () => this.unpublishVersion(route.packageName, route.version, token)
);
case 'unpublishPackage':
this.logger.log('debug', 'unpublishPackageMatch', {
packageName: route.packageName,
rev: route.rev,
});
return this.withPackageContext(
route.packageName,
actor,
async () => this.unpublishPackage(route.packageName, token)
);
case 'packageVersion':
this.logger.log('debug', 'versionMatch', {
packageName: route.packageName,
version: route.version,
});
return this.withPackageVersionContext(
route.packageName,
route.version,
actor,
async () => this.handlePackageVersion(route.packageName, route.version, token, actor)
);
case 'package':
this.logger.log('debug', 'packageMatch', { packageName: route.packageName });
return this.withPackageContext(
route.packageName,
actor,
async () => this.handlePackage(context.method, route.packageName, context.body, context.query, token, actor)
);
}
// User authentication: /-/user/org.couchdb.user:{username} });
const userMatch = path.match(/^\/-\/user\/org\.couchdb\.user:(.+)$/);
if (userMatch) {
return this.handleUserAuth(context.method, userMatch[1], context.body, token);
}
// Token operations: /-/npm/v1/tokens
if (path.startsWith('/-/npm/v1/tokens')) {
return this.handleTokens(context.method, path, context.body, token);
}
// Dist-tags: /-/package/{package}/dist-tags
const distTagsMatch = path.match(/^\/-\/package\/(@?[^\/]+(?:\/[^\/]+)?)\/dist-tags(?:\/(.+))?$/);
if (distTagsMatch) {
const [, packageName, tag] = distTagsMatch;
return this.handleDistTags(context.method, packageName, tag, context.body, token);
}
// Tarball download: /{package}/-/{filename}.tgz
const tarballMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)\/-\/(.+\.tgz)$/);
if (tarballMatch) {
const [, packageName, filename] = tarballMatch;
return this.handleTarballDownload(packageName, filename, token);
}
// Unpublish specific version: DELETE /{package}/-/{version}
const unpublishVersionMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)\/-\/([^\/]+)$/);
if (unpublishVersionMatch && context.method === 'DELETE') {
const [, packageName, version] = unpublishVersionMatch;
this.logger.log('debug', 'unpublishVersionMatch', { packageName, version });
return this.unpublishVersion(packageName, version, token);
}
// Unpublish entire package: DELETE /{package}/-rev/{rev}
const unpublishPackageMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)\/-rev\/([^\/]+)$/);
if (unpublishPackageMatch && context.method === 'DELETE') {
const [, packageName, rev] = unpublishPackageMatch;
this.logger.log('debug', 'unpublishPackageMatch', { packageName, rev });
return this.unpublishPackage(packageName, token);
}
// Package version: /{package}/{version}
const versionMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)\/([^\/]+)$/);
if (versionMatch) {
const [, packageName, version] = versionMatch;
this.logger.log('debug', 'versionMatch', { packageName, version });
return this.handlePackageVersion(packageName, version, token);
}
// Package operations: /{package}
const packageMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)$/);
if (packageMatch) {
const packageName = packageMatch[1];
this.logger.log('debug', 'packageMatch', { packageName });
return this.handlePackage(context.method, packageName, context.body, context.query, token);
}
return {
status: 404,
headers: { 'Content-Type': 'application/json' },
body: this.createError('E404', 'Not found'),
};
} }
protected async checkPermission( protected async checkPermission(
@@ -198,11 +227,12 @@ export class NpmRegistry extends BaseRegistry {
packageName: string, packageName: string,
body: any, body: any,
query: Record<string, string>, query: Record<string, string>,
token: IAuthToken | null token: IAuthToken | null,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
switch (method) { switch (method) {
case 'GET': case 'GET':
return this.getPackument(packageName, token, query); return this.getPackument(packageName, token, query, actor);
case 'PUT': case 'PUT':
return this.publishPackage(packageName, body, token); return this.publishPackage(packageName, body, token);
case 'DELETE': case 'DELETE':
@@ -219,29 +249,10 @@ export class NpmRegistry extends BaseRegistry {
private async getPackument( private async getPackument(
packageName: string, packageName: string,
token: IAuthToken | null, token: IAuthToken | null,
query: Record<string, string> query: Record<string, string>,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
let packument = await this.storage.getNpmPackument(packageName); const packument = await this.getLocalOrUpstreamPackument(packageName, actor, 'getPackument');
this.logger.log('debug', `getPackument: ${packageName}`, {
packageName,
found: !!packument,
versions: packument ? Object.keys(packument.versions).length : 0
});
// If not found locally, try upstream
if (!packument && this.upstream) {
this.logger.log('debug', `getPackument: fetching from upstream`, { packageName });
const upstreamPackument = await this.upstream.fetchPackument(packageName);
if (upstreamPackument) {
this.logger.log('debug', `getPackument: found in upstream`, {
packageName,
versions: Object.keys(upstreamPackument.versions || {}).length
});
packument = upstreamPackument;
// Optionally cache the packument locally (without tarballs)
// We don't store tarballs here - they'll be fetched on demand
}
}
if (!packument) { if (!packument) {
return { return {
@@ -279,24 +290,16 @@ export class NpmRegistry extends BaseRegistry {
private async handlePackageVersion( private async handlePackageVersion(
packageName: string, packageName: string,
version: string, version: string,
token: IAuthToken | null token: IAuthToken | null,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
this.logger.log('debug', 'handlePackageVersion', { packageName, version }); this.logger.log('debug', 'handlePackageVersion', { packageName, version });
let packument = await this.storage.getNpmPackument(packageName); const packument = await this.getLocalOrUpstreamPackument(packageName, actor, 'handlePackageVersion');
this.logger.log('debug', 'handlePackageVersion packument', { found: !!packument }); this.logger.log('debug', 'handlePackageVersion packument', { found: !!packument });
if (packument) { if (packument) {
this.logger.log('debug', 'handlePackageVersion versions', { versions: Object.keys(packument.versions || {}) }); this.logger.log('debug', 'handlePackageVersion versions', { versions: Object.keys(packument.versions || {}) });
} }
// If not found locally, try upstream
if (!packument && this.upstream) {
this.logger.log('debug', 'handlePackageVersion: fetching from upstream', { packageName });
const upstreamPackument = await this.upstream.fetchPackument(packageName);
if (upstreamPackument) {
packument = upstreamPackument;
}
}
if (!packument) { if (!packument) {
return { return {
status: 404, status: 404,
@@ -370,19 +373,7 @@ export class NpmRegistry extends BaseRegistry {
const isNew = !packument; const isNew = !packument;
if (isNew) { if (isNew) {
packument = { packument = createNewPackument(packageName, body, new Date().toISOString());
_id: packageName,
name: packageName,
description: body.description,
'dist-tags': body['dist-tags'] || { latest: Object.keys(body.versions)[0] },
versions: {},
time: {
created: new Date().toISOString(),
modified: new Date().toISOString(),
},
maintainers: body.maintainers || [],
readme: body.readme,
};
} }
// Process each new version // Process each new version
@@ -396,12 +387,8 @@ export class NpmRegistry extends BaseRegistry {
}; };
} }
// Find attachment for this version const attachment = getAttachmentForVersion(body, version);
const attachmentKey = Object.keys(body._attachments).find(key => if (!attachment) {
key.includes(version)
);
if (!attachmentKey) {
return { return {
status: 400, status: 400,
headers: {}, headers: {},
@@ -409,38 +396,24 @@ export class NpmRegistry extends BaseRegistry {
}; };
} }
const attachment = body._attachments[attachmentKey]; const preparedVersion = preparePublishedVersion({
packageName,
// Decode base64 tarball version,
const tarballBuffer = Buffer.from(attachment.data, 'base64'); versionData,
attachment,
// Calculate shasum registryUrl: this.registryUrl,
const crypto = await import('crypto'); userId: token?.userId,
const shasum = crypto.createHash('sha1').update(tarballBuffer).digest('hex'); });
const integrity = `sha512-${crypto.createHash('sha512').update(tarballBuffer).digest('base64')}`;
// Store tarball // Store tarball
await this.storage.putNpmTarball(packageName, version, tarballBuffer); await this.withPackageVersionContext(
packageName,
version,
undefined,
async () => this.storage.putNpmTarball(packageName, version, preparedVersion.tarballBuffer)
);
// Update version data with dist info recordPublishedVersion(packument, version, preparedVersion.versionData, new Date().toISOString());
const safeName = packageName.replace('@', '').replace('/', '-');
versionData.dist = {
tarball: `${this.registryUrl}/${packageName}/-/${safeName}-${version}.tgz`,
shasum,
integrity,
fileCount: 0,
unpackedSize: tarballBuffer.length,
};
versionData._id = `${packageName}@${version}`;
versionData._npmUser = token ? { name: token.userId, email: '' } : undefined;
// Add version to packument
packument.versions[version] = versionData;
if (packument.time) {
packument.time[version] = new Date().toISOString();
packument.time.modified = new Date().toISOString();
}
} }
// Update dist-tags // Update dist-tags
@@ -563,7 +536,8 @@ export class NpmRegistry extends BaseRegistry {
private async handleTarballDownload( private async handleTarballDownload(
packageName: string, packageName: string,
filename: string, filename: string,
token: IAuthToken | null token: IAuthToken | null,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
// Extract version from filename: package-name-1.0.0.tgz // Extract version from filename: package-name-1.0.0.tgz
const versionMatch = filename.match(/-([\d.]+(?:-[a-z0-9.]+)?)\.tgz$/i); const versionMatch = filename.match(/-([\d.]+(?:-[a-z0-9.]+)?)\.tgz$/i);
@@ -576,43 +550,120 @@ export class NpmRegistry extends BaseRegistry {
} }
const version = versionMatch[1]; const version = versionMatch[1];
let tarball = await this.storage.getNpmTarball(packageName, version);
// If not found locally, try upstream return this.withPackageVersionContext(
if (!tarball && this.upstream) { packageName,
this.logger.log('debug', 'handleTarballDownload: fetching from upstream', { version,
packageName, actor,
version, async (): Promise<IResponse> => {
}); // Try local storage first (streaming)
const upstreamTarball = await this.upstream.fetchTarball(packageName, version); const streamResult = await this.storage.getNpmTarballStream(packageName, version);
if (upstreamTarball) { if (streamResult) {
tarball = upstreamTarball; return {
// Cache the tarball locally for future requests status: 200,
await this.storage.putNpmTarball(packageName, version, tarball); headers: {
this.logger.log('debug', 'handleTarballDownload: cached tarball locally', { 'Content-Type': 'application/octet-stream',
packageName, 'Content-Length': streamResult.size.toString(),
version, },
size: tarball.length, body: streamResult.stream,
}); };
}
// If not found locally, try upstream
let tarball: Buffer | null = null;
const upstream = await this.getUpstreamForRequest(packageName, 'tarball', 'GET', actor);
if (upstream) {
this.logger.log('debug', 'handleTarballDownload: fetching from upstream', {
packageName,
version,
});
const upstreamTarball = await upstream.fetchTarball(packageName, version);
if (upstreamTarball) {
tarball = upstreamTarball;
// Cache the tarball locally for future requests
await this.storage.putNpmTarball(packageName, version, tarball);
this.logger.log('debug', 'handleTarballDownload: cached tarball locally', {
packageName,
version,
size: tarball.length,
});
}
}
if (!tarball) {
return {
status: 404,
headers: {},
body: this.createError('E404', 'Tarball not found'),
};
}
return {
status: 200,
headers: {
'Content-Type': 'application/octet-stream',
'Content-Length': tarball.length.toString(),
},
body: tarball,
};
} }
);
}
private async withPackageContext<T>(
packageName: string,
actor: IRequestActor | undefined,
fn: () => Promise<T>
): Promise<T> {
return this.storage.withContext(
{ protocol: 'npm', actor, metadata: { packageName } },
fn
);
}
private async getLocalOrUpstreamPackument(
packageName: string,
actor: IRequestActor | undefined,
logPrefix: string
): Promise<IPackument | null> {
const localPackument = await this.storage.getNpmPackument(packageName);
this.logger.log('debug', `${logPrefix}: ${packageName}`, {
packageName,
found: !!localPackument,
versions: localPackument ? Object.keys(localPackument.versions).length : 0,
});
if (localPackument) {
return localPackument;
} }
if (!tarball) { const upstream = await this.getUpstreamForRequest(packageName, 'packument', 'GET', actor);
return { if (!upstream) {
status: 404, return null;
headers: {},
body: this.createError('E404', 'Tarball not found'),
};
} }
return { this.logger.log('debug', `${logPrefix}: fetching from upstream`, { packageName });
status: 200, const upstreamPackument = await upstream.fetchPackument(packageName);
headers: { if (upstreamPackument) {
'Content-Type': 'application/octet-stream', this.logger.log('debug', `${logPrefix}: found in upstream`, {
'Content-Length': tarball.length.toString(), packageName,
}, versions: Object.keys(upstreamPackument.versions || {}).length,
body: tarball, });
}; }
return upstreamPackument;
}
private async withPackageVersionContext<T>(
packageName: string,
version: string,
actor: IRequestActor | undefined,
fn: () => Promise<T>
): Promise<T> {
return this.storage.withContext(
{ protocol: 'npm', actor, metadata: { packageName, version } },
fn
);
} }
private async handleSearch(query: Record<string, string>): Promise<IResponse> { private async handleSearch(query: Record<string, string>): Promise<IResponse> {
@@ -680,6 +731,22 @@ export class NpmRegistry extends BaseRegistry {
this.logger.log('error', 'handleSearch failed', { error: (error as Error).message }); this.logger.log('error', 'handleSearch failed', { error: (error as Error).message });
} }
// Sort results by relevance: exact match first, then prefix match, then substring match
if (text) {
const lowerText = text.toLowerCase();
results.sort((a, b) => {
const aName = a.package.name.toLowerCase();
const bName = b.package.name.toLowerCase();
const aExact = aName === lowerText ? 0 : 1;
const bExact = bName === lowerText ? 0 : 1;
if (aExact !== bExact) return aExact - bExact;
const aPrefix = aName.startsWith(lowerText) ? 0 : 1;
const bPrefix = bName.startsWith(lowerText) ? 0 : 1;
if (aPrefix !== bPrefix) return aPrefix - bPrefix;
return aName.localeCompare(bName);
});
}
// Apply pagination // Apply pagination
const paginatedResults = results.slice(from, from + size); const paginatedResults = results.slice(from, from + size);
+79
View File
@@ -0,0 +1,79 @@
import * as crypto from 'node:crypto';
import type { IPackument, IPublishRequest, INpmVersion } from './interfaces.npm.js';
function getTarballFileName(packageName: string, version: string): string {
const safeName = packageName.replace('@', '').replace('/', '-');
return `${safeName}-${version}.tgz`;
}
export function createNewPackument(
packageName: string,
body: IPublishRequest,
timestamp: string
): IPackument {
return {
_id: packageName,
name: packageName,
description: body.description,
'dist-tags': body['dist-tags'] || { latest: Object.keys(body.versions)[0] },
versions: {},
time: {
created: timestamp,
modified: timestamp,
},
maintainers: body.maintainers || [],
readme: body.readme,
};
}
export function getAttachmentForVersion(
body: IPublishRequest,
version: string
): IPublishRequest['_attachments'][string] | null {
const attachmentKey = Object.keys(body._attachments).find((key) => key.includes(version));
return attachmentKey ? body._attachments[attachmentKey] : null;
}
export function preparePublishedVersion(options: {
packageName: string;
version: string;
versionData: INpmVersion;
attachment: IPublishRequest['_attachments'][string];
registryUrl: string;
userId?: string;
}): { tarballBuffer: Buffer; versionData: INpmVersion } {
const tarballBuffer = Buffer.from(options.attachment.data, 'base64');
const shasum = crypto.createHash('sha1').update(tarballBuffer).digest('hex');
const integrity = `sha512-${crypto.createHash('sha512').update(tarballBuffer).digest('base64')}`;
const tarballFileName = getTarballFileName(options.packageName, options.version);
return {
tarballBuffer,
versionData: {
...options.versionData,
dist: {
...options.versionData.dist,
tarball: `${options.registryUrl}/${options.packageName}/-/${tarballFileName}`,
shasum,
integrity,
fileCount: 0,
unpackedSize: tarballBuffer.length,
},
_id: `${options.packageName}@${options.version}`,
...(options.userId ? { _npmUser: { name: options.userId, email: '' } } : {}),
},
};
}
export function recordPublishedVersion(
packument: IPackument,
version: string,
versionData: INpmVersion,
timestamp: string
): void {
packument.versions[version] = versionData;
if (packument.time) {
packument.time[version] = timestamp;
packument.time.modified = timestamp;
}
}
+110
View File
@@ -0,0 +1,110 @@
export type TNpmRequestRoute =
| { type: 'root' }
| { type: 'search' }
| { type: 'userAuth'; username: string }
| { type: 'tokens'; path: string }
| { type: 'distTags'; packageName: string; tag?: string }
| { type: 'tarball'; packageName: string; filename: string }
| { type: 'unpublishVersion'; packageName: string; version: string }
| { type: 'unpublishPackage'; packageName: string; rev: string }
| { type: 'packageVersion'; packageName: string; version: string }
| { type: 'package'; packageName: string };
function decodePackageName(rawPackageName: string): string {
return decodeURIComponent(rawPackageName);
}
export function parseNpmRequestRoute(path: string, method: string): TNpmRequestRoute | null {
if (path === '/' || path === '') {
return { type: 'root' };
}
if (path.startsWith('/-/v1/search')) {
return { type: 'search' };
}
const userMatch = path.match(/^\/-\/user\/org\.couchdb\.user:(.+)$/);
if (userMatch) {
return {
type: 'userAuth',
username: userMatch[1],
};
}
if (path.startsWith('/-/npm/v1/tokens')) {
return {
type: 'tokens',
path,
};
}
const distTagsMatch = path.match(/^\/-\/package\/(@?[^\/]+(?:\/[^\/]+)?)\/dist-tags(?:\/(.+))?$/);
if (distTagsMatch) {
const [, rawPackageName, tag] = distTagsMatch;
return {
type: 'distTags',
packageName: decodePackageName(rawPackageName),
tag,
};
}
const tarballMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)\/-\/(.+\.tgz)$/);
if (tarballMatch) {
const [, rawPackageName, filename] = tarballMatch;
return {
type: 'tarball',
packageName: decodePackageName(rawPackageName),
filename,
};
}
if (method === 'DELETE') {
const unpublishVersionMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)\/-\/([^\/]+)$/);
if (unpublishVersionMatch) {
const [, rawPackageName, version] = unpublishVersionMatch;
return {
type: 'unpublishVersion',
packageName: decodePackageName(rawPackageName),
version,
};
}
const unpublishPackageMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)\/-rev\/([^\/]+)$/);
if (unpublishPackageMatch) {
const [, rawPackageName, rev] = unpublishPackageMatch;
return {
type: 'unpublishPackage',
packageName: decodePackageName(rawPackageName),
rev,
};
}
}
const unencodedScopedPackageMatch = path.match(/^\/@[^\/]+\/[^\/]+$/);
if (unencodedScopedPackageMatch) {
return {
type: 'package',
packageName: decodePackageName(path.substring(1)),
};
}
const versionMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)\/([^\/]+)$/);
if (versionMatch) {
const [, rawPackageName, version] = versionMatch;
return {
type: 'packageVersion',
packageName: decodePackageName(rawPackageName),
version,
};
}
const packageMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)$/);
if (packageMatch) {
return {
type: 'package',
packageName: decodePackageName(packageMatch[1]),
};
}
return null;
}
+236 -124
View File
@@ -2,8 +2,9 @@ import { Smartlog } from '@push.rocks/smartlog';
import { BaseRegistry } from '../core/classes.baseregistry.js'; import { BaseRegistry } from '../core/classes.baseregistry.js';
import { RegistryStorage } from '../core/classes.registrystorage.js'; import { RegistryStorage } from '../core/classes.registrystorage.js';
import { AuthManager } from '../core/classes.authmanager.js'; import { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken, IRegistryError } from '../core/interfaces.core.js'; import type { IRequestContext, IResponse, IAuthToken, IRegistryError, IRequestActor } from '../core/interfaces.core.js';
import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js'; import { createHashTransform, streamToBuffer } from '../core/helpers.stream.js';
import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
import { OciUpstream } from './classes.ociupstream.js'; import { OciUpstream } from './classes.ociupstream.js';
import type { import type {
IUploadSession, IUploadSession,
@@ -24,7 +25,7 @@ export class OciRegistry extends BaseRegistry {
private basePath: string = '/oci'; private basePath: string = '/oci';
private cleanupInterval?: NodeJS.Timeout; private cleanupInterval?: NodeJS.Timeout;
private ociTokens?: { realm: string; service: string }; private ociTokens?: { realm: string; service: string };
private upstream: OciUpstream | null = null; private upstreamProvider: IUpstreamProvider | null = null;
private logger: Smartlog; private logger: Smartlog;
constructor( constructor(
@@ -32,36 +33,61 @@ export class OciRegistry extends BaseRegistry {
authManager: AuthManager, authManager: AuthManager,
basePath: string = '/oci', basePath: string = '/oci',
ociTokens?: { realm: string; service: string }, ociTokens?: { realm: string; service: string },
upstreamConfig?: IProtocolUpstreamConfig upstreamProvider?: IUpstreamProvider
) { ) {
super(); super();
this.storage = storage; this.storage = storage;
this.authManager = authManager; this.authManager = authManager;
this.basePath = basePath; this.basePath = basePath;
this.ociTokens = ociTokens; this.ociTokens = ociTokens;
this.upstreamProvider = upstreamProvider || null;
// Initialize logger this.logger = this.createProtocolLogger('oci-registry', 'oci');
this.logger = new Smartlog({
logContext: {
company: 'push.rocks',
companyunit: 'smartregistry',
containerName: 'oci-registry',
environment: (process.env.NODE_ENV as any) || 'development',
runtime: 'node',
zone: 'oci'
}
});
this.logger.enableConsole();
// Initialize upstream if configured if (upstreamProvider) {
if (upstreamConfig?.enabled) { this.logger.log('info', 'OCI upstream provider configured');
this.upstream = new OciUpstream(upstreamConfig, basePath, this.logger);
this.logger.log('info', 'OCI upstream initialized', {
upstreams: upstreamConfig.upstreams.map(u => u.name),
});
} }
} }
/**
* Extract scope from OCI repository name.
* @example "myorg/myimage" -> "myorg"
* @example "library/nginx" -> "library"
* @example "nginx" -> null
*/
private extractScope(repository: string): string | null {
const slashIndex = repository.indexOf('/');
if (slashIndex > 0) {
return repository.substring(0, slashIndex);
}
return null;
}
/**
* Get upstream for a specific request.
* Calls the provider to resolve upstream config dynamically.
*/
private async getUpstreamForRequest(
resource: string,
resourceType: string,
method: string,
actor?: IRequestActor
): Promise<OciUpstream | null> {
if (!this.upstreamProvider) return null;
const config = await this.upstreamProvider.resolveUpstreamConfig({
protocol: 'oci',
resource,
scope: this.extractScope(resource),
actor,
method,
resourceType,
});
if (!config?.enabled) return null;
return new OciUpstream(config, this.basePath, this.logger);
}
public async init(): Promise<void> { public async init(): Promise<void> {
// Start cleanup of stale upload sessions // Start cleanup of stale upload sessions
this.startUploadSessionCleanup(); this.startUploadSessionCleanup();
@@ -75,67 +101,70 @@ export class OciRegistry extends BaseRegistry {
// Remove base path from URL // Remove base path from URL
const path = context.path.replace(this.basePath, ''); const path = context.path.replace(this.basePath, '');
// Extract token from Authorization header const tokenString = this.extractBearerToken(context);
const authHeader = context.headers['authorization'] || context.headers['Authorization'];
const tokenString = authHeader?.replace(/^Bearer\s+/i, '');
const token = tokenString ? await this.authManager.validateToken(tokenString, 'oci') : null; const token = tokenString ? await this.authManager.validateToken(tokenString, 'oci') : null;
// Route to appropriate handler const actor: IRequestActor = this.buildRequestActor(context, token);
if (path === '/v2/' || path === '/v2') {
return this.handleVersionCheck();
}
// Manifest operations: /v2/{name}/manifests/{reference} return this.storage.withContext({ protocol: 'oci', actor }, async () => {
const manifestMatch = path.match(/^\/v2\/([^\/]+(?:\/[^\/]+)*)\/manifests\/([^\/]+)$/); // Route to appropriate handler
if (manifestMatch) { // OCI spec: GET /v2/ is the version check endpoint
const [, name, reference] = manifestMatch; if (path === '/' || path === '' || path === '/v2/' || path === '/v2') {
// Prefer rawBody for content-addressable operations to preserve exact bytes return this.handleVersionCheck();
const bodyData = context.rawBody || context.body; }
return this.handleManifestRequest(context.method, name, reference, token, bodyData, context.headers);
}
// Blob operations: /v2/{name}/blobs/{digest} // Manifest operations: /{name}/manifests/{reference}
const blobMatch = path.match(/^\/v2\/([^\/]+(?:\/[^\/]+)*)\/blobs\/(sha256:[a-f0-9]{64})$/); const manifestMatch = path.match(/^\/([^\/]+(?:\/[^\/]+)*)\/manifests\/([^\/]+)$/);
if (blobMatch) { if (manifestMatch) {
const [, name, digest] = blobMatch; const [, name, reference] = manifestMatch;
return this.handleBlobRequest(context.method, name, digest, token, context.headers); // Prefer rawBody for content-addressable operations to preserve exact bytes
} const bodyData = context.rawBody || context.body;
return this.handleManifestRequest(context.method, name, reference, token, bodyData, context.headers, actor);
}
// Blob upload operations: /v2/{name}/blobs/uploads/ // Blob operations: /{name}/blobs/{digest}
const uploadInitMatch = path.match(/^\/v2\/([^\/]+(?:\/[^\/]+)*)\/blobs\/uploads\/?$/); const blobMatch = path.match(/^\/([^\/]+(?:\/[^\/]+)*)\/blobs\/(sha256:[a-f0-9]{64})$/);
if (uploadInitMatch && context.method === 'POST') { if (blobMatch) {
const [, name] = uploadInitMatch; const [, name, digest] = blobMatch;
// Prefer rawBody for content-addressable operations to preserve exact bytes return this.handleBlobRequest(context.method, name, digest, token, context.headers, actor);
const bodyData = context.rawBody || context.body; }
return this.handleUploadInit(name, token, context.query, bodyData);
}
// Blob upload operations: /v2/{name}/blobs/uploads/{uuid} // Blob upload operations: /{name}/blobs/uploads/
const uploadMatch = path.match(/^\/v2\/([^\/]+(?:\/[^\/]+)*)\/blobs\/uploads\/([^\/]+)$/); const uploadInitMatch = path.match(/^\/([^\/]+(?:\/[^\/]+)*)\/blobs\/uploads\/?$/);
if (uploadMatch) { if (uploadInitMatch && context.method === 'POST') {
const [, name, uploadId] = uploadMatch; const [, name] = uploadInitMatch;
return this.handleUploadSession(context.method, uploadId, token, context); // Prefer rawBody for content-addressable operations to preserve exact bytes
} const bodyData = context.rawBody || context.body;
return this.handleUploadInit(name, token, context.query, bodyData);
}
// Tags list: /v2/{name}/tags/list // Blob upload operations: /{name}/blobs/uploads/{uuid}
const tagsMatch = path.match(/^\/v2\/([^\/]+(?:\/[^\/]+)*)\/tags\/list$/); const uploadMatch = path.match(/^\/([^\/]+(?:\/[^\/]+)*)\/blobs\/uploads\/([^\/]+)$/);
if (tagsMatch) { if (uploadMatch) {
const [, name] = tagsMatch; const [, name, uploadId] = uploadMatch;
return this.handleTagsList(name, token, context.query); return this.handleUploadSession(context.method, uploadId, token, context);
} }
// Referrers: /v2/{name}/referrers/{digest} // Tags list: /{name}/tags/list
const referrersMatch = path.match(/^\/v2\/([^\/]+(?:\/[^\/]+)*)\/referrers\/(sha256:[a-f0-9]{64})$/); const tagsMatch = path.match(/^\/([^\/]+(?:\/[^\/]+)*)\/tags\/list$/);
if (referrersMatch) { if (tagsMatch) {
const [, name, digest] = referrersMatch; const [, name] = tagsMatch;
return this.handleReferrers(name, digest, token, context.query); return this.handleTagsList(name, token, context.query);
} }
return { // Referrers: /{name}/referrers/{digest}
status: 404, const referrersMatch = path.match(/^\/([^\/]+(?:\/[^\/]+)*)\/referrers\/(sha256:[a-f0-9]{64})$/);
headers: { 'Content-Type': 'application/json' }, if (referrersMatch) {
body: this.createError('NOT_FOUND', 'Endpoint not found'), const [, name, digest] = referrersMatch;
}; return this.handleReferrers(name, digest, token, context.query);
}
return {
status: 404,
headers: { 'Content-Type': 'application/json' },
body: this.createError('NOT_FOUND', 'Endpoint not found'),
};
});
} }
protected async checkPermission( protected async checkPermission(
@@ -168,11 +197,12 @@ export class OciRegistry extends BaseRegistry {
reference: string, reference: string,
token: IAuthToken | null, token: IAuthToken | null,
body?: Buffer | any, body?: Buffer | any,
headers?: Record<string, string> headers?: Record<string, string>,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
switch (method) { switch (method) {
case 'GET': case 'GET':
return this.getManifest(repository, reference, token, headers); return this.getManifest(repository, reference, token, headers, actor);
case 'HEAD': case 'HEAD':
return this.headManifest(repository, reference, token); return this.headManifest(repository, reference, token);
case 'PUT': case 'PUT':
@@ -193,11 +223,12 @@ export class OciRegistry extends BaseRegistry {
repository: string, repository: string,
digest: string, digest: string,
token: IAuthToken | null, token: IAuthToken | null,
headers: Record<string, string> headers: Record<string, string>,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
switch (method) { switch (method) {
case 'GET': case 'GET':
return this.getBlob(repository, digest, token, headers['range'] || headers['Range']); return this.getBlob(repository, digest, token, headers['range'] || headers['Range'], actor);
case 'HEAD': case 'HEAD':
return this.headBlob(repository, digest, token); return this.headBlob(repository, digest, token);
case 'DELETE': case 'DELETE':
@@ -243,7 +274,7 @@ export class OciRegistry extends BaseRegistry {
return { return {
status: 201, status: 201,
headers: { headers: {
'Location': `${this.basePath}/v2/${repository}/blobs/${digest}`, 'Location': `${this.basePath}/${repository}/blobs/${digest}`,
'Docker-Content-Digest': digest, 'Docker-Content-Digest': digest,
}, },
body: null, body: null,
@@ -256,6 +287,8 @@ export class OciRegistry extends BaseRegistry {
uploadId, uploadId,
repository, repository,
chunks: [], chunks: [],
chunkPaths: [],
chunkIndex: 0,
totalSize: 0, totalSize: 0,
createdAt: new Date(), createdAt: new Date(),
lastActivity: new Date(), lastActivity: new Date(),
@@ -266,7 +299,7 @@ export class OciRegistry extends BaseRegistry {
return { return {
status: 202, status: 202,
headers: { headers: {
'Location': `${this.basePath}/v2/${repository}/blobs/uploads/${uploadId}`, 'Location': `${this.basePath}/${repository}/blobs/uploads/${uploadId}`,
'Docker-Upload-UUID': uploadId, 'Docker-Upload-UUID': uploadId,
}, },
body: null, body: null,
@@ -318,7 +351,8 @@ export class OciRegistry extends BaseRegistry {
repository: string, repository: string,
reference: string, reference: string,
token: IAuthToken | null, token: IAuthToken | null,
headers?: Record<string, string> headers?: Record<string, string>,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
if (!await this.checkPermission(token, repository, 'pull')) { if (!await this.checkPermission(token, repository, 'pull')) {
return this.createUnauthorizedResponse(repository, 'pull'); return this.createUnauthorizedResponse(repository, 'pull');
@@ -346,30 +380,33 @@ export class OciRegistry extends BaseRegistry {
} }
// If not found locally, try upstream // If not found locally, try upstream
if (!manifestData && this.upstream) { if (!manifestData) {
this.logger.log('debug', 'getManifest: fetching from upstream', { repository, reference }); const upstream = await this.getUpstreamForRequest(repository, 'manifest', 'GET', actor);
const upstreamResult = await this.upstream.fetchManifest(repository, reference); if (upstream) {
if (upstreamResult) { this.logger.log('debug', 'getManifest: fetching from upstream', { repository, reference });
manifestData = Buffer.from(JSON.stringify(upstreamResult.manifest), 'utf8'); const upstreamResult = await upstream.fetchManifest(repository, reference);
contentType = upstreamResult.contentType; if (upstreamResult) {
digest = upstreamResult.digest; manifestData = Buffer.from(JSON.stringify(upstreamResult.manifest), 'utf8');
contentType = upstreamResult.contentType;
digest = upstreamResult.digest;
// Cache the manifest locally // Cache the manifest locally
await this.storage.putOciManifest(repository, digest, manifestData, contentType); await this.storage.putOciManifest(repository, digest, manifestData, contentType);
// If reference is a tag, update tags mapping // If reference is a tag, update tags mapping
if (!reference.startsWith('sha256:')) { if (!reference.startsWith('sha256:')) {
const tags = await this.getTagsData(repository); const tags = await this.getTagsData(repository);
tags[reference] = digest; tags[reference] = digest;
const tagsPath = `oci/tags/${repository}/tags.json`; const tagsPath = `oci/tags/${repository}/tags.json`;
await this.storage.putObject(tagsPath, Buffer.from(JSON.stringify(tags), 'utf-8')); await this.storage.putObject(tagsPath, Buffer.from(JSON.stringify(tags), 'utf-8'));
}
this.logger.log('debug', 'getManifest: cached manifest locally', {
repository,
reference,
digest,
});
} }
this.logger.log('debug', 'getManifest: cached manifest locally', {
repository,
reference,
digest,
});
} }
} }
@@ -477,7 +514,7 @@ export class OciRegistry extends BaseRegistry {
return { return {
status: 201, status: 201,
headers: { headers: {
'Location': `${this.basePath}/v2/${repository}/manifests/${digest}`, 'Location': `${this.basePath}/${repository}/manifests/${digest}`,
'Docker-Content-Digest': digest, 'Docker-Content-Digest': digest,
}, },
body: null, body: null,
@@ -514,19 +551,33 @@ export class OciRegistry extends BaseRegistry {
repository: string, repository: string,
digest: string, digest: string,
token: IAuthToken | null, token: IAuthToken | null,
range?: string range?: string,
actor?: IRequestActor
): Promise<IResponse> { ): Promise<IResponse> {
if (!await this.checkPermission(token, repository, 'pull')) { if (!await this.checkPermission(token, repository, 'pull')) {
return this.createUnauthorizedResponse(repository, 'pull'); return this.createUnauthorizedResponse(repository, 'pull');
} }
// Try local storage first // Try local storage first (streaming)
let data = await this.storage.getOciBlob(digest); const streamResult = await this.storage.getOciBlobStream(digest);
if (streamResult) {
return {
status: 200,
headers: {
'Content-Type': 'application/octet-stream',
'Content-Length': streamResult.size.toString(),
'Docker-Content-Digest': digest,
},
body: streamResult.stream,
};
}
// If not found locally, try upstream // If not found locally, try upstream
if (!data && this.upstream) { let data: Buffer | null = null;
const upstream = await this.getUpstreamForRequest(repository, 'blob', 'GET', actor);
if (upstream) {
this.logger.log('debug', 'getBlob: fetching from upstream', { repository, digest }); this.logger.log('debug', 'getBlob: fetching from upstream', { repository, digest });
const upstreamBlob = await this.upstream.fetchBlob(repository, digest); const upstreamBlob = await upstream.fetchBlob(repository, digest);
if (upstreamBlob) { if (upstreamBlob) {
data = upstreamBlob; data = upstreamBlob;
// Cache the blob locally (blobs are content-addressable and immutable) // Cache the blob locally (blobs are content-addressable and immutable)
@@ -566,17 +617,15 @@ export class OciRegistry extends BaseRegistry {
return this.createUnauthorizedHeadResponse(repository, 'pull'); return this.createUnauthorizedHeadResponse(repository, 'pull');
} }
const exists = await this.storage.ociBlobExists(digest); const blobSize = await this.storage.getOciBlobSize(digest);
if (!exists) { if (blobSize === null) {
return { status: 404, headers: {}, body: null }; return { status: 404, headers: {}, body: null };
} }
const blob = await this.storage.getOciBlob(digest);
return { return {
status: 200, status: 200,
headers: { headers: {
'Content-Length': blob ? blob.length.toString() : '0', 'Content-Length': blobSize.toString(),
'Docker-Content-Digest': digest, 'Docker-Content-Digest': digest,
}, },
body: null, body: null,
@@ -616,14 +665,19 @@ export class OciRegistry extends BaseRegistry {
} }
const chunkData = this.toBuffer(data); const chunkData = this.toBuffer(data);
session.chunks.push(chunkData);
// Write chunk to temp S3 object instead of accumulating in memory
const chunkPath = `oci/uploads/${uploadId}/chunk-${session.chunkIndex}`;
await this.storage.putObject(chunkPath, chunkData);
session.chunkPaths.push(chunkPath);
session.chunkIndex++;
session.totalSize += chunkData.length; session.totalSize += chunkData.length;
session.lastActivity = new Date(); session.lastActivity = new Date();
return { return {
status: 202, status: 202,
headers: { headers: {
'Location': `${this.basePath}/v2/${session.repository}/blobs/uploads/${uploadId}`, 'Location': `${this.basePath}/${session.repository}/blobs/uploads/${uploadId}`,
'Range': `0-${session.totalSize - 1}`, 'Range': `0-${session.totalSize - 1}`,
'Docker-Upload-UUID': uploadId, 'Docker-Upload-UUID': uploadId,
}, },
@@ -645,13 +699,52 @@ export class OciRegistry extends BaseRegistry {
}; };
} }
const chunks = [...session.chunks]; // If there's final data in the PUT body, write it as the last chunk
if (finalData) chunks.push(this.toBuffer(finalData)); if (finalData) {
const blobData = Buffer.concat(chunks); const buf = this.toBuffer(finalData);
const chunkPath = `oci/uploads/${uploadId}/chunk-${session.chunkIndex}`;
await this.storage.putObject(chunkPath, buf);
session.chunkPaths.push(chunkPath);
session.chunkIndex++;
session.totalSize += buf.length;
}
// Verify digest // Create a ReadableStream that assembles all chunks from S3 sequentially
const calculatedDigest = await this.calculateDigest(blobData); const chunkPaths = [...session.chunkPaths];
const storage = this.storage;
let chunkIdx = 0;
const assembledStream = new ReadableStream<Uint8Array>({
async pull(controller) {
if (chunkIdx >= chunkPaths.length) {
controller.close();
return;
}
const result = await storage.getObjectStream(chunkPaths[chunkIdx++]);
if (result) {
const reader = result.stream.getReader();
while (true) {
const { done, value } = await reader.read();
if (done) break;
if (value) controller.enqueue(value);
}
}
},
});
// Pipe through hash transform for incremental digest verification
const { transform: hashTransform, getDigest } = createHashTransform('sha256');
const hashedStream = assembledStream.pipeThrough(hashTransform);
// Consume stream to buffer for S3 upload
// (AWS SDK PutObjectCommand requires known content-length for streams;
// the key win is chunks are NOT accumulated in memory during PATCH — they live in S3)
const blobData = await streamToBuffer(hashedStream);
// Verify digest before storing
const calculatedDigest = `sha256:${getDigest()}`;
if (calculatedDigest !== digest) { if (calculatedDigest !== digest) {
await this.cleanupUploadChunks(session);
this.uploadSessions.delete(uploadId);
return { return {
status: 400, status: 400,
headers: {}, headers: {},
@@ -659,19 +752,36 @@ export class OciRegistry extends BaseRegistry {
}; };
} }
// Store verified blob
await this.storage.putOciBlob(digest, blobData); await this.storage.putOciBlob(digest, blobData);
// Cleanup temp chunks and session
await this.cleanupUploadChunks(session);
this.uploadSessions.delete(uploadId); this.uploadSessions.delete(uploadId);
return { return {
status: 201, status: 201,
headers: { headers: {
'Location': `${this.basePath}/v2/${session.repository}/blobs/${digest}`, 'Location': `${this.basePath}/${session.repository}/blobs/${digest}`,
'Docker-Content-Digest': digest, 'Docker-Content-Digest': digest,
}, },
body: null, body: null,
}; };
} }
/**
* Delete all temp S3 chunk objects for an upload session.
*/
private async cleanupUploadChunks(session: IUploadSession): Promise<void> {
for (const chunkPath of session.chunkPaths) {
try {
await this.storage.deleteObject(chunkPath);
} catch {
// Best-effort cleanup
}
}
}
private async getUploadStatus(uploadId: string): Promise<IResponse> { private async getUploadStatus(uploadId: string): Promise<IResponse> {
const session = this.uploadSessions.get(uploadId); const session = this.uploadSessions.get(uploadId);
if (!session) { if (!session) {
@@ -685,7 +795,7 @@ export class OciRegistry extends BaseRegistry {
return { return {
status: 204, status: 204,
headers: { headers: {
'Location': `${this.basePath}/v2/${session.repository}/blobs/uploads/${uploadId}`, 'Location': `${this.basePath}/${session.repository}/blobs/uploads/${uploadId}`,
'Range': session.totalSize > 0 ? `0-${session.totalSize - 1}` : '0-0', 'Range': session.totalSize > 0 ? `0-${session.totalSize - 1}` : '0-0',
'Docker-Upload-UUID': uploadId, 'Docker-Upload-UUID': uploadId,
}, },
@@ -830,7 +940,7 @@ export class OciRegistry extends BaseRegistry {
* Per OCI Distribution Spec, 401 responses MUST include WWW-Authenticate header. * Per OCI Distribution Spec, 401 responses MUST include WWW-Authenticate header.
*/ */
private createUnauthorizedResponse(repository: string, action: string): IResponse { private createUnauthorizedResponse(repository: string, action: string): IResponse {
const realm = this.ociTokens?.realm || `${this.basePath}/v2/token`; const realm = this.ociTokens?.realm || `${this.basePath}/token`;
const service = this.ociTokens?.service || 'registry'; const service = this.ociTokens?.service || 'registry';
return { return {
status: 401, status: 401,
@@ -845,7 +955,7 @@ export class OciRegistry extends BaseRegistry {
* Create an unauthorized HEAD response (no body per HTTP spec). * Create an unauthorized HEAD response (no body per HTTP spec).
*/ */
private createUnauthorizedHeadResponse(repository: string, action: string): IResponse { private createUnauthorizedHeadResponse(repository: string, action: string): IResponse {
const realm = this.ociTokens?.realm || `${this.basePath}/v2/token`; const realm = this.ociTokens?.realm || `${this.basePath}/token`;
const service = this.ociTokens?.service || 'registry'; const service = this.ociTokens?.service || 'registry';
return { return {
status: 401, status: 401,
@@ -863,6 +973,8 @@ export class OciRegistry extends BaseRegistry {
for (const [uploadId, session] of this.uploadSessions.entries()) { for (const [uploadId, session] of this.uploadSessions.entries()) {
if (now.getTime() - session.lastActivity.getTime() > maxAge) { if (now.getTime() - session.lastActivity.getTime() > maxAge) {
// Clean up temp S3 chunks for stale sessions
this.cleanupUploadChunks(session).catch(() => {});
this.uploadSessions.delete(uploadId); this.uploadSessions.delete(uploadId);
} }
} }
+18 -8
View File
@@ -24,13 +24,18 @@ export class OciUpstream extends BaseUpstream {
/** Local registry base path for URL building */ /** Local registry base path for URL building */
private readonly localBasePath: string; private readonly localBasePath: string;
/** API prefix for outbound OCI requests (default: /v2) */
private readonly apiPrefix: string;
constructor( constructor(
config: IProtocolUpstreamConfig, config: IProtocolUpstreamConfig,
localBasePath: string = '/oci', localBasePath: string = '/oci',
logger?: plugins.smartlog.Smartlog, logger?: plugins.smartlog.Smartlog,
apiPrefix: string = '/v2',
) { ) {
super(config, logger); super(config, logger);
this.localBasePath = localBasePath; this.localBasePath = localBasePath;
this.apiPrefix = apiPrefix;
} }
/** /**
@@ -44,7 +49,7 @@ export class OciUpstream extends BaseUpstream {
protocol: 'oci', protocol: 'oci',
resource: repository, resource: repository,
resourceType: 'manifest', resourceType: 'manifest',
path: `/v2/${repository}/manifests/${reference}`, path: `${this.apiPrefix}/${repository}/manifests/${reference}`,
method: 'GET', method: 'GET',
headers: { headers: {
'accept': [ 'accept': [
@@ -88,7 +93,7 @@ export class OciUpstream extends BaseUpstream {
protocol: 'oci', protocol: 'oci',
resource: repository, resource: repository,
resourceType: 'manifest', resourceType: 'manifest',
path: `/v2/${repository}/manifests/${reference}`, path: `${this.apiPrefix}/${repository}/manifests/${reference}`,
method: 'HEAD', method: 'HEAD',
headers: { headers: {
'accept': [ 'accept': [
@@ -127,7 +132,7 @@ export class OciUpstream extends BaseUpstream {
protocol: 'oci', protocol: 'oci',
resource: repository, resource: repository,
resourceType: 'blob', resourceType: 'blob',
path: `/v2/${repository}/blobs/${digest}`, path: `${this.apiPrefix}/${repository}/blobs/${digest}`,
method: 'GET', method: 'GET',
headers: { headers: {
'accept': 'application/octet-stream', 'accept': 'application/octet-stream',
@@ -155,7 +160,7 @@ export class OciUpstream extends BaseUpstream {
protocol: 'oci', protocol: 'oci',
resource: repository, resource: repository,
resourceType: 'blob', resourceType: 'blob',
path: `/v2/${repository}/blobs/${digest}`, path: `${this.apiPrefix}/${repository}/blobs/${digest}`,
method: 'HEAD', method: 'HEAD',
headers: {}, headers: {},
query: {}, query: {},
@@ -189,7 +194,7 @@ export class OciUpstream extends BaseUpstream {
protocol: 'oci', protocol: 'oci',
resource: repository, resource: repository,
resourceType: 'tags', resourceType: 'tags',
path: `/v2/${repository}/tags/list`, path: `${this.apiPrefix}/${repository}/tags/list`,
method: 'GET', method: 'GET',
headers: { headers: {
'accept': 'application/json', 'accept': 'application/json',
@@ -215,7 +220,8 @@ export class OciUpstream extends BaseUpstream {
/** /**
* Override URL building for OCI-specific handling. * Override URL building for OCI-specific handling.
* OCI registries use /v2/ prefix and may require special handling for Docker Hub. * OCI registries use a configurable API prefix (default /v2/) and may require
* special handling for Docker Hub.
*/ */
protected buildUpstreamUrl( protected buildUpstreamUrl(
upstream: IUpstreamRegistryConfig, upstream: IUpstreamRegistryConfig,
@@ -228,16 +234,20 @@ export class OciUpstream extends BaseUpstream {
baseUrl = baseUrl.slice(0, -1); baseUrl = baseUrl.slice(0, -1);
} }
// Use per-upstream apiPrefix if configured, otherwise use the instance default
const prefix = upstream.apiPrefix || this.apiPrefix;
// Handle Docker Hub special case // Handle Docker Hub special case
// Docker Hub uses registry-1.docker.io but library images need special handling // Docker Hub uses registry-1.docker.io but library images need special handling
if (baseUrl.includes('docker.io') || baseUrl.includes('registry-1.docker.io')) { if (baseUrl.includes('docker.io') || baseUrl.includes('registry-1.docker.io')) {
// For library images (e.g., "nginx" -> "library/nginx") // For library images (e.g., "nginx" -> "library/nginx")
const pathParts = context.path.match(/^\/v2\/([^\/]+)\/(.+)$/); const escapedPrefix = prefix.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
const pathParts = context.path.match(new RegExp(`^${escapedPrefix}\\/([^\\/]+)\\/(.+)$`));
if (pathParts) { if (pathParts) {
const [, repository, rest] = pathParts; const [, repository, rest] = pathParts;
// If repository doesn't contain a slash, it's a library image // If repository doesn't contain a slash, it's a library image
if (!repository.includes('/')) { if (!repository.includes('/')) {
return `${baseUrl}/v2/library/${repository}/${rest}`; return `${baseUrl}${prefix}/library/${repository}/${rest}`;
} }
} }
} }
+4
View File
@@ -62,6 +62,10 @@ export interface IUploadSession {
uploadId: string; uploadId: string;
repository: string; repository: string;
chunks: Buffer[]; chunks: Buffer[];
/** S3 paths to temp chunk objects (streaming mode) */
chunkPaths: string[];
/** Index counter for naming temp chunk objects */
chunkIndex: number;
totalSize: number; totalSize: number;
createdAt: Date; createdAt: Date;
lastActivity: Date; lastActivity: Date;
+2 -1
View File
@@ -1,7 +1,8 @@
// native scope // native scope
import * as asyncHooks from 'node:async_hooks';
import * as path from 'path'; import * as path from 'path';
export { path }; export { asyncHooks, path };
// @push.rocks scope // @push.rocks scope
import * as smartarchive from '@push.rocks/smartarchive'; import * as smartarchive from '@push.rocks/smartarchive';
+147 -117
View File
@@ -2,8 +2,8 @@ import { Smartlog } from '@push.rocks/smartlog';
import { BaseRegistry } from '../core/classes.baseregistry.js'; import { BaseRegistry } from '../core/classes.baseregistry.js';
import { RegistryStorage } from '../core/classes.registrystorage.js'; import { RegistryStorage } from '../core/classes.registrystorage.js';
import { AuthManager } from '../core/classes.authmanager.js'; import { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js'; import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from '../core/interfaces.core.js';
import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js'; import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
import { isBinaryData, toBuffer } from '../core/helpers.buffer.js'; import { isBinaryData, toBuffer } from '../core/helpers.buffer.js';
import type { import type {
IPypiPackageMetadata, IPypiPackageMetadata,
@@ -24,47 +24,55 @@ export class PypiRegistry extends BaseRegistry {
private basePath: string = '/pypi'; private basePath: string = '/pypi';
private registryUrl: string; private registryUrl: string;
private logger: Smartlog; private logger: Smartlog;
private upstream: PypiUpstream | null = null; private upstreamProvider: IUpstreamProvider | null = null;
constructor( constructor(
storage: RegistryStorage, storage: RegistryStorage,
authManager: AuthManager, authManager: AuthManager,
basePath: string = '/pypi', basePath: string = '/pypi',
registryUrl: string = 'http://localhost:5000', registryUrl: string = 'http://localhost:5000',
upstreamConfig?: IProtocolUpstreamConfig upstreamProvider?: IUpstreamProvider
) { ) {
super(); super();
this.storage = storage; this.storage = storage;
this.authManager = authManager; this.authManager = authManager;
this.basePath = basePath; this.basePath = basePath;
this.registryUrl = registryUrl; this.registryUrl = registryUrl;
this.upstreamProvider = upstreamProvider || null;
// Initialize logger this.logger = this.createProtocolLogger('pypi-registry', 'pypi');
this.logger = new Smartlog({ }
logContext: {
company: 'push.rocks', /**
companyunit: 'smartregistry', * Get upstream for a specific request.
containerName: 'pypi-registry', * Calls the provider to resolve upstream config dynamically.
environment: (process.env.NODE_ENV as any) || 'development', */
runtime: 'node', private async getUpstreamForRequest(
zone: 'pypi' resource: string,
} resourceType: string,
method: string,
actor?: IRequestActor
): Promise<PypiUpstream | null> {
if (!this.upstreamProvider) return null;
const config = await this.upstreamProvider.resolveUpstreamConfig({
protocol: 'pypi',
resource,
scope: resource, // For PyPI, package name is the scope
actor,
method,
resourceType,
}); });
this.logger.enableConsole();
// Initialize upstream if configured if (!config?.enabled) return null;
if (upstreamConfig?.enabled) { return new PypiUpstream(config, this.registryUrl, this.logger);
this.upstream = new PypiUpstream(upstreamConfig, registryUrl, this.logger);
}
} }
/** /**
* Clean up resources (timers, connections, etc.) * Clean up resources (timers, connections, etc.)
*/ */
public destroy(): void { public destroy(): void {
if (this.upstream) { // No persistent upstream to clean up with dynamic provider
this.upstream.stop();
}
} }
public async init(): Promise<void> { public async init(): Promise<void> {
@@ -84,61 +92,65 @@ export class PypiRegistry extends BaseRegistry {
public async handleRequest(context: IRequestContext): Promise<IResponse> { public async handleRequest(context: IRequestContext): Promise<IResponse> {
let path = context.path.replace(this.basePath, ''); let path = context.path.replace(this.basePath, '');
// Also handle /simple path prefix
if (path.startsWith('/simple')) {
path = path.replace('/simple', '');
return this.handleSimpleRequest(path, context);
}
// Extract token (Basic Auth or Bearer) // Extract token (Basic Auth or Bearer)
const token = await this.extractToken(context); const token = await this.extractToken(context);
this.logger.log('debug', `handleRequest: ${context.method} ${path}`, { const actor: IRequestActor = this.buildRequestActor(context, token);
method: context.method,
path, return this.storage.withContext({ protocol: 'pypi', actor }, async () => {
hasAuth: !!token // Also handle /simple path prefix
if (path.startsWith('/simple')) {
path = path.replace('/simple', '');
return this.handleSimpleRequest(path, context, actor);
}
this.logger.log('debug', `handleRequest: ${context.method} ${path}`, {
method: context.method,
path,
hasAuth: !!token
});
// Root upload endpoint (POST /)
if ((path === '/' || path === '') && context.method === 'POST') {
return this.handleUpload(context, token);
}
// Package metadata JSON API: GET /{package}/json
const jsonMatch = path.match(/^\/([^\/]+)\/json$/);
if (jsonMatch && context.method === 'GET') {
return this.handlePackageJson(jsonMatch[1]);
}
// Version-specific JSON API: GET /{package}/{version}/json
const versionJsonMatch = path.match(/^\/([^\/]+)\/([^\/]+)\/json$/);
if (versionJsonMatch && context.method === 'GET') {
return this.handleVersionJson(versionJsonMatch[1], versionJsonMatch[2]);
}
// Package file download: GET /packages/{package}/{filename}
const downloadMatch = path.match(/^\/packages\/([^\/]+)\/(.+)$/);
if (downloadMatch && context.method === 'GET') {
return this.handleDownload(downloadMatch[1], downloadMatch[2], actor);
}
// Delete package: DELETE /packages/{package}
if (path.match(/^\/packages\/([^\/]+)$/) && context.method === 'DELETE') {
const packageName = path.match(/^\/packages\/([^\/]+)$/)?.[1];
return this.handleDeletePackage(packageName!, token);
}
// Delete version: DELETE /packages/{package}/{version}
const deleteVersionMatch = path.match(/^\/packages\/([^\/]+)\/([^\/]+)$/);
if (deleteVersionMatch && context.method === 'DELETE') {
return this.handleDeleteVersion(deleteVersionMatch[1], deleteVersionMatch[2], token);
}
return {
status: 404,
headers: { 'Content-Type': 'application/json' },
body: { error: 'Not Found' },
};
}); });
// Root upload endpoint (POST /)
if ((path === '/' || path === '') && context.method === 'POST') {
return this.handleUpload(context, token);
}
// Package metadata JSON API: GET /{package}/json
const jsonMatch = path.match(/^\/([^\/]+)\/json$/);
if (jsonMatch && context.method === 'GET') {
return this.handlePackageJson(jsonMatch[1]);
}
// Version-specific JSON API: GET /{package}/{version}/json
const versionJsonMatch = path.match(/^\/([^\/]+)\/([^\/]+)\/json$/);
if (versionJsonMatch && context.method === 'GET') {
return this.handleVersionJson(versionJsonMatch[1], versionJsonMatch[2]);
}
// Package file download: GET /packages/{package}/{filename}
const downloadMatch = path.match(/^\/packages\/([^\/]+)\/(.+)$/);
if (downloadMatch && context.method === 'GET') {
return this.handleDownload(downloadMatch[1], downloadMatch[2]);
}
// Delete package: DELETE /packages/{package}
if (path.match(/^\/packages\/([^\/]+)$/) && context.method === 'DELETE') {
const packageName = path.match(/^\/packages\/([^\/]+)$/)?.[1];
return this.handleDeletePackage(packageName!, token);
}
// Delete version: DELETE /packages/{package}/{version}
const deleteVersionMatch = path.match(/^\/packages\/([^\/]+)\/([^\/]+)$/);
if (deleteVersionMatch && context.method === 'DELETE') {
return this.handleDeleteVersion(deleteVersionMatch[1], deleteVersionMatch[2], token);
}
return {
status: 404,
headers: { 'Content-Type': 'application/json' },
body: { error: 'Not Found' },
};
} }
/** /**
@@ -156,7 +168,7 @@ export class PypiRegistry extends BaseRegistry {
/** /**
* Handle Simple API requests (PEP 503 HTML or PEP 691 JSON) * Handle Simple API requests (PEP 503 HTML or PEP 691 JSON)
*/ */
private async handleSimpleRequest(path: string, context: IRequestContext): Promise<IResponse> { private async handleSimpleRequest(path: string, context: IRequestContext, actor?: IRequestActor): Promise<IResponse> {
// Ensure path ends with / (PEP 503 requirement) // Ensure path ends with / (PEP 503 requirement)
if (!path.endsWith('/') && !path.includes('.')) { if (!path.endsWith('/') && !path.includes('.')) {
return { return {
@@ -174,7 +186,7 @@ export class PypiRegistry extends BaseRegistry {
// Package index: /simple/{package}/ // Package index: /simple/{package}/
const packageMatch = path.match(/^\/([^\/]+)\/$/); const packageMatch = path.match(/^\/([^\/]+)\/$/);
if (packageMatch) { if (packageMatch) {
return this.handleSimplePackage(packageMatch[1], context); return this.handleSimplePackage(packageMatch[1], context, actor);
} }
return { return {
@@ -228,46 +240,49 @@ export class PypiRegistry extends BaseRegistry {
* Handle Simple API package index * Handle Simple API package index
* Returns HTML (PEP 503) or JSON (PEP 691) based on Accept header * Returns HTML (PEP 503) or JSON (PEP 691) based on Accept header
*/ */
private async handleSimplePackage(packageName: string, context: IRequestContext): Promise<IResponse> { private async handleSimplePackage(packageName: string, context: IRequestContext, actor?: IRequestActor): Promise<IResponse> {
const normalized = helpers.normalizePypiPackageName(packageName); const normalized = helpers.normalizePypiPackageName(packageName);
// Get package metadata // Get package metadata
let metadata = await this.storage.getPypiPackageMetadata(normalized); let metadata = await this.storage.getPypiPackageMetadata(normalized);
// Try upstream if not found locally // Try upstream if not found locally
if (!metadata && this.upstream) { if (!metadata) {
const upstreamHtml = await this.upstream.fetchSimplePackage(normalized); const upstream = await this.getUpstreamForRequest(normalized, 'simple', 'GET', actor);
if (upstreamHtml) { if (upstream) {
// Parse the HTML to extract file information and cache it const upstreamHtml = await upstream.fetchSimplePackage(normalized);
// For now, just return the upstream HTML directly (caching can be improved later) if (upstreamHtml) {
const acceptHeader = context.headers['accept'] || context.headers['Accept'] || ''; // Parse the HTML to extract file information and cache it
const preferJson = acceptHeader.includes('application/vnd.pypi.simple') && // For now, just return the upstream HTML directly (caching can be improved later)
acceptHeader.includes('json'); const acceptHeader = context.headers['accept'] || context.headers['Accept'] || '';
const preferJson = acceptHeader.includes('application/vnd.pypi.simple') &&
acceptHeader.includes('json');
if (preferJson) { if (preferJson) {
// Try to get JSON format from upstream // Try to get JSON format from upstream
const upstreamJson = await this.upstream.fetchPackageJson(normalized); const upstreamJson = await upstream.fetchPackageJson(normalized);
if (upstreamJson) { if (upstreamJson) {
return { return {
status: 200, status: 200,
headers: { headers: {
'Content-Type': 'application/vnd.pypi.simple.v1+json', 'Content-Type': 'application/vnd.pypi.simple.v1+json',
'Cache-Control': 'public, max-age=300' 'Cache-Control': 'public, max-age=300'
}, },
body: upstreamJson, body: upstreamJson,
}; };
}
} }
}
// Return HTML format // Return HTML format
return { return {
status: 200, status: 200,
headers: { headers: {
'Content-Type': 'text/html; charset=utf-8', 'Content-Type': 'text/html; charset=utf-8',
'Cache-Control': 'public, max-age=300' 'Cache-Control': 'public, max-age=300'
}, },
body: upstreamHtml, body: upstreamHtml,
}; };
}
} }
} }
@@ -328,14 +343,13 @@ export class PypiRegistry extends BaseRegistry {
* Extract authentication token from request * Extract authentication token from request
*/ */
private async extractToken(context: IRequestContext): Promise<IAuthToken | null> { private async extractToken(context: IRequestContext): Promise<IAuthToken | null> {
const authHeader = context.headers['authorization'] || context.headers['Authorization']; const authHeader = this.getAuthorizationHeader(context);
if (!authHeader) return null; if (!authHeader) return null;
// Handle Basic Auth (username:password or __token__:token) // Handle Basic Auth (username:password or __token__:token)
if (authHeader.startsWith('Basic ')) { const basicCredentials = this.parseBasicAuthHeader(authHeader);
const base64 = authHeader.substring(6); if (basicCredentials) {
const decoded = Buffer.from(base64, 'base64').toString('utf-8'); const { username, password } = basicCredentials;
const [username, password] = decoded.split(':');
// PyPI token authentication: username = __token__ // PyPI token authentication: username = __token__
if (username === '__token__') { if (username === '__token__') {
@@ -348,8 +362,8 @@ export class PypiRegistry extends BaseRegistry {
} }
// Handle Bearer token // Handle Bearer token
if (authHeader.startsWith('Bearer ')) { const token = this.extractBearerToken(authHeader);
const token = authHeader.substring(7); if (token) {
return this.authManager.validateToken(token, 'pypi'); return this.authManager.validateToken(token, 'pypi');
} }
@@ -503,13 +517,29 @@ export class PypiRegistry extends BaseRegistry {
/** /**
* Handle package download * Handle package download
*/ */
private async handleDownload(packageName: string, filename: string): Promise<IResponse> { private async handleDownload(packageName: string, filename: string, actor?: IRequestActor): Promise<IResponse> {
const normalized = helpers.normalizePypiPackageName(packageName); const normalized = helpers.normalizePypiPackageName(packageName);
let fileData = await this.storage.getPypiPackageFile(normalized, filename);
// Try streaming from local storage first
const streamResult = await this.storage.getPypiPackageFileStream(normalized, filename);
if (streamResult) {
return {
status: 200,
headers: {
'Content-Type': 'application/octet-stream',
'Content-Disposition': `attachment; filename="${filename}"`,
'Content-Length': streamResult.size.toString()
},
body: streamResult.stream,
};
}
// Try upstream if not found locally // Try upstream if not found locally
if (!fileData && this.upstream) { let fileData: Buffer | null = null;
fileData = await this.upstream.fetchPackageFile(normalized, filename); const upstream = await this.getUpstreamForRequest(normalized, 'file', 'GET', actor);
if (upstream) {
fileData = await upstream.fetchPackageFile(normalized, filename);
if (fileData) { if (fileData) {
// Cache locally // Cache locally
await this.storage.putPypiPackageFile(normalized, filename, fileData); await this.storage.putPypiPackageFile(normalized, filename, fileData);
+102 -72
View File
@@ -2,8 +2,8 @@ import { Smartlog } from '@push.rocks/smartlog';
import { BaseRegistry } from '../core/classes.baseregistry.js'; import { BaseRegistry } from '../core/classes.baseregistry.js';
import { RegistryStorage } from '../core/classes.registrystorage.js'; import { RegistryStorage } from '../core/classes.registrystorage.js';
import { AuthManager } from '../core/classes.authmanager.js'; import { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js'; import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from '../core/interfaces.core.js';
import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js'; import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
import type { import type {
IRubyGemsMetadata, IRubyGemsMetadata,
IRubyGemsVersionMetadata, IRubyGemsVersionMetadata,
@@ -25,47 +25,55 @@ export class RubyGemsRegistry extends BaseRegistry {
private basePath: string = '/rubygems'; private basePath: string = '/rubygems';
private registryUrl: string; private registryUrl: string;
private logger: Smartlog; private logger: Smartlog;
private upstream: RubygemsUpstream | null = null; private upstreamProvider: IUpstreamProvider | null = null;
constructor( constructor(
storage: RegistryStorage, storage: RegistryStorage,
authManager: AuthManager, authManager: AuthManager,
basePath: string = '/rubygems', basePath: string = '/rubygems',
registryUrl: string = 'http://localhost:5000/rubygems', registryUrl: string = 'http://localhost:5000/rubygems',
upstreamConfig?: IProtocolUpstreamConfig upstreamProvider?: IUpstreamProvider
) { ) {
super(); super();
this.storage = storage; this.storage = storage;
this.authManager = authManager; this.authManager = authManager;
this.basePath = basePath; this.basePath = basePath;
this.registryUrl = registryUrl; this.registryUrl = registryUrl;
this.upstreamProvider = upstreamProvider || null;
// Initialize logger this.logger = this.createProtocolLogger('rubygems-registry', 'rubygems');
this.logger = new Smartlog({ }
logContext: {
company: 'push.rocks', /**
companyunit: 'smartregistry', * Get upstream for a specific request.
containerName: 'rubygems-registry', * Calls the provider to resolve upstream config dynamically.
environment: (process.env.NODE_ENV as any) || 'development', */
runtime: 'node', private async getUpstreamForRequest(
zone: 'rubygems' resource: string,
} resourceType: string,
method: string,
actor?: IRequestActor
): Promise<RubygemsUpstream | null> {
if (!this.upstreamProvider) return null;
const config = await this.upstreamProvider.resolveUpstreamConfig({
protocol: 'rubygems',
resource,
scope: resource, // gem name is the scope
actor,
method,
resourceType,
}); });
this.logger.enableConsole();
// Initialize upstream if configured if (!config?.enabled) return null;
if (upstreamConfig?.enabled) { return new RubygemsUpstream(config, this.logger);
this.upstream = new RubygemsUpstream(upstreamConfig, this.logger);
}
} }
/** /**
* Clean up resources (timers, connections, etc.) * Clean up resources (timers, connections, etc.)
*/ */
public destroy(): void { public destroy(): void {
if (this.upstream) { // No persistent upstream to clean up with dynamic provider
this.upstream.stop();
}
} }
public async init(): Promise<void> { public async init(): Promise<void> {
@@ -95,58 +103,62 @@ export class RubyGemsRegistry extends BaseRegistry {
// Extract token (Authorization header) // Extract token (Authorization header)
const token = await this.extractToken(context); const token = await this.extractToken(context);
const actor: IRequestActor = this.buildRequestActor(context, token);
this.logger.log('debug', `handleRequest: ${context.method} ${path}`, { this.logger.log('debug', `handleRequest: ${context.method} ${path}`, {
method: context.method, method: context.method,
path, path,
hasAuth: !!token hasAuth: !!token
}); });
// Compact Index endpoints return this.storage.withContext({ protocol: 'rubygems', actor }, async () => {
if (path === '/versions' && context.method === 'GET') { // Compact Index endpoints
return this.handleVersionsFile(context); if (path === '/versions' && context.method === 'GET') {
} return this.handleVersionsFile(context);
}
if (path === '/names' && context.method === 'GET') { if (path === '/names' && context.method === 'GET') {
return this.handleNamesFile(); return this.handleNamesFile();
} }
// Info file: GET /info/{gem} // Info file: GET /info/{gem}
const infoMatch = path.match(/^\/info\/([^\/]+)$/); const infoMatch = path.match(/^\/info\/([^\/]+)$/);
if (infoMatch && context.method === 'GET') { if (infoMatch && context.method === 'GET') {
return this.handleInfoFile(infoMatch[1]); return this.handleInfoFile(infoMatch[1], actor);
} }
// Gem download: GET /gems/{gem}-{version}[-{platform}].gem // Gem download: GET /gems/{gem}-{version}[-{platform}].gem
const downloadMatch = path.match(/^\/gems\/(.+\.gem)$/); const downloadMatch = path.match(/^\/gems\/(.+\.gem)$/);
if (downloadMatch && context.method === 'GET') { if (downloadMatch && context.method === 'GET') {
return this.handleDownload(downloadMatch[1]); return this.handleDownload(downloadMatch[1], actor);
} }
// Legacy specs endpoints (Marshal format) // Legacy specs endpoints (Marshal format)
if (path === '/specs.4.8.gz' && context.method === 'GET') { if (path === '/specs.4.8.gz' && context.method === 'GET') {
return this.handleSpecs(false); return this.handleSpecs(false);
} }
if (path === '/latest_specs.4.8.gz' && context.method === 'GET') { if (path === '/latest_specs.4.8.gz' && context.method === 'GET') {
return this.handleSpecs(true); return this.handleSpecs(true);
} }
// Quick gemspec endpoint: GET /quick/Marshal.4.8/{gem}-{version}.gemspec.rz // Quick gemspec endpoint: GET /quick/Marshal.4.8/{gem}-{version}.gemspec.rz
const quickMatch = path.match(/^\/quick\/Marshal\.4\.8\/(.+)\.gemspec\.rz$/); const quickMatch = path.match(/^\/quick\/Marshal\.4\.8\/(.+)\.gemspec\.rz$/);
if (quickMatch && context.method === 'GET') { if (quickMatch && context.method === 'GET') {
return this.handleQuickGemspec(quickMatch[1]); return this.handleQuickGemspec(quickMatch[1]);
} }
// API v1 endpoints // API v1 endpoints
if (path.startsWith('/api/v1/')) { if (path.startsWith('/api/v1/')) {
return this.handleApiRequest(path.substring(7), context, token); return this.handleApiRequest(path.substring(7), context, token);
} }
return { return {
status: 404, status: 404,
headers: { 'Content-Type': 'application/json' }, headers: { 'Content-Type': 'application/json' },
body: { error: 'Not Found' }, body: { error: 'Not Found' },
}; };
});
} }
/** /**
@@ -165,7 +177,7 @@ export class RubyGemsRegistry extends BaseRegistry {
* Extract authentication token from request * Extract authentication token from request
*/ */
private async extractToken(context: IRequestContext): Promise<IAuthToken | null> { private async extractToken(context: IRequestContext): Promise<IAuthToken | null> {
const authHeader = context.headers['authorization'] || context.headers['Authorization']; const authHeader = this.getAuthorizationHeader(context);
if (!authHeader) return null; if (!authHeader) return null;
// RubyGems typically uses plain API key in Authorization header // RubyGems typically uses plain API key in Authorization header
@@ -232,16 +244,19 @@ export class RubyGemsRegistry extends BaseRegistry {
/** /**
* Handle /info/{gem} endpoint (Compact Index) * Handle /info/{gem} endpoint (Compact Index)
*/ */
private async handleInfoFile(gemName: string): Promise<IResponse> { private async handleInfoFile(gemName: string, actor?: IRequestActor): Promise<IResponse> {
let content = await this.storage.getRubyGemsInfo(gemName); let content = await this.storage.getRubyGemsInfo(gemName);
// Try upstream if not found locally // Try upstream if not found locally
if (!content && this.upstream) { if (!content) {
const upstreamInfo = await this.upstream.fetchInfo(gemName); const upstream = await this.getUpstreamForRequest(gemName, 'info', 'GET', actor);
if (upstreamInfo) { if (upstream) {
// Cache locally const upstreamInfo = await upstream.fetchInfo(gemName);
await this.storage.putRubyGemsInfo(gemName, upstreamInfo); if (upstreamInfo) {
content = upstreamInfo; // Cache locally
await this.storage.putRubyGemsInfo(gemName, upstreamInfo);
content = upstreamInfo;
}
} }
} }
@@ -267,21 +282,36 @@ export class RubyGemsRegistry extends BaseRegistry {
/** /**
* Handle gem file download * Handle gem file download
*/ */
private async handleDownload(filename: string): Promise<IResponse> { private async handleDownload(filename: string, actor?: IRequestActor): Promise<IResponse> {
const parsed = helpers.parseGemFilename(filename); const parsed = helpers.parseGemFilename(filename);
if (!parsed) { if (!parsed) {
return this.errorResponse(400, 'Invalid gem filename'); return this.errorResponse(400, 'Invalid gem filename');
} }
let gemData = await this.storage.getRubyGemsGem( // Try streaming from local storage first
const streamResult = await this.storage.getRubyGemsGemStream(
parsed.name, parsed.name,
parsed.version, parsed.version,
parsed.platform parsed.platform
); );
if (streamResult) {
return {
status: 200,
headers: {
'Content-Type': 'application/octet-stream',
'Content-Disposition': `attachment; filename="${filename}"`,
'Content-Length': streamResult.size.toString()
},
body: streamResult.stream,
};
}
// Try upstream if not found locally // Try upstream if not found locally
if (!gemData && this.upstream) { let gemData: Buffer | null = null;
gemData = await this.upstream.fetchGem(parsed.name, parsed.version); const upstream = await this.getUpstreamForRequest(parsed.name, 'gem', 'GET', actor);
if (upstream) {
gemData = await upstream.fetchGem(parsed.name, parsed.version);
if (gemData) { if (gemData) {
// Cache locally // Cache locally
await this.storage.putRubyGemsGem(parsed.name, parsed.version, gemData, parsed.platform); await this.storage.putRubyGemsGem(parsed.name, parsed.version, gemData, parsed.platform);
+8 -10
View File
@@ -254,14 +254,12 @@ export function generateVersionsJson(
uploadTime?: string; uploadTime?: string;
}> }>
): any { ): any {
return { // RubyGems.org API returns a flat array at /api/v1/versions/{gem}.json
name: gemName, return versions.map(v => ({
versions: versions.map(v => ({ number: v.version,
number: v.version, platform: v.platform || 'ruby',
platform: v.platform || 'ruby', built_at: v.uploadTime,
built_at: v.uploadTime, }));
})),
};
} }
/** /**
@@ -427,7 +425,7 @@ export async function extractGemMetadata(gemData: Buffer): Promise<{
// Step 2: Decompress the gzipped metadata // Step 2: Decompress the gzipped metadata
const gzipTools = new plugins.smartarchive.GzipTools(); const gzipTools = new plugins.smartarchive.GzipTools();
const metadataYaml = await gzipTools.decompress(metadataFile.contentBuffer); const metadataYaml = await gzipTools.decompress(metadataFile.contentBuffer);
const yamlContent = metadataYaml.toString('utf-8'); const yamlContent = Buffer.from(metadataYaml).toString('utf-8');
// Step 3: Parse the YAML to extract name, version, platform // Step 3: Parse the YAML to extract name, version, platform
// Look for name: field in YAML // Look for name: field in YAML
@@ -503,7 +501,7 @@ export async function generateSpecsGz(specs: Array<[string, string, string]>): P
} }
const uncompressed = Buffer.concat(parts); const uncompressed = Buffer.concat(parts);
return gzipTools.compress(uncompressed); return Buffer.from(await gzipTools.compress(uncompressed));
} }
/** /**
+1 -1
View File
@@ -105,7 +105,7 @@ export class UpstreamCache {
// If not in memory and we have storage, check S3 // If not in memory and we have storage, check S3
if (!entry && this.storage) { if (!entry && this.storage) {
entry = await this.loadFromStorage(key); entry = (await this.loadFromStorage(key)) ?? undefined;
if (entry) { if (entry) {
// Promote to memory cache // Promote to memory cache
this.memoryCache.set(key, entry); this.memoryCache.set(key, entry);
+82 -1
View File
@@ -1,4 +1,4 @@
import type { TRegistryProtocol } from '../core/interfaces.core.js'; import type { TRegistryProtocol, IRequestActor } from '../core/interfaces.core.js';
/** /**
* Scope rule for routing requests to specific upstreams. * Scope rule for routing requests to specific upstreams.
@@ -86,6 +86,8 @@ export interface IUpstreamRegistryConfig {
cache?: Partial<IUpstreamCacheConfig>; cache?: Partial<IUpstreamCacheConfig>;
/** Resilience configuration overrides */ /** Resilience configuration overrides */
resilience?: Partial<IUpstreamResilienceConfig>; resilience?: Partial<IUpstreamResilienceConfig>;
/** API path prefix for OCI registries (default: /v2). Useful for registries behind reverse proxies. */
apiPrefix?: string;
} }
/** /**
@@ -146,6 +148,8 @@ export interface IUpstreamFetchContext {
headers: Record<string, string>; headers: Record<string, string>;
/** Query parameters */ /** Query parameters */
query: Record<string, string>; query: Record<string, string>;
/** Actor performing the request (for cache key isolation) */
actor?: IRequestActor;
} }
/** /**
@@ -193,3 +197,80 @@ export const DEFAULT_RESILIENCE_CONFIG: IUpstreamResilienceConfig = {
circuitBreakerThreshold: 5, circuitBreakerThreshold: 5,
circuitBreakerResetMs: 30000, circuitBreakerResetMs: 30000,
}; };
// ============================================================================
// Upstream Provider Interfaces
// ============================================================================
/**
* Context for resolving upstream configuration.
* Passed to IUpstreamProvider per-request to enable dynamic upstream routing.
*/
export interface IUpstreamResolutionContext {
/** Protocol being accessed */
protocol: TRegistryProtocol;
/** Resource identifier (package name, repository, coordinates, etc.) */
resource: string;
/** Extracted scope (e.g., "company" from "@company/pkg", "myorg" from "myorg/image") */
scope: string | null;
/** Actor performing the request */
actor?: IRequestActor;
/** HTTP method */
method: string;
/** Resource type (packument, tarball, manifest, blob, etc.) */
resourceType: string;
}
/**
* Dynamic upstream configuration provider.
* Implement this interface to provide per-request upstream routing
* based on actor context (user, organization, etc.)
*
* @example
* ```typescript
* class OrgUpstreamProvider implements IUpstreamProvider {
* constructor(private db: Database) {}
*
* async resolveUpstreamConfig(ctx: IUpstreamResolutionContext) {
* if (ctx.actor?.orgId) {
* const orgConfig = await this.db.getOrgUpstream(ctx.actor.orgId, ctx.protocol);
* if (orgConfig) return orgConfig;
* }
* return this.db.getDefaultUpstream(ctx.protocol);
* }
* }
* ```
*/
export interface IUpstreamProvider {
/** Optional initialization */
init?(): Promise<void>;
/**
* Resolve upstream configuration for a request.
* @param context - Information about the current request
* @returns Upstream config to use, or null to skip upstream lookup
*/
resolveUpstreamConfig(context: IUpstreamResolutionContext): Promise<IProtocolUpstreamConfig | null>;
}
/**
* Static upstream provider for simple configurations.
* Use this when you have fixed upstream registries that don't change per-request.
*
* @example
* ```typescript
* const provider = new StaticUpstreamProvider({
* npm: {
* enabled: true,
* upstreams: [{ id: 'npmjs', url: 'https://registry.npmjs.org', priority: 1, enabled: true, auth: { type: 'none' } }],
* },
* });
* ```
*/
export class StaticUpstreamProvider implements IUpstreamProvider {
constructor(private configs: Partial<Record<TRegistryProtocol, IProtocolUpstreamConfig>>) {}
async resolveUpstreamConfig(ctx: IUpstreamResolutionContext): Promise<IProtocolUpstreamConfig | null> {
return this.configs[ctx.protocol] ?? null;
}
}
+1 -3
View File
@@ -4,9 +4,7 @@
"module": "NodeNext", "module": "NodeNext",
"moduleResolution": "NodeNext", "moduleResolution": "NodeNext",
"esModuleInterop": true, "esModuleInterop": true,
"verbatimModuleSyntax": true, "verbatimModuleSyntax": true
"baseUrl": ".",
"paths": {}
}, },
"exclude": ["dist_*/**/*.d.ts"] "exclude": ["dist_*/**/*.d.ts"]
} }