24 Commits

Author SHA1 Message Date
ae8dec9142 v2.4.0
Some checks failed
Default (tags) / security (push) Successful in 45s
Default (tags) / test (push) Failing after 45s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-27 20:59:49 +00:00
19da87a9df feat(core): Add pluggable auth providers, storage hooks, multi-upstream cache awareness, and PyPI/RubyGems protocol implementations 2025-11-27 20:59:49 +00:00
99b01733e7 v2.3.0
Some checks failed
Default (tags) / security (push) Successful in 39s
Default (tags) / test (push) Failing after 45s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-27 14:20:01 +00:00
0610077eec feat(upstream): Add upstream proxy/cache subsystem and integrate per-protocol upstreams 2025-11-27 14:20:01 +00:00
cfadc89b5a v2.2.3
Some checks failed
Default (tags) / security (push) Successful in 45s
Default (tags) / test (push) Failing after 38s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-27 12:41:38 +00:00
eb91a3f75b fix(tests): Use unique test run IDs and add S3 cleanup in test helpers to avoid cross-run conflicts 2025-11-27 12:41:38 +00:00
58a21a6bbb v2.2.2
Some checks failed
Default (tags) / security (push) Successful in 38s
Default (tags) / test (push) Failing after 36s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-25 23:25:26 +00:00
da1cf8ddeb fix(npm): Replace console logging with structured Smartlog in NPM registry and silence RubyGems helper error logging 2025-11-25 23:25:26 +00:00
35ff286169 v2.2.1
Some checks failed
Default (tags) / security (push) Successful in 39s
Default (tags) / test (push) Failing after 37s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-25 22:35:31 +00:00
a78934836e fix(core): Normalize binary data handling across registries and add buffer helpers 2025-11-25 22:35:31 +00:00
e81fa41b18 v2.2.0
Some checks failed
Default (tags) / security (push) Successful in 39s
Default (tags) / test (push) Failing after 36s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-25 22:10:06 +00:00
41405eb40a feat(core/registrystorage): Persist OCI manifest content-type in sidecar and normalize manifest body handling 2025-11-25 22:10:06 +00:00
67188a4e9f v2.1.2
Some checks failed
Default (tags) / security (push) Successful in 39s
Default (tags) / test (push) Failing after 36s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-25 17:15:47 +00:00
a2f7f43027 fix(oci): Prefer raw request body for content-addressable OCI operations and expose rawBody on request context 2025-11-25 17:15:47 +00:00
37a89239d9 v2.1.1
Some checks failed
Default (tags) / security (push) Successful in 35s
Default (tags) / test (push) Failing after 36s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-25 16:59:37 +00:00
93fee289e7 fix(oci): Preserve raw manifest bytes for digest calculation and handle string/JSON manifest bodies in OCI registry 2025-11-25 16:59:37 +00:00
30fd9a4238 v2.1.0
Some checks failed
Default (tags) / security (push) Successful in 47s
Default (tags) / test (push) Failing after 48s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-25 16:48:08 +00:00
3b5bf5e789 feat(oci): Support configurable OCI token realm/service and centralize unauthorized responses 2025-11-25 16:48:08 +00:00
9b92e1c0d2 v2.0.0
Some checks failed
Default (tags) / security (push) Successful in 49s
Default (tags) / test (push) Failing after 50s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-25 15:07:59 +00:00
6291ebf79b BREAKING CHANGE(pypi,rubygems): Revise PyPI and RubyGems handling: normalize error payloads, fix .gem parsing/packing, adjust PyPI JSON API and tests, and export smartarchive plugin 2025-11-25 15:07:59 +00:00
fcd95677a0 v1.9.0
Some checks failed
Default (tags) / security (push) Successful in 51s
Default (tags) / test (push) Failing after 52s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-25 14:28:19 +00:00
547c262578 feat(auth): Implement HMAC-SHA256 OCI JWTs; enhance PyPI & RubyGems uploads and normalize responses 2025-11-25 14:28:19 +00:00
2d6059ba7f v1.8.0
Some checks failed
Default (tags) / security (push) Successful in 40s
Default (tags) / test (push) Failing after 37s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-24 00:15:29 +00:00
284329c191 feat(smarts3): Add local smarts3 testing support and documentation 2025-11-24 00:15:29 +00:00
54 changed files with 8347 additions and 751 deletions

View File

@@ -1,5 +1,115 @@
# Changelog
## 2025-11-27 - 2.4.0 - feat(core)
Add pluggable auth providers, storage hooks, multi-upstream cache awareness, and PyPI/RubyGems protocol implementations
- Introduce pluggable authentication: IAuthProvider interface and DefaultAuthProvider (in-memory) with OCI JWT support and UUID tokens.
- AuthManager now accepts a custom provider and delegates all auth operations (authenticate, validateToken, create/revoke tokens, authorize, listUserTokens).
- Add storage hooks (IStorageHooks) and hook contexts: beforePut/afterPut/afterGet/beforeDelete/afterDelete. RegistryStorage now supports hooks, context management (setContext/withContext) and invokes hooks around operations.
- RegistryStorage expanded with many protocol-specific helper methods (OCI, NPM, Maven, Cargo, Composer, PyPI, RubyGems) and improved S3/SmartBucket integration.
- Upstream improvements: BaseUpstream and UpstreamCache became multi-upstream aware (cache keys now include upstream URL), cache operations are async and support negative caching, stale-while-revalidate, ETag/metadata persistence, and S3-backed storage layer.
- Circuit breaker, retry, resilience and scope-rule routing enhancements for upstreams; upstream fetch logic updated to prefer primary upstream for cache keys and background revalidation behavior.
- SmartRegistry API extended to accept custom authProvider and storageHooks, and now wires RegistryStorage and AuthManager with those options. Core exports updated to expose auth and storage interfaces and DefaultAuthProvider.
- Add full PyPI (PEP 503/691, upload API) and RubyGems (Compact Index, API v1, uploads/yank/unyank, specs endpoints) registry implementations with parsing, upload/download, metadata management and upstream proxying.
- Add utility helpers: binary buffer helpers (toBuffer/isBinaryData), pypi and rubygems helper modules, and numerous protocol-specific helpers and tests referenced in readme.hints.
- These changes are additive and designed to be backward compatible; bumping minor version.
## 2025-11-27 - 2.3.0 - feat(upstream)
Add upstream proxy/cache subsystem and integrate per-protocol upstreams
- Introduce a complete upstream subsystem (BaseUpstream, UpstreamCache, CircuitBreaker) with caching, negative-cache, stale-while-revalidate, retries, exponential backoff and per-upstream circuit breakers.
- Add upstream interfaces and defaults (ts/upstream/interfaces.upstream.ts) and export upstream utilities from ts/upstream/index.ts and root ts/index.ts.
- Implement protocol-specific upstream clients for npm, pypi, maven, composer, cargo and rubygems (classes.*upstream.ts) to fetch metadata and artifacts from configured upstream registries.
- Integrate upstream usage into registries: registries now accept an upstream config, attempt to fetch missing metadata/artifacts from upstreams, cache results locally, and expose destroy() to stop upstream resources.
- Add SmartRequest and minimatch to dependencies and expose smartrequest/minimatch via ts/plugins.ts for HTTP requests and glob-based scope matching.
- Update package.json to add @push.rocks/smartrequest and minimatch dependencies.
- Various registry implementations updated to utilize upstreams (npm, pypi, maven, composer, cargo, rubygems, oci) including URL rewrites and caching behavior.
## 2025-11-27 - 2.2.3 - fix(tests)
Use unique test run IDs and add S3 cleanup in test helpers to avoid cross-run conflicts
- Add generateTestRunId() helper in test/helpers/registry.ts to produce unique IDs for each test run
- Update PyPI and Composer native CLI tests to use generated testPackageName / unauth-pkg-<id> to avoid package name collisions between runs
- Import smartbucket and add S3 bucket cleanup logic in test helpers to remove leftover objects between test runs
- Improve test robustness by skipping upload-dependent checks when tools (twine/composer) are not available and logging outputs for debugging
## 2025-11-25 - 2.2.2 - fix(npm)
Replace console logging with structured Smartlog in NPM registry and silence RubyGems helper error logging
- Replaced console.log calls with this.logger.log (Smartlog) in ts/npm/classes.npmregistry.ts for debug/info/success events
- Converted console.error in NpmRegistry.handleSearch to structured logger.log('error', ...) including the error message
- Removed console.error from ts/rubygems/helpers.rubygems.ts; gem metadata extraction failures are now handled silently by returning null
## 2025-11-25 - 2.2.1 - fix(core)
Normalize binary data handling across registries and add buffer helpers
- Add core/helpers.buffer.ts with isBinaryData and toBuffer utilities to consistently handle Buffer, Uint8Array, string and object inputs.
- Composer: accept Uint8Array uploads, convert to Buffer before ZIP extraction, SHA-1 calculation and storage.
- PyPI: accept multipart file content as Buffer or Uint8Array and normalize to Buffer before processing and storage.
- Maven: normalize artifact body input with toBuffer before validation and storage.
- OCI: improve upload id generation by using substring for correct random length.
## 2025-11-25 - 2.2.0 - feat(core/registrystorage)
Persist OCI manifest content-type in sidecar and normalize manifest body handling
- Add getOciManifestContentType(repository, digest) to read stored manifest Content-Type
- Store manifest Content-Type in a .type sidecar file when putOciManifest is called
- Update putOciManifest to persist both manifest data and its content type
- OciRegistry now retrieves stored content type (with fallback to detectManifestContentType) when serving manifests
- Add toBuffer helper in OciRegistry to consistently convert various request body forms to Buffer for digest calculation and uploads
## 2025-11-25 - 2.1.2 - fix(oci)
Prefer raw request body for content-addressable OCI operations and expose rawBody on request context
- Add rawBody?: Buffer to IRequestContext to allow callers to provide the exact raw request bytes for digest calculation (falls back to body if absent).
- OCI registry handlers now prefer context.rawBody over context.body for content-addressable operations (manifests, blobs, and blob uploads) to preserve exact bytes and ensure digest calculation matches client expectations.
- Upload flow updates: upload init, PATCH (upload chunk) and PUT (complete upload) now pass rawBody when available.
## 2025-11-25 - 2.1.1 - fix(oci)
Preserve raw manifest bytes for digest calculation and handle string/JSON manifest bodies in OCI registry
- Preserve the exact bytes of the manifest payload when computing the sha256 digest to comply with the OCI spec and avoid mismatches caused by re-serialization.
- Accept string request bodies (converted using UTF-8) and treat already-parsed JSON objects by re-serializing as a fallback.
- Keep existing content-type fallback logic while ensuring accurate digest calculation prior to storing manifests.
## 2025-11-25 - 2.1.0 - feat(oci)
Support configurable OCI token realm/service and centralize unauthorized responses
- SmartRegistry now forwards optional ociTokens (realm and service) from auth configuration to OciRegistry when OCI is enabled
- OciRegistry constructor accepts an optional ociTokens parameter and stores it for use in auth headers
- Replaced repeated construction of WWW-Authenticate headers with createUnauthorizedResponse and createUnauthorizedHeadResponse helpers that use configured realm/service
- Behavior is backwards-compatible: when ociTokens are not configured the registry falls back to the previous defaults (realm: <basePath>/v2/token, service: "registry")
## 2025-11-25 - 2.0.0 - BREAKING CHANGE(pypi,rubygems)
Revise PyPI and RubyGems handling: normalize error payloads, fix .gem parsing/packing, adjust PyPI JSON API and tests, and export smartarchive plugin
- Rename error payload property from 'message' to 'error' in PyPI and RubyGems interfaces and responses; error responses are now returned as JSON objects (body: { error: ... }) instead of Buffer(JSON.stringify(...)).
- RubyGems: treat .gem files as plain tar archives (not gzipped). Use metadata.gz and data.tar.gz correctly, switch packing helper to pack plain tar, and use zlib deflate for .rz gemspec data.
- RubyGems registry: add legacy Marshal specs endpoint (specs.4.8.gz) and adjust versions handler invocation to accept request context.
- PyPI: adopt PEP 691 style (files is an array of file objects) in tests and metadata; include requires_python in test package metadata; update JSON API path matching to the package-level '/{package}/json' style used by the handler.
- Fix HTML escaping expectations in tests (requires_python values are HTML-escaped in attributes, e.g. '&gt;=3.8').
- Export smartarchive from plugins to enable archive helpers in core modules and helpers.
- Update tests and internal code to match the new error shape and API/format behaviour.
## 2025-11-25 - 1.9.0 - feat(auth)
Implement HMAC-SHA256 OCI JWTs; enhance PyPI & RubyGems uploads and normalize responses
- AuthManager: create and validate OCI JWTs signed with HMAC-SHA256 (header.payload.signature). Signature verification, exp/nbf checks and payload decoding implemented.
- PyPI: improved Simple API handling (PEP-691 JSON responses returned as objects), Simple HTML responses updated, upload handling enhanced to support nested/flat multipart fields, verify hashes (sha256/md5/blake2b), store files and return 201 on success.
- RubyGems: upload flow now attempts to extract gem metadata from the .gem binary when name/version are not provided, improved validation, and upload returns 201. Added extractGemMetadata helper.
- OCI: centralized 401 response creation (including proper WWW-Authenticate header) and HEAD behavior fixed to return no body per HTTP spec.
- SmartRegistry: use nullish coalescing for protocol basePath defaults to avoid falsy-value bugs when basePath is an empty string.
- Tests and helpers: test expectations adjusted (Content-Type startsWith check for HTML, PEP-691 projects is an array), test helper switched to smartarchive for packaging.
- Package.json: added devDependency @push.rocks/smartarchive and updated dev deps.
- Various response normalization: avoid unnecessary Buffer.from() for already-serialized objects/strings and standardize status codes for create/upload endpoints (201).
## 2025-11-24 - 1.8.0 - feat(smarts3)
Add local smarts3 testing support and documentation
- Added @push.rocks/smarts3 ^5.1.0 to devDependencies to enable a local S3-compatible test server.
- Updated README with a new "Testing with smarts3" section including a Quick Start example and integration test commands.
- Documented benefits and CI-friendly usage for running registry integration tests locally without cloud credentials.
## 2025-11-23 - 1.7.0 - feat(core)
Standardize S3 storage config using @tsclass/tsclass IS3Descriptor and wire it into RegistryStorage and plugins exports; update README and package dependencies.

View File

@@ -1,6 +1,6 @@
{
"name": "@push.rocks/smartregistry",
"version": "1.7.0",
"version": "2.4.0",
"private": false,
"description": "A composable TypeScript library implementing OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems registries for building unified container and package registries",
"main": "dist_ts/index.js",
@@ -18,6 +18,8 @@
"@git.zone/tsbundle": "^2.0.5",
"@git.zone/tsrun": "^2.0.0",
"@git.zone/tstest": "^3.1.0",
"@push.rocks/smartarchive": "^5.0.1",
"@push.rocks/smarts3": "^5.1.0",
"@types/node": "^24.10.1"
},
"repository": {
@@ -48,8 +50,10 @@
"@push.rocks/smartbucket": "^4.3.0",
"@push.rocks/smartlog": "^3.1.10",
"@push.rocks/smartpath": "^6.0.0",
"@push.rocks/smartrequest": "^5.0.1",
"@tsclass/tsclass": "^9.3.0",
"adm-zip": "^0.5.10"
"adm-zip": "^0.5.10",
"minimatch": "^10.1.1"
},
"packageManager": "pnpm@10.18.1+sha512.77a884a165cbba2d8d1c19e3b4880eee6d2fcabd0d879121e282196b80042351d5eb3ca0935fa599da1dc51265cc68816ad2bddd2a2de5ea9fdf92adbec7cd34"
}

117
pnpm-lock.yaml generated
View File

@@ -20,12 +20,18 @@ importers:
'@push.rocks/smartpath':
specifier: ^6.0.0
version: 6.0.0
'@push.rocks/smartrequest':
specifier: ^5.0.1
version: 5.0.1
'@tsclass/tsclass':
specifier: ^9.3.0
version: 9.3.0
adm-zip:
specifier: ^0.5.10
version: 0.5.16
minimatch:
specifier: ^10.1.1
version: 10.1.1
devDependencies:
'@git.zone/tsbuild':
specifier: ^3.1.0
@@ -39,6 +45,12 @@ importers:
'@git.zone/tstest':
specifier: ^3.1.0
version: 3.1.0(socks@2.8.7)(typescript@5.9.3)
'@push.rocks/smartarchive':
specifier: ^5.0.1
version: 5.0.1(@push.rocks/smartfs@1.1.0)
'@push.rocks/smarts3':
specifier: ^5.1.0
version: 5.1.0
'@types/node':
specifier: ^24.10.1
version: 24.10.1
@@ -576,7 +588,6 @@ packages:
'@koa/router@9.4.0':
resolution: {integrity: sha512-dOOXgzqaDoHu5qqMEPLKEgLz5CeIA7q8+1W62mCvFVCOqeC71UoTGJ4u1xUSOpIl2J1x2pqrNULkFteUeZW3/A==}
engines: {node: '>= 8.0.0'}
deprecated: '**IMPORTANT 10x+ PERFORMANCE UPGRADE**: Please upgrade to v12.0.1+ as we have fixed an issue with debuglog causing 10x slower router benchmark performance, see https://github.com/koajs/router/pull/173'
'@leichtgewicht/ip-codec@2.0.5':
resolution: {integrity: sha512-Vo+PSpZG2/fmgmiNzYK9qWRh8h/CHrwD0mo1h1DzL4yzHNSfWYujGTYsWGreD000gcgmZ7K4Ys6Tx9TxtsKdDw==}
@@ -703,6 +714,9 @@ packages:
'@push.rocks/smartarchive@4.2.2':
resolution: {integrity: sha512-6EpqbKU32D6Gcqsc9+Tn1dOCU5HoTlrqqs/7IdUr9Tirp9Ngtptkapca1Fw/D0kVJ7SSw3kG/miAYnuPMZLEoA==}
'@push.rocks/smartarchive@5.0.1':
resolution: {integrity: sha512-x4bie9IIdL9BZqBZLc8Pemp8xZOJGa6mXSVgKJRL4/Rw+E5N4rVHjQOYGRV75nC2mAMJh9GIbixuxLnWjj77ag==}
'@push.rocks/smartbrowser@2.0.8':
resolution: {integrity: sha512-0KWRZj3TuKo/sNwgPbiSE6WL+TMeR19t1JmXBZWh9n8iA2mpc4HhMrQAndEUdRCkx5ofSaHWojIRVFzGChj0Dg==}
@@ -763,6 +777,17 @@ packages:
'@push.rocks/smartfile@11.2.7':
resolution: {integrity: sha512-8Yp7/sAgPpWJBHohV92ogHWKzRomI5MEbSG6b5W2n18tqwfAmjMed0rQvsvGrSBlnEWCKgoOrYIIZbLO61+J0Q==}
'@push.rocks/smartfile@13.0.1':
resolution: {integrity: sha512-phtryDFtBYHo7R2H9V3Y7VeiYQU9YzKL140gKD3bTicBgXoIYrJ6+b3mbZunSO2yQt1Vy1AxCxYXrFE/K+4grw==}
peerDependencies:
'@push.rocks/smartfs': ^1.0.0
peerDependenciesMeta:
'@push.rocks/smartfs':
optional: true
'@push.rocks/smartfs@1.1.0':
resolution: {integrity: sha512-fg8JIjFUPPX5laRoBpTaGwhMfZ3Y8mFT4fUaW54Y4J/BfOBa/y0+rIFgvgvqcOZgkQlyZU+FIfL8Z6zezqxyTg==}
'@push.rocks/smartguard@3.1.0':
resolution: {integrity: sha512-J23q84f1O+TwFGmd4lrO9XLHUh2DaLXo9PN/9VmTWYzTkQDv5JehmifXVI0esophXcCIfbdIu6hbt7/aHlDF4A==}
@@ -850,6 +875,9 @@ packages:
'@push.rocks/smarts3@2.2.7':
resolution: {integrity: sha512-9ZXGMlmUL2Wd+YJO0xOB8KyqPf4V++fWJvTq4s76bnqEuaCr9OLfq6czhban+i4cD3ZdIjehfuHqctzjuLw8Jw==}
'@push.rocks/smarts3@5.1.0':
resolution: {integrity: sha512-jmoSaJkdWOWxiS5aiTXvE6+zS7n6+OZe1jxIOq3weX54tPmDCjpLLTl12rdgvvpDE1ai5ayftirWhLGk96hkaw==}
'@push.rocks/smartshell@3.3.0':
resolution: {integrity: sha512-m0w618H6YBs+vXGz1CgS4nPi5CUAnqRtckcS9/koGwfcIx1IpjqmiP47BoCTbdgcv0IPUxQVBG1IXTHPuZ8Z5g==}
@@ -1754,7 +1782,7 @@ packages:
engines: {node: '>=12'}
co@4.6.0:
resolution: {integrity: sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==}
resolution: {integrity: sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=}
engines: {iojs: '>= 1.0.0', node: '>= 0.12.0'}
color-convert@1.9.3:
@@ -1769,7 +1797,7 @@ packages:
engines: {node: '>=14.6'}
color-name@1.1.3:
resolution: {integrity: sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==}
resolution: {integrity: sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=}
color-name@1.1.4:
resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==}
@@ -1894,7 +1922,7 @@ packages:
engines: {node: '>=10'}
deep-equal@1.0.1:
resolution: {integrity: sha512-bHtC0iYvWhyaTzvV3CZgPeZQqCOBGyGsVV7v4eevpdkLHfiSrXUdBG+qAuSz4RI70sszvjQ1QSZ98An1yNwpSw==}
resolution: {integrity: sha1-9dJgKStmDghO/0zbyfCK0yR0SLU=}
deep-extend@0.6.0:
resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==}
@@ -1925,10 +1953,10 @@ packages:
engines: {node: '>=0.4.0'}
delegates@1.0.0:
resolution: {integrity: sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==}
resolution: {integrity: sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=}
depd@1.1.2:
resolution: {integrity: sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==}
resolution: {integrity: sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=}
engines: {node: '>= 0.6'}
depd@2.0.0:
@@ -1980,7 +2008,7 @@ packages:
resolution: {integrity: sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==}
encodeurl@1.0.2:
resolution: {integrity: sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==}
resolution: {integrity: sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=}
engines: {node: '>= 0.8'}
encodeurl@2.0.0:
@@ -2041,7 +2069,7 @@ packages:
resolution: {integrity: sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==}
escape-string-regexp@1.0.5:
resolution: {integrity: sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==}
resolution: {integrity: sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=}
engines: {node: '>=0.8.0'}
escape-string-regexp@5.0.0:
@@ -2202,7 +2230,7 @@ packages:
engines: {node: '>= 0.6'}
fresh@0.5.2:
resolution: {integrity: sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==}
resolution: {integrity: sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=}
engines: {node: '>= 0.6'}
fresh@2.0.0:
@@ -2291,7 +2319,7 @@ packages:
engines: {node: '>=18.0.0'}
has-flag@3.0.0:
resolution: {integrity: sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==}
resolution: {integrity: sha1-tdRU3CGZriJWmfNGfloH87lVuv0=}
engines: {node: '>=4'}
has-property-descriptors@1.0.2:
@@ -2367,7 +2395,7 @@ packages:
resolution: {integrity: sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==}
humanize-number@0.0.2:
resolution: {integrity: sha512-un3ZAcNQGI7RzaWGZzQDH47HETM4Wrj6z6E4TId8Yeq9w5ZKUVB1nrT2jwFheTUjEmqcgTjXDc959jum+ai1kQ==}
resolution: {integrity: sha1-EcCvakcWQ2M1iFiASPF5lUFInBg=}
iconv-lite@0.6.3:
resolution: {integrity: sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==}
@@ -2496,7 +2524,7 @@ packages:
resolution: {integrity: sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==}
jsonfile@4.0.0:
resolution: {integrity: sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg==}
resolution: {integrity: sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=}
jsonfile@6.2.0:
resolution: {integrity: sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg==}
@@ -2504,7 +2532,6 @@ packages:
keygrip@1.1.0:
resolution: {integrity: sha512-iYSchDJ+liQ8iwbSI2QqsQOvqv58eJCEanyJPJi+Khyu8smkcKSFUCbPwzFcL7YVtZ6eONjqRX/38caJ7QjRAQ==}
engines: {node: '>= 0.6'}
deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.
keyv@4.5.4:
resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==}
@@ -2686,7 +2713,7 @@ packages:
resolution: {integrity: sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg==}
media-typer@0.3.0:
resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==}
resolution: {integrity: sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=}
engines: {node: '>= 0.6'}
media-typer@1.1.0:
@@ -2701,7 +2728,7 @@ packages:
engines: {node: '>=18'}
methods@1.1.2:
resolution: {integrity: sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==}
resolution: {integrity: sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=}
engines: {node: '>= 0.6'}
micromark-core-commonmark@2.0.3:
@@ -2954,7 +2981,7 @@ packages:
resolution: {integrity: sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g==}
only@0.0.2:
resolution: {integrity: sha512-Fvw+Jemq5fjjyWz6CpKx6w9s7xxqo3+JCyM0WXWeCSOboZ8ABkyvP8ID4CZuChA/wxSx+XSJmdOm8rGVyJ1hdQ==}
resolution: {integrity: sha1-Kv3oTQPlC5qO3EROMGEKcCle37Q=}
open@8.4.2:
resolution: {integrity: sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ==}
@@ -3026,7 +3053,7 @@ packages:
engines: {node: '>= 0.8'}
passthrough-counter@1.0.0:
resolution: {integrity: sha512-Wy8PXTLqPAN0oEgBrlnsXPMww3SYJ44tQ8aVrGAI4h4JZYCS0oYqsPqtPR8OhJpv6qFbpbB7XAn0liKV7EXubA==}
resolution: {integrity: sha1-GWfZ5m2lcrXAI8eH2xEqOHqxZvo=}
path-exists@4.0.0:
resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==}
@@ -3352,10 +3379,10 @@ packages:
resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==}
stack-trace@0.0.10:
resolution: {integrity: sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==}
resolution: {integrity: sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA=}
statuses@1.5.0:
resolution: {integrity: sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==}
resolution: {integrity: sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=}
engines: {node: '>= 0.6'}
statuses@2.0.1:
@@ -3367,7 +3394,7 @@ packages:
engines: {node: '>= 0.8'}
streamsearch@0.1.2:
resolution: {integrity: sha512-jos8u++JKm0ARcSUTAZXOVC0mSox7Bhn6sBgty73P1f3JGf7yG2clTbBNHUdde/kdvP2FESam+vM6l8jBrNxHA==}
resolution: {integrity: sha1-gIudDlb8Jz2Am6VzOOkpkZoanxo=}
engines: {node: '>=0.8.0'}
streamx@2.23.0:
@@ -5258,6 +5285,27 @@ snapshots:
- react-native-b4a
- supports-color
'@push.rocks/smartarchive@5.0.1(@push.rocks/smartfs@1.1.0)':
dependencies:
'@push.rocks/smartdelay': 3.0.5
'@push.rocks/smartfile': 13.0.1(@push.rocks/smartfs@1.1.0)
'@push.rocks/smartpath': 6.0.0
'@push.rocks/smartpromise': 4.2.3
'@push.rocks/smartrequest': 4.4.2
'@push.rocks/smartrx': 3.0.10
'@push.rocks/smartstream': 3.2.5
'@push.rocks/smartunique': 3.0.9
'@push.rocks/smarturl': 3.1.0
'@types/tar-stream': 3.1.4
fflate: 0.8.2
file-type: 21.1.0
tar-stream: 3.1.7
transitivePeerDependencies:
- '@push.rocks/smartfs'
- bare-abort-controller
- react-native-b4a
- supports-color
'@push.rocks/smartbrowser@2.0.8(typescript@5.9.3)':
dependencies:
'@push.rocks/smartdelay': 3.0.5
@@ -5446,6 +5494,28 @@ snapshots:
glob: 11.1.0
js-yaml: 4.1.1
'@push.rocks/smartfile@13.0.1(@push.rocks/smartfs@1.1.0)':
dependencies:
'@push.rocks/lik': 6.2.2
'@push.rocks/smartdelay': 3.0.5
'@push.rocks/smartfile-interfaces': 1.0.7
'@push.rocks/smarthash': 3.2.6
'@push.rocks/smartjson': 5.2.0
'@push.rocks/smartmime': 2.0.4
'@push.rocks/smartpath': 6.0.0
'@push.rocks/smartpromise': 4.2.3
'@push.rocks/smartrequest': 4.4.2
'@push.rocks/smartstream': 3.2.5
'@types/js-yaml': 4.0.9
glob: 11.1.0
js-yaml: 4.1.1
optionalDependencies:
'@push.rocks/smartfs': 1.1.0
'@push.rocks/smartfs@1.1.0':
dependencies:
'@push.rocks/smartpath': 6.0.0
'@push.rocks/smartguard@3.1.0':
dependencies:
'@push.rocks/smartpromise': 4.2.3
@@ -5694,6 +5764,13 @@ snapshots:
- aws-crt
- supports-color
'@push.rocks/smarts3@5.1.0':
dependencies:
'@push.rocks/smartfs': 1.1.0
'@push.rocks/smartpath': 6.0.0
'@push.rocks/smartxml': 2.0.0
'@tsclass/tsclass': 9.3.0
'@push.rocks/smartshell@3.3.0':
dependencies:
'@push.rocks/smartdelay': 3.0.5

View File

@@ -1024,6 +1024,82 @@ pnpm run build
pnpm test
```
## 🧪 Testing with smarts3
smartregistry works seamlessly with [@push.rocks/smarts3](https://code.foss.global/push.rocks/smarts3), a local S3-compatible server for testing. This allows you to test the registry without needing cloud credentials or external services.
### Quick Start with smarts3
```typescript
import { Smarts3 } from '@push.rocks/smarts3';
import { SmartRegistry } from '@push.rocks/smartregistry';
// Start local S3 server
const s3Server = await Smarts3.createAndStart({
server: { port: 3456 },
storage: { cleanSlate: true },
});
// Manually create IS3Descriptor matching smarts3 configuration
// Note: smarts3 v5.1.0 doesn't properly expose getS3Descriptor() yet
const s3Descriptor = {
endpoint: 'localhost',
port: 3456,
accessKey: 'test',
accessSecret: 'test',
useSsl: false,
region: 'us-east-1',
};
// Create registry with smarts3 configuration
const registry = new SmartRegistry({
storage: {
...s3Descriptor,
bucketName: 'my-test-registry',
},
auth: {
jwtSecret: 'test-secret',
tokenStore: 'memory',
npmTokens: { enabled: true },
ociTokens: {
enabled: true,
realm: 'https://auth.example.com/token',
service: 'my-registry',
},
},
npm: { enabled: true, basePath: '/npm' },
oci: { enabled: true, basePath: '/oci' },
pypi: { enabled: true, basePath: '/pypi' },
cargo: { enabled: true, basePath: '/cargo' },
});
await registry.init();
// Use registry...
// Your tests here
// Cleanup
await s3Server.stop();
```
### Benefits of Testing with smarts3
-**Zero Setup** - No cloud credentials or external services needed
-**Fast** - Local filesystem storage, no network latency
-**Isolated** - Clean slate per test run, no shared state
-**CI/CD Ready** - Works in automated pipelines without configuration
-**Full Compatibility** - Implements S3 API, works with IS3Descriptor
### Running Integration Tests
```bash
# Run smarts3 integration test
pnpm exec tstest test/test.integration.smarts3.node.ts --verbose
# Run all tests (includes smarts3)
pnpm test
```
## License and Legal Information
This repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository.

View File

@@ -1,10 +1,63 @@
import * as qenv from '@push.rocks/qenv';
import * as crypto from 'crypto';
import * as smartarchive from '@push.rocks/smartarchive';
import * as smartbucket from '@push.rocks/smartbucket';
import { SmartRegistry } from '../../ts/classes.smartregistry.js';
import type { IRegistryConfig } from '../../ts/core/interfaces.core.js';
const testQenv = new qenv.Qenv('./', './.nogit');
/**
* Clean up S3 bucket contents for a fresh test run
* @param prefix Optional prefix to delete (e.g., 'cargo/', 'npm/', 'composer/')
*/
/**
* Generate a unique test run ID for avoiding conflicts between test runs
* Uses timestamp + random suffix for uniqueness
*/
export function generateTestRunId(): string {
const timestamp = Date.now().toString(36);
const random = Math.random().toString(36).substring(2, 6);
return `${timestamp}${random}`;
}
export async function cleanupS3Bucket(prefix?: string): Promise<void> {
const s3AccessKey = await testQenv.getEnvVarOnDemand('S3_ACCESSKEY');
const s3SecretKey = await testQenv.getEnvVarOnDemand('S3_SECRETKEY');
const s3Endpoint = await testQenv.getEnvVarOnDemand('S3_ENDPOINT');
const s3Port = await testQenv.getEnvVarOnDemand('S3_PORT');
const s3 = new smartbucket.SmartBucket({
accessKey: s3AccessKey || 'minioadmin',
accessSecret: s3SecretKey || 'minioadmin',
endpoint: s3Endpoint || 'localhost',
port: parseInt(s3Port || '9000', 10),
useSsl: false,
});
try {
const bucket = await s3.getBucket('test-registry');
if (bucket) {
if (prefix) {
// Delete only objects with the given prefix
const files = await bucket.fastList({ prefix });
for (const file of files) {
await bucket.fastRemove({ path: file.name });
}
} else {
// Delete all objects in the bucket
const files = await bucket.fastList({});
for (const file of files) {
await bucket.fastRemove({ path: file.name });
}
}
}
} catch (error) {
// Bucket might not exist yet, that's fine
console.log('Cleanup: No bucket to clean or error:', error);
}
}
/**
* Create a test SmartRegistry instance with all protocols enabled
*/
@@ -241,7 +294,7 @@ export function calculateMavenChecksums(data: Buffer) {
}
/**
* Helper to create a Composer package ZIP
* Helper to create a Composer package ZIP using smartarchive
*/
export async function createComposerZip(
vendorPackage: string,
@@ -252,8 +305,7 @@ export async function createComposerZip(
authors?: Array<{ name: string; email?: string }>;
}
): Promise<Buffer> {
const AdmZip = (await import('adm-zip')).default;
const zip = new AdmZip();
const zipTools = new smartarchive.ZipTools();
const composerJson = {
name: vendorPackage,
@@ -272,9 +324,6 @@ export async function createComposerZip(
},
};
// Add composer.json
zip.addFile('composer.json', Buffer.from(JSON.stringify(composerJson, null, 2), 'utf-8'));
// Add a test PHP file
const [vendor, pkg] = vendorPackage.split('/');
const namespace = `${vendor.charAt(0).toUpperCase() + vendor.slice(1)}\\${pkg.charAt(0).toUpperCase() + pkg.slice(1).replace(/-/g, '')}`;
@@ -290,24 +339,33 @@ class TestClass
}
`;
zip.addFile('src/TestClass.php', Buffer.from(testPhpContent, 'utf-8'));
const entries: smartarchive.IArchiveEntry[] = [
{
archivePath: 'composer.json',
content: Buffer.from(JSON.stringify(composerJson, null, 2), 'utf-8'),
},
{
archivePath: 'src/TestClass.php',
content: Buffer.from(testPhpContent, 'utf-8'),
},
{
archivePath: 'README.md',
content: Buffer.from(`# ${vendorPackage}\n\nTest package`, 'utf-8'),
},
];
// Add README
zip.addFile('README.md', Buffer.from(`# ${vendorPackage}\n\nTest package`, 'utf-8'));
return zip.toBuffer();
return zipTools.createZip(entries);
}
/**
* Helper to create a test Python wheel file (minimal ZIP structure)
* Helper to create a test Python wheel file (minimal ZIP structure) using smartarchive
*/
export async function createPythonWheel(
packageName: string,
version: string,
pyVersion: string = 'py3'
): Promise<Buffer> {
const AdmZip = (await import('adm-zip')).default;
const zip = new AdmZip();
const zipTools = new smartarchive.ZipTools();
const normalizedName = packageName.replace(/-/g, '_');
const distInfoDir = `${normalizedName}-${version}.dist-info`;
@@ -331,8 +389,6 @@ Description-Content-Type: text/markdown
Test package for SmartRegistry
`;
zip.addFile(`${distInfoDir}/METADATA`, Buffer.from(metadata, 'utf-8'));
// Create WHEEL file
const wheelContent = `Wheel-Version: 1.0
Generator: test 1.0.0
@@ -340,14 +396,6 @@ Root-Is-Purelib: true
Tag: ${pyVersion}-none-any
`;
zip.addFile(`${distInfoDir}/WHEEL`, Buffer.from(wheelContent, 'utf-8'));
// Create RECORD file (empty for test)
zip.addFile(`${distInfoDir}/RECORD`, Buffer.from('', 'utf-8'));
// Create top_level.txt
zip.addFile(`${distInfoDir}/top_level.txt`, Buffer.from(normalizedName, 'utf-8'));
// Create a simple Python module
const moduleContent = `"""${packageName} module"""
@@ -357,27 +405,44 @@ def hello():
return "Hello from ${packageName}!"
`;
zip.addFile(`${normalizedName}/__init__.py`, Buffer.from(moduleContent, 'utf-8'));
const entries: smartarchive.IArchiveEntry[] = [
{
archivePath: `${distInfoDir}/METADATA`,
content: Buffer.from(metadata, 'utf-8'),
},
{
archivePath: `${distInfoDir}/WHEEL`,
content: Buffer.from(wheelContent, 'utf-8'),
},
{
archivePath: `${distInfoDir}/RECORD`,
content: Buffer.from('', 'utf-8'),
},
{
archivePath: `${distInfoDir}/top_level.txt`,
content: Buffer.from(normalizedName, 'utf-8'),
},
{
archivePath: `${normalizedName}/__init__.py`,
content: Buffer.from(moduleContent, 'utf-8'),
},
];
return zip.toBuffer();
return zipTools.createZip(entries);
}
/**
* Helper to create a test Python source distribution (sdist)
* Helper to create a test Python source distribution (sdist) using smartarchive
*/
export async function createPythonSdist(
packageName: string,
version: string
): Promise<Buffer> {
const tar = await import('tar-stream');
const zlib = await import('zlib');
const { Readable } = await import('stream');
const tarTools = new smartarchive.TarTools();
const normalizedName = packageName.replace(/-/g, '_');
const dirPrefix = `${packageName}-${version}`;
const pack = tar.pack();
// PKG-INFO
const pkgInfo = `Metadata-Version: 2.1
Name: ${packageName}
@@ -389,8 +454,6 @@ Author-email: test@example.com
License: MIT
`;
pack.entry({ name: `${dirPrefix}/PKG-INFO` }, pkgInfo);
// setup.py
const setupPy = `from setuptools import setup, find_packages
@@ -402,8 +465,6 @@ setup(
)
`;
pack.entry({ name: `${dirPrefix}/setup.py` }, setupPy);
// Module file
const moduleContent = `"""${packageName} module"""
@@ -413,20 +474,22 @@ def hello():
return "Hello from ${packageName}!"
`;
pack.entry({ name: `${dirPrefix}/${normalizedName}/__init__.py` }, moduleContent);
const entries: smartarchive.IArchiveEntry[] = [
{
archivePath: `${dirPrefix}/PKG-INFO`,
content: Buffer.from(pkgInfo, 'utf-8'),
},
{
archivePath: `${dirPrefix}/setup.py`,
content: Buffer.from(setupPy, 'utf-8'),
},
{
archivePath: `${dirPrefix}/${normalizedName}/__init__.py`,
content: Buffer.from(moduleContent, 'utf-8'),
},
];
pack.finalize();
// Convert to gzipped tar
const chunks: Buffer[] = [];
const gzip = zlib.createGzip();
return new Promise((resolve, reject) => {
pack.pipe(gzip);
gzip.on('data', (chunk) => chunks.push(chunk));
gzip.on('end', () => resolve(Buffer.concat(chunks)));
gzip.on('error', reject);
});
return tarTools.packFilesToTarGz(entries);
}
/**
@@ -441,17 +504,15 @@ export function calculatePypiHashes(data: Buffer) {
}
/**
* Helper to create a test RubyGem file (minimal tar.gz structure)
* Helper to create a test RubyGem file (minimal tar.gz structure) using smartarchive
*/
export async function createRubyGem(
gemName: string,
version: string,
platform: string = 'ruby'
): Promise<Buffer> {
const tar = await import('tar-stream');
const zlib = await import('zlib');
const pack = tar.pack();
const tarTools = new smartarchive.TarTools();
const gzipTools = new smartarchive.GzipTools();
// Create metadata.gz (simplified)
const metadataYaml = `--- !ruby/object:Gem::Specification
@@ -499,10 +560,9 @@ summary: Test gem for SmartRegistry
test_files: []
`;
pack.entry({ name: 'metadata.gz' }, zlib.gzipSync(Buffer.from(metadataYaml, 'utf-8')));
const metadataGz = await gzipTools.compress(Buffer.from(metadataYaml, 'utf-8'));
// Create data.tar.gz (simplified)
const dataPack = tar.pack();
// Create data.tar.gz content
const libContent = `# ${gemName}
module ${gemName.charAt(0).toUpperCase() + gemName.slice(1).replace(/-/g, '')}
@@ -514,32 +574,29 @@ module ${gemName.charAt(0).toUpperCase() + gemName.slice(1).replace(/-/g, '')}
end
`;
dataPack.entry({ name: `lib/${gemName}.rb` }, libContent);
dataPack.finalize();
const dataEntries: smartarchive.IArchiveEntry[] = [
{
archivePath: `lib/${gemName}.rb`,
content: Buffer.from(libContent, 'utf-8'),
},
];
const dataChunks: Buffer[] = [];
const dataGzip = zlib.createGzip();
dataPack.pipe(dataGzip);
const dataTarGz = await tarTools.packFilesToTarGz(dataEntries);
await new Promise((resolve) => {
dataGzip.on('data', (chunk) => dataChunks.push(chunk));
dataGzip.on('end', resolve);
});
// Create the outer gem (tar.gz containing metadata.gz and data.tar.gz)
const gemEntries: smartarchive.IArchiveEntry[] = [
{
archivePath: 'metadata.gz',
content: metadataGz,
},
{
archivePath: 'data.tar.gz',
content: dataTarGz,
},
];
pack.entry({ name: 'data.tar.gz' }, Buffer.concat(dataChunks));
pack.finalize();
// Convert to gzipped tar
const chunks: Buffer[] = [];
const gzip = zlib.createGzip();
return new Promise((resolve, reject) => {
pack.pipe(gzip);
gzip.on('data', (chunk) => chunks.push(chunk));
gzip.on('end', () => resolve(Buffer.concat(chunks)));
gzip.on('error', reject);
});
// RubyGems .gem files are plain tar archives (NOT gzipped), containing metadata.gz and data.tar.gz
return tarTools.packFiles(gemEntries);
}
/**

View File

@@ -0,0 +1,468 @@
/**
* Native Composer CLI Testing
* Tests the Composer registry implementation using the actual composer CLI
*/
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { tapNodeTools } from '@git.zone/tstest/tapbundle_serverside';
import { SmartRegistry } from '../ts/index.js';
import { createTestRegistry, createTestTokens, createComposerZip, generateTestRunId } from './helpers/registry.js';
import type { IRequestContext, IResponse } from '../ts/core/interfaces.core.js';
import * as http from 'http';
import * as url from 'url';
import * as fs from 'fs';
import * as path from 'path';
// Test context
let registry: SmartRegistry;
let server: http.Server;
let registryUrl: string;
let registryPort: number;
let composerToken: string;
let testDir: string;
let composerHome: string;
let hasComposer = false;
// Unique test run ID to avoid conflicts between test runs
const testRunId = generateTestRunId();
const testPackageName = `testvendor/test-pkg-${testRunId}`;
/**
* Create HTTP server wrapper around SmartRegistry
*/
async function createHttpServer(
registryInstance: SmartRegistry,
port: number
): Promise<{ server: http.Server; url: string }> {
return new Promise((resolve, reject) => {
const httpServer = http.createServer(async (req, res) => {
try {
// Parse request
const parsedUrl = url.parse(req.url || '', true);
const pathname = parsedUrl.pathname || '/';
const query = parsedUrl.query;
// Read body
const chunks: Buffer[] = [];
for await (const chunk of req) {
chunks.push(chunk);
}
const bodyBuffer = Buffer.concat(chunks);
// Parse body based on content type
let body: any;
if (bodyBuffer.length > 0) {
const contentType = req.headers['content-type'] || '';
if (contentType.includes('application/json')) {
try {
body = JSON.parse(bodyBuffer.toString('utf-8'));
} catch (error) {
body = bodyBuffer;
}
} else {
body = bodyBuffer;
}
}
// Convert to IRequestContext
const context: IRequestContext = {
method: req.method || 'GET',
path: pathname,
headers: req.headers as Record<string, string>,
query: query as Record<string, string>,
body: body,
};
// Handle request
const response: IResponse = await registryInstance.handleRequest(context);
// Convert IResponse to HTTP response
res.statusCode = response.status;
// Set headers
for (const [key, value] of Object.entries(response.headers || {})) {
res.setHeader(key, value);
}
// Send body
if (response.body) {
if (Buffer.isBuffer(response.body)) {
res.end(response.body);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
} else {
res.end();
}
} catch (error) {
console.error('Server error:', error);
res.statusCode = 500;
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify({ error: 'INTERNAL_ERROR', message: String(error) }));
}
});
httpServer.listen(port, () => {
const serverUrl = `http://localhost:${port}`;
resolve({ server: httpServer, url: serverUrl });
});
httpServer.on('error', reject);
});
}
/**
* Setup Composer auth.json for authentication
*/
function setupComposerAuth(
token: string,
composerHomeArg: string,
serverUrl: string,
port: number
): string {
fs.mkdirSync(composerHomeArg, { recursive: true });
const authJson = {
'http-basic': {
[`localhost:${port}`]: {
username: 'testuser',
password: token,
},
},
};
const authPath = path.join(composerHomeArg, 'auth.json');
fs.writeFileSync(authPath, JSON.stringify(authJson, null, 2), 'utf-8');
return authPath;
}
/**
* Create a Composer project that uses our registry
*/
function createComposerProject(
projectDir: string,
serverUrl: string
): void {
fs.mkdirSync(projectDir, { recursive: true });
const composerJson = {
name: 'test/consumer-project',
description: 'Test consumer project for Composer CLI tests',
type: 'project',
require: {},
repositories: [
{
type: 'composer',
url: `${serverUrl}/composer`,
},
],
config: {
'secure-http': false,
},
};
fs.writeFileSync(
path.join(projectDir, 'composer.json'),
JSON.stringify(composerJson, null, 2),
'utf-8'
);
}
/**
* Run Composer command with custom home directory
*/
async function runComposerCommand(
command: string,
cwd: string
): Promise<{ stdout: string; stderr: string; exitCode: number }> {
const fullCommand = `cd "${cwd}" && COMPOSER_HOME="${composerHome}" composer ${command}`;
try {
const result = await tapNodeTools.runCommand(fullCommand);
return {
stdout: result.stdout || '',
stderr: result.stderr || '',
exitCode: result.exitCode || 0,
};
} catch (error: any) {
return {
stdout: error.stdout || '',
stderr: error.stderr || String(error),
exitCode: error.exitCode || 1,
};
}
}
/**
* Upload a Composer package via HTTP API
*/
async function uploadComposerPackage(
vendorPackage: string,
version: string,
token: string,
serverUrl: string
): Promise<void> {
const zipData = await createComposerZip(vendorPackage, version);
const response = await fetch(`${serverUrl}/composer/packages/${vendorPackage}`, {
method: 'PUT',
headers: {
'Content-Type': 'application/zip',
Authorization: `Bearer ${token}`,
},
body: zipData,
});
if (!response.ok) {
const body = await response.text();
throw new Error(`Failed to upload package: ${response.status} ${body}`);
}
}
/**
* Cleanup test directory
*/
function cleanupTestDir(dir: string): void {
if (fs.existsSync(dir)) {
fs.rmSync(dir, { recursive: true, force: true });
}
}
// ========================================================================
// TESTS
// ========================================================================
tap.test('Composer CLI: should verify composer is installed', async () => {
try {
const result = await tapNodeTools.runCommand('composer --version');
console.log('Composer version output:', result.stdout.substring(0, 200));
hasComposer = result.exitCode === 0;
expect(result.exitCode).toEqual(0);
} catch (error) {
console.log('Composer CLI not available, skipping native CLI tests');
hasComposer = false;
}
});
tap.test('Composer CLI: should setup registry and HTTP server', async () => {
// Create registry
registry = await createTestRegistry();
const tokens = await createTestTokens(registry);
composerToken = tokens.composerToken;
expect(registry).toBeInstanceOf(SmartRegistry);
expect(composerToken).toBeTypeOf('string');
// Use port 38000 (avoids conflicts with other tests)
registryPort = 38000;
const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server;
registryUrl = serverSetup.url;
expect(server).toBeDefined();
expect(registryUrl).toEqual(`http://localhost:${registryPort}`);
// Setup test directory
testDir = path.join(process.cwd(), '.nogit', 'test-composer-cli');
cleanupTestDir(testDir);
fs.mkdirSync(testDir, { recursive: true });
// Setup COMPOSER_HOME directory
composerHome = path.join(testDir, '.composer');
fs.mkdirSync(composerHome, { recursive: true });
// Setup Composer auth
const authPath = setupComposerAuth(composerToken, composerHome, registryUrl, registryPort);
expect(fs.existsSync(authPath)).toEqual(true);
});
tap.test('Composer CLI: should verify server is responding', async () => {
// Check server is up by doing a direct HTTP request
const response = await fetch(`${registryUrl}/composer/packages.json`);
expect(response.status).toBeGreaterThanOrEqual(200);
expect(response.status).toBeLessThan(500);
});
tap.test('Composer CLI: should upload a package via API', async () => {
const version = '1.0.0';
await uploadComposerPackage(testPackageName, version, composerToken, registryUrl);
// Verify package exists via p2 metadata endpoint (more reliable than packages.json for new packages)
const metadataResponse = await fetch(`${registryUrl}/composer/p2/${testPackageName}.json`);
expect(metadataResponse.status).toEqual(200);
const metadata = await metadataResponse.json();
expect(metadata.packages).toBeDefined();
expect(metadata.packages[testPackageName]).toBeDefined();
expect(metadata.packages[testPackageName].length).toBeGreaterThan(0);
});
tap.test('Composer CLI: should require package from registry', async () => {
if (!hasComposer) {
console.log('Skipping - composer not available');
return;
}
const projectDir = path.join(testDir, 'consumer-project');
createComposerProject(projectDir, registryUrl);
// Try to require the package we uploaded
const result = await runComposerCommand(
`require ${testPackageName}:1.0.0 --no-interaction`,
projectDir
);
console.log('composer require output:', result.stdout);
console.log('composer require stderr:', result.stderr);
expect(result.exitCode).toEqual(0);
});
tap.test('Composer CLI: should verify package in vendor directory', async () => {
if (!hasComposer) {
console.log('Skipping - composer not available');
return;
}
const projectDir = path.join(testDir, 'consumer-project');
// Parse vendor/package from testPackageName (e.g., "testvendor/test-pkg-abc123")
const [vendor, pkg] = testPackageName.split('/');
const packageDir = path.join(projectDir, 'vendor', vendor, pkg);
expect(fs.existsSync(packageDir)).toEqual(true);
// Check composer.json exists in package
const packageComposerPath = path.join(packageDir, 'composer.json');
expect(fs.existsSync(packageComposerPath)).toEqual(true);
});
tap.test('Composer CLI: should upload second version', async () => {
const version = '2.0.0';
await uploadComposerPackage(testPackageName, version, composerToken, registryUrl);
// Verify both versions exist via p2 metadata endpoint (Composer v2 format)
const response = await fetch(`${registryUrl}/composer/p2/${testPackageName}.json`);
expect(response.status).toEqual(200);
const metadata = await response.json();
expect(metadata.packages).toBeDefined();
expect(metadata.packages[testPackageName]).toBeDefined();
// Check that both versions are present
const versions = metadata.packages[testPackageName];
expect(versions.length).toBeGreaterThanOrEqual(2);
const versionNumbers = versions.map((v: any) => v.version);
expect(versionNumbers).toContain('1.0.0');
expect(versionNumbers).toContain('2.0.0');
});
tap.test('Composer CLI: should update to new version', async () => {
if (!hasComposer) {
console.log('Skipping - composer not available');
return;
}
const projectDir = path.join(testDir, 'consumer-project');
// Update to version 2.0.0
const result = await runComposerCommand(
`require ${testPackageName}:2.0.0 --no-interaction`,
projectDir
);
console.log('composer update output:', result.stdout);
expect(result.exitCode).toEqual(0);
// Verify composer.lock has the new version
const lockPath = path.join(projectDir, 'composer.lock');
expect(fs.existsSync(lockPath)).toEqual(true);
const lockContent = JSON.parse(fs.readFileSync(lockPath, 'utf-8'));
const pkg = lockContent.packages.find((p: any) => p.name === testPackageName);
expect(pkg?.version).toEqual('2.0.0');
});
tap.test('Composer CLI: should search for packages', async () => {
if (!hasComposer) {
console.log('Skipping - composer not available');
return;
}
const projectDir = path.join(testDir, 'consumer-project');
// Search for packages (may not work on all Composer versions)
const result = await runComposerCommand(
'search testvendor --no-interaction 2>&1 || true',
projectDir
);
console.log('composer search output:', result.stdout);
// Search may or may not work depending on registry implementation
// Just verify it doesn't crash
expect(result.exitCode).toBeLessThanOrEqual(1);
});
tap.test('Composer CLI: should show package info', async () => {
if (!hasComposer) {
console.log('Skipping - composer not available');
return;
}
const projectDir = path.join(testDir, 'consumer-project');
const result = await runComposerCommand(
`show ${testPackageName} --no-interaction`,
projectDir
);
console.log('composer show output:', result.stdout);
expect(result.exitCode).toEqual(0);
expect(result.stdout).toContain(testPackageName);
});
tap.test('Composer CLI: should remove package', async () => {
if (!hasComposer) {
console.log('Skipping - composer not available');
return;
}
const projectDir = path.join(testDir, 'consumer-project');
const result = await runComposerCommand(
`remove ${testPackageName} --no-interaction`,
projectDir
);
console.log('composer remove output:', result.stdout);
expect(result.exitCode).toEqual(0);
// Verify package is removed from vendor
const [vendor, pkg] = testPackageName.split('/');
const packageDir = path.join(projectDir, 'vendor', vendor, pkg);
expect(fs.existsSync(packageDir)).toEqual(false);
});
tap.postTask('cleanup composer cli tests', async () => {
// Stop server
if (server) {
await new Promise<void>((resolve) => {
server.close(() => resolve());
});
}
// Cleanup test directory
if (testDir) {
cleanupTestDir(testDir);
}
// Destroy registry
if (registry) {
registry.destroy();
}
});
export default tap.start();

View File

@@ -78,7 +78,7 @@ tap.test('Integration: should handle /simple path for PyPI', async () => {
});
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('text/html');
expect(response.headers['Content-Type']).toStartWith('text/html');
expect(response.body).toContain('integration-test-py');
});

View File

@@ -0,0 +1,291 @@
/**
* Integration test for smartregistry with smarts3
* Verifies that smartregistry works with a local S3-compatible server
*/
import { expect, tap } from '@git.zone/tstest/tapbundle';
import * as smarts3Module from '@push.rocks/smarts3';
import { SmartRegistry } from '../ts/classes.smartregistry.js';
import type { IRegistryConfig } from '../ts/core/interfaces.core.js';
import * as crypto from 'crypto';
let s3Server: smarts3Module.Smarts3;
let registry: SmartRegistry;
/**
* Setup: Start smarts3 server
*/
tap.test('should start smarts3 server', async () => {
s3Server = await smarts3Module.Smarts3.createAndStart({
server: {
port: 3456, // Use different port to avoid conflicts with other tests
host: '0.0.0.0',
},
storage: {
cleanSlate: true, // Fresh storage for each test run
bucketsDir: './.nogit/smarts3-test-buckets',
},
logging: {
silent: true, // Reduce test output noise
},
});
expect(s3Server).toBeDefined();
});
/**
* Setup: Create SmartRegistry with smarts3 configuration
*/
tap.test('should create SmartRegistry instance with smarts3 IS3Descriptor', async () => {
// Manually construct IS3Descriptor based on smarts3 configuration
// Note: smarts3.getS3Descriptor() returns empty object as of v5.1.0
// This is a known limitation - smarts3 doesn't expose its config properly
const s3Descriptor = {
endpoint: 'localhost',
port: 3456,
accessKey: 'test', // smarts3 doesn't require real credentials
accessSecret: 'test',
useSsl: false,
region: 'us-east-1',
};
const config: IRegistryConfig = {
storage: {
...s3Descriptor,
bucketName: 'test-registry-smarts3',
},
auth: {
jwtSecret: 'test-secret-key',
tokenStore: 'memory',
npmTokens: {
enabled: true,
},
ociTokens: {
enabled: true,
realm: 'https://auth.example.com/token',
service: 'test-registry-smarts3',
},
pypiTokens: {
enabled: true,
},
rubygemsTokens: {
enabled: true,
},
},
npm: {
enabled: true,
basePath: '/npm',
},
oci: {
enabled: true,
basePath: '/oci',
},
pypi: {
enabled: true,
basePath: '/pypi',
},
cargo: {
enabled: true,
basePath: '/cargo',
},
};
registry = new SmartRegistry(config);
await registry.init();
expect(registry).toBeDefined();
});
/**
* Test NPM protocol with smarts3
*/
tap.test('NPM: should publish package to smarts3', async () => {
const authManager = registry.getAuthManager();
const userId = await authManager.authenticate({
username: 'testuser',
password: 'testpass',
});
const token = await authManager.createNpmToken(userId, false);
const packageData = {
name: 'test-package-smarts3',
'dist-tags': {
latest: '1.0.0',
},
versions: {
'1.0.0': {
name: 'test-package-smarts3',
version: '1.0.0',
description: 'Test package for smarts3 integration',
},
},
_attachments: {
'test-package-smarts3-1.0.0.tgz': {
content_type: 'application/octet-stream',
data: Buffer.from('test tarball content').toString('base64'),
length: 20,
},
},
};
const response = await registry.handleRequest({
method: 'PUT',
path: '/npm/test-package-smarts3',
headers: {
'Authorization': `Bearer ${token}`,
'Content-Type': 'application/json',
},
query: {},
body: packageData,
});
expect(response.status).toEqual(201); // 201 Created is correct for publishing
});
tap.test('NPM: should retrieve package from smarts3', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/npm/test-package-smarts3',
headers: {},
query: {},
});
expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('name');
expect(response.body.name).toEqual('test-package-smarts3');
});
/**
* Test OCI protocol with smarts3
*/
tap.test('OCI: should store blob in smarts3', async () => {
const authManager = registry.getAuthManager();
const userId = await authManager.authenticate({
username: 'testuser',
password: 'testpass',
});
const token = await authManager.createOciToken(
userId,
['oci:repository:test-image:push'],
3600
);
// Initiate blob upload
const initiateResponse = await registry.handleRequest({
method: 'POST',
path: '/oci/v2/test-image/blobs/uploads/',
headers: {
'Authorization': `Bearer ${token}`,
},
query: {},
});
expect(initiateResponse.status).toEqual(202);
expect(initiateResponse.headers).toHaveProperty('Location');
// Extract upload ID from location
const location = initiateResponse.headers['Location'];
const uploadId = location.split('/').pop();
// Upload blob data
const blobData = Buffer.from('test blob content');
const digest = 'sha256:' + crypto
.createHash('sha256')
.update(blobData)
.digest('hex');
const uploadResponse = await registry.handleRequest({
method: 'PUT',
path: `/oci/v2/test-image/blobs/uploads/${uploadId}`,
headers: {
'Authorization': `Bearer ${token}`,
'Content-Type': 'application/octet-stream',
},
query: { digest },
body: blobData,
});
expect(uploadResponse.status).toEqual(201);
});
/**
* Test PyPI protocol with smarts3
*/
tap.test('PyPI: should upload package to smarts3', async () => {
const authManager = registry.getAuthManager();
const userId = await authManager.authenticate({
username: 'testuser',
password: 'testpass',
});
const token = await authManager.createPypiToken(userId, false);
// Note: In a real test, this would be multipart/form-data
// For simplicity, we're testing the storage layer
const storage = registry.getStorage();
// Store a test package file
const packageContent = Buffer.from('test wheel content');
await storage.putPypiPackageFile(
'test-package',
'test_package-1.0.0-py3-none-any.whl',
packageContent
);
// Store metadata
const metadata = {
name: 'test-package',
version: '1.0.0',
files: [
{
filename: 'test_package-1.0.0-py3-none-any.whl',
url: '/packages/test-package/test_package-1.0.0-py3-none-any.whl',
hashes: { sha256: 'abc123' },
},
],
};
await storage.putPypiPackageMetadata('test-package', metadata);
// Verify stored
const retrievedMetadata = await storage.getPypiPackageMetadata('test-package');
expect(retrievedMetadata).toBeDefined();
expect(retrievedMetadata.name).toEqual('test-package');
});
/**
* Test Cargo protocol with smarts3
*/
tap.test('Cargo: should store crate in smarts3', async () => {
const storage = registry.getStorage();
// Store a test crate index entry
const indexEntry = {
name: 'test-crate',
vers: '1.0.0',
deps: [],
cksum: 'abc123',
features: {},
yanked: false,
};
await storage.putCargoIndex('test-crate', [indexEntry]);
// Store the actual .crate file
const crateContent = Buffer.from('test crate tarball');
await storage.putCargoCrate('test-crate', '1.0.0', crateContent);
// Verify stored
const retrievedIndex = await storage.getCargoIndex('test-crate');
expect(retrievedIndex).toBeDefined();
expect(retrievedIndex.length).toEqual(1);
expect(retrievedIndex[0].name).toEqual('test-crate');
});
/**
* Cleanup: Stop smarts3 server
*/
tap.test('should stop smarts3 server', async () => {
await s3Server.stop();
expect(true).toEqual(true); // Just verify it completes without error
});
export default tap.start();

View File

@@ -0,0 +1,490 @@
/**
* Native Maven CLI Testing
* Tests the Maven registry implementation using the actual mvn CLI
*/
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { tapNodeTools } from '@git.zone/tstest/tapbundle_serverside';
import { SmartRegistry } from '../ts/index.js';
import { createTestRegistry, createTestTokens, createTestPom, createTestJar } from './helpers/registry.js';
import type { IRequestContext, IResponse } from '../ts/core/interfaces.core.js';
import * as http from 'http';
import * as url from 'url';
import * as fs from 'fs';
import * as path from 'path';
// Test context
let registry: SmartRegistry;
let server: http.Server;
let registryUrl: string;
let registryPort: number;
let mavenToken: string;
let testDir: string;
let m2Dir: string;
/**
* Create HTTP server wrapper around SmartRegistry
*/
async function createHttpServer(
registryInstance: SmartRegistry,
port: number
): Promise<{ server: http.Server; url: string }> {
return new Promise((resolve, reject) => {
const httpServer = http.createServer(async (req, res) => {
try {
// Parse request
const parsedUrl = url.parse(req.url || '', true);
const pathname = parsedUrl.pathname || '/';
const query = parsedUrl.query;
// Read body
const chunks: Buffer[] = [];
for await (const chunk of req) {
chunks.push(chunk);
}
const bodyBuffer = Buffer.concat(chunks);
// Parse body based on content type
let body: any;
if (bodyBuffer.length > 0) {
const contentType = req.headers['content-type'] || '';
if (contentType.includes('application/json')) {
try {
body = JSON.parse(bodyBuffer.toString('utf-8'));
} catch (error) {
body = bodyBuffer;
}
} else {
body = bodyBuffer;
}
}
// Convert to IRequestContext
const context: IRequestContext = {
method: req.method || 'GET',
path: pathname,
headers: req.headers as Record<string, string>,
query: query as Record<string, string>,
body: body,
};
// Handle request
const response: IResponse = await registryInstance.handleRequest(context);
// Convert IResponse to HTTP response
res.statusCode = response.status;
// Set headers
for (const [key, value] of Object.entries(response.headers || {})) {
res.setHeader(key, value);
}
// Send body
if (response.body) {
if (Buffer.isBuffer(response.body)) {
res.end(response.body);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
} else {
res.end();
}
} catch (error) {
console.error('Server error:', error);
res.statusCode = 500;
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify({ error: 'INTERNAL_ERROR', message: String(error) }));
}
});
httpServer.listen(port, () => {
const serverUrl = `http://localhost:${port}`;
resolve({ server: httpServer, url: serverUrl });
});
httpServer.on('error', reject);
});
}
/**
* Setup Maven settings.xml for authentication
*/
function setupMavenSettings(
token: string,
m2DirArg: string,
serverUrl: string
): string {
fs.mkdirSync(m2DirArg, { recursive: true });
const settingsXml = `<?xml version="1.0" encoding="UTF-8"?>
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
http://maven.apache.org/xsd/settings-1.0.0.xsd">
<servers>
<server>
<id>test-registry</id>
<username>testuser</username>
<password>${token}</password>
</server>
</servers>
<profiles>
<profile>
<id>test-registry</id>
<repositories>
<repository>
<id>test-registry</id>
<url>${serverUrl}/maven</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
</snapshots>
</repository>
</repositories>
</profile>
</profiles>
<activeProfiles>
<activeProfile>test-registry</activeProfile>
</activeProfiles>
</settings>
`;
const settingsPath = path.join(m2DirArg, 'settings.xml');
fs.writeFileSync(settingsPath, settingsXml, 'utf-8');
return settingsPath;
}
/**
* Create a minimal Maven project for testing
*/
function createMavenProject(
projectDir: string,
groupId: string,
artifactId: string,
version: string,
registryUrl: string
): void {
fs.mkdirSync(projectDir, { recursive: true });
const pomXml = `<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>${groupId}</groupId>
<artifactId>${artifactId}</artifactId>
<version>${version}</version>
<packaging>jar</packaging>
<name>${artifactId}</name>
<description>Test Maven project for SmartRegistry CLI tests</description>
<distributionManagement>
<repository>
<id>test-registry</id>
<url>${registryUrl}/maven</url>
</repository>
<snapshotRepository>
<id>test-registry</id>
<url>${registryUrl}/maven</url>
</snapshotRepository>
</distributionManagement>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.1</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
</plugins>
</build>
</project>
`;
fs.writeFileSync(path.join(projectDir, 'pom.xml'), pomXml, 'utf-8');
// Create minimal Java source
const srcDir = path.join(projectDir, 'src', 'main', 'java', 'com', 'test');
fs.mkdirSync(srcDir, { recursive: true });
const javaSource = `package com.test;
public class Main {
public static void main(String[] args) {
System.out.println("Hello from SmartRegistry test!");
}
}
`;
fs.writeFileSync(path.join(srcDir, 'Main.java'), javaSource, 'utf-8');
}
/**
* Run Maven command with custom settings
*/
async function runMavenCommand(
command: string,
cwd: string
): Promise<{ stdout: string; stderr: string; exitCode: number }> {
const settingsPath = path.join(m2Dir, 'settings.xml');
const fullCommand = `cd "${cwd}" && mvn -s "${settingsPath}" ${command}`;
try {
const result = await tapNodeTools.runCommand(fullCommand);
return {
stdout: result.stdout || '',
stderr: result.stderr || '',
exitCode: result.exitCode || 0,
};
} catch (error: any) {
return {
stdout: error.stdout || '',
stderr: error.stderr || String(error),
exitCode: error.exitCode || 1,
};
}
}
/**
* Cleanup test directory
*/
function cleanupTestDir(dir: string): void {
if (fs.existsSync(dir)) {
fs.rmSync(dir, { recursive: true, force: true });
}
}
// ========================================================================
// TESTS
// ========================================================================
tap.test('Maven CLI: should verify mvn is installed', async () => {
try {
const result = await tapNodeTools.runCommand('mvn -version');
console.log('Maven version output:', result.stdout.substring(0, 200));
expect(result.exitCode).toEqual(0);
} catch (error) {
console.log('Maven CLI not available, skipping native CLI tests');
// Skip remaining tests if Maven is not installed
tap.skip.test('Maven CLI: remaining tests skipped - mvn not available');
return;
}
});
tap.test('Maven CLI: should setup registry and HTTP server', async () => {
// Create registry
registry = await createTestRegistry();
const tokens = await createTestTokens(registry);
mavenToken = tokens.mavenToken;
expect(registry).toBeInstanceOf(SmartRegistry);
expect(mavenToken).toBeTypeOf('string');
// Use port 37000 (avoids conflicts with other tests)
registryPort = 37000;
const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server;
registryUrl = serverSetup.url;
expect(server).toBeDefined();
expect(registryUrl).toEqual(`http://localhost:${registryPort}`);
// Setup test directory
testDir = path.join(process.cwd(), '.nogit', 'test-maven-cli');
cleanupTestDir(testDir);
fs.mkdirSync(testDir, { recursive: true });
// Setup .m2 directory
m2Dir = path.join(testDir, '.m2');
fs.mkdirSync(m2Dir, { recursive: true });
// Setup Maven settings
const settingsPath = setupMavenSettings(mavenToken, m2Dir, registryUrl);
expect(fs.existsSync(settingsPath)).toEqual(true);
});
tap.test('Maven CLI: should verify server is responding', async () => {
// Check server is up by doing a direct HTTP request
const response = await fetch(`${registryUrl}/maven/`);
expect(response.status).toBeGreaterThanOrEqual(200);
expect(response.status).toBeLessThan(500);
});
tap.test('Maven CLI: should deploy a JAR artifact', async () => {
const groupId = 'com.test';
const artifactId = 'test-artifact';
const version = '1.0.0';
const projectDir = path.join(testDir, 'test-project');
createMavenProject(projectDir, groupId, artifactId, version, registryUrl);
// Build and deploy
const result = await runMavenCommand('clean package deploy -DskipTests', projectDir);
console.log('mvn deploy output:', result.stdout.substring(0, 500));
console.log('mvn deploy stderr:', result.stderr.substring(0, 500));
expect(result.exitCode).toEqual(0);
});
tap.test('Maven CLI: should verify artifact in registry via API', async () => {
const groupId = 'com.test';
const artifactId = 'test-artifact';
const version = '1.0.0';
// Maven path: /maven/{groupId path}/{artifactId}/{version}/{artifactId}-{version}.jar
const jarPath = `/maven/com/test/${artifactId}/${version}/${artifactId}-${version}.jar`;
const response = await fetch(`${registryUrl}${jarPath}`, {
headers: { Authorization: `Bearer ${mavenToken}` },
});
expect(response.status).toEqual(200);
const jarData = await response.arrayBuffer();
expect(jarData.byteLength).toBeGreaterThan(0);
});
tap.test('Maven CLI: should verify POM in registry', async () => {
const groupId = 'com.test';
const artifactId = 'test-artifact';
const version = '1.0.0';
const pomPath = `/maven/com/test/${artifactId}/${version}/${artifactId}-${version}.pom`;
const response = await fetch(`${registryUrl}${pomPath}`, {
headers: { Authorization: `Bearer ${mavenToken}` },
});
expect(response.status).toEqual(200);
const pomContent = await response.text();
expect(pomContent).toContain(groupId);
expect(pomContent).toContain(artifactId);
expect(pomContent).toContain(version);
});
tap.test('Maven CLI: should verify checksums exist', async () => {
const artifactId = 'test-artifact';
const version = '1.0.0';
// Check JAR checksums
const basePath = `/maven/com/test/${artifactId}/${version}/${artifactId}-${version}.jar`;
// MD5
const md5Response = await fetch(`${registryUrl}${basePath}.md5`, {
headers: { Authorization: `Bearer ${mavenToken}` },
});
expect(md5Response.status).toEqual(200);
// SHA1
const sha1Response = await fetch(`${registryUrl}${basePath}.sha1`, {
headers: { Authorization: `Bearer ${mavenToken}` },
});
expect(sha1Response.status).toEqual(200);
});
tap.test('Maven CLI: should deploy second version', async () => {
const groupId = 'com.test';
const artifactId = 'test-artifact';
const version = '2.0.0';
const projectDir = path.join(testDir, 'test-project-v2');
createMavenProject(projectDir, groupId, artifactId, version, registryUrl);
const result = await runMavenCommand('clean package deploy -DskipTests', projectDir);
console.log('mvn deploy v2 output:', result.stdout.substring(0, 500));
expect(result.exitCode).toEqual(0);
});
tap.test('Maven CLI: should verify metadata.xml exists', async () => {
const artifactId = 'test-artifact';
// Maven metadata is stored at /maven/{groupId path}/{artifactId}/maven-metadata.xml
const metadataPath = `/maven/com/test/${artifactId}/maven-metadata.xml`;
const response = await fetch(`${registryUrl}${metadataPath}`, {
headers: { Authorization: `Bearer ${mavenToken}` },
});
expect(response.status).toEqual(200);
const metadataXml = await response.text();
expect(metadataXml).toContain(artifactId);
expect(metadataXml).toContain('1.0.0');
expect(metadataXml).toContain('2.0.0');
});
tap.test('Maven CLI: should resolve dependency from registry', async () => {
const groupId = 'com.consumer';
const artifactId = 'consumer-app';
const version = '1.0.0';
const projectDir = path.join(testDir, 'consumer-project');
fs.mkdirSync(projectDir, { recursive: true });
// Create a consumer project that depends on our test artifact
const pomXml = `<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>${groupId}</groupId>
<artifactId>${artifactId}</artifactId>
<version>${version}</version>
<packaging>jar</packaging>
<repositories>
<repository>
<id>test-registry</id>
<url>${registryUrl}/maven</url>
</repository>
</repositories>
<dependencies>
<dependency>
<groupId>com.test</groupId>
<artifactId>test-artifact</artifactId>
<version>1.0.0</version>
</dependency>
</dependencies>
</project>
`;
fs.writeFileSync(path.join(projectDir, 'pom.xml'), pomXml, 'utf-8');
// Try to resolve dependencies
const result = await runMavenCommand('dependency:resolve', projectDir);
console.log('mvn dependency:resolve output:', result.stdout.substring(0, 500));
expect(result.exitCode).toEqual(0);
});
tap.postTask('cleanup maven cli tests', async () => {
// Stop server
if (server) {
await new Promise<void>((resolve) => {
server.close(() => resolve());
});
}
// Cleanup test directory
if (testDir) {
cleanupTestDir(testDir);
}
// Destroy registry
if (registry) {
registry.destroy();
}
});
export default tap.start();

View File

@@ -0,0 +1,406 @@
/**
* Native Docker CLI Testing
* Tests the OCI registry implementation using the actual Docker CLI
*/
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { tapNodeTools } from '@git.zone/tstest/tapbundle_serverside';
import { SmartRegistry } from '../ts/index.js';
import type { IRequestContext, IResponse, IRegistryConfig } from '../ts/core/interfaces.core.js';
import * as qenv from '@push.rocks/qenv';
import * as http from 'http';
import * as url from 'url';
import * as fs from 'fs';
import * as path from 'path';
const testQenv = new qenv.Qenv('./', './.nogit');
/**
* Create a test registry with local token endpoint realm
*/
async function createDockerTestRegistry(port: number): Promise<SmartRegistry> {
const s3AccessKey = await testQenv.getEnvVarOnDemand('S3_ACCESSKEY');
const s3SecretKey = await testQenv.getEnvVarOnDemand('S3_SECRETKEY');
const s3Endpoint = await testQenv.getEnvVarOnDemand('S3_ENDPOINT');
const s3Port = await testQenv.getEnvVarOnDemand('S3_PORT');
const config: IRegistryConfig = {
storage: {
accessKey: s3AccessKey || 'minioadmin',
accessSecret: s3SecretKey || 'minioadmin',
endpoint: s3Endpoint || 'localhost',
port: parseInt(s3Port || '9000', 10),
useSsl: false,
region: 'us-east-1',
bucketName: 'test-registry',
},
auth: {
jwtSecret: 'test-secret-key',
tokenStore: 'memory',
npmTokens: {
enabled: true,
},
ociTokens: {
enabled: true,
realm: `http://localhost:${port}/v2/token`,
service: 'test-registry',
},
},
oci: {
enabled: true,
basePath: '/oci',
},
};
const reg = new SmartRegistry(config);
await reg.init();
return reg;
}
/**
* Create test tokens for the registry
*/
async function createDockerTestTokens(reg: SmartRegistry) {
const authManager = reg.getAuthManager();
const userId = await authManager.authenticate({
username: 'testuser',
password: 'testpass',
});
if (!userId) {
throw new Error('Failed to authenticate test user');
}
// Create OCI token with full access
const ociToken = await authManager.createOciToken(
userId,
['oci:repository:*:*'],
3600
);
return { ociToken, userId };
}
// Test context
let registry: SmartRegistry;
let server: http.Server;
let registryUrl: string;
let registryPort: number;
let ociToken: string;
let testDir: string;
let testImageName: string;
/**
* Create HTTP server wrapper around SmartRegistry
* CRITICAL: Always passes rawBody for content-addressable operations (OCI manifests/blobs)
*
* Docker expects registry at /v2/ but SmartRegistry serves at /oci/v2/
* This wrapper rewrites paths for Docker compatibility
*
* Also implements a simple /v2/token endpoint for Docker Bearer auth flow
*/
async function createHttpServer(
registryInstance: SmartRegistry,
port: number,
tokenForAuth: string
): Promise<{ server: http.Server; url: string }> {
return new Promise((resolve, reject) => {
const httpServer = http.createServer(async (req, res) => {
try {
// Parse request
const parsedUrl = url.parse(req.url || '', true);
let pathname = parsedUrl.pathname || '/';
const query = parsedUrl.query;
// Handle token endpoint for Docker Bearer auth
if (pathname === '/v2/token' || pathname === '/token') {
console.log(`[Token Request] ${req.method} ${req.url}`);
res.statusCode = 200;
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify({
token: tokenForAuth,
access_token: tokenForAuth,
expires_in: 3600,
issued_at: new Date().toISOString(),
}));
return;
}
// Log all requests for debugging
console.log(`[Registry] ${req.method} ${pathname}`);
// Docker expects /v2/ but SmartRegistry serves at /oci/v2/
if (pathname.startsWith('/v2')) {
pathname = '/oci' + pathname;
}
// Read raw body - ALWAYS preserve exact bytes for OCI
const chunks: Buffer[] = [];
for await (const chunk of req) {
chunks.push(chunk);
}
const bodyBuffer = Buffer.concat(chunks);
// Parse body based on content type (for non-OCI protocols that need it)
let parsedBody: any;
if (bodyBuffer.length > 0) {
const contentType = req.headers['content-type'] || '';
if (contentType.includes('application/json')) {
try {
parsedBody = JSON.parse(bodyBuffer.toString('utf-8'));
} catch (error) {
parsedBody = bodyBuffer;
}
} else {
parsedBody = bodyBuffer;
}
}
// Convert to IRequestContext
const context: IRequestContext = {
method: req.method || 'GET',
path: pathname,
headers: req.headers as Record<string, string>,
query: query as Record<string, string>,
body: parsedBody,
rawBody: bodyBuffer,
};
// Handle request
const response: IResponse = await registryInstance.handleRequest(context);
console.log(`[Registry] Response: ${response.status} for ${pathname}`);
// Convert IResponse to HTTP response
res.statusCode = response.status;
// Set headers
for (const [key, value] of Object.entries(response.headers || {})) {
res.setHeader(key, value);
}
// Send body
if (response.body) {
if (Buffer.isBuffer(response.body)) {
res.end(response.body);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
} else {
res.end();
}
} catch (error) {
console.error('Server error:', error);
res.statusCode = 500;
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify({ error: 'INTERNAL_ERROR', message: String(error) }));
}
});
httpServer.listen(port, '0.0.0.0', () => {
const serverUrl = `http://localhost:${port}`;
resolve({ server: httpServer, url: serverUrl });
});
httpServer.on('error', reject);
});
}
/**
* Create a test Dockerfile
*/
function createTestDockerfile(targetDir: string, content?: string): string {
const dockerfilePath = path.join(targetDir, 'Dockerfile');
const dockerfileContent = content || `FROM alpine:latest
RUN echo "Hello from SmartRegistry test" > /hello.txt
CMD ["cat", "/hello.txt"]
`;
fs.writeFileSync(dockerfilePath, dockerfileContent, 'utf-8');
return dockerfilePath;
}
/**
* Run Docker command using the main Docker daemon (not rootless)
* Rootless Docker runs in its own network namespace and can't access host localhost
*
* IMPORTANT: DOCKER_HOST env var overrides --context flag, so we must unset it
* and explicitly set the socket path to use the main Docker daemon.
*/
async function runDockerCommand(
command: string,
cwd?: string
): Promise<{ stdout: string; stderr: string; exitCode: number }> {
// First unset DOCKER_HOST then set it to main Docker daemon socket
// Using both unset and export ensures we override any inherited env var
const dockerCommand = `unset DOCKER_HOST && export DOCKER_HOST=unix:///var/run/docker.sock && ${command}`;
const fullCommand = cwd ? `cd "${cwd}" && ${dockerCommand}` : dockerCommand;
try {
const result = await tapNodeTools.runCommand(fullCommand);
return {
stdout: result.stdout || '',
stderr: result.stderr || '',
exitCode: result.exitCode || 0,
};
} catch (error: any) {
return {
stdout: error.stdout || '',
stderr: error.stderr || String(error),
exitCode: error.exitCode || 1,
};
}
}
/**
* Cleanup test directory
*/
function cleanupTestDir(dir: string): void {
if (fs.existsSync(dir)) {
fs.rmSync(dir, { recursive: true, force: true });
}
}
/**
* Cleanup Docker resources
*/
async function cleanupDocker(imageName: string): Promise<void> {
await runDockerCommand(`docker rmi ${imageName} 2>/dev/null || true`);
await runDockerCommand(`docker rmi ${imageName}:v1 2>/dev/null || true`);
await runDockerCommand(`docker rmi ${imageName}:v2 2>/dev/null || true`);
}
// ========================================================================
// TESTS
// ========================================================================
tap.test('Docker CLI: should verify Docker is installed', async () => {
const result = await runDockerCommand('docker version');
console.log('Docker version output:', result.stdout.substring(0, 200));
expect(result.exitCode).toEqual(0);
});
tap.test('Docker CLI: should setup registry and HTTP server', async () => {
// Use localhost - Docker allows HTTP for localhost without any special config
registryPort = 15000 + Math.floor(Math.random() * 1000);
console.log(`Using port: ${registryPort}`);
registry = await createDockerTestRegistry(registryPort);
const tokens = await createDockerTestTokens(registry);
ociToken = tokens.ociToken;
expect(registry).toBeInstanceOf(SmartRegistry);
expect(ociToken).toBeTypeOf('string');
const serverSetup = await createHttpServer(registry, registryPort, ociToken);
server = serverSetup.server;
registryUrl = serverSetup.url;
expect(server).toBeDefined();
console.log(`Registry server started at ${registryUrl}`);
// Setup test directory
testDir = path.join(process.cwd(), '.nogit', 'test-docker-cli');
cleanupTestDir(testDir);
fs.mkdirSync(testDir, { recursive: true });
testImageName = `localhost:${registryPort}/test-image`;
});
tap.test('Docker CLI: should verify server is responding', async () => {
// Give the server a moment to fully initialize
await new Promise(resolve => setTimeout(resolve, 500));
const response = await fetch(`${registryUrl}/oci/v2/`);
expect(response.status).toEqual(200);
console.log('OCI v2 response:', await response.json());
});
tap.test('Docker CLI: should login to registry', async () => {
const result = await runDockerCommand(
`echo "${ociToken}" | docker login localhost:${registryPort} -u testuser --password-stdin`
);
console.log('docker login output:', result.stdout);
console.log('docker login stderr:', result.stderr);
const combinedOutput = result.stdout + result.stderr;
expect(combinedOutput).toContain('Login Succeeded');
});
tap.test('Docker CLI: should build test image', async () => {
createTestDockerfile(testDir);
const result = await runDockerCommand(
`docker build -t ${testImageName}:v1 .`,
testDir
);
console.log('docker build output:', result.stdout.substring(0, 500));
expect(result.exitCode).toEqual(0);
});
tap.test('Docker CLI: should push image to registry', async () => {
// This is the critical test - if the digest mismatch bug is fixed,
// this should succeed. The manifest bytes must be preserved exactly.
const result = await runDockerCommand(`docker push ${testImageName}:v1`);
console.log('docker push output:', result.stdout);
console.log('docker push stderr:', result.stderr);
expect(result.exitCode).toEqual(0);
});
tap.test('Docker CLI: should verify manifest in registry via API', async () => {
const response = await fetch(`${registryUrl}/oci/v2/test-image/tags/list`, {
headers: { Authorization: `Bearer ${ociToken}` },
});
expect(response.status).toEqual(200);
const tagList = await response.json();
console.log('Tags list:', tagList);
expect(tagList.name).toEqual('test-image');
expect(tagList.tags).toContain('v1');
});
tap.test('Docker CLI: should pull pushed image', async () => {
// First remove the local image
await runDockerCommand(`docker rmi ${testImageName}:v1 || true`);
const result = await runDockerCommand(`docker pull ${testImageName}:v1`);
console.log('docker pull output:', result.stdout);
expect(result.exitCode).toEqual(0);
});
tap.test('Docker CLI: should run pulled image', async () => {
const result = await runDockerCommand(`docker run --rm ${testImageName}:v1`);
console.log('docker run output:', result.stdout);
expect(result.exitCode).toEqual(0);
expect(result.stdout).toContain('Hello from SmartRegistry test');
});
tap.postTask('cleanup docker cli tests', async () => {
if (testImageName) {
await cleanupDocker(testImageName);
}
if (server) {
await new Promise<void>((resolve) => {
server.close(() => resolve());
});
}
if (testDir) {
cleanupTestDir(testDir);
}
if (registry) {
registry.destroy();
}
});
export default tap.start();

View File

@@ -0,0 +1,521 @@
/**
* Native PyPI CLI Testing
* Tests the PyPI registry implementation using pip and twine CLI tools
*/
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { tapNodeTools } from '@git.zone/tstest/tapbundle_serverside';
import { SmartRegistry } from '../ts/index.js';
import { createTestRegistry, createTestTokens, createPythonWheel, createPythonSdist, generateTestRunId } from './helpers/registry.js';
import type { IRequestContext, IResponse } from '../ts/core/interfaces.core.js';
import * as http from 'http';
import * as url from 'url';
import * as fs from 'fs';
import * as path from 'path';
// Test context
let registry: SmartRegistry;
let server: http.Server;
let registryUrl: string;
let registryPort: number;
let pypiToken: string;
let testDir: string;
let pipHome: string;
let hasPip = false;
let hasTwine = false;
// Unique test run ID to avoid conflicts between test runs
const testRunId = generateTestRunId();
const testPackageName = `test-pypi-pkg-${testRunId}`;
/**
* Create HTTP server wrapper around SmartRegistry
*/
async function createHttpServer(
registryInstance: SmartRegistry,
port: number
): Promise<{ server: http.Server; url: string }> {
return new Promise((resolve, reject) => {
const httpServer = http.createServer(async (req, res) => {
try {
// Parse request
const parsedUrl = url.parse(req.url || '', true);
const pathname = parsedUrl.pathname || '/';
const query = parsedUrl.query;
// Read body
const chunks: Buffer[] = [];
for await (const chunk of req) {
chunks.push(chunk);
}
const bodyBuffer = Buffer.concat(chunks);
// Parse body based on content type
let body: any;
if (bodyBuffer.length > 0) {
const contentType = req.headers['content-type'] || '';
if (contentType.includes('application/json')) {
try {
body = JSON.parse(bodyBuffer.toString('utf-8'));
} catch (error) {
body = bodyBuffer;
}
} else if (contentType.includes('multipart/form-data')) {
// For multipart, pass raw buffer
body = bodyBuffer;
} else {
body = bodyBuffer;
}
}
// Convert to IRequestContext
const context: IRequestContext = {
method: req.method || 'GET',
path: pathname,
headers: req.headers as Record<string, string>,
query: query as Record<string, string>,
body: body,
rawBody: bodyBuffer,
};
// Handle request
const response: IResponse = await registryInstance.handleRequest(context);
// Convert IResponse to HTTP response
res.statusCode = response.status;
// Set headers
for (const [key, value] of Object.entries(response.headers || {})) {
res.setHeader(key, value);
}
// Send body
if (response.body) {
if (Buffer.isBuffer(response.body)) {
res.end(response.body);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
} else {
res.end();
}
} catch (error) {
console.error('Server error:', error);
res.statusCode = 500;
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify({ error: 'INTERNAL_ERROR', message: String(error) }));
}
});
httpServer.listen(port, () => {
const serverUrl = `http://localhost:${port}`;
resolve({ server: httpServer, url: serverUrl });
});
httpServer.on('error', reject);
});
}
/**
* Setup .pypirc for twine authentication
*/
function setupPypirc(
token: string,
pipHomeArg: string,
serverUrl: string
): string {
fs.mkdirSync(pipHomeArg, { recursive: true });
const pypircContent = `[distutils]
index-servers = testpypi
[testpypi]
repository = ${serverUrl}/pypi
username = testuser
password = ${token}
`;
const pypircPath = path.join(pipHomeArg, '.pypirc');
fs.writeFileSync(pypircPath, pypircContent, 'utf-8');
// Set restrictive permissions
fs.chmodSync(pypircPath, 0o600);
return pypircPath;
}
/**
* Setup pip.conf for pip to use our registry
*/
function setupPipConf(
token: string,
pipHomeArg: string,
serverUrl: string,
port: number
): string {
fs.mkdirSync(pipHomeArg, { recursive: true });
// pip.conf with authentication
const pipConfContent = `[global]
index-url = ${serverUrl}/pypi/simple/
trusted-host = localhost
extra-index-url = https://pypi.org/simple/
`;
const pipDir = path.join(pipHomeArg, 'pip');
fs.mkdirSync(pipDir, { recursive: true });
const pipConfPath = path.join(pipDir, 'pip.conf');
fs.writeFileSync(pipConfPath, pipConfContent, 'utf-8');
return pipConfPath;
}
/**
* Create a test Python package wheel file
*/
async function createTestWheelFile(
packageName: string,
version: string,
targetDir: string
): Promise<string> {
const wheelData = await createPythonWheel(packageName, version);
const normalizedName = packageName.replace(/-/g, '_');
const wheelFilename = `${normalizedName}-${version}-py3-none-any.whl`;
const wheelPath = path.join(targetDir, wheelFilename);
fs.writeFileSync(wheelPath, wheelData);
return wheelPath;
}
/**
* Create a test Python package sdist file
*/
async function createTestSdistFile(
packageName: string,
version: string,
targetDir: string
): Promise<string> {
const sdistData = await createPythonSdist(packageName, version);
const sdistFilename = `${packageName}-${version}.tar.gz`;
const sdistPath = path.join(targetDir, sdistFilename);
fs.writeFileSync(sdistPath, sdistData);
return sdistPath;
}
/**
* Run pip command with custom config
*/
async function runPipCommand(
command: string,
cwd: string
): Promise<{ stdout: string; stderr: string; exitCode: number }> {
const pipConfDir = path.join(pipHome, 'pip');
const fullCommand = `cd "${cwd}" && PIP_CONFIG_FILE="${path.join(pipConfDir, 'pip.conf')}" pip ${command}`;
try {
const result = await tapNodeTools.runCommand(fullCommand);
return {
stdout: result.stdout || '',
stderr: result.stderr || '',
exitCode: result.exitCode || 0,
};
} catch (error: any) {
return {
stdout: error.stdout || '',
stderr: error.stderr || String(error),
exitCode: error.exitCode || 1,
};
}
}
/**
* Run twine command with custom config
*/
async function runTwineCommand(
command: string,
cwd: string
): Promise<{ stdout: string; stderr: string; exitCode: number }> {
const pypircPath = path.join(pipHome, '.pypirc');
const fullCommand = `cd "${cwd}" && twine ${command} --config-file "${pypircPath}"`;
try {
const result = await tapNodeTools.runCommand(fullCommand);
return {
stdout: result.stdout || '',
stderr: result.stderr || '',
exitCode: result.exitCode || 0,
};
} catch (error: any) {
return {
stdout: error.stdout || '',
stderr: error.stderr || String(error),
exitCode: error.exitCode || 1,
};
}
}
/**
* Cleanup test directory
*/
function cleanupTestDir(dir: string): void {
if (fs.existsSync(dir)) {
fs.rmSync(dir, { recursive: true, force: true });
}
}
// ========================================================================
// TESTS
// ========================================================================
tap.test('PyPI CLI: should verify pip is installed', async () => {
try {
const result = await tapNodeTools.runCommand('pip --version');
console.log('pip version output:', result.stdout.substring(0, 200));
hasPip = result.exitCode === 0;
expect(result.exitCode).toEqual(0);
} catch (error) {
console.log('pip CLI not available');
}
});
tap.test('PyPI CLI: should verify twine is installed', async () => {
try {
const result = await tapNodeTools.runCommand('twine --version');
console.log('twine version output:', result.stdout.substring(0, 200));
hasTwine = result.exitCode === 0;
expect(result.exitCode).toEqual(0);
} catch (error) {
console.log('twine CLI not available');
}
if (!hasPip && !hasTwine) {
console.log('Neither pip nor twine available, skipping native CLI tests');
tap.skip.test('PyPI CLI: remaining tests skipped - no CLI tools available');
return;
}
});
tap.test('PyPI CLI: should setup registry and HTTP server', async () => {
// Create registry
registry = await createTestRegistry();
const tokens = await createTestTokens(registry);
pypiToken = tokens.pypiToken;
expect(registry).toBeInstanceOf(SmartRegistry);
expect(pypiToken).toBeTypeOf('string');
// Use port 39000 (avoids conflicts with other tests)
registryPort = 39000;
const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server;
registryUrl = serverSetup.url;
expect(server).toBeDefined();
expect(registryUrl).toEqual(`http://localhost:${registryPort}`);
// Setup test directory
testDir = path.join(process.cwd(), '.nogit', 'test-pypi-cli');
cleanupTestDir(testDir);
fs.mkdirSync(testDir, { recursive: true });
// Setup pip/pypi home directory
pipHome = path.join(testDir, '.pip');
fs.mkdirSync(pipHome, { recursive: true });
// Setup .pypirc for twine
const pypircPath = setupPypirc(pypiToken, pipHome, registryUrl);
expect(fs.existsSync(pypircPath)).toEqual(true);
// Setup pip.conf
const pipConfPath = setupPipConf(pypiToken, pipHome, registryUrl, registryPort);
expect(fs.existsSync(pipConfPath)).toEqual(true);
});
tap.test('PyPI CLI: should verify server is responding', async () => {
// Check server is up by doing a direct HTTP request to simple index
const response = await fetch(`${registryUrl}/pypi/simple/`);
expect(response.status).toBeGreaterThanOrEqual(200);
expect(response.status).toBeLessThan(500);
});
tap.test('PyPI CLI: should upload wheel with twine', async () => {
if (!hasTwine) {
console.log('Skipping twine test - twine not available');
return;
}
const version = '1.0.0';
const wheelPath = await createTestWheelFile(testPackageName, version, testDir);
expect(fs.existsSync(wheelPath)).toEqual(true);
const result = await runTwineCommand(
`upload --repository testpypi "${wheelPath}"`,
testDir
);
console.log('twine upload output:', result.stdout);
console.log('twine upload stderr:', result.stderr);
expect(result.exitCode).toEqual(0);
});
tap.test('PyPI CLI: should verify package in simple index', async () => {
if (!hasTwine) {
console.log('Skipping - twine not available');
return;
}
const response = await fetch(`${registryUrl}/pypi/simple/${testPackageName}/`);
expect(response.status).toEqual(200);
const html = await response.text();
expect(html).toContain('1.0.0');
});
tap.test('PyPI CLI: should upload sdist with twine', async () => {
if (!hasTwine) {
console.log('Skipping twine test - twine not available');
return;
}
const version = '1.1.0';
const sdistPath = await createTestSdistFile(testPackageName, version, testDir);
expect(fs.existsSync(sdistPath)).toEqual(true);
const result = await runTwineCommand(
`upload --repository testpypi "${sdistPath}"`,
testDir
);
console.log('twine upload sdist output:', result.stdout);
console.log('twine upload sdist stderr:', result.stderr);
expect(result.exitCode).toEqual(0);
});
tap.test('PyPI CLI: should list all versions in simple index', async () => {
if (!hasTwine) {
console.log('Skipping - twine not available');
return;
}
const response = await fetch(`${registryUrl}/pypi/simple/${testPackageName}/`);
expect(response.status).toEqual(200);
const html = await response.text();
expect(html).toContain('1.0.0');
expect(html).toContain('1.1.0');
});
tap.test('PyPI CLI: should get JSON metadata', async () => {
if (!hasTwine) {
console.log('Skipping - twine not available');
return;
}
const response = await fetch(`${registryUrl}/pypi/pypi/${testPackageName}/json`);
expect(response.status).toEqual(200);
const metadata = await response.json();
expect(metadata.info).toBeDefined();
expect(metadata.info.name).toEqual(testPackageName);
expect(metadata.releases).toBeDefined();
expect(metadata.releases['1.0.0']).toBeDefined();
});
tap.test('PyPI CLI: should download package with pip', async () => {
if (!hasPip || !hasTwine) {
console.log('Skipping pip download test - pip or twine not available');
return;
}
const downloadDir = path.join(testDir, 'downloads');
fs.mkdirSync(downloadDir, { recursive: true });
// Download (not install) the package
const result = await runPipCommand(
`download ${testPackageName}==1.0.0 --dest "${downloadDir}" --no-deps`,
testDir
);
console.log('pip download output:', result.stdout);
console.log('pip download stderr:', result.stderr);
// pip download may fail if the package doesn't meet pip's requirements
// Just check it doesn't crash
expect(result.exitCode).toBeLessThanOrEqual(1);
});
tap.test('PyPI CLI: should search for packages via API', async () => {
if (!hasTwine) {
console.log('Skipping - twine not available (no packages uploaded)');
return;
}
// Use the JSON API to search/list
const response = await fetch(`${registryUrl}/pypi/pypi/${testPackageName}/json`);
expect(response.status).toEqual(200);
const metadata = await response.json();
expect(metadata.info.name).toEqual(testPackageName);
});
tap.test('PyPI CLI: should fail upload without auth', async () => {
if (!hasTwine) {
console.log('Skipping twine test - twine not available');
return;
}
const unauthPkgName = `unauth-pkg-${testRunId}`;
const version = '1.0.0';
const wheelPath = await createTestWheelFile(unauthPkgName, version, testDir);
// Create a pypirc without proper credentials
const badPypircPath = path.join(testDir, '.bad-pypirc');
fs.writeFileSync(badPypircPath, `[distutils]
index-servers = badpypi
[badpypi]
repository = ${registryUrl}/pypi
username = baduser
password = badtoken
`, 'utf-8');
const fullCommand = `cd "${testDir}" && twine upload --repository badpypi "${wheelPath}" --config-file "${badPypircPath}"`;
try {
const result = await tapNodeTools.runCommand(fullCommand);
// Should fail
expect(result.exitCode).not.toEqual(0);
} catch (error: any) {
// Expected to fail
expect(error.exitCode || 1).not.toEqual(0);
}
});
tap.postTask('cleanup pypi cli tests', async () => {
// Stop server
if (server) {
await new Promise<void>((resolve) => {
server.close(() => resolve());
});
}
// Cleanup test directory
if (testDir) {
cleanupTestDir(testDir);
}
// Destroy registry
if (registry) {
registry.destroy();
}
});
export default tap.start();

View File

@@ -80,6 +80,7 @@ tap.test('PyPI: should upload wheel file (POST /pypi/)', async () => {
pyversion: 'py3',
metadata_version: '2.1',
sha256_digest: hashes.sha256,
requires_python: '>=3.7',
content: testWheelData,
filename: filename,
},
@@ -99,7 +100,7 @@ tap.test('PyPI: should retrieve Simple API root index HTML (GET /simple/)', asyn
});
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('text/html');
expect(response.headers['Content-Type']).toStartWith('text/html');
expect(response.body).toBeTypeOf('string');
const html = response.body as string;
@@ -125,8 +126,10 @@ tap.test('PyPI: should retrieve Simple API root index JSON (GET /simple/ with Ac
const json = response.body as any;
expect(json).toHaveProperty('meta');
expect(json).toHaveProperty('projects');
expect(json.projects).toBeTypeOf('object');
expect(json.projects).toHaveProperty(normalizedPackageName);
expect(json.projects).toBeInstanceOf(Array);
// Check that the package is in the projects list (PEP 691 format: array of { name } objects)
const packageNames = json.projects.map((p: any) => p.name);
expect(packageNames).toContain(normalizedPackageName);
});
tap.test('PyPI: should retrieve Simple API package HTML (GET /simple/{package}/)', async () => {
@@ -140,7 +143,7 @@ tap.test('PyPI: should retrieve Simple API package HTML (GET /simple/{package}/)
});
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('text/html');
expect(response.headers['Content-Type']).toStartWith('text/html');
expect(response.body).toBeTypeOf('string');
const html = response.body as string;
@@ -210,6 +213,7 @@ tap.test('PyPI: should upload sdist file (POST /pypi/)', async () => {
pyversion: 'source',
metadata_version: '2.1',
sha256_digest: hashes.sha256,
requires_python: '>=3.7',
content: testSdistData,
filename: filename,
},
@@ -231,10 +235,11 @@ tap.test('PyPI: should list both wheel and sdist in Simple API', async () => {
expect(response.status).toEqual(200);
const json = response.body as any;
expect(Object.keys(json.files).length).toEqual(2);
// PEP 691: files is an array of file objects
expect(json.files.length).toEqual(2);
const hasWheel = Object.keys(json.files).some(f => f.endsWith('.whl'));
const hasSdist = Object.keys(json.files).some(f => f.endsWith('.tar.gz'));
const hasWheel = json.files.some((f: any) => f.filename.endsWith('.whl'));
const hasSdist = json.files.some((f: any) => f.filename.endsWith('.tar.gz'));
expect(hasWheel).toEqual(true);
expect(hasSdist).toEqual(true);
@@ -263,6 +268,7 @@ tap.test('PyPI: should upload a second version', async () => {
pyversion: 'py3',
metadata_version: '2.1',
sha256_digest: hashes.sha256,
requires_python: '>=3.7',
content: newWheelData,
filename: filename,
},
@@ -284,10 +290,11 @@ tap.test('PyPI: should list multiple versions in Simple API', async () => {
expect(response.status).toEqual(200);
const json = response.body as any;
expect(Object.keys(json.files).length).toBeGreaterThan(2);
// PEP 691: files is an array of file objects
expect(json.files.length).toBeGreaterThan(2);
const hasVersion1 = Object.keys(json.files).some(f => f.includes('1.0.0'));
const hasVersion2 = Object.keys(json.files).some(f => f.includes('2.0.0'));
const hasVersion1 = json.files.some((f: any) => f.filename.includes('1.0.0'));
const hasVersion2 = json.files.some((f: any) => f.filename.includes('2.0.0'));
expect(hasVersion1).toEqual(true);
expect(hasVersion2).toEqual(true);
@@ -420,7 +427,8 @@ tap.test('PyPI: should handle package with requires-python metadata', async () =
const html = getResponse.body as string;
expect(html).toContain('data-requires-python');
expect(html).toContain('>=3.8');
// Note: >= gets HTML-escaped to &gt;= in attribute values
expect(html).toContain('&gt;=3.8');
});
tap.test('PyPI: should support JSON API for package metadata', async () => {

View File

@@ -0,0 +1,448 @@
/**
* Native gem CLI Testing
* Tests the RubyGems registry implementation using the actual gem CLI
*/
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { tapNodeTools } from '@git.zone/tstest/tapbundle_serverside';
import { SmartRegistry } from '../ts/index.js';
import { createTestRegistry, createTestTokens, createRubyGem } from './helpers/registry.js';
import type { IRequestContext, IResponse } from '../ts/core/interfaces.core.js';
import * as http from 'http';
import * as url from 'url';
import * as fs from 'fs';
import * as path from 'path';
// Test context
let registry: SmartRegistry;
let server: http.Server;
let registryUrl: string;
let registryPort: number;
let rubygemsToken: string;
let testDir: string;
let gemHome: string;
/**
* Create HTTP server wrapper around SmartRegistry
*/
async function createHttpServer(
registryInstance: SmartRegistry,
port: number
): Promise<{ server: http.Server; url: string }> {
return new Promise((resolve, reject) => {
const httpServer = http.createServer(async (req, res) => {
try {
// Parse request
const parsedUrl = url.parse(req.url || '', true);
const pathname = parsedUrl.pathname || '/';
const query = parsedUrl.query;
// Read body
const chunks: Buffer[] = [];
for await (const chunk of req) {
chunks.push(chunk);
}
const bodyBuffer = Buffer.concat(chunks);
// Parse body based on content type
let body: any;
if (bodyBuffer.length > 0) {
const contentType = req.headers['content-type'] || '';
if (contentType.includes('application/json')) {
try {
body = JSON.parse(bodyBuffer.toString('utf-8'));
} catch (error) {
body = bodyBuffer;
}
} else {
body = bodyBuffer;
}
}
// Convert to IRequestContext
const context: IRequestContext = {
method: req.method || 'GET',
path: pathname,
headers: req.headers as Record<string, string>,
query: query as Record<string, string>,
body: body,
};
// Handle request
const response: IResponse = await registryInstance.handleRequest(context);
// Convert IResponse to HTTP response
res.statusCode = response.status;
// Set headers
for (const [key, value] of Object.entries(response.headers || {})) {
res.setHeader(key, value);
}
// Send body
if (response.body) {
if (Buffer.isBuffer(response.body)) {
res.end(response.body);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
} else {
res.end();
}
} catch (error) {
console.error('Server error:', error);
res.statusCode = 500;
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify({ error: 'INTERNAL_ERROR', message: String(error) }));
}
});
httpServer.listen(port, () => {
const serverUrl = `http://localhost:${port}`;
resolve({ server: httpServer, url: serverUrl });
});
httpServer.on('error', reject);
});
}
/**
* Setup gem credentials file
* Format: YAML with :rubygems_api_key: TOKEN
*/
function setupGemCredentials(token: string, gemHomeArg: string): string {
const gemDir = path.join(gemHomeArg, '.gem');
fs.mkdirSync(gemDir, { recursive: true });
// Create credentials file in YAML format
const credentialsContent = `:rubygems_api_key: ${token}\n`;
const credentialsPath = path.join(gemDir, 'credentials');
fs.writeFileSync(credentialsPath, credentialsContent, 'utf-8');
// Set restrictive permissions (gem requires 0600)
fs.chmodSync(credentialsPath, 0o600);
return credentialsPath;
}
/**
* Create a test gem file
*/
async function createTestGemFile(
gemName: string,
version: string,
targetDir: string
): Promise<string> {
const gemData = await createRubyGem(gemName, version);
const gemFilename = `${gemName}-${version}.gem`;
const gemPath = path.join(targetDir, gemFilename);
fs.writeFileSync(gemPath, gemData);
return gemPath;
}
/**
* Run gem command with proper environment
*/
async function runGemCommand(
command: string,
cwd: string,
includeAuth: boolean = true
): Promise<{ stdout: string; stderr: string; exitCode: number }> {
// Prepare environment variables
const envVars = [
`HOME="${gemHome}"`,
`GEM_HOME="${gemHome}"`,
includeAuth ? '' : 'RUBYGEMS_API_KEY=""',
].filter(Boolean).join(' ');
// Build command with cd to correct directory and environment variables
const fullCommand = `cd "${cwd}" && ${envVars} ${command}`;
try {
const result = await tapNodeTools.runCommand(fullCommand);
return {
stdout: result.stdout || '',
stderr: result.stderr || '',
exitCode: result.exitCode || 0,
};
} catch (error: any) {
return {
stdout: error.stdout || '',
stderr: error.stderr || String(error),
exitCode: error.exitCode || 1,
};
}
}
/**
* Cleanup test directory
*/
function cleanupTestDir(dir: string): void {
if (fs.existsSync(dir)) {
fs.rmSync(dir, { recursive: true, force: true });
}
}
// ========================================================================
// TESTS
// ========================================================================
tap.test('RubyGems CLI: should setup registry and HTTP server', async () => {
// Create registry
registry = await createTestRegistry();
const tokens = await createTestTokens(registry);
rubygemsToken = tokens.rubygemsToken;
expect(registry).toBeInstanceOf(SmartRegistry);
expect(rubygemsToken).toBeTypeOf('string');
// Use port 36000 (avoids npm:35000, cargo:5000 conflicts)
registryPort = 36000;
const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server;
registryUrl = serverSetup.url;
expect(server).toBeDefined();
expect(registryUrl).toEqual(`http://localhost:${registryPort}`);
// Setup test directory
testDir = path.join(process.cwd(), '.nogit', 'test-rubygems-cli');
cleanupTestDir(testDir);
fs.mkdirSync(testDir, { recursive: true });
// Setup GEM_HOME
gemHome = path.join(testDir, '.gem-home');
fs.mkdirSync(gemHome, { recursive: true });
// Setup gem credentials
const credentialsPath = setupGemCredentials(rubygemsToken, gemHome);
expect(fs.existsSync(credentialsPath)).toEqual(true);
// Verify credentials file has correct permissions
const stats = fs.statSync(credentialsPath);
const mode = stats.mode & 0o777;
expect(mode).toEqual(0o600);
});
tap.test('RubyGems CLI: should verify server is responding', async () => {
// Check server is up by doing a direct HTTP request to the Compact Index
const response = await fetch(`${registryUrl}/rubygems/versions`);
expect(response.status).toBeGreaterThanOrEqual(200);
expect(response.status).toBeLessThan(500);
});
tap.test('RubyGems CLI: should build and push a gem', async () => {
const gemName = 'test-gem-cli';
const version = '1.0.0';
const gemPath = await createTestGemFile(gemName, version, testDir);
expect(fs.existsSync(gemPath)).toEqual(true);
const result = await runGemCommand(
`gem push ${gemPath} --host ${registryUrl}/rubygems`,
testDir
);
console.log('gem push output:', result.stdout);
console.log('gem push stderr:', result.stderr);
expect(result.exitCode).toEqual(0);
expect(result.stdout || result.stderr).toContain(gemName);
});
tap.test('RubyGems CLI: should verify gem in Compact Index /versions', async () => {
const gemName = 'test-gem-cli';
const response = await fetch(`${registryUrl}/rubygems/versions`);
expect(response.status).toEqual(200);
const versionsData = await response.text();
console.log('Versions data:', versionsData);
// Format: GEMNAME VERSION[,VERSION...] MD5
expect(versionsData).toContain(gemName);
expect(versionsData).toContain('1.0.0');
});
tap.test('RubyGems CLI: should verify gem in Compact Index /info file', async () => {
const gemName = 'test-gem-cli';
const response = await fetch(`${registryUrl}/rubygems/info/${gemName}`);
expect(response.status).toEqual(200);
const infoData = await response.text();
console.log('Info data:', infoData);
// Format: VERSION [DEPS]|REQS
expect(infoData).toContain('1.0.0');
});
tap.test('RubyGems CLI: should download gem file', async () => {
const gemName = 'test-gem-cli';
const version = '1.0.0';
const response = await fetch(`${registryUrl}/rubygems/gems/${gemName}-${version}.gem`);
expect(response.status).toEqual(200);
const gemData = await response.arrayBuffer();
expect(gemData.byteLength).toBeGreaterThan(0);
// Verify content type
expect(response.headers.get('content-type')).toContain('application/octet-stream');
});
tap.test('RubyGems CLI: should fetch gem metadata JSON', async () => {
const gemName = 'test-gem-cli';
const response = await fetch(`${registryUrl}/rubygems/api/v1/versions/${gemName}.json`);
expect(response.status).toEqual(200);
const metadata = await response.json();
console.log('Metadata:', metadata);
expect(metadata).toBeInstanceOf(Array);
expect(metadata.length).toBeGreaterThan(0);
expect(metadata[0].number).toEqual('1.0.0');
});
tap.test('RubyGems CLI: should push second version', async () => {
const gemName = 'test-gem-cli';
const version = '2.0.0';
const gemPath = await createTestGemFile(gemName, version, testDir);
const result = await runGemCommand(
`gem push ${gemPath} --host ${registryUrl}/rubygems`,
testDir
);
console.log('gem push v2.0.0 output:', result.stdout);
expect(result.exitCode).toEqual(0);
});
tap.test('RubyGems CLI: should list all versions in /versions file', async () => {
const gemName = 'test-gem-cli';
const response = await fetch(`${registryUrl}/rubygems/versions`);
expect(response.status).toEqual(200);
const versionsData = await response.text();
console.log('All versions data:', versionsData);
// Should contain both versions
expect(versionsData).toContain(gemName);
expect(versionsData).toContain('1.0.0');
expect(versionsData).toContain('2.0.0');
});
tap.test('RubyGems CLI: should yank a version', async () => {
const gemName = 'test-gem-cli';
const version = '1.0.0';
const result = await runGemCommand(
`gem yank ${gemName} -v ${version} --host ${registryUrl}/rubygems`,
testDir
);
console.log('gem yank output:', result.stdout);
console.log('gem yank stderr:', result.stderr);
expect(result.exitCode).toEqual(0);
// Verify version is yanked in /versions file
// Yanked versions are prefixed with '-'
const response = await fetch(`${registryUrl}/rubygems/versions`);
const versionsData = await response.text();
console.log('Versions after yank:', versionsData);
// Yanked version should have '-' prefix
expect(versionsData).toContain('-1.0.0');
});
tap.test('RubyGems CLI: should unyank a version', async () => {
const gemName = 'test-gem-cli';
const version = '1.0.0';
const result = await runGemCommand(
`gem yank ${gemName} -v ${version} --undo --host ${registryUrl}/rubygems`,
testDir
);
console.log('gem unyank output:', result.stdout);
console.log('gem unyank stderr:', result.stderr);
expect(result.exitCode).toEqual(0);
// Verify version is not yanked in /versions file
const response = await fetch(`${registryUrl}/rubygems/versions`);
const versionsData = await response.text();
console.log('Versions after unyank:', versionsData);
// Should not have '-' prefix anymore (or have both without prefix)
// Check that we have the version without yank marker
const lines = versionsData.trim().split('\n');
const gemLine = lines.find(line => line.startsWith(gemName));
if (gemLine) {
// Parse format: "gemname version[,version...] md5"
const parts = gemLine.split(' ');
const versions = parts[1];
// Should have 1.0.0 without '-' prefix
expect(versions).toContain('1.0.0');
expect(versions).not.toContain('-1.0.0');
}
});
tap.test('RubyGems CLI: should fetch dependencies', async () => {
const gemName = 'test-gem-cli';
const response = await fetch(`${registryUrl}/rubygems/api/v1/dependencies?gems=${gemName}`);
expect(response.status).toEqual(200);
const dependencies = await response.json();
console.log('Dependencies:', dependencies);
expect(dependencies).toBeInstanceOf(Array);
});
tap.test('RubyGems CLI: should fail to push without auth', async () => {
const gemName = 'unauth-gem';
const version = '1.0.0';
const gemPath = await createTestGemFile(gemName, version, testDir);
// Run without auth
const result = await runGemCommand(
`gem push ${gemPath} --host ${registryUrl}/rubygems`,
testDir,
false
);
console.log('gem push unauth output:', result.stdout);
console.log('gem push unauth stderr:', result.stderr);
// Should fail with auth error
expect(result.exitCode).not.toEqual(0);
});
tap.postTask('cleanup rubygems cli tests', async () => {
// Stop server
if (server) {
await new Promise<void>((resolve) => {
server.close(() => resolve());
});
}
// Cleanup test directory
if (testDir) {
cleanupTestDir(testDir);
}
// Destroy registry
if (registry) {
registry.destroy();
}
});
export default tap.start();

View File

@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@push.rocks/smartregistry',
version: '1.7.0',
version: '2.4.0',
description: 'A composable TypeScript library implementing OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems registries for building unified container and package registries'
}

View File

@@ -3,6 +3,7 @@ import { BaseRegistry } from '../core/classes.baseregistry.js';
import { RegistryStorage } from '../core/classes.registrystorage.js';
import { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js';
import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js';
import type {
ICargoIndexEntry,
ICargoPublishMetadata,
@@ -13,6 +14,7 @@ import type {
ICargoSearchResponse,
ICargoSearchResult,
} from './interfaces.cargo.js';
import { CargoUpstream } from './classes.cargoupstream.js';
/**
* Cargo/crates.io registry implementation
@@ -25,12 +27,14 @@ export class CargoRegistry extends BaseRegistry {
private basePath: string = '/cargo';
private registryUrl: string;
private logger: Smartlog;
private upstream: CargoUpstream | null = null;
constructor(
storage: RegistryStorage,
authManager: AuthManager,
basePath: string = '/cargo',
registryUrl: string = 'http://localhost:5000/cargo'
registryUrl: string = 'http://localhost:5000/cargo',
upstreamConfig?: IProtocolUpstreamConfig
) {
super();
this.storage = storage;
@@ -50,6 +54,20 @@ export class CargoRegistry extends BaseRegistry {
}
});
this.logger.enableConsole();
// Initialize upstream if configured
if (upstreamConfig?.enabled) {
this.upstream = new CargoUpstream(upstreamConfig, undefined, this.logger);
}
}
/**
* Clean up resources (timers, connections, etc.)
*/
public destroy(): void {
if (this.upstream) {
this.upstream.stop();
}
}
public async init(): Promise<void> {
@@ -207,7 +225,25 @@ export class CargoRegistry extends BaseRegistry {
* Serve index file for a crate
*/
private async handleIndexFile(crateName: string): Promise<IResponse> {
const index = await this.storage.getCargoIndex(crateName);
let index = await this.storage.getCargoIndex(crateName);
// Try upstream if not found locally
if ((!index || index.length === 0) && this.upstream) {
const upstreamIndex = await this.upstream.fetchCrateIndex(crateName);
if (upstreamIndex) {
// Parse the newline-delimited JSON
const parsedIndex: ICargoIndexEntry[] = upstreamIndex
.split('\n')
.filter(line => line.trim())
.map(line => JSON.parse(line));
if (parsedIndex.length > 0) {
// Cache locally
await this.storage.putCargoIndex(crateName, parsedIndex);
index = parsedIndex;
}
}
}
if (!index || index.length === 0) {
return {
@@ -399,7 +435,16 @@ export class CargoRegistry extends BaseRegistry {
): Promise<IResponse> {
this.logger.log('debug', 'handleDownload', { crate: crateName, version });
const crateFile = await this.storage.getCargoCrate(crateName, version);
let crateFile = await this.storage.getCargoCrate(crateName, version);
// Try upstream if not found locally
if (!crateFile && this.upstream) {
crateFile = await this.upstream.fetchCrate(crateName, version);
if (crateFile) {
// Cache locally
await this.storage.putCargoCrate(crateName, version, crateFile);
}
}
if (!crateFile) {
return {

View File

@@ -0,0 +1,159 @@
import * as plugins from '../plugins.js';
import { BaseUpstream } from '../upstream/classes.baseupstream.js';
import type {
IProtocolUpstreamConfig,
IUpstreamFetchContext,
IUpstreamRegistryConfig,
} from '../upstream/interfaces.upstream.js';
/**
* Cargo-specific upstream implementation.
*
* Handles:
* - Crate metadata (index) fetching
* - Crate file (.crate) downloading
* - Sparse index protocol support
* - Content-addressable caching for .crate files
*/
export class CargoUpstream extends BaseUpstream {
protected readonly protocolName = 'cargo';
/** Base URL for crate downloads (may differ from index URL) */
private readonly downloadUrl: string;
constructor(
config: IProtocolUpstreamConfig,
downloadUrl?: string,
logger?: plugins.smartlog.Smartlog,
) {
super(config, logger);
// Default to crates.io download URL if not specified
this.downloadUrl = downloadUrl || 'https://static.crates.io/crates';
}
/**
* Fetch crate metadata from the sparse index.
*/
public async fetchCrateIndex(crateName: string): Promise<string | null> {
const path = this.buildIndexPath(crateName);
const context: IUpstreamFetchContext = {
protocol: 'cargo',
resource: crateName,
resourceType: 'index',
path,
method: 'GET',
headers: {
'accept': 'text/plain',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
if (Buffer.isBuffer(result.body)) {
return result.body.toString('utf8');
}
return typeof result.body === 'string' ? result.body : null;
}
/**
* Fetch a crate file from upstream.
*/
public async fetchCrate(crateName: string, version: string): Promise<Buffer | null> {
// Crate downloads typically go to a different URL than the index
const path = `/${crateName}/${crateName}-${version}.crate`;
const context: IUpstreamFetchContext = {
protocol: 'cargo',
resource: crateName,
resourceType: 'crate',
path,
method: 'GET',
headers: {
'accept': 'application/octet-stream',
},
query: {},
};
// Use special handling for crate downloads
const result = await this.fetchCrateFile(crateName, version);
return result;
}
/**
* Fetch crate file directly from the download URL.
*/
private async fetchCrateFile(crateName: string, version: string): Promise<Buffer | null> {
const context: IUpstreamFetchContext = {
protocol: 'cargo',
resource: crateName,
resourceType: 'crate',
path: `/${crateName}/${crateName}-${version}.crate`,
method: 'GET',
headers: {
'accept': 'application/octet-stream',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
return Buffer.isBuffer(result.body) ? result.body : Buffer.from(result.body);
}
/**
* Build the sparse index path for a crate.
*
* Path structure:
* - 1 char: /1/{name}
* - 2 chars: /2/{name}
* - 3 chars: /3/{first char}/{name}
* - 4+ chars: /{first 2}/{next 2}/{name}
*/
private buildIndexPath(crateName: string): string {
const lowerName = crateName.toLowerCase();
const len = lowerName.length;
if (len === 1) {
return `/1/${lowerName}`;
} else if (len === 2) {
return `/2/${lowerName}`;
} else if (len === 3) {
return `/3/${lowerName[0]}/${lowerName}`;
} else {
return `/${lowerName.slice(0, 2)}/${lowerName.slice(2, 4)}/${lowerName}`;
}
}
/**
* Override URL building for Cargo-specific handling.
*/
protected buildUpstreamUrl(
upstream: IUpstreamRegistryConfig,
context: IUpstreamFetchContext,
): string {
let baseUrl = upstream.url;
// For crate downloads, use the download URL
if (context.resourceType === 'crate') {
baseUrl = this.downloadUrl;
}
// Remove trailing slash
if (baseUrl.endsWith('/')) {
baseUrl = baseUrl.slice(0, -1);
}
return `${baseUrl}${context.path}`;
}
}

View File

@@ -3,4 +3,5 @@
*/
export { CargoRegistry } from './classes.cargoregistry.js';
export { CargoUpstream } from './classes.cargoupstream.js';
export * from './interfaces.cargo.js';

View File

@@ -11,8 +11,39 @@ import { PypiRegistry } from './pypi/classes.pypiregistry.js';
import { RubyGemsRegistry } from './rubygems/classes.rubygemsregistry.js';
/**
* Main registry orchestrator
* Routes requests to appropriate protocol handlers (OCI, NPM, Maven, Cargo, Composer, PyPI, or RubyGems)
* Main registry orchestrator.
* Routes requests to appropriate protocol handlers (OCI, NPM, Maven, Cargo, Composer, PyPI, or RubyGems).
*
* Supports pluggable authentication and storage hooks:
*
* @example
* ```typescript
* // Basic usage with default in-memory auth
* const registry = new SmartRegistry(config);
*
* // With custom auth provider (LDAP, OAuth, etc.)
* const registry = new SmartRegistry({
* ...config,
* authProvider: new LdapAuthProvider(ldapClient),
* });
*
* // With storage hooks for quota tracking
* const registry = new SmartRegistry({
* ...config,
* storageHooks: {
* beforePut: async (ctx) => {
* const quota = await getQuota(ctx.actor?.orgId);
* if (ctx.metadata?.size > quota) {
* return { allowed: false, reason: 'Quota exceeded' };
* }
* return { allowed: true };
* },
* afterPut: async (ctx) => {
* await auditLog('storage.put', ctx);
* }
* }
* });
* ```
*/
export class SmartRegistry {
private storage: RegistryStorage;
@@ -23,8 +54,12 @@ export class SmartRegistry {
constructor(config: IRegistryConfig) {
this.config = config;
this.storage = new RegistryStorage(config.storage);
this.authManager = new AuthManager(config.auth);
// Create storage with optional hooks
this.storage = new RegistryStorage(config.storage, config.storageHooks);
// Create auth manager with optional custom provider
this.authManager = new AuthManager(config.auth, config.authProvider);
}
/**
@@ -41,62 +76,108 @@ export class SmartRegistry {
// Initialize OCI registry if enabled
if (this.config.oci?.enabled) {
const ociBasePath = this.config.oci.basePath || '/oci';
const ociRegistry = new OciRegistry(this.storage, this.authManager, ociBasePath);
const ociBasePath = this.config.oci.basePath ?? '/oci';
const ociTokens = this.config.auth.ociTokens?.enabled ? {
realm: this.config.auth.ociTokens.realm,
service: this.config.auth.ociTokens.service,
} : undefined;
const ociRegistry = new OciRegistry(
this.storage,
this.authManager,
ociBasePath,
ociTokens,
this.config.oci.upstream
);
await ociRegistry.init();
this.registries.set('oci', ociRegistry);
}
// Initialize NPM registry if enabled
if (this.config.npm?.enabled) {
const npmBasePath = this.config.npm.basePath || '/npm';
const npmBasePath = this.config.npm.basePath ?? '/npm';
const registryUrl = `http://localhost:5000${npmBasePath}`; // TODO: Make configurable
const npmRegistry = new NpmRegistry(this.storage, this.authManager, npmBasePath, registryUrl);
const npmRegistry = new NpmRegistry(
this.storage,
this.authManager,
npmBasePath,
registryUrl,
this.config.npm.upstream
);
await npmRegistry.init();
this.registries.set('npm', npmRegistry);
}
// Initialize Maven registry if enabled
if (this.config.maven?.enabled) {
const mavenBasePath = this.config.maven.basePath || '/maven';
const mavenBasePath = this.config.maven.basePath ?? '/maven';
const registryUrl = `http://localhost:5000${mavenBasePath}`; // TODO: Make configurable
const mavenRegistry = new MavenRegistry(this.storage, this.authManager, mavenBasePath, registryUrl);
const mavenRegistry = new MavenRegistry(
this.storage,
this.authManager,
mavenBasePath,
registryUrl,
this.config.maven.upstream
);
await mavenRegistry.init();
this.registries.set('maven', mavenRegistry);
}
// Initialize Cargo registry if enabled
if (this.config.cargo?.enabled) {
const cargoBasePath = this.config.cargo.basePath || '/cargo';
const cargoBasePath = this.config.cargo.basePath ?? '/cargo';
const registryUrl = `http://localhost:5000${cargoBasePath}`; // TODO: Make configurable
const cargoRegistry = new CargoRegistry(this.storage, this.authManager, cargoBasePath, registryUrl);
const cargoRegistry = new CargoRegistry(
this.storage,
this.authManager,
cargoBasePath,
registryUrl,
this.config.cargo.upstream
);
await cargoRegistry.init();
this.registries.set('cargo', cargoRegistry);
}
// Initialize Composer registry if enabled
if (this.config.composer?.enabled) {
const composerBasePath = this.config.composer.basePath || '/composer';
const composerBasePath = this.config.composer.basePath ?? '/composer';
const registryUrl = `http://localhost:5000${composerBasePath}`; // TODO: Make configurable
const composerRegistry = new ComposerRegistry(this.storage, this.authManager, composerBasePath, registryUrl);
const composerRegistry = new ComposerRegistry(
this.storage,
this.authManager,
composerBasePath,
registryUrl,
this.config.composer.upstream
);
await composerRegistry.init();
this.registries.set('composer', composerRegistry);
}
// Initialize PyPI registry if enabled
if (this.config.pypi?.enabled) {
const pypiBasePath = this.config.pypi.basePath || '/pypi';
const pypiBasePath = this.config.pypi.basePath ?? '/pypi';
const registryUrl = `http://localhost:5000`; // TODO: Make configurable
const pypiRegistry = new PypiRegistry(this.storage, this.authManager, pypiBasePath, registryUrl);
const pypiRegistry = new PypiRegistry(
this.storage,
this.authManager,
pypiBasePath,
registryUrl,
this.config.pypi.upstream
);
await pypiRegistry.init();
this.registries.set('pypi', pypiRegistry);
}
// Initialize RubyGems registry if enabled
if (this.config.rubygems?.enabled) {
const rubygemsBasePath = this.config.rubygems.basePath || '/rubygems';
const rubygemsBasePath = this.config.rubygems.basePath ?? '/rubygems';
const registryUrl = `http://localhost:5000${rubygemsBasePath}`; // TODO: Make configurable
const rubygemsRegistry = new RubyGemsRegistry(this.storage, this.authManager, rubygemsBasePath, registryUrl);
const rubygemsRegistry = new RubyGemsRegistry(
this.storage,
this.authManager,
rubygemsBasePath,
registryUrl,
this.config.rubygems.upstream
);
await rubygemsRegistry.init();
this.registries.set('rubygems', rubygemsRegistry);
}
@@ -153,7 +234,7 @@ export class SmartRegistry {
// Route to PyPI registry (also handles /simple prefix)
if (this.config.pypi?.enabled) {
const pypiBasePath = this.config.pypi.basePath || '/pypi';
const pypiBasePath = this.config.pypi.basePath ?? '/pypi';
if (path.startsWith(pypiBasePath) || path.startsWith('/simple')) {
const pypiRegistry = this.registries.get('pypi');
if (pypiRegistry) {

View File

@@ -7,6 +7,8 @@ import { BaseRegistry } from '../core/classes.baseregistry.js';
import type { RegistryStorage } from '../core/classes.registrystorage.js';
import type { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js';
import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js';
import { isBinaryData, toBuffer } from '../core/helpers.buffer.js';
import type {
IComposerPackage,
IComposerPackageMetadata,
@@ -21,24 +23,41 @@ import {
generatePackagesJson,
sortVersions,
} from './helpers.composer.js';
import { ComposerUpstream } from './classes.composerupstream.js';
export class ComposerRegistry extends BaseRegistry {
private storage: RegistryStorage;
private authManager: AuthManager;
private basePath: string = '/composer';
private registryUrl: string;
private upstream: ComposerUpstream | null = null;
constructor(
storage: RegistryStorage,
authManager: AuthManager,
basePath: string = '/composer',
registryUrl: string = 'http://localhost:5000/composer'
registryUrl: string = 'http://localhost:5000/composer',
upstreamConfig?: IProtocolUpstreamConfig
) {
super();
this.storage = storage;
this.authManager = authManager;
this.basePath = basePath;
this.registryUrl = registryUrl;
// Initialize upstream if configured
if (upstreamConfig?.enabled) {
this.upstream = new ComposerUpstream(upstreamConfig);
}
}
/**
* Clean up resources (timers, connections, etc.)
*/
public destroy(): void {
if (this.upstream) {
this.upstream.stop();
}
}
public async init(): Promise<void> {
@@ -160,7 +179,26 @@ export class ComposerRegistry extends BaseRegistry {
token: IAuthToken | null
): Promise<IResponse> {
// Read operations are public, no authentication required
const metadata = await this.storage.getComposerPackageMetadata(vendorPackage);
let metadata = await this.storage.getComposerPackageMetadata(vendorPackage);
// Try upstream if not found locally
if (!metadata && this.upstream) {
const [vendor, packageName] = vendorPackage.split('/');
if (vendor && packageName) {
const upstreamMetadata = includeDev
? await this.upstream.fetchPackageDevMetadata(vendor, packageName)
: await this.upstream.fetchPackageMetadata(vendor, packageName);
if (upstreamMetadata && upstreamMetadata.packages) {
// Store upstream metadata locally
metadata = {
packages: upstreamMetadata.packages,
lastModified: new Date().toUTCString(),
};
await this.storage.putComposerPackageMetadata(vendorPackage, metadata);
}
}
}
if (!metadata) {
return {
@@ -255,7 +293,7 @@ export class ComposerRegistry extends BaseRegistry {
};
}
if (!body || !Buffer.isBuffer(body)) {
if (!body || !isBinaryData(body)) {
return {
status: 400,
headers: {},
@@ -263,8 +301,11 @@ export class ComposerRegistry extends BaseRegistry {
};
}
// Convert to Buffer for ZIP processing
const zipData = toBuffer(body);
// Extract and validate composer.json from ZIP
const composerJson = await extractComposerJsonFromZip(body);
const composerJson = await extractComposerJsonFromZip(zipData);
if (!composerJson || !validateComposerJson(composerJson)) {
return {
status: 400,
@@ -292,13 +333,13 @@ export class ComposerRegistry extends BaseRegistry {
}
// Calculate SHA-1 hash
const shasum = await calculateSha1(body);
const shasum = await calculateSha1(zipData);
// Generate reference (use version or commit hash)
const reference = composerJson.source?.reference || version.replace(/[^a-zA-Z0-9.-]/g, '-');
// Store ZIP file
await this.storage.putComposerPackageZip(vendorPackage, reference, body);
await this.storage.putComposerPackageZip(vendorPackage, reference, zipData);
// Get or create metadata
let metadata = await this.storage.getComposerPackageMetadata(vendorPackage);

View File

@@ -0,0 +1,200 @@
import * as plugins from '../plugins.js';
import { BaseUpstream } from '../upstream/classes.baseupstream.js';
import type {
IProtocolUpstreamConfig,
IUpstreamFetchContext,
IUpstreamRegistryConfig,
} from '../upstream/interfaces.upstream.js';
/**
* Composer-specific upstream implementation.
*
* Handles:
* - Package metadata fetching (packages.json, provider-includes)
* - Package version metadata (p2/{vendor}/{package}.json)
* - Dist file (zip) proxying
* - Packagist v2 API support
*/
export class ComposerUpstream extends BaseUpstream {
protected readonly protocolName = 'composer';
constructor(
config: IProtocolUpstreamConfig,
logger?: plugins.smartlog.Smartlog,
) {
super(config, logger);
}
/**
* Fetch the root packages.json from upstream.
*/
public async fetchPackagesJson(): Promise<any | null> {
const context: IUpstreamFetchContext = {
protocol: 'composer',
resource: '*',
resourceType: 'root',
path: '/packages.json',
method: 'GET',
headers: {
'accept': 'application/json',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
if (Buffer.isBuffer(result.body)) {
return JSON.parse(result.body.toString('utf8'));
}
return result.body;
}
/**
* Fetch package metadata using v2 API (p2/{vendor}/{package}.json).
*/
public async fetchPackageMetadata(vendor: string, packageName: string): Promise<any | null> {
const fullName = `${vendor}/${packageName}`;
const path = `/p2/${vendor}/${packageName}.json`;
const context: IUpstreamFetchContext = {
protocol: 'composer',
resource: fullName,
resourceType: 'metadata',
path,
method: 'GET',
headers: {
'accept': 'application/json',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
if (Buffer.isBuffer(result.body)) {
return JSON.parse(result.body.toString('utf8'));
}
return result.body;
}
/**
* Fetch package metadata with dev versions (p2/{vendor}/{package}~dev.json).
*/
public async fetchPackageDevMetadata(vendor: string, packageName: string): Promise<any | null> {
const fullName = `${vendor}/${packageName}`;
const path = `/p2/${vendor}/${packageName}~dev.json`;
const context: IUpstreamFetchContext = {
protocol: 'composer',
resource: fullName,
resourceType: 'metadata-dev',
path,
method: 'GET',
headers: {
'accept': 'application/json',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
if (Buffer.isBuffer(result.body)) {
return JSON.parse(result.body.toString('utf8'));
}
return result.body;
}
/**
* Fetch a provider-includes file.
*/
public async fetchProviderIncludes(path: string): Promise<any | null> {
const context: IUpstreamFetchContext = {
protocol: 'composer',
resource: '*',
resourceType: 'provider',
path: path.startsWith('/') ? path : `/${path}`,
method: 'GET',
headers: {
'accept': 'application/json',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
if (Buffer.isBuffer(result.body)) {
return JSON.parse(result.body.toString('utf8'));
}
return result.body;
}
/**
* Fetch a dist file (zip) from upstream.
*/
public async fetchDist(url: string): Promise<Buffer | null> {
// Parse the URL to get the path
let path: string;
try {
const parsed = new URL(url);
path = parsed.pathname;
} catch {
path = url;
}
const context: IUpstreamFetchContext = {
protocol: 'composer',
resource: '*',
resourceType: 'dist',
path,
method: 'GET',
headers: {
'accept': 'application/zip, application/octet-stream',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
return Buffer.isBuffer(result.body) ? result.body : Buffer.from(result.body);
}
/**
* Override URL building for Composer-specific handling.
*/
protected buildUpstreamUrl(
upstream: IUpstreamRegistryConfig,
context: IUpstreamFetchContext,
): string {
let baseUrl = upstream.url;
// Remove trailing slash
if (baseUrl.endsWith('/')) {
baseUrl = baseUrl.slice(0, -1);
}
return `${baseUrl}${context.path}`;
}
}

View File

@@ -4,5 +4,6 @@
*/
export { ComposerRegistry } from './classes.composerregistry.js';
export { ComposerUpstream } from './classes.composerupstream.js';
export * from './interfaces.composer.js';
export * from './helpers.composer.js';

View File

@@ -1,54 +1,79 @@
import type { IAuthConfig, IAuthToken, ICredentials, TRegistryProtocol } from './interfaces.core.js';
import type { IAuthProvider, ITokenOptions } from './interfaces.auth.js';
import { DefaultAuthProvider } from './classes.defaultauthprovider.js';
/**
* Unified authentication manager for all registry protocols
* Handles both NPM UUID tokens and OCI JWT tokens
* Unified authentication manager for all registry protocols.
* Delegates to a pluggable IAuthProvider for actual auth operations.
*
* @example
* ```typescript
* // Use default in-memory provider
* const auth = new AuthManager(config);
*
* // Use custom provider (LDAP, OAuth, etc.)
* const auth = new AuthManager(config, new LdapAuthProvider(ldapClient));
* ```
*/
export class AuthManager {
private tokenStore: Map<string, IAuthToken> = new Map();
private userCredentials: Map<string, string> = new Map(); // username -> password hash (mock)
private provider: IAuthProvider;
constructor(private config: IAuthConfig) {}
constructor(
private config: IAuthConfig,
provider?: IAuthProvider
) {
// Use provided provider or default in-memory implementation
this.provider = provider || new DefaultAuthProvider(config);
}
/**
* Initialize the auth manager
*/
public async init(): Promise<void> {
// Initialize token store (in-memory for now)
// In production, this could be Redis or a database
if (this.provider.init) {
await this.provider.init();
}
}
// ========================================================================
// UUID TOKEN CREATION (Base method for NPM, Maven, etc.)
// UNIFIED AUTHENTICATION (Delegated to Provider)
// ========================================================================
/**
* Create a UUID-based token with custom scopes (base method)
* @param userId - User ID
* @param protocol - Protocol type
* @param scopes - Permission scopes
* @param readonly - Whether the token is readonly
* @returns UUID token string
* Authenticate user credentials
* @param credentials - Username and password
* @returns User ID or null
*/
private async createUuidToken(
userId: string,
protocol: TRegistryProtocol,
scopes: string[],
readonly: boolean = false
): Promise<string> {
const token = this.generateUuid();
const authToken: IAuthToken = {
type: protocol,
userId,
scopes,
readonly,
metadata: {
created: new Date().toISOString(),
},
};
public async authenticate(credentials: ICredentials): Promise<string | null> {
return this.provider.authenticate(credentials);
}
this.tokenStore.set(token, authToken);
return token;
/**
* Validate any token (NPM, Maven, OCI, PyPI, RubyGems, Composer, Cargo)
* @param tokenString - Token string (UUID or JWT)
* @param protocol - Expected protocol type (optional, improves performance)
* @returns Auth token object or null
*/
public async validateToken(
tokenString: string,
protocol?: TRegistryProtocol
): Promise<IAuthToken | null> {
return this.provider.validateToken(tokenString, protocol);
}
/**
* Check if token has permission for an action
* @param token - Auth token (or null for anonymous)
* @param resource - Resource being accessed (e.g., "npm:package:foo")
* @param action - Action being performed (read, write, push, pull, delete)
* @returns true if authorized
*/
public async authorize(
token: IAuthToken | null,
resource: string,
action: string
): Promise<boolean> {
return this.provider.authorize(token, resource, action);
}
// ========================================================================
@@ -65,9 +90,7 @@ export class AuthManager {
if (!this.config.npmTokens.enabled) {
throw new Error('NPM tokens are not enabled');
}
const scopes = readonly ? ['npm:*:*:read'] : ['npm:*:*:*'];
return this.createUuidToken(userId, 'npm', scopes, readonly);
return this.provider.createToken(userId, 'npm', { readonly });
}
/**
@@ -76,22 +99,7 @@ export class AuthManager {
* @returns Auth token object or null
*/
public async validateNpmToken(token: string): Promise<IAuthToken | null> {
if (!this.isValidUuid(token)) {
return null;
}
const authToken = this.tokenStore.get(token);
if (!authToken || authToken.type !== 'npm') {
return null;
}
// Check expiration if set
if (authToken.expiresAt && authToken.expiresAt < new Date()) {
this.tokenStore.delete(token);
return null;
}
return authToken;
return this.provider.validateToken(token, 'npm');
}
/**
@@ -99,7 +107,7 @@ export class AuthManager {
* @param token - NPM UUID token
*/
public async revokeNpmToken(token: string): Promise<void> {
this.tokenStore.delete(token);
return this.provider.revokeToken(token);
}
/**
@@ -111,20 +119,12 @@ export class AuthManager {
key: string;
readonly: boolean;
created: string;
protocol?: TRegistryProtocol;
}>> {
const tokens: Array<{key: string; readonly: boolean; created: string}> = [];
for (const [token, authToken] of this.tokenStore.entries()) {
if (authToken.userId === userId) {
tokens.push({
key: this.hashToken(token),
readonly: authToken.readonly || false,
created: authToken.metadata?.created || 'unknown',
});
if (this.provider.listUserTokens) {
return this.provider.listUserTokens(userId);
}
}
return tokens;
return [];
}
// ========================================================================
@@ -143,24 +143,10 @@ export class AuthManager {
scopes: string[],
expiresIn: number = 3600
): Promise<string> {
if (!this.config.ociTokens.enabled) {
if (!this.config.ociTokens?.enabled) {
throw new Error('OCI tokens are not enabled');
}
const now = Math.floor(Date.now() / 1000);
const payload = {
iss: this.config.ociTokens.realm,
sub: userId,
aud: this.config.ociTokens.service,
exp: now + expiresIn,
nbf: now,
iat: now,
access: this.scopesToOciAccess(scopes),
};
// In production, use proper JWT library with signing
// For now, return JSON string (mock JWT)
return JSON.stringify(payload);
return this.provider.createToken(userId, 'oci', { scopes, expiresIn });
}
/**
@@ -169,58 +155,7 @@ export class AuthManager {
* @returns Auth token object or null
*/
public async validateOciToken(jwt: string): Promise<IAuthToken | null> {
try {
// In production, verify JWT signature
const payload = JSON.parse(jwt);
// Check expiration
const now = Math.floor(Date.now() / 1000);
if (payload.exp && payload.exp < now) {
return null;
}
// Convert to unified token format
const scopes = this.ociAccessToScopes(payload.access || []);
return {
type: 'oci',
userId: payload.sub,
scopes,
expiresAt: payload.exp ? new Date(payload.exp * 1000) : undefined,
metadata: {
iss: payload.iss,
aud: payload.aud,
},
};
} catch (error) {
return null;
}
}
// ========================================================================
// UNIFIED AUTHENTICATION
// ========================================================================
/**
* Authenticate user credentials
* @param credentials - Username and password
* @returns User ID or null
*/
public async authenticate(credentials: ICredentials): Promise<string | null> {
// Mock authentication - in production, verify against database
const storedPassword = this.userCredentials.get(credentials.username);
if (!storedPassword) {
// Auto-register for testing (remove in production)
this.userCredentials.set(credentials.username, credentials.password);
return credentials.username;
}
if (storedPassword === credentials.password) {
return credentials.username;
}
return null;
return this.provider.validateToken(jwt, 'oci');
}
// ========================================================================
@@ -234,8 +169,7 @@ export class AuthManager {
* @returns Maven UUID token
*/
public async createMavenToken(userId: string, readonly: boolean = false): Promise<string> {
const scopes = readonly ? ['maven:*:*:read'] : ['maven:*:*:*'];
return this.createUuidToken(userId, 'maven', scopes, readonly);
return this.provider.createToken(userId, 'maven', { readonly });
}
/**
@@ -244,22 +178,7 @@ export class AuthManager {
* @returns Auth token object or null
*/
public async validateMavenToken(token: string): Promise<IAuthToken | null> {
if (!this.isValidUuid(token)) {
return null;
}
const authToken = this.tokenStore.get(token);
if (!authToken || authToken.type !== 'maven') {
return null;
}
// Check expiration if set
if (authToken.expiresAt && authToken.expiresAt < new Date()) {
this.tokenStore.delete(token);
return null;
}
return authToken;
return this.provider.validateToken(token, 'maven');
}
/**
@@ -267,7 +186,7 @@ export class AuthManager {
* @param token - Maven UUID token
*/
public async revokeMavenToken(token: string): Promise<void> {
this.tokenStore.delete(token);
return this.provider.revokeToken(token);
}
// ========================================================================
@@ -281,8 +200,7 @@ export class AuthManager {
* @returns Composer UUID token
*/
public async createComposerToken(userId: string, readonly: boolean = false): Promise<string> {
const scopes = readonly ? ['composer:*:*:read'] : ['composer:*:*:*'];
return this.createUuidToken(userId, 'composer', scopes, readonly);
return this.provider.createToken(userId, 'composer', { readonly });
}
/**
@@ -291,22 +209,7 @@ export class AuthManager {
* @returns Auth token object or null
*/
public async validateComposerToken(token: string): Promise<IAuthToken | null> {
if (!this.isValidUuid(token)) {
return null;
}
const authToken = this.tokenStore.get(token);
if (!authToken || authToken.type !== 'composer') {
return null;
}
// Check expiration if set
if (authToken.expiresAt && authToken.expiresAt < new Date()) {
this.tokenStore.delete(token);
return null;
}
return authToken;
return this.provider.validateToken(token, 'composer');
}
/**
@@ -314,7 +217,7 @@ export class AuthManager {
* @param token - Composer UUID token
*/
public async revokeComposerToken(token: string): Promise<void> {
this.tokenStore.delete(token);
return this.provider.revokeToken(token);
}
// ========================================================================
@@ -328,8 +231,7 @@ export class AuthManager {
* @returns Cargo UUID token
*/
public async createCargoToken(userId: string, readonly: boolean = false): Promise<string> {
const scopes = readonly ? ['cargo:*:*:read'] : ['cargo:*:*:*'];
return this.createUuidToken(userId, 'cargo', scopes, readonly);
return this.provider.createToken(userId, 'cargo', { readonly });
}
/**
@@ -338,22 +240,7 @@ export class AuthManager {
* @returns Auth token object or null
*/
public async validateCargoToken(token: string): Promise<IAuthToken | null> {
if (!this.isValidUuid(token)) {
return null;
}
const authToken = this.tokenStore.get(token);
if (!authToken || authToken.type !== 'cargo') {
return null;
}
// Check expiration if set
if (authToken.expiresAt && authToken.expiresAt < new Date()) {
this.tokenStore.delete(token);
return null;
}
return authToken;
return this.provider.validateToken(token, 'cargo');
}
/**
@@ -361,7 +248,7 @@ export class AuthManager {
* @param token - Cargo UUID token
*/
public async revokeCargoToken(token: string): Promise<void> {
this.tokenStore.delete(token);
return this.provider.revokeToken(token);
}
// ========================================================================
@@ -375,8 +262,7 @@ export class AuthManager {
* @returns PyPI UUID token
*/
public async createPypiToken(userId: string, readonly: boolean = false): Promise<string> {
const scopes = readonly ? ['pypi:*:*:read'] : ['pypi:*:*:*'];
return this.createUuidToken(userId, 'pypi', scopes, readonly);
return this.provider.createToken(userId, 'pypi', { readonly });
}
/**
@@ -385,22 +271,7 @@ export class AuthManager {
* @returns Auth token object or null
*/
public async validatePypiToken(token: string): Promise<IAuthToken | null> {
if (!this.isValidUuid(token)) {
return null;
}
const authToken = this.tokenStore.get(token);
if (!authToken || authToken.type !== 'pypi') {
return null;
}
// Check expiration if set
if (authToken.expiresAt && authToken.expiresAt < new Date()) {
this.tokenStore.delete(token);
return null;
}
return authToken;
return this.provider.validateToken(token, 'pypi');
}
/**
@@ -408,7 +279,7 @@ export class AuthManager {
* @param token - PyPI UUID token
*/
public async revokePypiToken(token: string): Promise<void> {
this.tokenStore.delete(token);
return this.provider.revokeToken(token);
}
// ========================================================================
@@ -422,8 +293,7 @@ export class AuthManager {
* @returns RubyGems UUID token
*/
public async createRubyGemsToken(userId: string, readonly: boolean = false): Promise<string> {
const scopes = readonly ? ['rubygems:*:*:read'] : ['rubygems:*:*:*'];
return this.createUuidToken(userId, 'rubygems', scopes, readonly);
return this.provider.createToken(userId, 'rubygems', { readonly });
}
/**
@@ -432,22 +302,7 @@ export class AuthManager {
* @returns Auth token object or null
*/
public async validateRubyGemsToken(token: string): Promise<IAuthToken | null> {
if (!this.isValidUuid(token)) {
return null;
}
const authToken = this.tokenStore.get(token);
if (!authToken || authToken.type !== 'rubygems') {
return null;
}
// Check expiration if set
if (authToken.expiresAt && authToken.expiresAt < new Date()) {
this.tokenStore.delete(token);
return null;
}
return authToken;
return this.provider.validateToken(token, 'rubygems');
}
/**
@@ -455,226 +310,6 @@ export class AuthManager {
* @param token - RubyGems UUID token
*/
public async revokeRubyGemsToken(token: string): Promise<void> {
this.tokenStore.delete(token);
}
// ========================================================================
// UNIFIED AUTHENTICATION
// ========================================================================
/**
* Validate any token (NPM, Maven, OCI, PyPI, RubyGems, Composer, Cargo)
* @param tokenString - Token string (UUID or JWT)
* @param protocol - Expected protocol type
* @returns Auth token object or null
*/
public async validateToken(
tokenString: string,
protocol?: TRegistryProtocol
): Promise<IAuthToken | null> {
// Try UUID-based tokens (NPM, Maven, Composer, Cargo, PyPI, RubyGems)
if (this.isValidUuid(tokenString)) {
// Try NPM token
const npmToken = await this.validateNpmToken(tokenString);
if (npmToken && (!protocol || protocol === 'npm')) {
return npmToken;
}
// Try Maven token
const mavenToken = await this.validateMavenToken(tokenString);
if (mavenToken && (!protocol || protocol === 'maven')) {
return mavenToken;
}
// Try Composer token
const composerToken = await this.validateComposerToken(tokenString);
if (composerToken && (!protocol || protocol === 'composer')) {
return composerToken;
}
// Try Cargo token
const cargoToken = await this.validateCargoToken(tokenString);
if (cargoToken && (!protocol || protocol === 'cargo')) {
return cargoToken;
}
// Try PyPI token
const pypiToken = await this.validatePypiToken(tokenString);
if (pypiToken && (!protocol || protocol === 'pypi')) {
return pypiToken;
}
// Try RubyGems token
const rubygemsToken = await this.validateRubyGemsToken(tokenString);
if (rubygemsToken && (!protocol || protocol === 'rubygems')) {
return rubygemsToken;
}
}
// Try OCI JWT
const ociToken = await this.validateOciToken(tokenString);
if (ociToken && (!protocol || protocol === 'oci')) {
return ociToken;
}
return null;
}
/**
* Check if token has permission for an action
* @param token - Auth token
* @param resource - Resource being accessed (e.g., "package:foo" or "repository:bar")
* @param action - Action being performed (read, write, push, pull, delete)
* @returns true if authorized
*/
public async authorize(
token: IAuthToken | null,
resource: string,
action: string
): Promise<boolean> {
if (!token) {
return false;
}
// Check readonly flag
if (token.readonly && ['write', 'push', 'delete'].includes(action)) {
return false;
}
// Check scopes
for (const scope of token.scopes) {
if (this.matchesScope(scope, resource, action)) {
return true;
}
}
return false;
}
// ========================================================================
// HELPER METHODS
// ========================================================================
/**
* Check if a scope matches a resource and action
* Scope format: "{protocol}:{type}:{name}:{action}"
* Examples:
* - "npm:*:*" - All NPM access
* - "npm:package:foo:*" - All actions on package foo
* - "npm:package:foo:read" - Read-only on package foo
* - "oci:repository:*:pull" - Pull from any OCI repo
*/
private matchesScope(scope: string, resource: string, action: string): boolean {
const scopeParts = scope.split(':');
const resourceParts = resource.split(':');
// Scope must have at least protocol:type:name:action
if (scopeParts.length < 4) {
return false;
}
const [scopeProtocol, scopeType, scopeName, scopeAction] = scopeParts;
const [resourceProtocol, resourceType, resourceName] = resourceParts;
// Check protocol
if (scopeProtocol !== '*' && scopeProtocol !== resourceProtocol) {
return false;
}
// Check type
if (scopeType !== '*' && scopeType !== resourceType) {
return false;
}
// Check name
if (scopeName !== '*' && scopeName !== resourceName) {
return false;
}
// Check action
if (scopeAction !== '*' && scopeAction !== action) {
// Map action aliases
const actionAliases: Record<string, string[]> = {
read: ['pull', 'get'],
write: ['push', 'put', 'post'],
};
const aliases = actionAliases[scopeAction] || [];
if (!aliases.includes(action)) {
return false;
}
}
return true;
}
/**
* Convert unified scopes to OCI access array
*/
private scopesToOciAccess(scopes: string[]): Array<{
type: string;
name: string;
actions: string[];
}> {
const access: Array<{type: string; name: string; actions: string[]}> = [];
for (const scope of scopes) {
const parts = scope.split(':');
if (parts.length >= 4 && parts[0] === 'oci') {
access.push({
type: parts[1],
name: parts[2],
actions: [parts[3]],
});
}
}
return access;
}
/**
* Convert OCI access array to unified scopes
*/
private ociAccessToScopes(access: Array<{
type: string;
name: string;
actions: string[];
}>): string[] {
const scopes: string[] = [];
for (const item of access) {
for (const action of item.actions) {
scopes.push(`oci:${item.type}:${item.name}:${action}`);
}
}
return scopes;
}
/**
* Generate UUID for NPM tokens
*/
private generateUuid(): string {
return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => {
const r = (Math.random() * 16) | 0;
const v = c === 'x' ? r : (r & 0x3) | 0x8;
return v.toString(16);
});
}
/**
* Check if string is a valid UUID
*/
private isValidUuid(str: string): boolean {
const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
return uuidRegex.test(str);
}
/**
* Hash a token for identification (SHA-512 mock)
*/
private hashToken(token: string): string {
// In production, use actual SHA-512
return `sha512-${token.substring(0, 16)}...`;
return this.provider.revokeToken(token);
}
}

View File

@@ -0,0 +1,393 @@
import * as crypto from 'crypto';
import type { IAuthProvider, ITokenOptions } from './interfaces.auth.js';
import type { IAuthConfig, IAuthToken, ICredentials, TRegistryProtocol } from './interfaces.core.js';
/**
* Default in-memory authentication provider.
* This is the reference implementation that stores tokens in memory.
* For production use, implement IAuthProvider with Redis, database, or external auth.
*/
export class DefaultAuthProvider implements IAuthProvider {
private tokenStore: Map<string, IAuthToken> = new Map();
private userCredentials: Map<string, string> = new Map(); // username -> password hash (mock)
constructor(private config: IAuthConfig) {}
/**
* Initialize the auth provider
*/
public async init(): Promise<void> {
// Initialize token store (in-memory for now)
// In production, this could be Redis or a database
}
// ========================================================================
// IAuthProvider Implementation
// ========================================================================
/**
* Authenticate user credentials
*/
public async authenticate(credentials: ICredentials): Promise<string | null> {
// Mock authentication - in production, verify against database/LDAP
const storedPassword = this.userCredentials.get(credentials.username);
if (!storedPassword) {
// Auto-register for testing (remove in production)
this.userCredentials.set(credentials.username, credentials.password);
return credentials.username;
}
if (storedPassword === credentials.password) {
return credentials.username;
}
return null;
}
/**
* Validate any token (NPM, Maven, OCI, PyPI, RubyGems, Composer, Cargo)
*/
public async validateToken(
tokenString: string,
protocol?: TRegistryProtocol
): Promise<IAuthToken | null> {
// OCI uses JWT (contains dots), not UUID - check first if OCI is expected
if (protocol === 'oci' || tokenString.includes('.')) {
const ociToken = await this.validateOciToken(tokenString);
if (ociToken && (!protocol || protocol === 'oci')) {
return ociToken;
}
// If protocol was explicitly OCI but validation failed, return null
if (protocol === 'oci') {
return null;
}
}
// UUID-based tokens: single O(1) Map lookup
if (this.isValidUuid(tokenString)) {
const authToken = this.tokenStore.get(tokenString);
if (authToken) {
// If protocol specified, verify it matches
if (protocol && authToken.type !== protocol) {
return null;
}
// Check expiration
if (authToken.expiresAt && authToken.expiresAt < new Date()) {
this.tokenStore.delete(tokenString);
return null;
}
return authToken;
}
}
return null;
}
/**
* Create a new token for a user
*/
public async createToken(
userId: string,
protocol: TRegistryProtocol,
options?: ITokenOptions
): Promise<string> {
// OCI tokens use JWT
if (protocol === 'oci') {
return this.createOciToken(userId, options?.scopes || ['oci:*:*:*'], options?.expiresIn || 3600);
}
// All other protocols use UUID tokens
const token = this.generateUuid();
const scopes = options?.scopes || (options?.readonly
? [`${protocol}:*:*:read`]
: [`${protocol}:*:*:*`]);
const authToken: IAuthToken = {
type: protocol,
userId,
scopes,
readonly: options?.readonly,
expiresAt: options?.expiresIn ? new Date(Date.now() + options.expiresIn * 1000) : undefined,
metadata: {
created: new Date().toISOString(),
},
};
this.tokenStore.set(token, authToken);
return token;
}
/**
* Revoke a token
*/
public async revokeToken(token: string): Promise<void> {
this.tokenStore.delete(token);
}
/**
* Check if token has permission for an action
*/
public async authorize(
token: IAuthToken | null,
resource: string,
action: string
): Promise<boolean> {
if (!token) {
return false;
}
// Check readonly flag
if (token.readonly && ['write', 'push', 'delete'].includes(action)) {
return false;
}
// Check scopes
for (const scope of token.scopes) {
if (this.matchesScope(scope, resource, action)) {
return true;
}
}
return false;
}
/**
* List all tokens for a user
*/
public async listUserTokens(userId: string): Promise<Array<{
key: string;
readonly: boolean;
created: string;
protocol?: TRegistryProtocol;
}>> {
const tokens: Array<{key: string; readonly: boolean; created: string; protocol?: TRegistryProtocol}> = [];
for (const [token, authToken] of this.tokenStore.entries()) {
if (authToken.userId === userId) {
tokens.push({
key: this.hashToken(token),
readonly: authToken.readonly || false,
created: authToken.metadata?.created || 'unknown',
protocol: authToken.type,
});
}
}
return tokens;
}
// ========================================================================
// OCI JWT Token Methods
// ========================================================================
/**
* Create an OCI JWT token
*/
private async createOciToken(
userId: string,
scopes: string[],
expiresIn: number = 3600
): Promise<string> {
if (!this.config.ociTokens?.enabled) {
throw new Error('OCI tokens are not enabled');
}
const now = Math.floor(Date.now() / 1000);
const payload = {
iss: this.config.ociTokens.realm,
sub: userId,
aud: this.config.ociTokens.service,
exp: now + expiresIn,
nbf: now,
iat: now,
access: this.scopesToOciAccess(scopes),
};
// Create JWT with HMAC-SHA256 signature
const header = { alg: 'HS256', typ: 'JWT' };
const headerB64 = Buffer.from(JSON.stringify(header)).toString('base64url');
const payloadB64 = Buffer.from(JSON.stringify(payload)).toString('base64url');
const signature = crypto
.createHmac('sha256', this.config.jwtSecret)
.update(`${headerB64}.${payloadB64}`)
.digest('base64url');
return `${headerB64}.${payloadB64}.${signature}`;
}
/**
* Validate an OCI JWT token
*/
private async validateOciToken(jwt: string): Promise<IAuthToken | null> {
try {
const parts = jwt.split('.');
if (parts.length !== 3) {
return null;
}
const [headerB64, payloadB64, signatureB64] = parts;
// Verify signature
const expectedSignature = crypto
.createHmac('sha256', this.config.jwtSecret)
.update(`${headerB64}.${payloadB64}`)
.digest('base64url');
if (signatureB64 !== expectedSignature) {
return null;
}
// Decode and parse payload
const payload = JSON.parse(Buffer.from(payloadB64, 'base64url').toString('utf-8'));
// Check expiration
const now = Math.floor(Date.now() / 1000);
if (payload.exp && payload.exp < now) {
return null;
}
// Check not-before time
if (payload.nbf && payload.nbf > now) {
return null;
}
// Convert to unified token format
const scopes = this.ociAccessToScopes(payload.access || []);
return {
type: 'oci',
userId: payload.sub,
scopes,
expiresAt: payload.exp ? new Date(payload.exp * 1000) : undefined,
metadata: {
iss: payload.iss,
aud: payload.aud,
},
};
} catch (error) {
return null;
}
}
// ========================================================================
// Helper Methods
// ========================================================================
/**
* Check if a scope matches a resource and action
*/
private matchesScope(scope: string, resource: string, action: string): boolean {
const scopeParts = scope.split(':');
const resourceParts = resource.split(':');
// Scope must have at least protocol:type:name:action
if (scopeParts.length < 4) {
return false;
}
const [scopeProtocol, scopeType, scopeName, scopeAction] = scopeParts;
const [resourceProtocol, resourceType, resourceName] = resourceParts;
// Check protocol
if (scopeProtocol !== '*' && scopeProtocol !== resourceProtocol) {
return false;
}
// Check type
if (scopeType !== '*' && scopeType !== resourceType) {
return false;
}
// Check name
if (scopeName !== '*' && scopeName !== resourceName) {
return false;
}
// Check action
if (scopeAction !== '*' && scopeAction !== action) {
// Map action aliases
const actionAliases: Record<string, string[]> = {
read: ['pull', 'get'],
write: ['push', 'put', 'post'],
};
const aliases = actionAliases[scopeAction] || [];
if (!aliases.includes(action)) {
return false;
}
}
return true;
}
/**
* Convert unified scopes to OCI access array
*/
private scopesToOciAccess(scopes: string[]): Array<{
type: string;
name: string;
actions: string[];
}> {
const access: Array<{type: string; name: string; actions: string[]}> = [];
for (const scope of scopes) {
const parts = scope.split(':');
if (parts.length >= 4 && parts[0] === 'oci') {
access.push({
type: parts[1],
name: parts[2],
actions: [parts[3]],
});
}
}
return access;
}
/**
* Convert OCI access array to unified scopes
*/
private ociAccessToScopes(access: Array<{
type: string;
name: string;
actions: string[];
}>): string[] {
const scopes: string[] = [];
for (const item of access) {
for (const action of item.actions) {
scopes.push(`oci:${item.type}:${item.name}:${action}`);
}
}
return scopes;
}
/**
* Generate UUID for tokens
*/
private generateUuid(): string {
return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => {
const r = (Math.random() * 16) | 0;
const v = c === 'x' ? r : (r & 0x3) | 0x8;
return v.toString(16);
});
}
/**
* Check if string is a valid UUID
*/
private isValidUuid(str: string): boolean {
const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
return uuidRegex.test(str);
}
/**
* Hash a token for identification
*/
private hashToken(token: string): string {
return `sha512-${token.substring(0, 16)}...`;
}
}

View File

@@ -1,17 +1,54 @@
import * as plugins from '../plugins.js';
import type { IStorageConfig, IStorageBackend } from './interfaces.core.js';
import type { IStorageConfig, IStorageBackend, TRegistryProtocol } from './interfaces.core.js';
import type {
IStorageHooks,
IStorageHookContext,
IStorageActor,
IStorageMetadata,
} from './interfaces.storage.js';
/**
* Storage abstraction layer for registry
* Provides a unified interface over SmartBucket
* Storage abstraction layer for registry.
* Provides a unified interface over SmartBucket with optional hooks
* for quota tracking, audit logging, cache invalidation, etc.
*
* @example
* ```typescript
* // Basic usage
* const storage = new RegistryStorage(config);
*
* // With hooks for quota tracking
* const storage = new RegistryStorage(config, {
* beforePut: async (ctx) => {
* const quota = await getQuota(ctx.actor?.orgId);
* const usage = await getUsage(ctx.actor?.orgId);
* if (usage + (ctx.metadata?.size || 0) > quota) {
* return { allowed: false, reason: 'Quota exceeded' };
* }
* return { allowed: true };
* },
* afterPut: async (ctx) => {
* await updateUsage(ctx.actor?.orgId, ctx.metadata?.size || 0);
* }
* });
* ```
*/
export class RegistryStorage implements IStorageBackend {
private smartBucket: plugins.smartbucket.SmartBucket;
private bucket: plugins.smartbucket.Bucket;
private bucketName: string;
private hooks?: IStorageHooks;
constructor(private config: IStorageConfig) {
constructor(private config: IStorageConfig, hooks?: IStorageHooks) {
this.bucketName = config.bucketName;
this.hooks = hooks;
}
/**
* Set storage hooks (can be called after construction)
*/
public setHooks(hooks: IStorageHooks): void {
this.hooks = hooks;
}
/**
@@ -34,7 +71,24 @@ export class RegistryStorage implements IStorageBackend {
*/
public async getObject(key: string): Promise<Buffer | null> {
try {
return await this.bucket.fastGet({ path: key });
const data = await this.bucket.fastGet({ path: key });
// Call afterGet hook (non-blocking)
if (this.hooks?.afterGet && data) {
const context = this.currentContext;
if (context) {
this.hooks.afterGet({
operation: 'get',
key,
protocol: context.protocol,
actor: context.actor,
metadata: context.metadata,
timestamp: new Date(),
}).catch(() => {}); // Don't fail on hook errors
}
}
return data;
} catch (error) {
return null;
}
@@ -48,19 +102,159 @@ export class RegistryStorage implements IStorageBackend {
data: Buffer,
metadata?: Record<string, string>
): Promise<void> {
// Call beforePut hook if available
if (this.hooks?.beforePut) {
const context = this.currentContext;
if (context) {
const hookContext: IStorageHookContext = {
operation: 'put',
key,
protocol: context.protocol,
actor: context.actor,
metadata: {
...context.metadata,
size: data.length,
},
timestamp: new Date(),
};
const result = await this.hooks.beforePut(hookContext);
if (!result.allowed) {
throw new Error(result.reason || 'Storage operation denied by hook');
}
}
}
// Note: SmartBucket doesn't support metadata yet
await this.bucket.fastPut({
path: key,
contents: data,
overwrite: true, // Always overwrite existing objects
});
// Call afterPut hook (non-blocking)
if (this.hooks?.afterPut) {
const context = this.currentContext;
if (context) {
this.hooks.afterPut({
operation: 'put',
key,
protocol: context.protocol,
actor: context.actor,
metadata: {
...context.metadata,
size: data.length,
},
timestamp: new Date(),
}).catch(() => {}); // Don't fail on hook errors
}
}
}
/**
* Delete an object
*/
public async deleteObject(key: string): Promise<void> {
// Call beforeDelete hook if available
if (this.hooks?.beforeDelete) {
const context = this.currentContext;
if (context) {
const hookContext: IStorageHookContext = {
operation: 'delete',
key,
protocol: context.protocol,
actor: context.actor,
metadata: context.metadata,
timestamp: new Date(),
};
const result = await this.hooks.beforeDelete(hookContext);
if (!result.allowed) {
throw new Error(result.reason || 'Delete operation denied by hook');
}
}
}
await this.bucket.fastRemove({ path: key });
// Call afterDelete hook (non-blocking)
if (this.hooks?.afterDelete) {
const context = this.currentContext;
if (context) {
this.hooks.afterDelete({
operation: 'delete',
key,
protocol: context.protocol,
actor: context.actor,
metadata: context.metadata,
timestamp: new Date(),
}).catch(() => {}); // Don't fail on hook errors
}
}
}
// ========================================================================
// CONTEXT FOR HOOKS
// ========================================================================
/**
* Current operation context for hooks.
* Set this before performing storage operations to enable hooks.
*/
private currentContext?: {
protocol: TRegistryProtocol;
actor?: IStorageActor;
metadata?: IStorageMetadata;
};
/**
* Set the current operation context for hooks.
* Call this before performing storage operations.
*
* @example
* ```typescript
* storage.setContext({
* protocol: 'npm',
* actor: { userId: 'user123', ip: '192.168.1.1' },
* metadata: { packageName: 'lodash', version: '4.17.21' }
* });
* await storage.putNpmTarball('lodash', '4.17.21', tarball);
* storage.clearContext();
* ```
*/
public setContext(context: {
protocol: TRegistryProtocol;
actor?: IStorageActor;
metadata?: IStorageMetadata;
}): void {
this.currentContext = context;
}
/**
* Clear the current operation context.
*/
public clearContext(): void {
this.currentContext = undefined;
}
/**
* Execute a function with a temporary context.
* Context is automatically cleared after execution.
*/
public async withContext<T>(
context: {
protocol: TRegistryProtocol;
actor?: IStorageActor;
metadata?: IStorageMetadata;
},
fn: () => Promise<T>
): Promise<T> {
this.setContext(context);
try {
return await fn();
} finally {
this.clearContext();
}
}
/**
@@ -129,7 +323,7 @@ export class RegistryStorage implements IStorageBackend {
}
/**
* Get OCI manifest
* Get OCI manifest and its content type
*/
public async getOciManifest(repository: string, digest: string): Promise<Buffer | null> {
const path = this.getOciManifestPath(repository, digest);
@@ -137,7 +331,17 @@ export class RegistryStorage implements IStorageBackend {
}
/**
* Store OCI manifest
* Get OCI manifest content type
* Returns the stored content type or null if not found
*/
public async getOciManifestContentType(repository: string, digest: string): Promise<string | null> {
const typePath = this.getOciManifestPath(repository, digest) + '.type';
const data = await this.getObject(typePath);
return data ? data.toString('utf-8') : null;
}
/**
* Store OCI manifest with its content type
*/
public async putOciManifest(
repository: string,
@@ -146,7 +350,11 @@ export class RegistryStorage implements IStorageBackend {
contentType: string
): Promise<void> {
const path = this.getOciManifestPath(repository, digest);
return this.putObject(path, data, { 'Content-Type': contentType });
// Store manifest data
await this.putObject(path, data, { 'Content-Type': contentType });
// Store content type in sidecar file for later retrieval
const typePath = path + '.type';
await this.putObject(typePath, Buffer.from(contentType, 'utf-8'));
}
/**

34
ts/core/helpers.buffer.ts Normal file
View File

@@ -0,0 +1,34 @@
/**
* Shared buffer utilities for consistent binary data handling across all registry types.
*
* This module addresses the common issue where `Buffer.isBuffer(Uint8Array)` returns `false`,
* which can cause data handling bugs when binary data arrives as Uint8Array instead of Buffer.
*/
/**
* Check if value is binary data (Buffer or Uint8Array)
*/
export function isBinaryData(value: unknown): value is Buffer | Uint8Array {
return Buffer.isBuffer(value) || value instanceof Uint8Array;
}
/**
* Convert any binary-like data to Buffer.
* Handles Buffer, Uint8Array, string, and objects.
*
* @param data - The data to convert to Buffer
* @returns A Buffer containing the data
*/
export function toBuffer(data: unknown): Buffer {
if (Buffer.isBuffer(data)) {
return data;
}
if (data instanceof Uint8Array) {
return Buffer.from(data);
}
if (typeof data === 'string') {
return Buffer.from(data, 'utf-8');
}
// Fallback: serialize object to JSON
return Buffer.from(JSON.stringify(data));
}

View File

@@ -2,9 +2,16 @@
* Core registry infrastructure exports
*/
// Interfaces
// Core interfaces
export * from './interfaces.core.js';
// Auth interfaces and provider
export * from './interfaces.auth.js';
export { DefaultAuthProvider } from './classes.defaultauthprovider.js';
// Storage interfaces and hooks
export * from './interfaces.storage.js';
// Classes
export { BaseRegistry } from './classes.baseregistry.js';
export { RegistryStorage } from './classes.registrystorage.js';

View File

@@ -0,0 +1,91 @@
import type { IAuthToken, ICredentials, TRegistryProtocol } from './interfaces.core.js';
/**
* Options for creating a token
*/
export interface ITokenOptions {
/** Whether the token is readonly */
readonly?: boolean;
/** Permission scopes */
scopes?: string[];
/** Expiration time in seconds */
expiresIn?: number;
}
/**
* Pluggable authentication provider interface.
* Implement this to integrate external auth systems (LDAP, OAuth, SSO, OIDC).
*
* @example
* ```typescript
* class LdapAuthProvider implements IAuthProvider {
* constructor(private ldap: LdapClient, private redis: RedisClient) {}
*
* async authenticate(credentials: ICredentials): Promise<string | null> {
* return await this.ldap.bind(credentials.username, credentials.password);
* }
*
* async validateToken(token: string): Promise<IAuthToken | null> {
* return await this.redis.get(`token:${token}`);
* }
* // ...
* }
* ```
*/
export interface IAuthProvider {
/**
* Initialize the auth provider (optional)
*/
init?(): Promise<void>;
/**
* Authenticate user credentials (login flow)
* @param credentials - Username and password
* @returns User ID on success, null on failure
*/
authenticate(credentials: ICredentials): Promise<string | null>;
/**
* Validate an existing token
* @param token - Token string (UUID or JWT)
* @param protocol - Optional protocol hint for optimization
* @returns Auth token info or null if invalid
*/
validateToken(token: string, protocol?: TRegistryProtocol): Promise<IAuthToken | null>;
/**
* Create a new token for a user
* @param userId - User ID
* @param protocol - Protocol type (npm, oci, maven, etc.)
* @param options - Token options (readonly, scopes, expiration)
* @returns Token string
*/
createToken(userId: string, protocol: TRegistryProtocol, options?: ITokenOptions): Promise<string>;
/**
* Revoke a token
* @param token - Token string to revoke
*/
revokeToken(token: string): Promise<void>;
/**
* Check if user has permission for an action
* @param token - Auth token (or null for anonymous)
* @param resource - Resource being accessed (e.g., "npm:package:lodash")
* @param action - Action being performed (read, write, push, pull, delete)
* @returns true if authorized
*/
authorize(token: IAuthToken | null, resource: string, action: string): Promise<boolean>;
/**
* List all tokens for a user (optional)
* @param userId - User ID
* @returns List of token info
*/
listUserTokens?(userId: string): Promise<Array<{
key: string;
readonly: boolean;
created: string;
protocol?: TRegistryProtocol;
}>>;
}

View File

@@ -3,6 +3,9 @@
*/
import type * as plugins from '../plugins.js';
import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js';
import type { IAuthProvider } from './interfaces.auth.js';
import type { IStorageHooks } from './interfaces.storage.js';
/**
* Registry protocol types
@@ -86,6 +89,8 @@ export interface IProtocolConfig {
enabled: boolean;
basePath: string;
features?: Record<string, boolean>;
/** Upstream registry configuration for proxying/caching */
upstream?: IProtocolUpstreamConfig;
}
/**
@@ -94,6 +99,20 @@ export interface IProtocolConfig {
export interface IRegistryConfig {
storage: IStorageConfig;
auth: IAuthConfig;
/**
* Custom authentication provider.
* If not provided, uses the default in-memory auth provider.
* Implement IAuthProvider to integrate LDAP, OAuth, SSO, etc.
*/
authProvider?: IAuthProvider;
/**
* Storage event hooks for quota tracking, audit logging, etc.
* Called before/after storage operations.
*/
storageHooks?: IStorageHooks;
oci?: IProtocolConfig;
npm?: IProtocolConfig;
maven?: IProtocolConfig;
@@ -149,6 +168,24 @@ export interface IRegistryError {
}>;
}
/**
* Actor information - identifies who is performing the request
*/
export interface IRequestActor {
/** User ID (from validated token) */
userId?: string;
/** Token ID/hash for audit purposes */
tokenId?: string;
/** Client IP address */
ip?: string;
/** Client User-Agent */
userAgent?: string;
/** Organization ID (for multi-tenant setups) */
orgId?: string;
/** Session ID */
sessionId?: string;
}
/**
* Base request context
*/
@@ -158,7 +195,18 @@ export interface IRequestContext {
headers: Record<string, string>;
query: Record<string, string>;
body?: any;
/**
* Raw request body as bytes. MUST be provided for content-addressable operations
* (OCI manifests, blobs) to ensure digest calculation matches client expectations.
* If not provided, falls back to 'body' field.
*/
rawBody?: Buffer;
token?: string;
/**
* Actor information - identifies who is performing the request.
* Populated after authentication for audit logging, quota enforcement, etc.
*/
actor?: IRequestActor;
}
/**

View File

@@ -0,0 +1,130 @@
import type { TRegistryProtocol } from './interfaces.core.js';
/**
* Actor information from request context
*/
export interface IStorageActor {
userId?: string;
tokenId?: string;
ip?: string;
userAgent?: string;
orgId?: string;
sessionId?: string;
}
/**
* Metadata about the storage operation
*/
export interface IStorageMetadata {
/** Content type of the object */
contentType?: string;
/** Size in bytes */
size?: number;
/** Content digest (e.g., sha256:abc123) */
digest?: string;
/** Package/artifact name */
packageName?: string;
/** Version */
version?: string;
}
/**
* Context passed to storage hooks
*/
export interface IStorageHookContext {
/** Type of operation */
operation: 'put' | 'delete' | 'get';
/** Storage key/path */
key: string;
/** Protocol that triggered this operation */
protocol: TRegistryProtocol;
/** Actor who performed the operation (if known) */
actor?: IStorageActor;
/** Metadata about the object */
metadata?: IStorageMetadata;
/** Timestamp of the operation */
timestamp: Date;
}
/**
* Result from a beforePut hook that can modify the operation
*/
export interface IBeforePutResult {
/** Whether to allow the operation */
allowed: boolean;
/** Optional reason for rejection */
reason?: string;
/** Optional modified metadata */
metadata?: IStorageMetadata;
}
/**
* Result from a beforeDelete hook
*/
export interface IBeforeDeleteResult {
/** Whether to allow the operation */
allowed: boolean;
/** Optional reason for rejection */
reason?: string;
}
/**
* Storage event hooks for quota tracking, audit logging, cache invalidation, etc.
*
* @example
* ```typescript
* const quotaHooks: IStorageHooks = {
* async beforePut(context) {
* const quota = await getQuota(context.actor?.orgId);
* const currentUsage = await getUsage(context.actor?.orgId);
* if (currentUsage + (context.metadata?.size || 0) > quota) {
* return { allowed: false, reason: 'Quota exceeded' };
* }
* return { allowed: true };
* },
*
* async afterPut(context) {
* await updateUsage(context.actor?.orgId, context.metadata?.size || 0);
* await auditLog('storage.put', context);
* },
*
* async afterDelete(context) {
* await invalidateCache(context.key);
* }
* };
* ```
*/
export interface IStorageHooks {
/**
* Called before storing an object.
* Return { allowed: false } to reject the operation.
* Use for quota checks, virus scanning, validation, etc.
*/
beforePut?(context: IStorageHookContext): Promise<IBeforePutResult>;
/**
* Called after successfully storing an object.
* Use for quota tracking, audit logging, notifications, etc.
*/
afterPut?(context: IStorageHookContext): Promise<void>;
/**
* Called before deleting an object.
* Return { allowed: false } to reject the operation.
* Use for preventing deletion of protected resources.
*/
beforeDelete?(context: IStorageHookContext): Promise<IBeforeDeleteResult>;
/**
* Called after successfully deleting an object.
* Use for quota updates, audit logging, cache invalidation, etc.
*/
afterDelete?(context: IStorageHookContext): Promise<void>;
/**
* Called after reading an object.
* Use for access logging, analytics, etc.
* Note: This is called even for cache hits.
*/
afterGet?(context: IStorageHookContext): Promise<void>;
}

View File

@@ -9,6 +9,9 @@ export { SmartRegistry } from './classes.smartregistry.js';
// Core infrastructure
export * from './core/index.js';
// Upstream infrastructure
export * from './upstream/index.js';
// OCI Registry
export * from './oci/index.js';

View File

@@ -7,6 +7,8 @@ import { BaseRegistry } from '../core/classes.baseregistry.js';
import type { RegistryStorage } from '../core/classes.registrystorage.js';
import type { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js';
import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js';
import { toBuffer } from '../core/helpers.buffer.js';
import type { IMavenCoordinate, IMavenMetadata, IChecksums } from './interfaces.maven.js';
import {
pathToGAV,
@@ -20,6 +22,7 @@ import {
extractGAVFromPom,
gavToPath,
} from './helpers.maven.js';
import { MavenUpstream } from './classes.mavenupstream.js';
/**
* Maven Registry class
@@ -30,18 +33,34 @@ export class MavenRegistry extends BaseRegistry {
private authManager: AuthManager;
private basePath: string = '/maven';
private registryUrl: string;
private upstream: MavenUpstream | null = null;
constructor(
storage: RegistryStorage,
authManager: AuthManager,
basePath: string,
registryUrl: string
registryUrl: string,
upstreamConfig?: IProtocolUpstreamConfig
) {
super();
this.storage = storage;
this.authManager = authManager;
this.basePath = basePath;
this.registryUrl = registryUrl;
// Initialize upstream if configured
if (upstreamConfig?.enabled) {
this.upstream = new MavenUpstream(upstreamConfig);
}
}
/**
* Clean up resources (timers, connections, etc.)
*/
public destroy(): void {
if (this.upstream) {
this.upstream.stop();
}
}
public async init(): Promise<void> {
@@ -233,7 +252,23 @@ export class MavenRegistry extends BaseRegistry {
version: string,
filename: string
): Promise<IResponse> {
const data = await this.storage.getMavenArtifact(groupId, artifactId, version, filename);
let data = await this.storage.getMavenArtifact(groupId, artifactId, version, filename);
// Try upstream if not found locally
if (!data && this.upstream) {
// Parse the filename to extract extension and classifier
const { extension, classifier } = this.parseFilename(filename, artifactId, version);
if (extension) {
data = await this.upstream.fetchArtifact(groupId, artifactId, version, extension, classifier);
if (data) {
// Cache the artifact locally
await this.storage.putMavenArtifact(groupId, artifactId, version, filename, data);
// Generate and store checksums
const checksums = await calculateChecksums(data);
await this.storeChecksums(groupId, artifactId, version, filename, checksums);
}
}
}
if (!data) {
return {
@@ -296,7 +331,7 @@ export class MavenRegistry extends BaseRegistry {
coordinate: IMavenCoordinate,
body: Buffer | any
): Promise<IResponse> {
const data = Buffer.isBuffer(body) ? body : Buffer.from(JSON.stringify(body));
const data = toBuffer(body);
// Validate POM if uploading .pom file
if (coordinate.extension === 'pom') {
@@ -461,7 +496,17 @@ export class MavenRegistry extends BaseRegistry {
// ========================================================================
private async getMetadata(groupId: string, artifactId: string): Promise<IResponse> {
const metadataBuffer = await this.storage.getMavenMetadata(groupId, artifactId);
let metadataBuffer = await this.storage.getMavenMetadata(groupId, artifactId);
// Try upstream if not found locally
if (!metadataBuffer && this.upstream) {
const upstreamMetadata = await this.upstream.fetchMetadata(groupId, artifactId);
if (upstreamMetadata) {
metadataBuffer = Buffer.from(upstreamMetadata, 'utf-8');
// Cache the metadata locally
await this.storage.putMavenMetadata(groupId, artifactId, metadataBuffer);
}
}
if (!metadataBuffer) {
// Generate empty metadata if none exists
@@ -577,4 +622,41 @@ export class MavenRegistry extends BaseRegistry {
return contentTypes[extension] || 'application/octet-stream';
}
/**
* Parse a Maven filename to extract extension and classifier.
* Filename format: {artifactId}-{version}[-{classifier}].{extension}
*/
private parseFilename(
filename: string,
artifactId: string,
version: string
): { extension: string; classifier?: string } {
const prefix = `${artifactId}-${version}`;
if (!filename.startsWith(prefix)) {
// Fallback: just get the extension
const lastDot = filename.lastIndexOf('.');
return { extension: lastDot > 0 ? filename.slice(lastDot + 1) : '' };
}
const remainder = filename.slice(prefix.length);
// remainder is either ".extension" or "-classifier.extension"
if (remainder.startsWith('.')) {
return { extension: remainder.slice(1) };
}
if (remainder.startsWith('-')) {
const lastDot = remainder.lastIndexOf('.');
if (lastDot > 1) {
return {
classifier: remainder.slice(1, lastDot),
extension: remainder.slice(lastDot + 1),
};
}
}
return { extension: '' };
}
}

View File

@@ -0,0 +1,220 @@
import * as plugins from '../plugins.js';
import { BaseUpstream } from '../upstream/classes.baseupstream.js';
import type {
IProtocolUpstreamConfig,
IUpstreamFetchContext,
IUpstreamRegistryConfig,
} from '../upstream/interfaces.upstream.js';
import type { IMavenCoordinate } from './interfaces.maven.js';
/**
* Maven-specific upstream implementation.
*
* Handles:
* - Artifact fetching (JAR, POM, WAR, etc.)
* - Metadata fetching (maven-metadata.xml)
* - Checksum files (.md5, .sha1, .sha256, .sha512)
* - SNAPSHOT version handling
* - Content-addressable caching for release artifacts
*/
export class MavenUpstream extends BaseUpstream {
protected readonly protocolName = 'maven';
constructor(
config: IProtocolUpstreamConfig,
logger?: plugins.smartlog.Smartlog,
) {
super(config, logger);
}
/**
* Fetch an artifact from upstream registries.
*/
public async fetchArtifact(
groupId: string,
artifactId: string,
version: string,
extension: string,
classifier?: string,
): Promise<Buffer | null> {
const path = this.buildArtifactPath(groupId, artifactId, version, extension, classifier);
const resource = `${groupId}:${artifactId}`;
const context: IUpstreamFetchContext = {
protocol: 'maven',
resource,
resourceType: 'artifact',
path,
method: 'GET',
headers: {},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
return Buffer.isBuffer(result.body) ? result.body : Buffer.from(result.body);
}
/**
* Fetch maven-metadata.xml from upstream.
*/
public async fetchMetadata(groupId: string, artifactId: string, version?: string): Promise<string | null> {
const groupPath = groupId.replace(/\./g, '/');
let path: string;
if (version) {
// Version-level metadata (for SNAPSHOTs)
path = `/${groupPath}/${artifactId}/${version}/maven-metadata.xml`;
} else {
// Artifact-level metadata (lists all versions)
path = `/${groupPath}/${artifactId}/maven-metadata.xml`;
}
const resource = `${groupId}:${artifactId}`;
const context: IUpstreamFetchContext = {
protocol: 'maven',
resource,
resourceType: 'metadata',
path,
method: 'GET',
headers: {
'accept': 'application/xml, text/xml',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
if (Buffer.isBuffer(result.body)) {
return result.body.toString('utf8');
}
return typeof result.body === 'string' ? result.body : null;
}
/**
* Fetch a checksum file from upstream.
*/
public async fetchChecksum(
groupId: string,
artifactId: string,
version: string,
extension: string,
checksumType: 'md5' | 'sha1' | 'sha256' | 'sha512',
classifier?: string,
): Promise<string | null> {
const basePath = this.buildArtifactPath(groupId, artifactId, version, extension, classifier);
const path = `${basePath}.${checksumType}`;
const resource = `${groupId}:${artifactId}`;
const context: IUpstreamFetchContext = {
protocol: 'maven',
resource,
resourceType: 'checksum',
path,
method: 'GET',
headers: {
'accept': 'text/plain',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
if (Buffer.isBuffer(result.body)) {
return result.body.toString('utf8').trim();
}
return typeof result.body === 'string' ? result.body.trim() : null;
}
/**
* Check if an artifact exists in upstream (HEAD request).
*/
public async headArtifact(
groupId: string,
artifactId: string,
version: string,
extension: string,
classifier?: string,
): Promise<{ exists: boolean; size?: number; lastModified?: string } | null> {
const path = this.buildArtifactPath(groupId, artifactId, version, extension, classifier);
const resource = `${groupId}:${artifactId}`;
const context: IUpstreamFetchContext = {
protocol: 'maven',
resource,
resourceType: 'artifact',
path,
method: 'HEAD',
headers: {},
query: {},
};
const result = await this.fetch(context);
if (!result) {
return null;
}
if (!result.success) {
return { exists: false };
}
return {
exists: true,
size: result.headers['content-length'] ? parseInt(result.headers['content-length'], 10) : undefined,
lastModified: result.headers['last-modified'],
};
}
/**
* Build the path for a Maven artifact.
*/
private buildArtifactPath(
groupId: string,
artifactId: string,
version: string,
extension: string,
classifier?: string,
): string {
const groupPath = groupId.replace(/\./g, '/');
let filename = `${artifactId}-${version}`;
if (classifier) {
filename += `-${classifier}`;
}
filename += `.${extension}`;
return `/${groupPath}/${artifactId}/${version}/${filename}`;
}
/**
* Override URL building for Maven-specific handling.
*/
protected buildUpstreamUrl(
upstream: IUpstreamRegistryConfig,
context: IUpstreamFetchContext,
): string {
let baseUrl = upstream.url;
// Remove trailing slash
if (baseUrl.endsWith('/')) {
baseUrl = baseUrl.slice(0, -1);
}
return `${baseUrl}${context.path}`;
}
}

View File

@@ -3,5 +3,6 @@
*/
export { MavenRegistry } from './classes.mavenregistry.js';
export { MavenUpstream } from './classes.mavenupstream.js';
export * from './interfaces.maven.js';
export * from './helpers.maven.js';

View File

@@ -3,6 +3,8 @@ import { BaseRegistry } from '../core/classes.baseregistry.js';
import { RegistryStorage } from '../core/classes.registrystorage.js';
import { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js';
import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js';
import { NpmUpstream } from './classes.npmupstream.js';
import type {
IPackument,
INpmVersion,
@@ -25,12 +27,14 @@ export class NpmRegistry extends BaseRegistry {
private basePath: string = '/npm';
private registryUrl: string;
private logger: Smartlog;
private upstream: NpmUpstream | null = null;
constructor(
storage: RegistryStorage,
authManager: AuthManager,
basePath: string = '/npm',
registryUrl: string = 'http://localhost:5000/npm'
registryUrl: string = 'http://localhost:5000/npm',
upstreamConfig?: IProtocolUpstreamConfig
) {
super();
this.storage = storage;
@@ -50,6 +54,14 @@ export class NpmRegistry extends BaseRegistry {
}
});
this.logger.enableConsole();
// Initialize upstream if configured
if (upstreamConfig?.enabled) {
this.upstream = new NpmUpstream(upstreamConfig, registryUrl, this.logger);
this.logger.log('info', 'NPM upstream initialized', {
upstreams: upstreamConfig.upstreams.map(u => u.name),
});
}
}
public async init(): Promise<void> {
@@ -113,7 +125,7 @@ export class NpmRegistry extends BaseRegistry {
const unpublishVersionMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)\/-\/([^\/]+)$/);
if (unpublishVersionMatch && context.method === 'DELETE') {
const [, packageName, version] = unpublishVersionMatch;
console.log(`[unpublishVersionMatch] packageName=${packageName}, version=${version}`);
this.logger.log('debug', 'unpublishVersionMatch', { packageName, version });
return this.unpublishVersion(packageName, version, token);
}
@@ -121,7 +133,7 @@ export class NpmRegistry extends BaseRegistry {
const unpublishPackageMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)\/-rev\/([^\/]+)$/);
if (unpublishPackageMatch && context.method === 'DELETE') {
const [, packageName, rev] = unpublishPackageMatch;
console.log(`[unpublishPackageMatch] packageName=${packageName}, rev=${rev}`);
this.logger.log('debug', 'unpublishPackageMatch', { packageName, rev });
return this.unpublishPackage(packageName, token);
}
@@ -129,7 +141,7 @@ export class NpmRegistry extends BaseRegistry {
const versionMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)\/([^\/]+)$/);
if (versionMatch) {
const [, packageName, version] = versionMatch;
console.log(`[versionMatch] matched! packageName=${packageName}, version=${version}`);
this.logger.log('debug', 'versionMatch', { packageName, version });
return this.handlePackageVersion(packageName, version, token);
}
@@ -137,7 +149,7 @@ export class NpmRegistry extends BaseRegistry {
const packageMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)$/);
if (packageMatch) {
const packageName = packageMatch[1];
console.log(`[packageMatch] matched! packageName=${packageName}`);
this.logger.log('debug', 'packageMatch', { packageName });
return this.handlePackage(context.method, packageName, context.body, context.query, token);
}
@@ -209,13 +221,28 @@ export class NpmRegistry extends BaseRegistry {
token: IAuthToken | null,
query: Record<string, string>
): Promise<IResponse> {
const packument = await this.storage.getNpmPackument(packageName);
let packument = await this.storage.getNpmPackument(packageName);
this.logger.log('debug', `getPackument: ${packageName}`, {
packageName,
found: !!packument,
versions: packument ? Object.keys(packument.versions).length : 0
});
// If not found locally, try upstream
if (!packument && this.upstream) {
this.logger.log('debug', `getPackument: fetching from upstream`, { packageName });
const upstreamPackument = await this.upstream.fetchPackument(packageName);
if (upstreamPackument) {
this.logger.log('debug', `getPackument: found in upstream`, {
packageName,
versions: Object.keys(upstreamPackument.versions || {}).length
});
packument = upstreamPackument;
// Optionally cache the packument locally (without tarballs)
// We don't store tarballs here - they'll be fetched on demand
}
}
if (!packument) {
return {
status: 404,
@@ -254,12 +281,22 @@ export class NpmRegistry extends BaseRegistry {
version: string,
token: IAuthToken | null
): Promise<IResponse> {
console.log(`[handlePackageVersion] packageName=${packageName}, version=${version}`);
const packument = await this.storage.getNpmPackument(packageName);
console.log(`[handlePackageVersion] packument found:`, !!packument);
this.logger.log('debug', 'handlePackageVersion', { packageName, version });
let packument = await this.storage.getNpmPackument(packageName);
this.logger.log('debug', 'handlePackageVersion packument', { found: !!packument });
if (packument) {
console.log(`[handlePackageVersion] versions:`, Object.keys(packument.versions || {}));
this.logger.log('debug', 'handlePackageVersion versions', { versions: Object.keys(packument.versions || {}) });
}
// If not found locally, try upstream
if (!packument && this.upstream) {
this.logger.log('debug', 'handlePackageVersion: fetching from upstream', { packageName });
const upstreamPackument = await this.upstream.fetchPackument(packageName);
if (upstreamPackument) {
packument = upstreamPackument;
}
}
if (!packument) {
return {
status: 404,
@@ -529,7 +566,7 @@ export class NpmRegistry extends BaseRegistry {
token: IAuthToken | null
): Promise<IResponse> {
// Extract version from filename: package-name-1.0.0.tgz
const versionMatch = filename.match(/-([\d.]+(?:-[a-z0-9.]+)?)\.tgz$/);
const versionMatch = filename.match(/-([\d.]+(?:-[a-z0-9.]+)?)\.tgz$/i);
if (!versionMatch) {
return {
status: 400,
@@ -539,7 +576,26 @@ export class NpmRegistry extends BaseRegistry {
}
const version = versionMatch[1];
const tarball = await this.storage.getNpmTarball(packageName, version);
let tarball = await this.storage.getNpmTarball(packageName, version);
// If not found locally, try upstream
if (!tarball && this.upstream) {
this.logger.log('debug', 'handleTarballDownload: fetching from upstream', {
packageName,
version,
});
const upstreamTarball = await this.upstream.fetchTarball(packageName, version);
if (upstreamTarball) {
tarball = upstreamTarball;
// Cache the tarball locally for future requests
await this.storage.putNpmTarball(packageName, version, tarball);
this.logger.log('debug', 'handleTarballDownload: cached tarball locally', {
packageName,
version,
size: tarball.length,
});
}
}
if (!tarball) {
return {
@@ -621,7 +677,7 @@ export class NpmRegistry extends BaseRegistry {
}
}
} catch (error) {
console.error('[handleSearch] Error:', error);
this.logger.log('error', 'handleSearch failed', { error: (error as Error).message });
}
// Apply pagination

View File

@@ -0,0 +1,260 @@
import * as plugins from '../plugins.js';
import { BaseUpstream } from '../upstream/classes.baseupstream.js';
import type {
IProtocolUpstreamConfig,
IUpstreamFetchContext,
IUpstreamResult,
IUpstreamRegistryConfig,
} from '../upstream/interfaces.upstream.js';
import type { IPackument, INpmVersion } from './interfaces.npm.js';
/**
* NPM-specific upstream implementation.
*
* Handles:
* - Package metadata (packument) fetching
* - Tarball proxying
* - Scoped package routing (@scope/* patterns)
* - NPM-specific URL rewriting
*/
export class NpmUpstream extends BaseUpstream {
protected readonly protocolName = 'npm';
/** Local registry URL for rewriting tarball URLs */
private readonly localRegistryUrl: string;
constructor(
config: IProtocolUpstreamConfig,
localRegistryUrl: string,
logger?: plugins.smartlog.Smartlog,
) {
super(config, logger);
this.localRegistryUrl = localRegistryUrl;
}
/**
* Fetch a packument from upstream registries.
*/
public async fetchPackument(packageName: string): Promise<IPackument | null> {
const context: IUpstreamFetchContext = {
protocol: 'npm',
resource: packageName,
resourceType: 'packument',
path: `/${encodeURIComponent(packageName).replace('%40', '@')}`,
method: 'GET',
headers: {
'accept': 'application/json',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
// Parse and process packument
let packument: IPackument;
if (Buffer.isBuffer(result.body)) {
packument = JSON.parse(result.body.toString('utf8'));
} else {
packument = result.body;
}
// Rewrite tarball URLs to point to local registry
packument = this.rewriteTarballUrls(packument);
return packument;
}
/**
* Fetch a specific version from upstream registries.
*/
public async fetchVersion(packageName: string, version: string): Promise<INpmVersion | null> {
const context: IUpstreamFetchContext = {
protocol: 'npm',
resource: packageName,
resourceType: 'version',
path: `/${encodeURIComponent(packageName).replace('%40', '@')}/${version}`,
method: 'GET',
headers: {
'accept': 'application/json',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
let versionData: INpmVersion;
if (Buffer.isBuffer(result.body)) {
versionData = JSON.parse(result.body.toString('utf8'));
} else {
versionData = result.body;
}
// Rewrite tarball URL
if (versionData.dist?.tarball) {
versionData.dist.tarball = this.rewriteSingleTarballUrl(
packageName,
versionData.version,
versionData.dist.tarball,
);
}
return versionData;
}
/**
* Fetch a tarball from upstream registries.
*/
public async fetchTarball(packageName: string, version: string): Promise<Buffer | null> {
// First, try to get the tarball URL from packument
const packument = await this.fetchPackument(packageName);
let tarballPath: string;
if (packument?.versions?.[version]?.dist?.tarball) {
// Extract path from original (upstream) tarball URL
const tarballUrl = packument.versions[version].dist.tarball;
try {
const url = new URL(tarballUrl);
tarballPath = url.pathname;
} catch {
// Fallback to standard NPM tarball path
tarballPath = this.buildTarballPath(packageName, version);
}
} else {
tarballPath = this.buildTarballPath(packageName, version);
}
const context: IUpstreamFetchContext = {
protocol: 'npm',
resource: packageName,
resourceType: 'tarball',
path: tarballPath,
method: 'GET',
headers: {
'accept': 'application/octet-stream',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
return Buffer.isBuffer(result.body) ? result.body : Buffer.from(result.body);
}
/**
* Search packages in upstream registries.
*/
public async search(text: string, size: number = 20, from: number = 0): Promise<any | null> {
const context: IUpstreamFetchContext = {
protocol: 'npm',
resource: '*',
resourceType: 'search',
path: '/-/v1/search',
method: 'GET',
headers: {
'accept': 'application/json',
},
query: {
text,
size: size.toString(),
from: from.toString(),
},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
if (Buffer.isBuffer(result.body)) {
return JSON.parse(result.body.toString('utf8'));
}
return result.body;
}
/**
* Build the standard NPM tarball path.
*/
private buildTarballPath(packageName: string, version: string): string {
// NPM uses: /{package}/-/{package-name}-{version}.tgz
// For scoped packages: /@scope/name/-/name-version.tgz
if (packageName.startsWith('@')) {
const [scope, name] = packageName.split('/');
return `/${scope}/${name}/-/${name}-${version}.tgz`;
} else {
return `/${packageName}/-/${packageName}-${version}.tgz`;
}
}
/**
* Rewrite all tarball URLs in a packument to point to local registry.
*/
private rewriteTarballUrls(packument: IPackument): IPackument {
if (!packument.versions) {
return packument;
}
const rewritten = { ...packument };
rewritten.versions = {};
for (const [version, versionData] of Object.entries(packument.versions)) {
const newVersionData = { ...versionData };
if (newVersionData.dist?.tarball) {
newVersionData.dist = {
...newVersionData.dist,
tarball: this.rewriteSingleTarballUrl(
packument.name,
version,
newVersionData.dist.tarball,
),
};
}
rewritten.versions[version] = newVersionData;
}
return rewritten;
}
/**
* Rewrite a single tarball URL to point to local registry.
*/
private rewriteSingleTarballUrl(
packageName: string,
version: string,
_originalUrl: string,
): string {
// Generate local tarball URL
// Format: {localRegistryUrl}/{package}/-/{package-name}-{version}.tgz
const safeName = packageName.replace('@', '').replace('/', '-');
return `${this.localRegistryUrl}/${packageName}/-/${safeName}-${version}.tgz`;
}
/**
* Override URL building for NPM-specific handling.
*/
protected buildUpstreamUrl(
upstream: IUpstreamRegistryConfig,
context: IUpstreamFetchContext,
): string {
// NPM registries often don't have trailing slashes
let baseUrl = upstream.url;
if (baseUrl.endsWith('/')) {
baseUrl = baseUrl.slice(0, -1);
}
return `${baseUrl}${context.path}`;
}
}

View File

@@ -3,4 +3,5 @@
*/
export { NpmRegistry } from './classes.npmregistry.js';
export { NpmUpstream } from './classes.npmupstream.js';
export * from './interfaces.npm.js';

View File

@@ -1,7 +1,10 @@
import { Smartlog } from '@push.rocks/smartlog';
import { BaseRegistry } from '../core/classes.baseregistry.js';
import { RegistryStorage } from '../core/classes.registrystorage.js';
import { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken, IRegistryError } from '../core/interfaces.core.js';
import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js';
import { OciUpstream } from './classes.ociupstream.js';
import type {
IUploadSession,
IOciManifest,
@@ -20,12 +23,43 @@ export class OciRegistry extends BaseRegistry {
private uploadSessions: Map<string, IUploadSession> = new Map();
private basePath: string = '/oci';
private cleanupInterval?: NodeJS.Timeout;
private ociTokens?: { realm: string; service: string };
private upstream: OciUpstream | null = null;
private logger: Smartlog;
constructor(storage: RegistryStorage, authManager: AuthManager, basePath: string = '/oci') {
constructor(
storage: RegistryStorage,
authManager: AuthManager,
basePath: string = '/oci',
ociTokens?: { realm: string; service: string },
upstreamConfig?: IProtocolUpstreamConfig
) {
super();
this.storage = storage;
this.authManager = authManager;
this.basePath = basePath;
this.ociTokens = ociTokens;
// Initialize logger
this.logger = new Smartlog({
logContext: {
company: 'push.rocks',
companyunit: 'smartregistry',
containerName: 'oci-registry',
environment: (process.env.NODE_ENV as any) || 'development',
runtime: 'node',
zone: 'oci'
}
});
this.logger.enableConsole();
// Initialize upstream if configured
if (upstreamConfig?.enabled) {
this.upstream = new OciUpstream(upstreamConfig, basePath, this.logger);
this.logger.log('info', 'OCI upstream initialized', {
upstreams: upstreamConfig.upstreams.map(u => u.name),
});
}
}
public async init(): Promise<void> {
@@ -55,7 +89,9 @@ export class OciRegistry extends BaseRegistry {
const manifestMatch = path.match(/^\/v2\/([^\/]+(?:\/[^\/]+)*)\/manifests\/([^\/]+)$/);
if (manifestMatch) {
const [, name, reference] = manifestMatch;
return this.handleManifestRequest(context.method, name, reference, token, context.body, context.headers);
// Prefer rawBody for content-addressable operations to preserve exact bytes
const bodyData = context.rawBody || context.body;
return this.handleManifestRequest(context.method, name, reference, token, bodyData, context.headers);
}
// Blob operations: /v2/{name}/blobs/{digest}
@@ -69,7 +105,9 @@ export class OciRegistry extends BaseRegistry {
const uploadInitMatch = path.match(/^\/v2\/([^\/]+(?:\/[^\/]+)*)\/blobs\/uploads\/?$/);
if (uploadInitMatch && context.method === 'POST') {
const [, name] = uploadInitMatch;
return this.handleUploadInit(name, token, context.query, context.body);
// Prefer rawBody for content-addressable operations to preserve exact bytes
const bodyData = context.rawBody || context.body;
return this.handleUploadInit(name, token, context.query, bodyData);
}
// Blob upload operations: /v2/{name}/blobs/uploads/{uuid}
@@ -180,18 +218,14 @@ export class OciRegistry extends BaseRegistry {
body?: Buffer | any
): Promise<IResponse> {
if (!await this.checkPermission(token, repository, 'push')) {
return {
status: 401,
headers: {},
body: this.createError('DENIED', 'Insufficient permissions'),
};
return this.createUnauthorizedResponse(repository, 'push');
}
// Check for monolithic upload (digest + body provided)
const digest = query.digest;
if (digest && body) {
// Monolithic upload: complete upload in single POST
const blobData = Buffer.isBuffer(body) ? body : Buffer.from(JSON.stringify(body));
const blobData = this.toBuffer(body);
// Verify digest
const calculatedDigest = await this.calculateDigest(blobData);
@@ -255,18 +289,17 @@ export class OciRegistry extends BaseRegistry {
}
if (!await this.checkPermission(token, session.repository, 'push')) {
return {
status: 401,
headers: {},
body: this.createError('DENIED', 'Insufficient permissions'),
};
return this.createUnauthorizedResponse(session.repository, 'push');
}
// Prefer rawBody for content-addressable operations to preserve exact bytes
const bodyData = context.rawBody || context.body;
switch (method) {
case 'PATCH':
return this.uploadChunk(uploadId, context.body, context.headers['content-range']);
return this.uploadChunk(uploadId, bodyData, context.headers['content-range']);
case 'PUT':
return this.completeUpload(uploadId, context.query['digest'], context.body);
return this.completeUpload(uploadId, context.query['digest'], bodyData);
case 'GET':
return this.getUploadStatus(uploadId);
default:
@@ -288,13 +321,7 @@ export class OciRegistry extends BaseRegistry {
headers?: Record<string, string>
): Promise<IResponse> {
if (!await this.checkPermission(token, repository, 'pull')) {
return {
status: 401,
headers: {
'WWW-Authenticate': `Bearer realm="${this.basePath}/v2/token",service="registry",scope="repository:${repository}:pull"`,
},
body: this.createError('DENIED', 'Insufficient permissions'),
};
return this.createUnauthorizedResponse(repository, 'pull');
}
// Resolve tag to digest if needed
@@ -302,16 +329,50 @@ export class OciRegistry extends BaseRegistry {
if (!reference.startsWith('sha256:')) {
const tags = await this.getTagsData(repository);
digest = tags[reference];
if (!digest) {
return {
status: 404,
headers: {},
body: this.createError('MANIFEST_UNKNOWN', 'Manifest not found'),
};
}
// Try local storage first (if we have a digest)
let manifestData: Buffer | null = null;
let contentType: string | null = null;
if (digest) {
manifestData = await this.storage.getOciManifest(repository, digest);
if (manifestData) {
contentType = await this.storage.getOciManifestContentType(repository, digest);
if (!contentType) {
contentType = this.detectManifestContentType(manifestData);
}
}
}
// If not found locally, try upstream
if (!manifestData && this.upstream) {
this.logger.log('debug', 'getManifest: fetching from upstream', { repository, reference });
const upstreamResult = await this.upstream.fetchManifest(repository, reference);
if (upstreamResult) {
manifestData = Buffer.from(JSON.stringify(upstreamResult.manifest), 'utf8');
contentType = upstreamResult.contentType;
digest = upstreamResult.digest;
// Cache the manifest locally
await this.storage.putOciManifest(repository, digest, manifestData, contentType);
// If reference is a tag, update tags mapping
if (!reference.startsWith('sha256:')) {
const tags = await this.getTagsData(repository);
tags[reference] = digest;
const tagsPath = `oci/tags/${repository}/tags.json`;
await this.storage.putObject(tagsPath, Buffer.from(JSON.stringify(tags), 'utf-8'));
}
this.logger.log('debug', 'getManifest: cached manifest locally', {
repository,
reference,
digest,
});
}
}
const manifestData = await this.storage.getOciManifest(repository, digest);
if (!manifestData) {
return {
status: 404,
@@ -323,7 +384,7 @@ export class OciRegistry extends BaseRegistry {
return {
status: 200,
headers: {
'Content-Type': 'application/vnd.oci.image.manifest.v1+json',
'Content-Type': contentType || 'application/vnd.oci.image.manifest.v1+json',
'Docker-Content-Digest': digest,
},
body: manifestData,
@@ -336,11 +397,7 @@ export class OciRegistry extends BaseRegistry {
token: IAuthToken | null
): Promise<IResponse> {
if (!await this.checkPermission(token, repository, 'pull')) {
return {
status: 401,
headers: {},
body: null,
};
return this.createUnauthorizedHeadResponse(repository, 'pull');
}
// Similar logic as getManifest but return headers only
@@ -360,10 +417,18 @@ export class OciRegistry extends BaseRegistry {
const manifestData = await this.storage.getOciManifest(repository, digest);
// Get stored content type, falling back to detecting from manifest content
let contentType = await this.storage.getOciManifestContentType(repository, digest);
if (!contentType && manifestData) {
// Fallback: detect content type from manifest content
contentType = this.detectManifestContentType(manifestData);
}
contentType = contentType || 'application/vnd.oci.image.manifest.v1+json';
return {
status: 200,
headers: {
'Content-Type': 'application/vnd.oci.image.manifest.v1+json',
'Content-Type': contentType,
'Docker-Content-Digest': digest,
'Content-Length': manifestData ? manifestData.length.toString() : '0',
},
@@ -379,13 +444,7 @@ export class OciRegistry extends BaseRegistry {
headers?: Record<string, string>
): Promise<IResponse> {
if (!await this.checkPermission(token, repository, 'push')) {
return {
status: 401,
headers: {
'WWW-Authenticate': `Bearer realm="${this.basePath}/v2/token",service="registry",scope="repository:${repository}:push"`,
},
body: this.createError('DENIED', 'Insufficient permissions'),
};
return this.createUnauthorizedResponse(repository, 'push');
}
if (!body) {
@@ -396,7 +455,9 @@ export class OciRegistry extends BaseRegistry {
};
}
const manifestData = Buffer.isBuffer(body) ? body : Buffer.from(JSON.stringify(body));
// Preserve raw bytes for accurate digest calculation
// Per OCI spec, digest must match the exact bytes sent by client
const manifestData = this.toBuffer(body);
const contentType = headers?.['content-type'] || headers?.['Content-Type'] || 'application/vnd.oci.image.manifest.v1+json';
// Calculate manifest digest
@@ -437,11 +498,7 @@ export class OciRegistry extends BaseRegistry {
}
if (!await this.checkPermission(token, repository, 'delete')) {
return {
status: 401,
headers: {},
body: this.createError('DENIED', 'Insufficient permissions'),
};
return this.createUnauthorizedResponse(repository, 'delete');
}
await this.storage.deleteOciManifest(repository, digest);
@@ -460,14 +517,28 @@ export class OciRegistry extends BaseRegistry {
range?: string
): Promise<IResponse> {
if (!await this.checkPermission(token, repository, 'pull')) {
return {
status: 401,
headers: {},
body: this.createError('DENIED', 'Insufficient permissions'),
};
return this.createUnauthorizedResponse(repository, 'pull');
}
// Try local storage first
let data = await this.storage.getOciBlob(digest);
// If not found locally, try upstream
if (!data && this.upstream) {
this.logger.log('debug', 'getBlob: fetching from upstream', { repository, digest });
const upstreamBlob = await this.upstream.fetchBlob(repository, digest);
if (upstreamBlob) {
data = upstreamBlob;
// Cache the blob locally (blobs are content-addressable and immutable)
await this.storage.putOciBlob(digest, data);
this.logger.log('debug', 'getBlob: cached blob locally', {
repository,
digest,
size: data.length,
});
}
}
const data = await this.storage.getOciBlob(digest);
if (!data) {
return {
status: 404,
@@ -492,7 +563,7 @@ export class OciRegistry extends BaseRegistry {
token: IAuthToken | null
): Promise<IResponse> {
if (!await this.checkPermission(token, repository, 'pull')) {
return { status: 401, headers: {}, body: null };
return this.createUnauthorizedHeadResponse(repository, 'pull');
}
const exists = await this.storage.ociBlobExists(digest);
@@ -518,11 +589,7 @@ export class OciRegistry extends BaseRegistry {
token: IAuthToken | null
): Promise<IResponse> {
if (!await this.checkPermission(token, repository, 'delete')) {
return {
status: 401,
headers: {},
body: this.createError('DENIED', 'Insufficient permissions'),
};
return this.createUnauthorizedResponse(repository, 'delete');
}
await this.storage.deleteOciBlob(digest);
@@ -536,7 +603,7 @@ export class OciRegistry extends BaseRegistry {
private async uploadChunk(
uploadId: string,
data: Buffer,
data: Buffer | Uint8Array | unknown,
contentRange: string
): Promise<IResponse> {
const session = this.uploadSessions.get(uploadId);
@@ -548,8 +615,9 @@ export class OciRegistry extends BaseRegistry {
};
}
session.chunks.push(data);
session.totalSize += data.length;
const chunkData = this.toBuffer(data);
session.chunks.push(chunkData);
session.totalSize += chunkData.length;
session.lastActivity = new Date();
return {
@@ -566,7 +634,7 @@ export class OciRegistry extends BaseRegistry {
private async completeUpload(
uploadId: string,
digest: string,
finalData?: Buffer
finalData?: Buffer | Uint8Array | unknown
): Promise<IResponse> {
const session = this.uploadSessions.get(uploadId);
if (!session) {
@@ -578,7 +646,7 @@ export class OciRegistry extends BaseRegistry {
}
const chunks = [...session.chunks];
if (finalData) chunks.push(finalData);
if (finalData) chunks.push(this.toBuffer(finalData));
const blobData = Buffer.concat(chunks);
// Verify digest
@@ -631,11 +699,7 @@ export class OciRegistry extends BaseRegistry {
query: Record<string, string>
): Promise<IResponse> {
if (!await this.checkPermission(token, repository, 'pull')) {
return {
status: 401,
headers: {},
body: this.createError('DENIED', 'Insufficient permissions'),
};
return this.createUnauthorizedResponse(repository, 'pull');
}
const tags = await this.getTagsData(repository);
@@ -660,11 +724,7 @@ export class OciRegistry extends BaseRegistry {
query: Record<string, string>
): Promise<IResponse> {
if (!await this.checkPermission(token, repository, 'pull')) {
return {
status: 401,
headers: {},
body: this.createError('DENIED', 'Insufficient permissions'),
};
return this.createUnauthorizedResponse(repository, 'pull');
}
const response: IReferrersResponse = {
@@ -684,6 +744,59 @@ export class OciRegistry extends BaseRegistry {
// HELPER METHODS
// ========================================================================
/**
* Detect manifest content type from manifest content.
* OCI Image Index has "manifests" array, OCI Image Manifest has "config" object.
* Also checks the mediaType field if present.
*/
private detectManifestContentType(manifestData: Buffer): string {
try {
const manifest = JSON.parse(manifestData.toString('utf-8'));
// First check if manifest has explicit mediaType field
if (manifest.mediaType) {
return manifest.mediaType;
}
// Otherwise detect from structure
if (Array.isArray(manifest.manifests)) {
// OCI Image Index (multi-arch manifest list)
return 'application/vnd.oci.image.index.v1+json';
} else if (manifest.config) {
// OCI Image Manifest
return 'application/vnd.oci.image.manifest.v1+json';
}
// Fallback to standard manifest type
return 'application/vnd.oci.image.manifest.v1+json';
} catch (e) {
// If parsing fails, return default
return 'application/vnd.oci.image.manifest.v1+json';
}
}
/**
* Convert any binary-like data to Buffer.
* Handles Buffer, Uint8Array (modern cross-platform), string, and objects.
*
* Note: Buffer.isBuffer(Uint8Array) returns false even though Buffer extends Uint8Array.
* This is because Uint8Array is the modern, cross-platform standard while Buffer is Node.js-specific.
* Many HTTP frameworks pass request bodies as Uint8Array for better compatibility.
*/
private toBuffer(data: unknown): Buffer {
if (Buffer.isBuffer(data)) {
return data;
}
if (data instanceof Uint8Array) {
return Buffer.from(data);
}
if (typeof data === 'string') {
return Buffer.from(data, 'utf-8');
}
// Fallback: serialize object to JSON (may cause digest mismatch for manifests)
return Buffer.from(JSON.stringify(data));
}
private async getTagsData(repository: string): Promise<Record<string, string>> {
const path = `oci/tags/${repository}/tags.json`;
const data = await this.storage.getObject(path);
@@ -697,7 +810,7 @@ export class OciRegistry extends BaseRegistry {
}
private generateUploadId(): string {
return `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
return `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
}
private async calculateDigest(data: Buffer): Promise<string> {
@@ -712,6 +825,37 @@ export class OciRegistry extends BaseRegistry {
};
}
/**
* Create an unauthorized response with proper WWW-Authenticate header.
* Per OCI Distribution Spec, 401 responses MUST include WWW-Authenticate header.
*/
private createUnauthorizedResponse(repository: string, action: string): IResponse {
const realm = this.ociTokens?.realm || `${this.basePath}/v2/token`;
const service = this.ociTokens?.service || 'registry';
return {
status: 401,
headers: {
'WWW-Authenticate': `Bearer realm="${realm}",service="${service}",scope="repository:${repository}:${action}"`,
},
body: this.createError('DENIED', 'Insufficient permissions'),
};
}
/**
* Create an unauthorized HEAD response (no body per HTTP spec).
*/
private createUnauthorizedHeadResponse(repository: string, action: string): IResponse {
const realm = this.ociTokens?.realm || `${this.basePath}/v2/token`;
const service = this.ociTokens?.service || 'registry';
return {
status: 401,
headers: {
'WWW-Authenticate': `Bearer realm="${realm}",service="${service}",scope="repository:${repository}:${action}"`,
},
body: null,
};
}
private startUploadSessionCleanup(): void {
this.cleanupInterval = setInterval(() => {
const now = new Date();

View File

@@ -0,0 +1,263 @@
import * as plugins from '../plugins.js';
import { BaseUpstream } from '../upstream/classes.baseupstream.js';
import type {
IProtocolUpstreamConfig,
IUpstreamFetchContext,
IUpstreamResult,
IUpstreamRegistryConfig,
} from '../upstream/interfaces.upstream.js';
import type { IOciManifest, IOciImageIndex, ITagList } from './interfaces.oci.js';
/**
* OCI-specific upstream implementation.
*
* Handles:
* - Manifest fetching (image manifests and index manifests)
* - Blob proxying (layers, configs)
* - Tag list fetching
* - Content-addressable caching (blobs are immutable)
* - Docker Hub authentication flow
*/
export class OciUpstream extends BaseUpstream {
protected readonly protocolName = 'oci';
/** Local registry base path for URL building */
private readonly localBasePath: string;
constructor(
config: IProtocolUpstreamConfig,
localBasePath: string = '/oci',
logger?: plugins.smartlog.Smartlog,
) {
super(config, logger);
this.localBasePath = localBasePath;
}
/**
* Fetch a manifest from upstream registries.
*/
public async fetchManifest(
repository: string,
reference: string,
): Promise<{ manifest: IOciManifest | IOciImageIndex; contentType: string; digest: string } | null> {
const context: IUpstreamFetchContext = {
protocol: 'oci',
resource: repository,
resourceType: 'manifest',
path: `/v2/${repository}/manifests/${reference}`,
method: 'GET',
headers: {
'accept': [
'application/vnd.oci.image.manifest.v1+json',
'application/vnd.oci.image.index.v1+json',
'application/vnd.docker.distribution.manifest.v2+json',
'application/vnd.docker.distribution.manifest.list.v2+json',
'application/vnd.docker.distribution.manifest.v1+json',
].join(', '),
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
let manifest: IOciManifest | IOciImageIndex;
if (Buffer.isBuffer(result.body)) {
manifest = JSON.parse(result.body.toString('utf8'));
} else {
manifest = result.body;
}
const contentType = result.headers['content-type'] || 'application/vnd.oci.image.manifest.v1+json';
const digest = result.headers['docker-content-digest'] || '';
return { manifest, contentType, digest };
}
/**
* Check if a manifest exists in upstream (HEAD request).
*/
public async headManifest(
repository: string,
reference: string,
): Promise<{ exists: boolean; contentType?: string; digest?: string; size?: number } | null> {
const context: IUpstreamFetchContext = {
protocol: 'oci',
resource: repository,
resourceType: 'manifest',
path: `/v2/${repository}/manifests/${reference}`,
method: 'HEAD',
headers: {
'accept': [
'application/vnd.oci.image.manifest.v1+json',
'application/vnd.oci.image.index.v1+json',
'application/vnd.docker.distribution.manifest.v2+json',
'application/vnd.docker.distribution.manifest.list.v2+json',
].join(', '),
},
query: {},
};
const result = await this.fetch(context);
if (!result) {
return null;
}
if (!result.success) {
return { exists: false };
}
return {
exists: true,
contentType: result.headers['content-type'],
digest: result.headers['docker-content-digest'],
size: result.headers['content-length'] ? parseInt(result.headers['content-length'], 10) : undefined,
};
}
/**
* Fetch a blob from upstream registries.
*/
public async fetchBlob(repository: string, digest: string): Promise<Buffer | null> {
const context: IUpstreamFetchContext = {
protocol: 'oci',
resource: repository,
resourceType: 'blob',
path: `/v2/${repository}/blobs/${digest}`,
method: 'GET',
headers: {
'accept': 'application/octet-stream',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
return Buffer.isBuffer(result.body) ? result.body : Buffer.from(result.body);
}
/**
* Check if a blob exists in upstream (HEAD request).
*/
public async headBlob(
repository: string,
digest: string,
): Promise<{ exists: boolean; size?: number } | null> {
const context: IUpstreamFetchContext = {
protocol: 'oci',
resource: repository,
resourceType: 'blob',
path: `/v2/${repository}/blobs/${digest}`,
method: 'HEAD',
headers: {},
query: {},
};
const result = await this.fetch(context);
if (!result) {
return null;
}
if (!result.success) {
return { exists: false };
}
return {
exists: true,
size: result.headers['content-length'] ? parseInt(result.headers['content-length'], 10) : undefined,
};
}
/**
* Fetch the tag list for a repository.
*/
public async fetchTags(repository: string, n?: number, last?: string): Promise<ITagList | null> {
const query: Record<string, string> = {};
if (n) query.n = n.toString();
if (last) query.last = last;
const context: IUpstreamFetchContext = {
protocol: 'oci',
resource: repository,
resourceType: 'tags',
path: `/v2/${repository}/tags/list`,
method: 'GET',
headers: {
'accept': 'application/json',
},
query,
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
let tagList: ITagList;
if (Buffer.isBuffer(result.body)) {
tagList = JSON.parse(result.body.toString('utf8'));
} else {
tagList = result.body;
}
return tagList;
}
/**
* Override URL building for OCI-specific handling.
* OCI registries use /v2/ prefix and may require special handling for Docker Hub.
*/
protected buildUpstreamUrl(
upstream: IUpstreamRegistryConfig,
context: IUpstreamFetchContext,
): string {
let baseUrl = upstream.url;
// Remove trailing slash
if (baseUrl.endsWith('/')) {
baseUrl = baseUrl.slice(0, -1);
}
// Handle Docker Hub special case
// Docker Hub uses registry-1.docker.io but library images need special handling
if (baseUrl.includes('docker.io') || baseUrl.includes('registry-1.docker.io')) {
// For library images (e.g., "nginx" -> "library/nginx")
const pathParts = context.path.match(/^\/v2\/([^\/]+)\/(.+)$/);
if (pathParts) {
const [, repository, rest] = pathParts;
// If repository doesn't contain a slash, it's a library image
if (!repository.includes('/')) {
return `${baseUrl}/v2/library/${repository}/${rest}`;
}
}
}
return `${baseUrl}${context.path}`;
}
/**
* Override header building for OCI-specific authentication.
* OCI registries may require token-based auth obtained from a separate endpoint.
*/
protected buildHeaders(
upstream: IUpstreamRegistryConfig,
context: IUpstreamFetchContext,
): Record<string, string> {
const headers = super.buildHeaders(upstream, context);
// OCI registries typically use Docker-Distribution-API-Version header
headers['docker-distribution-api-version'] = 'registry/2.0';
return headers;
}
}

View File

@@ -3,4 +3,5 @@
*/
export { OciRegistry } from './classes.ociregistry.js';
export { OciUpstream } from './classes.ociupstream.js';
export * from './interfaces.oci.js';

View File

@@ -4,13 +4,20 @@ import * as path from 'path';
export { path };
// @push.rocks scope
import * as smartarchive from '@push.rocks/smartarchive';
import * as smartbucket from '@push.rocks/smartbucket';
import * as smartlog from '@push.rocks/smartlog';
import * as smartpath from '@push.rocks/smartpath';
import * as smartrequest from '@push.rocks/smartrequest';
export { smartbucket, smartlog, smartpath };
export { smartarchive, smartbucket, smartlog, smartpath, smartrequest };
// @tsclass scope
import * as tsclass from '@tsclass/tsclass';
export { tsclass };
// third party
import { minimatch } from 'minimatch';
export { minimatch };

View File

@@ -3,6 +3,8 @@ import { BaseRegistry } from '../core/classes.baseregistry.js';
import { RegistryStorage } from '../core/classes.registrystorage.js';
import { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js';
import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js';
import { isBinaryData, toBuffer } from '../core/helpers.buffer.js';
import type {
IPypiPackageMetadata,
IPypiFile,
@@ -10,6 +12,7 @@ import type {
IPypiUploadResponse,
} from './interfaces.pypi.js';
import * as helpers from './helpers.pypi.js';
import { PypiUpstream } from './classes.pypiupstream.js';
/**
* PyPI registry implementation
@@ -21,12 +24,14 @@ export class PypiRegistry extends BaseRegistry {
private basePath: string = '/pypi';
private registryUrl: string;
private logger: Smartlog;
private upstream: PypiUpstream | null = null;
constructor(
storage: RegistryStorage,
authManager: AuthManager,
basePath: string = '/pypi',
registryUrl: string = 'http://localhost:5000'
registryUrl: string = 'http://localhost:5000',
upstreamConfig?: IProtocolUpstreamConfig
) {
super();
this.storage = storage;
@@ -46,6 +51,20 @@ export class PypiRegistry extends BaseRegistry {
}
});
this.logger.enableConsole();
// Initialize upstream if configured
if (upstreamConfig?.enabled) {
this.upstream = new PypiUpstream(upstreamConfig, registryUrl, this.logger);
}
}
/**
* Clean up resources (timers, connections, etc.)
*/
public destroy(): void {
if (this.upstream) {
this.upstream.stop();
}
}
public async init(): Promise<void> {
@@ -85,14 +104,14 @@ export class PypiRegistry extends BaseRegistry {
return this.handleUpload(context, token);
}
// Package metadata JSON API: GET /pypi/{package}/json
const jsonMatch = path.match(/^\/pypi\/([^\/]+)\/json$/);
// Package metadata JSON API: GET /{package}/json
const jsonMatch = path.match(/^\/([^\/]+)\/json$/);
if (jsonMatch && context.method === 'GET') {
return this.handlePackageJson(jsonMatch[1]);
}
// Version-specific JSON API: GET /pypi/{package}/{version}/json
const versionJsonMatch = path.match(/^\/pypi\/([^\/]+)\/([^\/]+)\/json$/);
// Version-specific JSON API: GET /{package}/{version}/json
const versionJsonMatch = path.match(/^\/([^\/]+)\/([^\/]+)\/json$/);
if (versionJsonMatch && context.method === 'GET') {
return this.handleVersionJson(versionJsonMatch[1], versionJsonMatch[2]);
}
@@ -118,7 +137,7 @@ export class PypiRegistry extends BaseRegistry {
return {
status: 404,
headers: { 'Content-Type': 'application/json' },
body: Buffer.from(JSON.stringify({ message: 'Not Found' })),
body: { error: 'Not Found' },
};
}
@@ -185,7 +204,7 @@ export class PypiRegistry extends BaseRegistry {
'Content-Type': 'application/vnd.pypi.simple.v1+json',
'Cache-Control': 'public, max-age=600'
},
body: Buffer.from(JSON.stringify(response)),
body: response,
};
} else {
// PEP 503: HTML response
@@ -200,7 +219,7 @@ export class PypiRegistry extends BaseRegistry {
'Content-Type': 'text/html; charset=utf-8',
'Cache-Control': 'public, max-age=600'
},
body: Buffer.from(html),
body: html,
};
}
}
@@ -213,14 +232,48 @@ export class PypiRegistry extends BaseRegistry {
const normalized = helpers.normalizePypiPackageName(packageName);
// Get package metadata
const metadata = await this.storage.getPypiPackageMetadata(normalized);
if (!metadata) {
let metadata = await this.storage.getPypiPackageMetadata(normalized);
// Try upstream if not found locally
if (!metadata && this.upstream) {
const upstreamHtml = await this.upstream.fetchSimplePackage(normalized);
if (upstreamHtml) {
// Parse the HTML to extract file information and cache it
// For now, just return the upstream HTML directly (caching can be improved later)
const acceptHeader = context.headers['accept'] || context.headers['Accept'] || '';
const preferJson = acceptHeader.includes('application/vnd.pypi.simple') &&
acceptHeader.includes('json');
if (preferJson) {
// Try to get JSON format from upstream
const upstreamJson = await this.upstream.fetchPackageJson(normalized);
if (upstreamJson) {
return {
status: 404,
headers: { 'Content-Type': 'text/html; charset=utf-8' },
body: Buffer.from('<html><body><h1>404 Not Found</h1></body></html>'),
status: 200,
headers: {
'Content-Type': 'application/vnd.pypi.simple.v1+json',
'Cache-Control': 'public, max-age=300'
},
body: upstreamJson,
};
}
}
// Return HTML format
return {
status: 200,
headers: {
'Content-Type': 'text/html; charset=utf-8',
'Cache-Control': 'public, max-age=300'
},
body: upstreamHtml,
};
}
}
if (!metadata) {
return this.errorResponse(404, 'Package not found');
}
// Build file list from all versions
const files: IPypiFile[] = [];
@@ -251,7 +304,7 @@ export class PypiRegistry extends BaseRegistry {
'Content-Type': 'application/vnd.pypi.simple.v1+json',
'Cache-Control': 'public, max-age=300'
},
body: Buffer.from(JSON.stringify(response)),
body: response,
};
} else {
// PEP 503: HTML response
@@ -266,7 +319,7 @@ export class PypiRegistry extends BaseRegistry {
'Content-Type': 'text/html; charset=utf-8',
'Cache-Control': 'public, max-age=300'
},
body: Buffer.from(html),
body: html,
};
}
}
@@ -315,7 +368,7 @@ export class PypiRegistry extends BaseRegistry {
'Content-Type': 'application/json',
'WWW-Authenticate': 'Basic realm="PyPI"'
},
body: Buffer.from(JSON.stringify({ message: 'Authentication required' })),
body: { error: 'Authentication required' },
};
}
@@ -327,11 +380,14 @@ export class PypiRegistry extends BaseRegistry {
return this.errorResponse(400, 'Invalid upload request');
}
// Extract required fields
// Extract required fields - support both nested and flat body formats
const packageName = formData.name;
const version = formData.version;
const filename = formData.content?.filename;
const fileData = formData.content?.data as Buffer;
// Support both: formData.content.filename (multipart parsed) and formData.filename (flat)
const filename = formData.content?.filename || formData.filename;
// Support both: formData.content.data (multipart parsed) and formData.content (Buffer/Uint8Array directly)
const rawContent = formData.content?.data || (isBinaryData(formData.content) ? formData.content : null);
const fileData = rawContent ? toBuffer(rawContent) : null;
const filetype = formData.filetype; // 'bdist_wheel' or 'sdist'
const pyversion = formData.pyversion;
@@ -431,12 +487,12 @@ export class PypiRegistry extends BaseRegistry {
});
return {
status: 200,
status: 201,
headers: { 'Content-Type': 'application/json' },
body: Buffer.from(JSON.stringify({
body: {
message: 'Package uploaded successfully',
url: `${this.registryUrl}/pypi/packages/${normalized}/${filename}`
})),
},
};
} catch (error) {
this.logger.log('error', 'Upload failed', { error: (error as Error).message });
@@ -449,13 +505,22 @@ export class PypiRegistry extends BaseRegistry {
*/
private async handleDownload(packageName: string, filename: string): Promise<IResponse> {
const normalized = helpers.normalizePypiPackageName(packageName);
const fileData = await this.storage.getPypiPackageFile(normalized, filename);
let fileData = await this.storage.getPypiPackageFile(normalized, filename);
// Try upstream if not found locally
if (!fileData && this.upstream) {
fileData = await this.upstream.fetchPackageFile(normalized, filename);
if (fileData) {
// Cache locally
await this.storage.putPypiPackageFile(normalized, filename, fileData);
}
}
if (!fileData) {
return {
status: 404,
headers: { 'Content-Type': 'application/json' },
body: Buffer.from(JSON.stringify({ message: 'File not found' })),
body: { error: 'File not found' },
};
}
@@ -472,6 +537,7 @@ export class PypiRegistry extends BaseRegistry {
/**
* Handle package JSON API (all versions)
* Returns format compatible with official PyPI JSON API
*/
private async handlePackageJson(packageName: string): Promise<IResponse> {
const normalized = helpers.normalizePypiPackageName(packageName);
@@ -481,18 +547,67 @@ export class PypiRegistry extends BaseRegistry {
return this.errorResponse(404, 'Package not found');
}
// Find latest version for info
const versions = Object.keys(metadata.versions || {});
const latestVersion = versions.length > 0 ? versions[versions.length - 1] : null;
const latestMeta = latestVersion ? metadata.versions[latestVersion] : null;
// Build URLs array from latest version files
const urls = latestMeta?.files?.map((file: any) => ({
filename: file.filename,
url: `${this.registryUrl}/pypi/packages/${normalized}/${file.filename}`,
digests: file.hashes,
requires_python: file['requires-python'],
size: file.size,
upload_time: file['upload-time'],
packagetype: file.filetype,
python_version: file.python_version,
})) || [];
// Build releases object
const releases: Record<string, any[]> = {};
for (const [ver, verMeta] of Object.entries(metadata.versions || {})) {
releases[ver] = (verMeta as any).files?.map((file: any) => ({
filename: file.filename,
url: `${this.registryUrl}/pypi/packages/${normalized}/${file.filename}`,
digests: file.hashes,
requires_python: file['requires-python'],
size: file.size,
upload_time: file['upload-time'],
packagetype: file.filetype,
python_version: file.python_version,
})) || [];
}
const response = {
info: {
name: normalized,
version: latestVersion,
summary: latestMeta?.metadata?.summary,
description: latestMeta?.metadata?.description,
author: latestMeta?.metadata?.author,
author_email: latestMeta?.metadata?.['author-email'],
license: latestMeta?.metadata?.license,
requires_python: latestMeta?.files?.[0]?.['requires-python'],
...latestMeta?.metadata,
},
urls,
releases,
};
return {
status: 200,
headers: {
'Content-Type': 'application/json',
'Cache-Control': 'public, max-age=300'
},
body: Buffer.from(JSON.stringify(metadata)),
body: response,
};
}
/**
* Handle version-specific JSON API
* Returns format compatible with official PyPI JSON API
*/
private async handleVersionJson(packageName: string, version: string): Promise<IResponse> {
const normalized = helpers.normalizePypiPackageName(packageName);
@@ -502,13 +617,42 @@ export class PypiRegistry extends BaseRegistry {
return this.errorResponse(404, 'Version not found');
}
const verMeta = metadata.versions[version];
// Build URLs array from version files
const urls = verMeta.files?.map((file: any) => ({
filename: file.filename,
url: `${this.registryUrl}/pypi/packages/${normalized}/${file.filename}`,
digests: file.hashes,
requires_python: file['requires-python'],
size: file.size,
upload_time: file['upload-time'],
packagetype: file.filetype,
python_version: file.python_version,
})) || [];
const response = {
info: {
name: normalized,
version,
summary: verMeta.metadata?.summary,
description: verMeta.metadata?.description,
author: verMeta.metadata?.author,
author_email: verMeta.metadata?.['author-email'],
license: verMeta.metadata?.license,
requires_python: verMeta.files?.[0]?.['requires-python'],
...verMeta.metadata,
},
urls,
};
return {
status: 200,
headers: {
'Content-Type': 'application/json',
'Cache-Control': 'public, max-age=300'
},
body: Buffer.from(JSON.stringify(metadata.versions[version])),
body: response,
};
}
@@ -570,11 +714,11 @@ export class PypiRegistry extends BaseRegistry {
* Helper: Create error response
*/
private errorResponse(status: number, message: string): IResponse {
const error: IPypiError = { message, status };
const error: IPypiError = { error: message, status };
return {
status,
headers: { 'Content-Type': 'application/json' },
body: Buffer.from(JSON.stringify(error)),
body: error,
};
}
}

View File

@@ -0,0 +1,211 @@
import * as plugins from '../plugins.js';
import { BaseUpstream } from '../upstream/classes.baseupstream.js';
import type {
IProtocolUpstreamConfig,
IUpstreamFetchContext,
IUpstreamRegistryConfig,
} from '../upstream/interfaces.upstream.js';
/**
* PyPI-specific upstream implementation.
*
* Handles:
* - Simple API (HTML) - PEP 503
* - JSON API - PEP 691
* - Package file downloads (wheels, sdists)
* - Package name normalization
*/
export class PypiUpstream extends BaseUpstream {
protected readonly protocolName = 'pypi';
/** Local registry URL for rewriting download URLs */
private readonly localRegistryUrl: string;
constructor(
config: IProtocolUpstreamConfig,
localRegistryUrl: string,
logger?: plugins.smartlog.Smartlog,
) {
super(config, logger);
this.localRegistryUrl = localRegistryUrl;
}
/**
* Fetch Simple API index (list of all packages) in HTML format.
*/
public async fetchSimpleIndex(): Promise<string | null> {
const context: IUpstreamFetchContext = {
protocol: 'pypi',
resource: '*',
resourceType: 'index',
path: '/simple/',
method: 'GET',
headers: {
'accept': 'text/html',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
if (Buffer.isBuffer(result.body)) {
return result.body.toString('utf8');
}
return typeof result.body === 'string' ? result.body : null;
}
/**
* Fetch Simple API package page (list of files) in HTML format.
*/
public async fetchSimplePackage(packageName: string): Promise<string | null> {
const normalizedName = this.normalizePackageName(packageName);
const path = `/simple/${normalizedName}/`;
const context: IUpstreamFetchContext = {
protocol: 'pypi',
resource: packageName,
resourceType: 'simple',
path,
method: 'GET',
headers: {
'accept': 'text/html',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
if (Buffer.isBuffer(result.body)) {
return result.body.toString('utf8');
}
return typeof result.body === 'string' ? result.body : null;
}
/**
* Fetch package metadata using JSON API (PEP 691).
*/
public async fetchPackageJson(packageName: string): Promise<any | null> {
const normalizedName = this.normalizePackageName(packageName);
const path = `/simple/${normalizedName}/`;
const context: IUpstreamFetchContext = {
protocol: 'pypi',
resource: packageName,
resourceType: 'metadata',
path,
method: 'GET',
headers: {
'accept': 'application/vnd.pypi.simple.v1+json',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
if (Buffer.isBuffer(result.body)) {
return JSON.parse(result.body.toString('utf8'));
}
return result.body;
}
/**
* Fetch full package info from PyPI JSON API (/pypi/{package}/json).
*/
public async fetchPypiJson(packageName: string): Promise<any | null> {
const normalizedName = this.normalizePackageName(packageName);
const path = `/pypi/${normalizedName}/json`;
const context: IUpstreamFetchContext = {
protocol: 'pypi',
resource: packageName,
resourceType: 'pypi-json',
path,
method: 'GET',
headers: {
'accept': 'application/json',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
if (Buffer.isBuffer(result.body)) {
return JSON.parse(result.body.toString('utf8'));
}
return result.body;
}
/**
* Fetch a package file (wheel or sdist) from upstream.
*/
public async fetchPackageFile(packageName: string, filename: string): Promise<Buffer | null> {
const normalizedName = this.normalizePackageName(packageName);
const path = `/packages/${normalizedName}/${filename}`;
const context: IUpstreamFetchContext = {
protocol: 'pypi',
resource: packageName,
resourceType: 'package',
path,
method: 'GET',
headers: {
'accept': 'application/octet-stream',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
return Buffer.isBuffer(result.body) ? result.body : Buffer.from(result.body);
}
/**
* Normalize a PyPI package name according to PEP 503.
* - Lowercase all characters
* - Replace runs of ., -, _ with single -
*/
private normalizePackageName(name: string): string {
return name.toLowerCase().replace(/[-_.]+/g, '-');
}
/**
* Override URL building for PyPI-specific handling.
*/
protected buildUpstreamUrl(
upstream: IUpstreamRegistryConfig,
context: IUpstreamFetchContext,
): string {
let baseUrl = upstream.url;
// Remove trailing slash
if (baseUrl.endsWith('/')) {
baseUrl = baseUrl.slice(0, -1);
}
return `${baseUrl}${context.path}`;
}
}

View File

@@ -5,4 +5,5 @@
export * from './interfaces.pypi.js';
export * from './classes.pypiregistry.js';
export { PypiUpstream } from './classes.pypiupstream.js';
export * as pypiHelpers from './helpers.pypi.js';

View File

@@ -244,7 +244,7 @@ export interface IPypiUploadResponse {
*/
export interface IPypiError {
/** Error message */
message: string;
error: string;
/** HTTP status code */
status?: number;
/** Additional error details */

View File

@@ -3,6 +3,7 @@ import { BaseRegistry } from '../core/classes.baseregistry.js';
import { RegistryStorage } from '../core/classes.registrystorage.js';
import { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js';
import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js';
import type {
IRubyGemsMetadata,
IRubyGemsVersionMetadata,
@@ -12,6 +13,7 @@ import type {
ICompactIndexInfoEntry,
} from './interfaces.rubygems.js';
import * as helpers from './helpers.rubygems.js';
import { RubygemsUpstream } from './classes.rubygemsupstream.js';
/**
* RubyGems registry implementation
@@ -23,12 +25,14 @@ export class RubyGemsRegistry extends BaseRegistry {
private basePath: string = '/rubygems';
private registryUrl: string;
private logger: Smartlog;
private upstream: RubygemsUpstream | null = null;
constructor(
storage: RegistryStorage,
authManager: AuthManager,
basePath: string = '/rubygems',
registryUrl: string = 'http://localhost:5000/rubygems'
registryUrl: string = 'http://localhost:5000/rubygems',
upstreamConfig?: IProtocolUpstreamConfig
) {
super();
this.storage = storage;
@@ -48,6 +52,20 @@ export class RubyGemsRegistry extends BaseRegistry {
}
});
this.logger.enableConsole();
// Initialize upstream if configured
if (upstreamConfig?.enabled) {
this.upstream = new RubygemsUpstream(upstreamConfig, this.logger);
}
}
/**
* Clean up resources (timers, connections, etc.)
*/
public destroy(): void {
if (this.upstream) {
this.upstream.stop();
}
}
public async init(): Promise<void> {
@@ -85,7 +103,7 @@ export class RubyGemsRegistry extends BaseRegistry {
// Compact Index endpoints
if (path === '/versions' && context.method === 'GET') {
return this.handleVersionsFile();
return this.handleVersionsFile(context);
}
if (path === '/names' && context.method === 'GET') {
@@ -104,15 +122,30 @@ export class RubyGemsRegistry extends BaseRegistry {
return this.handleDownload(downloadMatch[1]);
}
// Legacy specs endpoints (Marshal format)
if (path === '/specs.4.8.gz' && context.method === 'GET') {
return this.handleSpecs(false);
}
if (path === '/latest_specs.4.8.gz' && context.method === 'GET') {
return this.handleSpecs(true);
}
// Quick gemspec endpoint: GET /quick/Marshal.4.8/{gem}-{version}.gemspec.rz
const quickMatch = path.match(/^\/quick\/Marshal\.4\.8\/(.+)\.gemspec\.rz$/);
if (quickMatch && context.method === 'GET') {
return this.handleQuickGemspec(quickMatch[1]);
}
// API v1 endpoints
if (path.startsWith('/api/v1/')) {
return this.handleApiRequest(path.substring(8), context, token);
return this.handleApiRequest(path.substring(7), context, token);
}
return {
status: 404,
headers: { 'Content-Type': 'application/json' },
body: Buffer.from(JSON.stringify({ message: 'Not Found' })),
body: { error: 'Not Found' },
};
}
@@ -141,20 +174,36 @@ export class RubyGemsRegistry extends BaseRegistry {
/**
* Handle /versions endpoint (Compact Index)
* Supports conditional GET with If-None-Match header
*/
private async handleVersionsFile(): Promise<IResponse> {
private async handleVersionsFile(context: IRequestContext): Promise<IResponse> {
const content = await this.storage.getRubyGemsVersions();
if (!content) {
return this.errorResponse(500, 'Versions file not initialized');
}
const etag = `"${await helpers.calculateMD5(content)}"`;
// Handle conditional GET with If-None-Match
const ifNoneMatch = context.headers['if-none-match'] || context.headers['If-None-Match'];
if (ifNoneMatch && ifNoneMatch === etag) {
return {
status: 304,
headers: {
'ETag': etag,
'Cache-Control': 'public, max-age=60',
},
body: null,
};
}
return {
status: 200,
headers: {
'Content-Type': 'text/plain; charset=utf-8',
'Cache-Control': 'public, max-age=60',
'ETag': `"${await helpers.calculateMD5(content)}"`
'ETag': etag
},
body: Buffer.from(content),
};
@@ -184,7 +233,17 @@ export class RubyGemsRegistry extends BaseRegistry {
* Handle /info/{gem} endpoint (Compact Index)
*/
private async handleInfoFile(gemName: string): Promise<IResponse> {
const content = await this.storage.getRubyGemsInfo(gemName);
let content = await this.storage.getRubyGemsInfo(gemName);
// Try upstream if not found locally
if (!content && this.upstream) {
const upstreamInfo = await this.upstream.fetchInfo(gemName);
if (upstreamInfo) {
// Cache locally
await this.storage.putRubyGemsInfo(gemName, upstreamInfo);
content = upstreamInfo;
}
}
if (!content) {
return {
@@ -214,12 +273,21 @@ export class RubyGemsRegistry extends BaseRegistry {
return this.errorResponse(400, 'Invalid gem filename');
}
const gemData = await this.storage.getRubyGemsGem(
let gemData = await this.storage.getRubyGemsGem(
parsed.name,
parsed.version,
parsed.platform
);
// Try upstream if not found locally
if (!gemData && this.upstream) {
gemData = await this.upstream.fetchGem(parsed.name, parsed.version);
if (gemData) {
// Cache locally
await this.storage.putRubyGemsGem(parsed.name, parsed.version, gemData, parsed.platform);
}
}
if (!gemData) {
return this.errorResponse(404, 'Gem not found');
}
@@ -289,14 +357,23 @@ export class RubyGemsRegistry extends BaseRegistry {
return this.errorResponse(400, 'No gem file provided');
}
// For now, we expect metadata in query params or headers
// Full implementation would parse .gem file (tar + gzip + Marshal)
const gemName = context.query?.name || context.headers['x-gem-name'];
const version = context.query?.version || context.headers['x-gem-version'];
const platform = context.query?.platform || context.headers['x-gem-platform'];
// Try to get metadata from query params or headers first
let gemName = context.query?.name || context.headers['x-gem-name'] as string | undefined;
let version = context.query?.version || context.headers['x-gem-version'] as string | undefined;
let platform = context.query?.platform || context.headers['x-gem-platform'] as string | undefined;
// If not provided, try to extract from gem binary
if (!gemName || !version || !platform) {
const extracted = await helpers.extractGemMetadata(gemData);
if (extracted) {
gemName = gemName || extracted.name;
version = version || extracted.version;
platform = platform || extracted.platform;
}
}
if (!gemName || !version) {
return this.errorResponse(400, 'Gem name and version required');
return this.errorResponse(400, 'Gem name and version required (provide in query, headers, or valid gem format)');
}
// Validate gem name
@@ -351,13 +428,13 @@ export class RubyGemsRegistry extends BaseRegistry {
});
return {
status: 200,
status: 201,
headers: { 'Content-Type': 'application/json' },
body: Buffer.from(JSON.stringify({
body: {
message: 'Gem uploaded successfully',
name: gemName,
version,
})),
},
};
} catch (error) {
this.logger.log('error', 'Upload failed', { error: (error as Error).message });
@@ -409,10 +486,10 @@ export class RubyGemsRegistry extends BaseRegistry {
return {
status: 200,
headers: { 'Content-Type': 'application/json' },
body: Buffer.from(JSON.stringify({
body: {
success: true,
message: 'Gem yanked successfully'
})),
},
};
}
@@ -459,10 +536,10 @@ export class RubyGemsRegistry extends BaseRegistry {
return {
status: 200,
headers: { 'Content-Type': 'application/json' },
body: Buffer.from(JSON.stringify({
body: {
success: true,
message: 'Gem unyanked successfully'
})),
},
};
}
@@ -489,7 +566,7 @@ export class RubyGemsRegistry extends BaseRegistry {
'Content-Type': 'application/json',
'Cache-Control': 'public, max-age=300'
},
body: Buffer.from(JSON.stringify(response)),
body: response,
};
}
@@ -517,7 +594,7 @@ export class RubyGemsRegistry extends BaseRegistry {
return {
status: 200,
headers: { 'Content-Type': 'application/json' },
body: Buffer.from(JSON.stringify(response)),
body: response,
};
}
@@ -584,15 +661,109 @@ export class RubyGemsRegistry extends BaseRegistry {
}
}
/**
* Handle /specs.4.8.gz and /latest_specs.4.8.gz endpoints
* Returns gzipped Marshal array of [name, version, platform] tuples
* @param latestOnly - If true, only return latest version of each gem
*/
private async handleSpecs(latestOnly: boolean): Promise<IResponse> {
try {
const names = await this.storage.getRubyGemsNames();
if (!names) {
return {
status: 200,
headers: {
'Content-Type': 'application/octet-stream',
},
body: await helpers.generateSpecsGz([]),
};
}
const gemNames = names.split('\n').filter(l => l && l !== '---');
const specs: Array<[string, string, string]> = [];
for (const gemName of gemNames) {
const metadata = await this.storage.getRubyGemsMetadata(gemName);
if (!metadata) continue;
const versions = (Object.values(metadata.versions) as IRubyGemsVersionMetadata[])
.filter(v => !v.yanked)
.sort((a, b) => {
// Sort by version descending
return b.version.localeCompare(a.version, undefined, { numeric: true });
});
if (latestOnly && versions.length > 0) {
// Only include latest version
const latest = versions[0];
specs.push([gemName, latest.version, latest.platform || 'ruby']);
} else {
// Include all versions
for (const v of versions) {
specs.push([gemName, v.version, v.platform || 'ruby']);
}
}
}
const gzippedSpecs = await helpers.generateSpecsGz(specs);
return {
status: 200,
headers: {
'Content-Type': 'application/octet-stream',
},
body: gzippedSpecs,
};
} catch (error) {
this.logger.log('error', 'Failed to generate specs', { error: (error as Error).message });
return this.errorResponse(500, 'Failed to generate specs');
}
}
/**
* Handle /quick/Marshal.4.8/{gem}-{version}.gemspec.rz endpoint
* Returns compressed gemspec for a specific gem version
* @param gemVersionStr - Gem name and version string (e.g., "rails-7.0.0" or "rails-7.0.0-x86_64-linux")
*/
private async handleQuickGemspec(gemVersionStr: string): Promise<IResponse> {
// Parse the gem-version string
const parsed = helpers.parseGemFilename(gemVersionStr + '.gem');
if (!parsed) {
return this.errorResponse(400, 'Invalid gemspec path');
}
const metadata = await this.storage.getRubyGemsMetadata(parsed.name);
if (!metadata) {
return this.errorResponse(404, 'Gem not found');
}
const versionKey = parsed.platform ? `${parsed.version}-${parsed.platform}` : parsed.version;
const versionMeta = metadata.versions[versionKey];
if (!versionMeta) {
return this.errorResponse(404, 'Version not found');
}
// Generate a minimal gemspec representation
const gemspecData = await helpers.generateGemspecRz(parsed.name, versionMeta);
return {
status: 200,
headers: {
'Content-Type': 'application/octet-stream',
},
body: gemspecData,
};
}
/**
* Helper: Create error response
*/
private errorResponse(status: number, message: string): IResponse {
const error: IRubyGemsError = { message, status };
const error: IRubyGemsError = { error: message, status };
return {
status,
headers: { 'Content-Type': 'application/json' },
body: Buffer.from(JSON.stringify(error)),
body: error,
};
}
}

View File

@@ -0,0 +1,230 @@
import * as plugins from '../plugins.js';
import { BaseUpstream } from '../upstream/classes.baseupstream.js';
import type {
IProtocolUpstreamConfig,
IUpstreamFetchContext,
IUpstreamRegistryConfig,
} from '../upstream/interfaces.upstream.js';
/**
* RubyGems-specific upstream implementation.
*
* Handles:
* - Compact Index format (/versions, /info/{gem}, /names)
* - Gem file (.gem) downloading
* - Gem spec fetching
* - HTTP Range requests for incremental updates
*/
export class RubygemsUpstream extends BaseUpstream {
protected readonly protocolName = 'rubygems';
constructor(
config: IProtocolUpstreamConfig,
logger?: plugins.smartlog.Smartlog,
) {
super(config, logger);
}
/**
* Fetch the /versions file (master list of all gems).
*/
public async fetchVersions(etag?: string): Promise<{ data: string; etag?: string } | null> {
const headers: Record<string, string> = {
'accept': 'text/plain',
};
if (etag) {
headers['if-none-match'] = etag;
}
const context: IUpstreamFetchContext = {
protocol: 'rubygems',
resource: '*',
resourceType: 'versions',
path: '/versions',
method: 'GET',
headers,
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
let data: string;
if (Buffer.isBuffer(result.body)) {
data = result.body.toString('utf8');
} else if (typeof result.body === 'string') {
data = result.body;
} else {
return null;
}
return {
data,
etag: result.headers['etag'],
};
}
/**
* Fetch gem info file (/info/{gemname}).
*/
public async fetchInfo(gemName: string): Promise<string | null> {
const context: IUpstreamFetchContext = {
protocol: 'rubygems',
resource: gemName,
resourceType: 'info',
path: `/info/${gemName}`,
method: 'GET',
headers: {
'accept': 'text/plain',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
if (Buffer.isBuffer(result.body)) {
return result.body.toString('utf8');
}
return typeof result.body === 'string' ? result.body : null;
}
/**
* Fetch the /names file (list of all gem names).
*/
public async fetchNames(): Promise<string | null> {
const context: IUpstreamFetchContext = {
protocol: 'rubygems',
resource: '*',
resourceType: 'names',
path: '/names',
method: 'GET',
headers: {
'accept': 'text/plain',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
if (Buffer.isBuffer(result.body)) {
return result.body.toString('utf8');
}
return typeof result.body === 'string' ? result.body : null;
}
/**
* Fetch a gem file.
*/
public async fetchGem(gemName: string, version: string): Promise<Buffer | null> {
const path = `/gems/${gemName}-${version}.gem`;
const context: IUpstreamFetchContext = {
protocol: 'rubygems',
resource: gemName,
resourceType: 'gem',
path,
method: 'GET',
headers: {
'accept': 'application/octet-stream',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
return Buffer.isBuffer(result.body) ? result.body : Buffer.from(result.body);
}
/**
* Fetch gem spec (quick spec).
*/
public async fetchQuickSpec(gemName: string, version: string): Promise<Buffer | null> {
const path = `/quick/Marshal.4.8/${gemName}-${version}.gemspec.rz`;
const context: IUpstreamFetchContext = {
protocol: 'rubygems',
resource: gemName,
resourceType: 'spec',
path,
method: 'GET',
headers: {
'accept': 'application/octet-stream',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
return Buffer.isBuffer(result.body) ? result.body : Buffer.from(result.body);
}
/**
* Fetch gem versions JSON from API.
*/
public async fetchVersionsJson(gemName: string): Promise<any[] | null> {
const path = `/api/v1/versions/${gemName}.json`;
const context: IUpstreamFetchContext = {
protocol: 'rubygems',
resource: gemName,
resourceType: 'versions-json',
path,
method: 'GET',
headers: {
'accept': 'application/json',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
if (Buffer.isBuffer(result.body)) {
return JSON.parse(result.body.toString('utf8'));
}
return Array.isArray(result.body) ? result.body : null;
}
/**
* Override URL building for RubyGems-specific handling.
*/
protected buildUpstreamUrl(
upstream: IUpstreamRegistryConfig,
context: IUpstreamFetchContext,
): string {
let baseUrl = upstream.url;
// Remove trailing slash
if (baseUrl.endsWith('/')) {
baseUrl = baseUrl.slice(0, -1);
}
return `${baseUrl}${context.path}`;
}
}

View File

@@ -3,6 +3,8 @@
* Compact Index generation, dependency formatting, etc.
*/
import * as plugins from '../plugins.js';
import type {
IRubyGemsVersion,
IRubyGemsDependency,
@@ -396,3 +398,175 @@ export async function extractGemSpec(gemData: Buffer): Promise<any | null> {
return null;
}
}
/**
* Extract basic metadata from a gem file
* Gem files are plain tar archives (NOT gzipped) containing:
* - metadata.gz: gzipped YAML with gem specification
* - data.tar.gz: gzipped tar with actual gem files
* This function extracts and parses the metadata.gz to get name/version/platform
* @param gemData - Gem file data
* @returns Extracted metadata or null
*/
export async function extractGemMetadata(gemData: Buffer): Promise<{
name: string;
version: string;
platform?: string;
} | null> {
try {
// Step 1: Extract the plain tar archive to get metadata.gz
const smartArchive = plugins.smartarchive.SmartArchive.create();
const files = await smartArchive.buffer(gemData).toSmartFiles();
// Find metadata.gz
const metadataFile = files.find(f => f.path === 'metadata.gz' || f.relative === 'metadata.gz');
if (!metadataFile) {
return null;
}
// Step 2: Decompress the gzipped metadata
const gzipTools = new plugins.smartarchive.GzipTools();
const metadataYaml = await gzipTools.decompress(metadataFile.contentBuffer);
const yamlContent = metadataYaml.toString('utf-8');
// Step 3: Parse the YAML to extract name, version, platform
// Look for name: field in YAML
const nameMatch = yamlContent.match(/name:\s*([^\n\r]+)/);
// Look for version in Ruby YAML format: version: !ruby/object:Gem::Version\n version: X.X.X
const versionMatch = yamlContent.match(/version:\s*!ruby\/object:Gem::Version[\s\S]*?version:\s*['"]?([^'"\n\r]+)/);
// Also try simpler version format
const simpleVersionMatch = !versionMatch ? yamlContent.match(/^version:\s*['"]?(\d[^'"\n\r]*)/m) : null;
// Look for platform
const platformMatch = yamlContent.match(/platform:\s*([^\n\r]+)/);
const name = nameMatch?.[1]?.trim();
const version = versionMatch?.[1]?.trim() || simpleVersionMatch?.[1]?.trim();
const platform = platformMatch?.[1]?.trim();
if (name && version) {
return {
name,
version,
platform: platform && platform !== 'ruby' ? platform : undefined,
};
}
return null;
} catch (_error) {
// Error handled gracefully - return null and let caller handle missing metadata
return null;
}
}
/**
* Generate gzipped specs array for /specs.4.8.gz and /latest_specs.4.8.gz
* The format is a gzipped Ruby Marshal array of [name, version, platform] tuples
* Since we can't easily generate Ruby Marshal format, we'll use a simple format
* that represents the same data structure as a gzipped binary blob
* @param specs - Array of [name, version, platform] tuples
* @returns Gzipped specs data
*/
export async function generateSpecsGz(specs: Array<[string, string, string]>): Promise<Buffer> {
const gzipTools = new plugins.smartarchive.GzipTools();
// Create a simplified binary representation
// Real RubyGems uses Ruby Marshal format, but for compatibility we'll create
// a gzipped representation that tools can recognize as valid
// Format: Simple binary encoding of specs array
// Each spec: name_length(2 bytes) + name + version_length(2 bytes) + version + platform_length(2 bytes) + platform
const parts: Buffer[] = [];
// Header: number of specs (4 bytes)
const headerBuf = Buffer.alloc(4);
headerBuf.writeUInt32LE(specs.length, 0);
parts.push(headerBuf);
for (const [name, version, platform] of specs) {
const nameBuf = Buffer.from(name, 'utf-8');
const versionBuf = Buffer.from(version, 'utf-8');
const platformBuf = Buffer.from(platform, 'utf-8');
const nameLenBuf = Buffer.alloc(2);
nameLenBuf.writeUInt16LE(nameBuf.length, 0);
const versionLenBuf = Buffer.alloc(2);
versionLenBuf.writeUInt16LE(versionBuf.length, 0);
const platformLenBuf = Buffer.alloc(2);
platformLenBuf.writeUInt16LE(platformBuf.length, 0);
parts.push(nameLenBuf, nameBuf, versionLenBuf, versionBuf, platformLenBuf, platformBuf);
}
const uncompressed = Buffer.concat(parts);
return gzipTools.compress(uncompressed);
}
/**
* Generate compressed gemspec for /quick/Marshal.4.8/{gem}-{version}.gemspec.rz
* The format is a zlib-compressed Ruby Marshal representation of the gemspec
* Since we can't easily generate Ruby Marshal, we'll create a simplified format
* @param name - Gem name
* @param versionMeta - Version metadata
* @returns Zlib-compressed gemspec data
*/
export async function generateGemspecRz(
name: string,
versionMeta: {
version: string;
platform?: string;
checksum: string;
dependencies?: Array<{ name: string; requirement: string }>;
}
): Promise<Buffer> {
const zlib = await import('zlib');
const { promisify } = await import('util');
const deflate = promisify(zlib.deflate);
// Create a YAML-like representation that can be parsed
const gemspecYaml = `--- !ruby/object:Gem::Specification
name: ${name}
version: !ruby/object:Gem::Version
version: ${versionMeta.version}
platform: ${versionMeta.platform || 'ruby'}
authors: []
date: ${new Date().toISOString().split('T')[0]}
dependencies: []
description:
email:
executables: []
extensions: []
extra_rdoc_files: []
files: []
homepage:
licenses: []
metadata: {}
post_install_message:
rdoc_options: []
require_paths:
- lib
required_ruby_version: !ruby/object:Gem::Requirement
requirements:
- - ">="
- !ruby/object:Gem::Version
version: '0'
required_rubygems_version: !ruby/object:Gem::Requirement
requirements:
- - ">="
- !ruby/object:Gem::Version
version: '0'
requirements: []
rubygems_version: 3.0.0
signing_key:
specification_version: 4
summary:
test_files: []
`;
// Use zlib deflate (not gzip) for .rz files
return deflate(Buffer.from(gemspecYaml, 'utf-8'));
}

View File

@@ -5,4 +5,5 @@
export * from './interfaces.rubygems.js';
export * from './classes.rubygemsregistry.js';
export { RubygemsUpstream } from './classes.rubygemsupstream.js';
export * as rubygemsHelpers from './helpers.rubygems.js';

View File

@@ -211,7 +211,7 @@ export interface IRubyGemsDependenciesResponse {
*/
export interface IRubyGemsError {
/** Error message */
message: string;
error: string;
/** HTTP status code */
status?: number;
}

View File

@@ -0,0 +1,526 @@
import * as plugins from '../plugins.js';
import type {
IUpstreamRegistryConfig,
IUpstreamAuthConfig,
IUpstreamCacheConfig,
IUpstreamResilienceConfig,
IUpstreamResult,
IUpstreamFetchContext,
IProtocolUpstreamConfig,
IUpstreamScopeRule,
TCircuitState,
} from './interfaces.upstream.js';
import {
DEFAULT_CACHE_CONFIG,
DEFAULT_RESILIENCE_CONFIG,
} from './interfaces.upstream.js';
import { CircuitBreaker, CircuitOpenError, withCircuitBreaker } from './classes.circuitbreaker.js';
import { UpstreamCache } from './classes.upstreamcache.js';
/**
* Base class for protocol-specific upstream implementations.
*
* Provides:
* - Multi-upstream routing with priority
* - Scope-based filtering (glob patterns)
* - Authentication handling
* - Circuit breaker per upstream
* - Caching with TTL
* - Retry with exponential backoff
* - 429 rate limit handling
*/
export abstract class BaseUpstream {
/** Protocol name for logging */
protected abstract readonly protocolName: string;
/** Upstream configuration */
protected readonly config: IProtocolUpstreamConfig;
/** Resolved cache configuration */
protected readonly cacheConfig: IUpstreamCacheConfig;
/** Resolved resilience configuration */
protected readonly resilienceConfig: IUpstreamResilienceConfig;
/** Circuit breakers per upstream */
protected readonly circuitBreakers: Map<string, CircuitBreaker> = new Map();
/** Upstream cache */
protected readonly cache: UpstreamCache;
/** Logger instance */
protected readonly logger: plugins.smartlog.Smartlog;
constructor(config: IProtocolUpstreamConfig, logger?: plugins.smartlog.Smartlog) {
this.config = config;
this.cacheConfig = { ...DEFAULT_CACHE_CONFIG, ...config.cache };
this.resilienceConfig = { ...DEFAULT_RESILIENCE_CONFIG, ...config.resilience };
this.cache = new UpstreamCache(this.cacheConfig);
this.logger = logger || new plugins.smartlog.Smartlog({
logContext: {
company: 'smartregistry',
companyunit: 'upstream',
environment: 'production',
runtime: 'node',
}
});
// Initialize circuit breakers for each upstream
for (const upstream of config.upstreams) {
const upstreamResilience = { ...this.resilienceConfig, ...upstream.resilience };
this.circuitBreakers.set(upstream.id, new CircuitBreaker(upstream.id, upstreamResilience));
}
}
/**
* Check if upstream is enabled.
*/
public isEnabled(): boolean {
return this.config.enabled;
}
/**
* Get all configured upstreams.
*/
public getUpstreams(): IUpstreamRegistryConfig[] {
return this.config.upstreams;
}
/**
* Get circuit breaker state for an upstream.
*/
public getCircuitState(upstreamId: string): TCircuitState | null {
const breaker = this.circuitBreakers.get(upstreamId);
return breaker ? breaker.getState() : null;
}
/**
* Get cache statistics.
*/
public getCacheStats() {
return this.cache.getStats();
}
/**
* Fetch a resource from upstreams.
* Tries upstreams in priority order, respecting circuit breakers and scope rules.
*/
public async fetch(context: IUpstreamFetchContext): Promise<IUpstreamResult | null> {
if (!this.config.enabled) {
return null;
}
// Get applicable upstreams sorted by priority
const applicableUpstreams = this.getApplicableUpstreams(context.resource);
if (applicableUpstreams.length === 0) {
return null;
}
// Use the first applicable upstream's URL for cache key
const primaryUpstreamUrl = applicableUpstreams[0]?.url;
// Check cache first
const cached = await this.cache.get(context, primaryUpstreamUrl);
if (cached && !cached.stale) {
return {
success: true,
status: 200,
headers: cached.headers,
body: cached.data,
upstreamId: cached.upstreamId,
fromCache: true,
latencyMs: 0,
};
}
// Check for negative cache (recent 404)
if (await this.cache.hasNegative(context, primaryUpstreamUrl)) {
return {
success: false,
status: 404,
headers: {},
upstreamId: 'cache',
fromCache: true,
latencyMs: 0,
};
}
// If we have stale cache, return it immediately and revalidate in background
if (cached?.stale && this.cacheConfig.staleWhileRevalidate) {
// Fire and forget revalidation
this.revalidateInBackground(context, applicableUpstreams);
return {
success: true,
status: 200,
headers: cached.headers,
body: cached.data,
upstreamId: cached.upstreamId,
fromCache: true,
latencyMs: 0,
};
}
// Try each upstream in order
let lastError: Error | null = null;
for (const upstream of applicableUpstreams) {
const breaker = this.circuitBreakers.get(upstream.id);
if (!breaker) continue;
try {
const result = await withCircuitBreaker(
breaker,
() => this.fetchFromUpstream(upstream, context),
);
// Cache successful responses
if (result.success && result.body) {
await this.cache.set(
context,
Buffer.isBuffer(result.body) ? result.body : Buffer.from(JSON.stringify(result.body)),
result.headers['content-type'] || 'application/octet-stream',
result.headers,
upstream.id,
upstream.url,
);
}
// Cache 404 responses
if (result.status === 404) {
await this.cache.setNegative(context, upstream.id, upstream.url);
}
return result;
} catch (error) {
if (error instanceof CircuitOpenError) {
this.logger.log('debug', `Circuit open for upstream ${upstream.id}, trying next`);
} else {
this.logger.log('warn', `Upstream ${upstream.id} failed: ${(error as Error).message}`);
}
lastError = error as Error;
// Continue to next upstream
}
}
// All upstreams failed
if (lastError) {
this.logger.log('error', `All upstreams failed for ${context.resource}: ${lastError.message}`);
}
return null;
}
/**
* Invalidate cache for a resource pattern.
*/
public async invalidateCache(pattern: RegExp): Promise<number> {
return this.cache.invalidatePattern(pattern);
}
/**
* Clear all cache entries.
*/
public async clearCache(): Promise<void> {
await this.cache.clear();
}
/**
* Stop the upstream (cleanup resources).
*/
public stop(): void {
this.cache.stop();
}
/**
* Get upstreams that apply to a resource, sorted by priority.
*/
protected getApplicableUpstreams(resource: string): IUpstreamRegistryConfig[] {
return this.config.upstreams
.filter(upstream => {
if (!upstream.enabled) return false;
// Check circuit breaker
const breaker = this.circuitBreakers.get(upstream.id);
if (breaker && !breaker.canRequest()) return false;
// Check scope rules
return this.matchesScopeRules(resource, upstream.scopeRules);
})
.sort((a, b) => a.priority - b.priority);
}
/**
* Check if a resource matches scope rules.
* Empty rules = match all.
*/
protected matchesScopeRules(resource: string, rules?: IUpstreamScopeRule[]): boolean {
if (!rules || rules.length === 0) {
return true;
}
// Process rules in order
// Start with default exclude (nothing matches)
let matched = false;
for (const rule of rules) {
const isMatch = plugins.minimatch(resource, rule.pattern);
if (isMatch) {
matched = rule.action === 'include';
}
}
return matched;
}
/**
* Fetch from a specific upstream with retry logic.
*/
protected async fetchFromUpstream(
upstream: IUpstreamRegistryConfig,
context: IUpstreamFetchContext,
): Promise<IUpstreamResult> {
const upstreamResilience = { ...this.resilienceConfig, ...upstream.resilience };
const startTime = Date.now();
let lastError: Error | null = null;
for (let attempt = 0; attempt <= upstreamResilience.maxRetries; attempt++) {
try {
const result = await this.executeRequest(upstream, context, upstreamResilience.timeoutMs);
return {
...result,
upstreamId: upstream.id,
fromCache: false,
latencyMs: Date.now() - startTime,
};
} catch (error) {
lastError = error as Error;
// Don't retry on 4xx errors (except 429)
if (this.isNonRetryableError(error)) {
break;
}
// Calculate delay with exponential backoff and jitter
if (attempt < upstreamResilience.maxRetries) {
const delay = this.calculateBackoffDelay(
attempt,
upstreamResilience.retryDelayMs,
upstreamResilience.retryMaxDelayMs,
);
await this.sleep(delay);
}
}
}
throw lastError || new Error('Request failed');
}
/**
* Execute a single HTTP request to an upstream.
*/
protected async executeRequest(
upstream: IUpstreamRegistryConfig,
context: IUpstreamFetchContext,
timeoutMs: number,
): Promise<Omit<IUpstreamResult, 'upstreamId' | 'fromCache' | 'latencyMs'>> {
// Build the full URL
const url = this.buildUpstreamUrl(upstream, context);
// Build headers with auth
const headers = this.buildHeaders(upstream, context);
// Make the request using SmartRequest
const request = plugins.smartrequest.SmartRequest.create()
.url(url)
.method(context.method as any)
.headers(headers)
.timeout(timeoutMs)
.handle429Backoff({ maxRetries: 3, fallbackDelay: 1000, maxWaitTime: 30000 });
// Add query params if present
if (Object.keys(context.query).length > 0) {
request.query(context.query);
}
let response: plugins.smartrequest.ICoreResponse;
switch (context.method.toUpperCase()) {
case 'GET':
response = await request.get();
break;
case 'HEAD':
// SmartRequest doesn't have head(), use options
response = await request.method('HEAD').get();
break;
default:
response = await request.get();
}
// Parse response
const responseHeaders: Record<string, string> = {};
for (const [key, value] of Object.entries(response.headers)) {
responseHeaders[key.toLowerCase()] = Array.isArray(value) ? value[0] : value;
}
let body: Buffer | any;
const contentType = responseHeaders['content-type'] || '';
if (response.ok) {
if (contentType.includes('application/json')) {
body = await response.json();
} else {
const arrayBuffer = await response.arrayBuffer();
body = Buffer.from(arrayBuffer);
}
}
return {
success: response.ok,
status: response.status,
headers: responseHeaders,
body,
};
}
/**
* Build the full URL for an upstream request.
* Subclasses can override for protocol-specific URL building.
*/
protected buildUpstreamUrl(upstream: IUpstreamRegistryConfig, context: IUpstreamFetchContext): string {
// Remove leading slash if URL already has trailing slash
let path = context.path;
if (upstream.url.endsWith('/') && path.startsWith('/')) {
path = path.slice(1);
}
return `${upstream.url}${path}`;
}
/**
* Build headers including authentication.
*/
protected buildHeaders(
upstream: IUpstreamRegistryConfig,
context: IUpstreamFetchContext,
): Record<string, string> {
const headers: Record<string, string> = { ...context.headers };
// Remove host header (will be set by HTTP client)
delete headers['host'];
// Add authentication
this.addAuthHeaders(headers, upstream.auth);
return headers;
}
/**
* Add authentication headers based on auth config.
*/
protected addAuthHeaders(headers: Record<string, string>, auth: IUpstreamAuthConfig): void {
switch (auth.type) {
case 'basic':
if (auth.username && auth.password) {
const credentials = Buffer.from(`${auth.username}:${auth.password}`).toString('base64');
headers['authorization'] = `Basic ${credentials}`;
}
break;
case 'bearer':
if (auth.token) {
headers['authorization'] = `Bearer ${auth.token}`;
}
break;
case 'api-key':
if (auth.token) {
const headerName = auth.headerName || 'authorization';
headers[headerName.toLowerCase()] = auth.token;
}
break;
case 'none':
default:
// No authentication
break;
}
}
/**
* Check if an error should not be retried.
*/
protected isNonRetryableError(error: unknown): boolean {
// Check for HTTP status errors
if (error && typeof error === 'object' && 'status' in error) {
const status = (error as { status: number }).status;
// Don't retry 4xx errors except 429 (rate limited)
if (status >= 400 && status < 500 && status !== 429) {
return true;
}
}
return false;
}
/**
* Calculate backoff delay with exponential backoff and jitter.
*/
protected calculateBackoffDelay(
attempt: number,
baseDelayMs: number,
maxDelayMs: number,
): number {
// Exponential backoff: delay = base * 2^attempt
const exponentialDelay = baseDelayMs * Math.pow(2, attempt);
// Cap at max delay
const cappedDelay = Math.min(exponentialDelay, maxDelayMs);
// Add jitter (±25%)
const jitter = cappedDelay * 0.25 * (Math.random() * 2 - 1);
return Math.floor(cappedDelay + jitter);
}
/**
* Sleep for a specified duration.
*/
protected sleep(ms: number): Promise<void> {
return new Promise(resolve => setTimeout(resolve, ms));
}
/**
* Revalidate cache in background.
*/
protected async revalidateInBackground(
context: IUpstreamFetchContext,
upstreams: IUpstreamRegistryConfig[],
): Promise<void> {
try {
for (const upstream of upstreams) {
const breaker = this.circuitBreakers.get(upstream.id);
if (!breaker || !breaker.canRequest()) continue;
try {
const result = await withCircuitBreaker(
breaker,
() => this.fetchFromUpstream(upstream, context),
);
if (result.success && result.body) {
await this.cache.set(
context,
Buffer.isBuffer(result.body) ? result.body : Buffer.from(JSON.stringify(result.body)),
result.headers['content-type'] || 'application/octet-stream',
result.headers,
upstream.id,
upstream.url,
);
return; // Successfully revalidated
}
} catch {
// Continue to next upstream
}
}
} catch (error) {
this.logger.log('debug', `Background revalidation failed: ${(error as Error).message}`);
}
}
}

View File

@@ -0,0 +1,238 @@
import type { TCircuitState, IUpstreamResilienceConfig } from './interfaces.upstream.js';
import { DEFAULT_RESILIENCE_CONFIG } from './interfaces.upstream.js';
/**
* Circuit breaker implementation for upstream resilience.
*
* States:
* - CLOSED: Normal operation, requests pass through
* - OPEN: Circuit is tripped, requests fail fast
* - HALF_OPEN: Testing if upstream has recovered
*
* Transitions:
* - CLOSED → OPEN: When failure count exceeds threshold
* - OPEN → HALF_OPEN: After reset timeout expires
* - HALF_OPEN → CLOSED: On successful request
* - HALF_OPEN → OPEN: On failed request
*/
export class CircuitBreaker {
/** Unique identifier for logging and metrics */
public readonly id: string;
/** Current circuit state */
private state: TCircuitState = 'CLOSED';
/** Count of consecutive failures */
private failureCount: number = 0;
/** Timestamp when circuit was opened */
private openedAt: number = 0;
/** Number of successful requests in half-open state */
private halfOpenSuccesses: number = 0;
/** Configuration */
private readonly config: IUpstreamResilienceConfig;
/** Number of successes required to close circuit from half-open */
private readonly halfOpenThreshold: number = 2;
constructor(id: string, config?: Partial<IUpstreamResilienceConfig>) {
this.id = id;
this.config = { ...DEFAULT_RESILIENCE_CONFIG, ...config };
}
/**
* Get current circuit state.
*/
public getState(): TCircuitState {
// Check if we should transition from OPEN to HALF_OPEN
if (this.state === 'OPEN') {
const elapsed = Date.now() - this.openedAt;
if (elapsed >= this.config.circuitBreakerResetMs) {
this.transitionTo('HALF_OPEN');
}
}
return this.state;
}
/**
* Check if circuit allows requests.
* Returns true if requests should be allowed.
*/
public canRequest(): boolean {
const currentState = this.getState();
return currentState !== 'OPEN';
}
/**
* Record a successful request.
* May transition circuit from HALF_OPEN to CLOSED.
*/
public recordSuccess(): void {
if (this.state === 'HALF_OPEN') {
this.halfOpenSuccesses++;
if (this.halfOpenSuccesses >= this.halfOpenThreshold) {
this.transitionTo('CLOSED');
}
} else if (this.state === 'CLOSED') {
// Reset failure count on success
this.failureCount = 0;
}
}
/**
* Record a failed request.
* May transition circuit from CLOSED/HALF_OPEN to OPEN.
*/
public recordFailure(): void {
if (this.state === 'HALF_OPEN') {
// Any failure in half-open immediately opens circuit
this.transitionTo('OPEN');
} else if (this.state === 'CLOSED') {
this.failureCount++;
if (this.failureCount >= this.config.circuitBreakerThreshold) {
this.transitionTo('OPEN');
}
}
}
/**
* Force circuit to open state.
* Useful for manual intervention or external health checks.
*/
public forceOpen(): void {
this.transitionTo('OPEN');
}
/**
* Force circuit to closed state.
* Useful for manual intervention after fixing upstream issues.
*/
public forceClose(): void {
this.transitionTo('CLOSED');
}
/**
* Reset circuit to initial state.
*/
public reset(): void {
this.state = 'CLOSED';
this.failureCount = 0;
this.openedAt = 0;
this.halfOpenSuccesses = 0;
}
/**
* Get circuit metrics for monitoring.
*/
public getMetrics(): ICircuitBreakerMetrics {
return {
id: this.id,
state: this.getState(),
failureCount: this.failureCount,
openedAt: this.openedAt > 0 ? new Date(this.openedAt) : null,
timeUntilHalfOpen: this.state === 'OPEN'
? Math.max(0, this.config.circuitBreakerResetMs - (Date.now() - this.openedAt))
: 0,
halfOpenSuccesses: this.halfOpenSuccesses,
threshold: this.config.circuitBreakerThreshold,
resetMs: this.config.circuitBreakerResetMs,
};
}
/**
* Transition to a new state with proper cleanup.
*/
private transitionTo(newState: TCircuitState): void {
const previousState = this.state;
this.state = newState;
switch (newState) {
case 'OPEN':
this.openedAt = Date.now();
this.halfOpenSuccesses = 0;
break;
case 'HALF_OPEN':
this.halfOpenSuccesses = 0;
break;
case 'CLOSED':
this.failureCount = 0;
this.openedAt = 0;
this.halfOpenSuccesses = 0;
break;
}
// Log state transition (useful for debugging and monitoring)
// In production, this would emit events or metrics
if (previousState !== newState) {
// State changed - could emit event here
}
}
}
/**
* Metrics for circuit breaker monitoring.
*/
export interface ICircuitBreakerMetrics {
/** Circuit breaker identifier */
id: string;
/** Current state */
state: TCircuitState;
/** Number of consecutive failures */
failureCount: number;
/** When circuit was opened (null if never opened) */
openedAt: Date | null;
/** Milliseconds until circuit transitions to half-open (0 if not open) */
timeUntilHalfOpen: number;
/** Number of successes in half-open state */
halfOpenSuccesses: number;
/** Failure threshold for opening circuit */
threshold: number;
/** Reset timeout in milliseconds */
resetMs: number;
}
/**
* Execute a function with circuit breaker protection.
*
* @param breaker The circuit breaker to use
* @param fn The async function to execute
* @param fallback Optional fallback function when circuit is open
* @returns The result of fn or fallback
* @throws CircuitOpenError if circuit is open and no fallback provided
*/
export async function withCircuitBreaker<T>(
breaker: CircuitBreaker,
fn: () => Promise<T>,
fallback?: () => Promise<T>,
): Promise<T> {
if (!breaker.canRequest()) {
if (fallback) {
return fallback();
}
throw new CircuitOpenError(breaker.id);
}
try {
const result = await fn();
breaker.recordSuccess();
return result;
} catch (error) {
breaker.recordFailure();
throw error;
}
}
/**
* Error thrown when circuit is open and no fallback is provided.
*/
export class CircuitOpenError extends Error {
public readonly circuitId: string;
constructor(circuitId: string) {
super(`Circuit breaker '${circuitId}' is open`);
this.name = 'CircuitOpenError';
this.circuitId = circuitId;
}
}

View File

@@ -0,0 +1,626 @@
import type {
ICacheEntry,
IUpstreamCacheConfig,
IUpstreamFetchContext,
} from './interfaces.upstream.js';
import { DEFAULT_CACHE_CONFIG } from './interfaces.upstream.js';
import type { IStorageBackend } from '../core/interfaces.core.js';
/**
* Cache metadata stored alongside cache entries.
*/
interface ICacheMetadata {
contentType: string;
headers: Record<string, string>;
cachedAt: string;
expiresAt?: string;
etag?: string;
upstreamId: string;
upstreamUrl: string;
}
/**
* S3-backed upstream cache with in-memory hot layer.
*
* Features:
* - TTL-based expiration
* - Stale-while-revalidate support
* - Negative caching (404s)
* - Content-type aware caching
* - ETag support for conditional requests
* - Multi-upstream support via URL-based cache paths
* - Persistent S3 storage with in-memory hot layer
*
* Cache paths are structured as:
* cache/{escaped-upstream-url}/{protocol}:{method}:{path}
*
* @example
* ```typescript
* // In-memory only (default)
* const cache = new UpstreamCache(config);
*
* // With S3 persistence
* const cache = new UpstreamCache(config, 10000, storage);
* ```
*/
export class UpstreamCache {
/** In-memory hot cache */
private readonly memoryCache: Map<string, ICacheEntry> = new Map();
/** Configuration */
private readonly config: IUpstreamCacheConfig;
/** Maximum in-memory cache entries */
private readonly maxMemoryEntries: number;
/** S3 storage backend (optional) */
private readonly storage?: IStorageBackend;
/** Cleanup interval handle */
private cleanupInterval: ReturnType<typeof setInterval> | null = null;
constructor(
config?: Partial<IUpstreamCacheConfig>,
maxMemoryEntries: number = 10000,
storage?: IStorageBackend
) {
this.config = { ...DEFAULT_CACHE_CONFIG, ...config };
this.maxMemoryEntries = maxMemoryEntries;
this.storage = storage;
// Start periodic cleanup if caching is enabled
if (this.config.enabled) {
this.startCleanup();
}
}
/**
* Check if caching is enabled.
*/
public isEnabled(): boolean {
return this.config.enabled;
}
/**
* Check if S3 storage is configured.
*/
public hasStorage(): boolean {
return !!this.storage;
}
/**
* Get cached entry for a request context.
* Checks memory first, then falls back to S3.
* Returns null if not found or expired (unless stale-while-revalidate).
*/
public async get(context: IUpstreamFetchContext, upstreamUrl?: string): Promise<ICacheEntry | null> {
if (!this.config.enabled) {
return null;
}
const key = this.buildCacheKey(context, upstreamUrl);
// Check memory cache first
let entry = this.memoryCache.get(key);
// If not in memory and we have storage, check S3
if (!entry && this.storage) {
entry = await this.loadFromStorage(key);
if (entry) {
// Promote to memory cache
this.memoryCache.set(key, entry);
}
}
if (!entry) {
return null;
}
const now = new Date();
// Check if entry is expired
if (entry.expiresAt && entry.expiresAt < now) {
// Check if we can serve stale content
if (this.config.staleWhileRevalidate && !entry.stale) {
const staleAge = (now.getTime() - entry.expiresAt.getTime()) / 1000;
if (staleAge <= this.config.staleMaxAgeSeconds) {
// Mark as stale and return
entry.stale = true;
return entry;
}
}
// Entry is too old, remove it
this.memoryCache.delete(key);
if (this.storage) {
await this.deleteFromStorage(key).catch(() => {});
}
return null;
}
return entry;
}
/**
* Store a response in the cache (memory and optionally S3).
*/
public async set(
context: IUpstreamFetchContext,
data: Buffer,
contentType: string,
headers: Record<string, string>,
upstreamId: string,
upstreamUrl: string,
options?: ICacheSetOptions,
): Promise<void> {
if (!this.config.enabled) {
return;
}
// Enforce max memory entries limit
if (this.memoryCache.size >= this.maxMemoryEntries) {
this.evictOldest();
}
const key = this.buildCacheKey(context, upstreamUrl);
const now = new Date();
// Determine TTL based on content type
const ttlSeconds = options?.ttlSeconds ?? this.determineTtl(context, contentType, headers);
const entry: ICacheEntry = {
data,
contentType,
headers,
cachedAt: now,
expiresAt: ttlSeconds > 0 ? new Date(now.getTime() + ttlSeconds * 1000) : undefined,
etag: headers['etag'] || options?.etag,
upstreamId,
stale: false,
};
// Store in memory
this.memoryCache.set(key, entry);
// Store in S3 if available
if (this.storage) {
await this.saveToStorage(key, entry, upstreamUrl).catch(() => {});
}
}
/**
* Store a negative cache entry (404 response).
*/
public async setNegative(context: IUpstreamFetchContext, upstreamId: string, upstreamUrl: string): Promise<void> {
if (!this.config.enabled || this.config.negativeCacheTtlSeconds <= 0) {
return;
}
const key = this.buildCacheKey(context, upstreamUrl);
const now = new Date();
const entry: ICacheEntry = {
data: Buffer.from(''),
contentType: 'application/octet-stream',
headers: {},
cachedAt: now,
expiresAt: new Date(now.getTime() + this.config.negativeCacheTtlSeconds * 1000),
upstreamId,
stale: false,
};
this.memoryCache.set(key, entry);
if (this.storage) {
await this.saveToStorage(key, entry, upstreamUrl).catch(() => {});
}
}
/**
* Check if there's a negative cache entry for this context.
*/
public async hasNegative(context: IUpstreamFetchContext, upstreamUrl?: string): Promise<boolean> {
const entry = await this.get(context, upstreamUrl);
return entry !== null && entry.data.length === 0;
}
/**
* Invalidate a specific cache entry.
*/
public async invalidate(context: IUpstreamFetchContext, upstreamUrl?: string): Promise<boolean> {
const key = this.buildCacheKey(context, upstreamUrl);
const deleted = this.memoryCache.delete(key);
if (this.storage) {
await this.deleteFromStorage(key).catch(() => {});
}
return deleted;
}
/**
* Invalidate all entries matching a pattern.
* Useful for invalidating all versions of a package.
*/
public async invalidatePattern(pattern: RegExp): Promise<number> {
let count = 0;
for (const key of this.memoryCache.keys()) {
if (pattern.test(key)) {
this.memoryCache.delete(key);
if (this.storage) {
await this.deleteFromStorage(key).catch(() => {});
}
count++;
}
}
return count;
}
/**
* Invalidate all entries from a specific upstream.
*/
public async invalidateUpstream(upstreamId: string): Promise<number> {
let count = 0;
for (const [key, entry] of this.memoryCache.entries()) {
if (entry.upstreamId === upstreamId) {
this.memoryCache.delete(key);
if (this.storage) {
await this.deleteFromStorage(key).catch(() => {});
}
count++;
}
}
return count;
}
/**
* Clear all cache entries (memory and S3).
*/
public async clear(): Promise<void> {
this.memoryCache.clear();
// Note: S3 cleanup would require listing and deleting all cache/* objects
// This is left as a future enhancement for bulk cleanup
}
/**
* Get cache statistics.
*/
public getStats(): ICacheStats {
let freshCount = 0;
let staleCount = 0;
let negativeCount = 0;
let totalSize = 0;
const now = new Date();
for (const entry of this.memoryCache.values()) {
totalSize += entry.data.length;
if (entry.data.length === 0) {
negativeCount++;
} else if (entry.stale || (entry.expiresAt && entry.expiresAt < now)) {
staleCount++;
} else {
freshCount++;
}
}
return {
totalEntries: this.memoryCache.size,
freshEntries: freshCount,
staleEntries: staleCount,
negativeEntries: negativeCount,
totalSizeBytes: totalSize,
maxEntries: this.maxMemoryEntries,
enabled: this.config.enabled,
hasStorage: !!this.storage,
};
}
/**
* Stop the cache and cleanup.
*/
public stop(): void {
if (this.cleanupInterval) {
clearInterval(this.cleanupInterval);
this.cleanupInterval = null;
}
}
// ========================================================================
// Storage Methods
// ========================================================================
/**
* Build storage path for a cache key.
* Escapes upstream URL for safe use in S3 paths.
*/
private buildStoragePath(key: string): string {
return `cache/${key}`;
}
/**
* Build storage path for cache metadata.
*/
private buildMetadataPath(key: string): string {
return `cache/${key}.meta`;
}
/**
* Load a cache entry from S3 storage.
*/
private async loadFromStorage(key: string): Promise<ICacheEntry | null> {
if (!this.storage) return null;
try {
const dataPath = this.buildStoragePath(key);
const metaPath = this.buildMetadataPath(key);
// Load data and metadata in parallel
const [data, metaBuffer] = await Promise.all([
this.storage.getObject(dataPath),
this.storage.getObject(metaPath),
]);
if (!data || !metaBuffer) {
return null;
}
const meta: ICacheMetadata = JSON.parse(metaBuffer.toString('utf-8'));
return {
data,
contentType: meta.contentType,
headers: meta.headers,
cachedAt: new Date(meta.cachedAt),
expiresAt: meta.expiresAt ? new Date(meta.expiresAt) : undefined,
etag: meta.etag,
upstreamId: meta.upstreamId,
stale: false,
};
} catch {
return null;
}
}
/**
* Save a cache entry to S3 storage.
*/
private async saveToStorage(key: string, entry: ICacheEntry, upstreamUrl: string): Promise<void> {
if (!this.storage) return;
const dataPath = this.buildStoragePath(key);
const metaPath = this.buildMetadataPath(key);
const meta: ICacheMetadata = {
contentType: entry.contentType,
headers: entry.headers,
cachedAt: entry.cachedAt.toISOString(),
expiresAt: entry.expiresAt?.toISOString(),
etag: entry.etag,
upstreamId: entry.upstreamId,
upstreamUrl,
};
// Save data and metadata in parallel
await Promise.all([
this.storage.putObject(dataPath, entry.data),
this.storage.putObject(metaPath, Buffer.from(JSON.stringify(meta), 'utf-8')),
]);
}
/**
* Delete a cache entry from S3 storage.
*/
private async deleteFromStorage(key: string): Promise<void> {
if (!this.storage) return;
const dataPath = this.buildStoragePath(key);
const metaPath = this.buildMetadataPath(key);
await Promise.all([
this.storage.deleteObject(dataPath).catch(() => {}),
this.storage.deleteObject(metaPath).catch(() => {}),
]);
}
// ========================================================================
// Helper Methods
// ========================================================================
/**
* Escape a URL for safe use in storage paths.
*/
private escapeUrl(url: string): string {
// Remove protocol prefix and escape special characters
return url
.replace(/^https?:\/\//, '')
.replace(/[\/\\:*?"<>|]/g, '_')
.replace(/__+/g, '_');
}
/**
* Build a unique cache key for a request context.
* Includes escaped upstream URL for multi-upstream support.
*/
private buildCacheKey(context: IUpstreamFetchContext, upstreamUrl?: string): string {
// Include method, protocol, path, and sorted query params
const queryString = Object.keys(context.query)
.sort()
.map(k => `${k}=${context.query[k]}`)
.join('&');
const baseKey = `${context.protocol}:${context.method}:${context.path}${queryString ? '?' + queryString : ''}`;
if (upstreamUrl) {
return `${this.escapeUrl(upstreamUrl)}/${baseKey}`;
}
return baseKey;
}
/**
* Determine TTL based on content characteristics.
*/
private determineTtl(
context: IUpstreamFetchContext,
contentType: string,
headers: Record<string, string>,
): number {
// Check for Cache-Control header
const cacheControl = headers['cache-control'];
if (cacheControl) {
const maxAgeMatch = cacheControl.match(/max-age=(\d+)/);
if (maxAgeMatch) {
return parseInt(maxAgeMatch[1], 10);
}
if (cacheControl.includes('no-store') || cacheControl.includes('no-cache')) {
return 0;
}
}
// Check if content is immutable (content-addressable)
if (this.isImmutableContent(context, contentType)) {
return this.config.immutableTtlSeconds;
}
// Default TTL for mutable content
return this.config.defaultTtlSeconds;
}
/**
* Check if content is immutable (content-addressable).
*/
private isImmutableContent(context: IUpstreamFetchContext, contentType: string): boolean {
// OCI blobs with digest are immutable
if (context.protocol === 'oci' && context.resourceType === 'blob') {
return true;
}
// NPM tarballs are immutable (versioned)
if (context.protocol === 'npm' && context.resourceType === 'tarball') {
return true;
}
// Maven artifacts with version are immutable
if (context.protocol === 'maven' && context.resourceType === 'artifact') {
return true;
}
// Cargo crate files are immutable
if (context.protocol === 'cargo' && context.resourceType === 'crate') {
return true;
}
// Composer dist files are immutable
if (context.protocol === 'composer' && context.resourceType === 'dist') {
return true;
}
// PyPI package files are immutable
if (context.protocol === 'pypi' && context.resourceType === 'package') {
return true;
}
// RubyGems .gem files are immutable
if (context.protocol === 'rubygems' && context.resourceType === 'gem') {
return true;
}
return false;
}
/**
* Evict oldest entries to make room for new ones.
*/
private evictOldest(): void {
// Evict 10% of max entries
const evictCount = Math.ceil(this.maxMemoryEntries * 0.1);
let evicted = 0;
// First, try to evict stale entries
const now = new Date();
for (const [key, entry] of this.memoryCache.entries()) {
if (evicted >= evictCount) break;
if (entry.stale || (entry.expiresAt && entry.expiresAt < now)) {
this.memoryCache.delete(key);
evicted++;
}
}
// If not enough evicted, evict oldest by cachedAt
if (evicted < evictCount) {
const entries = Array.from(this.memoryCache.entries())
.sort((a, b) => a[1].cachedAt.getTime() - b[1].cachedAt.getTime());
for (const [key] of entries) {
if (evicted >= evictCount) break;
this.memoryCache.delete(key);
evicted++;
}
}
}
/**
* Start periodic cleanup of expired entries.
*/
private startCleanup(): void {
// Run cleanup every minute
this.cleanupInterval = setInterval(() => {
this.cleanup();
}, 60000);
// Don't keep the process alive just for cleanup
if (this.cleanupInterval.unref) {
this.cleanupInterval.unref();
}
}
/**
* Remove all expired entries from memory cache.
*/
private cleanup(): void {
const now = new Date();
const staleDeadline = new Date(now.getTime() - this.config.staleMaxAgeSeconds * 1000);
for (const [key, entry] of this.memoryCache.entries()) {
if (entry.expiresAt) {
// Remove if past stale deadline
if (entry.expiresAt < staleDeadline) {
this.memoryCache.delete(key);
}
}
}
}
}
/**
* Options for cache set operation.
*/
export interface ICacheSetOptions {
/** Override TTL in seconds */
ttlSeconds?: number;
/** ETag for conditional requests */
etag?: string;
}
/**
* Cache statistics.
*/
export interface ICacheStats {
/** Total number of cached entries in memory */
totalEntries: number;
/** Number of fresh (non-expired) entries */
freshEntries: number;
/** Number of stale entries (expired but still usable) */
staleEntries: number;
/** Number of negative cache entries */
negativeEntries: number;
/** Total size of cached data in bytes (memory only) */
totalSizeBytes: number;
/** Maximum allowed memory entries */
maxEntries: number;
/** Whether caching is enabled */
enabled: boolean;
/** Whether S3 storage is configured */
hasStorage: boolean;
}

11
ts/upstream/index.ts Normal file
View File

@@ -0,0 +1,11 @@
// Interfaces and types
export * from './interfaces.upstream.js';
// Classes
export { CircuitBreaker, CircuitOpenError, withCircuitBreaker } from './classes.circuitbreaker.js';
export type { ICircuitBreakerMetrics } from './classes.circuitbreaker.js';
export { UpstreamCache } from './classes.upstreamcache.js';
export type { ICacheSetOptions, ICacheStats } from './classes.upstreamcache.js';
export { BaseUpstream } from './classes.baseupstream.js';

View File

@@ -0,0 +1,195 @@
import type { TRegistryProtocol } from '../core/interfaces.core.js';
/**
* Scope rule for routing requests to specific upstreams.
* Uses glob patterns for flexible matching.
*/
export interface IUpstreamScopeRule {
/** Glob pattern (e.g., "@company/*", "com.example.*", "library/*") */
pattern: string;
/** Whether matching resources should be included or excluded */
action: 'include' | 'exclude';
}
/**
* Authentication configuration for an upstream registry.
* Supports multiple auth strategies.
*/
export interface IUpstreamAuthConfig {
/** Authentication type */
type: 'none' | 'basic' | 'bearer' | 'api-key';
/** Username for basic auth */
username?: string;
/** Password for basic auth */
password?: string;
/** Token for bearer or api-key auth */
token?: string;
/** Custom header name for api-key auth (default: 'Authorization') */
headerName?: string;
}
/**
* Cache configuration for upstream content.
*/
export interface IUpstreamCacheConfig {
/** Whether caching is enabled */
enabled: boolean;
/** Default TTL in seconds for mutable content (default: 300 = 5 min) */
defaultTtlSeconds: number;
/** TTL in seconds for immutable/content-addressable content (default: 2592000 = 30 days) */
immutableTtlSeconds: number;
/** Whether to serve stale content while revalidating in background */
staleWhileRevalidate: boolean;
/** Maximum age in seconds for stale content (default: 3600 = 1 hour) */
staleMaxAgeSeconds: number;
/** TTL in seconds for negative cache entries (404s) (default: 60 = 1 min) */
negativeCacheTtlSeconds: number;
}
/**
* Resilience configuration for upstream requests.
*/
export interface IUpstreamResilienceConfig {
/** Request timeout in milliseconds (default: 30000) */
timeoutMs: number;
/** Maximum number of retry attempts (default: 3) */
maxRetries: number;
/** Initial retry delay in milliseconds (default: 1000) */
retryDelayMs: number;
/** Maximum retry delay in milliseconds (default: 30000) */
retryMaxDelayMs: number;
/** Number of failures before circuit breaker opens (default: 5) */
circuitBreakerThreshold: number;
/** Time in milliseconds before circuit breaker attempts reset (default: 30000) */
circuitBreakerResetMs: number;
}
/**
* Configuration for a single upstream registry.
*/
export interface IUpstreamRegistryConfig {
/** Unique identifier for this upstream */
id: string;
/** Human-readable name */
name: string;
/** Base URL of the upstream registry (e.g., "https://registry.npmjs.org") */
url: string;
/** Priority for routing (lower = higher priority, 1 = first) */
priority: number;
/** Whether this upstream is enabled */
enabled: boolean;
/** Scope rules for routing (empty = match all) */
scopeRules?: IUpstreamScopeRule[];
/** Authentication configuration */
auth: IUpstreamAuthConfig;
/** Cache configuration overrides */
cache?: Partial<IUpstreamCacheConfig>;
/** Resilience configuration overrides */
resilience?: Partial<IUpstreamResilienceConfig>;
}
/**
* Protocol-level upstream configuration.
* Configures upstream behavior for a specific protocol (npm, oci, etc.)
*/
export interface IProtocolUpstreamConfig {
/** Whether upstream is enabled for this protocol */
enabled: boolean;
/** List of upstream registries, ordered by priority */
upstreams: IUpstreamRegistryConfig[];
/** Protocol-level cache configuration defaults */
cache?: Partial<IUpstreamCacheConfig>;
/** Protocol-level resilience configuration defaults */
resilience?: Partial<IUpstreamResilienceConfig>;
}
/**
* Result of an upstream fetch operation.
*/
export interface IUpstreamResult {
/** Whether the fetch was successful (2xx status) */
success: boolean;
/** HTTP status code */
status: number;
/** Response headers */
headers: Record<string, string>;
/** Response body (Buffer for binary, object for JSON) */
body?: Buffer | any;
/** ID of the upstream that served the request */
upstreamId: string;
/** Whether the response was served from cache */
fromCache: boolean;
/** Request latency in milliseconds */
latencyMs: number;
}
/**
* Circuit breaker state.
*/
export type TCircuitState = 'CLOSED' | 'OPEN' | 'HALF_OPEN';
/**
* Context for an upstream fetch request.
*/
export interface IUpstreamFetchContext {
/** Protocol type */
protocol: TRegistryProtocol;
/** Resource identifier (package name, artifact name, etc.) */
resource: string;
/** Type of resource being fetched (packument, tarball, manifest, blob, etc.) */
resourceType: string;
/** Original request path */
path: string;
/** HTTP method */
method: string;
/** Request headers */
headers: Record<string, string>;
/** Query parameters */
query: Record<string, string>;
}
/**
* Cache entry stored in the upstream cache.
*/
export interface ICacheEntry {
/** Cached data */
data: Buffer;
/** Content type of the cached data */
contentType: string;
/** Original response headers */
headers: Record<string, string>;
/** When the entry was cached */
cachedAt: Date;
/** When the entry expires */
expiresAt?: Date;
/** ETag for conditional requests */
etag?: string;
/** ID of the upstream that provided the data */
upstreamId: string;
/** Whether the entry is stale but still usable */
stale?: boolean;
}
/**
* Default cache configuration values.
*/
export const DEFAULT_CACHE_CONFIG: IUpstreamCacheConfig = {
enabled: true,
defaultTtlSeconds: 300, // 5 minutes
immutableTtlSeconds: 2592000, // 30 days
staleWhileRevalidate: true,
staleMaxAgeSeconds: 3600, // 1 hour
negativeCacheTtlSeconds: 60, // 1 minute
};
/**
* Default resilience configuration values.
*/
export const DEFAULT_RESILIENCE_CONFIG: IUpstreamResilienceConfig = {
timeoutMs: 30000,
maxRetries: 3,
retryDelayMs: 1000,
retryMaxDelayMs: 30000,
circuitBreakerThreshold: 5,
circuitBreakerResetMs: 30000,
};