52 Commits

Author SHA1 Message Date
09335d41f3 v2.8.2
Some checks failed
Default (tags) / security (push) Failing after 0s
Default (tags) / test (push) Failing after 0s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-03-27 17:37:24 +00:00
2221eef722 fix(maven,tests): handle Maven Basic auth and accept deploy-plugin metadata/checksum uploads while stabilizing npm CLI test cleanup 2026-03-27 17:37:24 +00:00
26ddf1a59f v2.8.1
Some checks failed
Default (tags) / security (push) Failing after 0s
Default (tags) / test (push) Failing after 0s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-03-24 23:23:03 +00:00
5acd1d6166 fix(registry): align OCI and RubyGems API behavior and improve npm search result ordering 2026-03-24 23:23:03 +00:00
abf7605e14 v2.8.0
Some checks failed
Default (tags) / security (push) Failing after 0s
Default (tags) / test (push) Failing after 0s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-03-24 22:59:37 +00:00
7da1a35efe feat(core,storage,oci,registry-config): add streaming response support and configurable registry URLs across protocols 2026-03-24 22:59:37 +00:00
1f0acf2825 fix(oci): remove /v2/ from internal route patterns and make upstream apiPrefix configurable
The OCI handler had /v2/ baked into all regex patterns and Location headers.
When basePath was set to /v2 (as in stack.gallery), stripping it removed the
prefix that patterns expected, causing all OCI endpoints to 404.

Now patterns match on bare paths after basePath stripping, working correctly
regardless of the basePath value.

Also adds configurable apiPrefix to OCI upstream class (default /v2) for
registries behind reverse proxies with custom path prefixes.
2026-03-21 16:17:52 +00:00
37e4c5be4a fix(npm): decode URL-encoded package names after regex extraction
Scoped npm packages use %2f encoding for the slash in URLs (e.g. @scope%2fpackage).
Previously, the encoded name was used as-is for storage and packument metadata,
causing npm install to fail with EINVALIDPACKAGENAME. Now each regex extraction
point decodes the package name via decodeURIComponent while keeping the path
encoded for correct regex matching.
2026-03-21 11:59:52 +00:00
9bbc3da484 v2.7.0
Some checks failed
Default (tags) / security (push) Successful in 43s
Default (tags) / test (push) Failing after 41s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-12-03 22:16:40 +00:00
e9af3f8328 feat(upstream): Add dynamic per-request upstream provider and integrate into registries 2025-12-03 22:16:40 +00:00
351680159b v2.6.0
Some checks failed
Default (tags) / security (push) Successful in 44s
Default (tags) / test (push) Failing after 43s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-27 22:12:52 +00:00
0cabf284ed feat(core): Add core registry infrastructure: storage, auth, upstream cache, and protocol handlers 2025-11-27 22:12:52 +00:00
dbc8566aad v2.5.0
Some checks failed
Default (tags) / security (push) Successful in 43s
Default (tags) / test (push) Failing after 44s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-27 21:11:04 +00:00
bd64a7b140 feat(pypi,rubygems): Add PyPI and RubyGems protocol implementations, upstream caching, and auth/storage improvements 2025-11-27 21:11:04 +00:00
ae8dec9142 v2.4.0
Some checks failed
Default (tags) / security (push) Successful in 45s
Default (tags) / test (push) Failing after 45s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-27 20:59:49 +00:00
19da87a9df feat(core): Add pluggable auth providers, storage hooks, multi-upstream cache awareness, and PyPI/RubyGems protocol implementations 2025-11-27 20:59:49 +00:00
99b01733e7 v2.3.0
Some checks failed
Default (tags) / security (push) Successful in 39s
Default (tags) / test (push) Failing after 45s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-27 14:20:01 +00:00
0610077eec feat(upstream): Add upstream proxy/cache subsystem and integrate per-protocol upstreams 2025-11-27 14:20:01 +00:00
cfadc89b5a v2.2.3
Some checks failed
Default (tags) / security (push) Successful in 45s
Default (tags) / test (push) Failing after 38s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-27 12:41:38 +00:00
eb91a3f75b fix(tests): Use unique test run IDs and add S3 cleanup in test helpers to avoid cross-run conflicts 2025-11-27 12:41:38 +00:00
58a21a6bbb v2.2.2
Some checks failed
Default (tags) / security (push) Successful in 38s
Default (tags) / test (push) Failing after 36s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-25 23:25:26 +00:00
da1cf8ddeb fix(npm): Replace console logging with structured Smartlog in NPM registry and silence RubyGems helper error logging 2025-11-25 23:25:26 +00:00
35ff286169 v2.2.1
Some checks failed
Default (tags) / security (push) Successful in 39s
Default (tags) / test (push) Failing after 37s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-25 22:35:31 +00:00
a78934836e fix(core): Normalize binary data handling across registries and add buffer helpers 2025-11-25 22:35:31 +00:00
e81fa41b18 v2.2.0
Some checks failed
Default (tags) / security (push) Successful in 39s
Default (tags) / test (push) Failing after 36s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-25 22:10:06 +00:00
41405eb40a feat(core/registrystorage): Persist OCI manifest content-type in sidecar and normalize manifest body handling 2025-11-25 22:10:06 +00:00
67188a4e9f v2.1.2
Some checks failed
Default (tags) / security (push) Successful in 39s
Default (tags) / test (push) Failing after 36s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-25 17:15:47 +00:00
a2f7f43027 fix(oci): Prefer raw request body for content-addressable OCI operations and expose rawBody on request context 2025-11-25 17:15:47 +00:00
37a89239d9 v2.1.1
Some checks failed
Default (tags) / security (push) Successful in 35s
Default (tags) / test (push) Failing after 36s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-25 16:59:37 +00:00
93fee289e7 fix(oci): Preserve raw manifest bytes for digest calculation and handle string/JSON manifest bodies in OCI registry 2025-11-25 16:59:37 +00:00
30fd9a4238 v2.1.0
Some checks failed
Default (tags) / security (push) Successful in 47s
Default (tags) / test (push) Failing after 48s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-25 16:48:08 +00:00
3b5bf5e789 feat(oci): Support configurable OCI token realm/service and centralize unauthorized responses 2025-11-25 16:48:08 +00:00
9b92e1c0d2 v2.0.0
Some checks failed
Default (tags) / security (push) Successful in 49s
Default (tags) / test (push) Failing after 50s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-25 15:07:59 +00:00
6291ebf79b BREAKING CHANGE(pypi,rubygems): Revise PyPI and RubyGems handling: normalize error payloads, fix .gem parsing/packing, adjust PyPI JSON API and tests, and export smartarchive plugin 2025-11-25 15:07:59 +00:00
fcd95677a0 v1.9.0
Some checks failed
Default (tags) / security (push) Successful in 51s
Default (tags) / test (push) Failing after 52s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-25 14:28:19 +00:00
547c262578 feat(auth): Implement HMAC-SHA256 OCI JWTs; enhance PyPI & RubyGems uploads and normalize responses 2025-11-25 14:28:19 +00:00
2d6059ba7f v1.8.0
Some checks failed
Default (tags) / security (push) Successful in 40s
Default (tags) / test (push) Failing after 37s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-24 00:15:29 +00:00
284329c191 feat(smarts3): Add local smarts3 testing support and documentation 2025-11-24 00:15:29 +00:00
4f662ff611 v1.7.0
Some checks failed
Default (tags) / security (push) Successful in 40s
Default (tags) / test (push) Failing after 36s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-23 23:54:42 +00:00
b3da95e6c1 feat(core): Standardize S3 storage config using @tsclass/tsclass IS3Descriptor and wire it into RegistryStorage and plugins exports; update README and package dependencies. 2025-11-23 23:54:41 +00:00
b1bb6af312 v1.6.0
Some checks failed
Default (tags) / security (push) Successful in 27s
Default (tags) / test (push) Failing after 36s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-21 17:13:06 +00:00
0d73230d5a feat(core): Add PyPI and RubyGems registries, integrate into SmartRegistry, extend storage and auth 2025-11-21 17:13:06 +00:00
ac51a94c8b v1.5.0
Some checks failed
Default (tags) / security (push) Successful in 40s
Default (tags) / test (push) Failing after 37s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-21 14:23:18 +00:00
9ca1e670ef feat(core): Add PyPI and RubyGems protocol support, Cargo token management, and storage helpers 2025-11-21 14:23:18 +00:00
fb8d6897e3 v1.4.1
Some checks failed
Default (tags) / security (push) Successful in 35s
Default (tags) / test (push) Failing after 35s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-21 09:36:02 +00:00
81ae4f2d59 fix(devcontainer): Simplify devcontainer configuration and rename container image 2025-11-21 09:36:02 +00:00
374469e37e v1.4.0
Some checks failed
Default (tags) / security (push) Successful in 39s
Default (tags) / test (push) Failing after 36s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-21 09:25:19 +00:00
9039613f7a feat(registrystorage): Add deleteMavenMetadata to RegistryStorage and update Maven DELETE test to expect 204 No Content 2025-11-21 09:25:19 +00:00
4d13fac9f1 v1.3.1
Some checks failed
Default (tags) / security (push) Successful in 25s
Default (tags) / test (push) Failing after 44s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-21 09:17:35 +00:00
42209d235d fix(maven): Pass request path to Maven checksum handler so checksum files are resolved correctly 2025-11-21 09:17:35 +00:00
80005af576 v1.3.0
Some checks failed
Default (tags) / security (push) Successful in 33s
Default (tags) / test (push) Failing after 36s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-21 09:13:02 +00:00
8d48627301 feat(core): Add Cargo and Composer registries with storage, auth and helpers 2025-11-21 09:13:02 +00:00
79 changed files with 22395 additions and 5023 deletions

View File

@@ -1,35 +1,5 @@
// The Dev Container format allows you to configure your environment. At the heart of it
// is a Docker image or Dockerfile which controls the tools available in your environment.
//
// See https://aka.ms/devcontainer.json for more information.
{
"name": "Ona",
// This universal image (~10GB) includes many development tools and languages,
// providing a convenient all-in-one development environment.
//
// This image is already available on remote runners for fast startup. On desktop
// and linux runners, it will need to be downloaded, which may take longer.
//
// For faster startup on desktop/linux, consider a smaller, language-specific image:
// • For Python: mcr.microsoft.com/devcontainers/python:3.13
// • For Node.js: mcr.microsoft.com/devcontainers/javascript-node:24
// • For Go: mcr.microsoft.com/devcontainers/go:1.24
// • For Java: mcr.microsoft.com/devcontainers/java:21
//
// Browse more options at: https://hub.docker.com/r/microsoft/devcontainers
// or build your own using the Dockerfile option below.
"name": "gitzone.universal",
"image": "mcr.microsoft.com/devcontainers/universal:4.0.1-noble"
// Use "build":
// instead of the image to use a Dockerfile to build an image.
// "build": {
// "context": ".",
// "dockerfile": "Dockerfile"
// }
// Features add additional features to your environment. See https://containers.dev/features
// Beware: features are not supported on all platforms and may have unintended side-effects.
// "features": {
// "ghcr.io/devcontainers/features/docker-in-docker": {
// "moby": false
// }
// }
}

24
.smartconfig.json Normal file
View File

@@ -0,0 +1,24 @@
{
"@git.zone/cli": {
"projectType": "npm",
"module": {
"githost": "code.foss.global",
"gitscope": "push.rocks",
"gitrepo": "smartregistry",
"description": "A composable TypeScript library implementing OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems registries for building unified container and package registries",
"npmPackagename": "@push.rocks/smartregistry",
"license": "MIT",
"projectDomain": "push.rocks"
},
"release": {
"registries": [
"https://verdaccio.lossless.digital",
"https://registry.npmjs.org"
],
"accessLevel": "public"
}
},
"@ship.zone/szci": {
"npmGlobalTools": []
}
}

View File

@@ -1,7 +1,7 @@
{
"json.schemas": [
{
"fileMatch": ["/npmextra.json"],
"fileMatch": ["/.smartconfig.json"],
"schema": {
"type": "object",
"properties": {

View File

@@ -1,5 +1,235 @@
# Changelog
## 2026-03-27 - 2.8.2 - fix(maven,tests)
handle Maven Basic auth and accept deploy-plugin metadata/checksum uploads while stabilizing npm CLI test cleanup
- Validate Maven tokens from Basic auth credentials by extracting the password portion before token validation.
- Return successful responses for PUT requests to checksum and maven-metadata endpoints so Maven deploy uploads do not fail when files are auto-generated.
- Improve npm CLI integration test isolation and cleanup by using a temporary test directory, copying per-package .npmrc files, and cleaning stale published packages before test runs.
- Tighten test teardown by destroying the registry explicitly and simplifying package/install fixture generation.
## 2026-03-24 - 2.8.1 - fix(registry)
align OCI and RubyGems API behavior and improve npm search result ordering
- handle OCI version checks on /v2 and /v2/ endpoints
- return RubyGems versions JSON in the expected flat array format and update unyank coverage to use the HTTP endpoint
- prioritize exact and prefix matches in npm search results
- update documentation to reflect full upstream proxy support
## 2026-03-24 - 2.8.0 - feat(core,storage,oci,registry-config)
add streaming response support and configurable registry URLs across protocols
- Normalize SmartRegistry responses to ReadableStream bodies at the public API boundary and add stream helper utilities for buffers, JSON, and hashing
- Add streaming storage accessors for OCI, npm, Maven, Cargo, Composer, PyPI, and RubyGems downloads to reduce in-memory buffering
- Make per-protocol registryUrl configurable so CLI and integration tests can use correct host and port values
- Refactor OCI blob uploads to persist chunks in storage during upload and clean up temporary chunk objects after completion or expiry
- Update tests and storage integration to use the new stream-based response model and smartstorage backend
## 2025-12-03 - 2.7.0 - feat(upstream)
Add dynamic per-request upstream provider and integrate into registries
- Introduce IUpstreamProvider and IUpstreamResolutionContext to resolve upstream configs per request.
- Add StaticUpstreamProvider implementation for simple static upstream configurations.
- Propagate dynamic upstream provider through SmartRegistry and wire into protocol handlers (npm, oci, maven, cargo, composer, pypi, rubygems).
- Replace persistent per-protocol upstream instances with per-request resolution: registries now call provider.resolveUpstreamConfig(...) and instantiate protocol-specific Upstream when needed.
- Add IRequestActor to core interfaces and pass actor context (userId, ip, userAgent, etc.) to upstream resolution and storage/auth hooks.
- Update many protocol registries to accept an upstreamProvider instead of IProtocolUpstreamConfig and to attempt upstream fetches only when provider returns enabled config.
- Add utilities and tests: test helpers to create registries with upstream provider, a tracking upstream provider helper, StaticUpstreamProvider tests and extensive upstream/provider integration tests.
- Improve upstream interfaces and cache/fetch contexts (IUpstreamFetchContext includes actor) and add StaticUpstreamProvider class to upstream module.
## 2025-11-27 - 2.6.0 - feat(core)
Add core registry infrastructure: storage, auth, upstream cache, and protocol handlers
- Introduce RegistryStorage: unified storage abstraction with hook support (before/after put/delete/get) and helpers for OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems paths and operations
- Add DefaultAuthProvider and AuthManager: in-memory token store, UUID tokens for package protocols, OCI JWT creation/validation, token lifecycle (create/validate/revoke) and authorization checking
- Add SmartRegistry orchestrator to initialize and route requests to protocol handlers (OCI, NPM, Maven, Cargo, Composer, PyPI, RubyGems)
- Implement upstream subsystem: UpstreamCache (in-memory + optional S3 persistence), BaseUpstream with multi-upstream routing, scope rules, retries, TTLs, stale-while-revalidate and negative caching
- Add circuit breaker implementation for upstream resilience with exponential backoff and per-upstream breakers
- Add protocol implementations and helpers: NpmRegistry/NpmUpstream (packument/tarball handling and tarball URL rewriting), PypiRegistry (PEP 503/691 support, uploads, metadata), MavenRegistry (artifact/metadata handling and checksum generation), CargoRegistry (sparse index, publish/download/yank)
- Utility exports and helpers: buffer helpers, plugins aggregator, path helpers, and various protocol-specific helper modules
## 2025-11-27 - 2.5.0 - feat(pypi,rubygems)
Add PyPI and RubyGems protocol implementations, upstream caching, and auth/storage improvements
- Implemented full PyPI support (PEP 503 Simple API HTML, PEP 691 JSON API, legacy upload handling, name normalization, hash verification, content negotiation, package/file storage and metadata management).
- Implemented RubyGems support (compact index, /versions, /info, /names endpoints, gem upload, yank/unyank, platform handling and file storage).
- Expanded RegistryStorage with protocol-specific helpers for OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems (get/put/delete/list helpers, metadata handling, context-aware hooks).
- Added AuthManager and DefaultAuthProvider improvements: unified token creation/validation for multiple protocols (npm, oci, maven, composer, cargo, pypi, rubygems) and OCI JWT support.
- Added upstream infrastructure: BaseUpstream, UpstreamCache (S3-backed optional, stale-while-revalidate, negative caching), circuit breaker with retries/backoff and resilience defaults.
- Added various protocol registries (NPM, Maven, Cargo, OCI, PyPI) with request routing, permission checks, and optional upstream proxying/caching.
## 2025-11-27 - 2.4.0 - feat(core)
Add pluggable auth providers, storage hooks, multi-upstream cache awareness, and PyPI/RubyGems protocol implementations
- Introduce pluggable authentication: IAuthProvider interface and DefaultAuthProvider (in-memory) with OCI JWT support and UUID tokens.
- AuthManager now accepts a custom provider and delegates all auth operations (authenticate, validateToken, create/revoke tokens, authorize, listUserTokens).
- Add storage hooks (IStorageHooks) and hook contexts: beforePut/afterPut/afterGet/beforeDelete/afterDelete. RegistryStorage now supports hooks, context management (setContext/withContext) and invokes hooks around operations.
- RegistryStorage expanded with many protocol-specific helper methods (OCI, NPM, Maven, Cargo, Composer, PyPI, RubyGems) and improved S3/SmartBucket integration.
- Upstream improvements: BaseUpstream and UpstreamCache became multi-upstream aware (cache keys now include upstream URL), cache operations are async and support negative caching, stale-while-revalidate, ETag/metadata persistence, and S3-backed storage layer.
- Circuit breaker, retry, resilience and scope-rule routing enhancements for upstreams; upstream fetch logic updated to prefer primary upstream for cache keys and background revalidation behavior.
- SmartRegistry API extended to accept custom authProvider and storageHooks, and now wires RegistryStorage and AuthManager with those options. Core exports updated to expose auth and storage interfaces and DefaultAuthProvider.
- Add full PyPI (PEP 503/691, upload API) and RubyGems (Compact Index, API v1, uploads/yank/unyank, specs endpoints) registry implementations with parsing, upload/download, metadata management and upstream proxying.
- Add utility helpers: binary buffer helpers (toBuffer/isBinaryData), pypi and rubygems helper modules, and numerous protocol-specific helpers and tests referenced in readme.hints.
- These changes are additive and designed to be backward compatible; bumping minor version.
## 2025-11-27 - 2.3.0 - feat(upstream)
Add upstream proxy/cache subsystem and integrate per-protocol upstreams
- Introduce a complete upstream subsystem (BaseUpstream, UpstreamCache, CircuitBreaker) with caching, negative-cache, stale-while-revalidate, retries, exponential backoff and per-upstream circuit breakers.
- Add upstream interfaces and defaults (ts/upstream/interfaces.upstream.ts) and export upstream utilities from ts/upstream/index.ts and root ts/index.ts.
- Implement protocol-specific upstream clients for npm, pypi, maven, composer, cargo and rubygems (classes.*upstream.ts) to fetch metadata and artifacts from configured upstream registries.
- Integrate upstream usage into registries: registries now accept an upstream config, attempt to fetch missing metadata/artifacts from upstreams, cache results locally, and expose destroy() to stop upstream resources.
- Add SmartRequest and minimatch to dependencies and expose smartrequest/minimatch via ts/plugins.ts for HTTP requests and glob-based scope matching.
- Update package.json to add @push.rocks/smartrequest and minimatch dependencies.
- Various registry implementations updated to utilize upstreams (npm, pypi, maven, composer, cargo, rubygems, oci) including URL rewrites and caching behavior.
## 2025-11-27 - 2.2.3 - fix(tests)
Use unique test run IDs and add S3 cleanup in test helpers to avoid cross-run conflicts
- Add generateTestRunId() helper in test/helpers/registry.ts to produce unique IDs for each test run
- Update PyPI and Composer native CLI tests to use generated testPackageName / unauth-pkg-<id> to avoid package name collisions between runs
- Import smartbucket and add S3 bucket cleanup logic in test helpers to remove leftover objects between test runs
- Improve test robustness by skipping upload-dependent checks when tools (twine/composer) are not available and logging outputs for debugging
## 2025-11-25 - 2.2.2 - fix(npm)
Replace console logging with structured Smartlog in NPM registry and silence RubyGems helper error logging
- Replaced console.log calls with this.logger.log (Smartlog) in ts/npm/classes.npmregistry.ts for debug/info/success events
- Converted console.error in NpmRegistry.handleSearch to structured logger.log('error', ...) including the error message
- Removed console.error from ts/rubygems/helpers.rubygems.ts; gem metadata extraction failures are now handled silently by returning null
## 2025-11-25 - 2.2.1 - fix(core)
Normalize binary data handling across registries and add buffer helpers
- Add core/helpers.buffer.ts with isBinaryData and toBuffer utilities to consistently handle Buffer, Uint8Array, string and object inputs.
- Composer: accept Uint8Array uploads, convert to Buffer before ZIP extraction, SHA-1 calculation and storage.
- PyPI: accept multipart file content as Buffer or Uint8Array and normalize to Buffer before processing and storage.
- Maven: normalize artifact body input with toBuffer before validation and storage.
- OCI: improve upload id generation by using substring for correct random length.
## 2025-11-25 - 2.2.0 - feat(core/registrystorage)
Persist OCI manifest content-type in sidecar and normalize manifest body handling
- Add getOciManifestContentType(repository, digest) to read stored manifest Content-Type
- Store manifest Content-Type in a .type sidecar file when putOciManifest is called
- Update putOciManifest to persist both manifest data and its content type
- OciRegistry now retrieves stored content type (with fallback to detectManifestContentType) when serving manifests
- Add toBuffer helper in OciRegistry to consistently convert various request body forms to Buffer for digest calculation and uploads
## 2025-11-25 - 2.1.2 - fix(oci)
Prefer raw request body for content-addressable OCI operations and expose rawBody on request context
- Add rawBody?: Buffer to IRequestContext to allow callers to provide the exact raw request bytes for digest calculation (falls back to body if absent).
- OCI registry handlers now prefer context.rawBody over context.body for content-addressable operations (manifests, blobs, and blob uploads) to preserve exact bytes and ensure digest calculation matches client expectations.
- Upload flow updates: upload init, PATCH (upload chunk) and PUT (complete upload) now pass rawBody when available.
## 2025-11-25 - 2.1.1 - fix(oci)
Preserve raw manifest bytes for digest calculation and handle string/JSON manifest bodies in OCI registry
- Preserve the exact bytes of the manifest payload when computing the sha256 digest to comply with the OCI spec and avoid mismatches caused by re-serialization.
- Accept string request bodies (converted using UTF-8) and treat already-parsed JSON objects by re-serializing as a fallback.
- Keep existing content-type fallback logic while ensuring accurate digest calculation prior to storing manifests.
## 2025-11-25 - 2.1.0 - feat(oci)
Support configurable OCI token realm/service and centralize unauthorized responses
- SmartRegistry now forwards optional ociTokens (realm and service) from auth configuration to OciRegistry when OCI is enabled
- OciRegistry constructor accepts an optional ociTokens parameter and stores it for use in auth headers
- Replaced repeated construction of WWW-Authenticate headers with createUnauthorizedResponse and createUnauthorizedHeadResponse helpers that use configured realm/service
- Behavior is backwards-compatible: when ociTokens are not configured the registry falls back to the previous defaults (realm: <basePath>/v2/token, service: "registry")
## 2025-11-25 - 2.0.0 - BREAKING CHANGE(pypi,rubygems)
Revise PyPI and RubyGems handling: normalize error payloads, fix .gem parsing/packing, adjust PyPI JSON API and tests, and export smartarchive plugin
- Rename error payload property from 'message' to 'error' in PyPI and RubyGems interfaces and responses; error responses are now returned as JSON objects (body: { error: ... }) instead of Buffer(JSON.stringify(...)).
- RubyGems: treat .gem files as plain tar archives (not gzipped). Use metadata.gz and data.tar.gz correctly, switch packing helper to pack plain tar, and use zlib deflate for .rz gemspec data.
- RubyGems registry: add legacy Marshal specs endpoint (specs.4.8.gz) and adjust versions handler invocation to accept request context.
- PyPI: adopt PEP 691 style (files is an array of file objects) in tests and metadata; include requires_python in test package metadata; update JSON API path matching to the package-level '/{package}/json' style used by the handler.
- Fix HTML escaping expectations in tests (requires_python values are HTML-escaped in attributes, e.g. '&gt;=3.8').
- Export smartarchive from plugins to enable archive helpers in core modules and helpers.
- Update tests and internal code to match the new error shape and API/format behaviour.
## 2025-11-25 - 1.9.0 - feat(auth)
Implement HMAC-SHA256 OCI JWTs; enhance PyPI & RubyGems uploads and normalize responses
- AuthManager: create and validate OCI JWTs signed with HMAC-SHA256 (header.payload.signature). Signature verification, exp/nbf checks and payload decoding implemented.
- PyPI: improved Simple API handling (PEP-691 JSON responses returned as objects), Simple HTML responses updated, upload handling enhanced to support nested/flat multipart fields, verify hashes (sha256/md5/blake2b), store files and return 201 on success.
- RubyGems: upload flow now attempts to extract gem metadata from the .gem binary when name/version are not provided, improved validation, and upload returns 201. Added extractGemMetadata helper.
- OCI: centralized 401 response creation (including proper WWW-Authenticate header) and HEAD behavior fixed to return no body per HTTP spec.
- SmartRegistry: use nullish coalescing for protocol basePath defaults to avoid falsy-value bugs when basePath is an empty string.
- Tests and helpers: test expectations adjusted (Content-Type startsWith check for HTML, PEP-691 projects is an array), test helper switched to smartarchive for packaging.
- Package.json: added devDependency @push.rocks/smartarchive and updated dev deps.
- Various response normalization: avoid unnecessary Buffer.from() for already-serialized objects/strings and standardize status codes for create/upload endpoints (201).
## 2025-11-24 - 1.8.0 - feat(smarts3)
Add local smarts3 testing support and documentation
- Added @push.rocks/smarts3 ^5.1.0 to devDependencies to enable a local S3-compatible test server.
- Updated README with a new "Testing with smarts3" section including a Quick Start example and integration test commands.
- Documented benefits and CI-friendly usage for running registry integration tests locally without cloud credentials.
## 2025-11-23 - 1.7.0 - feat(core)
Standardize S3 storage config using @tsclass/tsclass IS3Descriptor and wire it into RegistryStorage and plugins exports; update README and package dependencies.
- Add @tsclass/tsclass dependency to package.json to provide a standardized IS3Descriptor for S3 configuration.
- Export tsclass from ts/plugins.ts so plugin types are available to core modules.
- Update IStorageConfig to extend plugins.tsclass.storage.IS3Descriptor, consolidating storage configuration typing.
- Change RegistryStorage.init() to pass the storage config directly as an IS3Descriptor to SmartBucket (bucketName remains part of IStorageConfig).
- Update README storage section with example config and mention IS3Descriptor integration.
## 2025-11-21 - 1.6.0 - feat(core)
Add PyPI and RubyGems registries, integrate into SmartRegistry, extend storage and auth
- Introduce PyPI registry implementation with PEP 503 (Simple API) and PEP 691 (JSON API), legacy upload support, content negotiation and HTML/JSON generators (ts/pypi/*).
- Introduce RubyGems registry implementation with Compact Index support, API v1 endpoints (upload, yank/unyank), versions/names files and helpers (ts/rubygems/*).
- Wire PyPI and RubyGems into the main orchestrator: SmartRegistry now initializes, exposes and routes requests to pypi and rubygems handlers.
- Extend RegistryStorage with PyPI and RubyGems storage helpers (metadata, simple index, package files, compact index files, gem files).
- Extend AuthManager to support PyPI and RubyGems UUID token creation, validation and revocation and include them in unified token validation.
- Add verification of client-provided hashes during PyPI uploads (SHA256 always calculated and verified; MD5 and Blake2b verified when provided) to prevent corrupted uploads.
- Export new modules from library entry point (ts/index.ts) and add lightweight rubygems index file export.
- Add helper utilities for PyPI and RubyGems (name normalization, HTML generation, hash calculations, compact index generation/parsing).
- Update documentation hints/readme to reflect implementation status and configuration examples for pypi and rubygems.
## 2025-11-21 - 1.5.0 - feat(core)
Add PyPI and RubyGems protocol support, Cargo token management, and storage helpers
- Extend core protocol types to include 'pypi' and 'rubygems' and add protocol config entries for pypi and rubygems.
- Add PyPI storage methods for metadata, Simple API HTML/JSON indexes, package files, version listing and deletion in RegistryStorage.
- Add Cargo-specific storage helpers (index paths, crate storage) and ensure Cargo registry initialization and endpoints are wired into SmartRegistry.
- Extend AuthManager with Cargo, PyPI and RubyGems token creation, validation and revocation methods; update unified validateToken to check these token types.
- Update test helpers to create Cargo tokens and return cargoToken from registry setup.
## 2025-11-21 - 1.4.1 - fix(devcontainer)
Simplify devcontainer configuration and rename container image
- Rename Dev Container name to 'gitzone.universal' and set image to mcr.microsoft.com/devcontainers/universal:4.0.1-noble
- Remove large inline comments and example 'build'/'features' blocks to simplify the devcontainer.json
## 2025-11-21 - 1.4.0 - feat(registrystorage)
Add deleteMavenMetadata to RegistryStorage and update Maven DELETE test to expect 204 No Content
- Add deleteMavenMetadata(groupId, artifactId) to RegistryStorage to remove maven-metadata.xml.
- Update Maven test to assert 204 No Content for DELETE responses (previously expected 200).
## 2025-11-21 - 1.3.1 - fix(maven)
Pass request path to Maven checksum handler so checksum files are resolved correctly
- Call handleChecksumRequest with the full request path from MavenRegistry.handleRequest
- Allows getChecksum to extract the checksum filename from the URL and fetch the correct checksum file from storage
- Fixes 404s when requesting artifact checksum files (md5, sha1, sha256, sha512)
## 2025-11-21 - 1.3.0 - feat(core)
Add Cargo and Composer registries with storage, auth and helpers
- Add Cargo registry implementation (ts/cargo) including index, publish, download, yank/unyank and search handlers
- Add Composer registry implementation (ts/composer) including package upload/download, metadata, packages.json and helpers
- Extend RegistryStorage with Cargo and Composer-specific storage helpers and path conventions
- Extend AuthManager with Composer token creation/validation and unified token validation support
- Wire SmartRegistry to initialize and route requests to cargo and composer handlers
- Add adm-zip dependency and Composer ZIP parsing helpers (extractComposerJsonFromZip, sha1 calculation, version sorting)
- Add tests for Cargo index path calculation and config handling
- Export new modules from ts/index.ts and add module entry files for composer and cargo
## 2025-11-21 - 1.2.0 - feat(maven)
Add Maven registry protocol support (storage, auth, routing, interfaces, and exports)

View File

@@ -1,18 +0,0 @@
{
"gitzone": {
"projectType": "npm",
"module": {
"githost": "code.foss.global",
"gitscope": "push.rocks",
"gitrepo": "smartregistry",
"description": "a registry for npm modules and oci images",
"npmPackagename": "@push.rocks/smartregistry",
"license": "MIT",
"projectDomain": "push.rocks"
}
},
"npmci": {
"npmGlobalTools": [],
"npmAccessLevel": "public"
}
}

View File

@@ -1,8 +1,8 @@
{
"name": "@push.rocks/smartregistry",
"version": "1.2.0",
"version": "2.8.2",
"private": false,
"description": "a registry for npm modules and oci images",
"description": "A composable TypeScript library implementing OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems registries for building unified container and package registries",
"main": "dist_ts/index.js",
"typings": "dist_ts/index.d.ts",
"type": "module",
@@ -10,15 +10,17 @@
"license": "MIT",
"scripts": {
"test": "(tstest test/ --verbose --logfile --timeout 240)",
"build": "(tsbuild --web --allowimplicitany)",
"build": "(tsbuild --allowimplicitany)",
"buildDocs": "(tsdoc)"
},
"devDependencies": {
"@git.zone/tsbuild": "^3.1.0",
"@git.zone/tsbundle": "^2.0.5",
"@git.zone/tsrun": "^2.0.0",
"@git.zone/tstest": "^3.1.0",
"@types/node": "^24.10.1"
"@git.zone/tsbuild": "^4.4.0",
"@git.zone/tsbundle": "^2.10.0",
"@git.zone/tsrun": "^2.0.2",
"@git.zone/tstest": "^3.6.0",
"@push.rocks/smartarchive": "^5.2.1",
"@push.rocks/smartstorage": "^6.3.2",
"@types/node": "^25.5.0"
},
"repository": {
"type": "git",
@@ -37,7 +39,7 @@
"dist_ts_web/**/*",
"assets/**/*",
"cli.js",
"npmextra.json",
".smartconfig.json",
"readme.md"
],
"pnpm": {
@@ -45,9 +47,13 @@
},
"dependencies": {
"@push.rocks/qenv": "^6.1.3",
"@push.rocks/smartbucket": "^4.3.0",
"@push.rocks/smartlog": "^3.1.10",
"@push.rocks/smartpath": "^6.0.0"
"@push.rocks/smartbucket": "^4.5.1",
"@push.rocks/smartlog": "^3.2.1",
"@push.rocks/smartpath": "^6.0.0",
"@push.rocks/smartrequest": "^5.0.1",
"@tsclass/tsclass": "^9.5.0",
"adm-zip": "^0.5.16",
"minimatch": "^10.2.4"
},
"packageManager": "pnpm@10.18.1+sha512.77a884a165cbba2d8d1c19e3b4880eee6d2fcabd0d879121e282196b80042351d5eb3ca0935fa599da1dc51265cc68816ad2bddd2a2de5ea9fdf92adbec7cd34"
}

7077
pnpm-lock.yaml generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,3 +1,439 @@
# Project Readme Hints
# Project Implementation Notes
This is the initial readme hints file.
This file contains technical implementation details for PyPI and RubyGems protocols.
## Python (PyPI) Protocol Implementation ✅
### PEP 503: Simple Repository API (HTML-based)
**URL Structure:**
- Root: `/<base>/` - Lists all projects
- Project: `/<base>/<project>/` - Lists all files for a project
- All URLs MUST end with `/` (redirect if missing)
**Package Name Normalization:**
- Lowercase all characters
- Replace runs of `.`, `-`, `_` with single `-`
- Implementation: `re.sub(r"[-_.]+", "-", name).lower()`
**HTML Format:**
- Root: One anchor per project
- Project: One anchor per file
- Anchor text must match final filename
- Anchor href links to download URL
**Hash Fragments:**
Format: `#<hashname>=<hashvalue>`
- hashname: lowercase hash function name (recommend `sha256`)
- hashvalue: hex-encoded digest
**Data Attributes:**
- `data-gpg-sig`: `true`/`false` for GPG signature presence
- `data-requires-python`: PEP 345 requirement string (HTML-encode `<` as `&lt;`, `>` as `&gt;`)
### PEP 691: JSON-based Simple API
**Content Types:**
- `application/vnd.pypi.simple.v1+json` - JSON format
- `application/vnd.pypi.simple.v1+html` - HTML format
- `text/html` - Alias for HTML (backwards compat)
**Root Endpoint JSON:**
```json
{
"meta": {"api-version": "1.0"},
"projects": [{"name": "ProjectName"}]
}
```
**Project Endpoint JSON:**
```json
{
"name": "normalized-name",
"meta": {"api-version": "1.0"},
"files": [
{
"filename": "package-1.0-py3-none-any.whl",
"url": "https://example.com/path/to/file",
"hashes": {"sha256": "..."},
"requires-python": ">=3.7",
"dist-info-metadata": true | {"sha256": "..."},
"gpg-sig": true,
"yanked": false | "reason string"
}
]
}
```
**Content Negotiation:**
- Use `Accept` header for format selection
- Server responds with `Content-Type` header
- Support both JSON and HTML formats
### PyPI Upload API (Legacy /legacy/)
**Endpoint:**
- URL: `https://upload.pypi.org/legacy/`
- Method: `POST`
- Content-Type: `multipart/form-data`
**Required Form Fields:**
- `:action` = `file_upload`
- `protocol_version` = `1`
- `content` = Binary file data with filename
- `filetype` = `bdist_wheel` | `sdist`
- `pyversion` = Python tag (e.g., `py3`, `py2.py3`) or `source` for sdist
- `metadata_version` = Metadata standard version
- `name` = Package name
- `version` = Version string
**Hash Digest (one required):**
- `md5_digest`: urlsafe base64 without padding
- `sha256_digest`: hexadecimal
- `blake2_256_digest`: hexadecimal
**Optional Fields:**
- `attestations`: JSON array of attestation objects
- Any Core Metadata fields (lowercase, hyphens → underscores)
- Example: `Description-Content-Type``description_content_type`
**Authentication:**
- Username/password or API token in HTTP Basic Auth
- API tokens: username = `__token__`, password = token value
**Behavior:**
- First file uploaded creates the release
- Multiple files uploaded sequentially for same version
### PEP 694: Upload 2.0 API
**Status:** Draft (not yet required, legacy API still supported)
- Multi-step workflow with sessions
- Async upload support with resumption
- JSON-based API
- Standard HTTP auth (RFC 7235)
- Not implementing initially (legacy API sufficient)
---
## Ruby (RubyGems) Protocol Implementation ✅
### Compact Index Format
**Endpoints:**
- `/versions` - Master list of all gems and versions
- `/info/<RUBYGEM>` - Detailed info for specific gem
- `/names` - Simple list of gem names
**Authentication:**
- UUID tokens similar to NPM pattern
- API key in `Authorization` header
- Scope format: `rubygems:gem:{name}:{read|write|yank}`
### `/versions` File Format
**Structure:**
```
created_at: 2024-04-01T00:00:05Z
---
RUBYGEM [-]VERSION_PLATFORM[,VERSION_PLATFORM,...] MD5
```
**Details:**
- Metadata lines before `---` delimiter
- One line per gem with comma-separated versions
- `[-]` prefix indicates yanked version
- `MD5`: Checksum of corresponding `/info/<RUBYGEM>` file
- Append-only during month, recalculated monthly
### `/info/<RUBYGEM>` File Format
**Structure:**
```
---
VERSION[-PLATFORM] [DEPENDENCY[,DEPENDENCY,...]]|REQUIREMENT[,REQUIREMENT,...]
```
**Dependency Format:**
```
GEM:CONSTRAINT[&CONSTRAINT]
```
- Examples: `actionmailer:= 2.2.2`, `parser:>= 3.2.2.3`
- Operators: `=`, `>`, `<`, `>=`, `<=`, `~>`, `!=`
- Multiple constraints: `unicode-display_width:< 3.0&>= 2.4.0`
**Requirement Format:**
```
checksum:SHA256_HEX
ruby:CONSTRAINT
rubygems:CONSTRAINT
```
**Platform:**
- Default platform is `ruby`
- Non-default platforms: `VERSION-PLATFORM` (e.g., `3.2.1-arm64-darwin`)
**Yanked Gems:**
- Listed with `-` prefix in `/versions`
- Excluded entirely from `/info/<RUBYGEM>` file
### `/names` File Format
```
---
gemname1
gemname2
gemname3
```
### HTTP Range Support
**Headers:**
- `Range: bytes=#{start}-`: Request from byte position
- `If-None-Match`: ETag conditional request
- `Repr-Digest`: SHA256 checksum in response
**Caching Strategy:**
1. Store file with last byte position
2. Request range from last position
3. Append response to existing file
4. Verify SHA256 against `Repr-Digest`
### RubyGems Upload/Management API
**Upload Gem:**
- `POST /api/v1/gems`
- Binary `.gem` file in request body
- `Authorization` header with API key
**Yank Version:**
- `DELETE /api/v1/gems/yank`
- Parameters: `gem_name`, `version`
**Unyank Version:**
- `PUT /api/v1/gems/unyank`
- Parameters: `gem_name`, `version`
**Version Metadata:**
- `GET /api/v1/versions/<gem>.json`
- Returns JSON array of versions
**Dependencies:**
- `GET /api/v1/dependencies?gems=<comma-list>`
- Returns dependency information for resolution
---
## Implementation Details
### Completed Protocols
- ✅ OCI Distribution Spec v1.1
- ✅ NPM Registry API
- ✅ Maven Repository
- ✅ Cargo/crates.io Registry
- ✅ Composer/Packagist
- ✅ PyPI (Python Package Index) - PEP 503/691
- ✅ RubyGems - Compact Index
### Storage Paths
**PyPI:**
```
pypi/
├── simple/ # PEP 503 HTML files
│ ├── index.html # All packages list
│ └── {package}/index.html # Package versions list
├── packages/
│ └── {package}/{filename} # .whl and .tar.gz files
└── metadata/
└── {package}/metadata.json # Package metadata
```
**RubyGems:**
```
rubygems/
├── versions # Master versions file
├── info/{gemname} # Per-gem info files
├── names # All gem names
└── gems/{gemname}-{version}.gem # .gem files
```
### Authentication Pattern
Both protocols should follow the existing UUID token pattern used by NPM, Maven, Cargo, Composer:
```typescript
// AuthManager additions
createPypiToken(userId: string, readonly: boolean): string
validatePypiToken(token: string): ITokenInfo | null
revokePypiToken(token: string): boolean
createRubyGemsToken(userId: string, readonly: boolean): string
validateRubyGemsToken(token: string): ITokenInfo | null
revokeRubyGemsToken(token: string): boolean
```
### Scope Format
```
pypi:package:{name}:{read|write}
rubygems:gem:{name}:{read|write|yank}
```
### Common Patterns
1. **Package name normalization** - Critical for PyPI
2. **Checksum calculation** - SHA256 for both protocols
3. **Append-only files** - RubyGems compact index
4. **Content negotiation** - PyPI JSON vs HTML
5. **Multipart upload parsing** - PyPI file uploads
6. **Binary file handling** - Both protocols (.whl, .tar.gz, .gem)
---
## Key Differences from Existing Protocols
**PyPI vs NPM:**
- PyPI uses Simple API (HTML) + JSON API
- PyPI requires package name normalization
- PyPI uses multipart form data for uploads (not JSON)
- PyPI supports multiple file types per release (wheel + sdist)
**RubyGems vs Cargo:**
- RubyGems uses compact index (append-only text files)
- RubyGems uses checksums in index files (not just filenames)
- RubyGems has HTTP Range support for incremental updates
- RubyGems uses MD5 for index checksums, SHA256 for .gem files
---
## Testing Requirements
### PyPI Tests Must Cover:
- Package upload (wheel and sdist)
- Package name normalization
- Simple API HTML generation (PEP 503)
- JSON API responses (PEP 691)
- Content negotiation
- Hash calculation and verification
- Authentication (tokens)
- Multi-file releases
- Yanked packages
### RubyGems Tests Must Cover:
- Gem upload
- Compact index generation
- `/versions` file updates (append-only)
- `/info/<gem>` file generation
- `/names` file generation
- Checksum calculations (MD5 and SHA256)
- Platform-specific gems
- Yanking/unyanking
- HTTP Range requests
- Authentication (API keys)
---
## Security Considerations
1. **Package name validation** - Prevent path traversal
2. **File size limits** - Prevent DoS via large uploads
3. **Content-Type validation** - Verify file types
4. **Checksum verification** - Ensure file integrity
5. **Token scope enforcement** - Read vs write permissions
6. **HTML escaping** - Prevent XSS in generated HTML
7. **Metadata sanitization** - Clean user-provided strings
8. **Rate limiting** - Consider upload frequency limits
---
## Implementation Status (Completed)
### PyPI Implementation ✅
- **Files Created:**
- `ts/pypi/interfaces.pypi.ts` - Type definitions (354 lines)
- `ts/pypi/helpers.pypi.ts` - Helper functions (280 lines)
- `ts/pypi/classes.pypiregistry.ts` - Main registry (650 lines)
- `ts/pypi/index.ts` - Module exports
- **Features Implemented:**
- ✅ PEP 503 Simple API (HTML)
- ✅ PEP 691 JSON API
- ✅ Content negotiation (Accept header)
- ✅ Package name normalization
- ✅ File upload with multipart/form-data
- ✅ Hash verification (SHA256, MD5, Blake2b)
- ✅ Package metadata management
- ✅ JSON API endpoints (/pypi/{package}/json)
- ✅ Token-based authentication
- ✅ Scope-based permissions (read/write/delete)
- **Security Enhancements:**
- ✅ Hash verification on upload (validates client-provided hashes)
- ✅ Package name validation (regex check)
- ✅ HTML escaping in generated pages
- ✅ Permission checks on all mutating operations
### RubyGems Implementation ✅
- **Files Created:**
- `ts/rubygems/interfaces.rubygems.ts` - Type definitions (215 lines)
- `ts/rubygems/helpers.rubygems.ts` - Helper functions (350 lines)
- `ts/rubygems/classes.rubygemsregistry.ts` - Main registry (580 lines)
- `ts/rubygems/index.ts` - Module exports
- **Features Implemented:**
- ✅ Compact Index format (modern Bundler)
- ✅ /versions endpoint (all gems list)
- ✅ /info/{gem} endpoint (gem-specific metadata)
- ✅ /names endpoint (gem names list)
- ✅ Gem upload API
- ✅ Yank/unyank functionality
- ✅ Platform-specific gems support
- ✅ JSON API endpoints
- ✅ Legacy endpoints (specs.4.8.gz, Marshal.4.8)
- ✅ Token-based authentication
- ✅ Scope-based permissions
### Integration ✅
- **Core Updates:**
- ✅ Updated `IRegistryConfig` interface
- ✅ Updated `TRegistryProtocol` type
- ✅ Added authentication methods to `AuthManager`
- ✅ Added 30+ storage methods to `RegistryStorage`
- ✅ Updated `SmartRegistry` initialization and routing
- ✅ Module exports from `ts/index.ts`
- **Test Coverage:**
-`test/test.pypi.ts` - 25+ tests covering all PyPI endpoints
-`test/test.rubygems.ts` - 30+ tests covering all RubyGems endpoints
-`test/test.integration.pypi-rubygems.ts` - Integration tests
- ✅ Updated test helpers with PyPI and RubyGems support
### Known Limitations
1. **PyPI:**
- Does not implement legacy XML-RPC API
- No support for PGP signatures (data-gpg-sig always false)
- Metadata extraction from wheel files not implemented
2. **RubyGems:**
- Gem spec extraction from .gem files returns placeholder (Ruby Marshal parsing not implemented)
- Legacy Marshal endpoints return basic data only
- No support for gem dependencies resolution
### Configuration Example
```typescript
{
pypi: {
enabled: true,
basePath: '/pypi', // Also handles /simple
},
rubygems: {
enabled: true,
basePath: '/rubygems',
},
auth: {
pypiTokens: { enabled: true },
rubygemsTokens: { enabled: true },
}
}
```

836
readme.md

File diff suppressed because it is too large Load Diff

133
test/cargo.test.node.ts Normal file
View File

@@ -0,0 +1,133 @@
import { tap, expect } from '@git.zone/tstest';
import { RegistryStorage } from '../ts/core/classes.registrystorage.js';
import { CargoRegistry } from '../ts/cargo/classes.cargoregistry.js';
import { AuthManager } from '../ts/core/classes.authmanager.js';
import { streamToJson } from '../ts/core/helpers.stream.js';
// Test index path calculation
tap.test('should calculate correct index paths for different crate names', async () => {
const storage = new RegistryStorage({
accessKey: 'test',
accessSecret: 'test',
endpoint: 's3.test.com',
bucketName: 'test-bucket',
});
// Access private method for testing
const getPath = (storage as any).getCargoIndexPath.bind(storage);
// 1-character names
expect(getPath('a')).to.equal('cargo/index/1/a');
expect(getPath('z')).to.equal('cargo/index/1/z');
// 2-character names
expect(getPath('io')).to.equal('cargo/index/2/io');
expect(getPath('ab')).to.equal('cargo/index/2/ab');
// 3-character names
expect(getPath('axo')).to.equal('cargo/index/3/a/axo');
expect(getPath('foo')).to.equal('cargo/index/3/f/foo');
// 4+ character names
expect(getPath('serde')).to.equal('cargo/index/se/rd/serde');
expect(getPath('tokio')).to.equal('cargo/index/to/ki/tokio');
expect(getPath('my-crate')).to.equal('cargo/index/my/--/my-crate');
});
// Test crate file path calculation
tap.test('should calculate correct crate file paths', async () => {
const storage = new RegistryStorage({
accessKey: 'test',
accessSecret: 'test',
endpoint: 's3.test.com',
bucketName: 'test-bucket',
});
// Access private method for testing
const getPath = (storage as any).getCargoCratePath.bind(storage);
expect(getPath('serde', '1.0.0')).to.equal('cargo/crates/serde/serde-1.0.0.crate');
expect(getPath('tokio', '1.28.0')).to.equal('cargo/crates/tokio/tokio-1.28.0.crate');
expect(getPath('my-crate', '0.1.0')).to.equal('cargo/crates/my-crate/my-crate-0.1.0.crate');
});
// Test crate name validation
tap.test('should validate crate names correctly', async () => {
const storage = new RegistryStorage({
accessKey: 'test',
accessSecret: 'test',
endpoint: 's3.test.com',
bucketName: 'test-bucket',
});
const authManager = new AuthManager({
jwtSecret: 'test-secret',
tokenStore: 'memory',
npmTokens: { enabled: true },
ociTokens: { enabled: false, realm: '', service: '' },
});
const registry = new CargoRegistry(storage, authManager, '/cargo', 'http://localhost:5000/cargo');
// Access private method for testing
const validate = (registry as any).validateCrateName.bind(registry);
// Valid names
expect(validate('serde')).to.be.true;
expect(validate('tokio')).to.be.true;
expect(validate('my-crate')).to.be.true;
expect(validate('my_crate')).to.be.true;
expect(validate('crate123')).to.be.true;
expect(validate('a')).to.be.true;
// Invalid names (uppercase not allowed)
expect(validate('Serde')).to.be.false;
expect(validate('MyCreate')).to.be.false;
// Invalid names (special characters)
expect(validate('my.crate')).to.be.false;
expect(validate('my@crate')).to.be.false;
expect(validate('my crate')).to.be.false;
// Invalid names (too long)
const longName = 'a'.repeat(65);
expect(validate(longName)).to.be.false;
// Invalid names (empty)
expect(validate('')).to.be.false;
});
// Test config.json response
tap.test('should return valid config.json', async () => {
const storage = new RegistryStorage({
accessKey: 'test',
accessSecret: 'test',
endpoint: 's3.test.com',
bucketName: 'test-bucket',
});
const authManager = new AuthManager({
jwtSecret: 'test-secret',
tokenStore: 'memory',
npmTokens: { enabled: true },
ociTokens: { enabled: false, realm: '', service: '' },
});
const registry = new CargoRegistry(storage, authManager, '/cargo', 'http://localhost:5000/cargo');
const response = await registry.handleRequest({
method: 'GET',
path: '/cargo/config.json',
headers: {},
query: {},
});
expect(response.status).to.equal(200);
expect(response.headers['Content-Type']).to.equal('application/json');
const body = await streamToJson(response.body);
expect(body).to.be.an('object');
expect(body.dl).to.include('/api/v1/crates/{crate}/{version}/download');
expect(body.api).to.equal('http://localhost:5000/cargo');
});
export default tap.start();

View File

@@ -1,14 +1,73 @@
import * as qenv from '@push.rocks/qenv';
import * as crypto from 'crypto';
import * as smartarchive from '@push.rocks/smartarchive';
import * as smartbucket from '@push.rocks/smartbucket';
import { SmartRegistry } from '../../ts/classes.smartregistry.js';
import type { IRegistryConfig } from '../../ts/core/interfaces.core.js';
import type { IRegistryConfig, IAuthToken, TRegistryProtocol } from '../../ts/core/interfaces.core.js';
import type { IAuthProvider, ITokenOptions } from '../../ts/core/interfaces.auth.js';
import type { IStorageHooks, IStorageHookContext, IBeforePutResult, IBeforeDeleteResult } from '../../ts/core/interfaces.storage.js';
import { StaticUpstreamProvider } from '../../ts/upstream/interfaces.upstream.js';
import type { IUpstreamProvider, IUpstreamResolutionContext, IProtocolUpstreamConfig } from '../../ts/upstream/interfaces.upstream.js';
const testQenv = new qenv.Qenv('./', './.nogit');
/**
* Create a test SmartRegistry instance with OCI, NPM, and Maven enabled
* Clean up S3 bucket contents for a fresh test run
* @param prefix Optional prefix to delete (e.g., 'cargo/', 'npm/', 'composer/')
*/
export async function createTestRegistry(): Promise<SmartRegistry> {
/**
* Generate a unique test run ID for avoiding conflicts between test runs
* Uses timestamp + random suffix for uniqueness
*/
export function generateTestRunId(): string {
const timestamp = Date.now().toString(36);
const random = Math.random().toString(36).substring(2, 6);
return `${timestamp}${random}`;
}
export async function cleanupS3Bucket(prefix?: string): Promise<void> {
const s3AccessKey = await testQenv.getEnvVarOnDemand('S3_ACCESSKEY');
const s3SecretKey = await testQenv.getEnvVarOnDemand('S3_SECRETKEY');
const s3Endpoint = await testQenv.getEnvVarOnDemand('S3_ENDPOINT');
const s3Port = await testQenv.getEnvVarOnDemand('S3_PORT');
const s3 = new smartbucket.SmartBucket({
accessKey: s3AccessKey || 'minioadmin',
accessSecret: s3SecretKey || 'minioadmin',
endpoint: s3Endpoint || 'localhost',
port: parseInt(s3Port || '9000', 10),
useSsl: false,
});
try {
const bucket = await s3.getBucket('test-registry');
if (bucket) {
if (prefix) {
// Delete only objects with the given prefix
const files = await bucket.fastList({ prefix });
for (const file of files) {
await bucket.fastRemove({ path: file.name });
}
} else {
// Delete all objects in the bucket
const files = await bucket.fastList({});
for (const file of files) {
await bucket.fastRemove({ path: file.name });
}
}
}
} catch (error) {
// Bucket might not exist yet, that's fine
console.log('Cleanup: No bucket to clean or error:', error);
}
}
/**
* Create a test SmartRegistry instance with all protocols enabled
*/
export async function createTestRegistry(options?: {
registryUrl?: string;
}): Promise<SmartRegistry> {
// Read S3 config from env.json
const s3AccessKey = await testQenv.getEnvVarOnDemand('S3_ACCESSKEY');
const s3SecretKey = await testQenv.getEnvVarOnDemand('S3_SECRETKEY');
@@ -36,18 +95,47 @@ export async function createTestRegistry(): Promise<SmartRegistry> {
realm: 'https://auth.example.com/token',
service: 'test-registry',
},
pypiTokens: {
enabled: true,
},
rubygemsTokens: {
enabled: true,
},
},
oci: {
enabled: true,
basePath: '/oci',
...(options?.registryUrl ? { registryUrl: `${options.registryUrl}/oci` } : {}),
},
npm: {
enabled: true,
basePath: '/npm',
...(options?.registryUrl ? { registryUrl: `${options.registryUrl}/npm` } : {}),
},
maven: {
enabled: true,
basePath: '/maven',
...(options?.registryUrl ? { registryUrl: `${options.registryUrl}/maven` } : {}),
},
composer: {
enabled: true,
basePath: '/composer',
...(options?.registryUrl ? { registryUrl: `${options.registryUrl}/composer` } : {}),
},
cargo: {
enabled: true,
basePath: '/cargo',
...(options?.registryUrl ? { registryUrl: `${options.registryUrl}/cargo` } : {}),
},
pypi: {
enabled: true,
basePath: '/pypi',
...(options?.registryUrl ? { registryUrl: options.registryUrl } : {}),
},
rubygems: {
enabled: true,
basePath: '/rubygems',
...(options?.registryUrl ? { registryUrl: `${options.registryUrl}/rubygems` } : {}),
},
};
@@ -57,6 +145,89 @@ export async function createTestRegistry(): Promise<SmartRegistry> {
return registry;
}
/**
* Create a test SmartRegistry instance with upstream provider configured
*/
export async function createTestRegistryWithUpstream(
upstreamProvider?: IUpstreamProvider
): Promise<SmartRegistry> {
// Read S3 config from env.json
const s3AccessKey = await testQenv.getEnvVarOnDemand('S3_ACCESSKEY');
const s3SecretKey = await testQenv.getEnvVarOnDemand('S3_SECRETKEY');
const s3Endpoint = await testQenv.getEnvVarOnDemand('S3_ENDPOINT');
const s3Port = await testQenv.getEnvVarOnDemand('S3_PORT');
// Default to StaticUpstreamProvider with npm.js configured
const defaultProvider = new StaticUpstreamProvider({
npm: {
enabled: true,
upstreams: [{ id: 'npmjs', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
},
oci: {
enabled: true,
upstreams: [{ id: 'dockerhub', url: 'https://registry-1.docker.io', priority: 1, enabled: true }],
},
});
const config: IRegistryConfig = {
storage: {
accessKey: s3AccessKey || 'minioadmin',
accessSecret: s3SecretKey || 'minioadmin',
endpoint: s3Endpoint || 'localhost',
port: parseInt(s3Port || '9000', 10),
useSsl: false,
region: 'us-east-1',
bucketName: 'test-registry',
},
auth: {
jwtSecret: 'test-secret-key',
tokenStore: 'memory',
npmTokens: { enabled: true },
ociTokens: {
enabled: true,
realm: 'https://auth.example.com/token',
service: 'test-registry',
},
pypiTokens: { enabled: true },
rubygemsTokens: { enabled: true },
},
upstreamProvider: upstreamProvider || defaultProvider,
oci: { enabled: true, basePath: '/oci' },
npm: { enabled: true, basePath: '/npm' },
maven: { enabled: true, basePath: '/maven' },
composer: { enabled: true, basePath: '/composer' },
cargo: { enabled: true, basePath: '/cargo' },
pypi: { enabled: true, basePath: '/pypi' },
rubygems: { enabled: true, basePath: '/rubygems' },
};
const registry = new SmartRegistry(config);
await registry.init();
return registry;
}
/**
* Create a mock upstream provider that tracks all calls for testing
*/
export function createTrackingUpstreamProvider(
baseConfig?: Partial<Record<TRegistryProtocol, IProtocolUpstreamConfig>>
): {
provider: IUpstreamProvider;
calls: IUpstreamResolutionContext[];
} {
const calls: IUpstreamResolutionContext[] = [];
const provider: IUpstreamProvider = {
async resolveUpstreamConfig(context: IUpstreamResolutionContext) {
calls.push({ ...context });
return baseConfig?.[context.protocol] ?? null;
},
};
return { provider, calls };
}
/**
* Helper to create test authentication tokens
*/
@@ -86,7 +257,19 @@ export async function createTestTokens(registry: SmartRegistry) {
// Create Maven token with full access
const mavenToken = await authManager.createMavenToken(userId, false);
return { npmToken, ociToken, mavenToken, userId };
// Create Composer token with full access
const composerToken = await authManager.createComposerToken(userId, false);
// Create Cargo token with full access
const cargoToken = await authManager.createCargoToken(userId, false);
// Create PyPI token with full access
const pypiToken = await authManager.createPypiToken(userId, false);
// Create RubyGems token with full access
const rubygemsToken = await authManager.createRubyGemsToken(userId, false);
return { npmToken, ociToken, mavenToken, composerToken, cargoToken, pypiToken, rubygemsToken, userId };
}
/**
@@ -205,3 +388,544 @@ export function calculateMavenChecksums(data: Buffer) {
sha512: crypto.createHash('sha512').update(data).digest('hex'),
};
}
/**
* Helper to create a Composer package ZIP using smartarchive
*/
export async function createComposerZip(
vendorPackage: string,
version: string,
options?: {
description?: string;
license?: string[];
authors?: Array<{ name: string; email?: string }>;
}
): Promise<Buffer> {
const zipTools = new smartarchive.ZipTools();
const composerJson = {
name: vendorPackage,
version: version,
type: 'library',
description: options?.description || 'Test Composer package',
license: options?.license || ['MIT'],
authors: options?.authors || [{ name: 'Test Author', email: 'test@example.com' }],
require: {
php: '>=7.4',
},
autoload: {
'psr-4': {
'Vendor\\TestPackage\\': 'src/',
},
},
};
// Add a test PHP file
const [vendor, pkg] = vendorPackage.split('/');
const namespace = `${vendor.charAt(0).toUpperCase() + vendor.slice(1)}\\${pkg.charAt(0).toUpperCase() + pkg.slice(1).replace(/-/g, '')}`;
const testPhpContent = `<?php
namespace ${namespace};
class TestClass
{
public function greet(): string
{
return "Hello from ${vendorPackage}!";
}
}
`;
const entries: smartarchive.IArchiveEntry[] = [
{
archivePath: 'composer.json',
content: Buffer.from(JSON.stringify(composerJson, null, 2), 'utf-8'),
},
{
archivePath: 'src/TestClass.php',
content: Buffer.from(testPhpContent, 'utf-8'),
},
{
archivePath: 'README.md',
content: Buffer.from(`# ${vendorPackage}\n\nTest package`, 'utf-8'),
},
];
return Buffer.from(await zipTools.createZip(entries));
}
/**
* Helper to create a test Python wheel file (minimal ZIP structure) using smartarchive
*/
export async function createPythonWheel(
packageName: string,
version: string,
pyVersion: string = 'py3'
): Promise<Buffer> {
const zipTools = new smartarchive.ZipTools();
const normalizedName = packageName.replace(/-/g, '_');
const distInfoDir = `${normalizedName}-${version}.dist-info`;
// Create METADATA file
const metadata = `Metadata-Version: 2.1
Name: ${packageName}
Version: ${version}
Summary: Test Python package
Home-page: https://example.com
Author: Test Author
Author-email: test@example.com
License: MIT
Platform: UNKNOWN
Classifier: Programming Language :: Python :: 3
Requires-Python: >=3.7
Description-Content-Type: text/markdown
# ${packageName}
Test package for SmartRegistry
`;
// Create WHEEL file
const wheelContent = `Wheel-Version: 1.0
Generator: test 1.0.0
Root-Is-Purelib: true
Tag: ${pyVersion}-none-any
`;
// Create a simple Python module
const moduleContent = `"""${packageName} module"""
__version__ = "${version}"
def hello():
return "Hello from ${packageName}!"
`;
const entries: smartarchive.IArchiveEntry[] = [
{
archivePath: `${distInfoDir}/METADATA`,
content: Buffer.from(metadata, 'utf-8'),
},
{
archivePath: `${distInfoDir}/WHEEL`,
content: Buffer.from(wheelContent, 'utf-8'),
},
{
archivePath: `${distInfoDir}/RECORD`,
content: Buffer.from('', 'utf-8'),
},
{
archivePath: `${distInfoDir}/top_level.txt`,
content: Buffer.from(normalizedName, 'utf-8'),
},
{
archivePath: `${normalizedName}/__init__.py`,
content: Buffer.from(moduleContent, 'utf-8'),
},
];
return Buffer.from(await zipTools.createZip(entries));
}
/**
* Helper to create a test Python source distribution (sdist) using smartarchive
*/
export async function createPythonSdist(
packageName: string,
version: string
): Promise<Buffer> {
const tarTools = new smartarchive.TarTools();
const normalizedName = packageName.replace(/-/g, '_');
const dirPrefix = `${packageName}-${version}`;
// PKG-INFO
const pkgInfo = `Metadata-Version: 2.1
Name: ${packageName}
Version: ${version}
Summary: Test Python package
Home-page: https://example.com
Author: Test Author
Author-email: test@example.com
License: MIT
`;
// setup.py
const setupPy = `from setuptools import setup, find_packages
setup(
name="${packageName}",
version="${version}",
packages=find_packages(),
python_requires=">=3.7",
)
`;
// Module file
const moduleContent = `"""${packageName} module"""
__version__ = "${version}"
def hello():
return "Hello from ${packageName}!"
`;
const entries: smartarchive.IArchiveEntry[] = [
{
archivePath: `${dirPrefix}/PKG-INFO`,
content: Buffer.from(pkgInfo, 'utf-8'),
},
{
archivePath: `${dirPrefix}/setup.py`,
content: Buffer.from(setupPy, 'utf-8'),
},
{
archivePath: `${dirPrefix}/${normalizedName}/__init__.py`,
content: Buffer.from(moduleContent, 'utf-8'),
},
];
return Buffer.from(await tarTools.packFilesToTarGz(entries));
}
/**
* Helper to calculate PyPI file hashes
*/
export function calculatePypiHashes(data: Buffer) {
return {
md5: crypto.createHash('md5').update(data).digest('hex'),
sha256: crypto.createHash('sha256').update(data).digest('hex'),
blake2b: crypto.createHash('blake2b512').update(data).digest('hex'),
};
}
/**
* Helper to create a test RubyGem file (minimal tar.gz structure) using smartarchive
*/
export async function createRubyGem(
gemName: string,
version: string,
platform: string = 'ruby'
): Promise<Buffer> {
const tarTools = new smartarchive.TarTools();
const gzipTools = new smartarchive.GzipTools();
// Create metadata.gz (simplified)
const metadataYaml = `--- !ruby/object:Gem::Specification
name: ${gemName}
version: !ruby/object:Gem::Version
version: ${version}
platform: ${platform}
authors:
- Test Author
autorequire:
bindir: bin
cert_chain: []
date: ${new Date().toISOString().split('T')[0]}
dependencies: []
description: Test RubyGem
email: test@example.com
executables: []
extensions: []
extra_rdoc_files: []
files:
- lib/${gemName}.rb
homepage: https://example.com
licenses:
- MIT
metadata: {}
post_install_message:
rdoc_options: []
require_paths:
- lib
required_ruby_version: !ruby/object:Gem::Requirement
requirements:
- - ">="
- !ruby/object:Gem::Version
version: '2.7'
required_rubygems_version: !ruby/object:Gem::Requirement
requirements:
- - ">="
- !ruby/object:Gem::Version
version: '0'
requirements: []
rubygems_version: 3.0.0
signing_key:
specification_version: 4
summary: Test gem for SmartRegistry
test_files: []
`;
const metadataGz = Buffer.from(await gzipTools.compress(Buffer.from(metadataYaml, 'utf-8')));
// Create data.tar.gz content
const libContent = `# ${gemName}
module ${gemName.charAt(0).toUpperCase() + gemName.slice(1).replace(/-/g, '')}
VERSION = "${version}"
def self.hello
"Hello from #{gemName}!"
end
end
`;
const dataEntries: smartarchive.IArchiveEntry[] = [
{
archivePath: `lib/${gemName}.rb`,
content: Buffer.from(libContent, 'utf-8'),
},
];
const dataTarGz = Buffer.from(await tarTools.packFilesToTarGz(dataEntries));
// Create the outer gem (tar.gz containing metadata.gz and data.tar.gz)
const gemEntries: smartarchive.IArchiveEntry[] = [
{
archivePath: 'metadata.gz',
content: metadataGz,
},
{
archivePath: 'data.tar.gz',
content: dataTarGz,
},
];
// RubyGems .gem files are plain tar archives (NOT gzipped), containing metadata.gz and data.tar.gz
return Buffer.from(await tarTools.packFiles(gemEntries));
}
/**
* Helper to calculate RubyGems checksums
*/
export function calculateRubyGemsChecksums(data: Buffer) {
return {
md5: crypto.createHash('md5').update(data).digest('hex'),
sha256: crypto.createHash('sha256').update(data).digest('hex'),
};
}
// ============================================================================
// Enterprise Extensibility Test Helpers
// ============================================================================
/**
* Create a mock auth provider for testing pluggable authentication.
* Allows customizing behavior for different test scenarios.
*/
export function createMockAuthProvider(overrides?: Partial<IAuthProvider>): IAuthProvider {
const tokens = new Map<string, IAuthToken>();
return {
init: async () => {},
authenticate: async (credentials) => {
// Default: always authenticate successfully
return credentials.username;
},
validateToken: async (token, protocol) => {
const stored = tokens.get(token);
if (stored && (!protocol || stored.type === protocol)) {
return stored;
}
// Mock token for tests
if (token === 'valid-mock-token') {
return {
type: 'npm' as TRegistryProtocol,
userId: 'mock-user',
scopes: ['npm:*:*:*'],
};
}
return null;
},
createToken: async (userId, protocol, options) => {
const tokenId = `mock-${protocol}-${Date.now()}`;
const authToken: IAuthToken = {
type: protocol,
userId,
scopes: options?.scopes || [`${protocol}:*:*:*`],
readonly: options?.readonly,
expiresAt: options?.expiresIn ? new Date(Date.now() + options.expiresIn * 1000) : undefined,
};
tokens.set(tokenId, authToken);
return tokenId;
},
revokeToken: async (token) => {
tokens.delete(token);
},
authorize: async (token, resource, action) => {
if (!token) return false;
if (token.readonly && ['write', 'push', 'delete'].includes(action)) {
return false;
}
return true;
},
listUserTokens: async (userId) => {
const result: Array<{ key: string; readonly: boolean; created: string; protocol?: TRegistryProtocol }> = [];
for (const [key, token] of tokens.entries()) {
if (token.userId === userId) {
result.push({
key: `hash-${key.substring(0, 8)}`,
readonly: token.readonly || false,
created: new Date().toISOString(),
protocol: token.type,
});
}
}
return result;
},
...overrides,
};
}
/**
* Create test storage hooks that track all calls.
* Useful for verifying hook invocation order and parameters.
*/
export function createTrackingHooks(options?: {
beforePutAllowed?: boolean;
beforeDeleteAllowed?: boolean;
throwOnAfterPut?: boolean;
throwOnAfterGet?: boolean;
}): {
hooks: IStorageHooks;
calls: Array<{ method: string; context: IStorageHookContext; timestamp: number }>;
} {
const calls: Array<{ method: string; context: IStorageHookContext; timestamp: number }> = [];
return {
calls,
hooks: {
beforePut: async (ctx) => {
calls.push({ method: 'beforePut', context: ctx, timestamp: Date.now() });
return {
allowed: options?.beforePutAllowed !== false,
reason: options?.beforePutAllowed === false ? 'Blocked by test' : undefined,
};
},
afterPut: async (ctx) => {
calls.push({ method: 'afterPut', context: ctx, timestamp: Date.now() });
if (options?.throwOnAfterPut) {
throw new Error('Test error in afterPut');
}
},
beforeDelete: async (ctx) => {
calls.push({ method: 'beforeDelete', context: ctx, timestamp: Date.now() });
return {
allowed: options?.beforeDeleteAllowed !== false,
reason: options?.beforeDeleteAllowed === false ? 'Blocked by test' : undefined,
};
},
afterDelete: async (ctx) => {
calls.push({ method: 'afterDelete', context: ctx, timestamp: Date.now() });
},
afterGet: async (ctx) => {
calls.push({ method: 'afterGet', context: ctx, timestamp: Date.now() });
if (options?.throwOnAfterGet) {
throw new Error('Test error in afterGet');
}
},
},
};
}
/**
* Create a blocking storage hooks implementation for quota testing.
*/
export function createQuotaHooks(maxSizeBytes: number): {
hooks: IStorageHooks;
currentUsage: { bytes: number };
} {
const currentUsage = { bytes: 0 };
return {
currentUsage,
hooks: {
beforePut: async (ctx) => {
const size = ctx.metadata?.size || 0;
if (currentUsage.bytes + size > maxSizeBytes) {
return { allowed: false, reason: `Quota exceeded: ${currentUsage.bytes + size} > ${maxSizeBytes}` };
}
return { allowed: true };
},
afterPut: async (ctx) => {
currentUsage.bytes += ctx.metadata?.size || 0;
},
afterDelete: async (ctx) => {
currentUsage.bytes -= ctx.metadata?.size || 0;
if (currentUsage.bytes < 0) currentUsage.bytes = 0;
},
},
};
}
/**
* Create a SmartBucket storage backend for upstream cache testing.
*/
export async function createTestStorageBackend(): Promise<{
storage: {
getObject: (key: string) => Promise<Buffer | null>;
putObject: (key: string, data: Buffer) => Promise<void>;
deleteObject: (key: string) => Promise<void>;
listObjects: (prefix: string) => Promise<string[]>;
};
bucket: smartbucket.Bucket;
cleanup: () => Promise<void>;
}> {
const s3AccessKey = await testQenv.getEnvVarOnDemand('S3_ACCESSKEY');
const s3SecretKey = await testQenv.getEnvVarOnDemand('S3_SECRETKEY');
const s3Endpoint = await testQenv.getEnvVarOnDemand('S3_ENDPOINT');
const s3Port = await testQenv.getEnvVarOnDemand('S3_PORT');
const s3 = new smartbucket.SmartBucket({
accessKey: s3AccessKey || 'minioadmin',
accessSecret: s3SecretKey || 'minioadmin',
endpoint: s3Endpoint || 'localhost',
port: parseInt(s3Port || '9000', 10),
useSsl: false,
});
const testRunId = generateTestRunId();
const bucketName = 'test-cache-' + testRunId.substring(0, 8);
const bucket = await s3.createBucket(bucketName);
const storage = {
getObject: async (key: string): Promise<Buffer | null> => {
try {
const file = await bucket.fastGet({ path: key });
if (!file) return null;
const stream = await file.createReadStream();
const chunks: Buffer[] = [];
for await (const chunk of stream) {
chunks.push(Buffer.from(chunk));
}
return Buffer.concat(chunks);
} catch {
return null;
}
},
putObject: async (key: string, data: Buffer): Promise<void> => {
await bucket.fastPut({ path: key, contents: data, overwrite: true });
},
deleteObject: async (key: string): Promise<void> => {
await bucket.fastRemove({ path: key });
},
listObjects: async (prefix: string): Promise<string[]> => {
const files = await bucket.fastList({ prefix });
return files.map(f => f.name);
},
};
const cleanup = async () => {
try {
const files = await bucket.fastList({});
for (const file of files) {
await bucket.fastRemove({ path: file.name });
}
await s3.removeBucket(bucketName);
} catch {
// Ignore cleanup errors
}
};
return { storage, bucket, cleanup };
}

412
test/test.auth.provider.ts Normal file
View File

@@ -0,0 +1,412 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { DefaultAuthProvider } from '../ts/core/classes.defaultauthprovider.js';
import { AuthManager } from '../ts/core/classes.authmanager.js';
import type { IAuthProvider } from '../ts/core/interfaces.auth.js';
import type { IAuthConfig, IAuthToken } from '../ts/core/interfaces.core.js';
import { createMockAuthProvider } from './helpers/registry.js';
// ============================================================================
// Test State
// ============================================================================
let provider: DefaultAuthProvider;
let authConfig: IAuthConfig;
// ============================================================================
// Setup
// ============================================================================
tap.test('setup: should create DefaultAuthProvider', async () => {
authConfig = {
jwtSecret: 'test-secret-key-for-jwt-signing',
tokenStore: 'memory',
npmTokens: { enabled: true },
ociTokens: {
enabled: true,
realm: 'https://auth.example.com/token',
service: 'test-registry',
},
mavenTokens: { enabled: true },
cargoTokens: { enabled: true },
composerTokens: { enabled: true },
pypiTokens: { enabled: true },
rubygemsTokens: { enabled: true },
};
provider = new DefaultAuthProvider(authConfig);
await provider.init();
expect(provider).toBeInstanceOf(DefaultAuthProvider);
});
// ============================================================================
// Authentication Tests
// ============================================================================
tap.test('authenticate: should authenticate new user (auto-registration)', async () => {
const userId = await provider.authenticate({
username: 'newuser',
password: 'newpassword',
});
expect(userId).toEqual('newuser');
});
tap.test('authenticate: should authenticate existing user with correct password', async () => {
// First registration
await provider.authenticate({
username: 'existinguser',
password: 'correctpass',
});
// Second authentication with same credentials
const userId = await provider.authenticate({
username: 'existinguser',
password: 'correctpass',
});
expect(userId).toEqual('existinguser');
});
tap.test('authenticate: should reject authentication with wrong password', async () => {
// First registration
await provider.authenticate({
username: 'passworduser',
password: 'originalpass',
});
// Attempt with wrong password
const userId = await provider.authenticate({
username: 'passworduser',
password: 'wrongpass',
});
expect(userId).toBeNull();
});
// ============================================================================
// Token Creation Tests
// ============================================================================
tap.test('createToken: should create NPM token with correct scopes', async () => {
const token = await provider.createToken('testuser', 'npm', {
scopes: ['npm:package:*:*'],
});
expect(token).toBeTruthy();
expect(typeof token).toEqual('string');
// Validate the token
const validated = await provider.validateToken(token, 'npm');
expect(validated).toBeTruthy();
expect(validated!.type).toEqual('npm');
expect(validated!.userId).toEqual('testuser');
expect(validated!.scopes).toContain('npm:package:*:*');
});
tap.test('createToken: should create Maven token', async () => {
const token = await provider.createToken('mavenuser', 'maven', {
readonly: true,
});
expect(token).toBeTruthy();
const validated = await provider.validateToken(token, 'maven');
expect(validated).toBeTruthy();
expect(validated!.type).toEqual('maven');
expect(validated!.readonly).toBeTrue();
});
tap.test('createToken: should create OCI JWT token with correct claims', async () => {
const token = await provider.createToken('ociuser', 'oci', {
scopes: ['oci:repository:myrepo:push', 'oci:repository:myrepo:pull'],
expiresIn: 3600,
});
expect(token).toBeTruthy();
// OCI tokens are JWTs (contain dots)
expect(token.split('.').length).toEqual(3);
const validated = await provider.validateToken(token, 'oci');
expect(validated).toBeTruthy();
expect(validated!.type).toEqual('oci');
expect(validated!.userId).toEqual('ociuser');
expect(validated!.scopes.length).toBeGreaterThan(0);
});
tap.test('createToken: should create token with expiration', async () => {
const token = await provider.createToken('expiryuser', 'npm', {
expiresIn: 60, // 60 seconds
});
const validated = await provider.validateToken(token, 'npm');
expect(validated).toBeTruthy();
expect(validated!.expiresAt).toBeTruthy();
expect(validated!.expiresAt!.getTime()).toBeGreaterThan(Date.now());
});
// ============================================================================
// Token Validation Tests
// ============================================================================
tap.test('validateToken: should validate UUID token (NPM, Maven, etc.)', async () => {
const npmToken = await provider.createToken('validateuser', 'npm');
const validated = await provider.validateToken(npmToken);
expect(validated).toBeTruthy();
expect(validated!.type).toEqual('npm');
expect(validated!.userId).toEqual('validateuser');
});
tap.test('validateToken: should validate OCI JWT token', async () => {
const ociToken = await provider.createToken('ocivalidate', 'oci', {
scopes: ['oci:repository:*:*'],
});
const validated = await provider.validateToken(ociToken, 'oci');
expect(validated).toBeTruthy();
expect(validated!.type).toEqual('oci');
expect(validated!.userId).toEqual('ocivalidate');
});
tap.test('validateToken: should reject expired tokens', async () => {
const token = await provider.createToken('expireduser', 'npm', {
expiresIn: -1, // Already expired (in the past)
});
// The token should be created but will fail validation due to expiry
const validated = await provider.validateToken(token, 'npm');
// Token should be rejected because it's expired
expect(validated).toBeNull();
});
tap.test('validateToken: should reject invalid token', async () => {
const validated = await provider.validateToken('invalid-random-token');
expect(validated).toBeNull();
});
tap.test('validateToken: should reject token with wrong protocol', async () => {
const npmToken = await provider.createToken('protocoluser', 'npm');
// Try to validate as Maven token
const validated = await provider.validateToken(npmToken, 'maven');
expect(validated).toBeNull();
});
// ============================================================================
// Token Revocation Tests
// ============================================================================
tap.test('revokeToken: should revoke tokens', async () => {
const token = await provider.createToken('revokeuser', 'npm');
// Verify token works before revocation
let validated = await provider.validateToken(token);
expect(validated).toBeTruthy();
// Revoke the token
await provider.revokeToken(token);
// Token should no longer be valid
validated = await provider.validateToken(token);
expect(validated).toBeNull();
});
// ============================================================================
// Authorization Tests
// ============================================================================
tap.test('authorize: should authorize read actions for readonly tokens', async () => {
const token = await provider.createToken('readonlyuser', 'npm', {
readonly: true,
scopes: ['npm:package:*:read'],
});
const validated = await provider.validateToken(token);
const canRead = await provider.authorize(validated, 'npm:package:lodash', 'read');
expect(canRead).toBeTrue();
const canPull = await provider.authorize(validated, 'npm:package:lodash', 'pull');
expect(canPull).toBeTrue();
});
tap.test('authorize: should deny write actions for readonly tokens', async () => {
const token = await provider.createToken('readonlyuser2', 'npm', {
readonly: true,
scopes: ['npm:package:*:*'],
});
const validated = await provider.validateToken(token);
const canWrite = await provider.authorize(validated, 'npm:package:lodash', 'write');
expect(canWrite).toBeFalse();
const canPush = await provider.authorize(validated, 'npm:package:lodash', 'push');
expect(canPush).toBeFalse();
const canDelete = await provider.authorize(validated, 'npm:package:lodash', 'delete');
expect(canDelete).toBeFalse();
});
tap.test('authorize: should match scopes with wildcards', async () => {
// The scope system uses literal * as wildcard, not glob patterns
// npm:*:*:* means "all types, all names, all actions under npm"
const token = await provider.createToken('wildcarduser', 'npm', {
scopes: ['npm:*:*:*'],
});
const validated = await provider.validateToken(token);
// Should match any npm resource with full wildcard scope
const canAccessAnyPackage = await provider.authorize(validated, 'npm:package:lodash', 'read');
expect(canAccessAnyPackage).toBeTrue();
const canAccessScopedPackage = await provider.authorize(validated, 'npm:package:@myorg/foo', 'write');
expect(canAccessScopedPackage).toBeTrue();
});
tap.test('authorize: should deny access with null token', async () => {
const canAccess = await provider.authorize(null, 'npm:package:lodash', 'read');
expect(canAccess).toBeFalse();
});
// ============================================================================
// List Tokens Tests
// ============================================================================
tap.test('listUserTokens: should list user tokens', async () => {
// Create multiple tokens for the same user
const userId = 'listtokenuser';
await provider.createToken(userId, 'npm');
await provider.createToken(userId, 'maven', { readonly: true });
await provider.createToken(userId, 'cargo');
const tokens = await provider.listUserTokens!(userId);
expect(tokens.length).toBeGreaterThanOrEqual(3);
// Check that tokens have expected properties
for (const token of tokens) {
expect(token.key).toBeTruthy();
expect(typeof token.readonly).toEqual('boolean');
expect(token.created).toBeTruthy();
}
// Verify we have different protocols
const protocols = tokens.map(t => t.protocol);
expect(protocols).toContain('npm');
expect(protocols).toContain('maven');
expect(protocols).toContain('cargo');
});
// ============================================================================
// AuthManager Integration Tests
// ============================================================================
tap.test('AuthManager: should accept custom IAuthProvider', async () => {
const mockProvider = createMockAuthProvider({
authenticate: async (credentials) => {
if (credentials.username === 'custom' && credentials.password === 'pass') {
return 'custom-user-id';
}
return null;
},
});
const manager = new AuthManager(authConfig, mockProvider);
await manager.init();
// Use the custom provider
const userId = await manager.authenticate({
username: 'custom',
password: 'pass',
});
expect(userId).toEqual('custom-user-id');
// Wrong credentials should fail
const failed = await manager.authenticate({
username: 'custom',
password: 'wrong',
});
expect(failed).toBeNull();
});
tap.test('AuthManager: should use default provider when none specified', async () => {
const manager = new AuthManager(authConfig);
await manager.init();
// Should use DefaultAuthProvider internally
const userId = await manager.authenticate({
username: 'defaultuser',
password: 'defaultpass',
});
expect(userId).toEqual('defaultuser');
});
tap.test('AuthManager: should delegate token creation to provider', async () => {
let tokenCreated = false;
const mockProvider = createMockAuthProvider({
createToken: async (userId, protocol, options) => {
tokenCreated = true;
return `mock-token-${protocol}-${userId}`;
},
});
const manager = new AuthManager(authConfig, mockProvider);
await manager.init();
const token = await manager.createNpmToken('delegateuser', false);
expect(tokenCreated).toBeTrue();
expect(token).toContain('mock-token-npm');
});
// ============================================================================
// Edge Cases
// ============================================================================
tap.test('edge: should handle concurrent token operations', async () => {
const promises: Promise<string>[] = [];
// Create 10 tokens concurrently
for (let i = 0; i < 10; i++) {
promises.push(provider.createToken(`concurrent-user-${i}`, 'npm'));
}
const tokens = await Promise.all(promises);
// All tokens should be unique
const uniqueTokens = new Set(tokens);
expect(uniqueTokens.size).toEqual(10);
// All tokens should be valid
for (const token of tokens) {
const validated = await provider.validateToken(token);
expect(validated).toBeTruthy();
}
});
tap.test('edge: should handle empty scopes', async () => {
const token = await provider.createToken('emptyuser', 'npm', {
scopes: [],
});
const validated = await provider.validateToken(token);
expect(validated).toBeTruthy();
// Even with empty scopes, token should be valid
});
// ============================================================================
// Cleanup
// ============================================================================
tap.test('cleanup', async () => {
// No cleanup needed for in-memory provider
});
export default tap.start();

View File

@@ -0,0 +1,468 @@
/**
* Native cargo CLI Testing
* Tests the Cargo registry implementation using the actual cargo CLI
*/
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { tapNodeTools } from '@git.zone/tstest/tapbundle_serverside';
import { SmartRegistry } from '../ts/index.js';
import { createTestRegistry, createTestTokens } from './helpers/registry.js';
import type { IRequestContext, IResponse } from '../ts/core/interfaces.core.js';
import * as http from 'http';
import * as url from 'url';
import * as fs from 'fs';
import * as path from 'path';
// Test context
let registry: SmartRegistry;
let server: http.Server;
let registryUrl: string;
let registryPort: number;
let cargoToken: string;
let testDir: string;
let cargoHome: string;
/**
* Create HTTP server wrapper around SmartRegistry
*/
async function createHttpServer(
registryInstance: SmartRegistry,
port: number
): Promise<{ server: http.Server; url: string }> {
return new Promise((resolve, reject) => {
const httpServer = http.createServer(async (req, res) => {
try {
// Parse request
const parsedUrl = url.parse(req.url || '', true);
const pathname = parsedUrl.pathname || '/';
const query = parsedUrl.query;
// Read body
const chunks: Buffer[] = [];
for await (const chunk of req) {
chunks.push(chunk);
}
const bodyBuffer = Buffer.concat(chunks);
// Parse body based on content type
let body: any;
if (bodyBuffer.length > 0) {
const contentType = req.headers['content-type'] || '';
if (contentType.includes('application/json')) {
try {
body = JSON.parse(bodyBuffer.toString('utf-8'));
} catch (error) {
body = bodyBuffer;
}
} else {
body = bodyBuffer;
}
}
// Convert to IRequestContext
const context: IRequestContext = {
method: req.method || 'GET',
path: pathname,
headers: req.headers as Record<string, string>,
query: query as Record<string, string>,
body: body,
};
// Handle request
const response: IResponse = await registryInstance.handleRequest(context);
// Convert IResponse to HTTP response
res.statusCode = response.status;
// Set headers
for (const [key, value] of Object.entries(response.headers || {})) {
res.setHeader(key, value);
}
// Send body (response.body is always ReadableStream<Uint8Array> or undefined)
if (response.body) {
const { Readable } = await import('stream');
Readable.fromWeb(response.body).pipe(res);
} else {
res.end();
}
} catch (error) {
console.error('Server error:', error);
res.statusCode = 500;
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify({ error: 'INTERNAL_ERROR', message: String(error) }));
}
});
httpServer.listen(port, () => {
const serverUrl = `http://localhost:${port}`;
resolve({ server: httpServer, url: serverUrl });
});
httpServer.on('error', reject);
});
}
/**
* Setup Cargo configuration
*/
function setupCargoConfig(registryUrlArg: string, token: string, cargoHomeArg: string): void {
const cargoConfigDir = path.join(cargoHomeArg, '.cargo');
fs.mkdirSync(cargoConfigDir, { recursive: true });
// Create config.toml with sparse protocol
const configContent = `[registries.test-registry]
index = "sparse+${registryUrlArg}/cargo/"
[source.crates-io]
replace-with = "test-registry"
[net]
retry = 0
`;
fs.writeFileSync(path.join(cargoConfigDir, 'config.toml'), configContent, 'utf-8');
// Create credentials.toml (Cargo uses plain token, no "Bearer" prefix)
const credentialsContent = `[registries.test-registry]
token = "${token}"
`;
fs.writeFileSync(path.join(cargoConfigDir, 'credentials.toml'), credentialsContent, 'utf-8');
}
/**
* Create a test Cargo crate
*/
function createTestCrate(
crateName: string,
version: string,
targetDir: string
): string {
const crateDir = path.join(targetDir, crateName);
fs.mkdirSync(crateDir, { recursive: true });
// Create Cargo.toml
const cargoToml = `[package]
name = "${crateName}"
version = "${version}"
edition = "2021"
description = "Test crate ${crateName}"
license = "MIT"
authors = ["Test Author <test@example.com>"]
[dependencies]
`;
fs.writeFileSync(path.join(crateDir, 'Cargo.toml'), cargoToml, 'utf-8');
// Create src directory
const srcDir = path.join(crateDir, 'src');
fs.mkdirSync(srcDir, { recursive: true });
// Create lib.rs
const libRs = `//! Test crate ${crateName}
/// Returns a greeting message
pub fn greet() -> String {
format!("Hello from {}@{}", "${crateName}", "${version}")
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_greet() {
let greeting = greet();
assert!(greeting.contains("${crateName}"));
}
}
`;
fs.writeFileSync(path.join(srcDir, 'lib.rs'), libRs, 'utf-8');
// Create README.md
const readme = `# ${crateName}
Test crate for SmartRegistry.
Version: ${version}
`;
fs.writeFileSync(path.join(crateDir, 'README.md'), readme, 'utf-8');
return crateDir;
}
/**
* Run cargo command with proper environment
*/
async function runCargoCommand(
command: string,
cwd: string,
includeToken: boolean = true
): Promise<{ stdout: string; stderr: string; exitCode: number }> {
// Prepare environment variables
// NOTE: Cargo converts registry name "test-registry" to "TEST_REGISTRY" for env vars
const envVars = [
`CARGO_HOME="${cargoHome}"`,
`CARGO_REGISTRIES_TEST_REGISTRY_INDEX="sparse+${registryUrl}/cargo/"`,
includeToken ? `CARGO_REGISTRIES_TEST_REGISTRY_TOKEN="${cargoToken}"` : '',
`CARGO_NET_RETRY="0"`,
].filter(Boolean).join(' ');
// Build command with cd to correct directory and environment variables
const fullCommand = `cd "${cwd}" && ${envVars} ${command}`;
try {
const result = await tapNodeTools.runCommand(fullCommand);
return {
stdout: result.stdout || '',
stderr: result.stderr || '',
exitCode: result.exitCode || 0,
};
} catch (error: any) {
return {
stdout: error.stdout || '',
stderr: error.stderr || String(error),
exitCode: error.exitCode || 1,
};
}
}
/**
* Cleanup test directory
*/
function cleanupTestDir(dir: string): void {
if (fs.existsSync(dir)) {
fs.rmSync(dir, { recursive: true, force: true });
}
}
// ========================================================================
// TESTS
// ========================================================================
tap.test('Cargo CLI: should setup registry and HTTP server', async () => {
// Use port 5000
registryPort = 5000;
// Create registry with correct registryUrl for CLI tests
registry = await createTestRegistry({ registryUrl: `http://localhost:${registryPort}` });
const tokens = await createTestTokens(registry);
cargoToken = tokens.cargoToken;
expect(registry).toBeInstanceOf(SmartRegistry);
expect(cargoToken).toBeTypeOf('string');
// Clean up any existing index from previous test runs
const storage = registry.getStorage();
try {
await storage.putCargoIndex('test-crate-cli', []);
} catch (error) {
// Ignore error if operation fails
}
const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server;
registryUrl = serverSetup.url;
expect(server).toBeDefined();
expect(registryUrl).toEqual(`http://localhost:${registryPort}`);
// Setup test directory
testDir = path.join(process.cwd(), '.nogit', 'test-cargo-cli');
cleanupTestDir(testDir);
fs.mkdirSync(testDir, { recursive: true });
// Setup CARGO_HOME
cargoHome = path.join(testDir, '.cargo-home');
fs.mkdirSync(cargoHome, { recursive: true });
// Setup Cargo config
setupCargoConfig(registryUrl, cargoToken, cargoHome);
expect(fs.existsSync(path.join(cargoHome, '.cargo', 'config.toml'))).toEqual(true);
expect(fs.existsSync(path.join(cargoHome, '.cargo', 'credentials.toml'))).toEqual(true);
});
tap.test('Cargo CLI: should verify server is responding', async () => {
// Check server is up by doing a direct HTTP request to the cargo index
const response = await fetch(`${registryUrl}/cargo/`);
expect(response.status).toBeGreaterThanOrEqual(200);
expect(response.status).toBeLessThan(500);
});
tap.test('Cargo CLI: should publish a crate', async () => {
const crateName = 'test-crate-cli';
const version = '0.1.0';
const crateDir = createTestCrate(crateName, version, testDir);
const result = await runCargoCommand('cargo publish --registry test-registry --allow-dirty', crateDir);
console.log('cargo publish output:', result.stdout);
console.log('cargo publish stderr:', result.stderr);
expect(result.exitCode).toEqual(0);
expect(result.stdout || result.stderr).toContain(crateName);
});
tap.test('Cargo CLI: should verify crate in index', async () => {
const crateName = 'test-crate-cli';
// Cargo uses a specific index structure
// For crate "test-crate-cli", the index path is based on the first characters
// 1 char: <name>
// 2 char: 2/<name>
// 3 char: 3/<first-char>/<name>
// 4+ char: <first-2-chars>/<second-2-chars>/<name>
// "test-crate-cli" is 14 chars, so it should be at: te/st/test-crate-cli
const indexPath = `/cargo/te/st/${crateName}`;
const response = await fetch(`${registryUrl}${indexPath}`);
expect(response.status).toEqual(200);
const indexData = await response.text();
console.log('Index data:', indexData);
// Index should contain JSON line with crate info
expect(indexData).toContain(crateName);
expect(indexData).toContain('0.1.0');
});
tap.test('Cargo CLI: should download published crate', async () => {
const crateName = 'test-crate-cli';
const version = '0.1.0';
// Cargo downloads crates from /cargo/api/v1/crates/{name}/{version}/download
const downloadPath = `/cargo/api/v1/crates/${crateName}/${version}/download`;
const response = await fetch(`${registryUrl}${downloadPath}`);
expect(response.status).toEqual(200);
const crateData = await response.arrayBuffer();
expect(crateData.byteLength).toBeGreaterThan(0);
});
tap.test('Cargo CLI: should publish second version', async () => {
const crateName = 'test-crate-cli';
const version = '0.2.0';
const crateDir = createTestCrate(crateName, version, testDir);
const result = await runCargoCommand('cargo publish --registry test-registry --allow-dirty', crateDir);
console.log('cargo publish v0.2.0 output:', result.stdout);
expect(result.exitCode).toEqual(0);
});
tap.test('Cargo CLI: should list versions in index', async () => {
const crateName = 'test-crate-cli';
const indexPath = `/cargo/te/st/${crateName}`;
const response = await fetch(`${registryUrl}${indexPath}`);
expect(response.status).toEqual(200);
const indexData = await response.text();
const lines = indexData.trim().split('\n');
// Should have 2 lines (2 versions)
expect(lines.length).toEqual(2);
// Parse JSON lines
const version1 = JSON.parse(lines[0]);
const version2 = JSON.parse(lines[1]);
expect(version1.vers).toEqual('0.1.0');
expect(version2.vers).toEqual('0.2.0');
});
tap.test('Cargo CLI: should search for crate', async () => {
const crateName = 'test-crate-cli';
// Cargo search endpoint: /cargo/api/v1/crates?q={query}
const response = await fetch(`${registryUrl}/cargo/api/v1/crates?q=${crateName}`);
expect(response.status).toEqual(200);
const searchResults = await response.json();
console.log('Search results:', searchResults);
expect(searchResults).toHaveProperty('crates');
expect(searchResults.crates).toBeInstanceOf(Array);
expect(searchResults.crates.length).toBeGreaterThan(0);
expect(searchResults.crates[0].name).toEqual(crateName);
});
tap.test('Cargo CLI: should yank a version', async () => {
const crateName = 'test-crate-cli';
const crateDir = path.join(testDir, crateName);
const result = await runCargoCommand('cargo yank --registry test-registry --vers 0.1.0', crateDir);
console.log('cargo yank output:', result.stdout);
console.log('cargo yank stderr:', result.stderr);
expect(result.exitCode).toEqual(0);
// Verify version is yanked in index
const indexPath = `/cargo/te/st/${crateName}`;
const response = await fetch(`${registryUrl}${indexPath}`);
const indexData = await response.text();
const lines = indexData.trim().split('\n');
const version1 = JSON.parse(lines[0]);
expect(version1.yanked).toEqual(true);
});
tap.test('Cargo CLI: should unyank a version', async () => {
const crateName = 'test-crate-cli';
const crateDir = path.join(testDir, crateName);
const result = await runCargoCommand('cargo yank --registry test-registry --vers 0.1.0 --undo', crateDir);
console.log('cargo unyank output:', result.stdout);
console.log('cargo unyank stderr:', result.stderr);
expect(result.exitCode).toEqual(0);
// Verify version is not yanked in index
const indexPath = `/cargo/te/st/${crateName}`;
const response = await fetch(`${registryUrl}${indexPath}`);
const indexData = await response.text();
const lines = indexData.trim().split('\n');
const version1 = JSON.parse(lines[0]);
expect(version1.yanked).toEqual(false);
});
tap.test('Cargo CLI: should fail to publish without auth', async () => {
const crateName = 'unauth-crate';
const version = '0.1.0';
const crateDir = createTestCrate(crateName, version, testDir);
// Run without token (includeToken: false)
const result = await runCargoCommand('cargo publish --registry test-registry --allow-dirty', crateDir, false);
console.log('cargo publish unauth output:', result.stdout);
console.log('cargo publish unauth stderr:', result.stderr);
// Should fail with auth error
expect(result.exitCode).not.toEqual(0);
expect(result.stderr).toContain('token');
});
tap.postTask('cleanup cargo cli tests', async () => {
// Stop server
if (server) {
await new Promise<void>((resolve) => {
server.close(() => resolve());
});
}
// Cleanup test directory
if (testDir) {
cleanupTestDir(testDir);
}
// Destroy registry
if (registry) {
registry.destroy();
}
});
export default tap.start();

View File

@@ -0,0 +1,462 @@
/**
* Native Composer CLI Testing
* Tests the Composer registry implementation using the actual composer CLI
*/
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { tapNodeTools } from '@git.zone/tstest/tapbundle_serverside';
import { SmartRegistry } from '../ts/index.js';
import { createTestRegistry, createTestTokens, createComposerZip, generateTestRunId } from './helpers/registry.js';
import type { IRequestContext, IResponse } from '../ts/core/interfaces.core.js';
import * as http from 'http';
import * as url from 'url';
import * as fs from 'fs';
import * as path from 'path';
// Test context
let registry: SmartRegistry;
let server: http.Server;
let registryUrl: string;
let registryPort: number;
let composerToken: string;
let testDir: string;
let composerHome: string;
let hasComposer = false;
// Unique test run ID to avoid conflicts between test runs
const testRunId = generateTestRunId();
const testPackageName = `testvendor/test-pkg-${testRunId}`;
/**
* Create HTTP server wrapper around SmartRegistry
*/
async function createHttpServer(
registryInstance: SmartRegistry,
port: number
): Promise<{ server: http.Server; url: string }> {
return new Promise((resolve, reject) => {
const httpServer = http.createServer(async (req, res) => {
try {
// Parse request
const parsedUrl = url.parse(req.url || '', true);
const pathname = parsedUrl.pathname || '/';
const query = parsedUrl.query;
// Read body
const chunks: Buffer[] = [];
for await (const chunk of req) {
chunks.push(chunk);
}
const bodyBuffer = Buffer.concat(chunks);
// Parse body based on content type
let body: any;
if (bodyBuffer.length > 0) {
const contentType = req.headers['content-type'] || '';
if (contentType.includes('application/json')) {
try {
body = JSON.parse(bodyBuffer.toString('utf-8'));
} catch (error) {
body = bodyBuffer;
}
} else {
body = bodyBuffer;
}
}
// Convert to IRequestContext
const context: IRequestContext = {
method: req.method || 'GET',
path: pathname,
headers: req.headers as Record<string, string>,
query: query as Record<string, string>,
body: body,
};
// Handle request
const response: IResponse = await registryInstance.handleRequest(context);
// Convert IResponse to HTTP response
res.statusCode = response.status;
// Set headers
for (const [key, value] of Object.entries(response.headers || {})) {
res.setHeader(key, value);
}
// Send body (response.body is always ReadableStream<Uint8Array> or undefined)
if (response.body) {
const { Readable } = await import('stream');
Readable.fromWeb(response.body).pipe(res);
} else {
res.end();
}
} catch (error) {
console.error('Server error:', error);
res.statusCode = 500;
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify({ error: 'INTERNAL_ERROR', message: String(error) }));
}
});
httpServer.listen(port, () => {
const serverUrl = `http://localhost:${port}`;
resolve({ server: httpServer, url: serverUrl });
});
httpServer.on('error', reject);
});
}
/**
* Setup Composer auth.json for authentication
*/
function setupComposerAuth(
token: string,
composerHomeArg: string,
serverUrl: string,
port: number
): string {
fs.mkdirSync(composerHomeArg, { recursive: true });
const authJson = {
'http-basic': {
[`localhost:${port}`]: {
username: 'testuser',
password: token,
},
},
};
const authPath = path.join(composerHomeArg, 'auth.json');
fs.writeFileSync(authPath, JSON.stringify(authJson, null, 2), 'utf-8');
return authPath;
}
/**
* Create a Composer project that uses our registry
*/
function createComposerProject(
projectDir: string,
serverUrl: string
): void {
fs.mkdirSync(projectDir, { recursive: true });
const composerJson = {
name: 'test/consumer-project',
description: 'Test consumer project for Composer CLI tests',
type: 'project',
require: {},
repositories: [
{
type: 'composer',
url: `${serverUrl}/composer`,
},
],
config: {
'secure-http': false,
},
};
fs.writeFileSync(
path.join(projectDir, 'composer.json'),
JSON.stringify(composerJson, null, 2),
'utf-8'
);
}
/**
* Run Composer command with custom home directory
*/
async function runComposerCommand(
command: string,
cwd: string
): Promise<{ stdout: string; stderr: string; exitCode: number }> {
const fullCommand = `cd "${cwd}" && COMPOSER_HOME="${composerHome}" composer ${command}`;
try {
const result = await tapNodeTools.runCommand(fullCommand);
return {
stdout: result.stdout || '',
stderr: result.stderr || '',
exitCode: result.exitCode || 0,
};
} catch (error: any) {
return {
stdout: error.stdout || '',
stderr: error.stderr || String(error),
exitCode: error.exitCode || 1,
};
}
}
/**
* Upload a Composer package via HTTP API
*/
async function uploadComposerPackage(
vendorPackage: string,
version: string,
token: string,
serverUrl: string
): Promise<void> {
const zipData = await createComposerZip(vendorPackage, version);
const response = await fetch(`${serverUrl}/composer/packages/${vendorPackage}`, {
method: 'PUT',
headers: {
'Content-Type': 'application/zip',
Authorization: `Bearer ${token}`,
},
body: zipData,
});
if (!response.ok) {
const body = await response.text();
throw new Error(`Failed to upload package: ${response.status} ${body}`);
}
}
/**
* Cleanup test directory
*/
function cleanupTestDir(dir: string): void {
if (fs.existsSync(dir)) {
fs.rmSync(dir, { recursive: true, force: true });
}
}
// ========================================================================
// TESTS
// ========================================================================
tap.test('Composer CLI: should verify composer is installed', async () => {
try {
const result = await tapNodeTools.runCommand('composer --version');
console.log('Composer version output:', result.stdout.substring(0, 200));
hasComposer = result.exitCode === 0;
expect(result.exitCode).toEqual(0);
} catch (error) {
console.log('Composer CLI not available, skipping native CLI tests');
hasComposer = false;
}
});
tap.test('Composer CLI: should setup registry and HTTP server', async () => {
// Use port 38000 (avoids conflicts with other tests)
registryPort = 38000;
// Create registry with correct registryUrl for CLI tests
registry = await createTestRegistry({ registryUrl: `http://localhost:${registryPort}` });
const tokens = await createTestTokens(registry);
composerToken = tokens.composerToken;
expect(registry).toBeInstanceOf(SmartRegistry);
expect(composerToken).toBeTypeOf('string');
const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server;
registryUrl = serverSetup.url;
expect(server).toBeDefined();
expect(registryUrl).toEqual(`http://localhost:${registryPort}`);
// Setup test directory
testDir = path.join(process.cwd(), '.nogit', 'test-composer-cli');
cleanupTestDir(testDir);
fs.mkdirSync(testDir, { recursive: true });
// Setup COMPOSER_HOME directory
composerHome = path.join(testDir, '.composer');
fs.mkdirSync(composerHome, { recursive: true });
// Setup Composer auth
const authPath = setupComposerAuth(composerToken, composerHome, registryUrl, registryPort);
expect(fs.existsSync(authPath)).toEqual(true);
});
tap.test('Composer CLI: should verify server is responding', async () => {
// Check server is up by doing a direct HTTP request
const response = await fetch(`${registryUrl}/composer/packages.json`);
expect(response.status).toBeGreaterThanOrEqual(200);
expect(response.status).toBeLessThan(500);
});
tap.test('Composer CLI: should upload a package via API', async () => {
const version = '1.0.0';
await uploadComposerPackage(testPackageName, version, composerToken, registryUrl);
// Verify package exists via p2 metadata endpoint (more reliable than packages.json for new packages)
const metadataResponse = await fetch(`${registryUrl}/composer/p2/${testPackageName}.json`);
expect(metadataResponse.status).toEqual(200);
const metadata = await metadataResponse.json();
expect(metadata.packages).toBeDefined();
expect(metadata.packages[testPackageName]).toBeDefined();
expect(metadata.packages[testPackageName].length).toBeGreaterThan(0);
});
tap.test('Composer CLI: should require package from registry', async () => {
if (!hasComposer) {
console.log('Skipping - composer not available');
return;
}
const projectDir = path.join(testDir, 'consumer-project');
createComposerProject(projectDir, registryUrl);
// Try to require the package we uploaded
const result = await runComposerCommand(
`require ${testPackageName}:1.0.0 --no-interaction`,
projectDir
);
console.log('composer require output:', result.stdout);
console.log('composer require stderr:', result.stderr);
expect(result.exitCode).toEqual(0);
});
tap.test('Composer CLI: should verify package in vendor directory', async () => {
if (!hasComposer) {
console.log('Skipping - composer not available');
return;
}
const projectDir = path.join(testDir, 'consumer-project');
// Parse vendor/package from testPackageName (e.g., "testvendor/test-pkg-abc123")
const [vendor, pkg] = testPackageName.split('/');
const packageDir = path.join(projectDir, 'vendor', vendor, pkg);
expect(fs.existsSync(packageDir)).toEqual(true);
// Check composer.json exists in package
const packageComposerPath = path.join(packageDir, 'composer.json');
expect(fs.existsSync(packageComposerPath)).toEqual(true);
});
tap.test('Composer CLI: should upload second version', async () => {
const version = '2.0.0';
await uploadComposerPackage(testPackageName, version, composerToken, registryUrl);
// Verify both versions exist via p2 metadata endpoint (Composer v2 format)
const response = await fetch(`${registryUrl}/composer/p2/${testPackageName}.json`);
expect(response.status).toEqual(200);
const metadata = await response.json();
expect(metadata.packages).toBeDefined();
expect(metadata.packages[testPackageName]).toBeDefined();
// Check that both versions are present
const versions = metadata.packages[testPackageName];
expect(versions.length).toBeGreaterThanOrEqual(2);
const versionNumbers = versions.map((v: any) => v.version);
expect(versionNumbers).toContain('1.0.0');
expect(versionNumbers).toContain('2.0.0');
});
tap.test('Composer CLI: should update to new version', async () => {
if (!hasComposer) {
console.log('Skipping - composer not available');
return;
}
const projectDir = path.join(testDir, 'consumer-project');
// Update to version 2.0.0
const result = await runComposerCommand(
`require ${testPackageName}:2.0.0 --no-interaction`,
projectDir
);
console.log('composer update output:', result.stdout);
expect(result.exitCode).toEqual(0);
// Verify composer.lock has the new version
const lockPath = path.join(projectDir, 'composer.lock');
expect(fs.existsSync(lockPath)).toEqual(true);
const lockContent = JSON.parse(fs.readFileSync(lockPath, 'utf-8'));
const pkg = lockContent.packages.find((p: any) => p.name === testPackageName);
expect(pkg?.version).toEqual('2.0.0');
});
tap.test('Composer CLI: should search for packages', async () => {
if (!hasComposer) {
console.log('Skipping - composer not available');
return;
}
const projectDir = path.join(testDir, 'consumer-project');
// Search for packages (may not work on all Composer versions)
const result = await runComposerCommand(
'search testvendor --no-interaction 2>&1 || true',
projectDir
);
console.log('composer search output:', result.stdout);
// Search may or may not work depending on registry implementation
// Just verify it doesn't crash
expect(result.exitCode).toBeLessThanOrEqual(1);
});
tap.test('Composer CLI: should show package info', async () => {
if (!hasComposer) {
console.log('Skipping - composer not available');
return;
}
const projectDir = path.join(testDir, 'consumer-project');
const result = await runComposerCommand(
`show ${testPackageName} --no-interaction`,
projectDir
);
console.log('composer show output:', result.stdout);
expect(result.exitCode).toEqual(0);
expect(result.stdout).toContain(testPackageName);
});
tap.test('Composer CLI: should remove package', async () => {
if (!hasComposer) {
console.log('Skipping - composer not available');
return;
}
const projectDir = path.join(testDir, 'consumer-project');
const result = await runComposerCommand(
`remove ${testPackageName} --no-interaction`,
projectDir
);
console.log('composer remove output:', result.stdout);
expect(result.exitCode).toEqual(0);
// Verify package is removed from vendor
const [vendor, pkg] = testPackageName.split('/');
const packageDir = path.join(projectDir, 'vendor', vendor, pkg);
expect(fs.existsSync(packageDir)).toEqual(false);
});
tap.postTask('cleanup composer cli tests', async () => {
// Stop server
if (server) {
await new Promise<void>((resolve) => {
server.close(() => resolve());
});
}
// Cleanup test directory
if (testDir) {
cleanupTestDir(testDir);
}
// Destroy registry
if (registry) {
registry.destroy();
}
});
export default tap.start();

310
test/test.composer.ts Normal file
View File

@@ -0,0 +1,310 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js';
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
import { createTestRegistry, createTestTokens, createComposerZip } from './helpers/registry.js';
let registry: SmartRegistry;
let composerToken: string;
let userId: string;
// Test data
const testPackageName = 'vendor/test-package';
const testVersion = '1.0.0';
let testZipData: Buffer;
tap.test('Composer: should create registry instance', async () => {
registry = await createTestRegistry();
const tokens = await createTestTokens(registry);
composerToken = tokens.composerToken;
userId = tokens.userId;
expect(registry).toBeInstanceOf(SmartRegistry);
expect(composerToken).toBeTypeOf('string');
});
tap.test('Composer: should create test ZIP package', async () => {
testZipData = await createComposerZip(testPackageName, testVersion, {
description: 'Test Composer package for registry',
license: ['MIT'],
authors: [{ name: 'Test Author', email: 'test@example.com' }],
});
expect(testZipData).toBeInstanceOf(Buffer);
expect(testZipData.length).toBeGreaterThan(0);
});
tap.test('Composer: should return packages.json (GET /packages.json)', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/composer/packages.json',
headers: {},
query: {},
});
expect(response.status).toEqual(200);
const body = await streamToJson(response.body);
expect(body).toHaveProperty('metadata-url');
expect(body).toHaveProperty('available-packages');
expect(body['available-packages']).toBeInstanceOf(Array);
});
tap.test('Composer: should upload a package (PUT /packages/{vendor/package})', async () => {
const response = await registry.handleRequest({
method: 'PUT',
path: `/composer/packages/${testPackageName}`,
headers: {
Authorization: `Bearer ${composerToken}`,
'Content-Type': 'application/zip',
},
query: {},
body: testZipData,
});
expect(response.status).toEqual(201);
const body = await streamToJson(response.body);
expect(body.status).toEqual('success');
expect(body.package).toEqual(testPackageName);
expect(body.version).toEqual(testVersion);
});
tap.test('Composer: should retrieve package metadata (GET /p2/{vendor/package}.json)', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: `/composer/p2/${testPackageName}.json`,
headers: {},
query: {},
});
expect(response.status).toEqual(200);
const body = await streamToJson(response.body);
expect(body).toHaveProperty('packages');
expect(body.packages[testPackageName]).toBeInstanceOf(Array);
expect(body.packages[testPackageName].length).toEqual(1);
const packageData = body.packages[testPackageName][0];
expect(packageData.name).toEqual(testPackageName);
expect(packageData.version).toEqual(testVersion);
expect(packageData.version_normalized).toEqual('1.0.0.0');
expect(packageData).toHaveProperty('dist');
expect(packageData.dist.type).toEqual('zip');
expect(packageData.dist).toHaveProperty('url');
expect(packageData.dist).toHaveProperty('shasum');
expect(packageData.dist).toHaveProperty('reference');
});
tap.test('Composer: should download package ZIP (GET /dists/{vendor/package}/{ref}.zip)', async () => {
// First get metadata to find reference
const metadataResponse = await registry.handleRequest({
method: 'GET',
path: `/composer/p2/${testPackageName}.json`,
headers: {},
query: {},
});
const metaBody = await streamToJson(metadataResponse.body);
const reference = metaBody.packages[testPackageName][0].dist.reference;
const response = await registry.handleRequest({
method: 'GET',
path: `/composer/dists/${testPackageName}/${reference}.zip`,
headers: {},
query: {},
});
expect(response.status).toEqual(200);
const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
expect(response.headers['Content-Type']).toEqual('application/zip');
expect(response.headers['Content-Disposition']).toContain('attachment');
});
tap.test('Composer: should list packages (GET /packages/list.json)', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/composer/packages/list.json',
headers: {},
query: {},
});
expect(response.status).toEqual(200);
const body = await streamToJson(response.body);
expect(body).toHaveProperty('packageNames');
expect(body.packageNames).toBeInstanceOf(Array);
expect(body.packageNames).toContain(testPackageName);
});
tap.test('Composer: should filter package list (GET /packages/list.json?filter=vendor/*)', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/composer/packages/list.json',
headers: {},
query: { filter: 'vendor/*' },
});
expect(response.status).toEqual(200);
const body = await streamToJson(response.body);
expect(body.packageNames).toBeInstanceOf(Array);
expect(body.packageNames).toContain(testPackageName);
});
tap.test('Composer: should prevent duplicate version upload', async () => {
const response = await registry.handleRequest({
method: 'PUT',
path: `/composer/packages/${testPackageName}`,
headers: {
Authorization: `Bearer ${composerToken}`,
'Content-Type': 'application/zip',
},
query: {},
body: testZipData,
});
expect(response.status).toEqual(409);
const body = await streamToJson(response.body);
expect(body.status).toEqual('error');
expect(body.message).toContain('already exists');
});
tap.test('Composer: should upload a second version', async () => {
const testVersion2 = '1.1.0';
const testZipData2 = await createComposerZip(testPackageName, testVersion2);
const response = await registry.handleRequest({
method: 'PUT',
path: `/composer/packages/${testPackageName}`,
headers: {
Authorization: `Bearer ${composerToken}`,
'Content-Type': 'application/zip',
},
query: {},
body: testZipData2,
});
expect(response.status).toEqual(201);
const body = await streamToJson(response.body);
expect(body.status).toEqual('success');
expect(body.version).toEqual(testVersion2);
});
tap.test('Composer: should return multiple versions in metadata', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: `/composer/p2/${testPackageName}.json`,
headers: {},
query: {},
});
expect(response.status).toEqual(200);
const body = await streamToJson(response.body);
expect(body.packages[testPackageName]).toBeInstanceOf(Array);
expect(body.packages[testPackageName].length).toEqual(2);
const versions = body.packages[testPackageName].map((p: any) => p.version);
expect(versions).toContain('1.0.0');
expect(versions).toContain('1.1.0');
});
tap.test('Composer: should delete a specific version (DELETE /packages/{vendor/package}/{version})', async () => {
const response = await registry.handleRequest({
method: 'DELETE',
path: `/composer/packages/${testPackageName}/1.0.0`,
headers: {
Authorization: `Bearer ${composerToken}`,
},
query: {},
});
expect(response.status).toEqual(204);
// Verify version was removed
const metadataResponse = await registry.handleRequest({
method: 'GET',
path: `/composer/p2/${testPackageName}.json`,
headers: {},
query: {},
});
const metaBody = await streamToJson(metadataResponse.body);
expect(metaBody.packages[testPackageName].length).toEqual(1);
expect(metaBody.packages[testPackageName][0].version).toEqual('1.1.0');
});
tap.test('Composer: should require auth for package upload', async () => {
const testZipData3 = await createComposerZip('vendor/unauth-package', '1.0.0');
const response = await registry.handleRequest({
method: 'PUT',
path: '/composer/packages/vendor/unauth-package',
headers: {
'Content-Type': 'application/zip',
},
query: {},
body: testZipData3,
});
expect(response.status).toEqual(401);
const body = await streamToJson(response.body);
expect(body.status).toEqual('error');
});
tap.test('Composer: should reject invalid ZIP (no composer.json)', async () => {
const invalidZip = Buffer.from('invalid zip content');
const response = await registry.handleRequest({
method: 'PUT',
path: `/composer/packages/${testPackageName}`,
headers: {
Authorization: `Bearer ${composerToken}`,
'Content-Type': 'application/zip',
},
query: {},
body: invalidZip,
});
expect(response.status).toEqual(400);
const body = await streamToJson(response.body);
expect(body.status).toEqual('error');
expect(body.message).toContain('composer.json');
});
tap.test('Composer: should delete entire package (DELETE /packages/{vendor/package})', async () => {
const response = await registry.handleRequest({
method: 'DELETE',
path: `/composer/packages/${testPackageName}`,
headers: {
Authorization: `Bearer ${composerToken}`,
},
query: {},
});
expect(response.status).toEqual(204);
// Verify package was removed
const metadataResponse = await registry.handleRequest({
method: 'GET',
path: `/composer/p2/${testPackageName}.json`,
headers: {},
query: {},
});
expect(metadataResponse.status).toEqual(404);
});
tap.test('Composer: should return 404 for non-existent package', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/composer/p2/non/existent.json',
headers: {},
query: {},
});
expect(response.status).toEqual(404);
});
tap.postTask('cleanup registry', async () => {
if (registry) {
registry.destroy();
}
});
export default tap.start();

View File

@@ -0,0 +1,292 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js';
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
import {
createTestRegistry,
createTestTokens,
createPythonWheel,
createRubyGem,
} from './helpers/registry.js';
let registry: SmartRegistry;
let pypiToken: string;
let rubygemsToken: string;
tap.test('Integration: should initialize registry with all protocols', async () => {
registry = await createTestRegistry();
const tokens = await createTestTokens(registry);
pypiToken = tokens.pypiToken;
rubygemsToken = tokens.rubygemsToken;
expect(registry).toBeInstanceOf(SmartRegistry);
expect(registry.isInitialized()).toEqual(true);
expect(pypiToken).toBeTypeOf('string');
expect(rubygemsToken).toBeTypeOf('string');
});
tap.test('Integration: should correctly route PyPI requests', async () => {
const wheelData = await createPythonWheel('integration-test-py', '1.0.0');
const response = await registry.handleRequest({
method: 'POST',
path: '/pypi/',
headers: {
Authorization: `Bearer ${pypiToken}`,
'Content-Type': 'multipart/form-data',
},
query: {},
body: {
':action': 'file_upload',
protocol_version: '1',
name: 'integration-test-py',
version: '1.0.0',
filetype: 'bdist_wheel',
pyversion: 'py3',
metadata_version: '2.1',
content: wheelData,
filename: 'integration_test_py-1.0.0-py3-none-any.whl',
},
});
expect(response.status).toEqual(201);
});
tap.test('Integration: should correctly route RubyGems requests', async () => {
const gemData = await createRubyGem('integration-test-gem', '1.0.0');
const response = await registry.handleRequest({
method: 'POST',
path: '/rubygems/api/v1/gems',
headers: {
Authorization: rubygemsToken,
'Content-Type': 'application/octet-stream',
},
query: {},
body: gemData,
});
expect(response.status).toEqual(201);
});
tap.test('Integration: should handle /simple path for PyPI', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/simple/',
headers: {
Accept: 'text/html',
},
query: {},
});
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toStartWith('text/html');
const body = await streamToBuffer(response.body);
const text = body.toString('utf-8');
expect(text).toContain('integration-test-py');
});
tap.test('Integration: should reject PyPI token for RubyGems endpoint', async () => {
const gemData = await createRubyGem('unauthorized-gem', '1.0.0');
const response = await registry.handleRequest({
method: 'POST',
path: '/rubygems/api/v1/gems',
headers: {
Authorization: pypiToken, // Using PyPI token for RubyGems endpoint
'Content-Type': 'application/octet-stream',
},
query: {},
body: gemData,
});
expect(response.status).toEqual(401);
});
tap.test('Integration: should reject RubyGems token for PyPI endpoint', async () => {
const wheelData = await createPythonWheel('unauthorized-py', '1.0.0');
const response = await registry.handleRequest({
method: 'POST',
path: '/pypi/',
headers: {
Authorization: `Bearer ${rubygemsToken}`, // Using RubyGems token for PyPI endpoint
'Content-Type': 'multipart/form-data',
},
query: {},
body: {
':action': 'file_upload',
protocol_version: '1',
name: 'unauthorized-py',
version: '1.0.0',
filetype: 'bdist_wheel',
pyversion: 'py3',
metadata_version: '2.1',
content: wheelData,
filename: 'unauthorized_py-1.0.0-py3-none-any.whl',
},
});
expect(response.status).toEqual(401);
});
tap.test('Integration: should return 404 for unknown paths', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/unknown-protocol/endpoint',
headers: {},
query: {},
});
expect(response.status).toEqual(404);
const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
expect(body.error).toEqual('NOT_FOUND');
});
tap.test('Integration: should retrieve PyPI registry instance', async () => {
const pypiRegistry = registry.getRegistry('pypi');
expect(pypiRegistry).toBeDefined();
expect(pypiRegistry).not.toBeNull();
});
tap.test('Integration: should retrieve RubyGems registry instance', async () => {
const rubygemsRegistry = registry.getRegistry('rubygems');
expect(rubygemsRegistry).toBeDefined();
expect(rubygemsRegistry).not.toBeNull();
});
tap.test('Integration: should retrieve all other protocol instances', async () => {
const ociRegistry = registry.getRegistry('oci');
const npmRegistry = registry.getRegistry('npm');
const mavenRegistry = registry.getRegistry('maven');
const composerRegistry = registry.getRegistry('composer');
const cargoRegistry = registry.getRegistry('cargo');
expect(ociRegistry).toBeDefined();
expect(npmRegistry).toBeDefined();
expect(mavenRegistry).toBeDefined();
expect(composerRegistry).toBeDefined();
expect(cargoRegistry).toBeDefined();
});
tap.test('Integration: should share storage across protocols', async () => {
const storage = registry.getStorage();
expect(storage).toBeDefined();
// Verify storage has methods for all protocols
expect(typeof storage.getPypiPackageMetadata).toEqual('function');
expect(typeof storage.getRubyGemsVersions).toEqual('function');
expect(typeof storage.getNpmPackument).toEqual('function');
expect(typeof storage.getOciBlob).toEqual('function');
});
tap.test('Integration: should share auth manager across protocols', async () => {
const authManager = registry.getAuthManager();
expect(authManager).toBeDefined();
// Verify auth manager has methods for all protocols
expect(typeof authManager.createPypiToken).toEqual('function');
expect(typeof authManager.createRubyGemsToken).toEqual('function');
expect(typeof authManager.createNpmToken).toEqual('function');
expect(typeof authManager.createOciToken).toEqual('function');
});
tap.test('Integration: should handle concurrent requests to different protocols', async () => {
const pypiRequest = registry.handleRequest({
method: 'GET',
path: '/simple/',
headers: {},
query: {},
});
const rubygemsRequest = registry.handleRequest({
method: 'GET',
path: '/rubygems/versions',
headers: {},
query: {},
});
const [pypiResponse, rubygemsResponse] = await Promise.all([pypiRequest, rubygemsRequest]);
expect(pypiResponse.status).toEqual(200);
expect(rubygemsResponse.status).toEqual(200);
});
tap.test('Integration: should handle package name conflicts across protocols', async () => {
const packageName = 'conflict-test';
// Upload PyPI package
const wheelData = await createPythonWheel(packageName, '1.0.0');
const pypiResponse = await registry.handleRequest({
method: 'POST',
path: '/pypi/',
headers: {
Authorization: `Bearer ${pypiToken}`,
'Content-Type': 'multipart/form-data',
},
query: {},
body: {
':action': 'file_upload',
protocol_version: '1',
name: packageName,
version: '1.0.0',
filetype: 'bdist_wheel',
pyversion: 'py3',
metadata_version: '2.1',
content: wheelData,
filename: `${packageName.replace(/-/g, '_')}-1.0.0-py3-none-any.whl`,
},
});
expect(pypiResponse.status).toEqual(201);
// Upload RubyGems package with same name
const gemData = await createRubyGem(packageName, '1.0.0');
const rubygemsResponse = await registry.handleRequest({
method: 'POST',
path: '/rubygems/api/v1/gems',
headers: {
Authorization: rubygemsToken,
'Content-Type': 'application/octet-stream',
},
query: {},
body: gemData,
});
expect(rubygemsResponse.status).toEqual(201);
// Both should exist independently
const pypiGetResponse = await registry.handleRequest({
method: 'GET',
path: `/simple/${packageName}/`,
headers: {},
query: {},
});
const rubygemsGetResponse = await registry.handleRequest({
method: 'GET',
path: `/rubygems/gems/${packageName}-1.0.0.gem`,
headers: {},
query: {},
});
expect(pypiGetResponse.status).toEqual(200);
expect(rubygemsGetResponse.status).toEqual(200);
});
tap.test('Integration: should properly clean up resources on destroy', async () => {
// Destroy should clean up all registries
expect(() => registry.destroy()).not.toThrow();
});
tap.postTask('cleanup registry', async () => {
if (registry && registry.isInitialized()) {
registry.destroy();
}
});
export default tap.start();

View File

@@ -0,0 +1,275 @@
/**
* Integration test for smartregistry with smartstorage
* Verifies that smartregistry works with a local S3-compatible server
*/
import { expect, tap } from '@git.zone/tstest/tapbundle';
import * as smartstorageModule from '@push.rocks/smartstorage';
import { SmartRegistry } from '../ts/classes.smartregistry.js';
import type { IRegistryConfig } from '../ts/core/interfaces.core.js';
import { streamToJson } from '../ts/core/helpers.stream.js';
import * as crypto from 'crypto';
let s3Server: smartstorageModule.SmartStorage;
let registry: SmartRegistry;
/**
* Setup: Start smartstorage server
*/
tap.test('should start smartstorage server', async () => {
s3Server = await smartstorageModule.SmartStorage.createAndStart({
server: {
port: 3456,
address: '0.0.0.0',
silent: true,
},
storage: {
cleanSlate: true,
directory: './.nogit/smartstorage-test-buckets',
},
logging: {
enabled: false,
},
});
expect(s3Server).toBeDefined();
});
/**
* Setup: Create SmartRegistry with smartstorage configuration
*/
tap.test('should create SmartRegistry instance with smarts3 IS3Descriptor', async () => {
const s3Descriptor = await s3Server.getStorageDescriptor();
const config: IRegistryConfig = {
storage: {
...s3Descriptor,
bucketName: 'test-registry-smarts3',
},
auth: {
jwtSecret: 'test-secret-key',
tokenStore: 'memory',
npmTokens: {
enabled: true,
},
ociTokens: {
enabled: true,
realm: 'https://auth.example.com/token',
service: 'test-registry-smarts3',
},
pypiTokens: {
enabled: true,
},
rubygemsTokens: {
enabled: true,
},
},
npm: {
enabled: true,
basePath: '/npm',
},
oci: {
enabled: true,
basePath: '/oci',
},
pypi: {
enabled: true,
basePath: '/pypi',
},
cargo: {
enabled: true,
basePath: '/cargo',
},
};
registry = new SmartRegistry(config);
await registry.init();
expect(registry).toBeDefined();
});
/**
* Test NPM protocol with smartstorage
*/
tap.test('NPM: should publish package to smarts3', async () => {
const authManager = registry.getAuthManager();
const userId = await authManager.authenticate({
username: 'testuser',
password: 'testpass',
});
const token = await authManager.createNpmToken(userId, false);
const packageData = {
name: 'test-package-smarts3',
'dist-tags': {
latest: '1.0.0',
},
versions: {
'1.0.0': {
name: 'test-package-smarts3',
version: '1.0.0',
description: 'Test package for smarts3 integration',
},
},
_attachments: {
'test-package-smarts3-1.0.0.tgz': {
content_type: 'application/octet-stream',
data: Buffer.from('test tarball content').toString('base64'),
length: 20,
},
},
};
const response = await registry.handleRequest({
method: 'PUT',
path: '/npm/test-package-smarts3',
headers: {
'Authorization': `Bearer ${token}`,
'Content-Type': 'application/json',
},
query: {},
body: packageData,
});
expect(response.status).toEqual(201);
});
tap.test('NPM: should retrieve package from smarts3', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/npm/test-package-smarts3',
headers: {},
query: {},
});
expect(response.status).toEqual(200);
const body = await streamToJson(response.body);
expect(body).toHaveProperty('name');
expect(body.name).toEqual('test-package-smarts3');
});
/**
* Test OCI protocol with smartstorage
*/
tap.test('OCI: should store blob in smarts3', async () => {
const authManager = registry.getAuthManager();
const userId = await authManager.authenticate({
username: 'testuser',
password: 'testpass',
});
const token = await authManager.createOciToken(
userId,
['oci:repository:test-image:push'],
3600
);
// Initiate blob upload
const initiateResponse = await registry.handleRequest({
method: 'POST',
path: '/oci/test-image/blobs/uploads/',
headers: {
'Authorization': `Bearer ${token}`,
},
query: {},
});
expect(initiateResponse.status).toEqual(202);
expect(initiateResponse.headers).toHaveProperty('Location');
// Extract upload ID from location
const location = initiateResponse.headers['Location'];
const uploadId = location.split('/').pop();
// Upload blob data
const blobData = Buffer.from('test blob content');
const digest = 'sha256:' + crypto
.createHash('sha256')
.update(blobData)
.digest('hex');
const uploadResponse = await registry.handleRequest({
method: 'PUT',
path: `/oci/test-image/blobs/uploads/${uploadId}`,
headers: {
'Authorization': `Bearer ${token}`,
'Content-Type': 'application/octet-stream',
},
query: { digest },
body: blobData,
});
expect(uploadResponse.status).toEqual(201);
});
/**
* Test PyPI protocol with smartstorage
*/
tap.test('PyPI: should upload package to smarts3', async () => {
const storage = registry.getStorage();
// Store a test package file
const packageContent = Buffer.from('test wheel content');
await storage.putPypiPackageFile(
'test-package',
'test_package-1.0.0-py3-none-any.whl',
packageContent
);
// Store metadata
const metadata = {
name: 'test-package',
version: '1.0.0',
files: [
{
filename: 'test_package-1.0.0-py3-none-any.whl',
url: '/packages/test-package/test_package-1.0.0-py3-none-any.whl',
hashes: { sha256: 'abc123' },
},
],
};
await storage.putPypiPackageMetadata('test-package', metadata);
// Verify stored
const retrievedMetadata = await storage.getPypiPackageMetadata('test-package');
expect(retrievedMetadata).toBeDefined();
expect(retrievedMetadata.name).toEqual('test-package');
});
/**
* Test Cargo protocol with smartstorage
*/
tap.test('Cargo: should store crate in smarts3', async () => {
const storage = registry.getStorage();
// Store a test crate index entry
const indexEntry = {
name: 'test-crate',
vers: '1.0.0',
deps: [],
cksum: 'abc123',
features: {},
yanked: false,
};
await storage.putCargoIndex('test-crate', [indexEntry]);
// Store the actual .crate file
const crateContent = Buffer.from('test crate tarball');
await storage.putCargoCrate('test-crate', '1.0.0', crateContent);
// Verify stored
const retrievedIndex = await storage.getCargoIndex('test-crate');
expect(retrievedIndex).toBeDefined();
expect(retrievedIndex.length).toEqual(1);
expect(retrievedIndex[0].name).toEqual('test-crate');
});
/**
* Cleanup: Stop smartstorage server
*/
tap.test('should stop smartstorage server', async () => {
registry.destroy();
await s3Server.stop();
});
export default tap.start();

View File

@@ -0,0 +1,484 @@
/**
* Native Maven CLI Testing
* Tests the Maven registry implementation using the actual mvn CLI
*/
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { tapNodeTools } from '@git.zone/tstest/tapbundle_serverside';
import { SmartRegistry } from '../ts/index.js';
import { createTestRegistry, createTestTokens, createTestPom, createTestJar } from './helpers/registry.js';
import type { IRequestContext, IResponse } from '../ts/core/interfaces.core.js';
import * as http from 'http';
import * as url from 'url';
import * as fs from 'fs';
import * as path from 'path';
// Test context
let registry: SmartRegistry;
let server: http.Server;
let registryUrl: string;
let registryPort: number;
let mavenToken: string;
let testDir: string;
let m2Dir: string;
/**
* Create HTTP server wrapper around SmartRegistry
*/
async function createHttpServer(
registryInstance: SmartRegistry,
port: number
): Promise<{ server: http.Server; url: string }> {
return new Promise((resolve, reject) => {
const httpServer = http.createServer(async (req, res) => {
try {
// Parse request
const parsedUrl = url.parse(req.url || '', true);
const pathname = parsedUrl.pathname || '/';
const query = parsedUrl.query;
// Read body
const chunks: Buffer[] = [];
for await (const chunk of req) {
chunks.push(chunk);
}
const bodyBuffer = Buffer.concat(chunks);
// Parse body based on content type
let body: any;
if (bodyBuffer.length > 0) {
const contentType = req.headers['content-type'] || '';
if (contentType.includes('application/json')) {
try {
body = JSON.parse(bodyBuffer.toString('utf-8'));
} catch (error) {
body = bodyBuffer;
}
} else {
body = bodyBuffer;
}
}
// Convert to IRequestContext
const context: IRequestContext = {
method: req.method || 'GET',
path: pathname,
headers: req.headers as Record<string, string>,
query: query as Record<string, string>,
body: body,
};
// Handle request
const response: IResponse = await registryInstance.handleRequest(context);
// Convert IResponse to HTTP response
res.statusCode = response.status;
// Set headers
for (const [key, value] of Object.entries(response.headers || {})) {
res.setHeader(key, value);
}
// Send body (response.body is always ReadableStream<Uint8Array> or undefined)
if (response.body) {
const { Readable } = await import('stream');
Readable.fromWeb(response.body).pipe(res);
} else {
res.end();
}
} catch (error) {
console.error('Server error:', error);
res.statusCode = 500;
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify({ error: 'INTERNAL_ERROR', message: String(error) }));
}
});
httpServer.listen(port, () => {
const serverUrl = `http://localhost:${port}`;
resolve({ server: httpServer, url: serverUrl });
});
httpServer.on('error', reject);
});
}
/**
* Setup Maven settings.xml for authentication
*/
function setupMavenSettings(
token: string,
m2DirArg: string,
serverUrl: string
): string {
fs.mkdirSync(m2DirArg, { recursive: true });
const settingsXml = `<?xml version="1.0" encoding="UTF-8"?>
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
http://maven.apache.org/xsd/settings-1.0.0.xsd">
<servers>
<server>
<id>test-registry</id>
<username>testuser</username>
<password>${token}</password>
</server>
</servers>
<profiles>
<profile>
<id>test-registry</id>
<repositories>
<repository>
<id>test-registry</id>
<url>${serverUrl}/maven</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
</snapshots>
</repository>
</repositories>
</profile>
</profiles>
<activeProfiles>
<activeProfile>test-registry</activeProfile>
</activeProfiles>
</settings>
`;
const settingsPath = path.join(m2DirArg, 'settings.xml');
fs.writeFileSync(settingsPath, settingsXml, 'utf-8');
return settingsPath;
}
/**
* Create a minimal Maven project for testing
*/
function createMavenProject(
projectDir: string,
groupId: string,
artifactId: string,
version: string,
registryUrl: string
): void {
fs.mkdirSync(projectDir, { recursive: true });
const pomXml = `<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>${groupId}</groupId>
<artifactId>${artifactId}</artifactId>
<version>${version}</version>
<packaging>jar</packaging>
<name>${artifactId}</name>
<description>Test Maven project for SmartRegistry CLI tests</description>
<distributionManagement>
<repository>
<id>test-registry</id>
<url>${registryUrl}/maven</url>
</repository>
<snapshotRepository>
<id>test-registry</id>
<url>${registryUrl}/maven</url>
</snapshotRepository>
</distributionManagement>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.1</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
</plugins>
</build>
</project>
`;
fs.writeFileSync(path.join(projectDir, 'pom.xml'), pomXml, 'utf-8');
// Create minimal Java source
const srcDir = path.join(projectDir, 'src', 'main', 'java', 'com', 'test');
fs.mkdirSync(srcDir, { recursive: true });
const javaSource = `package com.test;
public class Main {
public static void main(String[] args) {
System.out.println("Hello from SmartRegistry test!");
}
}
`;
fs.writeFileSync(path.join(srcDir, 'Main.java'), javaSource, 'utf-8');
}
/**
* Run Maven command with custom settings
*/
async function runMavenCommand(
command: string,
cwd: string
): Promise<{ stdout: string; stderr: string; exitCode: number }> {
const settingsPath = path.join(m2Dir, 'settings.xml');
const fullCommand = `cd "${cwd}" && mvn -s "${settingsPath}" ${command}`;
try {
const result = await tapNodeTools.runCommand(fullCommand);
return {
stdout: result.stdout || '',
stderr: result.stderr || '',
exitCode: result.exitCode || 0,
};
} catch (error: any) {
return {
stdout: error.stdout || '',
stderr: error.stderr || String(error),
exitCode: error.exitCode || 1,
};
}
}
/**
* Cleanup test directory
*/
function cleanupTestDir(dir: string): void {
if (fs.existsSync(dir)) {
fs.rmSync(dir, { recursive: true, force: true });
}
}
// ========================================================================
// TESTS
// ========================================================================
tap.test('Maven CLI: should verify mvn is installed', async () => {
try {
const result = await tapNodeTools.runCommand('mvn -version');
console.log('Maven version output:', result.stdout.substring(0, 200));
expect(result.exitCode).toEqual(0);
} catch (error) {
console.log('Maven CLI not available, skipping native CLI tests');
// Skip remaining tests if Maven is not installed
tap.skip.test('Maven CLI: remaining tests skipped - mvn not available');
return;
}
});
tap.test('Maven CLI: should setup registry and HTTP server', async () => {
// Use port 37000 (avoids conflicts with other tests)
registryPort = 37000;
// Create registry with correct registryUrl for CLI tests
registry = await createTestRegistry({ registryUrl: `http://localhost:${registryPort}` });
const tokens = await createTestTokens(registry);
mavenToken = tokens.mavenToken;
expect(registry).toBeInstanceOf(SmartRegistry);
expect(mavenToken).toBeTypeOf('string');
const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server;
registryUrl = serverSetup.url;
expect(server).toBeDefined();
expect(registryUrl).toEqual(`http://localhost:${registryPort}`);
// Setup test directory
testDir = path.join(process.cwd(), '.nogit', 'test-maven-cli');
cleanupTestDir(testDir);
fs.mkdirSync(testDir, { recursive: true });
// Setup .m2 directory
m2Dir = path.join(testDir, '.m2');
fs.mkdirSync(m2Dir, { recursive: true });
// Setup Maven settings
const settingsPath = setupMavenSettings(mavenToken, m2Dir, registryUrl);
expect(fs.existsSync(settingsPath)).toEqual(true);
});
tap.test('Maven CLI: should verify server is responding', async () => {
// Check server is up by doing a direct HTTP request
const response = await fetch(`${registryUrl}/maven/`);
expect(response.status).toBeGreaterThanOrEqual(200);
expect(response.status).toBeLessThan(500);
});
tap.test('Maven CLI: should deploy a JAR artifact', async () => {
const groupId = 'com.test';
const artifactId = 'test-artifact';
const version = '1.0.0';
const projectDir = path.join(testDir, 'test-project');
createMavenProject(projectDir, groupId, artifactId, version, registryUrl);
// Build and deploy
const result = await runMavenCommand('clean package deploy -DskipTests', projectDir);
console.log('mvn deploy output:', result.stdout.substring(0, 500));
console.log('mvn deploy stderr:', result.stderr.substring(0, 500));
expect(result.exitCode).toEqual(0);
});
tap.test('Maven CLI: should verify artifact in registry via API', async () => {
const groupId = 'com.test';
const artifactId = 'test-artifact';
const version = '1.0.0';
// Maven path: /maven/{groupId path}/{artifactId}/{version}/{artifactId}-{version}.jar
const jarPath = `/maven/com/test/${artifactId}/${version}/${artifactId}-${version}.jar`;
const response = await fetch(`${registryUrl}${jarPath}`, {
headers: { Authorization: `Bearer ${mavenToken}` },
});
expect(response.status).toEqual(200);
const jarData = await response.arrayBuffer();
expect(jarData.byteLength).toBeGreaterThan(0);
});
tap.test('Maven CLI: should verify POM in registry', async () => {
const groupId = 'com.test';
const artifactId = 'test-artifact';
const version = '1.0.0';
const pomPath = `/maven/com/test/${artifactId}/${version}/${artifactId}-${version}.pom`;
const response = await fetch(`${registryUrl}${pomPath}`, {
headers: { Authorization: `Bearer ${mavenToken}` },
});
expect(response.status).toEqual(200);
const pomContent = await response.text();
expect(pomContent).toContain(groupId);
expect(pomContent).toContain(artifactId);
expect(pomContent).toContain(version);
});
tap.test('Maven CLI: should verify checksums exist', async () => {
const artifactId = 'test-artifact';
const version = '1.0.0';
// Check JAR checksums
const basePath = `/maven/com/test/${artifactId}/${version}/${artifactId}-${version}.jar`;
// MD5
const md5Response = await fetch(`${registryUrl}${basePath}.md5`, {
headers: { Authorization: `Bearer ${mavenToken}` },
});
expect(md5Response.status).toEqual(200);
// SHA1
const sha1Response = await fetch(`${registryUrl}${basePath}.sha1`, {
headers: { Authorization: `Bearer ${mavenToken}` },
});
expect(sha1Response.status).toEqual(200);
});
tap.test('Maven CLI: should deploy second version', async () => {
const groupId = 'com.test';
const artifactId = 'test-artifact';
const version = '2.0.0';
const projectDir = path.join(testDir, 'test-project-v2');
createMavenProject(projectDir, groupId, artifactId, version, registryUrl);
const result = await runMavenCommand('clean package deploy -DskipTests', projectDir);
console.log('mvn deploy v2 output:', result.stdout.substring(0, 500));
expect(result.exitCode).toEqual(0);
});
tap.test('Maven CLI: should verify metadata.xml exists', async () => {
const artifactId = 'test-artifact';
// Maven metadata is stored at /maven/{groupId path}/{artifactId}/maven-metadata.xml
const metadataPath = `/maven/com/test/${artifactId}/maven-metadata.xml`;
const response = await fetch(`${registryUrl}${metadataPath}`, {
headers: { Authorization: `Bearer ${mavenToken}` },
});
expect(response.status).toEqual(200);
const metadataXml = await response.text();
expect(metadataXml).toContain(artifactId);
expect(metadataXml).toContain('1.0.0');
expect(metadataXml).toContain('2.0.0');
});
tap.test('Maven CLI: should resolve dependency from registry', async () => {
const groupId = 'com.consumer';
const artifactId = 'consumer-app';
const version = '1.0.0';
const projectDir = path.join(testDir, 'consumer-project');
fs.mkdirSync(projectDir, { recursive: true });
// Create a consumer project that depends on our test artifact
const pomXml = `<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>${groupId}</groupId>
<artifactId>${artifactId}</artifactId>
<version>${version}</version>
<packaging>jar</packaging>
<repositories>
<repository>
<id>test-registry</id>
<url>${registryUrl}/maven</url>
</repository>
</repositories>
<dependencies>
<dependency>
<groupId>com.test</groupId>
<artifactId>test-artifact</artifactId>
<version>1.0.0</version>
</dependency>
</dependencies>
</project>
`;
fs.writeFileSync(path.join(projectDir, 'pom.xml'), pomXml, 'utf-8');
// Try to resolve dependencies
const result = await runMavenCommand('dependency:resolve', projectDir);
console.log('mvn dependency:resolve output:', result.stdout.substring(0, 500));
expect(result.exitCode).toEqual(0);
});
tap.postTask('cleanup maven cli tests', async () => {
// Stop server
if (server) {
await new Promise<void>((resolve) => {
server.close(() => resolve());
});
}
// Cleanup test directory
if (testDir) {
cleanupTestDir(testDir);
}
// Destroy registry
if (registry) {
registry.destroy();
}
});
export default tap.start();

View File

@@ -1,5 +1,6 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js';
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
import {
createTestRegistry,
createTestTokens,
@@ -30,6 +31,14 @@ tap.test('Maven: should create registry instance', async () => {
expect(registry).toBeInstanceOf(SmartRegistry);
expect(mavenToken).toBeTypeOf('string');
// Clean up any existing metadata from previous test runs
const storage = registry.getStorage();
try {
await storage.deleteMavenMetadata(testGroupId, testArtifactId);
} catch (error) {
// Ignore error if metadata doesn't exist
}
});
tap.test('Maven: should upload POM file (PUT /{groupPath}/{artifactId}/{version}/*.pom)', async () => {
@@ -80,10 +89,11 @@ tap.test('Maven: should retrieve uploaded POM file (GET)', async () => {
});
expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer);
expect((response.body as Buffer).toString('utf-8')).toContain(testGroupId);
expect((response.body as Buffer).toString('utf-8')).toContain(testArtifactId);
expect((response.body as Buffer).toString('utf-8')).toContain(testVersion);
const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
expect(body.toString('utf-8')).toContain(testGroupId);
expect(body.toString('utf-8')).toContain(testArtifactId);
expect(body.toString('utf-8')).toContain(testVersion);
expect(response.headers['Content-Type']).toEqual('application/xml');
});
@@ -99,7 +109,8 @@ tap.test('Maven: should retrieve uploaded JAR file (GET)', async () => {
});
expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer);
const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
expect(response.headers['Content-Type']).toEqual('application/java-archive');
});
@@ -116,8 +127,9 @@ tap.test('Maven: should retrieve MD5 checksum for JAR (GET *.jar.md5)', async ()
});
expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer);
expect((response.body as Buffer).toString('utf-8')).toEqual(checksums.md5);
const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
expect(body.toString('utf-8')).toEqual(checksums.md5);
expect(response.headers['Content-Type']).toEqual('text/plain');
});
@@ -134,8 +146,9 @@ tap.test('Maven: should retrieve SHA1 checksum for JAR (GET *.jar.sha1)', async
});
expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer);
expect((response.body as Buffer).toString('utf-8')).toEqual(checksums.sha1);
const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
expect(body.toString('utf-8')).toEqual(checksums.sha1);
expect(response.headers['Content-Type']).toEqual('text/plain');
});
@@ -152,8 +165,9 @@ tap.test('Maven: should retrieve SHA256 checksum for JAR (GET *.jar.sha256)', as
});
expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer);
expect((response.body as Buffer).toString('utf-8')).toEqual(checksums.sha256);
const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
expect(body.toString('utf-8')).toEqual(checksums.sha256);
expect(response.headers['Content-Type']).toEqual('text/plain');
});
@@ -170,8 +184,9 @@ tap.test('Maven: should retrieve SHA512 checksum for JAR (GET *.jar.sha512)', as
});
expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer);
expect((response.body as Buffer).toString('utf-8')).toEqual(checksums.sha512);
const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
expect(body.toString('utf-8')).toEqual(checksums.sha512);
expect(response.headers['Content-Type']).toEqual('text/plain');
});
@@ -186,8 +201,9 @@ tap.test('Maven: should retrieve maven-metadata.xml (GET)', async () => {
});
expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer);
const xml = (response.body as Buffer).toString('utf-8');
const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
const xml = body.toString('utf-8');
expect(xml).toContain('<groupId>');
expect(xml).toContain('<artifactId>');
expect(xml).toContain('<version>1.0.0</version>');
@@ -239,7 +255,8 @@ tap.test('Maven: should upload a second version and update metadata', async () =
});
expect(response.status).toEqual(200);
const xml = (response.body as Buffer).toString('utf-8');
const metaBody = await streamToBuffer(response.body);
const xml = metaBody.toString('utf-8');
expect(xml).toContain('<version>1.0.0</version>');
expect(xml).toContain('<version>2.0.0</version>');
expect(xml).toContain('<latest>2.0.0</latest>');
@@ -277,7 +294,8 @@ tap.test('Maven: should return 404 for non-existent artifact', async () => {
});
expect(response.status).toEqual(404);
expect(response.body).toHaveProperty('error');
const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
});
tap.test('Maven: should return 401 for unauthorized upload', async () => {
@@ -296,7 +314,8 @@ tap.test('Maven: should return 401 for unauthorized upload', async () => {
});
expect(response.status).toEqual(401);
expect(response.body).toHaveProperty('error');
const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
});
tap.test('Maven: should reject POM upload with mismatched GAV', async () => {
@@ -320,7 +339,8 @@ tap.test('Maven: should reject POM upload with mismatched GAV', async () => {
});
expect(response.status).toEqual(400);
expect(response.body).toHaveProperty('error');
const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
});
tap.test('Maven: should delete an artifact (DELETE)', async () => {
@@ -336,7 +356,7 @@ tap.test('Maven: should delete an artifact (DELETE)', async () => {
query: {},
});
expect(response.status).toEqual(200);
expect(response.status).toEqual(204); // 204 No Content is correct for DELETE
// Verify artifact was deleted
const getResponse = await registry.handleRequest({

View File

@@ -0,0 +1,387 @@
/**
* Native npm CLI Testing
* Tests the NPM registry implementation using the actual npm CLI
*/
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { tapNodeTools } from '@git.zone/tstest/tapbundle_serverside';
import { SmartRegistry } from '../ts/index.js';
import { createTestRegistry, createTestTokens, cleanupS3Bucket } from './helpers/registry.js';
import type { IRequestContext, IResponse } from '../ts/core/interfaces.core.js';
import * as http from 'http';
import * as url from 'url';
import * as fs from 'fs';
import * as path from 'path';
// Test state
let registry: SmartRegistry;
let server: http.Server;
let registryUrl: string;
let registryPort: number;
let npmToken: string;
let testDir: string;
let npmrcPath: string;
/**
* Create HTTP server wrapper around SmartRegistry
*/
async function createHttpServer(
registryInstance: SmartRegistry,
port: number
): Promise<{ server: http.Server; url: string }> {
return new Promise((resolve, reject) => {
const httpServer = http.createServer(async (req, res) => {
try {
const parsedUrl = new url.URL(req.url || '/', `http://localhost:${port}`);
const pathname = parsedUrl.pathname;
const query: Record<string, string> = {};
parsedUrl.searchParams.forEach((value, key) => {
query[key] = value;
});
// Read body
let body: any = undefined;
if (req.method === 'PUT' || req.method === 'POST' || req.method === 'PATCH') {
const chunks: Buffer[] = [];
for await (const chunk of req) {
chunks.push(Buffer.from(chunk));
}
const bodyBuffer = Buffer.concat(chunks);
const contentType = req.headers['content-type'] || '';
if (contentType.includes('application/json')) {
try {
body = JSON.parse(bodyBuffer.toString('utf-8'));
} catch (error) {
body = bodyBuffer;
}
} else {
body = bodyBuffer;
}
}
// Convert to IRequestContext
const context: IRequestContext = {
method: req.method || 'GET',
path: pathname,
headers: req.headers as Record<string, string>,
query: query as Record<string, string>,
body: body,
};
// Handle request
const response: IResponse = await registryInstance.handleRequest(context);
// Convert IResponse to HTTP response
res.statusCode = response.status;
// Set headers
for (const [key, value] of Object.entries(response.headers || {})) {
res.setHeader(key, value);
}
// Send body (response.body is always ReadableStream<Uint8Array> or undefined)
if (response.body) {
const { Readable } = await import('stream');
Readable.fromWeb(response.body).pipe(res);
} else {
res.end();
}
} catch (error) {
console.error('Server error:', error);
res.statusCode = 500;
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify({ error: 'INTERNAL_ERROR', message: String(error) }));
}
});
httpServer.listen(port, () => {
const serverUrl = `http://localhost:${port}`;
resolve({ server: httpServer, url: serverUrl });
});
httpServer.on('error', reject);
});
}
/**
* Setup .npmrc configuration
*/
function setupNpmrc(registryUrlArg: string, token: string, testDirArg: string): string {
const npmrcContent = `registry=${registryUrlArg}/npm/
//localhost:${registryPort}/npm/:_authToken=${token}
`;
const npmrcFilePath = path.join(testDirArg, '.npmrc');
fs.writeFileSync(npmrcFilePath, npmrcContent, 'utf-8');
return npmrcFilePath;
}
/**
* Create a test package
*/
function createTestPackage(
packageName: string,
version: string,
targetDir: string
): string {
const packageDir = path.join(targetDir, packageName.replace(/\//g, '-'));
fs.mkdirSync(packageDir, { recursive: true });
// Create package.json
const packageJson = {
name: packageName,
version: version,
description: `Test package ${packageName}`,
main: 'index.js',
scripts: {},
};
fs.writeFileSync(
path.join(packageDir, 'package.json'),
JSON.stringify(packageJson, null, 2),
'utf-8'
);
// Create a simple index.js
fs.writeFileSync(
path.join(packageDir, 'index.js'),
`module.exports = { name: '${packageName}', version: '${version}' };\n`,
'utf-8'
);
// Create README.md
fs.writeFileSync(
path.join(packageDir, 'README.md'),
`# ${packageName}\n\nTest package version ${version}\n`,
'utf-8'
);
// Copy .npmrc into the package directory
if (npmrcPath && fs.existsSync(npmrcPath)) {
fs.copyFileSync(npmrcPath, path.join(packageDir, '.npmrc'));
}
return packageDir;
}
/**
* Run npm command with proper environment
*/
async function runNpmCommand(
command: string,
cwd: string
): Promise<{ stdout: string; stderr: string; exitCode: number }> {
const { exec } = await import('child_process');
// Build isolated env that prevents npm from reading ~/.npmrc
const env: Record<string, string> = {};
// Copy only essential env vars (PATH, etc.) — exclude HOME to prevent ~/.npmrc reading
for (const key of ['PATH', 'NODE', 'NVM_DIR', 'NVM_BIN', 'LANG', 'TERM', 'SHELL']) {
if (process.env[key]) env[key] = process.env[key]!;
}
env.HOME = testDir;
env.NPM_CONFIG_USERCONFIG = npmrcPath;
env.NPM_CONFIG_GLOBALCONFIG = '/dev/null';
env.NPM_CONFIG_CACHE = path.join(testDir, '.npm-cache');
env.NPM_CONFIG_PREFIX = path.join(testDir, '.npm-global');
env.NPM_CONFIG_REGISTRY = `${registryUrl}/npm/`;
return new Promise((resolve) => {
exec(command, { cwd, env, timeout: 30000 }, (error, stdout, stderr) => {
resolve({
stdout: stdout || '',
stderr: stderr || '',
exitCode: error ? (error as any).code ?? 1 : 0,
});
});
});
}
/**
* Cleanup test directory
*/
function cleanupTestDir(dir: string): void {
if (fs.existsSync(dir)) {
fs.rmSync(dir, { recursive: true, force: true });
}
}
// ========================================================================
// TESTS
// ========================================================================
tap.test('NPM CLI: should setup registry and HTTP server', async () => {
// Find available port
registryPort = 35000;
// Create registry with correct registryUrl for CLI tests
registry = await createTestRegistry({ registryUrl: `http://localhost:${registryPort}` });
const tokens = await createTestTokens(registry);
npmToken = tokens.npmToken;
// Clean up stale npm CLI test data via unpublish API
for (const pkg of ['test-package-cli', '@testscope%2fscoped-package']) {
await registry.handleRequest({
method: 'DELETE',
path: `/npm/${pkg}/-rev/cleanup`,
headers: { Authorization: `Bearer ${npmToken}` },
query: {},
});
}
expect(registry).toBeInstanceOf(SmartRegistry);
expect(npmToken).toBeTypeOf('string');
const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server;
registryUrl = serverSetup.url;
expect(server).toBeDefined();
expect(registryUrl).toEqual(`http://localhost:${registryPort}`);
// Setup test directory — use /tmp to isolate from project tree
testDir = path.join('/tmp', 'smartregistry-test-npm-cli');
cleanupTestDir(testDir);
fs.mkdirSync(testDir, { recursive: true });
// Setup .npmrc
npmrcPath = setupNpmrc(registryUrl, npmToken, testDir);
expect(fs.existsSync(npmrcPath)).toEqual(true);
});
tap.test('NPM CLI: should verify server is responding', async () => {
const result = await runNpmCommand('npm ping', testDir);
console.log('npm ping output:', result.stdout, result.stderr);
// npm ping may not work with custom registries, so just check server is up
// by doing a direct HTTP request
const response = await fetch(`${registryUrl}/npm/`);
expect(response.status).toBeGreaterThanOrEqual(200);
expect(response.status).toBeLessThan(500);
});
tap.test('NPM CLI: should publish a package', async () => {
const packageName = 'test-package-cli';
const version = '1.0.0';
const packageDir = createTestPackage(packageName, version, testDir);
const result = await runNpmCommand('npm publish', packageDir);
console.log('npm publish output:', result.stdout);
console.log('npm publish stderr:', result.stderr);
expect(result.exitCode).toEqual(0);
expect(result.stdout || result.stderr).toContain(packageName);
});
tap.test('NPM CLI: should view published package', async () => {
const packageName = 'test-package-cli';
const result = await runNpmCommand(`npm view ${packageName}`, testDir);
console.log('npm view output:', result.stdout);
expect(result.exitCode).toEqual(0);
expect(result.stdout).toContain(packageName);
expect(result.stdout).toContain('1.0.0');
});
tap.test('NPM CLI: should install published package', async () => {
const packageName = 'test-package-cli';
const installDir = path.join(testDir, 'install-test');
fs.mkdirSync(installDir, { recursive: true });
// Create a minimal package.json for install target
fs.writeFileSync(
path.join(installDir, 'package.json'),
JSON.stringify({ name: 'install-test', version: '1.0.0', dependencies: { [packageName]: '1.0.0' } }),
'utf-8'
);
// Copy .npmrc
if (npmrcPath && fs.existsSync(npmrcPath)) {
fs.copyFileSync(npmrcPath, path.join(installDir, '.npmrc'));
}
const result = await runNpmCommand('npm install', installDir);
console.log('npm install output:', result.stdout);
console.log('npm install stderr:', result.stderr);
expect(result.exitCode).toEqual(0);
// Verify package was installed
const installed = fs.existsSync(path.join(installDir, 'node_modules', packageName, 'package.json'));
expect(installed).toEqual(true);
});
tap.test('NPM CLI: should publish second version', async () => {
const packageName = 'test-package-cli';
const version = '1.1.0';
const packageDir = createTestPackage(packageName, version, testDir);
const result = await runNpmCommand('npm publish', packageDir);
console.log('npm publish v1.1.0 output:', result.stdout);
expect(result.exitCode).toEqual(0);
});
tap.test('NPM CLI: should list versions', async () => {
const packageName = 'test-package-cli';
const result = await runNpmCommand(`npm view ${packageName} versions`, testDir);
console.log('npm view versions output:', result.stdout);
expect(result.exitCode).toEqual(0);
expect(result.stdout).toContain('1.0.0');
expect(result.stdout).toContain('1.1.0');
});
tap.test('NPM CLI: should publish scoped package', async () => {
const packageName = '@testscope/scoped-package';
const version = '1.0.0';
const packageDir = createTestPackage(packageName, version, testDir);
const result = await runNpmCommand('npm publish --access public', packageDir);
console.log('npm publish scoped output:', result.stdout);
console.log('npm publish scoped stderr:', result.stderr);
expect(result.exitCode).toEqual(0);
});
tap.test('NPM CLI: should view scoped package', async () => {
const packageName = '@testscope/scoped-package';
const result = await runNpmCommand(`npm view ${packageName}`, testDir);
console.log('npm view scoped output:', result.stdout);
expect(result.exitCode).toEqual(0);
expect(result.stdout).toContain('scoped-package');
});
tap.test('NPM CLI: should fail to publish without auth', async () => {
const packageName = 'unauth-package';
const version = '1.0.0';
const packageDir = createTestPackage(packageName, version, testDir);
// Temporarily remove .npmrc (write one without auth)
const noAuthNpmrc = path.join(packageDir, '.npmrc');
fs.writeFileSync(noAuthNpmrc, `registry=${registryUrl}/npm/\n`, 'utf-8');
const result = await runNpmCommand('npm publish', packageDir);
console.log('npm publish unauth output:', result.stdout);
console.log('npm publish unauth stderr:', result.stderr);
// Should fail with auth error
expect(result.exitCode).not.toEqual(0);
});
tap.postTask('cleanup npm cli tests', async () => {
// Stop server
if (server) {
await new Promise<void>((resolve) => {
server.close(() => resolve());
});
}
// Cleanup test directory
cleanupTestDir(testDir);
});
export default tap.start();

View File

@@ -1,5 +1,6 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js';
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
import { createTestRegistry, createTestTokens, createTestPackument } from './helpers/registry.js';
let registry: SmartRegistry;
@@ -34,8 +35,9 @@ tap.test('NPM: should handle user authentication (PUT /-/user/org.couchdb.user:{
});
expect(response.status).toEqual(201);
expect(response.body).toHaveProperty('token');
expect((response.body as any).token).toBeTypeOf('string');
const body = await streamToJson(response.body);
expect(body).toHaveProperty('token');
expect(body.token).toBeTypeOf('string');
});
tap.test('NPM: should publish a package (PUT /{package})', async () => {
@@ -53,8 +55,9 @@ tap.test('NPM: should publish a package (PUT /{package})', async () => {
});
expect(response.status).toEqual(201);
expect(response.body).toHaveProperty('ok');
expect((response.body as any).ok).toEqual(true);
const body = await streamToJson(response.body);
expect(body).toHaveProperty('ok');
expect(body.ok).toEqual(true);
});
tap.test('NPM: should retrieve package metadata (GET /{package})', async () => {
@@ -66,10 +69,11 @@ tap.test('NPM: should retrieve package metadata (GET /{package})', async () => {
});
expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('name');
expect((response.body as any).name).toEqual(testPackageName);
expect((response.body as any).versions).toHaveProperty(testVersion);
expect((response.body as any)['dist-tags'].latest).toEqual(testVersion);
const body = await streamToJson(response.body);
expect(body).toHaveProperty('name');
expect(body.name).toEqual(testPackageName);
expect(body.versions).toHaveProperty(testVersion);
expect(body['dist-tags'].latest).toEqual(testVersion);
});
tap.test('NPM: should retrieve specific version metadata (GET /{package}/{version})', async () => {
@@ -81,9 +85,10 @@ tap.test('NPM: should retrieve specific version metadata (GET /{package}/{versio
});
expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('version');
expect((response.body as any).version).toEqual(testVersion);
expect((response.body as any).name).toEqual(testPackageName);
const body = await streamToJson(response.body);
expect(body).toHaveProperty('version');
expect(body.version).toEqual(testVersion);
expect(body.name).toEqual(testPackageName);
});
tap.test('NPM: should download tarball (GET /{package}/-/{tarball})', async () => {
@@ -95,8 +100,9 @@ tap.test('NPM: should download tarball (GET /{package}/-/{tarball})', async () =
});
expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer);
expect((response.body as Buffer).toString('utf-8')).toEqual('fake tarball content');
const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
expect(body.toString('utf-8')).toEqual('fake tarball content');
expect(response.headers['Content-Type']).toEqual('application/octet-stream');
});
@@ -127,7 +133,8 @@ tap.test('NPM: should publish a new version of the package', async () => {
});
expect(getResponse.status).toEqual(200);
expect((getResponse.body as any).versions).toHaveProperty(newVersion);
const getBody = await streamToJson(getResponse.body);
expect(getBody.versions).toHaveProperty(newVersion);
});
tap.test('NPM: should get dist-tags (GET /-/package/{pkg}/dist-tags)', async () => {
@@ -139,8 +146,9 @@ tap.test('NPM: should get dist-tags (GET /-/package/{pkg}/dist-tags)', async ()
});
expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('latest');
expect((response.body as any).latest).toBeTypeOf('string');
const body = await streamToJson(response.body);
expect(body).toHaveProperty('latest');
expect(body.latest).toBeTypeOf('string');
});
tap.test('NPM: should update dist-tag (PUT /-/package/{pkg}/dist-tags/{tag})', async () => {
@@ -165,7 +173,8 @@ tap.test('NPM: should update dist-tag (PUT /-/package/{pkg}/dist-tags/{tag})', a
query: {},
});
expect((getResponse.body as any)['dist-tags'].beta).toEqual('1.1.0');
const getBody2 = await streamToJson(getResponse.body);
expect(getBody2['dist-tags'].beta).toEqual('1.1.0');
});
tap.test('NPM: should delete dist-tag (DELETE /-/package/{pkg}/dist-tags/{tag})', async () => {
@@ -188,7 +197,8 @@ tap.test('NPM: should delete dist-tag (DELETE /-/package/{pkg}/dist-tags/{tag})'
query: {},
});
expect((getResponse.body as any)['dist-tags']).not.toHaveProperty('beta');
const getBody3 = await streamToJson(getResponse.body);
expect(getBody3['dist-tags']).not.toHaveProperty('beta');
});
tap.test('NPM: should create a new token (POST /-/npm/v1/tokens)', async () => {
@@ -208,8 +218,9 @@ tap.test('NPM: should create a new token (POST /-/npm/v1/tokens)', async () => {
});
expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('token');
expect((response.body as any).readonly).toEqual(true);
const body = await streamToJson(response.body);
expect(body).toHaveProperty('token');
expect(body.readonly).toEqual(true);
});
tap.test('NPM: should list tokens (GET /-/npm/v1/tokens)', async () => {
@@ -223,9 +234,10 @@ tap.test('NPM: should list tokens (GET /-/npm/v1/tokens)', async () => {
});
expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('objects');
expect((response.body as any).objects).toBeInstanceOf(Array);
expect((response.body as any).objects.length).toBeGreaterThan(0);
const body = await streamToJson(response.body);
expect(body).toHaveProperty('objects');
expect(body.objects).toBeInstanceOf(Array);
expect(body.objects.length).toBeGreaterThan(0);
});
tap.test('NPM: should search packages (GET /-/v1/search)', async () => {
@@ -240,9 +252,10 @@ tap.test('NPM: should search packages (GET /-/v1/search)', async () => {
});
expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('objects');
expect((response.body as any).objects).toBeInstanceOf(Array);
expect((response.body as any).total).toBeGreaterThan(0);
const body = await streamToJson(response.body);
expect(body).toHaveProperty('objects');
expect(body.objects).toBeInstanceOf(Array);
expect(body.total).toBeGreaterThan(0);
});
tap.test('NPM: should search packages with specific query', async () => {
@@ -256,7 +269,8 @@ tap.test('NPM: should search packages with specific query', async () => {
});
expect(response.status).toEqual(200);
const results = (response.body as any).objects;
const body = await streamToJson(response.body);
const results = body.objects;
expect(results.length).toBeGreaterThan(0);
expect(results[0].package.name).toEqual(testPackageName);
});
@@ -281,7 +295,8 @@ tap.test('NPM: should unpublish a specific version (DELETE /{package}/-/{version
query: {},
});
expect((getResponse.body as any).versions).not.toHaveProperty(testVersion);
const getBody4 = await streamToJson(getResponse.body);
expect(getBody4.versions).not.toHaveProperty(testVersion);
});
tap.test('NPM: should unpublish entire package (DELETE /{package}/-rev/{rev})', async () => {
@@ -316,7 +331,8 @@ tap.test('NPM: should return 404 for non-existent package', async () => {
});
expect(response.status).toEqual(404);
expect(response.body).toHaveProperty('error');
const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
});
tap.test('NPM: should return 401 for unauthorized publish', async () => {
@@ -334,7 +350,8 @@ tap.test('NPM: should return 401 for unauthorized publish', async () => {
});
expect(response.status).toEqual(401);
expect(response.body).toHaveProperty('error');
const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
});
tap.test('NPM: should reject readonly token for write operations', async () => {

View File

@@ -0,0 +1,396 @@
/**
* Native Docker CLI Testing
* Tests the OCI registry implementation using the actual Docker CLI
*/
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { tapNodeTools } from '@git.zone/tstest/tapbundle_serverside';
import { SmartRegistry } from '../ts/index.js';
import type { IRequestContext, IResponse, IRegistryConfig } from '../ts/core/interfaces.core.js';
import * as qenv from '@push.rocks/qenv';
import * as http from 'http';
import * as url from 'url';
import * as fs from 'fs';
import * as path from 'path';
const testQenv = new qenv.Qenv('./', './.nogit');
/**
* Create a test registry with local token endpoint realm
*/
async function createDockerTestRegistry(port: number): Promise<SmartRegistry> {
const s3AccessKey = await testQenv.getEnvVarOnDemand('S3_ACCESSKEY');
const s3SecretKey = await testQenv.getEnvVarOnDemand('S3_SECRETKEY');
const s3Endpoint = await testQenv.getEnvVarOnDemand('S3_ENDPOINT');
const s3Port = await testQenv.getEnvVarOnDemand('S3_PORT');
const config: IRegistryConfig = {
storage: {
accessKey: s3AccessKey || 'minioadmin',
accessSecret: s3SecretKey || 'minioadmin',
endpoint: s3Endpoint || 'localhost',
port: parseInt(s3Port || '9000', 10),
useSsl: false,
region: 'us-east-1',
bucketName: 'test-registry',
},
auth: {
jwtSecret: 'test-secret-key',
tokenStore: 'memory',
npmTokens: {
enabled: true,
},
ociTokens: {
enabled: true,
realm: `http://localhost:${port}/v2/token`,
service: 'test-registry',
},
},
oci: {
enabled: true,
basePath: '/v2',
},
};
const reg = new SmartRegistry(config);
await reg.init();
return reg;
}
/**
* Create test tokens for the registry
*/
async function createDockerTestTokens(reg: SmartRegistry) {
const authManager = reg.getAuthManager();
const userId = await authManager.authenticate({
username: 'testuser',
password: 'testpass',
});
if (!userId) {
throw new Error('Failed to authenticate test user');
}
// Create OCI token with full access
const ociToken = await authManager.createOciToken(
userId,
['oci:repository:*:*'],
3600
);
return { ociToken, userId };
}
// Test context
let registry: SmartRegistry;
let server: http.Server;
let registryUrl: string;
let registryPort: number;
let ociToken: string;
let testDir: string;
let testImageName: string;
/**
* Create HTTP server wrapper around SmartRegistry
* CRITICAL: Always passes rawBody for content-addressable operations (OCI manifests/blobs)
*
* SmartRegistry OCI is configured with basePath '/v2' matching Docker's native /v2/ prefix.
*
* Also implements a simple /v2/token endpoint for Docker Bearer auth flow
*/
async function createHttpServer(
registryInstance: SmartRegistry,
port: number,
tokenForAuth: string
): Promise<{ server: http.Server; url: string }> {
return new Promise((resolve, reject) => {
const httpServer = http.createServer(async (req, res) => {
try {
// Parse request
const parsedUrl = url.parse(req.url || '', true);
let pathname = parsedUrl.pathname || '/';
const query = parsedUrl.query;
// Handle token endpoint for Docker Bearer auth
if (pathname === '/v2/token' || pathname === '/token') {
console.log(`[Token Request] ${req.method} ${req.url}`);
res.statusCode = 200;
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify({
token: tokenForAuth,
access_token: tokenForAuth,
expires_in: 3600,
issued_at: new Date().toISOString(),
}));
return;
}
// Log all requests for debugging
console.log(`[Registry] ${req.method} ${pathname}`);
// basePath is /v2 which matches Docker's native /v2/ prefix — no rewrite needed
// Read raw body - ALWAYS preserve exact bytes for OCI
const chunks: Buffer[] = [];
for await (const chunk of req) {
chunks.push(chunk);
}
const bodyBuffer = Buffer.concat(chunks);
// Parse body based on content type (for non-OCI protocols that need it)
let parsedBody: any;
if (bodyBuffer.length > 0) {
const contentType = req.headers['content-type'] || '';
if (contentType.includes('application/json')) {
try {
parsedBody = JSON.parse(bodyBuffer.toString('utf-8'));
} catch (error) {
parsedBody = bodyBuffer;
}
} else {
parsedBody = bodyBuffer;
}
}
// Convert to IRequestContext
const context: IRequestContext = {
method: req.method || 'GET',
path: pathname,
headers: req.headers as Record<string, string>,
query: query as Record<string, string>,
body: parsedBody,
rawBody: bodyBuffer,
};
// Handle request
const response: IResponse = await registryInstance.handleRequest(context);
console.log(`[Registry] Response: ${response.status} for ${pathname}`);
// Convert IResponse to HTTP response
res.statusCode = response.status;
// Set headers
for (const [key, value] of Object.entries(response.headers || {})) {
res.setHeader(key, value);
}
// Send body (response.body is always ReadableStream<Uint8Array> or undefined)
if (response.body) {
const { Readable } = await import('stream');
Readable.fromWeb(response.body).pipe(res);
} else {
res.end();
}
} catch (error) {
console.error('Server error:', error);
res.statusCode = 500;
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify({ error: 'INTERNAL_ERROR', message: String(error) }));
}
});
httpServer.listen(port, '0.0.0.0', () => {
const serverUrl = `http://localhost:${port}`;
resolve({ server: httpServer, url: serverUrl });
});
httpServer.on('error', reject);
});
}
/**
* Create a test Dockerfile
*/
function createTestDockerfile(targetDir: string, content?: string): string {
const dockerfilePath = path.join(targetDir, 'Dockerfile');
const dockerfileContent = content || `FROM alpine:latest
RUN echo "Hello from SmartRegistry test" > /hello.txt
CMD ["cat", "/hello.txt"]
`;
fs.writeFileSync(dockerfilePath, dockerfileContent, 'utf-8');
return dockerfilePath;
}
/**
* Run Docker command using the main Docker daemon (not rootless)
* Rootless Docker runs in its own network namespace and can't access host localhost
*
* IMPORTANT: DOCKER_HOST env var overrides --context flag, so we must unset it
* and explicitly set the socket path to use the main Docker daemon.
*/
async function runDockerCommand(
command: string,
cwd?: string
): Promise<{ stdout: string; stderr: string; exitCode: number }> {
// First unset DOCKER_HOST then set it to main Docker daemon socket
// Using both unset and export ensures we override any inherited env var
const dockerCommand = `unset DOCKER_HOST && export DOCKER_HOST=unix:///var/run/docker.sock && ${command}`;
const fullCommand = cwd ? `cd "${cwd}" && ${dockerCommand}` : dockerCommand;
try {
const result = await tapNodeTools.runCommand(fullCommand);
return {
stdout: result.stdout || '',
stderr: result.stderr || '',
exitCode: result.exitCode || 0,
};
} catch (error: any) {
return {
stdout: error.stdout || '',
stderr: error.stderr || String(error),
exitCode: error.exitCode || 1,
};
}
}
/**
* Cleanup test directory
*/
function cleanupTestDir(dir: string): void {
if (fs.existsSync(dir)) {
fs.rmSync(dir, { recursive: true, force: true });
}
}
/**
* Cleanup Docker resources
*/
async function cleanupDocker(imageName: string): Promise<void> {
await runDockerCommand(`docker rmi ${imageName} 2>/dev/null || true`);
await runDockerCommand(`docker rmi ${imageName}:v1 2>/dev/null || true`);
await runDockerCommand(`docker rmi ${imageName}:v2 2>/dev/null || true`);
}
// ========================================================================
// TESTS
// ========================================================================
tap.test('Docker CLI: should verify Docker is installed', async () => {
const result = await runDockerCommand('docker version');
console.log('Docker version output:', result.stdout.substring(0, 200));
expect(result.exitCode).toEqual(0);
});
tap.test('Docker CLI: should setup registry and HTTP server', async () => {
// Use localhost - Docker allows HTTP for localhost without any special config
registryPort = 15000 + Math.floor(Math.random() * 1000);
console.log(`Using port: ${registryPort}`);
registry = await createDockerTestRegistry(registryPort);
const tokens = await createDockerTestTokens(registry);
ociToken = tokens.ociToken;
expect(registry).toBeInstanceOf(SmartRegistry);
expect(ociToken).toBeTypeOf('string');
const serverSetup = await createHttpServer(registry, registryPort, ociToken);
server = serverSetup.server;
registryUrl = serverSetup.url;
expect(server).toBeDefined();
console.log(`Registry server started at ${registryUrl}`);
// Setup test directory
testDir = path.join(process.cwd(), '.nogit', 'test-docker-cli');
cleanupTestDir(testDir);
fs.mkdirSync(testDir, { recursive: true });
testImageName = `localhost:${registryPort}/test-image`;
});
tap.test('Docker CLI: should verify server is responding', async () => {
// Give the server a moment to fully initialize
await new Promise(resolve => setTimeout(resolve, 500));
const response = await fetch(`${registryUrl}/v2/`);
expect(response.status).toEqual(200);
console.log('OCI v2 response:', await response.json());
});
tap.test('Docker CLI: should login to registry', async () => {
const result = await runDockerCommand(
`echo "${ociToken}" | docker login localhost:${registryPort} -u testuser --password-stdin`
);
console.log('docker login output:', result.stdout);
console.log('docker login stderr:', result.stderr);
const combinedOutput = result.stdout + result.stderr;
expect(combinedOutput).toContain('Login Succeeded');
});
tap.test('Docker CLI: should build test image', async () => {
createTestDockerfile(testDir);
const result = await runDockerCommand(
`docker build -t ${testImageName}:v1 .`,
testDir
);
console.log('docker build output:', result.stdout.substring(0, 500));
expect(result.exitCode).toEqual(0);
});
tap.test('Docker CLI: should push image to registry', async () => {
// This is the critical test - if the digest mismatch bug is fixed,
// this should succeed. The manifest bytes must be preserved exactly.
const result = await runDockerCommand(`docker push ${testImageName}:v1`);
console.log('docker push output:', result.stdout);
console.log('docker push stderr:', result.stderr);
expect(result.exitCode).toEqual(0);
});
tap.test('Docker CLI: should verify manifest in registry via API', async () => {
const response = await fetch(`${registryUrl}/v2/test-image/tags/list`, {
headers: { Authorization: `Bearer ${ociToken}` },
});
expect(response.status).toEqual(200);
const tagList = await response.json();
console.log('Tags list:', tagList);
expect(tagList.name).toEqual('test-image');
expect(tagList.tags).toContain('v1');
});
tap.test('Docker CLI: should pull pushed image', async () => {
// First remove the local image
await runDockerCommand(`docker rmi ${testImageName}:v1 || true`);
const result = await runDockerCommand(`docker pull ${testImageName}:v1`);
console.log('docker pull output:', result.stdout);
expect(result.exitCode).toEqual(0);
});
tap.test('Docker CLI: should run pulled image', async () => {
const result = await runDockerCommand(`docker run --rm ${testImageName}:v1`);
console.log('docker run output:', result.stdout);
expect(result.exitCode).toEqual(0);
expect(result.stdout).toContain('Hello from SmartRegistry test');
});
tap.postTask('cleanup docker cli tests', async () => {
if (testImageName) {
await cleanupDocker(testImageName);
}
if (server) {
await new Promise<void>((resolve) => {
server.close(() => resolve());
});
}
if (testDir) {
cleanupTestDir(testDir);
}
if (registry) {
registry.destroy();
}
});
export default tap.start();

View File

@@ -1,5 +1,6 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js';
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
import { createTestRegistry, createTestTokens, calculateDigest, createTestManifest } from './helpers/registry.js';
let registry: SmartRegistry;
@@ -24,7 +25,7 @@ tap.test('OCI: should create registry instance', async () => {
tap.test('OCI: should handle version check (GET /v2/)', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/oci/v2/',
path: '/oci/',
headers: {},
query: {},
});
@@ -36,7 +37,7 @@ tap.test('OCI: should handle version check (GET /v2/)', async () => {
tap.test('OCI: should initiate blob upload (POST /v2/{name}/blobs/uploads/)', async () => {
const response = await registry.handleRequest({
method: 'POST',
path: '/oci/v2/test-repo/blobs/uploads/',
path: '/oci/test-repo/blobs/uploads/',
headers: {
Authorization: `Bearer ${ociToken}`,
},
@@ -53,7 +54,7 @@ tap.test('OCI: should upload blob in single PUT', async () => {
const response = await registry.handleRequest({
method: 'POST',
path: '/oci/v2/test-repo/blobs/uploads/',
path: '/oci/test-repo/blobs/uploads/',
headers: {
Authorization: `Bearer ${ociToken}`,
},
@@ -73,7 +74,7 @@ tap.test('OCI: should upload config blob', async () => {
const response = await registry.handleRequest({
method: 'POST',
path: '/oci/v2/test-repo/blobs/uploads/',
path: '/oci/test-repo/blobs/uploads/',
headers: {
Authorization: `Bearer ${ociToken}`,
},
@@ -90,7 +91,7 @@ tap.test('OCI: should upload config blob', async () => {
tap.test('OCI: should check if blob exists (HEAD /v2/{name}/blobs/{digest})', async () => {
const response = await registry.handleRequest({
method: 'HEAD',
path: `/oci/v2/test-repo/blobs/${testBlobDigest}`,
path: `/oci/test-repo/blobs/${testBlobDigest}`,
headers: {
Authorization: `Bearer ${ociToken}`,
},
@@ -105,7 +106,7 @@ tap.test('OCI: should check if blob exists (HEAD /v2/{name}/blobs/{digest})', as
tap.test('OCI: should retrieve blob (GET /v2/{name}/blobs/{digest})', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: `/oci/v2/test-repo/blobs/${testBlobDigest}`,
path: `/oci/test-repo/blobs/${testBlobDigest}`,
headers: {
Authorization: `Bearer ${ociToken}`,
},
@@ -113,8 +114,9 @@ tap.test('OCI: should retrieve blob (GET /v2/{name}/blobs/{digest})', async () =
});
expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer);
expect((response.body as Buffer).toString('utf-8')).toEqual('Hello from OCI test blob!');
const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
expect(body.toString('utf-8')).toEqual('Hello from OCI test blob!');
expect(response.headers['Docker-Content-Digest']).toEqual(testBlobDigest);
});
@@ -126,7 +128,7 @@ tap.test('OCI: should upload manifest (PUT /v2/{name}/manifests/{reference})', a
const response = await registry.handleRequest({
method: 'PUT',
path: '/oci/v2/test-repo/manifests/v1.0.0',
path: '/oci/test-repo/manifests/v1.0.0',
headers: {
Authorization: `Bearer ${ociToken}`,
'Content-Type': 'application/vnd.oci.image.manifest.v1+json',
@@ -143,7 +145,7 @@ tap.test('OCI: should upload manifest (PUT /v2/{name}/manifests/{reference})', a
tap.test('OCI: should retrieve manifest by tag (GET /v2/{name}/manifests/{reference})', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/oci/v2/test-repo/manifests/v1.0.0',
path: '/oci/test-repo/manifests/v1.0.0',
headers: {
Authorization: `Bearer ${ociToken}`,
Accept: 'application/vnd.oci.image.manifest.v1+json',
@@ -152,9 +154,10 @@ tap.test('OCI: should retrieve manifest by tag (GET /v2/{name}/manifests/{refere
});
expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer);
const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
const manifest = JSON.parse((response.body as Buffer).toString('utf-8'));
const manifest = JSON.parse(body.toString('utf-8'));
expect(manifest.schemaVersion).toEqual(2);
expect(manifest.config.digest).toEqual(testConfigDigest);
expect(manifest.layers[0].digest).toEqual(testBlobDigest);
@@ -163,7 +166,7 @@ tap.test('OCI: should retrieve manifest by tag (GET /v2/{name}/manifests/{refere
tap.test('OCI: should retrieve manifest by digest (GET /v2/{name}/manifests/{digest})', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: `/oci/v2/test-repo/manifests/${testManifestDigest}`,
path: `/oci/test-repo/manifests/${testManifestDigest}`,
headers: {
Authorization: `Bearer ${ociToken}`,
Accept: 'application/vnd.oci.image.manifest.v1+json',
@@ -178,7 +181,7 @@ tap.test('OCI: should retrieve manifest by digest (GET /v2/{name}/manifests/{dig
tap.test('OCI: should check if manifest exists (HEAD /v2/{name}/manifests/{reference})', async () => {
const response = await registry.handleRequest({
method: 'HEAD',
path: '/oci/v2/test-repo/manifests/v1.0.0',
path: '/oci/test-repo/manifests/v1.0.0',
headers: {
Authorization: `Bearer ${ociToken}`,
Accept: 'application/vnd.oci.image.manifest.v1+json',
@@ -193,7 +196,7 @@ tap.test('OCI: should check if manifest exists (HEAD /v2/{name}/manifests/{refer
tap.test('OCI: should list tags (GET /v2/{name}/tags/list)', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/oci/v2/test-repo/tags/list',
path: '/oci/test-repo/tags/list',
headers: {
Authorization: `Bearer ${ociToken}`,
},
@@ -201,9 +204,9 @@ tap.test('OCI: should list tags (GET /v2/{name}/tags/list)', async () => {
});
expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('tags');
const tagList = await streamToJson(response.body);
expect(tagList).toHaveProperty('tags');
const tagList = response.body as any;
expect(tagList.name).toEqual('test-repo');
expect(tagList.tags).toBeInstanceOf(Array);
expect(tagList.tags).toContain('v1.0.0');
@@ -212,7 +215,7 @@ tap.test('OCI: should list tags (GET /v2/{name}/tags/list)', async () => {
tap.test('OCI: should handle pagination for tag list', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/oci/v2/test-repo/tags/list',
path: '/oci/test-repo/tags/list',
headers: {
Authorization: `Bearer ${ociToken}`,
},
@@ -222,13 +225,14 @@ tap.test('OCI: should handle pagination for tag list', async () => {
});
expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('tags');
const body = await streamToJson(response.body);
expect(body).toHaveProperty('tags');
});
tap.test('OCI: should return 404 for non-existent blob', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/oci/v2/test-repo/blobs/sha256:0000000000000000000000000000000000000000000000000000000000000000',
path: '/oci/test-repo/blobs/sha256:0000000000000000000000000000000000000000000000000000000000000000',
headers: {
Authorization: `Bearer ${ociToken}`,
},
@@ -236,13 +240,14 @@ tap.test('OCI: should return 404 for non-existent blob', async () => {
});
expect(response.status).toEqual(404);
expect(response.body).toHaveProperty('errors');
const body = await streamToJson(response.body);
expect(body).toHaveProperty('errors');
});
tap.test('OCI: should return 404 for non-existent manifest', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/oci/v2/test-repo/manifests/non-existent-tag',
path: '/oci/test-repo/manifests/non-existent-tag',
headers: {
Authorization: `Bearer ${ociToken}`,
Accept: 'application/vnd.oci.image.manifest.v1+json',
@@ -251,13 +256,14 @@ tap.test('OCI: should return 404 for non-existent manifest', async () => {
});
expect(response.status).toEqual(404);
expect(response.body).toHaveProperty('errors');
const body = await streamToJson(response.body);
expect(body).toHaveProperty('errors');
});
tap.test('OCI: should delete manifest (DELETE /v2/{name}/manifests/{digest})', async () => {
const response = await registry.handleRequest({
method: 'DELETE',
path: `/oci/v2/test-repo/manifests/${testManifestDigest}`,
path: `/oci/test-repo/manifests/${testManifestDigest}`,
headers: {
Authorization: `Bearer ${ociToken}`,
},
@@ -270,7 +276,7 @@ tap.test('OCI: should delete manifest (DELETE /v2/{name}/manifests/{digest})', a
tap.test('OCI: should delete blob (DELETE /v2/{name}/blobs/{digest})', async () => {
const response = await registry.handleRequest({
method: 'DELETE',
path: `/oci/v2/test-repo/blobs/${testBlobDigest}`,
path: `/oci/test-repo/blobs/${testBlobDigest}`,
headers: {
Authorization: `Bearer ${ociToken}`,
},
@@ -283,7 +289,7 @@ tap.test('OCI: should delete blob (DELETE /v2/{name}/blobs/{digest})', async ()
tap.test('OCI: should handle unauthorized requests', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/oci/v2/test-repo/manifests/v1.0.0',
path: '/oci/test-repo/manifests/v1.0.0',
headers: {
// No authorization header
},

View File

@@ -0,0 +1,515 @@
/**
* Native PyPI CLI Testing
* Tests the PyPI registry implementation using pip and twine CLI tools
*/
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { tapNodeTools } from '@git.zone/tstest/tapbundle_serverside';
import { SmartRegistry } from '../ts/index.js';
import { createTestRegistry, createTestTokens, createPythonWheel, createPythonSdist, generateTestRunId } from './helpers/registry.js';
import type { IRequestContext, IResponse } from '../ts/core/interfaces.core.js';
import * as http from 'http';
import * as url from 'url';
import * as fs from 'fs';
import * as path from 'path';
// Test context
let registry: SmartRegistry;
let server: http.Server;
let registryUrl: string;
let registryPort: number;
let pypiToken: string;
let testDir: string;
let pipHome: string;
let hasPip = false;
let hasTwine = false;
// Unique test run ID to avoid conflicts between test runs
const testRunId = generateTestRunId();
const testPackageName = `test-pypi-pkg-${testRunId}`;
/**
* Create HTTP server wrapper around SmartRegistry
*/
async function createHttpServer(
registryInstance: SmartRegistry,
port: number
): Promise<{ server: http.Server; url: string }> {
return new Promise((resolve, reject) => {
const httpServer = http.createServer(async (req, res) => {
try {
// Parse request
const parsedUrl = url.parse(req.url || '', true);
const pathname = parsedUrl.pathname || '/';
const query = parsedUrl.query;
// Read body
const chunks: Buffer[] = [];
for await (const chunk of req) {
chunks.push(chunk);
}
const bodyBuffer = Buffer.concat(chunks);
// Parse body based on content type
let body: any;
if (bodyBuffer.length > 0) {
const contentType = req.headers['content-type'] || '';
if (contentType.includes('application/json')) {
try {
body = JSON.parse(bodyBuffer.toString('utf-8'));
} catch (error) {
body = bodyBuffer;
}
} else if (contentType.includes('multipart/form-data')) {
// For multipart, pass raw buffer
body = bodyBuffer;
} else {
body = bodyBuffer;
}
}
// Convert to IRequestContext
const context: IRequestContext = {
method: req.method || 'GET',
path: pathname,
headers: req.headers as Record<string, string>,
query: query as Record<string, string>,
body: body,
rawBody: bodyBuffer,
};
// Handle request
const response: IResponse = await registryInstance.handleRequest(context);
// Convert IResponse to HTTP response
res.statusCode = response.status;
// Set headers
for (const [key, value] of Object.entries(response.headers || {})) {
res.setHeader(key, value);
}
// Send body (response.body is always ReadableStream<Uint8Array> or undefined)
if (response.body) {
const { Readable } = await import('stream');
Readable.fromWeb(response.body).pipe(res);
} else {
res.end();
}
} catch (error) {
console.error('Server error:', error);
res.statusCode = 500;
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify({ error: 'INTERNAL_ERROR', message: String(error) }));
}
});
httpServer.listen(port, () => {
const serverUrl = `http://localhost:${port}`;
resolve({ server: httpServer, url: serverUrl });
});
httpServer.on('error', reject);
});
}
/**
* Setup .pypirc for twine authentication
*/
function setupPypirc(
token: string,
pipHomeArg: string,
serverUrl: string
): string {
fs.mkdirSync(pipHomeArg, { recursive: true });
const pypircContent = `[distutils]
index-servers = testpypi
[testpypi]
repository = ${serverUrl}/pypi
username = testuser
password = ${token}
`;
const pypircPath = path.join(pipHomeArg, '.pypirc');
fs.writeFileSync(pypircPath, pypircContent, 'utf-8');
// Set restrictive permissions
fs.chmodSync(pypircPath, 0o600);
return pypircPath;
}
/**
* Setup pip.conf for pip to use our registry
*/
function setupPipConf(
token: string,
pipHomeArg: string,
serverUrl: string,
port: number
): string {
fs.mkdirSync(pipHomeArg, { recursive: true });
// pip.conf with authentication
const pipConfContent = `[global]
index-url = ${serverUrl}/pypi/simple/
trusted-host = localhost
extra-index-url = https://pypi.org/simple/
`;
const pipDir = path.join(pipHomeArg, 'pip');
fs.mkdirSync(pipDir, { recursive: true });
const pipConfPath = path.join(pipDir, 'pip.conf');
fs.writeFileSync(pipConfPath, pipConfContent, 'utf-8');
return pipConfPath;
}
/**
* Create a test Python package wheel file
*/
async function createTestWheelFile(
packageName: string,
version: string,
targetDir: string
): Promise<string> {
const wheelData = await createPythonWheel(packageName, version);
const normalizedName = packageName.replace(/-/g, '_');
const wheelFilename = `${normalizedName}-${version}-py3-none-any.whl`;
const wheelPath = path.join(targetDir, wheelFilename);
fs.writeFileSync(wheelPath, wheelData);
return wheelPath;
}
/**
* Create a test Python package sdist file
*/
async function createTestSdistFile(
packageName: string,
version: string,
targetDir: string
): Promise<string> {
const sdistData = await createPythonSdist(packageName, version);
const sdistFilename = `${packageName}-${version}.tar.gz`;
const sdistPath = path.join(targetDir, sdistFilename);
fs.writeFileSync(sdistPath, sdistData);
return sdistPath;
}
/**
* Run pip command with custom config
*/
async function runPipCommand(
command: string,
cwd: string
): Promise<{ stdout: string; stderr: string; exitCode: number }> {
const pipConfDir = path.join(pipHome, 'pip');
const fullCommand = `cd "${cwd}" && PIP_CONFIG_FILE="${path.join(pipConfDir, 'pip.conf')}" pip ${command}`;
try {
const result = await tapNodeTools.runCommand(fullCommand);
return {
stdout: result.stdout || '',
stderr: result.stderr || '',
exitCode: result.exitCode || 0,
};
} catch (error: any) {
return {
stdout: error.stdout || '',
stderr: error.stderr || String(error),
exitCode: error.exitCode || 1,
};
}
}
/**
* Run twine command with custom config
*/
async function runTwineCommand(
command: string,
cwd: string
): Promise<{ stdout: string; stderr: string; exitCode: number }> {
const pypircPath = path.join(pipHome, '.pypirc');
const fullCommand = `cd "${cwd}" && twine ${command} --config-file "${pypircPath}"`;
try {
const result = await tapNodeTools.runCommand(fullCommand);
return {
stdout: result.stdout || '',
stderr: result.stderr || '',
exitCode: result.exitCode || 0,
};
} catch (error: any) {
return {
stdout: error.stdout || '',
stderr: error.stderr || String(error),
exitCode: error.exitCode || 1,
};
}
}
/**
* Cleanup test directory
*/
function cleanupTestDir(dir: string): void {
if (fs.existsSync(dir)) {
fs.rmSync(dir, { recursive: true, force: true });
}
}
// ========================================================================
// TESTS
// ========================================================================
tap.test('PyPI CLI: should verify pip is installed', async () => {
try {
const result = await tapNodeTools.runCommand('pip --version');
console.log('pip version output:', result.stdout.substring(0, 200));
hasPip = result.exitCode === 0;
expect(result.exitCode).toEqual(0);
} catch (error) {
console.log('pip CLI not available');
}
});
tap.test('PyPI CLI: should verify twine is installed', async () => {
try {
const result = await tapNodeTools.runCommand('twine --version');
console.log('twine version output:', result.stdout.substring(0, 200));
hasTwine = result.exitCode === 0;
expect(result.exitCode).toEqual(0);
} catch (error) {
console.log('twine CLI not available');
}
if (!hasPip && !hasTwine) {
console.log('Neither pip nor twine available, skipping native CLI tests');
tap.skip.test('PyPI CLI: remaining tests skipped - no CLI tools available');
return;
}
});
tap.test('PyPI CLI: should setup registry and HTTP server', async () => {
// Use port 39000 (avoids conflicts with other tests)
registryPort = 39000;
// Create registry with correct registryUrl for CLI tests
registry = await createTestRegistry({ registryUrl: `http://localhost:${registryPort}` });
const tokens = await createTestTokens(registry);
pypiToken = tokens.pypiToken;
expect(registry).toBeInstanceOf(SmartRegistry);
expect(pypiToken).toBeTypeOf('string');
const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server;
registryUrl = serverSetup.url;
expect(server).toBeDefined();
expect(registryUrl).toEqual(`http://localhost:${registryPort}`);
// Setup test directory
testDir = path.join(process.cwd(), '.nogit', 'test-pypi-cli');
cleanupTestDir(testDir);
fs.mkdirSync(testDir, { recursive: true });
// Setup pip/pypi home directory
pipHome = path.join(testDir, '.pip');
fs.mkdirSync(pipHome, { recursive: true });
// Setup .pypirc for twine
const pypircPath = setupPypirc(pypiToken, pipHome, registryUrl);
expect(fs.existsSync(pypircPath)).toEqual(true);
// Setup pip.conf
const pipConfPath = setupPipConf(pypiToken, pipHome, registryUrl, registryPort);
expect(fs.existsSync(pipConfPath)).toEqual(true);
});
tap.test('PyPI CLI: should verify server is responding', async () => {
// Check server is up by doing a direct HTTP request to simple index
const response = await fetch(`${registryUrl}/pypi/simple/`);
expect(response.status).toBeGreaterThanOrEqual(200);
expect(response.status).toBeLessThan(500);
});
tap.test('PyPI CLI: should upload wheel with twine', async () => {
if (!hasTwine) {
console.log('Skipping twine test - twine not available');
return;
}
const version = '1.0.0';
const wheelPath = await createTestWheelFile(testPackageName, version, testDir);
expect(fs.existsSync(wheelPath)).toEqual(true);
const result = await runTwineCommand(
`upload --repository testpypi "${wheelPath}"`,
testDir
);
console.log('twine upload output:', result.stdout);
console.log('twine upload stderr:', result.stderr);
expect(result.exitCode).toEqual(0);
});
tap.test('PyPI CLI: should verify package in simple index', async () => {
if (!hasTwine) {
console.log('Skipping - twine not available');
return;
}
const response = await fetch(`${registryUrl}/pypi/simple/${testPackageName}/`);
expect(response.status).toEqual(200);
const html = await response.text();
expect(html).toContain('1.0.0');
});
tap.test('PyPI CLI: should upload sdist with twine', async () => {
if (!hasTwine) {
console.log('Skipping twine test - twine not available');
return;
}
const version = '1.1.0';
const sdistPath = await createTestSdistFile(testPackageName, version, testDir);
expect(fs.existsSync(sdistPath)).toEqual(true);
const result = await runTwineCommand(
`upload --repository testpypi "${sdistPath}"`,
testDir
);
console.log('twine upload sdist output:', result.stdout);
console.log('twine upload sdist stderr:', result.stderr);
expect(result.exitCode).toEqual(0);
});
tap.test('PyPI CLI: should list all versions in simple index', async () => {
if (!hasTwine) {
console.log('Skipping - twine not available');
return;
}
const response = await fetch(`${registryUrl}/pypi/simple/${testPackageName}/`);
expect(response.status).toEqual(200);
const html = await response.text();
expect(html).toContain('1.0.0');
expect(html).toContain('1.1.0');
});
tap.test('PyPI CLI: should get JSON metadata', async () => {
if (!hasTwine) {
console.log('Skipping - twine not available');
return;
}
const response = await fetch(`${registryUrl}/pypi/pypi/${testPackageName}/json`);
expect(response.status).toEqual(200);
const metadata = await response.json();
expect(metadata.info).toBeDefined();
expect(metadata.info.name).toEqual(testPackageName);
expect(metadata.releases).toBeDefined();
expect(metadata.releases['1.0.0']).toBeDefined();
});
tap.test('PyPI CLI: should download package with pip', async () => {
if (!hasPip || !hasTwine) {
console.log('Skipping pip download test - pip or twine not available');
return;
}
const downloadDir = path.join(testDir, 'downloads');
fs.mkdirSync(downloadDir, { recursive: true });
// Download (not install) the package
const result = await runPipCommand(
`download ${testPackageName}==1.0.0 --dest "${downloadDir}" --no-deps`,
testDir
);
console.log('pip download output:', result.stdout);
console.log('pip download stderr:', result.stderr);
// pip download may fail if the package doesn't meet pip's requirements
// Just check it doesn't crash
expect(result.exitCode).toBeLessThanOrEqual(1);
});
tap.test('PyPI CLI: should search for packages via API', async () => {
if (!hasTwine) {
console.log('Skipping - twine not available (no packages uploaded)');
return;
}
// Use the JSON API to search/list
const response = await fetch(`${registryUrl}/pypi/pypi/${testPackageName}/json`);
expect(response.status).toEqual(200);
const metadata = await response.json();
expect(metadata.info.name).toEqual(testPackageName);
});
tap.test('PyPI CLI: should fail upload without auth', async () => {
if (!hasTwine) {
console.log('Skipping twine test - twine not available');
return;
}
const unauthPkgName = `unauth-pkg-${testRunId}`;
const version = '1.0.0';
const wheelPath = await createTestWheelFile(unauthPkgName, version, testDir);
// Create a pypirc without proper credentials
const badPypircPath = path.join(testDir, '.bad-pypirc');
fs.writeFileSync(badPypircPath, `[distutils]
index-servers = badpypi
[badpypi]
repository = ${registryUrl}/pypi
username = baduser
password = badtoken
`, 'utf-8');
const fullCommand = `cd "${testDir}" && twine upload --repository badpypi "${wheelPath}" --config-file "${badPypircPath}"`;
try {
const result = await tapNodeTools.runCommand(fullCommand);
// Should fail
expect(result.exitCode).not.toEqual(0);
} catch (error: any) {
// Expected to fail
expect(error.exitCode || 1).not.toEqual(0);
}
});
tap.postTask('cleanup pypi cli tests', async () => {
// Stop server
if (server) {
await new Promise<void>((resolve) => {
server.close(() => resolve());
});
}
// Cleanup test directory
if (testDir) {
cleanupTestDir(testDir);
}
// Destroy registry
if (registry) {
registry.destroy();
}
});
export default tap.start();

485
test/test.pypi.ts Normal file
View File

@@ -0,0 +1,485 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js';
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
import {
createTestRegistry,
createTestTokens,
createPythonWheel,
createPythonSdist,
calculatePypiHashes,
} from './helpers/registry.js';
import { normalizePypiPackageName } from '../ts/pypi/helpers.pypi.js';
let registry: SmartRegistry;
let pypiToken: string;
let userId: string;
// Test data
const testPackageName = 'test-package';
const normalizedPackageName = normalizePypiPackageName(testPackageName);
const testVersion = '1.0.0';
let testWheelData: Buffer;
let testSdistData: Buffer;
tap.test('PyPI: should create registry instance', async () => {
registry = await createTestRegistry();
const tokens = await createTestTokens(registry);
pypiToken = tokens.pypiToken;
userId = tokens.userId;
expect(registry).toBeInstanceOf(SmartRegistry);
expect(pypiToken).toBeTypeOf('string');
// Clean up any existing metadata from previous test runs
const storage = registry.getStorage();
try {
await storage.deletePypiPackage(normalizedPackageName);
} catch (error) {
// Ignore error if package doesn't exist
}
});
tap.test('PyPI: should create test package files', async () => {
testWheelData = await createPythonWheel(testPackageName, testVersion);
testSdistData = await createPythonSdist(testPackageName, testVersion);
expect(testWheelData).toBeInstanceOf(Buffer);
expect(testWheelData.length).toBeGreaterThan(0);
expect(testSdistData).toBeInstanceOf(Buffer);
expect(testSdistData.length).toBeGreaterThan(0);
});
tap.test('PyPI: should upload wheel file (POST /pypi/)', async () => {
const hashes = calculatePypiHashes(testWheelData);
const filename = `${testPackageName.replace(/-/g, '_')}-${testVersion}-py3-none-any.whl`;
const formData = new FormData();
formData.append(':action', 'file_upload');
formData.append('protocol_version', '1');
formData.append('name', testPackageName);
formData.append('version', testVersion);
formData.append('filetype', 'bdist_wheel');
formData.append('pyversion', 'py3');
formData.append('metadata_version', '2.1');
formData.append('sha256_digest', hashes.sha256);
formData.append('content', new Blob([testWheelData]), filename);
const response = await registry.handleRequest({
method: 'POST',
path: '/pypi/',
headers: {
Authorization: `Bearer ${pypiToken}`,
'Content-Type': 'multipart/form-data',
},
query: {},
body: {
':action': 'file_upload',
protocol_version: '1',
name: testPackageName,
version: testVersion,
filetype: 'bdist_wheel',
pyversion: 'py3',
metadata_version: '2.1',
sha256_digest: hashes.sha256,
requires_python: '>=3.7',
content: testWheelData,
filename: filename,
},
});
expect(response.status).toEqual(201);
});
tap.test('PyPI: should retrieve Simple API root index HTML (GET /simple/)', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/simple/',
headers: {
Accept: 'text/html',
},
query: {},
});
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toStartWith('text/html');
const body = await streamToBuffer(response.body);
const html = body.toString('utf-8');
expect(html).toBeTypeOf('string');
expect(html).toContain('<!DOCTYPE html>');
expect(html).toContain('<title>Simple Index</title>');
expect(html).toContain(normalizedPackageName);
});
tap.test('PyPI: should retrieve Simple API root index JSON (GET /simple/ with Accept: application/vnd.pypi.simple.v1+json)', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/simple/',
headers: {
Accept: 'application/vnd.pypi.simple.v1+json',
},
query: {},
});
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/vnd.pypi.simple.v1+json');
const json = await streamToJson(response.body);
expect(json).toBeTypeOf('object');
expect(json).toHaveProperty('meta');
expect(json).toHaveProperty('projects');
expect(json.projects).toBeInstanceOf(Array);
// Check that the package is in the projects list (PEP 691 format: array of { name } objects)
const packageNames = json.projects.map((p: any) => p.name);
expect(packageNames).toContain(normalizedPackageName);
});
tap.test('PyPI: should retrieve Simple API package HTML (GET /simple/{package}/)', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: `/simple/${normalizedPackageName}/`,
headers: {
Accept: 'text/html',
},
query: {},
});
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toStartWith('text/html');
const body = await streamToBuffer(response.body);
const html = body.toString('utf-8');
expect(html).toBeTypeOf('string');
expect(html).toContain('<!DOCTYPE html>');
expect(html).toContain(`<title>Links for ${normalizedPackageName}</title>`);
expect(html).toContain('.whl');
expect(html).toContain('data-requires-python');
});
tap.test('PyPI: should retrieve Simple API package JSON (GET /simple/{package}/ with Accept: application/vnd.pypi.simple.v1+json)', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: `/simple/${normalizedPackageName}/`,
headers: {
Accept: 'application/vnd.pypi.simple.v1+json',
},
query: {},
});
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/vnd.pypi.simple.v1+json');
const json = await streamToJson(response.body);
expect(json).toBeTypeOf('object');
expect(json).toHaveProperty('meta');
expect(json).toHaveProperty('name');
expect(json.name).toEqual(normalizedPackageName);
expect(json).toHaveProperty('files');
expect(json.files).toBeTypeOf('object');
expect(Object.keys(json.files).length).toBeGreaterThan(0);
});
tap.test('PyPI: should download wheel file (GET /pypi/packages/{package}/{filename})', async () => {
const filename = `${testPackageName.replace(/-/g, '_')}-${testVersion}-py3-none-any.whl`;
const response = await registry.handleRequest({
method: 'GET',
path: `/pypi/packages/${normalizedPackageName}/${filename}`,
headers: {},
query: {},
});
expect(response.status).toEqual(200);
const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
expect(body.length).toEqual(testWheelData.length);
expect(response.headers['Content-Type']).toEqual('application/octet-stream');
});
tap.test('PyPI: should upload sdist file (POST /pypi/)', async () => {
const hashes = calculatePypiHashes(testSdistData);
const filename = `${testPackageName}-${testVersion}.tar.gz`;
const response = await registry.handleRequest({
method: 'POST',
path: '/pypi/',
headers: {
Authorization: `Bearer ${pypiToken}`,
'Content-Type': 'multipart/form-data',
},
query: {},
body: {
':action': 'file_upload',
protocol_version: '1',
name: testPackageName,
version: testVersion,
filetype: 'sdist',
pyversion: 'source',
metadata_version: '2.1',
sha256_digest: hashes.sha256,
requires_python: '>=3.7',
content: testSdistData,
filename: filename,
},
});
expect(response.status).toEqual(201);
});
tap.test('PyPI: should list both wheel and sdist in Simple API', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: `/simple/${normalizedPackageName}/`,
headers: {
Accept: 'application/vnd.pypi.simple.v1+json',
},
query: {},
});
expect(response.status).toEqual(200);
const json = await streamToJson(response.body);
// PEP 691: files is an array of file objects
expect(json.files.length).toEqual(2);
const hasWheel = json.files.some((f: any) => f.filename.endsWith('.whl'));
const hasSdist = json.files.some((f: any) => f.filename.endsWith('.tar.gz'));
expect(hasWheel).toEqual(true);
expect(hasSdist).toEqual(true);
});
tap.test('PyPI: should upload a second version', async () => {
const newVersion = '2.0.0';
const newWheelData = await createPythonWheel(testPackageName, newVersion);
const hashes = calculatePypiHashes(newWheelData);
const filename = `${testPackageName.replace(/-/g, '_')}-${newVersion}-py3-none-any.whl`;
const response = await registry.handleRequest({
method: 'POST',
path: '/pypi/',
headers: {
Authorization: `Bearer ${pypiToken}`,
'Content-Type': 'multipart/form-data',
},
query: {},
body: {
':action': 'file_upload',
protocol_version: '1',
name: testPackageName,
version: newVersion,
filetype: 'bdist_wheel',
pyversion: 'py3',
metadata_version: '2.1',
sha256_digest: hashes.sha256,
requires_python: '>=3.7',
content: newWheelData,
filename: filename,
},
});
expect(response.status).toEqual(201);
});
tap.test('PyPI: should list multiple versions in Simple API', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: `/simple/${normalizedPackageName}/`,
headers: {
Accept: 'application/vnd.pypi.simple.v1+json',
},
query: {},
});
expect(response.status).toEqual(200);
const json = await streamToJson(response.body);
// PEP 691: files is an array of file objects
expect(json.files.length).toBeGreaterThan(2);
const hasVersion1 = json.files.some((f: any) => f.filename.includes('1.0.0'));
const hasVersion2 = json.files.some((f: any) => f.filename.includes('2.0.0'));
expect(hasVersion1).toEqual(true);
expect(hasVersion2).toEqual(true);
});
tap.test('PyPI: should normalize package names correctly', async () => {
const testNames = [
{ input: 'Test-Package', expected: 'test-package' },
{ input: 'Test_Package', expected: 'test-package' },
{ input: 'Test..Package', expected: 'test-package' },
{ input: 'Test---Package', expected: 'test-package' },
];
for (const { input, expected } of testNames) {
const normalized = normalizePypiPackageName(input);
expect(normalized).toEqual(expected);
}
});
tap.test('PyPI: should return 404 for non-existent package', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/simple/nonexistent-package/',
headers: {},
query: {},
});
expect(response.status).toEqual(404);
const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
});
tap.test('PyPI: should return 401 for unauthorized upload', async () => {
const wheelData = await createPythonWheel('unauthorized-test', '1.0.0');
const hashes = calculatePypiHashes(wheelData);
const response = await registry.handleRequest({
method: 'POST',
path: '/pypi/',
headers: {
// No authorization header
'Content-Type': 'multipart/form-data',
},
query: {},
body: {
':action': 'file_upload',
protocol_version: '1',
name: 'unauthorized-test',
version: '1.0.0',
filetype: 'bdist_wheel',
pyversion: 'py3',
metadata_version: '2.1',
sha256_digest: hashes.sha256,
content: wheelData,
filename: 'unauthorized_test-1.0.0-py3-none-any.whl',
},
});
expect(response.status).toEqual(401);
const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
});
tap.test('PyPI: should reject upload with mismatched hash', async () => {
const wheelData = await createPythonWheel('hash-test', '1.0.0');
const response = await registry.handleRequest({
method: 'POST',
path: '/pypi/',
headers: {
Authorization: `Bearer ${pypiToken}`,
'Content-Type': 'multipart/form-data',
},
query: {},
body: {
':action': 'file_upload',
protocol_version: '1',
name: 'hash-test',
version: '1.0.0',
filetype: 'bdist_wheel',
pyversion: 'py3',
metadata_version: '2.1',
sha256_digest: 'wrong_hash_value',
content: wheelData,
filename: 'hash_test-1.0.0-py3-none-any.whl',
},
});
expect(response.status).toEqual(400);
const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
});
tap.test('PyPI: should handle package with requires-python metadata', async () => {
const packageName = 'python-version-test';
const wheelData = await createPythonWheel(packageName, '1.0.0');
const hashes = calculatePypiHashes(wheelData);
const response = await registry.handleRequest({
method: 'POST',
path: '/pypi/',
headers: {
Authorization: `Bearer ${pypiToken}`,
'Content-Type': 'multipart/form-data',
},
query: {},
body: {
':action': 'file_upload',
protocol_version: '1',
name: packageName,
version: '1.0.0',
filetype: 'bdist_wheel',
pyversion: 'py3',
metadata_version: '2.1',
sha256_digest: hashes.sha256,
'requires_python': '>=3.8',
content: wheelData,
filename: `${packageName.replace(/-/g, '_')}-1.0.0-py3-none-any.whl`,
},
});
expect(response.status).toEqual(201);
// Verify requires-python is in Simple API
const getResponse = await registry.handleRequest({
method: 'GET',
path: `/simple/${normalizePypiPackageName(packageName)}/`,
headers: {
Accept: 'text/html',
},
query: {},
});
const getBody = await streamToBuffer(getResponse.body);
const html = getBody.toString('utf-8');
expect(html).toContain('data-requires-python');
// Note: >= gets HTML-escaped to &gt;= in attribute values
expect(html).toContain('&gt;=3.8');
});
tap.test('PyPI: should support JSON API for package metadata', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: `/pypi/${normalizedPackageName}/json`,
headers: {},
query: {},
});
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/json');
const json = await streamToJson(response.body);
expect(json).toBeTypeOf('object');
expect(json).toHaveProperty('info');
expect(json.info).toHaveProperty('name');
expect(json.info.name).toEqual(normalizedPackageName);
expect(json).toHaveProperty('urls');
});
tap.test('PyPI: should support JSON API for specific version', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: `/pypi/${normalizedPackageName}/${testVersion}/json`,
headers: {},
query: {},
});
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/json');
const json = await streamToJson(response.body);
expect(json).toBeTypeOf('object');
expect(json).toHaveProperty('info');
expect(json.info.version).toEqual(testVersion);
expect(json).toHaveProperty('urls');
});
tap.postTask('cleanup registry', async () => {
if (registry) {
registry.destroy();
}
});
export default tap.start();

View File

@@ -0,0 +1,449 @@
/**
* Native gem CLI Testing
* Tests the RubyGems registry implementation using the actual gem CLI
*/
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { tapNodeTools } from '@git.zone/tstest/tapbundle_serverside';
import { SmartRegistry } from '../ts/index.js';
import { createTestRegistry, createTestTokens, createRubyGem } from './helpers/registry.js';
import type { IRequestContext, IResponse } from '../ts/core/interfaces.core.js';
import * as http from 'http';
import * as url from 'url';
import * as fs from 'fs';
import * as path from 'path';
// Test context
let registry: SmartRegistry;
let server: http.Server;
let registryUrl: string;
let registryPort: number;
let rubygemsToken: string;
let testDir: string;
let gemHome: string;
/**
* Create HTTP server wrapper around SmartRegistry
*/
async function createHttpServer(
registryInstance: SmartRegistry,
port: number
): Promise<{ server: http.Server; url: string }> {
return new Promise((resolve, reject) => {
const httpServer = http.createServer(async (req, res) => {
try {
// Parse request
const parsedUrl = url.parse(req.url || '', true);
const pathname = parsedUrl.pathname || '/';
const query = parsedUrl.query;
// Read body
const chunks: Buffer[] = [];
for await (const chunk of req) {
chunks.push(chunk);
}
const bodyBuffer = Buffer.concat(chunks);
// Parse body based on content type
let body: any;
if (bodyBuffer.length > 0) {
const contentType = req.headers['content-type'] || '';
if (contentType.includes('application/json')) {
try {
body = JSON.parse(bodyBuffer.toString('utf-8'));
} catch (error) {
body = bodyBuffer;
}
} else {
body = bodyBuffer;
}
}
// Convert to IRequestContext
const context: IRequestContext = {
method: req.method || 'GET',
path: pathname,
headers: req.headers as Record<string, string>,
query: query as Record<string, string>,
body: body,
};
// Handle request
const response: IResponse = await registryInstance.handleRequest(context);
// Convert IResponse to HTTP response
res.statusCode = response.status;
// Set headers
for (const [key, value] of Object.entries(response.headers || {})) {
res.setHeader(key, value);
}
// Send body (response.body is always ReadableStream<Uint8Array> or undefined)
if (response.body) {
const { Readable } = await import('stream');
Readable.fromWeb(response.body).pipe(res);
} else {
res.end();
}
} catch (error) {
console.error('Server error:', error);
res.statusCode = 500;
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify({ error: 'INTERNAL_ERROR', message: String(error) }));
}
});
httpServer.listen(port, () => {
const serverUrl = `http://localhost:${port}`;
resolve({ server: httpServer, url: serverUrl });
});
httpServer.on('error', reject);
});
}
/**
* Setup gem credentials file
* Format: YAML with :rubygems_api_key: TOKEN
*/
function setupGemCredentials(token: string, gemHomeArg: string): string {
const gemDir = path.join(gemHomeArg, '.gem');
fs.mkdirSync(gemDir, { recursive: true });
// Create credentials file in YAML format
const credentialsContent = `:rubygems_api_key: ${token}\n`;
const credentialsPath = path.join(gemDir, 'credentials');
fs.writeFileSync(credentialsPath, credentialsContent, 'utf-8');
// Set restrictive permissions (gem requires 0600)
fs.chmodSync(credentialsPath, 0o600);
return credentialsPath;
}
/**
* Create a test gem file
*/
async function createTestGemFile(
gemName: string,
version: string,
targetDir: string
): Promise<string> {
const gemData = await createRubyGem(gemName, version);
const gemFilename = `${gemName}-${version}.gem`;
const gemPath = path.join(targetDir, gemFilename);
fs.writeFileSync(gemPath, gemData);
return gemPath;
}
/**
* Run gem command with proper environment
*/
async function runGemCommand(
command: string,
cwd: string,
includeAuth: boolean = true
): Promise<{ stdout: string; stderr: string; exitCode: number }> {
// When not including auth, use a temp HOME without credentials
const effectiveHome = includeAuth ? gemHome : path.join(gemHome, 'noauth');
if (!includeAuth) {
fs.mkdirSync(effectiveHome, { recursive: true });
}
// Prepare environment variables
const envVars = [
`HOME="${effectiveHome}"`,
`GEM_HOME="${gemHome}"`,
].filter(Boolean).join(' ');
// Build command with cd to correct directory and environment variables
const fullCommand = `cd "${cwd}" && ${envVars} ${command}`;
try {
const result = await tapNodeTools.runCommand(fullCommand);
return {
stdout: result.stdout || '',
stderr: result.stderr || '',
exitCode: result.exitCode || 0,
};
} catch (error: any) {
return {
stdout: error.stdout || '',
stderr: error.stderr || String(error),
exitCode: error.exitCode || 1,
};
}
}
/**
* Cleanup test directory
*/
function cleanupTestDir(dir: string): void {
if (fs.existsSync(dir)) {
fs.rmSync(dir, { recursive: true, force: true });
}
}
// ========================================================================
// TESTS
// ========================================================================
tap.test('RubyGems CLI: should setup registry and HTTP server', async () => {
// Use port 36000 (avoids npm:35000, cargo:5000 conflicts)
registryPort = 36000;
// Create registry with correct registryUrl for CLI tests
registry = await createTestRegistry({ registryUrl: `http://localhost:${registryPort}` });
const tokens = await createTestTokens(registry);
rubygemsToken = tokens.rubygemsToken;
expect(registry).toBeInstanceOf(SmartRegistry);
expect(rubygemsToken).toBeTypeOf('string');
const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server;
registryUrl = serverSetup.url;
expect(server).toBeDefined();
expect(registryUrl).toEqual(`http://localhost:${registryPort}`);
// Setup test directory
testDir = path.join(process.cwd(), '.nogit', 'test-rubygems-cli');
cleanupTestDir(testDir);
fs.mkdirSync(testDir, { recursive: true });
// Setup GEM_HOME
gemHome = path.join(testDir, '.gem-home');
fs.mkdirSync(gemHome, { recursive: true });
// Setup gem credentials
const credentialsPath = setupGemCredentials(rubygemsToken, gemHome);
expect(fs.existsSync(credentialsPath)).toEqual(true);
// Verify credentials file has correct permissions
const stats = fs.statSync(credentialsPath);
const mode = stats.mode & 0o777;
expect(mode).toEqual(0o600);
});
tap.test('RubyGems CLI: should verify server is responding', async () => {
// Check server is up by doing a direct HTTP request to the Compact Index
const response = await fetch(`${registryUrl}/rubygems/versions`);
expect(response.status).toBeGreaterThanOrEqual(200);
expect(response.status).toBeLessThan(500);
});
tap.test('RubyGems CLI: should build and push a gem', async () => {
const gemName = 'test-gem-cli';
const version = '1.0.0';
const gemPath = await createTestGemFile(gemName, version, testDir);
expect(fs.existsSync(gemPath)).toEqual(true);
const result = await runGemCommand(
`gem push ${gemPath} --host ${registryUrl}/rubygems`,
testDir
);
console.log('gem push output:', result.stdout);
console.log('gem push stderr:', result.stderr);
expect(result.exitCode).toEqual(0);
expect(result.stdout || result.stderr).toContain(gemName);
});
tap.test('RubyGems CLI: should verify gem in Compact Index /versions', async () => {
const gemName = 'test-gem-cli';
const response = await fetch(`${registryUrl}/rubygems/versions`);
expect(response.status).toEqual(200);
const versionsData = await response.text();
console.log('Versions data:', versionsData);
// Format: GEMNAME VERSION[,VERSION...] MD5
expect(versionsData).toContain(gemName);
expect(versionsData).toContain('1.0.0');
});
tap.test('RubyGems CLI: should verify gem in Compact Index /info file', async () => {
const gemName = 'test-gem-cli';
const response = await fetch(`${registryUrl}/rubygems/info/${gemName}`);
expect(response.status).toEqual(200);
const infoData = await response.text();
console.log('Info data:', infoData);
// Format: VERSION [DEPS]|REQS
expect(infoData).toContain('1.0.0');
});
tap.test('RubyGems CLI: should download gem file', async () => {
const gemName = 'test-gem-cli';
const version = '1.0.0';
const response = await fetch(`${registryUrl}/rubygems/gems/${gemName}-${version}.gem`);
expect(response.status).toEqual(200);
const gemData = await response.arrayBuffer();
expect(gemData.byteLength).toBeGreaterThan(0);
// Verify content type
expect(response.headers.get('content-type')).toContain('application/octet-stream');
});
tap.test('RubyGems CLI: should fetch gem metadata JSON', async () => {
const gemName = 'test-gem-cli';
const response = await fetch(`${registryUrl}/rubygems/api/v1/versions/${gemName}.json`);
expect(response.status).toEqual(200);
const metadata = await response.json();
console.log('Metadata:', metadata);
expect(metadata).toBeInstanceOf(Array);
expect(metadata.length).toBeGreaterThan(0);
expect(metadata[0].number).toEqual('1.0.0');
});
tap.test('RubyGems CLI: should push second version', async () => {
const gemName = 'test-gem-cli';
const version = '2.0.0';
const gemPath = await createTestGemFile(gemName, version, testDir);
const result = await runGemCommand(
`gem push ${gemPath} --host ${registryUrl}/rubygems`,
testDir
);
console.log('gem push v2.0.0 output:', result.stdout);
expect(result.exitCode).toEqual(0);
});
tap.test('RubyGems CLI: should list all versions in /versions file', async () => {
const gemName = 'test-gem-cli';
const response = await fetch(`${registryUrl}/rubygems/versions`);
expect(response.status).toEqual(200);
const versionsData = await response.text();
console.log('All versions data:', versionsData);
// Should contain both versions
expect(versionsData).toContain(gemName);
expect(versionsData).toContain('1.0.0');
expect(versionsData).toContain('2.0.0');
});
tap.test('RubyGems CLI: should yank a version', async () => {
const gemName = 'test-gem-cli';
const version = '1.0.0';
const result = await runGemCommand(
`gem yank ${gemName} -v ${version} --host ${registryUrl}/rubygems`,
testDir
);
console.log('gem yank output:', result.stdout);
console.log('gem yank stderr:', result.stderr);
expect(result.exitCode).toEqual(0);
// Verify version is yanked in /versions file
// Yanked versions are prefixed with '-'
const response = await fetch(`${registryUrl}/rubygems/versions`);
const versionsData = await response.text();
console.log('Versions after yank:', versionsData);
// Yanked version should have '-' prefix
expect(versionsData).toContain('-1.0.0');
});
tap.test('RubyGems CLI: should unyank a version', async () => {
const gemName = 'test-gem-cli';
const version = '1.0.0';
// Use PUT /api/v1/gems/unyank via HTTP API (gem yank --undo removed in Ruby 4.0)
const response = await fetch(
`${registryUrl}/rubygems/api/v1/gems/unyank?gem_name=${gemName}&version=${version}`,
{
method: 'PUT',
headers: {
'Authorization': rubygemsToken,
},
}
);
console.log('gem unyank status:', response.status);
expect(response.status).toEqual(200);
// Verify version is not yanked in /versions file
const versionsResponse = await fetch(`${registryUrl}/rubygems/versions`);
const versionsData = await versionsResponse.text();
console.log('Versions after unyank:', versionsData);
// Should not have '-' prefix anymore
const lines = versionsData.trim().split('\n');
const gemLine = lines.find(line => line.startsWith(gemName));
if (gemLine) {
const parts = gemLine.split(' ');
const versions = parts[1];
expect(versions).toContain('1.0.0');
expect(versions).not.toContain('-1.0.0');
}
});
tap.test('RubyGems CLI: should fetch dependencies', async () => {
const gemName = 'test-gem-cli';
const response = await fetch(`${registryUrl}/rubygems/api/v1/dependencies?gems=${gemName}`);
expect(response.status).toEqual(200);
const dependencies = await response.json();
console.log('Dependencies:', dependencies);
expect(dependencies).toBeInstanceOf(Array);
});
tap.test('RubyGems CLI: should fail to push without auth', async () => {
const gemName = 'unauth-gem';
const version = '1.0.0';
const gemPath = await createTestGemFile(gemName, version, testDir);
// Run without auth
const result = await runGemCommand(
`gem push ${gemPath} --host ${registryUrl}/rubygems`,
testDir,
false
);
console.log('gem push unauth output:', result.stdout);
console.log('gem push unauth stderr:', result.stderr);
// Should fail with auth error
expect(result.exitCode).not.toEqual(0);
});
tap.postTask('cleanup rubygems cli tests', async () => {
// Stop server
if (server) {
await new Promise<void>((resolve) => {
server.close(() => resolve());
});
}
// Cleanup test directory
if (testDir) {
cleanupTestDir(testDir);
}
// Destroy registry
if (registry) {
registry.destroy();
}
});
export default tap.start();

523
test/test.rubygems.ts Normal file
View File

@@ -0,0 +1,523 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js';
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
import {
createTestRegistry,
createTestTokens,
createRubyGem,
calculateRubyGemsChecksums,
} from './helpers/registry.js';
let registry: SmartRegistry;
let rubygemsToken: string;
let userId: string;
// Test data
const testGemName = 'test-gem';
const testVersion = '1.0.0';
let testGemData: Buffer;
tap.test('RubyGems: should create registry instance', async () => {
registry = await createTestRegistry();
const tokens = await createTestTokens(registry);
rubygemsToken = tokens.rubygemsToken;
userId = tokens.userId;
expect(registry).toBeInstanceOf(SmartRegistry);
expect(rubygemsToken).toBeTypeOf('string');
// Clean up any existing metadata from previous test runs
const storage = registry.getStorage();
try {
await storage.deleteRubyGem(testGemName);
} catch (error) {
// Ignore error if gem doesn't exist
}
});
tap.test('RubyGems: should create test gem file', async () => {
testGemData = await createRubyGem(testGemName, testVersion);
expect(testGemData).toBeInstanceOf(Buffer);
expect(testGemData.length).toBeGreaterThan(0);
});
tap.test('RubyGems: should upload gem file (POST /rubygems/api/v1/gems)', async () => {
const response = await registry.handleRequest({
method: 'POST',
path: '/rubygems/api/v1/gems',
headers: {
Authorization: rubygemsToken,
'Content-Type': 'application/octet-stream',
},
query: {},
body: testGemData,
});
expect(response.status).toEqual(201);
const body = await streamToJson(response.body);
expect(body).toHaveProperty('message');
});
tap.test('RubyGems: should retrieve Compact Index versions file (GET /rubygems/versions)', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/rubygems/versions',
headers: {},
query: {},
});
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('text/plain; charset=utf-8');
const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
const content = body.toString('utf-8');
expect(content).toContain('created_at:');
expect(content).toContain('---');
expect(content).toContain(testGemName);
expect(content).toContain(testVersion);
});
tap.test('RubyGems: should retrieve Compact Index info file (GET /rubygems/info/{gem})', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: `/rubygems/info/${testGemName}`,
headers: {},
query: {},
});
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('text/plain; charset=utf-8');
const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
const content = body.toString('utf-8');
expect(content).toContain('---');
expect(content).toContain(testVersion);
expect(content).toContain('checksum:');
});
tap.test('RubyGems: should retrieve Compact Index names file (GET /rubygems/names)', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/rubygems/names',
headers: {},
query: {},
});
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('text/plain; charset=utf-8');
const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
const content = body.toString('utf-8');
expect(content).toContain('---');
expect(content).toContain(testGemName);
});
tap.test('RubyGems: should download gem file (GET /rubygems/gems/{gem}-{version}.gem)', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: `/rubygems/gems/${testGemName}-${testVersion}.gem`,
headers: {},
query: {},
});
expect(response.status).toEqual(200);
const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
expect(body.length).toEqual(testGemData.length);
expect(response.headers['Content-Type']).toEqual('application/octet-stream');
});
tap.test('RubyGems: should upload a second version', async () => {
const newVersion = '2.0.0';
const newGemData = await createRubyGem(testGemName, newVersion);
const response = await registry.handleRequest({
method: 'POST',
path: '/rubygems/api/v1/gems',
headers: {
Authorization: rubygemsToken,
'Content-Type': 'application/octet-stream',
},
query: {},
body: newGemData,
});
expect(response.status).toEqual(201);
});
tap.test('RubyGems: should list multiple versions in Compact Index', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/rubygems/versions',
headers: {},
query: {},
});
expect(response.status).toEqual(200);
const body = await streamToBuffer(response.body);
const content = body.toString('utf-8');
const lines = content.split('\n');
const gemLine = lines.find(l => l.startsWith(`${testGemName} `));
expect(gemLine).toBeDefined();
expect(gemLine).toContain('1.0.0');
expect(gemLine).toContain('2.0.0');
});
tap.test('RubyGems: should list multiple versions in info file', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: `/rubygems/info/${testGemName}`,
headers: {},
query: {},
});
expect(response.status).toEqual(200);
const body = await streamToBuffer(response.body);
const content = body.toString('utf-8');
expect(content).toContain('1.0.0');
expect(content).toContain('2.0.0');
});
tap.test('RubyGems: should support platform-specific gems', async () => {
const platformVersion = '1.5.0';
const platform = 'x86_64-linux';
const platformGemData = await createRubyGem(testGemName, platformVersion, platform);
const response = await registry.handleRequest({
method: 'POST',
path: '/rubygems/api/v1/gems',
headers: {
Authorization: rubygemsToken,
'Content-Type': 'application/octet-stream',
},
query: {},
body: platformGemData,
});
expect(response.status).toEqual(201);
// Verify platform is listed in versions
const versionsResponse = await registry.handleRequest({
method: 'GET',
path: '/rubygems/versions',
headers: {},
query: {},
});
const versionsBody = await streamToBuffer(versionsResponse.body);
const content = versionsBody.toString('utf-8');
const lines = content.split('\n');
const gemLine = lines.find(l => l.startsWith(`${testGemName} `));
expect(gemLine).toContain(`${platformVersion}_${platform}`);
});
tap.test('RubyGems: should yank a gem version (DELETE /rubygems/api/v1/gems/yank)', async () => {
const response = await registry.handleRequest({
method: 'DELETE',
path: '/rubygems/api/v1/gems/yank',
headers: {
Authorization: rubygemsToken,
},
query: {
gem_name: testGemName,
version: testVersion,
},
});
expect(response.status).toEqual(200);
const body = await streamToJson(response.body);
expect(body).toHaveProperty('message');
expect(body.message).toContain('yanked');
});
tap.test('RubyGems: should mark yanked version in Compact Index', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/rubygems/versions',
headers: {},
query: {},
});
expect(response.status).toEqual(200);
const body = await streamToBuffer(response.body);
const content = body.toString('utf-8');
const lines = content.split('\n');
const gemLine = lines.find(l => l.startsWith(`${testGemName} `));
// Yanked versions are prefixed with '-'
expect(gemLine).toContain(`-${testVersion}`);
});
tap.test('RubyGems: should still allow downloading yanked gem', async () => {
// Yanked gems can still be downloaded if explicitly requested
const response = await registry.handleRequest({
method: 'GET',
path: `/rubygems/gems/${testGemName}-${testVersion}.gem`,
headers: {},
query: {},
});
expect(response.status).toEqual(200);
const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
});
tap.test('RubyGems: should unyank a gem version (PUT /rubygems/api/v1/gems/unyank)', async () => {
const response = await registry.handleRequest({
method: 'PUT',
path: '/rubygems/api/v1/gems/unyank',
headers: {
Authorization: rubygemsToken,
},
query: {
gem_name: testGemName,
version: testVersion,
},
});
expect(response.status).toEqual(200);
const body = await streamToJson(response.body);
expect(body).toHaveProperty('message');
expect(body.message).toContain('unyanked');
});
tap.test('RubyGems: should remove yank marker after unyank', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/rubygems/versions',
headers: {},
query: {},
});
expect(response.status).toEqual(200);
const body = await streamToBuffer(response.body);
const content = body.toString('utf-8');
const lines = content.split('\n');
const gemLine = lines.find(l => l.startsWith(`${testGemName} `));
// After unyank, version should not have '-' prefix
const versions = gemLine!.split(' ')[1].split(',');
const version1 = versions.find(v => v.includes('1.0.0'));
expect(version1).not.toStartWith('-');
expect(version1).toContain('1.0.0');
});
tap.test('RubyGems: should retrieve versions JSON (GET /rubygems/api/v1/versions/{gem}.json)', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: `/rubygems/api/v1/versions/${testGemName}.json`,
headers: {},
query: {},
});
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/json');
const json = await streamToJson(response.body);
expect(json).toBeInstanceOf(Array);
expect(json.length).toBeGreaterThan(0);
expect(json[0]).toHaveProperty('number');
});
tap.test('RubyGems: should retrieve dependencies JSON (GET /rubygems/api/v1/dependencies)', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/rubygems/api/v1/dependencies',
headers: {},
query: {
gems: `${testGemName}`,
},
});
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/json');
const json = await streamToJson(response.body);
expect(json).toBeTypeOf('object');
expect(Array.isArray(json)).toEqual(true);
});
tap.test('RubyGems: should retrieve gem spec (GET /rubygems/quick/Marshal.4.8/{gem}-{version}.gemspec.rz)', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: `/rubygems/quick/Marshal.4.8/${testGemName}-${testVersion}.gemspec.rz`,
headers: {},
query: {},
});
expect(response.status).toEqual(200);
const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
});
tap.test('RubyGems: should support latest specs endpoint (GET /rubygems/latest_specs.4.8.gz)', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/rubygems/latest_specs.4.8.gz',
headers: {},
query: {},
});
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/octet-stream');
const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
});
tap.test('RubyGems: should support specs endpoint (GET /rubygems/specs.4.8.gz)', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/rubygems/specs.4.8.gz',
headers: {},
query: {},
});
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/octet-stream');
const body = await streamToBuffer(response.body);
expect(body).toBeInstanceOf(Buffer);
});
tap.test('RubyGems: should return 404 for non-existent gem', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/rubygems/gems/nonexistent-gem-1.0.0.gem',
headers: {},
query: {},
});
expect(response.status).toEqual(404);
const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
});
tap.test('RubyGems: should return 401 for unauthorized upload', async () => {
const gemData = await createRubyGem('unauthorized-gem', '1.0.0');
const response = await registry.handleRequest({
method: 'POST',
path: '/rubygems/api/v1/gems',
headers: {
// No authorization header
'Content-Type': 'application/octet-stream',
},
query: {},
body: gemData,
});
expect(response.status).toEqual(401);
const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
});
tap.test('RubyGems: should return 401 for unauthorized yank', async () => {
const response = await registry.handleRequest({
method: 'DELETE',
path: '/rubygems/api/v1/gems/yank',
headers: {
// No authorization header
},
query: {
gem_name: testGemName,
version: '2.0.0',
},
});
expect(response.status).toEqual(401);
const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
});
tap.test('RubyGems: should handle gem with dependencies', async () => {
const gemWithDeps = 'gem-with-deps';
const version = '1.0.0';
const gemData = await createRubyGem(gemWithDeps, version);
const response = await registry.handleRequest({
method: 'POST',
path: '/rubygems/api/v1/gems',
headers: {
Authorization: rubygemsToken,
'Content-Type': 'application/octet-stream',
},
query: {},
body: gemData,
});
expect(response.status).toEqual(201);
// Check info file contains dependency info
const infoResponse = await registry.handleRequest({
method: 'GET',
path: `/rubygems/info/${gemWithDeps}`,
headers: {},
query: {},
});
expect(infoResponse.status).toEqual(200);
const infoBody = await streamToBuffer(infoResponse.body);
const content = infoBody.toString('utf-8');
expect(content).toContain('checksum:');
});
tap.test('RubyGems: should validate gem filename format', async () => {
const invalidGemData = Buffer.from('invalid gem data');
const response = await registry.handleRequest({
method: 'POST',
path: '/rubygems/api/v1/gems',
headers: {
Authorization: rubygemsToken,
'Content-Type': 'application/octet-stream',
},
query: {},
body: invalidGemData,
});
// Should fail validation
expect(response.status).toBeGreaterThanOrEqual(400);
});
tap.test('RubyGems: should support conditional GET with ETag', async () => {
// First request to get ETag
const response1 = await registry.handleRequest({
method: 'GET',
path: '/rubygems/versions',
headers: {},
query: {},
});
const etag = response1.headers['ETag'];
expect(etag).toBeDefined();
// Second request with If-None-Match
const response2 = await registry.handleRequest({
method: 'GET',
path: '/rubygems/versions',
headers: {
'If-None-Match': etag as string,
},
query: {},
});
expect(response2.status).toEqual(304);
});
tap.postTask('cleanup registry', async () => {
if (registry) {
registry.destroy();
}
});
export default tap.start();

506
test/test.storage.hooks.ts Normal file
View File

@@ -0,0 +1,506 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import * as qenv from '@push.rocks/qenv';
import { RegistryStorage } from '../ts/core/classes.registrystorage.js';
import type { IStorageConfig } from '../ts/core/interfaces.core.js';
import type { IStorageHooks, IStorageHookContext } from '../ts/core/interfaces.storage.js';
import { createTrackingHooks, createQuotaHooks, generateTestRunId } from './helpers/registry.js';
const testQenv = new qenv.Qenv('./', './.nogit');
// ============================================================================
// Test State
// ============================================================================
let storage: RegistryStorage;
let storageConfig: IStorageConfig;
let testRunId: string;
// ============================================================================
// Setup
// ============================================================================
tap.test('setup: should create storage config', async () => {
testRunId = generateTestRunId();
const s3AccessKey = await testQenv.getEnvVarOnDemand('S3_ACCESSKEY');
const s3SecretKey = await testQenv.getEnvVarOnDemand('S3_SECRETKEY');
const s3Endpoint = await testQenv.getEnvVarOnDemand('S3_ENDPOINT');
const s3Port = await testQenv.getEnvVarOnDemand('S3_PORT');
storageConfig = {
accessKey: s3AccessKey || 'minioadmin',
accessSecret: s3SecretKey || 'minioadmin',
endpoint: s3Endpoint || 'localhost',
port: parseInt(s3Port || '9000', 10),
useSsl: false,
region: 'us-east-1',
bucketName: `test-hooks-${testRunId}`,
};
expect(storageConfig.bucketName).toBeTruthy();
});
// ============================================================================
// beforePut Hook Tests
// ============================================================================
tap.test('beforePut: should be called before storage', async () => {
const tracker = createTrackingHooks();
storage = new RegistryStorage(storageConfig, tracker.hooks);
await storage.init();
// Set context and put object
storage.setContext({
protocol: 'npm',
actor: { userId: 'testuser' },
metadata: { packageName: 'test-package' },
});
await storage.putObject('test/beforeput-called.txt', Buffer.from('test data'));
storage.clearContext();
// Verify beforePut was called
const beforePutCalls = tracker.calls.filter(c => c.method === 'beforePut');
expect(beforePutCalls.length).toEqual(1);
expect(beforePutCalls[0].context.operation).toEqual('put');
expect(beforePutCalls[0].context.key).toEqual('test/beforeput-called.txt');
expect(beforePutCalls[0].context.protocol).toEqual('npm');
});
tap.test('beforePut: returning {allowed: false} should block storage', async () => {
const tracker = createTrackingHooks({ beforePutAllowed: false });
const blockingStorage = new RegistryStorage(storageConfig, tracker.hooks);
await blockingStorage.init();
blockingStorage.setContext({
protocol: 'npm',
actor: { userId: 'testuser' },
});
let errorThrown = false;
try {
await blockingStorage.putObject('test/should-not-exist.txt', Buffer.from('blocked data'));
} catch (error) {
errorThrown = true;
expect((error as Error).message).toContain('Blocked by test');
}
blockingStorage.clearContext();
expect(errorThrown).toBeTrue();
// Verify object was NOT stored
const result = await blockingStorage.getObject('test/should-not-exist.txt');
expect(result).toBeNull();
});
// ============================================================================
// afterPut Hook Tests
// ============================================================================
tap.test('afterPut: should be called after successful storage', async () => {
const tracker = createTrackingHooks();
const trackedStorage = new RegistryStorage(storageConfig, tracker.hooks);
await trackedStorage.init();
trackedStorage.setContext({
protocol: 'maven',
actor: { userId: 'maven-user' },
});
await trackedStorage.putObject('test/afterput-test.txt', Buffer.from('after put test'));
trackedStorage.clearContext();
// Give async hook time to complete
await new Promise(resolve => setTimeout(resolve, 100));
const afterPutCalls = tracker.calls.filter(c => c.method === 'afterPut');
expect(afterPutCalls.length).toEqual(1);
expect(afterPutCalls[0].context.operation).toEqual('put');
});
tap.test('afterPut: should receive correct metadata (size, key, protocol)', async () => {
const tracker = createTrackingHooks();
const metadataStorage = new RegistryStorage(storageConfig, tracker.hooks);
await metadataStorage.init();
const testData = Buffer.from('metadata test data - some content here');
metadataStorage.setContext({
protocol: 'cargo',
actor: { userId: 'cargo-user', ip: '192.168.1.100' },
metadata: { packageName: 'my-crate', version: '1.0.0' },
});
await metadataStorage.putObject('test/metadata-test.txt', testData);
metadataStorage.clearContext();
await new Promise(resolve => setTimeout(resolve, 100));
const afterPutCalls = tracker.calls.filter(c => c.method === 'afterPut');
expect(afterPutCalls.length).toBeGreaterThanOrEqual(1);
const call = afterPutCalls[afterPutCalls.length - 1];
expect(call.context.metadata?.size).toEqual(testData.length);
expect(call.context.key).toEqual('test/metadata-test.txt');
expect(call.context.protocol).toEqual('cargo');
expect(call.context.actor?.userId).toEqual('cargo-user');
expect(call.context.actor?.ip).toEqual('192.168.1.100');
});
// ============================================================================
// beforeDelete Hook Tests
// ============================================================================
tap.test('beforeDelete: should be called before deletion', async () => {
const tracker = createTrackingHooks();
const deleteStorage = new RegistryStorage(storageConfig, tracker.hooks);
await deleteStorage.init();
// First, store an object
deleteStorage.setContext({ protocol: 'npm' });
await deleteStorage.putObject('test/to-delete.txt', Buffer.from('delete me'));
// Now delete it
await deleteStorage.deleteObject('test/to-delete.txt');
deleteStorage.clearContext();
const beforeDeleteCalls = tracker.calls.filter(c => c.method === 'beforeDelete');
expect(beforeDeleteCalls.length).toEqual(1);
expect(beforeDeleteCalls[0].context.operation).toEqual('delete');
expect(beforeDeleteCalls[0].context.key).toEqual('test/to-delete.txt');
});
tap.test('beforeDelete: returning {allowed: false} should block deletion', async () => {
const tracker = createTrackingHooks({ beforeDeleteAllowed: false });
const protectedStorage = new RegistryStorage(storageConfig, tracker.hooks);
await protectedStorage.init();
// First store an object
protectedStorage.setContext({ protocol: 'npm' });
await protectedStorage.putObject('test/protected.txt', Buffer.from('protected data'));
// Try to delete - should be blocked
let errorThrown = false;
try {
await protectedStorage.deleteObject('test/protected.txt');
} catch (error) {
errorThrown = true;
expect((error as Error).message).toContain('Blocked by test');
}
protectedStorage.clearContext();
expect(errorThrown).toBeTrue();
// Verify object still exists
const result = await protectedStorage.getObject('test/protected.txt');
expect(result).toBeTruthy();
});
// ============================================================================
// afterDelete Hook Tests
// ============================================================================
tap.test('afterDelete: should be called after successful deletion', async () => {
const tracker = createTrackingHooks();
const afterDeleteStorage = new RegistryStorage(storageConfig, tracker.hooks);
await afterDeleteStorage.init();
afterDeleteStorage.setContext({ protocol: 'pypi' });
await afterDeleteStorage.putObject('test/delete-tracked.txt', Buffer.from('to be deleted'));
await afterDeleteStorage.deleteObject('test/delete-tracked.txt');
afterDeleteStorage.clearContext();
await new Promise(resolve => setTimeout(resolve, 100));
const afterDeleteCalls = tracker.calls.filter(c => c.method === 'afterDelete');
expect(afterDeleteCalls.length).toEqual(1);
expect(afterDeleteCalls[0].context.operation).toEqual('delete');
});
// ============================================================================
// afterGet Hook Tests
// ============================================================================
tap.test('afterGet: should be called after reading object', async () => {
const tracker = createTrackingHooks();
const getStorage = new RegistryStorage(storageConfig, tracker.hooks);
await getStorage.init();
// Store an object first
getStorage.setContext({ protocol: 'rubygems' });
await getStorage.putObject('test/read-test.txt', Buffer.from('read me'));
// Clear calls to focus on the get
tracker.calls.length = 0;
// Read the object
const data = await getStorage.getObject('test/read-test.txt');
getStorage.clearContext();
await new Promise(resolve => setTimeout(resolve, 100));
expect(data).toBeTruthy();
expect(data!.toString()).toEqual('read me');
const afterGetCalls = tracker.calls.filter(c => c.method === 'afterGet');
expect(afterGetCalls.length).toEqual(1);
expect(afterGetCalls[0].context.operation).toEqual('get');
});
// ============================================================================
// Context Tests
// ============================================================================
tap.test('context: hooks should receive actor information', async () => {
const tracker = createTrackingHooks();
const actorStorage = new RegistryStorage(storageConfig, tracker.hooks);
await actorStorage.init();
actorStorage.setContext({
protocol: 'composer',
actor: {
userId: 'user-123',
tokenId: 'token-abc',
ip: '10.0.0.1',
userAgent: 'composer/2.0',
orgId: 'org-456',
sessionId: 'session-xyz',
},
});
await actorStorage.putObject('test/actor-test.txt', Buffer.from('actor test'));
actorStorage.clearContext();
const beforePutCall = tracker.calls.find(c => c.method === 'beforePut');
expect(beforePutCall).toBeTruthy();
expect(beforePutCall!.context.actor?.userId).toEqual('user-123');
expect(beforePutCall!.context.actor?.tokenId).toEqual('token-abc');
expect(beforePutCall!.context.actor?.ip).toEqual('10.0.0.1');
expect(beforePutCall!.context.actor?.userAgent).toEqual('composer/2.0');
expect(beforePutCall!.context.actor?.orgId).toEqual('org-456');
expect(beforePutCall!.context.actor?.sessionId).toEqual('session-xyz');
});
tap.test('withContext: should set and clear context correctly', async () => {
const tracker = createTrackingHooks();
const contextStorage = new RegistryStorage(storageConfig, tracker.hooks);
await contextStorage.init();
// Use withContext to ensure automatic cleanup
await contextStorage.withContext(
{
protocol: 'oci',
actor: { userId: 'oci-user' },
},
async () => {
await contextStorage.putObject('test/with-context.txt', Buffer.from('context managed'));
}
);
const call = tracker.calls.find(c => c.method === 'beforePut');
expect(call).toBeTruthy();
expect(call!.context.protocol).toEqual('oci');
expect(call!.context.actor?.userId).toEqual('oci-user');
});
tap.test('withContext: should clear context even on error', async () => {
const tracker = createTrackingHooks({ beforePutAllowed: false });
const errorStorage = new RegistryStorage(storageConfig, tracker.hooks);
await errorStorage.init();
let errorThrown = false;
try {
await errorStorage.withContext(
{
protocol: 'npm',
actor: { userId: 'error-user' },
},
async () => {
await errorStorage.putObject('test/error-context.txt', Buffer.from('will fail'));
}
);
} catch {
errorThrown = true;
}
expect(errorThrown).toBeTrue();
// Verify context was cleared - next operation without context should work
// (hooks won't be called without context)
tracker.hooks.beforePut = async () => ({ allowed: true });
await errorStorage.putObject('test/after-error.txt', Buffer.from('ok'));
});
// ============================================================================
// Graceful Degradation Tests
// ============================================================================
tap.test('graceful: hooks should not fail the operation if afterPut throws', async () => {
const tracker = createTrackingHooks({ throwOnAfterPut: true });
const gracefulStorage = new RegistryStorage(storageConfig, tracker.hooks);
await gracefulStorage.init();
gracefulStorage.setContext({ protocol: 'npm' });
// This should NOT throw even though afterPut throws
let errorThrown = false;
try {
await gracefulStorage.putObject('test/graceful-afterput.txt', Buffer.from('should succeed'));
} catch {
errorThrown = true;
}
gracefulStorage.clearContext();
expect(errorThrown).toBeFalse();
// Verify object was stored
const data = await gracefulStorage.getObject('test/graceful-afterput.txt');
expect(data).toBeTruthy();
});
tap.test('graceful: hooks should not fail the operation if afterGet throws', async () => {
const tracker = createTrackingHooks({ throwOnAfterGet: true });
const gracefulGetStorage = new RegistryStorage(storageConfig, tracker.hooks);
await gracefulGetStorage.init();
// Store first
gracefulGetStorage.setContext({ protocol: 'maven' });
await gracefulGetStorage.putObject('test/graceful-afterget.txt', Buffer.from('read me gracefully'));
// Read should succeed even though afterGet throws
let errorThrown = false;
try {
const data = await gracefulGetStorage.getObject('test/graceful-afterget.txt');
expect(data).toBeTruthy();
} catch {
errorThrown = true;
}
gracefulGetStorage.clearContext();
expect(errorThrown).toBeFalse();
});
// ============================================================================
// Quota Hooks Tests
// ============================================================================
tap.test('quota: should block storage when quota exceeded', async () => {
const maxSize = 100; // 100 bytes max
const quotaTracker = createQuotaHooks(maxSize);
const quotaStorage = new RegistryStorage(storageConfig, quotaTracker.hooks);
await quotaStorage.init();
quotaStorage.setContext({
protocol: 'npm',
actor: { userId: 'quota-user' },
});
// Store 50 bytes - should succeed
await quotaStorage.putObject('test/quota-1.txt', Buffer.from('x'.repeat(50)));
expect(quotaTracker.currentUsage.bytes).toEqual(50);
// Try to store 60 more bytes - should fail (total would be 110)
let errorThrown = false;
try {
await quotaStorage.putObject('test/quota-2.txt', Buffer.from('x'.repeat(60)));
} catch (error) {
errorThrown = true;
expect((error as Error).message).toContain('Quota exceeded');
}
quotaStorage.clearContext();
expect(errorThrown).toBeTrue();
expect(quotaTracker.currentUsage.bytes).toEqual(50); // Still 50, not 110
});
tap.test('quota: should update usage after delete', async () => {
const maxSize = 200;
const quotaTracker = createQuotaHooks(maxSize);
const quotaDelStorage = new RegistryStorage(storageConfig, quotaTracker.hooks);
await quotaDelStorage.init();
quotaDelStorage.setContext({
protocol: 'npm',
metadata: { size: 75 },
});
// Store and track
await quotaDelStorage.putObject('test/quota-del.txt', Buffer.from('x'.repeat(75)));
expect(quotaTracker.currentUsage.bytes).toEqual(75);
// Delete and verify usage decreases
await quotaDelStorage.deleteObject('test/quota-del.txt');
await new Promise(resolve => setTimeout(resolve, 100));
quotaDelStorage.clearContext();
// Usage should be reduced (though exact value depends on metadata)
expect(quotaTracker.currentUsage.bytes).toBeLessThanOrEqual(75);
});
// ============================================================================
// setHooks Tests
// ============================================================================
tap.test('setHooks: should allow setting hooks after construction', async () => {
const lateStorage = new RegistryStorage(storageConfig);
await lateStorage.init();
// Initially no hooks
await lateStorage.putObject('test/no-hooks.txt', Buffer.from('no hooks yet'));
// Add hooks later
const tracker = createTrackingHooks();
lateStorage.setHooks(tracker.hooks);
lateStorage.setContext({ protocol: 'npm' });
await lateStorage.putObject('test/with-late-hooks.txt', Buffer.from('now with hooks'));
lateStorage.clearContext();
const beforePutCalls = tracker.calls.filter(c => c.method === 'beforePut');
expect(beforePutCalls.length).toEqual(1);
});
// ============================================================================
// Cleanup
// ============================================================================
tap.test('cleanup: should clean up test bucket', async () => {
if (storage) {
// Clean up test objects
const prefixes = ['test/'];
for (const prefix of prefixes) {
try {
const objects = await storage.listObjects(prefix);
for (const obj of objects) {
await storage.deleteObject(obj);
}
} catch {
// Ignore cleanup errors
}
}
}
});
export default tap.start();

View File

@@ -1,5 +1,6 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js';
import { streamToJson } from '../ts/core/helpers.stream.js';
import { createTestRegistry, createTestTokens } from './helpers/registry.js';
let registry: SmartRegistry;
@@ -54,8 +55,9 @@ tap.test('Integration: should return 404 for unknown paths', async () => {
});
expect(response.status).toEqual(404);
expect(response.body).toHaveProperty('error');
expect((response.body as any).error).toEqual('NOT_FOUND');
const body = await streamToJson(response.body);
expect(body).toHaveProperty('error');
expect(body.error).toEqual('NOT_FOUND');
});
tap.test('Integration: should create and validate tokens', async () => {

View File

@@ -0,0 +1,598 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import * as qenv from '@push.rocks/qenv';
import * as smartbucket from '@push.rocks/smartbucket';
import { UpstreamCache } from '../ts/upstream/classes.upstreamcache.js';
import type { IUpstreamFetchContext, IUpstreamCacheConfig } from '../ts/upstream/interfaces.upstream.js';
import type { IStorageBackend } from '../ts/core/interfaces.core.js';
import { generateTestRunId } from './helpers/registry.js';
const testQenv = new qenv.Qenv('./', './.nogit');
// ============================================================================
// Test State
// ============================================================================
let cache: UpstreamCache;
let storageBackend: IStorageBackend;
let s3Bucket: smartbucket.Bucket;
let smartBucket: smartbucket.SmartBucket;
let testRunId: string;
let bucketName: string;
// ============================================================================
// Helper Functions
// ============================================================================
function createFetchContext(overrides?: Partial<IUpstreamFetchContext>): IUpstreamFetchContext {
// Use resource name as path to ensure unique cache keys
const resource = overrides?.resource || 'lodash';
return {
protocol: 'npm',
resource,
resourceType: 'packument',
path: `/${resource}`,
method: 'GET',
headers: {},
query: {},
...overrides,
};
}
// ============================================================================
// Setup
// ============================================================================
tap.test('setup: should create S3 storage backend', async () => {
testRunId = generateTestRunId();
bucketName = `test-ucache-${testRunId.substring(0, 8)}`;
const s3AccessKey = await testQenv.getEnvVarOnDemand('S3_ACCESSKEY');
const s3SecretKey = await testQenv.getEnvVarOnDemand('S3_SECRETKEY');
const s3Endpoint = await testQenv.getEnvVarOnDemand('S3_ENDPOINT');
const s3Port = await testQenv.getEnvVarOnDemand('S3_PORT');
smartBucket = new smartbucket.SmartBucket({
accessKey: s3AccessKey || 'minioadmin',
accessSecret: s3SecretKey || 'minioadmin',
endpoint: s3Endpoint || 'localhost',
port: parseInt(s3Port || '9000', 10),
useSsl: false,
});
s3Bucket = await smartBucket.createBucket(bucketName);
// Create storage backend adapter
storageBackend = {
getObject: async (key: string): Promise<Buffer | null> => {
try {
// fastGet returns Buffer directly (or undefined if not found)
const data = await s3Bucket.fastGet({ path: key });
if (!data) {
return null;
}
return data;
} catch (error) {
// fastGet throws if object doesn't exist
return null;
}
},
putObject: async (key: string, data: Buffer): Promise<void> => {
await s3Bucket.fastPut({ path: key, contents: data, overwrite: true });
},
deleteObject: async (key: string): Promise<void> => {
await s3Bucket.fastRemove({ path: key });
},
listObjects: async (prefix: string): Promise<string[]> => {
const paths: string[] = [];
for await (const path of s3Bucket.listAllObjects(prefix)) {
paths.push(path);
}
return paths;
},
};
expect(storageBackend).toBeTruthy();
});
tap.test('setup: should create UpstreamCache with S3 storage', async () => {
cache = new UpstreamCache(
{ enabled: true, defaultTtlSeconds: 300 },
10000,
storageBackend
);
expect(cache.isEnabled()).toBeTrue();
expect(cache.hasStorage()).toBeTrue();
});
// ============================================================================
// Basic Cache Operations
// ============================================================================
tap.test('cache: should store cache entry in S3', async () => {
const context = createFetchContext({ resource: 'store-test' });
const testData = Buffer.from(JSON.stringify({ name: 'store-test', version: '1.0.0' }));
const upstreamUrl = 'https://registry.npmjs.org';
await cache.set(
context,
testData,
'application/json',
{ 'etag': '"abc123"' },
'npmjs',
upstreamUrl
);
// Verify in S3
const stats = cache.getStats();
expect(stats.totalEntries).toBeGreaterThanOrEqual(1);
});
tap.test('cache: should retrieve cache entry from S3', async () => {
const context = createFetchContext({ resource: 'retrieve-test' });
const testData = Buffer.from('retrieve test data');
const upstreamUrl = 'https://registry.npmjs.org';
await cache.set(
context,
testData,
'application/octet-stream',
{},
'npmjs',
upstreamUrl
);
const entry = await cache.get(context, upstreamUrl);
expect(entry).toBeTruthy();
expect(entry!.data.toString()).toEqual('retrieve test data');
expect(entry!.contentType).toEqual('application/octet-stream');
expect(entry!.upstreamId).toEqual('npmjs');
});
// ============================================================================
// Multi-Upstream URL Tests
// ============================================================================
tap.test('cache: should include upstream URL in cache path', async () => {
const context = createFetchContext({ resource: 'url-path-test' });
const testData = Buffer.from('url path test');
await cache.set(
context,
testData,
'text/plain',
{},
'npmjs',
'https://registry.npmjs.org'
);
// The cache key should include the escaped URL
const entry = await cache.get(context, 'https://registry.npmjs.org');
expect(entry).toBeTruthy();
expect(entry!.data.toString()).toEqual('url path test');
});
tap.test('cache: should handle multiple upstreams with different URLs', async () => {
const context = createFetchContext({ resource: '@company/private-pkg' });
// Store from private upstream
const privateData = Buffer.from('private package data');
await cache.set(
context,
privateData,
'application/json',
{},
'private-npm',
'https://npm.company.com'
);
// Store from public upstream (same resource name, different upstream)
const publicData = Buffer.from('public package data');
await cache.set(
context,
publicData,
'application/json',
{},
'public-npm',
'https://registry.npmjs.org'
);
// Retrieve from private - should get private data
const privateEntry = await cache.get(context, 'https://npm.company.com');
expect(privateEntry).toBeTruthy();
expect(privateEntry!.data.toString()).toEqual('private package data');
expect(privateEntry!.upstreamId).toEqual('private-npm');
// Retrieve from public - should get public data
const publicEntry = await cache.get(context, 'https://registry.npmjs.org');
expect(publicEntry).toBeTruthy();
expect(publicEntry!.data.toString()).toEqual('public package data');
expect(publicEntry!.upstreamId).toEqual('public-npm');
});
// ============================================================================
// TTL and Expiration Tests
// ============================================================================
tap.test('cache: should respect TTL expiration', async () => {
// Create cache with very short TTL
const shortTtlCache = new UpstreamCache(
{
enabled: true,
defaultTtlSeconds: 1, // 1 second TTL
staleWhileRevalidate: false,
staleMaxAgeSeconds: 0,
immutableTtlSeconds: 1,
negativeCacheTtlSeconds: 1,
},
1000,
storageBackend
);
const context = createFetchContext({ resource: 'ttl-test' });
const testData = Buffer.from('expires soon');
await shortTtlCache.set(
context,
testData,
'text/plain',
{},
'test-upstream',
'https://test.example.com'
);
// Should exist immediately
let entry = await shortTtlCache.get(context, 'https://test.example.com');
expect(entry).toBeTruthy();
// Wait for expiration
await new Promise(resolve => setTimeout(resolve, 1500));
// Should be expired now
entry = await shortTtlCache.get(context, 'https://test.example.com');
expect(entry).toBeNull();
shortTtlCache.stop();
});
tap.test('cache: should serve stale content during stale-while-revalidate window', async () => {
const staleCache = new UpstreamCache(
{
enabled: true,
defaultTtlSeconds: 1, // 1 second fresh
staleWhileRevalidate: true,
staleMaxAgeSeconds: 60, // 60 seconds stale window
immutableTtlSeconds: 1,
negativeCacheTtlSeconds: 1,
},
1000,
storageBackend
);
const context = createFetchContext({ resource: 'stale-test' });
const testData = Buffer.from('stale but usable');
await staleCache.set(
context,
testData,
'text/plain',
{},
'stale-upstream',
'https://stale.example.com'
);
// Wait for fresh period to expire
await new Promise(resolve => setTimeout(resolve, 1500));
// Should still be available but marked as stale
const entry = await staleCache.get(context, 'https://stale.example.com');
expect(entry).toBeTruthy();
expect(entry!.stale).toBeTrue();
expect(entry!.data.toString()).toEqual('stale but usable');
staleCache.stop();
});
tap.test('cache: should reject content past stale deadline', async () => {
const veryShortCache = new UpstreamCache(
{
enabled: true,
defaultTtlSeconds: 1,
staleWhileRevalidate: true,
staleMaxAgeSeconds: 1, // Only 1 second stale window
immutableTtlSeconds: 1,
negativeCacheTtlSeconds: 1,
},
1000,
storageBackend
);
const context = createFetchContext({ resource: 'very-stale-test' });
await veryShortCache.set(
context,
Buffer.from('will expire completely'),
'text/plain',
{},
'short-upstream',
'https://short.example.com'
);
// Wait for both fresh AND stale periods to expire
await new Promise(resolve => setTimeout(resolve, 2500));
const entry = await veryShortCache.get(context, 'https://short.example.com');
expect(entry).toBeNull();
veryShortCache.stop();
});
// ============================================================================
// Negative Cache Tests
// ============================================================================
tap.test('cache: should store negative cache entries (404)', async () => {
const context = createFetchContext({ resource: 'not-found-pkg' });
const upstreamUrl = 'https://registry.npmjs.org';
await cache.setNegative(context, 'npmjs', upstreamUrl);
const hasNegative = await cache.hasNegative(context, upstreamUrl);
expect(hasNegative).toBeTrue();
});
tap.test('cache: should retrieve negative cache entries', async () => {
const context = createFetchContext({ resource: 'negative-retrieve-test' });
const upstreamUrl = 'https://registry.npmjs.org';
await cache.setNegative(context, 'npmjs', upstreamUrl);
const entry = await cache.get(context, upstreamUrl);
expect(entry).toBeTruthy();
expect(entry!.data.length).toEqual(0); // Empty buffer indicates 404
});
// ============================================================================
// Eviction Tests
// ============================================================================
tap.test('cache: should evict oldest entries when memory limit reached', async () => {
// Create cache with very small limit
const smallCache = new UpstreamCache(
{ enabled: true, defaultTtlSeconds: 300 },
5, // Only 5 entries
storageBackend
);
// Add 10 entries
for (let i = 0; i < 10; i++) {
const context = createFetchContext({ resource: `evict-test-${i}` });
await smallCache.set(
context,
Buffer.from(`data ${i}`),
'text/plain',
{},
'evict-upstream',
'https://evict.example.com'
);
}
const stats = smallCache.getStats();
// Should have evicted some entries
expect(stats.totalEntries).toBeLessThanOrEqual(5);
smallCache.stop();
});
// ============================================================================
// Query Parameter Tests
// ============================================================================
tap.test('cache: cache key should include query parameters', async () => {
const context1 = createFetchContext({
resource: 'query-test',
query: { version: '1.0.0' },
});
const context2 = createFetchContext({
resource: 'query-test',
query: { version: '2.0.0' },
});
const upstreamUrl = 'https://registry.npmjs.org';
// Store with v1 query
await cache.set(
context1,
Buffer.from('version 1 data'),
'text/plain',
{},
'query-upstream',
upstreamUrl
);
// Store with v2 query
await cache.set(
context2,
Buffer.from('version 2 data'),
'text/plain',
{},
'query-upstream',
upstreamUrl
);
// Retrieve v1 - should get v1 data
const entry1 = await cache.get(context1, upstreamUrl);
expect(entry1).toBeTruthy();
expect(entry1!.data.toString()).toEqual('version 1 data');
// Retrieve v2 - should get v2 data
const entry2 = await cache.get(context2, upstreamUrl);
expect(entry2).toBeTruthy();
expect(entry2!.data.toString()).toEqual('version 2 data');
});
// ============================================================================
// S3 Persistence Tests
// ============================================================================
tap.test('cache: should load from S3 on memory cache miss', async () => {
// Use a unique resource name for this test
const uniqueResource = `persist-test-${Date.now()}`;
const persistContext = createFetchContext({ resource: uniqueResource });
const upstreamUrl = 'https://persist.example.com';
// Store in first cache instance
await cache.set(
persistContext,
Buffer.from('persisted data'),
'text/plain',
{},
'persist-upstream',
upstreamUrl
);
// Wait for S3 write to complete
await new Promise(resolve => setTimeout(resolve, 200));
// Verify the entry is in the original cache's memory
const originalEntry = await cache.get(persistContext, upstreamUrl);
expect(originalEntry).toBeTruthy();
// Create a new cache instance (simulates restart) with SAME storage backend
const freshCache = new UpstreamCache(
{ enabled: true, defaultTtlSeconds: 300 },
10000,
storageBackend
);
// Fresh cache has empty memory, should load from S3
const entry = await freshCache.get(persistContext, upstreamUrl);
expect(entry).toBeTruthy();
expect(entry!.data.toString()).toEqual('persisted data');
freshCache.stop();
});
// ============================================================================
// Cache Stats Tests
// ============================================================================
tap.test('cache: should return accurate stats', async () => {
const statsCache = new UpstreamCache(
{ enabled: true, defaultTtlSeconds: 300 },
1000,
storageBackend
);
// Add some entries
for (let i = 0; i < 3; i++) {
const context = createFetchContext({ resource: `stats-test-${i}` });
await statsCache.set(
context,
Buffer.from(`stats data ${i}`),
'text/plain',
{},
'stats-upstream',
'https://stats.example.com'
);
}
// Add a negative entry
const negContext = createFetchContext({ resource: 'stats-negative' });
await statsCache.setNegative(negContext, 'stats-upstream', 'https://stats.example.com');
const stats = statsCache.getStats();
expect(stats.totalEntries).toBeGreaterThanOrEqual(4);
expect(stats.enabled).toBeTrue();
expect(stats.hasStorage).toBeTrue();
expect(stats.maxEntries).toEqual(1000);
statsCache.stop();
});
// ============================================================================
// Invalidation Tests
// ============================================================================
tap.test('cache: should invalidate specific cache entry', async () => {
const invalidateContext = createFetchContext({ resource: 'invalidate-test' });
const upstreamUrl = 'https://invalidate.example.com';
await cache.set(
invalidateContext,
Buffer.from('to be invalidated'),
'text/plain',
{},
'inv-upstream',
upstreamUrl
);
// Verify it exists
let entry = await cache.get(invalidateContext, upstreamUrl);
expect(entry).toBeTruthy();
// Invalidate
const deleted = await cache.invalidate(invalidateContext, upstreamUrl);
expect(deleted).toBeTrue();
// Should be gone
entry = await cache.get(invalidateContext, upstreamUrl);
expect(entry).toBeNull();
});
tap.test('cache: should invalidate entries matching pattern', async () => {
const upstreamUrl = 'https://pattern.example.com';
// Add multiple entries
for (const name of ['pattern-a', 'pattern-b', 'other-c']) {
const context = createFetchContext({ resource: name });
await cache.set(
context,
Buffer.from(`data for ${name}`),
'text/plain',
{},
'pattern-upstream',
upstreamUrl
);
}
// Invalidate entries matching 'pattern-*'
const count = await cache.invalidatePattern(/pattern-/);
expect(count).toBeGreaterThanOrEqual(2);
// pattern-a should be gone
const entryA = await cache.get(createFetchContext({ resource: 'pattern-a' }), upstreamUrl);
expect(entryA).toBeNull();
// other-c should still exist
const entryC = await cache.get(createFetchContext({ resource: 'other-c' }), upstreamUrl);
expect(entryC).toBeTruthy();
});
// ============================================================================
// Cleanup
// ============================================================================
tap.test('cleanup: should stop cache and clean up bucket', async () => {
if (cache) {
cache.stop();
}
// Clean up test bucket
if (s3Bucket) {
try {
const files = await s3Bucket.fastList({});
for (const file of files) {
await s3Bucket.fastRemove({ path: file.name });
}
await smartBucket.removeBucket(bucketName);
} catch {
// Ignore cleanup errors
}
}
});
export default tap.start();

View File

@@ -0,0 +1,343 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js';
import {
createTestRegistryWithUpstream,
createTrackingUpstreamProvider,
} from './helpers/registry.js';
import { StaticUpstreamProvider } from '../ts/upstream/interfaces.upstream.js';
import type {
IUpstreamProvider,
IUpstreamResolutionContext,
IProtocolUpstreamConfig,
} from '../ts/upstream/interfaces.upstream.js';
import type { TRegistryProtocol } from '../ts/core/interfaces.core.js';
// =============================================================================
// StaticUpstreamProvider Tests
// =============================================================================
tap.test('StaticUpstreamProvider: should return config for configured protocol', async () => {
const npmConfig: IProtocolUpstreamConfig = {
enabled: true,
upstreams: [{ id: 'npmjs', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
};
const provider = new StaticUpstreamProvider({
npm: npmConfig,
});
const result = await provider.resolveUpstreamConfig({
protocol: 'npm',
resource: 'lodash',
scope: null,
method: 'GET',
resourceType: 'packument',
});
expect(result).toBeDefined();
expect(result?.enabled).toEqual(true);
expect(result?.upstreams[0].id).toEqual('npmjs');
});
tap.test('StaticUpstreamProvider: should return null for unconfigured protocol', async () => {
const provider = new StaticUpstreamProvider({
npm: {
enabled: true,
upstreams: [{ id: 'npmjs', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
},
});
const result = await provider.resolveUpstreamConfig({
protocol: 'maven',
resource: 'com.example:lib',
scope: 'com.example',
method: 'GET',
resourceType: 'pom',
});
expect(result).toBeNull();
});
tap.test('StaticUpstreamProvider: should support multiple protocols', async () => {
const provider = new StaticUpstreamProvider({
npm: {
enabled: true,
upstreams: [{ id: 'npmjs', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
},
oci: {
enabled: true,
upstreams: [{ id: 'dockerhub', url: 'https://registry-1.docker.io', priority: 1, enabled: true }],
},
maven: {
enabled: true,
upstreams: [{ id: 'central', url: 'https://repo1.maven.org/maven2', priority: 1, enabled: true }],
},
});
const npmResult = await provider.resolveUpstreamConfig({
protocol: 'npm',
resource: 'lodash',
scope: null,
method: 'GET',
resourceType: 'packument',
});
expect(npmResult?.upstreams[0].id).toEqual('npmjs');
const ociResult = await provider.resolveUpstreamConfig({
protocol: 'oci',
resource: 'library/nginx',
scope: 'library',
method: 'GET',
resourceType: 'manifest',
});
expect(ociResult?.upstreams[0].id).toEqual('dockerhub');
const mavenResult = await provider.resolveUpstreamConfig({
protocol: 'maven',
resource: 'com.example:lib',
scope: 'com.example',
method: 'GET',
resourceType: 'pom',
});
expect(mavenResult?.upstreams[0].id).toEqual('central');
});
// =============================================================================
// Registry with Provider Integration Tests
// =============================================================================
let registry: SmartRegistry;
let trackingProvider: ReturnType<typeof createTrackingUpstreamProvider>;
tap.test('Provider Integration: should create registry with upstream provider', async () => {
trackingProvider = createTrackingUpstreamProvider({
npm: {
enabled: true,
upstreams: [{ id: 'test-npm', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
},
});
registry = await createTestRegistryWithUpstream(trackingProvider.provider);
expect(registry).toBeInstanceOf(SmartRegistry);
expect(registry.isInitialized()).toEqual(true);
});
tap.test('Provider Integration: should call provider when fetching unknown npm package', async () => {
// Clear previous calls
trackingProvider.calls.length = 0;
// Request a package that doesn't exist locally - should trigger upstream lookup
const response = await registry.handleRequest({
method: 'GET',
path: '/npm/@test-scope/nonexistent-package',
headers: {},
query: {},
});
// Provider should have been called for the packument lookup
const npmCalls = trackingProvider.calls.filter(c => c.protocol === 'npm');
// The package doesn't exist locally, so upstream should be consulted
// Note: actual upstream fetch may fail since the package doesn't exist
expect(response.status).toBeOneOf([404, 200, 502]); // 404 if not found, 502 if upstream error
});
tap.test('Provider Integration: provider receives correct context for scoped npm package', async () => {
trackingProvider.calls.length = 0;
// Use URL-encoded path for scoped packages as npm client does
await registry.handleRequest({
method: 'GET',
path: '/npm/@myorg%2fmy-package',
headers: {},
query: {},
});
// Find any npm call - the exact resource type depends on routing
const npmCalls = trackingProvider.calls.filter(c => c.protocol === 'npm');
// Provider should be called for upstream lookup
if (npmCalls.length > 0) {
const call = npmCalls[0];
expect(call.protocol).toEqual('npm');
// The resource should include the scoped name
expect(call.resource).toInclude('myorg');
expect(call.method).toEqual('GET');
}
});
tap.test('Provider Integration: provider receives correct context for unscoped npm package', async () => {
trackingProvider.calls.length = 0;
await registry.handleRequest({
method: 'GET',
path: '/npm/lodash',
headers: {},
query: {},
});
const packumentCall = trackingProvider.calls.find(
c => c.protocol === 'npm' && c.resourceType === 'packument'
);
if (packumentCall) {
expect(packumentCall.protocol).toEqual('npm');
expect(packumentCall.resource).toEqual('lodash');
expect(packumentCall.scope).toBeNull(); // No scope for unscoped package
}
});
// =============================================================================
// Custom Provider Implementation Tests
// =============================================================================
tap.test('Custom Provider: should support dynamic resolution based on context', async () => {
// Create a provider that returns different configs based on scope
const dynamicProvider: IUpstreamProvider = {
async resolveUpstreamConfig(context: IUpstreamResolutionContext) {
if (context.scope === 'internal') {
// Internal packages go to private registry
return {
enabled: true,
upstreams: [{ id: 'private', url: 'https://private.registry.com', priority: 1, enabled: true }],
};
}
// Everything else goes to public registry
return {
enabled: true,
upstreams: [{ id: 'public', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
};
},
};
const internalResult = await dynamicProvider.resolveUpstreamConfig({
protocol: 'npm',
resource: '@internal/utils',
scope: 'internal',
method: 'GET',
resourceType: 'packument',
});
expect(internalResult?.upstreams[0].id).toEqual('private');
const publicResult = await dynamicProvider.resolveUpstreamConfig({
protocol: 'npm',
resource: '@public/utils',
scope: 'public',
method: 'GET',
resourceType: 'packument',
});
expect(publicResult?.upstreams[0].id).toEqual('public');
});
tap.test('Custom Provider: should support actor-based resolution', async () => {
const actorAwareProvider: IUpstreamProvider = {
async resolveUpstreamConfig(context: IUpstreamResolutionContext) {
// Different upstreams based on user's organization
if (context.actor?.orgId === 'enterprise-org') {
return {
enabled: true,
upstreams: [{ id: 'enterprise', url: 'https://enterprise.registry.com', priority: 1, enabled: true }],
};
}
return {
enabled: true,
upstreams: [{ id: 'default', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
};
},
};
const enterpriseResult = await actorAwareProvider.resolveUpstreamConfig({
protocol: 'npm',
resource: 'lodash',
scope: null,
actor: { orgId: 'enterprise-org', userId: 'user1' },
method: 'GET',
resourceType: 'packument',
});
expect(enterpriseResult?.upstreams[0].id).toEqual('enterprise');
const defaultResult = await actorAwareProvider.resolveUpstreamConfig({
protocol: 'npm',
resource: 'lodash',
scope: null,
actor: { orgId: 'free-org', userId: 'user2' },
method: 'GET',
resourceType: 'packument',
});
expect(defaultResult?.upstreams[0].id).toEqual('default');
});
tap.test('Custom Provider: should support disabling upstream for specific resources', async () => {
const selectiveProvider: IUpstreamProvider = {
async resolveUpstreamConfig(context: IUpstreamResolutionContext) {
// Block upstream for internal packages
if (context.scope === 'internal') {
return null; // No upstream for internal packages
}
return {
enabled: true,
upstreams: [{ id: 'public', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
};
},
};
const internalResult = await selectiveProvider.resolveUpstreamConfig({
protocol: 'npm',
resource: '@internal/secret',
scope: 'internal',
method: 'GET',
resourceType: 'packument',
});
expect(internalResult).toBeNull();
const publicResult = await selectiveProvider.resolveUpstreamConfig({
protocol: 'npm',
resource: 'lodash',
scope: null,
method: 'GET',
resourceType: 'packument',
});
expect(publicResult).not.toBeNull();
});
// =============================================================================
// Registry without Provider Tests
// =============================================================================
tap.test('No Provider: registry should work without upstream provider', async () => {
const registryWithoutUpstream = await createTestRegistryWithUpstream(
// Pass a provider that always returns null
{
async resolveUpstreamConfig() {
return null;
},
}
);
expect(registryWithoutUpstream).toBeInstanceOf(SmartRegistry);
// Should return 404 for non-existent package (no upstream to check)
const response = await registryWithoutUpstream.handleRequest({
method: 'GET',
path: '/npm/nonexistent-package-xyz',
headers: {},
query: {},
});
expect(response.status).toEqual(404);
registryWithoutUpstream.destroy();
});
// =============================================================================
// Cleanup
// =============================================================================
tap.postTask('cleanup registry', async () => {
if (registry) {
registry.destroy();
}
});
export default tap.start();

View File

@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@push.rocks/smartregistry',
version: '1.2.0',
description: 'a registry for npm modules and oci images'
version: '2.8.2',
description: 'A composable TypeScript library implementing OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems registries for building unified container and package registries'
}

View File

@@ -2,7 +2,8 @@ import { Smartlog } from '@push.rocks/smartlog';
import { BaseRegistry } from '../core/classes.baseregistry.js';
import { RegistryStorage } from '../core/classes.registrystorage.js';
import { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js';
import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from '../core/interfaces.core.js';
import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
import type {
ICargoIndexEntry,
ICargoPublishMetadata,
@@ -13,6 +14,7 @@ import type {
ICargoSearchResponse,
ICargoSearchResult,
} from './interfaces.cargo.js';
import { CargoUpstream } from './classes.cargoupstream.js';
/**
* Cargo/crates.io registry implementation
@@ -25,18 +27,21 @@ export class CargoRegistry extends BaseRegistry {
private basePath: string = '/cargo';
private registryUrl: string;
private logger: Smartlog;
private upstreamProvider: IUpstreamProvider | null = null;
constructor(
storage: RegistryStorage,
authManager: AuthManager,
basePath: string = '/cargo',
registryUrl: string = 'http://localhost:5000/cargo'
registryUrl: string = 'http://localhost:5000/cargo',
upstreamProvider?: IUpstreamProvider
) {
super();
this.storage = storage;
this.authManager = authManager;
this.basePath = basePath;
this.registryUrl = registryUrl;
this.upstreamProvider = upstreamProvider || null;
// Initialize logger
this.logger = new Smartlog({
@@ -52,6 +57,38 @@ export class CargoRegistry extends BaseRegistry {
this.logger.enableConsole();
}
/**
* Get upstream for a specific request.
* Calls the provider to resolve upstream config dynamically.
*/
private async getUpstreamForRequest(
resource: string,
resourceType: string,
method: string,
actor?: IRequestActor
): Promise<CargoUpstream | null> {
if (!this.upstreamProvider) return null;
const config = await this.upstreamProvider.resolveUpstreamConfig({
protocol: 'cargo',
resource,
scope: resource, // For Cargo, crate name is the scope
actor,
method,
resourceType,
});
if (!config?.enabled) return null;
return new CargoUpstream(config, undefined, this.logger);
}
/**
* Clean up resources (timers, connections, etc.)
*/
public destroy(): void {
// No persistent upstream to clean up with dynamic provider
}
public async init(): Promise<void> {
// Initialize config.json if not exists
const existingConfig = await this.storage.getCargoConfig();
@@ -76,6 +113,14 @@ export class CargoRegistry extends BaseRegistry {
const authHeader = context.headers['authorization'] || context.headers['Authorization'];
const token = authHeader ? await this.authManager.validateToken(authHeader, 'cargo') : null;
// Build actor from context and validated token
const actor: IRequestActor = {
...context.actor,
userId: token?.userId,
ip: context.headers['x-forwarded-for'] || context.headers['X-Forwarded-For'],
userAgent: context.headers['user-agent'] || context.headers['User-Agent'],
};
this.logger.log('debug', `handleRequest: ${context.method} ${path}`, {
method: context.method,
path,
@@ -89,11 +134,11 @@ export class CargoRegistry extends BaseRegistry {
// API endpoints
if (path.startsWith('/api/v1/')) {
return this.handleApiRequest(path, context, token);
return this.handleApiRequest(path, context, token, actor);
}
// Index files (sparse protocol)
return this.handleIndexRequest(path);
return this.handleIndexRequest(path, actor);
}
/**
@@ -114,7 +159,8 @@ export class CargoRegistry extends BaseRegistry {
private async handleApiRequest(
path: string,
context: IRequestContext,
token: IAuthToken | null
token: IAuthToken | null,
actor?: IRequestActor
): Promise<IResponse> {
// Publish: PUT /api/v1/crates/new
if (path === '/api/v1/crates/new' && context.method === 'PUT') {
@@ -124,7 +170,7 @@ export class CargoRegistry extends BaseRegistry {
// Download: GET /api/v1/crates/{crate}/{version}/download
const downloadMatch = path.match(/^\/api\/v1\/crates\/([^\/]+)\/([^\/]+)\/download$/);
if (downloadMatch && context.method === 'GET') {
return this.handleDownload(downloadMatch[1], downloadMatch[2]);
return this.handleDownload(downloadMatch[1], downloadMatch[2], actor);
}
// Yank: DELETE /api/v1/crates/{crate}/{version}/yank
@@ -157,7 +203,7 @@ export class CargoRegistry extends BaseRegistry {
* Handle index file requests
* Paths: /1/{name}, /2/{name}, /3/{c}/{name}, /{p1}/{p2}/{name}
*/
private async handleIndexRequest(path: string): Promise<IResponse> {
private async handleIndexRequest(path: string, actor?: IRequestActor): Promise<IResponse> {
// Parse index paths to extract crate name
const pathParts = path.split('/').filter(p => p);
let crateName: string | null = null;
@@ -184,7 +230,7 @@ export class CargoRegistry extends BaseRegistry {
};
}
return this.handleIndexFile(crateName);
return this.handleIndexFile(crateName, actor);
}
/**
@@ -206,8 +252,29 @@ export class CargoRegistry extends BaseRegistry {
/**
* Serve index file for a crate
*/
private async handleIndexFile(crateName: string): Promise<IResponse> {
const index = await this.storage.getCargoIndex(crateName);
private async handleIndexFile(crateName: string, actor?: IRequestActor): Promise<IResponse> {
let index = await this.storage.getCargoIndex(crateName);
// Try upstream if not found locally
if (!index || index.length === 0) {
const upstream = await this.getUpstreamForRequest(crateName, 'index', 'GET', actor);
if (upstream) {
const upstreamIndex = await upstream.fetchCrateIndex(crateName);
if (upstreamIndex) {
// Parse the newline-delimited JSON
const parsedIndex: ICargoIndexEntry[] = upstreamIndex
.split('\n')
.filter(line => line.trim())
.map(line => JSON.parse(line));
if (parsedIndex.length > 0) {
// Cache locally
await this.storage.putCargoIndex(crateName, parsedIndex);
index = parsedIndex;
}
}
}
}
if (!index || index.length === 0) {
return {
@@ -303,7 +370,7 @@ export class CargoRegistry extends BaseRegistry {
const parsed = this.parsePublishRequest(body);
metadata = parsed.metadata;
crateFile = parsed.crateFile;
} catch (error) {
} catch (error: any) {
this.logger.log('error', 'handlePublish: parse error', { error: error.message });
return {
status: 400,
@@ -395,11 +462,36 @@ export class CargoRegistry extends BaseRegistry {
*/
private async handleDownload(
crateName: string,
version: string
version: string,
actor?: IRequestActor
): Promise<IResponse> {
this.logger.log('debug', 'handleDownload', { crate: crateName, version });
const crateFile = await this.storage.getCargoCrate(crateName, version);
// Try streaming from local storage first
const streamResult = await this.storage.getCargoCrateStream(crateName, version);
if (streamResult) {
return {
status: 200,
headers: {
'Content-Type': 'application/gzip',
'Content-Length': streamResult.size.toString(),
'Content-Disposition': `attachment; filename="${crateName}-${version}.crate"`,
},
body: streamResult.stream,
};
}
// Try upstream if not found locally
let crateFile: Buffer | null = null;
const upstream = await this.getUpstreamForRequest(crateName, 'crate', 'GET', actor);
if (upstream) {
crateFile = await upstream.fetchCrate(crateName, version);
if (crateFile) {
// Cache locally
await this.storage.putCargoCrate(crateName, version, crateFile);
}
}
if (!crateFile) {
return {
@@ -567,7 +659,7 @@ export class CargoRegistry extends BaseRegistry {
}
}
}
} catch (error) {
} catch (error: any) {
this.logger.log('error', 'handleSearch: error', { error: error.message });
}

View File

@@ -0,0 +1,159 @@
import * as plugins from '../plugins.js';
import { BaseUpstream } from '../upstream/classes.baseupstream.js';
import type {
IProtocolUpstreamConfig,
IUpstreamFetchContext,
IUpstreamRegistryConfig,
} from '../upstream/interfaces.upstream.js';
/**
* Cargo-specific upstream implementation.
*
* Handles:
* - Crate metadata (index) fetching
* - Crate file (.crate) downloading
* - Sparse index protocol support
* - Content-addressable caching for .crate files
*/
export class CargoUpstream extends BaseUpstream {
protected readonly protocolName = 'cargo';
/** Base URL for crate downloads (may differ from index URL) */
private readonly downloadUrl: string;
constructor(
config: IProtocolUpstreamConfig,
downloadUrl?: string,
logger?: plugins.smartlog.Smartlog,
) {
super(config, logger);
// Default to crates.io download URL if not specified
this.downloadUrl = downloadUrl || 'https://static.crates.io/crates';
}
/**
* Fetch crate metadata from the sparse index.
*/
public async fetchCrateIndex(crateName: string): Promise<string | null> {
const path = this.buildIndexPath(crateName);
const context: IUpstreamFetchContext = {
protocol: 'cargo',
resource: crateName,
resourceType: 'index',
path,
method: 'GET',
headers: {
'accept': 'text/plain',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
if (Buffer.isBuffer(result.body)) {
return result.body.toString('utf8');
}
return typeof result.body === 'string' ? result.body : null;
}
/**
* Fetch a crate file from upstream.
*/
public async fetchCrate(crateName: string, version: string): Promise<Buffer | null> {
// Crate downloads typically go to a different URL than the index
const path = `/${crateName}/${crateName}-${version}.crate`;
const context: IUpstreamFetchContext = {
protocol: 'cargo',
resource: crateName,
resourceType: 'crate',
path,
method: 'GET',
headers: {
'accept': 'application/octet-stream',
},
query: {},
};
// Use special handling for crate downloads
const result = await this.fetchCrateFile(crateName, version);
return result;
}
/**
* Fetch crate file directly from the download URL.
*/
private async fetchCrateFile(crateName: string, version: string): Promise<Buffer | null> {
const context: IUpstreamFetchContext = {
protocol: 'cargo',
resource: crateName,
resourceType: 'crate',
path: `/${crateName}/${crateName}-${version}.crate`,
method: 'GET',
headers: {
'accept': 'application/octet-stream',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
return Buffer.isBuffer(result.body) ? result.body : Buffer.from(result.body);
}
/**
* Build the sparse index path for a crate.
*
* Path structure:
* - 1 char: /1/{name}
* - 2 chars: /2/{name}
* - 3 chars: /3/{first char}/{name}
* - 4+ chars: /{first 2}/{next 2}/{name}
*/
private buildIndexPath(crateName: string): string {
const lowerName = crateName.toLowerCase();
const len = lowerName.length;
if (len === 1) {
return `/1/${lowerName}`;
} else if (len === 2) {
return `/2/${lowerName}`;
} else if (len === 3) {
return `/3/${lowerName[0]}/${lowerName}`;
} else {
return `/${lowerName.slice(0, 2)}/${lowerName.slice(2, 4)}/${lowerName}`;
}
}
/**
* Override URL building for Cargo-specific handling.
*/
protected buildUpstreamUrl(
upstream: IUpstreamRegistryConfig,
context: IUpstreamFetchContext,
): string {
let baseUrl = upstream.url;
// For crate downloads, use the download URL
if (context.resourceType === 'crate') {
baseUrl = this.downloadUrl;
}
// Remove trailing slash
if (baseUrl.endsWith('/')) {
baseUrl = baseUrl.slice(0, -1);
}
return `${baseUrl}${context.path}`;
}
}

View File

@@ -3,4 +3,5 @@
*/
export { CargoRegistry } from './classes.cargoregistry.js';
export { CargoUpstream } from './classes.cargoupstream.js';
export * from './interfaces.cargo.js';

View File

@@ -2,13 +2,49 @@ import { RegistryStorage } from './core/classes.registrystorage.js';
import { AuthManager } from './core/classes.authmanager.js';
import { BaseRegistry } from './core/classes.baseregistry.js';
import type { IRegistryConfig, IRequestContext, IResponse } from './core/interfaces.core.js';
import { toReadableStream } from './core/helpers.stream.js';
import { OciRegistry } from './oci/classes.ociregistry.js';
import { NpmRegistry } from './npm/classes.npmregistry.js';
import { MavenRegistry } from './maven/classes.mavenregistry.js';
import { CargoRegistry } from './cargo/classes.cargoregistry.js';
import { ComposerRegistry } from './composer/classes.composerregistry.js';
import { PypiRegistry } from './pypi/classes.pypiregistry.js';
import { RubyGemsRegistry } from './rubygems/classes.rubygemsregistry.js';
/**
* Main registry orchestrator
* Routes requests to appropriate protocol handlers (OCI, NPM, or Maven)
* Main registry orchestrator.
* Routes requests to appropriate protocol handlers (OCI, NPM, Maven, Cargo, Composer, PyPI, or RubyGems).
*
* Supports pluggable authentication and storage hooks:
*
* @example
* ```typescript
* // Basic usage with default in-memory auth
* const registry = new SmartRegistry(config);
*
* // With custom auth provider (LDAP, OAuth, etc.)
* const registry = new SmartRegistry({
* ...config,
* authProvider: new LdapAuthProvider(ldapClient),
* });
*
* // With storage hooks for quota tracking
* const registry = new SmartRegistry({
* ...config,
* storageHooks: {
* beforePut: async (ctx) => {
* const quota = await getQuota(ctx.actor?.orgId);
* if (ctx.metadata?.size > quota) {
* return { allowed: false, reason: 'Quota exceeded' };
* }
* return { allowed: true };
* },
* afterPut: async (ctx) => {
* await auditLog('storage.put', ctx);
* }
* }
* });
* ```
*/
export class SmartRegistry {
private storage: RegistryStorage;
@@ -19,8 +55,12 @@ export class SmartRegistry {
constructor(config: IRegistryConfig) {
this.config = config;
this.storage = new RegistryStorage(config.storage);
this.authManager = new AuthManager(config.auth);
// Create storage with optional hooks
this.storage = new RegistryStorage(config.storage, config.storageHooks);
// Create auth manager with optional custom provider
this.authManager = new AuthManager(config.auth, config.authProvider);
}
/**
@@ -37,30 +77,112 @@ export class SmartRegistry {
// Initialize OCI registry if enabled
if (this.config.oci?.enabled) {
const ociBasePath = this.config.oci.basePath || '/oci';
const ociRegistry = new OciRegistry(this.storage, this.authManager, ociBasePath);
const ociBasePath = this.config.oci.basePath ?? '/oci';
const ociTokens = this.config.auth.ociTokens?.enabled ? {
realm: this.config.auth.ociTokens.realm,
service: this.config.auth.ociTokens.service,
} : undefined;
const ociRegistry = new OciRegistry(
this.storage,
this.authManager,
ociBasePath,
ociTokens,
this.config.upstreamProvider
);
await ociRegistry.init();
this.registries.set('oci', ociRegistry);
}
// Initialize NPM registry if enabled
if (this.config.npm?.enabled) {
const npmBasePath = this.config.npm.basePath || '/npm';
const registryUrl = `http://localhost:5000${npmBasePath}`; // TODO: Make configurable
const npmRegistry = new NpmRegistry(this.storage, this.authManager, npmBasePath, registryUrl);
const npmBasePath = this.config.npm.basePath ?? '/npm';
const registryUrl = this.config.npm.registryUrl ?? `http://localhost:5000${npmBasePath}`;
const npmRegistry = new NpmRegistry(
this.storage,
this.authManager,
npmBasePath,
registryUrl,
this.config.upstreamProvider
);
await npmRegistry.init();
this.registries.set('npm', npmRegistry);
}
// Initialize Maven registry if enabled
if (this.config.maven?.enabled) {
const mavenBasePath = this.config.maven.basePath || '/maven';
const registryUrl = `http://localhost:5000${mavenBasePath}`; // TODO: Make configurable
const mavenRegistry = new MavenRegistry(this.storage, this.authManager, mavenBasePath, registryUrl);
const mavenBasePath = this.config.maven.basePath ?? '/maven';
const registryUrl = this.config.maven.registryUrl ?? `http://localhost:5000${mavenBasePath}`;
const mavenRegistry = new MavenRegistry(
this.storage,
this.authManager,
mavenBasePath,
registryUrl,
this.config.upstreamProvider
);
await mavenRegistry.init();
this.registries.set('maven', mavenRegistry);
}
// Initialize Cargo registry if enabled
if (this.config.cargo?.enabled) {
const cargoBasePath = this.config.cargo.basePath ?? '/cargo';
const registryUrl = this.config.cargo.registryUrl ?? `http://localhost:5000${cargoBasePath}`;
const cargoRegistry = new CargoRegistry(
this.storage,
this.authManager,
cargoBasePath,
registryUrl,
this.config.upstreamProvider
);
await cargoRegistry.init();
this.registries.set('cargo', cargoRegistry);
}
// Initialize Composer registry if enabled
if (this.config.composer?.enabled) {
const composerBasePath = this.config.composer.basePath ?? '/composer';
const registryUrl = this.config.composer.registryUrl ?? `http://localhost:5000${composerBasePath}`;
const composerRegistry = new ComposerRegistry(
this.storage,
this.authManager,
composerBasePath,
registryUrl,
this.config.upstreamProvider
);
await composerRegistry.init();
this.registries.set('composer', composerRegistry);
}
// Initialize PyPI registry if enabled
if (this.config.pypi?.enabled) {
const pypiBasePath = this.config.pypi.basePath ?? '/pypi';
const registryUrl = this.config.pypi.registryUrl ?? `http://localhost:5000`;
const pypiRegistry = new PypiRegistry(
this.storage,
this.authManager,
pypiBasePath,
registryUrl,
this.config.upstreamProvider
);
await pypiRegistry.init();
this.registries.set('pypi', pypiRegistry);
}
// Initialize RubyGems registry if enabled
if (this.config.rubygems?.enabled) {
const rubygemsBasePath = this.config.rubygems.basePath ?? '/rubygems';
const registryUrl = this.config.rubygems.registryUrl ?? `http://localhost:5000${rubygemsBasePath}`;
const rubygemsRegistry = new RubyGemsRegistry(
this.storage,
this.authManager,
rubygemsBasePath,
registryUrl,
this.config.upstreamProvider
);
await rubygemsRegistry.init();
this.registries.set('rubygems', rubygemsRegistry);
}
this.initialized = true;
}
@@ -70,40 +192,88 @@ export class SmartRegistry {
*/
public async handleRequest(context: IRequestContext): Promise<IResponse> {
const path = context.path;
let response: IResponse | undefined;
// Route to OCI registry
if (this.config.oci?.enabled && path.startsWith(this.config.oci.basePath)) {
if (!response && this.config.oci?.enabled && path.startsWith(this.config.oci.basePath)) {
const ociRegistry = this.registries.get('oci');
if (ociRegistry) {
return ociRegistry.handleRequest(context);
response = await ociRegistry.handleRequest(context);
}
}
// Route to NPM registry
if (this.config.npm?.enabled && path.startsWith(this.config.npm.basePath)) {
if (!response && this.config.npm?.enabled && path.startsWith(this.config.npm.basePath)) {
const npmRegistry = this.registries.get('npm');
if (npmRegistry) {
return npmRegistry.handleRequest(context);
response = await npmRegistry.handleRequest(context);
}
}
// Route to Maven registry
if (this.config.maven?.enabled && path.startsWith(this.config.maven.basePath)) {
if (!response && this.config.maven?.enabled && path.startsWith(this.config.maven.basePath)) {
const mavenRegistry = this.registries.get('maven');
if (mavenRegistry) {
return mavenRegistry.handleRequest(context);
response = await mavenRegistry.handleRequest(context);
}
}
// Route to Cargo registry
if (!response && this.config.cargo?.enabled && path.startsWith(this.config.cargo.basePath)) {
const cargoRegistry = this.registries.get('cargo');
if (cargoRegistry) {
response = await cargoRegistry.handleRequest(context);
}
}
// Route to Composer registry
if (!response && this.config.composer?.enabled && path.startsWith(this.config.composer.basePath)) {
const composerRegistry = this.registries.get('composer');
if (composerRegistry) {
response = await composerRegistry.handleRequest(context);
}
}
// Route to PyPI registry (also handles /simple prefix)
if (!response && this.config.pypi?.enabled) {
const pypiBasePath = this.config.pypi.basePath ?? '/pypi';
if (path.startsWith(pypiBasePath) || path.startsWith('/simple')) {
const pypiRegistry = this.registries.get('pypi');
if (pypiRegistry) {
response = await pypiRegistry.handleRequest(context);
}
}
}
// Route to RubyGems registry
if (!response && this.config.rubygems?.enabled && path.startsWith(this.config.rubygems.basePath)) {
const rubygemsRegistry = this.registries.get('rubygems');
if (rubygemsRegistry) {
response = await rubygemsRegistry.handleRequest(context);
}
}
// No matching registry
return {
status: 404,
headers: { 'Content-Type': 'application/json' },
body: {
error: 'NOT_FOUND',
message: 'No registry handler for this path',
},
};
if (!response) {
response = {
status: 404,
headers: { 'Content-Type': 'application/json' },
body: {
error: 'NOT_FOUND',
message: 'No registry handler for this path',
},
};
}
// Normalize body to ReadableStream<Uint8Array> at the API boundary
if (response.body != null && !(response.body instanceof ReadableStream)) {
if (!Buffer.isBuffer(response.body) && typeof response.body === 'object' && !(response.body instanceof Uint8Array)) {
response.headers['Content-Type'] ??= 'application/json';
}
response.body = toReadableStream(response.body);
}
return response;
}
/**
@@ -123,7 +293,7 @@ export class SmartRegistry {
/**
* Get a specific registry handler
*/
public getRegistry(protocol: 'oci' | 'npm' | 'maven'): BaseRegistry | undefined {
public getRegistry(protocol: 'oci' | 'npm' | 'maven' | 'cargo' | 'composer' | 'pypi' | 'rubygems'): BaseRegistry | undefined {
return this.registries.get(protocol);
}

View File

@@ -0,0 +1,544 @@
/**
* Composer Registry Implementation
* Compliant with Composer v2 repository API
*/
import { BaseRegistry } from '../core/classes.baseregistry.js';
import type { RegistryStorage } from '../core/classes.registrystorage.js';
import type { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from '../core/interfaces.core.js';
import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
import { isBinaryData, toBuffer } from '../core/helpers.buffer.js';
import type {
IComposerPackage,
IComposerPackageMetadata,
IComposerRepository,
} from './interfaces.composer.js';
import {
normalizeVersion,
validateComposerJson,
extractComposerJsonFromZip,
calculateSha1,
parseVendorPackage,
generatePackagesJson,
sortVersions,
} from './helpers.composer.js';
import { ComposerUpstream } from './classes.composerupstream.js';
export class ComposerRegistry extends BaseRegistry {
private storage: RegistryStorage;
private authManager: AuthManager;
private basePath: string = '/composer';
private registryUrl: string;
private upstreamProvider: IUpstreamProvider | null = null;
constructor(
storage: RegistryStorage,
authManager: AuthManager,
basePath: string = '/composer',
registryUrl: string = 'http://localhost:5000/composer',
upstreamProvider?: IUpstreamProvider
) {
super();
this.storage = storage;
this.authManager = authManager;
this.basePath = basePath;
this.registryUrl = registryUrl;
this.upstreamProvider = upstreamProvider || null;
}
/**
* Extract scope from Composer package name.
* For Composer, vendor is the scope.
* @example "symfony" from "symfony/console"
*/
private extractScope(vendorPackage: string): string | null {
const slashIndex = vendorPackage.indexOf('/');
if (slashIndex > 0) {
return vendorPackage.substring(0, slashIndex);
}
return null;
}
/**
* Get upstream for a specific request.
* Calls the provider to resolve upstream config dynamically.
*/
private async getUpstreamForRequest(
resource: string,
resourceType: string,
method: string,
actor?: IRequestActor
): Promise<ComposerUpstream | null> {
if (!this.upstreamProvider) return null;
const config = await this.upstreamProvider.resolveUpstreamConfig({
protocol: 'composer',
resource,
scope: this.extractScope(resource),
actor,
method,
resourceType,
});
if (!config?.enabled) return null;
return new ComposerUpstream(config);
}
/**
* Clean up resources (timers, connections, etc.)
*/
public destroy(): void {
// No persistent upstream to clean up with dynamic provider
}
public async init(): Promise<void> {
// Composer registry initialization
}
public getBasePath(): string {
return this.basePath;
}
public async handleRequest(context: IRequestContext): Promise<IResponse> {
const path = context.path.replace(this.basePath, '');
// Extract token from Authorization header
const authHeader = context.headers['authorization'] || context.headers['Authorization'];
let token: IAuthToken | null = null;
if (authHeader) {
if (authHeader.startsWith('Bearer ')) {
const tokenString = authHeader.replace(/^Bearer\s+/i, '');
token = await this.authManager.validateToken(tokenString, 'composer');
} else if (authHeader.startsWith('Basic ')) {
// Handle HTTP Basic Auth
const credentials = Buffer.from(authHeader.replace(/^Basic\s+/i, ''), 'base64').toString('utf-8');
const [username, password] = credentials.split(':');
const userId = await this.authManager.authenticate({ username, password });
if (userId) {
// Create temporary token for this request
token = {
type: 'composer',
userId,
scopes: ['composer:*:*:read'],
readonly: true,
};
}
}
}
// Build actor from context and validated token
const actor: IRequestActor = {
...context.actor,
userId: token?.userId,
ip: context.headers['x-forwarded-for'] || context.headers['X-Forwarded-For'],
userAgent: context.headers['user-agent'] || context.headers['User-Agent'],
};
// Root packages.json
if (path === '/packages.json' || path === '' || path === '/') {
return this.handlePackagesJson();
}
// Package metadata: /p2/{vendor}/{package}.json or /p2/{vendor}/{package}~dev.json
const metadataMatch = path.match(/^\/p2\/([^\/]+\/[^\/]+?)(~dev)?\.json$/);
if (metadataMatch) {
const [, vendorPackage, devSuffix] = metadataMatch;
const includeDev = !!devSuffix;
return this.handlePackageMetadata(vendorPackage, includeDev, token, actor);
}
// Package list: /packages/list.json?filter=vendor/*
if (path.startsWith('/packages/list.json')) {
const filter = context.query['filter'];
return this.handlePackageList(filter, token);
}
// Package ZIP download: /dists/{vendor}/{package}/{reference}.zip
const distMatch = path.match(/^\/dists\/([^\/]+\/[^\/]+)\/([^\/]+)\.zip$/);
if (distMatch) {
const [, vendorPackage, reference] = distMatch;
return this.handlePackageDownload(vendorPackage, reference, token);
}
// Package upload: PUT /packages/{vendor}/{package}
const uploadMatch = path.match(/^\/packages\/([^\/]+\/[^\/]+)$/);
if (uploadMatch && context.method === 'PUT') {
const vendorPackage = uploadMatch[1];
return this.handlePackageUpload(vendorPackage, context.body, token);
}
// Package delete: DELETE /packages/{vendor}/{package}
if (uploadMatch && context.method === 'DELETE') {
const vendorPackage = uploadMatch[1];
return this.handlePackageDelete(vendorPackage, token);
}
// Version delete: DELETE /packages/{vendor}/{package}/{version}
const versionDeleteMatch = path.match(/^\/packages\/([^\/]+\/[^\/]+)\/(.+)$/);
if (versionDeleteMatch && context.method === 'DELETE') {
const [, vendorPackage, version] = versionDeleteMatch;
return this.handleVersionDelete(vendorPackage, version, token);
}
return {
status: 404,
headers: { 'Content-Type': 'application/json' },
body: { status: 'error', message: 'Not found' },
};
}
protected async checkPermission(
token: IAuthToken | null,
resource: string,
action: string
): Promise<boolean> {
if (!token) return false;
return this.authManager.authorize(token, `composer:package:${resource}`, action);
}
// ========================================================================
// REQUEST HANDLERS
// ========================================================================
private async handlePackagesJson(): Promise<IResponse> {
const availablePackages = await this.storage.listComposerPackages();
const packagesJson = generatePackagesJson(this.registryUrl, availablePackages);
return {
status: 200,
headers: { 'Content-Type': 'application/json' },
body: packagesJson,
};
}
private async handlePackageMetadata(
vendorPackage: string,
includeDev: boolean,
token: IAuthToken | null,
actor?: IRequestActor
): Promise<IResponse> {
// Read operations are public, no authentication required
let metadata = await this.storage.getComposerPackageMetadata(vendorPackage);
// Try upstream if not found locally
if (!metadata) {
const upstream = await this.getUpstreamForRequest(vendorPackage, 'metadata', 'GET', actor);
if (upstream) {
const [vendor, packageName] = vendorPackage.split('/');
if (vendor && packageName) {
const upstreamMetadata = includeDev
? await upstream.fetchPackageDevMetadata(vendor, packageName)
: await upstream.fetchPackageMetadata(vendor, packageName);
if (upstreamMetadata && upstreamMetadata.packages) {
// Store upstream metadata locally
metadata = {
packages: upstreamMetadata.packages,
lastModified: new Date().toUTCString(),
};
await this.storage.putComposerPackageMetadata(vendorPackage, metadata);
}
}
}
}
if (!metadata) {
return {
status: 404,
headers: { 'Content-Type': 'application/json' },
body: { status: 'error', message: 'Package not found' },
};
}
// Filter dev versions if needed
let packages = metadata.packages[vendorPackage] || [];
if (!includeDev) {
packages = packages.filter((pkg: IComposerPackage) =>
!pkg.version.includes('dev') && !pkg.version.includes('alpha') && !pkg.version.includes('beta')
);
}
const response: IComposerPackageMetadata = {
minified: 'composer/2.0',
packages: {
[vendorPackage]: packages,
},
};
return {
status: 200,
headers: {
'Content-Type': 'application/json',
'Last-Modified': metadata.lastModified || new Date().toUTCString(),
},
body: response,
};
}
private async handlePackageList(
filter: string | undefined,
token: IAuthToken | null
): Promise<IResponse> {
let packages = await this.storage.listComposerPackages();
// Apply filter if provided
if (filter) {
const regex = new RegExp('^' + filter.replace(/\*/g, '.*') + '$');
packages = packages.filter(pkg => regex.test(pkg));
}
return {
status: 200,
headers: { 'Content-Type': 'application/json' },
body: { packageNames: packages },
};
}
private async handlePackageDownload(
vendorPackage: string,
reference: string,
token: IAuthToken | null
): Promise<IResponse> {
// Read operations are public, no authentication required
const streamResult = await this.storage.getComposerPackageZipStream(vendorPackage, reference);
if (!streamResult) {
return {
status: 404,
headers: {},
body: { status: 'error', message: 'Package file not found' },
};
}
return {
status: 200,
headers: {
'Content-Type': 'application/zip',
'Content-Length': streamResult.size.toString(),
'Content-Disposition': `attachment; filename="${reference}.zip"`,
},
body: streamResult.stream,
};
}
private async handlePackageUpload(
vendorPackage: string,
body: any,
token: IAuthToken | null
): Promise<IResponse> {
// Check write permission
if (!await this.checkPermission(token, vendorPackage, 'write')) {
return {
status: 401,
headers: {},
body: { status: 'error', message: 'Write permission required' },
};
}
if (!body || !isBinaryData(body)) {
return {
status: 400,
headers: {},
body: { status: 'error', message: 'ZIP file required' },
};
}
// Convert to Buffer for ZIP processing
const zipData = toBuffer(body);
// Extract and validate composer.json from ZIP
const composerJson = await extractComposerJsonFromZip(zipData);
if (!composerJson || !validateComposerJson(composerJson)) {
return {
status: 400,
headers: {},
body: { status: 'error', message: 'Invalid composer.json in ZIP' },
};
}
// Verify package name matches
if (composerJson.name !== vendorPackage) {
return {
status: 400,
headers: {},
body: { status: 'error', message: 'Package name mismatch' },
};
}
const version = composerJson.version;
if (!version) {
return {
status: 400,
headers: {},
body: { status: 'error', message: 'Version required in composer.json' },
};
}
// Calculate SHA-1 hash
const shasum = await calculateSha1(zipData);
// Generate reference (use version or commit hash)
const reference = composerJson.source?.reference || version.replace(/[^a-zA-Z0-9.-]/g, '-');
// Store ZIP file
await this.storage.putComposerPackageZip(vendorPackage, reference, zipData);
// Get or create metadata
let metadata = await this.storage.getComposerPackageMetadata(vendorPackage);
if (!metadata) {
metadata = {
packages: {
[vendorPackage]: [],
},
lastModified: new Date().toUTCString(),
};
}
// Build package entry
const packageEntry: IComposerPackage = {
...composerJson,
version_normalized: normalizeVersion(version),
dist: {
type: 'zip',
url: `${this.registryUrl}/dists/${vendorPackage}/${reference}.zip`,
reference,
shasum,
},
time: new Date().toISOString(),
};
// Add to metadata (check if version already exists)
const packages = metadata.packages[vendorPackage] || [];
const existingIndex = packages.findIndex((p: IComposerPackage) => p.version === version);
if (existingIndex >= 0) {
return {
status: 409,
headers: {},
body: { status: 'error', message: 'Version already exists' },
};
}
packages.push(packageEntry);
// Sort by version
const sortedVersions = sortVersions(packages.map((p: IComposerPackage) => p.version));
packages.sort((a: IComposerPackage, b: IComposerPackage) => {
return sortedVersions.indexOf(a.version) - sortedVersions.indexOf(b.version);
});
metadata.packages[vendorPackage] = packages;
metadata.lastModified = new Date().toUTCString();
// Store updated metadata
await this.storage.putComposerPackageMetadata(vendorPackage, metadata);
return {
status: 201,
headers: {},
body: {
status: 'success',
message: 'Package uploaded successfully',
package: vendorPackage,
version,
},
};
}
private async handlePackageDelete(
vendorPackage: string,
token: IAuthToken | null
): Promise<IResponse> {
// Check delete permission
if (!await this.checkPermission(token, vendorPackage, 'delete')) {
return {
status: 401,
headers: {},
body: { status: 'error', message: 'Delete permission required' },
};
}
const metadata = await this.storage.getComposerPackageMetadata(vendorPackage);
if (!metadata) {
return {
status: 404,
headers: {},
body: { status: 'error', message: 'Package not found' },
};
}
// Delete all ZIP files
const packages = metadata.packages[vendorPackage] || [];
for (const pkg of packages) {
if (pkg.dist?.reference) {
await this.storage.deleteComposerPackageZip(vendorPackage, pkg.dist.reference);
}
}
// Delete metadata
await this.storage.deleteComposerPackageMetadata(vendorPackage);
return {
status: 204,
headers: {},
body: null,
};
}
private async handleVersionDelete(
vendorPackage: string,
version: string,
token: IAuthToken | null
): Promise<IResponse> {
// Check delete permission
if (!await this.checkPermission(token, vendorPackage, 'delete')) {
return {
status: 401,
headers: {},
body: { status: 'error', message: 'Delete permission required' },
};
}
const metadata = await this.storage.getComposerPackageMetadata(vendorPackage);
if (!metadata) {
return {
status: 404,
headers: {},
body: { status: 'error', message: 'Package not found' },
};
}
const packages = metadata.packages[vendorPackage] || [];
const versionIndex = packages.findIndex((p: IComposerPackage) => p.version === version);
if (versionIndex === -1) {
return {
status: 404,
headers: {},
body: { status: 'error', message: 'Version not found' },
};
}
// Delete ZIP file
const pkg = packages[versionIndex];
if (pkg.dist?.reference) {
await this.storage.deleteComposerPackageZip(vendorPackage, pkg.dist.reference);
}
// Remove from metadata
packages.splice(versionIndex, 1);
metadata.packages[vendorPackage] = packages;
metadata.lastModified = new Date().toUTCString();
// Save updated metadata
await this.storage.putComposerPackageMetadata(vendorPackage, metadata);
return {
status: 204,
headers: {},
body: null,
};
}
}

View File

@@ -0,0 +1,200 @@
import * as plugins from '../plugins.js';
import { BaseUpstream } from '../upstream/classes.baseupstream.js';
import type {
IProtocolUpstreamConfig,
IUpstreamFetchContext,
IUpstreamRegistryConfig,
} from '../upstream/interfaces.upstream.js';
/**
* Composer-specific upstream implementation.
*
* Handles:
* - Package metadata fetching (packages.json, provider-includes)
* - Package version metadata (p2/{vendor}/{package}.json)
* - Dist file (zip) proxying
* - Packagist v2 API support
*/
export class ComposerUpstream extends BaseUpstream {
protected readonly protocolName = 'composer';
constructor(
config: IProtocolUpstreamConfig,
logger?: plugins.smartlog.Smartlog,
) {
super(config, logger);
}
/**
* Fetch the root packages.json from upstream.
*/
public async fetchPackagesJson(): Promise<any | null> {
const context: IUpstreamFetchContext = {
protocol: 'composer',
resource: '*',
resourceType: 'root',
path: '/packages.json',
method: 'GET',
headers: {
'accept': 'application/json',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
if (Buffer.isBuffer(result.body)) {
return JSON.parse(result.body.toString('utf8'));
}
return result.body;
}
/**
* Fetch package metadata using v2 API (p2/{vendor}/{package}.json).
*/
public async fetchPackageMetadata(vendor: string, packageName: string): Promise<any | null> {
const fullName = `${vendor}/${packageName}`;
const path = `/p2/${vendor}/${packageName}.json`;
const context: IUpstreamFetchContext = {
protocol: 'composer',
resource: fullName,
resourceType: 'metadata',
path,
method: 'GET',
headers: {
'accept': 'application/json',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
if (Buffer.isBuffer(result.body)) {
return JSON.parse(result.body.toString('utf8'));
}
return result.body;
}
/**
* Fetch package metadata with dev versions (p2/{vendor}/{package}~dev.json).
*/
public async fetchPackageDevMetadata(vendor: string, packageName: string): Promise<any | null> {
const fullName = `${vendor}/${packageName}`;
const path = `/p2/${vendor}/${packageName}~dev.json`;
const context: IUpstreamFetchContext = {
protocol: 'composer',
resource: fullName,
resourceType: 'metadata-dev',
path,
method: 'GET',
headers: {
'accept': 'application/json',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
if (Buffer.isBuffer(result.body)) {
return JSON.parse(result.body.toString('utf8'));
}
return result.body;
}
/**
* Fetch a provider-includes file.
*/
public async fetchProviderIncludes(path: string): Promise<any | null> {
const context: IUpstreamFetchContext = {
protocol: 'composer',
resource: '*',
resourceType: 'provider',
path: path.startsWith('/') ? path : `/${path}`,
method: 'GET',
headers: {
'accept': 'application/json',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
if (Buffer.isBuffer(result.body)) {
return JSON.parse(result.body.toString('utf8'));
}
return result.body;
}
/**
* Fetch a dist file (zip) from upstream.
*/
public async fetchDist(url: string): Promise<Buffer | null> {
// Parse the URL to get the path
let path: string;
try {
const parsed = new URL(url);
path = parsed.pathname;
} catch {
path = url;
}
const context: IUpstreamFetchContext = {
protocol: 'composer',
resource: '*',
resourceType: 'dist',
path,
method: 'GET',
headers: {
'accept': 'application/zip, application/octet-stream',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
return Buffer.isBuffer(result.body) ? result.body : Buffer.from(result.body);
}
/**
* Override URL building for Composer-specific handling.
*/
protected buildUpstreamUrl(
upstream: IUpstreamRegistryConfig,
context: IUpstreamFetchContext,
): string {
let baseUrl = upstream.url;
// Remove trailing slash
if (baseUrl.endsWith('/')) {
baseUrl = baseUrl.slice(0, -1);
}
return `${baseUrl}${context.path}`;
}
}

View File

@@ -0,0 +1,139 @@
/**
* Composer Registry Helper Functions
*/
import type { IComposerPackage } from './interfaces.composer.js';
/**
* Normalize version string to Composer format
* Example: "1.0.0" -> "1.0.0.0", "v2.3.1" -> "2.3.1.0"
*/
export function normalizeVersion(version: string): string {
// Remove 'v' prefix if present
let normalized = version.replace(/^v/i, '');
// Handle special versions (dev, alpha, beta, rc)
if (normalized.includes('dev') || normalized.includes('alpha') || normalized.includes('beta') || normalized.includes('RC')) {
// For dev versions, just return as-is with .0 appended if needed
const parts = normalized.split(/[-+]/)[0].split('.');
while (parts.length < 4) {
parts.push('0');
}
return parts.slice(0, 4).join('.');
}
// Split by dots
const parts = normalized.split('.');
// Ensure 4 parts (major.minor.patch.build)
while (parts.length < 4) {
parts.push('0');
}
return parts.slice(0, 4).join('.');
}
/**
* Validate composer.json structure
*/
export function validateComposerJson(composerJson: any): boolean {
return !!(
composerJson &&
typeof composerJson.name === 'string' &&
composerJson.name.includes('/') &&
(composerJson.version || composerJson.require)
);
}
/**
* Extract composer.json from ZIP buffer
*/
export async function extractComposerJsonFromZip(zipBuffer: Buffer): Promise<any | null> {
try {
const AdmZip = (await import('adm-zip')).default;
const zip = new AdmZip(zipBuffer);
const entries = zip.getEntries();
// Look for composer.json in root or first-level directory
for (const entry of entries) {
if (entry.entryName.endsWith('composer.json')) {
const parts = entry.entryName.split('/');
if (parts.length <= 2) { // Root or first-level dir
const content = entry.getData().toString('utf-8');
return JSON.parse(content);
}
}
}
return null;
} catch (error) {
return null;
}
}
/**
* Calculate SHA-1 hash for ZIP file
*/
export async function calculateSha1(data: Buffer): Promise<string> {
const crypto = await import('crypto');
return crypto.createHash('sha1').update(data).digest('hex');
}
/**
* Parse vendor/package format
*/
export function parseVendorPackage(name: string): { vendor: string; package: string } | null {
const parts = name.split('/');
if (parts.length !== 2) {
return null;
}
return { vendor: parts[0], package: parts[1] };
}
/**
* Generate packages.json root repository file
*/
export function generatePackagesJson(
registryUrl: string,
availablePackages: string[]
): any {
return {
'metadata-url': `${registryUrl}/p2/%package%.json`,
'available-packages': availablePackages,
};
}
/**
* Sort versions in semantic version order
*/
export function sortVersions(versions: string[]): string[] {
return versions.sort((a, b) => {
const aParts = a.replace(/^v/i, '').split(/[.-]/).map(part => {
const num = parseInt(part, 10);
return isNaN(num) ? part : num;
});
const bParts = b.replace(/^v/i, '').split(/[.-]/).map(part => {
const num = parseInt(part, 10);
return isNaN(num) ? part : num;
});
for (let i = 0; i < Math.max(aParts.length, bParts.length); i++) {
const aPart = aParts[i] ?? 0;
const bPart = bParts[i] ?? 0;
// Compare numbers numerically, strings lexicographically
if (typeof aPart === 'number' && typeof bPart === 'number') {
if (aPart !== bPart) {
return aPart - bPart;
}
} else {
const aStr = String(aPart);
const bStr = String(bPart);
if (aStr !== bStr) {
return aStr.localeCompare(bStr);
}
}
}
return 0;
});
}

9
ts/composer/index.ts Normal file
View File

@@ -0,0 +1,9 @@
/**
* Composer Registry Module
* Export all public interfaces, classes, and helpers
*/
export { ComposerRegistry } from './classes.composerregistry.js';
export { ComposerUpstream } from './classes.composerupstream.js';
export * from './interfaces.composer.js';
export * from './helpers.composer.js';

View File

@@ -0,0 +1,111 @@
/**
* Composer Registry Type Definitions
* Compliant with Composer v2 repository API
*/
/**
* Composer package metadata
*/
export interface IComposerPackage {
name: string; // vendor/package-name
version: string; // 1.0.0
version_normalized: string; // 1.0.0.0
type?: string; // library, project, metapackage
description?: string;
keywords?: string[];
homepage?: string;
license?: string[];
authors?: IComposerAuthor[];
require?: Record<string, string>;
'require-dev'?: Record<string, string>;
suggest?: Record<string, string>;
provide?: Record<string, string>;
conflict?: Record<string, string>;
replace?: Record<string, string>;
autoload?: IComposerAutoload;
'autoload-dev'?: IComposerAutoload;
dist?: IComposerDist;
source?: IComposerSource;
time?: string; // ISO 8601 timestamp
support?: Record<string, string>;
funding?: IComposerFunding[];
extra?: Record<string, any>;
}
/**
* Author information
*/
export interface IComposerAuthor {
name: string;
email?: string;
homepage?: string;
role?: string;
}
/**
* PSR-4/PSR-0 autoloading configuration
*/
export interface IComposerAutoload {
'psr-4'?: Record<string, string | string[]>;
'psr-0'?: Record<string, string | string[]>;
classmap?: string[];
files?: string[];
'exclude-from-classmap'?: string[];
}
/**
* Distribution information (ZIP download)
*/
export interface IComposerDist {
type: 'zip' | 'tar' | 'phar';
url: string;
reference?: string; // commit hash or tag
shasum?: string; // SHA-1 hash
}
/**
* Source repository information
*/
export interface IComposerSource {
type: 'git' | 'svn' | 'hg';
url: string;
reference: string; // commit hash, branch, or tag
}
/**
* Funding information
*/
export interface IComposerFunding {
type: string; // github, patreon, etc.
url: string;
}
/**
* Repository metadata (packages.json)
*/
export interface IComposerRepository {
packages?: Record<string, Record<string, IComposerPackage>>;
'metadata-url'?: string; // /p2/%package%.json
'available-packages'?: string[];
'available-package-patterns'?: string[];
'providers-url'?: string;
'notify-batch'?: string;
minified?: string; // "composer/2.0"
}
/**
* Package metadata response (/p2/vendor/package.json)
*/
export interface IComposerPackageMetadata {
packages: Record<string, IComposerPackage[]>;
minified?: string;
lastModified?: string;
}
/**
* Error structure
*/
export interface IComposerError {
status: string;
message: string;
}

View File

@@ -1,54 +1,79 @@
import type { IAuthConfig, IAuthToken, ICredentials, TRegistryProtocol } from './interfaces.core.js';
import type { IAuthProvider, ITokenOptions } from './interfaces.auth.js';
import { DefaultAuthProvider } from './classes.defaultauthprovider.js';
/**
* Unified authentication manager for all registry protocols
* Handles both NPM UUID tokens and OCI JWT tokens
* Unified authentication manager for all registry protocols.
* Delegates to a pluggable IAuthProvider for actual auth operations.
*
* @example
* ```typescript
* // Use default in-memory provider
* const auth = new AuthManager(config);
*
* // Use custom provider (LDAP, OAuth, etc.)
* const auth = new AuthManager(config, new LdapAuthProvider(ldapClient));
* ```
*/
export class AuthManager {
private tokenStore: Map<string, IAuthToken> = new Map();
private userCredentials: Map<string, string> = new Map(); // username -> password hash (mock)
private provider: IAuthProvider;
constructor(private config: IAuthConfig) {}
constructor(
private config: IAuthConfig,
provider?: IAuthProvider
) {
// Use provided provider or default in-memory implementation
this.provider = provider || new DefaultAuthProvider(config);
}
/**
* Initialize the auth manager
*/
public async init(): Promise<void> {
// Initialize token store (in-memory for now)
// In production, this could be Redis or a database
if (this.provider.init) {
await this.provider.init();
}
}
// ========================================================================
// UUID TOKEN CREATION (Base method for NPM, Maven, etc.)
// UNIFIED AUTHENTICATION (Delegated to Provider)
// ========================================================================
/**
* Create a UUID-based token with custom scopes (base method)
* @param userId - User ID
* @param protocol - Protocol type
* @param scopes - Permission scopes
* @param readonly - Whether the token is readonly
* @returns UUID token string
* Authenticate user credentials
* @param credentials - Username and password
* @returns User ID or null
*/
private async createUuidToken(
userId: string,
protocol: TRegistryProtocol,
scopes: string[],
readonly: boolean = false
): Promise<string> {
const token = this.generateUuid();
const authToken: IAuthToken = {
type: protocol,
userId,
scopes,
readonly,
metadata: {
created: new Date().toISOString(),
},
};
public async authenticate(credentials: ICredentials): Promise<string | null> {
return this.provider.authenticate(credentials);
}
this.tokenStore.set(token, authToken);
return token;
/**
* Validate any token (NPM, Maven, OCI, PyPI, RubyGems, Composer, Cargo)
* @param tokenString - Token string (UUID or JWT)
* @param protocol - Expected protocol type (optional, improves performance)
* @returns Auth token object or null
*/
public async validateToken(
tokenString: string,
protocol?: TRegistryProtocol
): Promise<IAuthToken | null> {
return this.provider.validateToken(tokenString, protocol);
}
/**
* Check if token has permission for an action
* @param token - Auth token (or null for anonymous)
* @param resource - Resource being accessed (e.g., "npm:package:foo")
* @param action - Action being performed (read, write, push, pull, delete)
* @returns true if authorized
*/
public async authorize(
token: IAuthToken | null,
resource: string,
action: string
): Promise<boolean> {
return this.provider.authorize(token, resource, action);
}
// ========================================================================
@@ -65,9 +90,7 @@ export class AuthManager {
if (!this.config.npmTokens.enabled) {
throw new Error('NPM tokens are not enabled');
}
const scopes = readonly ? ['npm:*:*:read'] : ['npm:*:*:*'];
return this.createUuidToken(userId, 'npm', scopes, readonly);
return this.provider.createToken(userId, 'npm', { readonly });
}
/**
@@ -76,22 +99,7 @@ export class AuthManager {
* @returns Auth token object or null
*/
public async validateNpmToken(token: string): Promise<IAuthToken | null> {
if (!this.isValidUuid(token)) {
return null;
}
const authToken = this.tokenStore.get(token);
if (!authToken || authToken.type !== 'npm') {
return null;
}
// Check expiration if set
if (authToken.expiresAt && authToken.expiresAt < new Date()) {
this.tokenStore.delete(token);
return null;
}
return authToken;
return this.provider.validateToken(token, 'npm');
}
/**
@@ -99,7 +107,7 @@ export class AuthManager {
* @param token - NPM UUID token
*/
public async revokeNpmToken(token: string): Promise<void> {
this.tokenStore.delete(token);
return this.provider.revokeToken(token);
}
/**
@@ -111,20 +119,12 @@ export class AuthManager {
key: string;
readonly: boolean;
created: string;
protocol?: TRegistryProtocol;
}>> {
const tokens: Array<{key: string; readonly: boolean; created: string}> = [];
for (const [token, authToken] of this.tokenStore.entries()) {
if (authToken.userId === userId) {
tokens.push({
key: this.hashToken(token),
readonly: authToken.readonly || false,
created: authToken.metadata?.created || 'unknown',
});
}
if (this.provider.listUserTokens) {
return this.provider.listUserTokens(userId);
}
return tokens;
return [];
}
// ========================================================================
@@ -143,24 +143,10 @@ export class AuthManager {
scopes: string[],
expiresIn: number = 3600
): Promise<string> {
if (!this.config.ociTokens.enabled) {
if (!this.config.ociTokens?.enabled) {
throw new Error('OCI tokens are not enabled');
}
const now = Math.floor(Date.now() / 1000);
const payload = {
iss: this.config.ociTokens.realm,
sub: userId,
aud: this.config.ociTokens.service,
exp: now + expiresIn,
nbf: now,
iat: now,
access: this.scopesToOciAccess(scopes),
};
// In production, use proper JWT library with signing
// For now, return JSON string (mock JWT)
return JSON.stringify(payload);
return this.provider.createToken(userId, 'oci', { scopes, expiresIn });
}
/**
@@ -169,58 +155,7 @@ export class AuthManager {
* @returns Auth token object or null
*/
public async validateOciToken(jwt: string): Promise<IAuthToken | null> {
try {
// In production, verify JWT signature
const payload = JSON.parse(jwt);
// Check expiration
const now = Math.floor(Date.now() / 1000);
if (payload.exp && payload.exp < now) {
return null;
}
// Convert to unified token format
const scopes = this.ociAccessToScopes(payload.access || []);
return {
type: 'oci',
userId: payload.sub,
scopes,
expiresAt: payload.exp ? new Date(payload.exp * 1000) : undefined,
metadata: {
iss: payload.iss,
aud: payload.aud,
},
};
} catch (error) {
return null;
}
}
// ========================================================================
// UNIFIED AUTHENTICATION
// ========================================================================
/**
* Authenticate user credentials
* @param credentials - Username and password
* @returns User ID or null
*/
public async authenticate(credentials: ICredentials): Promise<string | null> {
// Mock authentication - in production, verify against database
const storedPassword = this.userCredentials.get(credentials.username);
if (!storedPassword) {
// Auto-register for testing (remove in production)
this.userCredentials.set(credentials.username, credentials.password);
return credentials.username;
}
if (storedPassword === credentials.password) {
return credentials.username;
}
return null;
return this.provider.validateToken(jwt, 'oci');
}
// ========================================================================
@@ -234,8 +169,7 @@ export class AuthManager {
* @returns Maven UUID token
*/
public async createMavenToken(userId: string, readonly: boolean = false): Promise<string> {
const scopes = readonly ? ['maven:*:*:read'] : ['maven:*:*:*'];
return this.createUuidToken(userId, 'maven', scopes, readonly);
return this.provider.createToken(userId, 'maven', { readonly });
}
/**
@@ -244,22 +178,7 @@ export class AuthManager {
* @returns Auth token object or null
*/
public async validateMavenToken(token: string): Promise<IAuthToken | null> {
if (!this.isValidUuid(token)) {
return null;
}
const authToken = this.tokenStore.get(token);
if (!authToken || authToken.type !== 'maven') {
return null;
}
// Check expiration if set
if (authToken.expiresAt && authToken.expiresAt < new Date()) {
this.tokenStore.delete(token);
return null;
}
return authToken;
return this.provider.validateToken(token, 'maven');
}
/**
@@ -267,202 +186,130 @@ export class AuthManager {
* @param token - Maven UUID token
*/
public async revokeMavenToken(token: string): Promise<void> {
this.tokenStore.delete(token);
return this.provider.revokeToken(token);
}
// ========================================================================
// UNIFIED AUTHENTICATION
// COMPOSER TOKEN MANAGEMENT
// ========================================================================
/**
* Validate any token (NPM, Maven, or OCI)
* @param tokenString - Token string (UUID or JWT)
* @param protocol - Expected protocol type
* Create a Composer token
* @param userId - User ID
* @param readonly - Whether the token is readonly
* @returns Composer UUID token
*/
public async createComposerToken(userId: string, readonly: boolean = false): Promise<string> {
return this.provider.createToken(userId, 'composer', { readonly });
}
/**
* Validate a Composer token
* @param token - Composer UUID token
* @returns Auth token object or null
*/
public async validateToken(
tokenString: string,
protocol?: TRegistryProtocol
): Promise<IAuthToken | null> {
// Try UUID-based tokens (NPM, Maven)
if (this.isValidUuid(tokenString)) {
// Try NPM token
const npmToken = await this.validateNpmToken(tokenString);
if (npmToken && (!protocol || protocol === 'npm')) {
return npmToken;
}
// Try Maven token
const mavenToken = await this.validateMavenToken(tokenString);
if (mavenToken && (!protocol || protocol === 'maven')) {
return mavenToken;
}
}
// Try OCI JWT
const ociToken = await this.validateOciToken(tokenString);
if (ociToken && (!protocol || protocol === 'oci')) {
return ociToken;
}
return null;
public async validateComposerToken(token: string): Promise<IAuthToken | null> {
return this.provider.validateToken(token, 'composer');
}
/**
* Check if token has permission for an action
* @param token - Auth token
* @param resource - Resource being accessed (e.g., "package:foo" or "repository:bar")
* @param action - Action being performed (read, write, push, pull, delete)
* @returns true if authorized
* Revoke a Composer token
* @param token - Composer UUID token
*/
public async authorize(
token: IAuthToken | null,
resource: string,
action: string
): Promise<boolean> {
if (!token) {
return false;
}
// Check readonly flag
if (token.readonly && ['write', 'push', 'delete'].includes(action)) {
return false;
}
// Check scopes
for (const scope of token.scopes) {
if (this.matchesScope(scope, resource, action)) {
return true;
}
}
return false;
public async revokeComposerToken(token: string): Promise<void> {
return this.provider.revokeToken(token);
}
// ========================================================================
// HELPER METHODS
// CARGO TOKEN MANAGEMENT
// ========================================================================
/**
* Check if a scope matches a resource and action
* Scope format: "{protocol}:{type}:{name}:{action}"
* Examples:
* - "npm:*:*" - All NPM access
* - "npm:package:foo:*" - All actions on package foo
* - "npm:package:foo:read" - Read-only on package foo
* - "oci:repository:*:pull" - Pull from any OCI repo
* Create a Cargo token
* @param userId - User ID
* @param readonly - Whether the token is readonly
* @returns Cargo UUID token
*/
private matchesScope(scope: string, resource: string, action: string): boolean {
const scopeParts = scope.split(':');
const resourceParts = resource.split(':');
// Scope must have at least protocol:type:name:action
if (scopeParts.length < 4) {
return false;
}
const [scopeProtocol, scopeType, scopeName, scopeAction] = scopeParts;
const [resourceProtocol, resourceType, resourceName] = resourceParts;
// Check protocol
if (scopeProtocol !== '*' && scopeProtocol !== resourceProtocol) {
return false;
}
// Check type
if (scopeType !== '*' && scopeType !== resourceType) {
return false;
}
// Check name
if (scopeName !== '*' && scopeName !== resourceName) {
return false;
}
// Check action
if (scopeAction !== '*' && scopeAction !== action) {
// Map action aliases
const actionAliases: Record<string, string[]> = {
read: ['pull', 'get'],
write: ['push', 'put', 'post'],
};
const aliases = actionAliases[scopeAction] || [];
if (!aliases.includes(action)) {
return false;
}
}
return true;
public async createCargoToken(userId: string, readonly: boolean = false): Promise<string> {
return this.provider.createToken(userId, 'cargo', { readonly });
}
/**
* Convert unified scopes to OCI access array
* Validate a Cargo token
* @param token - Cargo UUID token
* @returns Auth token object or null
*/
private scopesToOciAccess(scopes: string[]): Array<{
type: string;
name: string;
actions: string[];
}> {
const access: Array<{type: string; name: string; actions: string[]}> = [];
for (const scope of scopes) {
const parts = scope.split(':');
if (parts.length >= 4 && parts[0] === 'oci') {
access.push({
type: parts[1],
name: parts[2],
actions: [parts[3]],
});
}
}
return access;
public async validateCargoToken(token: string): Promise<IAuthToken | null> {
return this.provider.validateToken(token, 'cargo');
}
/**
* Convert OCI access array to unified scopes
* Revoke a Cargo token
* @param token - Cargo UUID token
*/
private ociAccessToScopes(access: Array<{
type: string;
name: string;
actions: string[];
}>): string[] {
const scopes: string[] = [];
public async revokeCargoToken(token: string): Promise<void> {
return this.provider.revokeToken(token);
}
for (const item of access) {
for (const action of item.actions) {
scopes.push(`oci:${item.type}:${item.name}:${action}`);
}
}
// ========================================================================
// PYPI AUTHENTICATION
// ========================================================================
return scopes;
/**
* Create a PyPI token
* @param userId - User ID
* @param readonly - Whether the token is readonly
* @returns PyPI UUID token
*/
public async createPypiToken(userId: string, readonly: boolean = false): Promise<string> {
return this.provider.createToken(userId, 'pypi', { readonly });
}
/**
* Generate UUID for NPM tokens
* Validate a PyPI token
* @param token - PyPI UUID token
* @returns Auth token object or null
*/
private generateUuid(): string {
return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => {
const r = (Math.random() * 16) | 0;
const v = c === 'x' ? r : (r & 0x3) | 0x8;
return v.toString(16);
});
public async validatePypiToken(token: string): Promise<IAuthToken | null> {
return this.provider.validateToken(token, 'pypi');
}
/**
* Check if string is a valid UUID
* Revoke a PyPI token
* @param token - PyPI UUID token
*/
private isValidUuid(str: string): boolean {
const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
return uuidRegex.test(str);
public async revokePypiToken(token: string): Promise<void> {
return this.provider.revokeToken(token);
}
// ========================================================================
// RUBYGEMS AUTHENTICATION
// ========================================================================
/**
* Create a RubyGems token
* @param userId - User ID
* @param readonly - Whether the token is readonly
* @returns RubyGems UUID token
*/
public async createRubyGemsToken(userId: string, readonly: boolean = false): Promise<string> {
return this.provider.createToken(userId, 'rubygems', { readonly });
}
/**
* Hash a token for identification (SHA-512 mock)
* Validate a RubyGems token
* @param token - RubyGems UUID token
* @returns Auth token object or null
*/
private hashToken(token: string): string {
// In production, use actual SHA-512
return `sha512-${token.substring(0, 16)}...`;
public async validateRubyGemsToken(token: string): Promise<IAuthToken | null> {
return this.provider.validateToken(token, 'rubygems');
}
/**
* Revoke a RubyGems token
* @param token - RubyGems UUID token
*/
public async revokeRubyGemsToken(token: string): Promise<void> {
return this.provider.revokeToken(token);
}
}

View File

@@ -0,0 +1,393 @@
import * as crypto from 'crypto';
import type { IAuthProvider, ITokenOptions } from './interfaces.auth.js';
import type { IAuthConfig, IAuthToken, ICredentials, TRegistryProtocol } from './interfaces.core.js';
/**
* Default in-memory authentication provider.
* This is the reference implementation that stores tokens in memory.
* For production use, implement IAuthProvider with Redis, database, or external auth.
*/
export class DefaultAuthProvider implements IAuthProvider {
private tokenStore: Map<string, IAuthToken> = new Map();
private userCredentials: Map<string, string> = new Map(); // username -> password hash (mock)
constructor(private config: IAuthConfig) {}
/**
* Initialize the auth provider
*/
public async init(): Promise<void> {
// Initialize token store (in-memory for now)
// In production, this could be Redis or a database
}
// ========================================================================
// IAuthProvider Implementation
// ========================================================================
/**
* Authenticate user credentials
*/
public async authenticate(credentials: ICredentials): Promise<string | null> {
// Mock authentication - in production, verify against database/LDAP
const storedPassword = this.userCredentials.get(credentials.username);
if (!storedPassword) {
// Auto-register for testing (remove in production)
this.userCredentials.set(credentials.username, credentials.password);
return credentials.username;
}
if (storedPassword === credentials.password) {
return credentials.username;
}
return null;
}
/**
* Validate any token (NPM, Maven, OCI, PyPI, RubyGems, Composer, Cargo)
*/
public async validateToken(
tokenString: string,
protocol?: TRegistryProtocol
): Promise<IAuthToken | null> {
// OCI uses JWT (contains dots), not UUID - check first if OCI is expected
if (protocol === 'oci' || tokenString.includes('.')) {
const ociToken = await this.validateOciToken(tokenString);
if (ociToken && (!protocol || protocol === 'oci')) {
return ociToken;
}
// If protocol was explicitly OCI but validation failed, return null
if (protocol === 'oci') {
return null;
}
}
// UUID-based tokens: single O(1) Map lookup
if (this.isValidUuid(tokenString)) {
const authToken = this.tokenStore.get(tokenString);
if (authToken) {
// If protocol specified, verify it matches
if (protocol && authToken.type !== protocol) {
return null;
}
// Check expiration
if (authToken.expiresAt && authToken.expiresAt < new Date()) {
this.tokenStore.delete(tokenString);
return null;
}
return authToken;
}
}
return null;
}
/**
* Create a new token for a user
*/
public async createToken(
userId: string,
protocol: TRegistryProtocol,
options?: ITokenOptions
): Promise<string> {
// OCI tokens use JWT
if (protocol === 'oci') {
return this.createOciToken(userId, options?.scopes || ['oci:*:*:*'], options?.expiresIn || 3600);
}
// All other protocols use UUID tokens
const token = this.generateUuid();
const scopes = options?.scopes || (options?.readonly
? [`${protocol}:*:*:read`]
: [`${protocol}:*:*:*`]);
const authToken: IAuthToken = {
type: protocol,
userId,
scopes,
readonly: options?.readonly,
expiresAt: options?.expiresIn ? new Date(Date.now() + options.expiresIn * 1000) : undefined,
metadata: {
created: new Date().toISOString(),
},
};
this.tokenStore.set(token, authToken);
return token;
}
/**
* Revoke a token
*/
public async revokeToken(token: string): Promise<void> {
this.tokenStore.delete(token);
}
/**
* Check if token has permission for an action
*/
public async authorize(
token: IAuthToken | null,
resource: string,
action: string
): Promise<boolean> {
if (!token) {
return false;
}
// Check readonly flag
if (token.readonly && ['write', 'push', 'delete'].includes(action)) {
return false;
}
// Check scopes
for (const scope of token.scopes) {
if (this.matchesScope(scope, resource, action)) {
return true;
}
}
return false;
}
/**
* List all tokens for a user
*/
public async listUserTokens(userId: string): Promise<Array<{
key: string;
readonly: boolean;
created: string;
protocol?: TRegistryProtocol;
}>> {
const tokens: Array<{key: string; readonly: boolean; created: string; protocol?: TRegistryProtocol}> = [];
for (const [token, authToken] of this.tokenStore.entries()) {
if (authToken.userId === userId) {
tokens.push({
key: this.hashToken(token),
readonly: authToken.readonly || false,
created: authToken.metadata?.created || 'unknown',
protocol: authToken.type,
});
}
}
return tokens;
}
// ========================================================================
// OCI JWT Token Methods
// ========================================================================
/**
* Create an OCI JWT token
*/
private async createOciToken(
userId: string,
scopes: string[],
expiresIn: number = 3600
): Promise<string> {
if (!this.config.ociTokens?.enabled) {
throw new Error('OCI tokens are not enabled');
}
const now = Math.floor(Date.now() / 1000);
const payload = {
iss: this.config.ociTokens.realm,
sub: userId,
aud: this.config.ociTokens.service,
exp: now + expiresIn,
nbf: now,
iat: now,
access: this.scopesToOciAccess(scopes),
};
// Create JWT with HMAC-SHA256 signature
const header = { alg: 'HS256', typ: 'JWT' };
const headerB64 = Buffer.from(JSON.stringify(header)).toString('base64url');
const payloadB64 = Buffer.from(JSON.stringify(payload)).toString('base64url');
const signature = crypto
.createHmac('sha256', this.config.jwtSecret)
.update(`${headerB64}.${payloadB64}`)
.digest('base64url');
return `${headerB64}.${payloadB64}.${signature}`;
}
/**
* Validate an OCI JWT token
*/
private async validateOciToken(jwt: string): Promise<IAuthToken | null> {
try {
const parts = jwt.split('.');
if (parts.length !== 3) {
return null;
}
const [headerB64, payloadB64, signatureB64] = parts;
// Verify signature
const expectedSignature = crypto
.createHmac('sha256', this.config.jwtSecret)
.update(`${headerB64}.${payloadB64}`)
.digest('base64url');
if (signatureB64 !== expectedSignature) {
return null;
}
// Decode and parse payload
const payload = JSON.parse(Buffer.from(payloadB64, 'base64url').toString('utf-8'));
// Check expiration
const now = Math.floor(Date.now() / 1000);
if (payload.exp && payload.exp < now) {
return null;
}
// Check not-before time
if (payload.nbf && payload.nbf > now) {
return null;
}
// Convert to unified token format
const scopes = this.ociAccessToScopes(payload.access || []);
return {
type: 'oci',
userId: payload.sub,
scopes,
expiresAt: payload.exp ? new Date(payload.exp * 1000) : undefined,
metadata: {
iss: payload.iss,
aud: payload.aud,
},
};
} catch (error) {
return null;
}
}
// ========================================================================
// Helper Methods
// ========================================================================
/**
* Check if a scope matches a resource and action
*/
private matchesScope(scope: string, resource: string, action: string): boolean {
const scopeParts = scope.split(':');
const resourceParts = resource.split(':');
// Scope must have at least protocol:type:name:action
if (scopeParts.length < 4) {
return false;
}
const [scopeProtocol, scopeType, scopeName, scopeAction] = scopeParts;
const [resourceProtocol, resourceType, resourceName] = resourceParts;
// Check protocol
if (scopeProtocol !== '*' && scopeProtocol !== resourceProtocol) {
return false;
}
// Check type
if (scopeType !== '*' && scopeType !== resourceType) {
return false;
}
// Check name
if (scopeName !== '*' && scopeName !== resourceName) {
return false;
}
// Check action
if (scopeAction !== '*' && scopeAction !== action) {
// Map action aliases
const actionAliases: Record<string, string[]> = {
read: ['pull', 'get'],
write: ['push', 'put', 'post'],
};
const aliases = actionAliases[scopeAction] || [];
if (!aliases.includes(action)) {
return false;
}
}
return true;
}
/**
* Convert unified scopes to OCI access array
*/
private scopesToOciAccess(scopes: string[]): Array<{
type: string;
name: string;
actions: string[];
}> {
const access: Array<{type: string; name: string; actions: string[]}> = [];
for (const scope of scopes) {
const parts = scope.split(':');
if (parts.length >= 4 && parts[0] === 'oci') {
access.push({
type: parts[1],
name: parts[2],
actions: [parts[3]],
});
}
}
return access;
}
/**
* Convert OCI access array to unified scopes
*/
private ociAccessToScopes(access: Array<{
type: string;
name: string;
actions: string[];
}>): string[] {
const scopes: string[] = [];
for (const item of access) {
for (const action of item.actions) {
scopes.push(`oci:${item.type}:${item.name}:${action}`);
}
}
return scopes;
}
/**
* Generate UUID for tokens
*/
private generateUuid(): string {
return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => {
const r = (Math.random() * 16) | 0;
const v = c === 'x' ? r : (r & 0x3) | 0x8;
return v.toString(16);
});
}
/**
* Check if string is a valid UUID
*/
private isValidUuid(str: string): boolean {
const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
return uuidRegex.test(str);
}
/**
* Hash a token for identification
*/
private hashToken(token: string): string {
return `sha512-${token.substring(0, 16)}...`;
}
}

File diff suppressed because it is too large Load Diff

34
ts/core/helpers.buffer.ts Normal file
View File

@@ -0,0 +1,34 @@
/**
* Shared buffer utilities for consistent binary data handling across all registry types.
*
* This module addresses the common issue where `Buffer.isBuffer(Uint8Array)` returns `false`,
* which can cause data handling bugs when binary data arrives as Uint8Array instead of Buffer.
*/
/**
* Check if value is binary data (Buffer or Uint8Array)
*/
export function isBinaryData(value: unknown): value is Buffer | Uint8Array {
return Buffer.isBuffer(value) || value instanceof Uint8Array;
}
/**
* Convert any binary-like data to Buffer.
* Handles Buffer, Uint8Array, string, and objects.
*
* @param data - The data to convert to Buffer
* @returns A Buffer containing the data
*/
export function toBuffer(data: unknown): Buffer {
if (Buffer.isBuffer(data)) {
return data;
}
if (data instanceof Uint8Array) {
return Buffer.from(data);
}
if (typeof data === 'string') {
return Buffer.from(data, 'utf-8');
}
// Fallback: serialize object to JSON
return Buffer.from(JSON.stringify(data));
}

63
ts/core/helpers.stream.ts Normal file
View File

@@ -0,0 +1,63 @@
import * as crypto from 'crypto';
/**
* Convert Buffer, Uint8Array, string, or JSON object to a ReadableStream<Uint8Array>.
*/
export function toReadableStream(data: Buffer | Uint8Array | string | object): ReadableStream<Uint8Array> {
const buf = Buffer.isBuffer(data)
? data
: data instanceof Uint8Array
? Buffer.from(data)
: typeof data === 'string'
? Buffer.from(data, 'utf-8')
: Buffer.from(JSON.stringify(data), 'utf-8');
return new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(new Uint8Array(buf));
controller.close();
},
});
}
/**
* Consume a ReadableStream into a Buffer.
*/
export async function streamToBuffer(stream: ReadableStream<Uint8Array>): Promise<Buffer> {
const reader = stream.getReader();
const chunks: Uint8Array[] = [];
while (true) {
const { done, value } = await reader.read();
if (done) break;
if (value) chunks.push(value);
}
return Buffer.concat(chunks);
}
/**
* Consume a ReadableStream into a parsed JSON object.
*/
export async function streamToJson<T = any>(stream: ReadableStream<Uint8Array>): Promise<T> {
const buf = await streamToBuffer(stream);
return JSON.parse(buf.toString('utf-8'));
}
/**
* Create a TransformStream that incrementally hashes data passing through.
* Data flows through unchanged; the digest is available after the stream completes.
*/
export function createHashTransform(algorithm: string = 'sha256'): {
transform: TransformStream<Uint8Array, Uint8Array>;
getDigest: () => string;
} {
const hash = crypto.createHash(algorithm);
const transform = new TransformStream<Uint8Array, Uint8Array>({
transform(chunk, controller) {
hash.update(chunk);
controller.enqueue(chunk);
},
});
return {
transform,
getDigest: () => hash.digest('hex'),
};
}

View File

@@ -2,9 +2,19 @@
* Core registry infrastructure exports
*/
// Interfaces
// Core interfaces
export * from './interfaces.core.js';
// Auth interfaces and provider
export * from './interfaces.auth.js';
export { DefaultAuthProvider } from './classes.defaultauthprovider.js';
// Storage interfaces and hooks
export * from './interfaces.storage.js';
// Stream helpers
export { toReadableStream, streamToBuffer, streamToJson, createHashTransform } from './helpers.stream.js';
// Classes
export { BaseRegistry } from './classes.baseregistry.js';
export { RegistryStorage } from './classes.registrystorage.js';

View File

@@ -0,0 +1,91 @@
import type { IAuthToken, ICredentials, TRegistryProtocol } from './interfaces.core.js';
/**
* Options for creating a token
*/
export interface ITokenOptions {
/** Whether the token is readonly */
readonly?: boolean;
/** Permission scopes */
scopes?: string[];
/** Expiration time in seconds */
expiresIn?: number;
}
/**
* Pluggable authentication provider interface.
* Implement this to integrate external auth systems (LDAP, OAuth, SSO, OIDC).
*
* @example
* ```typescript
* class LdapAuthProvider implements IAuthProvider {
* constructor(private ldap: LdapClient, private redis: RedisClient) {}
*
* async authenticate(credentials: ICredentials): Promise<string | null> {
* return await this.ldap.bind(credentials.username, credentials.password);
* }
*
* async validateToken(token: string): Promise<IAuthToken | null> {
* return await this.redis.get(`token:${token}`);
* }
* // ...
* }
* ```
*/
export interface IAuthProvider {
/**
* Initialize the auth provider (optional)
*/
init?(): Promise<void>;
/**
* Authenticate user credentials (login flow)
* @param credentials - Username and password
* @returns User ID on success, null on failure
*/
authenticate(credentials: ICredentials): Promise<string | null>;
/**
* Validate an existing token
* @param token - Token string (UUID or JWT)
* @param protocol - Optional protocol hint for optimization
* @returns Auth token info or null if invalid
*/
validateToken(token: string, protocol?: TRegistryProtocol): Promise<IAuthToken | null>;
/**
* Create a new token for a user
* @param userId - User ID
* @param protocol - Protocol type (npm, oci, maven, etc.)
* @param options - Token options (readonly, scopes, expiration)
* @returns Token string
*/
createToken(userId: string, protocol: TRegistryProtocol, options?: ITokenOptions): Promise<string>;
/**
* Revoke a token
* @param token - Token string to revoke
*/
revokeToken(token: string): Promise<void>;
/**
* Check if user has permission for an action
* @param token - Auth token (or null for anonymous)
* @param resource - Resource being accessed (e.g., "npm:package:lodash")
* @param action - Action being performed (read, write, push, pull, delete)
* @returns true if authorized
*/
authorize(token: IAuthToken | null, resource: string, action: string): Promise<boolean>;
/**
* List all tokens for a user (optional)
* @param userId - User ID
* @returns List of token info
*/
listUserTokens?(userId: string): Promise<Array<{
key: string;
readonly: boolean;
created: string;
protocol?: TRegistryProtocol;
}>>;
}

View File

@@ -2,10 +2,15 @@
* Core interfaces for the composable registry system
*/
import type * as plugins from '../plugins.js';
import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
import type { IAuthProvider } from './interfaces.auth.js';
import type { IStorageHooks } from './interfaces.storage.js';
/**
* Registry protocol types
*/
export type TRegistryProtocol = 'oci' | 'npm' | 'maven';
export type TRegistryProtocol = 'oci' | 'npm' | 'maven' | 'cargo' | 'composer' | 'pypi' | 'rubygems';
/**
* Unified action types across protocols
@@ -40,14 +45,9 @@ export interface ICredentials {
/**
* Storage backend configuration
* Extends IS3Descriptor from @tsclass/tsclass with bucketName
*/
export interface IStorageConfig {
accessKey: string;
accessSecret: string;
endpoint: string;
port?: number;
useSsl?: boolean;
region?: string;
export interface IStorageConfig extends plugins.tsclass.storage.IS3Descriptor {
bucketName: string;
}
@@ -70,6 +70,16 @@ export interface IAuthConfig {
realm: string;
service: string;
};
/** PyPI token settings */
pypiTokens?: {
enabled: boolean;
defaultReadonly?: boolean;
};
/** RubyGems token settings */
rubygemsTokens?: {
enabled: boolean;
defaultReadonly?: boolean;
};
}
/**
@@ -78,6 +88,7 @@ export interface IAuthConfig {
export interface IProtocolConfig {
enabled: boolean;
basePath: string;
registryUrl?: string;
features?: Record<string, boolean>;
}
@@ -87,9 +98,34 @@ export interface IProtocolConfig {
export interface IRegistryConfig {
storage: IStorageConfig;
auth: IAuthConfig;
/**
* Custom authentication provider.
* If not provided, uses the default in-memory auth provider.
* Implement IAuthProvider to integrate LDAP, OAuth, SSO, etc.
*/
authProvider?: IAuthProvider;
/**
* Storage event hooks for quota tracking, audit logging, etc.
* Called before/after storage operations.
*/
storageHooks?: IStorageHooks;
/**
* Dynamic upstream configuration provider.
* Called per-request to resolve which upstream registries to use.
* Use StaticUpstreamProvider for simple static configurations.
*/
upstreamProvider?: IUpstreamProvider;
oci?: IProtocolConfig;
npm?: IProtocolConfig;
maven?: IProtocolConfig;
cargo?: IProtocolConfig;
composer?: IProtocolConfig;
pypi?: IProtocolConfig;
rubygems?: IProtocolConfig;
}
/**
@@ -125,6 +161,21 @@ export interface IStorageBackend {
* Get object metadata
*/
getMetadata(key: string): Promise<Record<string, string> | null>;
/**
* Get an object as a ReadableStream. Returns null if not found.
*/
getObjectStream?(key: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null>;
/**
* Store an object from a ReadableStream.
*/
putObjectStream?(key: string, stream: ReadableStream<Uint8Array>): Promise<void>;
/**
* Get object size without reading data (S3 HEAD request).
*/
getObjectSize?(key: string): Promise<number | null>;
}
/**
@@ -138,6 +189,24 @@ export interface IRegistryError {
}>;
}
/**
* Actor information - identifies who is performing the request
*/
export interface IRequestActor {
/** User ID (from validated token) */
userId?: string;
/** Token ID/hash for audit purposes */
tokenId?: string;
/** Client IP address */
ip?: string;
/** Client User-Agent */
userAgent?: string;
/** Organization ID (for multi-tenant setups) */
orgId?: string;
/** Session ID */
sessionId?: string;
}
/**
* Base request context
*/
@@ -147,14 +216,28 @@ export interface IRequestContext {
headers: Record<string, string>;
query: Record<string, string>;
body?: any;
/**
* Raw request body as bytes. MUST be provided for content-addressable operations
* (OCI manifests, blobs) to ensure digest calculation matches client expectations.
* If not provided, falls back to 'body' field.
*/
rawBody?: Buffer;
token?: string;
/**
* Actor information - identifies who is performing the request.
* Populated after authentication for audit logging, quota enforcement, etc.
*/
actor?: IRequestActor;
}
/**
* Base response structure
* Base response structure.
* `body` is always a `ReadableStream<Uint8Array>` at the public API boundary.
* Internal handlers may return Buffer/string/object — the SmartRegistry orchestrator
* auto-wraps them via `toReadableStream()` before returning to the caller.
*/
export interface IResponse {
status: number;
headers: Record<string, string>;
body?: any;
body?: ReadableStream<Uint8Array> | any;
}

View File

@@ -0,0 +1,130 @@
import type { TRegistryProtocol } from './interfaces.core.js';
/**
* Actor information from request context
*/
export interface IStorageActor {
userId?: string;
tokenId?: string;
ip?: string;
userAgent?: string;
orgId?: string;
sessionId?: string;
}
/**
* Metadata about the storage operation
*/
export interface IStorageMetadata {
/** Content type of the object */
contentType?: string;
/** Size in bytes */
size?: number;
/** Content digest (e.g., sha256:abc123) */
digest?: string;
/** Package/artifact name */
packageName?: string;
/** Version */
version?: string;
}
/**
* Context passed to storage hooks
*/
export interface IStorageHookContext {
/** Type of operation */
operation: 'put' | 'delete' | 'get';
/** Storage key/path */
key: string;
/** Protocol that triggered this operation */
protocol: TRegistryProtocol;
/** Actor who performed the operation (if known) */
actor?: IStorageActor;
/** Metadata about the object */
metadata?: IStorageMetadata;
/** Timestamp of the operation */
timestamp: Date;
}
/**
* Result from a beforePut hook that can modify the operation
*/
export interface IBeforePutResult {
/** Whether to allow the operation */
allowed: boolean;
/** Optional reason for rejection */
reason?: string;
/** Optional modified metadata */
metadata?: IStorageMetadata;
}
/**
* Result from a beforeDelete hook
*/
export interface IBeforeDeleteResult {
/** Whether to allow the operation */
allowed: boolean;
/** Optional reason for rejection */
reason?: string;
}
/**
* Storage event hooks for quota tracking, audit logging, cache invalidation, etc.
*
* @example
* ```typescript
* const quotaHooks: IStorageHooks = {
* async beforePut(context) {
* const quota = await getQuota(context.actor?.orgId);
* const currentUsage = await getUsage(context.actor?.orgId);
* if (currentUsage + (context.metadata?.size || 0) > quota) {
* return { allowed: false, reason: 'Quota exceeded' };
* }
* return { allowed: true };
* },
*
* async afterPut(context) {
* await updateUsage(context.actor?.orgId, context.metadata?.size || 0);
* await auditLog('storage.put', context);
* },
*
* async afterDelete(context) {
* await invalidateCache(context.key);
* }
* };
* ```
*/
export interface IStorageHooks {
/**
* Called before storing an object.
* Return { allowed: false } to reject the operation.
* Use for quota checks, virus scanning, validation, etc.
*/
beforePut?(context: IStorageHookContext): Promise<IBeforePutResult>;
/**
* Called after successfully storing an object.
* Use for quota tracking, audit logging, notifications, etc.
*/
afterPut?(context: IStorageHookContext): Promise<void>;
/**
* Called before deleting an object.
* Return { allowed: false } to reject the operation.
* Use for preventing deletion of protected resources.
*/
beforeDelete?(context: IStorageHookContext): Promise<IBeforeDeleteResult>;
/**
* Called after successfully deleting an object.
* Use for quota updates, audit logging, cache invalidation, etc.
*/
afterDelete?(context: IStorageHookContext): Promise<void>;
/**
* Called after reading an object.
* Use for access logging, analytics, etc.
* Note: This is called even for cache hits.
*/
afterGet?(context: IStorageHookContext): Promise<void>;
}

View File

@@ -1,6 +1,6 @@
/**
* @push.rocks/smartregistry
* Composable registry supporting OCI, NPM, and Maven protocols
* Composable registry supporting OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems protocols
*/
// Main orchestrator
@@ -9,6 +9,9 @@ export { SmartRegistry } from './classes.smartregistry.js';
// Core infrastructure
export * from './core/index.js';
// Upstream infrastructure
export * from './upstream/index.js';
// OCI Registry
export * from './oci/index.js';
@@ -17,3 +20,15 @@ export * from './npm/index.js';
// Maven Registry
export * from './maven/index.js';
// Cargo Registry
export * from './cargo/index.js';
// Composer Registry
export * from './composer/index.js';
// PyPI Registry
export * from './pypi/index.js';
// RubyGems Registry
export * from './rubygems/index.js';

View File

@@ -6,7 +6,9 @@
import { BaseRegistry } from '../core/classes.baseregistry.js';
import type { RegistryStorage } from '../core/classes.registrystorage.js';
import type { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js';
import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from '../core/interfaces.core.js';
import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
import { toBuffer } from '../core/helpers.buffer.js';
import type { IMavenCoordinate, IMavenMetadata, IChecksums } from './interfaces.maven.js';
import {
pathToGAV,
@@ -20,6 +22,7 @@ import {
extractGAVFromPom,
gavToPath,
} from './helpers.maven.js';
import { MavenUpstream } from './classes.mavenupstream.js';
/**
* Maven Registry class
@@ -30,18 +33,64 @@ export class MavenRegistry extends BaseRegistry {
private authManager: AuthManager;
private basePath: string = '/maven';
private registryUrl: string;
private upstreamProvider: IUpstreamProvider | null = null;
constructor(
storage: RegistryStorage,
authManager: AuthManager,
basePath: string,
registryUrl: string
registryUrl: string,
upstreamProvider?: IUpstreamProvider
) {
super();
this.storage = storage;
this.authManager = authManager;
this.basePath = basePath;
this.registryUrl = registryUrl;
this.upstreamProvider = upstreamProvider || null;
}
/**
* Extract scope from Maven coordinates.
* For Maven, the groupId is the scope.
* @example "com.example" from "com.example:my-lib"
*/
private extractScope(groupId: string): string | null {
return groupId || null;
}
/**
* Get upstream for a specific request.
* Calls the provider to resolve upstream config dynamically.
*/
private async getUpstreamForRequest(
resource: string,
resourceType: string,
method: string,
actor?: IRequestActor
): Promise<MavenUpstream | null> {
if (!this.upstreamProvider) return null;
// For Maven, resource is "groupId:artifactId"
const [groupId] = resource.split(':');
const config = await this.upstreamProvider.resolveUpstreamConfig({
protocol: 'maven',
resource,
scope: this.extractScope(groupId),
actor,
method,
resourceType,
});
if (!config?.enabled) return null;
return new MavenUpstream(config);
}
/**
* Clean up resources (timers, connections, etc.)
*/
public destroy(): void {
// No persistent upstream to clean up with dynamic provider
}
public async init(): Promise<void> {
@@ -61,18 +110,34 @@ export class MavenRegistry extends BaseRegistry {
let token: IAuthToken | null = null;
if (authHeader) {
const tokenString = authHeader.replace(/^(Bearer|Basic)\s+/i, '');
// For now, try to validate as Maven token (reuse npm token type)
token = await this.authManager.validateToken(tokenString, 'maven');
if (/^Basic\s+/i.test(authHeader)) {
// Maven sends Basic Auth: base64(username:password) — extract the password as token
const base64 = authHeader.replace(/^Basic\s+/i, '');
const decoded = Buffer.from(base64, 'base64').toString('utf-8');
const colonIndex = decoded.indexOf(':');
const password = colonIndex >= 0 ? decoded.substring(colonIndex + 1) : decoded;
token = await this.authManager.validateToken(password, 'maven');
} else {
const tokenString = authHeader.replace(/^Bearer\s+/i, '');
token = await this.authManager.validateToken(tokenString, 'maven');
}
}
// Build actor from context and validated token
const actor: IRequestActor = {
...context.actor,
userId: token?.userId,
ip: context.headers['x-forwarded-for'] || context.headers['X-Forwarded-For'],
userAgent: context.headers['user-agent'] || context.headers['User-Agent'],
};
// Parse path to determine request type
const coordinate = pathToGAV(path);
if (!coordinate) {
// Not a valid artifact path, could be metadata or root
if (path.endsWith('/maven-metadata.xml')) {
return this.handleMetadataRequest(context.method, path, token);
return this.handleMetadataRequest(context.method, path, token, actor);
}
return {
@@ -85,11 +150,11 @@ export class MavenRegistry extends BaseRegistry {
// Check if it's a checksum file
if (coordinate.extension === 'md5' || coordinate.extension === 'sha1' ||
coordinate.extension === 'sha256' || coordinate.extension === 'sha512') {
return this.handleChecksumRequest(context.method, coordinate, token);
return this.handleChecksumRequest(context.method, coordinate, token, path);
}
// Handle artifact requests (JAR, POM, WAR, etc.)
return this.handleArtifactRequest(context.method, coordinate, token, context.body);
return this.handleArtifactRequest(context.method, coordinate, token, context.body, actor);
}
protected async checkPermission(
@@ -109,7 +174,8 @@ export class MavenRegistry extends BaseRegistry {
method: string,
coordinate: IMavenCoordinate,
token: IAuthToken | null,
body?: Buffer | any
body?: Buffer | any,
actor?: IRequestActor
): Promise<IResponse> {
const { groupId, artifactId, version } = coordinate;
const filename = buildFilename(coordinate);
@@ -118,19 +184,9 @@ export class MavenRegistry extends BaseRegistry {
switch (method) {
case 'GET':
case 'HEAD':
// Read permission required
if (!await this.checkPermission(token, resource, 'read')) {
return {
status: 401,
headers: {
'WWW-Authenticate': `Bearer realm="${this.basePath}",service="maven-registry"`,
},
body: { error: 'UNAUTHORIZED', message: 'Authentication required' },
};
}
// Maven repositories typically allow anonymous reads
return method === 'GET'
? this.getArtifact(groupId, artifactId, version, filename)
? this.getArtifact(groupId, artifactId, version, filename, actor)
: this.headArtifact(groupId, artifactId, version, filename);
case 'PUT':
@@ -181,29 +237,30 @@ export class MavenRegistry extends BaseRegistry {
private async handleChecksumRequest(
method: string,
coordinate: IMavenCoordinate,
token: IAuthToken | null
token: IAuthToken | null,
path: string
): Promise<IResponse> {
const { groupId, artifactId, version, extension } = coordinate;
const resource = `${groupId}:${artifactId}`;
// Checksums follow the same permissions as their artifacts
// Checksums follow the same permissions as their artifacts (public read)
if (method === 'GET' || method === 'HEAD') {
if (!await this.checkPermission(token, resource, 'read')) {
return {
status: 401,
headers: {
'WWW-Authenticate': `Bearer realm="${this.basePath}",service="maven-registry"`,
},
body: { error: 'UNAUTHORIZED', message: 'Authentication required' },
};
}
return this.getChecksum(groupId, artifactId, version, coordinate, path);
}
return this.getChecksum(groupId, artifactId, version, coordinate);
// Accept PUT silently — Maven deploy-plugin uploads checksums alongside artifacts,
// but our registry auto-generates them, so we just acknowledge the upload
if (method === 'PUT') {
return {
status: 200,
headers: {},
body: { status: 'ok' },
};
}
return {
status: 405,
headers: { 'Allow': 'GET, HEAD' },
headers: { 'Allow': 'GET, HEAD, PUT' },
body: { error: 'METHOD_NOT_ALLOWED', message: 'Checksums are auto-generated' },
};
}
@@ -211,7 +268,8 @@ export class MavenRegistry extends BaseRegistry {
private async handleMetadataRequest(
method: string,
path: string,
token: IAuthToken | null
token: IAuthToken | null,
actor?: IRequestActor
): Promise<IResponse> {
// Parse path to extract groupId and artifactId
// Path format: /com/example/my-lib/maven-metadata.xml
@@ -232,12 +290,22 @@ export class MavenRegistry extends BaseRegistry {
if (method === 'GET') {
// Metadata is usually public (read permission optional)
// Some registries allow anonymous metadata access
return this.getMetadata(groupId, artifactId);
return this.getMetadata(groupId, artifactId, actor);
}
// Accept PUT silently — Maven deploy-plugin uploads maven-metadata.xml,
// but our registry auto-generates it, so we just acknowledge the upload
if (method === 'PUT') {
return {
status: 200,
headers: {},
body: { status: 'ok' },
};
}
return {
status: 405,
headers: { 'Allow': 'GET' },
headers: { 'Allow': 'GET, PUT' },
body: { error: 'METHOD_NOT_ALLOWED', message: 'Metadata is auto-generated' },
};
}
@@ -250,9 +318,42 @@ export class MavenRegistry extends BaseRegistry {
groupId: string,
artifactId: string,
version: string,
filename: string
filename: string,
actor?: IRequestActor
): Promise<IResponse> {
const data = await this.storage.getMavenArtifact(groupId, artifactId, version, filename);
// Try local storage first (streaming)
const streamResult = await this.storage.getMavenArtifactStream(groupId, artifactId, version, filename);
if (streamResult) {
const ext = filename.split('.').pop() || '';
const contentType = this.getContentType(ext);
return {
status: 200,
headers: {
'Content-Type': contentType,
'Content-Length': streamResult.size.toString(),
},
body: streamResult.stream,
};
}
// Try upstream if not found locally
let data: Buffer | null = null;
const resource = `${groupId}:${artifactId}`;
const upstream = await this.getUpstreamForRequest(resource, 'artifact', 'GET', actor);
if (upstream) {
// Parse the filename to extract extension and classifier
const { extension, classifier } = this.parseFilename(filename, artifactId, version);
if (extension) {
data = await upstream.fetchArtifact(groupId, artifactId, version, extension, classifier);
if (data) {
// Cache the artifact locally
await this.storage.putMavenArtifact(groupId, artifactId, version, filename, data);
// Generate and store checksums
const checksums = await calculateChecksums(data);
await this.storeChecksums(groupId, artifactId, version, filename, checksums);
}
}
}
if (!data) {
return {
@@ -315,7 +416,7 @@ export class MavenRegistry extends BaseRegistry {
coordinate: IMavenCoordinate,
body: Buffer | any
): Promise<IResponse> {
const data = Buffer.isBuffer(body) ? body : Buffer.from(JSON.stringify(body));
const data = toBuffer(body);
// Validate POM if uploading .pom file
if (coordinate.extension === 'pom') {
@@ -402,9 +503,14 @@ export class MavenRegistry extends BaseRegistry {
groupId: string,
artifactId: string,
version: string,
coordinate: IMavenCoordinate
coordinate: IMavenCoordinate,
fullPath: string
): Promise<IResponse> {
const checksumFilename = buildFilename(coordinate);
// Extract the filename from the full path (last component)
// The fullPath might be something like /com/example/test/test-artifact/1.0.0/test-artifact-1.0.0.jar.md5
const pathParts = fullPath.split('/');
const checksumFilename = pathParts[pathParts.length - 1];
const data = await this.storage.getMavenArtifact(groupId, artifactId, version, checksumFilename);
if (!data) {
@@ -474,8 +580,22 @@ export class MavenRegistry extends BaseRegistry {
// METADATA OPERATIONS
// ========================================================================
private async getMetadata(groupId: string, artifactId: string): Promise<IResponse> {
const metadataBuffer = await this.storage.getMavenMetadata(groupId, artifactId);
private async getMetadata(groupId: string, artifactId: string, actor?: IRequestActor): Promise<IResponse> {
let metadataBuffer = await this.storage.getMavenMetadata(groupId, artifactId);
// Try upstream if not found locally
if (!metadataBuffer) {
const resource = `${groupId}:${artifactId}`;
const upstream = await this.getUpstreamForRequest(resource, 'metadata', 'GET', actor);
if (upstream) {
const upstreamMetadata = await upstream.fetchMetadata(groupId, artifactId);
if (upstreamMetadata) {
metadataBuffer = Buffer.from(upstreamMetadata, 'utf-8');
// Cache the metadata locally
await this.storage.putMavenMetadata(groupId, artifactId, metadataBuffer);
}
}
}
if (!metadataBuffer) {
// Generate empty metadata if none exists
@@ -567,10 +687,8 @@ export class MavenRegistry extends BaseRegistry {
const xml = generateMetadataXml(metadata);
await this.storage.putMavenMetadata(groupId, artifactId, Buffer.from(xml, 'utf-8'));
// Also store checksums for metadata
const checksums = await calculateChecksums(Buffer.from(xml, 'utf-8'));
const metadataFilename = 'maven-metadata.xml';
await this.storeChecksums(groupId, artifactId, '', metadataFilename, checksums);
// Note: Checksums for maven-metadata.xml are optional and not critical
// They would need special handling since metadata uses a different storage path
}
// ========================================================================
@@ -593,4 +711,41 @@ export class MavenRegistry extends BaseRegistry {
return contentTypes[extension] || 'application/octet-stream';
}
/**
* Parse a Maven filename to extract extension and classifier.
* Filename format: {artifactId}-{version}[-{classifier}].{extension}
*/
private parseFilename(
filename: string,
artifactId: string,
version: string
): { extension: string; classifier?: string } {
const prefix = `${artifactId}-${version}`;
if (!filename.startsWith(prefix)) {
// Fallback: just get the extension
const lastDot = filename.lastIndexOf('.');
return { extension: lastDot > 0 ? filename.slice(lastDot + 1) : '' };
}
const remainder = filename.slice(prefix.length);
// remainder is either ".extension" or "-classifier.extension"
if (remainder.startsWith('.')) {
return { extension: remainder.slice(1) };
}
if (remainder.startsWith('-')) {
const lastDot = remainder.lastIndexOf('.');
if (lastDot > 1) {
return {
classifier: remainder.slice(1, lastDot),
extension: remainder.slice(lastDot + 1),
};
}
}
return { extension: '' };
}
}

View File

@@ -0,0 +1,220 @@
import * as plugins from '../plugins.js';
import { BaseUpstream } from '../upstream/classes.baseupstream.js';
import type {
IProtocolUpstreamConfig,
IUpstreamFetchContext,
IUpstreamRegistryConfig,
} from '../upstream/interfaces.upstream.js';
import type { IMavenCoordinate } from './interfaces.maven.js';
/**
* Maven-specific upstream implementation.
*
* Handles:
* - Artifact fetching (JAR, POM, WAR, etc.)
* - Metadata fetching (maven-metadata.xml)
* - Checksum files (.md5, .sha1, .sha256, .sha512)
* - SNAPSHOT version handling
* - Content-addressable caching for release artifacts
*/
export class MavenUpstream extends BaseUpstream {
protected readonly protocolName = 'maven';
constructor(
config: IProtocolUpstreamConfig,
logger?: plugins.smartlog.Smartlog,
) {
super(config, logger);
}
/**
* Fetch an artifact from upstream registries.
*/
public async fetchArtifact(
groupId: string,
artifactId: string,
version: string,
extension: string,
classifier?: string,
): Promise<Buffer | null> {
const path = this.buildArtifactPath(groupId, artifactId, version, extension, classifier);
const resource = `${groupId}:${artifactId}`;
const context: IUpstreamFetchContext = {
protocol: 'maven',
resource,
resourceType: 'artifact',
path,
method: 'GET',
headers: {},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
return Buffer.isBuffer(result.body) ? result.body : Buffer.from(result.body);
}
/**
* Fetch maven-metadata.xml from upstream.
*/
public async fetchMetadata(groupId: string, artifactId: string, version?: string): Promise<string | null> {
const groupPath = groupId.replace(/\./g, '/');
let path: string;
if (version) {
// Version-level metadata (for SNAPSHOTs)
path = `/${groupPath}/${artifactId}/${version}/maven-metadata.xml`;
} else {
// Artifact-level metadata (lists all versions)
path = `/${groupPath}/${artifactId}/maven-metadata.xml`;
}
const resource = `${groupId}:${artifactId}`;
const context: IUpstreamFetchContext = {
protocol: 'maven',
resource,
resourceType: 'metadata',
path,
method: 'GET',
headers: {
'accept': 'application/xml, text/xml',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
if (Buffer.isBuffer(result.body)) {
return result.body.toString('utf8');
}
return typeof result.body === 'string' ? result.body : null;
}
/**
* Fetch a checksum file from upstream.
*/
public async fetchChecksum(
groupId: string,
artifactId: string,
version: string,
extension: string,
checksumType: 'md5' | 'sha1' | 'sha256' | 'sha512',
classifier?: string,
): Promise<string | null> {
const basePath = this.buildArtifactPath(groupId, artifactId, version, extension, classifier);
const path = `${basePath}.${checksumType}`;
const resource = `${groupId}:${artifactId}`;
const context: IUpstreamFetchContext = {
protocol: 'maven',
resource,
resourceType: 'checksum',
path,
method: 'GET',
headers: {
'accept': 'text/plain',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
if (Buffer.isBuffer(result.body)) {
return result.body.toString('utf8').trim();
}
return typeof result.body === 'string' ? result.body.trim() : null;
}
/**
* Check if an artifact exists in upstream (HEAD request).
*/
public async headArtifact(
groupId: string,
artifactId: string,
version: string,
extension: string,
classifier?: string,
): Promise<{ exists: boolean; size?: number; lastModified?: string } | null> {
const path = this.buildArtifactPath(groupId, artifactId, version, extension, classifier);
const resource = `${groupId}:${artifactId}`;
const context: IUpstreamFetchContext = {
protocol: 'maven',
resource,
resourceType: 'artifact',
path,
method: 'HEAD',
headers: {},
query: {},
};
const result = await this.fetch(context);
if (!result) {
return null;
}
if (!result.success) {
return { exists: false };
}
return {
exists: true,
size: result.headers['content-length'] ? parseInt(result.headers['content-length'], 10) : undefined,
lastModified: result.headers['last-modified'],
};
}
/**
* Build the path for a Maven artifact.
*/
private buildArtifactPath(
groupId: string,
artifactId: string,
version: string,
extension: string,
classifier?: string,
): string {
const groupPath = groupId.replace(/\./g, '/');
let filename = `${artifactId}-${version}`;
if (classifier) {
filename += `-${classifier}`;
}
filename += `.${extension}`;
return `/${groupPath}/${artifactId}/${version}/${filename}`;
}
/**
* Override URL building for Maven-specific handling.
*/
protected buildUpstreamUrl(
upstream: IUpstreamRegistryConfig,
context: IUpstreamFetchContext,
): string {
let baseUrl = upstream.url;
// Remove trailing slash
if (baseUrl.endsWith('/')) {
baseUrl = baseUrl.slice(0, -1);
}
return `${baseUrl}${context.path}`;
}
}

View File

@@ -65,22 +65,35 @@ export function pathToGAV(path: string): IMavenCoordinate | null {
/**
* Parse Maven artifact filename
* Example: my-lib-1.0.0-sources.jar → {classifier: 'sources', extension: 'jar'}
* Example: my-lib-1.0.0.jar.md5 → {extension: 'md5'}
*/
export function parseFilename(
filename: string,
artifactId: string,
version: string
): { classifier?: string; extension: string } | null {
// Expected format: {artifactId}-{version}[-{classifier}].{extension}
// Expected format: {artifactId}-{version}[-{classifier}].{extension}[.checksum]
const prefix = `${artifactId}-${version}`;
if (!filename.startsWith(prefix)) {
return null;
}
const remainder = filename.substring(prefix.length);
let remainder = filename.substring(prefix.length);
// Check for classifier
// Check if this is a checksum file (double extension like .jar.md5)
const checksumExtensions = ['md5', 'sha1', 'sha256', 'sha512'];
const lastDotIndex = remainder.lastIndexOf('.');
if (lastDotIndex !== -1) {
const possibleChecksum = remainder.substring(lastDotIndex + 1);
if (checksumExtensions.includes(possibleChecksum)) {
// This is a checksum file - just return the checksum extension
// The base artifact extension doesn't matter for checksum retrieval
return { extension: possibleChecksum };
}
}
// Regular artifact file parsing
const dotIndex = remainder.lastIndexOf('.');
if (dotIndex === -1) {
return null; // No extension

View File

@@ -3,5 +3,6 @@
*/
export { MavenRegistry } from './classes.mavenregistry.js';
export { MavenUpstream } from './classes.mavenupstream.js';
export * from './interfaces.maven.js';
export * from './helpers.maven.js';

View File

@@ -2,7 +2,9 @@ import { Smartlog } from '@push.rocks/smartlog';
import { BaseRegistry } from '../core/classes.baseregistry.js';
import { RegistryStorage } from '../core/classes.registrystorage.js';
import { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js';
import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from '../core/interfaces.core.js';
import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
import { NpmUpstream } from './classes.npmupstream.js';
import type {
IPackument,
INpmVersion,
@@ -25,18 +27,21 @@ export class NpmRegistry extends BaseRegistry {
private basePath: string = '/npm';
private registryUrl: string;
private logger: Smartlog;
private upstreamProvider: IUpstreamProvider | null = null;
constructor(
storage: RegistryStorage,
authManager: AuthManager,
basePath: string = '/npm',
registryUrl: string = 'http://localhost:5000/npm'
registryUrl: string = 'http://localhost:5000/npm',
upstreamProvider?: IUpstreamProvider
) {
super();
this.storage = storage;
this.authManager = authManager;
this.basePath = basePath;
this.registryUrl = registryUrl;
this.upstreamProvider = upstreamProvider || null;
// Initialize logger
this.logger = new Smartlog({
@@ -50,6 +55,50 @@ export class NpmRegistry extends BaseRegistry {
}
});
this.logger.enableConsole();
if (upstreamProvider) {
this.logger.log('info', 'NPM upstream provider configured');
}
}
/**
* Extract scope from npm package name.
* @example "@company/utils" -> "company"
* @example "lodash" -> null
*/
private extractScope(packageName: string): string | null {
if (packageName.startsWith('@')) {
const slashIndex = packageName.indexOf('/');
if (slashIndex > 1) {
return packageName.substring(1, slashIndex);
}
}
return null;
}
/**
* Get upstream for a specific request.
* Calls the provider to resolve upstream config dynamically.
*/
private async getUpstreamForRequest(
resource: string,
resourceType: string,
method: string,
actor?: IRequestActor
): Promise<NpmUpstream | null> {
if (!this.upstreamProvider) return null;
const config = await this.upstreamProvider.resolveUpstreamConfig({
protocol: 'npm',
resource,
scope: this.extractScope(resource),
actor,
method,
resourceType,
});
if (!config?.enabled) return null;
return new NpmUpstream(config, this.registryUrl, this.logger);
}
public async init(): Promise<void> {
@@ -68,6 +117,14 @@ export class NpmRegistry extends BaseRegistry {
const tokenString = authHeader?.replace(/^Bearer\s+/i, '');
const token = tokenString ? await this.authManager.validateToken(tokenString, 'npm') : null;
// Build actor context for upstream resolution
const actor: IRequestActor = {
userId: token?.userId,
ip: context.headers['x-forwarded-for'] || context.headers['x-real-ip'],
userAgent: context.headers['user-agent'],
...context.actor, // Include any pre-populated actor info
};
this.logger.log('debug', `handleRequest: ${context.method} ${path}`, {
method: context.method,
path,
@@ -98,47 +155,47 @@ export class NpmRegistry extends BaseRegistry {
// Dist-tags: /-/package/{package}/dist-tags
const distTagsMatch = path.match(/^\/-\/package\/(@?[^\/]+(?:\/[^\/]+)?)\/dist-tags(?:\/(.+))?$/);
if (distTagsMatch) {
const [, packageName, tag] = distTagsMatch;
return this.handleDistTags(context.method, packageName, tag, context.body, token);
const [, rawPkgName, tag] = distTagsMatch;
return this.handleDistTags(context.method, decodeURIComponent(rawPkgName), tag, context.body, token);
}
// Tarball download: /{package}/-/{filename}.tgz
const tarballMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)\/-\/(.+\.tgz)$/);
if (tarballMatch) {
const [, packageName, filename] = tarballMatch;
return this.handleTarballDownload(packageName, filename, token);
const [, rawPkgName, filename] = tarballMatch;
return this.handleTarballDownload(decodeURIComponent(rawPkgName), filename, token, actor);
}
// Unpublish specific version: DELETE /{package}/-/{version}
const unpublishVersionMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)\/-\/([^\/]+)$/);
if (unpublishVersionMatch && context.method === 'DELETE') {
const [, packageName, version] = unpublishVersionMatch;
console.log(`[unpublishVersionMatch] packageName=${packageName}, version=${version}`);
return this.unpublishVersion(packageName, version, token);
const [, rawPkgName, version] = unpublishVersionMatch;
this.logger.log('debug', 'unpublishVersionMatch', { packageName: decodeURIComponent(rawPkgName), version });
return this.unpublishVersion(decodeURIComponent(rawPkgName), version, token);
}
// Unpublish entire package: DELETE /{package}/-rev/{rev}
const unpublishPackageMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)\/-rev\/([^\/]+)$/);
if (unpublishPackageMatch && context.method === 'DELETE') {
const [, packageName, rev] = unpublishPackageMatch;
console.log(`[unpublishPackageMatch] packageName=${packageName}, rev=${rev}`);
return this.unpublishPackage(packageName, token);
const [, rawPkgName, rev] = unpublishPackageMatch;
this.logger.log('debug', 'unpublishPackageMatch', { packageName: decodeURIComponent(rawPkgName), rev });
return this.unpublishPackage(decodeURIComponent(rawPkgName), token);
}
// Package version: /{package}/{version}
const versionMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)\/([^\/]+)$/);
if (versionMatch) {
const [, packageName, version] = versionMatch;
console.log(`[versionMatch] matched! packageName=${packageName}, version=${version}`);
return this.handlePackageVersion(packageName, version, token);
const [, rawPkgName, version] = versionMatch;
this.logger.log('debug', 'versionMatch', { packageName: decodeURIComponent(rawPkgName), version });
return this.handlePackageVersion(decodeURIComponent(rawPkgName), version, token, actor);
}
// Package operations: /{package}
const packageMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)$/);
if (packageMatch) {
const packageName = packageMatch[1];
console.log(`[packageMatch] matched! packageName=${packageName}`);
return this.handlePackage(context.method, packageName, context.body, context.query, token);
const packageName = decodeURIComponent(packageMatch[1]);
this.logger.log('debug', 'packageMatch', { packageName });
return this.handlePackage(context.method, packageName, context.body, context.query, token, actor);
}
return {
@@ -186,11 +243,12 @@ export class NpmRegistry extends BaseRegistry {
packageName: string,
body: any,
query: Record<string, string>,
token: IAuthToken | null
token: IAuthToken | null,
actor?: IRequestActor
): Promise<IResponse> {
switch (method) {
case 'GET':
return this.getPackument(packageName, token, query);
return this.getPackument(packageName, token, query, actor);
case 'PUT':
return this.publishPackage(packageName, body, token);
case 'DELETE':
@@ -207,15 +265,34 @@ export class NpmRegistry extends BaseRegistry {
private async getPackument(
packageName: string,
token: IAuthToken | null,
query: Record<string, string>
query: Record<string, string>,
actor?: IRequestActor
): Promise<IResponse> {
const packument = await this.storage.getNpmPackument(packageName);
let packument = await this.storage.getNpmPackument(packageName);
this.logger.log('debug', `getPackument: ${packageName}`, {
packageName,
found: !!packument,
versions: packument ? Object.keys(packument.versions).length : 0
});
// If not found locally, try upstream
if (!packument) {
const upstream = await this.getUpstreamForRequest(packageName, 'packument', 'GET', actor);
if (upstream) {
this.logger.log('debug', `getPackument: fetching from upstream`, { packageName });
const upstreamPackument = await upstream.fetchPackument(packageName);
if (upstreamPackument) {
this.logger.log('debug', `getPackument: found in upstream`, {
packageName,
versions: Object.keys(upstreamPackument.versions || {}).length
});
packument = upstreamPackument;
// Optionally cache the packument locally (without tarballs)
// We don't store tarballs here - they'll be fetched on demand
}
}
}
if (!packument) {
return {
status: 404,
@@ -252,14 +329,28 @@ export class NpmRegistry extends BaseRegistry {
private async handlePackageVersion(
packageName: string,
version: string,
token: IAuthToken | null
token: IAuthToken | null,
actor?: IRequestActor
): Promise<IResponse> {
console.log(`[handlePackageVersion] packageName=${packageName}, version=${version}`);
const packument = await this.storage.getNpmPackument(packageName);
console.log(`[handlePackageVersion] packument found:`, !!packument);
this.logger.log('debug', 'handlePackageVersion', { packageName, version });
let packument = await this.storage.getNpmPackument(packageName);
this.logger.log('debug', 'handlePackageVersion packument', { found: !!packument });
if (packument) {
console.log(`[handlePackageVersion] versions:`, Object.keys(packument.versions || {}));
this.logger.log('debug', 'handlePackageVersion versions', { versions: Object.keys(packument.versions || {}) });
}
// If not found locally, try upstream
if (!packument) {
const upstream = await this.getUpstreamForRequest(packageName, 'packument', 'GET', actor);
if (upstream) {
this.logger.log('debug', 'handlePackageVersion: fetching from upstream', { packageName });
const upstreamPackument = await upstream.fetchPackument(packageName);
if (upstreamPackument) {
packument = upstreamPackument;
}
}
}
if (!packument) {
return {
status: 404,
@@ -526,10 +617,11 @@ export class NpmRegistry extends BaseRegistry {
private async handleTarballDownload(
packageName: string,
filename: string,
token: IAuthToken | null
token: IAuthToken | null,
actor?: IRequestActor
): Promise<IResponse> {
// Extract version from filename: package-name-1.0.0.tgz
const versionMatch = filename.match(/-([\d.]+(?:-[a-z0-9.]+)?)\.tgz$/);
const versionMatch = filename.match(/-([\d.]+(?:-[a-z0-9.]+)?)\.tgz$/i);
if (!versionMatch) {
return {
status: 400,
@@ -539,7 +631,40 @@ export class NpmRegistry extends BaseRegistry {
}
const version = versionMatch[1];
const tarball = await this.storage.getNpmTarball(packageName, version);
// Try local storage first (streaming)
const streamResult = await this.storage.getNpmTarballStream(packageName, version);
if (streamResult) {
return {
status: 200,
headers: {
'Content-Type': 'application/octet-stream',
'Content-Length': streamResult.size.toString(),
},
body: streamResult.stream,
};
}
// If not found locally, try upstream
let tarball: Buffer | null = null;
const upstream = await this.getUpstreamForRequest(packageName, 'tarball', 'GET', actor);
if (upstream) {
this.logger.log('debug', 'handleTarballDownload: fetching from upstream', {
packageName,
version,
});
const upstreamTarball = await upstream.fetchTarball(packageName, version);
if (upstreamTarball) {
tarball = upstreamTarball;
// Cache the tarball locally for future requests
await this.storage.putNpmTarball(packageName, version, tarball);
this.logger.log('debug', 'handleTarballDownload: cached tarball locally', {
packageName,
version,
size: tarball.length,
});
}
}
if (!tarball) {
return {
@@ -621,7 +746,23 @@ export class NpmRegistry extends BaseRegistry {
}
}
} catch (error) {
console.error('[handleSearch] Error:', error);
this.logger.log('error', 'handleSearch failed', { error: (error as Error).message });
}
// Sort results by relevance: exact match first, then prefix match, then substring match
if (text) {
const lowerText = text.toLowerCase();
results.sort((a, b) => {
const aName = a.package.name.toLowerCase();
const bName = b.package.name.toLowerCase();
const aExact = aName === lowerText ? 0 : 1;
const bExact = bName === lowerText ? 0 : 1;
if (aExact !== bExact) return aExact - bExact;
const aPrefix = aName.startsWith(lowerText) ? 0 : 1;
const bPrefix = bName.startsWith(lowerText) ? 0 : 1;
if (aPrefix !== bPrefix) return aPrefix - bPrefix;
return aName.localeCompare(bName);
});
}
// Apply pagination

View File

@@ -0,0 +1,260 @@
import * as plugins from '../plugins.js';
import { BaseUpstream } from '../upstream/classes.baseupstream.js';
import type {
IProtocolUpstreamConfig,
IUpstreamFetchContext,
IUpstreamResult,
IUpstreamRegistryConfig,
} from '../upstream/interfaces.upstream.js';
import type { IPackument, INpmVersion } from './interfaces.npm.js';
/**
* NPM-specific upstream implementation.
*
* Handles:
* - Package metadata (packument) fetching
* - Tarball proxying
* - Scoped package routing (@scope/* patterns)
* - NPM-specific URL rewriting
*/
export class NpmUpstream extends BaseUpstream {
protected readonly protocolName = 'npm';
/** Local registry URL for rewriting tarball URLs */
private readonly localRegistryUrl: string;
constructor(
config: IProtocolUpstreamConfig,
localRegistryUrl: string,
logger?: plugins.smartlog.Smartlog,
) {
super(config, logger);
this.localRegistryUrl = localRegistryUrl;
}
/**
* Fetch a packument from upstream registries.
*/
public async fetchPackument(packageName: string): Promise<IPackument | null> {
const context: IUpstreamFetchContext = {
protocol: 'npm',
resource: packageName,
resourceType: 'packument',
path: `/${encodeURIComponent(packageName).replace('%40', '@')}`,
method: 'GET',
headers: {
'accept': 'application/json',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
// Parse and process packument
let packument: IPackument;
if (Buffer.isBuffer(result.body)) {
packument = JSON.parse(result.body.toString('utf8'));
} else {
packument = result.body;
}
// Rewrite tarball URLs to point to local registry
packument = this.rewriteTarballUrls(packument);
return packument;
}
/**
* Fetch a specific version from upstream registries.
*/
public async fetchVersion(packageName: string, version: string): Promise<INpmVersion | null> {
const context: IUpstreamFetchContext = {
protocol: 'npm',
resource: packageName,
resourceType: 'version',
path: `/${encodeURIComponent(packageName).replace('%40', '@')}/${version}`,
method: 'GET',
headers: {
'accept': 'application/json',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
let versionData: INpmVersion;
if (Buffer.isBuffer(result.body)) {
versionData = JSON.parse(result.body.toString('utf8'));
} else {
versionData = result.body;
}
// Rewrite tarball URL
if (versionData.dist?.tarball) {
versionData.dist.tarball = this.rewriteSingleTarballUrl(
packageName,
versionData.version,
versionData.dist.tarball,
);
}
return versionData;
}
/**
* Fetch a tarball from upstream registries.
*/
public async fetchTarball(packageName: string, version: string): Promise<Buffer | null> {
// First, try to get the tarball URL from packument
const packument = await this.fetchPackument(packageName);
let tarballPath: string;
if (packument?.versions?.[version]?.dist?.tarball) {
// Extract path from original (upstream) tarball URL
const tarballUrl = packument.versions[version].dist.tarball;
try {
const url = new URL(tarballUrl);
tarballPath = url.pathname;
} catch {
// Fallback to standard NPM tarball path
tarballPath = this.buildTarballPath(packageName, version);
}
} else {
tarballPath = this.buildTarballPath(packageName, version);
}
const context: IUpstreamFetchContext = {
protocol: 'npm',
resource: packageName,
resourceType: 'tarball',
path: tarballPath,
method: 'GET',
headers: {
'accept': 'application/octet-stream',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
return Buffer.isBuffer(result.body) ? result.body : Buffer.from(result.body);
}
/**
* Search packages in upstream registries.
*/
public async search(text: string, size: number = 20, from: number = 0): Promise<any | null> {
const context: IUpstreamFetchContext = {
protocol: 'npm',
resource: '*',
resourceType: 'search',
path: '/-/v1/search',
method: 'GET',
headers: {
'accept': 'application/json',
},
query: {
text,
size: size.toString(),
from: from.toString(),
},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
if (Buffer.isBuffer(result.body)) {
return JSON.parse(result.body.toString('utf8'));
}
return result.body;
}
/**
* Build the standard NPM tarball path.
*/
private buildTarballPath(packageName: string, version: string): string {
// NPM uses: /{package}/-/{package-name}-{version}.tgz
// For scoped packages: /@scope/name/-/name-version.tgz
if (packageName.startsWith('@')) {
const [scope, name] = packageName.split('/');
return `/${scope}/${name}/-/${name}-${version}.tgz`;
} else {
return `/${packageName}/-/${packageName}-${version}.tgz`;
}
}
/**
* Rewrite all tarball URLs in a packument to point to local registry.
*/
private rewriteTarballUrls(packument: IPackument): IPackument {
if (!packument.versions) {
return packument;
}
const rewritten = { ...packument };
rewritten.versions = {};
for (const [version, versionData] of Object.entries(packument.versions)) {
const newVersionData = { ...versionData };
if (newVersionData.dist?.tarball) {
newVersionData.dist = {
...newVersionData.dist,
tarball: this.rewriteSingleTarballUrl(
packument.name,
version,
newVersionData.dist.tarball,
),
};
}
rewritten.versions[version] = newVersionData;
}
return rewritten;
}
/**
* Rewrite a single tarball URL to point to local registry.
*/
private rewriteSingleTarballUrl(
packageName: string,
version: string,
_originalUrl: string,
): string {
// Generate local tarball URL
// Format: {localRegistryUrl}/{package}/-/{package-name}-{version}.tgz
const safeName = packageName.replace('@', '').replace('/', '-');
return `${this.localRegistryUrl}/${packageName}/-/${safeName}-${version}.tgz`;
}
/**
* Override URL building for NPM-specific handling.
*/
protected buildUpstreamUrl(
upstream: IUpstreamRegistryConfig,
context: IUpstreamFetchContext,
): string {
// NPM registries often don't have trailing slashes
let baseUrl = upstream.url;
if (baseUrl.endsWith('/')) {
baseUrl = baseUrl.slice(0, -1);
}
return `${baseUrl}${context.path}`;
}
}

View File

@@ -3,4 +3,5 @@
*/
export { NpmRegistry } from './classes.npmregistry.js';
export { NpmUpstream } from './classes.npmupstream.js';
export * from './interfaces.npm.js';

View File

@@ -1,7 +1,11 @@
import { Smartlog } from '@push.rocks/smartlog';
import { BaseRegistry } from '../core/classes.baseregistry.js';
import { RegistryStorage } from '../core/classes.registrystorage.js';
import { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken, IRegistryError } from '../core/interfaces.core.js';
import type { IRequestContext, IResponse, IAuthToken, IRegistryError, IRequestActor } from '../core/interfaces.core.js';
import { createHashTransform, streamToBuffer } from '../core/helpers.stream.js';
import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
import { OciUpstream } from './classes.ociupstream.js';
import type {
IUploadSession,
IOciManifest,
@@ -20,12 +24,79 @@ export class OciRegistry extends BaseRegistry {
private uploadSessions: Map<string, IUploadSession> = new Map();
private basePath: string = '/oci';
private cleanupInterval?: NodeJS.Timeout;
private ociTokens?: { realm: string; service: string };
private upstreamProvider: IUpstreamProvider | null = null;
private logger: Smartlog;
constructor(storage: RegistryStorage, authManager: AuthManager, basePath: string = '/oci') {
constructor(
storage: RegistryStorage,
authManager: AuthManager,
basePath: string = '/oci',
ociTokens?: { realm: string; service: string },
upstreamProvider?: IUpstreamProvider
) {
super();
this.storage = storage;
this.authManager = authManager;
this.basePath = basePath;
this.ociTokens = ociTokens;
this.upstreamProvider = upstreamProvider || null;
// Initialize logger
this.logger = new Smartlog({
logContext: {
company: 'push.rocks',
companyunit: 'smartregistry',
containerName: 'oci-registry',
environment: (process.env.NODE_ENV as any) || 'development',
runtime: 'node',
zone: 'oci'
}
});
this.logger.enableConsole();
if (upstreamProvider) {
this.logger.log('info', 'OCI upstream provider configured');
}
}
/**
* Extract scope from OCI repository name.
* @example "myorg/myimage" -> "myorg"
* @example "library/nginx" -> "library"
* @example "nginx" -> null
*/
private extractScope(repository: string): string | null {
const slashIndex = repository.indexOf('/');
if (slashIndex > 0) {
return repository.substring(0, slashIndex);
}
return null;
}
/**
* Get upstream for a specific request.
* Calls the provider to resolve upstream config dynamically.
*/
private async getUpstreamForRequest(
resource: string,
resourceType: string,
method: string,
actor?: IRequestActor
): Promise<OciUpstream | null> {
if (!this.upstreamProvider) return null;
const config = await this.upstreamProvider.resolveUpstreamConfig({
protocol: 'oci',
resource,
scope: this.extractScope(resource),
actor,
method,
resourceType,
});
if (!config?.enabled) return null;
return new OciUpstream(config, this.basePath, this.logger);
}
public async init(): Promise<void> {
@@ -46,48 +117,61 @@ export class OciRegistry extends BaseRegistry {
const tokenString = authHeader?.replace(/^Bearer\s+/i, '');
const token = tokenString ? await this.authManager.validateToken(tokenString, 'oci') : null;
// Build actor from context and validated token
const actor: IRequestActor = {
...context.actor,
userId: token?.userId,
ip: context.headers['x-forwarded-for'] || context.headers['X-Forwarded-For'],
userAgent: context.headers['user-agent'] || context.headers['User-Agent'],
};
// Route to appropriate handler
if (path === '/v2/' || path === '/v2') {
// OCI spec: GET /v2/ is the version check endpoint
if (path === '/' || path === '' || path === '/v2/' || path === '/v2') {
return this.handleVersionCheck();
}
// Manifest operations: /v2/{name}/manifests/{reference}
const manifestMatch = path.match(/^\/v2\/([^\/]+(?:\/[^\/]+)*)\/manifests\/([^\/]+)$/);
// Manifest operations: /{name}/manifests/{reference}
const manifestMatch = path.match(/^\/([^\/]+(?:\/[^\/]+)*)\/manifests\/([^\/]+)$/);
if (manifestMatch) {
const [, name, reference] = manifestMatch;
return this.handleManifestRequest(context.method, name, reference, token, context.body, context.headers);
// Prefer rawBody for content-addressable operations to preserve exact bytes
const bodyData = context.rawBody || context.body;
return this.handleManifestRequest(context.method, name, reference, token, bodyData, context.headers, actor);
}
// Blob operations: /v2/{name}/blobs/{digest}
const blobMatch = path.match(/^\/v2\/([^\/]+(?:\/[^\/]+)*)\/blobs\/(sha256:[a-f0-9]{64})$/);
// Blob operations: /{name}/blobs/{digest}
const blobMatch = path.match(/^\/([^\/]+(?:\/[^\/]+)*)\/blobs\/(sha256:[a-f0-9]{64})$/);
if (blobMatch) {
const [, name, digest] = blobMatch;
return this.handleBlobRequest(context.method, name, digest, token, context.headers);
return this.handleBlobRequest(context.method, name, digest, token, context.headers, actor);
}
// Blob upload operations: /v2/{name}/blobs/uploads/
const uploadInitMatch = path.match(/^\/v2\/([^\/]+(?:\/[^\/]+)*)\/blobs\/uploads\/?$/);
// Blob upload operations: /{name}/blobs/uploads/
const uploadInitMatch = path.match(/^\/([^\/]+(?:\/[^\/]+)*)\/blobs\/uploads\/?$/);
if (uploadInitMatch && context.method === 'POST') {
const [, name] = uploadInitMatch;
return this.handleUploadInit(name, token, context.query, context.body);
// Prefer rawBody for content-addressable operations to preserve exact bytes
const bodyData = context.rawBody || context.body;
return this.handleUploadInit(name, token, context.query, bodyData);
}
// Blob upload operations: /v2/{name}/blobs/uploads/{uuid}
const uploadMatch = path.match(/^\/v2\/([^\/]+(?:\/[^\/]+)*)\/blobs\/uploads\/([^\/]+)$/);
// Blob upload operations: /{name}/blobs/uploads/{uuid}
const uploadMatch = path.match(/^\/([^\/]+(?:\/[^\/]+)*)\/blobs\/uploads\/([^\/]+)$/);
if (uploadMatch) {
const [, name, uploadId] = uploadMatch;
return this.handleUploadSession(context.method, uploadId, token, context);
}
// Tags list: /v2/{name}/tags/list
const tagsMatch = path.match(/^\/v2\/([^\/]+(?:\/[^\/]+)*)\/tags\/list$/);
// Tags list: /{name}/tags/list
const tagsMatch = path.match(/^\/([^\/]+(?:\/[^\/]+)*)\/tags\/list$/);
if (tagsMatch) {
const [, name] = tagsMatch;
return this.handleTagsList(name, token, context.query);
}
// Referrers: /v2/{name}/referrers/{digest}
const referrersMatch = path.match(/^\/v2\/([^\/]+(?:\/[^\/]+)*)\/referrers\/(sha256:[a-f0-9]{64})$/);
// Referrers: /{name}/referrers/{digest}
const referrersMatch = path.match(/^\/([^\/]+(?:\/[^\/]+)*)\/referrers\/(sha256:[a-f0-9]{64})$/);
if (referrersMatch) {
const [, name, digest] = referrersMatch;
return this.handleReferrers(name, digest, token, context.query);
@@ -130,11 +214,12 @@ export class OciRegistry extends BaseRegistry {
reference: string,
token: IAuthToken | null,
body?: Buffer | any,
headers?: Record<string, string>
headers?: Record<string, string>,
actor?: IRequestActor
): Promise<IResponse> {
switch (method) {
case 'GET':
return this.getManifest(repository, reference, token, headers);
return this.getManifest(repository, reference, token, headers, actor);
case 'HEAD':
return this.headManifest(repository, reference, token);
case 'PUT':
@@ -155,11 +240,12 @@ export class OciRegistry extends BaseRegistry {
repository: string,
digest: string,
token: IAuthToken | null,
headers: Record<string, string>
headers: Record<string, string>,
actor?: IRequestActor
): Promise<IResponse> {
switch (method) {
case 'GET':
return this.getBlob(repository, digest, token, headers['range'] || headers['Range']);
return this.getBlob(repository, digest, token, headers['range'] || headers['Range'], actor);
case 'HEAD':
return this.headBlob(repository, digest, token);
case 'DELETE':
@@ -180,18 +266,14 @@ export class OciRegistry extends BaseRegistry {
body?: Buffer | any
): Promise<IResponse> {
if (!await this.checkPermission(token, repository, 'push')) {
return {
status: 401,
headers: {},
body: this.createError('DENIED', 'Insufficient permissions'),
};
return this.createUnauthorizedResponse(repository, 'push');
}
// Check for monolithic upload (digest + body provided)
const digest = query.digest;
if (digest && body) {
// Monolithic upload: complete upload in single POST
const blobData = Buffer.isBuffer(body) ? body : Buffer.from(JSON.stringify(body));
const blobData = this.toBuffer(body);
// Verify digest
const calculatedDigest = await this.calculateDigest(blobData);
@@ -209,7 +291,7 @@ export class OciRegistry extends BaseRegistry {
return {
status: 201,
headers: {
'Location': `${this.basePath}/v2/${repository}/blobs/${digest}`,
'Location': `${this.basePath}/${repository}/blobs/${digest}`,
'Docker-Content-Digest': digest,
},
body: null,
@@ -222,6 +304,8 @@ export class OciRegistry extends BaseRegistry {
uploadId,
repository,
chunks: [],
chunkPaths: [],
chunkIndex: 0,
totalSize: 0,
createdAt: new Date(),
lastActivity: new Date(),
@@ -232,7 +316,7 @@ export class OciRegistry extends BaseRegistry {
return {
status: 202,
headers: {
'Location': `${this.basePath}/v2/${repository}/blobs/uploads/${uploadId}`,
'Location': `${this.basePath}/${repository}/blobs/uploads/${uploadId}`,
'Docker-Upload-UUID': uploadId,
},
body: null,
@@ -255,18 +339,17 @@ export class OciRegistry extends BaseRegistry {
}
if (!await this.checkPermission(token, session.repository, 'push')) {
return {
status: 401,
headers: {},
body: this.createError('DENIED', 'Insufficient permissions'),
};
return this.createUnauthorizedResponse(session.repository, 'push');
}
// Prefer rawBody for content-addressable operations to preserve exact bytes
const bodyData = context.rawBody || context.body;
switch (method) {
case 'PATCH':
return this.uploadChunk(uploadId, context.body, context.headers['content-range']);
return this.uploadChunk(uploadId, bodyData, context.headers['content-range']);
case 'PUT':
return this.completeUpload(uploadId, context.query['digest'], context.body);
return this.completeUpload(uploadId, context.query['digest'], bodyData);
case 'GET':
return this.getUploadStatus(uploadId);
default:
@@ -285,16 +368,11 @@ export class OciRegistry extends BaseRegistry {
repository: string,
reference: string,
token: IAuthToken | null,
headers?: Record<string, string>
headers?: Record<string, string>,
actor?: IRequestActor
): Promise<IResponse> {
if (!await this.checkPermission(token, repository, 'pull')) {
return {
status: 401,
headers: {
'WWW-Authenticate': `Bearer realm="${this.basePath}/v2/token",service="registry",scope="repository:${repository}:pull"`,
},
body: this.createError('DENIED', 'Insufficient permissions'),
};
return this.createUnauthorizedResponse(repository, 'pull');
}
// Resolve tag to digest if needed
@@ -302,16 +380,53 @@ export class OciRegistry extends BaseRegistry {
if (!reference.startsWith('sha256:')) {
const tags = await this.getTagsData(repository);
digest = tags[reference];
if (!digest) {
return {
status: 404,
headers: {},
body: this.createError('MANIFEST_UNKNOWN', 'Manifest not found'),
};
}
// Try local storage first (if we have a digest)
let manifestData: Buffer | null = null;
let contentType: string | null = null;
if (digest) {
manifestData = await this.storage.getOciManifest(repository, digest);
if (manifestData) {
contentType = await this.storage.getOciManifestContentType(repository, digest);
if (!contentType) {
contentType = this.detectManifestContentType(manifestData);
}
}
}
// If not found locally, try upstream
if (!manifestData) {
const upstream = await this.getUpstreamForRequest(repository, 'manifest', 'GET', actor);
if (upstream) {
this.logger.log('debug', 'getManifest: fetching from upstream', { repository, reference });
const upstreamResult = await upstream.fetchManifest(repository, reference);
if (upstreamResult) {
manifestData = Buffer.from(JSON.stringify(upstreamResult.manifest), 'utf8');
contentType = upstreamResult.contentType;
digest = upstreamResult.digest;
// Cache the manifest locally
await this.storage.putOciManifest(repository, digest, manifestData, contentType);
// If reference is a tag, update tags mapping
if (!reference.startsWith('sha256:')) {
const tags = await this.getTagsData(repository);
tags[reference] = digest;
const tagsPath = `oci/tags/${repository}/tags.json`;
await this.storage.putObject(tagsPath, Buffer.from(JSON.stringify(tags), 'utf-8'));
}
this.logger.log('debug', 'getManifest: cached manifest locally', {
repository,
reference,
digest,
});
}
}
}
const manifestData = await this.storage.getOciManifest(repository, digest);
if (!manifestData) {
return {
status: 404,
@@ -323,7 +438,7 @@ export class OciRegistry extends BaseRegistry {
return {
status: 200,
headers: {
'Content-Type': 'application/vnd.oci.image.manifest.v1+json',
'Content-Type': contentType || 'application/vnd.oci.image.manifest.v1+json',
'Docker-Content-Digest': digest,
},
body: manifestData,
@@ -336,11 +451,7 @@ export class OciRegistry extends BaseRegistry {
token: IAuthToken | null
): Promise<IResponse> {
if (!await this.checkPermission(token, repository, 'pull')) {
return {
status: 401,
headers: {},
body: null,
};
return this.createUnauthorizedHeadResponse(repository, 'pull');
}
// Similar logic as getManifest but return headers only
@@ -360,10 +471,18 @@ export class OciRegistry extends BaseRegistry {
const manifestData = await this.storage.getOciManifest(repository, digest);
// Get stored content type, falling back to detecting from manifest content
let contentType = await this.storage.getOciManifestContentType(repository, digest);
if (!contentType && manifestData) {
// Fallback: detect content type from manifest content
contentType = this.detectManifestContentType(manifestData);
}
contentType = contentType || 'application/vnd.oci.image.manifest.v1+json';
return {
status: 200,
headers: {
'Content-Type': 'application/vnd.oci.image.manifest.v1+json',
'Content-Type': contentType,
'Docker-Content-Digest': digest,
'Content-Length': manifestData ? manifestData.length.toString() : '0',
},
@@ -379,13 +498,7 @@ export class OciRegistry extends BaseRegistry {
headers?: Record<string, string>
): Promise<IResponse> {
if (!await this.checkPermission(token, repository, 'push')) {
return {
status: 401,
headers: {
'WWW-Authenticate': `Bearer realm="${this.basePath}/v2/token",service="registry",scope="repository:${repository}:push"`,
},
body: this.createError('DENIED', 'Insufficient permissions'),
};
return this.createUnauthorizedResponse(repository, 'push');
}
if (!body) {
@@ -396,7 +509,9 @@ export class OciRegistry extends BaseRegistry {
};
}
const manifestData = Buffer.isBuffer(body) ? body : Buffer.from(JSON.stringify(body));
// Preserve raw bytes for accurate digest calculation
// Per OCI spec, digest must match the exact bytes sent by client
const manifestData = this.toBuffer(body);
const contentType = headers?.['content-type'] || headers?.['Content-Type'] || 'application/vnd.oci.image.manifest.v1+json';
// Calculate manifest digest
@@ -416,7 +531,7 @@ export class OciRegistry extends BaseRegistry {
return {
status: 201,
headers: {
'Location': `${this.basePath}/v2/${repository}/manifests/${digest}`,
'Location': `${this.basePath}/${repository}/manifests/${digest}`,
'Docker-Content-Digest': digest,
},
body: null,
@@ -437,11 +552,7 @@ export class OciRegistry extends BaseRegistry {
}
if (!await this.checkPermission(token, repository, 'delete')) {
return {
status: 401,
headers: {},
body: this.createError('DENIED', 'Insufficient permissions'),
};
return this.createUnauthorizedResponse(repository, 'delete');
}
await this.storage.deleteOciManifest(repository, digest);
@@ -457,17 +568,45 @@ export class OciRegistry extends BaseRegistry {
repository: string,
digest: string,
token: IAuthToken | null,
range?: string
range?: string,
actor?: IRequestActor
): Promise<IResponse> {
if (!await this.checkPermission(token, repository, 'pull')) {
return this.createUnauthorizedResponse(repository, 'pull');
}
// Try local storage first (streaming)
const streamResult = await this.storage.getOciBlobStream(digest);
if (streamResult) {
return {
status: 401,
headers: {},
body: this.createError('DENIED', 'Insufficient permissions'),
status: 200,
headers: {
'Content-Type': 'application/octet-stream',
'Content-Length': streamResult.size.toString(),
'Docker-Content-Digest': digest,
},
body: streamResult.stream,
};
}
const data = await this.storage.getOciBlob(digest);
// If not found locally, try upstream
let data: Buffer | null = null;
const upstream = await this.getUpstreamForRequest(repository, 'blob', 'GET', actor);
if (upstream) {
this.logger.log('debug', 'getBlob: fetching from upstream', { repository, digest });
const upstreamBlob = await upstream.fetchBlob(repository, digest);
if (upstreamBlob) {
data = upstreamBlob;
// Cache the blob locally (blobs are content-addressable and immutable)
await this.storage.putOciBlob(digest, data);
this.logger.log('debug', 'getBlob: cached blob locally', {
repository,
digest,
size: data.length,
});
}
}
if (!data) {
return {
status: 404,
@@ -492,20 +631,18 @@ export class OciRegistry extends BaseRegistry {
token: IAuthToken | null
): Promise<IResponse> {
if (!await this.checkPermission(token, repository, 'pull')) {
return { status: 401, headers: {}, body: null };
return this.createUnauthorizedHeadResponse(repository, 'pull');
}
const exists = await this.storage.ociBlobExists(digest);
if (!exists) {
const blobSize = await this.storage.getOciBlobSize(digest);
if (blobSize === null) {
return { status: 404, headers: {}, body: null };
}
const blob = await this.storage.getOciBlob(digest);
return {
status: 200,
headers: {
'Content-Length': blob ? blob.length.toString() : '0',
'Content-Length': blobSize.toString(),
'Docker-Content-Digest': digest,
},
body: null,
@@ -518,11 +655,7 @@ export class OciRegistry extends BaseRegistry {
token: IAuthToken | null
): Promise<IResponse> {
if (!await this.checkPermission(token, repository, 'delete')) {
return {
status: 401,
headers: {},
body: this.createError('DENIED', 'Insufficient permissions'),
};
return this.createUnauthorizedResponse(repository, 'delete');
}
await this.storage.deleteOciBlob(digest);
@@ -536,7 +669,7 @@ export class OciRegistry extends BaseRegistry {
private async uploadChunk(
uploadId: string,
data: Buffer,
data: Buffer | Uint8Array | unknown,
contentRange: string
): Promise<IResponse> {
const session = this.uploadSessions.get(uploadId);
@@ -548,14 +681,20 @@ export class OciRegistry extends BaseRegistry {
};
}
session.chunks.push(data);
session.totalSize += data.length;
const chunkData = this.toBuffer(data);
// Write chunk to temp S3 object instead of accumulating in memory
const chunkPath = `oci/uploads/${uploadId}/chunk-${session.chunkIndex}`;
await this.storage.putObject(chunkPath, chunkData);
session.chunkPaths.push(chunkPath);
session.chunkIndex++;
session.totalSize += chunkData.length;
session.lastActivity = new Date();
return {
status: 202,
headers: {
'Location': `${this.basePath}/v2/${session.repository}/blobs/uploads/${uploadId}`,
'Location': `${this.basePath}/${session.repository}/blobs/uploads/${uploadId}`,
'Range': `0-${session.totalSize - 1}`,
'Docker-Upload-UUID': uploadId,
},
@@ -566,7 +705,7 @@ export class OciRegistry extends BaseRegistry {
private async completeUpload(
uploadId: string,
digest: string,
finalData?: Buffer
finalData?: Buffer | Uint8Array | unknown
): Promise<IResponse> {
const session = this.uploadSessions.get(uploadId);
if (!session) {
@@ -577,13 +716,52 @@ export class OciRegistry extends BaseRegistry {
};
}
const chunks = [...session.chunks];
if (finalData) chunks.push(finalData);
const blobData = Buffer.concat(chunks);
// If there's final data in the PUT body, write it as the last chunk
if (finalData) {
const buf = this.toBuffer(finalData);
const chunkPath = `oci/uploads/${uploadId}/chunk-${session.chunkIndex}`;
await this.storage.putObject(chunkPath, buf);
session.chunkPaths.push(chunkPath);
session.chunkIndex++;
session.totalSize += buf.length;
}
// Verify digest
const calculatedDigest = await this.calculateDigest(blobData);
// Create a ReadableStream that assembles all chunks from S3 sequentially
const chunkPaths = [...session.chunkPaths];
const storage = this.storage;
let chunkIdx = 0;
const assembledStream = new ReadableStream<Uint8Array>({
async pull(controller) {
if (chunkIdx >= chunkPaths.length) {
controller.close();
return;
}
const result = await storage.getObjectStream(chunkPaths[chunkIdx++]);
if (result) {
const reader = result.stream.getReader();
while (true) {
const { done, value } = await reader.read();
if (done) break;
if (value) controller.enqueue(value);
}
}
},
});
// Pipe through hash transform for incremental digest verification
const { transform: hashTransform, getDigest } = createHashTransform('sha256');
const hashedStream = assembledStream.pipeThrough(hashTransform);
// Consume stream to buffer for S3 upload
// (AWS SDK PutObjectCommand requires known content-length for streams;
// the key win is chunks are NOT accumulated in memory during PATCH — they live in S3)
const blobData = await streamToBuffer(hashedStream);
// Verify digest before storing
const calculatedDigest = `sha256:${getDigest()}`;
if (calculatedDigest !== digest) {
await this.cleanupUploadChunks(session);
this.uploadSessions.delete(uploadId);
return {
status: 400,
headers: {},
@@ -591,19 +769,36 @@ export class OciRegistry extends BaseRegistry {
};
}
// Store verified blob
await this.storage.putOciBlob(digest, blobData);
// Cleanup temp chunks and session
await this.cleanupUploadChunks(session);
this.uploadSessions.delete(uploadId);
return {
status: 201,
headers: {
'Location': `${this.basePath}/v2/${session.repository}/blobs/${digest}`,
'Location': `${this.basePath}/${session.repository}/blobs/${digest}`,
'Docker-Content-Digest': digest,
},
body: null,
};
}
/**
* Delete all temp S3 chunk objects for an upload session.
*/
private async cleanupUploadChunks(session: IUploadSession): Promise<void> {
for (const chunkPath of session.chunkPaths) {
try {
await this.storage.deleteObject(chunkPath);
} catch {
// Best-effort cleanup
}
}
}
private async getUploadStatus(uploadId: string): Promise<IResponse> {
const session = this.uploadSessions.get(uploadId);
if (!session) {
@@ -617,7 +812,7 @@ export class OciRegistry extends BaseRegistry {
return {
status: 204,
headers: {
'Location': `${this.basePath}/v2/${session.repository}/blobs/uploads/${uploadId}`,
'Location': `${this.basePath}/${session.repository}/blobs/uploads/${uploadId}`,
'Range': session.totalSize > 0 ? `0-${session.totalSize - 1}` : '0-0',
'Docker-Upload-UUID': uploadId,
},
@@ -631,11 +826,7 @@ export class OciRegistry extends BaseRegistry {
query: Record<string, string>
): Promise<IResponse> {
if (!await this.checkPermission(token, repository, 'pull')) {
return {
status: 401,
headers: {},
body: this.createError('DENIED', 'Insufficient permissions'),
};
return this.createUnauthorizedResponse(repository, 'pull');
}
const tags = await this.getTagsData(repository);
@@ -660,11 +851,7 @@ export class OciRegistry extends BaseRegistry {
query: Record<string, string>
): Promise<IResponse> {
if (!await this.checkPermission(token, repository, 'pull')) {
return {
status: 401,
headers: {},
body: this.createError('DENIED', 'Insufficient permissions'),
};
return this.createUnauthorizedResponse(repository, 'pull');
}
const response: IReferrersResponse = {
@@ -684,6 +871,59 @@ export class OciRegistry extends BaseRegistry {
// HELPER METHODS
// ========================================================================
/**
* Detect manifest content type from manifest content.
* OCI Image Index has "manifests" array, OCI Image Manifest has "config" object.
* Also checks the mediaType field if present.
*/
private detectManifestContentType(manifestData: Buffer): string {
try {
const manifest = JSON.parse(manifestData.toString('utf-8'));
// First check if manifest has explicit mediaType field
if (manifest.mediaType) {
return manifest.mediaType;
}
// Otherwise detect from structure
if (Array.isArray(manifest.manifests)) {
// OCI Image Index (multi-arch manifest list)
return 'application/vnd.oci.image.index.v1+json';
} else if (manifest.config) {
// OCI Image Manifest
return 'application/vnd.oci.image.manifest.v1+json';
}
// Fallback to standard manifest type
return 'application/vnd.oci.image.manifest.v1+json';
} catch (e) {
// If parsing fails, return default
return 'application/vnd.oci.image.manifest.v1+json';
}
}
/**
* Convert any binary-like data to Buffer.
* Handles Buffer, Uint8Array (modern cross-platform), string, and objects.
*
* Note: Buffer.isBuffer(Uint8Array) returns false even though Buffer extends Uint8Array.
* This is because Uint8Array is the modern, cross-platform standard while Buffer is Node.js-specific.
* Many HTTP frameworks pass request bodies as Uint8Array for better compatibility.
*/
private toBuffer(data: unknown): Buffer {
if (Buffer.isBuffer(data)) {
return data;
}
if (data instanceof Uint8Array) {
return Buffer.from(data);
}
if (typeof data === 'string') {
return Buffer.from(data, 'utf-8');
}
// Fallback: serialize object to JSON (may cause digest mismatch for manifests)
return Buffer.from(JSON.stringify(data));
}
private async getTagsData(repository: string): Promise<Record<string, string>> {
const path = `oci/tags/${repository}/tags.json`;
const data = await this.storage.getObject(path);
@@ -697,7 +937,7 @@ export class OciRegistry extends BaseRegistry {
}
private generateUploadId(): string {
return `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
return `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
}
private async calculateDigest(data: Buffer): Promise<string> {
@@ -712,6 +952,37 @@ export class OciRegistry extends BaseRegistry {
};
}
/**
* Create an unauthorized response with proper WWW-Authenticate header.
* Per OCI Distribution Spec, 401 responses MUST include WWW-Authenticate header.
*/
private createUnauthorizedResponse(repository: string, action: string): IResponse {
const realm = this.ociTokens?.realm || `${this.basePath}/token`;
const service = this.ociTokens?.service || 'registry';
return {
status: 401,
headers: {
'WWW-Authenticate': `Bearer realm="${realm}",service="${service}",scope="repository:${repository}:${action}"`,
},
body: this.createError('DENIED', 'Insufficient permissions'),
};
}
/**
* Create an unauthorized HEAD response (no body per HTTP spec).
*/
private createUnauthorizedHeadResponse(repository: string, action: string): IResponse {
const realm = this.ociTokens?.realm || `${this.basePath}/token`;
const service = this.ociTokens?.service || 'registry';
return {
status: 401,
headers: {
'WWW-Authenticate': `Bearer realm="${realm}",service="${service}",scope="repository:${repository}:${action}"`,
},
body: null,
};
}
private startUploadSessionCleanup(): void {
this.cleanupInterval = setInterval(() => {
const now = new Date();
@@ -719,6 +990,8 @@ export class OciRegistry extends BaseRegistry {
for (const [uploadId, session] of this.uploadSessions.entries()) {
if (now.getTime() - session.lastActivity.getTime() > maxAge) {
// Clean up temp S3 chunks for stale sessions
this.cleanupUploadChunks(session).catch(() => {});
this.uploadSessions.delete(uploadId);
}
}

View File

@@ -0,0 +1,273 @@
import * as plugins from '../plugins.js';
import { BaseUpstream } from '../upstream/classes.baseupstream.js';
import type {
IProtocolUpstreamConfig,
IUpstreamFetchContext,
IUpstreamResult,
IUpstreamRegistryConfig,
} from '../upstream/interfaces.upstream.js';
import type { IOciManifest, IOciImageIndex, ITagList } from './interfaces.oci.js';
/**
* OCI-specific upstream implementation.
*
* Handles:
* - Manifest fetching (image manifests and index manifests)
* - Blob proxying (layers, configs)
* - Tag list fetching
* - Content-addressable caching (blobs are immutable)
* - Docker Hub authentication flow
*/
export class OciUpstream extends BaseUpstream {
protected readonly protocolName = 'oci';
/** Local registry base path for URL building */
private readonly localBasePath: string;
/** API prefix for outbound OCI requests (default: /v2) */
private readonly apiPrefix: string;
constructor(
config: IProtocolUpstreamConfig,
localBasePath: string = '/oci',
logger?: plugins.smartlog.Smartlog,
apiPrefix: string = '/v2',
) {
super(config, logger);
this.localBasePath = localBasePath;
this.apiPrefix = apiPrefix;
}
/**
* Fetch a manifest from upstream registries.
*/
public async fetchManifest(
repository: string,
reference: string,
): Promise<{ manifest: IOciManifest | IOciImageIndex; contentType: string; digest: string } | null> {
const context: IUpstreamFetchContext = {
protocol: 'oci',
resource: repository,
resourceType: 'manifest',
path: `${this.apiPrefix}/${repository}/manifests/${reference}`,
method: 'GET',
headers: {
'accept': [
'application/vnd.oci.image.manifest.v1+json',
'application/vnd.oci.image.index.v1+json',
'application/vnd.docker.distribution.manifest.v2+json',
'application/vnd.docker.distribution.manifest.list.v2+json',
'application/vnd.docker.distribution.manifest.v1+json',
].join(', '),
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
let manifest: IOciManifest | IOciImageIndex;
if (Buffer.isBuffer(result.body)) {
manifest = JSON.parse(result.body.toString('utf8'));
} else {
manifest = result.body;
}
const contentType = result.headers['content-type'] || 'application/vnd.oci.image.manifest.v1+json';
const digest = result.headers['docker-content-digest'] || '';
return { manifest, contentType, digest };
}
/**
* Check if a manifest exists in upstream (HEAD request).
*/
public async headManifest(
repository: string,
reference: string,
): Promise<{ exists: boolean; contentType?: string; digest?: string; size?: number } | null> {
const context: IUpstreamFetchContext = {
protocol: 'oci',
resource: repository,
resourceType: 'manifest',
path: `${this.apiPrefix}/${repository}/manifests/${reference}`,
method: 'HEAD',
headers: {
'accept': [
'application/vnd.oci.image.manifest.v1+json',
'application/vnd.oci.image.index.v1+json',
'application/vnd.docker.distribution.manifest.v2+json',
'application/vnd.docker.distribution.manifest.list.v2+json',
].join(', '),
},
query: {},
};
const result = await this.fetch(context);
if (!result) {
return null;
}
if (!result.success) {
return { exists: false };
}
return {
exists: true,
contentType: result.headers['content-type'],
digest: result.headers['docker-content-digest'],
size: result.headers['content-length'] ? parseInt(result.headers['content-length'], 10) : undefined,
};
}
/**
* Fetch a blob from upstream registries.
*/
public async fetchBlob(repository: string, digest: string): Promise<Buffer | null> {
const context: IUpstreamFetchContext = {
protocol: 'oci',
resource: repository,
resourceType: 'blob',
path: `${this.apiPrefix}/${repository}/blobs/${digest}`,
method: 'GET',
headers: {
'accept': 'application/octet-stream',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
return Buffer.isBuffer(result.body) ? result.body : Buffer.from(result.body);
}
/**
* Check if a blob exists in upstream (HEAD request).
*/
public async headBlob(
repository: string,
digest: string,
): Promise<{ exists: boolean; size?: number } | null> {
const context: IUpstreamFetchContext = {
protocol: 'oci',
resource: repository,
resourceType: 'blob',
path: `${this.apiPrefix}/${repository}/blobs/${digest}`,
method: 'HEAD',
headers: {},
query: {},
};
const result = await this.fetch(context);
if (!result) {
return null;
}
if (!result.success) {
return { exists: false };
}
return {
exists: true,
size: result.headers['content-length'] ? parseInt(result.headers['content-length'], 10) : undefined,
};
}
/**
* Fetch the tag list for a repository.
*/
public async fetchTags(repository: string, n?: number, last?: string): Promise<ITagList | null> {
const query: Record<string, string> = {};
if (n) query.n = n.toString();
if (last) query.last = last;
const context: IUpstreamFetchContext = {
protocol: 'oci',
resource: repository,
resourceType: 'tags',
path: `${this.apiPrefix}/${repository}/tags/list`,
method: 'GET',
headers: {
'accept': 'application/json',
},
query,
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
let tagList: ITagList;
if (Buffer.isBuffer(result.body)) {
tagList = JSON.parse(result.body.toString('utf8'));
} else {
tagList = result.body;
}
return tagList;
}
/**
* Override URL building for OCI-specific handling.
* OCI registries use a configurable API prefix (default /v2/) and may require
* special handling for Docker Hub.
*/
protected buildUpstreamUrl(
upstream: IUpstreamRegistryConfig,
context: IUpstreamFetchContext,
): string {
let baseUrl = upstream.url;
// Remove trailing slash
if (baseUrl.endsWith('/')) {
baseUrl = baseUrl.slice(0, -1);
}
// Use per-upstream apiPrefix if configured, otherwise use the instance default
const prefix = upstream.apiPrefix || this.apiPrefix;
// Handle Docker Hub special case
// Docker Hub uses registry-1.docker.io but library images need special handling
if (baseUrl.includes('docker.io') || baseUrl.includes('registry-1.docker.io')) {
// For library images (e.g., "nginx" -> "library/nginx")
const escapedPrefix = prefix.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
const pathParts = context.path.match(new RegExp(`^${escapedPrefix}\\/([^\\/]+)\\/(.+)$`));
if (pathParts) {
const [, repository, rest] = pathParts;
// If repository doesn't contain a slash, it's a library image
if (!repository.includes('/')) {
return `${baseUrl}${prefix}/library/${repository}/${rest}`;
}
}
}
return `${baseUrl}${context.path}`;
}
/**
* Override header building for OCI-specific authentication.
* OCI registries may require token-based auth obtained from a separate endpoint.
*/
protected buildHeaders(
upstream: IUpstreamRegistryConfig,
context: IUpstreamFetchContext,
): Record<string, string> {
const headers = super.buildHeaders(upstream, context);
// OCI registries typically use Docker-Distribution-API-Version header
headers['docker-distribution-api-version'] = 'registry/2.0';
return headers;
}
}

View File

@@ -3,4 +3,5 @@
*/
export { OciRegistry } from './classes.ociregistry.js';
export { OciUpstream } from './classes.ociupstream.js';
export * from './interfaces.oci.js';

View File

@@ -62,6 +62,10 @@ export interface IUploadSession {
uploadId: string;
repository: string;
chunks: Buffer[];
/** S3 paths to temp chunk objects (streaming mode) */
chunkPaths: string[];
/** Index counter for naming temp chunk objects */
chunkIndex: number;
totalSize: number;
createdAt: Date;
lastActivity: Date;

View File

@@ -4,8 +4,20 @@ import * as path from 'path';
export { path };
// @push.rocks scope
import * as smartarchive from '@push.rocks/smartarchive';
import * as smartbucket from '@push.rocks/smartbucket';
import * as smartlog from '@push.rocks/smartlog';
import * as smartpath from '@push.rocks/smartpath';
import * as smartrequest from '@push.rocks/smartrequest';
export { smartbucket, smartlog, smartpath };
export { smartarchive, smartbucket, smartlog, smartpath, smartrequest };
// @tsclass scope
import * as tsclass from '@tsclass/tsclass';
export { tsclass };
// third party
import { minimatch } from 'minimatch';
export { minimatch };

View File

@@ -0,0 +1,770 @@
import { Smartlog } from '@push.rocks/smartlog';
import { BaseRegistry } from '../core/classes.baseregistry.js';
import { RegistryStorage } from '../core/classes.registrystorage.js';
import { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from '../core/interfaces.core.js';
import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
import { isBinaryData, toBuffer } from '../core/helpers.buffer.js';
import type {
IPypiPackageMetadata,
IPypiFile,
IPypiError,
IPypiUploadResponse,
} from './interfaces.pypi.js';
import * as helpers from './helpers.pypi.js';
import { PypiUpstream } from './classes.pypiupstream.js';
/**
* PyPI registry implementation
* Implements PEP 503 (Simple API), PEP 691 (JSON API), and legacy upload API
*/
export class PypiRegistry extends BaseRegistry {
private storage: RegistryStorage;
private authManager: AuthManager;
private basePath: string = '/pypi';
private registryUrl: string;
private logger: Smartlog;
private upstreamProvider: IUpstreamProvider | null = null;
constructor(
storage: RegistryStorage,
authManager: AuthManager,
basePath: string = '/pypi',
registryUrl: string = 'http://localhost:5000',
upstreamProvider?: IUpstreamProvider
) {
super();
this.storage = storage;
this.authManager = authManager;
this.basePath = basePath;
this.registryUrl = registryUrl;
this.upstreamProvider = upstreamProvider || null;
// Initialize logger
this.logger = new Smartlog({
logContext: {
company: 'push.rocks',
companyunit: 'smartregistry',
containerName: 'pypi-registry',
environment: (process.env.NODE_ENV as any) || 'development',
runtime: 'node',
zone: 'pypi'
}
});
this.logger.enableConsole();
}
/**
* Get upstream for a specific request.
* Calls the provider to resolve upstream config dynamically.
*/
private async getUpstreamForRequest(
resource: string,
resourceType: string,
method: string,
actor?: IRequestActor
): Promise<PypiUpstream | null> {
if (!this.upstreamProvider) return null;
const config = await this.upstreamProvider.resolveUpstreamConfig({
protocol: 'pypi',
resource,
scope: resource, // For PyPI, package name is the scope
actor,
method,
resourceType,
});
if (!config?.enabled) return null;
return new PypiUpstream(config, this.registryUrl, this.logger);
}
/**
* Clean up resources (timers, connections, etc.)
*/
public destroy(): void {
// No persistent upstream to clean up with dynamic provider
}
public async init(): Promise<void> {
// Initialize root Simple API index if not exists
const existingIndex = await this.storage.getPypiSimpleRootIndex();
if (!existingIndex) {
const html = helpers.generateSimpleRootHtml([]);
await this.storage.putPypiSimpleRootIndex(html);
this.logger.log('info', 'Initialized PyPI root index');
}
}
public getBasePath(): string {
return this.basePath;
}
public async handleRequest(context: IRequestContext): Promise<IResponse> {
let path = context.path.replace(this.basePath, '');
// Extract token (Basic Auth or Bearer)
const token = await this.extractToken(context);
// Build actor from context and validated token
const actor: IRequestActor = {
...context.actor,
userId: token?.userId,
ip: context.headers['x-forwarded-for'] || context.headers['X-Forwarded-For'],
userAgent: context.headers['user-agent'] || context.headers['User-Agent'],
};
// Also handle /simple path prefix
if (path.startsWith('/simple')) {
path = path.replace('/simple', '');
return this.handleSimpleRequest(path, context, actor);
}
this.logger.log('debug', `handleRequest: ${context.method} ${path}`, {
method: context.method,
path,
hasAuth: !!token
});
// Root upload endpoint (POST /)
if ((path === '/' || path === '') && context.method === 'POST') {
return this.handleUpload(context, token);
}
// Package metadata JSON API: GET /{package}/json
const jsonMatch = path.match(/^\/([^\/]+)\/json$/);
if (jsonMatch && context.method === 'GET') {
return this.handlePackageJson(jsonMatch[1]);
}
// Version-specific JSON API: GET /{package}/{version}/json
const versionJsonMatch = path.match(/^\/([^\/]+)\/([^\/]+)\/json$/);
if (versionJsonMatch && context.method === 'GET') {
return this.handleVersionJson(versionJsonMatch[1], versionJsonMatch[2]);
}
// Package file download: GET /packages/{package}/{filename}
const downloadMatch = path.match(/^\/packages\/([^\/]+)\/(.+)$/);
if (downloadMatch && context.method === 'GET') {
return this.handleDownload(downloadMatch[1], downloadMatch[2], actor);
}
// Delete package: DELETE /packages/{package}
if (path.match(/^\/packages\/([^\/]+)$/) && context.method === 'DELETE') {
const packageName = path.match(/^\/packages\/([^\/]+)$/)?.[1];
return this.handleDeletePackage(packageName!, token);
}
// Delete version: DELETE /packages/{package}/{version}
const deleteVersionMatch = path.match(/^\/packages\/([^\/]+)\/([^\/]+)$/);
if (deleteVersionMatch && context.method === 'DELETE') {
return this.handleDeleteVersion(deleteVersionMatch[1], deleteVersionMatch[2], token);
}
return {
status: 404,
headers: { 'Content-Type': 'application/json' },
body: { error: 'Not Found' },
};
}
/**
* Check if token has permission for resource
*/
protected async checkPermission(
token: IAuthToken | null,
resource: string,
action: string
): Promise<boolean> {
if (!token) return false;
return this.authManager.authorize(token, `pypi:package:${resource}`, action);
}
/**
* Handle Simple API requests (PEP 503 HTML or PEP 691 JSON)
*/
private async handleSimpleRequest(path: string, context: IRequestContext, actor?: IRequestActor): Promise<IResponse> {
// Ensure path ends with / (PEP 503 requirement)
if (!path.endsWith('/') && !path.includes('.')) {
return {
status: 301,
headers: { 'Location': `${this.basePath}/simple${path}/` },
body: Buffer.from(''),
};
}
// Root index: /simple/
if (path === '/' || path === '') {
return this.handleSimpleRoot(context);
}
// Package index: /simple/{package}/
const packageMatch = path.match(/^\/([^\/]+)\/$/);
if (packageMatch) {
return this.handleSimplePackage(packageMatch[1], context, actor);
}
return {
status: 404,
headers: { 'Content-Type': 'text/html; charset=utf-8' },
body: Buffer.from('<html><body><h1>404 Not Found</h1></body></html>'),
};
}
/**
* Handle Simple API root index
* Returns HTML (PEP 503) or JSON (PEP 691) based on Accept header
*/
private async handleSimpleRoot(context: IRequestContext): Promise<IResponse> {
const acceptHeader = context.headers['accept'] || context.headers['Accept'] || '';
const preferJson = acceptHeader.includes('application/vnd.pypi.simple') &&
acceptHeader.includes('json');
const packages = await this.storage.listPypiPackages();
if (preferJson) {
// PEP 691: JSON response
const response = helpers.generateJsonRootResponse(packages);
return {
status: 200,
headers: {
'Content-Type': 'application/vnd.pypi.simple.v1+json',
'Cache-Control': 'public, max-age=600'
},
body: response,
};
} else {
// PEP 503: HTML response
const html = helpers.generateSimpleRootHtml(packages);
// Update stored index
await this.storage.putPypiSimpleRootIndex(html);
return {
status: 200,
headers: {
'Content-Type': 'text/html; charset=utf-8',
'Cache-Control': 'public, max-age=600'
},
body: html,
};
}
}
/**
* Handle Simple API package index
* Returns HTML (PEP 503) or JSON (PEP 691) based on Accept header
*/
private async handleSimplePackage(packageName: string, context: IRequestContext, actor?: IRequestActor): Promise<IResponse> {
const normalized = helpers.normalizePypiPackageName(packageName);
// Get package metadata
let metadata = await this.storage.getPypiPackageMetadata(normalized);
// Try upstream if not found locally
if (!metadata) {
const upstream = await this.getUpstreamForRequest(normalized, 'simple', 'GET', actor);
if (upstream) {
const upstreamHtml = await upstream.fetchSimplePackage(normalized);
if (upstreamHtml) {
// Parse the HTML to extract file information and cache it
// For now, just return the upstream HTML directly (caching can be improved later)
const acceptHeader = context.headers['accept'] || context.headers['Accept'] || '';
const preferJson = acceptHeader.includes('application/vnd.pypi.simple') &&
acceptHeader.includes('json');
if (preferJson) {
// Try to get JSON format from upstream
const upstreamJson = await upstream.fetchPackageJson(normalized);
if (upstreamJson) {
return {
status: 200,
headers: {
'Content-Type': 'application/vnd.pypi.simple.v1+json',
'Cache-Control': 'public, max-age=300'
},
body: upstreamJson,
};
}
}
// Return HTML format
return {
status: 200,
headers: {
'Content-Type': 'text/html; charset=utf-8',
'Cache-Control': 'public, max-age=300'
},
body: upstreamHtml,
};
}
}
}
if (!metadata) {
return this.errorResponse(404, 'Package not found');
}
// Build file list from all versions
const files: IPypiFile[] = [];
for (const [version, versionMeta] of Object.entries(metadata.versions || {})) {
for (const file of (versionMeta as any).files || []) {
files.push({
filename: file.filename,
url: `${this.registryUrl}/pypi/packages/${normalized}/${file.filename}`,
hashes: file.hashes,
'requires-python': file['requires-python'],
yanked: file.yanked || (versionMeta as any).yanked,
size: file.size,
'upload-time': file['upload-time'],
});
}
}
const acceptHeader = context.headers['accept'] || context.headers['Accept'] || '';
const preferJson = acceptHeader.includes('application/vnd.pypi.simple') &&
acceptHeader.includes('json');
if (preferJson) {
// PEP 691: JSON response
const response = helpers.generateJsonPackageResponse(normalized, files);
return {
status: 200,
headers: {
'Content-Type': 'application/vnd.pypi.simple.v1+json',
'Cache-Control': 'public, max-age=300'
},
body: response,
};
} else {
// PEP 503: HTML response
const html = helpers.generateSimplePackageHtml(normalized, files, this.registryUrl);
// Update stored index
await this.storage.putPypiSimpleIndex(normalized, html);
return {
status: 200,
headers: {
'Content-Type': 'text/html; charset=utf-8',
'Cache-Control': 'public, max-age=300'
},
body: html,
};
}
}
/**
* Extract authentication token from request
*/
private async extractToken(context: IRequestContext): Promise<IAuthToken | null> {
const authHeader = context.headers['authorization'] || context.headers['Authorization'];
if (!authHeader) return null;
// Handle Basic Auth (username:password or __token__:token)
if (authHeader.startsWith('Basic ')) {
const base64 = authHeader.substring(6);
const decoded = Buffer.from(base64, 'base64').toString('utf-8');
const [username, password] = decoded.split(':');
// PyPI token authentication: username = __token__
if (username === '__token__') {
return this.authManager.validateToken(password, 'pypi');
}
// Username/password authentication (would need user lookup)
// For now, not implemented
return null;
}
// Handle Bearer token
if (authHeader.startsWith('Bearer ')) {
const token = authHeader.substring(7);
return this.authManager.validateToken(token, 'pypi');
}
return null;
}
/**
* Handle package upload (multipart/form-data)
* POST / with :action=file_upload
*/
private async handleUpload(context: IRequestContext, token: IAuthToken | null): Promise<IResponse> {
if (!token) {
return {
status: 401,
headers: {
'Content-Type': 'application/json',
'WWW-Authenticate': 'Basic realm="PyPI"'
},
body: { error: 'Authentication required' },
};
}
try {
// Parse multipart form data (context.body should be parsed by server)
const formData = context.body as any; // Assuming parsed multipart data
if (!formData || formData[':action'] !== 'file_upload') {
return this.errorResponse(400, 'Invalid upload request');
}
// Extract required fields - support both nested and flat body formats
const packageName = formData.name;
const version = formData.version;
// Support both: formData.content.filename (multipart parsed) and formData.filename (flat)
const filename = formData.content?.filename || formData.filename;
// Support both: formData.content.data (multipart parsed) and formData.content (Buffer/Uint8Array directly)
const rawContent = formData.content?.data || (isBinaryData(formData.content) ? formData.content : null);
const fileData = rawContent ? toBuffer(rawContent) : null;
const filetype = formData.filetype; // 'bdist_wheel' or 'sdist'
const pyversion = formData.pyversion;
if (!packageName || !version || !filename || !fileData) {
return this.errorResponse(400, 'Missing required fields');
}
// Validate package name
if (!helpers.isValidPackageName(packageName)) {
return this.errorResponse(400, 'Invalid package name');
}
const normalized = helpers.normalizePypiPackageName(packageName);
// Check permission
if (!(await this.checkPermission(token, normalized, 'write'))) {
return this.errorResponse(403, 'Insufficient permissions');
}
// Calculate and verify hashes
const hashes: Record<string, string> = {};
// Always calculate SHA256
const actualSha256 = await helpers.calculateHash(fileData, 'sha256');
hashes.sha256 = actualSha256;
// Verify client-provided SHA256 if present
if (formData.sha256_digest && formData.sha256_digest !== actualSha256) {
return this.errorResponse(400, 'SHA256 hash mismatch');
}
// Calculate MD5 if requested
if (formData.md5_digest) {
const actualMd5 = await helpers.calculateHash(fileData, 'md5');
hashes.md5 = actualMd5;
// Verify if client provided MD5
if (formData.md5_digest !== actualMd5) {
return this.errorResponse(400, 'MD5 hash mismatch');
}
}
// Calculate Blake2b if requested
if (formData.blake2_256_digest) {
const actualBlake2b = await helpers.calculateHash(fileData, 'blake2b');
hashes.blake2b = actualBlake2b;
// Verify if client provided Blake2b
if (formData.blake2_256_digest !== actualBlake2b) {
return this.errorResponse(400, 'Blake2b hash mismatch');
}
}
// Store file
await this.storage.putPypiPackageFile(normalized, filename, fileData);
// Update metadata
let metadata = await this.storage.getPypiPackageMetadata(normalized);
if (!metadata) {
metadata = {
name: normalized,
versions: {},
};
}
if (!metadata.versions[version]) {
metadata.versions[version] = {
version,
files: [],
};
}
// Add file to version
metadata.versions[version].files.push({
filename,
path: `pypi/packages/${normalized}/${filename}`,
filetype,
python_version: pyversion,
hashes,
size: fileData.length,
'requires-python': formData.requires_python,
'upload-time': new Date().toISOString(),
'uploaded-by': token.userId,
});
// Store core metadata if provided
if (formData.summary || formData.description) {
metadata.versions[version].metadata = helpers.extractCoreMetadata(formData);
}
metadata['last-modified'] = new Date().toISOString();
await this.storage.putPypiPackageMetadata(normalized, metadata);
this.logger.log('info', `Package uploaded: ${normalized} ${version}`, {
filename,
size: fileData.length
});
return {
status: 201,
headers: { 'Content-Type': 'application/json' },
body: {
message: 'Package uploaded successfully',
url: `${this.registryUrl}/pypi/packages/${normalized}/${filename}`
},
};
} catch (error) {
this.logger.log('error', 'Upload failed', { error: (error as Error).message });
return this.errorResponse(500, 'Upload failed: ' + (error as Error).message);
}
}
/**
* Handle package download
*/
private async handleDownload(packageName: string, filename: string, actor?: IRequestActor): Promise<IResponse> {
const normalized = helpers.normalizePypiPackageName(packageName);
// Try streaming from local storage first
const streamResult = await this.storage.getPypiPackageFileStream(normalized, filename);
if (streamResult) {
return {
status: 200,
headers: {
'Content-Type': 'application/octet-stream',
'Content-Disposition': `attachment; filename="${filename}"`,
'Content-Length': streamResult.size.toString()
},
body: streamResult.stream,
};
}
// Try upstream if not found locally
let fileData: Buffer | null = null;
const upstream = await this.getUpstreamForRequest(normalized, 'file', 'GET', actor);
if (upstream) {
fileData = await upstream.fetchPackageFile(normalized, filename);
if (fileData) {
// Cache locally
await this.storage.putPypiPackageFile(normalized, filename, fileData);
}
}
if (!fileData) {
return {
status: 404,
headers: { 'Content-Type': 'application/json' },
body: { error: 'File not found' },
};
}
return {
status: 200,
headers: {
'Content-Type': 'application/octet-stream',
'Content-Disposition': `attachment; filename="${filename}"`,
'Content-Length': fileData.length.toString()
},
body: fileData,
};
}
/**
* Handle package JSON API (all versions)
* Returns format compatible with official PyPI JSON API
*/
private async handlePackageJson(packageName: string): Promise<IResponse> {
const normalized = helpers.normalizePypiPackageName(packageName);
const metadata = await this.storage.getPypiPackageMetadata(normalized);
if (!metadata) {
return this.errorResponse(404, 'Package not found');
}
// Find latest version for info
const versions = Object.keys(metadata.versions || {});
const latestVersion = versions.length > 0 ? versions[versions.length - 1] : null;
const latestMeta = latestVersion ? metadata.versions[latestVersion] : null;
// Build URLs array from latest version files
const urls = latestMeta?.files?.map((file: any) => ({
filename: file.filename,
url: `${this.registryUrl}/pypi/packages/${normalized}/${file.filename}`,
digests: file.hashes,
requires_python: file['requires-python'],
size: file.size,
upload_time: file['upload-time'],
packagetype: file.filetype,
python_version: file.python_version,
})) || [];
// Build releases object
const releases: Record<string, any[]> = {};
for (const [ver, verMeta] of Object.entries(metadata.versions || {})) {
releases[ver] = (verMeta as any).files?.map((file: any) => ({
filename: file.filename,
url: `${this.registryUrl}/pypi/packages/${normalized}/${file.filename}`,
digests: file.hashes,
requires_python: file['requires-python'],
size: file.size,
upload_time: file['upload-time'],
packagetype: file.filetype,
python_version: file.python_version,
})) || [];
}
const response = {
info: {
name: normalized,
version: latestVersion,
summary: latestMeta?.metadata?.summary,
description: latestMeta?.metadata?.description,
author: latestMeta?.metadata?.author,
author_email: latestMeta?.metadata?.['author-email'],
license: latestMeta?.metadata?.license,
requires_python: latestMeta?.files?.[0]?.['requires-python'],
...latestMeta?.metadata,
},
urls,
releases,
};
return {
status: 200,
headers: {
'Content-Type': 'application/json',
'Cache-Control': 'public, max-age=300'
},
body: response,
};
}
/**
* Handle version-specific JSON API
* Returns format compatible with official PyPI JSON API
*/
private async handleVersionJson(packageName: string, version: string): Promise<IResponse> {
const normalized = helpers.normalizePypiPackageName(packageName);
const metadata = await this.storage.getPypiPackageMetadata(normalized);
if (!metadata || !metadata.versions[version]) {
return this.errorResponse(404, 'Version not found');
}
const verMeta = metadata.versions[version];
// Build URLs array from version files
const urls = verMeta.files?.map((file: any) => ({
filename: file.filename,
url: `${this.registryUrl}/pypi/packages/${normalized}/${file.filename}`,
digests: file.hashes,
requires_python: file['requires-python'],
size: file.size,
upload_time: file['upload-time'],
packagetype: file.filetype,
python_version: file.python_version,
})) || [];
const response = {
info: {
name: normalized,
version,
summary: verMeta.metadata?.summary,
description: verMeta.metadata?.description,
author: verMeta.metadata?.author,
author_email: verMeta.metadata?.['author-email'],
license: verMeta.metadata?.license,
requires_python: verMeta.files?.[0]?.['requires-python'],
...verMeta.metadata,
},
urls,
};
return {
status: 200,
headers: {
'Content-Type': 'application/json',
'Cache-Control': 'public, max-age=300'
},
body: response,
};
}
/**
* Handle package deletion
*/
private async handleDeletePackage(packageName: string, token: IAuthToken | null): Promise<IResponse> {
if (!token) {
return this.errorResponse(401, 'Authentication required');
}
const normalized = helpers.normalizePypiPackageName(packageName);
if (!(await this.checkPermission(token, normalized, 'delete'))) {
return this.errorResponse(403, 'Insufficient permissions');
}
await this.storage.deletePypiPackage(normalized);
this.logger.log('info', `Package deleted: ${normalized}`);
return {
status: 204,
headers: {},
body: Buffer.from(''),
};
}
/**
* Handle version deletion
*/
private async handleDeleteVersion(
packageName: string,
version: string,
token: IAuthToken | null
): Promise<IResponse> {
if (!token) {
return this.errorResponse(401, 'Authentication required');
}
const normalized = helpers.normalizePypiPackageName(packageName);
if (!(await this.checkPermission(token, normalized, 'delete'))) {
return this.errorResponse(403, 'Insufficient permissions');
}
await this.storage.deletePypiPackageVersion(normalized, version);
this.logger.log('info', `Version deleted: ${normalized} ${version}`);
return {
status: 204,
headers: {},
body: Buffer.from(''),
};
}
/**
* Helper: Create error response
*/
private errorResponse(status: number, message: string): IResponse {
const error: IPypiError = { error: message, status };
return {
status,
headers: { 'Content-Type': 'application/json' },
body: error,
};
}
}

View File

@@ -0,0 +1,211 @@
import * as plugins from '../plugins.js';
import { BaseUpstream } from '../upstream/classes.baseupstream.js';
import type {
IProtocolUpstreamConfig,
IUpstreamFetchContext,
IUpstreamRegistryConfig,
} from '../upstream/interfaces.upstream.js';
/**
* PyPI-specific upstream implementation.
*
* Handles:
* - Simple API (HTML) - PEP 503
* - JSON API - PEP 691
* - Package file downloads (wheels, sdists)
* - Package name normalization
*/
export class PypiUpstream extends BaseUpstream {
protected readonly protocolName = 'pypi';
/** Local registry URL for rewriting download URLs */
private readonly localRegistryUrl: string;
constructor(
config: IProtocolUpstreamConfig,
localRegistryUrl: string,
logger?: plugins.smartlog.Smartlog,
) {
super(config, logger);
this.localRegistryUrl = localRegistryUrl;
}
/**
* Fetch Simple API index (list of all packages) in HTML format.
*/
public async fetchSimpleIndex(): Promise<string | null> {
const context: IUpstreamFetchContext = {
protocol: 'pypi',
resource: '*',
resourceType: 'index',
path: '/simple/',
method: 'GET',
headers: {
'accept': 'text/html',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
if (Buffer.isBuffer(result.body)) {
return result.body.toString('utf8');
}
return typeof result.body === 'string' ? result.body : null;
}
/**
* Fetch Simple API package page (list of files) in HTML format.
*/
public async fetchSimplePackage(packageName: string): Promise<string | null> {
const normalizedName = this.normalizePackageName(packageName);
const path = `/simple/${normalizedName}/`;
const context: IUpstreamFetchContext = {
protocol: 'pypi',
resource: packageName,
resourceType: 'simple',
path,
method: 'GET',
headers: {
'accept': 'text/html',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
if (Buffer.isBuffer(result.body)) {
return result.body.toString('utf8');
}
return typeof result.body === 'string' ? result.body : null;
}
/**
* Fetch package metadata using JSON API (PEP 691).
*/
public async fetchPackageJson(packageName: string): Promise<any | null> {
const normalizedName = this.normalizePackageName(packageName);
const path = `/simple/${normalizedName}/`;
const context: IUpstreamFetchContext = {
protocol: 'pypi',
resource: packageName,
resourceType: 'metadata',
path,
method: 'GET',
headers: {
'accept': 'application/vnd.pypi.simple.v1+json',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
if (Buffer.isBuffer(result.body)) {
return JSON.parse(result.body.toString('utf8'));
}
return result.body;
}
/**
* Fetch full package info from PyPI JSON API (/pypi/{package}/json).
*/
public async fetchPypiJson(packageName: string): Promise<any | null> {
const normalizedName = this.normalizePackageName(packageName);
const path = `/pypi/${normalizedName}/json`;
const context: IUpstreamFetchContext = {
protocol: 'pypi',
resource: packageName,
resourceType: 'pypi-json',
path,
method: 'GET',
headers: {
'accept': 'application/json',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
if (Buffer.isBuffer(result.body)) {
return JSON.parse(result.body.toString('utf8'));
}
return result.body;
}
/**
* Fetch a package file (wheel or sdist) from upstream.
*/
public async fetchPackageFile(packageName: string, filename: string): Promise<Buffer | null> {
const normalizedName = this.normalizePackageName(packageName);
const path = `/packages/${normalizedName}/${filename}`;
const context: IUpstreamFetchContext = {
protocol: 'pypi',
resource: packageName,
resourceType: 'package',
path,
method: 'GET',
headers: {
'accept': 'application/octet-stream',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
return Buffer.isBuffer(result.body) ? result.body : Buffer.from(result.body);
}
/**
* Normalize a PyPI package name according to PEP 503.
* - Lowercase all characters
* - Replace runs of ., -, _ with single -
*/
private normalizePackageName(name: string): string {
return name.toLowerCase().replace(/[-_.]+/g, '-');
}
/**
* Override URL building for PyPI-specific handling.
*/
protected buildUpstreamUrl(
upstream: IUpstreamRegistryConfig,
context: IUpstreamFetchContext,
): string {
let baseUrl = upstream.url;
// Remove trailing slash
if (baseUrl.endsWith('/')) {
baseUrl = baseUrl.slice(0, -1);
}
return `${baseUrl}${context.path}`;
}
}

299
ts/pypi/helpers.pypi.ts Normal file
View File

@@ -0,0 +1,299 @@
/**
* Helper functions for PyPI registry
* Package name normalization, HTML generation, etc.
*/
import type { IPypiFile, IPypiPackageMetadata } from './interfaces.pypi.js';
/**
* Normalize package name according to PEP 503
* Lowercase and replace runs of [._-] with a single dash
* @param name - Package name
* @returns Normalized name
*/
export function normalizePypiPackageName(name: string): string {
return name
.toLowerCase()
.replace(/[-_.]+/g, '-');
}
/**
* Escape HTML special characters to prevent XSS
* @param str - String to escape
* @returns Escaped string
*/
export function escapeHtml(str: string): string {
return str
.replace(/&/g, '&amp;')
.replace(/</g, '&lt;')
.replace(/>/g, '&gt;')
.replace(/"/g, '&quot;')
.replace(/'/g, '&#039;');
}
/**
* Generate PEP 503 compliant HTML for root index (all packages)
* @param packages - List of package names
* @returns HTML string
*/
export function generateSimpleRootHtml(packages: string[]): string {
const links = packages
.map(pkg => {
const normalized = normalizePypiPackageName(pkg);
return ` <a href="${escapeHtml(normalized)}/">${escapeHtml(pkg)}</a>`;
})
.join('\n');
return `<!DOCTYPE html>
<html>
<head>
<meta name="pypi:repository-version" content="1.0">
<title>Simple Index</title>
</head>
<body>
<h1>Simple Index</h1>
${links}
</body>
</html>`;
}
/**
* Generate PEP 503 compliant HTML for package index (file list)
* @param packageName - Package name (normalized)
* @param files - List of files
* @param baseUrl - Base URL for downloads
* @returns HTML string
*/
export function generateSimplePackageHtml(
packageName: string,
files: IPypiFile[],
baseUrl: string
): string {
const links = files
.map(file => {
// Build URL
let url = file.url;
if (!url.startsWith('http://') && !url.startsWith('https://')) {
// Relative URL - make it absolute
url = `${baseUrl}/packages/${packageName}/${file.filename}`;
}
// Add hash fragment
const hashName = Object.keys(file.hashes)[0];
const hashValue = file.hashes[hashName];
const fragment = hashName && hashValue ? `#${hashName}=${hashValue}` : '';
// Build data attributes
const dataAttrs: string[] = [];
if (file['requires-python']) {
const escaped = escapeHtml(file['requires-python']);
dataAttrs.push(`data-requires-python="${escaped}"`);
}
if (file['gpg-sig'] !== undefined) {
dataAttrs.push(`data-gpg-sig="${file['gpg-sig'] ? 'true' : 'false'}"`);
}
if (file.yanked) {
const reason = typeof file.yanked === 'string' ? file.yanked : '';
if (reason) {
dataAttrs.push(`data-yanked="${escapeHtml(reason)}"`);
} else {
dataAttrs.push(`data-yanked=""`);
}
}
const dataAttrStr = dataAttrs.length > 0 ? ' ' + dataAttrs.join(' ') : '';
return ` <a href="${escapeHtml(url)}${fragment}"${dataAttrStr}>${escapeHtml(file.filename)}</a>`;
})
.join('\n');
return `<!DOCTYPE html>
<html>
<head>
<meta name="pypi:repository-version" content="1.0">
<title>Links for ${escapeHtml(packageName)}</title>
</head>
<body>
<h1>Links for ${escapeHtml(packageName)}</h1>
${links}
</body>
</html>`;
}
/**
* Parse filename to extract package info
* Supports wheel and sdist formats
* @param filename - Package filename
* @returns Parsed info or null
*/
export function parsePackageFilename(filename: string): {
name: string;
version: string;
filetype: 'bdist_wheel' | 'sdist';
pythonVersion?: string;
} | null {
// Wheel format: {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl
const wheelMatch = filename.match(/^([a-zA-Z0-9_.-]+?)-([a-zA-Z0-9_.]+?)(?:-(\d+))?-([^-]+)-([^-]+)-([^-]+)\.whl$/);
if (wheelMatch) {
return {
name: wheelMatch[1],
version: wheelMatch[2],
filetype: 'bdist_wheel',
pythonVersion: wheelMatch[4],
};
}
// Sdist tar.gz format: {name}-{version}.tar.gz
const sdistTarMatch = filename.match(/^([a-zA-Z0-9_.-]+?)-([a-zA-Z0-9_.]+)\.tar\.gz$/);
if (sdistTarMatch) {
return {
name: sdistTarMatch[1],
version: sdistTarMatch[2],
filetype: 'sdist',
pythonVersion: 'source',
};
}
// Sdist zip format: {name}-{version}.zip
const sdistZipMatch = filename.match(/^([a-zA-Z0-9_.-]+?)-([a-zA-Z0-9_.]+)\.zip$/);
if (sdistZipMatch) {
return {
name: sdistZipMatch[1],
version: sdistZipMatch[2],
filetype: 'sdist',
pythonVersion: 'source',
};
}
return null;
}
/**
* Calculate hash digest for a buffer
* @param data - Data to hash
* @param algorithm - Hash algorithm (sha256, md5, blake2b)
* @returns Hex-encoded hash
*/
export async function calculateHash(data: Buffer, algorithm: 'sha256' | 'md5' | 'blake2b'): Promise<string> {
const crypto = await import('crypto');
let hash: any;
if (algorithm === 'blake2b') {
// Node.js uses 'blake2b512' for blake2b
hash = crypto.createHash('blake2b512');
} else {
hash = crypto.createHash(algorithm);
}
hash.update(data);
return hash.digest('hex');
}
/**
* Validate package name
* Must contain only ASCII letters, numbers, ., -, and _
* @param name - Package name
* @returns true if valid
*/
export function isValidPackageName(name: string): boolean {
return /^[a-zA-Z0-9._-]+$/.test(name);
}
/**
* Validate version string (basic check)
* @param version - Version string
* @returns true if valid
*/
export function isValidVersion(version: string): boolean {
// Basic check - allows numbers, letters, dots, hyphens, underscores
// More strict validation would follow PEP 440
return /^[a-zA-Z0-9._-]+$/.test(version);
}
/**
* Extract metadata from package metadata
* Filters and normalizes metadata fields
* @param metadata - Raw metadata object
* @returns Filtered metadata
*/
export function extractCoreMetadata(metadata: Record<string, any>): Record<string, any> {
const coreFields = [
'metadata-version',
'name',
'version',
'platform',
'supported-platform',
'summary',
'description',
'description-content-type',
'keywords',
'home-page',
'download-url',
'author',
'author-email',
'maintainer',
'maintainer-email',
'license',
'classifier',
'requires-python',
'requires-dist',
'requires-external',
'provides-dist',
'project-url',
'provides-extra',
];
const result: Record<string, any> = {};
for (const [key, value] of Object.entries(metadata)) {
const normalizedKey = key.toLowerCase().replace(/_/g, '-');
if (coreFields.includes(normalizedKey)) {
result[normalizedKey] = value;
}
}
return result;
}
/**
* Generate JSON API response for package list (PEP 691)
* @param packages - List of package names
* @returns JSON object
*/
export function generateJsonRootResponse(packages: string[]): any {
return {
meta: {
'api-version': '1.0',
},
projects: packages.map(name => ({ name })),
};
}
/**
* Generate JSON API response for package files (PEP 691)
* @param packageName - Package name (normalized)
* @param files - List of files
* @returns JSON object
*/
export function generateJsonPackageResponse(packageName: string, files: IPypiFile[]): any {
return {
meta: {
'api-version': '1.0',
},
name: packageName,
files: files.map(file => ({
filename: file.filename,
url: file.url,
hashes: file.hashes,
'requires-python': file['requires-python'],
'dist-info-metadata': file['dist-info-metadata'],
'gpg-sig': file['gpg-sig'],
yanked: file.yanked,
size: file.size,
'upload-time': file['upload-time'],
})),
};
}

9
ts/pypi/index.ts Normal file
View File

@@ -0,0 +1,9 @@
/**
* PyPI Registry Module
* Python Package Index implementation
*/
export * from './interfaces.pypi.js';
export * from './classes.pypiregistry.js';
export { PypiUpstream } from './classes.pypiupstream.js';
export * as pypiHelpers from './helpers.pypi.js';

316
ts/pypi/interfaces.pypi.ts Normal file
View File

@@ -0,0 +1,316 @@
/**
* PyPI Registry Type Definitions
* Compliant with PEP 503 (Simple API), PEP 691 (JSON API), and PyPI upload API
*/
/**
* File information for a package distribution
* Used in both PEP 503 HTML and PEP 691 JSON responses
*/
export interface IPypiFile {
/** Filename (e.g., "package-1.0.0-py3-none-any.whl") */
filename: string;
/** Download URL (absolute or relative) */
url: string;
/** Hash digests (multiple algorithms supported in JSON) */
hashes: Record<string, string>;
/** Python version requirement (PEP 345 format) */
'requires-python'?: string;
/** Whether distribution info metadata is available (PEP 658) */
'dist-info-metadata'?: boolean | { sha256: string };
/** Whether GPG signature is available */
'gpg-sig'?: boolean;
/** Yank status: false or reason string */
yanked?: boolean | string;
/** File size in bytes */
size?: number;
/** Upload timestamp */
'upload-time'?: string;
}
/**
* Package metadata stored internally
* Consolidated from multiple file uploads
*/
export interface IPypiPackageMetadata {
/** Normalized package name */
name: string;
/** Map of version to file list */
versions: Record<string, IPypiVersionMetadata>;
/** Timestamp of last update */
'last-modified'?: string;
}
/**
* Metadata for a specific version
*/
export interface IPypiVersionMetadata {
/** Version string */
version: string;
/** Files for this version (wheels, sdists) */
files: IPypiFileMetadata[];
/** Core metadata fields */
metadata?: IPypiCoreMetadata;
/** Whether entire version is yanked */
yanked?: boolean | string;
/** Upload timestamp */
'upload-time'?: string;
}
/**
* Internal file metadata
*/
export interface IPypiFileMetadata {
filename: string;
/** Storage key/path */
path: string;
/** File type: bdist_wheel or sdist */
filetype: 'bdist_wheel' | 'sdist';
/** Python version tag */
python_version: string;
/** Hash digests */
hashes: Record<string, string>;
/** File size in bytes */
size: number;
/** Python version requirement */
'requires-python'?: string;
/** Whether this file is yanked */
yanked?: boolean | string;
/** Upload timestamp */
'upload-time': string;
/** Uploader user ID */
'uploaded-by': string;
}
/**
* Core metadata fields (subset of PEP 566)
* These are extracted from package uploads
*/
export interface IPypiCoreMetadata {
/** Metadata version */
'metadata-version': string;
/** Package name */
name: string;
/** Version string */
version: string;
/** Platform compatibility */
platform?: string;
/** Supported platforms */
'supported-platform'?: string;
/** Summary/description */
summary?: string;
/** Long description */
description?: string;
/** Description content type (text/plain, text/markdown, text/x-rst) */
'description-content-type'?: string;
/** Keywords */
keywords?: string;
/** Homepage URL */
'home-page'?: string;
/** Download URL */
'download-url'?: string;
/** Author name */
author?: string;
/** Author email */
'author-email'?: string;
/** Maintainer name */
maintainer?: string;
/** Maintainer email */
'maintainer-email'?: string;
/** License */
license?: string;
/** Classifiers (Trove classifiers) */
classifier?: string[];
/** Python version requirement */
'requires-python'?: string;
/** Dist name requirement */
'requires-dist'?: string[];
/** External requirement */
'requires-external'?: string[];
/** Provides dist */
'provides-dist'?: string[];
/** Project URLs */
'project-url'?: string[];
/** Provides extra */
'provides-extra'?: string[];
}
/**
* PEP 503: Simple API root response (project list)
*/
export interface IPypiSimpleRootHtml {
/** List of project names */
projects: string[];
}
/**
* PEP 503: Simple API project response (file list)
*/
export interface IPypiSimpleProjectHtml {
/** Normalized project name */
name: string;
/** List of files */
files: IPypiFile[];
}
/**
* PEP 691: JSON API root response
*/
export interface IPypiJsonRoot {
/** API metadata */
meta: {
/** API version (e.g., "1.0") */
'api-version': string;
};
/** List of projects */
projects: Array<{
/** Project name */
name: string;
}>;
}
/**
* PEP 691: JSON API project response
*/
export interface IPypiJsonProject {
/** Normalized project name */
name: string;
/** API metadata */
meta: {
/** API version (e.g., "1.0") */
'api-version': string;
};
/** List of files */
files: IPypiFile[];
}
/**
* Upload form data (multipart/form-data fields)
* Based on PyPI legacy upload API
*/
export interface IPypiUploadForm {
/** Action type (always "file_upload") */
':action': 'file_upload';
/** Protocol version (always "1") */
protocol_version: '1';
/** File content (binary) */
content: Buffer;
/** File type */
filetype: 'bdist_wheel' | 'sdist';
/** Python version tag */
pyversion: string;
/** Package name */
name: string;
/** Version string */
version: string;
/** Metadata version */
metadata_version: string;
/** Hash digests (at least one required) */
md5_digest?: string;
sha256_digest?: string;
blake2_256_digest?: string;
/** Optional attestations */
attestations?: string; // JSON array
/** Optional core metadata fields */
summary?: string;
description?: string;
description_content_type?: string;
author?: string;
author_email?: string;
maintainer?: string;
maintainer_email?: string;
license?: string;
keywords?: string;
home_page?: string;
download_url?: string;
requires_python?: string;
classifiers?: string[];
platform?: string;
[key: string]: any; // Allow additional metadata fields
}
/**
* JSON API upload response
*/
export interface IPypiUploadResponse {
/** Success message */
message?: string;
/** URL of uploaded file */
url?: string;
}
/**
* Error response structure
*/
export interface IPypiError {
/** Error message */
error: string;
/** HTTP status code */
status?: number;
/** Additional error details */
details?: string[];
}
/**
* Search query parameters
*/
export interface IPypiSearchQuery {
/** Search term */
q?: string;
/** Page number */
page?: number;
/** Results per page */
per_page?: number;
}
/**
* Search result for a single package
*/
export interface IPypiSearchResult {
/** Package name */
name: string;
/** Latest version */
version: string;
/** Summary */
summary: string;
/** Description */
description?: string;
}
/**
* Search response structure
*/
export interface IPypiSearchResponse {
/** Search results */
results: IPypiSearchResult[];
/** Result count */
count: number;
/** Current page */
page: number;
/** Total pages */
pages: number;
}
/**
* Yank request
*/
export interface IPypiYankRequest {
/** Package name */
name: string;
/** Version to yank */
version: string;
/** Optional filename (specific file) */
filename?: string;
/** Reason for yanking */
reason?: string;
}
/**
* Yank response
*/
export interface IPypiYankResponse {
/** Success indicator */
success: boolean;
/** Message */
message?: string;
}

View File

@@ -0,0 +1,814 @@
import { Smartlog } from '@push.rocks/smartlog';
import { BaseRegistry } from '../core/classes.baseregistry.js';
import { RegistryStorage } from '../core/classes.registrystorage.js';
import { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from '../core/interfaces.core.js';
import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
import type {
IRubyGemsMetadata,
IRubyGemsVersionMetadata,
IRubyGemsUploadResponse,
IRubyGemsYankResponse,
IRubyGemsError,
ICompactIndexInfoEntry,
} from './interfaces.rubygems.js';
import * as helpers from './helpers.rubygems.js';
import { RubygemsUpstream } from './classes.rubygemsupstream.js';
/**
* RubyGems registry implementation
* Implements Compact Index API and RubyGems protocol
*/
export class RubyGemsRegistry extends BaseRegistry {
private storage: RegistryStorage;
private authManager: AuthManager;
private basePath: string = '/rubygems';
private registryUrl: string;
private logger: Smartlog;
private upstreamProvider: IUpstreamProvider | null = null;
constructor(
storage: RegistryStorage,
authManager: AuthManager,
basePath: string = '/rubygems',
registryUrl: string = 'http://localhost:5000/rubygems',
upstreamProvider?: IUpstreamProvider
) {
super();
this.storage = storage;
this.authManager = authManager;
this.basePath = basePath;
this.registryUrl = registryUrl;
this.upstreamProvider = upstreamProvider || null;
// Initialize logger
this.logger = new Smartlog({
logContext: {
company: 'push.rocks',
companyunit: 'smartregistry',
containerName: 'rubygems-registry',
environment: (process.env.NODE_ENV as any) || 'development',
runtime: 'node',
zone: 'rubygems'
}
});
this.logger.enableConsole();
}
/**
* Get upstream for a specific request.
* Calls the provider to resolve upstream config dynamically.
*/
private async getUpstreamForRequest(
resource: string,
resourceType: string,
method: string,
actor?: IRequestActor
): Promise<RubygemsUpstream | null> {
if (!this.upstreamProvider) return null;
const config = await this.upstreamProvider.resolveUpstreamConfig({
protocol: 'rubygems',
resource,
scope: resource, // gem name is the scope
actor,
method,
resourceType,
});
if (!config?.enabled) return null;
return new RubygemsUpstream(config, this.logger);
}
/**
* Clean up resources (timers, connections, etc.)
*/
public destroy(): void {
// No persistent upstream to clean up with dynamic provider
}
public async init(): Promise<void> {
// Initialize Compact Index files if not exist
const existingVersions = await this.storage.getRubyGemsVersions();
if (!existingVersions) {
const versions = helpers.generateCompactIndexVersions([]);
await this.storage.putRubyGemsVersions(versions);
this.logger.log('info', 'Initialized RubyGems Compact Index');
}
const existingNames = await this.storage.getRubyGemsNames();
if (!existingNames) {
const names = helpers.generateNamesFile([]);
await this.storage.putRubyGemsNames(names);
this.logger.log('info', 'Initialized RubyGems names file');
}
}
public getBasePath(): string {
return this.basePath;
}
public async handleRequest(context: IRequestContext): Promise<IResponse> {
let path = context.path.replace(this.basePath, '');
// Extract token (Authorization header)
const token = await this.extractToken(context);
// Build actor from context and validated token
const actor: IRequestActor = {
...context.actor,
userId: token?.userId,
ip: context.headers['x-forwarded-for'] || context.headers['X-Forwarded-For'],
userAgent: context.headers['user-agent'] || context.headers['User-Agent'],
};
this.logger.log('debug', `handleRequest: ${context.method} ${path}`, {
method: context.method,
path,
hasAuth: !!token
});
// Compact Index endpoints
if (path === '/versions' && context.method === 'GET') {
return this.handleVersionsFile(context);
}
if (path === '/names' && context.method === 'GET') {
return this.handleNamesFile();
}
// Info file: GET /info/{gem}
const infoMatch = path.match(/^\/info\/([^\/]+)$/);
if (infoMatch && context.method === 'GET') {
return this.handleInfoFile(infoMatch[1], actor);
}
// Gem download: GET /gems/{gem}-{version}[-{platform}].gem
const downloadMatch = path.match(/^\/gems\/(.+\.gem)$/);
if (downloadMatch && context.method === 'GET') {
return this.handleDownload(downloadMatch[1], actor);
}
// Legacy specs endpoints (Marshal format)
if (path === '/specs.4.8.gz' && context.method === 'GET') {
return this.handleSpecs(false);
}
if (path === '/latest_specs.4.8.gz' && context.method === 'GET') {
return this.handleSpecs(true);
}
// Quick gemspec endpoint: GET /quick/Marshal.4.8/{gem}-{version}.gemspec.rz
const quickMatch = path.match(/^\/quick\/Marshal\.4\.8\/(.+)\.gemspec\.rz$/);
if (quickMatch && context.method === 'GET') {
return this.handleQuickGemspec(quickMatch[1]);
}
// API v1 endpoints
if (path.startsWith('/api/v1/')) {
return this.handleApiRequest(path.substring(7), context, token);
}
return {
status: 404,
headers: { 'Content-Type': 'application/json' },
body: { error: 'Not Found' },
};
}
/**
* Check if token has permission for resource
*/
protected async checkPermission(
token: IAuthToken | null,
resource: string,
action: string
): Promise<boolean> {
if (!token) return false;
return this.authManager.authorize(token, `rubygems:gem:${resource}`, action);
}
/**
* Extract authentication token from request
*/
private async extractToken(context: IRequestContext): Promise<IAuthToken | null> {
const authHeader = context.headers['authorization'] || context.headers['Authorization'];
if (!authHeader) return null;
// RubyGems typically uses plain API key in Authorization header
return this.authManager.validateToken(authHeader, 'rubygems');
}
/**
* Handle /versions endpoint (Compact Index)
* Supports conditional GET with If-None-Match header
*/
private async handleVersionsFile(context: IRequestContext): Promise<IResponse> {
const content = await this.storage.getRubyGemsVersions();
if (!content) {
return this.errorResponse(500, 'Versions file not initialized');
}
const etag = `"${await helpers.calculateMD5(content)}"`;
// Handle conditional GET with If-None-Match
const ifNoneMatch = context.headers['if-none-match'] || context.headers['If-None-Match'];
if (ifNoneMatch && ifNoneMatch === etag) {
return {
status: 304,
headers: {
'ETag': etag,
'Cache-Control': 'public, max-age=60',
},
body: null,
};
}
return {
status: 200,
headers: {
'Content-Type': 'text/plain; charset=utf-8',
'Cache-Control': 'public, max-age=60',
'ETag': etag
},
body: Buffer.from(content),
};
}
/**
* Handle /names endpoint (Compact Index)
*/
private async handleNamesFile(): Promise<IResponse> {
const content = await this.storage.getRubyGemsNames();
if (!content) {
return this.errorResponse(500, 'Names file not initialized');
}
return {
status: 200,
headers: {
'Content-Type': 'text/plain; charset=utf-8',
'Cache-Control': 'public, max-age=300'
},
body: Buffer.from(content),
};
}
/**
* Handle /info/{gem} endpoint (Compact Index)
*/
private async handleInfoFile(gemName: string, actor?: IRequestActor): Promise<IResponse> {
let content = await this.storage.getRubyGemsInfo(gemName);
// Try upstream if not found locally
if (!content) {
const upstream = await this.getUpstreamForRequest(gemName, 'info', 'GET', actor);
if (upstream) {
const upstreamInfo = await upstream.fetchInfo(gemName);
if (upstreamInfo) {
// Cache locally
await this.storage.putRubyGemsInfo(gemName, upstreamInfo);
content = upstreamInfo;
}
}
}
if (!content) {
return {
status: 404,
headers: { 'Content-Type': 'text/plain' },
body: Buffer.from('Not Found'),
};
}
return {
status: 200,
headers: {
'Content-Type': 'text/plain; charset=utf-8',
'Cache-Control': 'public, max-age=300',
'ETag': `"${await helpers.calculateMD5(content)}"`
},
body: Buffer.from(content),
};
}
/**
* Handle gem file download
*/
private async handleDownload(filename: string, actor?: IRequestActor): Promise<IResponse> {
const parsed = helpers.parseGemFilename(filename);
if (!parsed) {
return this.errorResponse(400, 'Invalid gem filename');
}
// Try streaming from local storage first
const streamResult = await this.storage.getRubyGemsGemStream(
parsed.name,
parsed.version,
parsed.platform
);
if (streamResult) {
return {
status: 200,
headers: {
'Content-Type': 'application/octet-stream',
'Content-Disposition': `attachment; filename="${filename}"`,
'Content-Length': streamResult.size.toString()
},
body: streamResult.stream,
};
}
// Try upstream if not found locally
let gemData: Buffer | null = null;
const upstream = await this.getUpstreamForRequest(parsed.name, 'gem', 'GET', actor);
if (upstream) {
gemData = await upstream.fetchGem(parsed.name, parsed.version);
if (gemData) {
// Cache locally
await this.storage.putRubyGemsGem(parsed.name, parsed.version, gemData, parsed.platform);
}
}
if (!gemData) {
return this.errorResponse(404, 'Gem not found');
}
return {
status: 200,
headers: {
'Content-Type': 'application/octet-stream',
'Content-Disposition': `attachment; filename="${filename}"`,
'Content-Length': gemData.length.toString()
},
body: gemData,
};
}
/**
* Handle API v1 requests
*/
private async handleApiRequest(
path: string,
context: IRequestContext,
token: IAuthToken | null
): Promise<IResponse> {
// Upload gem: POST /gems
if (path === '/gems' && context.method === 'POST') {
return this.handleUpload(context, token);
}
// Yank gem: DELETE /gems/yank
if (path === '/gems/yank' && context.method === 'DELETE') {
return this.handleYank(context, token);
}
// Unyank gem: PUT /gems/unyank
if (path === '/gems/unyank' && context.method === 'PUT') {
return this.handleUnyank(context, token);
}
// Version list: GET /versions/{gem}.json
const versionsMatch = path.match(/^\/versions\/([^\/]+)\.json$/);
if (versionsMatch && context.method === 'GET') {
return this.handleVersionsJson(versionsMatch[1]);
}
// Dependencies: GET /dependencies?gems={list}
if (path.startsWith('/dependencies') && context.method === 'GET') {
const gemsParam = context.query?.gems || '';
return this.handleDependencies(gemsParam);
}
return this.errorResponse(404, 'API endpoint not found');
}
/**
* Handle gem upload
* POST /api/v1/gems
*/
private async handleUpload(context: IRequestContext, token: IAuthToken | null): Promise<IResponse> {
if (!token) {
return this.errorResponse(401, 'Authentication required');
}
try {
// Extract gem data from request body
const gemData = context.body as Buffer;
if (!gemData || gemData.length === 0) {
return this.errorResponse(400, 'No gem file provided');
}
// Try to get metadata from query params or headers first
let gemName = context.query?.name || context.headers['x-gem-name'] as string | undefined;
let version = context.query?.version || context.headers['x-gem-version'] as string | undefined;
let platform = context.query?.platform || context.headers['x-gem-platform'] as string | undefined;
// If not provided, try to extract from gem binary
if (!gemName || !version || !platform) {
const extracted = await helpers.extractGemMetadata(gemData);
if (extracted) {
gemName = gemName || extracted.name;
version = version || extracted.version;
platform = platform || extracted.platform;
}
}
if (!gemName || !version) {
return this.errorResponse(400, 'Gem name and version required (provide in query, headers, or valid gem format)');
}
// Validate gem name
if (!helpers.isValidGemName(gemName)) {
return this.errorResponse(400, 'Invalid gem name');
}
// Check permission
if (!(await this.checkPermission(token, gemName, 'write'))) {
return this.errorResponse(403, 'Insufficient permissions');
}
// Calculate checksum
const checksum = await helpers.calculateSHA256(gemData);
// Store gem file
await this.storage.putRubyGemsGem(gemName, version, gemData, platform);
// Update metadata
let metadata: IRubyGemsMetadata = await this.storage.getRubyGemsMetadata(gemName) || {
name: gemName,
versions: {},
};
const versionKey = platform ? `${version}-${platform}` : version;
metadata.versions[versionKey] = {
version,
platform,
checksum,
size: gemData.length,
'upload-time': new Date().toISOString(),
'uploaded-by': token.userId,
dependencies: [], // Would extract from gem spec
requirements: [],
};
metadata['last-modified'] = new Date().toISOString();
await this.storage.putRubyGemsMetadata(gemName, metadata);
// Update Compact Index info file
await this.updateCompactIndexForGem(gemName, metadata);
// Update versions file
await this.updateVersionsFile(gemName, version, platform || 'ruby', false);
// Update names file
await this.updateNamesFile(gemName);
this.logger.log('info', `Gem uploaded: ${gemName} ${version}`, {
platform,
size: gemData.length
});
return {
status: 201,
headers: { 'Content-Type': 'application/json' },
body: {
message: 'Gem uploaded successfully',
name: gemName,
version,
},
};
} catch (error) {
this.logger.log('error', 'Upload failed', { error: (error as Error).message });
return this.errorResponse(500, 'Upload failed: ' + (error as Error).message);
}
}
/**
* Handle gem yanking
* DELETE /api/v1/gems/yank
*/
private async handleYank(context: IRequestContext, token: IAuthToken | null): Promise<IResponse> {
if (!token) {
return this.errorResponse(401, 'Authentication required');
}
const gemName = context.query?.gem_name;
const version = context.query?.version;
const platform = context.query?.platform;
if (!gemName || !version) {
return this.errorResponse(400, 'Gem name and version required');
}
if (!(await this.checkPermission(token, gemName, 'yank'))) {
return this.errorResponse(403, 'Insufficient permissions');
}
// Update metadata to mark as yanked
const metadata = await this.storage.getRubyGemsMetadata(gemName);
if (!metadata) {
return this.errorResponse(404, 'Gem not found');
}
const versionKey = platform ? `${version}-${platform}` : version;
if (!metadata.versions[versionKey]) {
return this.errorResponse(404, 'Version not found');
}
metadata.versions[versionKey].yanked = true;
await this.storage.putRubyGemsMetadata(gemName, metadata);
// Update Compact Index
await this.updateCompactIndexForGem(gemName, metadata);
await this.updateVersionsFile(gemName, version, platform || 'ruby', true);
this.logger.log('info', `Gem yanked: ${gemName} ${version}`);
return {
status: 200,
headers: { 'Content-Type': 'application/json' },
body: {
success: true,
message: 'Gem yanked successfully'
},
};
}
/**
* Handle gem unyanking
* PUT /api/v1/gems/unyank
*/
private async handleUnyank(context: IRequestContext, token: IAuthToken | null): Promise<IResponse> {
if (!token) {
return this.errorResponse(401, 'Authentication required');
}
const gemName = context.query?.gem_name;
const version = context.query?.version;
const platform = context.query?.platform;
if (!gemName || !version) {
return this.errorResponse(400, 'Gem name and version required');
}
if (!(await this.checkPermission(token, gemName, 'write'))) {
return this.errorResponse(403, 'Insufficient permissions');
}
const metadata = await this.storage.getRubyGemsMetadata(gemName);
if (!metadata) {
return this.errorResponse(404, 'Gem not found');
}
const versionKey = platform ? `${version}-${platform}` : version;
if (!metadata.versions[versionKey]) {
return this.errorResponse(404, 'Version not found');
}
metadata.versions[versionKey].yanked = false;
await this.storage.putRubyGemsMetadata(gemName, metadata);
// Update Compact Index
await this.updateCompactIndexForGem(gemName, metadata);
await this.updateVersionsFile(gemName, version, platform || 'ruby', false);
this.logger.log('info', `Gem unyanked: ${gemName} ${version}`);
return {
status: 200,
headers: { 'Content-Type': 'application/json' },
body: {
success: true,
message: 'Gem unyanked successfully'
},
};
}
/**
* Handle versions JSON API
*/
private async handleVersionsJson(gemName: string): Promise<IResponse> {
const metadata = await this.storage.getRubyGemsMetadata(gemName);
if (!metadata) {
return this.errorResponse(404, 'Gem not found');
}
const versions = Object.values(metadata.versions).map((v: any) => ({
version: v.version,
platform: v.platform,
uploadTime: v['upload-time'],
}));
const response = helpers.generateVersionsJson(gemName, versions);
return {
status: 200,
headers: {
'Content-Type': 'application/json',
'Cache-Control': 'public, max-age=300'
},
body: response,
};
}
/**
* Handle dependencies query
*/
private async handleDependencies(gemsParam: string): Promise<IResponse> {
const gemNames = gemsParam.split(',').filter(n => n.trim());
const result = new Map();
for (const gemName of gemNames) {
const metadata = await this.storage.getRubyGemsMetadata(gemName);
if (metadata) {
const versions = Object.values(metadata.versions).map((v: any) => ({
version: v.version,
platform: v.platform,
dependencies: v.dependencies || [],
}));
result.set(gemName, versions);
}
}
const response = helpers.generateDependenciesJson(result);
return {
status: 200,
headers: { 'Content-Type': 'application/json' },
body: response,
};
}
/**
* Update Compact Index info file for a gem
*/
private async updateCompactIndexForGem(
gemName: string,
metadata: IRubyGemsMetadata
): Promise<void> {
const entries: ICompactIndexInfoEntry[] = Object.values(metadata.versions)
.filter(v => !v.yanked) // Exclude yanked from info file
.map(v => ({
version: v.version,
platform: v.platform,
dependencies: v.dependencies || [],
requirements: v.requirements || [],
checksum: v.checksum,
}));
const content = helpers.generateCompactIndexInfo(entries);
await this.storage.putRubyGemsInfo(gemName, content);
}
/**
* Update versions file with new/updated gem
*/
private async updateVersionsFile(
gemName: string,
version: string,
platform: string,
yanked: boolean
): Promise<void> {
const existingVersions = await this.storage.getRubyGemsVersions();
if (!existingVersions) return;
// Calculate info file checksum
const infoContent = await this.storage.getRubyGemsInfo(gemName) || '';
const infoChecksum = await helpers.calculateMD5(infoContent);
const updated = helpers.updateCompactIndexVersions(
existingVersions,
gemName,
{ version, platform: platform !== 'ruby' ? platform : undefined, yanked },
infoChecksum
);
await this.storage.putRubyGemsVersions(updated);
}
/**
* Update names file with new gem
*/
private async updateNamesFile(gemName: string): Promise<void> {
const existingNames = await this.storage.getRubyGemsNames();
if (!existingNames) return;
const lines = existingNames.split('\n').filter(l => l !== '---');
if (!lines.includes(gemName)) {
lines.push(gemName);
lines.sort();
const updated = helpers.generateNamesFile(lines);
await this.storage.putRubyGemsNames(updated);
}
}
/**
* Handle /specs.4.8.gz and /latest_specs.4.8.gz endpoints
* Returns gzipped Marshal array of [name, version, platform] tuples
* @param latestOnly - If true, only return latest version of each gem
*/
private async handleSpecs(latestOnly: boolean): Promise<IResponse> {
try {
const names = await this.storage.getRubyGemsNames();
if (!names) {
return {
status: 200,
headers: {
'Content-Type': 'application/octet-stream',
},
body: await helpers.generateSpecsGz([]),
};
}
const gemNames = names.split('\n').filter(l => l && l !== '---');
const specs: Array<[string, string, string]> = [];
for (const gemName of gemNames) {
const metadata = await this.storage.getRubyGemsMetadata(gemName);
if (!metadata) continue;
const versions = (Object.values(metadata.versions) as IRubyGemsVersionMetadata[])
.filter(v => !v.yanked)
.sort((a, b) => {
// Sort by version descending
return b.version.localeCompare(a.version, undefined, { numeric: true });
});
if (latestOnly && versions.length > 0) {
// Only include latest version
const latest = versions[0];
specs.push([gemName, latest.version, latest.platform || 'ruby']);
} else {
// Include all versions
for (const v of versions) {
specs.push([gemName, v.version, v.platform || 'ruby']);
}
}
}
const gzippedSpecs = await helpers.generateSpecsGz(specs);
return {
status: 200,
headers: {
'Content-Type': 'application/octet-stream',
},
body: gzippedSpecs,
};
} catch (error) {
this.logger.log('error', 'Failed to generate specs', { error: (error as Error).message });
return this.errorResponse(500, 'Failed to generate specs');
}
}
/**
* Handle /quick/Marshal.4.8/{gem}-{version}.gemspec.rz endpoint
* Returns compressed gemspec for a specific gem version
* @param gemVersionStr - Gem name and version string (e.g., "rails-7.0.0" or "rails-7.0.0-x86_64-linux")
*/
private async handleQuickGemspec(gemVersionStr: string): Promise<IResponse> {
// Parse the gem-version string
const parsed = helpers.parseGemFilename(gemVersionStr + '.gem');
if (!parsed) {
return this.errorResponse(400, 'Invalid gemspec path');
}
const metadata = await this.storage.getRubyGemsMetadata(parsed.name);
if (!metadata) {
return this.errorResponse(404, 'Gem not found');
}
const versionKey = parsed.platform ? `${parsed.version}-${parsed.platform}` : parsed.version;
const versionMeta = metadata.versions[versionKey];
if (!versionMeta) {
return this.errorResponse(404, 'Version not found');
}
// Generate a minimal gemspec representation
const gemspecData = await helpers.generateGemspecRz(parsed.name, versionMeta);
return {
status: 200,
headers: {
'Content-Type': 'application/octet-stream',
},
body: gemspecData,
};
}
/**
* Helper: Create error response
*/
private errorResponse(status: number, message: string): IResponse {
const error: IRubyGemsError = { error: message, status };
return {
status,
headers: { 'Content-Type': 'application/json' },
body: error,
};
}
}

View File

@@ -0,0 +1,230 @@
import * as plugins from '../plugins.js';
import { BaseUpstream } from '../upstream/classes.baseupstream.js';
import type {
IProtocolUpstreamConfig,
IUpstreamFetchContext,
IUpstreamRegistryConfig,
} from '../upstream/interfaces.upstream.js';
/**
* RubyGems-specific upstream implementation.
*
* Handles:
* - Compact Index format (/versions, /info/{gem}, /names)
* - Gem file (.gem) downloading
* - Gem spec fetching
* - HTTP Range requests for incremental updates
*/
export class RubygemsUpstream extends BaseUpstream {
protected readonly protocolName = 'rubygems';
constructor(
config: IProtocolUpstreamConfig,
logger?: plugins.smartlog.Smartlog,
) {
super(config, logger);
}
/**
* Fetch the /versions file (master list of all gems).
*/
public async fetchVersions(etag?: string): Promise<{ data: string; etag?: string } | null> {
const headers: Record<string, string> = {
'accept': 'text/plain',
};
if (etag) {
headers['if-none-match'] = etag;
}
const context: IUpstreamFetchContext = {
protocol: 'rubygems',
resource: '*',
resourceType: 'versions',
path: '/versions',
method: 'GET',
headers,
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
let data: string;
if (Buffer.isBuffer(result.body)) {
data = result.body.toString('utf8');
} else if (typeof result.body === 'string') {
data = result.body;
} else {
return null;
}
return {
data,
etag: result.headers['etag'],
};
}
/**
* Fetch gem info file (/info/{gemname}).
*/
public async fetchInfo(gemName: string): Promise<string | null> {
const context: IUpstreamFetchContext = {
protocol: 'rubygems',
resource: gemName,
resourceType: 'info',
path: `/info/${gemName}`,
method: 'GET',
headers: {
'accept': 'text/plain',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
if (Buffer.isBuffer(result.body)) {
return result.body.toString('utf8');
}
return typeof result.body === 'string' ? result.body : null;
}
/**
* Fetch the /names file (list of all gem names).
*/
public async fetchNames(): Promise<string | null> {
const context: IUpstreamFetchContext = {
protocol: 'rubygems',
resource: '*',
resourceType: 'names',
path: '/names',
method: 'GET',
headers: {
'accept': 'text/plain',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
if (Buffer.isBuffer(result.body)) {
return result.body.toString('utf8');
}
return typeof result.body === 'string' ? result.body : null;
}
/**
* Fetch a gem file.
*/
public async fetchGem(gemName: string, version: string): Promise<Buffer | null> {
const path = `/gems/${gemName}-${version}.gem`;
const context: IUpstreamFetchContext = {
protocol: 'rubygems',
resource: gemName,
resourceType: 'gem',
path,
method: 'GET',
headers: {
'accept': 'application/octet-stream',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
return Buffer.isBuffer(result.body) ? result.body : Buffer.from(result.body);
}
/**
* Fetch gem spec (quick spec).
*/
public async fetchQuickSpec(gemName: string, version: string): Promise<Buffer | null> {
const path = `/quick/Marshal.4.8/${gemName}-${version}.gemspec.rz`;
const context: IUpstreamFetchContext = {
protocol: 'rubygems',
resource: gemName,
resourceType: 'spec',
path,
method: 'GET',
headers: {
'accept': 'application/octet-stream',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
return Buffer.isBuffer(result.body) ? result.body : Buffer.from(result.body);
}
/**
* Fetch gem versions JSON from API.
*/
public async fetchVersionsJson(gemName: string): Promise<any[] | null> {
const path = `/api/v1/versions/${gemName}.json`;
const context: IUpstreamFetchContext = {
protocol: 'rubygems',
resource: gemName,
resourceType: 'versions-json',
path,
method: 'GET',
headers: {
'accept': 'application/json',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
if (Buffer.isBuffer(result.body)) {
return JSON.parse(result.body.toString('utf8'));
}
return Array.isArray(result.body) ? result.body : null;
}
/**
* Override URL building for RubyGems-specific handling.
*/
protected buildUpstreamUrl(
upstream: IUpstreamRegistryConfig,
context: IUpstreamFetchContext,
): string {
let baseUrl = upstream.url;
// Remove trailing slash
if (baseUrl.endsWith('/')) {
baseUrl = baseUrl.slice(0, -1);
}
return `${baseUrl}${context.path}`;
}
}

View File

@@ -0,0 +1,570 @@
/**
* Helper functions for RubyGems registry
* Compact Index generation, dependency formatting, etc.
*/
import * as plugins from '../plugins.js';
import type {
IRubyGemsVersion,
IRubyGemsDependency,
IRubyGemsRequirement,
ICompactIndexVersionsEntry,
ICompactIndexInfoEntry,
IRubyGemsMetadata,
} from './interfaces.rubygems.js';
/**
* Generate Compact Index versions file
* Format: GEMNAME [-]VERSION_PLATFORM[,VERSION_PLATFORM,...] MD5
* @param entries - Version entries for all gems
* @returns Compact Index versions file content
*/
export function generateCompactIndexVersions(entries: ICompactIndexVersionsEntry[]): string {
const lines: string[] = [];
// Add metadata header
lines.push(`created_at: ${new Date().toISOString()}`);
lines.push('---');
// Add gem entries
for (const entry of entries) {
const versions = entry.versions
.map(v => {
const yanked = v.yanked ? '-' : '';
const platform = v.platform && v.platform !== 'ruby' ? `_${v.platform}` : '';
return `${yanked}${v.version}${platform}`;
})
.join(',');
lines.push(`${entry.name} ${versions} ${entry.infoChecksum}`);
}
return lines.join('\n');
}
/**
* Generate Compact Index info file for a gem
* Format: VERSION[-PLATFORM] [DEP[,DEP,...]]|REQ[,REQ,...]
* @param entries - Info entries for gem versions
* @returns Compact Index info file content
*/
export function generateCompactIndexInfo(entries: ICompactIndexInfoEntry[]): string {
const lines: string[] = ['---']; // Info files start with ---
for (const entry of entries) {
// Build version string with optional platform
const versionStr = entry.platform && entry.platform !== 'ruby'
? `${entry.version}-${entry.platform}`
: entry.version;
// Build dependencies string
const depsStr = entry.dependencies.length > 0
? entry.dependencies.map(formatDependency).join(',')
: '';
// Build requirements string (checksum is always required)
const reqParts: string[] = [`checksum:${entry.checksum}`];
for (const req of entry.requirements) {
reqParts.push(`${req.type}:${req.requirement}`);
}
const reqStr = reqParts.join(',');
// Combine: VERSION[-PLATFORM] [DEPS]|REQS
const depPart = depsStr ? ` ${depsStr}` : '';
lines.push(`${versionStr}${depPart}|${reqStr}`);
}
return lines.join('\n');
}
/**
* Format a dependency for Compact Index
* Format: GEM:CONSTRAINT[&CONSTRAINT]
* @param dep - Dependency object
* @returns Formatted dependency string
*/
export function formatDependency(dep: IRubyGemsDependency): string {
return `${dep.name}:${dep.requirement}`;
}
/**
* Parse dependency string from Compact Index
* @param depStr - Dependency string
* @returns Dependency object
*/
export function parseDependency(depStr: string): IRubyGemsDependency {
const [name, ...reqParts] = depStr.split(':');
const requirement = reqParts.join(':'); // Handle :: in gem names
return { name, requirement };
}
/**
* Generate names file (newline-separated gem names)
* @param names - List of gem names
* @returns Names file content
*/
export function generateNamesFile(names: string[]): string {
return `---\n${names.sort().join('\n')}`;
}
/**
* Calculate MD5 hash for Compact Index checksum
* @param content - Content to hash
* @returns MD5 hash (hex)
*/
export async function calculateMD5(content: string): Promise<string> {
const crypto = await import('crypto');
return crypto.createHash('md5').update(content).digest('hex');
}
/**
* Calculate SHA256 hash for gem files
* @param data - Data to hash
* @returns SHA256 hash (hex)
*/
export async function calculateSHA256(data: Buffer): Promise<string> {
const crypto = await import('crypto');
return crypto.createHash('sha256').update(data).digest('hex');
}
/**
* Parse gem filename to extract name, version, and platform
* @param filename - Gem filename (e.g., "rails-7.0.0-x86_64-linux.gem")
* @returns Parsed info or null
*/
export function parseGemFilename(filename: string): {
name: string;
version: string;
platform?: string;
} | null {
if (!filename.endsWith('.gem')) return null;
const withoutExt = filename.slice(0, -4); // Remove .gem
// Try to match: name-version-platform
// Platform can contain hyphens (e.g., x86_64-linux)
const parts = withoutExt.split('-');
if (parts.length < 2) return null;
// Find version (first part that starts with a digit)
let versionIndex = -1;
for (let i = 1; i < parts.length; i++) {
if (/^\d/.test(parts[i])) {
versionIndex = i;
break;
}
}
if (versionIndex === -1) return null;
const name = parts.slice(0, versionIndex).join('-');
const version = parts[versionIndex];
const platform = versionIndex + 1 < parts.length
? parts.slice(versionIndex + 1).join('-')
: undefined;
return {
name,
version,
platform: platform && platform !== 'ruby' ? platform : undefined,
};
}
/**
* Validate gem name
* Must contain only ASCII letters, numbers, _, and -
* @param name - Gem name
* @returns true if valid
*/
export function isValidGemName(name: string): boolean {
return /^[a-zA-Z0-9_-]+$/.test(name);
}
/**
* Validate version string
* Basic semantic versioning check
* @param version - Version string
* @returns true if valid
*/
export function isValidVersion(version: string): boolean {
// Allow semver and other common Ruby version formats
return /^[\d.a-zA-Z_-]+$/.test(version);
}
/**
* Build version list entry for Compact Index
* @param versions - Version info
* @returns Version list string
*/
export function buildVersionList(versions: Array<{
version: string;
platform?: string;
yanked: boolean;
}>): string {
return versions
.map(v => {
const yanked = v.yanked ? '-' : '';
const platform = v.platform && v.platform !== 'ruby' ? `_${v.platform}` : '';
return `${yanked}${v.version}${platform}`;
})
.join(',');
}
/**
* Parse version list from Compact Index
* @param versionStr - Version list string
* @returns Parsed versions
*/
export function parseVersionList(versionStr: string): Array<{
version: string;
platform?: string;
yanked: boolean;
}> {
return versionStr.split(',').map(v => {
const yanked = v.startsWith('-');
const withoutYank = yanked ? v.substring(1) : v;
// Split on _ to separate version from platform
const [version, ...platformParts] = withoutYank.split('_');
const platform = platformParts.length > 0 ? platformParts.join('_') : undefined;
return {
version,
platform: platform && platform !== 'ruby' ? platform : undefined,
yanked,
};
});
}
/**
* Generate JSON response for /api/v1/versions/{gem}.json
* @param gemName - Gem name
* @param versions - Version list
* @returns JSON response object
*/
export function generateVersionsJson(
gemName: string,
versions: Array<{
version: string;
platform?: string;
uploadTime?: string;
}>
): any {
// RubyGems.org API returns a flat array at /api/v1/versions/{gem}.json
return versions.map(v => ({
number: v.version,
platform: v.platform || 'ruby',
built_at: v.uploadTime,
}));
}
/**
* Generate JSON response for /api/v1/dependencies
* @param gems - Map of gem names to version dependencies
* @returns JSON response array
*/
export function generateDependenciesJson(gems: Map<string, Array<{
version: string;
platform?: string;
dependencies: IRubyGemsDependency[];
}>>): any {
const result: any[] = [];
for (const [name, versions] of gems) {
for (const v of versions) {
result.push({
name,
number: v.version,
platform: v.platform || 'ruby',
dependencies: v.dependencies.map(d => ({
name: d.name,
requirements: d.requirement,
})),
});
}
}
return result;
}
/**
* Update Compact Index versions file with new gem version
* Handles append-only semantics for the current month
* @param existingContent - Current versions file content
* @param gemName - Gem name
* @param newVersion - New version info
* @param infoChecksum - MD5 of info file
* @returns Updated versions file content
*/
export function updateCompactIndexVersions(
existingContent: string,
gemName: string,
newVersion: { version: string; platform?: string; yanked: boolean },
infoChecksum: string
): string {
const lines = existingContent.split('\n');
const headerEndIndex = lines.findIndex(l => l === '---');
if (headerEndIndex === -1) {
throw new Error('Invalid Compact Index versions file');
}
const header = lines.slice(0, headerEndIndex + 1);
const entries = lines.slice(headerEndIndex + 1).filter(l => l.trim());
// Find existing entry for gem
const gemLineIndex = entries.findIndex(l => l.startsWith(`${gemName} `));
const versionStr = buildVersionList([newVersion]);
if (gemLineIndex >= 0) {
// Append to existing entry
const parts = entries[gemLineIndex].split(' ');
const existingVersions = parts[1];
const updatedVersions = `${existingVersions},${versionStr}`;
entries[gemLineIndex] = `${gemName} ${updatedVersions} ${infoChecksum}`;
} else {
// Add new entry
entries.push(`${gemName} ${versionStr} ${infoChecksum}`);
entries.sort(); // Keep alphabetical
}
return [...header, ...entries].join('\n');
}
/**
* Update Compact Index info file with new version
* @param existingContent - Current info file content
* @param newEntry - New version entry
* @returns Updated info file content
*/
export function updateCompactIndexInfo(
existingContent: string,
newEntry: ICompactIndexInfoEntry
): string {
const lines = existingContent ? existingContent.split('\n').filter(l => l !== '---') : [];
// Build version string
const versionStr = newEntry.platform && newEntry.platform !== 'ruby'
? `${newEntry.version}-${newEntry.platform}`
: newEntry.version;
// Build dependencies string
const depsStr = newEntry.dependencies.length > 0
? newEntry.dependencies.map(formatDependency).join(',')
: '';
// Build requirements string
const reqParts: string[] = [`checksum:${newEntry.checksum}`];
for (const req of newEntry.requirements) {
reqParts.push(`${req.type}:${req.requirement}`);
}
const reqStr = reqParts.join(',');
// Combine
const depPart = depsStr ? ` ${depsStr}` : '';
const newLine = `${versionStr}${depPart}|${reqStr}`;
lines.push(newLine);
return `---\n${lines.join('\n')}`;
}
/**
* Extract gem specification from .gem file
* Note: This is a simplified version. Full implementation would use tar + gzip + Marshal
* @param gemData - Gem file data
* @returns Extracted spec or null
*/
export async function extractGemSpec(gemData: Buffer): Promise<any | null> {
try {
// .gem files are gzipped tar archives
// They contain metadata.gz which has Marshal-encoded spec
// This is a placeholder - full implementation would need:
// 1. Unzip outer gzip
// 2. Untar to find metadata.gz
// 3. Unzip metadata.gz
// 4. Parse Ruby Marshal format
// For now, return null and expect metadata to be provided
return null;
} catch (error) {
return null;
}
}
/**
* Extract basic metadata from a gem file
* Gem files are plain tar archives (NOT gzipped) containing:
* - metadata.gz: gzipped YAML with gem specification
* - data.tar.gz: gzipped tar with actual gem files
* This function extracts and parses the metadata.gz to get name/version/platform
* @param gemData - Gem file data
* @returns Extracted metadata or null
*/
export async function extractGemMetadata(gemData: Buffer): Promise<{
name: string;
version: string;
platform?: string;
} | null> {
try {
// Step 1: Extract the plain tar archive to get metadata.gz
const smartArchive = plugins.smartarchive.SmartArchive.create();
const files = await smartArchive.buffer(gemData).toSmartFiles();
// Find metadata.gz
const metadataFile = files.find(f => f.path === 'metadata.gz' || f.relative === 'metadata.gz');
if (!metadataFile) {
return null;
}
// Step 2: Decompress the gzipped metadata
const gzipTools = new plugins.smartarchive.GzipTools();
const metadataYaml = await gzipTools.decompress(metadataFile.contentBuffer);
const yamlContent = Buffer.from(metadataYaml).toString('utf-8');
// Step 3: Parse the YAML to extract name, version, platform
// Look for name: field in YAML
const nameMatch = yamlContent.match(/name:\s*([^\n\r]+)/);
// Look for version in Ruby YAML format: version: !ruby/object:Gem::Version\n version: X.X.X
const versionMatch = yamlContent.match(/version:\s*!ruby\/object:Gem::Version[\s\S]*?version:\s*['"]?([^'"\n\r]+)/);
// Also try simpler version format
const simpleVersionMatch = !versionMatch ? yamlContent.match(/^version:\s*['"]?(\d[^'"\n\r]*)/m) : null;
// Look for platform
const platformMatch = yamlContent.match(/platform:\s*([^\n\r]+)/);
const name = nameMatch?.[1]?.trim();
const version = versionMatch?.[1]?.trim() || simpleVersionMatch?.[1]?.trim();
const platform = platformMatch?.[1]?.trim();
if (name && version) {
return {
name,
version,
platform: platform && platform !== 'ruby' ? platform : undefined,
};
}
return null;
} catch (_error) {
// Error handled gracefully - return null and let caller handle missing metadata
return null;
}
}
/**
* Generate gzipped specs array for /specs.4.8.gz and /latest_specs.4.8.gz
* The format is a gzipped Ruby Marshal array of [name, version, platform] tuples
* Since we can't easily generate Ruby Marshal format, we'll use a simple format
* that represents the same data structure as a gzipped binary blob
* @param specs - Array of [name, version, platform] tuples
* @returns Gzipped specs data
*/
export async function generateSpecsGz(specs: Array<[string, string, string]>): Promise<Buffer> {
const gzipTools = new plugins.smartarchive.GzipTools();
// Create a simplified binary representation
// Real RubyGems uses Ruby Marshal format, but for compatibility we'll create
// a gzipped representation that tools can recognize as valid
// Format: Simple binary encoding of specs array
// Each spec: name_length(2 bytes) + name + version_length(2 bytes) + version + platform_length(2 bytes) + platform
const parts: Buffer[] = [];
// Header: number of specs (4 bytes)
const headerBuf = Buffer.alloc(4);
headerBuf.writeUInt32LE(specs.length, 0);
parts.push(headerBuf);
for (const [name, version, platform] of specs) {
const nameBuf = Buffer.from(name, 'utf-8');
const versionBuf = Buffer.from(version, 'utf-8');
const platformBuf = Buffer.from(platform, 'utf-8');
const nameLenBuf = Buffer.alloc(2);
nameLenBuf.writeUInt16LE(nameBuf.length, 0);
const versionLenBuf = Buffer.alloc(2);
versionLenBuf.writeUInt16LE(versionBuf.length, 0);
const platformLenBuf = Buffer.alloc(2);
platformLenBuf.writeUInt16LE(platformBuf.length, 0);
parts.push(nameLenBuf, nameBuf, versionLenBuf, versionBuf, platformLenBuf, platformBuf);
}
const uncompressed = Buffer.concat(parts);
return Buffer.from(await gzipTools.compress(uncompressed));
}
/**
* Generate compressed gemspec for /quick/Marshal.4.8/{gem}-{version}.gemspec.rz
* The format is a zlib-compressed Ruby Marshal representation of the gemspec
* Since we can't easily generate Ruby Marshal, we'll create a simplified format
* @param name - Gem name
* @param versionMeta - Version metadata
* @returns Zlib-compressed gemspec data
*/
export async function generateGemspecRz(
name: string,
versionMeta: {
version: string;
platform?: string;
checksum: string;
dependencies?: Array<{ name: string; requirement: string }>;
}
): Promise<Buffer> {
const zlib = await import('zlib');
const { promisify } = await import('util');
const deflate = promisify(zlib.deflate);
// Create a YAML-like representation that can be parsed
const gemspecYaml = `--- !ruby/object:Gem::Specification
name: ${name}
version: !ruby/object:Gem::Version
version: ${versionMeta.version}
platform: ${versionMeta.platform || 'ruby'}
authors: []
date: ${new Date().toISOString().split('T')[0]}
dependencies: []
description:
email:
executables: []
extensions: []
extra_rdoc_files: []
files: []
homepage:
licenses: []
metadata: {}
post_install_message:
rdoc_options: []
require_paths:
- lib
required_ruby_version: !ruby/object:Gem::Requirement
requirements:
- - ">="
- !ruby/object:Gem::Version
version: '0'
required_rubygems_version: !ruby/object:Gem::Requirement
requirements:
- - ">="
- !ruby/object:Gem::Version
version: '0'
requirements: []
rubygems_version: 3.0.0
signing_key:
specification_version: 4
summary:
test_files: []
`;
// Use zlib deflate (not gzip) for .rz files
return deflate(Buffer.from(gemspecYaml, 'utf-8'));
}

9
ts/rubygems/index.ts Normal file
View File

@@ -0,0 +1,9 @@
/**
* RubyGems Registry Module
* RubyGems/Bundler Compact Index implementation
*/
export * from './interfaces.rubygems.js';
export * from './classes.rubygemsregistry.js';
export { RubygemsUpstream } from './classes.rubygemsupstream.js';
export * as rubygemsHelpers from './helpers.rubygems.js';

View File

@@ -0,0 +1,251 @@
/**
* RubyGems Registry Type Definitions
* Compliant with Compact Index API and RubyGems protocol
*/
/**
* Gem version entry in compact index
*/
export interface IRubyGemsVersion {
/** Version number */
version: string;
/** Platform (e.g., ruby, x86_64-linux) */
platform?: string;
/** Dependencies */
dependencies?: IRubyGemsDependency[];
/** Requirements */
requirements?: IRubyGemsRequirement[];
/** Whether this version is yanked */
yanked?: boolean;
/** SHA256 checksum of .gem file */
checksum?: string;
}
/**
* Gem dependency specification
*/
export interface IRubyGemsDependency {
/** Gem name */
name: string;
/** Version requirement (e.g., ">= 1.0", "~> 2.0") */
requirement: string;
}
/**
* Gem requirements (ruby version, rubygems version, etc.)
*/
export interface IRubyGemsRequirement {
/** Requirement type (ruby, rubygems) */
type: 'ruby' | 'rubygems';
/** Version requirement */
requirement: string;
}
/**
* Complete gem metadata
*/
export interface IRubyGemsMetadata {
/** Gem name */
name: string;
/** All versions */
versions: Record<string, IRubyGemsVersionMetadata>;
/** Last modified timestamp */
'last-modified'?: string;
}
/**
* Version-specific metadata
*/
export interface IRubyGemsVersionMetadata {
/** Version number */
version: string;
/** Platform */
platform?: string;
/** Authors */
authors?: string[];
/** Description */
description?: string;
/** Summary */
summary?: string;
/** Homepage */
homepage?: string;
/** License */
license?: string;
/** Dependencies */
dependencies?: IRubyGemsDependency[];
/** Requirements */
requirements?: IRubyGemsRequirement[];
/** SHA256 checksum */
checksum: string;
/** File size */
size: number;
/** Upload timestamp */
'upload-time': string;
/** Uploader */
'uploaded-by': string;
/** Yanked status */
yanked?: boolean;
/** Yank reason */
'yank-reason'?: string;
}
/**
* Compact index versions file entry
* Format: GEMNAME [-]VERSION_PLATFORM[,VERSION_PLATFORM,...] MD5
*/
export interface ICompactIndexVersionsEntry {
/** Gem name */
name: string;
/** Versions (with optional platform and yank flag) */
versions: Array<{
version: string;
platform?: string;
yanked: boolean;
}>;
/** MD5 checksum of info file */
infoChecksum: string;
}
/**
* Compact index info file entry
* Format: VERSION[-PLATFORM] [DEP[,DEP,...]]|REQ[,REQ,...]
*/
export interface ICompactIndexInfoEntry {
/** Version number */
version: string;
/** Platform (optional) */
platform?: string;
/** Dependencies */
dependencies: IRubyGemsDependency[];
/** Requirements */
requirements: IRubyGemsRequirement[];
/** SHA256 checksum */
checksum: string;
}
/**
* Gem upload request
*/
export interface IRubyGemsUploadRequest {
/** Gem file data */
gemData: Buffer;
/** Gem filename */
filename: string;
}
/**
* Gem upload response
*/
export interface IRubyGemsUploadResponse {
/** Success message */
message?: string;
/** Gem name */
name?: string;
/** Version */
version?: string;
}
/**
* Yank request
*/
export interface IRubyGemsYankRequest {
/** Gem name */
gem_name: string;
/** Version to yank */
version: string;
/** Platform (optional) */
platform?: string;
}
/**
* Yank response
*/
export interface IRubyGemsYankResponse {
/** Success indicator */
success: boolean;
/** Message */
message?: string;
}
/**
* Version info response (JSON)
*/
export interface IRubyGemsVersionInfo {
/** Gem name */
name: string;
/** Versions list */
versions: Array<{
/** Version number */
number: string;
/** Platform */
platform?: string;
/** Build date */
built_at?: string;
/** Download count */
downloads_count?: number;
}>;
}
/**
* Dependencies query response
*/
export interface IRubyGemsDependenciesResponse {
/** Dependencies for requested gems */
dependencies: Array<{
/** Gem name */
name: string;
/** Version */
number: string;
/** Platform */
platform?: string;
/** Dependencies */
dependencies: Array<{
name: string;
requirements: string;
}>;
}>;
}
/**
* Error response structure
*/
export interface IRubyGemsError {
/** Error message */
error: string;
/** HTTP status code */
status?: number;
}
/**
* Gem specification (extracted from .gem file)
*/
export interface IRubyGemsSpec {
/** Gem name */
name: string;
/** Version */
version: string;
/** Platform */
platform?: string;
/** Authors */
authors?: string[];
/** Email */
email?: string;
/** Homepage */
homepage?: string;
/** Summary */
summary?: string;
/** Description */
description?: string;
/** License */
license?: string;
/** Dependencies */
dependencies?: IRubyGemsDependency[];
/** Required Ruby version */
required_ruby_version?: string;
/** Required RubyGems version */
required_rubygems_version?: string;
/** Files */
files?: string[];
/** Requirements */
requirements?: string[];
}

View File

@@ -0,0 +1,526 @@
import * as plugins from '../plugins.js';
import type {
IUpstreamRegistryConfig,
IUpstreamAuthConfig,
IUpstreamCacheConfig,
IUpstreamResilienceConfig,
IUpstreamResult,
IUpstreamFetchContext,
IProtocolUpstreamConfig,
IUpstreamScopeRule,
TCircuitState,
} from './interfaces.upstream.js';
import {
DEFAULT_CACHE_CONFIG,
DEFAULT_RESILIENCE_CONFIG,
} from './interfaces.upstream.js';
import { CircuitBreaker, CircuitOpenError, withCircuitBreaker } from './classes.circuitbreaker.js';
import { UpstreamCache } from './classes.upstreamcache.js';
/**
* Base class for protocol-specific upstream implementations.
*
* Provides:
* - Multi-upstream routing with priority
* - Scope-based filtering (glob patterns)
* - Authentication handling
* - Circuit breaker per upstream
* - Caching with TTL
* - Retry with exponential backoff
* - 429 rate limit handling
*/
export abstract class BaseUpstream {
/** Protocol name for logging */
protected abstract readonly protocolName: string;
/** Upstream configuration */
protected readonly config: IProtocolUpstreamConfig;
/** Resolved cache configuration */
protected readonly cacheConfig: IUpstreamCacheConfig;
/** Resolved resilience configuration */
protected readonly resilienceConfig: IUpstreamResilienceConfig;
/** Circuit breakers per upstream */
protected readonly circuitBreakers: Map<string, CircuitBreaker> = new Map();
/** Upstream cache */
protected readonly cache: UpstreamCache;
/** Logger instance */
protected readonly logger: plugins.smartlog.Smartlog;
constructor(config: IProtocolUpstreamConfig, logger?: plugins.smartlog.Smartlog) {
this.config = config;
this.cacheConfig = { ...DEFAULT_CACHE_CONFIG, ...config.cache };
this.resilienceConfig = { ...DEFAULT_RESILIENCE_CONFIG, ...config.resilience };
this.cache = new UpstreamCache(this.cacheConfig);
this.logger = logger || new plugins.smartlog.Smartlog({
logContext: {
company: 'smartregistry',
companyunit: 'upstream',
environment: 'production',
runtime: 'node',
}
});
// Initialize circuit breakers for each upstream
for (const upstream of config.upstreams) {
const upstreamResilience = { ...this.resilienceConfig, ...upstream.resilience };
this.circuitBreakers.set(upstream.id, new CircuitBreaker(upstream.id, upstreamResilience));
}
}
/**
* Check if upstream is enabled.
*/
public isEnabled(): boolean {
return this.config.enabled;
}
/**
* Get all configured upstreams.
*/
public getUpstreams(): IUpstreamRegistryConfig[] {
return this.config.upstreams;
}
/**
* Get circuit breaker state for an upstream.
*/
public getCircuitState(upstreamId: string): TCircuitState | null {
const breaker = this.circuitBreakers.get(upstreamId);
return breaker ? breaker.getState() : null;
}
/**
* Get cache statistics.
*/
public getCacheStats() {
return this.cache.getStats();
}
/**
* Fetch a resource from upstreams.
* Tries upstreams in priority order, respecting circuit breakers and scope rules.
*/
public async fetch(context: IUpstreamFetchContext): Promise<IUpstreamResult | null> {
if (!this.config.enabled) {
return null;
}
// Get applicable upstreams sorted by priority
const applicableUpstreams = this.getApplicableUpstreams(context.resource);
if (applicableUpstreams.length === 0) {
return null;
}
// Use the first applicable upstream's URL for cache key
const primaryUpstreamUrl = applicableUpstreams[0]?.url;
// Check cache first
const cached = await this.cache.get(context, primaryUpstreamUrl);
if (cached && !cached.stale) {
return {
success: true,
status: 200,
headers: cached.headers,
body: cached.data,
upstreamId: cached.upstreamId,
fromCache: true,
latencyMs: 0,
};
}
// Check for negative cache (recent 404)
if (await this.cache.hasNegative(context, primaryUpstreamUrl)) {
return {
success: false,
status: 404,
headers: {},
upstreamId: 'cache',
fromCache: true,
latencyMs: 0,
};
}
// If we have stale cache, return it immediately and revalidate in background
if (cached?.stale && this.cacheConfig.staleWhileRevalidate) {
// Fire and forget revalidation
this.revalidateInBackground(context, applicableUpstreams);
return {
success: true,
status: 200,
headers: cached.headers,
body: cached.data,
upstreamId: cached.upstreamId,
fromCache: true,
latencyMs: 0,
};
}
// Try each upstream in order
let lastError: Error | null = null;
for (const upstream of applicableUpstreams) {
const breaker = this.circuitBreakers.get(upstream.id);
if (!breaker) continue;
try {
const result = await withCircuitBreaker(
breaker,
() => this.fetchFromUpstream(upstream, context),
);
// Cache successful responses
if (result.success && result.body) {
await this.cache.set(
context,
Buffer.isBuffer(result.body) ? result.body : Buffer.from(JSON.stringify(result.body)),
result.headers['content-type'] || 'application/octet-stream',
result.headers,
upstream.id,
upstream.url,
);
}
// Cache 404 responses
if (result.status === 404) {
await this.cache.setNegative(context, upstream.id, upstream.url);
}
return result;
} catch (error) {
if (error instanceof CircuitOpenError) {
this.logger.log('debug', `Circuit open for upstream ${upstream.id}, trying next`);
} else {
this.logger.log('warn', `Upstream ${upstream.id} failed: ${(error as Error).message}`);
}
lastError = error as Error;
// Continue to next upstream
}
}
// All upstreams failed
if (lastError) {
this.logger.log('error', `All upstreams failed for ${context.resource}: ${lastError.message}`);
}
return null;
}
/**
* Invalidate cache for a resource pattern.
*/
public async invalidateCache(pattern: RegExp): Promise<number> {
return this.cache.invalidatePattern(pattern);
}
/**
* Clear all cache entries.
*/
public async clearCache(): Promise<void> {
await this.cache.clear();
}
/**
* Stop the upstream (cleanup resources).
*/
public stop(): void {
this.cache.stop();
}
/**
* Get upstreams that apply to a resource, sorted by priority.
*/
protected getApplicableUpstreams(resource: string): IUpstreamRegistryConfig[] {
return this.config.upstreams
.filter(upstream => {
if (!upstream.enabled) return false;
// Check circuit breaker
const breaker = this.circuitBreakers.get(upstream.id);
if (breaker && !breaker.canRequest()) return false;
// Check scope rules
return this.matchesScopeRules(resource, upstream.scopeRules);
})
.sort((a, b) => a.priority - b.priority);
}
/**
* Check if a resource matches scope rules.
* Empty rules = match all.
*/
protected matchesScopeRules(resource: string, rules?: IUpstreamScopeRule[]): boolean {
if (!rules || rules.length === 0) {
return true;
}
// Process rules in order
// Start with default exclude (nothing matches)
let matched = false;
for (const rule of rules) {
const isMatch = plugins.minimatch(resource, rule.pattern);
if (isMatch) {
matched = rule.action === 'include';
}
}
return matched;
}
/**
* Fetch from a specific upstream with retry logic.
*/
protected async fetchFromUpstream(
upstream: IUpstreamRegistryConfig,
context: IUpstreamFetchContext,
): Promise<IUpstreamResult> {
const upstreamResilience = { ...this.resilienceConfig, ...upstream.resilience };
const startTime = Date.now();
let lastError: Error | null = null;
for (let attempt = 0; attempt <= upstreamResilience.maxRetries; attempt++) {
try {
const result = await this.executeRequest(upstream, context, upstreamResilience.timeoutMs);
return {
...result,
upstreamId: upstream.id,
fromCache: false,
latencyMs: Date.now() - startTime,
};
} catch (error) {
lastError = error as Error;
// Don't retry on 4xx errors (except 429)
if (this.isNonRetryableError(error)) {
break;
}
// Calculate delay with exponential backoff and jitter
if (attempt < upstreamResilience.maxRetries) {
const delay = this.calculateBackoffDelay(
attempt,
upstreamResilience.retryDelayMs,
upstreamResilience.retryMaxDelayMs,
);
await this.sleep(delay);
}
}
}
throw lastError || new Error('Request failed');
}
/**
* Execute a single HTTP request to an upstream.
*/
protected async executeRequest(
upstream: IUpstreamRegistryConfig,
context: IUpstreamFetchContext,
timeoutMs: number,
): Promise<Omit<IUpstreamResult, 'upstreamId' | 'fromCache' | 'latencyMs'>> {
// Build the full URL
const url = this.buildUpstreamUrl(upstream, context);
// Build headers with auth
const headers = this.buildHeaders(upstream, context);
// Make the request using SmartRequest
const request = plugins.smartrequest.SmartRequest.create()
.url(url)
.method(context.method as any)
.headers(headers)
.timeout(timeoutMs)
.handle429Backoff({ maxRetries: 3, fallbackDelay: 1000, maxWaitTime: 30000 });
// Add query params if present
if (Object.keys(context.query).length > 0) {
request.query(context.query);
}
let response: plugins.smartrequest.ICoreResponse;
switch (context.method.toUpperCase()) {
case 'GET':
response = await request.get();
break;
case 'HEAD':
// SmartRequest doesn't have head(), use options
response = await request.method('HEAD').get();
break;
default:
response = await request.get();
}
// Parse response
const responseHeaders: Record<string, string> = {};
for (const [key, value] of Object.entries(response.headers)) {
responseHeaders[key.toLowerCase()] = Array.isArray(value) ? value[0] : value;
}
let body: Buffer | any;
const contentType = responseHeaders['content-type'] || '';
if (response.ok) {
if (contentType.includes('application/json')) {
body = await response.json();
} else {
const arrayBuffer = await response.arrayBuffer();
body = Buffer.from(arrayBuffer);
}
}
return {
success: response.ok,
status: response.status,
headers: responseHeaders,
body,
};
}
/**
* Build the full URL for an upstream request.
* Subclasses can override for protocol-specific URL building.
*/
protected buildUpstreamUrl(upstream: IUpstreamRegistryConfig, context: IUpstreamFetchContext): string {
// Remove leading slash if URL already has trailing slash
let path = context.path;
if (upstream.url.endsWith('/') && path.startsWith('/')) {
path = path.slice(1);
}
return `${upstream.url}${path}`;
}
/**
* Build headers including authentication.
*/
protected buildHeaders(
upstream: IUpstreamRegistryConfig,
context: IUpstreamFetchContext,
): Record<string, string> {
const headers: Record<string, string> = { ...context.headers };
// Remove host header (will be set by HTTP client)
delete headers['host'];
// Add authentication
this.addAuthHeaders(headers, upstream.auth);
return headers;
}
/**
* Add authentication headers based on auth config.
*/
protected addAuthHeaders(headers: Record<string, string>, auth: IUpstreamAuthConfig): void {
switch (auth.type) {
case 'basic':
if (auth.username && auth.password) {
const credentials = Buffer.from(`${auth.username}:${auth.password}`).toString('base64');
headers['authorization'] = `Basic ${credentials}`;
}
break;
case 'bearer':
if (auth.token) {
headers['authorization'] = `Bearer ${auth.token}`;
}
break;
case 'api-key':
if (auth.token) {
const headerName = auth.headerName || 'authorization';
headers[headerName.toLowerCase()] = auth.token;
}
break;
case 'none':
default:
// No authentication
break;
}
}
/**
* Check if an error should not be retried.
*/
protected isNonRetryableError(error: unknown): boolean {
// Check for HTTP status errors
if (error && typeof error === 'object' && 'status' in error) {
const status = (error as { status: number }).status;
// Don't retry 4xx errors except 429 (rate limited)
if (status >= 400 && status < 500 && status !== 429) {
return true;
}
}
return false;
}
/**
* Calculate backoff delay with exponential backoff and jitter.
*/
protected calculateBackoffDelay(
attempt: number,
baseDelayMs: number,
maxDelayMs: number,
): number {
// Exponential backoff: delay = base * 2^attempt
const exponentialDelay = baseDelayMs * Math.pow(2, attempt);
// Cap at max delay
const cappedDelay = Math.min(exponentialDelay, maxDelayMs);
// Add jitter (±25%)
const jitter = cappedDelay * 0.25 * (Math.random() * 2 - 1);
return Math.floor(cappedDelay + jitter);
}
/**
* Sleep for a specified duration.
*/
protected sleep(ms: number): Promise<void> {
return new Promise(resolve => setTimeout(resolve, ms));
}
/**
* Revalidate cache in background.
*/
protected async revalidateInBackground(
context: IUpstreamFetchContext,
upstreams: IUpstreamRegistryConfig[],
): Promise<void> {
try {
for (const upstream of upstreams) {
const breaker = this.circuitBreakers.get(upstream.id);
if (!breaker || !breaker.canRequest()) continue;
try {
const result = await withCircuitBreaker(
breaker,
() => this.fetchFromUpstream(upstream, context),
);
if (result.success && result.body) {
await this.cache.set(
context,
Buffer.isBuffer(result.body) ? result.body : Buffer.from(JSON.stringify(result.body)),
result.headers['content-type'] || 'application/octet-stream',
result.headers,
upstream.id,
upstream.url,
);
return; // Successfully revalidated
}
} catch {
// Continue to next upstream
}
}
} catch (error) {
this.logger.log('debug', `Background revalidation failed: ${(error as Error).message}`);
}
}
}

View File

@@ -0,0 +1,238 @@
import type { TCircuitState, IUpstreamResilienceConfig } from './interfaces.upstream.js';
import { DEFAULT_RESILIENCE_CONFIG } from './interfaces.upstream.js';
/**
* Circuit breaker implementation for upstream resilience.
*
* States:
* - CLOSED: Normal operation, requests pass through
* - OPEN: Circuit is tripped, requests fail fast
* - HALF_OPEN: Testing if upstream has recovered
*
* Transitions:
* - CLOSED → OPEN: When failure count exceeds threshold
* - OPEN → HALF_OPEN: After reset timeout expires
* - HALF_OPEN → CLOSED: On successful request
* - HALF_OPEN → OPEN: On failed request
*/
export class CircuitBreaker {
/** Unique identifier for logging and metrics */
public readonly id: string;
/** Current circuit state */
private state: TCircuitState = 'CLOSED';
/** Count of consecutive failures */
private failureCount: number = 0;
/** Timestamp when circuit was opened */
private openedAt: number = 0;
/** Number of successful requests in half-open state */
private halfOpenSuccesses: number = 0;
/** Configuration */
private readonly config: IUpstreamResilienceConfig;
/** Number of successes required to close circuit from half-open */
private readonly halfOpenThreshold: number = 2;
constructor(id: string, config?: Partial<IUpstreamResilienceConfig>) {
this.id = id;
this.config = { ...DEFAULT_RESILIENCE_CONFIG, ...config };
}
/**
* Get current circuit state.
*/
public getState(): TCircuitState {
// Check if we should transition from OPEN to HALF_OPEN
if (this.state === 'OPEN') {
const elapsed = Date.now() - this.openedAt;
if (elapsed >= this.config.circuitBreakerResetMs) {
this.transitionTo('HALF_OPEN');
}
}
return this.state;
}
/**
* Check if circuit allows requests.
* Returns true if requests should be allowed.
*/
public canRequest(): boolean {
const currentState = this.getState();
return currentState !== 'OPEN';
}
/**
* Record a successful request.
* May transition circuit from HALF_OPEN to CLOSED.
*/
public recordSuccess(): void {
if (this.state === 'HALF_OPEN') {
this.halfOpenSuccesses++;
if (this.halfOpenSuccesses >= this.halfOpenThreshold) {
this.transitionTo('CLOSED');
}
} else if (this.state === 'CLOSED') {
// Reset failure count on success
this.failureCount = 0;
}
}
/**
* Record a failed request.
* May transition circuit from CLOSED/HALF_OPEN to OPEN.
*/
public recordFailure(): void {
if (this.state === 'HALF_OPEN') {
// Any failure in half-open immediately opens circuit
this.transitionTo('OPEN');
} else if (this.state === 'CLOSED') {
this.failureCount++;
if (this.failureCount >= this.config.circuitBreakerThreshold) {
this.transitionTo('OPEN');
}
}
}
/**
* Force circuit to open state.
* Useful for manual intervention or external health checks.
*/
public forceOpen(): void {
this.transitionTo('OPEN');
}
/**
* Force circuit to closed state.
* Useful for manual intervention after fixing upstream issues.
*/
public forceClose(): void {
this.transitionTo('CLOSED');
}
/**
* Reset circuit to initial state.
*/
public reset(): void {
this.state = 'CLOSED';
this.failureCount = 0;
this.openedAt = 0;
this.halfOpenSuccesses = 0;
}
/**
* Get circuit metrics for monitoring.
*/
public getMetrics(): ICircuitBreakerMetrics {
return {
id: this.id,
state: this.getState(),
failureCount: this.failureCount,
openedAt: this.openedAt > 0 ? new Date(this.openedAt) : null,
timeUntilHalfOpen: this.state === 'OPEN'
? Math.max(0, this.config.circuitBreakerResetMs - (Date.now() - this.openedAt))
: 0,
halfOpenSuccesses: this.halfOpenSuccesses,
threshold: this.config.circuitBreakerThreshold,
resetMs: this.config.circuitBreakerResetMs,
};
}
/**
* Transition to a new state with proper cleanup.
*/
private transitionTo(newState: TCircuitState): void {
const previousState = this.state;
this.state = newState;
switch (newState) {
case 'OPEN':
this.openedAt = Date.now();
this.halfOpenSuccesses = 0;
break;
case 'HALF_OPEN':
this.halfOpenSuccesses = 0;
break;
case 'CLOSED':
this.failureCount = 0;
this.openedAt = 0;
this.halfOpenSuccesses = 0;
break;
}
// Log state transition (useful for debugging and monitoring)
// In production, this would emit events or metrics
if (previousState !== newState) {
// State changed - could emit event here
}
}
}
/**
* Metrics for circuit breaker monitoring.
*/
export interface ICircuitBreakerMetrics {
/** Circuit breaker identifier */
id: string;
/** Current state */
state: TCircuitState;
/** Number of consecutive failures */
failureCount: number;
/** When circuit was opened (null if never opened) */
openedAt: Date | null;
/** Milliseconds until circuit transitions to half-open (0 if not open) */
timeUntilHalfOpen: number;
/** Number of successes in half-open state */
halfOpenSuccesses: number;
/** Failure threshold for opening circuit */
threshold: number;
/** Reset timeout in milliseconds */
resetMs: number;
}
/**
* Execute a function with circuit breaker protection.
*
* @param breaker The circuit breaker to use
* @param fn The async function to execute
* @param fallback Optional fallback function when circuit is open
* @returns The result of fn or fallback
* @throws CircuitOpenError if circuit is open and no fallback provided
*/
export async function withCircuitBreaker<T>(
breaker: CircuitBreaker,
fn: () => Promise<T>,
fallback?: () => Promise<T>,
): Promise<T> {
if (!breaker.canRequest()) {
if (fallback) {
return fallback();
}
throw new CircuitOpenError(breaker.id);
}
try {
const result = await fn();
breaker.recordSuccess();
return result;
} catch (error) {
breaker.recordFailure();
throw error;
}
}
/**
* Error thrown when circuit is open and no fallback is provided.
*/
export class CircuitOpenError extends Error {
public readonly circuitId: string;
constructor(circuitId: string) {
super(`Circuit breaker '${circuitId}' is open`);
this.name = 'CircuitOpenError';
this.circuitId = circuitId;
}
}

View File

@@ -0,0 +1,626 @@
import type {
ICacheEntry,
IUpstreamCacheConfig,
IUpstreamFetchContext,
} from './interfaces.upstream.js';
import { DEFAULT_CACHE_CONFIG } from './interfaces.upstream.js';
import type { IStorageBackend } from '../core/interfaces.core.js';
/**
* Cache metadata stored alongside cache entries.
*/
interface ICacheMetadata {
contentType: string;
headers: Record<string, string>;
cachedAt: string;
expiresAt?: string;
etag?: string;
upstreamId: string;
upstreamUrl: string;
}
/**
* S3-backed upstream cache with in-memory hot layer.
*
* Features:
* - TTL-based expiration
* - Stale-while-revalidate support
* - Negative caching (404s)
* - Content-type aware caching
* - ETag support for conditional requests
* - Multi-upstream support via URL-based cache paths
* - Persistent S3 storage with in-memory hot layer
*
* Cache paths are structured as:
* cache/{escaped-upstream-url}/{protocol}:{method}:{path}
*
* @example
* ```typescript
* // In-memory only (default)
* const cache = new UpstreamCache(config);
*
* // With S3 persistence
* const cache = new UpstreamCache(config, 10000, storage);
* ```
*/
export class UpstreamCache {
/** In-memory hot cache */
private readonly memoryCache: Map<string, ICacheEntry> = new Map();
/** Configuration */
private readonly config: IUpstreamCacheConfig;
/** Maximum in-memory cache entries */
private readonly maxMemoryEntries: number;
/** S3 storage backend (optional) */
private readonly storage?: IStorageBackend;
/** Cleanup interval handle */
private cleanupInterval: ReturnType<typeof setInterval> | null = null;
constructor(
config?: Partial<IUpstreamCacheConfig>,
maxMemoryEntries: number = 10000,
storage?: IStorageBackend
) {
this.config = { ...DEFAULT_CACHE_CONFIG, ...config };
this.maxMemoryEntries = maxMemoryEntries;
this.storage = storage;
// Start periodic cleanup if caching is enabled
if (this.config.enabled) {
this.startCleanup();
}
}
/**
* Check if caching is enabled.
*/
public isEnabled(): boolean {
return this.config.enabled;
}
/**
* Check if S3 storage is configured.
*/
public hasStorage(): boolean {
return !!this.storage;
}
/**
* Get cached entry for a request context.
* Checks memory first, then falls back to S3.
* Returns null if not found or expired (unless stale-while-revalidate).
*/
public async get(context: IUpstreamFetchContext, upstreamUrl?: string): Promise<ICacheEntry | null> {
if (!this.config.enabled) {
return null;
}
const key = this.buildCacheKey(context, upstreamUrl);
// Check memory cache first
let entry = this.memoryCache.get(key);
// If not in memory and we have storage, check S3
if (!entry && this.storage) {
entry = (await this.loadFromStorage(key)) ?? undefined;
if (entry) {
// Promote to memory cache
this.memoryCache.set(key, entry);
}
}
if (!entry) {
return null;
}
const now = new Date();
// Check if entry is expired
if (entry.expiresAt && entry.expiresAt < now) {
// Check if we can serve stale content
if (this.config.staleWhileRevalidate && !entry.stale) {
const staleAge = (now.getTime() - entry.expiresAt.getTime()) / 1000;
if (staleAge <= this.config.staleMaxAgeSeconds) {
// Mark as stale and return
entry.stale = true;
return entry;
}
}
// Entry is too old, remove it
this.memoryCache.delete(key);
if (this.storage) {
await this.deleteFromStorage(key).catch(() => {});
}
return null;
}
return entry;
}
/**
* Store a response in the cache (memory and optionally S3).
*/
public async set(
context: IUpstreamFetchContext,
data: Buffer,
contentType: string,
headers: Record<string, string>,
upstreamId: string,
upstreamUrl: string,
options?: ICacheSetOptions,
): Promise<void> {
if (!this.config.enabled) {
return;
}
// Enforce max memory entries limit
if (this.memoryCache.size >= this.maxMemoryEntries) {
this.evictOldest();
}
const key = this.buildCacheKey(context, upstreamUrl);
const now = new Date();
// Determine TTL based on content type
const ttlSeconds = options?.ttlSeconds ?? this.determineTtl(context, contentType, headers);
const entry: ICacheEntry = {
data,
contentType,
headers,
cachedAt: now,
expiresAt: ttlSeconds > 0 ? new Date(now.getTime() + ttlSeconds * 1000) : undefined,
etag: headers['etag'] || options?.etag,
upstreamId,
stale: false,
};
// Store in memory
this.memoryCache.set(key, entry);
// Store in S3 if available
if (this.storage) {
await this.saveToStorage(key, entry, upstreamUrl).catch(() => {});
}
}
/**
* Store a negative cache entry (404 response).
*/
public async setNegative(context: IUpstreamFetchContext, upstreamId: string, upstreamUrl: string): Promise<void> {
if (!this.config.enabled || this.config.negativeCacheTtlSeconds <= 0) {
return;
}
const key = this.buildCacheKey(context, upstreamUrl);
const now = new Date();
const entry: ICacheEntry = {
data: Buffer.from(''),
contentType: 'application/octet-stream',
headers: {},
cachedAt: now,
expiresAt: new Date(now.getTime() + this.config.negativeCacheTtlSeconds * 1000),
upstreamId,
stale: false,
};
this.memoryCache.set(key, entry);
if (this.storage) {
await this.saveToStorage(key, entry, upstreamUrl).catch(() => {});
}
}
/**
* Check if there's a negative cache entry for this context.
*/
public async hasNegative(context: IUpstreamFetchContext, upstreamUrl?: string): Promise<boolean> {
const entry = await this.get(context, upstreamUrl);
return entry !== null && entry.data.length === 0;
}
/**
* Invalidate a specific cache entry.
*/
public async invalidate(context: IUpstreamFetchContext, upstreamUrl?: string): Promise<boolean> {
const key = this.buildCacheKey(context, upstreamUrl);
const deleted = this.memoryCache.delete(key);
if (this.storage) {
await this.deleteFromStorage(key).catch(() => {});
}
return deleted;
}
/**
* Invalidate all entries matching a pattern.
* Useful for invalidating all versions of a package.
*/
public async invalidatePattern(pattern: RegExp): Promise<number> {
let count = 0;
for (const key of this.memoryCache.keys()) {
if (pattern.test(key)) {
this.memoryCache.delete(key);
if (this.storage) {
await this.deleteFromStorage(key).catch(() => {});
}
count++;
}
}
return count;
}
/**
* Invalidate all entries from a specific upstream.
*/
public async invalidateUpstream(upstreamId: string): Promise<number> {
let count = 0;
for (const [key, entry] of this.memoryCache.entries()) {
if (entry.upstreamId === upstreamId) {
this.memoryCache.delete(key);
if (this.storage) {
await this.deleteFromStorage(key).catch(() => {});
}
count++;
}
}
return count;
}
/**
* Clear all cache entries (memory and S3).
*/
public async clear(): Promise<void> {
this.memoryCache.clear();
// Note: S3 cleanup would require listing and deleting all cache/* objects
// This is left as a future enhancement for bulk cleanup
}
/**
* Get cache statistics.
*/
public getStats(): ICacheStats {
let freshCount = 0;
let staleCount = 0;
let negativeCount = 0;
let totalSize = 0;
const now = new Date();
for (const entry of this.memoryCache.values()) {
totalSize += entry.data.length;
if (entry.data.length === 0) {
negativeCount++;
} else if (entry.stale || (entry.expiresAt && entry.expiresAt < now)) {
staleCount++;
} else {
freshCount++;
}
}
return {
totalEntries: this.memoryCache.size,
freshEntries: freshCount,
staleEntries: staleCount,
negativeEntries: negativeCount,
totalSizeBytes: totalSize,
maxEntries: this.maxMemoryEntries,
enabled: this.config.enabled,
hasStorage: !!this.storage,
};
}
/**
* Stop the cache and cleanup.
*/
public stop(): void {
if (this.cleanupInterval) {
clearInterval(this.cleanupInterval);
this.cleanupInterval = null;
}
}
// ========================================================================
// Storage Methods
// ========================================================================
/**
* Build storage path for a cache key.
* Escapes upstream URL for safe use in S3 paths.
*/
private buildStoragePath(key: string): string {
return `cache/${key}`;
}
/**
* Build storage path for cache metadata.
*/
private buildMetadataPath(key: string): string {
return `cache/${key}.meta`;
}
/**
* Load a cache entry from S3 storage.
*/
private async loadFromStorage(key: string): Promise<ICacheEntry | null> {
if (!this.storage) return null;
try {
const dataPath = this.buildStoragePath(key);
const metaPath = this.buildMetadataPath(key);
// Load data and metadata in parallel
const [data, metaBuffer] = await Promise.all([
this.storage.getObject(dataPath),
this.storage.getObject(metaPath),
]);
if (!data || !metaBuffer) {
return null;
}
const meta: ICacheMetadata = JSON.parse(metaBuffer.toString('utf-8'));
return {
data,
contentType: meta.contentType,
headers: meta.headers,
cachedAt: new Date(meta.cachedAt),
expiresAt: meta.expiresAt ? new Date(meta.expiresAt) : undefined,
etag: meta.etag,
upstreamId: meta.upstreamId,
stale: false,
};
} catch {
return null;
}
}
/**
* Save a cache entry to S3 storage.
*/
private async saveToStorage(key: string, entry: ICacheEntry, upstreamUrl: string): Promise<void> {
if (!this.storage) return;
const dataPath = this.buildStoragePath(key);
const metaPath = this.buildMetadataPath(key);
const meta: ICacheMetadata = {
contentType: entry.contentType,
headers: entry.headers,
cachedAt: entry.cachedAt.toISOString(),
expiresAt: entry.expiresAt?.toISOString(),
etag: entry.etag,
upstreamId: entry.upstreamId,
upstreamUrl,
};
// Save data and metadata in parallel
await Promise.all([
this.storage.putObject(dataPath, entry.data),
this.storage.putObject(metaPath, Buffer.from(JSON.stringify(meta), 'utf-8')),
]);
}
/**
* Delete a cache entry from S3 storage.
*/
private async deleteFromStorage(key: string): Promise<void> {
if (!this.storage) return;
const dataPath = this.buildStoragePath(key);
const metaPath = this.buildMetadataPath(key);
await Promise.all([
this.storage.deleteObject(dataPath).catch(() => {}),
this.storage.deleteObject(metaPath).catch(() => {}),
]);
}
// ========================================================================
// Helper Methods
// ========================================================================
/**
* Escape a URL for safe use in storage paths.
*/
private escapeUrl(url: string): string {
// Remove protocol prefix and escape special characters
return url
.replace(/^https?:\/\//, '')
.replace(/[\/\\:*?"<>|]/g, '_')
.replace(/__+/g, '_');
}
/**
* Build a unique cache key for a request context.
* Includes escaped upstream URL for multi-upstream support.
*/
private buildCacheKey(context: IUpstreamFetchContext, upstreamUrl?: string): string {
// Include method, protocol, path, and sorted query params
const queryString = Object.keys(context.query)
.sort()
.map(k => `${k}=${context.query[k]}`)
.join('&');
const baseKey = `${context.protocol}:${context.method}:${context.path}${queryString ? '?' + queryString : ''}`;
if (upstreamUrl) {
return `${this.escapeUrl(upstreamUrl)}/${baseKey}`;
}
return baseKey;
}
/**
* Determine TTL based on content characteristics.
*/
private determineTtl(
context: IUpstreamFetchContext,
contentType: string,
headers: Record<string, string>,
): number {
// Check for Cache-Control header
const cacheControl = headers['cache-control'];
if (cacheControl) {
const maxAgeMatch = cacheControl.match(/max-age=(\d+)/);
if (maxAgeMatch) {
return parseInt(maxAgeMatch[1], 10);
}
if (cacheControl.includes('no-store') || cacheControl.includes('no-cache')) {
return 0;
}
}
// Check if content is immutable (content-addressable)
if (this.isImmutableContent(context, contentType)) {
return this.config.immutableTtlSeconds;
}
// Default TTL for mutable content
return this.config.defaultTtlSeconds;
}
/**
* Check if content is immutable (content-addressable).
*/
private isImmutableContent(context: IUpstreamFetchContext, contentType: string): boolean {
// OCI blobs with digest are immutable
if (context.protocol === 'oci' && context.resourceType === 'blob') {
return true;
}
// NPM tarballs are immutable (versioned)
if (context.protocol === 'npm' && context.resourceType === 'tarball') {
return true;
}
// Maven artifacts with version are immutable
if (context.protocol === 'maven' && context.resourceType === 'artifact') {
return true;
}
// Cargo crate files are immutable
if (context.protocol === 'cargo' && context.resourceType === 'crate') {
return true;
}
// Composer dist files are immutable
if (context.protocol === 'composer' && context.resourceType === 'dist') {
return true;
}
// PyPI package files are immutable
if (context.protocol === 'pypi' && context.resourceType === 'package') {
return true;
}
// RubyGems .gem files are immutable
if (context.protocol === 'rubygems' && context.resourceType === 'gem') {
return true;
}
return false;
}
/**
* Evict oldest entries to make room for new ones.
*/
private evictOldest(): void {
// Evict 10% of max entries
const evictCount = Math.ceil(this.maxMemoryEntries * 0.1);
let evicted = 0;
// First, try to evict stale entries
const now = new Date();
for (const [key, entry] of this.memoryCache.entries()) {
if (evicted >= evictCount) break;
if (entry.stale || (entry.expiresAt && entry.expiresAt < now)) {
this.memoryCache.delete(key);
evicted++;
}
}
// If not enough evicted, evict oldest by cachedAt
if (evicted < evictCount) {
const entries = Array.from(this.memoryCache.entries())
.sort((a, b) => a[1].cachedAt.getTime() - b[1].cachedAt.getTime());
for (const [key] of entries) {
if (evicted >= evictCount) break;
this.memoryCache.delete(key);
evicted++;
}
}
}
/**
* Start periodic cleanup of expired entries.
*/
private startCleanup(): void {
// Run cleanup every minute
this.cleanupInterval = setInterval(() => {
this.cleanup();
}, 60000);
// Don't keep the process alive just for cleanup
if (this.cleanupInterval.unref) {
this.cleanupInterval.unref();
}
}
/**
* Remove all expired entries from memory cache.
*/
private cleanup(): void {
const now = new Date();
const staleDeadline = new Date(now.getTime() - this.config.staleMaxAgeSeconds * 1000);
for (const [key, entry] of this.memoryCache.entries()) {
if (entry.expiresAt) {
// Remove if past stale deadline
if (entry.expiresAt < staleDeadline) {
this.memoryCache.delete(key);
}
}
}
}
}
/**
* Options for cache set operation.
*/
export interface ICacheSetOptions {
/** Override TTL in seconds */
ttlSeconds?: number;
/** ETag for conditional requests */
etag?: string;
}
/**
* Cache statistics.
*/
export interface ICacheStats {
/** Total number of cached entries in memory */
totalEntries: number;
/** Number of fresh (non-expired) entries */
freshEntries: number;
/** Number of stale entries (expired but still usable) */
staleEntries: number;
/** Number of negative cache entries */
negativeEntries: number;
/** Total size of cached data in bytes (memory only) */
totalSizeBytes: number;
/** Maximum allowed memory entries */
maxEntries: number;
/** Whether caching is enabled */
enabled: boolean;
/** Whether S3 storage is configured */
hasStorage: boolean;
}

11
ts/upstream/index.ts Normal file
View File

@@ -0,0 +1,11 @@
// Interfaces and types
export * from './interfaces.upstream.js';
// Classes
export { CircuitBreaker, CircuitOpenError, withCircuitBreaker } from './classes.circuitbreaker.js';
export type { ICircuitBreakerMetrics } from './classes.circuitbreaker.js';
export { UpstreamCache } from './classes.upstreamcache.js';
export type { ICacheSetOptions, ICacheStats } from './classes.upstreamcache.js';
export { BaseUpstream } from './classes.baseupstream.js';

View File

@@ -0,0 +1,276 @@
import type { TRegistryProtocol, IRequestActor } from '../core/interfaces.core.js';
/**
* Scope rule for routing requests to specific upstreams.
* Uses glob patterns for flexible matching.
*/
export interface IUpstreamScopeRule {
/** Glob pattern (e.g., "@company/*", "com.example.*", "library/*") */
pattern: string;
/** Whether matching resources should be included or excluded */
action: 'include' | 'exclude';
}
/**
* Authentication configuration for an upstream registry.
* Supports multiple auth strategies.
*/
export interface IUpstreamAuthConfig {
/** Authentication type */
type: 'none' | 'basic' | 'bearer' | 'api-key';
/** Username for basic auth */
username?: string;
/** Password for basic auth */
password?: string;
/** Token for bearer or api-key auth */
token?: string;
/** Custom header name for api-key auth (default: 'Authorization') */
headerName?: string;
}
/**
* Cache configuration for upstream content.
*/
export interface IUpstreamCacheConfig {
/** Whether caching is enabled */
enabled: boolean;
/** Default TTL in seconds for mutable content (default: 300 = 5 min) */
defaultTtlSeconds: number;
/** TTL in seconds for immutable/content-addressable content (default: 2592000 = 30 days) */
immutableTtlSeconds: number;
/** Whether to serve stale content while revalidating in background */
staleWhileRevalidate: boolean;
/** Maximum age in seconds for stale content (default: 3600 = 1 hour) */
staleMaxAgeSeconds: number;
/** TTL in seconds for negative cache entries (404s) (default: 60 = 1 min) */
negativeCacheTtlSeconds: number;
}
/**
* Resilience configuration for upstream requests.
*/
export interface IUpstreamResilienceConfig {
/** Request timeout in milliseconds (default: 30000) */
timeoutMs: number;
/** Maximum number of retry attempts (default: 3) */
maxRetries: number;
/** Initial retry delay in milliseconds (default: 1000) */
retryDelayMs: number;
/** Maximum retry delay in milliseconds (default: 30000) */
retryMaxDelayMs: number;
/** Number of failures before circuit breaker opens (default: 5) */
circuitBreakerThreshold: number;
/** Time in milliseconds before circuit breaker attempts reset (default: 30000) */
circuitBreakerResetMs: number;
}
/**
* Configuration for a single upstream registry.
*/
export interface IUpstreamRegistryConfig {
/** Unique identifier for this upstream */
id: string;
/** Human-readable name */
name: string;
/** Base URL of the upstream registry (e.g., "https://registry.npmjs.org") */
url: string;
/** Priority for routing (lower = higher priority, 1 = first) */
priority: number;
/** Whether this upstream is enabled */
enabled: boolean;
/** Scope rules for routing (empty = match all) */
scopeRules?: IUpstreamScopeRule[];
/** Authentication configuration */
auth: IUpstreamAuthConfig;
/** Cache configuration overrides */
cache?: Partial<IUpstreamCacheConfig>;
/** Resilience configuration overrides */
resilience?: Partial<IUpstreamResilienceConfig>;
/** API path prefix for OCI registries (default: /v2). Useful for registries behind reverse proxies. */
apiPrefix?: string;
}
/**
* Protocol-level upstream configuration.
* Configures upstream behavior for a specific protocol (npm, oci, etc.)
*/
export interface IProtocolUpstreamConfig {
/** Whether upstream is enabled for this protocol */
enabled: boolean;
/** List of upstream registries, ordered by priority */
upstreams: IUpstreamRegistryConfig[];
/** Protocol-level cache configuration defaults */
cache?: Partial<IUpstreamCacheConfig>;
/** Protocol-level resilience configuration defaults */
resilience?: Partial<IUpstreamResilienceConfig>;
}
/**
* Result of an upstream fetch operation.
*/
export interface IUpstreamResult {
/** Whether the fetch was successful (2xx status) */
success: boolean;
/** HTTP status code */
status: number;
/** Response headers */
headers: Record<string, string>;
/** Response body (Buffer for binary, object for JSON) */
body?: Buffer | any;
/** ID of the upstream that served the request */
upstreamId: string;
/** Whether the response was served from cache */
fromCache: boolean;
/** Request latency in milliseconds */
latencyMs: number;
}
/**
* Circuit breaker state.
*/
export type TCircuitState = 'CLOSED' | 'OPEN' | 'HALF_OPEN';
/**
* Context for an upstream fetch request.
*/
export interface IUpstreamFetchContext {
/** Protocol type */
protocol: TRegistryProtocol;
/** Resource identifier (package name, artifact name, etc.) */
resource: string;
/** Type of resource being fetched (packument, tarball, manifest, blob, etc.) */
resourceType: string;
/** Original request path */
path: string;
/** HTTP method */
method: string;
/** Request headers */
headers: Record<string, string>;
/** Query parameters */
query: Record<string, string>;
/** Actor performing the request (for cache key isolation) */
actor?: IRequestActor;
}
/**
* Cache entry stored in the upstream cache.
*/
export interface ICacheEntry {
/** Cached data */
data: Buffer;
/** Content type of the cached data */
contentType: string;
/** Original response headers */
headers: Record<string, string>;
/** When the entry was cached */
cachedAt: Date;
/** When the entry expires */
expiresAt?: Date;
/** ETag for conditional requests */
etag?: string;
/** ID of the upstream that provided the data */
upstreamId: string;
/** Whether the entry is stale but still usable */
stale?: boolean;
}
/**
* Default cache configuration values.
*/
export const DEFAULT_CACHE_CONFIG: IUpstreamCacheConfig = {
enabled: true,
defaultTtlSeconds: 300, // 5 minutes
immutableTtlSeconds: 2592000, // 30 days
staleWhileRevalidate: true,
staleMaxAgeSeconds: 3600, // 1 hour
negativeCacheTtlSeconds: 60, // 1 minute
};
/**
* Default resilience configuration values.
*/
export const DEFAULT_RESILIENCE_CONFIG: IUpstreamResilienceConfig = {
timeoutMs: 30000,
maxRetries: 3,
retryDelayMs: 1000,
retryMaxDelayMs: 30000,
circuitBreakerThreshold: 5,
circuitBreakerResetMs: 30000,
};
// ============================================================================
// Upstream Provider Interfaces
// ============================================================================
/**
* Context for resolving upstream configuration.
* Passed to IUpstreamProvider per-request to enable dynamic upstream routing.
*/
export interface IUpstreamResolutionContext {
/** Protocol being accessed */
protocol: TRegistryProtocol;
/** Resource identifier (package name, repository, coordinates, etc.) */
resource: string;
/** Extracted scope (e.g., "company" from "@company/pkg", "myorg" from "myorg/image") */
scope: string | null;
/** Actor performing the request */
actor?: IRequestActor;
/** HTTP method */
method: string;
/** Resource type (packument, tarball, manifest, blob, etc.) */
resourceType: string;
}
/**
* Dynamic upstream configuration provider.
* Implement this interface to provide per-request upstream routing
* based on actor context (user, organization, etc.)
*
* @example
* ```typescript
* class OrgUpstreamProvider implements IUpstreamProvider {
* constructor(private db: Database) {}
*
* async resolveUpstreamConfig(ctx: IUpstreamResolutionContext) {
* if (ctx.actor?.orgId) {
* const orgConfig = await this.db.getOrgUpstream(ctx.actor.orgId, ctx.protocol);
* if (orgConfig) return orgConfig;
* }
* return this.db.getDefaultUpstream(ctx.protocol);
* }
* }
* ```
*/
export interface IUpstreamProvider {
/** Optional initialization */
init?(): Promise<void>;
/**
* Resolve upstream configuration for a request.
* @param context - Information about the current request
* @returns Upstream config to use, or null to skip upstream lookup
*/
resolveUpstreamConfig(context: IUpstreamResolutionContext): Promise<IProtocolUpstreamConfig | null>;
}
/**
* Static upstream provider for simple configurations.
* Use this when you have fixed upstream registries that don't change per-request.
*
* @example
* ```typescript
* const provider = new StaticUpstreamProvider({
* npm: {
* enabled: true,
* upstreams: [{ id: 'npmjs', url: 'https://registry.npmjs.org', priority: 1, enabled: true, auth: { type: 'none' } }],
* },
* });
* ```
*/
export class StaticUpstreamProvider implements IUpstreamProvider {
constructor(private configs: Partial<Record<TRegistryProtocol, IProtocolUpstreamConfig>>) {}
async resolveUpstreamConfig(ctx: IUpstreamResolutionContext): Promise<IProtocolUpstreamConfig | null> {
return this.configs[ctx.protocol] ?? null;
}
}

View File

@@ -4,9 +4,7 @@
"module": "NodeNext",
"moduleResolution": "NodeNext",
"esModuleInterop": true,
"verbatimModuleSyntax": true,
"baseUrl": ".",
"paths": {}
"verbatimModuleSyntax": true
},
"exclude": ["dist_*/**/*.d.ts"]
}