Compare commits
58 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 2e2726a4de | |||
| 0e35256062 | |||
| 10190a39fc | |||
| 9643ef98b9 | |||
| 09335d41f3 | |||
| 2221eef722 | |||
| 26ddf1a59f | |||
| 5acd1d6166 | |||
| abf7605e14 | |||
| 7da1a35efe | |||
| 1f0acf2825 | |||
| 37e4c5be4a | |||
| 9bbc3da484 | |||
| e9af3f8328 | |||
| 351680159b | |||
| 0cabf284ed | |||
| dbc8566aad | |||
| bd64a7b140 | |||
| ae8dec9142 | |||
| 19da87a9df | |||
| 99b01733e7 | |||
| 0610077eec | |||
| cfadc89b5a | |||
| eb91a3f75b | |||
| 58a21a6bbb | |||
| da1cf8ddeb | |||
| 35ff286169 | |||
| a78934836e | |||
| e81fa41b18 | |||
| 41405eb40a | |||
| 67188a4e9f | |||
| a2f7f43027 | |||
| 37a89239d9 | |||
| 93fee289e7 | |||
| 30fd9a4238 | |||
| 3b5bf5e789 | |||
| 9b92e1c0d2 | |||
| 6291ebf79b | |||
| fcd95677a0 | |||
| 547c262578 | |||
| 2d6059ba7f | |||
| 284329c191 | |||
| 4f662ff611 | |||
| b3da95e6c1 | |||
| b1bb6af312 | |||
| 0d73230d5a | |||
| ac51a94c8b | |||
| 9ca1e670ef | |||
| fb8d6897e3 | |||
| 81ae4f2d59 | |||
| 374469e37e | |||
| 9039613f7a | |||
| 4d13fac9f1 | |||
| 42209d235d | |||
| 80005af576 | |||
| 8d48627301 | |||
| 92d27d8b15 | |||
| 0b31219b7d |
@@ -1,35 +1,5 @@
|
||||
// The Dev Container format allows you to configure your environment. At the heart of it
|
||||
// is a Docker image or Dockerfile which controls the tools available in your environment.
|
||||
//
|
||||
// See https://aka.ms/devcontainer.json for more information.
|
||||
{
|
||||
"name": "Ona",
|
||||
// This universal image (~10GB) includes many development tools and languages,
|
||||
// providing a convenient all-in-one development environment.
|
||||
//
|
||||
// This image is already available on remote runners for fast startup. On desktop
|
||||
// and linux runners, it will need to be downloaded, which may take longer.
|
||||
//
|
||||
// For faster startup on desktop/linux, consider a smaller, language-specific image:
|
||||
// • For Python: mcr.microsoft.com/devcontainers/python:3.13
|
||||
// • For Node.js: mcr.microsoft.com/devcontainers/javascript-node:24
|
||||
// • For Go: mcr.microsoft.com/devcontainers/go:1.24
|
||||
// • For Java: mcr.microsoft.com/devcontainers/java:21
|
||||
//
|
||||
// Browse more options at: https://hub.docker.com/r/microsoft/devcontainers
|
||||
// or build your own using the Dockerfile option below.
|
||||
"name": "gitzone.universal",
|
||||
"image": "mcr.microsoft.com/devcontainers/universal:4.0.1-noble"
|
||||
// Use "build":
|
||||
// instead of the image to use a Dockerfile to build an image.
|
||||
// "build": {
|
||||
// "context": ".",
|
||||
// "dockerfile": "Dockerfile"
|
||||
// }
|
||||
// Features add additional features to your environment. See https://containers.dev/features
|
||||
// Beware: features are not supported on all platforms and may have unintended side-effects.
|
||||
// "features": {
|
||||
// "ghcr.io/devcontainers/features/docker-in-docker": {
|
||||
// "moby": false
|
||||
// }
|
||||
// }
|
||||
}
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"@git.zone/cli": {
|
||||
"projectType": "npm",
|
||||
"module": {
|
||||
"githost": "code.foss.global",
|
||||
"gitscope": "push.rocks",
|
||||
"gitrepo": "smartregistry",
|
||||
"description": "A composable TypeScript library implementing OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems registries for building unified container and package registries",
|
||||
"npmPackagename": "@push.rocks/smartregistry",
|
||||
"license": "MIT",
|
||||
"projectDomain": "push.rocks"
|
||||
},
|
||||
"release": {
|
||||
"registries": [
|
||||
"https://verdaccio.lossless.digital",
|
||||
"https://registry.npmjs.org"
|
||||
],
|
||||
"accessLevel": "public"
|
||||
}
|
||||
},
|
||||
"@ship.zone/szci": {
|
||||
"npmGlobalTools": []
|
||||
}
|
||||
}
|
||||
Vendored
+1
-1
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"json.schemas": [
|
||||
{
|
||||
"fileMatch": ["/npmextra.json"],
|
||||
"fileMatch": ["/.smartconfig.json"],
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
|
||||
+256
@@ -1,5 +1,261 @@
|
||||
# Changelog
|
||||
|
||||
## 2026-04-16 - 2.9.1 - fix(license)
|
||||
add missing MIT license file to repository
|
||||
|
||||
- Adds the project license file to align the repository contents with the package metadata license declaration.
|
||||
|
||||
## 2026-04-16 - 2.9.0 - feat(registry)
|
||||
add declarative protocol routing and request-scoped storage hook context across registries
|
||||
|
||||
- Refactors protocol registration and request dispatch in SmartRegistry around shared registry descriptors.
|
||||
- Wraps protocol request handling in storage context so hooks receive protocol, actor, package, and version metadata without cross-request leakage.
|
||||
- Adds shared base registry helpers for header parsing, bearer/basic auth extraction, actor construction, and protocol logger creation.
|
||||
- Improves NPM route parsing and publish helpers, including support for unencoded scoped package metadata and publish paths.
|
||||
- Introduces centralized registry storage path helpers and expands test helpers and coverage for concurrent context isolation and real request hook metadata.
|
||||
|
||||
## 2026-03-27 - 2.8.2 - fix(maven,tests)
|
||||
handle Maven Basic auth and accept deploy-plugin metadata/checksum uploads while stabilizing npm CLI test cleanup
|
||||
|
||||
- Validate Maven tokens from Basic auth credentials by extracting the password portion before token validation.
|
||||
- Return successful responses for PUT requests to checksum and maven-metadata endpoints so Maven deploy uploads do not fail when files are auto-generated.
|
||||
- Improve npm CLI integration test isolation and cleanup by using a temporary test directory, copying per-package .npmrc files, and cleaning stale published packages before test runs.
|
||||
- Tighten test teardown by destroying the registry explicitly and simplifying package/install fixture generation.
|
||||
|
||||
## 2026-03-24 - 2.8.1 - fix(registry)
|
||||
align OCI and RubyGems API behavior and improve npm search result ordering
|
||||
|
||||
- handle OCI version checks on /v2 and /v2/ endpoints
|
||||
- return RubyGems versions JSON in the expected flat array format and update unyank coverage to use the HTTP endpoint
|
||||
- prioritize exact and prefix matches in npm search results
|
||||
- update documentation to reflect full upstream proxy support
|
||||
|
||||
## 2026-03-24 - 2.8.0 - feat(core,storage,oci,registry-config)
|
||||
add streaming response support and configurable registry URLs across protocols
|
||||
|
||||
- Normalize SmartRegistry responses to ReadableStream bodies at the public API boundary and add stream helper utilities for buffers, JSON, and hashing
|
||||
- Add streaming storage accessors for OCI, npm, Maven, Cargo, Composer, PyPI, and RubyGems downloads to reduce in-memory buffering
|
||||
- Make per-protocol registryUrl configurable so CLI and integration tests can use correct host and port values
|
||||
- Refactor OCI blob uploads to persist chunks in storage during upload and clean up temporary chunk objects after completion or expiry
|
||||
- Update tests and storage integration to use the new stream-based response model and smartstorage backend
|
||||
|
||||
## 2025-12-03 - 2.7.0 - feat(upstream)
|
||||
Add dynamic per-request upstream provider and integrate into registries
|
||||
|
||||
- Introduce IUpstreamProvider and IUpstreamResolutionContext to resolve upstream configs per request.
|
||||
- Add StaticUpstreamProvider implementation for simple static upstream configurations.
|
||||
- Propagate dynamic upstream provider through SmartRegistry and wire into protocol handlers (npm, oci, maven, cargo, composer, pypi, rubygems).
|
||||
- Replace persistent per-protocol upstream instances with per-request resolution: registries now call provider.resolveUpstreamConfig(...) and instantiate protocol-specific Upstream when needed.
|
||||
- Add IRequestActor to core interfaces and pass actor context (userId, ip, userAgent, etc.) to upstream resolution and storage/auth hooks.
|
||||
- Update many protocol registries to accept an upstreamProvider instead of IProtocolUpstreamConfig and to attempt upstream fetches only when provider returns enabled config.
|
||||
- Add utilities and tests: test helpers to create registries with upstream provider, a tracking upstream provider helper, StaticUpstreamProvider tests and extensive upstream/provider integration tests.
|
||||
- Improve upstream interfaces and cache/fetch contexts (IUpstreamFetchContext includes actor) and add StaticUpstreamProvider class to upstream module.
|
||||
|
||||
## 2025-11-27 - 2.6.0 - feat(core)
|
||||
Add core registry infrastructure: storage, auth, upstream cache, and protocol handlers
|
||||
|
||||
- Introduce RegistryStorage: unified storage abstraction with hook support (before/after put/delete/get) and helpers for OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems paths and operations
|
||||
- Add DefaultAuthProvider and AuthManager: in-memory token store, UUID tokens for package protocols, OCI JWT creation/validation, token lifecycle (create/validate/revoke) and authorization checking
|
||||
- Add SmartRegistry orchestrator to initialize and route requests to protocol handlers (OCI, NPM, Maven, Cargo, Composer, PyPI, RubyGems)
|
||||
- Implement upstream subsystem: UpstreamCache (in-memory + optional S3 persistence), BaseUpstream with multi-upstream routing, scope rules, retries, TTLs, stale-while-revalidate and negative caching
|
||||
- Add circuit breaker implementation for upstream resilience with exponential backoff and per-upstream breakers
|
||||
- Add protocol implementations and helpers: NpmRegistry/NpmUpstream (packument/tarball handling and tarball URL rewriting), PypiRegistry (PEP 503/691 support, uploads, metadata), MavenRegistry (artifact/metadata handling and checksum generation), CargoRegistry (sparse index, publish/download/yank)
|
||||
- Utility exports and helpers: buffer helpers, plugins aggregator, path helpers, and various protocol-specific helper modules
|
||||
|
||||
## 2025-11-27 - 2.5.0 - feat(pypi,rubygems)
|
||||
Add PyPI and RubyGems protocol implementations, upstream caching, and auth/storage improvements
|
||||
|
||||
- Implemented full PyPI support (PEP 503 Simple API HTML, PEP 691 JSON API, legacy upload handling, name normalization, hash verification, content negotiation, package/file storage and metadata management).
|
||||
- Implemented RubyGems support (compact index, /versions, /info, /names endpoints, gem upload, yank/unyank, platform handling and file storage).
|
||||
- Expanded RegistryStorage with protocol-specific helpers for OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems (get/put/delete/list helpers, metadata handling, context-aware hooks).
|
||||
- Added AuthManager and DefaultAuthProvider improvements: unified token creation/validation for multiple protocols (npm, oci, maven, composer, cargo, pypi, rubygems) and OCI JWT support.
|
||||
- Added upstream infrastructure: BaseUpstream, UpstreamCache (S3-backed optional, stale-while-revalidate, negative caching), circuit breaker with retries/backoff and resilience defaults.
|
||||
- Added various protocol registries (NPM, Maven, Cargo, OCI, PyPI) with request routing, permission checks, and optional upstream proxying/caching.
|
||||
|
||||
## 2025-11-27 - 2.4.0 - feat(core)
|
||||
Add pluggable auth providers, storage hooks, multi-upstream cache awareness, and PyPI/RubyGems protocol implementations
|
||||
|
||||
- Introduce pluggable authentication: IAuthProvider interface and DefaultAuthProvider (in-memory) with OCI JWT support and UUID tokens.
|
||||
- AuthManager now accepts a custom provider and delegates all auth operations (authenticate, validateToken, create/revoke tokens, authorize, listUserTokens).
|
||||
- Add storage hooks (IStorageHooks) and hook contexts: beforePut/afterPut/afterGet/beforeDelete/afterDelete. RegistryStorage now supports hooks, context management (setContext/withContext) and invokes hooks around operations.
|
||||
- RegistryStorage expanded with many protocol-specific helper methods (OCI, NPM, Maven, Cargo, Composer, PyPI, RubyGems) and improved S3/SmartBucket integration.
|
||||
- Upstream improvements: BaseUpstream and UpstreamCache became multi-upstream aware (cache keys now include upstream URL), cache operations are async and support negative caching, stale-while-revalidate, ETag/metadata persistence, and S3-backed storage layer.
|
||||
- Circuit breaker, retry, resilience and scope-rule routing enhancements for upstreams; upstream fetch logic updated to prefer primary upstream for cache keys and background revalidation behavior.
|
||||
- SmartRegistry API extended to accept custom authProvider and storageHooks, and now wires RegistryStorage and AuthManager with those options. Core exports updated to expose auth and storage interfaces and DefaultAuthProvider.
|
||||
- Add full PyPI (PEP 503/691, upload API) and RubyGems (Compact Index, API v1, uploads/yank/unyank, specs endpoints) registry implementations with parsing, upload/download, metadata management and upstream proxying.
|
||||
- Add utility helpers: binary buffer helpers (toBuffer/isBinaryData), pypi and rubygems helper modules, and numerous protocol-specific helpers and tests referenced in readme.hints.
|
||||
- These changes are additive and designed to be backward compatible; bumping minor version.
|
||||
|
||||
## 2025-11-27 - 2.3.0 - feat(upstream)
|
||||
Add upstream proxy/cache subsystem and integrate per-protocol upstreams
|
||||
|
||||
- Introduce a complete upstream subsystem (BaseUpstream, UpstreamCache, CircuitBreaker) with caching, negative-cache, stale-while-revalidate, retries, exponential backoff and per-upstream circuit breakers.
|
||||
- Add upstream interfaces and defaults (ts/upstream/interfaces.upstream.ts) and export upstream utilities from ts/upstream/index.ts and root ts/index.ts.
|
||||
- Implement protocol-specific upstream clients for npm, pypi, maven, composer, cargo and rubygems (classes.*upstream.ts) to fetch metadata and artifacts from configured upstream registries.
|
||||
- Integrate upstream usage into registries: registries now accept an upstream config, attempt to fetch missing metadata/artifacts from upstreams, cache results locally, and expose destroy() to stop upstream resources.
|
||||
- Add SmartRequest and minimatch to dependencies and expose smartrequest/minimatch via ts/plugins.ts for HTTP requests and glob-based scope matching.
|
||||
- Update package.json to add @push.rocks/smartrequest and minimatch dependencies.
|
||||
- Various registry implementations updated to utilize upstreams (npm, pypi, maven, composer, cargo, rubygems, oci) including URL rewrites and caching behavior.
|
||||
|
||||
## 2025-11-27 - 2.2.3 - fix(tests)
|
||||
Use unique test run IDs and add S3 cleanup in test helpers to avoid cross-run conflicts
|
||||
|
||||
- Add generateTestRunId() helper in test/helpers/registry.ts to produce unique IDs for each test run
|
||||
- Update PyPI and Composer native CLI tests to use generated testPackageName / unauth-pkg-<id> to avoid package name collisions between runs
|
||||
- Import smartbucket and add S3 bucket cleanup logic in test helpers to remove leftover objects between test runs
|
||||
- Improve test robustness by skipping upload-dependent checks when tools (twine/composer) are not available and logging outputs for debugging
|
||||
|
||||
## 2025-11-25 - 2.2.2 - fix(npm)
|
||||
Replace console logging with structured Smartlog in NPM registry and silence RubyGems helper error logging
|
||||
|
||||
- Replaced console.log calls with this.logger.log (Smartlog) in ts/npm/classes.npmregistry.ts for debug/info/success events
|
||||
- Converted console.error in NpmRegistry.handleSearch to structured logger.log('error', ...) including the error message
|
||||
- Removed console.error from ts/rubygems/helpers.rubygems.ts; gem metadata extraction failures are now handled silently by returning null
|
||||
|
||||
## 2025-11-25 - 2.2.1 - fix(core)
|
||||
Normalize binary data handling across registries and add buffer helpers
|
||||
|
||||
- Add core/helpers.buffer.ts with isBinaryData and toBuffer utilities to consistently handle Buffer, Uint8Array, string and object inputs.
|
||||
- Composer: accept Uint8Array uploads, convert to Buffer before ZIP extraction, SHA-1 calculation and storage.
|
||||
- PyPI: accept multipart file content as Buffer or Uint8Array and normalize to Buffer before processing and storage.
|
||||
- Maven: normalize artifact body input with toBuffer before validation and storage.
|
||||
- OCI: improve upload id generation by using substring for correct random length.
|
||||
|
||||
## 2025-11-25 - 2.2.0 - feat(core/registrystorage)
|
||||
Persist OCI manifest content-type in sidecar and normalize manifest body handling
|
||||
|
||||
- Add getOciManifestContentType(repository, digest) to read stored manifest Content-Type
|
||||
- Store manifest Content-Type in a .type sidecar file when putOciManifest is called
|
||||
- Update putOciManifest to persist both manifest data and its content type
|
||||
- OciRegistry now retrieves stored content type (with fallback to detectManifestContentType) when serving manifests
|
||||
- Add toBuffer helper in OciRegistry to consistently convert various request body forms to Buffer for digest calculation and uploads
|
||||
|
||||
## 2025-11-25 - 2.1.2 - fix(oci)
|
||||
Prefer raw request body for content-addressable OCI operations and expose rawBody on request context
|
||||
|
||||
- Add rawBody?: Buffer to IRequestContext to allow callers to provide the exact raw request bytes for digest calculation (falls back to body if absent).
|
||||
- OCI registry handlers now prefer context.rawBody over context.body for content-addressable operations (manifests, blobs, and blob uploads) to preserve exact bytes and ensure digest calculation matches client expectations.
|
||||
- Upload flow updates: upload init, PATCH (upload chunk) and PUT (complete upload) now pass rawBody when available.
|
||||
|
||||
## 2025-11-25 - 2.1.1 - fix(oci)
|
||||
Preserve raw manifest bytes for digest calculation and handle string/JSON manifest bodies in OCI registry
|
||||
|
||||
- Preserve the exact bytes of the manifest payload when computing the sha256 digest to comply with the OCI spec and avoid mismatches caused by re-serialization.
|
||||
- Accept string request bodies (converted using UTF-8) and treat already-parsed JSON objects by re-serializing as a fallback.
|
||||
- Keep existing content-type fallback logic while ensuring accurate digest calculation prior to storing manifests.
|
||||
|
||||
## 2025-11-25 - 2.1.0 - feat(oci)
|
||||
Support configurable OCI token realm/service and centralize unauthorized responses
|
||||
|
||||
- SmartRegistry now forwards optional ociTokens (realm and service) from auth configuration to OciRegistry when OCI is enabled
|
||||
- OciRegistry constructor accepts an optional ociTokens parameter and stores it for use in auth headers
|
||||
- Replaced repeated construction of WWW-Authenticate headers with createUnauthorizedResponse and createUnauthorizedHeadResponse helpers that use configured realm/service
|
||||
- Behavior is backwards-compatible: when ociTokens are not configured the registry falls back to the previous defaults (realm: <basePath>/v2/token, service: "registry")
|
||||
|
||||
## 2025-11-25 - 2.0.0 - BREAKING CHANGE(pypi,rubygems)
|
||||
Revise PyPI and RubyGems handling: normalize error payloads, fix .gem parsing/packing, adjust PyPI JSON API and tests, and export smartarchive plugin
|
||||
|
||||
- Rename error payload property from 'message' to 'error' in PyPI and RubyGems interfaces and responses; error responses are now returned as JSON objects (body: { error: ... }) instead of Buffer(JSON.stringify(...)).
|
||||
- RubyGems: treat .gem files as plain tar archives (not gzipped). Use metadata.gz and data.tar.gz correctly, switch packing helper to pack plain tar, and use zlib deflate for .rz gemspec data.
|
||||
- RubyGems registry: add legacy Marshal specs endpoint (specs.4.8.gz) and adjust versions handler invocation to accept request context.
|
||||
- PyPI: adopt PEP 691 style (files is an array of file objects) in tests and metadata; include requires_python in test package metadata; update JSON API path matching to the package-level '/{package}/json' style used by the handler.
|
||||
- Fix HTML escaping expectations in tests (requires_python values are HTML-escaped in attributes, e.g. '>=3.8').
|
||||
- Export smartarchive from plugins to enable archive helpers in core modules and helpers.
|
||||
- Update tests and internal code to match the new error shape and API/format behaviour.
|
||||
|
||||
## 2025-11-25 - 1.9.0 - feat(auth)
|
||||
Implement HMAC-SHA256 OCI JWTs; enhance PyPI & RubyGems uploads and normalize responses
|
||||
|
||||
- AuthManager: create and validate OCI JWTs signed with HMAC-SHA256 (header.payload.signature). Signature verification, exp/nbf checks and payload decoding implemented.
|
||||
- PyPI: improved Simple API handling (PEP-691 JSON responses returned as objects), Simple HTML responses updated, upload handling enhanced to support nested/flat multipart fields, verify hashes (sha256/md5/blake2b), store files and return 201 on success.
|
||||
- RubyGems: upload flow now attempts to extract gem metadata from the .gem binary when name/version are not provided, improved validation, and upload returns 201. Added extractGemMetadata helper.
|
||||
- OCI: centralized 401 response creation (including proper WWW-Authenticate header) and HEAD behavior fixed to return no body per HTTP spec.
|
||||
- SmartRegistry: use nullish coalescing for protocol basePath defaults to avoid falsy-value bugs when basePath is an empty string.
|
||||
- Tests and helpers: test expectations adjusted (Content-Type startsWith check for HTML, PEP-691 projects is an array), test helper switched to smartarchive for packaging.
|
||||
- Package.json: added devDependency @push.rocks/smartarchive and updated dev deps.
|
||||
- Various response normalization: avoid unnecessary Buffer.from() for already-serialized objects/strings and standardize status codes for create/upload endpoints (201).
|
||||
|
||||
## 2025-11-24 - 1.8.0 - feat(smarts3)
|
||||
Add local smarts3 testing support and documentation
|
||||
|
||||
- Added @push.rocks/smarts3 ^5.1.0 to devDependencies to enable a local S3-compatible test server.
|
||||
- Updated README with a new "Testing with smarts3" section including a Quick Start example and integration test commands.
|
||||
- Documented benefits and CI-friendly usage for running registry integration tests locally without cloud credentials.
|
||||
|
||||
## 2025-11-23 - 1.7.0 - feat(core)
|
||||
Standardize S3 storage config using @tsclass/tsclass IS3Descriptor and wire it into RegistryStorage and plugins exports; update README and package dependencies.
|
||||
|
||||
- Add @tsclass/tsclass dependency to package.json to provide a standardized IS3Descriptor for S3 configuration.
|
||||
- Export tsclass from ts/plugins.ts so plugin types are available to core modules.
|
||||
- Update IStorageConfig to extend plugins.tsclass.storage.IS3Descriptor, consolidating storage configuration typing.
|
||||
- Change RegistryStorage.init() to pass the storage config directly as an IS3Descriptor to SmartBucket (bucketName remains part of IStorageConfig).
|
||||
- Update README storage section with example config and mention IS3Descriptor integration.
|
||||
|
||||
## 2025-11-21 - 1.6.0 - feat(core)
|
||||
Add PyPI and RubyGems registries, integrate into SmartRegistry, extend storage and auth
|
||||
|
||||
- Introduce PyPI registry implementation with PEP 503 (Simple API) and PEP 691 (JSON API), legacy upload support, content negotiation and HTML/JSON generators (ts/pypi/*).
|
||||
- Introduce RubyGems registry implementation with Compact Index support, API v1 endpoints (upload, yank/unyank), versions/names files and helpers (ts/rubygems/*).
|
||||
- Wire PyPI and RubyGems into the main orchestrator: SmartRegistry now initializes, exposes and routes requests to pypi and rubygems handlers.
|
||||
- Extend RegistryStorage with PyPI and RubyGems storage helpers (metadata, simple index, package files, compact index files, gem files).
|
||||
- Extend AuthManager to support PyPI and RubyGems UUID token creation, validation and revocation and include them in unified token validation.
|
||||
- Add verification of client-provided hashes during PyPI uploads (SHA256 always calculated and verified; MD5 and Blake2b verified when provided) to prevent corrupted uploads.
|
||||
- Export new modules from library entry point (ts/index.ts) and add lightweight rubygems index file export.
|
||||
- Add helper utilities for PyPI and RubyGems (name normalization, HTML generation, hash calculations, compact index generation/parsing).
|
||||
- Update documentation hints/readme to reflect implementation status and configuration examples for pypi and rubygems.
|
||||
|
||||
## 2025-11-21 - 1.5.0 - feat(core)
|
||||
Add PyPI and RubyGems protocol support, Cargo token management, and storage helpers
|
||||
|
||||
- Extend core protocol types to include 'pypi' and 'rubygems' and add protocol config entries for pypi and rubygems.
|
||||
- Add PyPI storage methods for metadata, Simple API HTML/JSON indexes, package files, version listing and deletion in RegistryStorage.
|
||||
- Add Cargo-specific storage helpers (index paths, crate storage) and ensure Cargo registry initialization and endpoints are wired into SmartRegistry.
|
||||
- Extend AuthManager with Cargo, PyPI and RubyGems token creation, validation and revocation methods; update unified validateToken to check these token types.
|
||||
- Update test helpers to create Cargo tokens and return cargoToken from registry setup.
|
||||
|
||||
## 2025-11-21 - 1.4.1 - fix(devcontainer)
|
||||
Simplify devcontainer configuration and rename container image
|
||||
|
||||
- Rename Dev Container name to 'gitzone.universal' and set image to mcr.microsoft.com/devcontainers/universal:4.0.1-noble
|
||||
- Remove large inline comments and example 'build'/'features' blocks to simplify the devcontainer.json
|
||||
|
||||
## 2025-11-21 - 1.4.0 - feat(registrystorage)
|
||||
Add deleteMavenMetadata to RegistryStorage and update Maven DELETE test to expect 204 No Content
|
||||
|
||||
- Add deleteMavenMetadata(groupId, artifactId) to RegistryStorage to remove maven-metadata.xml.
|
||||
- Update Maven test to assert 204 No Content for DELETE responses (previously expected 200).
|
||||
|
||||
## 2025-11-21 - 1.3.1 - fix(maven)
|
||||
Pass request path to Maven checksum handler so checksum files are resolved correctly
|
||||
|
||||
- Call handleChecksumRequest with the full request path from MavenRegistry.handleRequest
|
||||
- Allows getChecksum to extract the checksum filename from the URL and fetch the correct checksum file from storage
|
||||
- Fixes 404s when requesting artifact checksum files (md5, sha1, sha256, sha512)
|
||||
|
||||
## 2025-11-21 - 1.3.0 - feat(core)
|
||||
Add Cargo and Composer registries with storage, auth and helpers
|
||||
|
||||
- Add Cargo registry implementation (ts/cargo) including index, publish, download, yank/unyank and search handlers
|
||||
- Add Composer registry implementation (ts/composer) including package upload/download, metadata, packages.json and helpers
|
||||
- Extend RegistryStorage with Cargo and Composer-specific storage helpers and path conventions
|
||||
- Extend AuthManager with Composer token creation/validation and unified token validation support
|
||||
- Wire SmartRegistry to initialize and route requests to cargo and composer handlers
|
||||
- Add adm-zip dependency and Composer ZIP parsing helpers (extractComposerJsonFromZip, sha1 calculation, version sorting)
|
||||
- Add tests for Cargo index path calculation and config handling
|
||||
- Export new modules from ts/index.ts and add module entry files for composer and cargo
|
||||
|
||||
## 2025-11-21 - 1.2.0 - feat(maven)
|
||||
Add Maven registry protocol support (storage, auth, routing, interfaces, and exports)
|
||||
|
||||
- Add Maven protocol to core types (TRegistryProtocol) and IRegistryConfig
|
||||
- SmartRegistry: initialize Maven registry when enabled, route requests to /maven, and expose it via getRegistry
|
||||
- RegistryStorage: implement Maven storage helpers (get/put/delete artifact, metadata, list versions) and path helpers
|
||||
- AuthManager: add UUID token creation/validation/revocation for Maven and integrate into unified validateToken/authorize flow
|
||||
- New ts/maven module: exports, interfaces and helpers for Maven coordinates, metadata, and search results
|
||||
- Add basic Cargo (crates.io) scaffolding: ts/cargo exports and Cargo interfaces
|
||||
- Update top-level ts/index.ts and package exports to include Maven (and cargo) modules
|
||||
- Tests/helpers updated to enable Maven in test registry and add Maven artifact/checksum helpers
|
||||
|
||||
## 2025-11-20 - 1.1.1 - fix(oci)
|
||||
Improve OCI manifest permission response and tag handling: include WWW-Authenticate header on unauthorized manifest GETs, accept optional headers in manifest lookup, and persist tags as a unified tags.json mapping when pushing manifests.
|
||||
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
Copyright (c) 2025 Task Venture Capital GmbH (hello@task.vc)
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -1,18 +0,0 @@
|
||||
{
|
||||
"gitzone": {
|
||||
"projectType": "npm",
|
||||
"module": {
|
||||
"githost": "code.foss.global",
|
||||
"gitscope": "push.rocks",
|
||||
"gitrepo": "smartregistry",
|
||||
"description": "a registry for npm modules and oci images",
|
||||
"npmPackagename": "@push.rocks/smartregistry",
|
||||
"license": "MIT",
|
||||
"projectDomain": "push.rocks"
|
||||
}
|
||||
},
|
||||
"npmci": {
|
||||
"npmGlobalTools": [],
|
||||
"npmAccessLevel": "public"
|
||||
}
|
||||
}
|
||||
+18
-12
@@ -1,8 +1,8 @@
|
||||
{
|
||||
"name": "@push.rocks/smartregistry",
|
||||
"version": "1.1.1",
|
||||
"version": "2.9.1",
|
||||
"private": false,
|
||||
"description": "a registry for npm modules and oci images",
|
||||
"description": "A composable TypeScript library implementing OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems registries for building unified container and package registries",
|
||||
"main": "dist_ts/index.js",
|
||||
"typings": "dist_ts/index.d.ts",
|
||||
"type": "module",
|
||||
@@ -10,15 +10,17 @@
|
||||
"license": "MIT",
|
||||
"scripts": {
|
||||
"test": "(tstest test/ --verbose --logfile --timeout 240)",
|
||||
"build": "(tsbuild --web --allowimplicitany)",
|
||||
"build": "(tsbuild --allowimplicitany)",
|
||||
"buildDocs": "(tsdoc)"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@git.zone/tsbuild": "^3.1.0",
|
||||
"@git.zone/tsbundle": "^2.0.5",
|
||||
"@git.zone/tsrun": "^2.0.0",
|
||||
"@git.zone/tstest": "^3.1.0",
|
||||
"@types/node": "^24.10.1"
|
||||
"@git.zone/tsbuild": "^4.4.0",
|
||||
"@git.zone/tsbundle": "^2.10.0",
|
||||
"@git.zone/tsrun": "^2.0.2",
|
||||
"@git.zone/tstest": "^3.6.0",
|
||||
"@push.rocks/smartarchive": "^5.2.1",
|
||||
"@push.rocks/smartstorage": "^6.3.2",
|
||||
"@types/node": "^25.5.0"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
@@ -37,7 +39,7 @@
|
||||
"dist_ts_web/**/*",
|
||||
"assets/**/*",
|
||||
"cli.js",
|
||||
"npmextra.json",
|
||||
".smartconfig.json",
|
||||
"readme.md"
|
||||
],
|
||||
"pnpm": {
|
||||
@@ -45,9 +47,13 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@push.rocks/qenv": "^6.1.3",
|
||||
"@push.rocks/smartbucket": "^4.3.0",
|
||||
"@push.rocks/smartlog": "^3.1.10",
|
||||
"@push.rocks/smartpath": "^6.0.0"
|
||||
"@push.rocks/smartbucket": "^4.5.1",
|
||||
"@push.rocks/smartlog": "^3.2.1",
|
||||
"@push.rocks/smartpath": "^6.0.0",
|
||||
"@push.rocks/smartrequest": "^5.0.1",
|
||||
"@tsclass/tsclass": "^9.5.0",
|
||||
"adm-zip": "^0.5.16",
|
||||
"minimatch": "^10.2.4"
|
||||
},
|
||||
"packageManager": "pnpm@10.18.1+sha512.77a884a165cbba2d8d1c19e3b4880eee6d2fcabd0d879121e282196b80042351d5eb3ca0935fa599da1dc51265cc68816ad2bddd2a2de5ea9fdf92adbec7cd34"
|
||||
}
|
||||
|
||||
Generated
+3021
-4082
File diff suppressed because it is too large
Load Diff
+438
-2
@@ -1,3 +1,439 @@
|
||||
# Project Readme Hints
|
||||
# Project Implementation Notes
|
||||
|
||||
This is the initial readme hints file.
|
||||
This file contains technical implementation details for PyPI and RubyGems protocols.
|
||||
|
||||
## Python (PyPI) Protocol Implementation ✅
|
||||
|
||||
### PEP 503: Simple Repository API (HTML-based)
|
||||
|
||||
**URL Structure:**
|
||||
- Root: `/<base>/` - Lists all projects
|
||||
- Project: `/<base>/<project>/` - Lists all files for a project
|
||||
- All URLs MUST end with `/` (redirect if missing)
|
||||
|
||||
**Package Name Normalization:**
|
||||
- Lowercase all characters
|
||||
- Replace runs of `.`, `-`, `_` with single `-`
|
||||
- Implementation: `re.sub(r"[-_.]+", "-", name).lower()`
|
||||
|
||||
**HTML Format:**
|
||||
- Root: One anchor per project
|
||||
- Project: One anchor per file
|
||||
- Anchor text must match final filename
|
||||
- Anchor href links to download URL
|
||||
|
||||
**Hash Fragments:**
|
||||
Format: `#<hashname>=<hashvalue>`
|
||||
- hashname: lowercase hash function name (recommend `sha256`)
|
||||
- hashvalue: hex-encoded digest
|
||||
|
||||
**Data Attributes:**
|
||||
- `data-gpg-sig`: `true`/`false` for GPG signature presence
|
||||
- `data-requires-python`: PEP 345 requirement string (HTML-encode `<` as `<`, `>` as `>`)
|
||||
|
||||
### PEP 691: JSON-based Simple API
|
||||
|
||||
**Content Types:**
|
||||
- `application/vnd.pypi.simple.v1+json` - JSON format
|
||||
- `application/vnd.pypi.simple.v1+html` - HTML format
|
||||
- `text/html` - Alias for HTML (backwards compat)
|
||||
|
||||
**Root Endpoint JSON:**
|
||||
```json
|
||||
{
|
||||
"meta": {"api-version": "1.0"},
|
||||
"projects": [{"name": "ProjectName"}]
|
||||
}
|
||||
```
|
||||
|
||||
**Project Endpoint JSON:**
|
||||
```json
|
||||
{
|
||||
"name": "normalized-name",
|
||||
"meta": {"api-version": "1.0"},
|
||||
"files": [
|
||||
{
|
||||
"filename": "package-1.0-py3-none-any.whl",
|
||||
"url": "https://example.com/path/to/file",
|
||||
"hashes": {"sha256": "..."},
|
||||
"requires-python": ">=3.7",
|
||||
"dist-info-metadata": true | {"sha256": "..."},
|
||||
"gpg-sig": true,
|
||||
"yanked": false | "reason string"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Content Negotiation:**
|
||||
- Use `Accept` header for format selection
|
||||
- Server responds with `Content-Type` header
|
||||
- Support both JSON and HTML formats
|
||||
|
||||
### PyPI Upload API (Legacy /legacy/)
|
||||
|
||||
**Endpoint:**
|
||||
- URL: `https://upload.pypi.org/legacy/`
|
||||
- Method: `POST`
|
||||
- Content-Type: `multipart/form-data`
|
||||
|
||||
**Required Form Fields:**
|
||||
- `:action` = `file_upload`
|
||||
- `protocol_version` = `1`
|
||||
- `content` = Binary file data with filename
|
||||
- `filetype` = `bdist_wheel` | `sdist`
|
||||
- `pyversion` = Python tag (e.g., `py3`, `py2.py3`) or `source` for sdist
|
||||
- `metadata_version` = Metadata standard version
|
||||
- `name` = Package name
|
||||
- `version` = Version string
|
||||
|
||||
**Hash Digest (one required):**
|
||||
- `md5_digest`: urlsafe base64 without padding
|
||||
- `sha256_digest`: hexadecimal
|
||||
- `blake2_256_digest`: hexadecimal
|
||||
|
||||
**Optional Fields:**
|
||||
- `attestations`: JSON array of attestation objects
|
||||
- Any Core Metadata fields (lowercase, hyphens → underscores)
|
||||
- Example: `Description-Content-Type` → `description_content_type`
|
||||
|
||||
**Authentication:**
|
||||
- Username/password or API token in HTTP Basic Auth
|
||||
- API tokens: username = `__token__`, password = token value
|
||||
|
||||
**Behavior:**
|
||||
- First file uploaded creates the release
|
||||
- Multiple files uploaded sequentially for same version
|
||||
|
||||
### PEP 694: Upload 2.0 API
|
||||
|
||||
**Status:** Draft (not yet required, legacy API still supported)
|
||||
- Multi-step workflow with sessions
|
||||
- Async upload support with resumption
|
||||
- JSON-based API
|
||||
- Standard HTTP auth (RFC 7235)
|
||||
- Not implementing initially (legacy API sufficient)
|
||||
|
||||
---
|
||||
|
||||
## Ruby (RubyGems) Protocol Implementation ✅
|
||||
|
||||
### Compact Index Format
|
||||
|
||||
**Endpoints:**
|
||||
- `/versions` - Master list of all gems and versions
|
||||
- `/info/<RUBYGEM>` - Detailed info for specific gem
|
||||
- `/names` - Simple list of gem names
|
||||
|
||||
**Authentication:**
|
||||
- UUID tokens similar to NPM pattern
|
||||
- API key in `Authorization` header
|
||||
- Scope format: `rubygems:gem:{name}:{read|write|yank}`
|
||||
|
||||
### `/versions` File Format
|
||||
|
||||
**Structure:**
|
||||
```
|
||||
created_at: 2024-04-01T00:00:05Z
|
||||
---
|
||||
RUBYGEM [-]VERSION_PLATFORM[,VERSION_PLATFORM,...] MD5
|
||||
```
|
||||
|
||||
**Details:**
|
||||
- Metadata lines before `---` delimiter
|
||||
- One line per gem with comma-separated versions
|
||||
- `[-]` prefix indicates yanked version
|
||||
- `MD5`: Checksum of corresponding `/info/<RUBYGEM>` file
|
||||
- Append-only during month, recalculated monthly
|
||||
|
||||
### `/info/<RUBYGEM>` File Format
|
||||
|
||||
**Structure:**
|
||||
```
|
||||
---
|
||||
VERSION[-PLATFORM] [DEPENDENCY[,DEPENDENCY,...]]|REQUIREMENT[,REQUIREMENT,...]
|
||||
```
|
||||
|
||||
**Dependency Format:**
|
||||
```
|
||||
GEM:CONSTRAINT[&CONSTRAINT]
|
||||
```
|
||||
- Examples: `actionmailer:= 2.2.2`, `parser:>= 3.2.2.3`
|
||||
- Operators: `=`, `>`, `<`, `>=`, `<=`, `~>`, `!=`
|
||||
- Multiple constraints: `unicode-display_width:< 3.0&>= 2.4.0`
|
||||
|
||||
**Requirement Format:**
|
||||
```
|
||||
checksum:SHA256_HEX
|
||||
ruby:CONSTRAINT
|
||||
rubygems:CONSTRAINT
|
||||
```
|
||||
|
||||
**Platform:**
|
||||
- Default platform is `ruby`
|
||||
- Non-default platforms: `VERSION-PLATFORM` (e.g., `3.2.1-arm64-darwin`)
|
||||
|
||||
**Yanked Gems:**
|
||||
- Listed with `-` prefix in `/versions`
|
||||
- Excluded entirely from `/info/<RUBYGEM>` file
|
||||
|
||||
### `/names` File Format
|
||||
|
||||
```
|
||||
---
|
||||
gemname1
|
||||
gemname2
|
||||
gemname3
|
||||
```
|
||||
|
||||
### HTTP Range Support
|
||||
|
||||
**Headers:**
|
||||
- `Range: bytes=#{start}-`: Request from byte position
|
||||
- `If-None-Match`: ETag conditional request
|
||||
- `Repr-Digest`: SHA256 checksum in response
|
||||
|
||||
**Caching Strategy:**
|
||||
1. Store file with last byte position
|
||||
2. Request range from last position
|
||||
3. Append response to existing file
|
||||
4. Verify SHA256 against `Repr-Digest`
|
||||
|
||||
### RubyGems Upload/Management API
|
||||
|
||||
**Upload Gem:**
|
||||
- `POST /api/v1/gems`
|
||||
- Binary `.gem` file in request body
|
||||
- `Authorization` header with API key
|
||||
|
||||
**Yank Version:**
|
||||
- `DELETE /api/v1/gems/yank`
|
||||
- Parameters: `gem_name`, `version`
|
||||
|
||||
**Unyank Version:**
|
||||
- `PUT /api/v1/gems/unyank`
|
||||
- Parameters: `gem_name`, `version`
|
||||
|
||||
**Version Metadata:**
|
||||
- `GET /api/v1/versions/<gem>.json`
|
||||
- Returns JSON array of versions
|
||||
|
||||
**Dependencies:**
|
||||
- `GET /api/v1/dependencies?gems=<comma-list>`
|
||||
- Returns dependency information for resolution
|
||||
|
||||
---
|
||||
|
||||
## Implementation Details
|
||||
|
||||
### Completed Protocols
|
||||
- ✅ OCI Distribution Spec v1.1
|
||||
- ✅ NPM Registry API
|
||||
- ✅ Maven Repository
|
||||
- ✅ Cargo/crates.io Registry
|
||||
- ✅ Composer/Packagist
|
||||
- ✅ PyPI (Python Package Index) - PEP 503/691
|
||||
- ✅ RubyGems - Compact Index
|
||||
|
||||
### Storage Paths
|
||||
|
||||
**PyPI:**
|
||||
```
|
||||
pypi/
|
||||
├── simple/ # PEP 503 HTML files
|
||||
│ ├── index.html # All packages list
|
||||
│ └── {package}/index.html # Package versions list
|
||||
├── packages/
|
||||
│ └── {package}/{filename} # .whl and .tar.gz files
|
||||
└── metadata/
|
||||
└── {package}/metadata.json # Package metadata
|
||||
```
|
||||
|
||||
**RubyGems:**
|
||||
```
|
||||
rubygems/
|
||||
├── versions # Master versions file
|
||||
├── info/{gemname} # Per-gem info files
|
||||
├── names # All gem names
|
||||
└── gems/{gemname}-{version}.gem # .gem files
|
||||
```
|
||||
|
||||
### Authentication Pattern
|
||||
|
||||
Both protocols should follow the existing UUID token pattern used by NPM, Maven, Cargo, Composer:
|
||||
|
||||
```typescript
|
||||
// AuthManager additions
|
||||
createPypiToken(userId: string, readonly: boolean): string
|
||||
validatePypiToken(token: string): ITokenInfo | null
|
||||
revokePypiToken(token: string): boolean
|
||||
|
||||
createRubyGemsToken(userId: string, readonly: boolean): string
|
||||
validateRubyGemsToken(token: string): ITokenInfo | null
|
||||
revokeRubyGemsToken(token: string): boolean
|
||||
```
|
||||
|
||||
### Scope Format
|
||||
|
||||
```
|
||||
pypi:package:{name}:{read|write}
|
||||
rubygems:gem:{name}:{read|write|yank}
|
||||
```
|
||||
|
||||
### Common Patterns
|
||||
|
||||
1. **Package name normalization** - Critical for PyPI
|
||||
2. **Checksum calculation** - SHA256 for both protocols
|
||||
3. **Append-only files** - RubyGems compact index
|
||||
4. **Content negotiation** - PyPI JSON vs HTML
|
||||
5. **Multipart upload parsing** - PyPI file uploads
|
||||
6. **Binary file handling** - Both protocols (.whl, .tar.gz, .gem)
|
||||
|
||||
---
|
||||
|
||||
## Key Differences from Existing Protocols
|
||||
|
||||
**PyPI vs NPM:**
|
||||
- PyPI uses Simple API (HTML) + JSON API
|
||||
- PyPI requires package name normalization
|
||||
- PyPI uses multipart form data for uploads (not JSON)
|
||||
- PyPI supports multiple file types per release (wheel + sdist)
|
||||
|
||||
**RubyGems vs Cargo:**
|
||||
- RubyGems uses compact index (append-only text files)
|
||||
- RubyGems uses checksums in index files (not just filenames)
|
||||
- RubyGems has HTTP Range support for incremental updates
|
||||
- RubyGems uses MD5 for index checksums, SHA256 for .gem files
|
||||
|
||||
---
|
||||
|
||||
## Testing Requirements
|
||||
|
||||
### PyPI Tests Must Cover:
|
||||
- Package upload (wheel and sdist)
|
||||
- Package name normalization
|
||||
- Simple API HTML generation (PEP 503)
|
||||
- JSON API responses (PEP 691)
|
||||
- Content negotiation
|
||||
- Hash calculation and verification
|
||||
- Authentication (tokens)
|
||||
- Multi-file releases
|
||||
- Yanked packages
|
||||
|
||||
### RubyGems Tests Must Cover:
|
||||
- Gem upload
|
||||
- Compact index generation
|
||||
- `/versions` file updates (append-only)
|
||||
- `/info/<gem>` file generation
|
||||
- `/names` file generation
|
||||
- Checksum calculations (MD5 and SHA256)
|
||||
- Platform-specific gems
|
||||
- Yanking/unyanking
|
||||
- HTTP Range requests
|
||||
- Authentication (API keys)
|
||||
|
||||
---
|
||||
|
||||
## Security Considerations
|
||||
|
||||
1. **Package name validation** - Prevent path traversal
|
||||
2. **File size limits** - Prevent DoS via large uploads
|
||||
3. **Content-Type validation** - Verify file types
|
||||
4. **Checksum verification** - Ensure file integrity
|
||||
5. **Token scope enforcement** - Read vs write permissions
|
||||
6. **HTML escaping** - Prevent XSS in generated HTML
|
||||
7. **Metadata sanitization** - Clean user-provided strings
|
||||
8. **Rate limiting** - Consider upload frequency limits
|
||||
|
||||
---
|
||||
|
||||
## Implementation Status (Completed)
|
||||
|
||||
### PyPI Implementation ✅
|
||||
- **Files Created:**
|
||||
- `ts/pypi/interfaces.pypi.ts` - Type definitions (354 lines)
|
||||
- `ts/pypi/helpers.pypi.ts` - Helper functions (280 lines)
|
||||
- `ts/pypi/classes.pypiregistry.ts` - Main registry (650 lines)
|
||||
- `ts/pypi/index.ts` - Module exports
|
||||
|
||||
- **Features Implemented:**
|
||||
- ✅ PEP 503 Simple API (HTML)
|
||||
- ✅ PEP 691 JSON API
|
||||
- ✅ Content negotiation (Accept header)
|
||||
- ✅ Package name normalization
|
||||
- ✅ File upload with multipart/form-data
|
||||
- ✅ Hash verification (SHA256, MD5, Blake2b)
|
||||
- ✅ Package metadata management
|
||||
- ✅ JSON API endpoints (/pypi/{package}/json)
|
||||
- ✅ Token-based authentication
|
||||
- ✅ Scope-based permissions (read/write/delete)
|
||||
|
||||
- **Security Enhancements:**
|
||||
- ✅ Hash verification on upload (validates client-provided hashes)
|
||||
- ✅ Package name validation (regex check)
|
||||
- ✅ HTML escaping in generated pages
|
||||
- ✅ Permission checks on all mutating operations
|
||||
|
||||
### RubyGems Implementation ✅
|
||||
- **Files Created:**
|
||||
- `ts/rubygems/interfaces.rubygems.ts` - Type definitions (215 lines)
|
||||
- `ts/rubygems/helpers.rubygems.ts` - Helper functions (350 lines)
|
||||
- `ts/rubygems/classes.rubygemsregistry.ts` - Main registry (580 lines)
|
||||
- `ts/rubygems/index.ts` - Module exports
|
||||
|
||||
- **Features Implemented:**
|
||||
- ✅ Compact Index format (modern Bundler)
|
||||
- ✅ /versions endpoint (all gems list)
|
||||
- ✅ /info/{gem} endpoint (gem-specific metadata)
|
||||
- ✅ /names endpoint (gem names list)
|
||||
- ✅ Gem upload API
|
||||
- ✅ Yank/unyank functionality
|
||||
- ✅ Platform-specific gems support
|
||||
- ✅ JSON API endpoints
|
||||
- ✅ Legacy endpoints (specs.4.8.gz, Marshal.4.8)
|
||||
- ✅ Token-based authentication
|
||||
- ✅ Scope-based permissions
|
||||
|
||||
### Integration ✅
|
||||
- **Core Updates:**
|
||||
- ✅ Updated `IRegistryConfig` interface
|
||||
- ✅ Updated `TRegistryProtocol` type
|
||||
- ✅ Added authentication methods to `AuthManager`
|
||||
- ✅ Added 30+ storage methods to `RegistryStorage`
|
||||
- ✅ Updated `SmartRegistry` initialization and routing
|
||||
- ✅ Module exports from `ts/index.ts`
|
||||
|
||||
- **Test Coverage:**
|
||||
- ✅ `test/test.pypi.ts` - 25+ tests covering all PyPI endpoints
|
||||
- ✅ `test/test.rubygems.ts` - 30+ tests covering all RubyGems endpoints
|
||||
- ✅ `test/test.integration.pypi-rubygems.ts` - Integration tests
|
||||
- ✅ Updated test helpers with PyPI and RubyGems support
|
||||
|
||||
### Known Limitations
|
||||
1. **PyPI:**
|
||||
- Does not implement legacy XML-RPC API
|
||||
- No support for PGP signatures (data-gpg-sig always false)
|
||||
- Metadata extraction from wheel files not implemented
|
||||
|
||||
2. **RubyGems:**
|
||||
- Gem spec extraction from .gem files returns placeholder (Ruby Marshal parsing not implemented)
|
||||
- Legacy Marshal endpoints return basic data only
|
||||
- No support for gem dependencies resolution
|
||||
|
||||
### Configuration Example
|
||||
```typescript
|
||||
{
|
||||
pypi: {
|
||||
enabled: true,
|
||||
basePath: '/pypi', // Also handles /simple
|
||||
},
|
||||
rubygems: {
|
||||
enabled: true,
|
||||
basePath: '/rubygems',
|
||||
},
|
||||
auth: {
|
||||
pypiTokens: { enabled: true },
|
||||
rubygemsTokens: { enabled: true },
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
@@ -1,486 +1,423 @@
|
||||
# @push.rocks/smartregistry
|
||||
|
||||
> 🚀 A composable TypeScript library implementing both **OCI Distribution Specification v1.1** and **NPM Registry API** for building unified container and package registries.
|
||||
> One TypeScript registry core for OCI, npm, Maven, Cargo, Composer, PyPI, and RubyGems.
|
||||
|
||||
## ✨ Features
|
||||
`@push.rocks/smartregistry` is a composable library for building your own multi-protocol package registry. You hand it HTTP requests, it routes them to the right protocol handler, stores artifacts in S3-compatible object storage, enforces shared auth scopes, and can proxy/cache upstream registries when content is not local.
|
||||
|
||||
### 🔄 Dual Protocol Support
|
||||
- **OCI Distribution Spec v1.1**: Full container registry with manifest/blob operations
|
||||
- **NPM Registry API**: Complete package registry with publish/install/search
|
||||
## Issue Reporting and Security
|
||||
|
||||
### 🏗️ Unified Architecture
|
||||
- **Composable Design**: Core infrastructure with protocol plugins
|
||||
- **Shared Storage**: Cloud-agnostic S3-compatible backend ([@push.rocks/smartbucket](https://www.npmjs.com/package/@push.rocks/smartbucket))
|
||||
- **Unified Authentication**: Scope-based permissions across both protocols
|
||||
- **Path-based Routing**: `/oci/*` for containers, `/npm/*` for packages
|
||||
For reporting bugs, issues, or security vulnerabilities, please visit [community.foss.global/](https://community.foss.global/). This is the central community hub for all issue reporting. Developers who sign and comply with our contribution agreement and go through identification can also get a [code.foss.global/](https://code.foss.global/) account to submit Pull Requests directly.
|
||||
|
||||
### 🔐 Authentication & Authorization
|
||||
- NPM UUID tokens for package operations
|
||||
- OCI JWT tokens for container operations
|
||||
- Unified scope system: `npm:package:foo:write`, `oci:repository:bar:push`
|
||||
- Pluggable via async callbacks
|
||||
## Why It Hits
|
||||
|
||||
### 📦 Comprehensive Feature Set
|
||||
- One registry engine, seven ecosystems.
|
||||
- Shared S3-backed storage instead of protocol-specific silos.
|
||||
- Shared auth, token minting, and scope checks across protocols.
|
||||
- Optional upstream proxying with retries, stale caching, negative caching, and circuit breakers.
|
||||
- Clean public API: `handleRequest()` in, `IResponse` out.
|
||||
|
||||
**OCI Features:**
|
||||
- ✅ Pull operations (manifests, blobs)
|
||||
- ✅ Push operations (chunked uploads)
|
||||
- ✅ Content discovery (tags, referrers API)
|
||||
- ✅ Content management (deletion)
|
||||
## What It Actually Ships
|
||||
|
||||
**NPM Features:**
|
||||
- ✅ Package publish/unpublish
|
||||
- ✅ Package download (tarballs)
|
||||
- ✅ Metadata & search
|
||||
- ✅ Dist-tag management
|
||||
- ✅ Token management
|
||||
| Protocol | Paths | What works | Auth style |
|
||||
| --- | --- | --- | --- |
|
||||
| OCI | `/oci/*` or `/v2/*` | version check, blobs, manifests, tags, referrers, deletes | Bearer JWT |
|
||||
| npm | `/npm/*` | login, publish, packuments, version metadata, tarballs, dist-tags, search, token APIs, unpublish | Bearer token |
|
||||
| Maven | `/maven/*` | POM/JAR/WAR upload and download, `maven-metadata.xml`, auto-generated checksums, delete | Bearer token or Basic auth with the token as password |
|
||||
| Cargo | `/cargo/*` | sparse index, `config.json`, publish, download, search, yank, unyank | plain `Authorization` token |
|
||||
| Composer | `/composer/*` | `packages.json`, `p2` metadata, ZIP dists, filtered package lists, upload, version delete, package delete | Bearer token, plus Basic auth for credential-backed reads |
|
||||
| PyPI | `/simple/*` and `/pypi/*` | PEP 503 HTML, PEP 691 JSON, upload, JSON metadata API, downloads, package delete, version delete | Bearer token or Basic `__token__:<token>` |
|
||||
| RubyGems | `/rubygems/*` | Compact Index, gem downloads, versions/dependencies JSON, specs endpoints, upload, yank, unyank | plain `Authorization` token |
|
||||
|
||||
## 📥 Installation
|
||||
Use `oci.basePath = '/v2'` if you want native Docker/OCI client compatibility. The default `/oci` path is fine for app-level routing, but Docker expects `/v2`.
|
||||
|
||||
Set `pypi.registryUrl` to the host root, not `/pypi`, because the Simple API lives at `/simple/*` while uploads and JSON endpoints live under `/pypi/*`.
|
||||
|
||||
## Install
|
||||
|
||||
```bash
|
||||
# Using npm
|
||||
npm install @push.rocks/smartregistry
|
||||
|
||||
# Using pnpm (recommended)
|
||||
pnpm add @push.rocks/smartregistry
|
||||
```
|
||||
|
||||
## 🚀 Quick Start
|
||||
## Quick Start
|
||||
|
||||
```typescript
|
||||
import { SmartRegistry, IRegistryConfig } from '@push.rocks/smartregistry';
|
||||
```ts
|
||||
import { SmartRegistry, type IRegistryConfig } from '@push.rocks/smartregistry';
|
||||
|
||||
const publicUrl = 'https://registry.example.com';
|
||||
|
||||
const config: IRegistryConfig = {
|
||||
storage: {
|
||||
accessKey: 'your-s3-key',
|
||||
accessSecret: 'your-s3-secret',
|
||||
endpoint: 's3.amazonaws.com',
|
||||
accessKey: process.env.S3_ACCESS_KEY!,
|
||||
accessSecret: process.env.S3_ACCESS_SECRET!,
|
||||
endpoint: 's3.example.com',
|
||||
port: 443,
|
||||
useSsl: true,
|
||||
region: 'us-east-1',
|
||||
bucketName: 'my-registry',
|
||||
region: 'eu-central-1',
|
||||
bucketName: 'registry-artifacts',
|
||||
},
|
||||
auth: {
|
||||
jwtSecret: 'your-secret-key',
|
||||
jwtSecret: process.env.REGISTRY_JWT_SECRET!,
|
||||
tokenStore: 'memory',
|
||||
npmTokens: { enabled: true },
|
||||
ociTokens: {
|
||||
enabled: true,
|
||||
realm: 'https://auth.example.com/token',
|
||||
service: 'my-registry',
|
||||
realm: `${publicUrl}/v2/token`,
|
||||
service: 'smartregistry',
|
||||
},
|
||||
pypiTokens: { enabled: true },
|
||||
rubygemsTokens: { enabled: true },
|
||||
},
|
||||
oci: {
|
||||
enabled: true,
|
||||
basePath: '/oci',
|
||||
},
|
||||
npm: {
|
||||
enabled: true,
|
||||
basePath: '/npm',
|
||||
},
|
||||
oci: { enabled: true, basePath: '/v2' },
|
||||
npm: { enabled: true, basePath: '/npm', registryUrl: `${publicUrl}/npm` },
|
||||
maven: { enabled: true, basePath: '/maven', registryUrl: `${publicUrl}/maven` },
|
||||
cargo: { enabled: true, basePath: '/cargo', registryUrl: `${publicUrl}/cargo` },
|
||||
composer: { enabled: true, basePath: '/composer', registryUrl: `${publicUrl}/composer` },
|
||||
pypi: { enabled: true, basePath: '/pypi', registryUrl: publicUrl },
|
||||
rubygems: { enabled: true, basePath: '/rubygems', registryUrl: `${publicUrl}/rubygems` },
|
||||
};
|
||||
|
||||
const registry = new SmartRegistry(config);
|
||||
await registry.init();
|
||||
|
||||
// Handle requests
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/npm/express',
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
const auth = registry.getAuthManager();
|
||||
const npmToken = await auth.createNpmToken('ci-bot');
|
||||
const cargoToken = await auth.createCargoToken('ci-bot');
|
||||
const ociToken = await auth.createOciToken('ci-bot', ['oci:repository:myorg/myapp:*']);
|
||||
|
||||
console.log({ npmToken, cargoToken, ociToken });
|
||||
```
|
||||
|
||||
## 🏛️ Architecture
|
||||
If you do not pass `authProvider`, the library uses `DefaultAuthProvider`, an in-memory reference implementation. That is perfect for tests and local dev, but you will usually want a real `IAuthProvider` in production.
|
||||
|
||||
### Directory Structure
|
||||
## HTTP Integration
|
||||
|
||||
```
|
||||
ts/
|
||||
├── core/ # Shared infrastructure
|
||||
│ ├── classes.baseregistry.ts
|
||||
│ ├── classes.registrystorage.ts
|
||||
│ ├── classes.authmanager.ts
|
||||
│ └── interfaces.core.ts
|
||||
├── oci/ # OCI implementation
|
||||
│ ├── classes.ociregistry.ts
|
||||
│ └── interfaces.oci.ts
|
||||
├── npm/ # NPM implementation
|
||||
│ ├── classes.npmregistry.ts
|
||||
│ └── interfaces.npm.ts
|
||||
└── classes.smartregistry.ts # Main orchestrator
|
||||
```
|
||||
`SmartRegistry` is not a web framework. You own the HTTP server, request parsing, and response writing. The happy path is very small:
|
||||
|
||||
### Request Flow
|
||||
```ts
|
||||
import { createServer, type IncomingHttpHeaders } from 'node:http';
|
||||
import { Readable } from 'node:stream';
|
||||
|
||||
```
|
||||
HTTP Request
|
||||
↓
|
||||
SmartRegistry (orchestrator)
|
||||
↓
|
||||
Path-based routing
|
||||
├─→ /oci/* → OciRegistry
|
||||
└─→ /npm/* → NpmRegistry
|
||||
↓
|
||||
Shared Storage & Auth
|
||||
↓
|
||||
S3-compatible backend
|
||||
```
|
||||
|
||||
## 💡 Usage Examples
|
||||
|
||||
### 🐳 OCI Registry (Container Images)
|
||||
|
||||
```typescript
|
||||
// Pull an image
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/oci/v2/library/nginx/manifests/latest',
|
||||
headers: {
|
||||
'Authorization': 'Bearer <token>',
|
||||
},
|
||||
query: {},
|
||||
});
|
||||
|
||||
// Push a blob
|
||||
const uploadInit = await registry.handleRequest({
|
||||
method: 'POST',
|
||||
path: '/oci/v2/myapp/blobs/uploads/',
|
||||
headers: { 'Authorization': 'Bearer <token>' },
|
||||
query: {},
|
||||
});
|
||||
|
||||
const uploadId = uploadInit.headers['Docker-Upload-UUID'];
|
||||
|
||||
await registry.handleRequest({
|
||||
method: 'PUT',
|
||||
path: `/oci/v2/myapp/blobs/uploads/${uploadId}`,
|
||||
headers: { 'Authorization': 'Bearer <token>' },
|
||||
query: { digest: 'sha256:abc123...' },
|
||||
body: blobData,
|
||||
});
|
||||
```
|
||||
|
||||
### 📦 NPM Registry (Packages)
|
||||
|
||||
```typescript
|
||||
// Install a package (get metadata)
|
||||
const metadata = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/npm/express',
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
// Download tarball
|
||||
const tarball = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/npm/express/-/express-4.18.0.tgz',
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
// Publish a package
|
||||
const publishResponse = await registry.handleRequest({
|
||||
method: 'PUT',
|
||||
path: '/npm/my-package',
|
||||
headers: { 'Authorization': 'Bearer <npm-token>' },
|
||||
query: {},
|
||||
body: {
|
||||
name: 'my-package',
|
||||
versions: {
|
||||
'1.0.0': { /* version metadata */ },
|
||||
},
|
||||
'dist-tags': { latest: '1.0.0' },
|
||||
_attachments: {
|
||||
'my-package-1.0.0.tgz': {
|
||||
content_type: 'application/octet-stream',
|
||||
data: '<base64-tarball>',
|
||||
length: 12345,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
// Search packages
|
||||
const searchResults = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/npm/-/v1/search',
|
||||
headers: {},
|
||||
query: { text: 'express', size: '20' },
|
||||
});
|
||||
```
|
||||
|
||||
### 🔐 Authentication
|
||||
|
||||
```typescript
|
||||
// Get auth manager instance
|
||||
const authManager = registry.getAuthManager();
|
||||
|
||||
// Authenticate user
|
||||
const userId = await authManager.authenticate({
|
||||
username: 'user',
|
||||
password: 'pass',
|
||||
});
|
||||
|
||||
// Create NPM token
|
||||
const npmToken = await authManager.createNpmToken(userId, false);
|
||||
|
||||
// Create OCI token with scopes
|
||||
const ociToken = await authManager.createOciToken(
|
||||
userId,
|
||||
['oci:repository:myapp:push', 'oci:repository:myapp:pull'],
|
||||
3600
|
||||
);
|
||||
|
||||
// Validate any token
|
||||
const token = await authManager.validateToken(npmToken, 'npm');
|
||||
|
||||
// Check permissions
|
||||
const canWrite = await authManager.authorize(
|
||||
token,
|
||||
'npm:package:my-package',
|
||||
'write'
|
||||
);
|
||||
```
|
||||
|
||||
## ⚙️ Configuration
|
||||
|
||||
### Storage Configuration
|
||||
|
||||
```typescript
|
||||
storage: {
|
||||
accessKey: string; // S3 access key
|
||||
accessSecret: string; // S3 secret key
|
||||
endpoint: string; // S3 endpoint
|
||||
port?: number; // Default: 443
|
||||
useSsl?: boolean; // Default: true
|
||||
region?: string; // Default: 'us-east-1'
|
||||
bucketName: string; // Bucket name
|
||||
}
|
||||
```
|
||||
|
||||
### Authentication Configuration
|
||||
|
||||
```typescript
|
||||
auth: {
|
||||
jwtSecret: string; // Secret for signing JWTs
|
||||
tokenStore: 'memory' | 'redis' | 'database';
|
||||
npmTokens: {
|
||||
enabled: boolean;
|
||||
defaultReadonly?: boolean;
|
||||
};
|
||||
ociTokens: {
|
||||
enabled: boolean;
|
||||
realm: string; // Auth server URL
|
||||
service: string; // Service name
|
||||
};
|
||||
}
|
||||
```
|
||||
|
||||
### Protocol Configuration
|
||||
|
||||
```typescript
|
||||
oci?: {
|
||||
enabled: boolean;
|
||||
basePath: string; // Default: '/oci'
|
||||
features?: {
|
||||
referrers?: boolean;
|
||||
deletion?: boolean;
|
||||
};
|
||||
function headersToRecord(headers: IncomingHttpHeaders): Record<string, string> {
|
||||
return Object.fromEntries(
|
||||
Object.entries(headers).map(([key, value]) => [
|
||||
key,
|
||||
Array.isArray(value) ? value.join(', ') : value ?? '',
|
||||
])
|
||||
);
|
||||
}
|
||||
|
||||
npm?: {
|
||||
enabled: boolean;
|
||||
basePath: string; // Default: '/npm'
|
||||
features?: {
|
||||
publish?: boolean;
|
||||
unpublish?: boolean;
|
||||
search?: boolean;
|
||||
};
|
||||
}
|
||||
```
|
||||
createServer(async (req, res) => {
|
||||
const url = new URL(req.url ?? '/', 'http://localhost');
|
||||
const headers = headersToRecord(req.headers);
|
||||
|
||||
## 📚 API Reference
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of req) {
|
||||
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
||||
}
|
||||
const rawBody = Buffer.concat(chunks);
|
||||
|
||||
### Core Classes
|
||||
let body: unknown = rawBody.length ? rawBody : undefined;
|
||||
if ((headers['content-type'] ?? '').includes('application/json') && rawBody.length) {
|
||||
body = JSON.parse(rawBody.toString('utf8'));
|
||||
}
|
||||
|
||||
#### SmartRegistry
|
||||
|
||||
Main orchestrator class that routes requests to appropriate protocol handlers.
|
||||
|
||||
**Methods:**
|
||||
- `init()` - Initialize the registry
|
||||
- `handleRequest(context)` - Handle HTTP request
|
||||
- `getStorage()` - Get storage instance
|
||||
- `getAuthManager()` - Get auth manager
|
||||
- `getRegistry(protocol)` - Get protocol handler
|
||||
|
||||
#### RegistryStorage
|
||||
|
||||
Unified storage abstraction for both OCI and NPM content.
|
||||
|
||||
**OCI Methods:**
|
||||
- `getOciBlob(digest)` - Get blob
|
||||
- `putOciBlob(digest, data)` - Store blob
|
||||
- `getOciManifest(repo, digest)` - Get manifest
|
||||
- `putOciManifest(repo, digest, data, type)` - Store manifest
|
||||
|
||||
**NPM Methods:**
|
||||
- `getNpmPackument(name)` - Get package metadata
|
||||
- `putNpmPackument(name, data)` - Store package metadata
|
||||
- `getNpmTarball(name, version)` - Get tarball
|
||||
- `putNpmTarball(name, version, data)` - Store tarball
|
||||
|
||||
#### AuthManager
|
||||
|
||||
Unified authentication manager supporting both NPM and OCI authentication schemes.
|
||||
|
||||
**Methods:**
|
||||
- `authenticate(credentials)` - Validate user credentials
|
||||
- `createNpmToken(userId, readonly)` - Create NPM token
|
||||
- `createOciToken(userId, scopes, expiresIn)` - Create OCI JWT
|
||||
- `validateToken(token, protocol)` - Validate any token
|
||||
- `authorize(token, resource, action)` - Check permissions
|
||||
|
||||
### Protocol Handlers
|
||||
|
||||
#### OciRegistry
|
||||
|
||||
OCI Distribution Specification v1.1 compliant registry.
|
||||
|
||||
**Endpoints:**
|
||||
- `GET /v2/` - Version check
|
||||
- `GET /v2/{name}/manifests/{ref}` - Get manifest
|
||||
- `PUT /v2/{name}/manifests/{ref}` - Push manifest
|
||||
- `GET /v2/{name}/blobs/{digest}` - Get blob
|
||||
- `POST /v2/{name}/blobs/uploads/` - Initiate upload
|
||||
- `PUT /v2/{name}/blobs/uploads/{uuid}` - Complete upload
|
||||
- `GET /v2/{name}/tags/list` - List tags
|
||||
- `GET /v2/{name}/referrers/{digest}` - Get referrers
|
||||
|
||||
#### NpmRegistry
|
||||
|
||||
NPM registry API compliant implementation.
|
||||
|
||||
**Endpoints:**
|
||||
- `GET /{package}` - Get package metadata
|
||||
- `PUT /{package}` - Publish package
|
||||
- `GET /{package}/-/{tarball}` - Download tarball
|
||||
- `GET /-/v1/search` - Search packages
|
||||
- `PUT /-/user/org.couchdb.user:{user}` - Login
|
||||
- `GET /-/npm/v1/tokens` - List tokens
|
||||
- `POST /-/npm/v1/tokens` - Create token
|
||||
- `PUT /-/package/{pkg}/dist-tags/{tag}` - Update tag
|
||||
|
||||
## 🗄️ Storage Structure
|
||||
|
||||
```
|
||||
bucket/
|
||||
├── oci/
|
||||
│ ├── blobs/
|
||||
│ │ └── sha256/{hash}
|
||||
│ ├── manifests/
|
||||
│ │ └── {repository}/{digest}
|
||||
│ └── tags/
|
||||
│ └── {repository}/tags.json
|
||||
└── npm/
|
||||
├── packages/
|
||||
│ ├── {name}/
|
||||
│ │ ├── index.json # Packument
|
||||
│ │ └── {name}-{ver}.tgz # Tarball
|
||||
│ └── @{scope}/{name}/
|
||||
│ ├── index.json
|
||||
│ └── {name}-{ver}.tgz
|
||||
└── users/
|
||||
└── {username}.json
|
||||
```
|
||||
|
||||
## 🎯 Scope Format
|
||||
|
||||
Unified scope format across protocols:
|
||||
|
||||
```
|
||||
{protocol}:{type}:{name}:{action}
|
||||
|
||||
Examples:
|
||||
npm:package:express:read # Read express package
|
||||
npm:package:*:write # Write any package
|
||||
npm:*:*:* # Full NPM access
|
||||
oci:repository:nginx:pull # Pull nginx image
|
||||
oci:repository:*:push # Push any image
|
||||
oci:*:*:* # Full OCI access
|
||||
```
|
||||
|
||||
## 🔌 Integration Examples
|
||||
|
||||
### Express Server
|
||||
|
||||
```typescript
|
||||
import express from 'express';
|
||||
import { SmartRegistry } from '@push.rocks/smartregistry';
|
||||
|
||||
const app = express();
|
||||
const registry = new SmartRegistry(config);
|
||||
await registry.init();
|
||||
|
||||
app.all('*', async (req, res) => {
|
||||
const response = await registry.handleRequest({
|
||||
method: req.method,
|
||||
path: req.path,
|
||||
headers: req.headers as Record<string, string>,
|
||||
query: req.query as Record<string, string>,
|
||||
body: req.body,
|
||||
method: req.method ?? 'GET',
|
||||
path: url.pathname,
|
||||
query: Object.fromEntries(url.searchParams),
|
||||
headers,
|
||||
body,
|
||||
rawBody: rawBody.length ? rawBody : undefined,
|
||||
});
|
||||
|
||||
res.status(response.status);
|
||||
Object.entries(response.headers).forEach(([key, value]) => {
|
||||
res.setHeader(key, value);
|
||||
});
|
||||
res.writeHead(response.status, response.headers);
|
||||
|
||||
if (response.body) {
|
||||
if (Buffer.isBuffer(response.body)) {
|
||||
res.send(response.body);
|
||||
} else {
|
||||
res.json(response.body);
|
||||
}
|
||||
} else {
|
||||
if (!response.body) {
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
});
|
||||
|
||||
app.listen(5000);
|
||||
Readable.fromWeb(response.body).pipe(res);
|
||||
}).listen(3000);
|
||||
```
|
||||
|
||||
## 🛠️ Development
|
||||
Keep `rawBody` for OCI manifests, OCI blobs, and any other digest-sensitive request where exact bytes matter.
|
||||
|
||||
For PyPI uploads, parse `multipart/form-data` before calling `handleRequest()` and pass the parsed fields in `context.body`. The library expects the upload form fields, not a raw multipart buffer.
|
||||
|
||||
At the public API boundary, `response.body` is always a `ReadableStream<Uint8Array>`.
|
||||
|
||||
## Core API
|
||||
|
||||
| API | Why you use it |
|
||||
| --- | --- |
|
||||
| `new SmartRegistry(config)` | build the registry orchestrator |
|
||||
| `await registry.init()` | initialize storage, auth, and enabled protocols |
|
||||
| `await registry.handleRequest(context)` | route one incoming HTTP request |
|
||||
| `registry.getAuthManager()` | mint, validate, revoke, and authorize tokens |
|
||||
| `registry.getStorage()` | reach the shared storage abstraction directly |
|
||||
| `registry.getRegistry(protocol)` | access a specific protocol handler |
|
||||
| `registry.destroy()` | clean up timers and protocol resources |
|
||||
|
||||
The package also exports the protocol-specific registry classes, upstream classes, `RegistryStorage`, `AuthManager`, `DefaultAuthProvider`, `StaticUpstreamProvider`, `UpstreamCache`, `CircuitBreaker`, and stream helpers such as `streamToBuffer()` and `streamToJson()`.
|
||||
|
||||
## Configuration Reference
|
||||
|
||||
| Key | Purpose |
|
||||
| --- | --- |
|
||||
| `storage` | S3-compatible backend config. This extends `IS3Descriptor` and adds `bucketName`. |
|
||||
| `auth` | shared token settings across protocols |
|
||||
| `authProvider` | plug in LDAP, OAuth, OIDC, custom DB-backed auth, or anything else implementing `IAuthProvider` |
|
||||
| `storageHooks` | receive before/after put/get/delete callbacks with protocol, actor, package, and version context |
|
||||
| `upstreamProvider` | decide per request which upstream registries to consult |
|
||||
| `oci` / `npm` / `maven` / `cargo` / `composer` / `pypi` / `rubygems` | enable protocols, set base paths, and define the public registry URL they should emit |
|
||||
|
||||
## Upstream Proxying
|
||||
|
||||
If a package, image, crate, or artifact does not exist locally, a protocol handler can resolve an upstream config on the fly and fetch it from there.
|
||||
|
||||
```ts
|
||||
import {
|
||||
StaticUpstreamProvider,
|
||||
} from '@push.rocks/smartregistry';
|
||||
|
||||
const upstreamProvider = new StaticUpstreamProvider({
|
||||
npm: {
|
||||
enabled: true,
|
||||
upstreams: [
|
||||
{
|
||||
id: 'npmjs',
|
||||
name: 'npmjs',
|
||||
url: 'https://registry.npmjs.org',
|
||||
priority: 1,
|
||||
enabled: true,
|
||||
auth: { type: 'none' },
|
||||
},
|
||||
],
|
||||
},
|
||||
oci: {
|
||||
enabled: true,
|
||||
upstreams: [
|
||||
{
|
||||
id: 'dockerhub',
|
||||
name: 'dockerhub',
|
||||
url: 'https://registry-1.docker.io',
|
||||
priority: 1,
|
||||
enabled: true,
|
||||
auth: { type: 'none' },
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
Pass that provider as `upstreamProvider` in your `IRegistryConfig`.
|
||||
|
||||
The upstream layer supports scope rules, per-request routing, retries with backoff, circuit breakers, stale-while-revalidate caching, and negative caching for 404s.
|
||||
|
||||
## Custom Auth and Audit Hooks
|
||||
|
||||
Bring your own auth system by implementing `IAuthProvider` and passing it as `authProvider`.
|
||||
|
||||
```ts
|
||||
const registry = new SmartRegistry({
|
||||
...config,
|
||||
authProvider: myAuthProvider,
|
||||
});
|
||||
```
|
||||
|
||||
Use `storageHooks` when you need quota checks, audit logs, or side effects around artifact writes and deletes.
|
||||
|
||||
```ts
|
||||
const registry = new SmartRegistry({
|
||||
...config,
|
||||
storageHooks: {
|
||||
async beforePut(context) {
|
||||
if ((context.metadata?.size ?? 0) > 500 * 1024 * 1024) {
|
||||
return { allowed: false, reason: 'Artifact too large' };
|
||||
}
|
||||
return { allowed: true };
|
||||
},
|
||||
async afterPut(context) {
|
||||
await auditLog('storage.put', context);
|
||||
},
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
`handleRequest()` also accepts an `actor` object. That extra context flows into storage hooks and upstream resolution, which is great for multi-tenant routing, org-aware policy checks, and audit trails.
|
||||
|
||||
```ts
|
||||
await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/npm/@acme/internal-lib',
|
||||
headers: { authorization: `Bearer ${npmToken}` },
|
||||
query: {},
|
||||
actor: {
|
||||
orgId: 'acme',
|
||||
sessionId: 'sess_123',
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
## Client Cheatsheet
|
||||
|
||||
### npm
|
||||
|
||||
```ini
|
||||
registry=https://registry.example.com/npm/
|
||||
//registry.example.com/npm/:_authToken=<npm-token>
|
||||
```
|
||||
|
||||
The npm handler also implements the npm-compatible login and token endpoints, including `PUT /-/user/org.couchdb.user:<name>` and `/-/npm/v1/tokens`.
|
||||
|
||||
### Docker / OCI
|
||||
|
||||
Set `oci.basePath` to `/v2` and expose a token endpoint that returns `token`, `access_token`, and `expires_in`.
|
||||
|
||||
```ts
|
||||
if (url.pathname === '/v2/token') {
|
||||
const token = await registry.getAuthManager().createOciToken(
|
||||
'docker-user',
|
||||
['oci:repository:*:*'],
|
||||
3600,
|
||||
);
|
||||
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.end(JSON.stringify({
|
||||
token,
|
||||
access_token: token,
|
||||
expires_in: 3600,
|
||||
}));
|
||||
return;
|
||||
}
|
||||
```
|
||||
|
||||
```bash
|
||||
# Install dependencies
|
||||
pnpm install
|
||||
|
||||
# Build
|
||||
pnpm run build
|
||||
|
||||
# Test
|
||||
pnpm test
|
||||
docker login registry.example.com
|
||||
docker push registry.example.com/myorg/myimage:latest
|
||||
docker pull registry.example.com/myorg/myimage:latest
|
||||
```
|
||||
|
||||
### Maven
|
||||
|
||||
```xml
|
||||
<settings>
|
||||
<servers>
|
||||
<server>
|
||||
<id>smartregistry</id>
|
||||
<username>token</username>
|
||||
<password>YOUR_MAVEN_TOKEN</password>
|
||||
</server>
|
||||
</servers>
|
||||
</settings>
|
||||
```
|
||||
|
||||
Point repositories or distribution management at `https://registry.example.com/maven`.
|
||||
|
||||
### Cargo
|
||||
|
||||
```toml
|
||||
# .cargo/config.toml
|
||||
[registries.smartregistry]
|
||||
index = "sparse+https://registry.example.com/cargo/"
|
||||
```
|
||||
|
||||
```toml
|
||||
# .cargo/credentials.toml
|
||||
[registries.smartregistry]
|
||||
token = "YOUR_CARGO_TOKEN"
|
||||
```
|
||||
|
||||
Cargo sends the token as a plain `Authorization` header, not `Bearer <token>`.
|
||||
|
||||
### Composer
|
||||
|
||||
```json
|
||||
{
|
||||
"repositories": [
|
||||
{
|
||||
"type": "composer",
|
||||
"url": "https://registry.example.com/composer"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
```json
|
||||
{
|
||||
"http-basic": {
|
||||
"registry.example.com": {
|
||||
"username": "my-user",
|
||||
"password": "my-password"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Composer installs can use Basic auth if your `authProvider.authenticate()` supports it. Programmatic writes use Bearer tokens cleanly.
|
||||
|
||||
### PyPI
|
||||
|
||||
```ini
|
||||
[distutils]
|
||||
index-servers = smartregistry
|
||||
|
||||
[smartregistry]
|
||||
repository = https://registry.example.com/pypi
|
||||
username = __token__
|
||||
password = YOUR_PYPI_TOKEN
|
||||
```
|
||||
|
||||
```bash
|
||||
pip install --index-url https://registry.example.com/simple/ your-package
|
||||
twine upload --repository smartregistry dist/*
|
||||
```
|
||||
|
||||
### RubyGems
|
||||
|
||||
```yaml
|
||||
:rubygems_api_key: YOUR_RUBYGEMS_TOKEN
|
||||
```
|
||||
|
||||
```bash
|
||||
gem push your-gem-1.0.0.gem --host https://registry.example.com/rubygems
|
||||
bundle config set --global https://registry.example.com/rubygems YOUR_RUBYGEMS_TOKEN
|
||||
```
|
||||
|
||||
## Testing and Compatibility
|
||||
|
||||
The repository contains protocol-level tests, cross-protocol integration tests, upstream provider tests, storage hook tests, and native-client test suites that exercise the library through real ecosystem tooling.
|
||||
|
||||
Native-client coverage exists for:
|
||||
|
||||
- Docker / OCI
|
||||
- npm
|
||||
- Maven (`mvn`)
|
||||
- Cargo
|
||||
- Composer
|
||||
- PyPI (`pip` and `twine`)
|
||||
- RubyGems (`gem`)
|
||||
|
||||
There are also integration tests for S3-compatible storage and `IS3Descriptor`-based configuration.
|
||||
|
||||
## License and Legal Information
|
||||
|
||||
This repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository.
|
||||
This repository contains open-source code licensed under the MIT License. A copy of the license can be found in the [LICENSE](./license) file.
|
||||
|
||||
**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.
|
||||
|
||||
### Trademarks
|
||||
|
||||
This project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.
|
||||
This project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH or third parties, and are not included within the scope of the MIT license granted herein.
|
||||
|
||||
Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines or the guidelines of the respective third-party owners, and any usage must be approved in writing. Third-party trademarks used herein are the property of their respective owners and used only in a descriptive manner, e.g. for an implementation of an API or similar.
|
||||
|
||||
### Company Information
|
||||
|
||||
Task Venture Capital GmbH
|
||||
Registered at District court Bremen HRB 35230 HB, Germany
|
||||
Registered at District Court Bremen HRB 35230 HB, Germany
|
||||
|
||||
For any legal inquiries or if you require further information, please contact us via email at hello@task.vc.
|
||||
For any legal inquiries or further information, please contact us via email at hello@task.vc.
|
||||
|
||||
By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
import { tap, expect } from '@git.zone/tstest';
|
||||
import { RegistryStorage } from '../ts/core/classes.registrystorage.js';
|
||||
import { CargoRegistry } from '../ts/cargo/classes.cargoregistry.js';
|
||||
import { AuthManager } from '../ts/core/classes.authmanager.js';
|
||||
import { streamToJson } from '../ts/core/helpers.stream.js';
|
||||
|
||||
// Test index path calculation
|
||||
tap.test('should calculate correct index paths for different crate names', async () => {
|
||||
const storage = new RegistryStorage({
|
||||
accessKey: 'test',
|
||||
accessSecret: 'test',
|
||||
endpoint: 's3.test.com',
|
||||
bucketName: 'test-bucket',
|
||||
});
|
||||
|
||||
// Access private method for testing
|
||||
const getPath = (storage as any).getCargoIndexPath.bind(storage);
|
||||
|
||||
// 1-character names
|
||||
expect(getPath('a')).to.equal('cargo/index/1/a');
|
||||
expect(getPath('z')).to.equal('cargo/index/1/z');
|
||||
|
||||
// 2-character names
|
||||
expect(getPath('io')).to.equal('cargo/index/2/io');
|
||||
expect(getPath('ab')).to.equal('cargo/index/2/ab');
|
||||
|
||||
// 3-character names
|
||||
expect(getPath('axo')).to.equal('cargo/index/3/a/axo');
|
||||
expect(getPath('foo')).to.equal('cargo/index/3/f/foo');
|
||||
|
||||
// 4+ character names
|
||||
expect(getPath('serde')).to.equal('cargo/index/se/rd/serde');
|
||||
expect(getPath('tokio')).to.equal('cargo/index/to/ki/tokio');
|
||||
expect(getPath('my-crate')).to.equal('cargo/index/my/--/my-crate');
|
||||
});
|
||||
|
||||
// Test crate file path calculation
|
||||
tap.test('should calculate correct crate file paths', async () => {
|
||||
const storage = new RegistryStorage({
|
||||
accessKey: 'test',
|
||||
accessSecret: 'test',
|
||||
endpoint: 's3.test.com',
|
||||
bucketName: 'test-bucket',
|
||||
});
|
||||
|
||||
// Access private method for testing
|
||||
const getPath = (storage as any).getCargoCratePath.bind(storage);
|
||||
|
||||
expect(getPath('serde', '1.0.0')).to.equal('cargo/crates/serde/serde-1.0.0.crate');
|
||||
expect(getPath('tokio', '1.28.0')).to.equal('cargo/crates/tokio/tokio-1.28.0.crate');
|
||||
expect(getPath('my-crate', '0.1.0')).to.equal('cargo/crates/my-crate/my-crate-0.1.0.crate');
|
||||
});
|
||||
|
||||
// Test crate name validation
|
||||
tap.test('should validate crate names correctly', async () => {
|
||||
const storage = new RegistryStorage({
|
||||
accessKey: 'test',
|
||||
accessSecret: 'test',
|
||||
endpoint: 's3.test.com',
|
||||
bucketName: 'test-bucket',
|
||||
});
|
||||
|
||||
const authManager = new AuthManager({
|
||||
jwtSecret: 'test-secret',
|
||||
tokenStore: 'memory',
|
||||
npmTokens: { enabled: true },
|
||||
ociTokens: { enabled: false, realm: '', service: '' },
|
||||
});
|
||||
|
||||
const registry = new CargoRegistry(storage, authManager, '/cargo', 'http://localhost:5000/cargo');
|
||||
|
||||
// Access private method for testing
|
||||
const validate = (registry as any).validateCrateName.bind(registry);
|
||||
|
||||
// Valid names
|
||||
expect(validate('serde')).to.be.true;
|
||||
expect(validate('tokio')).to.be.true;
|
||||
expect(validate('my-crate')).to.be.true;
|
||||
expect(validate('my_crate')).to.be.true;
|
||||
expect(validate('crate123')).to.be.true;
|
||||
expect(validate('a')).to.be.true;
|
||||
|
||||
// Invalid names (uppercase not allowed)
|
||||
expect(validate('Serde')).to.be.false;
|
||||
expect(validate('MyCreate')).to.be.false;
|
||||
|
||||
// Invalid names (special characters)
|
||||
expect(validate('my.crate')).to.be.false;
|
||||
expect(validate('my@crate')).to.be.false;
|
||||
expect(validate('my crate')).to.be.false;
|
||||
|
||||
// Invalid names (too long)
|
||||
const longName = 'a'.repeat(65);
|
||||
expect(validate(longName)).to.be.false;
|
||||
|
||||
// Invalid names (empty)
|
||||
expect(validate('')).to.be.false;
|
||||
});
|
||||
|
||||
// Test config.json response
|
||||
tap.test('should return valid config.json', async () => {
|
||||
const storage = new RegistryStorage({
|
||||
accessKey: 'test',
|
||||
accessSecret: 'test',
|
||||
endpoint: 's3.test.com',
|
||||
bucketName: 'test-bucket',
|
||||
});
|
||||
|
||||
const authManager = new AuthManager({
|
||||
jwtSecret: 'test-secret',
|
||||
tokenStore: 'memory',
|
||||
npmTokens: { enabled: true },
|
||||
ociTokens: { enabled: false, realm: '', service: '' },
|
||||
});
|
||||
|
||||
const registry = new CargoRegistry(storage, authManager, '/cargo', 'http://localhost:5000/cargo');
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/cargo/config.json',
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).to.equal(200);
|
||||
expect(response.headers['Content-Type']).to.equal('application/json');
|
||||
const body = await streamToJson(response.body);
|
||||
expect(body).to.be.an('object');
|
||||
expect(body.dl).to.include('/api/v1/crates/{crate}/{version}/download');
|
||||
expect(body.api).to.equal('http://localhost:5000/cargo');
|
||||
});
|
||||
|
||||
export default tap.start();
|
||||
@@ -0,0 +1,420 @@
|
||||
import * as crypto from 'crypto';
|
||||
import * as smartarchive from '@push.rocks/smartarchive';
|
||||
|
||||
/**
|
||||
* Helper to calculate SHA-256 digest in OCI format
|
||||
*/
|
||||
export function calculateDigest(data: Buffer): string {
|
||||
const hash = crypto.createHash('sha256').update(data).digest('hex');
|
||||
return `sha256:${hash}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper to create a minimal valid OCI manifest
|
||||
*/
|
||||
export function createTestManifest(configDigest: string, layerDigest: string) {
|
||||
return {
|
||||
schemaVersion: 2,
|
||||
mediaType: 'application/vnd.oci.image.manifest.v1+json',
|
||||
config: {
|
||||
mediaType: 'application/vnd.oci.image.config.v1+json',
|
||||
size: 123,
|
||||
digest: configDigest,
|
||||
},
|
||||
layers: [
|
||||
{
|
||||
mediaType: 'application/vnd.oci.image.layer.v1.tar+gzip',
|
||||
size: 456,
|
||||
digest: layerDigest,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper to create a minimal valid NPM packument
|
||||
*/
|
||||
export function createTestPackument(packageName: string, version: string, tarballData: Buffer) {
|
||||
const shasum = crypto.createHash('sha1').update(tarballData).digest('hex');
|
||||
const integrity = `sha512-${crypto.createHash('sha512').update(tarballData).digest('base64')}`;
|
||||
|
||||
return {
|
||||
name: packageName,
|
||||
versions: {
|
||||
[version]: {
|
||||
name: packageName,
|
||||
version: version,
|
||||
description: 'Test package',
|
||||
main: 'index.js',
|
||||
scripts: {},
|
||||
dist: {
|
||||
shasum: shasum,
|
||||
integrity: integrity,
|
||||
tarball: `http://localhost:5000/npm/${packageName}/-/${packageName}-${version}.tgz`,
|
||||
},
|
||||
},
|
||||
},
|
||||
'dist-tags': {
|
||||
latest: version,
|
||||
},
|
||||
_attachments: {
|
||||
[`${packageName}-${version}.tgz`]: {
|
||||
content_type: 'application/octet-stream',
|
||||
data: tarballData.toString('base64'),
|
||||
length: tarballData.length,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper to create a minimal valid Maven POM file
|
||||
*/
|
||||
export function createTestPom(
|
||||
groupId: string,
|
||||
artifactId: string,
|
||||
version: string,
|
||||
packaging: string = 'jar'
|
||||
): string {
|
||||
return `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
|
||||
http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>${groupId}</groupId>
|
||||
<artifactId>${artifactId}</artifactId>
|
||||
<version>${version}</version>
|
||||
<packaging>${packaging}</packaging>
|
||||
<name>${artifactId}</name>
|
||||
<description>Test Maven artifact</description>
|
||||
</project>`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper to create a test JAR file (minimal ZIP with manifest)
|
||||
*/
|
||||
export function createTestJar(): Buffer {
|
||||
const manifestContent = `Manifest-Version: 1.0
|
||||
Created-By: SmartRegistry Test
|
||||
`;
|
||||
|
||||
return Buffer.from(manifestContent, 'utf-8');
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper to calculate Maven checksums
|
||||
*/
|
||||
export function calculateMavenChecksums(data: Buffer) {
|
||||
return {
|
||||
md5: crypto.createHash('md5').update(data).digest('hex'),
|
||||
sha1: crypto.createHash('sha1').update(data).digest('hex'),
|
||||
sha256: crypto.createHash('sha256').update(data).digest('hex'),
|
||||
sha512: crypto.createHash('sha512').update(data).digest('hex'),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper to create a Composer package ZIP using smartarchive
|
||||
*/
|
||||
export async function createComposerZip(
|
||||
vendorPackage: string,
|
||||
version: string,
|
||||
options?: {
|
||||
description?: string;
|
||||
license?: string[];
|
||||
authors?: Array<{ name: string; email?: string }>;
|
||||
}
|
||||
): Promise<Buffer> {
|
||||
const zipTools = new smartarchive.ZipTools();
|
||||
|
||||
const composerJson = {
|
||||
name: vendorPackage,
|
||||
version: version,
|
||||
type: 'library',
|
||||
description: options?.description || 'Test Composer package',
|
||||
license: options?.license || ['MIT'],
|
||||
authors: options?.authors || [{ name: 'Test Author', email: 'test@example.com' }],
|
||||
require: {
|
||||
php: '>=7.4',
|
||||
},
|
||||
autoload: {
|
||||
'psr-4': {
|
||||
'Vendor\\TestPackage\\': 'src/',
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const [vendor, pkg] = vendorPackage.split('/');
|
||||
const namespace = `${vendor.charAt(0).toUpperCase() + vendor.slice(1)}\\${pkg.charAt(0).toUpperCase() + pkg.slice(1).replace(/-/g, '')}`;
|
||||
const testPhpContent = `<?php
|
||||
namespace ${namespace};
|
||||
|
||||
class TestClass
|
||||
{
|
||||
public function greet(): string
|
||||
{
|
||||
return "Hello from ${vendorPackage}!";
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
const entries: smartarchive.IArchiveEntry[] = [
|
||||
{
|
||||
archivePath: 'composer.json',
|
||||
content: Buffer.from(JSON.stringify(composerJson, null, 2), 'utf-8'),
|
||||
},
|
||||
{
|
||||
archivePath: 'src/TestClass.php',
|
||||
content: Buffer.from(testPhpContent, 'utf-8'),
|
||||
},
|
||||
{
|
||||
archivePath: 'README.md',
|
||||
content: Buffer.from(`# ${vendorPackage}\n\nTest package`, 'utf-8'),
|
||||
},
|
||||
];
|
||||
|
||||
return Buffer.from(await zipTools.createZip(entries));
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper to create a test Python wheel file (minimal ZIP structure) using smartarchive
|
||||
*/
|
||||
export async function createPythonWheel(
|
||||
packageName: string,
|
||||
version: string,
|
||||
pyVersion: string = 'py3'
|
||||
): Promise<Buffer> {
|
||||
const zipTools = new smartarchive.ZipTools();
|
||||
|
||||
const normalizedName = packageName.replace(/-/g, '_');
|
||||
const distInfoDir = `${normalizedName}-${version}.dist-info`;
|
||||
|
||||
const metadata = `Metadata-Version: 2.1
|
||||
Name: ${packageName}
|
||||
Version: ${version}
|
||||
Summary: Test Python package
|
||||
Home-page: https://example.com
|
||||
Author: Test Author
|
||||
Author-email: test@example.com
|
||||
License: MIT
|
||||
Platform: UNKNOWN
|
||||
Classifier: Programming Language :: Python :: 3
|
||||
Requires-Python: >=3.7
|
||||
Description-Content-Type: text/markdown
|
||||
|
||||
# ${packageName}
|
||||
|
||||
Test package for SmartRegistry
|
||||
`;
|
||||
|
||||
const wheelContent = `Wheel-Version: 1.0
|
||||
Generator: test 1.0.0
|
||||
Root-Is-Purelib: true
|
||||
Tag: ${pyVersion}-none-any
|
||||
`;
|
||||
|
||||
const moduleContent = `"""${packageName} module"""
|
||||
|
||||
__version__ = "${version}"
|
||||
|
||||
def hello():
|
||||
return "Hello from ${packageName}!"
|
||||
`;
|
||||
|
||||
const entries: smartarchive.IArchiveEntry[] = [
|
||||
{
|
||||
archivePath: `${distInfoDir}/METADATA`,
|
||||
content: Buffer.from(metadata, 'utf-8'),
|
||||
},
|
||||
{
|
||||
archivePath: `${distInfoDir}/WHEEL`,
|
||||
content: Buffer.from(wheelContent, 'utf-8'),
|
||||
},
|
||||
{
|
||||
archivePath: `${distInfoDir}/RECORD`,
|
||||
content: Buffer.from('', 'utf-8'),
|
||||
},
|
||||
{
|
||||
archivePath: `${distInfoDir}/top_level.txt`,
|
||||
content: Buffer.from(normalizedName, 'utf-8'),
|
||||
},
|
||||
{
|
||||
archivePath: `${normalizedName}/__init__.py`,
|
||||
content: Buffer.from(moduleContent, 'utf-8'),
|
||||
},
|
||||
];
|
||||
|
||||
return Buffer.from(await zipTools.createZip(entries));
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper to create a test Python source distribution (sdist) using smartarchive
|
||||
*/
|
||||
export async function createPythonSdist(
|
||||
packageName: string,
|
||||
version: string
|
||||
): Promise<Buffer> {
|
||||
const tarTools = new smartarchive.TarTools();
|
||||
|
||||
const normalizedName = packageName.replace(/-/g, '_');
|
||||
const dirPrefix = `${packageName}-${version}`;
|
||||
|
||||
const pkgInfo = `Metadata-Version: 2.1
|
||||
Name: ${packageName}
|
||||
Version: ${version}
|
||||
Summary: Test Python package
|
||||
Home-page: https://example.com
|
||||
Author: Test Author
|
||||
Author-email: test@example.com
|
||||
License: MIT
|
||||
`;
|
||||
|
||||
const setupPy = `from setuptools import setup, find_packages
|
||||
|
||||
setup(
|
||||
name="${packageName}",
|
||||
version="${version}",
|
||||
packages=find_packages(),
|
||||
python_requires=">=3.7",
|
||||
)
|
||||
`;
|
||||
|
||||
const moduleContent = `"""${packageName} module"""
|
||||
|
||||
__version__ = "${version}"
|
||||
|
||||
def hello():
|
||||
return "Hello from ${packageName}!"
|
||||
`;
|
||||
|
||||
const entries: smartarchive.IArchiveEntry[] = [
|
||||
{
|
||||
archivePath: `${dirPrefix}/PKG-INFO`,
|
||||
content: Buffer.from(pkgInfo, 'utf-8'),
|
||||
},
|
||||
{
|
||||
archivePath: `${dirPrefix}/setup.py`,
|
||||
content: Buffer.from(setupPy, 'utf-8'),
|
||||
},
|
||||
{
|
||||
archivePath: `${dirPrefix}/${normalizedName}/__init__.py`,
|
||||
content: Buffer.from(moduleContent, 'utf-8'),
|
||||
},
|
||||
];
|
||||
|
||||
return Buffer.from(await tarTools.packFilesToTarGz(entries));
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper to calculate PyPI file hashes
|
||||
*/
|
||||
export function calculatePypiHashes(data: Buffer) {
|
||||
return {
|
||||
md5: crypto.createHash('md5').update(data).digest('hex'),
|
||||
sha256: crypto.createHash('sha256').update(data).digest('hex'),
|
||||
blake2b: crypto.createHash('blake2b512').update(data).digest('hex'),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper to create a test RubyGem file (minimal tar.gz structure) using smartarchive
|
||||
*/
|
||||
export async function createRubyGem(
|
||||
gemName: string,
|
||||
version: string,
|
||||
platform: string = 'ruby'
|
||||
): Promise<Buffer> {
|
||||
const tarTools = new smartarchive.TarTools();
|
||||
const gzipTools = new smartarchive.GzipTools();
|
||||
|
||||
const metadataYaml = `--- !ruby/object:Gem::Specification
|
||||
name: ${gemName}
|
||||
version: !ruby/object:Gem::Version
|
||||
version: ${version}
|
||||
platform: ${platform}
|
||||
authors:
|
||||
- Test Author
|
||||
autorequire:
|
||||
bindir: bin
|
||||
cert_chain: []
|
||||
date: ${new Date().toISOString().split('T')[0]}
|
||||
dependencies: []
|
||||
description: Test RubyGem
|
||||
email: test@example.com
|
||||
executables: []
|
||||
extensions: []
|
||||
extra_rdoc_files: []
|
||||
files:
|
||||
- lib/${gemName}.rb
|
||||
homepage: https://example.com
|
||||
licenses:
|
||||
- MIT
|
||||
metadata: {}
|
||||
post_install_message:
|
||||
rdoc_options: []
|
||||
require_paths:
|
||||
- lib
|
||||
required_ruby_version: !ruby/object:Gem::Requirement
|
||||
requirements:
|
||||
- - ">="
|
||||
- !ruby/object:Gem::Version
|
||||
version: '2.7'
|
||||
required_rubygems_version: !ruby/object:Gem::Requirement
|
||||
requirements:
|
||||
- - ">="
|
||||
- !ruby/object:Gem::Version
|
||||
version: '0'
|
||||
requirements: []
|
||||
rubygems_version: 3.0.0
|
||||
signing_key:
|
||||
specification_version: 4
|
||||
summary: Test gem for SmartRegistry
|
||||
test_files: []
|
||||
`;
|
||||
|
||||
const metadataGz = Buffer.from(await gzipTools.compress(Buffer.from(metadataYaml, 'utf-8')));
|
||||
|
||||
const libContent = `# ${gemName}
|
||||
|
||||
module ${gemName.charAt(0).toUpperCase() + gemName.slice(1).replace(/-/g, '')}
|
||||
VERSION = "${version}"
|
||||
|
||||
def self.hello
|
||||
"Hello from #{gemName}!"
|
||||
end
|
||||
end
|
||||
`;
|
||||
|
||||
const dataEntries: smartarchive.IArchiveEntry[] = [
|
||||
{
|
||||
archivePath: `lib/${gemName}.rb`,
|
||||
content: Buffer.from(libContent, 'utf-8'),
|
||||
},
|
||||
];
|
||||
|
||||
const dataTarGz = Buffer.from(await tarTools.packFilesToTarGz(dataEntries));
|
||||
|
||||
const gemEntries: smartarchive.IArchiveEntry[] = [
|
||||
{
|
||||
archivePath: 'metadata.gz',
|
||||
content: metadataGz,
|
||||
},
|
||||
{
|
||||
archivePath: 'data.tar.gz',
|
||||
content: dataTarGz,
|
||||
},
|
||||
];
|
||||
|
||||
return Buffer.from(await tarTools.packFiles(gemEntries));
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper to calculate RubyGems checksums
|
||||
*/
|
||||
export function calculateRubyGemsChecksums(data: Buffer) {
|
||||
return {
|
||||
md5: crypto.createHash('md5').update(data).digest('hex'),
|
||||
sha256: crypto.createHash('sha256').update(data).digest('hex'),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
/**
|
||||
* Generate a unique test run ID for avoiding conflicts between test runs.
|
||||
*/
|
||||
export function generateTestRunId(): string {
|
||||
const timestamp = Date.now().toString(36);
|
||||
const random = Math.random().toString(36).substring(2, 6);
|
||||
return `${timestamp}${random}`;
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
import type { IAuthToken, TRegistryProtocol } from '../../ts/core/interfaces.core.js';
|
||||
import type { IAuthProvider } from '../../ts/core/interfaces.auth.js';
|
||||
import type {
|
||||
IUpstreamProvider,
|
||||
IUpstreamRegistryConfig,
|
||||
IUpstreamResolutionContext,
|
||||
IProtocolUpstreamConfig,
|
||||
} from '../../ts/upstream/interfaces.upstream.js';
|
||||
|
||||
type TTestUpstreamRegistryConfig = Omit<Partial<IUpstreamRegistryConfig>, 'id' | 'url' | 'priority' | 'enabled'> &
|
||||
Pick<IUpstreamRegistryConfig, 'id' | 'url' | 'priority' | 'enabled'>;
|
||||
|
||||
type TTestProtocolUpstreamConfig = Omit<IProtocolUpstreamConfig, 'upstreams'> & {
|
||||
upstreams: TTestUpstreamRegistryConfig[];
|
||||
};
|
||||
|
||||
function normalizeUpstreamRegistryConfig(
|
||||
upstream: TTestUpstreamRegistryConfig
|
||||
): IUpstreamRegistryConfig {
|
||||
return {
|
||||
...upstream,
|
||||
name: upstream.name ?? upstream.id,
|
||||
auth: upstream.auth ?? { type: 'none' },
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeProtocolUpstreamConfig(
|
||||
config: TTestProtocolUpstreamConfig | undefined
|
||||
): IProtocolUpstreamConfig | null {
|
||||
if (!config) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
...config,
|
||||
upstreams: config.upstreams.map(normalizeUpstreamRegistryConfig),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a mock upstream provider that tracks all calls for testing
|
||||
*/
|
||||
export function createTrackingUpstreamProvider(
|
||||
baseConfig?: Partial<Record<TRegistryProtocol, TTestProtocolUpstreamConfig>>
|
||||
): {
|
||||
provider: IUpstreamProvider;
|
||||
calls: IUpstreamResolutionContext[];
|
||||
} {
|
||||
const calls: IUpstreamResolutionContext[] = [];
|
||||
|
||||
const provider: IUpstreamProvider = {
|
||||
async resolveUpstreamConfig(context: IUpstreamResolutionContext) {
|
||||
calls.push({ ...context });
|
||||
return normalizeProtocolUpstreamConfig(baseConfig?.[context.protocol]);
|
||||
},
|
||||
};
|
||||
|
||||
return { provider, calls };
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a mock auth provider for testing pluggable authentication.
|
||||
* Allows customizing behavior for different test scenarios.
|
||||
*/
|
||||
export function createMockAuthProvider(overrides?: Partial<IAuthProvider>): IAuthProvider {
|
||||
const tokens = new Map<string, IAuthToken>();
|
||||
|
||||
return {
|
||||
init: async () => {},
|
||||
authenticate: async (credentials) => {
|
||||
return credentials.username;
|
||||
},
|
||||
validateToken: async (token, protocol) => {
|
||||
const stored = tokens.get(token);
|
||||
if (stored && (!protocol || stored.type === protocol)) {
|
||||
return stored;
|
||||
}
|
||||
if (token === 'valid-mock-token') {
|
||||
return {
|
||||
type: 'npm' as TRegistryProtocol,
|
||||
userId: 'mock-user',
|
||||
scopes: ['npm:*:*:*'],
|
||||
};
|
||||
}
|
||||
return null;
|
||||
},
|
||||
createToken: async (userId, protocol, options) => {
|
||||
const tokenId = `mock-${protocol}-${Date.now()}`;
|
||||
const authToken: IAuthToken = {
|
||||
type: protocol,
|
||||
userId,
|
||||
scopes: options?.scopes || [`${protocol}:*:*:*`],
|
||||
readonly: options?.readonly,
|
||||
expiresAt: options?.expiresIn ? new Date(Date.now() + options.expiresIn * 1000) : undefined,
|
||||
};
|
||||
tokens.set(tokenId, authToken);
|
||||
return tokenId;
|
||||
},
|
||||
revokeToken: async (token) => {
|
||||
tokens.delete(token);
|
||||
},
|
||||
authorize: async (token, resource, action) => {
|
||||
if (!token) return false;
|
||||
if (token.readonly && ['write', 'push', 'delete'].includes(action)) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
},
|
||||
listUserTokens: async (userId) => {
|
||||
const result: Array<{ key: string; readonly: boolean; created: string; protocol?: TRegistryProtocol }> = [];
|
||||
for (const [key, token] of tokens.entries()) {
|
||||
if (token.userId === userId) {
|
||||
result.push({
|
||||
key: `hash-${key.substring(0, 8)}`,
|
||||
readonly: token.readonly || false,
|
||||
created: new Date().toISOString(),
|
||||
protocol: token.type,
|
||||
});
|
||||
}
|
||||
}
|
||||
return result;
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
+68
-121
@@ -1,51 +1,77 @@
|
||||
import * as qenv from '@push.rocks/qenv';
|
||||
import * as crypto from 'crypto';
|
||||
import * as smartbucket from '@push.rocks/smartbucket';
|
||||
import { SmartRegistry } from '../../ts/classes.smartregistry.js';
|
||||
import type { IRegistryConfig } from '../../ts/core/interfaces.core.js';
|
||||
import type { IStorageHooks } from '../../ts/core/interfaces.storage.js';
|
||||
import type { IUpstreamProvider } from '../../ts/upstream/interfaces.upstream.js';
|
||||
import { buildTestRegistryConfig, createDefaultTestUpstreamProvider } from './registryconfig.js';
|
||||
import { generateTestRunId } from './ids.js';
|
||||
|
||||
export {
|
||||
calculateDigest,
|
||||
createTestManifest,
|
||||
createTestPackument,
|
||||
createTestPom,
|
||||
createTestJar,
|
||||
calculateMavenChecksums,
|
||||
createComposerZip,
|
||||
createPythonWheel,
|
||||
createPythonSdist,
|
||||
calculatePypiHashes,
|
||||
createRubyGem,
|
||||
calculateRubyGemsChecksums,
|
||||
} from './fixtures.js';
|
||||
export { createMockAuthProvider, createTrackingUpstreamProvider } from './providers.js';
|
||||
export { buildTestRegistryConfig, createDefaultTestUpstreamProvider } from './registryconfig.js';
|
||||
export { createTestStorageBackend } from './storagebackend.js';
|
||||
export { generateTestRunId } from './ids.js';
|
||||
export { createTestTokens } from './tokens.js';
|
||||
export { createTrackingHooks, createQuotaHooks } from './storagehooks.js';
|
||||
|
||||
const testQenv = new qenv.Qenv('./', './.nogit');
|
||||
|
||||
/**
|
||||
* Create a test SmartRegistry instance with both OCI and NPM enabled
|
||||
*/
|
||||
export async function createTestRegistry(): Promise<SmartRegistry> {
|
||||
// Read S3 config from env.json
|
||||
export async function cleanupS3Bucket(prefix?: string): Promise<void> {
|
||||
const s3AccessKey = await testQenv.getEnvVarOnDemand('S3_ACCESSKEY');
|
||||
const s3SecretKey = await testQenv.getEnvVarOnDemand('S3_SECRETKEY');
|
||||
const s3Endpoint = await testQenv.getEnvVarOnDemand('S3_ENDPOINT');
|
||||
const s3Port = await testQenv.getEnvVarOnDemand('S3_PORT');
|
||||
|
||||
const config: IRegistryConfig = {
|
||||
storage: {
|
||||
const s3 = new smartbucket.SmartBucket({
|
||||
accessKey: s3AccessKey || 'minioadmin',
|
||||
accessSecret: s3SecretKey || 'minioadmin',
|
||||
endpoint: s3Endpoint || 'localhost',
|
||||
port: parseInt(s3Port || '9000', 10),
|
||||
useSsl: false,
|
||||
region: 'us-east-1',
|
||||
bucketName: 'test-registry',
|
||||
},
|
||||
auth: {
|
||||
jwtSecret: 'test-secret-key',
|
||||
tokenStore: 'memory',
|
||||
npmTokens: {
|
||||
enabled: true,
|
||||
},
|
||||
ociTokens: {
|
||||
enabled: true,
|
||||
realm: 'https://auth.example.com/token',
|
||||
service: 'test-registry',
|
||||
},
|
||||
},
|
||||
oci: {
|
||||
enabled: true,
|
||||
basePath: '/oci',
|
||||
},
|
||||
npm: {
|
||||
enabled: true,
|
||||
basePath: '/npm',
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
try {
|
||||
const bucket = await s3.getBucketByName('test-registry');
|
||||
if (bucket) {
|
||||
if (prefix) {
|
||||
// Delete only objects with the given prefix
|
||||
for await (const path of bucket.listAllObjects(prefix)) {
|
||||
await bucket.fastRemove({ path });
|
||||
}
|
||||
} else {
|
||||
// Delete all objects in the bucket
|
||||
for await (const path of bucket.listAllObjects()) {
|
||||
await bucket.fastRemove({ path });
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
// Bucket might not exist yet, that's fine
|
||||
console.log('Cleanup: No bucket to clean or error:', error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a test SmartRegistry instance with all protocols enabled
|
||||
*/
|
||||
export async function createTestRegistry(options?: {
|
||||
registryUrl?: string;
|
||||
storageHooks?: IStorageHooks;
|
||||
}): Promise<SmartRegistry> {
|
||||
const config = await buildTestRegistryConfig(options);
|
||||
|
||||
const registry = new SmartRegistry(config);
|
||||
await registry.init();
|
||||
@@ -54,96 +80,17 @@ export async function createTestRegistry(): Promise<SmartRegistry> {
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper to create test authentication tokens
|
||||
* Create a test SmartRegistry instance with upstream provider configured
|
||||
*/
|
||||
export async function createTestTokens(registry: SmartRegistry) {
|
||||
const authManager = registry.getAuthManager();
|
||||
|
||||
// Authenticate and create tokens
|
||||
const userId = await authManager.authenticate({
|
||||
username: 'testuser',
|
||||
password: 'testpass',
|
||||
export async function createTestRegistryWithUpstream(
|
||||
upstreamProvider?: IUpstreamProvider
|
||||
): Promise<SmartRegistry> {
|
||||
const config = await buildTestRegistryConfig({
|
||||
upstreamProvider: upstreamProvider || createDefaultTestUpstreamProvider(),
|
||||
});
|
||||
|
||||
if (!userId) {
|
||||
throw new Error('Failed to authenticate test user');
|
||||
}
|
||||
const registry = new SmartRegistry(config);
|
||||
await registry.init();
|
||||
|
||||
// Create NPM token
|
||||
const npmToken = await authManager.createNpmToken(userId, false);
|
||||
|
||||
// Create OCI token with full access
|
||||
const ociToken = await authManager.createOciToken(
|
||||
userId,
|
||||
['oci:repository:*:*'],
|
||||
3600
|
||||
);
|
||||
|
||||
return { npmToken, ociToken, userId };
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper to calculate SHA-256 digest in OCI format
|
||||
*/
|
||||
export function calculateDigest(data: Buffer): string {
|
||||
const hash = crypto.createHash('sha256').update(data).digest('hex');
|
||||
return `sha256:${hash}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper to create a minimal valid OCI manifest
|
||||
*/
|
||||
export function createTestManifest(configDigest: string, layerDigest: string) {
|
||||
return {
|
||||
schemaVersion: 2,
|
||||
mediaType: 'application/vnd.oci.image.manifest.v1+json',
|
||||
config: {
|
||||
mediaType: 'application/vnd.oci.image.config.v1+json',
|
||||
size: 123,
|
||||
digest: configDigest,
|
||||
},
|
||||
layers: [
|
||||
{
|
||||
mediaType: 'application/vnd.oci.image.layer.v1.tar+gzip',
|
||||
size: 456,
|
||||
digest: layerDigest,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper to create a minimal valid NPM packument
|
||||
*/
|
||||
export function createTestPackument(packageName: string, version: string, tarballData: Buffer) {
|
||||
const shasum = crypto.createHash('sha1').update(tarballData).digest('hex');
|
||||
const integrity = `sha512-${crypto.createHash('sha512').update(tarballData).digest('base64')}`;
|
||||
|
||||
return {
|
||||
name: packageName,
|
||||
versions: {
|
||||
[version]: {
|
||||
name: packageName,
|
||||
version: version,
|
||||
description: 'Test package',
|
||||
main: 'index.js',
|
||||
scripts: {},
|
||||
dist: {
|
||||
shasum: shasum,
|
||||
integrity: integrity,
|
||||
tarball: `http://localhost:5000/npm/${packageName}/-/${packageName}-${version}.tgz`,
|
||||
},
|
||||
},
|
||||
},
|
||||
'dist-tags': {
|
||||
latest: version,
|
||||
},
|
||||
_attachments: {
|
||||
[`${packageName}-${version}.tgz`]: {
|
||||
content_type: 'application/octet-stream',
|
||||
data: tarballData.toString('base64'),
|
||||
length: tarballData.length,
|
||||
},
|
||||
},
|
||||
};
|
||||
return registry;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
import * as qenv from '@push.rocks/qenv';
|
||||
import type { IRegistryConfig } from '../../ts/core/interfaces.core.js';
|
||||
import type { IStorageHooks } from '../../ts/core/interfaces.storage.js';
|
||||
import { StaticUpstreamProvider } from '../../ts/upstream/interfaces.upstream.js';
|
||||
import type { IUpstreamProvider } from '../../ts/upstream/interfaces.upstream.js';
|
||||
|
||||
const testQenv = new qenv.Qenv('./', './.nogit');
|
||||
|
||||
async function getTestStorageConfig(): Promise<IRegistryConfig['storage']> {
|
||||
const s3AccessKey = await testQenv.getEnvVarOnDemand('S3_ACCESSKEY');
|
||||
const s3SecretKey = await testQenv.getEnvVarOnDemand('S3_SECRETKEY');
|
||||
const s3Endpoint = await testQenv.getEnvVarOnDemand('S3_ENDPOINT');
|
||||
const s3Port = await testQenv.getEnvVarOnDemand('S3_PORT');
|
||||
|
||||
return {
|
||||
accessKey: s3AccessKey || 'minioadmin',
|
||||
accessSecret: s3SecretKey || 'minioadmin',
|
||||
endpoint: s3Endpoint || 'localhost',
|
||||
port: parseInt(s3Port || '9000', 10),
|
||||
useSsl: false,
|
||||
region: 'us-east-1',
|
||||
bucketName: 'test-registry',
|
||||
};
|
||||
}
|
||||
|
||||
function getTestAuthConfig(): IRegistryConfig['auth'] {
|
||||
return {
|
||||
jwtSecret: 'test-secret-key',
|
||||
tokenStore: 'memory',
|
||||
npmTokens: {
|
||||
enabled: true,
|
||||
},
|
||||
ociTokens: {
|
||||
enabled: true,
|
||||
realm: 'https://auth.example.com/token',
|
||||
service: 'test-registry',
|
||||
},
|
||||
pypiTokens: {
|
||||
enabled: true,
|
||||
},
|
||||
rubygemsTokens: {
|
||||
enabled: true,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function createDefaultTestUpstreamProvider(): IUpstreamProvider {
|
||||
return new StaticUpstreamProvider({
|
||||
npm: {
|
||||
enabled: true,
|
||||
upstreams: [{
|
||||
id: 'npmjs',
|
||||
name: 'npmjs',
|
||||
url: 'https://registry.npmjs.org',
|
||||
priority: 1,
|
||||
enabled: true,
|
||||
auth: { type: 'none' },
|
||||
}],
|
||||
},
|
||||
oci: {
|
||||
enabled: true,
|
||||
upstreams: [{
|
||||
id: 'dockerhub',
|
||||
name: 'dockerhub',
|
||||
url: 'https://registry-1.docker.io',
|
||||
priority: 1,
|
||||
enabled: true,
|
||||
auth: { type: 'none' },
|
||||
}],
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export async function buildTestRegistryConfig(options?: {
|
||||
registryUrl?: string;
|
||||
storageHooks?: IStorageHooks;
|
||||
upstreamProvider?: IUpstreamProvider;
|
||||
}): Promise<IRegistryConfig> {
|
||||
const config: IRegistryConfig = {
|
||||
storage: await getTestStorageConfig(),
|
||||
auth: getTestAuthConfig(),
|
||||
...(options?.storageHooks ? { storageHooks: options.storageHooks } : {}),
|
||||
...(options?.upstreamProvider ? { upstreamProvider: options.upstreamProvider } : {}),
|
||||
oci: {
|
||||
enabled: true,
|
||||
basePath: '/oci',
|
||||
...(options?.registryUrl ? { registryUrl: `${options.registryUrl}/oci` } : {}),
|
||||
},
|
||||
npm: {
|
||||
enabled: true,
|
||||
basePath: '/npm',
|
||||
...(options?.registryUrl ? { registryUrl: `${options.registryUrl}/npm` } : {}),
|
||||
},
|
||||
maven: {
|
||||
enabled: true,
|
||||
basePath: '/maven',
|
||||
...(options?.registryUrl ? { registryUrl: `${options.registryUrl}/maven` } : {}),
|
||||
},
|
||||
composer: {
|
||||
enabled: true,
|
||||
basePath: '/composer',
|
||||
...(options?.registryUrl ? { registryUrl: `${options.registryUrl}/composer` } : {}),
|
||||
},
|
||||
cargo: {
|
||||
enabled: true,
|
||||
basePath: '/cargo',
|
||||
...(options?.registryUrl ? { registryUrl: `${options.registryUrl}/cargo` } : {}),
|
||||
},
|
||||
pypi: {
|
||||
enabled: true,
|
||||
basePath: '/pypi',
|
||||
...(options?.registryUrl ? { registryUrl: options.registryUrl } : {}),
|
||||
},
|
||||
rubygems: {
|
||||
enabled: true,
|
||||
basePath: '/rubygems',
|
||||
...(options?.registryUrl ? { registryUrl: `${options.registryUrl}/rubygems` } : {}),
|
||||
},
|
||||
};
|
||||
|
||||
return config;
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
import * as qenv from '@push.rocks/qenv';
|
||||
import * as smartbucket from '@push.rocks/smartbucket';
|
||||
import { generateTestRunId } from './ids.js';
|
||||
|
||||
const testQenv = new qenv.Qenv('./', './.nogit');
|
||||
|
||||
/**
|
||||
* Create a SmartBucket storage backend for upstream cache testing.
|
||||
*/
|
||||
export async function createTestStorageBackend(): Promise<{
|
||||
storage: {
|
||||
getObject: (key: string) => Promise<Buffer | null>;
|
||||
putObject: (key: string, data: Buffer) => Promise<void>;
|
||||
deleteObject: (key: string) => Promise<void>;
|
||||
listObjects: (prefix: string) => Promise<string[]>;
|
||||
};
|
||||
bucket: smartbucket.Bucket;
|
||||
cleanup: () => Promise<void>;
|
||||
}> {
|
||||
const s3AccessKey = await testQenv.getEnvVarOnDemand('S3_ACCESSKEY');
|
||||
const s3SecretKey = await testQenv.getEnvVarOnDemand('S3_SECRETKEY');
|
||||
const s3Endpoint = await testQenv.getEnvVarOnDemand('S3_ENDPOINT');
|
||||
const s3Port = await testQenv.getEnvVarOnDemand('S3_PORT');
|
||||
|
||||
const s3 = new smartbucket.SmartBucket({
|
||||
accessKey: s3AccessKey || 'minioadmin',
|
||||
accessSecret: s3SecretKey || 'minioadmin',
|
||||
endpoint: s3Endpoint || 'localhost',
|
||||
port: parseInt(s3Port || '9000', 10),
|
||||
useSsl: false,
|
||||
});
|
||||
|
||||
const testRunId = generateTestRunId();
|
||||
const bucketName = 'test-cache-' + testRunId.substring(0, 8);
|
||||
const bucket = await s3.createBucket(bucketName);
|
||||
|
||||
const storage = {
|
||||
getObject: async (key: string): Promise<Buffer | null> => {
|
||||
try {
|
||||
return await bucket.fastGet({ path: key });
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
},
|
||||
putObject: async (key: string, data: Buffer): Promise<void> => {
|
||||
await bucket.fastPut({ path: key, contents: data, overwrite: true });
|
||||
},
|
||||
deleteObject: async (key: string): Promise<void> => {
|
||||
await bucket.fastRemove({ path: key });
|
||||
},
|
||||
listObjects: async (prefix: string): Promise<string[]> => {
|
||||
const paths: string[] = [];
|
||||
for await (const path of bucket.listAllObjects(prefix)) {
|
||||
paths.push(path);
|
||||
}
|
||||
return paths;
|
||||
},
|
||||
};
|
||||
|
||||
const cleanup = async () => {
|
||||
try {
|
||||
for await (const path of bucket.listAllObjects()) {
|
||||
await bucket.fastRemove({ path });
|
||||
}
|
||||
await s3.removeBucket(bucketName);
|
||||
} catch {
|
||||
// Ignore cleanup errors
|
||||
}
|
||||
};
|
||||
|
||||
return { storage, bucket, cleanup };
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
import type { IStorageHooks, IStorageHookContext } from '../../ts/core/interfaces.storage.js';
|
||||
|
||||
/**
|
||||
* Create test storage hooks that track all calls.
|
||||
* Useful for verifying hook invocation order and parameters.
|
||||
*/
|
||||
export function createTrackingHooks(options?: {
|
||||
beforePutAllowed?: boolean;
|
||||
beforeDeleteAllowed?: boolean;
|
||||
throwOnAfterPut?: boolean;
|
||||
throwOnAfterGet?: boolean;
|
||||
}): {
|
||||
hooks: IStorageHooks;
|
||||
calls: Array<{ method: string; context: IStorageHookContext; timestamp: number }>;
|
||||
} {
|
||||
const calls: Array<{ method: string; context: IStorageHookContext; timestamp: number }> = [];
|
||||
|
||||
return {
|
||||
calls,
|
||||
hooks: {
|
||||
beforePut: async (ctx) => {
|
||||
calls.push({ method: 'beforePut', context: ctx, timestamp: Date.now() });
|
||||
return {
|
||||
allowed: options?.beforePutAllowed !== false,
|
||||
reason: options?.beforePutAllowed === false ? 'Blocked by test' : undefined,
|
||||
};
|
||||
},
|
||||
afterPut: async (ctx) => {
|
||||
calls.push({ method: 'afterPut', context: ctx, timestamp: Date.now() });
|
||||
if (options?.throwOnAfterPut) {
|
||||
throw new Error('Test error in afterPut');
|
||||
}
|
||||
},
|
||||
beforeDelete: async (ctx) => {
|
||||
calls.push({ method: 'beforeDelete', context: ctx, timestamp: Date.now() });
|
||||
return {
|
||||
allowed: options?.beforeDeleteAllowed !== false,
|
||||
reason: options?.beforeDeleteAllowed === false ? 'Blocked by test' : undefined,
|
||||
};
|
||||
},
|
||||
afterDelete: async (ctx) => {
|
||||
calls.push({ method: 'afterDelete', context: ctx, timestamp: Date.now() });
|
||||
},
|
||||
afterGet: async (ctx) => {
|
||||
calls.push({ method: 'afterGet', context: ctx, timestamp: Date.now() });
|
||||
if (options?.throwOnAfterGet) {
|
||||
throw new Error('Test error in afterGet');
|
||||
}
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a blocking storage hooks implementation for quota testing.
|
||||
*/
|
||||
export function createQuotaHooks(maxSizeBytes: number): {
|
||||
hooks: IStorageHooks;
|
||||
currentUsage: { bytes: number };
|
||||
} {
|
||||
const currentUsage = { bytes: 0 };
|
||||
|
||||
return {
|
||||
currentUsage,
|
||||
hooks: {
|
||||
beforePut: async (ctx) => {
|
||||
const size = ctx.metadata?.size || 0;
|
||||
if (currentUsage.bytes + size > maxSizeBytes) {
|
||||
return { allowed: false, reason: `Quota exceeded: ${currentUsage.bytes + size} > ${maxSizeBytes}` };
|
||||
}
|
||||
return { allowed: true };
|
||||
},
|
||||
afterPut: async (ctx) => {
|
||||
currentUsage.bytes += ctx.metadata?.size || 0;
|
||||
},
|
||||
afterDelete: async (ctx) => {
|
||||
currentUsage.bytes -= ctx.metadata?.size || 0;
|
||||
if (currentUsage.bytes < 0) currentUsage.bytes = 0;
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
import type { SmartRegistry } from '../../ts/classes.smartregistry.js';
|
||||
|
||||
/**
|
||||
* Helper to create test authentication tokens.
|
||||
*/
|
||||
export async function createTestTokens(registry: SmartRegistry) {
|
||||
const authManager = registry.getAuthManager();
|
||||
|
||||
const userId = await authManager.authenticate({
|
||||
username: 'testuser',
|
||||
password: 'testpass',
|
||||
});
|
||||
|
||||
if (!userId) {
|
||||
throw new Error('Failed to authenticate test user');
|
||||
}
|
||||
|
||||
const npmToken = await authManager.createNpmToken(userId, false);
|
||||
const ociToken = await authManager.createOciToken(userId, ['oci:repository:*:*'], 3600);
|
||||
const mavenToken = await authManager.createMavenToken(userId, false);
|
||||
const composerToken = await authManager.createComposerToken(userId, false);
|
||||
const cargoToken = await authManager.createCargoToken(userId, false);
|
||||
const pypiToken = await authManager.createPypiToken(userId, false);
|
||||
const rubygemsToken = await authManager.createRubyGemsToken(userId, false);
|
||||
|
||||
return { npmToken, ociToken, mavenToken, composerToken, cargoToken, pypiToken, rubygemsToken, userId };
|
||||
}
|
||||
@@ -0,0 +1,412 @@
|
||||
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||
import { DefaultAuthProvider } from '../ts/core/classes.defaultauthprovider.js';
|
||||
import { AuthManager } from '../ts/core/classes.authmanager.js';
|
||||
import type { IAuthProvider } from '../ts/core/interfaces.auth.js';
|
||||
import type { IAuthConfig, IAuthToken } from '../ts/core/interfaces.core.js';
|
||||
import { createMockAuthProvider } from './helpers/registry.js';
|
||||
|
||||
// ============================================================================
|
||||
// Test State
|
||||
// ============================================================================
|
||||
|
||||
let provider: DefaultAuthProvider;
|
||||
let authConfig: IAuthConfig;
|
||||
|
||||
// ============================================================================
|
||||
// Setup
|
||||
// ============================================================================
|
||||
|
||||
tap.test('setup: should create DefaultAuthProvider', async () => {
|
||||
authConfig = {
|
||||
jwtSecret: 'test-secret-key-for-jwt-signing',
|
||||
tokenStore: 'memory',
|
||||
npmTokens: { enabled: true },
|
||||
ociTokens: {
|
||||
enabled: true,
|
||||
realm: 'https://auth.example.com/token',
|
||||
service: 'test-registry',
|
||||
},
|
||||
mavenTokens: { enabled: true },
|
||||
cargoTokens: { enabled: true },
|
||||
composerTokens: { enabled: true },
|
||||
pypiTokens: { enabled: true },
|
||||
rubygemsTokens: { enabled: true },
|
||||
};
|
||||
|
||||
provider = new DefaultAuthProvider(authConfig);
|
||||
await provider.init();
|
||||
expect(provider).toBeInstanceOf(DefaultAuthProvider);
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Authentication Tests
|
||||
// ============================================================================
|
||||
|
||||
tap.test('authenticate: should authenticate new user (auto-registration)', async () => {
|
||||
const userId = await provider.authenticate({
|
||||
username: 'newuser',
|
||||
password: 'newpassword',
|
||||
});
|
||||
|
||||
expect(userId).toEqual('newuser');
|
||||
});
|
||||
|
||||
tap.test('authenticate: should authenticate existing user with correct password', async () => {
|
||||
// First registration
|
||||
await provider.authenticate({
|
||||
username: 'existinguser',
|
||||
password: 'correctpass',
|
||||
});
|
||||
|
||||
// Second authentication with same credentials
|
||||
const userId = await provider.authenticate({
|
||||
username: 'existinguser',
|
||||
password: 'correctpass',
|
||||
});
|
||||
|
||||
expect(userId).toEqual('existinguser');
|
||||
});
|
||||
|
||||
tap.test('authenticate: should reject authentication with wrong password', async () => {
|
||||
// First registration
|
||||
await provider.authenticate({
|
||||
username: 'passworduser',
|
||||
password: 'originalpass',
|
||||
});
|
||||
|
||||
// Attempt with wrong password
|
||||
const userId = await provider.authenticate({
|
||||
username: 'passworduser',
|
||||
password: 'wrongpass',
|
||||
});
|
||||
|
||||
expect(userId).toBeNull();
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Token Creation Tests
|
||||
// ============================================================================
|
||||
|
||||
tap.test('createToken: should create NPM token with correct scopes', async () => {
|
||||
const token = await provider.createToken('testuser', 'npm', {
|
||||
scopes: ['npm:package:*:*'],
|
||||
});
|
||||
|
||||
expect(token).toBeTruthy();
|
||||
expect(typeof token).toEqual('string');
|
||||
|
||||
// Validate the token
|
||||
const validated = await provider.validateToken(token, 'npm');
|
||||
expect(validated).toBeTruthy();
|
||||
expect(validated!.type).toEqual('npm');
|
||||
expect(validated!.userId).toEqual('testuser');
|
||||
expect(validated!.scopes).toContain('npm:package:*:*');
|
||||
});
|
||||
|
||||
tap.test('createToken: should create Maven token', async () => {
|
||||
const token = await provider.createToken('mavenuser', 'maven', {
|
||||
readonly: true,
|
||||
});
|
||||
|
||||
expect(token).toBeTruthy();
|
||||
|
||||
const validated = await provider.validateToken(token, 'maven');
|
||||
expect(validated).toBeTruthy();
|
||||
expect(validated!.type).toEqual('maven');
|
||||
expect(validated!.readonly).toBeTrue();
|
||||
});
|
||||
|
||||
tap.test('createToken: should create OCI JWT token with correct claims', async () => {
|
||||
const token = await provider.createToken('ociuser', 'oci', {
|
||||
scopes: ['oci:repository:myrepo:push', 'oci:repository:myrepo:pull'],
|
||||
expiresIn: 3600,
|
||||
});
|
||||
|
||||
expect(token).toBeTruthy();
|
||||
// OCI tokens are JWTs (contain dots)
|
||||
expect(token.split('.').length).toEqual(3);
|
||||
|
||||
const validated = await provider.validateToken(token, 'oci');
|
||||
expect(validated).toBeTruthy();
|
||||
expect(validated!.type).toEqual('oci');
|
||||
expect(validated!.userId).toEqual('ociuser');
|
||||
expect(validated!.scopes.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
tap.test('createToken: should create token with expiration', async () => {
|
||||
const token = await provider.createToken('expiryuser', 'npm', {
|
||||
expiresIn: 60, // 60 seconds
|
||||
});
|
||||
|
||||
const validated = await provider.validateToken(token, 'npm');
|
||||
expect(validated).toBeTruthy();
|
||||
expect(validated!.expiresAt).toBeTruthy();
|
||||
expect(validated!.expiresAt!.getTime()).toBeGreaterThan(Date.now());
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Token Validation Tests
|
||||
// ============================================================================
|
||||
|
||||
tap.test('validateToken: should validate UUID token (NPM, Maven, etc.)', async () => {
|
||||
const npmToken = await provider.createToken('validateuser', 'npm');
|
||||
const validated = await provider.validateToken(npmToken);
|
||||
|
||||
expect(validated).toBeTruthy();
|
||||
expect(validated!.type).toEqual('npm');
|
||||
expect(validated!.userId).toEqual('validateuser');
|
||||
});
|
||||
|
||||
tap.test('validateToken: should validate OCI JWT token', async () => {
|
||||
const ociToken = await provider.createToken('ocivalidate', 'oci', {
|
||||
scopes: ['oci:repository:*:*'],
|
||||
});
|
||||
|
||||
const validated = await provider.validateToken(ociToken, 'oci');
|
||||
|
||||
expect(validated).toBeTruthy();
|
||||
expect(validated!.type).toEqual('oci');
|
||||
expect(validated!.userId).toEqual('ocivalidate');
|
||||
});
|
||||
|
||||
tap.test('validateToken: should reject expired tokens', async () => {
|
||||
const token = await provider.createToken('expireduser', 'npm', {
|
||||
expiresIn: -1, // Already expired (in the past)
|
||||
});
|
||||
|
||||
// The token should be created but will fail validation due to expiry
|
||||
const validated = await provider.validateToken(token, 'npm');
|
||||
|
||||
// Token should be rejected because it's expired
|
||||
expect(validated).toBeNull();
|
||||
});
|
||||
|
||||
tap.test('validateToken: should reject invalid token', async () => {
|
||||
const validated = await provider.validateToken('invalid-random-token');
|
||||
expect(validated).toBeNull();
|
||||
});
|
||||
|
||||
tap.test('validateToken: should reject token with wrong protocol', async () => {
|
||||
const npmToken = await provider.createToken('protocoluser', 'npm');
|
||||
|
||||
// Try to validate as Maven token
|
||||
const validated = await provider.validateToken(npmToken, 'maven');
|
||||
expect(validated).toBeNull();
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Token Revocation Tests
|
||||
// ============================================================================
|
||||
|
||||
tap.test('revokeToken: should revoke tokens', async () => {
|
||||
const token = await provider.createToken('revokeuser', 'npm');
|
||||
|
||||
// Verify token works before revocation
|
||||
let validated = await provider.validateToken(token);
|
||||
expect(validated).toBeTruthy();
|
||||
|
||||
// Revoke the token
|
||||
await provider.revokeToken(token);
|
||||
|
||||
// Token should no longer be valid
|
||||
validated = await provider.validateToken(token);
|
||||
expect(validated).toBeNull();
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Authorization Tests
|
||||
// ============================================================================
|
||||
|
||||
tap.test('authorize: should authorize read actions for readonly tokens', async () => {
|
||||
const token = await provider.createToken('readonlyuser', 'npm', {
|
||||
readonly: true,
|
||||
scopes: ['npm:package:*:read'],
|
||||
});
|
||||
|
||||
const validated = await provider.validateToken(token);
|
||||
|
||||
const canRead = await provider.authorize(validated, 'npm:package:lodash', 'read');
|
||||
expect(canRead).toBeTrue();
|
||||
|
||||
const canPull = await provider.authorize(validated, 'npm:package:lodash', 'pull');
|
||||
expect(canPull).toBeTrue();
|
||||
});
|
||||
|
||||
tap.test('authorize: should deny write actions for readonly tokens', async () => {
|
||||
const token = await provider.createToken('readonlyuser2', 'npm', {
|
||||
readonly: true,
|
||||
scopes: ['npm:package:*:*'],
|
||||
});
|
||||
|
||||
const validated = await provider.validateToken(token);
|
||||
|
||||
const canWrite = await provider.authorize(validated, 'npm:package:lodash', 'write');
|
||||
expect(canWrite).toBeFalse();
|
||||
|
||||
const canPush = await provider.authorize(validated, 'npm:package:lodash', 'push');
|
||||
expect(canPush).toBeFalse();
|
||||
|
||||
const canDelete = await provider.authorize(validated, 'npm:package:lodash', 'delete');
|
||||
expect(canDelete).toBeFalse();
|
||||
});
|
||||
|
||||
tap.test('authorize: should match scopes with wildcards', async () => {
|
||||
// The scope system uses literal * as wildcard, not glob patterns
|
||||
// npm:*:*:* means "all types, all names, all actions under npm"
|
||||
const token = await provider.createToken('wildcarduser', 'npm', {
|
||||
scopes: ['npm:*:*:*'],
|
||||
});
|
||||
|
||||
const validated = await provider.validateToken(token);
|
||||
|
||||
// Should match any npm resource with full wildcard scope
|
||||
const canAccessAnyPackage = await provider.authorize(validated, 'npm:package:lodash', 'read');
|
||||
expect(canAccessAnyPackage).toBeTrue();
|
||||
|
||||
const canAccessScopedPackage = await provider.authorize(validated, 'npm:package:@myorg/foo', 'write');
|
||||
expect(canAccessScopedPackage).toBeTrue();
|
||||
});
|
||||
|
||||
tap.test('authorize: should deny access with null token', async () => {
|
||||
const canAccess = await provider.authorize(null, 'npm:package:lodash', 'read');
|
||||
expect(canAccess).toBeFalse();
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// List Tokens Tests
|
||||
// ============================================================================
|
||||
|
||||
tap.test('listUserTokens: should list user tokens', async () => {
|
||||
// Create multiple tokens for the same user
|
||||
const userId = 'listtokenuser';
|
||||
await provider.createToken(userId, 'npm');
|
||||
await provider.createToken(userId, 'maven', { readonly: true });
|
||||
await provider.createToken(userId, 'cargo');
|
||||
|
||||
const tokens = await provider.listUserTokens!(userId);
|
||||
|
||||
expect(tokens.length).toBeGreaterThanOrEqual(3);
|
||||
|
||||
// Check that tokens have expected properties
|
||||
for (const token of tokens) {
|
||||
expect(token.key).toBeTruthy();
|
||||
expect(typeof token.readonly).toEqual('boolean');
|
||||
expect(token.created).toBeTruthy();
|
||||
}
|
||||
|
||||
// Verify we have different protocols
|
||||
const protocols = tokens.map(t => t.protocol);
|
||||
expect(protocols).toContain('npm');
|
||||
expect(protocols).toContain('maven');
|
||||
expect(protocols).toContain('cargo');
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// AuthManager Integration Tests
|
||||
// ============================================================================
|
||||
|
||||
tap.test('AuthManager: should accept custom IAuthProvider', async () => {
|
||||
const mockProvider = createMockAuthProvider({
|
||||
authenticate: async (credentials) => {
|
||||
if (credentials.username === 'custom' && credentials.password === 'pass') {
|
||||
return 'custom-user-id';
|
||||
}
|
||||
return null;
|
||||
},
|
||||
});
|
||||
|
||||
const manager = new AuthManager(authConfig, mockProvider);
|
||||
await manager.init();
|
||||
|
||||
// Use the custom provider
|
||||
const userId = await manager.authenticate({
|
||||
username: 'custom',
|
||||
password: 'pass',
|
||||
});
|
||||
|
||||
expect(userId).toEqual('custom-user-id');
|
||||
|
||||
// Wrong credentials should fail
|
||||
const failed = await manager.authenticate({
|
||||
username: 'custom',
|
||||
password: 'wrong',
|
||||
});
|
||||
|
||||
expect(failed).toBeNull();
|
||||
});
|
||||
|
||||
tap.test('AuthManager: should use default provider when none specified', async () => {
|
||||
const manager = new AuthManager(authConfig);
|
||||
await manager.init();
|
||||
|
||||
// Should use DefaultAuthProvider internally
|
||||
const userId = await manager.authenticate({
|
||||
username: 'defaultuser',
|
||||
password: 'defaultpass',
|
||||
});
|
||||
|
||||
expect(userId).toEqual('defaultuser');
|
||||
});
|
||||
|
||||
tap.test('AuthManager: should delegate token creation to provider', async () => {
|
||||
let tokenCreated = false;
|
||||
const mockProvider = createMockAuthProvider({
|
||||
createToken: async (userId, protocol, options) => {
|
||||
tokenCreated = true;
|
||||
return `mock-token-${protocol}-${userId}`;
|
||||
},
|
||||
});
|
||||
|
||||
const manager = new AuthManager(authConfig, mockProvider);
|
||||
await manager.init();
|
||||
|
||||
const token = await manager.createNpmToken('delegateuser', false);
|
||||
|
||||
expect(tokenCreated).toBeTrue();
|
||||
expect(token).toContain('mock-token-npm');
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Edge Cases
|
||||
// ============================================================================
|
||||
|
||||
tap.test('edge: should handle concurrent token operations', async () => {
|
||||
const promises: Promise<string>[] = [];
|
||||
|
||||
// Create 10 tokens concurrently
|
||||
for (let i = 0; i < 10; i++) {
|
||||
promises.push(provider.createToken(`concurrent-user-${i}`, 'npm'));
|
||||
}
|
||||
|
||||
const tokens = await Promise.all(promises);
|
||||
|
||||
// All tokens should be unique
|
||||
const uniqueTokens = new Set(tokens);
|
||||
expect(uniqueTokens.size).toEqual(10);
|
||||
|
||||
// All tokens should be valid
|
||||
for (const token of tokens) {
|
||||
const validated = await provider.validateToken(token);
|
||||
expect(validated).toBeTruthy();
|
||||
}
|
||||
});
|
||||
|
||||
tap.test('edge: should handle empty scopes', async () => {
|
||||
const token = await provider.createToken('emptyuser', 'npm', {
|
||||
scopes: [],
|
||||
});
|
||||
|
||||
const validated = await provider.validateToken(token);
|
||||
expect(validated).toBeTruthy();
|
||||
// Even with empty scopes, token should be valid
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Cleanup
|
||||
// ============================================================================
|
||||
|
||||
tap.test('cleanup', async () => {
|
||||
// No cleanup needed for in-memory provider
|
||||
});
|
||||
|
||||
export default tap.start();
|
||||
@@ -0,0 +1,468 @@
|
||||
/**
|
||||
* Native cargo CLI Testing
|
||||
* Tests the Cargo registry implementation using the actual cargo CLI
|
||||
*/
|
||||
|
||||
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||
import { tapNodeTools } from '@git.zone/tstest/tapbundle_serverside';
|
||||
import { SmartRegistry } from '../ts/index.js';
|
||||
import { createTestRegistry, createTestTokens } from './helpers/registry.js';
|
||||
import type { IRequestContext, IResponse } from '../ts/core/interfaces.core.js';
|
||||
import * as http from 'http';
|
||||
import * as url from 'url';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
// Test context
|
||||
let registry: SmartRegistry;
|
||||
let server: http.Server;
|
||||
let registryUrl: string;
|
||||
let registryPort: number;
|
||||
let cargoToken: string;
|
||||
let testDir: string;
|
||||
let cargoHome: string;
|
||||
|
||||
/**
|
||||
* Create HTTP server wrapper around SmartRegistry
|
||||
*/
|
||||
async function createHttpServer(
|
||||
registryInstance: SmartRegistry,
|
||||
port: number
|
||||
): Promise<{ server: http.Server; url: string }> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const httpServer = http.createServer(async (req, res) => {
|
||||
try {
|
||||
// Parse request
|
||||
const parsedUrl = url.parse(req.url || '', true);
|
||||
const pathname = parsedUrl.pathname || '/';
|
||||
const query = parsedUrl.query;
|
||||
|
||||
// Read body
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of req) {
|
||||
chunks.push(chunk);
|
||||
}
|
||||
const bodyBuffer = Buffer.concat(chunks);
|
||||
|
||||
// Parse body based on content type
|
||||
let body: any;
|
||||
if (bodyBuffer.length > 0) {
|
||||
const contentType = req.headers['content-type'] || '';
|
||||
if (contentType.includes('application/json')) {
|
||||
try {
|
||||
body = JSON.parse(bodyBuffer.toString('utf-8'));
|
||||
} catch (error) {
|
||||
body = bodyBuffer;
|
||||
}
|
||||
} else {
|
||||
body = bodyBuffer;
|
||||
}
|
||||
}
|
||||
|
||||
// Convert to IRequestContext
|
||||
const context: IRequestContext = {
|
||||
method: req.method || 'GET',
|
||||
path: pathname,
|
||||
headers: req.headers as Record<string, string>,
|
||||
query: query as Record<string, string>,
|
||||
body: body,
|
||||
};
|
||||
|
||||
// Handle request
|
||||
const response: IResponse = await registryInstance.handleRequest(context);
|
||||
|
||||
// Convert IResponse to HTTP response
|
||||
res.statusCode = response.status;
|
||||
|
||||
// Set headers
|
||||
for (const [key, value] of Object.entries(response.headers || {})) {
|
||||
res.setHeader(key, value);
|
||||
}
|
||||
|
||||
// Send body (response.body is always ReadableStream<Uint8Array> or undefined)
|
||||
if (response.body) {
|
||||
const { Readable } = await import('stream');
|
||||
Readable.fromWeb(response.body).pipe(res);
|
||||
} else {
|
||||
res.end();
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Server error:', error);
|
||||
res.statusCode = 500;
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.end(JSON.stringify({ error: 'INTERNAL_ERROR', message: String(error) }));
|
||||
}
|
||||
});
|
||||
|
||||
httpServer.listen(port, () => {
|
||||
const serverUrl = `http://localhost:${port}`;
|
||||
resolve({ server: httpServer, url: serverUrl });
|
||||
});
|
||||
|
||||
httpServer.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Setup Cargo configuration
|
||||
*/
|
||||
function setupCargoConfig(registryUrlArg: string, token: string, cargoHomeArg: string): void {
|
||||
const cargoConfigDir = path.join(cargoHomeArg, '.cargo');
|
||||
fs.mkdirSync(cargoConfigDir, { recursive: true });
|
||||
|
||||
// Create config.toml with sparse protocol
|
||||
const configContent = `[registries.test-registry]
|
||||
index = "sparse+${registryUrlArg}/cargo/"
|
||||
|
||||
[source.crates-io]
|
||||
replace-with = "test-registry"
|
||||
|
||||
[net]
|
||||
retry = 0
|
||||
`;
|
||||
|
||||
fs.writeFileSync(path.join(cargoConfigDir, 'config.toml'), configContent, 'utf-8');
|
||||
|
||||
// Create credentials.toml (Cargo uses plain token, no "Bearer" prefix)
|
||||
const credentialsContent = `[registries.test-registry]
|
||||
token = "${token}"
|
||||
`;
|
||||
|
||||
fs.writeFileSync(path.join(cargoConfigDir, 'credentials.toml'), credentialsContent, 'utf-8');
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a test Cargo crate
|
||||
*/
|
||||
function createTestCrate(
|
||||
crateName: string,
|
||||
version: string,
|
||||
targetDir: string
|
||||
): string {
|
||||
const crateDir = path.join(targetDir, crateName);
|
||||
fs.mkdirSync(crateDir, { recursive: true });
|
||||
|
||||
// Create Cargo.toml
|
||||
const cargoToml = `[package]
|
||||
name = "${crateName}"
|
||||
version = "${version}"
|
||||
edition = "2021"
|
||||
description = "Test crate ${crateName}"
|
||||
license = "MIT"
|
||||
authors = ["Test Author <test@example.com>"]
|
||||
|
||||
[dependencies]
|
||||
`;
|
||||
|
||||
fs.writeFileSync(path.join(crateDir, 'Cargo.toml'), cargoToml, 'utf-8');
|
||||
|
||||
// Create src directory
|
||||
const srcDir = path.join(crateDir, 'src');
|
||||
fs.mkdirSync(srcDir, { recursive: true });
|
||||
|
||||
// Create lib.rs
|
||||
const libRs = `//! Test crate ${crateName}
|
||||
|
||||
/// Returns a greeting message
|
||||
pub fn greet() -> String {
|
||||
format!("Hello from {}@{}", "${crateName}", "${version}")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_greet() {
|
||||
let greeting = greet();
|
||||
assert!(greeting.contains("${crateName}"));
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
fs.writeFileSync(path.join(srcDir, 'lib.rs'), libRs, 'utf-8');
|
||||
|
||||
// Create README.md
|
||||
const readme = `# ${crateName}
|
||||
|
||||
Test crate for SmartRegistry.
|
||||
|
||||
Version: ${version}
|
||||
`;
|
||||
|
||||
fs.writeFileSync(path.join(crateDir, 'README.md'), readme, 'utf-8');
|
||||
|
||||
return crateDir;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run cargo command with proper environment
|
||||
*/
|
||||
async function runCargoCommand(
|
||||
command: string,
|
||||
cwd: string,
|
||||
includeToken: boolean = true
|
||||
): Promise<{ stdout: string; stderr: string; exitCode: number }> {
|
||||
// Prepare environment variables
|
||||
// NOTE: Cargo converts registry name "test-registry" to "TEST_REGISTRY" for env vars
|
||||
const envVars = [
|
||||
`CARGO_HOME="${cargoHome}"`,
|
||||
`CARGO_REGISTRIES_TEST_REGISTRY_INDEX="sparse+${registryUrl}/cargo/"`,
|
||||
includeToken ? `CARGO_REGISTRIES_TEST_REGISTRY_TOKEN="${cargoToken}"` : '',
|
||||
`CARGO_NET_RETRY="0"`,
|
||||
].filter(Boolean).join(' ');
|
||||
|
||||
// Build command with cd to correct directory and environment variables
|
||||
const fullCommand = `cd "${cwd}" && ${envVars} ${command}`;
|
||||
|
||||
try {
|
||||
const result = await tapNodeTools.runCommand(fullCommand);
|
||||
return {
|
||||
stdout: result.stdout || '',
|
||||
stderr: result.stderr || '',
|
||||
exitCode: result.exitCode || 0,
|
||||
};
|
||||
} catch (error: any) {
|
||||
return {
|
||||
stdout: error.stdout || '',
|
||||
stderr: error.stderr || String(error),
|
||||
exitCode: error.exitCode || 1,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleanup test directory
|
||||
*/
|
||||
function cleanupTestDir(dir: string): void {
|
||||
if (fs.existsSync(dir)) {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// TESTS
|
||||
// ========================================================================
|
||||
|
||||
tap.test('Cargo CLI: should setup registry and HTTP server', async () => {
|
||||
// Use port 5000
|
||||
registryPort = 5000;
|
||||
|
||||
// Create registry with correct registryUrl for CLI tests
|
||||
registry = await createTestRegistry({ registryUrl: `http://localhost:${registryPort}` });
|
||||
const tokens = await createTestTokens(registry);
|
||||
cargoToken = tokens.cargoToken;
|
||||
|
||||
expect(registry).toBeInstanceOf(SmartRegistry);
|
||||
expect(cargoToken).toBeTypeOf('string');
|
||||
|
||||
// Clean up any existing index from previous test runs
|
||||
const storage = registry.getStorage();
|
||||
try {
|
||||
await storage.putCargoIndex('test-crate-cli', []);
|
||||
} catch (error) {
|
||||
// Ignore error if operation fails
|
||||
}
|
||||
const serverSetup = await createHttpServer(registry, registryPort);
|
||||
server = serverSetup.server;
|
||||
registryUrl = serverSetup.url;
|
||||
|
||||
expect(server).toBeDefined();
|
||||
expect(registryUrl).toEqual(`http://localhost:${registryPort}`);
|
||||
|
||||
// Setup test directory
|
||||
testDir = path.join(process.cwd(), '.nogit', 'test-cargo-cli');
|
||||
cleanupTestDir(testDir);
|
||||
fs.mkdirSync(testDir, { recursive: true });
|
||||
|
||||
// Setup CARGO_HOME
|
||||
cargoHome = path.join(testDir, '.cargo-home');
|
||||
fs.mkdirSync(cargoHome, { recursive: true });
|
||||
|
||||
// Setup Cargo config
|
||||
setupCargoConfig(registryUrl, cargoToken, cargoHome);
|
||||
expect(fs.existsSync(path.join(cargoHome, '.cargo', 'config.toml'))).toEqual(true);
|
||||
expect(fs.existsSync(path.join(cargoHome, '.cargo', 'credentials.toml'))).toEqual(true);
|
||||
});
|
||||
|
||||
tap.test('Cargo CLI: should verify server is responding', async () => {
|
||||
// Check server is up by doing a direct HTTP request to the cargo index
|
||||
const response = await fetch(`${registryUrl}/cargo/`);
|
||||
expect(response.status).toBeGreaterThanOrEqual(200);
|
||||
expect(response.status).toBeLessThan(500);
|
||||
});
|
||||
|
||||
tap.test('Cargo CLI: should publish a crate', async () => {
|
||||
const crateName = 'test-crate-cli';
|
||||
const version = '0.1.0';
|
||||
const crateDir = createTestCrate(crateName, version, testDir);
|
||||
|
||||
const result = await runCargoCommand('cargo publish --registry test-registry --allow-dirty', crateDir);
|
||||
console.log('cargo publish output:', result.stdout);
|
||||
console.log('cargo publish stderr:', result.stderr);
|
||||
|
||||
expect(result.exitCode).toEqual(0);
|
||||
expect(result.stdout || result.stderr).toContain(crateName);
|
||||
});
|
||||
|
||||
tap.test('Cargo CLI: should verify crate in index', async () => {
|
||||
const crateName = 'test-crate-cli';
|
||||
|
||||
// Cargo uses a specific index structure
|
||||
// For crate "test-crate-cli", the index path is based on the first characters
|
||||
// 1 char: <name>
|
||||
// 2 char: 2/<name>
|
||||
// 3 char: 3/<first-char>/<name>
|
||||
// 4+ char: <first-2-chars>/<second-2-chars>/<name>
|
||||
|
||||
// "test-crate-cli" is 14 chars, so it should be at: te/st/test-crate-cli
|
||||
const indexPath = `/cargo/te/st/${crateName}`;
|
||||
|
||||
const response = await fetch(`${registryUrl}${indexPath}`);
|
||||
expect(response.status).toEqual(200);
|
||||
|
||||
const indexData = await response.text();
|
||||
console.log('Index data:', indexData);
|
||||
|
||||
// Index should contain JSON line with crate info
|
||||
expect(indexData).toContain(crateName);
|
||||
expect(indexData).toContain('0.1.0');
|
||||
});
|
||||
|
||||
tap.test('Cargo CLI: should download published crate', async () => {
|
||||
const crateName = 'test-crate-cli';
|
||||
const version = '0.1.0';
|
||||
|
||||
// Cargo downloads crates from /cargo/api/v1/crates/{name}/{version}/download
|
||||
const downloadPath = `/cargo/api/v1/crates/${crateName}/${version}/download`;
|
||||
|
||||
const response = await fetch(`${registryUrl}${downloadPath}`);
|
||||
expect(response.status).toEqual(200);
|
||||
|
||||
const crateData = await response.arrayBuffer();
|
||||
expect(crateData.byteLength).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
tap.test('Cargo CLI: should publish second version', async () => {
|
||||
const crateName = 'test-crate-cli';
|
||||
const version = '0.2.0';
|
||||
const crateDir = createTestCrate(crateName, version, testDir);
|
||||
|
||||
const result = await runCargoCommand('cargo publish --registry test-registry --allow-dirty', crateDir);
|
||||
console.log('cargo publish v0.2.0 output:', result.stdout);
|
||||
|
||||
expect(result.exitCode).toEqual(0);
|
||||
});
|
||||
|
||||
tap.test('Cargo CLI: should list versions in index', async () => {
|
||||
const crateName = 'test-crate-cli';
|
||||
const indexPath = `/cargo/te/st/${crateName}`;
|
||||
|
||||
const response = await fetch(`${registryUrl}${indexPath}`);
|
||||
expect(response.status).toEqual(200);
|
||||
|
||||
const indexData = await response.text();
|
||||
const lines = indexData.trim().split('\n');
|
||||
|
||||
// Should have 2 lines (2 versions)
|
||||
expect(lines.length).toEqual(2);
|
||||
|
||||
// Parse JSON lines
|
||||
const version1 = JSON.parse(lines[0]);
|
||||
const version2 = JSON.parse(lines[1]);
|
||||
|
||||
expect(version1.vers).toEqual('0.1.0');
|
||||
expect(version2.vers).toEqual('0.2.0');
|
||||
});
|
||||
|
||||
tap.test('Cargo CLI: should search for crate', async () => {
|
||||
const crateName = 'test-crate-cli';
|
||||
|
||||
// Cargo search endpoint: /cargo/api/v1/crates?q={query}
|
||||
const response = await fetch(`${registryUrl}/cargo/api/v1/crates?q=${crateName}`);
|
||||
expect(response.status).toEqual(200);
|
||||
|
||||
const searchResults = await response.json();
|
||||
console.log('Search results:', searchResults);
|
||||
|
||||
expect(searchResults).toHaveProperty('crates');
|
||||
expect(searchResults.crates).toBeInstanceOf(Array);
|
||||
expect(searchResults.crates.length).toBeGreaterThan(0);
|
||||
expect(searchResults.crates[0].name).toEqual(crateName);
|
||||
});
|
||||
|
||||
tap.test('Cargo CLI: should yank a version', async () => {
|
||||
const crateName = 'test-crate-cli';
|
||||
const crateDir = path.join(testDir, crateName);
|
||||
|
||||
const result = await runCargoCommand('cargo yank --registry test-registry --vers 0.1.0', crateDir);
|
||||
console.log('cargo yank output:', result.stdout);
|
||||
console.log('cargo yank stderr:', result.stderr);
|
||||
|
||||
expect(result.exitCode).toEqual(0);
|
||||
|
||||
// Verify version is yanked in index
|
||||
const indexPath = `/cargo/te/st/${crateName}`;
|
||||
const response = await fetch(`${registryUrl}${indexPath}`);
|
||||
const indexData = await response.text();
|
||||
const lines = indexData.trim().split('\n');
|
||||
const version1 = JSON.parse(lines[0]);
|
||||
|
||||
expect(version1.yanked).toEqual(true);
|
||||
});
|
||||
|
||||
tap.test('Cargo CLI: should unyank a version', async () => {
|
||||
const crateName = 'test-crate-cli';
|
||||
const crateDir = path.join(testDir, crateName);
|
||||
|
||||
const result = await runCargoCommand('cargo yank --registry test-registry --vers 0.1.0 --undo', crateDir);
|
||||
console.log('cargo unyank output:', result.stdout);
|
||||
console.log('cargo unyank stderr:', result.stderr);
|
||||
|
||||
expect(result.exitCode).toEqual(0);
|
||||
|
||||
// Verify version is not yanked in index
|
||||
const indexPath = `/cargo/te/st/${crateName}`;
|
||||
const response = await fetch(`${registryUrl}${indexPath}`);
|
||||
const indexData = await response.text();
|
||||
const lines = indexData.trim().split('\n');
|
||||
const version1 = JSON.parse(lines[0]);
|
||||
|
||||
expect(version1.yanked).toEqual(false);
|
||||
});
|
||||
|
||||
tap.test('Cargo CLI: should fail to publish without auth', async () => {
|
||||
const crateName = 'unauth-crate';
|
||||
const version = '0.1.0';
|
||||
const crateDir = createTestCrate(crateName, version, testDir);
|
||||
|
||||
// Run without token (includeToken: false)
|
||||
const result = await runCargoCommand('cargo publish --registry test-registry --allow-dirty', crateDir, false);
|
||||
console.log('cargo publish unauth output:', result.stdout);
|
||||
console.log('cargo publish unauth stderr:', result.stderr);
|
||||
|
||||
// Should fail with auth error
|
||||
expect(result.exitCode).not.toEqual(0);
|
||||
expect(result.stderr).toContain('token');
|
||||
});
|
||||
|
||||
tap.postTask('cleanup cargo cli tests', async () => {
|
||||
// Stop server
|
||||
if (server) {
|
||||
await new Promise<void>((resolve) => {
|
||||
server.close(() => resolve());
|
||||
});
|
||||
}
|
||||
|
||||
// Cleanup test directory
|
||||
if (testDir) {
|
||||
cleanupTestDir(testDir);
|
||||
}
|
||||
|
||||
// Destroy registry
|
||||
if (registry) {
|
||||
registry.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
export default tap.start();
|
||||
@@ -0,0 +1,462 @@
|
||||
/**
|
||||
* Native Composer CLI Testing
|
||||
* Tests the Composer registry implementation using the actual composer CLI
|
||||
*/
|
||||
|
||||
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||
import { tapNodeTools } from '@git.zone/tstest/tapbundle_serverside';
|
||||
import { SmartRegistry } from '../ts/index.js';
|
||||
import { createTestRegistry, createTestTokens, createComposerZip, generateTestRunId } from './helpers/registry.js';
|
||||
import type { IRequestContext, IResponse } from '../ts/core/interfaces.core.js';
|
||||
import * as http from 'http';
|
||||
import * as url from 'url';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
// Test context
|
||||
let registry: SmartRegistry;
|
||||
let server: http.Server;
|
||||
let registryUrl: string;
|
||||
let registryPort: number;
|
||||
let composerToken: string;
|
||||
let testDir: string;
|
||||
let composerHome: string;
|
||||
let hasComposer = false;
|
||||
|
||||
// Unique test run ID to avoid conflicts between test runs
|
||||
const testRunId = generateTestRunId();
|
||||
const testPackageName = `testvendor/test-pkg-${testRunId}`;
|
||||
|
||||
/**
|
||||
* Create HTTP server wrapper around SmartRegistry
|
||||
*/
|
||||
async function createHttpServer(
|
||||
registryInstance: SmartRegistry,
|
||||
port: number
|
||||
): Promise<{ server: http.Server; url: string }> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const httpServer = http.createServer(async (req, res) => {
|
||||
try {
|
||||
// Parse request
|
||||
const parsedUrl = url.parse(req.url || '', true);
|
||||
const pathname = parsedUrl.pathname || '/';
|
||||
const query = parsedUrl.query;
|
||||
|
||||
// Read body
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of req) {
|
||||
chunks.push(chunk);
|
||||
}
|
||||
const bodyBuffer = Buffer.concat(chunks);
|
||||
|
||||
// Parse body based on content type
|
||||
let body: any;
|
||||
if (bodyBuffer.length > 0) {
|
||||
const contentType = req.headers['content-type'] || '';
|
||||
if (contentType.includes('application/json')) {
|
||||
try {
|
||||
body = JSON.parse(bodyBuffer.toString('utf-8'));
|
||||
} catch (error) {
|
||||
body = bodyBuffer;
|
||||
}
|
||||
} else {
|
||||
body = bodyBuffer;
|
||||
}
|
||||
}
|
||||
|
||||
// Convert to IRequestContext
|
||||
const context: IRequestContext = {
|
||||
method: req.method || 'GET',
|
||||
path: pathname,
|
||||
headers: req.headers as Record<string, string>,
|
||||
query: query as Record<string, string>,
|
||||
body: body,
|
||||
};
|
||||
|
||||
// Handle request
|
||||
const response: IResponse = await registryInstance.handleRequest(context);
|
||||
|
||||
// Convert IResponse to HTTP response
|
||||
res.statusCode = response.status;
|
||||
|
||||
// Set headers
|
||||
for (const [key, value] of Object.entries(response.headers || {})) {
|
||||
res.setHeader(key, value);
|
||||
}
|
||||
|
||||
// Send body (response.body is always ReadableStream<Uint8Array> or undefined)
|
||||
if (response.body) {
|
||||
const { Readable } = await import('stream');
|
||||
Readable.fromWeb(response.body).pipe(res);
|
||||
} else {
|
||||
res.end();
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Server error:', error);
|
||||
res.statusCode = 500;
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.end(JSON.stringify({ error: 'INTERNAL_ERROR', message: String(error) }));
|
||||
}
|
||||
});
|
||||
|
||||
httpServer.listen(port, () => {
|
||||
const serverUrl = `http://localhost:${port}`;
|
||||
resolve({ server: httpServer, url: serverUrl });
|
||||
});
|
||||
|
||||
httpServer.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Setup Composer auth.json for authentication
|
||||
*/
|
||||
function setupComposerAuth(
|
||||
token: string,
|
||||
composerHomeArg: string,
|
||||
serverUrl: string,
|
||||
port: number
|
||||
): string {
|
||||
fs.mkdirSync(composerHomeArg, { recursive: true });
|
||||
|
||||
const authJson = {
|
||||
'http-basic': {
|
||||
[`localhost:${port}`]: {
|
||||
username: 'testuser',
|
||||
password: token,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const authPath = path.join(composerHomeArg, 'auth.json');
|
||||
fs.writeFileSync(authPath, JSON.stringify(authJson, null, 2), 'utf-8');
|
||||
|
||||
return authPath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a Composer project that uses our registry
|
||||
*/
|
||||
function createComposerProject(
|
||||
projectDir: string,
|
||||
serverUrl: string
|
||||
): void {
|
||||
fs.mkdirSync(projectDir, { recursive: true });
|
||||
|
||||
const composerJson = {
|
||||
name: 'test/consumer-project',
|
||||
description: 'Test consumer project for Composer CLI tests',
|
||||
type: 'project',
|
||||
require: {},
|
||||
repositories: [
|
||||
{
|
||||
type: 'composer',
|
||||
url: `${serverUrl}/composer`,
|
||||
},
|
||||
],
|
||||
config: {
|
||||
'secure-http': false,
|
||||
},
|
||||
};
|
||||
|
||||
fs.writeFileSync(
|
||||
path.join(projectDir, 'composer.json'),
|
||||
JSON.stringify(composerJson, null, 2),
|
||||
'utf-8'
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run Composer command with custom home directory
|
||||
*/
|
||||
async function runComposerCommand(
|
||||
command: string,
|
||||
cwd: string
|
||||
): Promise<{ stdout: string; stderr: string; exitCode: number }> {
|
||||
const fullCommand = `cd "${cwd}" && COMPOSER_HOME="${composerHome}" composer ${command}`;
|
||||
|
||||
try {
|
||||
const result = await tapNodeTools.runCommand(fullCommand);
|
||||
return {
|
||||
stdout: result.stdout || '',
|
||||
stderr: result.stderr || '',
|
||||
exitCode: result.exitCode || 0,
|
||||
};
|
||||
} catch (error: any) {
|
||||
return {
|
||||
stdout: error.stdout || '',
|
||||
stderr: error.stderr || String(error),
|
||||
exitCode: error.exitCode || 1,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload a Composer package via HTTP API
|
||||
*/
|
||||
async function uploadComposerPackage(
|
||||
vendorPackage: string,
|
||||
version: string,
|
||||
token: string,
|
||||
serverUrl: string
|
||||
): Promise<void> {
|
||||
const zipData = await createComposerZip(vendorPackage, version);
|
||||
|
||||
const response = await fetch(`${serverUrl}/composer/packages/${vendorPackage}`, {
|
||||
method: 'PUT',
|
||||
headers: {
|
||||
'Content-Type': 'application/zip',
|
||||
Authorization: `Bearer ${token}`,
|
||||
},
|
||||
body: zipData,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const body = await response.text();
|
||||
throw new Error(`Failed to upload package: ${response.status} ${body}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleanup test directory
|
||||
*/
|
||||
function cleanupTestDir(dir: string): void {
|
||||
if (fs.existsSync(dir)) {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// TESTS
|
||||
// ========================================================================
|
||||
|
||||
tap.test('Composer CLI: should verify composer is installed', async () => {
|
||||
try {
|
||||
const result = await tapNodeTools.runCommand('composer --version');
|
||||
console.log('Composer version output:', result.stdout.substring(0, 200));
|
||||
hasComposer = result.exitCode === 0;
|
||||
expect(result.exitCode).toEqual(0);
|
||||
} catch (error) {
|
||||
console.log('Composer CLI not available, skipping native CLI tests');
|
||||
hasComposer = false;
|
||||
}
|
||||
});
|
||||
|
||||
tap.test('Composer CLI: should setup registry and HTTP server', async () => {
|
||||
// Use port 38000 (avoids conflicts with other tests)
|
||||
registryPort = 38000;
|
||||
|
||||
// Create registry with correct registryUrl for CLI tests
|
||||
registry = await createTestRegistry({ registryUrl: `http://localhost:${registryPort}` });
|
||||
const tokens = await createTestTokens(registry);
|
||||
composerToken = tokens.composerToken;
|
||||
|
||||
expect(registry).toBeInstanceOf(SmartRegistry);
|
||||
expect(composerToken).toBeTypeOf('string');
|
||||
const serverSetup = await createHttpServer(registry, registryPort);
|
||||
server = serverSetup.server;
|
||||
registryUrl = serverSetup.url;
|
||||
|
||||
expect(server).toBeDefined();
|
||||
expect(registryUrl).toEqual(`http://localhost:${registryPort}`);
|
||||
|
||||
// Setup test directory
|
||||
testDir = path.join(process.cwd(), '.nogit', 'test-composer-cli');
|
||||
cleanupTestDir(testDir);
|
||||
fs.mkdirSync(testDir, { recursive: true });
|
||||
|
||||
// Setup COMPOSER_HOME directory
|
||||
composerHome = path.join(testDir, '.composer');
|
||||
fs.mkdirSync(composerHome, { recursive: true });
|
||||
|
||||
// Setup Composer auth
|
||||
const authPath = setupComposerAuth(composerToken, composerHome, registryUrl, registryPort);
|
||||
expect(fs.existsSync(authPath)).toEqual(true);
|
||||
});
|
||||
|
||||
tap.test('Composer CLI: should verify server is responding', async () => {
|
||||
// Check server is up by doing a direct HTTP request
|
||||
const response = await fetch(`${registryUrl}/composer/packages.json`);
|
||||
expect(response.status).toBeGreaterThanOrEqual(200);
|
||||
expect(response.status).toBeLessThan(500);
|
||||
});
|
||||
|
||||
tap.test('Composer CLI: should upload a package via API', async () => {
|
||||
const version = '1.0.0';
|
||||
|
||||
await uploadComposerPackage(testPackageName, version, composerToken, registryUrl);
|
||||
|
||||
// Verify package exists via p2 metadata endpoint (more reliable than packages.json for new packages)
|
||||
const metadataResponse = await fetch(`${registryUrl}/composer/p2/${testPackageName}.json`);
|
||||
expect(metadataResponse.status).toEqual(200);
|
||||
|
||||
const metadata = await metadataResponse.json();
|
||||
expect(metadata.packages).toBeDefined();
|
||||
expect(metadata.packages[testPackageName]).toBeDefined();
|
||||
expect(metadata.packages[testPackageName].length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
tap.test('Composer CLI: should require package from registry', async () => {
|
||||
if (!hasComposer) {
|
||||
console.log('Skipping - composer not available');
|
||||
return;
|
||||
}
|
||||
|
||||
const projectDir = path.join(testDir, 'consumer-project');
|
||||
createComposerProject(projectDir, registryUrl);
|
||||
|
||||
// Try to require the package we uploaded
|
||||
const result = await runComposerCommand(
|
||||
`require ${testPackageName}:1.0.0 --no-interaction`,
|
||||
projectDir
|
||||
);
|
||||
console.log('composer require output:', result.stdout);
|
||||
console.log('composer require stderr:', result.stderr);
|
||||
|
||||
expect(result.exitCode).toEqual(0);
|
||||
});
|
||||
|
||||
tap.test('Composer CLI: should verify package in vendor directory', async () => {
|
||||
if (!hasComposer) {
|
||||
console.log('Skipping - composer not available');
|
||||
return;
|
||||
}
|
||||
|
||||
const projectDir = path.join(testDir, 'consumer-project');
|
||||
// Parse vendor/package from testPackageName (e.g., "testvendor/test-pkg-abc123")
|
||||
const [vendor, pkg] = testPackageName.split('/');
|
||||
const packageDir = path.join(projectDir, 'vendor', vendor, pkg);
|
||||
|
||||
expect(fs.existsSync(packageDir)).toEqual(true);
|
||||
|
||||
// Check composer.json exists in package
|
||||
const packageComposerPath = path.join(packageDir, 'composer.json');
|
||||
expect(fs.existsSync(packageComposerPath)).toEqual(true);
|
||||
});
|
||||
|
||||
tap.test('Composer CLI: should upload second version', async () => {
|
||||
const version = '2.0.0';
|
||||
|
||||
await uploadComposerPackage(testPackageName, version, composerToken, registryUrl);
|
||||
|
||||
// Verify both versions exist via p2 metadata endpoint (Composer v2 format)
|
||||
const response = await fetch(`${registryUrl}/composer/p2/${testPackageName}.json`);
|
||||
expect(response.status).toEqual(200);
|
||||
|
||||
const metadata = await response.json();
|
||||
expect(metadata.packages).toBeDefined();
|
||||
expect(metadata.packages[testPackageName]).toBeDefined();
|
||||
// Check that both versions are present
|
||||
const versions = metadata.packages[testPackageName];
|
||||
expect(versions.length).toBeGreaterThanOrEqual(2);
|
||||
const versionNumbers = versions.map((v: any) => v.version);
|
||||
expect(versionNumbers).toContain('1.0.0');
|
||||
expect(versionNumbers).toContain('2.0.0');
|
||||
});
|
||||
|
||||
tap.test('Composer CLI: should update to new version', async () => {
|
||||
if (!hasComposer) {
|
||||
console.log('Skipping - composer not available');
|
||||
return;
|
||||
}
|
||||
|
||||
const projectDir = path.join(testDir, 'consumer-project');
|
||||
|
||||
// Update to version 2.0.0
|
||||
const result = await runComposerCommand(
|
||||
`require ${testPackageName}:2.0.0 --no-interaction`,
|
||||
projectDir
|
||||
);
|
||||
console.log('composer update output:', result.stdout);
|
||||
|
||||
expect(result.exitCode).toEqual(0);
|
||||
|
||||
// Verify composer.lock has the new version
|
||||
const lockPath = path.join(projectDir, 'composer.lock');
|
||||
expect(fs.existsSync(lockPath)).toEqual(true);
|
||||
|
||||
const lockContent = JSON.parse(fs.readFileSync(lockPath, 'utf-8'));
|
||||
const pkg = lockContent.packages.find((p: any) => p.name === testPackageName);
|
||||
expect(pkg?.version).toEqual('2.0.0');
|
||||
});
|
||||
|
||||
tap.test('Composer CLI: should search for packages', async () => {
|
||||
if (!hasComposer) {
|
||||
console.log('Skipping - composer not available');
|
||||
return;
|
||||
}
|
||||
|
||||
const projectDir = path.join(testDir, 'consumer-project');
|
||||
|
||||
// Search for packages (may not work on all Composer versions)
|
||||
const result = await runComposerCommand(
|
||||
'search testvendor --no-interaction 2>&1 || true',
|
||||
projectDir
|
||||
);
|
||||
console.log('composer search output:', result.stdout);
|
||||
|
||||
// Search may or may not work depending on registry implementation
|
||||
// Just verify it doesn't crash
|
||||
expect(result.exitCode).toBeLessThanOrEqual(1);
|
||||
});
|
||||
|
||||
tap.test('Composer CLI: should show package info', async () => {
|
||||
if (!hasComposer) {
|
||||
console.log('Skipping - composer not available');
|
||||
return;
|
||||
}
|
||||
|
||||
const projectDir = path.join(testDir, 'consumer-project');
|
||||
|
||||
const result = await runComposerCommand(
|
||||
`show ${testPackageName} --no-interaction`,
|
||||
projectDir
|
||||
);
|
||||
console.log('composer show output:', result.stdout);
|
||||
|
||||
expect(result.exitCode).toEqual(0);
|
||||
expect(result.stdout).toContain(testPackageName);
|
||||
});
|
||||
|
||||
tap.test('Composer CLI: should remove package', async () => {
|
||||
if (!hasComposer) {
|
||||
console.log('Skipping - composer not available');
|
||||
return;
|
||||
}
|
||||
|
||||
const projectDir = path.join(testDir, 'consumer-project');
|
||||
|
||||
const result = await runComposerCommand(
|
||||
`remove ${testPackageName} --no-interaction`,
|
||||
projectDir
|
||||
);
|
||||
console.log('composer remove output:', result.stdout);
|
||||
|
||||
expect(result.exitCode).toEqual(0);
|
||||
|
||||
// Verify package is removed from vendor
|
||||
const [vendor, pkg] = testPackageName.split('/');
|
||||
const packageDir = path.join(projectDir, 'vendor', vendor, pkg);
|
||||
expect(fs.existsSync(packageDir)).toEqual(false);
|
||||
});
|
||||
|
||||
tap.postTask('cleanup composer cli tests', async () => {
|
||||
// Stop server
|
||||
if (server) {
|
||||
await new Promise<void>((resolve) => {
|
||||
server.close(() => resolve());
|
||||
});
|
||||
}
|
||||
|
||||
// Cleanup test directory
|
||||
if (testDir) {
|
||||
cleanupTestDir(testDir);
|
||||
}
|
||||
|
||||
// Destroy registry
|
||||
if (registry) {
|
||||
registry.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
export default tap.start();
|
||||
@@ -0,0 +1,310 @@
|
||||
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||
import { SmartRegistry } from '../ts/index.js';
|
||||
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
|
||||
import { createTestRegistry, createTestTokens, createComposerZip } from './helpers/registry.js';
|
||||
|
||||
let registry: SmartRegistry;
|
||||
let composerToken: string;
|
||||
let userId: string;
|
||||
|
||||
// Test data
|
||||
const testPackageName = 'vendor/test-package';
|
||||
const testVersion = '1.0.0';
|
||||
let testZipData: Buffer;
|
||||
|
||||
tap.test('Composer: should create registry instance', async () => {
|
||||
registry = await createTestRegistry();
|
||||
const tokens = await createTestTokens(registry);
|
||||
composerToken = tokens.composerToken;
|
||||
userId = tokens.userId;
|
||||
|
||||
expect(registry).toBeInstanceOf(SmartRegistry);
|
||||
expect(composerToken).toBeTypeOf('string');
|
||||
});
|
||||
|
||||
tap.test('Composer: should create test ZIP package', async () => {
|
||||
testZipData = await createComposerZip(testPackageName, testVersion, {
|
||||
description: 'Test Composer package for registry',
|
||||
license: ['MIT'],
|
||||
authors: [{ name: 'Test Author', email: 'test@example.com' }],
|
||||
});
|
||||
|
||||
expect(testZipData).toBeInstanceOf(Buffer);
|
||||
expect(testZipData.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
tap.test('Composer: should return packages.json (GET /packages.json)', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/composer/packages.json',
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
const body = await streamToJson(response.body);
|
||||
expect(body).toHaveProperty('metadata-url');
|
||||
expect(body).toHaveProperty('available-packages');
|
||||
expect(body['available-packages']).toBeInstanceOf(Array);
|
||||
});
|
||||
|
||||
tap.test('Composer: should upload a package (PUT /packages/{vendor/package})', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'PUT',
|
||||
path: `/composer/packages/${testPackageName}`,
|
||||
headers: {
|
||||
Authorization: `Bearer ${composerToken}`,
|
||||
'Content-Type': 'application/zip',
|
||||
},
|
||||
query: {},
|
||||
body: testZipData,
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(201);
|
||||
const body = await streamToJson(response.body);
|
||||
expect(body.status).toEqual('success');
|
||||
expect(body.package).toEqual(testPackageName);
|
||||
expect(body.version).toEqual(testVersion);
|
||||
});
|
||||
|
||||
tap.test('Composer: should retrieve package metadata (GET /p2/{vendor/package}.json)', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: `/composer/p2/${testPackageName}.json`,
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
const body = await streamToJson(response.body);
|
||||
expect(body).toHaveProperty('packages');
|
||||
expect(body.packages[testPackageName]).toBeInstanceOf(Array);
|
||||
expect(body.packages[testPackageName].length).toEqual(1);
|
||||
|
||||
const packageData = body.packages[testPackageName][0];
|
||||
expect(packageData.name).toEqual(testPackageName);
|
||||
expect(packageData.version).toEqual(testVersion);
|
||||
expect(packageData.version_normalized).toEqual('1.0.0.0');
|
||||
expect(packageData).toHaveProperty('dist');
|
||||
expect(packageData.dist.type).toEqual('zip');
|
||||
expect(packageData.dist).toHaveProperty('url');
|
||||
expect(packageData.dist).toHaveProperty('shasum');
|
||||
expect(packageData.dist).toHaveProperty('reference');
|
||||
});
|
||||
|
||||
tap.test('Composer: should download package ZIP (GET /dists/{vendor/package}/{ref}.zip)', async () => {
|
||||
// First get metadata to find reference
|
||||
const metadataResponse = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: `/composer/p2/${testPackageName}.json`,
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
const metaBody = await streamToJson(metadataResponse.body);
|
||||
const reference = metaBody.packages[testPackageName][0].dist.reference;
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: `/composer/dists/${testPackageName}/${reference}.zip`,
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
const body = await streamToBuffer(response.body);
|
||||
expect(body).toBeInstanceOf(Buffer);
|
||||
expect(response.headers['Content-Type']).toEqual('application/zip');
|
||||
expect(response.headers['Content-Disposition']).toContain('attachment');
|
||||
});
|
||||
|
||||
tap.test('Composer: should list packages (GET /packages/list.json)', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/composer/packages/list.json',
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
const body = await streamToJson(response.body);
|
||||
expect(body).toHaveProperty('packageNames');
|
||||
expect(body.packageNames).toBeInstanceOf(Array);
|
||||
expect(body.packageNames).toContain(testPackageName);
|
||||
});
|
||||
|
||||
tap.test('Composer: should filter package list (GET /packages/list.json?filter=vendor/*)', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/composer/packages/list.json',
|
||||
headers: {},
|
||||
query: { filter: 'vendor/*' },
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
const body = await streamToJson(response.body);
|
||||
expect(body.packageNames).toBeInstanceOf(Array);
|
||||
expect(body.packageNames).toContain(testPackageName);
|
||||
});
|
||||
|
||||
tap.test('Composer: should prevent duplicate version upload', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'PUT',
|
||||
path: `/composer/packages/${testPackageName}`,
|
||||
headers: {
|
||||
Authorization: `Bearer ${composerToken}`,
|
||||
'Content-Type': 'application/zip',
|
||||
},
|
||||
query: {},
|
||||
body: testZipData,
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(409);
|
||||
const body = await streamToJson(response.body);
|
||||
expect(body.status).toEqual('error');
|
||||
expect(body.message).toContain('already exists');
|
||||
});
|
||||
|
||||
tap.test('Composer: should upload a second version', async () => {
|
||||
const testVersion2 = '1.1.0';
|
||||
const testZipData2 = await createComposerZip(testPackageName, testVersion2);
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'PUT',
|
||||
path: `/composer/packages/${testPackageName}`,
|
||||
headers: {
|
||||
Authorization: `Bearer ${composerToken}`,
|
||||
'Content-Type': 'application/zip',
|
||||
},
|
||||
query: {},
|
||||
body: testZipData2,
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(201);
|
||||
const body = await streamToJson(response.body);
|
||||
expect(body.status).toEqual('success');
|
||||
expect(body.version).toEqual(testVersion2);
|
||||
});
|
||||
|
||||
tap.test('Composer: should return multiple versions in metadata', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: `/composer/p2/${testPackageName}.json`,
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
const body = await streamToJson(response.body);
|
||||
expect(body.packages[testPackageName]).toBeInstanceOf(Array);
|
||||
expect(body.packages[testPackageName].length).toEqual(2);
|
||||
|
||||
const versions = body.packages[testPackageName].map((p: any) => p.version);
|
||||
expect(versions).toContain('1.0.0');
|
||||
expect(versions).toContain('1.1.0');
|
||||
});
|
||||
|
||||
tap.test('Composer: should delete a specific version (DELETE /packages/{vendor/package}/{version})', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'DELETE',
|
||||
path: `/composer/packages/${testPackageName}/1.0.0`,
|
||||
headers: {
|
||||
Authorization: `Bearer ${composerToken}`,
|
||||
},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(204);
|
||||
|
||||
// Verify version was removed
|
||||
const metadataResponse = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: `/composer/p2/${testPackageName}.json`,
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
const metaBody = await streamToJson(metadataResponse.body);
|
||||
expect(metaBody.packages[testPackageName].length).toEqual(1);
|
||||
expect(metaBody.packages[testPackageName][0].version).toEqual('1.1.0');
|
||||
});
|
||||
|
||||
tap.test('Composer: should require auth for package upload', async () => {
|
||||
const testZipData3 = await createComposerZip('vendor/unauth-package', '1.0.0');
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'PUT',
|
||||
path: '/composer/packages/vendor/unauth-package',
|
||||
headers: {
|
||||
'Content-Type': 'application/zip',
|
||||
},
|
||||
query: {},
|
||||
body: testZipData3,
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(401);
|
||||
const body = await streamToJson(response.body);
|
||||
expect(body.status).toEqual('error');
|
||||
});
|
||||
|
||||
tap.test('Composer: should reject invalid ZIP (no composer.json)', async () => {
|
||||
const invalidZip = Buffer.from('invalid zip content');
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'PUT',
|
||||
path: `/composer/packages/${testPackageName}`,
|
||||
headers: {
|
||||
Authorization: `Bearer ${composerToken}`,
|
||||
'Content-Type': 'application/zip',
|
||||
},
|
||||
query: {},
|
||||
body: invalidZip,
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(400);
|
||||
const body = await streamToJson(response.body);
|
||||
expect(body.status).toEqual('error');
|
||||
expect(body.message).toContain('composer.json');
|
||||
});
|
||||
|
||||
tap.test('Composer: should delete entire package (DELETE /packages/{vendor/package})', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'DELETE',
|
||||
path: `/composer/packages/${testPackageName}`,
|
||||
headers: {
|
||||
Authorization: `Bearer ${composerToken}`,
|
||||
},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(204);
|
||||
|
||||
// Verify package was removed
|
||||
const metadataResponse = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: `/composer/p2/${testPackageName}.json`,
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(metadataResponse.status).toEqual(404);
|
||||
});
|
||||
|
||||
tap.test('Composer: should return 404 for non-existent package', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/composer/p2/non/existent.json',
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(404);
|
||||
});
|
||||
|
||||
tap.postTask('cleanup registry', async () => {
|
||||
if (registry) {
|
||||
registry.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
export default tap.start();
|
||||
@@ -0,0 +1,292 @@
|
||||
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||
import { SmartRegistry } from '../ts/index.js';
|
||||
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
|
||||
import {
|
||||
createTestRegistry,
|
||||
createTestTokens,
|
||||
createPythonWheel,
|
||||
createRubyGem,
|
||||
} from './helpers/registry.js';
|
||||
|
||||
let registry: SmartRegistry;
|
||||
let pypiToken: string;
|
||||
let rubygemsToken: string;
|
||||
|
||||
tap.test('Integration: should initialize registry with all protocols', async () => {
|
||||
registry = await createTestRegistry();
|
||||
const tokens = await createTestTokens(registry);
|
||||
pypiToken = tokens.pypiToken;
|
||||
rubygemsToken = tokens.rubygemsToken;
|
||||
|
||||
expect(registry).toBeInstanceOf(SmartRegistry);
|
||||
expect(registry.isInitialized()).toEqual(true);
|
||||
expect(pypiToken).toBeTypeOf('string');
|
||||
expect(rubygemsToken).toBeTypeOf('string');
|
||||
});
|
||||
|
||||
tap.test('Integration: should correctly route PyPI requests', async () => {
|
||||
const wheelData = await createPythonWheel('integration-test-py', '1.0.0');
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'POST',
|
||||
path: '/pypi/',
|
||||
headers: {
|
||||
Authorization: `Bearer ${pypiToken}`,
|
||||
'Content-Type': 'multipart/form-data',
|
||||
},
|
||||
query: {},
|
||||
body: {
|
||||
':action': 'file_upload',
|
||||
protocol_version: '1',
|
||||
name: 'integration-test-py',
|
||||
version: '1.0.0',
|
||||
filetype: 'bdist_wheel',
|
||||
pyversion: 'py3',
|
||||
metadata_version: '2.1',
|
||||
content: wheelData,
|
||||
filename: 'integration_test_py-1.0.0-py3-none-any.whl',
|
||||
},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(201);
|
||||
});
|
||||
|
||||
tap.test('Integration: should correctly route RubyGems requests', async () => {
|
||||
const gemData = await createRubyGem('integration-test-gem', '1.0.0');
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'POST',
|
||||
path: '/rubygems/api/v1/gems',
|
||||
headers: {
|
||||
Authorization: rubygemsToken,
|
||||
'Content-Type': 'application/octet-stream',
|
||||
},
|
||||
query: {},
|
||||
body: gemData,
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(201);
|
||||
});
|
||||
|
||||
tap.test('Integration: should handle /simple path for PyPI', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/simple/',
|
||||
headers: {
|
||||
Accept: 'text/html',
|
||||
},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.headers['Content-Type']).toStartWith('text/html');
|
||||
const body = await streamToBuffer(response.body);
|
||||
const text = body.toString('utf-8');
|
||||
expect(text).toContain('integration-test-py');
|
||||
});
|
||||
|
||||
tap.test('Integration: should reject PyPI token for RubyGems endpoint', async () => {
|
||||
const gemData = await createRubyGem('unauthorized-gem', '1.0.0');
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'POST',
|
||||
path: '/rubygems/api/v1/gems',
|
||||
headers: {
|
||||
Authorization: pypiToken, // Using PyPI token for RubyGems endpoint
|
||||
'Content-Type': 'application/octet-stream',
|
||||
},
|
||||
query: {},
|
||||
body: gemData,
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(401);
|
||||
});
|
||||
|
||||
tap.test('Integration: should reject RubyGems token for PyPI endpoint', async () => {
|
||||
const wheelData = await createPythonWheel('unauthorized-py', '1.0.0');
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'POST',
|
||||
path: '/pypi/',
|
||||
headers: {
|
||||
Authorization: `Bearer ${rubygemsToken}`, // Using RubyGems token for PyPI endpoint
|
||||
'Content-Type': 'multipart/form-data',
|
||||
},
|
||||
query: {},
|
||||
body: {
|
||||
':action': 'file_upload',
|
||||
protocol_version: '1',
|
||||
name: 'unauthorized-py',
|
||||
version: '1.0.0',
|
||||
filetype: 'bdist_wheel',
|
||||
pyversion: 'py3',
|
||||
metadata_version: '2.1',
|
||||
content: wheelData,
|
||||
filename: 'unauthorized_py-1.0.0-py3-none-any.whl',
|
||||
},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(401);
|
||||
});
|
||||
|
||||
tap.test('Integration: should return 404 for unknown paths', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/unknown-protocol/endpoint',
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(404);
|
||||
const body = await streamToJson(response.body);
|
||||
expect(body).toHaveProperty('error');
|
||||
expect(body.error).toEqual('NOT_FOUND');
|
||||
});
|
||||
|
||||
tap.test('Integration: should retrieve PyPI registry instance', async () => {
|
||||
const pypiRegistry = registry.getRegistry('pypi');
|
||||
|
||||
expect(pypiRegistry).toBeDefined();
|
||||
expect(pypiRegistry).not.toBeNull();
|
||||
});
|
||||
|
||||
tap.test('Integration: should retrieve RubyGems registry instance', async () => {
|
||||
const rubygemsRegistry = registry.getRegistry('rubygems');
|
||||
|
||||
expect(rubygemsRegistry).toBeDefined();
|
||||
expect(rubygemsRegistry).not.toBeNull();
|
||||
});
|
||||
|
||||
tap.test('Integration: should retrieve all other protocol instances', async () => {
|
||||
const ociRegistry = registry.getRegistry('oci');
|
||||
const npmRegistry = registry.getRegistry('npm');
|
||||
const mavenRegistry = registry.getRegistry('maven');
|
||||
const composerRegistry = registry.getRegistry('composer');
|
||||
const cargoRegistry = registry.getRegistry('cargo');
|
||||
|
||||
expect(ociRegistry).toBeDefined();
|
||||
expect(npmRegistry).toBeDefined();
|
||||
expect(mavenRegistry).toBeDefined();
|
||||
expect(composerRegistry).toBeDefined();
|
||||
expect(cargoRegistry).toBeDefined();
|
||||
});
|
||||
|
||||
tap.test('Integration: should share storage across protocols', async () => {
|
||||
const storage = registry.getStorage();
|
||||
|
||||
expect(storage).toBeDefined();
|
||||
|
||||
// Verify storage has methods for all protocols
|
||||
expect(typeof storage.getPypiPackageMetadata).toEqual('function');
|
||||
expect(typeof storage.getRubyGemsVersions).toEqual('function');
|
||||
expect(typeof storage.getNpmPackument).toEqual('function');
|
||||
expect(typeof storage.getOciBlob).toEqual('function');
|
||||
});
|
||||
|
||||
tap.test('Integration: should share auth manager across protocols', async () => {
|
||||
const authManager = registry.getAuthManager();
|
||||
|
||||
expect(authManager).toBeDefined();
|
||||
|
||||
// Verify auth manager has methods for all protocols
|
||||
expect(typeof authManager.createPypiToken).toEqual('function');
|
||||
expect(typeof authManager.createRubyGemsToken).toEqual('function');
|
||||
expect(typeof authManager.createNpmToken).toEqual('function');
|
||||
expect(typeof authManager.createOciToken).toEqual('function');
|
||||
});
|
||||
|
||||
tap.test('Integration: should handle concurrent requests to different protocols', async () => {
|
||||
const pypiRequest = registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/simple/',
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
const rubygemsRequest = registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/rubygems/versions',
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
const [pypiResponse, rubygemsResponse] = await Promise.all([pypiRequest, rubygemsRequest]);
|
||||
|
||||
expect(pypiResponse.status).toEqual(200);
|
||||
expect(rubygemsResponse.status).toEqual(200);
|
||||
});
|
||||
|
||||
tap.test('Integration: should handle package name conflicts across protocols', async () => {
|
||||
const packageName = 'conflict-test';
|
||||
|
||||
// Upload PyPI package
|
||||
const wheelData = await createPythonWheel(packageName, '1.0.0');
|
||||
const pypiResponse = await registry.handleRequest({
|
||||
method: 'POST',
|
||||
path: '/pypi/',
|
||||
headers: {
|
||||
Authorization: `Bearer ${pypiToken}`,
|
||||
'Content-Type': 'multipart/form-data',
|
||||
},
|
||||
query: {},
|
||||
body: {
|
||||
':action': 'file_upload',
|
||||
protocol_version: '1',
|
||||
name: packageName,
|
||||
version: '1.0.0',
|
||||
filetype: 'bdist_wheel',
|
||||
pyversion: 'py3',
|
||||
metadata_version: '2.1',
|
||||
content: wheelData,
|
||||
filename: `${packageName.replace(/-/g, '_')}-1.0.0-py3-none-any.whl`,
|
||||
},
|
||||
});
|
||||
|
||||
expect(pypiResponse.status).toEqual(201);
|
||||
|
||||
// Upload RubyGems package with same name
|
||||
const gemData = await createRubyGem(packageName, '1.0.0');
|
||||
const rubygemsResponse = await registry.handleRequest({
|
||||
method: 'POST',
|
||||
path: '/rubygems/api/v1/gems',
|
||||
headers: {
|
||||
Authorization: rubygemsToken,
|
||||
'Content-Type': 'application/octet-stream',
|
||||
},
|
||||
query: {},
|
||||
body: gemData,
|
||||
});
|
||||
|
||||
expect(rubygemsResponse.status).toEqual(201);
|
||||
|
||||
// Both should exist independently
|
||||
const pypiGetResponse = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: `/simple/${packageName}/`,
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
const rubygemsGetResponse = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: `/rubygems/gems/${packageName}-1.0.0.gem`,
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(pypiGetResponse.status).toEqual(200);
|
||||
expect(rubygemsGetResponse.status).toEqual(200);
|
||||
});
|
||||
|
||||
tap.test('Integration: should properly clean up resources on destroy', async () => {
|
||||
// Destroy should clean up all registries
|
||||
expect(() => registry.destroy()).not.toThrow();
|
||||
});
|
||||
|
||||
tap.postTask('cleanup registry', async () => {
|
||||
if (registry && registry.isInitialized()) {
|
||||
registry.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
export default tap.start();
|
||||
@@ -0,0 +1,275 @@
|
||||
/**
|
||||
* Integration test for smartregistry with smartstorage
|
||||
* Verifies that smartregistry works with a local S3-compatible server
|
||||
*/
|
||||
|
||||
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||
import * as smartstorageModule from '@push.rocks/smartstorage';
|
||||
import { SmartRegistry } from '../ts/classes.smartregistry.js';
|
||||
import type { IRegistryConfig } from '../ts/core/interfaces.core.js';
|
||||
import { streamToJson } from '../ts/core/helpers.stream.js';
|
||||
import * as crypto from 'crypto';
|
||||
|
||||
let s3Server: smartstorageModule.SmartStorage;
|
||||
let registry: SmartRegistry;
|
||||
|
||||
/**
|
||||
* Setup: Start smartstorage server
|
||||
*/
|
||||
tap.test('should start smartstorage server', async () => {
|
||||
s3Server = await smartstorageModule.SmartStorage.createAndStart({
|
||||
server: {
|
||||
port: 3456,
|
||||
address: '0.0.0.0',
|
||||
silent: true,
|
||||
},
|
||||
storage: {
|
||||
cleanSlate: true,
|
||||
directory: './.nogit/smartstorage-test-buckets',
|
||||
},
|
||||
logging: {
|
||||
enabled: false,
|
||||
},
|
||||
});
|
||||
|
||||
expect(s3Server).toBeDefined();
|
||||
});
|
||||
|
||||
/**
|
||||
* Setup: Create SmartRegistry with smartstorage configuration
|
||||
*/
|
||||
tap.test('should create SmartRegistry instance with smarts3 IS3Descriptor', async () => {
|
||||
const s3Descriptor = await s3Server.getStorageDescriptor();
|
||||
|
||||
const config: IRegistryConfig = {
|
||||
storage: {
|
||||
...s3Descriptor,
|
||||
bucketName: 'test-registry-smarts3',
|
||||
},
|
||||
auth: {
|
||||
jwtSecret: 'test-secret-key',
|
||||
tokenStore: 'memory',
|
||||
npmTokens: {
|
||||
enabled: true,
|
||||
},
|
||||
ociTokens: {
|
||||
enabled: true,
|
||||
realm: 'https://auth.example.com/token',
|
||||
service: 'test-registry-smarts3',
|
||||
},
|
||||
pypiTokens: {
|
||||
enabled: true,
|
||||
},
|
||||
rubygemsTokens: {
|
||||
enabled: true,
|
||||
},
|
||||
},
|
||||
npm: {
|
||||
enabled: true,
|
||||
basePath: '/npm',
|
||||
},
|
||||
oci: {
|
||||
enabled: true,
|
||||
basePath: '/oci',
|
||||
},
|
||||
pypi: {
|
||||
enabled: true,
|
||||
basePath: '/pypi',
|
||||
},
|
||||
cargo: {
|
||||
enabled: true,
|
||||
basePath: '/cargo',
|
||||
},
|
||||
};
|
||||
|
||||
registry = new SmartRegistry(config);
|
||||
await registry.init();
|
||||
|
||||
expect(registry).toBeDefined();
|
||||
});
|
||||
|
||||
/**
|
||||
* Test NPM protocol with smartstorage
|
||||
*/
|
||||
tap.test('NPM: should publish package to smarts3', async () => {
|
||||
const authManager = registry.getAuthManager();
|
||||
const userId = await authManager.authenticate({
|
||||
username: 'testuser',
|
||||
password: 'testpass',
|
||||
});
|
||||
const token = await authManager.createNpmToken(userId, false);
|
||||
|
||||
const packageData = {
|
||||
name: 'test-package-smarts3',
|
||||
'dist-tags': {
|
||||
latest: '1.0.0',
|
||||
},
|
||||
versions: {
|
||||
'1.0.0': {
|
||||
name: 'test-package-smarts3',
|
||||
version: '1.0.0',
|
||||
description: 'Test package for smarts3 integration',
|
||||
},
|
||||
},
|
||||
_attachments: {
|
||||
'test-package-smarts3-1.0.0.tgz': {
|
||||
content_type: 'application/octet-stream',
|
||||
data: Buffer.from('test tarball content').toString('base64'),
|
||||
length: 20,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'PUT',
|
||||
path: '/npm/test-package-smarts3',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${token}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
query: {},
|
||||
body: packageData,
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(201);
|
||||
});
|
||||
|
||||
tap.test('NPM: should retrieve package from smarts3', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/npm/test-package-smarts3',
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
const body = await streamToJson(response.body);
|
||||
expect(body).toHaveProperty('name');
|
||||
expect(body.name).toEqual('test-package-smarts3');
|
||||
});
|
||||
|
||||
/**
|
||||
* Test OCI protocol with smartstorage
|
||||
*/
|
||||
tap.test('OCI: should store blob in smarts3', async () => {
|
||||
const authManager = registry.getAuthManager();
|
||||
const userId = await authManager.authenticate({
|
||||
username: 'testuser',
|
||||
password: 'testpass',
|
||||
});
|
||||
const token = await authManager.createOciToken(
|
||||
userId,
|
||||
['oci:repository:test-image:push'],
|
||||
3600
|
||||
);
|
||||
|
||||
// Initiate blob upload
|
||||
const initiateResponse = await registry.handleRequest({
|
||||
method: 'POST',
|
||||
path: '/oci/test-image/blobs/uploads/',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${token}`,
|
||||
},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(initiateResponse.status).toEqual(202);
|
||||
expect(initiateResponse.headers).toHaveProperty('Location');
|
||||
|
||||
// Extract upload ID from location
|
||||
const location = initiateResponse.headers['Location'];
|
||||
const uploadId = location.split('/').pop();
|
||||
|
||||
// Upload blob data
|
||||
const blobData = Buffer.from('test blob content');
|
||||
const digest = 'sha256:' + crypto
|
||||
.createHash('sha256')
|
||||
.update(blobData)
|
||||
.digest('hex');
|
||||
|
||||
const uploadResponse = await registry.handleRequest({
|
||||
method: 'PUT',
|
||||
path: `/oci/test-image/blobs/uploads/${uploadId}`,
|
||||
headers: {
|
||||
'Authorization': `Bearer ${token}`,
|
||||
'Content-Type': 'application/octet-stream',
|
||||
},
|
||||
query: { digest },
|
||||
body: blobData,
|
||||
});
|
||||
|
||||
expect(uploadResponse.status).toEqual(201);
|
||||
});
|
||||
|
||||
/**
|
||||
* Test PyPI protocol with smartstorage
|
||||
*/
|
||||
tap.test('PyPI: should upload package to smarts3', async () => {
|
||||
const storage = registry.getStorage();
|
||||
|
||||
// Store a test package file
|
||||
const packageContent = Buffer.from('test wheel content');
|
||||
await storage.putPypiPackageFile(
|
||||
'test-package',
|
||||
'test_package-1.0.0-py3-none-any.whl',
|
||||
packageContent
|
||||
);
|
||||
|
||||
// Store metadata
|
||||
const metadata = {
|
||||
name: 'test-package',
|
||||
version: '1.0.0',
|
||||
files: [
|
||||
{
|
||||
filename: 'test_package-1.0.0-py3-none-any.whl',
|
||||
url: '/packages/test-package/test_package-1.0.0-py3-none-any.whl',
|
||||
hashes: { sha256: 'abc123' },
|
||||
},
|
||||
],
|
||||
};
|
||||
await storage.putPypiPackageMetadata('test-package', metadata);
|
||||
|
||||
// Verify stored
|
||||
const retrievedMetadata = await storage.getPypiPackageMetadata('test-package');
|
||||
expect(retrievedMetadata).toBeDefined();
|
||||
expect(retrievedMetadata.name).toEqual('test-package');
|
||||
});
|
||||
|
||||
/**
|
||||
* Test Cargo protocol with smartstorage
|
||||
*/
|
||||
tap.test('Cargo: should store crate in smarts3', async () => {
|
||||
const storage = registry.getStorage();
|
||||
|
||||
// Store a test crate index entry
|
||||
const indexEntry = {
|
||||
name: 'test-crate',
|
||||
vers: '1.0.0',
|
||||
deps: [],
|
||||
cksum: 'abc123',
|
||||
features: {},
|
||||
yanked: false,
|
||||
};
|
||||
|
||||
await storage.putCargoIndex('test-crate', [indexEntry]);
|
||||
|
||||
// Store the actual .crate file
|
||||
const crateContent = Buffer.from('test crate tarball');
|
||||
await storage.putCargoCrate('test-crate', '1.0.0', crateContent);
|
||||
|
||||
// Verify stored
|
||||
const retrievedIndex = await storage.getCargoIndex('test-crate');
|
||||
expect(retrievedIndex).toBeDefined();
|
||||
expect(retrievedIndex.length).toEqual(1);
|
||||
expect(retrievedIndex[0].name).toEqual('test-crate');
|
||||
});
|
||||
|
||||
/**
|
||||
* Cleanup: Stop smartstorage server
|
||||
*/
|
||||
tap.test('should stop smartstorage server', async () => {
|
||||
registry.destroy();
|
||||
await s3Server.stop();
|
||||
});
|
||||
|
||||
export default tap.start();
|
||||
@@ -0,0 +1,484 @@
|
||||
/**
|
||||
* Native Maven CLI Testing
|
||||
* Tests the Maven registry implementation using the actual mvn CLI
|
||||
*/
|
||||
|
||||
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||
import { tapNodeTools } from '@git.zone/tstest/tapbundle_serverside';
|
||||
import { SmartRegistry } from '../ts/index.js';
|
||||
import { createTestRegistry, createTestTokens, createTestPom, createTestJar } from './helpers/registry.js';
|
||||
import type { IRequestContext, IResponse } from '../ts/core/interfaces.core.js';
|
||||
import * as http from 'http';
|
||||
import * as url from 'url';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
// Test context
|
||||
let registry: SmartRegistry;
|
||||
let server: http.Server;
|
||||
let registryUrl: string;
|
||||
let registryPort: number;
|
||||
let mavenToken: string;
|
||||
let testDir: string;
|
||||
let m2Dir: string;
|
||||
|
||||
/**
|
||||
* Create HTTP server wrapper around SmartRegistry
|
||||
*/
|
||||
async function createHttpServer(
|
||||
registryInstance: SmartRegistry,
|
||||
port: number
|
||||
): Promise<{ server: http.Server; url: string }> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const httpServer = http.createServer(async (req, res) => {
|
||||
try {
|
||||
// Parse request
|
||||
const parsedUrl = url.parse(req.url || '', true);
|
||||
const pathname = parsedUrl.pathname || '/';
|
||||
const query = parsedUrl.query;
|
||||
|
||||
// Read body
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of req) {
|
||||
chunks.push(chunk);
|
||||
}
|
||||
const bodyBuffer = Buffer.concat(chunks);
|
||||
|
||||
// Parse body based on content type
|
||||
let body: any;
|
||||
if (bodyBuffer.length > 0) {
|
||||
const contentType = req.headers['content-type'] || '';
|
||||
if (contentType.includes('application/json')) {
|
||||
try {
|
||||
body = JSON.parse(bodyBuffer.toString('utf-8'));
|
||||
} catch (error) {
|
||||
body = bodyBuffer;
|
||||
}
|
||||
} else {
|
||||
body = bodyBuffer;
|
||||
}
|
||||
}
|
||||
|
||||
// Convert to IRequestContext
|
||||
const context: IRequestContext = {
|
||||
method: req.method || 'GET',
|
||||
path: pathname,
|
||||
headers: req.headers as Record<string, string>,
|
||||
query: query as Record<string, string>,
|
||||
body: body,
|
||||
};
|
||||
|
||||
// Handle request
|
||||
const response: IResponse = await registryInstance.handleRequest(context);
|
||||
|
||||
// Convert IResponse to HTTP response
|
||||
res.statusCode = response.status;
|
||||
|
||||
// Set headers
|
||||
for (const [key, value] of Object.entries(response.headers || {})) {
|
||||
res.setHeader(key, value);
|
||||
}
|
||||
|
||||
// Send body (response.body is always ReadableStream<Uint8Array> or undefined)
|
||||
if (response.body) {
|
||||
const { Readable } = await import('stream');
|
||||
Readable.fromWeb(response.body).pipe(res);
|
||||
} else {
|
||||
res.end();
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Server error:', error);
|
||||
res.statusCode = 500;
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.end(JSON.stringify({ error: 'INTERNAL_ERROR', message: String(error) }));
|
||||
}
|
||||
});
|
||||
|
||||
httpServer.listen(port, () => {
|
||||
const serverUrl = `http://localhost:${port}`;
|
||||
resolve({ server: httpServer, url: serverUrl });
|
||||
});
|
||||
|
||||
httpServer.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Setup Maven settings.xml for authentication
|
||||
*/
|
||||
function setupMavenSettings(
|
||||
token: string,
|
||||
m2DirArg: string,
|
||||
serverUrl: string
|
||||
): string {
|
||||
fs.mkdirSync(m2DirArg, { recursive: true });
|
||||
|
||||
const settingsXml = `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
|
||||
http://maven.apache.org/xsd/settings-1.0.0.xsd">
|
||||
<servers>
|
||||
<server>
|
||||
<id>test-registry</id>
|
||||
<username>testuser</username>
|
||||
<password>${token}</password>
|
||||
</server>
|
||||
</servers>
|
||||
<profiles>
|
||||
<profile>
|
||||
<id>test-registry</id>
|
||||
<repositories>
|
||||
<repository>
|
||||
<id>test-registry</id>
|
||||
<url>${serverUrl}/maven</url>
|
||||
<releases>
|
||||
<enabled>true</enabled>
|
||||
</releases>
|
||||
<snapshots>
|
||||
<enabled>true</enabled>
|
||||
</snapshots>
|
||||
</repository>
|
||||
</repositories>
|
||||
</profile>
|
||||
</profiles>
|
||||
<activeProfiles>
|
||||
<activeProfile>test-registry</activeProfile>
|
||||
</activeProfiles>
|
||||
</settings>
|
||||
`;
|
||||
|
||||
const settingsPath = path.join(m2DirArg, 'settings.xml');
|
||||
fs.writeFileSync(settingsPath, settingsXml, 'utf-8');
|
||||
|
||||
return settingsPath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a minimal Maven project for testing
|
||||
*/
|
||||
function createMavenProject(
|
||||
projectDir: string,
|
||||
groupId: string,
|
||||
artifactId: string,
|
||||
version: string,
|
||||
registryUrl: string
|
||||
): void {
|
||||
fs.mkdirSync(projectDir, { recursive: true });
|
||||
|
||||
const pomXml = `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
|
||||
http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>${groupId}</groupId>
|
||||
<artifactId>${artifactId}</artifactId>
|
||||
<version>${version}</version>
|
||||
<packaging>jar</packaging>
|
||||
<name>${artifactId}</name>
|
||||
<description>Test Maven project for SmartRegistry CLI tests</description>
|
||||
|
||||
<distributionManagement>
|
||||
<repository>
|
||||
<id>test-registry</id>
|
||||
<url>${registryUrl}/maven</url>
|
||||
</repository>
|
||||
<snapshotRepository>
|
||||
<id>test-registry</id>
|
||||
<url>${registryUrl}/maven</url>
|
||||
</snapshotRepository>
|
||||
</distributionManagement>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>3.8.1</version>
|
||||
<configuration>
|
||||
<source>1.8</source>
|
||||
<target>1.8</target>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
`;
|
||||
|
||||
fs.writeFileSync(path.join(projectDir, 'pom.xml'), pomXml, 'utf-8');
|
||||
|
||||
// Create minimal Java source
|
||||
const srcDir = path.join(projectDir, 'src', 'main', 'java', 'com', 'test');
|
||||
fs.mkdirSync(srcDir, { recursive: true });
|
||||
|
||||
const javaSource = `package com.test;
|
||||
|
||||
public class Main {
|
||||
public static void main(String[] args) {
|
||||
System.out.println("Hello from SmartRegistry test!");
|
||||
}
|
||||
}
|
||||
`;
|
||||
fs.writeFileSync(path.join(srcDir, 'Main.java'), javaSource, 'utf-8');
|
||||
}
|
||||
|
||||
/**
|
||||
* Run Maven command with custom settings
|
||||
*/
|
||||
async function runMavenCommand(
|
||||
command: string,
|
||||
cwd: string
|
||||
): Promise<{ stdout: string; stderr: string; exitCode: number }> {
|
||||
const settingsPath = path.join(m2Dir, 'settings.xml');
|
||||
const fullCommand = `cd "${cwd}" && mvn -s "${settingsPath}" ${command}`;
|
||||
|
||||
try {
|
||||
const result = await tapNodeTools.runCommand(fullCommand);
|
||||
return {
|
||||
stdout: result.stdout || '',
|
||||
stderr: result.stderr || '',
|
||||
exitCode: result.exitCode || 0,
|
||||
};
|
||||
} catch (error: any) {
|
||||
return {
|
||||
stdout: error.stdout || '',
|
||||
stderr: error.stderr || String(error),
|
||||
exitCode: error.exitCode || 1,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleanup test directory
|
||||
*/
|
||||
function cleanupTestDir(dir: string): void {
|
||||
if (fs.existsSync(dir)) {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// TESTS
|
||||
// ========================================================================
|
||||
|
||||
tap.test('Maven CLI: should verify mvn is installed', async () => {
|
||||
try {
|
||||
const result = await tapNodeTools.runCommand('mvn -version');
|
||||
console.log('Maven version output:', result.stdout.substring(0, 200));
|
||||
expect(result.exitCode).toEqual(0);
|
||||
} catch (error) {
|
||||
console.log('Maven CLI not available, skipping native CLI tests');
|
||||
// Skip remaining tests if Maven is not installed
|
||||
tap.skip.test('Maven CLI: remaining tests skipped - mvn not available');
|
||||
return;
|
||||
}
|
||||
});
|
||||
|
||||
tap.test('Maven CLI: should setup registry and HTTP server', async () => {
|
||||
// Use port 37000 (avoids conflicts with other tests)
|
||||
registryPort = 37000;
|
||||
|
||||
// Create registry with correct registryUrl for CLI tests
|
||||
registry = await createTestRegistry({ registryUrl: `http://localhost:${registryPort}` });
|
||||
const tokens = await createTestTokens(registry);
|
||||
mavenToken = tokens.mavenToken;
|
||||
|
||||
expect(registry).toBeInstanceOf(SmartRegistry);
|
||||
expect(mavenToken).toBeTypeOf('string');
|
||||
const serverSetup = await createHttpServer(registry, registryPort);
|
||||
server = serverSetup.server;
|
||||
registryUrl = serverSetup.url;
|
||||
|
||||
expect(server).toBeDefined();
|
||||
expect(registryUrl).toEqual(`http://localhost:${registryPort}`);
|
||||
|
||||
// Setup test directory
|
||||
testDir = path.join(process.cwd(), '.nogit', 'test-maven-cli');
|
||||
cleanupTestDir(testDir);
|
||||
fs.mkdirSync(testDir, { recursive: true });
|
||||
|
||||
// Setup .m2 directory
|
||||
m2Dir = path.join(testDir, '.m2');
|
||||
fs.mkdirSync(m2Dir, { recursive: true });
|
||||
|
||||
// Setup Maven settings
|
||||
const settingsPath = setupMavenSettings(mavenToken, m2Dir, registryUrl);
|
||||
expect(fs.existsSync(settingsPath)).toEqual(true);
|
||||
});
|
||||
|
||||
tap.test('Maven CLI: should verify server is responding', async () => {
|
||||
// Check server is up by doing a direct HTTP request
|
||||
const response = await fetch(`${registryUrl}/maven/`);
|
||||
expect(response.status).toBeGreaterThanOrEqual(200);
|
||||
expect(response.status).toBeLessThan(500);
|
||||
});
|
||||
|
||||
tap.test('Maven CLI: should deploy a JAR artifact', async () => {
|
||||
const groupId = 'com.test';
|
||||
const artifactId = 'test-artifact';
|
||||
const version = '1.0.0';
|
||||
|
||||
const projectDir = path.join(testDir, 'test-project');
|
||||
createMavenProject(projectDir, groupId, artifactId, version, registryUrl);
|
||||
|
||||
// Build and deploy
|
||||
const result = await runMavenCommand('clean package deploy -DskipTests', projectDir);
|
||||
console.log('mvn deploy output:', result.stdout.substring(0, 500));
|
||||
console.log('mvn deploy stderr:', result.stderr.substring(0, 500));
|
||||
|
||||
expect(result.exitCode).toEqual(0);
|
||||
});
|
||||
|
||||
tap.test('Maven CLI: should verify artifact in registry via API', async () => {
|
||||
const groupId = 'com.test';
|
||||
const artifactId = 'test-artifact';
|
||||
const version = '1.0.0';
|
||||
|
||||
// Maven path: /maven/{groupId path}/{artifactId}/{version}/{artifactId}-{version}.jar
|
||||
const jarPath = `/maven/com/test/${artifactId}/${version}/${artifactId}-${version}.jar`;
|
||||
const response = await fetch(`${registryUrl}${jarPath}`, {
|
||||
headers: { Authorization: `Bearer ${mavenToken}` },
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
|
||||
const jarData = await response.arrayBuffer();
|
||||
expect(jarData.byteLength).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
tap.test('Maven CLI: should verify POM in registry', async () => {
|
||||
const groupId = 'com.test';
|
||||
const artifactId = 'test-artifact';
|
||||
const version = '1.0.0';
|
||||
|
||||
const pomPath = `/maven/com/test/${artifactId}/${version}/${artifactId}-${version}.pom`;
|
||||
const response = await fetch(`${registryUrl}${pomPath}`, {
|
||||
headers: { Authorization: `Bearer ${mavenToken}` },
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
|
||||
const pomContent = await response.text();
|
||||
expect(pomContent).toContain(groupId);
|
||||
expect(pomContent).toContain(artifactId);
|
||||
expect(pomContent).toContain(version);
|
||||
});
|
||||
|
||||
tap.test('Maven CLI: should verify checksums exist', async () => {
|
||||
const artifactId = 'test-artifact';
|
||||
const version = '1.0.0';
|
||||
|
||||
// Check JAR checksums
|
||||
const basePath = `/maven/com/test/${artifactId}/${version}/${artifactId}-${version}.jar`;
|
||||
|
||||
// MD5
|
||||
const md5Response = await fetch(`${registryUrl}${basePath}.md5`, {
|
||||
headers: { Authorization: `Bearer ${mavenToken}` },
|
||||
});
|
||||
expect(md5Response.status).toEqual(200);
|
||||
|
||||
// SHA1
|
||||
const sha1Response = await fetch(`${registryUrl}${basePath}.sha1`, {
|
||||
headers: { Authorization: `Bearer ${mavenToken}` },
|
||||
});
|
||||
expect(sha1Response.status).toEqual(200);
|
||||
});
|
||||
|
||||
tap.test('Maven CLI: should deploy second version', async () => {
|
||||
const groupId = 'com.test';
|
||||
const artifactId = 'test-artifact';
|
||||
const version = '2.0.0';
|
||||
|
||||
const projectDir = path.join(testDir, 'test-project-v2');
|
||||
createMavenProject(projectDir, groupId, artifactId, version, registryUrl);
|
||||
|
||||
const result = await runMavenCommand('clean package deploy -DskipTests', projectDir);
|
||||
console.log('mvn deploy v2 output:', result.stdout.substring(0, 500));
|
||||
|
||||
expect(result.exitCode).toEqual(0);
|
||||
});
|
||||
|
||||
tap.test('Maven CLI: should verify metadata.xml exists', async () => {
|
||||
const artifactId = 'test-artifact';
|
||||
|
||||
// Maven metadata is stored at /maven/{groupId path}/{artifactId}/maven-metadata.xml
|
||||
const metadataPath = `/maven/com/test/${artifactId}/maven-metadata.xml`;
|
||||
const response = await fetch(`${registryUrl}${metadataPath}`, {
|
||||
headers: { Authorization: `Bearer ${mavenToken}` },
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
|
||||
const metadataXml = await response.text();
|
||||
expect(metadataXml).toContain(artifactId);
|
||||
expect(metadataXml).toContain('1.0.0');
|
||||
expect(metadataXml).toContain('2.0.0');
|
||||
});
|
||||
|
||||
tap.test('Maven CLI: should resolve dependency from registry', async () => {
|
||||
const groupId = 'com.consumer';
|
||||
const artifactId = 'consumer-app';
|
||||
const version = '1.0.0';
|
||||
|
||||
const projectDir = path.join(testDir, 'consumer-project');
|
||||
fs.mkdirSync(projectDir, { recursive: true });
|
||||
|
||||
// Create a consumer project that depends on our test artifact
|
||||
const pomXml = `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
|
||||
http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>${groupId}</groupId>
|
||||
<artifactId>${artifactId}</artifactId>
|
||||
<version>${version}</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<repositories>
|
||||
<repository>
|
||||
<id>test-registry</id>
|
||||
<url>${registryUrl}/maven</url>
|
||||
</repository>
|
||||
</repositories>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.test</groupId>
|
||||
<artifactId>test-artifact</artifactId>
|
||||
<version>1.0.0</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
`;
|
||||
|
||||
fs.writeFileSync(path.join(projectDir, 'pom.xml'), pomXml, 'utf-8');
|
||||
|
||||
// Try to resolve dependencies
|
||||
const result = await runMavenCommand('dependency:resolve', projectDir);
|
||||
console.log('mvn dependency:resolve output:', result.stdout.substring(0, 500));
|
||||
|
||||
expect(result.exitCode).toEqual(0);
|
||||
});
|
||||
|
||||
tap.postTask('cleanup maven cli tests', async () => {
|
||||
// Stop server
|
||||
if (server) {
|
||||
await new Promise<void>((resolve) => {
|
||||
server.close(() => resolve());
|
||||
});
|
||||
}
|
||||
|
||||
// Cleanup test directory
|
||||
if (testDir) {
|
||||
cleanupTestDir(testDir);
|
||||
}
|
||||
|
||||
// Destroy registry
|
||||
if (registry) {
|
||||
registry.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
export default tap.start();
|
||||
@@ -0,0 +1,392 @@
|
||||
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||
import { SmartRegistry } from '../ts/index.js';
|
||||
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
|
||||
import {
|
||||
createTestRegistry,
|
||||
createTestTokens,
|
||||
createTestPom,
|
||||
createTestJar,
|
||||
calculateMavenChecksums,
|
||||
} from './helpers/registry.js';
|
||||
|
||||
let registry: SmartRegistry;
|
||||
let mavenToken: string;
|
||||
let userId: string;
|
||||
|
||||
// Test data
|
||||
const testGroupId = 'com.example.test';
|
||||
const testArtifactId = 'test-artifact';
|
||||
const testVersion = '1.0.0';
|
||||
const testJarData = createTestJar();
|
||||
const testPomData = Buffer.from(
|
||||
createTestPom(testGroupId, testArtifactId, testVersion),
|
||||
'utf-8'
|
||||
);
|
||||
|
||||
tap.test('Maven: should create registry instance', async () => {
|
||||
registry = await createTestRegistry();
|
||||
const tokens = await createTestTokens(registry);
|
||||
mavenToken = tokens.mavenToken;
|
||||
userId = tokens.userId;
|
||||
|
||||
expect(registry).toBeInstanceOf(SmartRegistry);
|
||||
expect(mavenToken).toBeTypeOf('string');
|
||||
|
||||
// Clean up any existing metadata from previous test runs
|
||||
const storage = registry.getStorage();
|
||||
try {
|
||||
await storage.deleteMavenMetadata(testGroupId, testArtifactId);
|
||||
} catch (error) {
|
||||
// Ignore error if metadata doesn't exist
|
||||
}
|
||||
});
|
||||
|
||||
tap.test('Maven: should upload POM file (PUT /{groupPath}/{artifactId}/{version}/*.pom)', async () => {
|
||||
const groupPath = testGroupId.replace(/\./g, '/');
|
||||
const pomFilename = `${testArtifactId}-${testVersion}.pom`;
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'PUT',
|
||||
path: `/maven/${groupPath}/${testArtifactId}/${testVersion}/${pomFilename}`,
|
||||
headers: {
|
||||
Authorization: `Bearer ${mavenToken}`,
|
||||
'Content-Type': 'application/xml',
|
||||
},
|
||||
query: {},
|
||||
body: testPomData,
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(201);
|
||||
});
|
||||
|
||||
tap.test('Maven: should upload JAR file (PUT /{groupPath}/{artifactId}/{version}/*.jar)', async () => {
|
||||
const groupPath = testGroupId.replace(/\./g, '/');
|
||||
const jarFilename = `${testArtifactId}-${testVersion}.jar`;
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'PUT',
|
||||
path: `/maven/${groupPath}/${testArtifactId}/${testVersion}/${jarFilename}`,
|
||||
headers: {
|
||||
Authorization: `Bearer ${mavenToken}`,
|
||||
'Content-Type': 'application/java-archive',
|
||||
},
|
||||
query: {},
|
||||
body: testJarData,
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(201);
|
||||
});
|
||||
|
||||
tap.test('Maven: should retrieve uploaded POM file (GET)', async () => {
|
||||
const groupPath = testGroupId.replace(/\./g, '/');
|
||||
const pomFilename = `${testArtifactId}-${testVersion}.pom`;
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: `/maven/${groupPath}/${testArtifactId}/${testVersion}/${pomFilename}`,
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
const body = await streamToBuffer(response.body);
|
||||
expect(body).toBeInstanceOf(Buffer);
|
||||
expect(body.toString('utf-8')).toContain(testGroupId);
|
||||
expect(body.toString('utf-8')).toContain(testArtifactId);
|
||||
expect(body.toString('utf-8')).toContain(testVersion);
|
||||
expect(response.headers['Content-Type']).toEqual('application/xml');
|
||||
});
|
||||
|
||||
tap.test('Maven: should retrieve uploaded JAR file (GET)', async () => {
|
||||
const groupPath = testGroupId.replace(/\./g, '/');
|
||||
const jarFilename = `${testArtifactId}-${testVersion}.jar`;
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: `/maven/${groupPath}/${testArtifactId}/${testVersion}/${jarFilename}`,
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
const body = await streamToBuffer(response.body);
|
||||
expect(body).toBeInstanceOf(Buffer);
|
||||
expect(response.headers['Content-Type']).toEqual('application/java-archive');
|
||||
});
|
||||
|
||||
tap.test('Maven: should retrieve MD5 checksum for JAR (GET *.jar.md5)', async () => {
|
||||
const groupPath = testGroupId.replace(/\./g, '/');
|
||||
const jarFilename = `${testArtifactId}-${testVersion}.jar`;
|
||||
const checksums = calculateMavenChecksums(testJarData);
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: `/maven/${groupPath}/${testArtifactId}/${testVersion}/${jarFilename}.md5`,
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
const body = await streamToBuffer(response.body);
|
||||
expect(body).toBeInstanceOf(Buffer);
|
||||
expect(body.toString('utf-8')).toEqual(checksums.md5);
|
||||
expect(response.headers['Content-Type']).toEqual('text/plain');
|
||||
});
|
||||
|
||||
tap.test('Maven: should retrieve SHA1 checksum for JAR (GET *.jar.sha1)', async () => {
|
||||
const groupPath = testGroupId.replace(/\./g, '/');
|
||||
const jarFilename = `${testArtifactId}-${testVersion}.jar`;
|
||||
const checksums = calculateMavenChecksums(testJarData);
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: `/maven/${groupPath}/${testArtifactId}/${testVersion}/${jarFilename}.sha1`,
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
const body = await streamToBuffer(response.body);
|
||||
expect(body).toBeInstanceOf(Buffer);
|
||||
expect(body.toString('utf-8')).toEqual(checksums.sha1);
|
||||
expect(response.headers['Content-Type']).toEqual('text/plain');
|
||||
});
|
||||
|
||||
tap.test('Maven: should retrieve SHA256 checksum for JAR (GET *.jar.sha256)', async () => {
|
||||
const groupPath = testGroupId.replace(/\./g, '/');
|
||||
const jarFilename = `${testArtifactId}-${testVersion}.jar`;
|
||||
const checksums = calculateMavenChecksums(testJarData);
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: `/maven/${groupPath}/${testArtifactId}/${testVersion}/${jarFilename}.sha256`,
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
const body = await streamToBuffer(response.body);
|
||||
expect(body).toBeInstanceOf(Buffer);
|
||||
expect(body.toString('utf-8')).toEqual(checksums.sha256);
|
||||
expect(response.headers['Content-Type']).toEqual('text/plain');
|
||||
});
|
||||
|
||||
tap.test('Maven: should retrieve SHA512 checksum for JAR (GET *.jar.sha512)', async () => {
|
||||
const groupPath = testGroupId.replace(/\./g, '/');
|
||||
const jarFilename = `${testArtifactId}-${testVersion}.jar`;
|
||||
const checksums = calculateMavenChecksums(testJarData);
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: `/maven/${groupPath}/${testArtifactId}/${testVersion}/${jarFilename}.sha512`,
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
const body = await streamToBuffer(response.body);
|
||||
expect(body).toBeInstanceOf(Buffer);
|
||||
expect(body.toString('utf-8')).toEqual(checksums.sha512);
|
||||
expect(response.headers['Content-Type']).toEqual('text/plain');
|
||||
});
|
||||
|
||||
tap.test('Maven: should retrieve maven-metadata.xml (GET)', async () => {
|
||||
const groupPath = testGroupId.replace(/\./g, '/');
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: `/maven/${groupPath}/${testArtifactId}/maven-metadata.xml`,
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
const body = await streamToBuffer(response.body);
|
||||
expect(body).toBeInstanceOf(Buffer);
|
||||
const xml = body.toString('utf-8');
|
||||
expect(xml).toContain('<groupId>');
|
||||
expect(xml).toContain('<artifactId>');
|
||||
expect(xml).toContain('<version>1.0.0</version>');
|
||||
expect(xml).toContain('<latest>1.0.0</latest>');
|
||||
expect(xml).toContain('<release>1.0.0</release>');
|
||||
expect(response.headers['Content-Type']).toEqual('application/xml');
|
||||
});
|
||||
|
||||
tap.test('Maven: should upload a second version and update metadata', async () => {
|
||||
const groupPath = testGroupId.replace(/\./g, '/');
|
||||
const newVersion = '2.0.0';
|
||||
const pomFilename = `${testArtifactId}-${newVersion}.pom`;
|
||||
const jarFilename = `${testArtifactId}-${newVersion}.jar`;
|
||||
const newPomData = Buffer.from(
|
||||
createTestPom(testGroupId, testArtifactId, newVersion),
|
||||
'utf-8'
|
||||
);
|
||||
|
||||
// Upload POM
|
||||
await registry.handleRequest({
|
||||
method: 'PUT',
|
||||
path: `/maven/${groupPath}/${testArtifactId}/${newVersion}/${pomFilename}`,
|
||||
headers: {
|
||||
Authorization: `Bearer ${mavenToken}`,
|
||||
'Content-Type': 'application/xml',
|
||||
},
|
||||
query: {},
|
||||
body: newPomData,
|
||||
});
|
||||
|
||||
// Upload JAR
|
||||
await registry.handleRequest({
|
||||
method: 'PUT',
|
||||
path: `/maven/${groupPath}/${testArtifactId}/${newVersion}/${jarFilename}`,
|
||||
headers: {
|
||||
Authorization: `Bearer ${mavenToken}`,
|
||||
'Content-Type': 'application/java-archive',
|
||||
},
|
||||
query: {},
|
||||
body: testJarData,
|
||||
});
|
||||
|
||||
// Retrieve metadata and verify both versions are present
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: `/maven/${groupPath}/${testArtifactId}/maven-metadata.xml`,
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
const metaBody = await streamToBuffer(response.body);
|
||||
const xml = metaBody.toString('utf-8');
|
||||
expect(xml).toContain('<version>1.0.0</version>');
|
||||
expect(xml).toContain('<version>2.0.0</version>');
|
||||
expect(xml).toContain('<latest>2.0.0</latest>');
|
||||
expect(xml).toContain('<release>2.0.0</release>');
|
||||
});
|
||||
|
||||
tap.test('Maven: should upload WAR file with correct content type', async () => {
|
||||
const groupPath = testGroupId.replace(/\./g, '/');
|
||||
const warVersion = '1.0.0-war';
|
||||
const warFilename = `${testArtifactId}-${warVersion}.war`;
|
||||
const warData = Buffer.from('fake war content', 'utf-8');
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'PUT',
|
||||
path: `/maven/${groupPath}/${testArtifactId}/${warVersion}/${warFilename}`,
|
||||
headers: {
|
||||
Authorization: `Bearer ${mavenToken}`,
|
||||
'Content-Type': 'application/x-webarchive',
|
||||
},
|
||||
query: {},
|
||||
body: warData,
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(201);
|
||||
});
|
||||
|
||||
tap.test('Maven: should return 404 for non-existent artifact', async () => {
|
||||
const groupPath = 'com/example/nonexistent';
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: `/maven/${groupPath}/fake-artifact/1.0.0/fake-artifact-1.0.0.jar`,
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(404);
|
||||
const body = await streamToJson(response.body);
|
||||
expect(body).toHaveProperty('error');
|
||||
});
|
||||
|
||||
tap.test('Maven: should return 401 for unauthorized upload', async () => {
|
||||
const groupPath = testGroupId.replace(/\./g, '/');
|
||||
const jarFilename = `${testArtifactId}-3.0.0.jar`;
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'PUT',
|
||||
path: `/maven/${groupPath}/${testArtifactId}/3.0.0/${jarFilename}`,
|
||||
headers: {
|
||||
// No authorization header
|
||||
'Content-Type': 'application/java-archive',
|
||||
},
|
||||
query: {},
|
||||
body: testJarData,
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(401);
|
||||
const body = await streamToJson(response.body);
|
||||
expect(body).toHaveProperty('error');
|
||||
});
|
||||
|
||||
tap.test('Maven: should reject POM upload with mismatched GAV', async () => {
|
||||
const groupPath = 'com/mismatch/test';
|
||||
const pomFilename = `different-artifact-1.0.0.pom`;
|
||||
// POM contains different GAV than the path
|
||||
const mismatchedPom = Buffer.from(
|
||||
createTestPom('com.other.group', 'other-artifact', '1.0.0'),
|
||||
'utf-8'
|
||||
);
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'PUT',
|
||||
path: `/maven/${groupPath}/different-artifact/1.0.0/${pomFilename}`,
|
||||
headers: {
|
||||
Authorization: `Bearer ${mavenToken}`,
|
||||
'Content-Type': 'application/xml',
|
||||
},
|
||||
query: {},
|
||||
body: mismatchedPom,
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(400);
|
||||
const body = await streamToJson(response.body);
|
||||
expect(body).toHaveProperty('error');
|
||||
});
|
||||
|
||||
tap.test('Maven: should delete an artifact (DELETE)', async () => {
|
||||
const groupPath = testGroupId.replace(/\./g, '/');
|
||||
const jarFilename = `${testArtifactId}-${testVersion}.jar`;
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'DELETE',
|
||||
path: `/maven/${groupPath}/${testArtifactId}/${testVersion}/${jarFilename}`,
|
||||
headers: {
|
||||
Authorization: `Bearer ${mavenToken}`,
|
||||
},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(204); // 204 No Content is correct for DELETE
|
||||
|
||||
// Verify artifact was deleted
|
||||
const getResponse = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: `/maven/${groupPath}/${testArtifactId}/${testVersion}/${jarFilename}`,
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(getResponse.status).toEqual(404);
|
||||
});
|
||||
|
||||
tap.test('Maven: should return 404 for checksum of deleted artifact', async () => {
|
||||
const groupPath = testGroupId.replace(/\./g, '/');
|
||||
const jarFilename = `${testArtifactId}-${testVersion}.jar`;
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: `/maven/${groupPath}/${testArtifactId}/${testVersion}/${jarFilename}.md5`,
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(404);
|
||||
});
|
||||
|
||||
tap.postTask('cleanup registry', async () => {
|
||||
if (registry) {
|
||||
registry.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
export default tap.start();
|
||||
@@ -0,0 +1,387 @@
|
||||
/**
|
||||
* Native npm CLI Testing
|
||||
* Tests the NPM registry implementation using the actual npm CLI
|
||||
*/
|
||||
|
||||
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||
import { tapNodeTools } from '@git.zone/tstest/tapbundle_serverside';
|
||||
import { SmartRegistry } from '../ts/index.js';
|
||||
import { createTestRegistry, createTestTokens, cleanupS3Bucket } from './helpers/registry.js';
|
||||
import type { IRequestContext, IResponse } from '../ts/core/interfaces.core.js';
|
||||
import * as http from 'http';
|
||||
import * as url from 'url';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
// Test state
|
||||
let registry: SmartRegistry;
|
||||
let server: http.Server;
|
||||
let registryUrl: string;
|
||||
let registryPort: number;
|
||||
let npmToken: string;
|
||||
let testDir: string;
|
||||
let npmrcPath: string;
|
||||
|
||||
/**
|
||||
* Create HTTP server wrapper around SmartRegistry
|
||||
*/
|
||||
async function createHttpServer(
|
||||
registryInstance: SmartRegistry,
|
||||
port: number
|
||||
): Promise<{ server: http.Server; url: string }> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const httpServer = http.createServer(async (req, res) => {
|
||||
try {
|
||||
const parsedUrl = new url.URL(req.url || '/', `http://localhost:${port}`);
|
||||
const pathname = parsedUrl.pathname;
|
||||
const query: Record<string, string> = {};
|
||||
parsedUrl.searchParams.forEach((value, key) => {
|
||||
query[key] = value;
|
||||
});
|
||||
|
||||
// Read body
|
||||
let body: any = undefined;
|
||||
if (req.method === 'PUT' || req.method === 'POST' || req.method === 'PATCH') {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of req) {
|
||||
chunks.push(Buffer.from(chunk));
|
||||
}
|
||||
const bodyBuffer = Buffer.concat(chunks);
|
||||
|
||||
const contentType = req.headers['content-type'] || '';
|
||||
if (contentType.includes('application/json')) {
|
||||
try {
|
||||
body = JSON.parse(bodyBuffer.toString('utf-8'));
|
||||
} catch (error) {
|
||||
body = bodyBuffer;
|
||||
}
|
||||
} else {
|
||||
body = bodyBuffer;
|
||||
}
|
||||
}
|
||||
|
||||
// Convert to IRequestContext
|
||||
const context: IRequestContext = {
|
||||
method: req.method || 'GET',
|
||||
path: pathname,
|
||||
headers: req.headers as Record<string, string>,
|
||||
query: query as Record<string, string>,
|
||||
body: body,
|
||||
};
|
||||
|
||||
// Handle request
|
||||
const response: IResponse = await registryInstance.handleRequest(context);
|
||||
|
||||
// Convert IResponse to HTTP response
|
||||
res.statusCode = response.status;
|
||||
|
||||
// Set headers
|
||||
for (const [key, value] of Object.entries(response.headers || {})) {
|
||||
res.setHeader(key, value);
|
||||
}
|
||||
|
||||
// Send body (response.body is always ReadableStream<Uint8Array> or undefined)
|
||||
if (response.body) {
|
||||
const { Readable } = await import('stream');
|
||||
Readable.fromWeb(response.body).pipe(res);
|
||||
} else {
|
||||
res.end();
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Server error:', error);
|
||||
res.statusCode = 500;
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.end(JSON.stringify({ error: 'INTERNAL_ERROR', message: String(error) }));
|
||||
}
|
||||
});
|
||||
|
||||
httpServer.listen(port, () => {
|
||||
const serverUrl = `http://localhost:${port}`;
|
||||
resolve({ server: httpServer, url: serverUrl });
|
||||
});
|
||||
|
||||
httpServer.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Setup .npmrc configuration
|
||||
*/
|
||||
function setupNpmrc(registryUrlArg: string, token: string, testDirArg: string): string {
|
||||
const npmrcContent = `registry=${registryUrlArg}/npm/
|
||||
//localhost:${registryPort}/npm/:_authToken=${token}
|
||||
`;
|
||||
|
||||
const npmrcFilePath = path.join(testDirArg, '.npmrc');
|
||||
fs.writeFileSync(npmrcFilePath, npmrcContent, 'utf-8');
|
||||
return npmrcFilePath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a test package
|
||||
*/
|
||||
function createTestPackage(
|
||||
packageName: string,
|
||||
version: string,
|
||||
targetDir: string
|
||||
): string {
|
||||
const packageDir = path.join(targetDir, packageName.replace(/\//g, '-'));
|
||||
fs.mkdirSync(packageDir, { recursive: true });
|
||||
|
||||
// Create package.json
|
||||
const packageJson = {
|
||||
name: packageName,
|
||||
version: version,
|
||||
description: `Test package ${packageName}`,
|
||||
main: 'index.js',
|
||||
scripts: {},
|
||||
};
|
||||
|
||||
fs.writeFileSync(
|
||||
path.join(packageDir, 'package.json'),
|
||||
JSON.stringify(packageJson, null, 2),
|
||||
'utf-8'
|
||||
);
|
||||
|
||||
// Create a simple index.js
|
||||
fs.writeFileSync(
|
||||
path.join(packageDir, 'index.js'),
|
||||
`module.exports = { name: '${packageName}', version: '${version}' };\n`,
|
||||
'utf-8'
|
||||
);
|
||||
|
||||
// Create README.md
|
||||
fs.writeFileSync(
|
||||
path.join(packageDir, 'README.md'),
|
||||
`# ${packageName}\n\nTest package version ${version}\n`,
|
||||
'utf-8'
|
||||
);
|
||||
|
||||
// Copy .npmrc into the package directory
|
||||
if (npmrcPath && fs.existsSync(npmrcPath)) {
|
||||
fs.copyFileSync(npmrcPath, path.join(packageDir, '.npmrc'));
|
||||
}
|
||||
|
||||
return packageDir;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run npm command with proper environment
|
||||
*/
|
||||
async function runNpmCommand(
|
||||
command: string,
|
||||
cwd: string
|
||||
): Promise<{ stdout: string; stderr: string; exitCode: number }> {
|
||||
const { exec } = await import('child_process');
|
||||
|
||||
// Build isolated env that prevents npm from reading ~/.npmrc
|
||||
const env: Record<string, string> = {};
|
||||
// Copy only essential env vars (PATH, etc.) — exclude HOME to prevent ~/.npmrc reading
|
||||
for (const key of ['PATH', 'NODE', 'NVM_DIR', 'NVM_BIN', 'LANG', 'TERM', 'SHELL']) {
|
||||
if (process.env[key]) env[key] = process.env[key]!;
|
||||
}
|
||||
env.HOME = testDir;
|
||||
env.NPM_CONFIG_USERCONFIG = npmrcPath;
|
||||
env.NPM_CONFIG_GLOBALCONFIG = '/dev/null';
|
||||
env.NPM_CONFIG_CACHE = path.join(testDir, '.npm-cache');
|
||||
env.NPM_CONFIG_PREFIX = path.join(testDir, '.npm-global');
|
||||
env.NPM_CONFIG_REGISTRY = `${registryUrl}/npm/`;
|
||||
|
||||
return new Promise((resolve) => {
|
||||
exec(command, { cwd, env, timeout: 30000 }, (error, stdout, stderr) => {
|
||||
resolve({
|
||||
stdout: stdout || '',
|
||||
stderr: stderr || '',
|
||||
exitCode: error ? (error as any).code ?? 1 : 0,
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleanup test directory
|
||||
*/
|
||||
function cleanupTestDir(dir: string): void {
|
||||
if (fs.existsSync(dir)) {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// TESTS
|
||||
// ========================================================================
|
||||
|
||||
tap.test('NPM CLI: should setup registry and HTTP server', async () => {
|
||||
// Find available port
|
||||
registryPort = 35000;
|
||||
|
||||
// Create registry with correct registryUrl for CLI tests
|
||||
registry = await createTestRegistry({ registryUrl: `http://localhost:${registryPort}` });
|
||||
const tokens = await createTestTokens(registry);
|
||||
npmToken = tokens.npmToken;
|
||||
|
||||
// Clean up stale npm CLI test data via unpublish API
|
||||
for (const pkg of ['test-package-cli', '@testscope%2fscoped-package']) {
|
||||
await registry.handleRequest({
|
||||
method: 'DELETE',
|
||||
path: `/npm/${pkg}/-rev/cleanup`,
|
||||
headers: { Authorization: `Bearer ${npmToken}` },
|
||||
query: {},
|
||||
});
|
||||
}
|
||||
|
||||
expect(registry).toBeInstanceOf(SmartRegistry);
|
||||
expect(npmToken).toBeTypeOf('string');
|
||||
const serverSetup = await createHttpServer(registry, registryPort);
|
||||
server = serverSetup.server;
|
||||
registryUrl = serverSetup.url;
|
||||
|
||||
expect(server).toBeDefined();
|
||||
expect(registryUrl).toEqual(`http://localhost:${registryPort}`);
|
||||
|
||||
// Setup test directory — use /tmp to isolate from project tree
|
||||
testDir = path.join('/tmp', 'smartregistry-test-npm-cli');
|
||||
cleanupTestDir(testDir);
|
||||
fs.mkdirSync(testDir, { recursive: true });
|
||||
|
||||
// Setup .npmrc
|
||||
npmrcPath = setupNpmrc(registryUrl, npmToken, testDir);
|
||||
expect(fs.existsSync(npmrcPath)).toEqual(true);
|
||||
});
|
||||
|
||||
tap.test('NPM CLI: should verify server is responding', async () => {
|
||||
const result = await runNpmCommand('npm ping', testDir);
|
||||
console.log('npm ping output:', result.stdout, result.stderr);
|
||||
|
||||
// npm ping may not work with custom registries, so just check server is up
|
||||
// by doing a direct HTTP request
|
||||
const response = await fetch(`${registryUrl}/npm/`);
|
||||
expect(response.status).toBeGreaterThanOrEqual(200);
|
||||
expect(response.status).toBeLessThan(500);
|
||||
});
|
||||
|
||||
tap.test('NPM CLI: should publish a package', async () => {
|
||||
const packageName = 'test-package-cli';
|
||||
const version = '1.0.0';
|
||||
const packageDir = createTestPackage(packageName, version, testDir);
|
||||
|
||||
const result = await runNpmCommand('npm publish', packageDir);
|
||||
console.log('npm publish output:', result.stdout);
|
||||
console.log('npm publish stderr:', result.stderr);
|
||||
|
||||
expect(result.exitCode).toEqual(0);
|
||||
expect(result.stdout || result.stderr).toContain(packageName);
|
||||
});
|
||||
|
||||
tap.test('NPM CLI: should view published package', async () => {
|
||||
const packageName = 'test-package-cli';
|
||||
|
||||
const result = await runNpmCommand(`npm view ${packageName}`, testDir);
|
||||
console.log('npm view output:', result.stdout);
|
||||
|
||||
expect(result.exitCode).toEqual(0);
|
||||
expect(result.stdout).toContain(packageName);
|
||||
expect(result.stdout).toContain('1.0.0');
|
||||
});
|
||||
|
||||
tap.test('NPM CLI: should install published package', async () => {
|
||||
const packageName = 'test-package-cli';
|
||||
const installDir = path.join(testDir, 'install-test');
|
||||
fs.mkdirSync(installDir, { recursive: true });
|
||||
|
||||
// Create a minimal package.json for install target
|
||||
fs.writeFileSync(
|
||||
path.join(installDir, 'package.json'),
|
||||
JSON.stringify({ name: 'install-test', version: '1.0.0', dependencies: { [packageName]: '1.0.0' } }),
|
||||
'utf-8'
|
||||
);
|
||||
// Copy .npmrc
|
||||
if (npmrcPath && fs.existsSync(npmrcPath)) {
|
||||
fs.copyFileSync(npmrcPath, path.join(installDir, '.npmrc'));
|
||||
}
|
||||
|
||||
const result = await runNpmCommand('npm install', installDir);
|
||||
console.log('npm install output:', result.stdout);
|
||||
console.log('npm install stderr:', result.stderr);
|
||||
|
||||
expect(result.exitCode).toEqual(0);
|
||||
|
||||
// Verify package was installed
|
||||
const installed = fs.existsSync(path.join(installDir, 'node_modules', packageName, 'package.json'));
|
||||
expect(installed).toEqual(true);
|
||||
});
|
||||
|
||||
tap.test('NPM CLI: should publish second version', async () => {
|
||||
const packageName = 'test-package-cli';
|
||||
const version = '1.1.0';
|
||||
const packageDir = createTestPackage(packageName, version, testDir);
|
||||
|
||||
const result = await runNpmCommand('npm publish', packageDir);
|
||||
console.log('npm publish v1.1.0 output:', result.stdout);
|
||||
|
||||
expect(result.exitCode).toEqual(0);
|
||||
});
|
||||
|
||||
tap.test('NPM CLI: should list versions', async () => {
|
||||
const packageName = 'test-package-cli';
|
||||
|
||||
const result = await runNpmCommand(`npm view ${packageName} versions`, testDir);
|
||||
console.log('npm view versions output:', result.stdout);
|
||||
|
||||
expect(result.exitCode).toEqual(0);
|
||||
expect(result.stdout).toContain('1.0.0');
|
||||
expect(result.stdout).toContain('1.1.0');
|
||||
});
|
||||
|
||||
tap.test('NPM CLI: should publish scoped package', async () => {
|
||||
const packageName = '@testscope/scoped-package';
|
||||
const version = '1.0.0';
|
||||
const packageDir = createTestPackage(packageName, version, testDir);
|
||||
|
||||
const result = await runNpmCommand('npm publish --access public', packageDir);
|
||||
console.log('npm publish scoped output:', result.stdout);
|
||||
console.log('npm publish scoped stderr:', result.stderr);
|
||||
|
||||
expect(result.exitCode).toEqual(0);
|
||||
});
|
||||
|
||||
tap.test('NPM CLI: should view scoped package', async () => {
|
||||
const packageName = '@testscope/scoped-package';
|
||||
|
||||
const result = await runNpmCommand(`npm view ${packageName}`, testDir);
|
||||
console.log('npm view scoped output:', result.stdout);
|
||||
|
||||
expect(result.exitCode).toEqual(0);
|
||||
expect(result.stdout).toContain('scoped-package');
|
||||
});
|
||||
|
||||
tap.test('NPM CLI: should fail to publish without auth', async () => {
|
||||
const packageName = 'unauth-package';
|
||||
const version = '1.0.0';
|
||||
const packageDir = createTestPackage(packageName, version, testDir);
|
||||
|
||||
// Temporarily remove .npmrc (write one without auth)
|
||||
const noAuthNpmrc = path.join(packageDir, '.npmrc');
|
||||
fs.writeFileSync(noAuthNpmrc, `registry=${registryUrl}/npm/\n`, 'utf-8');
|
||||
|
||||
const result = await runNpmCommand('npm publish', packageDir);
|
||||
console.log('npm publish unauth output:', result.stdout);
|
||||
console.log('npm publish unauth stderr:', result.stderr);
|
||||
|
||||
// Should fail with auth error
|
||||
expect(result.exitCode).not.toEqual(0);
|
||||
});
|
||||
|
||||
tap.postTask('cleanup npm cli tests', async () => {
|
||||
// Stop server
|
||||
if (server) {
|
||||
await new Promise<void>((resolve) => {
|
||||
server.close(() => resolve());
|
||||
});
|
||||
}
|
||||
|
||||
// Cleanup test directory
|
||||
cleanupTestDir(testDir);
|
||||
});
|
||||
|
||||
export default tap.start();
|
||||
+92
-31
@@ -1,6 +1,7 @@
|
||||
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||
import { SmartRegistry } from '../ts/index.js';
|
||||
import { createTestRegistry, createTestTokens, createTestPackument } from './helpers/registry.js';
|
||||
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
|
||||
import { createTestRegistry, createTestTokens, createTestPackument, generateTestRunId } from './helpers/registry.js';
|
||||
|
||||
let registry: SmartRegistry;
|
||||
let npmToken: string;
|
||||
@@ -34,8 +35,9 @@ tap.test('NPM: should handle user authentication (PUT /-/user/org.couchdb.user:{
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(201);
|
||||
expect(response.body).toHaveProperty('token');
|
||||
expect((response.body as any).token).toBeTypeOf('string');
|
||||
const body = await streamToJson(response.body);
|
||||
expect(body).toHaveProperty('token');
|
||||
expect(body.token).toBeTypeOf('string');
|
||||
});
|
||||
|
||||
tap.test('NPM: should publish a package (PUT /{package})', async () => {
|
||||
@@ -53,8 +55,9 @@ tap.test('NPM: should publish a package (PUT /{package})', async () => {
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(201);
|
||||
expect(response.body).toHaveProperty('ok');
|
||||
expect((response.body as any).ok).toEqual(true);
|
||||
const body = await streamToJson(response.body);
|
||||
expect(body).toHaveProperty('ok');
|
||||
expect(body.ok).toEqual(true);
|
||||
});
|
||||
|
||||
tap.test('NPM: should retrieve package metadata (GET /{package})', async () => {
|
||||
@@ -66,10 +69,11 @@ tap.test('NPM: should retrieve package metadata (GET /{package})', async () => {
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.body).toHaveProperty('name');
|
||||
expect((response.body as any).name).toEqual(testPackageName);
|
||||
expect((response.body as any).versions).toHaveProperty(testVersion);
|
||||
expect((response.body as any)['dist-tags'].latest).toEqual(testVersion);
|
||||
const body = await streamToJson(response.body);
|
||||
expect(body).toHaveProperty('name');
|
||||
expect(body.name).toEqual(testPackageName);
|
||||
expect(body.versions).toHaveProperty(testVersion);
|
||||
expect(body['dist-tags'].latest).toEqual(testVersion);
|
||||
});
|
||||
|
||||
tap.test('NPM: should retrieve specific version metadata (GET /{package}/{version})', async () => {
|
||||
@@ -81,9 +85,10 @@ tap.test('NPM: should retrieve specific version metadata (GET /{package}/{versio
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.body).toHaveProperty('version');
|
||||
expect((response.body as any).version).toEqual(testVersion);
|
||||
expect((response.body as any).name).toEqual(testPackageName);
|
||||
const body = await streamToJson(response.body);
|
||||
expect(body).toHaveProperty('version');
|
||||
expect(body.version).toEqual(testVersion);
|
||||
expect(body.name).toEqual(testPackageName);
|
||||
});
|
||||
|
||||
tap.test('NPM: should download tarball (GET /{package}/-/{tarball})', async () => {
|
||||
@@ -95,8 +100,9 @@ tap.test('NPM: should download tarball (GET /{package}/-/{tarball})', async () =
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.body).toBeInstanceOf(Buffer);
|
||||
expect((response.body as Buffer).toString('utf-8')).toEqual('fake tarball content');
|
||||
const body = await streamToBuffer(response.body);
|
||||
expect(body).toBeInstanceOf(Buffer);
|
||||
expect(body.toString('utf-8')).toEqual('fake tarball content');
|
||||
expect(response.headers['Content-Type']).toEqual('application/octet-stream');
|
||||
});
|
||||
|
||||
@@ -127,7 +133,52 @@ tap.test('NPM: should publish a new version of the package', async () => {
|
||||
});
|
||||
|
||||
expect(getResponse.status).toEqual(200);
|
||||
expect((getResponse.body as any).versions).toHaveProperty(newVersion);
|
||||
const getBody = await streamToJson(getResponse.body);
|
||||
expect(getBody.versions).toHaveProperty(newVersion);
|
||||
});
|
||||
|
||||
tap.test('NPM: should support unencoded scoped package publish and metadata routes', async () => {
|
||||
const scopedPackageName = `@scope/test-package-${generateTestRunId()}`;
|
||||
const scopedVersion = '2.0.0';
|
||||
const scopedTarballData = Buffer.from('scoped tarball content', 'utf-8');
|
||||
const packument = createTestPackument(scopedPackageName, scopedVersion, scopedTarballData);
|
||||
|
||||
const publishResponse = await registry.handleRequest({
|
||||
method: 'PUT',
|
||||
path: `/npm/${scopedPackageName}`,
|
||||
headers: {
|
||||
Authorization: `Bearer ${npmToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
query: {},
|
||||
body: packument,
|
||||
});
|
||||
|
||||
expect(publishResponse.status).toEqual(201);
|
||||
|
||||
const metadataResponse = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: `/npm/${scopedPackageName}`,
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(metadataResponse.status).toEqual(200);
|
||||
const metadataBody = await streamToJson(metadataResponse.body);
|
||||
expect(metadataBody.name).toEqual(scopedPackageName);
|
||||
expect(metadataBody.versions).toHaveProperty(scopedVersion);
|
||||
|
||||
const versionResponse = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: `/npm/${scopedPackageName}/${scopedVersion}`,
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(versionResponse.status).toEqual(200);
|
||||
const versionBody = await streamToJson(versionResponse.body);
|
||||
expect(versionBody.name).toEqual(scopedPackageName);
|
||||
expect(versionBody.version).toEqual(scopedVersion);
|
||||
});
|
||||
|
||||
tap.test('NPM: should get dist-tags (GET /-/package/{pkg}/dist-tags)', async () => {
|
||||
@@ -139,8 +190,9 @@ tap.test('NPM: should get dist-tags (GET /-/package/{pkg}/dist-tags)', async ()
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.body).toHaveProperty('latest');
|
||||
expect((response.body as any).latest).toBeTypeOf('string');
|
||||
const body = await streamToJson(response.body);
|
||||
expect(body).toHaveProperty('latest');
|
||||
expect(body.latest).toBeTypeOf('string');
|
||||
});
|
||||
|
||||
tap.test('NPM: should update dist-tag (PUT /-/package/{pkg}/dist-tags/{tag})', async () => {
|
||||
@@ -165,7 +217,8 @@ tap.test('NPM: should update dist-tag (PUT /-/package/{pkg}/dist-tags/{tag})', a
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect((getResponse.body as any)['dist-tags'].beta).toEqual('1.1.0');
|
||||
const getBody2 = await streamToJson(getResponse.body);
|
||||
expect(getBody2['dist-tags'].beta).toEqual('1.1.0');
|
||||
});
|
||||
|
||||
tap.test('NPM: should delete dist-tag (DELETE /-/package/{pkg}/dist-tags/{tag})', async () => {
|
||||
@@ -188,7 +241,8 @@ tap.test('NPM: should delete dist-tag (DELETE /-/package/{pkg}/dist-tags/{tag})'
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect((getResponse.body as any)['dist-tags']).not.toHaveProperty('beta');
|
||||
const getBody3 = await streamToJson(getResponse.body);
|
||||
expect(getBody3['dist-tags']).not.toHaveProperty('beta');
|
||||
});
|
||||
|
||||
tap.test('NPM: should create a new token (POST /-/npm/v1/tokens)', async () => {
|
||||
@@ -208,8 +262,9 @@ tap.test('NPM: should create a new token (POST /-/npm/v1/tokens)', async () => {
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.body).toHaveProperty('token');
|
||||
expect((response.body as any).readonly).toEqual(true);
|
||||
const body = await streamToJson(response.body);
|
||||
expect(body).toHaveProperty('token');
|
||||
expect(body.readonly).toEqual(true);
|
||||
});
|
||||
|
||||
tap.test('NPM: should list tokens (GET /-/npm/v1/tokens)', async () => {
|
||||
@@ -223,9 +278,10 @@ tap.test('NPM: should list tokens (GET /-/npm/v1/tokens)', async () => {
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.body).toHaveProperty('objects');
|
||||
expect((response.body as any).objects).toBeInstanceOf(Array);
|
||||
expect((response.body as any).objects.length).toBeGreaterThan(0);
|
||||
const body = await streamToJson(response.body);
|
||||
expect(body).toHaveProperty('objects');
|
||||
expect(body.objects).toBeInstanceOf(Array);
|
||||
expect(body.objects.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
tap.test('NPM: should search packages (GET /-/v1/search)', async () => {
|
||||
@@ -240,9 +296,10 @@ tap.test('NPM: should search packages (GET /-/v1/search)', async () => {
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.body).toHaveProperty('objects');
|
||||
expect((response.body as any).objects).toBeInstanceOf(Array);
|
||||
expect((response.body as any).total).toBeGreaterThan(0);
|
||||
const body = await streamToJson(response.body);
|
||||
expect(body).toHaveProperty('objects');
|
||||
expect(body.objects).toBeInstanceOf(Array);
|
||||
expect(body.total).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
tap.test('NPM: should search packages with specific query', async () => {
|
||||
@@ -256,7 +313,8 @@ tap.test('NPM: should search packages with specific query', async () => {
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
const results = (response.body as any).objects;
|
||||
const body = await streamToJson(response.body);
|
||||
const results = body.objects;
|
||||
expect(results.length).toBeGreaterThan(0);
|
||||
expect(results[0].package.name).toEqual(testPackageName);
|
||||
});
|
||||
@@ -281,7 +339,8 @@ tap.test('NPM: should unpublish a specific version (DELETE /{package}/-/{version
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect((getResponse.body as any).versions).not.toHaveProperty(testVersion);
|
||||
const getBody4 = await streamToJson(getResponse.body);
|
||||
expect(getBody4.versions).not.toHaveProperty(testVersion);
|
||||
});
|
||||
|
||||
tap.test('NPM: should unpublish entire package (DELETE /{package}/-rev/{rev})', async () => {
|
||||
@@ -316,7 +375,8 @@ tap.test('NPM: should return 404 for non-existent package', async () => {
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(404);
|
||||
expect(response.body).toHaveProperty('error');
|
||||
const body = await streamToJson(response.body);
|
||||
expect(body).toHaveProperty('error');
|
||||
});
|
||||
|
||||
tap.test('NPM: should return 401 for unauthorized publish', async () => {
|
||||
@@ -334,7 +394,8 @@ tap.test('NPM: should return 401 for unauthorized publish', async () => {
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(401);
|
||||
expect(response.body).toHaveProperty('error');
|
||||
const body = await streamToJson(response.body);
|
||||
expect(body).toHaveProperty('error');
|
||||
});
|
||||
|
||||
tap.test('NPM: should reject readonly token for write operations', async () => {
|
||||
|
||||
@@ -0,0 +1,396 @@
|
||||
/**
|
||||
* Native Docker CLI Testing
|
||||
* Tests the OCI registry implementation using the actual Docker CLI
|
||||
*/
|
||||
|
||||
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||
import { tapNodeTools } from '@git.zone/tstest/tapbundle_serverside';
|
||||
import { SmartRegistry } from '../ts/index.js';
|
||||
import type { IRequestContext, IResponse, IRegistryConfig } from '../ts/core/interfaces.core.js';
|
||||
import * as qenv from '@push.rocks/qenv';
|
||||
import * as http from 'http';
|
||||
import * as url from 'url';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
const testQenv = new qenv.Qenv('./', './.nogit');
|
||||
|
||||
/**
|
||||
* Create a test registry with local token endpoint realm
|
||||
*/
|
||||
async function createDockerTestRegistry(port: number): Promise<SmartRegistry> {
|
||||
const s3AccessKey = await testQenv.getEnvVarOnDemand('S3_ACCESSKEY');
|
||||
const s3SecretKey = await testQenv.getEnvVarOnDemand('S3_SECRETKEY');
|
||||
const s3Endpoint = await testQenv.getEnvVarOnDemand('S3_ENDPOINT');
|
||||
const s3Port = await testQenv.getEnvVarOnDemand('S3_PORT');
|
||||
|
||||
const config: IRegistryConfig = {
|
||||
storage: {
|
||||
accessKey: s3AccessKey || 'minioadmin',
|
||||
accessSecret: s3SecretKey || 'minioadmin',
|
||||
endpoint: s3Endpoint || 'localhost',
|
||||
port: parseInt(s3Port || '9000', 10),
|
||||
useSsl: false,
|
||||
region: 'us-east-1',
|
||||
bucketName: 'test-registry',
|
||||
},
|
||||
auth: {
|
||||
jwtSecret: 'test-secret-key',
|
||||
tokenStore: 'memory',
|
||||
npmTokens: {
|
||||
enabled: true,
|
||||
},
|
||||
ociTokens: {
|
||||
enabled: true,
|
||||
realm: `http://localhost:${port}/v2/token`,
|
||||
service: 'test-registry',
|
||||
},
|
||||
},
|
||||
oci: {
|
||||
enabled: true,
|
||||
basePath: '/v2',
|
||||
},
|
||||
};
|
||||
|
||||
const reg = new SmartRegistry(config);
|
||||
await reg.init();
|
||||
return reg;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create test tokens for the registry
|
||||
*/
|
||||
async function createDockerTestTokens(reg: SmartRegistry) {
|
||||
const authManager = reg.getAuthManager();
|
||||
|
||||
const userId = await authManager.authenticate({
|
||||
username: 'testuser',
|
||||
password: 'testpass',
|
||||
});
|
||||
|
||||
if (!userId) {
|
||||
throw new Error('Failed to authenticate test user');
|
||||
}
|
||||
|
||||
// Create OCI token with full access
|
||||
const ociToken = await authManager.createOciToken(
|
||||
userId,
|
||||
['oci:repository:*:*'],
|
||||
3600
|
||||
);
|
||||
|
||||
return { ociToken, userId };
|
||||
}
|
||||
|
||||
// Test context
|
||||
let registry: SmartRegistry;
|
||||
let server: http.Server;
|
||||
let registryUrl: string;
|
||||
let registryPort: number;
|
||||
let ociToken: string;
|
||||
let testDir: string;
|
||||
let testImageName: string;
|
||||
|
||||
/**
|
||||
* Create HTTP server wrapper around SmartRegistry
|
||||
* CRITICAL: Always passes rawBody for content-addressable operations (OCI manifests/blobs)
|
||||
*
|
||||
* SmartRegistry OCI is configured with basePath '/v2' matching Docker's native /v2/ prefix.
|
||||
*
|
||||
* Also implements a simple /v2/token endpoint for Docker Bearer auth flow
|
||||
*/
|
||||
async function createHttpServer(
|
||||
registryInstance: SmartRegistry,
|
||||
port: number,
|
||||
tokenForAuth: string
|
||||
): Promise<{ server: http.Server; url: string }> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const httpServer = http.createServer(async (req, res) => {
|
||||
try {
|
||||
// Parse request
|
||||
const parsedUrl = url.parse(req.url || '', true);
|
||||
let pathname = parsedUrl.pathname || '/';
|
||||
const query = parsedUrl.query;
|
||||
|
||||
// Handle token endpoint for Docker Bearer auth
|
||||
if (pathname === '/v2/token' || pathname === '/token') {
|
||||
console.log(`[Token Request] ${req.method} ${req.url}`);
|
||||
res.statusCode = 200;
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.end(JSON.stringify({
|
||||
token: tokenForAuth,
|
||||
access_token: tokenForAuth,
|
||||
expires_in: 3600,
|
||||
issued_at: new Date().toISOString(),
|
||||
}));
|
||||
return;
|
||||
}
|
||||
|
||||
// Log all requests for debugging
|
||||
console.log(`[Registry] ${req.method} ${pathname}`);
|
||||
|
||||
// basePath is /v2 which matches Docker's native /v2/ prefix — no rewrite needed
|
||||
|
||||
// Read raw body - ALWAYS preserve exact bytes for OCI
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of req) {
|
||||
chunks.push(chunk);
|
||||
}
|
||||
const bodyBuffer = Buffer.concat(chunks);
|
||||
|
||||
// Parse body based on content type (for non-OCI protocols that need it)
|
||||
let parsedBody: any;
|
||||
if (bodyBuffer.length > 0) {
|
||||
const contentType = req.headers['content-type'] || '';
|
||||
if (contentType.includes('application/json')) {
|
||||
try {
|
||||
parsedBody = JSON.parse(bodyBuffer.toString('utf-8'));
|
||||
} catch (error) {
|
||||
parsedBody = bodyBuffer;
|
||||
}
|
||||
} else {
|
||||
parsedBody = bodyBuffer;
|
||||
}
|
||||
}
|
||||
|
||||
// Convert to IRequestContext
|
||||
const context: IRequestContext = {
|
||||
method: req.method || 'GET',
|
||||
path: pathname,
|
||||
headers: req.headers as Record<string, string>,
|
||||
query: query as Record<string, string>,
|
||||
body: parsedBody,
|
||||
rawBody: bodyBuffer,
|
||||
};
|
||||
|
||||
// Handle request
|
||||
const response: IResponse = await registryInstance.handleRequest(context);
|
||||
console.log(`[Registry] Response: ${response.status} for ${pathname}`);
|
||||
|
||||
// Convert IResponse to HTTP response
|
||||
res.statusCode = response.status;
|
||||
|
||||
// Set headers
|
||||
for (const [key, value] of Object.entries(response.headers || {})) {
|
||||
res.setHeader(key, value);
|
||||
}
|
||||
|
||||
// Send body (response.body is always ReadableStream<Uint8Array> or undefined)
|
||||
if (response.body) {
|
||||
const { Readable } = await import('stream');
|
||||
Readable.fromWeb(response.body).pipe(res);
|
||||
} else {
|
||||
res.end();
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Server error:', error);
|
||||
res.statusCode = 500;
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.end(JSON.stringify({ error: 'INTERNAL_ERROR', message: String(error) }));
|
||||
}
|
||||
});
|
||||
|
||||
httpServer.listen(port, '0.0.0.0', () => {
|
||||
const serverUrl = `http://localhost:${port}`;
|
||||
resolve({ server: httpServer, url: serverUrl });
|
||||
});
|
||||
|
||||
httpServer.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a test Dockerfile
|
||||
*/
|
||||
function createTestDockerfile(targetDir: string, content?: string): string {
|
||||
const dockerfilePath = path.join(targetDir, 'Dockerfile');
|
||||
const dockerfileContent = content || `FROM alpine:latest
|
||||
RUN echo "Hello from SmartRegistry test" > /hello.txt
|
||||
CMD ["cat", "/hello.txt"]
|
||||
`;
|
||||
fs.writeFileSync(dockerfilePath, dockerfileContent, 'utf-8');
|
||||
return dockerfilePath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run Docker command using the main Docker daemon (not rootless)
|
||||
* Rootless Docker runs in its own network namespace and can't access host localhost
|
||||
*
|
||||
* IMPORTANT: DOCKER_HOST env var overrides --context flag, so we must unset it
|
||||
* and explicitly set the socket path to use the main Docker daemon.
|
||||
*/
|
||||
async function runDockerCommand(
|
||||
command: string,
|
||||
cwd?: string
|
||||
): Promise<{ stdout: string; stderr: string; exitCode: number }> {
|
||||
// First unset DOCKER_HOST then set it to main Docker daemon socket
|
||||
// Using both unset and export ensures we override any inherited env var
|
||||
const dockerCommand = `unset DOCKER_HOST && export DOCKER_HOST=unix:///var/run/docker.sock && ${command}`;
|
||||
const fullCommand = cwd ? `cd "${cwd}" && ${dockerCommand}` : dockerCommand;
|
||||
|
||||
try {
|
||||
const result = await tapNodeTools.runCommand(fullCommand);
|
||||
return {
|
||||
stdout: result.stdout || '',
|
||||
stderr: result.stderr || '',
|
||||
exitCode: result.exitCode || 0,
|
||||
};
|
||||
} catch (error: any) {
|
||||
return {
|
||||
stdout: error.stdout || '',
|
||||
stderr: error.stderr || String(error),
|
||||
exitCode: error.exitCode || 1,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleanup test directory
|
||||
*/
|
||||
function cleanupTestDir(dir: string): void {
|
||||
if (fs.existsSync(dir)) {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleanup Docker resources
|
||||
*/
|
||||
async function cleanupDocker(imageName: string): Promise<void> {
|
||||
await runDockerCommand(`docker rmi ${imageName} 2>/dev/null || true`);
|
||||
await runDockerCommand(`docker rmi ${imageName}:v1 2>/dev/null || true`);
|
||||
await runDockerCommand(`docker rmi ${imageName}:v2 2>/dev/null || true`);
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// TESTS
|
||||
// ========================================================================
|
||||
|
||||
tap.test('Docker CLI: should verify Docker is installed', async () => {
|
||||
const result = await runDockerCommand('docker version');
|
||||
console.log('Docker version output:', result.stdout.substring(0, 200));
|
||||
expect(result.exitCode).toEqual(0);
|
||||
});
|
||||
|
||||
tap.test('Docker CLI: should setup registry and HTTP server', async () => {
|
||||
// Use localhost - Docker allows HTTP for localhost without any special config
|
||||
registryPort = 15000 + Math.floor(Math.random() * 1000);
|
||||
console.log(`Using port: ${registryPort}`);
|
||||
|
||||
registry = await createDockerTestRegistry(registryPort);
|
||||
const tokens = await createDockerTestTokens(registry);
|
||||
ociToken = tokens.ociToken;
|
||||
|
||||
expect(registry).toBeInstanceOf(SmartRegistry);
|
||||
expect(ociToken).toBeTypeOf('string');
|
||||
|
||||
const serverSetup = await createHttpServer(registry, registryPort, ociToken);
|
||||
server = serverSetup.server;
|
||||
registryUrl = serverSetup.url;
|
||||
|
||||
expect(server).toBeDefined();
|
||||
console.log(`Registry server started at ${registryUrl}`);
|
||||
|
||||
// Setup test directory
|
||||
testDir = path.join(process.cwd(), '.nogit', 'test-docker-cli');
|
||||
cleanupTestDir(testDir);
|
||||
fs.mkdirSync(testDir, { recursive: true });
|
||||
|
||||
testImageName = `localhost:${registryPort}/test-image`;
|
||||
});
|
||||
|
||||
tap.test('Docker CLI: should verify server is responding', async () => {
|
||||
// Give the server a moment to fully initialize
|
||||
await new Promise(resolve => setTimeout(resolve, 500));
|
||||
|
||||
const response = await fetch(`${registryUrl}/v2/`);
|
||||
expect(response.status).toEqual(200);
|
||||
console.log('OCI v2 response:', await response.json());
|
||||
});
|
||||
|
||||
tap.test('Docker CLI: should login to registry', async () => {
|
||||
const result = await runDockerCommand(
|
||||
`echo "${ociToken}" | docker login localhost:${registryPort} -u testuser --password-stdin`
|
||||
);
|
||||
console.log('docker login output:', result.stdout);
|
||||
console.log('docker login stderr:', result.stderr);
|
||||
|
||||
const combinedOutput = result.stdout + result.stderr;
|
||||
expect(combinedOutput).toContain('Login Succeeded');
|
||||
});
|
||||
|
||||
tap.test('Docker CLI: should build test image', async () => {
|
||||
createTestDockerfile(testDir);
|
||||
|
||||
const result = await runDockerCommand(
|
||||
`docker build -t ${testImageName}:v1 .`,
|
||||
testDir
|
||||
);
|
||||
console.log('docker build output:', result.stdout.substring(0, 500));
|
||||
|
||||
expect(result.exitCode).toEqual(0);
|
||||
});
|
||||
|
||||
tap.test('Docker CLI: should push image to registry', async () => {
|
||||
// This is the critical test - if the digest mismatch bug is fixed,
|
||||
// this should succeed. The manifest bytes must be preserved exactly.
|
||||
const result = await runDockerCommand(`docker push ${testImageName}:v1`);
|
||||
console.log('docker push output:', result.stdout);
|
||||
console.log('docker push stderr:', result.stderr);
|
||||
|
||||
expect(result.exitCode).toEqual(0);
|
||||
});
|
||||
|
||||
tap.test('Docker CLI: should verify manifest in registry via API', async () => {
|
||||
const response = await fetch(`${registryUrl}/v2/test-image/tags/list`, {
|
||||
headers: { Authorization: `Bearer ${ociToken}` },
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
|
||||
const tagList = await response.json();
|
||||
console.log('Tags list:', tagList);
|
||||
|
||||
expect(tagList.name).toEqual('test-image');
|
||||
expect(tagList.tags).toContain('v1');
|
||||
});
|
||||
|
||||
tap.test('Docker CLI: should pull pushed image', async () => {
|
||||
// First remove the local image
|
||||
await runDockerCommand(`docker rmi ${testImageName}:v1 || true`);
|
||||
|
||||
const result = await runDockerCommand(`docker pull ${testImageName}:v1`);
|
||||
console.log('docker pull output:', result.stdout);
|
||||
|
||||
expect(result.exitCode).toEqual(0);
|
||||
});
|
||||
|
||||
tap.test('Docker CLI: should run pulled image', async () => {
|
||||
const result = await runDockerCommand(`docker run --rm ${testImageName}:v1`);
|
||||
console.log('docker run output:', result.stdout);
|
||||
|
||||
expect(result.exitCode).toEqual(0);
|
||||
expect(result.stdout).toContain('Hello from SmartRegistry test');
|
||||
});
|
||||
|
||||
tap.postTask('cleanup docker cli tests', async () => {
|
||||
if (testImageName) {
|
||||
await cleanupDocker(testImageName);
|
||||
}
|
||||
|
||||
if (server) {
|
||||
await new Promise<void>((resolve) => {
|
||||
server.close(() => resolve());
|
||||
});
|
||||
}
|
||||
|
||||
if (testDir) {
|
||||
cleanupTestDir(testDir);
|
||||
}
|
||||
|
||||
if (registry) {
|
||||
registry.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
export default tap.start();
|
||||
+32
-26
@@ -1,5 +1,6 @@
|
||||
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||
import { SmartRegistry } from '../ts/index.js';
|
||||
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
|
||||
import { createTestRegistry, createTestTokens, calculateDigest, createTestManifest } from './helpers/registry.js';
|
||||
|
||||
let registry: SmartRegistry;
|
||||
@@ -24,7 +25,7 @@ tap.test('OCI: should create registry instance', async () => {
|
||||
tap.test('OCI: should handle version check (GET /v2/)', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/oci/v2/',
|
||||
path: '/oci/',
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
@@ -36,7 +37,7 @@ tap.test('OCI: should handle version check (GET /v2/)', async () => {
|
||||
tap.test('OCI: should initiate blob upload (POST /v2/{name}/blobs/uploads/)', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'POST',
|
||||
path: '/oci/v2/test-repo/blobs/uploads/',
|
||||
path: '/oci/test-repo/blobs/uploads/',
|
||||
headers: {
|
||||
Authorization: `Bearer ${ociToken}`,
|
||||
},
|
||||
@@ -53,7 +54,7 @@ tap.test('OCI: should upload blob in single PUT', async () => {
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'POST',
|
||||
path: '/oci/v2/test-repo/blobs/uploads/',
|
||||
path: '/oci/test-repo/blobs/uploads/',
|
||||
headers: {
|
||||
Authorization: `Bearer ${ociToken}`,
|
||||
},
|
||||
@@ -73,7 +74,7 @@ tap.test('OCI: should upload config blob', async () => {
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'POST',
|
||||
path: '/oci/v2/test-repo/blobs/uploads/',
|
||||
path: '/oci/test-repo/blobs/uploads/',
|
||||
headers: {
|
||||
Authorization: `Bearer ${ociToken}`,
|
||||
},
|
||||
@@ -90,7 +91,7 @@ tap.test('OCI: should upload config blob', async () => {
|
||||
tap.test('OCI: should check if blob exists (HEAD /v2/{name}/blobs/{digest})', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'HEAD',
|
||||
path: `/oci/v2/test-repo/blobs/${testBlobDigest}`,
|
||||
path: `/oci/test-repo/blobs/${testBlobDigest}`,
|
||||
headers: {
|
||||
Authorization: `Bearer ${ociToken}`,
|
||||
},
|
||||
@@ -105,7 +106,7 @@ tap.test('OCI: should check if blob exists (HEAD /v2/{name}/blobs/{digest})', as
|
||||
tap.test('OCI: should retrieve blob (GET /v2/{name}/blobs/{digest})', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: `/oci/v2/test-repo/blobs/${testBlobDigest}`,
|
||||
path: `/oci/test-repo/blobs/${testBlobDigest}`,
|
||||
headers: {
|
||||
Authorization: `Bearer ${ociToken}`,
|
||||
},
|
||||
@@ -113,8 +114,9 @@ tap.test('OCI: should retrieve blob (GET /v2/{name}/blobs/{digest})', async () =
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.body).toBeInstanceOf(Buffer);
|
||||
expect((response.body as Buffer).toString('utf-8')).toEqual('Hello from OCI test blob!');
|
||||
const body = await streamToBuffer(response.body);
|
||||
expect(body).toBeInstanceOf(Buffer);
|
||||
expect(body.toString('utf-8')).toEqual('Hello from OCI test blob!');
|
||||
expect(response.headers['Docker-Content-Digest']).toEqual(testBlobDigest);
|
||||
});
|
||||
|
||||
@@ -126,7 +128,7 @@ tap.test('OCI: should upload manifest (PUT /v2/{name}/manifests/{reference})', a
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'PUT',
|
||||
path: '/oci/v2/test-repo/manifests/v1.0.0',
|
||||
path: '/oci/test-repo/manifests/v1.0.0',
|
||||
headers: {
|
||||
Authorization: `Bearer ${ociToken}`,
|
||||
'Content-Type': 'application/vnd.oci.image.manifest.v1+json',
|
||||
@@ -143,7 +145,7 @@ tap.test('OCI: should upload manifest (PUT /v2/{name}/manifests/{reference})', a
|
||||
tap.test('OCI: should retrieve manifest by tag (GET /v2/{name}/manifests/{reference})', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/oci/v2/test-repo/manifests/v1.0.0',
|
||||
path: '/oci/test-repo/manifests/v1.0.0',
|
||||
headers: {
|
||||
Authorization: `Bearer ${ociToken}`,
|
||||
Accept: 'application/vnd.oci.image.manifest.v1+json',
|
||||
@@ -152,9 +154,10 @@ tap.test('OCI: should retrieve manifest by tag (GET /v2/{name}/manifests/{refere
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.body).toBeInstanceOf(Buffer);
|
||||
const body = await streamToBuffer(response.body);
|
||||
expect(body).toBeInstanceOf(Buffer);
|
||||
|
||||
const manifest = JSON.parse((response.body as Buffer).toString('utf-8'));
|
||||
const manifest = JSON.parse(body.toString('utf-8'));
|
||||
expect(manifest.schemaVersion).toEqual(2);
|
||||
expect(manifest.config.digest).toEqual(testConfigDigest);
|
||||
expect(manifest.layers[0].digest).toEqual(testBlobDigest);
|
||||
@@ -163,7 +166,7 @@ tap.test('OCI: should retrieve manifest by tag (GET /v2/{name}/manifests/{refere
|
||||
tap.test('OCI: should retrieve manifest by digest (GET /v2/{name}/manifests/{digest})', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: `/oci/v2/test-repo/manifests/${testManifestDigest}`,
|
||||
path: `/oci/test-repo/manifests/${testManifestDigest}`,
|
||||
headers: {
|
||||
Authorization: `Bearer ${ociToken}`,
|
||||
Accept: 'application/vnd.oci.image.manifest.v1+json',
|
||||
@@ -178,7 +181,7 @@ tap.test('OCI: should retrieve manifest by digest (GET /v2/{name}/manifests/{dig
|
||||
tap.test('OCI: should check if manifest exists (HEAD /v2/{name}/manifests/{reference})', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'HEAD',
|
||||
path: '/oci/v2/test-repo/manifests/v1.0.0',
|
||||
path: '/oci/test-repo/manifests/v1.0.0',
|
||||
headers: {
|
||||
Authorization: `Bearer ${ociToken}`,
|
||||
Accept: 'application/vnd.oci.image.manifest.v1+json',
|
||||
@@ -193,7 +196,7 @@ tap.test('OCI: should check if manifest exists (HEAD /v2/{name}/manifests/{refer
|
||||
tap.test('OCI: should list tags (GET /v2/{name}/tags/list)', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/oci/v2/test-repo/tags/list',
|
||||
path: '/oci/test-repo/tags/list',
|
||||
headers: {
|
||||
Authorization: `Bearer ${ociToken}`,
|
||||
},
|
||||
@@ -201,9 +204,9 @@ tap.test('OCI: should list tags (GET /v2/{name}/tags/list)', async () => {
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.body).toHaveProperty('tags');
|
||||
const tagList = await streamToJson(response.body);
|
||||
expect(tagList).toHaveProperty('tags');
|
||||
|
||||
const tagList = response.body as any;
|
||||
expect(tagList.name).toEqual('test-repo');
|
||||
expect(tagList.tags).toBeInstanceOf(Array);
|
||||
expect(tagList.tags).toContain('v1.0.0');
|
||||
@@ -212,7 +215,7 @@ tap.test('OCI: should list tags (GET /v2/{name}/tags/list)', async () => {
|
||||
tap.test('OCI: should handle pagination for tag list', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/oci/v2/test-repo/tags/list',
|
||||
path: '/oci/test-repo/tags/list',
|
||||
headers: {
|
||||
Authorization: `Bearer ${ociToken}`,
|
||||
},
|
||||
@@ -222,13 +225,14 @@ tap.test('OCI: should handle pagination for tag list', async () => {
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.body).toHaveProperty('tags');
|
||||
const body = await streamToJson(response.body);
|
||||
expect(body).toHaveProperty('tags');
|
||||
});
|
||||
|
||||
tap.test('OCI: should return 404 for non-existent blob', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/oci/v2/test-repo/blobs/sha256:0000000000000000000000000000000000000000000000000000000000000000',
|
||||
path: '/oci/test-repo/blobs/sha256:0000000000000000000000000000000000000000000000000000000000000000',
|
||||
headers: {
|
||||
Authorization: `Bearer ${ociToken}`,
|
||||
},
|
||||
@@ -236,13 +240,14 @@ tap.test('OCI: should return 404 for non-existent blob', async () => {
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(404);
|
||||
expect(response.body).toHaveProperty('errors');
|
||||
const body = await streamToJson(response.body);
|
||||
expect(body).toHaveProperty('errors');
|
||||
});
|
||||
|
||||
tap.test('OCI: should return 404 for non-existent manifest', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/oci/v2/test-repo/manifests/non-existent-tag',
|
||||
path: '/oci/test-repo/manifests/non-existent-tag',
|
||||
headers: {
|
||||
Authorization: `Bearer ${ociToken}`,
|
||||
Accept: 'application/vnd.oci.image.manifest.v1+json',
|
||||
@@ -251,13 +256,14 @@ tap.test('OCI: should return 404 for non-existent manifest', async () => {
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(404);
|
||||
expect(response.body).toHaveProperty('errors');
|
||||
const body = await streamToJson(response.body);
|
||||
expect(body).toHaveProperty('errors');
|
||||
});
|
||||
|
||||
tap.test('OCI: should delete manifest (DELETE /v2/{name}/manifests/{digest})', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'DELETE',
|
||||
path: `/oci/v2/test-repo/manifests/${testManifestDigest}`,
|
||||
path: `/oci/test-repo/manifests/${testManifestDigest}`,
|
||||
headers: {
|
||||
Authorization: `Bearer ${ociToken}`,
|
||||
},
|
||||
@@ -270,7 +276,7 @@ tap.test('OCI: should delete manifest (DELETE /v2/{name}/manifests/{digest})', a
|
||||
tap.test('OCI: should delete blob (DELETE /v2/{name}/blobs/{digest})', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'DELETE',
|
||||
path: `/oci/v2/test-repo/blobs/${testBlobDigest}`,
|
||||
path: `/oci/test-repo/blobs/${testBlobDigest}`,
|
||||
headers: {
|
||||
Authorization: `Bearer ${ociToken}`,
|
||||
},
|
||||
@@ -283,7 +289,7 @@ tap.test('OCI: should delete blob (DELETE /v2/{name}/blobs/{digest})', async ()
|
||||
tap.test('OCI: should handle unauthorized requests', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/oci/v2/test-repo/manifests/v1.0.0',
|
||||
path: '/oci/test-repo/manifests/v1.0.0',
|
||||
headers: {
|
||||
// No authorization header
|
||||
},
|
||||
|
||||
@@ -0,0 +1,515 @@
|
||||
/**
|
||||
* Native PyPI CLI Testing
|
||||
* Tests the PyPI registry implementation using pip and twine CLI tools
|
||||
*/
|
||||
|
||||
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||
import { tapNodeTools } from '@git.zone/tstest/tapbundle_serverside';
|
||||
import { SmartRegistry } from '../ts/index.js';
|
||||
import { createTestRegistry, createTestTokens, createPythonWheel, createPythonSdist, generateTestRunId } from './helpers/registry.js';
|
||||
import type { IRequestContext, IResponse } from '../ts/core/interfaces.core.js';
|
||||
import * as http from 'http';
|
||||
import * as url from 'url';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
// Test context
|
||||
let registry: SmartRegistry;
|
||||
let server: http.Server;
|
||||
let registryUrl: string;
|
||||
let registryPort: number;
|
||||
let pypiToken: string;
|
||||
let testDir: string;
|
||||
let pipHome: string;
|
||||
let hasPip = false;
|
||||
let hasTwine = false;
|
||||
|
||||
// Unique test run ID to avoid conflicts between test runs
|
||||
const testRunId = generateTestRunId();
|
||||
const testPackageName = `test-pypi-pkg-${testRunId}`;
|
||||
|
||||
/**
|
||||
* Create HTTP server wrapper around SmartRegistry
|
||||
*/
|
||||
async function createHttpServer(
|
||||
registryInstance: SmartRegistry,
|
||||
port: number
|
||||
): Promise<{ server: http.Server; url: string }> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const httpServer = http.createServer(async (req, res) => {
|
||||
try {
|
||||
// Parse request
|
||||
const parsedUrl = url.parse(req.url || '', true);
|
||||
const pathname = parsedUrl.pathname || '/';
|
||||
const query = parsedUrl.query;
|
||||
|
||||
// Read body
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of req) {
|
||||
chunks.push(chunk);
|
||||
}
|
||||
const bodyBuffer = Buffer.concat(chunks);
|
||||
|
||||
// Parse body based on content type
|
||||
let body: any;
|
||||
if (bodyBuffer.length > 0) {
|
||||
const contentType = req.headers['content-type'] || '';
|
||||
if (contentType.includes('application/json')) {
|
||||
try {
|
||||
body = JSON.parse(bodyBuffer.toString('utf-8'));
|
||||
} catch (error) {
|
||||
body = bodyBuffer;
|
||||
}
|
||||
} else if (contentType.includes('multipart/form-data')) {
|
||||
// For multipart, pass raw buffer
|
||||
body = bodyBuffer;
|
||||
} else {
|
||||
body = bodyBuffer;
|
||||
}
|
||||
}
|
||||
|
||||
// Convert to IRequestContext
|
||||
const context: IRequestContext = {
|
||||
method: req.method || 'GET',
|
||||
path: pathname,
|
||||
headers: req.headers as Record<string, string>,
|
||||
query: query as Record<string, string>,
|
||||
body: body,
|
||||
rawBody: bodyBuffer,
|
||||
};
|
||||
|
||||
// Handle request
|
||||
const response: IResponse = await registryInstance.handleRequest(context);
|
||||
|
||||
// Convert IResponse to HTTP response
|
||||
res.statusCode = response.status;
|
||||
|
||||
// Set headers
|
||||
for (const [key, value] of Object.entries(response.headers || {})) {
|
||||
res.setHeader(key, value);
|
||||
}
|
||||
|
||||
// Send body (response.body is always ReadableStream<Uint8Array> or undefined)
|
||||
if (response.body) {
|
||||
const { Readable } = await import('stream');
|
||||
Readable.fromWeb(response.body).pipe(res);
|
||||
} else {
|
||||
res.end();
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Server error:', error);
|
||||
res.statusCode = 500;
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.end(JSON.stringify({ error: 'INTERNAL_ERROR', message: String(error) }));
|
||||
}
|
||||
});
|
||||
|
||||
httpServer.listen(port, () => {
|
||||
const serverUrl = `http://localhost:${port}`;
|
||||
resolve({ server: httpServer, url: serverUrl });
|
||||
});
|
||||
|
||||
httpServer.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Setup .pypirc for twine authentication
|
||||
*/
|
||||
function setupPypirc(
|
||||
token: string,
|
||||
pipHomeArg: string,
|
||||
serverUrl: string
|
||||
): string {
|
||||
fs.mkdirSync(pipHomeArg, { recursive: true });
|
||||
|
||||
const pypircContent = `[distutils]
|
||||
index-servers = testpypi
|
||||
|
||||
[testpypi]
|
||||
repository = ${serverUrl}/pypi
|
||||
username = testuser
|
||||
password = ${token}
|
||||
`;
|
||||
|
||||
const pypircPath = path.join(pipHomeArg, '.pypirc');
|
||||
fs.writeFileSync(pypircPath, pypircContent, 'utf-8');
|
||||
|
||||
// Set restrictive permissions
|
||||
fs.chmodSync(pypircPath, 0o600);
|
||||
|
||||
return pypircPath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Setup pip.conf for pip to use our registry
|
||||
*/
|
||||
function setupPipConf(
|
||||
token: string,
|
||||
pipHomeArg: string,
|
||||
serverUrl: string,
|
||||
port: number
|
||||
): string {
|
||||
fs.mkdirSync(pipHomeArg, { recursive: true });
|
||||
|
||||
// pip.conf with authentication
|
||||
const pipConfContent = `[global]
|
||||
index-url = ${serverUrl}/pypi/simple/
|
||||
trusted-host = localhost
|
||||
extra-index-url = https://pypi.org/simple/
|
||||
`;
|
||||
|
||||
const pipDir = path.join(pipHomeArg, 'pip');
|
||||
fs.mkdirSync(pipDir, { recursive: true });
|
||||
|
||||
const pipConfPath = path.join(pipDir, 'pip.conf');
|
||||
fs.writeFileSync(pipConfPath, pipConfContent, 'utf-8');
|
||||
|
||||
return pipConfPath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a test Python package wheel file
|
||||
*/
|
||||
async function createTestWheelFile(
|
||||
packageName: string,
|
||||
version: string,
|
||||
targetDir: string
|
||||
): Promise<string> {
|
||||
const wheelData = await createPythonWheel(packageName, version);
|
||||
const normalizedName = packageName.replace(/-/g, '_');
|
||||
const wheelFilename = `${normalizedName}-${version}-py3-none-any.whl`;
|
||||
const wheelPath = path.join(targetDir, wheelFilename);
|
||||
|
||||
fs.writeFileSync(wheelPath, wheelData);
|
||||
|
||||
return wheelPath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a test Python package sdist file
|
||||
*/
|
||||
async function createTestSdistFile(
|
||||
packageName: string,
|
||||
version: string,
|
||||
targetDir: string
|
||||
): Promise<string> {
|
||||
const sdistData = await createPythonSdist(packageName, version);
|
||||
const sdistFilename = `${packageName}-${version}.tar.gz`;
|
||||
const sdistPath = path.join(targetDir, sdistFilename);
|
||||
|
||||
fs.writeFileSync(sdistPath, sdistData);
|
||||
|
||||
return sdistPath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run pip command with custom config
|
||||
*/
|
||||
async function runPipCommand(
|
||||
command: string,
|
||||
cwd: string
|
||||
): Promise<{ stdout: string; stderr: string; exitCode: number }> {
|
||||
const pipConfDir = path.join(pipHome, 'pip');
|
||||
const fullCommand = `cd "${cwd}" && PIP_CONFIG_FILE="${path.join(pipConfDir, 'pip.conf')}" pip ${command}`;
|
||||
|
||||
try {
|
||||
const result = await tapNodeTools.runCommand(fullCommand);
|
||||
return {
|
||||
stdout: result.stdout || '',
|
||||
stderr: result.stderr || '',
|
||||
exitCode: result.exitCode || 0,
|
||||
};
|
||||
} catch (error: any) {
|
||||
return {
|
||||
stdout: error.stdout || '',
|
||||
stderr: error.stderr || String(error),
|
||||
exitCode: error.exitCode || 1,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run twine command with custom config
|
||||
*/
|
||||
async function runTwineCommand(
|
||||
command: string,
|
||||
cwd: string
|
||||
): Promise<{ stdout: string; stderr: string; exitCode: number }> {
|
||||
const pypircPath = path.join(pipHome, '.pypirc');
|
||||
const fullCommand = `cd "${cwd}" && twine ${command} --config-file "${pypircPath}"`;
|
||||
|
||||
try {
|
||||
const result = await tapNodeTools.runCommand(fullCommand);
|
||||
return {
|
||||
stdout: result.stdout || '',
|
||||
stderr: result.stderr || '',
|
||||
exitCode: result.exitCode || 0,
|
||||
};
|
||||
} catch (error: any) {
|
||||
return {
|
||||
stdout: error.stdout || '',
|
||||
stderr: error.stderr || String(error),
|
||||
exitCode: error.exitCode || 1,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleanup test directory
|
||||
*/
|
||||
function cleanupTestDir(dir: string): void {
|
||||
if (fs.existsSync(dir)) {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// TESTS
|
||||
// ========================================================================
|
||||
|
||||
tap.test('PyPI CLI: should verify pip is installed', async () => {
|
||||
try {
|
||||
const result = await tapNodeTools.runCommand('pip --version');
|
||||
console.log('pip version output:', result.stdout.substring(0, 200));
|
||||
hasPip = result.exitCode === 0;
|
||||
expect(result.exitCode).toEqual(0);
|
||||
} catch (error) {
|
||||
console.log('pip CLI not available');
|
||||
}
|
||||
});
|
||||
|
||||
tap.test('PyPI CLI: should verify twine is installed', async () => {
|
||||
try {
|
||||
const result = await tapNodeTools.runCommand('twine --version');
|
||||
console.log('twine version output:', result.stdout.substring(0, 200));
|
||||
hasTwine = result.exitCode === 0;
|
||||
expect(result.exitCode).toEqual(0);
|
||||
} catch (error) {
|
||||
console.log('twine CLI not available');
|
||||
}
|
||||
|
||||
if (!hasPip && !hasTwine) {
|
||||
console.log('Neither pip nor twine available, skipping native CLI tests');
|
||||
tap.skip.test('PyPI CLI: remaining tests skipped - no CLI tools available');
|
||||
return;
|
||||
}
|
||||
});
|
||||
|
||||
tap.test('PyPI CLI: should setup registry and HTTP server', async () => {
|
||||
// Use port 39000 (avoids conflicts with other tests)
|
||||
registryPort = 39000;
|
||||
|
||||
// Create registry with correct registryUrl for CLI tests
|
||||
registry = await createTestRegistry({ registryUrl: `http://localhost:${registryPort}` });
|
||||
const tokens = await createTestTokens(registry);
|
||||
pypiToken = tokens.pypiToken;
|
||||
|
||||
expect(registry).toBeInstanceOf(SmartRegistry);
|
||||
expect(pypiToken).toBeTypeOf('string');
|
||||
const serverSetup = await createHttpServer(registry, registryPort);
|
||||
server = serverSetup.server;
|
||||
registryUrl = serverSetup.url;
|
||||
|
||||
expect(server).toBeDefined();
|
||||
expect(registryUrl).toEqual(`http://localhost:${registryPort}`);
|
||||
|
||||
// Setup test directory
|
||||
testDir = path.join(process.cwd(), '.nogit', 'test-pypi-cli');
|
||||
cleanupTestDir(testDir);
|
||||
fs.mkdirSync(testDir, { recursive: true });
|
||||
|
||||
// Setup pip/pypi home directory
|
||||
pipHome = path.join(testDir, '.pip');
|
||||
fs.mkdirSync(pipHome, { recursive: true });
|
||||
|
||||
// Setup .pypirc for twine
|
||||
const pypircPath = setupPypirc(pypiToken, pipHome, registryUrl);
|
||||
expect(fs.existsSync(pypircPath)).toEqual(true);
|
||||
|
||||
// Setup pip.conf
|
||||
const pipConfPath = setupPipConf(pypiToken, pipHome, registryUrl, registryPort);
|
||||
expect(fs.existsSync(pipConfPath)).toEqual(true);
|
||||
});
|
||||
|
||||
tap.test('PyPI CLI: should verify server is responding', async () => {
|
||||
// Check server is up by doing a direct HTTP request to simple index
|
||||
const response = await fetch(`${registryUrl}/pypi/simple/`);
|
||||
expect(response.status).toBeGreaterThanOrEqual(200);
|
||||
expect(response.status).toBeLessThan(500);
|
||||
});
|
||||
|
||||
tap.test('PyPI CLI: should upload wheel with twine', async () => {
|
||||
if (!hasTwine) {
|
||||
console.log('Skipping twine test - twine not available');
|
||||
return;
|
||||
}
|
||||
|
||||
const version = '1.0.0';
|
||||
const wheelPath = await createTestWheelFile(testPackageName, version, testDir);
|
||||
|
||||
expect(fs.existsSync(wheelPath)).toEqual(true);
|
||||
|
||||
const result = await runTwineCommand(
|
||||
`upload --repository testpypi "${wheelPath}"`,
|
||||
testDir
|
||||
);
|
||||
console.log('twine upload output:', result.stdout);
|
||||
console.log('twine upload stderr:', result.stderr);
|
||||
|
||||
expect(result.exitCode).toEqual(0);
|
||||
});
|
||||
|
||||
tap.test('PyPI CLI: should verify package in simple index', async () => {
|
||||
if (!hasTwine) {
|
||||
console.log('Skipping - twine not available');
|
||||
return;
|
||||
}
|
||||
|
||||
const response = await fetch(`${registryUrl}/pypi/simple/${testPackageName}/`);
|
||||
expect(response.status).toEqual(200);
|
||||
|
||||
const html = await response.text();
|
||||
expect(html).toContain('1.0.0');
|
||||
});
|
||||
|
||||
tap.test('PyPI CLI: should upload sdist with twine', async () => {
|
||||
if (!hasTwine) {
|
||||
console.log('Skipping twine test - twine not available');
|
||||
return;
|
||||
}
|
||||
|
||||
const version = '1.1.0';
|
||||
const sdistPath = await createTestSdistFile(testPackageName, version, testDir);
|
||||
|
||||
expect(fs.existsSync(sdistPath)).toEqual(true);
|
||||
|
||||
const result = await runTwineCommand(
|
||||
`upload --repository testpypi "${sdistPath}"`,
|
||||
testDir
|
||||
);
|
||||
console.log('twine upload sdist output:', result.stdout);
|
||||
console.log('twine upload sdist stderr:', result.stderr);
|
||||
|
||||
expect(result.exitCode).toEqual(0);
|
||||
});
|
||||
|
||||
tap.test('PyPI CLI: should list all versions in simple index', async () => {
|
||||
if (!hasTwine) {
|
||||
console.log('Skipping - twine not available');
|
||||
return;
|
||||
}
|
||||
|
||||
const response = await fetch(`${registryUrl}/pypi/simple/${testPackageName}/`);
|
||||
expect(response.status).toEqual(200);
|
||||
|
||||
const html = await response.text();
|
||||
expect(html).toContain('1.0.0');
|
||||
expect(html).toContain('1.1.0');
|
||||
});
|
||||
|
||||
tap.test('PyPI CLI: should get JSON metadata', async () => {
|
||||
if (!hasTwine) {
|
||||
console.log('Skipping - twine not available');
|
||||
return;
|
||||
}
|
||||
|
||||
const response = await fetch(`${registryUrl}/pypi/pypi/${testPackageName}/json`);
|
||||
expect(response.status).toEqual(200);
|
||||
|
||||
const metadata = await response.json();
|
||||
expect(metadata.info).toBeDefined();
|
||||
expect(metadata.info.name).toEqual(testPackageName);
|
||||
expect(metadata.releases).toBeDefined();
|
||||
expect(metadata.releases['1.0.0']).toBeDefined();
|
||||
});
|
||||
|
||||
tap.test('PyPI CLI: should download package with pip', async () => {
|
||||
if (!hasPip || !hasTwine) {
|
||||
console.log('Skipping pip download test - pip or twine not available');
|
||||
return;
|
||||
}
|
||||
|
||||
const downloadDir = path.join(testDir, 'downloads');
|
||||
fs.mkdirSync(downloadDir, { recursive: true });
|
||||
|
||||
// Download (not install) the package
|
||||
const result = await runPipCommand(
|
||||
`download ${testPackageName}==1.0.0 --dest "${downloadDir}" --no-deps`,
|
||||
testDir
|
||||
);
|
||||
console.log('pip download output:', result.stdout);
|
||||
console.log('pip download stderr:', result.stderr);
|
||||
|
||||
// pip download may fail if the package doesn't meet pip's requirements
|
||||
// Just check it doesn't crash
|
||||
expect(result.exitCode).toBeLessThanOrEqual(1);
|
||||
});
|
||||
|
||||
tap.test('PyPI CLI: should search for packages via API', async () => {
|
||||
if (!hasTwine) {
|
||||
console.log('Skipping - twine not available (no packages uploaded)');
|
||||
return;
|
||||
}
|
||||
|
||||
// Use the JSON API to search/list
|
||||
const response = await fetch(`${registryUrl}/pypi/pypi/${testPackageName}/json`);
|
||||
expect(response.status).toEqual(200);
|
||||
|
||||
const metadata = await response.json();
|
||||
expect(metadata.info.name).toEqual(testPackageName);
|
||||
});
|
||||
|
||||
tap.test('PyPI CLI: should fail upload without auth', async () => {
|
||||
if (!hasTwine) {
|
||||
console.log('Skipping twine test - twine not available');
|
||||
return;
|
||||
}
|
||||
|
||||
const unauthPkgName = `unauth-pkg-${testRunId}`;
|
||||
const version = '1.0.0';
|
||||
const wheelPath = await createTestWheelFile(unauthPkgName, version, testDir);
|
||||
|
||||
// Create a pypirc without proper credentials
|
||||
const badPypircPath = path.join(testDir, '.bad-pypirc');
|
||||
fs.writeFileSync(badPypircPath, `[distutils]
|
||||
index-servers = badpypi
|
||||
|
||||
[badpypi]
|
||||
repository = ${registryUrl}/pypi
|
||||
username = baduser
|
||||
password = badtoken
|
||||
`, 'utf-8');
|
||||
|
||||
const fullCommand = `cd "${testDir}" && twine upload --repository badpypi "${wheelPath}" --config-file "${badPypircPath}"`;
|
||||
|
||||
try {
|
||||
const result = await tapNodeTools.runCommand(fullCommand);
|
||||
// Should fail
|
||||
expect(result.exitCode).not.toEqual(0);
|
||||
} catch (error: any) {
|
||||
// Expected to fail
|
||||
expect(error.exitCode || 1).not.toEqual(0);
|
||||
}
|
||||
});
|
||||
|
||||
tap.postTask('cleanup pypi cli tests', async () => {
|
||||
// Stop server
|
||||
if (server) {
|
||||
await new Promise<void>((resolve) => {
|
||||
server.close(() => resolve());
|
||||
});
|
||||
}
|
||||
|
||||
// Cleanup test directory
|
||||
if (testDir) {
|
||||
cleanupTestDir(testDir);
|
||||
}
|
||||
|
||||
// Destroy registry
|
||||
if (registry) {
|
||||
registry.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
export default tap.start();
|
||||
@@ -0,0 +1,485 @@
|
||||
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||
import { SmartRegistry } from '../ts/index.js';
|
||||
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
|
||||
import {
|
||||
createTestRegistry,
|
||||
createTestTokens,
|
||||
createPythonWheel,
|
||||
createPythonSdist,
|
||||
calculatePypiHashes,
|
||||
} from './helpers/registry.js';
|
||||
import { normalizePypiPackageName } from '../ts/pypi/helpers.pypi.js';
|
||||
|
||||
let registry: SmartRegistry;
|
||||
let pypiToken: string;
|
||||
let userId: string;
|
||||
|
||||
// Test data
|
||||
const testPackageName = 'test-package';
|
||||
const normalizedPackageName = normalizePypiPackageName(testPackageName);
|
||||
const testVersion = '1.0.0';
|
||||
let testWheelData: Buffer;
|
||||
let testSdistData: Buffer;
|
||||
|
||||
tap.test('PyPI: should create registry instance', async () => {
|
||||
registry = await createTestRegistry();
|
||||
const tokens = await createTestTokens(registry);
|
||||
pypiToken = tokens.pypiToken;
|
||||
userId = tokens.userId;
|
||||
|
||||
expect(registry).toBeInstanceOf(SmartRegistry);
|
||||
expect(pypiToken).toBeTypeOf('string');
|
||||
|
||||
// Clean up any existing metadata from previous test runs
|
||||
const storage = registry.getStorage();
|
||||
try {
|
||||
await storage.deletePypiPackage(normalizedPackageName);
|
||||
} catch (error) {
|
||||
// Ignore error if package doesn't exist
|
||||
}
|
||||
});
|
||||
|
||||
tap.test('PyPI: should create test package files', async () => {
|
||||
testWheelData = await createPythonWheel(testPackageName, testVersion);
|
||||
testSdistData = await createPythonSdist(testPackageName, testVersion);
|
||||
|
||||
expect(testWheelData).toBeInstanceOf(Buffer);
|
||||
expect(testWheelData.length).toBeGreaterThan(0);
|
||||
expect(testSdistData).toBeInstanceOf(Buffer);
|
||||
expect(testSdistData.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
tap.test('PyPI: should upload wheel file (POST /pypi/)', async () => {
|
||||
const hashes = calculatePypiHashes(testWheelData);
|
||||
const filename = `${testPackageName.replace(/-/g, '_')}-${testVersion}-py3-none-any.whl`;
|
||||
|
||||
const formData = new FormData();
|
||||
formData.append(':action', 'file_upload');
|
||||
formData.append('protocol_version', '1');
|
||||
formData.append('name', testPackageName);
|
||||
formData.append('version', testVersion);
|
||||
formData.append('filetype', 'bdist_wheel');
|
||||
formData.append('pyversion', 'py3');
|
||||
formData.append('metadata_version', '2.1');
|
||||
formData.append('sha256_digest', hashes.sha256);
|
||||
formData.append('content', new Blob([testWheelData]), filename);
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'POST',
|
||||
path: '/pypi/',
|
||||
headers: {
|
||||
Authorization: `Bearer ${pypiToken}`,
|
||||
'Content-Type': 'multipart/form-data',
|
||||
},
|
||||
query: {},
|
||||
body: {
|
||||
':action': 'file_upload',
|
||||
protocol_version: '1',
|
||||
name: testPackageName,
|
||||
version: testVersion,
|
||||
filetype: 'bdist_wheel',
|
||||
pyversion: 'py3',
|
||||
metadata_version: '2.1',
|
||||
sha256_digest: hashes.sha256,
|
||||
requires_python: '>=3.7',
|
||||
content: testWheelData,
|
||||
filename: filename,
|
||||
},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(201);
|
||||
});
|
||||
|
||||
tap.test('PyPI: should retrieve Simple API root index HTML (GET /simple/)', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/simple/',
|
||||
headers: {
|
||||
Accept: 'text/html',
|
||||
},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.headers['Content-Type']).toStartWith('text/html');
|
||||
const body = await streamToBuffer(response.body);
|
||||
const html = body.toString('utf-8');
|
||||
expect(html).toBeTypeOf('string');
|
||||
|
||||
expect(html).toContain('<!DOCTYPE html>');
|
||||
expect(html).toContain('<title>Simple Index</title>');
|
||||
expect(html).toContain(normalizedPackageName);
|
||||
});
|
||||
|
||||
tap.test('PyPI: should retrieve Simple API root index JSON (GET /simple/ with Accept: application/vnd.pypi.simple.v1+json)', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/simple/',
|
||||
headers: {
|
||||
Accept: 'application/vnd.pypi.simple.v1+json',
|
||||
},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.headers['Content-Type']).toEqual('application/vnd.pypi.simple.v1+json');
|
||||
const json = await streamToJson(response.body);
|
||||
expect(json).toBeTypeOf('object');
|
||||
|
||||
expect(json).toHaveProperty('meta');
|
||||
expect(json).toHaveProperty('projects');
|
||||
expect(json.projects).toBeInstanceOf(Array);
|
||||
// Check that the package is in the projects list (PEP 691 format: array of { name } objects)
|
||||
const packageNames = json.projects.map((p: any) => p.name);
|
||||
expect(packageNames).toContain(normalizedPackageName);
|
||||
});
|
||||
|
||||
tap.test('PyPI: should retrieve Simple API package HTML (GET /simple/{package}/)', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: `/simple/${normalizedPackageName}/`,
|
||||
headers: {
|
||||
Accept: 'text/html',
|
||||
},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.headers['Content-Type']).toStartWith('text/html');
|
||||
const body = await streamToBuffer(response.body);
|
||||
const html = body.toString('utf-8');
|
||||
expect(html).toBeTypeOf('string');
|
||||
|
||||
expect(html).toContain('<!DOCTYPE html>');
|
||||
expect(html).toContain(`<title>Links for ${normalizedPackageName}</title>`);
|
||||
expect(html).toContain('.whl');
|
||||
expect(html).toContain('data-requires-python');
|
||||
});
|
||||
|
||||
tap.test('PyPI: should retrieve Simple API package JSON (GET /simple/{package}/ with Accept: application/vnd.pypi.simple.v1+json)', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: `/simple/${normalizedPackageName}/`,
|
||||
headers: {
|
||||
Accept: 'application/vnd.pypi.simple.v1+json',
|
||||
},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.headers['Content-Type']).toEqual('application/vnd.pypi.simple.v1+json');
|
||||
const json = await streamToJson(response.body);
|
||||
expect(json).toBeTypeOf('object');
|
||||
|
||||
expect(json).toHaveProperty('meta');
|
||||
expect(json).toHaveProperty('name');
|
||||
expect(json.name).toEqual(normalizedPackageName);
|
||||
expect(json).toHaveProperty('files');
|
||||
expect(json.files).toBeTypeOf('object');
|
||||
expect(Object.keys(json.files).length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
tap.test('PyPI: should download wheel file (GET /pypi/packages/{package}/{filename})', async () => {
|
||||
const filename = `${testPackageName.replace(/-/g, '_')}-${testVersion}-py3-none-any.whl`;
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: `/pypi/packages/${normalizedPackageName}/${filename}`,
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
const body = await streamToBuffer(response.body);
|
||||
expect(body).toBeInstanceOf(Buffer);
|
||||
expect(body.length).toEqual(testWheelData.length);
|
||||
expect(response.headers['Content-Type']).toEqual('application/octet-stream');
|
||||
});
|
||||
|
||||
tap.test('PyPI: should upload sdist file (POST /pypi/)', async () => {
|
||||
const hashes = calculatePypiHashes(testSdistData);
|
||||
const filename = `${testPackageName}-${testVersion}.tar.gz`;
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'POST',
|
||||
path: '/pypi/',
|
||||
headers: {
|
||||
Authorization: `Bearer ${pypiToken}`,
|
||||
'Content-Type': 'multipart/form-data',
|
||||
},
|
||||
query: {},
|
||||
body: {
|
||||
':action': 'file_upload',
|
||||
protocol_version: '1',
|
||||
name: testPackageName,
|
||||
version: testVersion,
|
||||
filetype: 'sdist',
|
||||
pyversion: 'source',
|
||||
metadata_version: '2.1',
|
||||
sha256_digest: hashes.sha256,
|
||||
requires_python: '>=3.7',
|
||||
content: testSdistData,
|
||||
filename: filename,
|
||||
},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(201);
|
||||
});
|
||||
|
||||
tap.test('PyPI: should list both wheel and sdist in Simple API', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: `/simple/${normalizedPackageName}/`,
|
||||
headers: {
|
||||
Accept: 'application/vnd.pypi.simple.v1+json',
|
||||
},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
|
||||
const json = await streamToJson(response.body);
|
||||
// PEP 691: files is an array of file objects
|
||||
expect(json.files.length).toEqual(2);
|
||||
|
||||
const hasWheel = json.files.some((f: any) => f.filename.endsWith('.whl'));
|
||||
const hasSdist = json.files.some((f: any) => f.filename.endsWith('.tar.gz'));
|
||||
|
||||
expect(hasWheel).toEqual(true);
|
||||
expect(hasSdist).toEqual(true);
|
||||
});
|
||||
|
||||
tap.test('PyPI: should upload a second version', async () => {
|
||||
const newVersion = '2.0.0';
|
||||
const newWheelData = await createPythonWheel(testPackageName, newVersion);
|
||||
const hashes = calculatePypiHashes(newWheelData);
|
||||
const filename = `${testPackageName.replace(/-/g, '_')}-${newVersion}-py3-none-any.whl`;
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'POST',
|
||||
path: '/pypi/',
|
||||
headers: {
|
||||
Authorization: `Bearer ${pypiToken}`,
|
||||
'Content-Type': 'multipart/form-data',
|
||||
},
|
||||
query: {},
|
||||
body: {
|
||||
':action': 'file_upload',
|
||||
protocol_version: '1',
|
||||
name: testPackageName,
|
||||
version: newVersion,
|
||||
filetype: 'bdist_wheel',
|
||||
pyversion: 'py3',
|
||||
metadata_version: '2.1',
|
||||
sha256_digest: hashes.sha256,
|
||||
requires_python: '>=3.7',
|
||||
content: newWheelData,
|
||||
filename: filename,
|
||||
},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(201);
|
||||
});
|
||||
|
||||
tap.test('PyPI: should list multiple versions in Simple API', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: `/simple/${normalizedPackageName}/`,
|
||||
headers: {
|
||||
Accept: 'application/vnd.pypi.simple.v1+json',
|
||||
},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
|
||||
const json = await streamToJson(response.body);
|
||||
// PEP 691: files is an array of file objects
|
||||
expect(json.files.length).toBeGreaterThan(2);
|
||||
|
||||
const hasVersion1 = json.files.some((f: any) => f.filename.includes('1.0.0'));
|
||||
const hasVersion2 = json.files.some((f: any) => f.filename.includes('2.0.0'));
|
||||
|
||||
expect(hasVersion1).toEqual(true);
|
||||
expect(hasVersion2).toEqual(true);
|
||||
});
|
||||
|
||||
tap.test('PyPI: should normalize package names correctly', async () => {
|
||||
const testNames = [
|
||||
{ input: 'Test-Package', expected: 'test-package' },
|
||||
{ input: 'Test_Package', expected: 'test-package' },
|
||||
{ input: 'Test..Package', expected: 'test-package' },
|
||||
{ input: 'Test---Package', expected: 'test-package' },
|
||||
];
|
||||
|
||||
for (const { input, expected } of testNames) {
|
||||
const normalized = normalizePypiPackageName(input);
|
||||
expect(normalized).toEqual(expected);
|
||||
}
|
||||
});
|
||||
|
||||
tap.test('PyPI: should return 404 for non-existent package', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/simple/nonexistent-package/',
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(404);
|
||||
const body = await streamToJson(response.body);
|
||||
expect(body).toHaveProperty('error');
|
||||
});
|
||||
|
||||
tap.test('PyPI: should return 401 for unauthorized upload', async () => {
|
||||
const wheelData = await createPythonWheel('unauthorized-test', '1.0.0');
|
||||
const hashes = calculatePypiHashes(wheelData);
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'POST',
|
||||
path: '/pypi/',
|
||||
headers: {
|
||||
// No authorization header
|
||||
'Content-Type': 'multipart/form-data',
|
||||
},
|
||||
query: {},
|
||||
body: {
|
||||
':action': 'file_upload',
|
||||
protocol_version: '1',
|
||||
name: 'unauthorized-test',
|
||||
version: '1.0.0',
|
||||
filetype: 'bdist_wheel',
|
||||
pyversion: 'py3',
|
||||
metadata_version: '2.1',
|
||||
sha256_digest: hashes.sha256,
|
||||
content: wheelData,
|
||||
filename: 'unauthorized_test-1.0.0-py3-none-any.whl',
|
||||
},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(401);
|
||||
const body = await streamToJson(response.body);
|
||||
expect(body).toHaveProperty('error');
|
||||
});
|
||||
|
||||
tap.test('PyPI: should reject upload with mismatched hash', async () => {
|
||||
const wheelData = await createPythonWheel('hash-test', '1.0.0');
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'POST',
|
||||
path: '/pypi/',
|
||||
headers: {
|
||||
Authorization: `Bearer ${pypiToken}`,
|
||||
'Content-Type': 'multipart/form-data',
|
||||
},
|
||||
query: {},
|
||||
body: {
|
||||
':action': 'file_upload',
|
||||
protocol_version: '1',
|
||||
name: 'hash-test',
|
||||
version: '1.0.0',
|
||||
filetype: 'bdist_wheel',
|
||||
pyversion: 'py3',
|
||||
metadata_version: '2.1',
|
||||
sha256_digest: 'wrong_hash_value',
|
||||
content: wheelData,
|
||||
filename: 'hash_test-1.0.0-py3-none-any.whl',
|
||||
},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(400);
|
||||
const body = await streamToJson(response.body);
|
||||
expect(body).toHaveProperty('error');
|
||||
});
|
||||
|
||||
tap.test('PyPI: should handle package with requires-python metadata', async () => {
|
||||
const packageName = 'python-version-test';
|
||||
const wheelData = await createPythonWheel(packageName, '1.0.0');
|
||||
const hashes = calculatePypiHashes(wheelData);
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'POST',
|
||||
path: '/pypi/',
|
||||
headers: {
|
||||
Authorization: `Bearer ${pypiToken}`,
|
||||
'Content-Type': 'multipart/form-data',
|
||||
},
|
||||
query: {},
|
||||
body: {
|
||||
':action': 'file_upload',
|
||||
protocol_version: '1',
|
||||
name: packageName,
|
||||
version: '1.0.0',
|
||||
filetype: 'bdist_wheel',
|
||||
pyversion: 'py3',
|
||||
metadata_version: '2.1',
|
||||
sha256_digest: hashes.sha256,
|
||||
'requires_python': '>=3.8',
|
||||
content: wheelData,
|
||||
filename: `${packageName.replace(/-/g, '_')}-1.0.0-py3-none-any.whl`,
|
||||
},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(201);
|
||||
|
||||
// Verify requires-python is in Simple API
|
||||
const getResponse = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: `/simple/${normalizePypiPackageName(packageName)}/`,
|
||||
headers: {
|
||||
Accept: 'text/html',
|
||||
},
|
||||
query: {},
|
||||
});
|
||||
|
||||
const getBody = await streamToBuffer(getResponse.body);
|
||||
const html = getBody.toString('utf-8');
|
||||
expect(html).toContain('data-requires-python');
|
||||
// Note: >= gets HTML-escaped to >= in attribute values
|
||||
expect(html).toContain('>=3.8');
|
||||
});
|
||||
|
||||
tap.test('PyPI: should support JSON API for package metadata', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: `/pypi/${normalizedPackageName}/json`,
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.headers['Content-Type']).toEqual('application/json');
|
||||
const json = await streamToJson(response.body);
|
||||
expect(json).toBeTypeOf('object');
|
||||
|
||||
expect(json).toHaveProperty('info');
|
||||
expect(json.info).toHaveProperty('name');
|
||||
expect(json.info.name).toEqual(normalizedPackageName);
|
||||
expect(json).toHaveProperty('urls');
|
||||
});
|
||||
|
||||
tap.test('PyPI: should support JSON API for specific version', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: `/pypi/${normalizedPackageName}/${testVersion}/json`,
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.headers['Content-Type']).toEqual('application/json');
|
||||
const json = await streamToJson(response.body);
|
||||
expect(json).toBeTypeOf('object');
|
||||
|
||||
expect(json).toHaveProperty('info');
|
||||
expect(json.info.version).toEqual(testVersion);
|
||||
expect(json).toHaveProperty('urls');
|
||||
});
|
||||
|
||||
tap.postTask('cleanup registry', async () => {
|
||||
if (registry) {
|
||||
registry.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
export default tap.start();
|
||||
@@ -0,0 +1,449 @@
|
||||
/**
|
||||
* Native gem CLI Testing
|
||||
* Tests the RubyGems registry implementation using the actual gem CLI
|
||||
*/
|
||||
|
||||
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||
import { tapNodeTools } from '@git.zone/tstest/tapbundle_serverside';
|
||||
import { SmartRegistry } from '../ts/index.js';
|
||||
import { createTestRegistry, createTestTokens, createRubyGem } from './helpers/registry.js';
|
||||
import type { IRequestContext, IResponse } from '../ts/core/interfaces.core.js';
|
||||
import * as http from 'http';
|
||||
import * as url from 'url';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
// Test context
|
||||
let registry: SmartRegistry;
|
||||
let server: http.Server;
|
||||
let registryUrl: string;
|
||||
let registryPort: number;
|
||||
let rubygemsToken: string;
|
||||
let testDir: string;
|
||||
let gemHome: string;
|
||||
|
||||
/**
|
||||
* Create HTTP server wrapper around SmartRegistry
|
||||
*/
|
||||
async function createHttpServer(
|
||||
registryInstance: SmartRegistry,
|
||||
port: number
|
||||
): Promise<{ server: http.Server; url: string }> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const httpServer = http.createServer(async (req, res) => {
|
||||
try {
|
||||
// Parse request
|
||||
const parsedUrl = url.parse(req.url || '', true);
|
||||
const pathname = parsedUrl.pathname || '/';
|
||||
const query = parsedUrl.query;
|
||||
|
||||
// Read body
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of req) {
|
||||
chunks.push(chunk);
|
||||
}
|
||||
const bodyBuffer = Buffer.concat(chunks);
|
||||
|
||||
// Parse body based on content type
|
||||
let body: any;
|
||||
if (bodyBuffer.length > 0) {
|
||||
const contentType = req.headers['content-type'] || '';
|
||||
if (contentType.includes('application/json')) {
|
||||
try {
|
||||
body = JSON.parse(bodyBuffer.toString('utf-8'));
|
||||
} catch (error) {
|
||||
body = bodyBuffer;
|
||||
}
|
||||
} else {
|
||||
body = bodyBuffer;
|
||||
}
|
||||
}
|
||||
|
||||
// Convert to IRequestContext
|
||||
const context: IRequestContext = {
|
||||
method: req.method || 'GET',
|
||||
path: pathname,
|
||||
headers: req.headers as Record<string, string>,
|
||||
query: query as Record<string, string>,
|
||||
body: body,
|
||||
};
|
||||
|
||||
// Handle request
|
||||
const response: IResponse = await registryInstance.handleRequest(context);
|
||||
|
||||
// Convert IResponse to HTTP response
|
||||
res.statusCode = response.status;
|
||||
|
||||
// Set headers
|
||||
for (const [key, value] of Object.entries(response.headers || {})) {
|
||||
res.setHeader(key, value);
|
||||
}
|
||||
|
||||
// Send body (response.body is always ReadableStream<Uint8Array> or undefined)
|
||||
if (response.body) {
|
||||
const { Readable } = await import('stream');
|
||||
Readable.fromWeb(response.body).pipe(res);
|
||||
} else {
|
||||
res.end();
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Server error:', error);
|
||||
res.statusCode = 500;
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.end(JSON.stringify({ error: 'INTERNAL_ERROR', message: String(error) }));
|
||||
}
|
||||
});
|
||||
|
||||
httpServer.listen(port, () => {
|
||||
const serverUrl = `http://localhost:${port}`;
|
||||
resolve({ server: httpServer, url: serverUrl });
|
||||
});
|
||||
|
||||
httpServer.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Setup gem credentials file
|
||||
* Format: YAML with :rubygems_api_key: TOKEN
|
||||
*/
|
||||
function setupGemCredentials(token: string, gemHomeArg: string): string {
|
||||
const gemDir = path.join(gemHomeArg, '.gem');
|
||||
fs.mkdirSync(gemDir, { recursive: true });
|
||||
|
||||
// Create credentials file in YAML format
|
||||
const credentialsContent = `:rubygems_api_key: ${token}\n`;
|
||||
|
||||
const credentialsPath = path.join(gemDir, 'credentials');
|
||||
fs.writeFileSync(credentialsPath, credentialsContent, 'utf-8');
|
||||
|
||||
// Set restrictive permissions (gem requires 0600)
|
||||
fs.chmodSync(credentialsPath, 0o600);
|
||||
|
||||
return credentialsPath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a test gem file
|
||||
*/
|
||||
async function createTestGemFile(
|
||||
gemName: string,
|
||||
version: string,
|
||||
targetDir: string
|
||||
): Promise<string> {
|
||||
const gemData = await createRubyGem(gemName, version);
|
||||
const gemFilename = `${gemName}-${version}.gem`;
|
||||
const gemPath = path.join(targetDir, gemFilename);
|
||||
|
||||
fs.writeFileSync(gemPath, gemData);
|
||||
|
||||
return gemPath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run gem command with proper environment
|
||||
*/
|
||||
async function runGemCommand(
|
||||
command: string,
|
||||
cwd: string,
|
||||
includeAuth: boolean = true
|
||||
): Promise<{ stdout: string; stderr: string; exitCode: number }> {
|
||||
// When not including auth, use a temp HOME without credentials
|
||||
const effectiveHome = includeAuth ? gemHome : path.join(gemHome, 'noauth');
|
||||
if (!includeAuth) {
|
||||
fs.mkdirSync(effectiveHome, { recursive: true });
|
||||
}
|
||||
|
||||
// Prepare environment variables
|
||||
const envVars = [
|
||||
`HOME="${effectiveHome}"`,
|
||||
`GEM_HOME="${gemHome}"`,
|
||||
].filter(Boolean).join(' ');
|
||||
|
||||
// Build command with cd to correct directory and environment variables
|
||||
const fullCommand = `cd "${cwd}" && ${envVars} ${command}`;
|
||||
|
||||
try {
|
||||
const result = await tapNodeTools.runCommand(fullCommand);
|
||||
return {
|
||||
stdout: result.stdout || '',
|
||||
stderr: result.stderr || '',
|
||||
exitCode: result.exitCode || 0,
|
||||
};
|
||||
} catch (error: any) {
|
||||
return {
|
||||
stdout: error.stdout || '',
|
||||
stderr: error.stderr || String(error),
|
||||
exitCode: error.exitCode || 1,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleanup test directory
|
||||
*/
|
||||
function cleanupTestDir(dir: string): void {
|
||||
if (fs.existsSync(dir)) {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// TESTS
|
||||
// ========================================================================
|
||||
|
||||
tap.test('RubyGems CLI: should setup registry and HTTP server', async () => {
|
||||
// Use port 36000 (avoids npm:35000, cargo:5000 conflicts)
|
||||
registryPort = 36000;
|
||||
|
||||
// Create registry with correct registryUrl for CLI tests
|
||||
registry = await createTestRegistry({ registryUrl: `http://localhost:${registryPort}` });
|
||||
const tokens = await createTestTokens(registry);
|
||||
rubygemsToken = tokens.rubygemsToken;
|
||||
|
||||
expect(registry).toBeInstanceOf(SmartRegistry);
|
||||
expect(rubygemsToken).toBeTypeOf('string');
|
||||
const serverSetup = await createHttpServer(registry, registryPort);
|
||||
server = serverSetup.server;
|
||||
registryUrl = serverSetup.url;
|
||||
|
||||
expect(server).toBeDefined();
|
||||
expect(registryUrl).toEqual(`http://localhost:${registryPort}`);
|
||||
|
||||
// Setup test directory
|
||||
testDir = path.join(process.cwd(), '.nogit', 'test-rubygems-cli');
|
||||
cleanupTestDir(testDir);
|
||||
fs.mkdirSync(testDir, { recursive: true });
|
||||
|
||||
// Setup GEM_HOME
|
||||
gemHome = path.join(testDir, '.gem-home');
|
||||
fs.mkdirSync(gemHome, { recursive: true });
|
||||
|
||||
// Setup gem credentials
|
||||
const credentialsPath = setupGemCredentials(rubygemsToken, gemHome);
|
||||
expect(fs.existsSync(credentialsPath)).toEqual(true);
|
||||
|
||||
// Verify credentials file has correct permissions
|
||||
const stats = fs.statSync(credentialsPath);
|
||||
const mode = stats.mode & 0o777;
|
||||
expect(mode).toEqual(0o600);
|
||||
});
|
||||
|
||||
tap.test('RubyGems CLI: should verify server is responding', async () => {
|
||||
// Check server is up by doing a direct HTTP request to the Compact Index
|
||||
const response = await fetch(`${registryUrl}/rubygems/versions`);
|
||||
expect(response.status).toBeGreaterThanOrEqual(200);
|
||||
expect(response.status).toBeLessThan(500);
|
||||
});
|
||||
|
||||
tap.test('RubyGems CLI: should build and push a gem', async () => {
|
||||
const gemName = 'test-gem-cli';
|
||||
const version = '1.0.0';
|
||||
const gemPath = await createTestGemFile(gemName, version, testDir);
|
||||
|
||||
expect(fs.existsSync(gemPath)).toEqual(true);
|
||||
|
||||
const result = await runGemCommand(
|
||||
`gem push ${gemPath} --host ${registryUrl}/rubygems`,
|
||||
testDir
|
||||
);
|
||||
console.log('gem push output:', result.stdout);
|
||||
console.log('gem push stderr:', result.stderr);
|
||||
|
||||
expect(result.exitCode).toEqual(0);
|
||||
expect(result.stdout || result.stderr).toContain(gemName);
|
||||
});
|
||||
|
||||
tap.test('RubyGems CLI: should verify gem in Compact Index /versions', async () => {
|
||||
const gemName = 'test-gem-cli';
|
||||
|
||||
const response = await fetch(`${registryUrl}/rubygems/versions`);
|
||||
expect(response.status).toEqual(200);
|
||||
|
||||
const versionsData = await response.text();
|
||||
console.log('Versions data:', versionsData);
|
||||
|
||||
// Format: GEMNAME VERSION[,VERSION...] MD5
|
||||
expect(versionsData).toContain(gemName);
|
||||
expect(versionsData).toContain('1.0.0');
|
||||
});
|
||||
|
||||
tap.test('RubyGems CLI: should verify gem in Compact Index /info file', async () => {
|
||||
const gemName = 'test-gem-cli';
|
||||
|
||||
const response = await fetch(`${registryUrl}/rubygems/info/${gemName}`);
|
||||
expect(response.status).toEqual(200);
|
||||
|
||||
const infoData = await response.text();
|
||||
console.log('Info data:', infoData);
|
||||
|
||||
// Format: VERSION [DEPS]|REQS
|
||||
expect(infoData).toContain('1.0.0');
|
||||
});
|
||||
|
||||
tap.test('RubyGems CLI: should download gem file', async () => {
|
||||
const gemName = 'test-gem-cli';
|
||||
const version = '1.0.0';
|
||||
|
||||
const response = await fetch(`${registryUrl}/rubygems/gems/${gemName}-${version}.gem`);
|
||||
expect(response.status).toEqual(200);
|
||||
|
||||
const gemData = await response.arrayBuffer();
|
||||
expect(gemData.byteLength).toBeGreaterThan(0);
|
||||
|
||||
// Verify content type
|
||||
expect(response.headers.get('content-type')).toContain('application/octet-stream');
|
||||
});
|
||||
|
||||
tap.test('RubyGems CLI: should fetch gem metadata JSON', async () => {
|
||||
const gemName = 'test-gem-cli';
|
||||
|
||||
const response = await fetch(`${registryUrl}/rubygems/api/v1/versions/${gemName}.json`);
|
||||
expect(response.status).toEqual(200);
|
||||
|
||||
const metadata = await response.json();
|
||||
console.log('Metadata:', metadata);
|
||||
|
||||
expect(metadata).toBeInstanceOf(Array);
|
||||
expect(metadata.length).toBeGreaterThan(0);
|
||||
expect(metadata[0].number).toEqual('1.0.0');
|
||||
});
|
||||
|
||||
tap.test('RubyGems CLI: should push second version', async () => {
|
||||
const gemName = 'test-gem-cli';
|
||||
const version = '2.0.0';
|
||||
const gemPath = await createTestGemFile(gemName, version, testDir);
|
||||
|
||||
const result = await runGemCommand(
|
||||
`gem push ${gemPath} --host ${registryUrl}/rubygems`,
|
||||
testDir
|
||||
);
|
||||
console.log('gem push v2.0.0 output:', result.stdout);
|
||||
|
||||
expect(result.exitCode).toEqual(0);
|
||||
});
|
||||
|
||||
tap.test('RubyGems CLI: should list all versions in /versions file', async () => {
|
||||
const gemName = 'test-gem-cli';
|
||||
|
||||
const response = await fetch(`${registryUrl}/rubygems/versions`);
|
||||
expect(response.status).toEqual(200);
|
||||
|
||||
const versionsData = await response.text();
|
||||
console.log('All versions data:', versionsData);
|
||||
|
||||
// Should contain both versions
|
||||
expect(versionsData).toContain(gemName);
|
||||
expect(versionsData).toContain('1.0.0');
|
||||
expect(versionsData).toContain('2.0.0');
|
||||
});
|
||||
|
||||
tap.test('RubyGems CLI: should yank a version', async () => {
|
||||
const gemName = 'test-gem-cli';
|
||||
const version = '1.0.0';
|
||||
|
||||
const result = await runGemCommand(
|
||||
`gem yank ${gemName} -v ${version} --host ${registryUrl}/rubygems`,
|
||||
testDir
|
||||
);
|
||||
console.log('gem yank output:', result.stdout);
|
||||
console.log('gem yank stderr:', result.stderr);
|
||||
|
||||
expect(result.exitCode).toEqual(0);
|
||||
|
||||
// Verify version is yanked in /versions file
|
||||
// Yanked versions are prefixed with '-'
|
||||
const response = await fetch(`${registryUrl}/rubygems/versions`);
|
||||
const versionsData = await response.text();
|
||||
console.log('Versions after yank:', versionsData);
|
||||
|
||||
// Yanked version should have '-' prefix
|
||||
expect(versionsData).toContain('-1.0.0');
|
||||
});
|
||||
|
||||
tap.test('RubyGems CLI: should unyank a version', async () => {
|
||||
const gemName = 'test-gem-cli';
|
||||
const version = '1.0.0';
|
||||
|
||||
// Use PUT /api/v1/gems/unyank via HTTP API (gem yank --undo removed in Ruby 4.0)
|
||||
const response = await fetch(
|
||||
`${registryUrl}/rubygems/api/v1/gems/unyank?gem_name=${gemName}&version=${version}`,
|
||||
{
|
||||
method: 'PUT',
|
||||
headers: {
|
||||
'Authorization': rubygemsToken,
|
||||
},
|
||||
}
|
||||
);
|
||||
console.log('gem unyank status:', response.status);
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
|
||||
// Verify version is not yanked in /versions file
|
||||
const versionsResponse = await fetch(`${registryUrl}/rubygems/versions`);
|
||||
const versionsData = await versionsResponse.text();
|
||||
console.log('Versions after unyank:', versionsData);
|
||||
|
||||
// Should not have '-' prefix anymore
|
||||
const lines = versionsData.trim().split('\n');
|
||||
const gemLine = lines.find(line => line.startsWith(gemName));
|
||||
|
||||
if (gemLine) {
|
||||
const parts = gemLine.split(' ');
|
||||
const versions = parts[1];
|
||||
|
||||
expect(versions).toContain('1.0.0');
|
||||
expect(versions).not.toContain('-1.0.0');
|
||||
}
|
||||
});
|
||||
|
||||
tap.test('RubyGems CLI: should fetch dependencies', async () => {
|
||||
const gemName = 'test-gem-cli';
|
||||
|
||||
const response = await fetch(`${registryUrl}/rubygems/api/v1/dependencies?gems=${gemName}`);
|
||||
expect(response.status).toEqual(200);
|
||||
|
||||
const dependencies = await response.json();
|
||||
console.log('Dependencies:', dependencies);
|
||||
|
||||
expect(dependencies).toBeInstanceOf(Array);
|
||||
});
|
||||
|
||||
tap.test('RubyGems CLI: should fail to push without auth', async () => {
|
||||
const gemName = 'unauth-gem';
|
||||
const version = '1.0.0';
|
||||
const gemPath = await createTestGemFile(gemName, version, testDir);
|
||||
|
||||
// Run without auth
|
||||
const result = await runGemCommand(
|
||||
`gem push ${gemPath} --host ${registryUrl}/rubygems`,
|
||||
testDir,
|
||||
false
|
||||
);
|
||||
console.log('gem push unauth output:', result.stdout);
|
||||
console.log('gem push unauth stderr:', result.stderr);
|
||||
|
||||
// Should fail with auth error
|
||||
expect(result.exitCode).not.toEqual(0);
|
||||
});
|
||||
|
||||
tap.postTask('cleanup rubygems cli tests', async () => {
|
||||
// Stop server
|
||||
if (server) {
|
||||
await new Promise<void>((resolve) => {
|
||||
server.close(() => resolve());
|
||||
});
|
||||
}
|
||||
|
||||
// Cleanup test directory
|
||||
if (testDir) {
|
||||
cleanupTestDir(testDir);
|
||||
}
|
||||
|
||||
// Destroy registry
|
||||
if (registry) {
|
||||
registry.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
export default tap.start();
|
||||
@@ -0,0 +1,523 @@
|
||||
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||
import { SmartRegistry } from '../ts/index.js';
|
||||
import { streamToBuffer, streamToJson } from '../ts/core/helpers.stream.js';
|
||||
import {
|
||||
createTestRegistry,
|
||||
createTestTokens,
|
||||
createRubyGem,
|
||||
calculateRubyGemsChecksums,
|
||||
} from './helpers/registry.js';
|
||||
|
||||
let registry: SmartRegistry;
|
||||
let rubygemsToken: string;
|
||||
let userId: string;
|
||||
|
||||
// Test data
|
||||
const testGemName = 'test-gem';
|
||||
const testVersion = '1.0.0';
|
||||
let testGemData: Buffer;
|
||||
|
||||
tap.test('RubyGems: should create registry instance', async () => {
|
||||
registry = await createTestRegistry();
|
||||
const tokens = await createTestTokens(registry);
|
||||
rubygemsToken = tokens.rubygemsToken;
|
||||
userId = tokens.userId;
|
||||
|
||||
expect(registry).toBeInstanceOf(SmartRegistry);
|
||||
expect(rubygemsToken).toBeTypeOf('string');
|
||||
|
||||
// Clean up any existing metadata from previous test runs
|
||||
const storage = registry.getStorage();
|
||||
try {
|
||||
await storage.deleteRubyGem(testGemName);
|
||||
} catch (error) {
|
||||
// Ignore error if gem doesn't exist
|
||||
}
|
||||
});
|
||||
|
||||
tap.test('RubyGems: should create test gem file', async () => {
|
||||
testGemData = await createRubyGem(testGemName, testVersion);
|
||||
|
||||
expect(testGemData).toBeInstanceOf(Buffer);
|
||||
expect(testGemData.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
tap.test('RubyGems: should upload gem file (POST /rubygems/api/v1/gems)', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'POST',
|
||||
path: '/rubygems/api/v1/gems',
|
||||
headers: {
|
||||
Authorization: rubygemsToken,
|
||||
'Content-Type': 'application/octet-stream',
|
||||
},
|
||||
query: {},
|
||||
body: testGemData,
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(201);
|
||||
const body = await streamToJson(response.body);
|
||||
expect(body).toHaveProperty('message');
|
||||
});
|
||||
|
||||
tap.test('RubyGems: should retrieve Compact Index versions file (GET /rubygems/versions)', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/rubygems/versions',
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.headers['Content-Type']).toEqual('text/plain; charset=utf-8');
|
||||
const body = await streamToBuffer(response.body);
|
||||
expect(body).toBeInstanceOf(Buffer);
|
||||
|
||||
const content = body.toString('utf-8');
|
||||
expect(content).toContain('created_at:');
|
||||
expect(content).toContain('---');
|
||||
expect(content).toContain(testGemName);
|
||||
expect(content).toContain(testVersion);
|
||||
});
|
||||
|
||||
tap.test('RubyGems: should retrieve Compact Index info file (GET /rubygems/info/{gem})', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: `/rubygems/info/${testGemName}`,
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.headers['Content-Type']).toEqual('text/plain; charset=utf-8');
|
||||
const body = await streamToBuffer(response.body);
|
||||
expect(body).toBeInstanceOf(Buffer);
|
||||
|
||||
const content = body.toString('utf-8');
|
||||
expect(content).toContain('---');
|
||||
expect(content).toContain(testVersion);
|
||||
expect(content).toContain('checksum:');
|
||||
});
|
||||
|
||||
tap.test('RubyGems: should retrieve Compact Index names file (GET /rubygems/names)', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/rubygems/names',
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.headers['Content-Type']).toEqual('text/plain; charset=utf-8');
|
||||
const body = await streamToBuffer(response.body);
|
||||
expect(body).toBeInstanceOf(Buffer);
|
||||
|
||||
const content = body.toString('utf-8');
|
||||
expect(content).toContain('---');
|
||||
expect(content).toContain(testGemName);
|
||||
});
|
||||
|
||||
tap.test('RubyGems: should download gem file (GET /rubygems/gems/{gem}-{version}.gem)', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: `/rubygems/gems/${testGemName}-${testVersion}.gem`,
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
const body = await streamToBuffer(response.body);
|
||||
expect(body).toBeInstanceOf(Buffer);
|
||||
expect(body.length).toEqual(testGemData.length);
|
||||
expect(response.headers['Content-Type']).toEqual('application/octet-stream');
|
||||
});
|
||||
|
||||
tap.test('RubyGems: should upload a second version', async () => {
|
||||
const newVersion = '2.0.0';
|
||||
const newGemData = await createRubyGem(testGemName, newVersion);
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'POST',
|
||||
path: '/rubygems/api/v1/gems',
|
||||
headers: {
|
||||
Authorization: rubygemsToken,
|
||||
'Content-Type': 'application/octet-stream',
|
||||
},
|
||||
query: {},
|
||||
body: newGemData,
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(201);
|
||||
});
|
||||
|
||||
tap.test('RubyGems: should list multiple versions in Compact Index', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/rubygems/versions',
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
|
||||
const body = await streamToBuffer(response.body);
|
||||
const content = body.toString('utf-8');
|
||||
const lines = content.split('\n');
|
||||
const gemLine = lines.find(l => l.startsWith(`${testGemName} `));
|
||||
|
||||
expect(gemLine).toBeDefined();
|
||||
expect(gemLine).toContain('1.0.0');
|
||||
expect(gemLine).toContain('2.0.0');
|
||||
});
|
||||
|
||||
tap.test('RubyGems: should list multiple versions in info file', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: `/rubygems/info/${testGemName}`,
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
|
||||
const body = await streamToBuffer(response.body);
|
||||
const content = body.toString('utf-8');
|
||||
expect(content).toContain('1.0.0');
|
||||
expect(content).toContain('2.0.0');
|
||||
});
|
||||
|
||||
tap.test('RubyGems: should support platform-specific gems', async () => {
|
||||
const platformVersion = '1.5.0';
|
||||
const platform = 'x86_64-linux';
|
||||
const platformGemData = await createRubyGem(testGemName, platformVersion, platform);
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'POST',
|
||||
path: '/rubygems/api/v1/gems',
|
||||
headers: {
|
||||
Authorization: rubygemsToken,
|
||||
'Content-Type': 'application/octet-stream',
|
||||
},
|
||||
query: {},
|
||||
body: platformGemData,
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(201);
|
||||
|
||||
// Verify platform is listed in versions
|
||||
const versionsResponse = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/rubygems/versions',
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
const versionsBody = await streamToBuffer(versionsResponse.body);
|
||||
const content = versionsBody.toString('utf-8');
|
||||
const lines = content.split('\n');
|
||||
const gemLine = lines.find(l => l.startsWith(`${testGemName} `));
|
||||
|
||||
expect(gemLine).toContain(`${platformVersion}_${platform}`);
|
||||
});
|
||||
|
||||
tap.test('RubyGems: should yank a gem version (DELETE /rubygems/api/v1/gems/yank)', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'DELETE',
|
||||
path: '/rubygems/api/v1/gems/yank',
|
||||
headers: {
|
||||
Authorization: rubygemsToken,
|
||||
},
|
||||
query: {
|
||||
gem_name: testGemName,
|
||||
version: testVersion,
|
||||
},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
const body = await streamToJson(response.body);
|
||||
expect(body).toHaveProperty('message');
|
||||
expect(body.message).toContain('yanked');
|
||||
});
|
||||
|
||||
tap.test('RubyGems: should mark yanked version in Compact Index', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/rubygems/versions',
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
|
||||
const body = await streamToBuffer(response.body);
|
||||
const content = body.toString('utf-8');
|
||||
const lines = content.split('\n');
|
||||
const gemLine = lines.find(l => l.startsWith(`${testGemName} `));
|
||||
|
||||
// Yanked versions are prefixed with '-'
|
||||
expect(gemLine).toContain(`-${testVersion}`);
|
||||
});
|
||||
|
||||
tap.test('RubyGems: should still allow downloading yanked gem', async () => {
|
||||
// Yanked gems can still be downloaded if explicitly requested
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: `/rubygems/gems/${testGemName}-${testVersion}.gem`,
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
const body = await streamToBuffer(response.body);
|
||||
expect(body).toBeInstanceOf(Buffer);
|
||||
});
|
||||
|
||||
tap.test('RubyGems: should unyank a gem version (PUT /rubygems/api/v1/gems/unyank)', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'PUT',
|
||||
path: '/rubygems/api/v1/gems/unyank',
|
||||
headers: {
|
||||
Authorization: rubygemsToken,
|
||||
},
|
||||
query: {
|
||||
gem_name: testGemName,
|
||||
version: testVersion,
|
||||
},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
const body = await streamToJson(response.body);
|
||||
expect(body).toHaveProperty('message');
|
||||
expect(body.message).toContain('unyanked');
|
||||
});
|
||||
|
||||
tap.test('RubyGems: should remove yank marker after unyank', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/rubygems/versions',
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
|
||||
const body = await streamToBuffer(response.body);
|
||||
const content = body.toString('utf-8');
|
||||
const lines = content.split('\n');
|
||||
const gemLine = lines.find(l => l.startsWith(`${testGemName} `));
|
||||
|
||||
// After unyank, version should not have '-' prefix
|
||||
const versions = gemLine!.split(' ')[1].split(',');
|
||||
const version1 = versions.find(v => v.includes('1.0.0'));
|
||||
|
||||
expect(version1).not.toStartWith('-');
|
||||
expect(version1).toContain('1.0.0');
|
||||
});
|
||||
|
||||
tap.test('RubyGems: should retrieve versions JSON (GET /rubygems/api/v1/versions/{gem}.json)', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: `/rubygems/api/v1/versions/${testGemName}.json`,
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.headers['Content-Type']).toEqual('application/json');
|
||||
const json = await streamToJson(response.body);
|
||||
expect(json).toBeInstanceOf(Array);
|
||||
expect(json.length).toBeGreaterThan(0);
|
||||
expect(json[0]).toHaveProperty('number');
|
||||
});
|
||||
|
||||
tap.test('RubyGems: should retrieve dependencies JSON (GET /rubygems/api/v1/dependencies)', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/rubygems/api/v1/dependencies',
|
||||
headers: {},
|
||||
query: {
|
||||
gems: `${testGemName}`,
|
||||
},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.headers['Content-Type']).toEqual('application/json');
|
||||
const json = await streamToJson(response.body);
|
||||
expect(json).toBeTypeOf('object');
|
||||
|
||||
expect(Array.isArray(json)).toEqual(true);
|
||||
});
|
||||
|
||||
tap.test('RubyGems: should retrieve gem spec (GET /rubygems/quick/Marshal.4.8/{gem}-{version}.gemspec.rz)', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: `/rubygems/quick/Marshal.4.8/${testGemName}-${testVersion}.gemspec.rz`,
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
const body = await streamToBuffer(response.body);
|
||||
expect(body).toBeInstanceOf(Buffer);
|
||||
});
|
||||
|
||||
tap.test('RubyGems: should support latest specs endpoint (GET /rubygems/latest_specs.4.8.gz)', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/rubygems/latest_specs.4.8.gz',
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.headers['Content-Type']).toEqual('application/octet-stream');
|
||||
const body = await streamToBuffer(response.body);
|
||||
expect(body).toBeInstanceOf(Buffer);
|
||||
});
|
||||
|
||||
tap.test('RubyGems: should support specs endpoint (GET /rubygems/specs.4.8.gz)', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/rubygems/specs.4.8.gz',
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.headers['Content-Type']).toEqual('application/octet-stream');
|
||||
const body = await streamToBuffer(response.body);
|
||||
expect(body).toBeInstanceOf(Buffer);
|
||||
});
|
||||
|
||||
tap.test('RubyGems: should return 404 for non-existent gem', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/rubygems/gems/nonexistent-gem-1.0.0.gem',
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(404);
|
||||
const body = await streamToJson(response.body);
|
||||
expect(body).toHaveProperty('error');
|
||||
});
|
||||
|
||||
tap.test('RubyGems: should return 401 for unauthorized upload', async () => {
|
||||
const gemData = await createRubyGem('unauthorized-gem', '1.0.0');
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'POST',
|
||||
path: '/rubygems/api/v1/gems',
|
||||
headers: {
|
||||
// No authorization header
|
||||
'Content-Type': 'application/octet-stream',
|
||||
},
|
||||
query: {},
|
||||
body: gemData,
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(401);
|
||||
const body = await streamToJson(response.body);
|
||||
expect(body).toHaveProperty('error');
|
||||
});
|
||||
|
||||
tap.test('RubyGems: should return 401 for unauthorized yank', async () => {
|
||||
const response = await registry.handleRequest({
|
||||
method: 'DELETE',
|
||||
path: '/rubygems/api/v1/gems/yank',
|
||||
headers: {
|
||||
// No authorization header
|
||||
},
|
||||
query: {
|
||||
gem_name: testGemName,
|
||||
version: '2.0.0',
|
||||
},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(401);
|
||||
const body = await streamToJson(response.body);
|
||||
expect(body).toHaveProperty('error');
|
||||
});
|
||||
|
||||
tap.test('RubyGems: should handle gem with dependencies', async () => {
|
||||
const gemWithDeps = 'gem-with-deps';
|
||||
const version = '1.0.0';
|
||||
const gemData = await createRubyGem(gemWithDeps, version);
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'POST',
|
||||
path: '/rubygems/api/v1/gems',
|
||||
headers: {
|
||||
Authorization: rubygemsToken,
|
||||
'Content-Type': 'application/octet-stream',
|
||||
},
|
||||
query: {},
|
||||
body: gemData,
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(201);
|
||||
|
||||
// Check info file contains dependency info
|
||||
const infoResponse = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: `/rubygems/info/${gemWithDeps}`,
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(infoResponse.status).toEqual(200);
|
||||
|
||||
const infoBody = await streamToBuffer(infoResponse.body);
|
||||
const content = infoBody.toString('utf-8');
|
||||
expect(content).toContain('checksum:');
|
||||
});
|
||||
|
||||
tap.test('RubyGems: should validate gem filename format', async () => {
|
||||
const invalidGemData = Buffer.from('invalid gem data');
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'POST',
|
||||
path: '/rubygems/api/v1/gems',
|
||||
headers: {
|
||||
Authorization: rubygemsToken,
|
||||
'Content-Type': 'application/octet-stream',
|
||||
},
|
||||
query: {},
|
||||
body: invalidGemData,
|
||||
});
|
||||
|
||||
// Should fail validation
|
||||
expect(response.status).toBeGreaterThanOrEqual(400);
|
||||
});
|
||||
|
||||
tap.test('RubyGems: should support conditional GET with ETag', async () => {
|
||||
// First request to get ETag
|
||||
const response1 = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/rubygems/versions',
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
const etag = response1.headers['ETag'];
|
||||
expect(etag).toBeDefined();
|
||||
|
||||
// Second request with If-None-Match
|
||||
const response2 = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/rubygems/versions',
|
||||
headers: {
|
||||
'If-None-Match': etag as string,
|
||||
},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response2.status).toEqual(304);
|
||||
});
|
||||
|
||||
tap.postTask('cleanup registry', async () => {
|
||||
if (registry) {
|
||||
registry.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
export default tap.start();
|
||||
@@ -0,0 +1,647 @@
|
||||
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||
import * as qenv from '@push.rocks/qenv';
|
||||
import { RegistryStorage } from '../ts/core/classes.registrystorage.js';
|
||||
import type { IStorageConfig } from '../ts/core/interfaces.core.js';
|
||||
import type { IStorageHooks, IStorageHookContext } from '../ts/core/interfaces.storage.js';
|
||||
import {
|
||||
createQuotaHooks,
|
||||
createTestPackument,
|
||||
createTestRegistry,
|
||||
createTestTokens,
|
||||
createTrackingHooks,
|
||||
generateTestRunId,
|
||||
} from './helpers/registry.js';
|
||||
|
||||
const testQenv = new qenv.Qenv('./', './.nogit');
|
||||
|
||||
// ============================================================================
|
||||
// Test State
|
||||
// ============================================================================
|
||||
|
||||
let storage: RegistryStorage;
|
||||
let storageConfig: IStorageConfig;
|
||||
let testRunId: string;
|
||||
|
||||
// ============================================================================
|
||||
// Setup
|
||||
// ============================================================================
|
||||
|
||||
tap.test('setup: should create storage config', async () => {
|
||||
testRunId = generateTestRunId();
|
||||
|
||||
const s3AccessKey = await testQenv.getEnvVarOnDemand('S3_ACCESSKEY');
|
||||
const s3SecretKey = await testQenv.getEnvVarOnDemand('S3_SECRETKEY');
|
||||
const s3Endpoint = await testQenv.getEnvVarOnDemand('S3_ENDPOINT');
|
||||
const s3Port = await testQenv.getEnvVarOnDemand('S3_PORT');
|
||||
|
||||
storageConfig = {
|
||||
accessKey: s3AccessKey || 'minioadmin',
|
||||
accessSecret: s3SecretKey || 'minioadmin',
|
||||
endpoint: s3Endpoint || 'localhost',
|
||||
port: parseInt(s3Port || '9000', 10),
|
||||
useSsl: false,
|
||||
region: 'us-east-1',
|
||||
bucketName: `test-hooks-${testRunId}`,
|
||||
};
|
||||
|
||||
expect(storageConfig.bucketName).toBeTruthy();
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// beforePut Hook Tests
|
||||
// ============================================================================
|
||||
|
||||
tap.test('beforePut: should be called before storage', async () => {
|
||||
const tracker = createTrackingHooks();
|
||||
|
||||
storage = new RegistryStorage(storageConfig, tracker.hooks);
|
||||
await storage.init();
|
||||
|
||||
// Set context and put object
|
||||
storage.setContext({
|
||||
protocol: 'npm',
|
||||
actor: { userId: 'testuser' },
|
||||
metadata: { packageName: 'test-package' },
|
||||
});
|
||||
|
||||
await storage.putObject('test/beforeput-called.txt', Buffer.from('test data'));
|
||||
storage.clearContext();
|
||||
|
||||
// Verify beforePut was called
|
||||
const beforePutCalls = tracker.calls.filter(c => c.method === 'beforePut');
|
||||
expect(beforePutCalls.length).toEqual(1);
|
||||
expect(beforePutCalls[0].context.operation).toEqual('put');
|
||||
expect(beforePutCalls[0].context.key).toEqual('test/beforeput-called.txt');
|
||||
expect(beforePutCalls[0].context.protocol).toEqual('npm');
|
||||
});
|
||||
|
||||
tap.test('beforePut: returning {allowed: false} should block storage', async () => {
|
||||
const tracker = createTrackingHooks({ beforePutAllowed: false });
|
||||
|
||||
const blockingStorage = new RegistryStorage(storageConfig, tracker.hooks);
|
||||
await blockingStorage.init();
|
||||
|
||||
blockingStorage.setContext({
|
||||
protocol: 'npm',
|
||||
actor: { userId: 'testuser' },
|
||||
});
|
||||
|
||||
let errorThrown = false;
|
||||
try {
|
||||
await blockingStorage.putObject('test/should-not-exist.txt', Buffer.from('blocked data'));
|
||||
} catch (error) {
|
||||
errorThrown = true;
|
||||
expect((error as Error).message).toContain('Blocked by test');
|
||||
}
|
||||
|
||||
blockingStorage.clearContext();
|
||||
|
||||
expect(errorThrown).toBeTrue();
|
||||
|
||||
// Verify object was NOT stored
|
||||
const result = await blockingStorage.getObject('test/should-not-exist.txt');
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// afterPut Hook Tests
|
||||
// ============================================================================
|
||||
|
||||
tap.test('afterPut: should be called after successful storage', async () => {
|
||||
const tracker = createTrackingHooks();
|
||||
|
||||
const trackedStorage = new RegistryStorage(storageConfig, tracker.hooks);
|
||||
await trackedStorage.init();
|
||||
|
||||
trackedStorage.setContext({
|
||||
protocol: 'maven',
|
||||
actor: { userId: 'maven-user' },
|
||||
});
|
||||
|
||||
await trackedStorage.putObject('test/afterput-test.txt', Buffer.from('after put test'));
|
||||
trackedStorage.clearContext();
|
||||
|
||||
// Give async hook time to complete
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
const afterPutCalls = tracker.calls.filter(c => c.method === 'afterPut');
|
||||
expect(afterPutCalls.length).toEqual(1);
|
||||
expect(afterPutCalls[0].context.operation).toEqual('put');
|
||||
});
|
||||
|
||||
tap.test('afterPut: should receive correct metadata (size, key, protocol)', async () => {
|
||||
const tracker = createTrackingHooks();
|
||||
|
||||
const metadataStorage = new RegistryStorage(storageConfig, tracker.hooks);
|
||||
await metadataStorage.init();
|
||||
|
||||
const testData = Buffer.from('metadata test data - some content here');
|
||||
|
||||
metadataStorage.setContext({
|
||||
protocol: 'cargo',
|
||||
actor: { userId: 'cargo-user', ip: '192.168.1.100' },
|
||||
metadata: { packageName: 'my-crate', version: '1.0.0' },
|
||||
});
|
||||
|
||||
await metadataStorage.putObject('test/metadata-test.txt', testData);
|
||||
metadataStorage.clearContext();
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
const afterPutCalls = tracker.calls.filter(c => c.method === 'afterPut');
|
||||
expect(afterPutCalls.length).toBeGreaterThanOrEqual(1);
|
||||
|
||||
const call = afterPutCalls[afterPutCalls.length - 1];
|
||||
expect(call.context.metadata?.size).toEqual(testData.length);
|
||||
expect(call.context.key).toEqual('test/metadata-test.txt');
|
||||
expect(call.context.protocol).toEqual('cargo');
|
||||
expect(call.context.actor?.userId).toEqual('cargo-user');
|
||||
expect(call.context.actor?.ip).toEqual('192.168.1.100');
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// beforeDelete Hook Tests
|
||||
// ============================================================================
|
||||
|
||||
tap.test('beforeDelete: should be called before deletion', async () => {
|
||||
const tracker = createTrackingHooks();
|
||||
|
||||
const deleteStorage = new RegistryStorage(storageConfig, tracker.hooks);
|
||||
await deleteStorage.init();
|
||||
|
||||
// First, store an object
|
||||
deleteStorage.setContext({ protocol: 'npm' });
|
||||
await deleteStorage.putObject('test/to-delete.txt', Buffer.from('delete me'));
|
||||
|
||||
// Now delete it
|
||||
await deleteStorage.deleteObject('test/to-delete.txt');
|
||||
deleteStorage.clearContext();
|
||||
|
||||
const beforeDeleteCalls = tracker.calls.filter(c => c.method === 'beforeDelete');
|
||||
expect(beforeDeleteCalls.length).toEqual(1);
|
||||
expect(beforeDeleteCalls[0].context.operation).toEqual('delete');
|
||||
expect(beforeDeleteCalls[0].context.key).toEqual('test/to-delete.txt');
|
||||
});
|
||||
|
||||
tap.test('beforeDelete: returning {allowed: false} should block deletion', async () => {
|
||||
const tracker = createTrackingHooks({ beforeDeleteAllowed: false });
|
||||
|
||||
const protectedStorage = new RegistryStorage(storageConfig, tracker.hooks);
|
||||
await protectedStorage.init();
|
||||
|
||||
// First store an object
|
||||
protectedStorage.setContext({ protocol: 'npm' });
|
||||
await protectedStorage.putObject('test/protected.txt', Buffer.from('protected data'));
|
||||
|
||||
// Try to delete - should be blocked
|
||||
let errorThrown = false;
|
||||
try {
|
||||
await protectedStorage.deleteObject('test/protected.txt');
|
||||
} catch (error) {
|
||||
errorThrown = true;
|
||||
expect((error as Error).message).toContain('Blocked by test');
|
||||
}
|
||||
|
||||
protectedStorage.clearContext();
|
||||
|
||||
expect(errorThrown).toBeTrue();
|
||||
|
||||
// Verify object still exists
|
||||
const result = await protectedStorage.getObject('test/protected.txt');
|
||||
expect(result).toBeTruthy();
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// afterDelete Hook Tests
|
||||
// ============================================================================
|
||||
|
||||
tap.test('afterDelete: should be called after successful deletion', async () => {
|
||||
const tracker = createTrackingHooks();
|
||||
|
||||
const afterDeleteStorage = new RegistryStorage(storageConfig, tracker.hooks);
|
||||
await afterDeleteStorage.init();
|
||||
|
||||
afterDeleteStorage.setContext({ protocol: 'pypi' });
|
||||
await afterDeleteStorage.putObject('test/delete-tracked.txt', Buffer.from('to be deleted'));
|
||||
await afterDeleteStorage.deleteObject('test/delete-tracked.txt');
|
||||
afterDeleteStorage.clearContext();
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
const afterDeleteCalls = tracker.calls.filter(c => c.method === 'afterDelete');
|
||||
expect(afterDeleteCalls.length).toEqual(1);
|
||||
expect(afterDeleteCalls[0].context.operation).toEqual('delete');
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// afterGet Hook Tests
|
||||
// ============================================================================
|
||||
|
||||
tap.test('afterGet: should be called after reading object', async () => {
|
||||
const tracker = createTrackingHooks();
|
||||
|
||||
const getStorage = new RegistryStorage(storageConfig, tracker.hooks);
|
||||
await getStorage.init();
|
||||
|
||||
// Store an object first
|
||||
getStorage.setContext({ protocol: 'rubygems' });
|
||||
await getStorage.putObject('test/read-test.txt', Buffer.from('read me'));
|
||||
|
||||
// Clear calls to focus on the get
|
||||
tracker.calls.length = 0;
|
||||
|
||||
// Read the object
|
||||
const data = await getStorage.getObject('test/read-test.txt');
|
||||
getStorage.clearContext();
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
expect(data).toBeTruthy();
|
||||
expect(data!.toString()).toEqual('read me');
|
||||
|
||||
const afterGetCalls = tracker.calls.filter(c => c.method === 'afterGet');
|
||||
expect(afterGetCalls.length).toEqual(1);
|
||||
expect(afterGetCalls[0].context.operation).toEqual('get');
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Context Tests
|
||||
// ============================================================================
|
||||
|
||||
tap.test('context: hooks should receive actor information', async () => {
|
||||
const tracker = createTrackingHooks();
|
||||
|
||||
const actorStorage = new RegistryStorage(storageConfig, tracker.hooks);
|
||||
await actorStorage.init();
|
||||
|
||||
actorStorage.setContext({
|
||||
protocol: 'composer',
|
||||
actor: {
|
||||
userId: 'user-123',
|
||||
tokenId: 'token-abc',
|
||||
ip: '10.0.0.1',
|
||||
userAgent: 'composer/2.0',
|
||||
orgId: 'org-456',
|
||||
sessionId: 'session-xyz',
|
||||
},
|
||||
});
|
||||
|
||||
await actorStorage.putObject('test/actor-test.txt', Buffer.from('actor test'));
|
||||
actorStorage.clearContext();
|
||||
|
||||
const beforePutCall = tracker.calls.find(c => c.method === 'beforePut');
|
||||
expect(beforePutCall).toBeTruthy();
|
||||
expect(beforePutCall!.context.actor?.userId).toEqual('user-123');
|
||||
expect(beforePutCall!.context.actor?.tokenId).toEqual('token-abc');
|
||||
expect(beforePutCall!.context.actor?.ip).toEqual('10.0.0.1');
|
||||
expect(beforePutCall!.context.actor?.userAgent).toEqual('composer/2.0');
|
||||
expect(beforePutCall!.context.actor?.orgId).toEqual('org-456');
|
||||
expect(beforePutCall!.context.actor?.sessionId).toEqual('session-xyz');
|
||||
});
|
||||
|
||||
tap.test('withContext: should set and clear context correctly', async () => {
|
||||
const tracker = createTrackingHooks();
|
||||
|
||||
const contextStorage = new RegistryStorage(storageConfig, tracker.hooks);
|
||||
await contextStorage.init();
|
||||
|
||||
// Use withContext to ensure automatic cleanup
|
||||
await contextStorage.withContext(
|
||||
{
|
||||
protocol: 'oci',
|
||||
actor: { userId: 'oci-user' },
|
||||
},
|
||||
async () => {
|
||||
await contextStorage.putObject('test/with-context.txt', Buffer.from('context managed'));
|
||||
}
|
||||
);
|
||||
|
||||
const call = tracker.calls.find(c => c.method === 'beforePut');
|
||||
expect(call).toBeTruthy();
|
||||
expect(call!.context.protocol).toEqual('oci');
|
||||
expect(call!.context.actor?.userId).toEqual('oci-user');
|
||||
});
|
||||
|
||||
tap.test('withContext: should clear context even on error', async () => {
|
||||
const tracker = createTrackingHooks({ beforePutAllowed: false });
|
||||
|
||||
const errorStorage = new RegistryStorage(storageConfig, tracker.hooks);
|
||||
await errorStorage.init();
|
||||
|
||||
let errorThrown = false;
|
||||
try {
|
||||
await errorStorage.withContext(
|
||||
{
|
||||
protocol: 'npm',
|
||||
actor: { userId: 'error-user' },
|
||||
},
|
||||
async () => {
|
||||
await errorStorage.putObject('test/error-context.txt', Buffer.from('will fail'));
|
||||
}
|
||||
);
|
||||
} catch {
|
||||
errorThrown = true;
|
||||
}
|
||||
|
||||
expect(errorThrown).toBeTrue();
|
||||
|
||||
// Verify context was cleared - next operation without context should work
|
||||
// (hooks won't be called without context)
|
||||
tracker.hooks.beforePut = async () => ({ allowed: true });
|
||||
await errorStorage.putObject('test/after-error.txt', Buffer.from('ok'));
|
||||
});
|
||||
|
||||
tap.test('withContext: should isolate concurrent async operations', async () => {
|
||||
const tracker = createTrackingHooks();
|
||||
|
||||
const concurrentStorage = new RegistryStorage(storageConfig, tracker.hooks);
|
||||
await concurrentStorage.init();
|
||||
|
||||
const bucket = (concurrentStorage as any).bucket;
|
||||
const originalFastPut = bucket.fastPut.bind(bucket);
|
||||
const pendingWrites: Array<() => void> = [];
|
||||
let startedWrites = 0;
|
||||
let waitingWrites = 0;
|
||||
let startedResolve: () => void;
|
||||
let waitingResolve: () => void;
|
||||
const bothWritesStarted = new Promise<void>((resolve) => {
|
||||
startedResolve = resolve;
|
||||
});
|
||||
const bothWritesWaiting = new Promise<void>((resolve) => {
|
||||
waitingResolve = resolve;
|
||||
});
|
||||
|
||||
bucket.fastPut = async (options: any) => {
|
||||
startedWrites += 1;
|
||||
if (startedWrites === 2) {
|
||||
startedResolve();
|
||||
}
|
||||
|
||||
await bothWritesStarted;
|
||||
|
||||
await new Promise<void>((resolve) => {
|
||||
pendingWrites.push(resolve);
|
||||
waitingWrites += 1;
|
||||
if (waitingWrites === 2) {
|
||||
waitingResolve();
|
||||
}
|
||||
});
|
||||
|
||||
return originalFastPut(options);
|
||||
};
|
||||
|
||||
try {
|
||||
const opA = concurrentStorage.withContext(
|
||||
{
|
||||
protocol: 'npm',
|
||||
actor: { userId: 'user-a' },
|
||||
metadata: { packageName: 'package-a' },
|
||||
},
|
||||
async () => {
|
||||
await concurrentStorage.putObject('test/concurrent-a.txt', Buffer.from('a'));
|
||||
}
|
||||
);
|
||||
|
||||
const opB = concurrentStorage.withContext(
|
||||
{
|
||||
protocol: 'npm',
|
||||
actor: { userId: 'user-b' },
|
||||
metadata: { packageName: 'package-b' },
|
||||
},
|
||||
async () => {
|
||||
await concurrentStorage.putObject('test/concurrent-b.txt', Buffer.from('b'));
|
||||
}
|
||||
);
|
||||
|
||||
await bothWritesWaiting;
|
||||
|
||||
pendingWrites[0]!();
|
||||
pendingWrites[1]!();
|
||||
|
||||
await Promise.all([opA, opB]);
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
} finally {
|
||||
bucket.fastPut = originalFastPut;
|
||||
}
|
||||
|
||||
const afterPutCalls = tracker.calls.filter(
|
||||
(call) => call.method === 'afterPut' && call.context.key.startsWith('test/concurrent-')
|
||||
);
|
||||
expect(afterPutCalls.length).toEqual(2);
|
||||
|
||||
const callByKey = new Map(afterPutCalls.map((call) => [call.context.key, call]));
|
||||
expect(callByKey.get('test/concurrent-a.txt')?.context.actor?.userId).toEqual('user-a');
|
||||
expect(callByKey.get('test/concurrent-a.txt')?.context.metadata?.packageName).toEqual('package-a');
|
||||
expect(callByKey.get('test/concurrent-b.txt')?.context.actor?.userId).toEqual('user-b');
|
||||
expect(callByKey.get('test/concurrent-b.txt')?.context.metadata?.packageName).toEqual('package-b');
|
||||
});
|
||||
|
||||
tap.test('request hooks: should receive context during real npm publish requests', async () => {
|
||||
const tracker = createTrackingHooks();
|
||||
const registry = await createTestRegistry({ storageHooks: tracker.hooks });
|
||||
|
||||
try {
|
||||
const tokens = await createTestTokens(registry);
|
||||
const packageName = `hooked-package-${generateTestRunId()}`;
|
||||
const version = '1.0.0';
|
||||
const tarball = Buffer.from('hooked tarball data', 'utf-8');
|
||||
const packument = createTestPackument(packageName, version, tarball);
|
||||
|
||||
const response = await registry.handleRequest({
|
||||
method: 'PUT',
|
||||
path: `/npm/${packageName}`,
|
||||
headers: {
|
||||
Authorization: `Bearer ${tokens.npmToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
query: {},
|
||||
body: packument,
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(201);
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
const npmWrites = tracker.calls.filter(
|
||||
(call) => call.method === 'beforePut' && call.context.metadata?.packageName === packageName
|
||||
);
|
||||
expect(npmWrites.length).toBeGreaterThanOrEqual(2);
|
||||
|
||||
const packumentWrite = npmWrites.find(
|
||||
(call) => call.context.key === `npm/packages/${packageName}/index.json`
|
||||
);
|
||||
expect(packumentWrite).toBeTruthy();
|
||||
expect(packumentWrite!.context.protocol).toEqual('npm');
|
||||
expect(packumentWrite!.context.actor?.userId).toEqual(tokens.userId);
|
||||
expect(packumentWrite!.context.metadata?.packageName).toEqual(packageName);
|
||||
|
||||
const tarballWrite = npmWrites.find(
|
||||
(call) => call.context.key.endsWith(`-${version}.tgz`)
|
||||
);
|
||||
expect(tarballWrite).toBeTruthy();
|
||||
expect(tarballWrite!.context.metadata?.packageName).toEqual(packageName);
|
||||
expect(tarballWrite!.context.metadata?.version).toEqual(version);
|
||||
} finally {
|
||||
registry.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Graceful Degradation Tests
|
||||
// ============================================================================
|
||||
|
||||
tap.test('graceful: hooks should not fail the operation if afterPut throws', async () => {
|
||||
const tracker = createTrackingHooks({ throwOnAfterPut: true });
|
||||
|
||||
const gracefulStorage = new RegistryStorage(storageConfig, tracker.hooks);
|
||||
await gracefulStorage.init();
|
||||
|
||||
gracefulStorage.setContext({ protocol: 'npm' });
|
||||
|
||||
// This should NOT throw even though afterPut throws
|
||||
let errorThrown = false;
|
||||
try {
|
||||
await gracefulStorage.putObject('test/graceful-afterput.txt', Buffer.from('should succeed'));
|
||||
} catch {
|
||||
errorThrown = true;
|
||||
}
|
||||
|
||||
gracefulStorage.clearContext();
|
||||
|
||||
expect(errorThrown).toBeFalse();
|
||||
|
||||
// Verify object was stored
|
||||
const data = await gracefulStorage.getObject('test/graceful-afterput.txt');
|
||||
expect(data).toBeTruthy();
|
||||
});
|
||||
|
||||
tap.test('graceful: hooks should not fail the operation if afterGet throws', async () => {
|
||||
const tracker = createTrackingHooks({ throwOnAfterGet: true });
|
||||
|
||||
const gracefulGetStorage = new RegistryStorage(storageConfig, tracker.hooks);
|
||||
await gracefulGetStorage.init();
|
||||
|
||||
// Store first
|
||||
gracefulGetStorage.setContext({ protocol: 'maven' });
|
||||
await gracefulGetStorage.putObject('test/graceful-afterget.txt', Buffer.from('read me gracefully'));
|
||||
|
||||
// Read should succeed even though afterGet throws
|
||||
let errorThrown = false;
|
||||
try {
|
||||
const data = await gracefulGetStorage.getObject('test/graceful-afterget.txt');
|
||||
expect(data).toBeTruthy();
|
||||
} catch {
|
||||
errorThrown = true;
|
||||
}
|
||||
|
||||
gracefulGetStorage.clearContext();
|
||||
|
||||
expect(errorThrown).toBeFalse();
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Quota Hooks Tests
|
||||
// ============================================================================
|
||||
|
||||
tap.test('quota: should block storage when quota exceeded', async () => {
|
||||
const maxSize = 100; // 100 bytes max
|
||||
const quotaTracker = createQuotaHooks(maxSize);
|
||||
|
||||
const quotaStorage = new RegistryStorage(storageConfig, quotaTracker.hooks);
|
||||
await quotaStorage.init();
|
||||
|
||||
quotaStorage.setContext({
|
||||
protocol: 'npm',
|
||||
actor: { userId: 'quota-user' },
|
||||
});
|
||||
|
||||
// Store 50 bytes - should succeed
|
||||
await quotaStorage.putObject('test/quota-1.txt', Buffer.from('x'.repeat(50)));
|
||||
expect(quotaTracker.currentUsage.bytes).toEqual(50);
|
||||
|
||||
// Try to store 60 more bytes - should fail (total would be 110)
|
||||
let errorThrown = false;
|
||||
try {
|
||||
await quotaStorage.putObject('test/quota-2.txt', Buffer.from('x'.repeat(60)));
|
||||
} catch (error) {
|
||||
errorThrown = true;
|
||||
expect((error as Error).message).toContain('Quota exceeded');
|
||||
}
|
||||
|
||||
quotaStorage.clearContext();
|
||||
|
||||
expect(errorThrown).toBeTrue();
|
||||
expect(quotaTracker.currentUsage.bytes).toEqual(50); // Still 50, not 110
|
||||
});
|
||||
|
||||
tap.test('quota: should update usage after delete', async () => {
|
||||
const maxSize = 200;
|
||||
const quotaTracker = createQuotaHooks(maxSize);
|
||||
|
||||
const quotaDelStorage = new RegistryStorage(storageConfig, quotaTracker.hooks);
|
||||
await quotaDelStorage.init();
|
||||
|
||||
quotaDelStorage.setContext({
|
||||
protocol: 'npm',
|
||||
metadata: { size: 75 },
|
||||
});
|
||||
|
||||
// Store and track
|
||||
await quotaDelStorage.putObject('test/quota-del.txt', Buffer.from('x'.repeat(75)));
|
||||
expect(quotaTracker.currentUsage.bytes).toEqual(75);
|
||||
|
||||
// Delete and verify usage decreases
|
||||
await quotaDelStorage.deleteObject('test/quota-del.txt');
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
quotaDelStorage.clearContext();
|
||||
|
||||
// Usage should be reduced (though exact value depends on metadata)
|
||||
expect(quotaTracker.currentUsage.bytes).toBeLessThanOrEqual(75);
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// setHooks Tests
|
||||
// ============================================================================
|
||||
|
||||
tap.test('setHooks: should allow setting hooks after construction', async () => {
|
||||
const lateStorage = new RegistryStorage(storageConfig);
|
||||
await lateStorage.init();
|
||||
|
||||
// Initially no hooks
|
||||
await lateStorage.putObject('test/no-hooks.txt', Buffer.from('no hooks yet'));
|
||||
|
||||
// Add hooks later
|
||||
const tracker = createTrackingHooks();
|
||||
lateStorage.setHooks(tracker.hooks);
|
||||
|
||||
lateStorage.setContext({ protocol: 'npm' });
|
||||
await lateStorage.putObject('test/with-late-hooks.txt', Buffer.from('now with hooks'));
|
||||
lateStorage.clearContext();
|
||||
|
||||
const beforePutCalls = tracker.calls.filter(c => c.method === 'beforePut');
|
||||
expect(beforePutCalls.length).toEqual(1);
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Cleanup
|
||||
// ============================================================================
|
||||
|
||||
tap.test('cleanup: should clean up test bucket', async () => {
|
||||
if (storage) {
|
||||
// Clean up test objects
|
||||
const prefixes = ['test/'];
|
||||
for (const prefix of prefixes) {
|
||||
try {
|
||||
const objects = await storage.listObjects(prefix);
|
||||
for (const obj of objects) {
|
||||
await storage.deleteObject(obj);
|
||||
}
|
||||
} catch {
|
||||
// Ignore cleanup errors
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
export default tap.start();
|
||||
+4
-2
@@ -1,5 +1,6 @@
|
||||
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||
import { SmartRegistry } from '../ts/index.js';
|
||||
import { streamToJson } from '../ts/core/helpers.stream.js';
|
||||
import { createTestRegistry, createTestTokens } from './helpers/registry.js';
|
||||
|
||||
let registry: SmartRegistry;
|
||||
@@ -54,8 +55,9 @@ tap.test('Integration: should return 404 for unknown paths', async () => {
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(404);
|
||||
expect(response.body).toHaveProperty('error');
|
||||
expect((response.body as any).error).toEqual('NOT_FOUND');
|
||||
const body = await streamToJson(response.body);
|
||||
expect(body).toHaveProperty('error');
|
||||
expect(body.error).toEqual('NOT_FOUND');
|
||||
});
|
||||
|
||||
tap.test('Integration: should create and validate tokens', async () => {
|
||||
|
||||
@@ -0,0 +1,598 @@
|
||||
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||
import * as qenv from '@push.rocks/qenv';
|
||||
import * as smartbucket from '@push.rocks/smartbucket';
|
||||
import { UpstreamCache } from '../ts/upstream/classes.upstreamcache.js';
|
||||
import type { IUpstreamFetchContext, IUpstreamCacheConfig } from '../ts/upstream/interfaces.upstream.js';
|
||||
import type { IStorageBackend } from '../ts/core/interfaces.core.js';
|
||||
import { generateTestRunId } from './helpers/registry.js';
|
||||
|
||||
const testQenv = new qenv.Qenv('./', './.nogit');
|
||||
|
||||
// ============================================================================
|
||||
// Test State
|
||||
// ============================================================================
|
||||
|
||||
let cache: UpstreamCache;
|
||||
let storageBackend: IStorageBackend;
|
||||
let s3Bucket: smartbucket.Bucket;
|
||||
let smartBucket: smartbucket.SmartBucket;
|
||||
let testRunId: string;
|
||||
let bucketName: string;
|
||||
|
||||
// ============================================================================
|
||||
// Helper Functions
|
||||
// ============================================================================
|
||||
|
||||
function createFetchContext(overrides?: Partial<IUpstreamFetchContext>): IUpstreamFetchContext {
|
||||
// Use resource name as path to ensure unique cache keys
|
||||
const resource = overrides?.resource || 'lodash';
|
||||
return {
|
||||
protocol: 'npm',
|
||||
resource,
|
||||
resourceType: 'packument',
|
||||
path: `/${resource}`,
|
||||
method: 'GET',
|
||||
headers: {},
|
||||
query: {},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Setup
|
||||
// ============================================================================
|
||||
|
||||
tap.test('setup: should create S3 storage backend', async () => {
|
||||
testRunId = generateTestRunId();
|
||||
bucketName = `test-ucache-${testRunId.substring(0, 8)}`;
|
||||
|
||||
const s3AccessKey = await testQenv.getEnvVarOnDemand('S3_ACCESSKEY');
|
||||
const s3SecretKey = await testQenv.getEnvVarOnDemand('S3_SECRETKEY');
|
||||
const s3Endpoint = await testQenv.getEnvVarOnDemand('S3_ENDPOINT');
|
||||
const s3Port = await testQenv.getEnvVarOnDemand('S3_PORT');
|
||||
|
||||
smartBucket = new smartbucket.SmartBucket({
|
||||
accessKey: s3AccessKey || 'minioadmin',
|
||||
accessSecret: s3SecretKey || 'minioadmin',
|
||||
endpoint: s3Endpoint || 'localhost',
|
||||
port: parseInt(s3Port || '9000', 10),
|
||||
useSsl: false,
|
||||
});
|
||||
|
||||
s3Bucket = await smartBucket.createBucket(bucketName);
|
||||
|
||||
// Create storage backend adapter
|
||||
storageBackend = {
|
||||
getObject: async (key: string): Promise<Buffer | null> => {
|
||||
try {
|
||||
// fastGet returns Buffer directly (or undefined if not found)
|
||||
const data = await s3Bucket.fastGet({ path: key });
|
||||
if (!data) {
|
||||
return null;
|
||||
}
|
||||
return data;
|
||||
} catch (error) {
|
||||
// fastGet throws if object doesn't exist
|
||||
return null;
|
||||
}
|
||||
},
|
||||
putObject: async (key: string, data: Buffer): Promise<void> => {
|
||||
await s3Bucket.fastPut({ path: key, contents: data, overwrite: true });
|
||||
},
|
||||
deleteObject: async (key: string): Promise<void> => {
|
||||
await s3Bucket.fastRemove({ path: key });
|
||||
},
|
||||
listObjects: async (prefix: string): Promise<string[]> => {
|
||||
const paths: string[] = [];
|
||||
for await (const path of s3Bucket.listAllObjects(prefix)) {
|
||||
paths.push(path);
|
||||
}
|
||||
return paths;
|
||||
},
|
||||
};
|
||||
|
||||
expect(storageBackend).toBeTruthy();
|
||||
});
|
||||
|
||||
tap.test('setup: should create UpstreamCache with S3 storage', async () => {
|
||||
cache = new UpstreamCache(
|
||||
{ enabled: true, defaultTtlSeconds: 300 },
|
||||
10000,
|
||||
storageBackend
|
||||
);
|
||||
|
||||
expect(cache.isEnabled()).toBeTrue();
|
||||
expect(cache.hasStorage()).toBeTrue();
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Basic Cache Operations
|
||||
// ============================================================================
|
||||
|
||||
tap.test('cache: should store cache entry in S3', async () => {
|
||||
const context = createFetchContext({ resource: 'store-test' });
|
||||
const testData = Buffer.from(JSON.stringify({ name: 'store-test', version: '1.0.0' }));
|
||||
const upstreamUrl = 'https://registry.npmjs.org';
|
||||
|
||||
await cache.set(
|
||||
context,
|
||||
testData,
|
||||
'application/json',
|
||||
{ 'etag': '"abc123"' },
|
||||
'npmjs',
|
||||
upstreamUrl
|
||||
);
|
||||
|
||||
// Verify in S3
|
||||
const stats = cache.getStats();
|
||||
expect(stats.totalEntries).toBeGreaterThanOrEqual(1);
|
||||
});
|
||||
|
||||
tap.test('cache: should retrieve cache entry from S3', async () => {
|
||||
const context = createFetchContext({ resource: 'retrieve-test' });
|
||||
const testData = Buffer.from('retrieve test data');
|
||||
const upstreamUrl = 'https://registry.npmjs.org';
|
||||
|
||||
await cache.set(
|
||||
context,
|
||||
testData,
|
||||
'application/octet-stream',
|
||||
{},
|
||||
'npmjs',
|
||||
upstreamUrl
|
||||
);
|
||||
|
||||
const entry = await cache.get(context, upstreamUrl);
|
||||
|
||||
expect(entry).toBeTruthy();
|
||||
expect(entry!.data.toString()).toEqual('retrieve test data');
|
||||
expect(entry!.contentType).toEqual('application/octet-stream');
|
||||
expect(entry!.upstreamId).toEqual('npmjs');
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Multi-Upstream URL Tests
|
||||
// ============================================================================
|
||||
|
||||
tap.test('cache: should include upstream URL in cache path', async () => {
|
||||
const context = createFetchContext({ resource: 'url-path-test' });
|
||||
const testData = Buffer.from('url path test');
|
||||
|
||||
await cache.set(
|
||||
context,
|
||||
testData,
|
||||
'text/plain',
|
||||
{},
|
||||
'npmjs',
|
||||
'https://registry.npmjs.org'
|
||||
);
|
||||
|
||||
// The cache key should include the escaped URL
|
||||
const entry = await cache.get(context, 'https://registry.npmjs.org');
|
||||
expect(entry).toBeTruthy();
|
||||
expect(entry!.data.toString()).toEqual('url path test');
|
||||
});
|
||||
|
||||
tap.test('cache: should handle multiple upstreams with different URLs', async () => {
|
||||
const context = createFetchContext({ resource: '@company/private-pkg' });
|
||||
|
||||
// Store from private upstream
|
||||
const privateData = Buffer.from('private package data');
|
||||
await cache.set(
|
||||
context,
|
||||
privateData,
|
||||
'application/json',
|
||||
{},
|
||||
'private-npm',
|
||||
'https://npm.company.com'
|
||||
);
|
||||
|
||||
// Store from public upstream (same resource name, different upstream)
|
||||
const publicData = Buffer.from('public package data');
|
||||
await cache.set(
|
||||
context,
|
||||
publicData,
|
||||
'application/json',
|
||||
{},
|
||||
'public-npm',
|
||||
'https://registry.npmjs.org'
|
||||
);
|
||||
|
||||
// Retrieve from private - should get private data
|
||||
const privateEntry = await cache.get(context, 'https://npm.company.com');
|
||||
expect(privateEntry).toBeTruthy();
|
||||
expect(privateEntry!.data.toString()).toEqual('private package data');
|
||||
expect(privateEntry!.upstreamId).toEqual('private-npm');
|
||||
|
||||
// Retrieve from public - should get public data
|
||||
const publicEntry = await cache.get(context, 'https://registry.npmjs.org');
|
||||
expect(publicEntry).toBeTruthy();
|
||||
expect(publicEntry!.data.toString()).toEqual('public package data');
|
||||
expect(publicEntry!.upstreamId).toEqual('public-npm');
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// TTL and Expiration Tests
|
||||
// ============================================================================
|
||||
|
||||
tap.test('cache: should respect TTL expiration', async () => {
|
||||
// Create cache with very short TTL
|
||||
const shortTtlCache = new UpstreamCache(
|
||||
{
|
||||
enabled: true,
|
||||
defaultTtlSeconds: 1, // 1 second TTL
|
||||
staleWhileRevalidate: false,
|
||||
staleMaxAgeSeconds: 0,
|
||||
immutableTtlSeconds: 1,
|
||||
negativeCacheTtlSeconds: 1,
|
||||
},
|
||||
1000,
|
||||
storageBackend
|
||||
);
|
||||
|
||||
const context = createFetchContext({ resource: 'ttl-test' });
|
||||
const testData = Buffer.from('expires soon');
|
||||
|
||||
await shortTtlCache.set(
|
||||
context,
|
||||
testData,
|
||||
'text/plain',
|
||||
{},
|
||||
'test-upstream',
|
||||
'https://test.example.com'
|
||||
);
|
||||
|
||||
// Should exist immediately
|
||||
let entry = await shortTtlCache.get(context, 'https://test.example.com');
|
||||
expect(entry).toBeTruthy();
|
||||
|
||||
// Wait for expiration
|
||||
await new Promise(resolve => setTimeout(resolve, 1500));
|
||||
|
||||
// Should be expired now
|
||||
entry = await shortTtlCache.get(context, 'https://test.example.com');
|
||||
expect(entry).toBeNull();
|
||||
|
||||
shortTtlCache.stop();
|
||||
});
|
||||
|
||||
tap.test('cache: should serve stale content during stale-while-revalidate window', async () => {
|
||||
const staleCache = new UpstreamCache(
|
||||
{
|
||||
enabled: true,
|
||||
defaultTtlSeconds: 1, // 1 second fresh
|
||||
staleWhileRevalidate: true,
|
||||
staleMaxAgeSeconds: 60, // 60 seconds stale window
|
||||
immutableTtlSeconds: 1,
|
||||
negativeCacheTtlSeconds: 1,
|
||||
},
|
||||
1000,
|
||||
storageBackend
|
||||
);
|
||||
|
||||
const context = createFetchContext({ resource: 'stale-test' });
|
||||
const testData = Buffer.from('stale but usable');
|
||||
|
||||
await staleCache.set(
|
||||
context,
|
||||
testData,
|
||||
'text/plain',
|
||||
{},
|
||||
'stale-upstream',
|
||||
'https://stale.example.com'
|
||||
);
|
||||
|
||||
// Wait for fresh period to expire
|
||||
await new Promise(resolve => setTimeout(resolve, 1500));
|
||||
|
||||
// Should still be available but marked as stale
|
||||
const entry = await staleCache.get(context, 'https://stale.example.com');
|
||||
expect(entry).toBeTruthy();
|
||||
expect(entry!.stale).toBeTrue();
|
||||
expect(entry!.data.toString()).toEqual('stale but usable');
|
||||
|
||||
staleCache.stop();
|
||||
});
|
||||
|
||||
tap.test('cache: should reject content past stale deadline', async () => {
|
||||
const veryShortCache = new UpstreamCache(
|
||||
{
|
||||
enabled: true,
|
||||
defaultTtlSeconds: 1,
|
||||
staleWhileRevalidate: true,
|
||||
staleMaxAgeSeconds: 1, // Only 1 second stale window
|
||||
immutableTtlSeconds: 1,
|
||||
negativeCacheTtlSeconds: 1,
|
||||
},
|
||||
1000,
|
||||
storageBackend
|
||||
);
|
||||
|
||||
const context = createFetchContext({ resource: 'very-stale-test' });
|
||||
await veryShortCache.set(
|
||||
context,
|
||||
Buffer.from('will expire completely'),
|
||||
'text/plain',
|
||||
{},
|
||||
'short-upstream',
|
||||
'https://short.example.com'
|
||||
);
|
||||
|
||||
// Wait for both fresh AND stale periods to expire
|
||||
await new Promise(resolve => setTimeout(resolve, 2500));
|
||||
|
||||
const entry = await veryShortCache.get(context, 'https://short.example.com');
|
||||
expect(entry).toBeNull();
|
||||
|
||||
veryShortCache.stop();
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Negative Cache Tests
|
||||
// ============================================================================
|
||||
|
||||
tap.test('cache: should store negative cache entries (404)', async () => {
|
||||
const context = createFetchContext({ resource: 'not-found-pkg' });
|
||||
const upstreamUrl = 'https://registry.npmjs.org';
|
||||
|
||||
await cache.setNegative(context, 'npmjs', upstreamUrl);
|
||||
|
||||
const hasNegative = await cache.hasNegative(context, upstreamUrl);
|
||||
expect(hasNegative).toBeTrue();
|
||||
});
|
||||
|
||||
tap.test('cache: should retrieve negative cache entries', async () => {
|
||||
const context = createFetchContext({ resource: 'negative-retrieve-test' });
|
||||
const upstreamUrl = 'https://registry.npmjs.org';
|
||||
|
||||
await cache.setNegative(context, 'npmjs', upstreamUrl);
|
||||
|
||||
const entry = await cache.get(context, upstreamUrl);
|
||||
expect(entry).toBeTruthy();
|
||||
expect(entry!.data.length).toEqual(0); // Empty buffer indicates 404
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Eviction Tests
|
||||
// ============================================================================
|
||||
|
||||
tap.test('cache: should evict oldest entries when memory limit reached', async () => {
|
||||
// Create cache with very small limit
|
||||
const smallCache = new UpstreamCache(
|
||||
{ enabled: true, defaultTtlSeconds: 300 },
|
||||
5, // Only 5 entries
|
||||
storageBackend
|
||||
);
|
||||
|
||||
// Add 10 entries
|
||||
for (let i = 0; i < 10; i++) {
|
||||
const context = createFetchContext({ resource: `evict-test-${i}` });
|
||||
await smallCache.set(
|
||||
context,
|
||||
Buffer.from(`data ${i}`),
|
||||
'text/plain',
|
||||
{},
|
||||
'evict-upstream',
|
||||
'https://evict.example.com'
|
||||
);
|
||||
}
|
||||
|
||||
const stats = smallCache.getStats();
|
||||
// Should have evicted some entries
|
||||
expect(stats.totalEntries).toBeLessThanOrEqual(5);
|
||||
|
||||
smallCache.stop();
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Query Parameter Tests
|
||||
// ============================================================================
|
||||
|
||||
tap.test('cache: cache key should include query parameters', async () => {
|
||||
const context1 = createFetchContext({
|
||||
resource: 'query-test',
|
||||
query: { version: '1.0.0' },
|
||||
});
|
||||
|
||||
const context2 = createFetchContext({
|
||||
resource: 'query-test',
|
||||
query: { version: '2.0.0' },
|
||||
});
|
||||
|
||||
const upstreamUrl = 'https://registry.npmjs.org';
|
||||
|
||||
// Store with v1 query
|
||||
await cache.set(
|
||||
context1,
|
||||
Buffer.from('version 1 data'),
|
||||
'text/plain',
|
||||
{},
|
||||
'query-upstream',
|
||||
upstreamUrl
|
||||
);
|
||||
|
||||
// Store with v2 query
|
||||
await cache.set(
|
||||
context2,
|
||||
Buffer.from('version 2 data'),
|
||||
'text/plain',
|
||||
{},
|
||||
'query-upstream',
|
||||
upstreamUrl
|
||||
);
|
||||
|
||||
// Retrieve v1 - should get v1 data
|
||||
const entry1 = await cache.get(context1, upstreamUrl);
|
||||
expect(entry1).toBeTruthy();
|
||||
expect(entry1!.data.toString()).toEqual('version 1 data');
|
||||
|
||||
// Retrieve v2 - should get v2 data
|
||||
const entry2 = await cache.get(context2, upstreamUrl);
|
||||
expect(entry2).toBeTruthy();
|
||||
expect(entry2!.data.toString()).toEqual('version 2 data');
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// S3 Persistence Tests
|
||||
// ============================================================================
|
||||
|
||||
tap.test('cache: should load from S3 on memory cache miss', async () => {
|
||||
// Use a unique resource name for this test
|
||||
const uniqueResource = `persist-test-${Date.now()}`;
|
||||
const persistContext = createFetchContext({ resource: uniqueResource });
|
||||
const upstreamUrl = 'https://persist.example.com';
|
||||
|
||||
// Store in first cache instance
|
||||
await cache.set(
|
||||
persistContext,
|
||||
Buffer.from('persisted data'),
|
||||
'text/plain',
|
||||
{},
|
||||
'persist-upstream',
|
||||
upstreamUrl
|
||||
);
|
||||
|
||||
// Wait for S3 write to complete
|
||||
await new Promise(resolve => setTimeout(resolve, 200));
|
||||
|
||||
// Verify the entry is in the original cache's memory
|
||||
const originalEntry = await cache.get(persistContext, upstreamUrl);
|
||||
expect(originalEntry).toBeTruthy();
|
||||
|
||||
// Create a new cache instance (simulates restart) with SAME storage backend
|
||||
const freshCache = new UpstreamCache(
|
||||
{ enabled: true, defaultTtlSeconds: 300 },
|
||||
10000,
|
||||
storageBackend
|
||||
);
|
||||
|
||||
// Fresh cache has empty memory, should load from S3
|
||||
const entry = await freshCache.get(persistContext, upstreamUrl);
|
||||
|
||||
expect(entry).toBeTruthy();
|
||||
expect(entry!.data.toString()).toEqual('persisted data');
|
||||
|
||||
freshCache.stop();
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Cache Stats Tests
|
||||
// ============================================================================
|
||||
|
||||
tap.test('cache: should return accurate stats', async () => {
|
||||
const statsCache = new UpstreamCache(
|
||||
{ enabled: true, defaultTtlSeconds: 300 },
|
||||
1000,
|
||||
storageBackend
|
||||
);
|
||||
|
||||
// Add some entries
|
||||
for (let i = 0; i < 3; i++) {
|
||||
const context = createFetchContext({ resource: `stats-test-${i}` });
|
||||
await statsCache.set(
|
||||
context,
|
||||
Buffer.from(`stats data ${i}`),
|
||||
'text/plain',
|
||||
{},
|
||||
'stats-upstream',
|
||||
'https://stats.example.com'
|
||||
);
|
||||
}
|
||||
|
||||
// Add a negative entry
|
||||
const negContext = createFetchContext({ resource: 'stats-negative' });
|
||||
await statsCache.setNegative(negContext, 'stats-upstream', 'https://stats.example.com');
|
||||
|
||||
const stats = statsCache.getStats();
|
||||
|
||||
expect(stats.totalEntries).toBeGreaterThanOrEqual(4);
|
||||
expect(stats.enabled).toBeTrue();
|
||||
expect(stats.hasStorage).toBeTrue();
|
||||
expect(stats.maxEntries).toEqual(1000);
|
||||
|
||||
statsCache.stop();
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Invalidation Tests
|
||||
// ============================================================================
|
||||
|
||||
tap.test('cache: should invalidate specific cache entry', async () => {
|
||||
const invalidateContext = createFetchContext({ resource: 'invalidate-test' });
|
||||
const upstreamUrl = 'https://invalidate.example.com';
|
||||
|
||||
await cache.set(
|
||||
invalidateContext,
|
||||
Buffer.from('to be invalidated'),
|
||||
'text/plain',
|
||||
{},
|
||||
'inv-upstream',
|
||||
upstreamUrl
|
||||
);
|
||||
|
||||
// Verify it exists
|
||||
let entry = await cache.get(invalidateContext, upstreamUrl);
|
||||
expect(entry).toBeTruthy();
|
||||
|
||||
// Invalidate
|
||||
const deleted = await cache.invalidate(invalidateContext, upstreamUrl);
|
||||
expect(deleted).toBeTrue();
|
||||
|
||||
// Should be gone
|
||||
entry = await cache.get(invalidateContext, upstreamUrl);
|
||||
expect(entry).toBeNull();
|
||||
});
|
||||
|
||||
tap.test('cache: should invalidate entries matching pattern', async () => {
|
||||
const upstreamUrl = 'https://pattern.example.com';
|
||||
|
||||
// Add multiple entries
|
||||
for (const name of ['pattern-a', 'pattern-b', 'other-c']) {
|
||||
const context = createFetchContext({ resource: name });
|
||||
await cache.set(
|
||||
context,
|
||||
Buffer.from(`data for ${name}`),
|
||||
'text/plain',
|
||||
{},
|
||||
'pattern-upstream',
|
||||
upstreamUrl
|
||||
);
|
||||
}
|
||||
|
||||
// Invalidate entries matching 'pattern-*'
|
||||
const count = await cache.invalidatePattern(/pattern-/);
|
||||
expect(count).toBeGreaterThanOrEqual(2);
|
||||
|
||||
// pattern-a should be gone
|
||||
const entryA = await cache.get(createFetchContext({ resource: 'pattern-a' }), upstreamUrl);
|
||||
expect(entryA).toBeNull();
|
||||
|
||||
// other-c should still exist
|
||||
const entryC = await cache.get(createFetchContext({ resource: 'other-c' }), upstreamUrl);
|
||||
expect(entryC).toBeTruthy();
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Cleanup
|
||||
// ============================================================================
|
||||
|
||||
tap.test('cleanup: should stop cache and clean up bucket', async () => {
|
||||
if (cache) {
|
||||
cache.stop();
|
||||
}
|
||||
|
||||
// Clean up test bucket
|
||||
if (s3Bucket) {
|
||||
try {
|
||||
const files = await s3Bucket.fastList({});
|
||||
for (const file of files) {
|
||||
await s3Bucket.fastRemove({ path: file.name });
|
||||
}
|
||||
await smartBucket.removeBucket(bucketName);
|
||||
} catch {
|
||||
// Ignore cleanup errors
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
export default tap.start();
|
||||
@@ -0,0 +1,343 @@
|
||||
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||
import { SmartRegistry } from '../ts/index.js';
|
||||
import {
|
||||
createTestRegistryWithUpstream,
|
||||
createTrackingUpstreamProvider,
|
||||
} from './helpers/registry.js';
|
||||
import { StaticUpstreamProvider } from '../ts/upstream/interfaces.upstream.js';
|
||||
import type {
|
||||
IUpstreamProvider,
|
||||
IUpstreamResolutionContext,
|
||||
IProtocolUpstreamConfig,
|
||||
} from '../ts/upstream/interfaces.upstream.js';
|
||||
import type { TRegistryProtocol } from '../ts/core/interfaces.core.js';
|
||||
|
||||
// =============================================================================
|
||||
// StaticUpstreamProvider Tests
|
||||
// =============================================================================
|
||||
|
||||
tap.test('StaticUpstreamProvider: should return config for configured protocol', async () => {
|
||||
const npmConfig: IProtocolUpstreamConfig = {
|
||||
enabled: true,
|
||||
upstreams: [{ id: 'npmjs', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
|
||||
};
|
||||
|
||||
const provider = new StaticUpstreamProvider({
|
||||
npm: npmConfig,
|
||||
});
|
||||
|
||||
const result = await provider.resolveUpstreamConfig({
|
||||
protocol: 'npm',
|
||||
resource: 'lodash',
|
||||
scope: null,
|
||||
method: 'GET',
|
||||
resourceType: 'packument',
|
||||
});
|
||||
|
||||
expect(result).toBeDefined();
|
||||
expect(result?.enabled).toEqual(true);
|
||||
expect(result?.upstreams[0].id).toEqual('npmjs');
|
||||
});
|
||||
|
||||
tap.test('StaticUpstreamProvider: should return null for unconfigured protocol', async () => {
|
||||
const provider = new StaticUpstreamProvider({
|
||||
npm: {
|
||||
enabled: true,
|
||||
upstreams: [{ id: 'npmjs', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
|
||||
},
|
||||
});
|
||||
|
||||
const result = await provider.resolveUpstreamConfig({
|
||||
protocol: 'maven',
|
||||
resource: 'com.example:lib',
|
||||
scope: 'com.example',
|
||||
method: 'GET',
|
||||
resourceType: 'pom',
|
||||
});
|
||||
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
tap.test('StaticUpstreamProvider: should support multiple protocols', async () => {
|
||||
const provider = new StaticUpstreamProvider({
|
||||
npm: {
|
||||
enabled: true,
|
||||
upstreams: [{ id: 'npmjs', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
|
||||
},
|
||||
oci: {
|
||||
enabled: true,
|
||||
upstreams: [{ id: 'dockerhub', url: 'https://registry-1.docker.io', priority: 1, enabled: true }],
|
||||
},
|
||||
maven: {
|
||||
enabled: true,
|
||||
upstreams: [{ id: 'central', url: 'https://repo1.maven.org/maven2', priority: 1, enabled: true }],
|
||||
},
|
||||
});
|
||||
|
||||
const npmResult = await provider.resolveUpstreamConfig({
|
||||
protocol: 'npm',
|
||||
resource: 'lodash',
|
||||
scope: null,
|
||||
method: 'GET',
|
||||
resourceType: 'packument',
|
||||
});
|
||||
expect(npmResult?.upstreams[0].id).toEqual('npmjs');
|
||||
|
||||
const ociResult = await provider.resolveUpstreamConfig({
|
||||
protocol: 'oci',
|
||||
resource: 'library/nginx',
|
||||
scope: 'library',
|
||||
method: 'GET',
|
||||
resourceType: 'manifest',
|
||||
});
|
||||
expect(ociResult?.upstreams[0].id).toEqual('dockerhub');
|
||||
|
||||
const mavenResult = await provider.resolveUpstreamConfig({
|
||||
protocol: 'maven',
|
||||
resource: 'com.example:lib',
|
||||
scope: 'com.example',
|
||||
method: 'GET',
|
||||
resourceType: 'pom',
|
||||
});
|
||||
expect(mavenResult?.upstreams[0].id).toEqual('central');
|
||||
});
|
||||
|
||||
// =============================================================================
|
||||
// Registry with Provider Integration Tests
|
||||
// =============================================================================
|
||||
|
||||
let registry: SmartRegistry;
|
||||
let trackingProvider: ReturnType<typeof createTrackingUpstreamProvider>;
|
||||
|
||||
tap.test('Provider Integration: should create registry with upstream provider', async () => {
|
||||
trackingProvider = createTrackingUpstreamProvider({
|
||||
npm: {
|
||||
enabled: true,
|
||||
upstreams: [{ id: 'test-npm', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
|
||||
},
|
||||
});
|
||||
|
||||
registry = await createTestRegistryWithUpstream(trackingProvider.provider);
|
||||
|
||||
expect(registry).toBeInstanceOf(SmartRegistry);
|
||||
expect(registry.isInitialized()).toEqual(true);
|
||||
});
|
||||
|
||||
tap.test('Provider Integration: should call provider when fetching unknown npm package', async () => {
|
||||
// Clear previous calls
|
||||
trackingProvider.calls.length = 0;
|
||||
|
||||
// Request a package that doesn't exist locally - should trigger upstream lookup
|
||||
const response = await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/npm/@test-scope/nonexistent-package',
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
// Provider should have been called for the packument lookup
|
||||
const npmCalls = trackingProvider.calls.filter(c => c.protocol === 'npm');
|
||||
|
||||
// The package doesn't exist locally, so upstream should be consulted
|
||||
// Note: actual upstream fetch may fail since the package doesn't exist
|
||||
expect(response.status).toBeOneOf([404, 200, 502]); // 404 if not found, 502 if upstream error
|
||||
});
|
||||
|
||||
tap.test('Provider Integration: provider receives correct context for scoped npm package', async () => {
|
||||
trackingProvider.calls.length = 0;
|
||||
|
||||
// Use URL-encoded path for scoped packages as npm client does
|
||||
await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/npm/@myorg%2fmy-package',
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
// Find any npm call - the exact resource type depends on routing
|
||||
const npmCalls = trackingProvider.calls.filter(c => c.protocol === 'npm');
|
||||
|
||||
// Provider should be called for upstream lookup
|
||||
if (npmCalls.length > 0) {
|
||||
const call = npmCalls[0];
|
||||
expect(call.protocol).toEqual('npm');
|
||||
// The resource should include the scoped name
|
||||
expect(call.resource).toInclude('myorg');
|
||||
expect(call.method).toEqual('GET');
|
||||
}
|
||||
});
|
||||
|
||||
tap.test('Provider Integration: provider receives correct context for unscoped npm package', async () => {
|
||||
trackingProvider.calls.length = 0;
|
||||
|
||||
await registry.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/npm/lodash',
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
const packumentCall = trackingProvider.calls.find(
|
||||
c => c.protocol === 'npm' && c.resourceType === 'packument'
|
||||
);
|
||||
|
||||
if (packumentCall) {
|
||||
expect(packumentCall.protocol).toEqual('npm');
|
||||
expect(packumentCall.resource).toEqual('lodash');
|
||||
expect(packumentCall.scope).toBeNull(); // No scope for unscoped package
|
||||
}
|
||||
});
|
||||
|
||||
// =============================================================================
|
||||
// Custom Provider Implementation Tests
|
||||
// =============================================================================
|
||||
|
||||
tap.test('Custom Provider: should support dynamic resolution based on context', async () => {
|
||||
// Create a provider that returns different configs based on scope
|
||||
const dynamicProvider: IUpstreamProvider = {
|
||||
async resolveUpstreamConfig(context: IUpstreamResolutionContext) {
|
||||
if (context.scope === 'internal') {
|
||||
// Internal packages go to private registry
|
||||
return {
|
||||
enabled: true,
|
||||
upstreams: [{ id: 'private', url: 'https://private.registry.com', priority: 1, enabled: true }],
|
||||
};
|
||||
}
|
||||
// Everything else goes to public registry
|
||||
return {
|
||||
enabled: true,
|
||||
upstreams: [{ id: 'public', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
const internalResult = await dynamicProvider.resolveUpstreamConfig({
|
||||
protocol: 'npm',
|
||||
resource: '@internal/utils',
|
||||
scope: 'internal',
|
||||
method: 'GET',
|
||||
resourceType: 'packument',
|
||||
});
|
||||
expect(internalResult?.upstreams[0].id).toEqual('private');
|
||||
|
||||
const publicResult = await dynamicProvider.resolveUpstreamConfig({
|
||||
protocol: 'npm',
|
||||
resource: '@public/utils',
|
||||
scope: 'public',
|
||||
method: 'GET',
|
||||
resourceType: 'packument',
|
||||
});
|
||||
expect(publicResult?.upstreams[0].id).toEqual('public');
|
||||
});
|
||||
|
||||
tap.test('Custom Provider: should support actor-based resolution', async () => {
|
||||
const actorAwareProvider: IUpstreamProvider = {
|
||||
async resolveUpstreamConfig(context: IUpstreamResolutionContext) {
|
||||
// Different upstreams based on user's organization
|
||||
if (context.actor?.orgId === 'enterprise-org') {
|
||||
return {
|
||||
enabled: true,
|
||||
upstreams: [{ id: 'enterprise', url: 'https://enterprise.registry.com', priority: 1, enabled: true }],
|
||||
};
|
||||
}
|
||||
return {
|
||||
enabled: true,
|
||||
upstreams: [{ id: 'default', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
const enterpriseResult = await actorAwareProvider.resolveUpstreamConfig({
|
||||
protocol: 'npm',
|
||||
resource: 'lodash',
|
||||
scope: null,
|
||||
actor: { orgId: 'enterprise-org', userId: 'user1' },
|
||||
method: 'GET',
|
||||
resourceType: 'packument',
|
||||
});
|
||||
expect(enterpriseResult?.upstreams[0].id).toEqual('enterprise');
|
||||
|
||||
const defaultResult = await actorAwareProvider.resolveUpstreamConfig({
|
||||
protocol: 'npm',
|
||||
resource: 'lodash',
|
||||
scope: null,
|
||||
actor: { orgId: 'free-org', userId: 'user2' },
|
||||
method: 'GET',
|
||||
resourceType: 'packument',
|
||||
});
|
||||
expect(defaultResult?.upstreams[0].id).toEqual('default');
|
||||
});
|
||||
|
||||
tap.test('Custom Provider: should support disabling upstream for specific resources', async () => {
|
||||
const selectiveProvider: IUpstreamProvider = {
|
||||
async resolveUpstreamConfig(context: IUpstreamResolutionContext) {
|
||||
// Block upstream for internal packages
|
||||
if (context.scope === 'internal') {
|
||||
return null; // No upstream for internal packages
|
||||
}
|
||||
return {
|
||||
enabled: true,
|
||||
upstreams: [{ id: 'public', url: 'https://registry.npmjs.org', priority: 1, enabled: true }],
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
const internalResult = await selectiveProvider.resolveUpstreamConfig({
|
||||
protocol: 'npm',
|
||||
resource: '@internal/secret',
|
||||
scope: 'internal',
|
||||
method: 'GET',
|
||||
resourceType: 'packument',
|
||||
});
|
||||
expect(internalResult).toBeNull();
|
||||
|
||||
const publicResult = await selectiveProvider.resolveUpstreamConfig({
|
||||
protocol: 'npm',
|
||||
resource: 'lodash',
|
||||
scope: null,
|
||||
method: 'GET',
|
||||
resourceType: 'packument',
|
||||
});
|
||||
expect(publicResult).not.toBeNull();
|
||||
});
|
||||
|
||||
// =============================================================================
|
||||
// Registry without Provider Tests
|
||||
// =============================================================================
|
||||
|
||||
tap.test('No Provider: registry should work without upstream provider', async () => {
|
||||
const registryWithoutUpstream = await createTestRegistryWithUpstream(
|
||||
// Pass a provider that always returns null
|
||||
{
|
||||
async resolveUpstreamConfig() {
|
||||
return null;
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
expect(registryWithoutUpstream).toBeInstanceOf(SmartRegistry);
|
||||
|
||||
// Should return 404 for non-existent package (no upstream to check)
|
||||
const response = await registryWithoutUpstream.handleRequest({
|
||||
method: 'GET',
|
||||
path: '/npm/nonexistent-package-xyz',
|
||||
headers: {},
|
||||
query: {},
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(404);
|
||||
|
||||
registryWithoutUpstream.destroy();
|
||||
});
|
||||
|
||||
// =============================================================================
|
||||
// Cleanup
|
||||
// =============================================================================
|
||||
|
||||
tap.postTask('cleanup registry', async () => {
|
||||
if (registry) {
|
||||
registry.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
export default tap.start();
|
||||
@@ -3,6 +3,6 @@
|
||||
*/
|
||||
export const commitinfo = {
|
||||
name: '@push.rocks/smartregistry',
|
||||
version: '1.1.1',
|
||||
description: 'a registry for npm modules and oci images'
|
||||
version: '2.9.1',
|
||||
description: 'A composable TypeScript library implementing OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems registries for building unified container and package registries'
|
||||
}
|
||||
|
||||
@@ -0,0 +1,681 @@
|
||||
import { Smartlog } from '@push.rocks/smartlog';
|
||||
import { BaseRegistry } from '../core/classes.baseregistry.js';
|
||||
import { RegistryStorage } from '../core/classes.registrystorage.js';
|
||||
import { AuthManager } from '../core/classes.authmanager.js';
|
||||
import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from '../core/interfaces.core.js';
|
||||
import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
|
||||
import type {
|
||||
ICargoIndexEntry,
|
||||
ICargoPublishMetadata,
|
||||
ICargoConfig,
|
||||
ICargoError,
|
||||
ICargoPublishResponse,
|
||||
ICargoYankResponse,
|
||||
ICargoSearchResponse,
|
||||
ICargoSearchResult,
|
||||
} from './interfaces.cargo.js';
|
||||
import { CargoUpstream } from './classes.cargoupstream.js';
|
||||
|
||||
/**
|
||||
* Cargo/crates.io registry implementation
|
||||
* Implements the sparse HTTP-based protocol
|
||||
* Spec: https://doc.rust-lang.org/cargo/reference/registry-index.html
|
||||
*/
|
||||
export class CargoRegistry extends BaseRegistry {
|
||||
private storage: RegistryStorage;
|
||||
private authManager: AuthManager;
|
||||
private basePath: string = '/cargo';
|
||||
private registryUrl: string;
|
||||
private logger: Smartlog;
|
||||
private upstreamProvider: IUpstreamProvider | null = null;
|
||||
|
||||
constructor(
|
||||
storage: RegistryStorage,
|
||||
authManager: AuthManager,
|
||||
basePath: string = '/cargo',
|
||||
registryUrl: string = 'http://localhost:5000/cargo',
|
||||
upstreamProvider?: IUpstreamProvider
|
||||
) {
|
||||
super();
|
||||
this.storage = storage;
|
||||
this.authManager = authManager;
|
||||
this.basePath = basePath;
|
||||
this.registryUrl = registryUrl;
|
||||
this.upstreamProvider = upstreamProvider || null;
|
||||
|
||||
this.logger = this.createProtocolLogger('cargo-registry', 'cargo');
|
||||
}
|
||||
|
||||
/**
|
||||
* Get upstream for a specific request.
|
||||
* Calls the provider to resolve upstream config dynamically.
|
||||
*/
|
||||
private async getUpstreamForRequest(
|
||||
resource: string,
|
||||
resourceType: string,
|
||||
method: string,
|
||||
actor?: IRequestActor
|
||||
): Promise<CargoUpstream | null> {
|
||||
if (!this.upstreamProvider) return null;
|
||||
|
||||
const config = await this.upstreamProvider.resolveUpstreamConfig({
|
||||
protocol: 'cargo',
|
||||
resource,
|
||||
scope: resource, // For Cargo, crate name is the scope
|
||||
actor,
|
||||
method,
|
||||
resourceType,
|
||||
});
|
||||
|
||||
if (!config?.enabled) return null;
|
||||
return new CargoUpstream(config, undefined, this.logger);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up resources (timers, connections, etc.)
|
||||
*/
|
||||
public destroy(): void {
|
||||
// No persistent upstream to clean up with dynamic provider
|
||||
}
|
||||
|
||||
public async init(): Promise<void> {
|
||||
// Initialize config.json if not exists
|
||||
const existingConfig = await this.storage.getCargoConfig();
|
||||
if (!existingConfig) {
|
||||
const config: ICargoConfig = {
|
||||
dl: `${this.registryUrl}/api/v1/crates/{crate}/{version}/download`,
|
||||
api: this.registryUrl,
|
||||
};
|
||||
await this.storage.putCargoConfig(config);
|
||||
this.logger.log('info', 'Initialized Cargo registry config', { config });
|
||||
}
|
||||
}
|
||||
|
||||
public getBasePath(): string {
|
||||
return this.basePath;
|
||||
}
|
||||
|
||||
public async handleRequest(context: IRequestContext): Promise<IResponse> {
|
||||
const path = context.path.replace(this.basePath, '');
|
||||
|
||||
// Extract token (Cargo uses Authorization header WITHOUT "Bearer" prefix)
|
||||
const authHeader = this.getAuthorizationHeader(context);
|
||||
const token = authHeader ? await this.authManager.validateToken(authHeader, 'cargo') : null;
|
||||
|
||||
const actor: IRequestActor = this.buildRequestActor(context, token);
|
||||
|
||||
this.logger.log('debug', `handleRequest: ${context.method} ${path}`, {
|
||||
method: context.method,
|
||||
path,
|
||||
hasAuth: !!token
|
||||
});
|
||||
|
||||
return this.storage.withContext({ protocol: 'cargo', actor }, async () => {
|
||||
// Config endpoint (required for sparse protocol)
|
||||
if (path === '/config.json') {
|
||||
return this.handleConfigJson();
|
||||
}
|
||||
|
||||
// API endpoints
|
||||
if (path.startsWith('/api/v1/')) {
|
||||
return this.handleApiRequest(path, context, token, actor);
|
||||
}
|
||||
|
||||
// Index files (sparse protocol)
|
||||
return this.handleIndexRequest(path, actor);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if token has permission for resource
|
||||
*/
|
||||
protected async checkPermission(
|
||||
token: IAuthToken | null,
|
||||
resource: string,
|
||||
action: string
|
||||
): Promise<boolean> {
|
||||
if (!token) return false;
|
||||
return this.authManager.authorize(token, `cargo:crate:${resource}`, action);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle API requests (/api/v1/*)
|
||||
*/
|
||||
private async handleApiRequest(
|
||||
path: string,
|
||||
context: IRequestContext,
|
||||
token: IAuthToken | null,
|
||||
actor?: IRequestActor
|
||||
): Promise<IResponse> {
|
||||
// Publish: PUT /api/v1/crates/new
|
||||
if (path === '/api/v1/crates/new' && context.method === 'PUT') {
|
||||
return this.handlePublish(context.body as Buffer, token);
|
||||
}
|
||||
|
||||
// Download: GET /api/v1/crates/{crate}/{version}/download
|
||||
const downloadMatch = path.match(/^\/api\/v1\/crates\/([^\/]+)\/([^\/]+)\/download$/);
|
||||
if (downloadMatch && context.method === 'GET') {
|
||||
return this.handleDownload(downloadMatch[1], downloadMatch[2], actor);
|
||||
}
|
||||
|
||||
// Yank: DELETE /api/v1/crates/{crate}/{version}/yank
|
||||
const yankMatch = path.match(/^\/api\/v1\/crates\/([^\/]+)\/([^\/]+)\/yank$/);
|
||||
if (yankMatch && context.method === 'DELETE') {
|
||||
return this.handleYank(yankMatch[1], yankMatch[2], token);
|
||||
}
|
||||
|
||||
// Unyank: PUT /api/v1/crates/{crate}/{version}/unyank
|
||||
const unyankMatch = path.match(/^\/api\/v1\/crates\/([^\/]+)\/([^\/]+)\/unyank$/);
|
||||
if (unyankMatch && context.method === 'PUT') {
|
||||
return this.handleUnyank(unyankMatch[1], unyankMatch[2], token);
|
||||
}
|
||||
|
||||
// Search: GET /api/v1/crates?q={query}
|
||||
if (path.startsWith('/api/v1/crates') && context.method === 'GET') {
|
||||
const query = context.query?.q || '';
|
||||
const perPage = parseInt(context.query?.per_page || '10', 10);
|
||||
return this.handleSearch(query, perPage);
|
||||
}
|
||||
|
||||
return {
|
||||
status: 404,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: this.createError('API endpoint not found'),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle index file requests
|
||||
* Paths: /1/{name}, /2/{name}, /3/{c}/{name}, /{p1}/{p2}/{name}
|
||||
*/
|
||||
private async handleIndexRequest(path: string, actor?: IRequestActor): Promise<IResponse> {
|
||||
// Parse index paths to extract crate name
|
||||
const pathParts = path.split('/').filter(p => p);
|
||||
let crateName: string | null = null;
|
||||
|
||||
if (pathParts.length === 2 && pathParts[0] === '1') {
|
||||
// 1-character names: /1/{name}
|
||||
crateName = pathParts[1];
|
||||
} else if (pathParts.length === 2 && pathParts[0] === '2') {
|
||||
// 2-character names: /2/{name}
|
||||
crateName = pathParts[1];
|
||||
} else if (pathParts.length === 3 && pathParts[0] === '3') {
|
||||
// 3-character names: /3/{c}/{name}
|
||||
crateName = pathParts[2];
|
||||
} else if (pathParts.length === 3) {
|
||||
// 4+ character names: /{p1}/{p2}/{name}
|
||||
crateName = pathParts[2];
|
||||
}
|
||||
|
||||
if (!crateName) {
|
||||
return {
|
||||
status: 404,
|
||||
headers: { 'Content-Type': 'text/plain' },
|
||||
body: Buffer.from(''),
|
||||
};
|
||||
}
|
||||
|
||||
return this.handleIndexFile(crateName, actor);
|
||||
}
|
||||
|
||||
/**
|
||||
* Serve config.json
|
||||
*/
|
||||
private async handleConfigJson(): Promise<IResponse> {
|
||||
const config = await this.storage.getCargoConfig();
|
||||
|
||||
return {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: config || {
|
||||
dl: `${this.registryUrl}/api/v1/crates/{crate}/{version}/download`,
|
||||
api: this.registryUrl,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Serve index file for a crate
|
||||
*/
|
||||
private async handleIndexFile(crateName: string, actor?: IRequestActor): Promise<IResponse> {
|
||||
let index = await this.storage.getCargoIndex(crateName);
|
||||
|
||||
// Try upstream if not found locally
|
||||
if (!index || index.length === 0) {
|
||||
const upstream = await this.getUpstreamForRequest(crateName, 'index', 'GET', actor);
|
||||
if (upstream) {
|
||||
const upstreamIndex = await upstream.fetchCrateIndex(crateName);
|
||||
if (upstreamIndex) {
|
||||
// Parse the newline-delimited JSON
|
||||
const parsedIndex: ICargoIndexEntry[] = upstreamIndex
|
||||
.split('\n')
|
||||
.filter(line => line.trim())
|
||||
.map(line => JSON.parse(line));
|
||||
|
||||
if (parsedIndex.length > 0) {
|
||||
// Cache locally
|
||||
await this.storage.putCargoIndex(crateName, parsedIndex);
|
||||
index = parsedIndex;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!index || index.length === 0) {
|
||||
return {
|
||||
status: 404,
|
||||
headers: { 'Content-Type': 'text/plain' },
|
||||
body: Buffer.from(''),
|
||||
};
|
||||
}
|
||||
|
||||
// Return newline-delimited JSON
|
||||
const data = index.map(e => JSON.stringify(e)).join('\n') + '\n';
|
||||
|
||||
// Calculate ETag for caching
|
||||
const crypto = await import('crypto');
|
||||
const etag = `"${crypto.createHash('sha256').update(data).digest('hex')}"`;
|
||||
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'text/plain',
|
||||
'ETag': etag,
|
||||
},
|
||||
body: Buffer.from(data, 'utf-8'),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse binary publish request
|
||||
* Format: [4 bytes JSON len][JSON][4 bytes crate len][.crate file]
|
||||
*/
|
||||
private parsePublishRequest(body: Buffer): {
|
||||
metadata: ICargoPublishMetadata;
|
||||
crateFile: Buffer;
|
||||
} {
|
||||
let offset = 0;
|
||||
|
||||
// Read JSON length (4 bytes, u32 little-endian)
|
||||
if (body.length < 4) {
|
||||
throw new Error('Invalid publish request: body too short');
|
||||
}
|
||||
const jsonLength = body.readUInt32LE(offset);
|
||||
offset += 4;
|
||||
|
||||
// Read JSON metadata
|
||||
if (body.length < offset + jsonLength) {
|
||||
throw new Error('Invalid publish request: JSON data incomplete');
|
||||
}
|
||||
const jsonBuffer = body.slice(offset, offset + jsonLength);
|
||||
const metadata = JSON.parse(jsonBuffer.toString('utf-8'));
|
||||
offset += jsonLength;
|
||||
|
||||
// Read crate file length (4 bytes, u32 little-endian)
|
||||
if (body.length < offset + 4) {
|
||||
throw new Error('Invalid publish request: crate length missing');
|
||||
}
|
||||
const crateLength = body.readUInt32LE(offset);
|
||||
offset += 4;
|
||||
|
||||
// Read crate file
|
||||
if (body.length < offset + crateLength) {
|
||||
throw new Error('Invalid publish request: crate data incomplete');
|
||||
}
|
||||
const crateFile = body.slice(offset, offset + crateLength);
|
||||
|
||||
return { metadata, crateFile };
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle crate publish
|
||||
*/
|
||||
private async handlePublish(
|
||||
body: Buffer,
|
||||
token: IAuthToken | null
|
||||
): Promise<IResponse> {
|
||||
this.logger.log('info', 'handlePublish: received publish request', {
|
||||
bodyLength: body?.length || 0,
|
||||
hasAuth: !!token
|
||||
});
|
||||
|
||||
// Check authorization
|
||||
if (!token) {
|
||||
return {
|
||||
status: 403,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: this.createError('Authentication required'),
|
||||
};
|
||||
}
|
||||
|
||||
// Parse binary request
|
||||
let metadata: ICargoPublishMetadata;
|
||||
let crateFile: Buffer;
|
||||
try {
|
||||
const parsed = this.parsePublishRequest(body);
|
||||
metadata = parsed.metadata;
|
||||
crateFile = parsed.crateFile;
|
||||
} catch (error: any) {
|
||||
this.logger.log('error', 'handlePublish: parse error', { error: error.message });
|
||||
return {
|
||||
status: 400,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: this.createError(`Invalid request format: ${error.message}`),
|
||||
};
|
||||
}
|
||||
|
||||
// Validate crate name
|
||||
if (!this.validateCrateName(metadata.name)) {
|
||||
return {
|
||||
status: 400,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: this.createError('Invalid crate name'),
|
||||
};
|
||||
}
|
||||
|
||||
// Check permission
|
||||
const hasPermission = await this.checkPermission(token, metadata.name, 'write');
|
||||
if (!hasPermission) {
|
||||
this.logger.log('warn', 'handlePublish: unauthorized', {
|
||||
crateName: metadata.name,
|
||||
userId: token.userId
|
||||
});
|
||||
return {
|
||||
status: 403,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: this.createError('Insufficient permissions'),
|
||||
};
|
||||
}
|
||||
|
||||
// Calculate SHA256 checksum
|
||||
const crypto = await import('crypto');
|
||||
const cksum = crypto.createHash('sha256').update(crateFile).digest('hex');
|
||||
|
||||
// Create index entry
|
||||
const indexEntry: ICargoIndexEntry = {
|
||||
name: metadata.name,
|
||||
vers: metadata.vers,
|
||||
deps: metadata.deps,
|
||||
cksum,
|
||||
features: metadata.features,
|
||||
yanked: false,
|
||||
links: metadata.links || null,
|
||||
v: 2,
|
||||
rust_version: metadata.rust_version,
|
||||
};
|
||||
|
||||
// Check for duplicate version
|
||||
const existingIndex = await this.storage.getCargoIndex(metadata.name) || [];
|
||||
if (existingIndex.some(e => e.vers === metadata.vers)) {
|
||||
return {
|
||||
status: 400,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: this.createError(`Version ${metadata.vers} already exists`),
|
||||
};
|
||||
}
|
||||
|
||||
// Store crate file
|
||||
await this.storage.putCargoCrate(metadata.name, metadata.vers, crateFile);
|
||||
|
||||
// Update index (append new version)
|
||||
existingIndex.push(indexEntry);
|
||||
await this.storage.putCargoIndex(metadata.name, existingIndex);
|
||||
|
||||
this.logger.log('success', 'handlePublish: published crate', {
|
||||
name: metadata.name,
|
||||
version: metadata.vers,
|
||||
checksum: cksum
|
||||
});
|
||||
|
||||
const response: ICargoPublishResponse = {
|
||||
warnings: {
|
||||
invalid_categories: [],
|
||||
invalid_badges: [],
|
||||
other: [],
|
||||
},
|
||||
};
|
||||
|
||||
return {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: response,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle crate download
|
||||
*/
|
||||
private async handleDownload(
|
||||
crateName: string,
|
||||
version: string,
|
||||
actor?: IRequestActor
|
||||
): Promise<IResponse> {
|
||||
this.logger.log('debug', 'handleDownload', { crate: crateName, version });
|
||||
|
||||
// Try streaming from local storage first
|
||||
const streamResult = await this.storage.getCargoCrateStream(crateName, version);
|
||||
|
||||
if (streamResult) {
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/gzip',
|
||||
'Content-Length': streamResult.size.toString(),
|
||||
'Content-Disposition': `attachment; filename="${crateName}-${version}.crate"`,
|
||||
},
|
||||
body: streamResult.stream,
|
||||
};
|
||||
}
|
||||
|
||||
// Try upstream if not found locally
|
||||
let crateFile: Buffer | null = null;
|
||||
const upstream = await this.getUpstreamForRequest(crateName, 'crate', 'GET', actor);
|
||||
if (upstream) {
|
||||
crateFile = await upstream.fetchCrate(crateName, version);
|
||||
if (crateFile) {
|
||||
// Cache locally
|
||||
await this.storage.putCargoCrate(crateName, version, crateFile);
|
||||
}
|
||||
}
|
||||
|
||||
if (!crateFile) {
|
||||
return {
|
||||
status: 404,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: this.createError('Crate not found'),
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/gzip',
|
||||
'Content-Length': crateFile.length.toString(),
|
||||
'Content-Disposition': `attachment; filename="${crateName}-${version}.crate"`,
|
||||
},
|
||||
body: crateFile,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle yank operation
|
||||
*/
|
||||
private async handleYank(
|
||||
crateName: string,
|
||||
version: string,
|
||||
token: IAuthToken | null
|
||||
): Promise<IResponse> {
|
||||
return this.handleYankOperation(crateName, version, token, true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle unyank operation
|
||||
*/
|
||||
private async handleUnyank(
|
||||
crateName: string,
|
||||
version: string,
|
||||
token: IAuthToken | null
|
||||
): Promise<IResponse> {
|
||||
return this.handleYankOperation(crateName, version, token, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle yank/unyank operation
|
||||
*/
|
||||
private async handleYankOperation(
|
||||
crateName: string,
|
||||
version: string,
|
||||
token: IAuthToken | null,
|
||||
yank: boolean
|
||||
): Promise<IResponse> {
|
||||
this.logger.log('info', `handle${yank ? 'Yank' : 'Unyank'}`, {
|
||||
crate: crateName,
|
||||
version,
|
||||
hasAuth: !!token
|
||||
});
|
||||
|
||||
// Check authorization
|
||||
if (!token) {
|
||||
return {
|
||||
status: 403,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: this.createError('Authentication required'),
|
||||
};
|
||||
}
|
||||
|
||||
// Check permission
|
||||
const hasPermission = await this.checkPermission(token, crateName, 'write');
|
||||
if (!hasPermission) {
|
||||
return {
|
||||
status: 403,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: this.createError('Insufficient permissions'),
|
||||
};
|
||||
}
|
||||
|
||||
// Load index
|
||||
const index = await this.storage.getCargoIndex(crateName);
|
||||
if (!index) {
|
||||
return {
|
||||
status: 404,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: this.createError('Crate not found'),
|
||||
};
|
||||
}
|
||||
|
||||
// Find version
|
||||
const entry = index.find(e => e.vers === version);
|
||||
if (!entry) {
|
||||
return {
|
||||
status: 404,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: this.createError('Version not found'),
|
||||
};
|
||||
}
|
||||
|
||||
// Update yank status
|
||||
entry.yanked = yank;
|
||||
|
||||
// Save index (NOTE: do NOT delete .crate file)
|
||||
await this.storage.putCargoIndex(crateName, index);
|
||||
|
||||
this.logger.log('success', `${yank ? 'Yanked' : 'Unyanked'} version`, {
|
||||
crate: crateName,
|
||||
version
|
||||
});
|
||||
|
||||
const response: ICargoYankResponse = { ok: true };
|
||||
|
||||
return {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: response,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle search
|
||||
*/
|
||||
private async handleSearch(query: string, perPage: number): Promise<IResponse> {
|
||||
this.logger.log('debug', 'handleSearch', { query, perPage });
|
||||
|
||||
const results: ICargoSearchResult[] = [];
|
||||
|
||||
try {
|
||||
// List all index paths
|
||||
const indexPaths = await this.storage.listObjects('cargo/index/');
|
||||
|
||||
// Extract unique crate names
|
||||
const crateNames = new Set<string>();
|
||||
for (const path of indexPaths) {
|
||||
// Parse path to extract crate name
|
||||
const parts = path.split('/');
|
||||
if (parts.length >= 3) {
|
||||
const name = parts[parts.length - 1];
|
||||
if (name && !name.includes('.')) {
|
||||
crateNames.add(name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.log('debug', `handleSearch: found ${crateNames.size} crates`, {
|
||||
totalCrates: crateNames.size
|
||||
});
|
||||
|
||||
// Filter and process matching crates
|
||||
for (const name of crateNames) {
|
||||
if (!query || name.toLowerCase().includes(query.toLowerCase())) {
|
||||
const index = await this.storage.getCargoIndex(name);
|
||||
if (index && index.length > 0) {
|
||||
// Find latest non-yanked version
|
||||
const nonYanked = index.filter(e => !e.yanked);
|
||||
if (nonYanked.length > 0) {
|
||||
// Sort by version (simplified - should use semver)
|
||||
const sorted = [...nonYanked].sort((a, b) => b.vers.localeCompare(a.vers));
|
||||
|
||||
results.push({
|
||||
name: sorted[0].name,
|
||||
max_version: sorted[0].vers,
|
||||
description: '', // Would need to store separately
|
||||
});
|
||||
|
||||
if (results.length >= perPage) break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
this.logger.log('error', 'handleSearch: error', { error: error.message });
|
||||
}
|
||||
|
||||
const response: ICargoSearchResponse = {
|
||||
crates: results,
|
||||
meta: {
|
||||
total: results.length,
|
||||
},
|
||||
};
|
||||
|
||||
return {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: response,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate crate name
|
||||
* Rules: lowercase alphanumeric + _ and -, length 1-64
|
||||
*/
|
||||
private validateCrateName(name: string): boolean {
|
||||
return /^[a-z0-9_-]+$/.test(name) && name.length >= 1 && name.length <= 64;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create error response
|
||||
*/
|
||||
private createError(detail: string): ICargoError {
|
||||
return {
|
||||
errors: [{ detail }],
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
import * as plugins from '../plugins.js';
|
||||
import { BaseUpstream } from '../upstream/classes.baseupstream.js';
|
||||
import type {
|
||||
IProtocolUpstreamConfig,
|
||||
IUpstreamFetchContext,
|
||||
IUpstreamRegistryConfig,
|
||||
} from '../upstream/interfaces.upstream.js';
|
||||
|
||||
/**
|
||||
* Cargo-specific upstream implementation.
|
||||
*
|
||||
* Handles:
|
||||
* - Crate metadata (index) fetching
|
||||
* - Crate file (.crate) downloading
|
||||
* - Sparse index protocol support
|
||||
* - Content-addressable caching for .crate files
|
||||
*/
|
||||
export class CargoUpstream extends BaseUpstream {
|
||||
protected readonly protocolName = 'cargo';
|
||||
|
||||
/** Base URL for crate downloads (may differ from index URL) */
|
||||
private readonly downloadUrl: string;
|
||||
|
||||
constructor(
|
||||
config: IProtocolUpstreamConfig,
|
||||
downloadUrl?: string,
|
||||
logger?: plugins.smartlog.Smartlog,
|
||||
) {
|
||||
super(config, logger);
|
||||
// Default to crates.io download URL if not specified
|
||||
this.downloadUrl = downloadUrl || 'https://static.crates.io/crates';
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch crate metadata from the sparse index.
|
||||
*/
|
||||
public async fetchCrateIndex(crateName: string): Promise<string | null> {
|
||||
const path = this.buildIndexPath(crateName);
|
||||
|
||||
const context: IUpstreamFetchContext = {
|
||||
protocol: 'cargo',
|
||||
resource: crateName,
|
||||
resourceType: 'index',
|
||||
path,
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'accept': 'text/plain',
|
||||
},
|
||||
query: {},
|
||||
};
|
||||
|
||||
const result = await this.fetch(context);
|
||||
|
||||
if (!result || !result.success) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (Buffer.isBuffer(result.body)) {
|
||||
return result.body.toString('utf8');
|
||||
}
|
||||
|
||||
return typeof result.body === 'string' ? result.body : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a crate file from upstream.
|
||||
*/
|
||||
public async fetchCrate(crateName: string, version: string): Promise<Buffer | null> {
|
||||
// Crate downloads typically go to a different URL than the index
|
||||
const path = `/${crateName}/${crateName}-${version}.crate`;
|
||||
|
||||
const context: IUpstreamFetchContext = {
|
||||
protocol: 'cargo',
|
||||
resource: crateName,
|
||||
resourceType: 'crate',
|
||||
path,
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'accept': 'application/octet-stream',
|
||||
},
|
||||
query: {},
|
||||
};
|
||||
|
||||
// Use special handling for crate downloads
|
||||
const result = await this.fetchCrateFile(crateName, version);
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch crate file directly from the download URL.
|
||||
*/
|
||||
private async fetchCrateFile(crateName: string, version: string): Promise<Buffer | null> {
|
||||
const context: IUpstreamFetchContext = {
|
||||
protocol: 'cargo',
|
||||
resource: crateName,
|
||||
resourceType: 'crate',
|
||||
path: `/${crateName}/${crateName}-${version}.crate`,
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'accept': 'application/octet-stream',
|
||||
},
|
||||
query: {},
|
||||
};
|
||||
|
||||
const result = await this.fetch(context);
|
||||
|
||||
if (!result || !result.success) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return Buffer.isBuffer(result.body) ? result.body : Buffer.from(result.body);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the sparse index path for a crate.
|
||||
*
|
||||
* Path structure:
|
||||
* - 1 char: /1/{name}
|
||||
* - 2 chars: /2/{name}
|
||||
* - 3 chars: /3/{first char}/{name}
|
||||
* - 4+ chars: /{first 2}/{next 2}/{name}
|
||||
*/
|
||||
private buildIndexPath(crateName: string): string {
|
||||
const lowerName = crateName.toLowerCase();
|
||||
const len = lowerName.length;
|
||||
|
||||
if (len === 1) {
|
||||
return `/1/${lowerName}`;
|
||||
} else if (len === 2) {
|
||||
return `/2/${lowerName}`;
|
||||
} else if (len === 3) {
|
||||
return `/3/${lowerName[0]}/${lowerName}`;
|
||||
} else {
|
||||
return `/${lowerName.slice(0, 2)}/${lowerName.slice(2, 4)}/${lowerName}`;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Override URL building for Cargo-specific handling.
|
||||
*/
|
||||
protected buildUpstreamUrl(
|
||||
upstream: IUpstreamRegistryConfig,
|
||||
context: IUpstreamFetchContext,
|
||||
): string {
|
||||
let baseUrl = upstream.url;
|
||||
|
||||
// For crate downloads, use the download URL
|
||||
if (context.resourceType === 'crate') {
|
||||
baseUrl = this.downloadUrl;
|
||||
}
|
||||
|
||||
// Remove trailing slash
|
||||
if (baseUrl.endsWith('/')) {
|
||||
baseUrl = baseUrl.slice(0, -1);
|
||||
}
|
||||
|
||||
return `${baseUrl}${context.path}`;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
/**
|
||||
* Cargo/crates.io Registry module exports
|
||||
*/
|
||||
|
||||
export { CargoRegistry } from './classes.cargoregistry.js';
|
||||
export { CargoUpstream } from './classes.cargoupstream.js';
|
||||
export * from './interfaces.cargo.js';
|
||||
@@ -0,0 +1,169 @@
|
||||
/**
|
||||
* Cargo/crates.io registry type definitions
|
||||
* Based on: https://doc.rust-lang.org/cargo/reference/registry-index.html
|
||||
*/
|
||||
|
||||
/**
|
||||
* Dependency specification in Cargo index
|
||||
*/
|
||||
export interface ICargoDepend {
|
||||
/** Dependency package name */
|
||||
name: string;
|
||||
/** Version requirement (e.g., "^0.6", ">=1.0.0") */
|
||||
req: string;
|
||||
/** Optional features to enable */
|
||||
features: string[];
|
||||
/** Whether this dependency is optional */
|
||||
optional: boolean;
|
||||
/** Whether to include default features */
|
||||
default_features: boolean;
|
||||
/** Platform-specific target (e.g., "cfg(unix)") */
|
||||
target: string | null;
|
||||
/** Dependency kind: normal, dev, or build */
|
||||
kind: 'normal' | 'dev' | 'build';
|
||||
/** Alternative registry URL */
|
||||
registry: string | null;
|
||||
/** Rename to different package name */
|
||||
package: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Single version entry in the Cargo index file
|
||||
* Each line in the index file is one of these as JSON
|
||||
*/
|
||||
export interface ICargoIndexEntry {
|
||||
/** Crate name */
|
||||
name: string;
|
||||
/** Version string */
|
||||
vers: string;
|
||||
/** Dependencies */
|
||||
deps: ICargoDepend[];
|
||||
/** SHA256 checksum of the .crate file (hex) */
|
||||
cksum: string;
|
||||
/** Features (legacy format) */
|
||||
features: Record<string, string[]>;
|
||||
/** Features (extended format for newer Cargo) */
|
||||
features2?: Record<string, string[]>;
|
||||
/** Whether this version is yanked (deprecated but not deleted) */
|
||||
yanked: boolean;
|
||||
/** Optional native library link */
|
||||
links?: string | null;
|
||||
/** Index format version (2 is current) */
|
||||
v?: number;
|
||||
/** Minimum Rust version required */
|
||||
rust_version?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Metadata sent during crate publication
|
||||
*/
|
||||
export interface ICargoPublishMetadata {
|
||||
/** Crate name */
|
||||
name: string;
|
||||
/** Version string */
|
||||
vers: string;
|
||||
/** Dependencies */
|
||||
deps: ICargoDepend[];
|
||||
/** Features */
|
||||
features: Record<string, string[]>;
|
||||
/** Authors */
|
||||
authors: string[];
|
||||
/** Short description */
|
||||
description?: string;
|
||||
/** Documentation URL */
|
||||
documentation?: string;
|
||||
/** Homepage URL */
|
||||
homepage?: string;
|
||||
/** README content */
|
||||
readme?: string;
|
||||
/** README file path */
|
||||
readme_file?: string;
|
||||
/** Keywords for search */
|
||||
keywords?: string[];
|
||||
/** Categories */
|
||||
categories?: string[];
|
||||
/** License identifier (SPDX) */
|
||||
license?: string;
|
||||
/** License file path */
|
||||
license_file?: string;
|
||||
/** Repository URL */
|
||||
repository?: string;
|
||||
/** Badges */
|
||||
badges?: Record<string, any>;
|
||||
/** Native library link */
|
||||
links?: string | null;
|
||||
/** Minimum Rust version */
|
||||
rust_version?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Registry configuration (config.json)
|
||||
* Required for sparse protocol support
|
||||
*/
|
||||
export interface ICargoConfig {
|
||||
/** Download URL template */
|
||||
dl: string;
|
||||
/** API base URL */
|
||||
api: string;
|
||||
/** Whether authentication is required for downloads */
|
||||
'auth-required'?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Search result for a single crate
|
||||
*/
|
||||
export interface ICargoSearchResult {
|
||||
/** Crate name */
|
||||
name: string;
|
||||
/** Latest/maximum version */
|
||||
max_version: string;
|
||||
/** Description */
|
||||
description: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Search response structure
|
||||
*/
|
||||
export interface ICargoSearchResponse {
|
||||
/** Array of matching crates */
|
||||
crates: ICargoSearchResult[];
|
||||
/** Metadata about results */
|
||||
meta: {
|
||||
/** Total number of results */
|
||||
total: number;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Error response structure
|
||||
*/
|
||||
export interface ICargoError {
|
||||
/** Array of error details */
|
||||
errors: Array<{
|
||||
/** Error message */
|
||||
detail: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Publish success response
|
||||
*/
|
||||
export interface ICargoPublishResponse {
|
||||
/** Warnings from validation */
|
||||
warnings: {
|
||||
/** Invalid categories */
|
||||
invalid_categories: string[];
|
||||
/** Invalid badges */
|
||||
invalid_badges: string[];
|
||||
/** Other warnings */
|
||||
other: string[];
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Yank/Unyank response
|
||||
*/
|
||||
export interface ICargoYankResponse {
|
||||
/** Success indicator */
|
||||
ok: boolean;
|
||||
}
|
||||
+215
-35
@@ -1,25 +1,195 @@
|
||||
import { RegistryStorage } from './core/classes.registrystorage.js';
|
||||
import { AuthManager } from './core/classes.authmanager.js';
|
||||
import { BaseRegistry } from './core/classes.baseregistry.js';
|
||||
import type { IRegistryConfig, IRequestContext, IResponse } from './core/interfaces.core.js';
|
||||
import type {
|
||||
IProtocolConfig,
|
||||
IRegistryConfig,
|
||||
IRequestContext,
|
||||
IResponse,
|
||||
TRegistryProtocol,
|
||||
} from './core/interfaces.core.js';
|
||||
import { toReadableStream } from './core/helpers.stream.js';
|
||||
import { OciRegistry } from './oci/classes.ociregistry.js';
|
||||
import { NpmRegistry } from './npm/classes.npmregistry.js';
|
||||
import { MavenRegistry } from './maven/classes.mavenregistry.js';
|
||||
import { CargoRegistry } from './cargo/classes.cargoregistry.js';
|
||||
import { ComposerRegistry } from './composer/classes.composerregistry.js';
|
||||
import { PypiRegistry } from './pypi/classes.pypiregistry.js';
|
||||
import { RubyGemsRegistry } from './rubygems/classes.rubygemsregistry.js';
|
||||
|
||||
type TRegistryDescriptor = {
|
||||
protocol: TRegistryProtocol;
|
||||
getConfig: (config: IRegistryConfig) => IProtocolConfig | undefined;
|
||||
matchesPath: (config: IRegistryConfig, path: string) => boolean;
|
||||
create: (args: {
|
||||
storage: RegistryStorage;
|
||||
authManager: AuthManager;
|
||||
config: IRegistryConfig;
|
||||
protocolConfig: IProtocolConfig;
|
||||
}) => BaseRegistry;
|
||||
};
|
||||
|
||||
const registryDescriptors: TRegistryDescriptor[] = [
|
||||
{
|
||||
protocol: 'oci',
|
||||
getConfig: (config) => config.oci,
|
||||
matchesPath: (config, path) => path.startsWith(config.oci?.basePath ?? '/oci'),
|
||||
create: ({ storage, authManager, config, protocolConfig }) => {
|
||||
const ociTokens = config.auth.ociTokens?.enabled ? {
|
||||
realm: config.auth.ociTokens.realm,
|
||||
service: config.auth.ociTokens.service,
|
||||
} : undefined;
|
||||
return new OciRegistry(
|
||||
storage,
|
||||
authManager,
|
||||
protocolConfig.basePath ?? '/oci',
|
||||
ociTokens,
|
||||
config.upstreamProvider
|
||||
);
|
||||
},
|
||||
},
|
||||
{
|
||||
protocol: 'npm',
|
||||
getConfig: (config) => config.npm,
|
||||
matchesPath: (config, path) => path.startsWith(config.npm?.basePath ?? '/npm'),
|
||||
create: ({ storage, authManager, config, protocolConfig }) => {
|
||||
const basePath = protocolConfig.basePath ?? '/npm';
|
||||
return new NpmRegistry(
|
||||
storage,
|
||||
authManager,
|
||||
basePath,
|
||||
protocolConfig.registryUrl ?? `http://localhost:5000${basePath}`,
|
||||
config.upstreamProvider
|
||||
);
|
||||
},
|
||||
},
|
||||
{
|
||||
protocol: 'maven',
|
||||
getConfig: (config) => config.maven,
|
||||
matchesPath: (config, path) => path.startsWith(config.maven?.basePath ?? '/maven'),
|
||||
create: ({ storage, authManager, config, protocolConfig }) => {
|
||||
const basePath = protocolConfig.basePath ?? '/maven';
|
||||
return new MavenRegistry(
|
||||
storage,
|
||||
authManager,
|
||||
basePath,
|
||||
protocolConfig.registryUrl ?? `http://localhost:5000${basePath}`,
|
||||
config.upstreamProvider
|
||||
);
|
||||
},
|
||||
},
|
||||
{
|
||||
protocol: 'cargo',
|
||||
getConfig: (config) => config.cargo,
|
||||
matchesPath: (config, path) => path.startsWith(config.cargo?.basePath ?? '/cargo'),
|
||||
create: ({ storage, authManager, config, protocolConfig }) => {
|
||||
const basePath = protocolConfig.basePath ?? '/cargo';
|
||||
return new CargoRegistry(
|
||||
storage,
|
||||
authManager,
|
||||
basePath,
|
||||
protocolConfig.registryUrl ?? `http://localhost:5000${basePath}`,
|
||||
config.upstreamProvider
|
||||
);
|
||||
},
|
||||
},
|
||||
{
|
||||
protocol: 'composer',
|
||||
getConfig: (config) => config.composer,
|
||||
matchesPath: (config, path) => path.startsWith(config.composer?.basePath ?? '/composer'),
|
||||
create: ({ storage, authManager, config, protocolConfig }) => {
|
||||
const basePath = protocolConfig.basePath ?? '/composer';
|
||||
return new ComposerRegistry(
|
||||
storage,
|
||||
authManager,
|
||||
basePath,
|
||||
protocolConfig.registryUrl ?? `http://localhost:5000${basePath}`,
|
||||
config.upstreamProvider
|
||||
);
|
||||
},
|
||||
},
|
||||
{
|
||||
protocol: 'pypi',
|
||||
getConfig: (config) => config.pypi,
|
||||
matchesPath: (config, path) => {
|
||||
const basePath = config.pypi?.basePath ?? '/pypi';
|
||||
return path.startsWith(basePath) || path.startsWith('/simple');
|
||||
},
|
||||
create: ({ storage, authManager, config, protocolConfig }) => new PypiRegistry(
|
||||
storage,
|
||||
authManager,
|
||||
protocolConfig.basePath ?? '/pypi',
|
||||
protocolConfig.registryUrl ?? 'http://localhost:5000',
|
||||
config.upstreamProvider
|
||||
),
|
||||
},
|
||||
{
|
||||
protocol: 'rubygems',
|
||||
getConfig: (config) => config.rubygems,
|
||||
matchesPath: (config, path) => path.startsWith(config.rubygems?.basePath ?? '/rubygems'),
|
||||
create: ({ storage, authManager, config, protocolConfig }) => {
|
||||
const basePath = protocolConfig.basePath ?? '/rubygems';
|
||||
return new RubyGemsRegistry(
|
||||
storage,
|
||||
authManager,
|
||||
basePath,
|
||||
protocolConfig.registryUrl ?? `http://localhost:5000${basePath}`,
|
||||
config.upstreamProvider
|
||||
);
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
/**
|
||||
* Main registry orchestrator
|
||||
* Routes requests to appropriate protocol handlers (OCI or NPM)
|
||||
* Main registry orchestrator.
|
||||
* Routes requests to appropriate protocol handlers (OCI, NPM, Maven, Cargo, Composer, PyPI, or RubyGems).
|
||||
*
|
||||
* Supports pluggable authentication and storage hooks:
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* // Basic usage with default in-memory auth
|
||||
* const registry = new SmartRegistry(config);
|
||||
*
|
||||
* // With custom auth provider (LDAP, OAuth, etc.)
|
||||
* const registry = new SmartRegistry({
|
||||
* ...config,
|
||||
* authProvider: new LdapAuthProvider(ldapClient),
|
||||
* });
|
||||
*
|
||||
* // With storage hooks for quota tracking
|
||||
* const registry = new SmartRegistry({
|
||||
* ...config,
|
||||
* storageHooks: {
|
||||
* beforePut: async (ctx) => {
|
||||
* const quota = await getQuota(ctx.actor?.orgId);
|
||||
* if (ctx.metadata?.size > quota) {
|
||||
* return { allowed: false, reason: 'Quota exceeded' };
|
||||
* }
|
||||
* return { allowed: true };
|
||||
* },
|
||||
* afterPut: async (ctx) => {
|
||||
* await auditLog('storage.put', ctx);
|
||||
* }
|
||||
* }
|
||||
* });
|
||||
* ```
|
||||
*/
|
||||
export class SmartRegistry {
|
||||
private storage: RegistryStorage;
|
||||
private authManager: AuthManager;
|
||||
private registries: Map<string, BaseRegistry> = new Map();
|
||||
private registries: Map<TRegistryProtocol, BaseRegistry> = new Map();
|
||||
private config: IRegistryConfig;
|
||||
private initialized: boolean = false;
|
||||
|
||||
constructor(config: IRegistryConfig) {
|
||||
this.config = config;
|
||||
this.storage = new RegistryStorage(config.storage);
|
||||
this.authManager = new AuthManager(config.auth);
|
||||
|
||||
// Create storage with optional hooks
|
||||
this.storage = new RegistryStorage(config.storage, config.storageHooks);
|
||||
|
||||
// Create auth manager with optional custom provider
|
||||
this.authManager = new AuthManager(config.auth, config.authProvider);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -34,21 +204,20 @@ export class SmartRegistry {
|
||||
// Initialize auth manager
|
||||
await this.authManager.init();
|
||||
|
||||
// Initialize OCI registry if enabled
|
||||
if (this.config.oci?.enabled) {
|
||||
const ociBasePath = this.config.oci.basePath || '/oci';
|
||||
const ociRegistry = new OciRegistry(this.storage, this.authManager, ociBasePath);
|
||||
await ociRegistry.init();
|
||||
this.registries.set('oci', ociRegistry);
|
||||
for (const descriptor of registryDescriptors) {
|
||||
const protocolConfig = descriptor.getConfig(this.config);
|
||||
if (!protocolConfig?.enabled) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Initialize NPM registry if enabled
|
||||
if (this.config.npm?.enabled) {
|
||||
const npmBasePath = this.config.npm.basePath || '/npm';
|
||||
const registryUrl = `http://localhost:5000${npmBasePath}`; // TODO: Make configurable
|
||||
const npmRegistry = new NpmRegistry(this.storage, this.authManager, npmBasePath, registryUrl);
|
||||
await npmRegistry.init();
|
||||
this.registries.set('npm', npmRegistry);
|
||||
const registry = descriptor.create({
|
||||
storage: this.storage,
|
||||
authManager: this.authManager,
|
||||
config: this.config,
|
||||
protocolConfig,
|
||||
});
|
||||
await registry.init();
|
||||
this.registries.set(descriptor.protocol, registry);
|
||||
}
|
||||
|
||||
this.initialized = true;
|
||||
@@ -60,25 +229,27 @@ export class SmartRegistry {
|
||||
*/
|
||||
public async handleRequest(context: IRequestContext): Promise<IResponse> {
|
||||
const path = context.path;
|
||||
let response: IResponse | undefined;
|
||||
|
||||
// Route to OCI registry
|
||||
if (this.config.oci?.enabled && path.startsWith(this.config.oci.basePath)) {
|
||||
const ociRegistry = this.registries.get('oci');
|
||||
if (ociRegistry) {
|
||||
return ociRegistry.handleRequest(context);
|
||||
}
|
||||
for (const descriptor of registryDescriptors) {
|
||||
if (response) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Route to NPM registry
|
||||
if (this.config.npm?.enabled && path.startsWith(this.config.npm.basePath)) {
|
||||
const npmRegistry = this.registries.get('npm');
|
||||
if (npmRegistry) {
|
||||
return npmRegistry.handleRequest(context);
|
||||
const protocolConfig = descriptor.getConfig(this.config);
|
||||
if (!protocolConfig?.enabled || !descriptor.matchesPath(this.config, path)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const registry = this.registries.get(descriptor.protocol);
|
||||
if (registry) {
|
||||
response = await registry.handleRequest(context);
|
||||
}
|
||||
}
|
||||
|
||||
// No matching registry
|
||||
return {
|
||||
if (!response) {
|
||||
response = {
|
||||
status: 404,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: {
|
||||
@@ -88,6 +259,17 @@ export class SmartRegistry {
|
||||
};
|
||||
}
|
||||
|
||||
// Normalize body to ReadableStream<Uint8Array> at the API boundary
|
||||
if (response.body != null && !(response.body instanceof ReadableStream)) {
|
||||
if (!Buffer.isBuffer(response.body) && typeof response.body === 'object' && !(response.body instanceof Uint8Array)) {
|
||||
response.headers['Content-Type'] ??= 'application/json';
|
||||
}
|
||||
response.body = toReadableStream(response.body);
|
||||
}
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the storage instance (for testing/advanced use)
|
||||
*/
|
||||
@@ -105,7 +287,7 @@ export class SmartRegistry {
|
||||
/**
|
||||
* Get a specific registry handler
|
||||
*/
|
||||
public getRegistry(protocol: 'oci' | 'npm'): BaseRegistry | undefined {
|
||||
public getRegistry(protocol: 'oci' | 'npm' | 'maven' | 'cargo' | 'composer' | 'pypi' | 'rubygems'): BaseRegistry | undefined {
|
||||
return this.registries.get(protocol);
|
||||
}
|
||||
|
||||
@@ -121,9 +303,7 @@ export class SmartRegistry {
|
||||
*/
|
||||
public destroy(): void {
|
||||
for (const registry of this.registries.values()) {
|
||||
if (typeof (registry as any).destroy === 'function') {
|
||||
(registry as any).destroy();
|
||||
}
|
||||
registry.destroy();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,541 @@
|
||||
/**
|
||||
* Composer Registry Implementation
|
||||
* Compliant with Composer v2 repository API
|
||||
*/
|
||||
|
||||
import { BaseRegistry } from '../core/classes.baseregistry.js';
|
||||
import type { RegistryStorage } from '../core/classes.registrystorage.js';
|
||||
import type { AuthManager } from '../core/classes.authmanager.js';
|
||||
import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from '../core/interfaces.core.js';
|
||||
import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
|
||||
import { isBinaryData, toBuffer } from '../core/helpers.buffer.js';
|
||||
import type {
|
||||
IComposerPackage,
|
||||
IComposerPackageMetadata,
|
||||
IComposerRepository,
|
||||
} from './interfaces.composer.js';
|
||||
import {
|
||||
normalizeVersion,
|
||||
validateComposerJson,
|
||||
extractComposerJsonFromZip,
|
||||
calculateSha1,
|
||||
parseVendorPackage,
|
||||
generatePackagesJson,
|
||||
sortVersions,
|
||||
} from './helpers.composer.js';
|
||||
import { ComposerUpstream } from './classes.composerupstream.js';
|
||||
|
||||
export class ComposerRegistry extends BaseRegistry {
|
||||
private storage: RegistryStorage;
|
||||
private authManager: AuthManager;
|
||||
private basePath: string = '/composer';
|
||||
private registryUrl: string;
|
||||
private upstreamProvider: IUpstreamProvider | null = null;
|
||||
|
||||
constructor(
|
||||
storage: RegistryStorage,
|
||||
authManager: AuthManager,
|
||||
basePath: string = '/composer',
|
||||
registryUrl: string = 'http://localhost:5000/composer',
|
||||
upstreamProvider?: IUpstreamProvider
|
||||
) {
|
||||
super();
|
||||
this.storage = storage;
|
||||
this.authManager = authManager;
|
||||
this.basePath = basePath;
|
||||
this.registryUrl = registryUrl;
|
||||
this.upstreamProvider = upstreamProvider || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract scope from Composer package name.
|
||||
* For Composer, vendor is the scope.
|
||||
* @example "symfony" from "symfony/console"
|
||||
*/
|
||||
private extractScope(vendorPackage: string): string | null {
|
||||
const slashIndex = vendorPackage.indexOf('/');
|
||||
if (slashIndex > 0) {
|
||||
return vendorPackage.substring(0, slashIndex);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get upstream for a specific request.
|
||||
* Calls the provider to resolve upstream config dynamically.
|
||||
*/
|
||||
private async getUpstreamForRequest(
|
||||
resource: string,
|
||||
resourceType: string,
|
||||
method: string,
|
||||
actor?: IRequestActor
|
||||
): Promise<ComposerUpstream | null> {
|
||||
if (!this.upstreamProvider) return null;
|
||||
|
||||
const config = await this.upstreamProvider.resolveUpstreamConfig({
|
||||
protocol: 'composer',
|
||||
resource,
|
||||
scope: this.extractScope(resource),
|
||||
actor,
|
||||
method,
|
||||
resourceType,
|
||||
});
|
||||
|
||||
if (!config?.enabled) return null;
|
||||
return new ComposerUpstream(config);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up resources (timers, connections, etc.)
|
||||
*/
|
||||
public destroy(): void {
|
||||
// No persistent upstream to clean up with dynamic provider
|
||||
}
|
||||
|
||||
public async init(): Promise<void> {
|
||||
// Composer registry initialization
|
||||
}
|
||||
|
||||
public getBasePath(): string {
|
||||
return this.basePath;
|
||||
}
|
||||
|
||||
public async handleRequest(context: IRequestContext): Promise<IResponse> {
|
||||
const path = context.path.replace(this.basePath, '');
|
||||
|
||||
// Extract token from Authorization header
|
||||
const authHeader = this.getAuthorizationHeader(context);
|
||||
let token: IAuthToken | null = null;
|
||||
|
||||
if (authHeader) {
|
||||
const tokenString = this.extractBearerToken(authHeader);
|
||||
if (tokenString) {
|
||||
token = await this.authManager.validateToken(tokenString, 'composer');
|
||||
} else {
|
||||
// Handle HTTP Basic Auth
|
||||
const basicCredentials = this.parseBasicAuthHeader(authHeader);
|
||||
if (basicCredentials) {
|
||||
const userId = await this.authManager.authenticate(basicCredentials);
|
||||
if (userId) {
|
||||
// Create temporary token for this request
|
||||
token = {
|
||||
type: 'composer',
|
||||
userId,
|
||||
scopes: ['composer:*:*:read'],
|
||||
readonly: true,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const actor: IRequestActor = this.buildRequestActor(context, token);
|
||||
|
||||
return this.storage.withContext({ protocol: 'composer', actor }, async () => {
|
||||
// Root packages.json
|
||||
if (path === '/packages.json' || path === '' || path === '/') {
|
||||
return this.handlePackagesJson();
|
||||
}
|
||||
|
||||
// Package metadata: /p2/{vendor}/{package}.json or /p2/{vendor}/{package}~dev.json
|
||||
const metadataMatch = path.match(/^\/p2\/([^\/]+\/[^\/]+?)(~dev)?\.json$/);
|
||||
if (metadataMatch) {
|
||||
const [, vendorPackage, devSuffix] = metadataMatch;
|
||||
const includeDev = !!devSuffix;
|
||||
return this.handlePackageMetadata(vendorPackage, includeDev, token, actor);
|
||||
}
|
||||
|
||||
// Package list: /packages/list.json?filter=vendor/*
|
||||
if (path.startsWith('/packages/list.json')) {
|
||||
const filter = context.query['filter'];
|
||||
return this.handlePackageList(filter, token);
|
||||
}
|
||||
|
||||
// Package ZIP download: /dists/{vendor}/{package}/{reference}.zip
|
||||
const distMatch = path.match(/^\/dists\/([^\/]+\/[^\/]+)\/([^\/]+)\.zip$/);
|
||||
if (distMatch) {
|
||||
const [, vendorPackage, reference] = distMatch;
|
||||
return this.handlePackageDownload(vendorPackage, reference, token);
|
||||
}
|
||||
|
||||
// Package upload: PUT /packages/{vendor}/{package}
|
||||
const uploadMatch = path.match(/^\/packages\/([^\/]+\/[^\/]+)$/);
|
||||
if (uploadMatch && context.method === 'PUT') {
|
||||
const vendorPackage = uploadMatch[1];
|
||||
return this.handlePackageUpload(vendorPackage, context.body, token);
|
||||
}
|
||||
|
||||
// Package delete: DELETE /packages/{vendor}/{package}
|
||||
if (uploadMatch && context.method === 'DELETE') {
|
||||
const vendorPackage = uploadMatch[1];
|
||||
return this.handlePackageDelete(vendorPackage, token);
|
||||
}
|
||||
|
||||
// Version delete: DELETE /packages/{vendor}/{package}/{version}
|
||||
const versionDeleteMatch = path.match(/^\/packages\/([^\/]+\/[^\/]+)\/(.+)$/);
|
||||
if (versionDeleteMatch && context.method === 'DELETE') {
|
||||
const [, vendorPackage, version] = versionDeleteMatch;
|
||||
return this.handleVersionDelete(vendorPackage, version, token);
|
||||
}
|
||||
|
||||
return {
|
||||
status: 404,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: { status: 'error', message: 'Not found' },
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
protected async checkPermission(
|
||||
token: IAuthToken | null,
|
||||
resource: string,
|
||||
action: string
|
||||
): Promise<boolean> {
|
||||
if (!token) return false;
|
||||
return this.authManager.authorize(token, `composer:package:${resource}`, action);
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// REQUEST HANDLERS
|
||||
// ========================================================================
|
||||
|
||||
private async handlePackagesJson(): Promise<IResponse> {
|
||||
const availablePackages = await this.storage.listComposerPackages();
|
||||
const packagesJson = generatePackagesJson(this.registryUrl, availablePackages);
|
||||
|
||||
return {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: packagesJson,
|
||||
};
|
||||
}
|
||||
|
||||
private async handlePackageMetadata(
|
||||
vendorPackage: string,
|
||||
includeDev: boolean,
|
||||
token: IAuthToken | null,
|
||||
actor?: IRequestActor
|
||||
): Promise<IResponse> {
|
||||
// Read operations are public, no authentication required
|
||||
let metadata = await this.storage.getComposerPackageMetadata(vendorPackage);
|
||||
|
||||
// Try upstream if not found locally
|
||||
if (!metadata) {
|
||||
const upstream = await this.getUpstreamForRequest(vendorPackage, 'metadata', 'GET', actor);
|
||||
if (upstream) {
|
||||
const [vendor, packageName] = vendorPackage.split('/');
|
||||
if (vendor && packageName) {
|
||||
const upstreamMetadata = includeDev
|
||||
? await upstream.fetchPackageDevMetadata(vendor, packageName)
|
||||
: await upstream.fetchPackageMetadata(vendor, packageName);
|
||||
|
||||
if (upstreamMetadata && upstreamMetadata.packages) {
|
||||
// Store upstream metadata locally
|
||||
metadata = {
|
||||
packages: upstreamMetadata.packages,
|
||||
lastModified: new Date().toUTCString(),
|
||||
};
|
||||
await this.storage.putComposerPackageMetadata(vendorPackage, metadata);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!metadata) {
|
||||
return {
|
||||
status: 404,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: { status: 'error', message: 'Package not found' },
|
||||
};
|
||||
}
|
||||
|
||||
// Filter dev versions if needed
|
||||
let packages = metadata.packages[vendorPackage] || [];
|
||||
if (!includeDev) {
|
||||
packages = packages.filter((pkg: IComposerPackage) =>
|
||||
!pkg.version.includes('dev') && !pkg.version.includes('alpha') && !pkg.version.includes('beta')
|
||||
);
|
||||
}
|
||||
|
||||
const response: IComposerPackageMetadata = {
|
||||
minified: 'composer/2.0',
|
||||
packages: {
|
||||
[vendorPackage]: packages,
|
||||
},
|
||||
};
|
||||
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Last-Modified': metadata.lastModified || new Date().toUTCString(),
|
||||
},
|
||||
body: response,
|
||||
};
|
||||
}
|
||||
|
||||
private async handlePackageList(
|
||||
filter: string | undefined,
|
||||
token: IAuthToken | null
|
||||
): Promise<IResponse> {
|
||||
let packages = await this.storage.listComposerPackages();
|
||||
|
||||
// Apply filter if provided
|
||||
if (filter) {
|
||||
const regex = new RegExp('^' + filter.replace(/\*/g, '.*') + '$');
|
||||
packages = packages.filter(pkg => regex.test(pkg));
|
||||
}
|
||||
|
||||
return {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: { packageNames: packages },
|
||||
};
|
||||
}
|
||||
|
||||
private async handlePackageDownload(
|
||||
vendorPackage: string,
|
||||
reference: string,
|
||||
token: IAuthToken | null
|
||||
): Promise<IResponse> {
|
||||
// Read operations are public, no authentication required
|
||||
const streamResult = await this.storage.getComposerPackageZipStream(vendorPackage, reference);
|
||||
|
||||
if (!streamResult) {
|
||||
return {
|
||||
status: 404,
|
||||
headers: {},
|
||||
body: { status: 'error', message: 'Package file not found' },
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/zip',
|
||||
'Content-Length': streamResult.size.toString(),
|
||||
'Content-Disposition': `attachment; filename="${reference}.zip"`,
|
||||
},
|
||||
body: streamResult.stream,
|
||||
};
|
||||
}
|
||||
|
||||
private async handlePackageUpload(
|
||||
vendorPackage: string,
|
||||
body: any,
|
||||
token: IAuthToken | null
|
||||
): Promise<IResponse> {
|
||||
// Check write permission
|
||||
if (!await this.checkPermission(token, vendorPackage, 'write')) {
|
||||
return {
|
||||
status: 401,
|
||||
headers: {},
|
||||
body: { status: 'error', message: 'Write permission required' },
|
||||
};
|
||||
}
|
||||
|
||||
if (!body || !isBinaryData(body)) {
|
||||
return {
|
||||
status: 400,
|
||||
headers: {},
|
||||
body: { status: 'error', message: 'ZIP file required' },
|
||||
};
|
||||
}
|
||||
|
||||
// Convert to Buffer for ZIP processing
|
||||
const zipData = toBuffer(body);
|
||||
|
||||
// Extract and validate composer.json from ZIP
|
||||
const composerJson = await extractComposerJsonFromZip(zipData);
|
||||
if (!composerJson || !validateComposerJson(composerJson)) {
|
||||
return {
|
||||
status: 400,
|
||||
headers: {},
|
||||
body: { status: 'error', message: 'Invalid composer.json in ZIP' },
|
||||
};
|
||||
}
|
||||
|
||||
// Verify package name matches
|
||||
if (composerJson.name !== vendorPackage) {
|
||||
return {
|
||||
status: 400,
|
||||
headers: {},
|
||||
body: { status: 'error', message: 'Package name mismatch' },
|
||||
};
|
||||
}
|
||||
|
||||
const version = composerJson.version;
|
||||
if (!version) {
|
||||
return {
|
||||
status: 400,
|
||||
headers: {},
|
||||
body: { status: 'error', message: 'Version required in composer.json' },
|
||||
};
|
||||
}
|
||||
|
||||
// Calculate SHA-1 hash
|
||||
const shasum = await calculateSha1(zipData);
|
||||
|
||||
// Generate reference (use version or commit hash)
|
||||
const reference = composerJson.source?.reference || version.replace(/[^a-zA-Z0-9.-]/g, '-');
|
||||
|
||||
// Store ZIP file
|
||||
await this.storage.putComposerPackageZip(vendorPackage, reference, zipData);
|
||||
|
||||
// Get or create metadata
|
||||
let metadata = await this.storage.getComposerPackageMetadata(vendorPackage);
|
||||
if (!metadata) {
|
||||
metadata = {
|
||||
packages: {
|
||||
[vendorPackage]: [],
|
||||
},
|
||||
lastModified: new Date().toUTCString(),
|
||||
};
|
||||
}
|
||||
|
||||
// Build package entry
|
||||
const packageEntry: IComposerPackage = {
|
||||
...composerJson,
|
||||
version_normalized: normalizeVersion(version),
|
||||
dist: {
|
||||
type: 'zip',
|
||||
url: `${this.registryUrl}/dists/${vendorPackage}/${reference}.zip`,
|
||||
reference,
|
||||
shasum,
|
||||
},
|
||||
time: new Date().toISOString(),
|
||||
};
|
||||
|
||||
// Add to metadata (check if version already exists)
|
||||
const packages = metadata.packages[vendorPackage] || [];
|
||||
const existingIndex = packages.findIndex((p: IComposerPackage) => p.version === version);
|
||||
|
||||
if (existingIndex >= 0) {
|
||||
return {
|
||||
status: 409,
|
||||
headers: {},
|
||||
body: { status: 'error', message: 'Version already exists' },
|
||||
};
|
||||
}
|
||||
|
||||
packages.push(packageEntry);
|
||||
|
||||
// Sort by version
|
||||
const sortedVersions = sortVersions(packages.map((p: IComposerPackage) => p.version));
|
||||
packages.sort((a: IComposerPackage, b: IComposerPackage) => {
|
||||
return sortedVersions.indexOf(a.version) - sortedVersions.indexOf(b.version);
|
||||
});
|
||||
|
||||
metadata.packages[vendorPackage] = packages;
|
||||
metadata.lastModified = new Date().toUTCString();
|
||||
|
||||
// Store updated metadata
|
||||
await this.storage.putComposerPackageMetadata(vendorPackage, metadata);
|
||||
|
||||
return {
|
||||
status: 201,
|
||||
headers: {},
|
||||
body: {
|
||||
status: 'success',
|
||||
message: 'Package uploaded successfully',
|
||||
package: vendorPackage,
|
||||
version,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private async handlePackageDelete(
|
||||
vendorPackage: string,
|
||||
token: IAuthToken | null
|
||||
): Promise<IResponse> {
|
||||
// Check delete permission
|
||||
if (!await this.checkPermission(token, vendorPackage, 'delete')) {
|
||||
return {
|
||||
status: 401,
|
||||
headers: {},
|
||||
body: { status: 'error', message: 'Delete permission required' },
|
||||
};
|
||||
}
|
||||
|
||||
const metadata = await this.storage.getComposerPackageMetadata(vendorPackage);
|
||||
if (!metadata) {
|
||||
return {
|
||||
status: 404,
|
||||
headers: {},
|
||||
body: { status: 'error', message: 'Package not found' },
|
||||
};
|
||||
}
|
||||
|
||||
// Delete all ZIP files
|
||||
const packages = metadata.packages[vendorPackage] || [];
|
||||
for (const pkg of packages) {
|
||||
if (pkg.dist?.reference) {
|
||||
await this.storage.deleteComposerPackageZip(vendorPackage, pkg.dist.reference);
|
||||
}
|
||||
}
|
||||
|
||||
// Delete metadata
|
||||
await this.storage.deleteComposerPackageMetadata(vendorPackage);
|
||||
|
||||
return {
|
||||
status: 204,
|
||||
headers: {},
|
||||
body: null,
|
||||
};
|
||||
}
|
||||
|
||||
private async handleVersionDelete(
|
||||
vendorPackage: string,
|
||||
version: string,
|
||||
token: IAuthToken | null
|
||||
): Promise<IResponse> {
|
||||
// Check delete permission
|
||||
if (!await this.checkPermission(token, vendorPackage, 'delete')) {
|
||||
return {
|
||||
status: 401,
|
||||
headers: {},
|
||||
body: { status: 'error', message: 'Delete permission required' },
|
||||
};
|
||||
}
|
||||
|
||||
const metadata = await this.storage.getComposerPackageMetadata(vendorPackage);
|
||||
if (!metadata) {
|
||||
return {
|
||||
status: 404,
|
||||
headers: {},
|
||||
body: { status: 'error', message: 'Package not found' },
|
||||
};
|
||||
}
|
||||
|
||||
const packages = metadata.packages[vendorPackage] || [];
|
||||
const versionIndex = packages.findIndex((p: IComposerPackage) => p.version === version);
|
||||
|
||||
if (versionIndex === -1) {
|
||||
return {
|
||||
status: 404,
|
||||
headers: {},
|
||||
body: { status: 'error', message: 'Version not found' },
|
||||
};
|
||||
}
|
||||
|
||||
// Delete ZIP file
|
||||
const pkg = packages[versionIndex];
|
||||
if (pkg.dist?.reference) {
|
||||
await this.storage.deleteComposerPackageZip(vendorPackage, pkg.dist.reference);
|
||||
}
|
||||
|
||||
// Remove from metadata
|
||||
packages.splice(versionIndex, 1);
|
||||
metadata.packages[vendorPackage] = packages;
|
||||
metadata.lastModified = new Date().toUTCString();
|
||||
|
||||
// Save updated metadata
|
||||
await this.storage.putComposerPackageMetadata(vendorPackage, metadata);
|
||||
|
||||
return {
|
||||
status: 204,
|
||||
headers: {},
|
||||
body: null,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
import * as plugins from '../plugins.js';
|
||||
import { BaseUpstream } from '../upstream/classes.baseupstream.js';
|
||||
import type {
|
||||
IProtocolUpstreamConfig,
|
||||
IUpstreamFetchContext,
|
||||
IUpstreamRegistryConfig,
|
||||
} from '../upstream/interfaces.upstream.js';
|
||||
|
||||
/**
|
||||
* Composer-specific upstream implementation.
|
||||
*
|
||||
* Handles:
|
||||
* - Package metadata fetching (packages.json, provider-includes)
|
||||
* - Package version metadata (p2/{vendor}/{package}.json)
|
||||
* - Dist file (zip) proxying
|
||||
* - Packagist v2 API support
|
||||
*/
|
||||
export class ComposerUpstream extends BaseUpstream {
|
||||
protected readonly protocolName = 'composer';
|
||||
|
||||
constructor(
|
||||
config: IProtocolUpstreamConfig,
|
||||
logger?: plugins.smartlog.Smartlog,
|
||||
) {
|
||||
super(config, logger);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch the root packages.json from upstream.
|
||||
*/
|
||||
public async fetchPackagesJson(): Promise<any | null> {
|
||||
const context: IUpstreamFetchContext = {
|
||||
protocol: 'composer',
|
||||
resource: '*',
|
||||
resourceType: 'root',
|
||||
path: '/packages.json',
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'accept': 'application/json',
|
||||
},
|
||||
query: {},
|
||||
};
|
||||
|
||||
const result = await this.fetch(context);
|
||||
|
||||
if (!result || !result.success) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (Buffer.isBuffer(result.body)) {
|
||||
return JSON.parse(result.body.toString('utf8'));
|
||||
}
|
||||
|
||||
return result.body;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch package metadata using v2 API (p2/{vendor}/{package}.json).
|
||||
*/
|
||||
public async fetchPackageMetadata(vendor: string, packageName: string): Promise<any | null> {
|
||||
const fullName = `${vendor}/${packageName}`;
|
||||
const path = `/p2/${vendor}/${packageName}.json`;
|
||||
|
||||
const context: IUpstreamFetchContext = {
|
||||
protocol: 'composer',
|
||||
resource: fullName,
|
||||
resourceType: 'metadata',
|
||||
path,
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'accept': 'application/json',
|
||||
},
|
||||
query: {},
|
||||
};
|
||||
|
||||
const result = await this.fetch(context);
|
||||
|
||||
if (!result || !result.success) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (Buffer.isBuffer(result.body)) {
|
||||
return JSON.parse(result.body.toString('utf8'));
|
||||
}
|
||||
|
||||
return result.body;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch package metadata with dev versions (p2/{vendor}/{package}~dev.json).
|
||||
*/
|
||||
public async fetchPackageDevMetadata(vendor: string, packageName: string): Promise<any | null> {
|
||||
const fullName = `${vendor}/${packageName}`;
|
||||
const path = `/p2/${vendor}/${packageName}~dev.json`;
|
||||
|
||||
const context: IUpstreamFetchContext = {
|
||||
protocol: 'composer',
|
||||
resource: fullName,
|
||||
resourceType: 'metadata-dev',
|
||||
path,
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'accept': 'application/json',
|
||||
},
|
||||
query: {},
|
||||
};
|
||||
|
||||
const result = await this.fetch(context);
|
||||
|
||||
if (!result || !result.success) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (Buffer.isBuffer(result.body)) {
|
||||
return JSON.parse(result.body.toString('utf8'));
|
||||
}
|
||||
|
||||
return result.body;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a provider-includes file.
|
||||
*/
|
||||
public async fetchProviderIncludes(path: string): Promise<any | null> {
|
||||
const context: IUpstreamFetchContext = {
|
||||
protocol: 'composer',
|
||||
resource: '*',
|
||||
resourceType: 'provider',
|
||||
path: path.startsWith('/') ? path : `/${path}`,
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'accept': 'application/json',
|
||||
},
|
||||
query: {},
|
||||
};
|
||||
|
||||
const result = await this.fetch(context);
|
||||
|
||||
if (!result || !result.success) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (Buffer.isBuffer(result.body)) {
|
||||
return JSON.parse(result.body.toString('utf8'));
|
||||
}
|
||||
|
||||
return result.body;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a dist file (zip) from upstream.
|
||||
*/
|
||||
public async fetchDist(url: string): Promise<Buffer | null> {
|
||||
// Parse the URL to get the path
|
||||
let path: string;
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
path = parsed.pathname;
|
||||
} catch {
|
||||
path = url;
|
||||
}
|
||||
|
||||
const context: IUpstreamFetchContext = {
|
||||
protocol: 'composer',
|
||||
resource: '*',
|
||||
resourceType: 'dist',
|
||||
path,
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'accept': 'application/zip, application/octet-stream',
|
||||
},
|
||||
query: {},
|
||||
};
|
||||
|
||||
const result = await this.fetch(context);
|
||||
|
||||
if (!result || !result.success) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return Buffer.isBuffer(result.body) ? result.body : Buffer.from(result.body);
|
||||
}
|
||||
|
||||
/**
|
||||
* Override URL building for Composer-specific handling.
|
||||
*/
|
||||
protected buildUpstreamUrl(
|
||||
upstream: IUpstreamRegistryConfig,
|
||||
context: IUpstreamFetchContext,
|
||||
): string {
|
||||
let baseUrl = upstream.url;
|
||||
|
||||
// Remove trailing slash
|
||||
if (baseUrl.endsWith('/')) {
|
||||
baseUrl = baseUrl.slice(0, -1);
|
||||
}
|
||||
|
||||
return `${baseUrl}${context.path}`;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
/**
|
||||
* Composer Registry Helper Functions
|
||||
*/
|
||||
|
||||
import type { IComposerPackage } from './interfaces.composer.js';
|
||||
|
||||
/**
|
||||
* Normalize version string to Composer format
|
||||
* Example: "1.0.0" -> "1.0.0.0", "v2.3.1" -> "2.3.1.0"
|
||||
*/
|
||||
export function normalizeVersion(version: string): string {
|
||||
// Remove 'v' prefix if present
|
||||
let normalized = version.replace(/^v/i, '');
|
||||
|
||||
// Handle special versions (dev, alpha, beta, rc)
|
||||
if (normalized.includes('dev') || normalized.includes('alpha') || normalized.includes('beta') || normalized.includes('RC')) {
|
||||
// For dev versions, just return as-is with .0 appended if needed
|
||||
const parts = normalized.split(/[-+]/)[0].split('.');
|
||||
while (parts.length < 4) {
|
||||
parts.push('0');
|
||||
}
|
||||
return parts.slice(0, 4).join('.');
|
||||
}
|
||||
|
||||
// Split by dots
|
||||
const parts = normalized.split('.');
|
||||
|
||||
// Ensure 4 parts (major.minor.patch.build)
|
||||
while (parts.length < 4) {
|
||||
parts.push('0');
|
||||
}
|
||||
|
||||
return parts.slice(0, 4).join('.');
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate composer.json structure
|
||||
*/
|
||||
export function validateComposerJson(composerJson: any): boolean {
|
||||
return !!(
|
||||
composerJson &&
|
||||
typeof composerJson.name === 'string' &&
|
||||
composerJson.name.includes('/') &&
|
||||
(composerJson.version || composerJson.require)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract composer.json from ZIP buffer
|
||||
*/
|
||||
export async function extractComposerJsonFromZip(zipBuffer: Buffer): Promise<any | null> {
|
||||
try {
|
||||
const AdmZip = (await import('adm-zip')).default;
|
||||
const zip = new AdmZip(zipBuffer);
|
||||
const entries = zip.getEntries();
|
||||
|
||||
// Look for composer.json in root or first-level directory
|
||||
for (const entry of entries) {
|
||||
if (entry.entryName.endsWith('composer.json')) {
|
||||
const parts = entry.entryName.split('/');
|
||||
if (parts.length <= 2) { // Root or first-level dir
|
||||
const content = entry.getData().toString('utf-8');
|
||||
return JSON.parse(content);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
} catch (error) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate SHA-1 hash for ZIP file
|
||||
*/
|
||||
export async function calculateSha1(data: Buffer): Promise<string> {
|
||||
const crypto = await import('crypto');
|
||||
return crypto.createHash('sha1').update(data).digest('hex');
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse vendor/package format
|
||||
*/
|
||||
export function parseVendorPackage(name: string): { vendor: string; package: string } | null {
|
||||
const parts = name.split('/');
|
||||
if (parts.length !== 2) {
|
||||
return null;
|
||||
}
|
||||
return { vendor: parts[0], package: parts[1] };
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate packages.json root repository file
|
||||
*/
|
||||
export function generatePackagesJson(
|
||||
registryUrl: string,
|
||||
availablePackages: string[]
|
||||
): any {
|
||||
return {
|
||||
'metadata-url': `${registryUrl}/p2/%package%.json`,
|
||||
'available-packages': availablePackages,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Sort versions in semantic version order
|
||||
*/
|
||||
export function sortVersions(versions: string[]): string[] {
|
||||
return versions.sort((a, b) => {
|
||||
const aParts = a.replace(/^v/i, '').split(/[.-]/).map(part => {
|
||||
const num = parseInt(part, 10);
|
||||
return isNaN(num) ? part : num;
|
||||
});
|
||||
const bParts = b.replace(/^v/i, '').split(/[.-]/).map(part => {
|
||||
const num = parseInt(part, 10);
|
||||
return isNaN(num) ? part : num;
|
||||
});
|
||||
|
||||
for (let i = 0; i < Math.max(aParts.length, bParts.length); i++) {
|
||||
const aPart = aParts[i] ?? 0;
|
||||
const bPart = bParts[i] ?? 0;
|
||||
|
||||
// Compare numbers numerically, strings lexicographically
|
||||
if (typeof aPart === 'number' && typeof bPart === 'number') {
|
||||
if (aPart !== bPart) {
|
||||
return aPart - bPart;
|
||||
}
|
||||
} else {
|
||||
const aStr = String(aPart);
|
||||
const bStr = String(bPart);
|
||||
if (aStr !== bStr) {
|
||||
return aStr.localeCompare(bStr);
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
/**
|
||||
* Composer Registry Module
|
||||
* Export all public interfaces, classes, and helpers
|
||||
*/
|
||||
|
||||
export { ComposerRegistry } from './classes.composerregistry.js';
|
||||
export { ComposerUpstream } from './classes.composerupstream.js';
|
||||
export * from './interfaces.composer.js';
|
||||
export * from './helpers.composer.js';
|
||||
@@ -0,0 +1,111 @@
|
||||
/**
|
||||
* Composer Registry Type Definitions
|
||||
* Compliant with Composer v2 repository API
|
||||
*/
|
||||
|
||||
/**
|
||||
* Composer package metadata
|
||||
*/
|
||||
export interface IComposerPackage {
|
||||
name: string; // vendor/package-name
|
||||
version: string; // 1.0.0
|
||||
version_normalized: string; // 1.0.0.0
|
||||
type?: string; // library, project, metapackage
|
||||
description?: string;
|
||||
keywords?: string[];
|
||||
homepage?: string;
|
||||
license?: string[];
|
||||
authors?: IComposerAuthor[];
|
||||
require?: Record<string, string>;
|
||||
'require-dev'?: Record<string, string>;
|
||||
suggest?: Record<string, string>;
|
||||
provide?: Record<string, string>;
|
||||
conflict?: Record<string, string>;
|
||||
replace?: Record<string, string>;
|
||||
autoload?: IComposerAutoload;
|
||||
'autoload-dev'?: IComposerAutoload;
|
||||
dist?: IComposerDist;
|
||||
source?: IComposerSource;
|
||||
time?: string; // ISO 8601 timestamp
|
||||
support?: Record<string, string>;
|
||||
funding?: IComposerFunding[];
|
||||
extra?: Record<string, any>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Author information
|
||||
*/
|
||||
export interface IComposerAuthor {
|
||||
name: string;
|
||||
email?: string;
|
||||
homepage?: string;
|
||||
role?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* PSR-4/PSR-0 autoloading configuration
|
||||
*/
|
||||
export interface IComposerAutoload {
|
||||
'psr-4'?: Record<string, string | string[]>;
|
||||
'psr-0'?: Record<string, string | string[]>;
|
||||
classmap?: string[];
|
||||
files?: string[];
|
||||
'exclude-from-classmap'?: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Distribution information (ZIP download)
|
||||
*/
|
||||
export interface IComposerDist {
|
||||
type: 'zip' | 'tar' | 'phar';
|
||||
url: string;
|
||||
reference?: string; // commit hash or tag
|
||||
shasum?: string; // SHA-1 hash
|
||||
}
|
||||
|
||||
/**
|
||||
* Source repository information
|
||||
*/
|
||||
export interface IComposerSource {
|
||||
type: 'git' | 'svn' | 'hg';
|
||||
url: string;
|
||||
reference: string; // commit hash, branch, or tag
|
||||
}
|
||||
|
||||
/**
|
||||
* Funding information
|
||||
*/
|
||||
export interface IComposerFunding {
|
||||
type: string; // github, patreon, etc.
|
||||
url: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Repository metadata (packages.json)
|
||||
*/
|
||||
export interface IComposerRepository {
|
||||
packages?: Record<string, Record<string, IComposerPackage>>;
|
||||
'metadata-url'?: string; // /p2/%package%.json
|
||||
'available-packages'?: string[];
|
||||
'available-package-patterns'?: string[];
|
||||
'providers-url'?: string;
|
||||
'notify-batch'?: string;
|
||||
minified?: string; // "composer/2.0"
|
||||
}
|
||||
|
||||
/**
|
||||
* Package metadata response (/p2/vendor/package.json)
|
||||
*/
|
||||
export interface IComposerPackageMetadata {
|
||||
packages: Record<string, IComposerPackage[]>;
|
||||
minified?: string;
|
||||
lastModified?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Error structure
|
||||
*/
|
||||
export interface IComposerError {
|
||||
status: string;
|
||||
message: string;
|
||||
}
|
||||
+185
-258
@@ -1,21 +1,79 @@
|
||||
import type { IAuthConfig, IAuthToken, ICredentials, TRegistryProtocol } from './interfaces.core.js';
|
||||
import type { IAuthProvider, ITokenOptions } from './interfaces.auth.js';
|
||||
import { DefaultAuthProvider } from './classes.defaultauthprovider.js';
|
||||
|
||||
/**
|
||||
* Unified authentication manager for all registry protocols
|
||||
* Handles both NPM UUID tokens and OCI JWT tokens
|
||||
* Unified authentication manager for all registry protocols.
|
||||
* Delegates to a pluggable IAuthProvider for actual auth operations.
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* // Use default in-memory provider
|
||||
* const auth = new AuthManager(config);
|
||||
*
|
||||
* // Use custom provider (LDAP, OAuth, etc.)
|
||||
* const auth = new AuthManager(config, new LdapAuthProvider(ldapClient));
|
||||
* ```
|
||||
*/
|
||||
export class AuthManager {
|
||||
private tokenStore: Map<string, IAuthToken> = new Map();
|
||||
private userCredentials: Map<string, string> = new Map(); // username -> password hash (mock)
|
||||
private provider: IAuthProvider;
|
||||
|
||||
constructor(private config: IAuthConfig) {}
|
||||
constructor(
|
||||
private config: IAuthConfig,
|
||||
provider?: IAuthProvider
|
||||
) {
|
||||
// Use provided provider or default in-memory implementation
|
||||
this.provider = provider || new DefaultAuthProvider(config);
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the auth manager
|
||||
*/
|
||||
public async init(): Promise<void> {
|
||||
// Initialize token store (in-memory for now)
|
||||
// In production, this could be Redis or a database
|
||||
if (this.provider.init) {
|
||||
await this.provider.init();
|
||||
}
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// UNIFIED AUTHENTICATION (Delegated to Provider)
|
||||
// ========================================================================
|
||||
|
||||
/**
|
||||
* Authenticate user credentials
|
||||
* @param credentials - Username and password
|
||||
* @returns User ID or null
|
||||
*/
|
||||
public async authenticate(credentials: ICredentials): Promise<string | null> {
|
||||
return this.provider.authenticate(credentials);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate any token (NPM, Maven, OCI, PyPI, RubyGems, Composer, Cargo)
|
||||
* @param tokenString - Token string (UUID or JWT)
|
||||
* @param protocol - Expected protocol type (optional, improves performance)
|
||||
* @returns Auth token object or null
|
||||
*/
|
||||
public async validateToken(
|
||||
tokenString: string,
|
||||
protocol?: TRegistryProtocol
|
||||
): Promise<IAuthToken | null> {
|
||||
return this.provider.validateToken(tokenString, protocol);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if token has permission for an action
|
||||
* @param token - Auth token (or null for anonymous)
|
||||
* @param resource - Resource being accessed (e.g., "npm:package:foo")
|
||||
* @param action - Action being performed (read, write, push, pull, delete)
|
||||
* @returns true if authorized
|
||||
*/
|
||||
public async authorize(
|
||||
token: IAuthToken | null,
|
||||
resource: string,
|
||||
action: string
|
||||
): Promise<boolean> {
|
||||
return this.provider.authorize(token, resource, action);
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
@@ -32,20 +90,7 @@ export class AuthManager {
|
||||
if (!this.config.npmTokens.enabled) {
|
||||
throw new Error('NPM tokens are not enabled');
|
||||
}
|
||||
|
||||
const token = this.generateUuid();
|
||||
const authToken: IAuthToken = {
|
||||
type: 'npm',
|
||||
userId,
|
||||
scopes: readonly ? ['npm:*:*:read'] : ['npm:*:*:*'],
|
||||
readonly,
|
||||
metadata: {
|
||||
created: new Date().toISOString(),
|
||||
},
|
||||
};
|
||||
|
||||
this.tokenStore.set(token, authToken);
|
||||
return token;
|
||||
return this.provider.createToken(userId, 'npm', { readonly });
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -54,22 +99,7 @@ export class AuthManager {
|
||||
* @returns Auth token object or null
|
||||
*/
|
||||
public async validateNpmToken(token: string): Promise<IAuthToken | null> {
|
||||
if (!this.isValidUuid(token)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const authToken = this.tokenStore.get(token);
|
||||
if (!authToken || authToken.type !== 'npm') {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Check expiration if set
|
||||
if (authToken.expiresAt && authToken.expiresAt < new Date()) {
|
||||
this.tokenStore.delete(token);
|
||||
return null;
|
||||
}
|
||||
|
||||
return authToken;
|
||||
return this.provider.validateToken(token, 'npm');
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -77,7 +107,7 @@ export class AuthManager {
|
||||
* @param token - NPM UUID token
|
||||
*/
|
||||
public async revokeNpmToken(token: string): Promise<void> {
|
||||
this.tokenStore.delete(token);
|
||||
return this.provider.revokeToken(token);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -89,20 +119,12 @@ export class AuthManager {
|
||||
key: string;
|
||||
readonly: boolean;
|
||||
created: string;
|
||||
protocol?: TRegistryProtocol;
|
||||
}>> {
|
||||
const tokens: Array<{key: string; readonly: boolean; created: string}> = [];
|
||||
|
||||
for (const [token, authToken] of this.tokenStore.entries()) {
|
||||
if (authToken.userId === userId) {
|
||||
tokens.push({
|
||||
key: this.hashToken(token),
|
||||
readonly: authToken.readonly || false,
|
||||
created: authToken.metadata?.created || 'unknown',
|
||||
});
|
||||
if (this.provider.listUserTokens) {
|
||||
return this.provider.listUserTokens(userId);
|
||||
}
|
||||
}
|
||||
|
||||
return tokens;
|
||||
return [];
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
@@ -121,24 +143,10 @@ export class AuthManager {
|
||||
scopes: string[],
|
||||
expiresIn: number = 3600
|
||||
): Promise<string> {
|
||||
if (!this.config.ociTokens.enabled) {
|
||||
if (!this.config.ociTokens?.enabled) {
|
||||
throw new Error('OCI tokens are not enabled');
|
||||
}
|
||||
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
const payload = {
|
||||
iss: this.config.ociTokens.realm,
|
||||
sub: userId,
|
||||
aud: this.config.ociTokens.service,
|
||||
exp: now + expiresIn,
|
||||
nbf: now,
|
||||
iat: now,
|
||||
access: this.scopesToOciAccess(scopes),
|
||||
};
|
||||
|
||||
// In production, use proper JWT library with signing
|
||||
// For now, return JSON string (mock JWT)
|
||||
return JSON.stringify(payload);
|
||||
return this.provider.createToken(userId, 'oci', { scopes, expiresIn });
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -147,242 +155,161 @@ export class AuthManager {
|
||||
* @returns Auth token object or null
|
||||
*/
|
||||
public async validateOciToken(jwt: string): Promise<IAuthToken | null> {
|
||||
try {
|
||||
// In production, verify JWT signature
|
||||
const payload = JSON.parse(jwt);
|
||||
|
||||
// Check expiration
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
if (payload.exp && payload.exp < now) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Convert to unified token format
|
||||
const scopes = this.ociAccessToScopes(payload.access || []);
|
||||
|
||||
return {
|
||||
type: 'oci',
|
||||
userId: payload.sub,
|
||||
scopes,
|
||||
expiresAt: payload.exp ? new Date(payload.exp * 1000) : undefined,
|
||||
metadata: {
|
||||
iss: payload.iss,
|
||||
aud: payload.aud,
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
return null;
|
||||
}
|
||||
return this.provider.validateToken(jwt, 'oci');
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// UNIFIED AUTHENTICATION
|
||||
// MAVEN AUTHENTICATION
|
||||
// ========================================================================
|
||||
|
||||
/**
|
||||
* Authenticate user credentials
|
||||
* @param credentials - Username and password
|
||||
* @returns User ID or null
|
||||
* Create a Maven token
|
||||
* @param userId - User ID
|
||||
* @param readonly - Whether the token is readonly
|
||||
* @returns Maven UUID token
|
||||
*/
|
||||
public async authenticate(credentials: ICredentials): Promise<string | null> {
|
||||
// Mock authentication - in production, verify against database
|
||||
const storedPassword = this.userCredentials.get(credentials.username);
|
||||
|
||||
if (!storedPassword) {
|
||||
// Auto-register for testing (remove in production)
|
||||
this.userCredentials.set(credentials.username, credentials.password);
|
||||
return credentials.username;
|
||||
}
|
||||
|
||||
if (storedPassword === credentials.password) {
|
||||
return credentials.username;
|
||||
}
|
||||
|
||||
return null;
|
||||
public async createMavenToken(userId: string, readonly: boolean = false): Promise<string> {
|
||||
return this.provider.createToken(userId, 'maven', { readonly });
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate any token (NPM or OCI)
|
||||
* @param tokenString - Token string (UUID or JWT)
|
||||
* @param protocol - Expected protocol type
|
||||
* Validate a Maven token
|
||||
* @param token - Maven UUID token
|
||||
* @returns Auth token object or null
|
||||
*/
|
||||
public async validateToken(
|
||||
tokenString: string,
|
||||
protocol?: TRegistryProtocol
|
||||
): Promise<IAuthToken | null> {
|
||||
// Try NPM token first (UUID format)
|
||||
if (this.isValidUuid(tokenString)) {
|
||||
const npmToken = await this.validateNpmToken(tokenString);
|
||||
if (npmToken && (!protocol || protocol === 'npm')) {
|
||||
return npmToken;
|
||||
}
|
||||
}
|
||||
|
||||
// Try OCI JWT
|
||||
const ociToken = await this.validateOciToken(tokenString);
|
||||
if (ociToken && (!protocol || protocol === 'oci')) {
|
||||
return ociToken;
|
||||
}
|
||||
|
||||
return null;
|
||||
public async validateMavenToken(token: string): Promise<IAuthToken | null> {
|
||||
return this.provider.validateToken(token, 'maven');
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if token has permission for an action
|
||||
* @param token - Auth token
|
||||
* @param resource - Resource being accessed (e.g., "package:foo" or "repository:bar")
|
||||
* @param action - Action being performed (read, write, push, pull, delete)
|
||||
* @returns true if authorized
|
||||
* Revoke a Maven token
|
||||
* @param token - Maven UUID token
|
||||
*/
|
||||
public async authorize(
|
||||
token: IAuthToken | null,
|
||||
resource: string,
|
||||
action: string
|
||||
): Promise<boolean> {
|
||||
if (!token) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check readonly flag
|
||||
if (token.readonly && ['write', 'push', 'delete'].includes(action)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check scopes
|
||||
for (const scope of token.scopes) {
|
||||
if (this.matchesScope(scope, resource, action)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
public async revokeMavenToken(token: string): Promise<void> {
|
||||
return this.provider.revokeToken(token);
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// HELPER METHODS
|
||||
// COMPOSER TOKEN MANAGEMENT
|
||||
// ========================================================================
|
||||
|
||||
/**
|
||||
* Check if a scope matches a resource and action
|
||||
* Scope format: "{protocol}:{type}:{name}:{action}"
|
||||
* Examples:
|
||||
* - "npm:*:*" - All NPM access
|
||||
* - "npm:package:foo:*" - All actions on package foo
|
||||
* - "npm:package:foo:read" - Read-only on package foo
|
||||
* - "oci:repository:*:pull" - Pull from any OCI repo
|
||||
* Create a Composer token
|
||||
* @param userId - User ID
|
||||
* @param readonly - Whether the token is readonly
|
||||
* @returns Composer UUID token
|
||||
*/
|
||||
private matchesScope(scope: string, resource: string, action: string): boolean {
|
||||
const scopeParts = scope.split(':');
|
||||
const resourceParts = resource.split(':');
|
||||
|
||||
// Scope must have at least protocol:type:name:action
|
||||
if (scopeParts.length < 4) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const [scopeProtocol, scopeType, scopeName, scopeAction] = scopeParts;
|
||||
const [resourceProtocol, resourceType, resourceName] = resourceParts;
|
||||
|
||||
// Check protocol
|
||||
if (scopeProtocol !== '*' && scopeProtocol !== resourceProtocol) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check type
|
||||
if (scopeType !== '*' && scopeType !== resourceType) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check name
|
||||
if (scopeName !== '*' && scopeName !== resourceName) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check action
|
||||
if (scopeAction !== '*' && scopeAction !== action) {
|
||||
// Map action aliases
|
||||
const actionAliases: Record<string, string[]> = {
|
||||
read: ['pull', 'get'],
|
||||
write: ['push', 'put', 'post'],
|
||||
};
|
||||
|
||||
const aliases = actionAliases[scopeAction] || [];
|
||||
if (!aliases.includes(action)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
public async createComposerToken(userId: string, readonly: boolean = false): Promise<string> {
|
||||
return this.provider.createToken(userId, 'composer', { readonly });
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert unified scopes to OCI access array
|
||||
* Validate a Composer token
|
||||
* @param token - Composer UUID token
|
||||
* @returns Auth token object or null
|
||||
*/
|
||||
private scopesToOciAccess(scopes: string[]): Array<{
|
||||
type: string;
|
||||
name: string;
|
||||
actions: string[];
|
||||
}> {
|
||||
const access: Array<{type: string; name: string; actions: string[]}> = [];
|
||||
|
||||
for (const scope of scopes) {
|
||||
const parts = scope.split(':');
|
||||
if (parts.length >= 4 && parts[0] === 'oci') {
|
||||
access.push({
|
||||
type: parts[1],
|
||||
name: parts[2],
|
||||
actions: [parts[3]],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return access;
|
||||
public async validateComposerToken(token: string): Promise<IAuthToken | null> {
|
||||
return this.provider.validateToken(token, 'composer');
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert OCI access array to unified scopes
|
||||
* Revoke a Composer token
|
||||
* @param token - Composer UUID token
|
||||
*/
|
||||
private ociAccessToScopes(access: Array<{
|
||||
type: string;
|
||||
name: string;
|
||||
actions: string[];
|
||||
}>): string[] {
|
||||
const scopes: string[] = [];
|
||||
|
||||
for (const item of access) {
|
||||
for (const action of item.actions) {
|
||||
scopes.push(`oci:${item.type}:${item.name}:${action}`);
|
||||
}
|
||||
public async revokeComposerToken(token: string): Promise<void> {
|
||||
return this.provider.revokeToken(token);
|
||||
}
|
||||
|
||||
return scopes;
|
||||
// ========================================================================
|
||||
// CARGO TOKEN MANAGEMENT
|
||||
// ========================================================================
|
||||
|
||||
/**
|
||||
* Create a Cargo token
|
||||
* @param userId - User ID
|
||||
* @param readonly - Whether the token is readonly
|
||||
* @returns Cargo UUID token
|
||||
*/
|
||||
public async createCargoToken(userId: string, readonly: boolean = false): Promise<string> {
|
||||
return this.provider.createToken(userId, 'cargo', { readonly });
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate UUID for NPM tokens
|
||||
* Validate a Cargo token
|
||||
* @param token - Cargo UUID token
|
||||
* @returns Auth token object or null
|
||||
*/
|
||||
private generateUuid(): string {
|
||||
return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => {
|
||||
const r = (Math.random() * 16) | 0;
|
||||
const v = c === 'x' ? r : (r & 0x3) | 0x8;
|
||||
return v.toString(16);
|
||||
});
|
||||
public async validateCargoToken(token: string): Promise<IAuthToken | null> {
|
||||
return this.provider.validateToken(token, 'cargo');
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if string is a valid UUID
|
||||
* Revoke a Cargo token
|
||||
* @param token - Cargo UUID token
|
||||
*/
|
||||
private isValidUuid(str: string): boolean {
|
||||
const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
||||
return uuidRegex.test(str);
|
||||
public async revokeCargoToken(token: string): Promise<void> {
|
||||
return this.provider.revokeToken(token);
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// PYPI AUTHENTICATION
|
||||
// ========================================================================
|
||||
|
||||
/**
|
||||
* Create a PyPI token
|
||||
* @param userId - User ID
|
||||
* @param readonly - Whether the token is readonly
|
||||
* @returns PyPI UUID token
|
||||
*/
|
||||
public async createPypiToken(userId: string, readonly: boolean = false): Promise<string> {
|
||||
return this.provider.createToken(userId, 'pypi', { readonly });
|
||||
}
|
||||
|
||||
/**
|
||||
* Hash a token for identification (SHA-512 mock)
|
||||
* Validate a PyPI token
|
||||
* @param token - PyPI UUID token
|
||||
* @returns Auth token object or null
|
||||
*/
|
||||
private hashToken(token: string): string {
|
||||
// In production, use actual SHA-512
|
||||
return `sha512-${token.substring(0, 16)}...`;
|
||||
public async validatePypiToken(token: string): Promise<IAuthToken | null> {
|
||||
return this.provider.validateToken(token, 'pypi');
|
||||
}
|
||||
|
||||
/**
|
||||
* Revoke a PyPI token
|
||||
* @param token - PyPI UUID token
|
||||
*/
|
||||
public async revokePypiToken(token: string): Promise<void> {
|
||||
return this.provider.revokeToken(token);
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// RUBYGEMS AUTHENTICATION
|
||||
// ========================================================================
|
||||
|
||||
/**
|
||||
* Create a RubyGems token
|
||||
* @param userId - User ID
|
||||
* @param readonly - Whether the token is readonly
|
||||
* @returns RubyGems UUID token
|
||||
*/
|
||||
public async createRubyGemsToken(userId: string, readonly: boolean = false): Promise<string> {
|
||||
return this.provider.createToken(userId, 'rubygems', { readonly });
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate a RubyGems token
|
||||
* @param token - RubyGems UUID token
|
||||
* @returns Auth token object or null
|
||||
*/
|
||||
public async validateRubyGemsToken(token: string): Promise<IAuthToken | null> {
|
||||
return this.provider.validateToken(token, 'rubygems');
|
||||
}
|
||||
|
||||
/**
|
||||
* Revoke a RubyGems token
|
||||
* @param token - RubyGems UUID token
|
||||
*/
|
||||
public async revokeRubyGemsToken(token: string): Promise<void> {
|
||||
return this.provider.revokeToken(token);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,14 +1,131 @@
|
||||
import type { IRequestContext, IResponse, IAuthToken } from './interfaces.core.js';
|
||||
import * as plugins from '../plugins.js';
|
||||
import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from './interfaces.core.js';
|
||||
|
||||
/**
|
||||
* Abstract base class for all registry protocol implementations
|
||||
*/
|
||||
export abstract class BaseRegistry {
|
||||
protected getHeader(contextOrHeaders: IRequestContext | Record<string, string>, name: string): string | undefined {
|
||||
const headers = 'headers' in contextOrHeaders ? contextOrHeaders.headers : contextOrHeaders;
|
||||
if (headers[name] !== undefined) {
|
||||
return headers[name];
|
||||
}
|
||||
|
||||
const lowerName = name.toLowerCase();
|
||||
for (const [headerName, value] of Object.entries(headers)) {
|
||||
if (headerName.toLowerCase() === lowerName) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
protected getAuthorizationHeader(context: IRequestContext): string | undefined {
|
||||
return this.getHeader(context, 'authorization');
|
||||
}
|
||||
|
||||
protected getClientIp(context: IRequestContext): string | undefined {
|
||||
const forwardedFor = this.getHeader(context, 'x-forwarded-for');
|
||||
if (forwardedFor) {
|
||||
return forwardedFor.split(',')[0]?.trim();
|
||||
}
|
||||
|
||||
return this.getHeader(context, 'x-real-ip');
|
||||
}
|
||||
|
||||
protected getUserAgent(context: IRequestContext): string | undefined {
|
||||
return this.getHeader(context, 'user-agent');
|
||||
}
|
||||
|
||||
protected extractBearerToken(contextOrHeader: IRequestContext | string | undefined): string | null {
|
||||
const authHeader = typeof contextOrHeader === 'string'
|
||||
? contextOrHeader
|
||||
: contextOrHeader
|
||||
? this.getAuthorizationHeader(contextOrHeader)
|
||||
: undefined;
|
||||
|
||||
if (!authHeader || !/^Bearer\s+/i.test(authHeader)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return authHeader.replace(/^Bearer\s+/i, '');
|
||||
}
|
||||
|
||||
protected parseBasicAuthHeader(authHeader: string | undefined): { username: string; password: string } | null {
|
||||
if (!authHeader || !/^Basic\s+/i.test(authHeader)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const base64 = authHeader.replace(/^Basic\s+/i, '');
|
||||
const decoded = Buffer.from(base64, 'base64').toString('utf-8');
|
||||
const separatorIndex = decoded.indexOf(':');
|
||||
|
||||
if (separatorIndex < 0) {
|
||||
return {
|
||||
username: decoded,
|
||||
password: '',
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
username: decoded.substring(0, separatorIndex),
|
||||
password: decoded.substring(separatorIndex + 1),
|
||||
};
|
||||
}
|
||||
|
||||
protected buildRequestActor(context: IRequestContext, token: IAuthToken | null): IRequestActor {
|
||||
const actor: IRequestActor = {
|
||||
...(context.actor ?? {}),
|
||||
};
|
||||
|
||||
if (token?.userId) {
|
||||
actor.userId = token.userId;
|
||||
}
|
||||
|
||||
const ip = this.getClientIp(context);
|
||||
if (ip) {
|
||||
actor.ip = ip;
|
||||
}
|
||||
|
||||
const userAgent = this.getUserAgent(context);
|
||||
if (userAgent) {
|
||||
actor.userAgent = userAgent;
|
||||
}
|
||||
|
||||
return actor;
|
||||
}
|
||||
|
||||
protected createProtocolLogger(
|
||||
containerName: string,
|
||||
zone: string
|
||||
): plugins.smartlog.Smartlog {
|
||||
const logger = new plugins.smartlog.Smartlog({
|
||||
logContext: {
|
||||
company: 'push.rocks',
|
||||
companyunit: 'smartregistry',
|
||||
containerName,
|
||||
environment: (process.env.NODE_ENV as any) || 'development',
|
||||
runtime: 'node',
|
||||
zone,
|
||||
}
|
||||
});
|
||||
logger.enableConsole();
|
||||
return logger;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the registry
|
||||
*/
|
||||
abstract init(): Promise<void>;
|
||||
|
||||
/**
|
||||
* Clean up timers, connections, and other registry resources.
|
||||
*/
|
||||
public destroy(): void {
|
||||
// Default no-op for registries without persistent resources.
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle an incoming HTTP request
|
||||
* @param context - Request context
|
||||
|
||||
@@ -0,0 +1,393 @@
|
||||
import * as crypto from 'crypto';
|
||||
import type { IAuthProvider, ITokenOptions } from './interfaces.auth.js';
|
||||
import type { IAuthConfig, IAuthToken, ICredentials, TRegistryProtocol } from './interfaces.core.js';
|
||||
|
||||
/**
|
||||
* Default in-memory authentication provider.
|
||||
* This is the reference implementation that stores tokens in memory.
|
||||
* For production use, implement IAuthProvider with Redis, database, or external auth.
|
||||
*/
|
||||
export class DefaultAuthProvider implements IAuthProvider {
|
||||
private tokenStore: Map<string, IAuthToken> = new Map();
|
||||
private userCredentials: Map<string, string> = new Map(); // username -> password hash (mock)
|
||||
|
||||
constructor(private config: IAuthConfig) {}
|
||||
|
||||
/**
|
||||
* Initialize the auth provider
|
||||
*/
|
||||
public async init(): Promise<void> {
|
||||
// Initialize token store (in-memory for now)
|
||||
// In production, this could be Redis or a database
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// IAuthProvider Implementation
|
||||
// ========================================================================
|
||||
|
||||
/**
|
||||
* Authenticate user credentials
|
||||
*/
|
||||
public async authenticate(credentials: ICredentials): Promise<string | null> {
|
||||
// Mock authentication - in production, verify against database/LDAP
|
||||
const storedPassword = this.userCredentials.get(credentials.username);
|
||||
|
||||
if (!storedPassword) {
|
||||
// Auto-register for testing (remove in production)
|
||||
this.userCredentials.set(credentials.username, credentials.password);
|
||||
return credentials.username;
|
||||
}
|
||||
|
||||
if (storedPassword === credentials.password) {
|
||||
return credentials.username;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate any token (NPM, Maven, OCI, PyPI, RubyGems, Composer, Cargo)
|
||||
*/
|
||||
public async validateToken(
|
||||
tokenString: string,
|
||||
protocol?: TRegistryProtocol
|
||||
): Promise<IAuthToken | null> {
|
||||
// OCI uses JWT (contains dots), not UUID - check first if OCI is expected
|
||||
if (protocol === 'oci' || tokenString.includes('.')) {
|
||||
const ociToken = await this.validateOciToken(tokenString);
|
||||
if (ociToken && (!protocol || protocol === 'oci')) {
|
||||
return ociToken;
|
||||
}
|
||||
// If protocol was explicitly OCI but validation failed, return null
|
||||
if (protocol === 'oci') {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// UUID-based tokens: single O(1) Map lookup
|
||||
if (this.isValidUuid(tokenString)) {
|
||||
const authToken = this.tokenStore.get(tokenString);
|
||||
if (authToken) {
|
||||
// If protocol specified, verify it matches
|
||||
if (protocol && authToken.type !== protocol) {
|
||||
return null;
|
||||
}
|
||||
// Check expiration
|
||||
if (authToken.expiresAt && authToken.expiresAt < new Date()) {
|
||||
this.tokenStore.delete(tokenString);
|
||||
return null;
|
||||
}
|
||||
return authToken;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new token for a user
|
||||
*/
|
||||
public async createToken(
|
||||
userId: string,
|
||||
protocol: TRegistryProtocol,
|
||||
options?: ITokenOptions
|
||||
): Promise<string> {
|
||||
// OCI tokens use JWT
|
||||
if (protocol === 'oci') {
|
||||
return this.createOciToken(userId, options?.scopes || ['oci:*:*:*'], options?.expiresIn || 3600);
|
||||
}
|
||||
|
||||
// All other protocols use UUID tokens
|
||||
const token = this.generateUuid();
|
||||
const scopes = options?.scopes || (options?.readonly
|
||||
? [`${protocol}:*:*:read`]
|
||||
: [`${protocol}:*:*:*`]);
|
||||
|
||||
const authToken: IAuthToken = {
|
||||
type: protocol,
|
||||
userId,
|
||||
scopes,
|
||||
readonly: options?.readonly,
|
||||
expiresAt: options?.expiresIn ? new Date(Date.now() + options.expiresIn * 1000) : undefined,
|
||||
metadata: {
|
||||
created: new Date().toISOString(),
|
||||
},
|
||||
};
|
||||
|
||||
this.tokenStore.set(token, authToken);
|
||||
return token;
|
||||
}
|
||||
|
||||
/**
|
||||
* Revoke a token
|
||||
*/
|
||||
public async revokeToken(token: string): Promise<void> {
|
||||
this.tokenStore.delete(token);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if token has permission for an action
|
||||
*/
|
||||
public async authorize(
|
||||
token: IAuthToken | null,
|
||||
resource: string,
|
||||
action: string
|
||||
): Promise<boolean> {
|
||||
if (!token) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check readonly flag
|
||||
if (token.readonly && ['write', 'push', 'delete'].includes(action)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check scopes
|
||||
for (const scope of token.scopes) {
|
||||
if (this.matchesScope(scope, resource, action)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* List all tokens for a user
|
||||
*/
|
||||
public async listUserTokens(userId: string): Promise<Array<{
|
||||
key: string;
|
||||
readonly: boolean;
|
||||
created: string;
|
||||
protocol?: TRegistryProtocol;
|
||||
}>> {
|
||||
const tokens: Array<{key: string; readonly: boolean; created: string; protocol?: TRegistryProtocol}> = [];
|
||||
|
||||
for (const [token, authToken] of this.tokenStore.entries()) {
|
||||
if (authToken.userId === userId) {
|
||||
tokens.push({
|
||||
key: this.hashToken(token),
|
||||
readonly: authToken.readonly || false,
|
||||
created: authToken.metadata?.created || 'unknown',
|
||||
protocol: authToken.type,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return tokens;
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// OCI JWT Token Methods
|
||||
// ========================================================================
|
||||
|
||||
/**
|
||||
* Create an OCI JWT token
|
||||
*/
|
||||
private async createOciToken(
|
||||
userId: string,
|
||||
scopes: string[],
|
||||
expiresIn: number = 3600
|
||||
): Promise<string> {
|
||||
if (!this.config.ociTokens?.enabled) {
|
||||
throw new Error('OCI tokens are not enabled');
|
||||
}
|
||||
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
const payload = {
|
||||
iss: this.config.ociTokens.realm,
|
||||
sub: userId,
|
||||
aud: this.config.ociTokens.service,
|
||||
exp: now + expiresIn,
|
||||
nbf: now,
|
||||
iat: now,
|
||||
access: this.scopesToOciAccess(scopes),
|
||||
};
|
||||
|
||||
// Create JWT with HMAC-SHA256 signature
|
||||
const header = { alg: 'HS256', typ: 'JWT' };
|
||||
const headerB64 = Buffer.from(JSON.stringify(header)).toString('base64url');
|
||||
const payloadB64 = Buffer.from(JSON.stringify(payload)).toString('base64url');
|
||||
|
||||
const signature = crypto
|
||||
.createHmac('sha256', this.config.jwtSecret)
|
||||
.update(`${headerB64}.${payloadB64}`)
|
||||
.digest('base64url');
|
||||
|
||||
return `${headerB64}.${payloadB64}.${signature}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate an OCI JWT token
|
||||
*/
|
||||
private async validateOciToken(jwt: string): Promise<IAuthToken | null> {
|
||||
try {
|
||||
const parts = jwt.split('.');
|
||||
if (parts.length !== 3) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const [headerB64, payloadB64, signatureB64] = parts;
|
||||
|
||||
// Verify signature
|
||||
const expectedSignature = crypto
|
||||
.createHmac('sha256', this.config.jwtSecret)
|
||||
.update(`${headerB64}.${payloadB64}`)
|
||||
.digest('base64url');
|
||||
|
||||
if (signatureB64 !== expectedSignature) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Decode and parse payload
|
||||
const payload = JSON.parse(Buffer.from(payloadB64, 'base64url').toString('utf-8'));
|
||||
|
||||
// Check expiration
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
if (payload.exp && payload.exp < now) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Check not-before time
|
||||
if (payload.nbf && payload.nbf > now) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Convert to unified token format
|
||||
const scopes = this.ociAccessToScopes(payload.access || []);
|
||||
|
||||
return {
|
||||
type: 'oci',
|
||||
userId: payload.sub,
|
||||
scopes,
|
||||
expiresAt: payload.exp ? new Date(payload.exp * 1000) : undefined,
|
||||
metadata: {
|
||||
iss: payload.iss,
|
||||
aud: payload.aud,
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// Helper Methods
|
||||
// ========================================================================
|
||||
|
||||
/**
|
||||
* Check if a scope matches a resource and action
|
||||
*/
|
||||
private matchesScope(scope: string, resource: string, action: string): boolean {
|
||||
const scopeParts = scope.split(':');
|
||||
const resourceParts = resource.split(':');
|
||||
|
||||
// Scope must have at least protocol:type:name:action
|
||||
if (scopeParts.length < 4) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const [scopeProtocol, scopeType, scopeName, scopeAction] = scopeParts;
|
||||
const [resourceProtocol, resourceType, resourceName] = resourceParts;
|
||||
|
||||
// Check protocol
|
||||
if (scopeProtocol !== '*' && scopeProtocol !== resourceProtocol) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check type
|
||||
if (scopeType !== '*' && scopeType !== resourceType) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check name
|
||||
if (scopeName !== '*' && scopeName !== resourceName) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check action
|
||||
if (scopeAction !== '*' && scopeAction !== action) {
|
||||
// Map action aliases
|
||||
const actionAliases: Record<string, string[]> = {
|
||||
read: ['pull', 'get'],
|
||||
write: ['push', 'put', 'post'],
|
||||
};
|
||||
|
||||
const aliases = actionAliases[scopeAction] || [];
|
||||
if (!aliases.includes(action)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert unified scopes to OCI access array
|
||||
*/
|
||||
private scopesToOciAccess(scopes: string[]): Array<{
|
||||
type: string;
|
||||
name: string;
|
||||
actions: string[];
|
||||
}> {
|
||||
const access: Array<{type: string; name: string; actions: string[]}> = [];
|
||||
|
||||
for (const scope of scopes) {
|
||||
const parts = scope.split(':');
|
||||
if (parts.length >= 4 && parts[0] === 'oci') {
|
||||
access.push({
|
||||
type: parts[1],
|
||||
name: parts[2],
|
||||
actions: [parts[3]],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return access;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert OCI access array to unified scopes
|
||||
*/
|
||||
private ociAccessToScopes(access: Array<{
|
||||
type: string;
|
||||
name: string;
|
||||
actions: string[];
|
||||
}>): string[] {
|
||||
const scopes: string[] = [];
|
||||
|
||||
for (const item of access) {
|
||||
for (const action of item.actions) {
|
||||
scopes.push(`oci:${item.type}:${item.name}:${action}`);
|
||||
}
|
||||
}
|
||||
|
||||
return scopes;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate UUID for tokens
|
||||
*/
|
||||
private generateUuid(): string {
|
||||
return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => {
|
||||
const r = (Math.random() * 16) | 0;
|
||||
const v = c === 'x' ? r : (r & 0x3) | 0x8;
|
||||
return v.toString(16);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if string is a valid UUID
|
||||
*/
|
||||
private isValidUuid(str: string): boolean {
|
||||
const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
||||
return uuidRegex.test(str);
|
||||
}
|
||||
|
||||
/**
|
||||
* Hash a token for identification
|
||||
*/
|
||||
private hashToken(token: string): string {
|
||||
return `sha512-${token.substring(0, 16)}...`;
|
||||
}
|
||||
}
|
||||
+1022
-46
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,34 @@
|
||||
/**
|
||||
* Shared buffer utilities for consistent binary data handling across all registry types.
|
||||
*
|
||||
* This module addresses the common issue where `Buffer.isBuffer(Uint8Array)` returns `false`,
|
||||
* which can cause data handling bugs when binary data arrives as Uint8Array instead of Buffer.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Check if value is binary data (Buffer or Uint8Array)
|
||||
*/
|
||||
export function isBinaryData(value: unknown): value is Buffer | Uint8Array {
|
||||
return Buffer.isBuffer(value) || value instanceof Uint8Array;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert any binary-like data to Buffer.
|
||||
* Handles Buffer, Uint8Array, string, and objects.
|
||||
*
|
||||
* @param data - The data to convert to Buffer
|
||||
* @returns A Buffer containing the data
|
||||
*/
|
||||
export function toBuffer(data: unknown): Buffer {
|
||||
if (Buffer.isBuffer(data)) {
|
||||
return data;
|
||||
}
|
||||
if (data instanceof Uint8Array) {
|
||||
return Buffer.from(data);
|
||||
}
|
||||
if (typeof data === 'string') {
|
||||
return Buffer.from(data, 'utf-8');
|
||||
}
|
||||
// Fallback: serialize object to JSON
|
||||
return Buffer.from(JSON.stringify(data));
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
function digestToHash(digest: string): string {
|
||||
return digest.split(':')[1];
|
||||
}
|
||||
|
||||
export function getOciBlobPath(digest: string): string {
|
||||
return `oci/blobs/sha256/${digestToHash(digest)}`;
|
||||
}
|
||||
|
||||
export function getOciManifestPath(repository: string, digest: string): string {
|
||||
return `oci/manifests/${repository}/${digestToHash(digest)}`;
|
||||
}
|
||||
|
||||
export function getNpmPackumentPath(packageName: string): string {
|
||||
return `npm/packages/${packageName}/index.json`;
|
||||
}
|
||||
|
||||
export function getNpmTarballPath(packageName: string, version: string): string {
|
||||
const safeName = packageName.replace('@', '').replace('/', '-');
|
||||
return `npm/packages/${packageName}/${safeName}-${version}.tgz`;
|
||||
}
|
||||
|
||||
export function getMavenArtifactPath(
|
||||
groupId: string,
|
||||
artifactId: string,
|
||||
version: string,
|
||||
filename: string
|
||||
): string {
|
||||
const groupPath = groupId.replace(/\./g, '/');
|
||||
return `maven/artifacts/${groupPath}/${artifactId}/${version}/${filename}`;
|
||||
}
|
||||
|
||||
export function getMavenMetadataPath(groupId: string, artifactId: string): string {
|
||||
const groupPath = groupId.replace(/\./g, '/');
|
||||
return `maven/metadata/${groupPath}/${artifactId}/maven-metadata.xml`;
|
||||
}
|
||||
|
||||
export function getCargoConfigPath(): string {
|
||||
return 'cargo/config.json';
|
||||
}
|
||||
|
||||
export function getCargoIndexPath(crateName: string): string {
|
||||
const lower = crateName.toLowerCase();
|
||||
const len = lower.length;
|
||||
|
||||
if (len === 1) {
|
||||
return `cargo/index/1/${lower}`;
|
||||
}
|
||||
|
||||
if (len === 2) {
|
||||
return `cargo/index/2/${lower}`;
|
||||
}
|
||||
|
||||
if (len === 3) {
|
||||
return `cargo/index/3/${lower.charAt(0)}/${lower}`;
|
||||
}
|
||||
|
||||
const prefix1 = lower.substring(0, 2);
|
||||
const prefix2 = lower.substring(2, 4);
|
||||
return `cargo/index/${prefix1}/${prefix2}/${lower}`;
|
||||
}
|
||||
|
||||
export function getCargoCratePath(crateName: string, version: string): string {
|
||||
return `cargo/crates/${crateName}/${crateName}-${version}.crate`;
|
||||
}
|
||||
|
||||
export function getComposerMetadataPath(vendorPackage: string): string {
|
||||
return `composer/packages/${vendorPackage}/metadata.json`;
|
||||
}
|
||||
|
||||
export function getComposerZipPath(vendorPackage: string, reference: string): string {
|
||||
return `composer/packages/${vendorPackage}/${reference}.zip`;
|
||||
}
|
||||
|
||||
export function getPypiMetadataPath(packageName: string): string {
|
||||
return `pypi/metadata/${packageName}/metadata.json`;
|
||||
}
|
||||
|
||||
export function getPypiSimpleIndexPath(packageName: string): string {
|
||||
return `pypi/simple/${packageName}/index.html`;
|
||||
}
|
||||
|
||||
export function getPypiSimpleRootIndexPath(): string {
|
||||
return 'pypi/simple/index.html';
|
||||
}
|
||||
|
||||
export function getPypiPackageFilePath(packageName: string, filename: string): string {
|
||||
return `pypi/packages/${packageName}/${filename}`;
|
||||
}
|
||||
|
||||
export function getRubyGemsVersionsPath(): string {
|
||||
return 'rubygems/versions';
|
||||
}
|
||||
|
||||
export function getRubyGemsInfoPath(gemName: string): string {
|
||||
return `rubygems/info/${gemName}`;
|
||||
}
|
||||
|
||||
export function getRubyGemsNamesPath(): string {
|
||||
return 'rubygems/names';
|
||||
}
|
||||
|
||||
export function getRubyGemsGemPath(gemName: string, version: string, platform?: string): string {
|
||||
const filename = platform ? `${gemName}-${version}-${platform}.gem` : `${gemName}-${version}.gem`;
|
||||
return `rubygems/gems/${filename}`;
|
||||
}
|
||||
|
||||
export function getRubyGemsMetadataPath(gemName: string): string {
|
||||
return `rubygems/metadata/${gemName}/metadata.json`;
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
import * as crypto from 'crypto';
|
||||
|
||||
/**
|
||||
* Convert Buffer, Uint8Array, string, or JSON object to a ReadableStream<Uint8Array>.
|
||||
*/
|
||||
export function toReadableStream(data: Buffer | Uint8Array | string | object): ReadableStream<Uint8Array> {
|
||||
const buf = Buffer.isBuffer(data)
|
||||
? data
|
||||
: data instanceof Uint8Array
|
||||
? Buffer.from(data)
|
||||
: typeof data === 'string'
|
||||
? Buffer.from(data, 'utf-8')
|
||||
: Buffer.from(JSON.stringify(data), 'utf-8');
|
||||
return new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(new Uint8Array(buf));
|
||||
controller.close();
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Consume a ReadableStream into a Buffer.
|
||||
*/
|
||||
export async function streamToBuffer(stream: ReadableStream<Uint8Array>): Promise<Buffer> {
|
||||
const reader = stream.getReader();
|
||||
const chunks: Uint8Array[] = [];
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
if (value) chunks.push(value);
|
||||
}
|
||||
return Buffer.concat(chunks);
|
||||
}
|
||||
|
||||
/**
|
||||
* Consume a ReadableStream into a parsed JSON object.
|
||||
*/
|
||||
export async function streamToJson<T = any>(stream: ReadableStream<Uint8Array>): Promise<T> {
|
||||
const buf = await streamToBuffer(stream);
|
||||
return JSON.parse(buf.toString('utf-8'));
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a TransformStream that incrementally hashes data passing through.
|
||||
* Data flows through unchanged; the digest is available after the stream completes.
|
||||
*/
|
||||
export function createHashTransform(algorithm: string = 'sha256'): {
|
||||
transform: TransformStream<Uint8Array, Uint8Array>;
|
||||
getDigest: () => string;
|
||||
} {
|
||||
const hash = crypto.createHash(algorithm);
|
||||
const transform = new TransformStream<Uint8Array, Uint8Array>({
|
||||
transform(chunk, controller) {
|
||||
hash.update(chunk);
|
||||
controller.enqueue(chunk);
|
||||
},
|
||||
});
|
||||
return {
|
||||
transform,
|
||||
getDigest: () => hash.digest('hex'),
|
||||
};
|
||||
}
|
||||
+11
-1
@@ -2,9 +2,19 @@
|
||||
* Core registry infrastructure exports
|
||||
*/
|
||||
|
||||
// Interfaces
|
||||
// Core interfaces
|
||||
export * from './interfaces.core.js';
|
||||
|
||||
// Auth interfaces and provider
|
||||
export * from './interfaces.auth.js';
|
||||
export { DefaultAuthProvider } from './classes.defaultauthprovider.js';
|
||||
|
||||
// Storage interfaces and hooks
|
||||
export * from './interfaces.storage.js';
|
||||
|
||||
// Stream helpers
|
||||
export { toReadableStream, streamToBuffer, streamToJson, createHashTransform } from './helpers.stream.js';
|
||||
|
||||
// Classes
|
||||
export { BaseRegistry } from './classes.baseregistry.js';
|
||||
export { RegistryStorage } from './classes.registrystorage.js';
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
import type { IAuthToken, ICredentials, TRegistryProtocol } from './interfaces.core.js';
|
||||
|
||||
/**
|
||||
* Options for creating a token
|
||||
*/
|
||||
export interface ITokenOptions {
|
||||
/** Whether the token is readonly */
|
||||
readonly?: boolean;
|
||||
/** Permission scopes */
|
||||
scopes?: string[];
|
||||
/** Expiration time in seconds */
|
||||
expiresIn?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pluggable authentication provider interface.
|
||||
* Implement this to integrate external auth systems (LDAP, OAuth, SSO, OIDC).
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* class LdapAuthProvider implements IAuthProvider {
|
||||
* constructor(private ldap: LdapClient, private redis: RedisClient) {}
|
||||
*
|
||||
* async authenticate(credentials: ICredentials): Promise<string | null> {
|
||||
* return await this.ldap.bind(credentials.username, credentials.password);
|
||||
* }
|
||||
*
|
||||
* async validateToken(token: string): Promise<IAuthToken | null> {
|
||||
* return await this.redis.get(`token:${token}`);
|
||||
* }
|
||||
* // ...
|
||||
* }
|
||||
* ```
|
||||
*/
|
||||
export interface IAuthProvider {
|
||||
/**
|
||||
* Initialize the auth provider (optional)
|
||||
*/
|
||||
init?(): Promise<void>;
|
||||
|
||||
/**
|
||||
* Authenticate user credentials (login flow)
|
||||
* @param credentials - Username and password
|
||||
* @returns User ID on success, null on failure
|
||||
*/
|
||||
authenticate(credentials: ICredentials): Promise<string | null>;
|
||||
|
||||
/**
|
||||
* Validate an existing token
|
||||
* @param token - Token string (UUID or JWT)
|
||||
* @param protocol - Optional protocol hint for optimization
|
||||
* @returns Auth token info or null if invalid
|
||||
*/
|
||||
validateToken(token: string, protocol?: TRegistryProtocol): Promise<IAuthToken | null>;
|
||||
|
||||
/**
|
||||
* Create a new token for a user
|
||||
* @param userId - User ID
|
||||
* @param protocol - Protocol type (npm, oci, maven, etc.)
|
||||
* @param options - Token options (readonly, scopes, expiration)
|
||||
* @returns Token string
|
||||
*/
|
||||
createToken(userId: string, protocol: TRegistryProtocol, options?: ITokenOptions): Promise<string>;
|
||||
|
||||
/**
|
||||
* Revoke a token
|
||||
* @param token - Token string to revoke
|
||||
*/
|
||||
revokeToken(token: string): Promise<void>;
|
||||
|
||||
/**
|
||||
* Check if user has permission for an action
|
||||
* @param token - Auth token (or null for anonymous)
|
||||
* @param resource - Resource being accessed (e.g., "npm:package:lodash")
|
||||
* @param action - Action being performed (read, write, push, pull, delete)
|
||||
* @returns true if authorized
|
||||
*/
|
||||
authorize(token: IAuthToken | null, resource: string, action: string): Promise<boolean>;
|
||||
|
||||
/**
|
||||
* List all tokens for a user (optional)
|
||||
* @param userId - User ID
|
||||
* @returns List of token info
|
||||
*/
|
||||
listUserTokens?(userId: string): Promise<Array<{
|
||||
key: string;
|
||||
readonly: boolean;
|
||||
created: string;
|
||||
protocol?: TRegistryProtocol;
|
||||
}>>;
|
||||
}
|
||||
+94
-10
@@ -2,10 +2,15 @@
|
||||
* Core interfaces for the composable registry system
|
||||
*/
|
||||
|
||||
import type * as plugins from '../plugins.js';
|
||||
import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
|
||||
import type { IAuthProvider } from './interfaces.auth.js';
|
||||
import type { IStorageHooks } from './interfaces.storage.js';
|
||||
|
||||
/**
|
||||
* Registry protocol types
|
||||
*/
|
||||
export type TRegistryProtocol = 'oci' | 'npm';
|
||||
export type TRegistryProtocol = 'oci' | 'npm' | 'maven' | 'cargo' | 'composer' | 'pypi' | 'rubygems';
|
||||
|
||||
/**
|
||||
* Unified action types across protocols
|
||||
@@ -40,14 +45,9 @@ export interface ICredentials {
|
||||
|
||||
/**
|
||||
* Storage backend configuration
|
||||
* Extends IS3Descriptor from @tsclass/tsclass with bucketName
|
||||
*/
|
||||
export interface IStorageConfig {
|
||||
accessKey: string;
|
||||
accessSecret: string;
|
||||
endpoint: string;
|
||||
port?: number;
|
||||
useSsl?: boolean;
|
||||
region?: string;
|
||||
export interface IStorageConfig extends plugins.tsclass.storage.IS3Descriptor {
|
||||
bucketName: string;
|
||||
}
|
||||
|
||||
@@ -70,6 +70,16 @@ export interface IAuthConfig {
|
||||
realm: string;
|
||||
service: string;
|
||||
};
|
||||
/** PyPI token settings */
|
||||
pypiTokens?: {
|
||||
enabled: boolean;
|
||||
defaultReadonly?: boolean;
|
||||
};
|
||||
/** RubyGems token settings */
|
||||
rubygemsTokens?: {
|
||||
enabled: boolean;
|
||||
defaultReadonly?: boolean;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -78,6 +88,7 @@ export interface IAuthConfig {
|
||||
export interface IProtocolConfig {
|
||||
enabled: boolean;
|
||||
basePath: string;
|
||||
registryUrl?: string;
|
||||
features?: Record<string, boolean>;
|
||||
}
|
||||
|
||||
@@ -87,8 +98,34 @@ export interface IProtocolConfig {
|
||||
export interface IRegistryConfig {
|
||||
storage: IStorageConfig;
|
||||
auth: IAuthConfig;
|
||||
|
||||
/**
|
||||
* Custom authentication provider.
|
||||
* If not provided, uses the default in-memory auth provider.
|
||||
* Implement IAuthProvider to integrate LDAP, OAuth, SSO, etc.
|
||||
*/
|
||||
authProvider?: IAuthProvider;
|
||||
|
||||
/**
|
||||
* Storage event hooks for quota tracking, audit logging, etc.
|
||||
* Called before/after storage operations.
|
||||
*/
|
||||
storageHooks?: IStorageHooks;
|
||||
|
||||
/**
|
||||
* Dynamic upstream configuration provider.
|
||||
* Called per-request to resolve which upstream registries to use.
|
||||
* Use StaticUpstreamProvider for simple static configurations.
|
||||
*/
|
||||
upstreamProvider?: IUpstreamProvider;
|
||||
|
||||
oci?: IProtocolConfig;
|
||||
npm?: IProtocolConfig;
|
||||
maven?: IProtocolConfig;
|
||||
cargo?: IProtocolConfig;
|
||||
composer?: IProtocolConfig;
|
||||
pypi?: IProtocolConfig;
|
||||
rubygems?: IProtocolConfig;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -124,6 +161,21 @@ export interface IStorageBackend {
|
||||
* Get object metadata
|
||||
*/
|
||||
getMetadata(key: string): Promise<Record<string, string> | null>;
|
||||
|
||||
/**
|
||||
* Get an object as a ReadableStream. Returns null if not found.
|
||||
*/
|
||||
getObjectStream?(key: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null>;
|
||||
|
||||
/**
|
||||
* Store an object from a ReadableStream.
|
||||
*/
|
||||
putObjectStream?(key: string, stream: ReadableStream<Uint8Array>): Promise<void>;
|
||||
|
||||
/**
|
||||
* Get object size without reading data (S3 HEAD request).
|
||||
*/
|
||||
getObjectSize?(key: string): Promise<number | null>;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -137,6 +189,24 @@ export interface IRegistryError {
|
||||
}>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Actor information - identifies who is performing the request
|
||||
*/
|
||||
export interface IRequestActor {
|
||||
/** User ID (from validated token) */
|
||||
userId?: string;
|
||||
/** Token ID/hash for audit purposes */
|
||||
tokenId?: string;
|
||||
/** Client IP address */
|
||||
ip?: string;
|
||||
/** Client User-Agent */
|
||||
userAgent?: string;
|
||||
/** Organization ID (for multi-tenant setups) */
|
||||
orgId?: string;
|
||||
/** Session ID */
|
||||
sessionId?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Base request context
|
||||
*/
|
||||
@@ -146,14 +216,28 @@ export interface IRequestContext {
|
||||
headers: Record<string, string>;
|
||||
query: Record<string, string>;
|
||||
body?: any;
|
||||
/**
|
||||
* Raw request body as bytes. MUST be provided for content-addressable operations
|
||||
* (OCI manifests, blobs) to ensure digest calculation matches client expectations.
|
||||
* If not provided, falls back to 'body' field.
|
||||
*/
|
||||
rawBody?: Buffer;
|
||||
token?: string;
|
||||
/**
|
||||
* Actor information - identifies who is performing the request.
|
||||
* Populated after authentication for audit logging, quota enforcement, etc.
|
||||
*/
|
||||
actor?: IRequestActor;
|
||||
}
|
||||
|
||||
/**
|
||||
* Base response structure
|
||||
* Base response structure.
|
||||
* `body` is always a `ReadableStream<Uint8Array>` at the public API boundary.
|
||||
* Internal handlers may return Buffer/string/object — the SmartRegistry orchestrator
|
||||
* auto-wraps them via `toReadableStream()` before returning to the caller.
|
||||
*/
|
||||
export interface IResponse {
|
||||
status: number;
|
||||
headers: Record<string, string>;
|
||||
body?: any;
|
||||
body?: ReadableStream<Uint8Array> | any;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
import type { TRegistryProtocol } from './interfaces.core.js';
|
||||
|
||||
/**
|
||||
* Actor information from request context
|
||||
*/
|
||||
export interface IStorageActor {
|
||||
userId?: string;
|
||||
tokenId?: string;
|
||||
ip?: string;
|
||||
userAgent?: string;
|
||||
orgId?: string;
|
||||
sessionId?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Metadata about the storage operation
|
||||
*/
|
||||
export interface IStorageMetadata {
|
||||
/** Content type of the object */
|
||||
contentType?: string;
|
||||
/** Size in bytes */
|
||||
size?: number;
|
||||
/** Content digest (e.g., sha256:abc123) */
|
||||
digest?: string;
|
||||
/** Package/artifact name */
|
||||
packageName?: string;
|
||||
/** Version */
|
||||
version?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Context passed to storage hooks
|
||||
*/
|
||||
export interface IStorageHookContext {
|
||||
/** Type of operation */
|
||||
operation: 'put' | 'delete' | 'get';
|
||||
/** Storage key/path */
|
||||
key: string;
|
||||
/** Protocol that triggered this operation */
|
||||
protocol: TRegistryProtocol;
|
||||
/** Actor who performed the operation (if known) */
|
||||
actor?: IStorageActor;
|
||||
/** Metadata about the object */
|
||||
metadata?: IStorageMetadata;
|
||||
/** Timestamp of the operation */
|
||||
timestamp: Date;
|
||||
}
|
||||
|
||||
/**
|
||||
* Result from a beforePut hook that can modify the operation
|
||||
*/
|
||||
export interface IBeforePutResult {
|
||||
/** Whether to allow the operation */
|
||||
allowed: boolean;
|
||||
/** Optional reason for rejection */
|
||||
reason?: string;
|
||||
/** Optional modified metadata */
|
||||
metadata?: IStorageMetadata;
|
||||
}
|
||||
|
||||
/**
|
||||
* Result from a beforeDelete hook
|
||||
*/
|
||||
export interface IBeforeDeleteResult {
|
||||
/** Whether to allow the operation */
|
||||
allowed: boolean;
|
||||
/** Optional reason for rejection */
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Storage event hooks for quota tracking, audit logging, cache invalidation, etc.
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const quotaHooks: IStorageHooks = {
|
||||
* async beforePut(context) {
|
||||
* const quota = await getQuota(context.actor?.orgId);
|
||||
* const currentUsage = await getUsage(context.actor?.orgId);
|
||||
* if (currentUsage + (context.metadata?.size || 0) > quota) {
|
||||
* return { allowed: false, reason: 'Quota exceeded' };
|
||||
* }
|
||||
* return { allowed: true };
|
||||
* },
|
||||
*
|
||||
* async afterPut(context) {
|
||||
* await updateUsage(context.actor?.orgId, context.metadata?.size || 0);
|
||||
* await auditLog('storage.put', context);
|
||||
* },
|
||||
*
|
||||
* async afterDelete(context) {
|
||||
* await invalidateCache(context.key);
|
||||
* }
|
||||
* };
|
||||
* ```
|
||||
*/
|
||||
export interface IStorageHooks {
|
||||
/**
|
||||
* Called before storing an object.
|
||||
* Return { allowed: false } to reject the operation.
|
||||
* Use for quota checks, virus scanning, validation, etc.
|
||||
*/
|
||||
beforePut?(context: IStorageHookContext): Promise<IBeforePutResult>;
|
||||
|
||||
/**
|
||||
* Called after successfully storing an object.
|
||||
* Use for quota tracking, audit logging, notifications, etc.
|
||||
*/
|
||||
afterPut?(context: IStorageHookContext): Promise<void>;
|
||||
|
||||
/**
|
||||
* Called before deleting an object.
|
||||
* Return { allowed: false } to reject the operation.
|
||||
* Use for preventing deletion of protected resources.
|
||||
*/
|
||||
beforeDelete?(context: IStorageHookContext): Promise<IBeforeDeleteResult>;
|
||||
|
||||
/**
|
||||
* Called after successfully deleting an object.
|
||||
* Use for quota updates, audit logging, cache invalidation, etc.
|
||||
*/
|
||||
afterDelete?(context: IStorageHookContext): Promise<void>;
|
||||
|
||||
/**
|
||||
* Called after reading an object.
|
||||
* Use for access logging, analytics, etc.
|
||||
* Note: This is called even for cache hits.
|
||||
*/
|
||||
afterGet?(context: IStorageHookContext): Promise<void>;
|
||||
}
|
||||
+19
-1
@@ -1,6 +1,6 @@
|
||||
/**
|
||||
* @push.rocks/smartregistry
|
||||
* Composable registry supporting OCI and NPM protocols
|
||||
* Composable registry supporting OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems protocols
|
||||
*/
|
||||
|
||||
// Main orchestrator
|
||||
@@ -9,8 +9,26 @@ export { SmartRegistry } from './classes.smartregistry.js';
|
||||
// Core infrastructure
|
||||
export * from './core/index.js';
|
||||
|
||||
// Upstream infrastructure
|
||||
export * from './upstream/index.js';
|
||||
|
||||
// OCI Registry
|
||||
export * from './oci/index.js';
|
||||
|
||||
// NPM Registry
|
||||
export * from './npm/index.js';
|
||||
|
||||
// Maven Registry
|
||||
export * from './maven/index.js';
|
||||
|
||||
// Cargo Registry
|
||||
export * from './cargo/index.js';
|
||||
|
||||
// Composer Registry
|
||||
export * from './composer/index.js';
|
||||
|
||||
// PyPI Registry
|
||||
export * from './pypi/index.js';
|
||||
|
||||
// RubyGems Registry
|
||||
export * from './rubygems/index.js';
|
||||
|
||||
@@ -0,0 +1,743 @@
|
||||
/**
|
||||
* Maven Registry Implementation
|
||||
* Implements Maven repository protocol for Java artifacts
|
||||
*/
|
||||
|
||||
import { BaseRegistry } from '../core/classes.baseregistry.js';
|
||||
import type { RegistryStorage } from '../core/classes.registrystorage.js';
|
||||
import type { AuthManager } from '../core/classes.authmanager.js';
|
||||
import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from '../core/interfaces.core.js';
|
||||
import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
|
||||
import { toBuffer } from '../core/helpers.buffer.js';
|
||||
import type { IMavenCoordinate, IMavenMetadata, IChecksums } from './interfaces.maven.js';
|
||||
import {
|
||||
pathToGAV,
|
||||
buildFilename,
|
||||
calculateChecksums,
|
||||
generateMetadataXml,
|
||||
parseMetadataXml,
|
||||
formatMavenTimestamp,
|
||||
isSnapshot,
|
||||
validatePom,
|
||||
extractGAVFromPom,
|
||||
gavToPath,
|
||||
} from './helpers.maven.js';
|
||||
import { MavenUpstream } from './classes.mavenupstream.js';
|
||||
|
||||
/**
|
||||
* Maven Registry class
|
||||
* Handles Maven repository HTTP protocol
|
||||
*/
|
||||
export class MavenRegistry extends BaseRegistry {
|
||||
private storage: RegistryStorage;
|
||||
private authManager: AuthManager;
|
||||
private basePath: string = '/maven';
|
||||
private registryUrl: string;
|
||||
private upstreamProvider: IUpstreamProvider | null = null;
|
||||
|
||||
constructor(
|
||||
storage: RegistryStorage,
|
||||
authManager: AuthManager,
|
||||
basePath: string,
|
||||
registryUrl: string,
|
||||
upstreamProvider?: IUpstreamProvider
|
||||
) {
|
||||
super();
|
||||
this.storage = storage;
|
||||
this.authManager = authManager;
|
||||
this.basePath = basePath;
|
||||
this.registryUrl = registryUrl;
|
||||
this.upstreamProvider = upstreamProvider || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract scope from Maven coordinates.
|
||||
* For Maven, the groupId is the scope.
|
||||
* @example "com.example" from "com.example:my-lib"
|
||||
*/
|
||||
private extractScope(groupId: string): string | null {
|
||||
return groupId || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get upstream for a specific request.
|
||||
* Calls the provider to resolve upstream config dynamically.
|
||||
*/
|
||||
private async getUpstreamForRequest(
|
||||
resource: string,
|
||||
resourceType: string,
|
||||
method: string,
|
||||
actor?: IRequestActor
|
||||
): Promise<MavenUpstream | null> {
|
||||
if (!this.upstreamProvider) return null;
|
||||
|
||||
// For Maven, resource is "groupId:artifactId"
|
||||
const [groupId] = resource.split(':');
|
||||
const config = await this.upstreamProvider.resolveUpstreamConfig({
|
||||
protocol: 'maven',
|
||||
resource,
|
||||
scope: this.extractScope(groupId),
|
||||
actor,
|
||||
method,
|
||||
resourceType,
|
||||
});
|
||||
|
||||
if (!config?.enabled) return null;
|
||||
return new MavenUpstream(config);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up resources (timers, connections, etc.)
|
||||
*/
|
||||
public destroy(): void {
|
||||
// No persistent upstream to clean up with dynamic provider
|
||||
}
|
||||
|
||||
public async init(): Promise<void> {
|
||||
// No special initialization needed for Maven
|
||||
}
|
||||
|
||||
public getBasePath(): string {
|
||||
return this.basePath;
|
||||
}
|
||||
|
||||
public async handleRequest(context: IRequestContext): Promise<IResponse> {
|
||||
// Remove base path from URL
|
||||
const path = context.path.replace(this.basePath, '');
|
||||
|
||||
const authHeader = this.getAuthorizationHeader(context);
|
||||
let token: IAuthToken | null = null;
|
||||
|
||||
if (authHeader) {
|
||||
const basicCredentials = this.parseBasicAuthHeader(authHeader);
|
||||
if (basicCredentials) {
|
||||
// Maven sends Basic Auth: base64(username:password) — extract the password as token
|
||||
token = await this.authManager.validateToken(basicCredentials.password, 'maven');
|
||||
} else {
|
||||
const tokenString = this.extractBearerToken(authHeader);
|
||||
token = tokenString ? await this.authManager.validateToken(tokenString, 'maven') : null;
|
||||
}
|
||||
}
|
||||
|
||||
const actor: IRequestActor = this.buildRequestActor(context, token);
|
||||
|
||||
return this.storage.withContext({ protocol: 'maven', actor }, async () => {
|
||||
// Parse path to determine request type
|
||||
const coordinate = pathToGAV(path);
|
||||
|
||||
if (!coordinate) {
|
||||
// Not a valid artifact path, could be metadata or root
|
||||
if (path.endsWith('/maven-metadata.xml')) {
|
||||
return this.handleMetadataRequest(context.method, path, token, actor);
|
||||
}
|
||||
|
||||
return {
|
||||
status: 404,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: { error: 'NOT_FOUND', message: 'Invalid Maven path' },
|
||||
};
|
||||
}
|
||||
|
||||
// Check if it's a checksum file
|
||||
if (coordinate.extension === 'md5' || coordinate.extension === 'sha1' ||
|
||||
coordinate.extension === 'sha256' || coordinate.extension === 'sha512') {
|
||||
return this.handleChecksumRequest(context.method, coordinate, token, path);
|
||||
}
|
||||
|
||||
// Handle artifact requests (JAR, POM, WAR, etc.)
|
||||
return this.handleArtifactRequest(context.method, coordinate, token, context.body, actor);
|
||||
});
|
||||
}
|
||||
|
||||
protected async checkPermission(
|
||||
token: IAuthToken | null,
|
||||
resource: string,
|
||||
action: string
|
||||
): Promise<boolean> {
|
||||
if (!token) return false;
|
||||
return this.authManager.authorize(token, `maven:artifact:${resource}`, action);
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// REQUEST HANDLERS
|
||||
// ========================================================================
|
||||
|
||||
private async handleArtifactRequest(
|
||||
method: string,
|
||||
coordinate: IMavenCoordinate,
|
||||
token: IAuthToken | null,
|
||||
body?: Buffer | any,
|
||||
actor?: IRequestActor
|
||||
): Promise<IResponse> {
|
||||
const { groupId, artifactId, version } = coordinate;
|
||||
const filename = buildFilename(coordinate);
|
||||
const resource = `${groupId}:${artifactId}`;
|
||||
|
||||
switch (method) {
|
||||
case 'GET':
|
||||
case 'HEAD':
|
||||
// Maven repositories typically allow anonymous reads
|
||||
return method === 'GET'
|
||||
? this.getArtifact(groupId, artifactId, version, filename, actor)
|
||||
: this.headArtifact(groupId, artifactId, version, filename);
|
||||
|
||||
case 'PUT':
|
||||
// Write permission required
|
||||
if (!await this.checkPermission(token, resource, 'write')) {
|
||||
return {
|
||||
status: 401,
|
||||
headers: {
|
||||
'WWW-Authenticate': `Bearer realm="${this.basePath}",service="maven-registry"`,
|
||||
},
|
||||
body: { error: 'UNAUTHORIZED', message: 'Write permission required' },
|
||||
};
|
||||
}
|
||||
|
||||
if (!body) {
|
||||
return {
|
||||
status: 400,
|
||||
headers: {},
|
||||
body: { error: 'BAD_REQUEST', message: 'Request body required' },
|
||||
};
|
||||
}
|
||||
|
||||
return this.putArtifact(groupId, artifactId, version, filename, coordinate, body);
|
||||
|
||||
case 'DELETE':
|
||||
// Delete permission required
|
||||
if (!await this.checkPermission(token, resource, 'delete')) {
|
||||
return {
|
||||
status: 401,
|
||||
headers: {
|
||||
'WWW-Authenticate': `Bearer realm="${this.basePath}",service="maven-registry"`,
|
||||
},
|
||||
body: { error: 'UNAUTHORIZED', message: 'Delete permission required' },
|
||||
};
|
||||
}
|
||||
|
||||
return this.deleteArtifact(groupId, artifactId, version, filename);
|
||||
|
||||
default:
|
||||
return {
|
||||
status: 405,
|
||||
headers: { 'Allow': 'GET, HEAD, PUT, DELETE' },
|
||||
body: { error: 'METHOD_NOT_ALLOWED', message: 'Method not allowed' },
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
private async handleChecksumRequest(
|
||||
method: string,
|
||||
coordinate: IMavenCoordinate,
|
||||
token: IAuthToken | null,
|
||||
path: string
|
||||
): Promise<IResponse> {
|
||||
const { groupId, artifactId, version, extension } = coordinate;
|
||||
const resource = `${groupId}:${artifactId}`;
|
||||
|
||||
// Checksums follow the same permissions as their artifacts (public read)
|
||||
if (method === 'GET' || method === 'HEAD') {
|
||||
return this.getChecksum(groupId, artifactId, version, coordinate, path);
|
||||
}
|
||||
|
||||
// Accept PUT silently — Maven deploy-plugin uploads checksums alongside artifacts,
|
||||
// but our registry auto-generates them, so we just acknowledge the upload
|
||||
if (method === 'PUT') {
|
||||
return {
|
||||
status: 200,
|
||||
headers: {},
|
||||
body: { status: 'ok' },
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
status: 405,
|
||||
headers: { 'Allow': 'GET, HEAD, PUT' },
|
||||
body: { error: 'METHOD_NOT_ALLOWED', message: 'Checksums are auto-generated' },
|
||||
};
|
||||
}
|
||||
|
||||
private async handleMetadataRequest(
|
||||
method: string,
|
||||
path: string,
|
||||
token: IAuthToken | null,
|
||||
actor?: IRequestActor
|
||||
): Promise<IResponse> {
|
||||
// Parse path to extract groupId and artifactId
|
||||
// Path format: /com/example/my-lib/maven-metadata.xml
|
||||
const parts = path.split('/').filter(p => p && p !== 'maven-metadata.xml');
|
||||
|
||||
if (parts.length < 2) {
|
||||
return {
|
||||
status: 400,
|
||||
headers: {},
|
||||
body: { error: 'BAD_REQUEST', message: 'Invalid metadata path' },
|
||||
};
|
||||
}
|
||||
|
||||
const artifactId = parts[parts.length - 1];
|
||||
const groupId = parts.slice(0, -1).join('.');
|
||||
const resource = `${groupId}:${artifactId}`;
|
||||
|
||||
if (method === 'GET') {
|
||||
// Metadata is usually public (read permission optional)
|
||||
// Some registries allow anonymous metadata access
|
||||
return this.getMetadata(groupId, artifactId, actor);
|
||||
}
|
||||
|
||||
// Accept PUT silently — Maven deploy-plugin uploads maven-metadata.xml,
|
||||
// but our registry auto-generates it, so we just acknowledge the upload
|
||||
if (method === 'PUT') {
|
||||
return {
|
||||
status: 200,
|
||||
headers: {},
|
||||
body: { status: 'ok' },
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
status: 405,
|
||||
headers: { 'Allow': 'GET, PUT' },
|
||||
body: { error: 'METHOD_NOT_ALLOWED', message: 'Metadata is auto-generated' },
|
||||
};
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// ARTIFACT OPERATIONS
|
||||
// ========================================================================
|
||||
|
||||
private async getArtifact(
|
||||
groupId: string,
|
||||
artifactId: string,
|
||||
version: string,
|
||||
filename: string,
|
||||
actor?: IRequestActor
|
||||
): Promise<IResponse> {
|
||||
// Try local storage first (streaming)
|
||||
const streamResult = await this.storage.getMavenArtifactStream(groupId, artifactId, version, filename);
|
||||
if (streamResult) {
|
||||
const ext = filename.split('.').pop() || '';
|
||||
const contentType = this.getContentType(ext);
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': contentType,
|
||||
'Content-Length': streamResult.size.toString(),
|
||||
},
|
||||
body: streamResult.stream,
|
||||
};
|
||||
}
|
||||
|
||||
// Try upstream if not found locally
|
||||
let data: Buffer | null = null;
|
||||
const resource = `${groupId}:${artifactId}`;
|
||||
const upstream = await this.getUpstreamForRequest(resource, 'artifact', 'GET', actor);
|
||||
if (upstream) {
|
||||
// Parse the filename to extract extension and classifier
|
||||
const { extension, classifier } = this.parseFilename(filename, artifactId, version);
|
||||
if (extension) {
|
||||
data = await upstream.fetchArtifact(groupId, artifactId, version, extension, classifier);
|
||||
if (data) {
|
||||
// Cache the artifact locally
|
||||
await this.storage.putMavenArtifact(groupId, artifactId, version, filename, data);
|
||||
// Generate and store checksums
|
||||
const checksums = await calculateChecksums(data);
|
||||
await this.storeChecksums(groupId, artifactId, version, filename, checksums);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!data) {
|
||||
return {
|
||||
status: 404,
|
||||
headers: {},
|
||||
body: { error: 'NOT_FOUND', message: 'Artifact not found' },
|
||||
};
|
||||
}
|
||||
|
||||
// Determine content type based on extension
|
||||
const extension = filename.split('.').pop() || '';
|
||||
const contentType = this.getContentType(extension);
|
||||
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': contentType,
|
||||
'Content-Length': data.length.toString(),
|
||||
},
|
||||
body: data,
|
||||
};
|
||||
}
|
||||
|
||||
private async headArtifact(
|
||||
groupId: string,
|
||||
artifactId: string,
|
||||
version: string,
|
||||
filename: string
|
||||
): Promise<IResponse> {
|
||||
const exists = await this.storage.mavenArtifactExists(groupId, artifactId, version, filename);
|
||||
|
||||
if (!exists) {
|
||||
return {
|
||||
status: 404,
|
||||
headers: {},
|
||||
body: null,
|
||||
};
|
||||
}
|
||||
|
||||
// Get file size for Content-Length header
|
||||
const data = await this.storage.getMavenArtifact(groupId, artifactId, version, filename);
|
||||
const extension = filename.split('.').pop() || '';
|
||||
const contentType = this.getContentType(extension);
|
||||
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': contentType,
|
||||
'Content-Length': data ? data.length.toString() : '0',
|
||||
},
|
||||
body: null,
|
||||
};
|
||||
}
|
||||
|
||||
private async putArtifact(
|
||||
groupId: string,
|
||||
artifactId: string,
|
||||
version: string,
|
||||
filename: string,
|
||||
coordinate: IMavenCoordinate,
|
||||
body: Buffer | any
|
||||
): Promise<IResponse> {
|
||||
const data = toBuffer(body);
|
||||
|
||||
// Validate POM if uploading .pom file
|
||||
if (coordinate.extension === 'pom') {
|
||||
const pomValid = validatePom(data.toString('utf-8'));
|
||||
if (!pomValid) {
|
||||
return {
|
||||
status: 400,
|
||||
headers: {},
|
||||
body: { error: 'INVALID_POM', message: 'Invalid POM file' },
|
||||
};
|
||||
}
|
||||
|
||||
// Verify GAV matches path
|
||||
const pomGAV = extractGAVFromPom(data.toString('utf-8'));
|
||||
if (pomGAV && (pomGAV.groupId !== groupId || pomGAV.artifactId !== artifactId || pomGAV.version !== version)) {
|
||||
return {
|
||||
status: 400,
|
||||
headers: {},
|
||||
body: { error: 'GAV_MISMATCH', message: 'POM coordinates do not match upload path' },
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Store the artifact
|
||||
await this.storage.putMavenArtifact(groupId, artifactId, version, filename, data);
|
||||
|
||||
// Generate and store checksums
|
||||
const checksums = await calculateChecksums(data);
|
||||
await this.storeChecksums(groupId, artifactId, version, filename, checksums);
|
||||
|
||||
// Update maven-metadata.xml if this is a primary artifact (jar, pom, war)
|
||||
if (['jar', 'pom', 'war', 'ear', 'aar'].includes(coordinate.extension)) {
|
||||
await this.updateMetadata(groupId, artifactId, version);
|
||||
}
|
||||
|
||||
return {
|
||||
status: 201,
|
||||
headers: {
|
||||
'Location': `${this.registryUrl}/${gavToPath(groupId, artifactId, version)}/${filename}`,
|
||||
},
|
||||
body: { success: true, message: 'Artifact uploaded successfully' },
|
||||
};
|
||||
}
|
||||
|
||||
private async deleteArtifact(
|
||||
groupId: string,
|
||||
artifactId: string,
|
||||
version: string,
|
||||
filename: string
|
||||
): Promise<IResponse> {
|
||||
const exists = await this.storage.mavenArtifactExists(groupId, artifactId, version, filename);
|
||||
|
||||
if (!exists) {
|
||||
return {
|
||||
status: 404,
|
||||
headers: {},
|
||||
body: { error: 'NOT_FOUND', message: 'Artifact not found' },
|
||||
};
|
||||
}
|
||||
|
||||
await this.storage.deleteMavenArtifact(groupId, artifactId, version, filename);
|
||||
|
||||
// Also delete checksums
|
||||
for (const ext of ['md5', 'sha1', 'sha256', 'sha512']) {
|
||||
const checksumFile = `${filename}.${ext}`;
|
||||
const checksumExists = await this.storage.mavenArtifactExists(groupId, artifactId, version, checksumFile);
|
||||
if (checksumExists) {
|
||||
await this.storage.deleteMavenArtifact(groupId, artifactId, version, checksumFile);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
status: 204,
|
||||
headers: {},
|
||||
body: null,
|
||||
};
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// CHECKSUM OPERATIONS
|
||||
// ========================================================================
|
||||
|
||||
private async getChecksum(
|
||||
groupId: string,
|
||||
artifactId: string,
|
||||
version: string,
|
||||
coordinate: IMavenCoordinate,
|
||||
fullPath: string
|
||||
): Promise<IResponse> {
|
||||
// Extract the filename from the full path (last component)
|
||||
// The fullPath might be something like /com/example/test/test-artifact/1.0.0/test-artifact-1.0.0.jar.md5
|
||||
const pathParts = fullPath.split('/');
|
||||
const checksumFilename = pathParts[pathParts.length - 1];
|
||||
|
||||
const data = await this.storage.getMavenArtifact(groupId, artifactId, version, checksumFilename);
|
||||
|
||||
if (!data) {
|
||||
return {
|
||||
status: 404,
|
||||
headers: {},
|
||||
body: { error: 'NOT_FOUND', message: 'Checksum not found' },
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'text/plain',
|
||||
'Content-Length': data.length.toString(),
|
||||
},
|
||||
body: data,
|
||||
};
|
||||
}
|
||||
|
||||
private async storeChecksums(
|
||||
groupId: string,
|
||||
artifactId: string,
|
||||
version: string,
|
||||
filename: string,
|
||||
checksums: IChecksums
|
||||
): Promise<void> {
|
||||
// Store each checksum as a separate file
|
||||
await this.storage.putMavenArtifact(
|
||||
groupId,
|
||||
artifactId,
|
||||
version,
|
||||
`${filename}.md5`,
|
||||
Buffer.from(checksums.md5, 'utf-8')
|
||||
);
|
||||
|
||||
await this.storage.putMavenArtifact(
|
||||
groupId,
|
||||
artifactId,
|
||||
version,
|
||||
`${filename}.sha1`,
|
||||
Buffer.from(checksums.sha1, 'utf-8')
|
||||
);
|
||||
|
||||
if (checksums.sha256) {
|
||||
await this.storage.putMavenArtifact(
|
||||
groupId,
|
||||
artifactId,
|
||||
version,
|
||||
`${filename}.sha256`,
|
||||
Buffer.from(checksums.sha256, 'utf-8')
|
||||
);
|
||||
}
|
||||
|
||||
if (checksums.sha512) {
|
||||
await this.storage.putMavenArtifact(
|
||||
groupId,
|
||||
artifactId,
|
||||
version,
|
||||
`${filename}.sha512`,
|
||||
Buffer.from(checksums.sha512, 'utf-8')
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// METADATA OPERATIONS
|
||||
// ========================================================================
|
||||
|
||||
private async getMetadata(groupId: string, artifactId: string, actor?: IRequestActor): Promise<IResponse> {
|
||||
let metadataBuffer = await this.storage.getMavenMetadata(groupId, artifactId);
|
||||
|
||||
// Try upstream if not found locally
|
||||
if (!metadataBuffer) {
|
||||
const resource = `${groupId}:${artifactId}`;
|
||||
const upstream = await this.getUpstreamForRequest(resource, 'metadata', 'GET', actor);
|
||||
if (upstream) {
|
||||
const upstreamMetadata = await upstream.fetchMetadata(groupId, artifactId);
|
||||
if (upstreamMetadata) {
|
||||
metadataBuffer = Buffer.from(upstreamMetadata, 'utf-8');
|
||||
// Cache the metadata locally
|
||||
await this.storage.putMavenMetadata(groupId, artifactId, metadataBuffer);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!metadataBuffer) {
|
||||
// Generate empty metadata if none exists
|
||||
const emptyMetadata: IMavenMetadata = {
|
||||
groupId,
|
||||
artifactId,
|
||||
versioning: {
|
||||
versions: [],
|
||||
lastUpdated: formatMavenTimestamp(new Date()),
|
||||
},
|
||||
};
|
||||
|
||||
const xml = generateMetadataXml(emptyMetadata);
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/xml',
|
||||
'Content-Length': xml.length.toString(),
|
||||
},
|
||||
body: Buffer.from(xml, 'utf-8'),
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/xml',
|
||||
'Content-Length': metadataBuffer.length.toString(),
|
||||
},
|
||||
body: metadataBuffer,
|
||||
};
|
||||
}
|
||||
|
||||
private async updateMetadata(
|
||||
groupId: string,
|
||||
artifactId: string,
|
||||
newVersion: string
|
||||
): Promise<void> {
|
||||
// Get existing metadata or create new
|
||||
const existingBuffer = await this.storage.getMavenMetadata(groupId, artifactId);
|
||||
let metadata: IMavenMetadata;
|
||||
|
||||
if (existingBuffer) {
|
||||
const parsed = parseMetadataXml(existingBuffer.toString('utf-8'));
|
||||
if (parsed) {
|
||||
metadata = parsed;
|
||||
} else {
|
||||
// Create new if parsing failed
|
||||
metadata = {
|
||||
groupId,
|
||||
artifactId,
|
||||
versioning: {
|
||||
versions: [],
|
||||
lastUpdated: formatMavenTimestamp(new Date()),
|
||||
},
|
||||
};
|
||||
}
|
||||
} else {
|
||||
metadata = {
|
||||
groupId,
|
||||
artifactId,
|
||||
versioning: {
|
||||
versions: [],
|
||||
lastUpdated: formatMavenTimestamp(new Date()),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// Add new version if not already present
|
||||
if (!metadata.versioning.versions.includes(newVersion)) {
|
||||
metadata.versioning.versions.push(newVersion);
|
||||
metadata.versioning.versions.sort(); // Sort versions
|
||||
}
|
||||
|
||||
// Update latest and release
|
||||
const versions = metadata.versioning.versions;
|
||||
metadata.versioning.latest = versions[versions.length - 1];
|
||||
|
||||
// Release is the latest non-SNAPSHOT version
|
||||
const releaseVersions = versions.filter(v => !isSnapshot(v));
|
||||
if (releaseVersions.length > 0) {
|
||||
metadata.versioning.release = releaseVersions[releaseVersions.length - 1];
|
||||
}
|
||||
|
||||
// Update timestamp
|
||||
metadata.versioning.lastUpdated = formatMavenTimestamp(new Date());
|
||||
|
||||
// Generate and store XML
|
||||
const xml = generateMetadataXml(metadata);
|
||||
await this.storage.putMavenMetadata(groupId, artifactId, Buffer.from(xml, 'utf-8'));
|
||||
|
||||
// Note: Checksums for maven-metadata.xml are optional and not critical
|
||||
// They would need special handling since metadata uses a different storage path
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// UTILITY METHODS
|
||||
// ========================================================================
|
||||
|
||||
private getContentType(extension: string): string {
|
||||
const contentTypes: Record<string, string> = {
|
||||
'jar': 'application/java-archive',
|
||||
'war': 'application/java-archive',
|
||||
'ear': 'application/java-archive',
|
||||
'aar': 'application/java-archive',
|
||||
'pom': 'application/xml',
|
||||
'xml': 'application/xml',
|
||||
'md5': 'text/plain',
|
||||
'sha1': 'text/plain',
|
||||
'sha256': 'text/plain',
|
||||
'sha512': 'text/plain',
|
||||
};
|
||||
|
||||
return contentTypes[extension] || 'application/octet-stream';
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a Maven filename to extract extension and classifier.
|
||||
* Filename format: {artifactId}-{version}[-{classifier}].{extension}
|
||||
*/
|
||||
private parseFilename(
|
||||
filename: string,
|
||||
artifactId: string,
|
||||
version: string
|
||||
): { extension: string; classifier?: string } {
|
||||
const prefix = `${artifactId}-${version}`;
|
||||
|
||||
if (!filename.startsWith(prefix)) {
|
||||
// Fallback: just get the extension
|
||||
const lastDot = filename.lastIndexOf('.');
|
||||
return { extension: lastDot > 0 ? filename.slice(lastDot + 1) : '' };
|
||||
}
|
||||
|
||||
const remainder = filename.slice(prefix.length);
|
||||
// remainder is either ".extension" or "-classifier.extension"
|
||||
|
||||
if (remainder.startsWith('.')) {
|
||||
return { extension: remainder.slice(1) };
|
||||
}
|
||||
|
||||
if (remainder.startsWith('-')) {
|
||||
const lastDot = remainder.lastIndexOf('.');
|
||||
if (lastDot > 1) {
|
||||
return {
|
||||
classifier: remainder.slice(1, lastDot),
|
||||
extension: remainder.slice(lastDot + 1),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return { extension: '' };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
import * as plugins from '../plugins.js';
|
||||
import { BaseUpstream } from '../upstream/classes.baseupstream.js';
|
||||
import type {
|
||||
IProtocolUpstreamConfig,
|
||||
IUpstreamFetchContext,
|
||||
IUpstreamRegistryConfig,
|
||||
} from '../upstream/interfaces.upstream.js';
|
||||
import type { IMavenCoordinate } from './interfaces.maven.js';
|
||||
|
||||
/**
|
||||
* Maven-specific upstream implementation.
|
||||
*
|
||||
* Handles:
|
||||
* - Artifact fetching (JAR, POM, WAR, etc.)
|
||||
* - Metadata fetching (maven-metadata.xml)
|
||||
* - Checksum files (.md5, .sha1, .sha256, .sha512)
|
||||
* - SNAPSHOT version handling
|
||||
* - Content-addressable caching for release artifacts
|
||||
*/
|
||||
export class MavenUpstream extends BaseUpstream {
|
||||
protected readonly protocolName = 'maven';
|
||||
|
||||
constructor(
|
||||
config: IProtocolUpstreamConfig,
|
||||
logger?: plugins.smartlog.Smartlog,
|
||||
) {
|
||||
super(config, logger);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch an artifact from upstream registries.
|
||||
*/
|
||||
public async fetchArtifact(
|
||||
groupId: string,
|
||||
artifactId: string,
|
||||
version: string,
|
||||
extension: string,
|
||||
classifier?: string,
|
||||
): Promise<Buffer | null> {
|
||||
const path = this.buildArtifactPath(groupId, artifactId, version, extension, classifier);
|
||||
const resource = `${groupId}:${artifactId}`;
|
||||
|
||||
const context: IUpstreamFetchContext = {
|
||||
protocol: 'maven',
|
||||
resource,
|
||||
resourceType: 'artifact',
|
||||
path,
|
||||
method: 'GET',
|
||||
headers: {},
|
||||
query: {},
|
||||
};
|
||||
|
||||
const result = await this.fetch(context);
|
||||
|
||||
if (!result || !result.success) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return Buffer.isBuffer(result.body) ? result.body : Buffer.from(result.body);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch maven-metadata.xml from upstream.
|
||||
*/
|
||||
public async fetchMetadata(groupId: string, artifactId: string, version?: string): Promise<string | null> {
|
||||
const groupPath = groupId.replace(/\./g, '/');
|
||||
let path: string;
|
||||
|
||||
if (version) {
|
||||
// Version-level metadata (for SNAPSHOTs)
|
||||
path = `/${groupPath}/${artifactId}/${version}/maven-metadata.xml`;
|
||||
} else {
|
||||
// Artifact-level metadata (lists all versions)
|
||||
path = `/${groupPath}/${artifactId}/maven-metadata.xml`;
|
||||
}
|
||||
|
||||
const resource = `${groupId}:${artifactId}`;
|
||||
|
||||
const context: IUpstreamFetchContext = {
|
||||
protocol: 'maven',
|
||||
resource,
|
||||
resourceType: 'metadata',
|
||||
path,
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'accept': 'application/xml, text/xml',
|
||||
},
|
||||
query: {},
|
||||
};
|
||||
|
||||
const result = await this.fetch(context);
|
||||
|
||||
if (!result || !result.success) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (Buffer.isBuffer(result.body)) {
|
||||
return result.body.toString('utf8');
|
||||
}
|
||||
|
||||
return typeof result.body === 'string' ? result.body : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a checksum file from upstream.
|
||||
*/
|
||||
public async fetchChecksum(
|
||||
groupId: string,
|
||||
artifactId: string,
|
||||
version: string,
|
||||
extension: string,
|
||||
checksumType: 'md5' | 'sha1' | 'sha256' | 'sha512',
|
||||
classifier?: string,
|
||||
): Promise<string | null> {
|
||||
const basePath = this.buildArtifactPath(groupId, artifactId, version, extension, classifier);
|
||||
const path = `${basePath}.${checksumType}`;
|
||||
const resource = `${groupId}:${artifactId}`;
|
||||
|
||||
const context: IUpstreamFetchContext = {
|
||||
protocol: 'maven',
|
||||
resource,
|
||||
resourceType: 'checksum',
|
||||
path,
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'accept': 'text/plain',
|
||||
},
|
||||
query: {},
|
||||
};
|
||||
|
||||
const result = await this.fetch(context);
|
||||
|
||||
if (!result || !result.success) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (Buffer.isBuffer(result.body)) {
|
||||
return result.body.toString('utf8').trim();
|
||||
}
|
||||
|
||||
return typeof result.body === 'string' ? result.body.trim() : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if an artifact exists in upstream (HEAD request).
|
||||
*/
|
||||
public async headArtifact(
|
||||
groupId: string,
|
||||
artifactId: string,
|
||||
version: string,
|
||||
extension: string,
|
||||
classifier?: string,
|
||||
): Promise<{ exists: boolean; size?: number; lastModified?: string } | null> {
|
||||
const path = this.buildArtifactPath(groupId, artifactId, version, extension, classifier);
|
||||
const resource = `${groupId}:${artifactId}`;
|
||||
|
||||
const context: IUpstreamFetchContext = {
|
||||
protocol: 'maven',
|
||||
resource,
|
||||
resourceType: 'artifact',
|
||||
path,
|
||||
method: 'HEAD',
|
||||
headers: {},
|
||||
query: {},
|
||||
};
|
||||
|
||||
const result = await this.fetch(context);
|
||||
|
||||
if (!result) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!result.success) {
|
||||
return { exists: false };
|
||||
}
|
||||
|
||||
return {
|
||||
exists: true,
|
||||
size: result.headers['content-length'] ? parseInt(result.headers['content-length'], 10) : undefined,
|
||||
lastModified: result.headers['last-modified'],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the path for a Maven artifact.
|
||||
*/
|
||||
private buildArtifactPath(
|
||||
groupId: string,
|
||||
artifactId: string,
|
||||
version: string,
|
||||
extension: string,
|
||||
classifier?: string,
|
||||
): string {
|
||||
const groupPath = groupId.replace(/\./g, '/');
|
||||
let filename = `${artifactId}-${version}`;
|
||||
if (classifier) {
|
||||
filename += `-${classifier}`;
|
||||
}
|
||||
filename += `.${extension}`;
|
||||
|
||||
return `/${groupPath}/${artifactId}/${version}/${filename}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Override URL building for Maven-specific handling.
|
||||
*/
|
||||
protected buildUpstreamUrl(
|
||||
upstream: IUpstreamRegistryConfig,
|
||||
context: IUpstreamFetchContext,
|
||||
): string {
|
||||
let baseUrl = upstream.url;
|
||||
|
||||
// Remove trailing slash
|
||||
if (baseUrl.endsWith('/')) {
|
||||
baseUrl = baseUrl.slice(0, -1);
|
||||
}
|
||||
|
||||
return `${baseUrl}${context.path}`;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,346 @@
|
||||
/**
|
||||
* Maven helper utilities
|
||||
* Path conversion, XML generation, checksum calculation
|
||||
*/
|
||||
|
||||
import * as plugins from '../plugins.js';
|
||||
import type {
|
||||
IMavenCoordinate,
|
||||
IMavenMetadata,
|
||||
IChecksums,
|
||||
IMavenPom,
|
||||
} from './interfaces.maven.js';
|
||||
|
||||
/**
|
||||
* Convert Maven GAV coordinates to storage path
|
||||
* Example: com.example:my-lib:1.0.0 → com/example/my-lib/1.0.0
|
||||
*/
|
||||
export function gavToPath(
|
||||
groupId: string,
|
||||
artifactId: string,
|
||||
version?: string
|
||||
): string {
|
||||
const groupPath = groupId.replace(/\./g, '/');
|
||||
if (version) {
|
||||
return `${groupPath}/${artifactId}/${version}`;
|
||||
}
|
||||
return `${groupPath}/${artifactId}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse Maven path to GAV coordinates
|
||||
* Example: com/example/my-lib/1.0.0/my-lib-1.0.0.jar → {groupId, artifactId, version, ...}
|
||||
*/
|
||||
export function pathToGAV(path: string): IMavenCoordinate | null {
|
||||
// Remove leading slash if present
|
||||
const cleanPath = path.startsWith('/') ? path.substring(1) : path;
|
||||
|
||||
// Split path into parts
|
||||
const parts = cleanPath.split('/');
|
||||
if (parts.length < 4) {
|
||||
return null; // Not a valid artifact path
|
||||
}
|
||||
|
||||
// Last part is filename
|
||||
const filename = parts[parts.length - 1];
|
||||
const version = parts[parts.length - 2];
|
||||
const artifactId = parts[parts.length - 3];
|
||||
const groupId = parts.slice(0, -3).join('.');
|
||||
|
||||
// Parse filename to extract classifier and extension
|
||||
const parsed = parseFilename(filename, artifactId, version);
|
||||
if (!parsed) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
groupId,
|
||||
artifactId,
|
||||
version,
|
||||
classifier: parsed.classifier,
|
||||
extension: parsed.extension,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse Maven artifact filename
|
||||
* Example: my-lib-1.0.0-sources.jar → {classifier: 'sources', extension: 'jar'}
|
||||
* Example: my-lib-1.0.0.jar.md5 → {extension: 'md5'}
|
||||
*/
|
||||
export function parseFilename(
|
||||
filename: string,
|
||||
artifactId: string,
|
||||
version: string
|
||||
): { classifier?: string; extension: string } | null {
|
||||
// Expected format: {artifactId}-{version}[-{classifier}].{extension}[.checksum]
|
||||
const prefix = `${artifactId}-${version}`;
|
||||
|
||||
if (!filename.startsWith(prefix)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
let remainder = filename.substring(prefix.length);
|
||||
|
||||
// Check if this is a checksum file (double extension like .jar.md5)
|
||||
const checksumExtensions = ['md5', 'sha1', 'sha256', 'sha512'];
|
||||
const lastDotIndex = remainder.lastIndexOf('.');
|
||||
if (lastDotIndex !== -1) {
|
||||
const possibleChecksum = remainder.substring(lastDotIndex + 1);
|
||||
if (checksumExtensions.includes(possibleChecksum)) {
|
||||
// This is a checksum file - just return the checksum extension
|
||||
// The base artifact extension doesn't matter for checksum retrieval
|
||||
return { extension: possibleChecksum };
|
||||
}
|
||||
}
|
||||
|
||||
// Regular artifact file parsing
|
||||
const dotIndex = remainder.lastIndexOf('.');
|
||||
if (dotIndex === -1) {
|
||||
return null; // No extension
|
||||
}
|
||||
|
||||
const extension = remainder.substring(dotIndex + 1);
|
||||
const classifierPart = remainder.substring(0, dotIndex);
|
||||
|
||||
if (classifierPart.length === 0) {
|
||||
// No classifier
|
||||
return { extension };
|
||||
}
|
||||
|
||||
if (classifierPart.startsWith('-')) {
|
||||
// Has classifier
|
||||
const classifier = classifierPart.substring(1);
|
||||
return { classifier, extension };
|
||||
}
|
||||
|
||||
return null; // Invalid format
|
||||
}
|
||||
|
||||
/**
|
||||
* Build Maven artifact filename
|
||||
* Example: {artifactId: 'my-lib', version: '1.0.0', classifier: 'sources', extension: 'jar'}
|
||||
* → 'my-lib-1.0.0-sources.jar'
|
||||
*/
|
||||
export function buildFilename(coordinate: IMavenCoordinate): string {
|
||||
const { artifactId, version, classifier, extension } = coordinate;
|
||||
|
||||
let filename = `${artifactId}-${version}`;
|
||||
if (classifier) {
|
||||
filename += `-${classifier}`;
|
||||
}
|
||||
filename += `.${extension}`;
|
||||
|
||||
return filename;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate checksums for Maven artifact
|
||||
* Returns MD5, SHA-1, SHA-256, SHA-512
|
||||
*/
|
||||
export async function calculateChecksums(data: Buffer): Promise<IChecksums> {
|
||||
const crypto = await import('crypto');
|
||||
|
||||
return {
|
||||
md5: crypto.createHash('md5').update(data).digest('hex'),
|
||||
sha1: crypto.createHash('sha1').update(data).digest('hex'),
|
||||
sha256: crypto.createHash('sha256').update(data).digest('hex'),
|
||||
sha512: crypto.createHash('sha512').update(data).digest('hex'),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate maven-metadata.xml from metadata object
|
||||
*/
|
||||
export function generateMetadataXml(metadata: IMavenMetadata): string {
|
||||
const { groupId, artifactId, versioning } = metadata;
|
||||
const { latest, release, versions, lastUpdated, snapshot, snapshotVersions } = versioning;
|
||||
|
||||
let xml = '<?xml version="1.0" encoding="UTF-8"?>\n';
|
||||
xml += '<metadata>\n';
|
||||
xml += ` <groupId>${escapeXml(groupId)}</groupId>\n`;
|
||||
xml += ` <artifactId>${escapeXml(artifactId)}</artifactId>\n`;
|
||||
|
||||
// Add version if SNAPSHOT
|
||||
if (snapshot) {
|
||||
const snapshotVersion = versions[versions.length - 1]; // Assume last version is the SNAPSHOT
|
||||
xml += ` <version>${escapeXml(snapshotVersion)}</version>\n`;
|
||||
}
|
||||
|
||||
xml += ' <versioning>\n';
|
||||
|
||||
if (latest) {
|
||||
xml += ` <latest>${escapeXml(latest)}</latest>\n`;
|
||||
}
|
||||
|
||||
if (release) {
|
||||
xml += ` <release>${escapeXml(release)}</release>\n`;
|
||||
}
|
||||
|
||||
xml += ' <versions>\n';
|
||||
for (const version of versions) {
|
||||
xml += ` <version>${escapeXml(version)}</version>\n`;
|
||||
}
|
||||
xml += ' </versions>\n';
|
||||
|
||||
xml += ` <lastUpdated>${lastUpdated}</lastUpdated>\n`;
|
||||
|
||||
// Add SNAPSHOT info if present
|
||||
if (snapshot) {
|
||||
xml += ' <snapshot>\n';
|
||||
xml += ` <timestamp>${escapeXml(snapshot.timestamp)}</timestamp>\n`;
|
||||
xml += ` <buildNumber>${snapshot.buildNumber}</buildNumber>\n`;
|
||||
xml += ' </snapshot>\n';
|
||||
}
|
||||
|
||||
// Add SNAPSHOT versions if present
|
||||
if (snapshotVersions && snapshotVersions.length > 0) {
|
||||
xml += ' <snapshotVersions>\n';
|
||||
for (const sv of snapshotVersions) {
|
||||
xml += ' <snapshotVersion>\n';
|
||||
if (sv.classifier) {
|
||||
xml += ` <classifier>${escapeXml(sv.classifier)}</classifier>\n`;
|
||||
}
|
||||
xml += ` <extension>${escapeXml(sv.extension)}</extension>\n`;
|
||||
xml += ` <value>${escapeXml(sv.value)}</value>\n`;
|
||||
xml += ` <updated>${sv.updated}</updated>\n`;
|
||||
xml += ' </snapshotVersion>\n';
|
||||
}
|
||||
xml += ' </snapshotVersions>\n';
|
||||
}
|
||||
|
||||
xml += ' </versioning>\n';
|
||||
xml += '</metadata>\n';
|
||||
|
||||
return xml;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse maven-metadata.xml to metadata object
|
||||
* Basic XML parsing for Maven metadata
|
||||
*/
|
||||
export function parseMetadataXml(xml: string): IMavenMetadata | null {
|
||||
try {
|
||||
// Simple regex-based parsing (for basic metadata)
|
||||
// In production, use a proper XML parser
|
||||
|
||||
const groupIdMatch = xml.match(/<groupId>([^<]+)<\/groupId>/);
|
||||
const artifactIdMatch = xml.match(/<artifactId>([^<]+)<\/artifactId>/);
|
||||
const latestMatch = xml.match(/<latest>([^<]+)<\/latest>/);
|
||||
const releaseMatch = xml.match(/<release>([^<]+)<\/release>/);
|
||||
const lastUpdatedMatch = xml.match(/<lastUpdated>([^<]+)<\/lastUpdated>/);
|
||||
|
||||
if (!groupIdMatch || !artifactIdMatch) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Parse versions
|
||||
const versionsMatch = xml.match(/<versions>([\s\S]*?)<\/versions>/);
|
||||
const versions: string[] = [];
|
||||
if (versionsMatch) {
|
||||
const versionMatches = versionsMatch[1].matchAll(/<version>([^<]+)<\/version>/g);
|
||||
for (const match of versionMatches) {
|
||||
versions.push(match[1]);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
groupId: groupIdMatch[1],
|
||||
artifactId: artifactIdMatch[1],
|
||||
versioning: {
|
||||
latest: latestMatch ? latestMatch[1] : undefined,
|
||||
release: releaseMatch ? releaseMatch[1] : undefined,
|
||||
versions,
|
||||
lastUpdated: lastUpdatedMatch ? lastUpdatedMatch[1] : formatMavenTimestamp(new Date()),
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Escape XML special characters
|
||||
*/
|
||||
function escapeXml(str: string): string {
|
||||
return str
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, ''');
|
||||
}
|
||||
|
||||
/**
|
||||
* Format timestamp in Maven format: yyyyMMddHHmmss
|
||||
*/
|
||||
export function formatMavenTimestamp(date: Date): string {
|
||||
const year = date.getUTCFullYear();
|
||||
const month = String(date.getUTCMonth() + 1).padStart(2, '0');
|
||||
const day = String(date.getUTCDate()).padStart(2, '0');
|
||||
const hours = String(date.getUTCHours()).padStart(2, '0');
|
||||
const minutes = String(date.getUTCMinutes()).padStart(2, '0');
|
||||
const seconds = String(date.getUTCSeconds()).padStart(2, '0');
|
||||
|
||||
return `${year}${month}${day}${hours}${minutes}${seconds}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format SNAPSHOT timestamp: yyyyMMdd.HHmmss
|
||||
*/
|
||||
export function formatSnapshotTimestamp(date: Date): string {
|
||||
const year = date.getUTCFullYear();
|
||||
const month = String(date.getUTCMonth() + 1).padStart(2, '0');
|
||||
const day = String(date.getUTCDate()).padStart(2, '0');
|
||||
const hours = String(date.getUTCHours()).padStart(2, '0');
|
||||
const minutes = String(date.getUTCMinutes()).padStart(2, '0');
|
||||
const seconds = String(date.getUTCSeconds()).padStart(2, '0');
|
||||
|
||||
return `${year}${month}${day}.${hours}${minutes}${seconds}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if version is a SNAPSHOT
|
||||
*/
|
||||
export function isSnapshot(version: string): boolean {
|
||||
return version.endsWith('-SNAPSHOT');
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate POM basic structure
|
||||
*/
|
||||
export function validatePom(pomXml: string): boolean {
|
||||
try {
|
||||
// Basic validation - check for required fields
|
||||
return (
|
||||
pomXml.includes('<groupId>') &&
|
||||
pomXml.includes('<artifactId>') &&
|
||||
pomXml.includes('<version>') &&
|
||||
pomXml.includes('<modelVersion>')
|
||||
);
|
||||
} catch (error) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract GAV from POM XML
|
||||
*/
|
||||
export function extractGAVFromPom(pomXml: string): { groupId: string; artifactId: string; version: string } | null {
|
||||
try {
|
||||
const groupIdMatch = pomXml.match(/<groupId>([^<]+)<\/groupId>/);
|
||||
const artifactIdMatch = pomXml.match(/<artifactId>([^<]+)<\/artifactId>/);
|
||||
const versionMatch = pomXml.match(/<version>([^<]+)<\/version>/);
|
||||
|
||||
if (groupIdMatch && artifactIdMatch && versionMatch) {
|
||||
return {
|
||||
groupId: groupIdMatch[1],
|
||||
artifactId: artifactIdMatch[1],
|
||||
version: versionMatch[1],
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
} catch (error) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
/**
|
||||
* Maven Registry module exports
|
||||
*/
|
||||
|
||||
export { MavenRegistry } from './classes.mavenregistry.js';
|
||||
export { MavenUpstream } from './classes.mavenupstream.js';
|
||||
export * from './interfaces.maven.js';
|
||||
export * from './helpers.maven.js';
|
||||
@@ -0,0 +1,127 @@
|
||||
/**
|
||||
* Maven registry type definitions
|
||||
* Supports Maven repository protocol for Java artifacts
|
||||
*/
|
||||
|
||||
/**
|
||||
* Maven coordinate system (GAV + optional classifier)
|
||||
* Example: com.example:my-library:1.0.0:sources:jar
|
||||
*/
|
||||
export interface IMavenCoordinate {
|
||||
groupId: string; // e.g., "com.example.myapp"
|
||||
artifactId: string; // e.g., "my-library"
|
||||
version: string; // e.g., "1.0.0" or "1.0-SNAPSHOT"
|
||||
classifier?: string; // e.g., "sources", "javadoc"
|
||||
extension: string; // e.g., "jar", "war", "pom"
|
||||
}
|
||||
|
||||
/**
|
||||
* Maven metadata (maven-metadata.xml) structure
|
||||
* Contains version list and latest/release information
|
||||
*/
|
||||
export interface IMavenMetadata {
|
||||
groupId: string;
|
||||
artifactId: string;
|
||||
versioning: IMavenVersioning;
|
||||
}
|
||||
|
||||
/**
|
||||
* Maven versioning information
|
||||
*/
|
||||
export interface IMavenVersioning {
|
||||
latest?: string; // Latest version (including SNAPSHOTs)
|
||||
release?: string; // Latest release version (excluding SNAPSHOTs)
|
||||
versions: string[]; // List of all versions
|
||||
lastUpdated: string; // Format: yyyyMMddHHmmss
|
||||
snapshot?: IMavenSnapshot; // For SNAPSHOT versions
|
||||
snapshotVersions?: IMavenSnapshotVersion[]; // For SNAPSHOT builds
|
||||
}
|
||||
|
||||
/**
|
||||
* SNAPSHOT build information
|
||||
*/
|
||||
export interface IMavenSnapshot {
|
||||
timestamp: string; // Format: yyyyMMdd.HHmmss
|
||||
buildNumber: number; // Incremental build number
|
||||
}
|
||||
|
||||
/**
|
||||
* SNAPSHOT version entry
|
||||
*/
|
||||
export interface IMavenSnapshotVersion {
|
||||
classifier?: string;
|
||||
extension: string;
|
||||
value: string; // Timestamped version
|
||||
updated: string; // Format: yyyyMMddHHmmss
|
||||
}
|
||||
|
||||
/**
|
||||
* Checksums for Maven artifacts
|
||||
* Maven requires separate checksum files for each artifact
|
||||
*/
|
||||
export interface IChecksums {
|
||||
md5: string; // MD5 hash
|
||||
sha1: string; // SHA-1 hash (required)
|
||||
sha256?: string; // SHA-256 hash (optional)
|
||||
sha512?: string; // SHA-512 hash (optional)
|
||||
}
|
||||
|
||||
/**
|
||||
* Maven artifact file information
|
||||
*/
|
||||
export interface IMavenArtifactFile {
|
||||
filename: string; // Full filename with extension
|
||||
data: Buffer; // File content
|
||||
coordinate: IMavenCoordinate; // Parsed GAV coordinates
|
||||
checksums?: IChecksums; // Calculated checksums
|
||||
}
|
||||
|
||||
/**
|
||||
* Maven upload request
|
||||
* Contains all files for a single version (JAR, POM, sources, etc.)
|
||||
*/
|
||||
export interface IMavenUploadRequest {
|
||||
groupId: string;
|
||||
artifactId: string;
|
||||
version: string;
|
||||
files: IMavenArtifactFile[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Maven protocol configuration
|
||||
*/
|
||||
export interface IMavenProtocolConfig {
|
||||
enabled: boolean;
|
||||
basePath: string; // Default: '/maven'
|
||||
features?: {
|
||||
snapshots?: boolean; // Support SNAPSHOT versions (default: true)
|
||||
checksums?: boolean; // Auto-generate checksums (default: true)
|
||||
metadata?: boolean; // Auto-generate maven-metadata.xml (default: true)
|
||||
allowedExtensions?: string[]; // Allowed file extensions (default: jar, war, pom, etc.)
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Maven POM (Project Object Model) minimal structure
|
||||
* Only essential fields for validation
|
||||
*/
|
||||
export interface IMavenPom {
|
||||
modelVersion: string; // Always "4.0.0"
|
||||
groupId: string;
|
||||
artifactId: string;
|
||||
version: string;
|
||||
packaging?: string; // jar, war, pom, etc.
|
||||
name?: string;
|
||||
description?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Maven repository search result
|
||||
*/
|
||||
export interface IMavenSearchResult {
|
||||
groupId: string;
|
||||
artifactId: string;
|
||||
latestVersion: string;
|
||||
versions: string[];
|
||||
lastUpdated: string;
|
||||
}
|
||||
+272
-149
@@ -2,10 +2,11 @@ import { Smartlog } from '@push.rocks/smartlog';
|
||||
import { BaseRegistry } from '../core/classes.baseregistry.js';
|
||||
import { RegistryStorage } from '../core/classes.registrystorage.js';
|
||||
import { AuthManager } from '../core/classes.authmanager.js';
|
||||
import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js';
|
||||
import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from '../core/interfaces.core.js';
|
||||
import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
|
||||
import { NpmUpstream } from './classes.npmupstream.js';
|
||||
import type {
|
||||
IPackument,
|
||||
INpmVersion,
|
||||
IPublishRequest,
|
||||
ISearchResponse,
|
||||
ISearchResult,
|
||||
@@ -14,6 +15,13 @@ import type {
|
||||
IUserAuthRequest,
|
||||
INpmError,
|
||||
} from './interfaces.npm.js';
|
||||
import {
|
||||
createNewPackument,
|
||||
getAttachmentForVersion,
|
||||
preparePublishedVersion,
|
||||
recordPublishedVersion,
|
||||
} from './helpers.npmpublish.js';
|
||||
import { parseNpmRequestRoute } from './helpers.npmroutes.js';
|
||||
|
||||
/**
|
||||
* NPM Registry implementation
|
||||
@@ -25,31 +33,67 @@ export class NpmRegistry extends BaseRegistry {
|
||||
private basePath: string = '/npm';
|
||||
private registryUrl: string;
|
||||
private logger: Smartlog;
|
||||
private upstreamProvider: IUpstreamProvider | null = null;
|
||||
|
||||
constructor(
|
||||
storage: RegistryStorage,
|
||||
authManager: AuthManager,
|
||||
basePath: string = '/npm',
|
||||
registryUrl: string = 'http://localhost:5000/npm'
|
||||
registryUrl: string = 'http://localhost:5000/npm',
|
||||
upstreamProvider?: IUpstreamProvider
|
||||
) {
|
||||
super();
|
||||
this.storage = storage;
|
||||
this.authManager = authManager;
|
||||
this.basePath = basePath;
|
||||
this.registryUrl = registryUrl;
|
||||
this.upstreamProvider = upstreamProvider || null;
|
||||
|
||||
// Initialize logger
|
||||
this.logger = new Smartlog({
|
||||
logContext: {
|
||||
company: 'push.rocks',
|
||||
companyunit: 'smartregistry',
|
||||
containerName: 'npm-registry',
|
||||
environment: (process.env.NODE_ENV as any) || 'development',
|
||||
runtime: 'node',
|
||||
zone: 'npm'
|
||||
this.logger = this.createProtocolLogger('npm-registry', 'npm');
|
||||
|
||||
if (upstreamProvider) {
|
||||
this.logger.log('info', 'NPM upstream provider configured');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract scope from npm package name.
|
||||
* @example "@company/utils" -> "company"
|
||||
* @example "lodash" -> null
|
||||
*/
|
||||
private extractScope(packageName: string): string | null {
|
||||
if (packageName.startsWith('@')) {
|
||||
const slashIndex = packageName.indexOf('/');
|
||||
if (slashIndex > 1) {
|
||||
return packageName.substring(1, slashIndex);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get upstream for a specific request.
|
||||
* Calls the provider to resolve upstream config dynamically.
|
||||
*/
|
||||
private async getUpstreamForRequest(
|
||||
resource: string,
|
||||
resourceType: string,
|
||||
method: string,
|
||||
actor?: IRequestActor
|
||||
): Promise<NpmUpstream | null> {
|
||||
if (!this.upstreamProvider) return null;
|
||||
|
||||
const config = await this.upstreamProvider.resolveUpstreamConfig({
|
||||
protocol: 'npm',
|
||||
resource,
|
||||
scope: this.extractScope(resource),
|
||||
actor,
|
||||
method,
|
||||
resourceType,
|
||||
});
|
||||
this.logger.enableConsole();
|
||||
|
||||
if (!config?.enabled) return null;
|
||||
return new NpmUpstream(config, this.registryUrl, this.logger);
|
||||
}
|
||||
|
||||
public async init(): Promise<void> {
|
||||
@@ -63,84 +107,20 @@ export class NpmRegistry extends BaseRegistry {
|
||||
public async handleRequest(context: IRequestContext): Promise<IResponse> {
|
||||
const path = context.path.replace(this.basePath, '');
|
||||
|
||||
// Extract token from Authorization header
|
||||
const authHeader = context.headers['authorization'] || context.headers['Authorization'];
|
||||
const tokenString = authHeader?.replace(/^Bearer\s+/i, '');
|
||||
const tokenString = this.extractBearerToken(context);
|
||||
const token = tokenString ? await this.authManager.validateToken(tokenString, 'npm') : null;
|
||||
|
||||
const actor: IRequestActor = this.buildRequestActor(context, token);
|
||||
|
||||
this.logger.log('debug', `handleRequest: ${context.method} ${path}`, {
|
||||
method: context.method,
|
||||
path,
|
||||
hasAuth: !!token
|
||||
});
|
||||
|
||||
// Registry root
|
||||
if (path === '/' || path === '') {
|
||||
return this.handleRegistryInfo();
|
||||
}
|
||||
|
||||
// Search: /-/v1/search
|
||||
if (path.startsWith('/-/v1/search')) {
|
||||
return this.handleSearch(context.query);
|
||||
}
|
||||
|
||||
// User authentication: /-/user/org.couchdb.user:{username}
|
||||
const userMatch = path.match(/^\/-\/user\/org\.couchdb\.user:(.+)$/);
|
||||
if (userMatch) {
|
||||
return this.handleUserAuth(context.method, userMatch[1], context.body, token);
|
||||
}
|
||||
|
||||
// Token operations: /-/npm/v1/tokens
|
||||
if (path.startsWith('/-/npm/v1/tokens')) {
|
||||
return this.handleTokens(context.method, path, context.body, token);
|
||||
}
|
||||
|
||||
// Dist-tags: /-/package/{package}/dist-tags
|
||||
const distTagsMatch = path.match(/^\/-\/package\/(@?[^\/]+(?:\/[^\/]+)?)\/dist-tags(?:\/(.+))?$/);
|
||||
if (distTagsMatch) {
|
||||
const [, packageName, tag] = distTagsMatch;
|
||||
return this.handleDistTags(context.method, packageName, tag, context.body, token);
|
||||
}
|
||||
|
||||
// Tarball download: /{package}/-/{filename}.tgz
|
||||
const tarballMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)\/-\/(.+\.tgz)$/);
|
||||
if (tarballMatch) {
|
||||
const [, packageName, filename] = tarballMatch;
|
||||
return this.handleTarballDownload(packageName, filename, token);
|
||||
}
|
||||
|
||||
// Unpublish specific version: DELETE /{package}/-/{version}
|
||||
const unpublishVersionMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)\/-\/([^\/]+)$/);
|
||||
if (unpublishVersionMatch && context.method === 'DELETE') {
|
||||
const [, packageName, version] = unpublishVersionMatch;
|
||||
console.log(`[unpublishVersionMatch] packageName=${packageName}, version=${version}`);
|
||||
return this.unpublishVersion(packageName, version, token);
|
||||
}
|
||||
|
||||
// Unpublish entire package: DELETE /{package}/-rev/{rev}
|
||||
const unpublishPackageMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)\/-rev\/([^\/]+)$/);
|
||||
if (unpublishPackageMatch && context.method === 'DELETE') {
|
||||
const [, packageName, rev] = unpublishPackageMatch;
|
||||
console.log(`[unpublishPackageMatch] packageName=${packageName}, rev=${rev}`);
|
||||
return this.unpublishPackage(packageName, token);
|
||||
}
|
||||
|
||||
// Package version: /{package}/{version}
|
||||
const versionMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)\/([^\/]+)$/);
|
||||
if (versionMatch) {
|
||||
const [, packageName, version] = versionMatch;
|
||||
console.log(`[versionMatch] matched! packageName=${packageName}, version=${version}`);
|
||||
return this.handlePackageVersion(packageName, version, token);
|
||||
}
|
||||
|
||||
// Package operations: /{package}
|
||||
const packageMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)$/);
|
||||
if (packageMatch) {
|
||||
const packageName = packageMatch[1];
|
||||
console.log(`[packageMatch] matched! packageName=${packageName}`);
|
||||
return this.handlePackage(context.method, packageName, context.body, context.query, token);
|
||||
}
|
||||
|
||||
return this.storage.withContext({ protocol: 'npm', actor }, async () => {
|
||||
const route = parseNpmRequestRoute(path, context.method);
|
||||
if (!route) {
|
||||
return {
|
||||
status: 404,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
@@ -148,6 +128,67 @@ export class NpmRegistry extends BaseRegistry {
|
||||
};
|
||||
}
|
||||
|
||||
switch (route.type) {
|
||||
case 'root':
|
||||
return this.handleRegistryInfo();
|
||||
case 'search':
|
||||
return this.handleSearch(context.query);
|
||||
case 'userAuth':
|
||||
return this.handleUserAuth(context.method, route.username, context.body, token);
|
||||
case 'tokens':
|
||||
return this.handleTokens(context.method, route.path, context.body, token);
|
||||
case 'distTags':
|
||||
return this.withPackageContext(
|
||||
route.packageName,
|
||||
actor,
|
||||
async () => this.handleDistTags(context.method, route.packageName, route.tag, context.body, token)
|
||||
);
|
||||
case 'tarball':
|
||||
return this.handleTarballDownload(route.packageName, route.filename, token, actor);
|
||||
case 'unpublishVersion':
|
||||
this.logger.log('debug', 'unpublishVersionMatch', {
|
||||
packageName: route.packageName,
|
||||
version: route.version,
|
||||
});
|
||||
return this.withPackageVersionContext(
|
||||
route.packageName,
|
||||
route.version,
|
||||
actor,
|
||||
async () => this.unpublishVersion(route.packageName, route.version, token)
|
||||
);
|
||||
case 'unpublishPackage':
|
||||
this.logger.log('debug', 'unpublishPackageMatch', {
|
||||
packageName: route.packageName,
|
||||
rev: route.rev,
|
||||
});
|
||||
return this.withPackageContext(
|
||||
route.packageName,
|
||||
actor,
|
||||
async () => this.unpublishPackage(route.packageName, token)
|
||||
);
|
||||
case 'packageVersion':
|
||||
this.logger.log('debug', 'versionMatch', {
|
||||
packageName: route.packageName,
|
||||
version: route.version,
|
||||
});
|
||||
return this.withPackageVersionContext(
|
||||
route.packageName,
|
||||
route.version,
|
||||
actor,
|
||||
async () => this.handlePackageVersion(route.packageName, route.version, token, actor)
|
||||
);
|
||||
case 'package':
|
||||
this.logger.log('debug', 'packageMatch', { packageName: route.packageName });
|
||||
return this.withPackageContext(
|
||||
route.packageName,
|
||||
actor,
|
||||
async () => this.handlePackage(context.method, route.packageName, context.body, context.query, token, actor)
|
||||
);
|
||||
}
|
||||
|
||||
});
|
||||
}
|
||||
|
||||
protected async checkPermission(
|
||||
token: IAuthToken | null,
|
||||
resource: string,
|
||||
@@ -186,11 +227,12 @@ export class NpmRegistry extends BaseRegistry {
|
||||
packageName: string,
|
||||
body: any,
|
||||
query: Record<string, string>,
|
||||
token: IAuthToken | null
|
||||
token: IAuthToken | null,
|
||||
actor?: IRequestActor
|
||||
): Promise<IResponse> {
|
||||
switch (method) {
|
||||
case 'GET':
|
||||
return this.getPackument(packageName, token, query);
|
||||
return this.getPackument(packageName, token, query, actor);
|
||||
case 'PUT':
|
||||
return this.publishPackage(packageName, body, token);
|
||||
case 'DELETE':
|
||||
@@ -207,14 +249,10 @@ export class NpmRegistry extends BaseRegistry {
|
||||
private async getPackument(
|
||||
packageName: string,
|
||||
token: IAuthToken | null,
|
||||
query: Record<string, string>
|
||||
query: Record<string, string>,
|
||||
actor?: IRequestActor
|
||||
): Promise<IResponse> {
|
||||
const packument = await this.storage.getNpmPackument(packageName);
|
||||
this.logger.log('debug', `getPackument: ${packageName}`, {
|
||||
packageName,
|
||||
found: !!packument,
|
||||
versions: packument ? Object.keys(packument.versions).length : 0
|
||||
});
|
||||
const packument = await this.getLocalOrUpstreamPackument(packageName, actor, 'getPackument');
|
||||
|
||||
if (!packument) {
|
||||
return {
|
||||
@@ -252,14 +290,16 @@ export class NpmRegistry extends BaseRegistry {
|
||||
private async handlePackageVersion(
|
||||
packageName: string,
|
||||
version: string,
|
||||
token: IAuthToken | null
|
||||
token: IAuthToken | null,
|
||||
actor?: IRequestActor
|
||||
): Promise<IResponse> {
|
||||
console.log(`[handlePackageVersion] packageName=${packageName}, version=${version}`);
|
||||
const packument = await this.storage.getNpmPackument(packageName);
|
||||
console.log(`[handlePackageVersion] packument found:`, !!packument);
|
||||
this.logger.log('debug', 'handlePackageVersion', { packageName, version });
|
||||
const packument = await this.getLocalOrUpstreamPackument(packageName, actor, 'handlePackageVersion');
|
||||
this.logger.log('debug', 'handlePackageVersion packument', { found: !!packument });
|
||||
if (packument) {
|
||||
console.log(`[handlePackageVersion] versions:`, Object.keys(packument.versions || {}));
|
||||
this.logger.log('debug', 'handlePackageVersion versions', { versions: Object.keys(packument.versions || {}) });
|
||||
}
|
||||
|
||||
if (!packument) {
|
||||
return {
|
||||
status: 404,
|
||||
@@ -333,19 +373,7 @@ export class NpmRegistry extends BaseRegistry {
|
||||
const isNew = !packument;
|
||||
|
||||
if (isNew) {
|
||||
packument = {
|
||||
_id: packageName,
|
||||
name: packageName,
|
||||
description: body.description,
|
||||
'dist-tags': body['dist-tags'] || { latest: Object.keys(body.versions)[0] },
|
||||
versions: {},
|
||||
time: {
|
||||
created: new Date().toISOString(),
|
||||
modified: new Date().toISOString(),
|
||||
},
|
||||
maintainers: body.maintainers || [],
|
||||
readme: body.readme,
|
||||
};
|
||||
packument = createNewPackument(packageName, body, new Date().toISOString());
|
||||
}
|
||||
|
||||
// Process each new version
|
||||
@@ -359,12 +387,8 @@ export class NpmRegistry extends BaseRegistry {
|
||||
};
|
||||
}
|
||||
|
||||
// Find attachment for this version
|
||||
const attachmentKey = Object.keys(body._attachments).find(key =>
|
||||
key.includes(version)
|
||||
);
|
||||
|
||||
if (!attachmentKey) {
|
||||
const attachment = getAttachmentForVersion(body, version);
|
||||
if (!attachment) {
|
||||
return {
|
||||
status: 400,
|
||||
headers: {},
|
||||
@@ -372,38 +396,24 @@ export class NpmRegistry extends BaseRegistry {
|
||||
};
|
||||
}
|
||||
|
||||
const attachment = body._attachments[attachmentKey];
|
||||
|
||||
// Decode base64 tarball
|
||||
const tarballBuffer = Buffer.from(attachment.data, 'base64');
|
||||
|
||||
// Calculate shasum
|
||||
const crypto = await import('crypto');
|
||||
const shasum = crypto.createHash('sha1').update(tarballBuffer).digest('hex');
|
||||
const integrity = `sha512-${crypto.createHash('sha512').update(tarballBuffer).digest('base64')}`;
|
||||
const preparedVersion = preparePublishedVersion({
|
||||
packageName,
|
||||
version,
|
||||
versionData,
|
||||
attachment,
|
||||
registryUrl: this.registryUrl,
|
||||
userId: token?.userId,
|
||||
});
|
||||
|
||||
// Store tarball
|
||||
await this.storage.putNpmTarball(packageName, version, tarballBuffer);
|
||||
await this.withPackageVersionContext(
|
||||
packageName,
|
||||
version,
|
||||
undefined,
|
||||
async () => this.storage.putNpmTarball(packageName, version, preparedVersion.tarballBuffer)
|
||||
);
|
||||
|
||||
// Update version data with dist info
|
||||
const safeName = packageName.replace('@', '').replace('/', '-');
|
||||
versionData.dist = {
|
||||
tarball: `${this.registryUrl}/${packageName}/-/${safeName}-${version}.tgz`,
|
||||
shasum,
|
||||
integrity,
|
||||
fileCount: 0,
|
||||
unpackedSize: tarballBuffer.length,
|
||||
};
|
||||
|
||||
versionData._id = `${packageName}@${version}`;
|
||||
versionData._npmUser = token ? { name: token.userId, email: '' } : undefined;
|
||||
|
||||
// Add version to packument
|
||||
packument.versions[version] = versionData;
|
||||
if (packument.time) {
|
||||
packument.time[version] = new Date().toISOString();
|
||||
packument.time.modified = new Date().toISOString();
|
||||
}
|
||||
recordPublishedVersion(packument, version, preparedVersion.versionData, new Date().toISOString());
|
||||
}
|
||||
|
||||
// Update dist-tags
|
||||
@@ -526,10 +536,11 @@ export class NpmRegistry extends BaseRegistry {
|
||||
private async handleTarballDownload(
|
||||
packageName: string,
|
||||
filename: string,
|
||||
token: IAuthToken | null
|
||||
token: IAuthToken | null,
|
||||
actor?: IRequestActor
|
||||
): Promise<IResponse> {
|
||||
// Extract version from filename: package-name-1.0.0.tgz
|
||||
const versionMatch = filename.match(/-([\d.]+(?:-[a-z0-9.]+)?)\.tgz$/);
|
||||
const versionMatch = filename.match(/-([\d.]+(?:-[a-z0-9.]+)?)\.tgz$/i);
|
||||
if (!versionMatch) {
|
||||
return {
|
||||
status: 400,
|
||||
@@ -539,7 +550,45 @@ export class NpmRegistry extends BaseRegistry {
|
||||
}
|
||||
|
||||
const version = versionMatch[1];
|
||||
const tarball = await this.storage.getNpmTarball(packageName, version);
|
||||
|
||||
return this.withPackageVersionContext(
|
||||
packageName,
|
||||
version,
|
||||
actor,
|
||||
async (): Promise<IResponse> => {
|
||||
// Try local storage first (streaming)
|
||||
const streamResult = await this.storage.getNpmTarballStream(packageName, version);
|
||||
if (streamResult) {
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/octet-stream',
|
||||
'Content-Length': streamResult.size.toString(),
|
||||
},
|
||||
body: streamResult.stream,
|
||||
};
|
||||
}
|
||||
|
||||
// If not found locally, try upstream
|
||||
let tarball: Buffer | null = null;
|
||||
const upstream = await this.getUpstreamForRequest(packageName, 'tarball', 'GET', actor);
|
||||
if (upstream) {
|
||||
this.logger.log('debug', 'handleTarballDownload: fetching from upstream', {
|
||||
packageName,
|
||||
version,
|
||||
});
|
||||
const upstreamTarball = await upstream.fetchTarball(packageName, version);
|
||||
if (upstreamTarball) {
|
||||
tarball = upstreamTarball;
|
||||
// Cache the tarball locally for future requests
|
||||
await this.storage.putNpmTarball(packageName, version, tarball);
|
||||
this.logger.log('debug', 'handleTarballDownload: cached tarball locally', {
|
||||
packageName,
|
||||
version,
|
||||
size: tarball.length,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (!tarball) {
|
||||
return {
|
||||
@@ -558,6 +607,64 @@ export class NpmRegistry extends BaseRegistry {
|
||||
body: tarball,
|
||||
};
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
private async withPackageContext<T>(
|
||||
packageName: string,
|
||||
actor: IRequestActor | undefined,
|
||||
fn: () => Promise<T>
|
||||
): Promise<T> {
|
||||
return this.storage.withContext(
|
||||
{ protocol: 'npm', actor, metadata: { packageName } },
|
||||
fn
|
||||
);
|
||||
}
|
||||
|
||||
private async getLocalOrUpstreamPackument(
|
||||
packageName: string,
|
||||
actor: IRequestActor | undefined,
|
||||
logPrefix: string
|
||||
): Promise<IPackument | null> {
|
||||
const localPackument = await this.storage.getNpmPackument(packageName);
|
||||
this.logger.log('debug', `${logPrefix}: ${packageName}`, {
|
||||
packageName,
|
||||
found: !!localPackument,
|
||||
versions: localPackument ? Object.keys(localPackument.versions).length : 0,
|
||||
});
|
||||
|
||||
if (localPackument) {
|
||||
return localPackument;
|
||||
}
|
||||
|
||||
const upstream = await this.getUpstreamForRequest(packageName, 'packument', 'GET', actor);
|
||||
if (!upstream) {
|
||||
return null;
|
||||
}
|
||||
|
||||
this.logger.log('debug', `${logPrefix}: fetching from upstream`, { packageName });
|
||||
const upstreamPackument = await upstream.fetchPackument(packageName);
|
||||
if (upstreamPackument) {
|
||||
this.logger.log('debug', `${logPrefix}: found in upstream`, {
|
||||
packageName,
|
||||
versions: Object.keys(upstreamPackument.versions || {}).length,
|
||||
});
|
||||
}
|
||||
|
||||
return upstreamPackument;
|
||||
}
|
||||
|
||||
private async withPackageVersionContext<T>(
|
||||
packageName: string,
|
||||
version: string,
|
||||
actor: IRequestActor | undefined,
|
||||
fn: () => Promise<T>
|
||||
): Promise<T> {
|
||||
return this.storage.withContext(
|
||||
{ protocol: 'npm', actor, metadata: { packageName, version } },
|
||||
fn
|
||||
);
|
||||
}
|
||||
|
||||
private async handleSearch(query: Record<string, string>): Promise<IResponse> {
|
||||
const text = query.text || '';
|
||||
@@ -621,7 +728,23 @@ export class NpmRegistry extends BaseRegistry {
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('[handleSearch] Error:', error);
|
||||
this.logger.log('error', 'handleSearch failed', { error: (error as Error).message });
|
||||
}
|
||||
|
||||
// Sort results by relevance: exact match first, then prefix match, then substring match
|
||||
if (text) {
|
||||
const lowerText = text.toLowerCase();
|
||||
results.sort((a, b) => {
|
||||
const aName = a.package.name.toLowerCase();
|
||||
const bName = b.package.name.toLowerCase();
|
||||
const aExact = aName === lowerText ? 0 : 1;
|
||||
const bExact = bName === lowerText ? 0 : 1;
|
||||
if (aExact !== bExact) return aExact - bExact;
|
||||
const aPrefix = aName.startsWith(lowerText) ? 0 : 1;
|
||||
const bPrefix = bName.startsWith(lowerText) ? 0 : 1;
|
||||
if (aPrefix !== bPrefix) return aPrefix - bPrefix;
|
||||
return aName.localeCompare(bName);
|
||||
});
|
||||
}
|
||||
|
||||
// Apply pagination
|
||||
|
||||
@@ -0,0 +1,260 @@
|
||||
import * as plugins from '../plugins.js';
|
||||
import { BaseUpstream } from '../upstream/classes.baseupstream.js';
|
||||
import type {
|
||||
IProtocolUpstreamConfig,
|
||||
IUpstreamFetchContext,
|
||||
IUpstreamResult,
|
||||
IUpstreamRegistryConfig,
|
||||
} from '../upstream/interfaces.upstream.js';
|
||||
import type { IPackument, INpmVersion } from './interfaces.npm.js';
|
||||
|
||||
/**
|
||||
* NPM-specific upstream implementation.
|
||||
*
|
||||
* Handles:
|
||||
* - Package metadata (packument) fetching
|
||||
* - Tarball proxying
|
||||
* - Scoped package routing (@scope/* patterns)
|
||||
* - NPM-specific URL rewriting
|
||||
*/
|
||||
export class NpmUpstream extends BaseUpstream {
|
||||
protected readonly protocolName = 'npm';
|
||||
|
||||
/** Local registry URL for rewriting tarball URLs */
|
||||
private readonly localRegistryUrl: string;
|
||||
|
||||
constructor(
|
||||
config: IProtocolUpstreamConfig,
|
||||
localRegistryUrl: string,
|
||||
logger?: plugins.smartlog.Smartlog,
|
||||
) {
|
||||
super(config, logger);
|
||||
this.localRegistryUrl = localRegistryUrl;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a packument from upstream registries.
|
||||
*/
|
||||
public async fetchPackument(packageName: string): Promise<IPackument | null> {
|
||||
const context: IUpstreamFetchContext = {
|
||||
protocol: 'npm',
|
||||
resource: packageName,
|
||||
resourceType: 'packument',
|
||||
path: `/${encodeURIComponent(packageName).replace('%40', '@')}`,
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'accept': 'application/json',
|
||||
},
|
||||
query: {},
|
||||
};
|
||||
|
||||
const result = await this.fetch(context);
|
||||
|
||||
if (!result || !result.success) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Parse and process packument
|
||||
let packument: IPackument;
|
||||
if (Buffer.isBuffer(result.body)) {
|
||||
packument = JSON.parse(result.body.toString('utf8'));
|
||||
} else {
|
||||
packument = result.body;
|
||||
}
|
||||
|
||||
// Rewrite tarball URLs to point to local registry
|
||||
packument = this.rewriteTarballUrls(packument);
|
||||
|
||||
return packument;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a specific version from upstream registries.
|
||||
*/
|
||||
public async fetchVersion(packageName: string, version: string): Promise<INpmVersion | null> {
|
||||
const context: IUpstreamFetchContext = {
|
||||
protocol: 'npm',
|
||||
resource: packageName,
|
||||
resourceType: 'version',
|
||||
path: `/${encodeURIComponent(packageName).replace('%40', '@')}/${version}`,
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'accept': 'application/json',
|
||||
},
|
||||
query: {},
|
||||
};
|
||||
|
||||
const result = await this.fetch(context);
|
||||
|
||||
if (!result || !result.success) {
|
||||
return null;
|
||||
}
|
||||
|
||||
let versionData: INpmVersion;
|
||||
if (Buffer.isBuffer(result.body)) {
|
||||
versionData = JSON.parse(result.body.toString('utf8'));
|
||||
} else {
|
||||
versionData = result.body;
|
||||
}
|
||||
|
||||
// Rewrite tarball URL
|
||||
if (versionData.dist?.tarball) {
|
||||
versionData.dist.tarball = this.rewriteSingleTarballUrl(
|
||||
packageName,
|
||||
versionData.version,
|
||||
versionData.dist.tarball,
|
||||
);
|
||||
}
|
||||
|
||||
return versionData;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a tarball from upstream registries.
|
||||
*/
|
||||
public async fetchTarball(packageName: string, version: string): Promise<Buffer | null> {
|
||||
// First, try to get the tarball URL from packument
|
||||
const packument = await this.fetchPackument(packageName);
|
||||
let tarballPath: string;
|
||||
|
||||
if (packument?.versions?.[version]?.dist?.tarball) {
|
||||
// Extract path from original (upstream) tarball URL
|
||||
const tarballUrl = packument.versions[version].dist.tarball;
|
||||
try {
|
||||
const url = new URL(tarballUrl);
|
||||
tarballPath = url.pathname;
|
||||
} catch {
|
||||
// Fallback to standard NPM tarball path
|
||||
tarballPath = this.buildTarballPath(packageName, version);
|
||||
}
|
||||
} else {
|
||||
tarballPath = this.buildTarballPath(packageName, version);
|
||||
}
|
||||
|
||||
const context: IUpstreamFetchContext = {
|
||||
protocol: 'npm',
|
||||
resource: packageName,
|
||||
resourceType: 'tarball',
|
||||
path: tarballPath,
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'accept': 'application/octet-stream',
|
||||
},
|
||||
query: {},
|
||||
};
|
||||
|
||||
const result = await this.fetch(context);
|
||||
|
||||
if (!result || !result.success) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return Buffer.isBuffer(result.body) ? result.body : Buffer.from(result.body);
|
||||
}
|
||||
|
||||
/**
|
||||
* Search packages in upstream registries.
|
||||
*/
|
||||
public async search(text: string, size: number = 20, from: number = 0): Promise<any | null> {
|
||||
const context: IUpstreamFetchContext = {
|
||||
protocol: 'npm',
|
||||
resource: '*',
|
||||
resourceType: 'search',
|
||||
path: '/-/v1/search',
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'accept': 'application/json',
|
||||
},
|
||||
query: {
|
||||
text,
|
||||
size: size.toString(),
|
||||
from: from.toString(),
|
||||
},
|
||||
};
|
||||
|
||||
const result = await this.fetch(context);
|
||||
|
||||
if (!result || !result.success) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (Buffer.isBuffer(result.body)) {
|
||||
return JSON.parse(result.body.toString('utf8'));
|
||||
}
|
||||
|
||||
return result.body;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the standard NPM tarball path.
|
||||
*/
|
||||
private buildTarballPath(packageName: string, version: string): string {
|
||||
// NPM uses: /{package}/-/{package-name}-{version}.tgz
|
||||
// For scoped packages: /@scope/name/-/name-version.tgz
|
||||
if (packageName.startsWith('@')) {
|
||||
const [scope, name] = packageName.split('/');
|
||||
return `/${scope}/${name}/-/${name}-${version}.tgz`;
|
||||
} else {
|
||||
return `/${packageName}/-/${packageName}-${version}.tgz`;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Rewrite all tarball URLs in a packument to point to local registry.
|
||||
*/
|
||||
private rewriteTarballUrls(packument: IPackument): IPackument {
|
||||
if (!packument.versions) {
|
||||
return packument;
|
||||
}
|
||||
|
||||
const rewritten = { ...packument };
|
||||
rewritten.versions = {};
|
||||
|
||||
for (const [version, versionData] of Object.entries(packument.versions)) {
|
||||
const newVersionData = { ...versionData };
|
||||
if (newVersionData.dist?.tarball) {
|
||||
newVersionData.dist = {
|
||||
...newVersionData.dist,
|
||||
tarball: this.rewriteSingleTarballUrl(
|
||||
packument.name,
|
||||
version,
|
||||
newVersionData.dist.tarball,
|
||||
),
|
||||
};
|
||||
}
|
||||
rewritten.versions[version] = newVersionData;
|
||||
}
|
||||
|
||||
return rewritten;
|
||||
}
|
||||
|
||||
/**
|
||||
* Rewrite a single tarball URL to point to local registry.
|
||||
*/
|
||||
private rewriteSingleTarballUrl(
|
||||
packageName: string,
|
||||
version: string,
|
||||
_originalUrl: string,
|
||||
): string {
|
||||
// Generate local tarball URL
|
||||
// Format: {localRegistryUrl}/{package}/-/{package-name}-{version}.tgz
|
||||
const safeName = packageName.replace('@', '').replace('/', '-');
|
||||
return `${this.localRegistryUrl}/${packageName}/-/${safeName}-${version}.tgz`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Override URL building for NPM-specific handling.
|
||||
*/
|
||||
protected buildUpstreamUrl(
|
||||
upstream: IUpstreamRegistryConfig,
|
||||
context: IUpstreamFetchContext,
|
||||
): string {
|
||||
// NPM registries often don't have trailing slashes
|
||||
let baseUrl = upstream.url;
|
||||
if (baseUrl.endsWith('/')) {
|
||||
baseUrl = baseUrl.slice(0, -1);
|
||||
}
|
||||
|
||||
return `${baseUrl}${context.path}`;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
import * as crypto from 'node:crypto';
|
||||
import type { IPackument, IPublishRequest, INpmVersion } from './interfaces.npm.js';
|
||||
|
||||
function getTarballFileName(packageName: string, version: string): string {
|
||||
const safeName = packageName.replace('@', '').replace('/', '-');
|
||||
return `${safeName}-${version}.tgz`;
|
||||
}
|
||||
|
||||
export function createNewPackument(
|
||||
packageName: string,
|
||||
body: IPublishRequest,
|
||||
timestamp: string
|
||||
): IPackument {
|
||||
return {
|
||||
_id: packageName,
|
||||
name: packageName,
|
||||
description: body.description,
|
||||
'dist-tags': body['dist-tags'] || { latest: Object.keys(body.versions)[0] },
|
||||
versions: {},
|
||||
time: {
|
||||
created: timestamp,
|
||||
modified: timestamp,
|
||||
},
|
||||
maintainers: body.maintainers || [],
|
||||
readme: body.readme,
|
||||
};
|
||||
}
|
||||
|
||||
export function getAttachmentForVersion(
|
||||
body: IPublishRequest,
|
||||
version: string
|
||||
): IPublishRequest['_attachments'][string] | null {
|
||||
const attachmentKey = Object.keys(body._attachments).find((key) => key.includes(version));
|
||||
return attachmentKey ? body._attachments[attachmentKey] : null;
|
||||
}
|
||||
|
||||
export function preparePublishedVersion(options: {
|
||||
packageName: string;
|
||||
version: string;
|
||||
versionData: INpmVersion;
|
||||
attachment: IPublishRequest['_attachments'][string];
|
||||
registryUrl: string;
|
||||
userId?: string;
|
||||
}): { tarballBuffer: Buffer; versionData: INpmVersion } {
|
||||
const tarballBuffer = Buffer.from(options.attachment.data, 'base64');
|
||||
const shasum = crypto.createHash('sha1').update(tarballBuffer).digest('hex');
|
||||
const integrity = `sha512-${crypto.createHash('sha512').update(tarballBuffer).digest('base64')}`;
|
||||
const tarballFileName = getTarballFileName(options.packageName, options.version);
|
||||
|
||||
return {
|
||||
tarballBuffer,
|
||||
versionData: {
|
||||
...options.versionData,
|
||||
dist: {
|
||||
...options.versionData.dist,
|
||||
tarball: `${options.registryUrl}/${options.packageName}/-/${tarballFileName}`,
|
||||
shasum,
|
||||
integrity,
|
||||
fileCount: 0,
|
||||
unpackedSize: tarballBuffer.length,
|
||||
},
|
||||
_id: `${options.packageName}@${options.version}`,
|
||||
...(options.userId ? { _npmUser: { name: options.userId, email: '' } } : {}),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function recordPublishedVersion(
|
||||
packument: IPackument,
|
||||
version: string,
|
||||
versionData: INpmVersion,
|
||||
timestamp: string
|
||||
): void {
|
||||
packument.versions[version] = versionData;
|
||||
if (packument.time) {
|
||||
packument.time[version] = timestamp;
|
||||
packument.time.modified = timestamp;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
export type TNpmRequestRoute =
|
||||
| { type: 'root' }
|
||||
| { type: 'search' }
|
||||
| { type: 'userAuth'; username: string }
|
||||
| { type: 'tokens'; path: string }
|
||||
| { type: 'distTags'; packageName: string; tag?: string }
|
||||
| { type: 'tarball'; packageName: string; filename: string }
|
||||
| { type: 'unpublishVersion'; packageName: string; version: string }
|
||||
| { type: 'unpublishPackage'; packageName: string; rev: string }
|
||||
| { type: 'packageVersion'; packageName: string; version: string }
|
||||
| { type: 'package'; packageName: string };
|
||||
|
||||
function decodePackageName(rawPackageName: string): string {
|
||||
return decodeURIComponent(rawPackageName);
|
||||
}
|
||||
|
||||
export function parseNpmRequestRoute(path: string, method: string): TNpmRequestRoute | null {
|
||||
if (path === '/' || path === '') {
|
||||
return { type: 'root' };
|
||||
}
|
||||
|
||||
if (path.startsWith('/-/v1/search')) {
|
||||
return { type: 'search' };
|
||||
}
|
||||
|
||||
const userMatch = path.match(/^\/-\/user\/org\.couchdb\.user:(.+)$/);
|
||||
if (userMatch) {
|
||||
return {
|
||||
type: 'userAuth',
|
||||
username: userMatch[1],
|
||||
};
|
||||
}
|
||||
|
||||
if (path.startsWith('/-/npm/v1/tokens')) {
|
||||
return {
|
||||
type: 'tokens',
|
||||
path,
|
||||
};
|
||||
}
|
||||
|
||||
const distTagsMatch = path.match(/^\/-\/package\/(@?[^\/]+(?:\/[^\/]+)?)\/dist-tags(?:\/(.+))?$/);
|
||||
if (distTagsMatch) {
|
||||
const [, rawPackageName, tag] = distTagsMatch;
|
||||
return {
|
||||
type: 'distTags',
|
||||
packageName: decodePackageName(rawPackageName),
|
||||
tag,
|
||||
};
|
||||
}
|
||||
|
||||
const tarballMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)\/-\/(.+\.tgz)$/);
|
||||
if (tarballMatch) {
|
||||
const [, rawPackageName, filename] = tarballMatch;
|
||||
return {
|
||||
type: 'tarball',
|
||||
packageName: decodePackageName(rawPackageName),
|
||||
filename,
|
||||
};
|
||||
}
|
||||
|
||||
if (method === 'DELETE') {
|
||||
const unpublishVersionMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)\/-\/([^\/]+)$/);
|
||||
if (unpublishVersionMatch) {
|
||||
const [, rawPackageName, version] = unpublishVersionMatch;
|
||||
return {
|
||||
type: 'unpublishVersion',
|
||||
packageName: decodePackageName(rawPackageName),
|
||||
version,
|
||||
};
|
||||
}
|
||||
|
||||
const unpublishPackageMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)\/-rev\/([^\/]+)$/);
|
||||
if (unpublishPackageMatch) {
|
||||
const [, rawPackageName, rev] = unpublishPackageMatch;
|
||||
return {
|
||||
type: 'unpublishPackage',
|
||||
packageName: decodePackageName(rawPackageName),
|
||||
rev,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const unencodedScopedPackageMatch = path.match(/^\/@[^\/]+\/[^\/]+$/);
|
||||
if (unencodedScopedPackageMatch) {
|
||||
return {
|
||||
type: 'package',
|
||||
packageName: decodePackageName(path.substring(1)),
|
||||
};
|
||||
}
|
||||
|
||||
const versionMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)\/([^\/]+)$/);
|
||||
if (versionMatch) {
|
||||
const [, rawPackageName, version] = versionMatch;
|
||||
return {
|
||||
type: 'packageVersion',
|
||||
packageName: decodePackageName(rawPackageName),
|
||||
version,
|
||||
};
|
||||
}
|
||||
|
||||
const packageMatch = path.match(/^\/(@?[^\/]+(?:\/[^\/]+)?)$/);
|
||||
if (packageMatch) {
|
||||
return {
|
||||
type: 'package',
|
||||
packageName: decodePackageName(packageMatch[1]),
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
@@ -3,4 +3,5 @@
|
||||
*/
|
||||
|
||||
export { NpmRegistry } from './classes.npmregistry.js';
|
||||
export { NpmUpstream } from './classes.npmupstream.js';
|
||||
export * from './interfaces.npm.js';
|
||||
|
||||
+371
-115
@@ -1,7 +1,11 @@
|
||||
import { Smartlog } from '@push.rocks/smartlog';
|
||||
import { BaseRegistry } from '../core/classes.baseregistry.js';
|
||||
import { RegistryStorage } from '../core/classes.registrystorage.js';
|
||||
import { AuthManager } from '../core/classes.authmanager.js';
|
||||
import type { IRequestContext, IResponse, IAuthToken, IRegistryError } from '../core/interfaces.core.js';
|
||||
import type { IRequestContext, IResponse, IAuthToken, IRegistryError, IRequestActor } from '../core/interfaces.core.js';
|
||||
import { createHashTransform, streamToBuffer } from '../core/helpers.stream.js';
|
||||
import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
|
||||
import { OciUpstream } from './classes.ociupstream.js';
|
||||
import type {
|
||||
IUploadSession,
|
||||
IOciManifest,
|
||||
@@ -20,12 +24,68 @@ export class OciRegistry extends BaseRegistry {
|
||||
private uploadSessions: Map<string, IUploadSession> = new Map();
|
||||
private basePath: string = '/oci';
|
||||
private cleanupInterval?: NodeJS.Timeout;
|
||||
private ociTokens?: { realm: string; service: string };
|
||||
private upstreamProvider: IUpstreamProvider | null = null;
|
||||
private logger: Smartlog;
|
||||
|
||||
constructor(storage: RegistryStorage, authManager: AuthManager, basePath: string = '/oci') {
|
||||
constructor(
|
||||
storage: RegistryStorage,
|
||||
authManager: AuthManager,
|
||||
basePath: string = '/oci',
|
||||
ociTokens?: { realm: string; service: string },
|
||||
upstreamProvider?: IUpstreamProvider
|
||||
) {
|
||||
super();
|
||||
this.storage = storage;
|
||||
this.authManager = authManager;
|
||||
this.basePath = basePath;
|
||||
this.ociTokens = ociTokens;
|
||||
this.upstreamProvider = upstreamProvider || null;
|
||||
|
||||
this.logger = this.createProtocolLogger('oci-registry', 'oci');
|
||||
|
||||
if (upstreamProvider) {
|
||||
this.logger.log('info', 'OCI upstream provider configured');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract scope from OCI repository name.
|
||||
* @example "myorg/myimage" -> "myorg"
|
||||
* @example "library/nginx" -> "library"
|
||||
* @example "nginx" -> null
|
||||
*/
|
||||
private extractScope(repository: string): string | null {
|
||||
const slashIndex = repository.indexOf('/');
|
||||
if (slashIndex > 0) {
|
||||
return repository.substring(0, slashIndex);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get upstream for a specific request.
|
||||
* Calls the provider to resolve upstream config dynamically.
|
||||
*/
|
||||
private async getUpstreamForRequest(
|
||||
resource: string,
|
||||
resourceType: string,
|
||||
method: string,
|
||||
actor?: IRequestActor
|
||||
): Promise<OciUpstream | null> {
|
||||
if (!this.upstreamProvider) return null;
|
||||
|
||||
const config = await this.upstreamProvider.resolveUpstreamConfig({
|
||||
protocol: 'oci',
|
||||
resource,
|
||||
scope: this.extractScope(resource),
|
||||
actor,
|
||||
method,
|
||||
resourceType,
|
||||
});
|
||||
|
||||
if (!config?.enabled) return null;
|
||||
return new OciUpstream(config, this.basePath, this.logger);
|
||||
}
|
||||
|
||||
public async init(): Promise<void> {
|
||||
@@ -41,53 +101,59 @@ export class OciRegistry extends BaseRegistry {
|
||||
// Remove base path from URL
|
||||
const path = context.path.replace(this.basePath, '');
|
||||
|
||||
// Extract token from Authorization header
|
||||
const authHeader = context.headers['authorization'] || context.headers['Authorization'];
|
||||
const tokenString = authHeader?.replace(/^Bearer\s+/i, '');
|
||||
const tokenString = this.extractBearerToken(context);
|
||||
const token = tokenString ? await this.authManager.validateToken(tokenString, 'oci') : null;
|
||||
|
||||
const actor: IRequestActor = this.buildRequestActor(context, token);
|
||||
|
||||
return this.storage.withContext({ protocol: 'oci', actor }, async () => {
|
||||
// Route to appropriate handler
|
||||
if (path === '/v2/' || path === '/v2') {
|
||||
// OCI spec: GET /v2/ is the version check endpoint
|
||||
if (path === '/' || path === '' || path === '/v2/' || path === '/v2') {
|
||||
return this.handleVersionCheck();
|
||||
}
|
||||
|
||||
// Manifest operations: /v2/{name}/manifests/{reference}
|
||||
const manifestMatch = path.match(/^\/v2\/([^\/]+(?:\/[^\/]+)*)\/manifests\/([^\/]+)$/);
|
||||
// Manifest operations: /{name}/manifests/{reference}
|
||||
const manifestMatch = path.match(/^\/([^\/]+(?:\/[^\/]+)*)\/manifests\/([^\/]+)$/);
|
||||
if (manifestMatch) {
|
||||
const [, name, reference] = manifestMatch;
|
||||
return this.handleManifestRequest(context.method, name, reference, token, context.body, context.headers);
|
||||
// Prefer rawBody for content-addressable operations to preserve exact bytes
|
||||
const bodyData = context.rawBody || context.body;
|
||||
return this.handleManifestRequest(context.method, name, reference, token, bodyData, context.headers, actor);
|
||||
}
|
||||
|
||||
// Blob operations: /v2/{name}/blobs/{digest}
|
||||
const blobMatch = path.match(/^\/v2\/([^\/]+(?:\/[^\/]+)*)\/blobs\/(sha256:[a-f0-9]{64})$/);
|
||||
// Blob operations: /{name}/blobs/{digest}
|
||||
const blobMatch = path.match(/^\/([^\/]+(?:\/[^\/]+)*)\/blobs\/(sha256:[a-f0-9]{64})$/);
|
||||
if (blobMatch) {
|
||||
const [, name, digest] = blobMatch;
|
||||
return this.handleBlobRequest(context.method, name, digest, token, context.headers);
|
||||
return this.handleBlobRequest(context.method, name, digest, token, context.headers, actor);
|
||||
}
|
||||
|
||||
// Blob upload operations: /v2/{name}/blobs/uploads/
|
||||
const uploadInitMatch = path.match(/^\/v2\/([^\/]+(?:\/[^\/]+)*)\/blobs\/uploads\/?$/);
|
||||
// Blob upload operations: /{name}/blobs/uploads/
|
||||
const uploadInitMatch = path.match(/^\/([^\/]+(?:\/[^\/]+)*)\/blobs\/uploads\/?$/);
|
||||
if (uploadInitMatch && context.method === 'POST') {
|
||||
const [, name] = uploadInitMatch;
|
||||
return this.handleUploadInit(name, token, context.query, context.body);
|
||||
// Prefer rawBody for content-addressable operations to preserve exact bytes
|
||||
const bodyData = context.rawBody || context.body;
|
||||
return this.handleUploadInit(name, token, context.query, bodyData);
|
||||
}
|
||||
|
||||
// Blob upload operations: /v2/{name}/blobs/uploads/{uuid}
|
||||
const uploadMatch = path.match(/^\/v2\/([^\/]+(?:\/[^\/]+)*)\/blobs\/uploads\/([^\/]+)$/);
|
||||
// Blob upload operations: /{name}/blobs/uploads/{uuid}
|
||||
const uploadMatch = path.match(/^\/([^\/]+(?:\/[^\/]+)*)\/blobs\/uploads\/([^\/]+)$/);
|
||||
if (uploadMatch) {
|
||||
const [, name, uploadId] = uploadMatch;
|
||||
return this.handleUploadSession(context.method, uploadId, token, context);
|
||||
}
|
||||
|
||||
// Tags list: /v2/{name}/tags/list
|
||||
const tagsMatch = path.match(/^\/v2\/([^\/]+(?:\/[^\/]+)*)\/tags\/list$/);
|
||||
// Tags list: /{name}/tags/list
|
||||
const tagsMatch = path.match(/^\/([^\/]+(?:\/[^\/]+)*)\/tags\/list$/);
|
||||
if (tagsMatch) {
|
||||
const [, name] = tagsMatch;
|
||||
return this.handleTagsList(name, token, context.query);
|
||||
}
|
||||
|
||||
// Referrers: /v2/{name}/referrers/{digest}
|
||||
const referrersMatch = path.match(/^\/v2\/([^\/]+(?:\/[^\/]+)*)\/referrers\/(sha256:[a-f0-9]{64})$/);
|
||||
// Referrers: /{name}/referrers/{digest}
|
||||
const referrersMatch = path.match(/^\/([^\/]+(?:\/[^\/]+)*)\/referrers\/(sha256:[a-f0-9]{64})$/);
|
||||
if (referrersMatch) {
|
||||
const [, name, digest] = referrersMatch;
|
||||
return this.handleReferrers(name, digest, token, context.query);
|
||||
@@ -98,6 +164,7 @@ export class OciRegistry extends BaseRegistry {
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: this.createError('NOT_FOUND', 'Endpoint not found'),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
protected async checkPermission(
|
||||
@@ -130,11 +197,12 @@ export class OciRegistry extends BaseRegistry {
|
||||
reference: string,
|
||||
token: IAuthToken | null,
|
||||
body?: Buffer | any,
|
||||
headers?: Record<string, string>
|
||||
headers?: Record<string, string>,
|
||||
actor?: IRequestActor
|
||||
): Promise<IResponse> {
|
||||
switch (method) {
|
||||
case 'GET':
|
||||
return this.getManifest(repository, reference, token, headers);
|
||||
return this.getManifest(repository, reference, token, headers, actor);
|
||||
case 'HEAD':
|
||||
return this.headManifest(repository, reference, token);
|
||||
case 'PUT':
|
||||
@@ -155,11 +223,12 @@ export class OciRegistry extends BaseRegistry {
|
||||
repository: string,
|
||||
digest: string,
|
||||
token: IAuthToken | null,
|
||||
headers: Record<string, string>
|
||||
headers: Record<string, string>,
|
||||
actor?: IRequestActor
|
||||
): Promise<IResponse> {
|
||||
switch (method) {
|
||||
case 'GET':
|
||||
return this.getBlob(repository, digest, token, headers['range'] || headers['Range']);
|
||||
return this.getBlob(repository, digest, token, headers['range'] || headers['Range'], actor);
|
||||
case 'HEAD':
|
||||
return this.headBlob(repository, digest, token);
|
||||
case 'DELETE':
|
||||
@@ -180,18 +249,14 @@ export class OciRegistry extends BaseRegistry {
|
||||
body?: Buffer | any
|
||||
): Promise<IResponse> {
|
||||
if (!await this.checkPermission(token, repository, 'push')) {
|
||||
return {
|
||||
status: 401,
|
||||
headers: {},
|
||||
body: this.createError('DENIED', 'Insufficient permissions'),
|
||||
};
|
||||
return this.createUnauthorizedResponse(repository, 'push');
|
||||
}
|
||||
|
||||
// Check for monolithic upload (digest + body provided)
|
||||
const digest = query.digest;
|
||||
if (digest && body) {
|
||||
// Monolithic upload: complete upload in single POST
|
||||
const blobData = Buffer.isBuffer(body) ? body : Buffer.from(JSON.stringify(body));
|
||||
const blobData = this.toBuffer(body);
|
||||
|
||||
// Verify digest
|
||||
const calculatedDigest = await this.calculateDigest(blobData);
|
||||
@@ -209,7 +274,7 @@ export class OciRegistry extends BaseRegistry {
|
||||
return {
|
||||
status: 201,
|
||||
headers: {
|
||||
'Location': `${this.basePath}/v2/${repository}/blobs/${digest}`,
|
||||
'Location': `${this.basePath}/${repository}/blobs/${digest}`,
|
||||
'Docker-Content-Digest': digest,
|
||||
},
|
||||
body: null,
|
||||
@@ -222,6 +287,8 @@ export class OciRegistry extends BaseRegistry {
|
||||
uploadId,
|
||||
repository,
|
||||
chunks: [],
|
||||
chunkPaths: [],
|
||||
chunkIndex: 0,
|
||||
totalSize: 0,
|
||||
createdAt: new Date(),
|
||||
lastActivity: new Date(),
|
||||
@@ -232,7 +299,7 @@ export class OciRegistry extends BaseRegistry {
|
||||
return {
|
||||
status: 202,
|
||||
headers: {
|
||||
'Location': `${this.basePath}/v2/${repository}/blobs/uploads/${uploadId}`,
|
||||
'Location': `${this.basePath}/${repository}/blobs/uploads/${uploadId}`,
|
||||
'Docker-Upload-UUID': uploadId,
|
||||
},
|
||||
body: null,
|
||||
@@ -255,18 +322,17 @@ export class OciRegistry extends BaseRegistry {
|
||||
}
|
||||
|
||||
if (!await this.checkPermission(token, session.repository, 'push')) {
|
||||
return {
|
||||
status: 401,
|
||||
headers: {},
|
||||
body: this.createError('DENIED', 'Insufficient permissions'),
|
||||
};
|
||||
return this.createUnauthorizedResponse(session.repository, 'push');
|
||||
}
|
||||
|
||||
// Prefer rawBody for content-addressable operations to preserve exact bytes
|
||||
const bodyData = context.rawBody || context.body;
|
||||
|
||||
switch (method) {
|
||||
case 'PATCH':
|
||||
return this.uploadChunk(uploadId, context.body, context.headers['content-range']);
|
||||
return this.uploadChunk(uploadId, bodyData, context.headers['content-range']);
|
||||
case 'PUT':
|
||||
return this.completeUpload(uploadId, context.query['digest'], context.body);
|
||||
return this.completeUpload(uploadId, context.query['digest'], bodyData);
|
||||
case 'GET':
|
||||
return this.getUploadStatus(uploadId);
|
||||
default:
|
||||
@@ -285,16 +351,11 @@ export class OciRegistry extends BaseRegistry {
|
||||
repository: string,
|
||||
reference: string,
|
||||
token: IAuthToken | null,
|
||||
headers?: Record<string, string>
|
||||
headers?: Record<string, string>,
|
||||
actor?: IRequestActor
|
||||
): Promise<IResponse> {
|
||||
if (!await this.checkPermission(token, repository, 'pull')) {
|
||||
return {
|
||||
status: 401,
|
||||
headers: {
|
||||
'WWW-Authenticate': `Bearer realm="${this.basePath}/v2/token",service="registry",scope="repository:${repository}:pull"`,
|
||||
},
|
||||
body: this.createError('DENIED', 'Insufficient permissions'),
|
||||
};
|
||||
return this.createUnauthorizedResponse(repository, 'pull');
|
||||
}
|
||||
|
||||
// Resolve tag to digest if needed
|
||||
@@ -302,16 +363,53 @@ export class OciRegistry extends BaseRegistry {
|
||||
if (!reference.startsWith('sha256:')) {
|
||||
const tags = await this.getTagsData(repository);
|
||||
digest = tags[reference];
|
||||
if (!digest) {
|
||||
return {
|
||||
status: 404,
|
||||
headers: {},
|
||||
body: this.createError('MANIFEST_UNKNOWN', 'Manifest not found'),
|
||||
};
|
||||
}
|
||||
|
||||
// Try local storage first (if we have a digest)
|
||||
let manifestData: Buffer | null = null;
|
||||
let contentType: string | null = null;
|
||||
|
||||
if (digest) {
|
||||
manifestData = await this.storage.getOciManifest(repository, digest);
|
||||
if (manifestData) {
|
||||
contentType = await this.storage.getOciManifestContentType(repository, digest);
|
||||
if (!contentType) {
|
||||
contentType = this.detectManifestContentType(manifestData);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If not found locally, try upstream
|
||||
if (!manifestData) {
|
||||
const upstream = await this.getUpstreamForRequest(repository, 'manifest', 'GET', actor);
|
||||
if (upstream) {
|
||||
this.logger.log('debug', 'getManifest: fetching from upstream', { repository, reference });
|
||||
const upstreamResult = await upstream.fetchManifest(repository, reference);
|
||||
if (upstreamResult) {
|
||||
manifestData = Buffer.from(JSON.stringify(upstreamResult.manifest), 'utf8');
|
||||
contentType = upstreamResult.contentType;
|
||||
digest = upstreamResult.digest;
|
||||
|
||||
// Cache the manifest locally
|
||||
await this.storage.putOciManifest(repository, digest, manifestData, contentType);
|
||||
|
||||
// If reference is a tag, update tags mapping
|
||||
if (!reference.startsWith('sha256:')) {
|
||||
const tags = await this.getTagsData(repository);
|
||||
tags[reference] = digest;
|
||||
const tagsPath = `oci/tags/${repository}/tags.json`;
|
||||
await this.storage.putObject(tagsPath, Buffer.from(JSON.stringify(tags), 'utf-8'));
|
||||
}
|
||||
|
||||
this.logger.log('debug', 'getManifest: cached manifest locally', {
|
||||
repository,
|
||||
reference,
|
||||
digest,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const manifestData = await this.storage.getOciManifest(repository, digest);
|
||||
if (!manifestData) {
|
||||
return {
|
||||
status: 404,
|
||||
@@ -323,7 +421,7 @@ export class OciRegistry extends BaseRegistry {
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/vnd.oci.image.manifest.v1+json',
|
||||
'Content-Type': contentType || 'application/vnd.oci.image.manifest.v1+json',
|
||||
'Docker-Content-Digest': digest,
|
||||
},
|
||||
body: manifestData,
|
||||
@@ -336,11 +434,7 @@ export class OciRegistry extends BaseRegistry {
|
||||
token: IAuthToken | null
|
||||
): Promise<IResponse> {
|
||||
if (!await this.checkPermission(token, repository, 'pull')) {
|
||||
return {
|
||||
status: 401,
|
||||
headers: {},
|
||||
body: null,
|
||||
};
|
||||
return this.createUnauthorizedHeadResponse(repository, 'pull');
|
||||
}
|
||||
|
||||
// Similar logic as getManifest but return headers only
|
||||
@@ -360,10 +454,18 @@ export class OciRegistry extends BaseRegistry {
|
||||
|
||||
const manifestData = await this.storage.getOciManifest(repository, digest);
|
||||
|
||||
// Get stored content type, falling back to detecting from manifest content
|
||||
let contentType = await this.storage.getOciManifestContentType(repository, digest);
|
||||
if (!contentType && manifestData) {
|
||||
// Fallback: detect content type from manifest content
|
||||
contentType = this.detectManifestContentType(manifestData);
|
||||
}
|
||||
contentType = contentType || 'application/vnd.oci.image.manifest.v1+json';
|
||||
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/vnd.oci.image.manifest.v1+json',
|
||||
'Content-Type': contentType,
|
||||
'Docker-Content-Digest': digest,
|
||||
'Content-Length': manifestData ? manifestData.length.toString() : '0',
|
||||
},
|
||||
@@ -379,13 +481,7 @@ export class OciRegistry extends BaseRegistry {
|
||||
headers?: Record<string, string>
|
||||
): Promise<IResponse> {
|
||||
if (!await this.checkPermission(token, repository, 'push')) {
|
||||
return {
|
||||
status: 401,
|
||||
headers: {
|
||||
'WWW-Authenticate': `Bearer realm="${this.basePath}/v2/token",service="registry",scope="repository:${repository}:push"`,
|
||||
},
|
||||
body: this.createError('DENIED', 'Insufficient permissions'),
|
||||
};
|
||||
return this.createUnauthorizedResponse(repository, 'push');
|
||||
}
|
||||
|
||||
if (!body) {
|
||||
@@ -396,7 +492,9 @@ export class OciRegistry extends BaseRegistry {
|
||||
};
|
||||
}
|
||||
|
||||
const manifestData = Buffer.isBuffer(body) ? body : Buffer.from(JSON.stringify(body));
|
||||
// Preserve raw bytes for accurate digest calculation
|
||||
// Per OCI spec, digest must match the exact bytes sent by client
|
||||
const manifestData = this.toBuffer(body);
|
||||
const contentType = headers?.['content-type'] || headers?.['Content-Type'] || 'application/vnd.oci.image.manifest.v1+json';
|
||||
|
||||
// Calculate manifest digest
|
||||
@@ -416,7 +514,7 @@ export class OciRegistry extends BaseRegistry {
|
||||
return {
|
||||
status: 201,
|
||||
headers: {
|
||||
'Location': `${this.basePath}/v2/${repository}/manifests/${digest}`,
|
||||
'Location': `${this.basePath}/${repository}/manifests/${digest}`,
|
||||
'Docker-Content-Digest': digest,
|
||||
},
|
||||
body: null,
|
||||
@@ -437,11 +535,7 @@ export class OciRegistry extends BaseRegistry {
|
||||
}
|
||||
|
||||
if (!await this.checkPermission(token, repository, 'delete')) {
|
||||
return {
|
||||
status: 401,
|
||||
headers: {},
|
||||
body: this.createError('DENIED', 'Insufficient permissions'),
|
||||
};
|
||||
return this.createUnauthorizedResponse(repository, 'delete');
|
||||
}
|
||||
|
||||
await this.storage.deleteOciManifest(repository, digest);
|
||||
@@ -457,17 +551,45 @@ export class OciRegistry extends BaseRegistry {
|
||||
repository: string,
|
||||
digest: string,
|
||||
token: IAuthToken | null,
|
||||
range?: string
|
||||
range?: string,
|
||||
actor?: IRequestActor
|
||||
): Promise<IResponse> {
|
||||
if (!await this.checkPermission(token, repository, 'pull')) {
|
||||
return this.createUnauthorizedResponse(repository, 'pull');
|
||||
}
|
||||
|
||||
// Try local storage first (streaming)
|
||||
const streamResult = await this.storage.getOciBlobStream(digest);
|
||||
if (streamResult) {
|
||||
return {
|
||||
status: 401,
|
||||
headers: {},
|
||||
body: this.createError('DENIED', 'Insufficient permissions'),
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/octet-stream',
|
||||
'Content-Length': streamResult.size.toString(),
|
||||
'Docker-Content-Digest': digest,
|
||||
},
|
||||
body: streamResult.stream,
|
||||
};
|
||||
}
|
||||
|
||||
const data = await this.storage.getOciBlob(digest);
|
||||
// If not found locally, try upstream
|
||||
let data: Buffer | null = null;
|
||||
const upstream = await this.getUpstreamForRequest(repository, 'blob', 'GET', actor);
|
||||
if (upstream) {
|
||||
this.logger.log('debug', 'getBlob: fetching from upstream', { repository, digest });
|
||||
const upstreamBlob = await upstream.fetchBlob(repository, digest);
|
||||
if (upstreamBlob) {
|
||||
data = upstreamBlob;
|
||||
// Cache the blob locally (blobs are content-addressable and immutable)
|
||||
await this.storage.putOciBlob(digest, data);
|
||||
this.logger.log('debug', 'getBlob: cached blob locally', {
|
||||
repository,
|
||||
digest,
|
||||
size: data.length,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (!data) {
|
||||
return {
|
||||
status: 404,
|
||||
@@ -492,20 +614,18 @@ export class OciRegistry extends BaseRegistry {
|
||||
token: IAuthToken | null
|
||||
): Promise<IResponse> {
|
||||
if (!await this.checkPermission(token, repository, 'pull')) {
|
||||
return { status: 401, headers: {}, body: null };
|
||||
return this.createUnauthorizedHeadResponse(repository, 'pull');
|
||||
}
|
||||
|
||||
const exists = await this.storage.ociBlobExists(digest);
|
||||
if (!exists) {
|
||||
const blobSize = await this.storage.getOciBlobSize(digest);
|
||||
if (blobSize === null) {
|
||||
return { status: 404, headers: {}, body: null };
|
||||
}
|
||||
|
||||
const blob = await this.storage.getOciBlob(digest);
|
||||
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Length': blob ? blob.length.toString() : '0',
|
||||
'Content-Length': blobSize.toString(),
|
||||
'Docker-Content-Digest': digest,
|
||||
},
|
||||
body: null,
|
||||
@@ -518,11 +638,7 @@ export class OciRegistry extends BaseRegistry {
|
||||
token: IAuthToken | null
|
||||
): Promise<IResponse> {
|
||||
if (!await this.checkPermission(token, repository, 'delete')) {
|
||||
return {
|
||||
status: 401,
|
||||
headers: {},
|
||||
body: this.createError('DENIED', 'Insufficient permissions'),
|
||||
};
|
||||
return this.createUnauthorizedResponse(repository, 'delete');
|
||||
}
|
||||
|
||||
await this.storage.deleteOciBlob(digest);
|
||||
@@ -536,7 +652,7 @@ export class OciRegistry extends BaseRegistry {
|
||||
|
||||
private async uploadChunk(
|
||||
uploadId: string,
|
||||
data: Buffer,
|
||||
data: Buffer | Uint8Array | unknown,
|
||||
contentRange: string
|
||||
): Promise<IResponse> {
|
||||
const session = this.uploadSessions.get(uploadId);
|
||||
@@ -548,14 +664,20 @@ export class OciRegistry extends BaseRegistry {
|
||||
};
|
||||
}
|
||||
|
||||
session.chunks.push(data);
|
||||
session.totalSize += data.length;
|
||||
const chunkData = this.toBuffer(data);
|
||||
|
||||
// Write chunk to temp S3 object instead of accumulating in memory
|
||||
const chunkPath = `oci/uploads/${uploadId}/chunk-${session.chunkIndex}`;
|
||||
await this.storage.putObject(chunkPath, chunkData);
|
||||
session.chunkPaths.push(chunkPath);
|
||||
session.chunkIndex++;
|
||||
session.totalSize += chunkData.length;
|
||||
session.lastActivity = new Date();
|
||||
|
||||
return {
|
||||
status: 202,
|
||||
headers: {
|
||||
'Location': `${this.basePath}/v2/${session.repository}/blobs/uploads/${uploadId}`,
|
||||
'Location': `${this.basePath}/${session.repository}/blobs/uploads/${uploadId}`,
|
||||
'Range': `0-${session.totalSize - 1}`,
|
||||
'Docker-Upload-UUID': uploadId,
|
||||
},
|
||||
@@ -566,7 +688,7 @@ export class OciRegistry extends BaseRegistry {
|
||||
private async completeUpload(
|
||||
uploadId: string,
|
||||
digest: string,
|
||||
finalData?: Buffer
|
||||
finalData?: Buffer | Uint8Array | unknown
|
||||
): Promise<IResponse> {
|
||||
const session = this.uploadSessions.get(uploadId);
|
||||
if (!session) {
|
||||
@@ -577,13 +699,52 @@ export class OciRegistry extends BaseRegistry {
|
||||
};
|
||||
}
|
||||
|
||||
const chunks = [...session.chunks];
|
||||
if (finalData) chunks.push(finalData);
|
||||
const blobData = Buffer.concat(chunks);
|
||||
// If there's final data in the PUT body, write it as the last chunk
|
||||
if (finalData) {
|
||||
const buf = this.toBuffer(finalData);
|
||||
const chunkPath = `oci/uploads/${uploadId}/chunk-${session.chunkIndex}`;
|
||||
await this.storage.putObject(chunkPath, buf);
|
||||
session.chunkPaths.push(chunkPath);
|
||||
session.chunkIndex++;
|
||||
session.totalSize += buf.length;
|
||||
}
|
||||
|
||||
// Verify digest
|
||||
const calculatedDigest = await this.calculateDigest(blobData);
|
||||
// Create a ReadableStream that assembles all chunks from S3 sequentially
|
||||
const chunkPaths = [...session.chunkPaths];
|
||||
const storage = this.storage;
|
||||
let chunkIdx = 0;
|
||||
const assembledStream = new ReadableStream<Uint8Array>({
|
||||
async pull(controller) {
|
||||
if (chunkIdx >= chunkPaths.length) {
|
||||
controller.close();
|
||||
return;
|
||||
}
|
||||
const result = await storage.getObjectStream(chunkPaths[chunkIdx++]);
|
||||
if (result) {
|
||||
const reader = result.stream.getReader();
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
if (value) controller.enqueue(value);
|
||||
}
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
// Pipe through hash transform for incremental digest verification
|
||||
const { transform: hashTransform, getDigest } = createHashTransform('sha256');
|
||||
const hashedStream = assembledStream.pipeThrough(hashTransform);
|
||||
|
||||
// Consume stream to buffer for S3 upload
|
||||
// (AWS SDK PutObjectCommand requires known content-length for streams;
|
||||
// the key win is chunks are NOT accumulated in memory during PATCH — they live in S3)
|
||||
const blobData = await streamToBuffer(hashedStream);
|
||||
|
||||
// Verify digest before storing
|
||||
const calculatedDigest = `sha256:${getDigest()}`;
|
||||
if (calculatedDigest !== digest) {
|
||||
await this.cleanupUploadChunks(session);
|
||||
this.uploadSessions.delete(uploadId);
|
||||
return {
|
||||
status: 400,
|
||||
headers: {},
|
||||
@@ -591,19 +752,36 @@ export class OciRegistry extends BaseRegistry {
|
||||
};
|
||||
}
|
||||
|
||||
// Store verified blob
|
||||
await this.storage.putOciBlob(digest, blobData);
|
||||
|
||||
// Cleanup temp chunks and session
|
||||
await this.cleanupUploadChunks(session);
|
||||
this.uploadSessions.delete(uploadId);
|
||||
|
||||
return {
|
||||
status: 201,
|
||||
headers: {
|
||||
'Location': `${this.basePath}/v2/${session.repository}/blobs/${digest}`,
|
||||
'Location': `${this.basePath}/${session.repository}/blobs/${digest}`,
|
||||
'Docker-Content-Digest': digest,
|
||||
},
|
||||
body: null,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete all temp S3 chunk objects for an upload session.
|
||||
*/
|
||||
private async cleanupUploadChunks(session: IUploadSession): Promise<void> {
|
||||
for (const chunkPath of session.chunkPaths) {
|
||||
try {
|
||||
await this.storage.deleteObject(chunkPath);
|
||||
} catch {
|
||||
// Best-effort cleanup
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async getUploadStatus(uploadId: string): Promise<IResponse> {
|
||||
const session = this.uploadSessions.get(uploadId);
|
||||
if (!session) {
|
||||
@@ -617,7 +795,7 @@ export class OciRegistry extends BaseRegistry {
|
||||
return {
|
||||
status: 204,
|
||||
headers: {
|
||||
'Location': `${this.basePath}/v2/${session.repository}/blobs/uploads/${uploadId}`,
|
||||
'Location': `${this.basePath}/${session.repository}/blobs/uploads/${uploadId}`,
|
||||
'Range': session.totalSize > 0 ? `0-${session.totalSize - 1}` : '0-0',
|
||||
'Docker-Upload-UUID': uploadId,
|
||||
},
|
||||
@@ -631,11 +809,7 @@ export class OciRegistry extends BaseRegistry {
|
||||
query: Record<string, string>
|
||||
): Promise<IResponse> {
|
||||
if (!await this.checkPermission(token, repository, 'pull')) {
|
||||
return {
|
||||
status: 401,
|
||||
headers: {},
|
||||
body: this.createError('DENIED', 'Insufficient permissions'),
|
||||
};
|
||||
return this.createUnauthorizedResponse(repository, 'pull');
|
||||
}
|
||||
|
||||
const tags = await this.getTagsData(repository);
|
||||
@@ -660,11 +834,7 @@ export class OciRegistry extends BaseRegistry {
|
||||
query: Record<string, string>
|
||||
): Promise<IResponse> {
|
||||
if (!await this.checkPermission(token, repository, 'pull')) {
|
||||
return {
|
||||
status: 401,
|
||||
headers: {},
|
||||
body: this.createError('DENIED', 'Insufficient permissions'),
|
||||
};
|
||||
return this.createUnauthorizedResponse(repository, 'pull');
|
||||
}
|
||||
|
||||
const response: IReferrersResponse = {
|
||||
@@ -684,6 +854,59 @@ export class OciRegistry extends BaseRegistry {
|
||||
// HELPER METHODS
|
||||
// ========================================================================
|
||||
|
||||
/**
|
||||
* Detect manifest content type from manifest content.
|
||||
* OCI Image Index has "manifests" array, OCI Image Manifest has "config" object.
|
||||
* Also checks the mediaType field if present.
|
||||
*/
|
||||
private detectManifestContentType(manifestData: Buffer): string {
|
||||
try {
|
||||
const manifest = JSON.parse(manifestData.toString('utf-8'));
|
||||
|
||||
// First check if manifest has explicit mediaType field
|
||||
if (manifest.mediaType) {
|
||||
return manifest.mediaType;
|
||||
}
|
||||
|
||||
// Otherwise detect from structure
|
||||
if (Array.isArray(manifest.manifests)) {
|
||||
// OCI Image Index (multi-arch manifest list)
|
||||
return 'application/vnd.oci.image.index.v1+json';
|
||||
} else if (manifest.config) {
|
||||
// OCI Image Manifest
|
||||
return 'application/vnd.oci.image.manifest.v1+json';
|
||||
}
|
||||
|
||||
// Fallback to standard manifest type
|
||||
return 'application/vnd.oci.image.manifest.v1+json';
|
||||
} catch (e) {
|
||||
// If parsing fails, return default
|
||||
return 'application/vnd.oci.image.manifest.v1+json';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert any binary-like data to Buffer.
|
||||
* Handles Buffer, Uint8Array (modern cross-platform), string, and objects.
|
||||
*
|
||||
* Note: Buffer.isBuffer(Uint8Array) returns false even though Buffer extends Uint8Array.
|
||||
* This is because Uint8Array is the modern, cross-platform standard while Buffer is Node.js-specific.
|
||||
* Many HTTP frameworks pass request bodies as Uint8Array for better compatibility.
|
||||
*/
|
||||
private toBuffer(data: unknown): Buffer {
|
||||
if (Buffer.isBuffer(data)) {
|
||||
return data;
|
||||
}
|
||||
if (data instanceof Uint8Array) {
|
||||
return Buffer.from(data);
|
||||
}
|
||||
if (typeof data === 'string') {
|
||||
return Buffer.from(data, 'utf-8');
|
||||
}
|
||||
// Fallback: serialize object to JSON (may cause digest mismatch for manifests)
|
||||
return Buffer.from(JSON.stringify(data));
|
||||
}
|
||||
|
||||
private async getTagsData(repository: string): Promise<Record<string, string>> {
|
||||
const path = `oci/tags/${repository}/tags.json`;
|
||||
const data = await this.storage.getObject(path);
|
||||
@@ -697,7 +920,7 @@ export class OciRegistry extends BaseRegistry {
|
||||
}
|
||||
|
||||
private generateUploadId(): string {
|
||||
return `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
|
||||
return `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
|
||||
}
|
||||
|
||||
private async calculateDigest(data: Buffer): Promise<string> {
|
||||
@@ -712,6 +935,37 @@ export class OciRegistry extends BaseRegistry {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an unauthorized response with proper WWW-Authenticate header.
|
||||
* Per OCI Distribution Spec, 401 responses MUST include WWW-Authenticate header.
|
||||
*/
|
||||
private createUnauthorizedResponse(repository: string, action: string): IResponse {
|
||||
const realm = this.ociTokens?.realm || `${this.basePath}/token`;
|
||||
const service = this.ociTokens?.service || 'registry';
|
||||
return {
|
||||
status: 401,
|
||||
headers: {
|
||||
'WWW-Authenticate': `Bearer realm="${realm}",service="${service}",scope="repository:${repository}:${action}"`,
|
||||
},
|
||||
body: this.createError('DENIED', 'Insufficient permissions'),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an unauthorized HEAD response (no body per HTTP spec).
|
||||
*/
|
||||
private createUnauthorizedHeadResponse(repository: string, action: string): IResponse {
|
||||
const realm = this.ociTokens?.realm || `${this.basePath}/token`;
|
||||
const service = this.ociTokens?.service || 'registry';
|
||||
return {
|
||||
status: 401,
|
||||
headers: {
|
||||
'WWW-Authenticate': `Bearer realm="${realm}",service="${service}",scope="repository:${repository}:${action}"`,
|
||||
},
|
||||
body: null,
|
||||
};
|
||||
}
|
||||
|
||||
private startUploadSessionCleanup(): void {
|
||||
this.cleanupInterval = setInterval(() => {
|
||||
const now = new Date();
|
||||
@@ -719,6 +973,8 @@ export class OciRegistry extends BaseRegistry {
|
||||
|
||||
for (const [uploadId, session] of this.uploadSessions.entries()) {
|
||||
if (now.getTime() - session.lastActivity.getTime() > maxAge) {
|
||||
// Clean up temp S3 chunks for stale sessions
|
||||
this.cleanupUploadChunks(session).catch(() => {});
|
||||
this.uploadSessions.delete(uploadId);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,273 @@
|
||||
import * as plugins from '../plugins.js';
|
||||
import { BaseUpstream } from '../upstream/classes.baseupstream.js';
|
||||
import type {
|
||||
IProtocolUpstreamConfig,
|
||||
IUpstreamFetchContext,
|
||||
IUpstreamResult,
|
||||
IUpstreamRegistryConfig,
|
||||
} from '../upstream/interfaces.upstream.js';
|
||||
import type { IOciManifest, IOciImageIndex, ITagList } from './interfaces.oci.js';
|
||||
|
||||
/**
|
||||
* OCI-specific upstream implementation.
|
||||
*
|
||||
* Handles:
|
||||
* - Manifest fetching (image manifests and index manifests)
|
||||
* - Blob proxying (layers, configs)
|
||||
* - Tag list fetching
|
||||
* - Content-addressable caching (blobs are immutable)
|
||||
* - Docker Hub authentication flow
|
||||
*/
|
||||
export class OciUpstream extends BaseUpstream {
|
||||
protected readonly protocolName = 'oci';
|
||||
|
||||
/** Local registry base path for URL building */
|
||||
private readonly localBasePath: string;
|
||||
|
||||
/** API prefix for outbound OCI requests (default: /v2) */
|
||||
private readonly apiPrefix: string;
|
||||
|
||||
constructor(
|
||||
config: IProtocolUpstreamConfig,
|
||||
localBasePath: string = '/oci',
|
||||
logger?: plugins.smartlog.Smartlog,
|
||||
apiPrefix: string = '/v2',
|
||||
) {
|
||||
super(config, logger);
|
||||
this.localBasePath = localBasePath;
|
||||
this.apiPrefix = apiPrefix;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a manifest from upstream registries.
|
||||
*/
|
||||
public async fetchManifest(
|
||||
repository: string,
|
||||
reference: string,
|
||||
): Promise<{ manifest: IOciManifest | IOciImageIndex; contentType: string; digest: string } | null> {
|
||||
const context: IUpstreamFetchContext = {
|
||||
protocol: 'oci',
|
||||
resource: repository,
|
||||
resourceType: 'manifest',
|
||||
path: `${this.apiPrefix}/${repository}/manifests/${reference}`,
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'accept': [
|
||||
'application/vnd.oci.image.manifest.v1+json',
|
||||
'application/vnd.oci.image.index.v1+json',
|
||||
'application/vnd.docker.distribution.manifest.v2+json',
|
||||
'application/vnd.docker.distribution.manifest.list.v2+json',
|
||||
'application/vnd.docker.distribution.manifest.v1+json',
|
||||
].join(', '),
|
||||
},
|
||||
query: {},
|
||||
};
|
||||
|
||||
const result = await this.fetch(context);
|
||||
|
||||
if (!result || !result.success) {
|
||||
return null;
|
||||
}
|
||||
|
||||
let manifest: IOciManifest | IOciImageIndex;
|
||||
if (Buffer.isBuffer(result.body)) {
|
||||
manifest = JSON.parse(result.body.toString('utf8'));
|
||||
} else {
|
||||
manifest = result.body;
|
||||
}
|
||||
|
||||
const contentType = result.headers['content-type'] || 'application/vnd.oci.image.manifest.v1+json';
|
||||
const digest = result.headers['docker-content-digest'] || '';
|
||||
|
||||
return { manifest, contentType, digest };
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a manifest exists in upstream (HEAD request).
|
||||
*/
|
||||
public async headManifest(
|
||||
repository: string,
|
||||
reference: string,
|
||||
): Promise<{ exists: boolean; contentType?: string; digest?: string; size?: number } | null> {
|
||||
const context: IUpstreamFetchContext = {
|
||||
protocol: 'oci',
|
||||
resource: repository,
|
||||
resourceType: 'manifest',
|
||||
path: `${this.apiPrefix}/${repository}/manifests/${reference}`,
|
||||
method: 'HEAD',
|
||||
headers: {
|
||||
'accept': [
|
||||
'application/vnd.oci.image.manifest.v1+json',
|
||||
'application/vnd.oci.image.index.v1+json',
|
||||
'application/vnd.docker.distribution.manifest.v2+json',
|
||||
'application/vnd.docker.distribution.manifest.list.v2+json',
|
||||
].join(', '),
|
||||
},
|
||||
query: {},
|
||||
};
|
||||
|
||||
const result = await this.fetch(context);
|
||||
|
||||
if (!result) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!result.success) {
|
||||
return { exists: false };
|
||||
}
|
||||
|
||||
return {
|
||||
exists: true,
|
||||
contentType: result.headers['content-type'],
|
||||
digest: result.headers['docker-content-digest'],
|
||||
size: result.headers['content-length'] ? parseInt(result.headers['content-length'], 10) : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a blob from upstream registries.
|
||||
*/
|
||||
public async fetchBlob(repository: string, digest: string): Promise<Buffer | null> {
|
||||
const context: IUpstreamFetchContext = {
|
||||
protocol: 'oci',
|
||||
resource: repository,
|
||||
resourceType: 'blob',
|
||||
path: `${this.apiPrefix}/${repository}/blobs/${digest}`,
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'accept': 'application/octet-stream',
|
||||
},
|
||||
query: {},
|
||||
};
|
||||
|
||||
const result = await this.fetch(context);
|
||||
|
||||
if (!result || !result.success) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return Buffer.isBuffer(result.body) ? result.body : Buffer.from(result.body);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a blob exists in upstream (HEAD request).
|
||||
*/
|
||||
public async headBlob(
|
||||
repository: string,
|
||||
digest: string,
|
||||
): Promise<{ exists: boolean; size?: number } | null> {
|
||||
const context: IUpstreamFetchContext = {
|
||||
protocol: 'oci',
|
||||
resource: repository,
|
||||
resourceType: 'blob',
|
||||
path: `${this.apiPrefix}/${repository}/blobs/${digest}`,
|
||||
method: 'HEAD',
|
||||
headers: {},
|
||||
query: {},
|
||||
};
|
||||
|
||||
const result = await this.fetch(context);
|
||||
|
||||
if (!result) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!result.success) {
|
||||
return { exists: false };
|
||||
}
|
||||
|
||||
return {
|
||||
exists: true,
|
||||
size: result.headers['content-length'] ? parseInt(result.headers['content-length'], 10) : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch the tag list for a repository.
|
||||
*/
|
||||
public async fetchTags(repository: string, n?: number, last?: string): Promise<ITagList | null> {
|
||||
const query: Record<string, string> = {};
|
||||
if (n) query.n = n.toString();
|
||||
if (last) query.last = last;
|
||||
|
||||
const context: IUpstreamFetchContext = {
|
||||
protocol: 'oci',
|
||||
resource: repository,
|
||||
resourceType: 'tags',
|
||||
path: `${this.apiPrefix}/${repository}/tags/list`,
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'accept': 'application/json',
|
||||
},
|
||||
query,
|
||||
};
|
||||
|
||||
const result = await this.fetch(context);
|
||||
|
||||
if (!result || !result.success) {
|
||||
return null;
|
||||
}
|
||||
|
||||
let tagList: ITagList;
|
||||
if (Buffer.isBuffer(result.body)) {
|
||||
tagList = JSON.parse(result.body.toString('utf8'));
|
||||
} else {
|
||||
tagList = result.body;
|
||||
}
|
||||
|
||||
return tagList;
|
||||
}
|
||||
|
||||
/**
|
||||
* Override URL building for OCI-specific handling.
|
||||
* OCI registries use a configurable API prefix (default /v2/) and may require
|
||||
* special handling for Docker Hub.
|
||||
*/
|
||||
protected buildUpstreamUrl(
|
||||
upstream: IUpstreamRegistryConfig,
|
||||
context: IUpstreamFetchContext,
|
||||
): string {
|
||||
let baseUrl = upstream.url;
|
||||
|
||||
// Remove trailing slash
|
||||
if (baseUrl.endsWith('/')) {
|
||||
baseUrl = baseUrl.slice(0, -1);
|
||||
}
|
||||
|
||||
// Use per-upstream apiPrefix if configured, otherwise use the instance default
|
||||
const prefix = upstream.apiPrefix || this.apiPrefix;
|
||||
|
||||
// Handle Docker Hub special case
|
||||
// Docker Hub uses registry-1.docker.io but library images need special handling
|
||||
if (baseUrl.includes('docker.io') || baseUrl.includes('registry-1.docker.io')) {
|
||||
// For library images (e.g., "nginx" -> "library/nginx")
|
||||
const escapedPrefix = prefix.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
const pathParts = context.path.match(new RegExp(`^${escapedPrefix}\\/([^\\/]+)\\/(.+)$`));
|
||||
if (pathParts) {
|
||||
const [, repository, rest] = pathParts;
|
||||
// If repository doesn't contain a slash, it's a library image
|
||||
if (!repository.includes('/')) {
|
||||
return `${baseUrl}${prefix}/library/${repository}/${rest}`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return `${baseUrl}${context.path}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Override header building for OCI-specific authentication.
|
||||
* OCI registries may require token-based auth obtained from a separate endpoint.
|
||||
*/
|
||||
protected buildHeaders(
|
||||
upstream: IUpstreamRegistryConfig,
|
||||
context: IUpstreamFetchContext,
|
||||
): Record<string, string> {
|
||||
const headers = super.buildHeaders(upstream, context);
|
||||
|
||||
// OCI registries typically use Docker-Distribution-API-Version header
|
||||
headers['docker-distribution-api-version'] = 'registry/2.0';
|
||||
|
||||
return headers;
|
||||
}
|
||||
}
|
||||
@@ -3,4 +3,5 @@
|
||||
*/
|
||||
|
||||
export { OciRegistry } from './classes.ociregistry.js';
|
||||
export { OciUpstream } from './classes.ociupstream.js';
|
||||
export * from './interfaces.oci.js';
|
||||
|
||||
@@ -62,6 +62,10 @@ export interface IUploadSession {
|
||||
uploadId: string;
|
||||
repository: string;
|
||||
chunks: Buffer[];
|
||||
/** S3 paths to temp chunk objects (streaming mode) */
|
||||
chunkPaths: string[];
|
||||
/** Index counter for naming temp chunk objects */
|
||||
chunkIndex: number;
|
||||
totalSize: number;
|
||||
createdAt: Date;
|
||||
lastActivity: Date;
|
||||
|
||||
+15
-2
@@ -1,11 +1,24 @@
|
||||
// native scope
|
||||
import * as asyncHooks from 'node:async_hooks';
|
||||
import * as path from 'path';
|
||||
|
||||
export { path };
|
||||
export { asyncHooks, path };
|
||||
|
||||
// @push.rocks scope
|
||||
import * as smartarchive from '@push.rocks/smartarchive';
|
||||
import * as smartbucket from '@push.rocks/smartbucket';
|
||||
import * as smartlog from '@push.rocks/smartlog';
|
||||
import * as smartpath from '@push.rocks/smartpath';
|
||||
import * as smartrequest from '@push.rocks/smartrequest';
|
||||
|
||||
export { smartbucket, smartlog, smartpath };
|
||||
export { smartarchive, smartbucket, smartlog, smartpath, smartrequest };
|
||||
|
||||
// @tsclass scope
|
||||
import * as tsclass from '@tsclass/tsclass';
|
||||
|
||||
export { tsclass };
|
||||
|
||||
// third party
|
||||
import { minimatch } from 'minimatch';
|
||||
|
||||
export { minimatch };
|
||||
|
||||
@@ -0,0 +1,754 @@
|
||||
import { Smartlog } from '@push.rocks/smartlog';
|
||||
import { BaseRegistry } from '../core/classes.baseregistry.js';
|
||||
import { RegistryStorage } from '../core/classes.registrystorage.js';
|
||||
import { AuthManager } from '../core/classes.authmanager.js';
|
||||
import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from '../core/interfaces.core.js';
|
||||
import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
|
||||
import { isBinaryData, toBuffer } from '../core/helpers.buffer.js';
|
||||
import type {
|
||||
IPypiPackageMetadata,
|
||||
IPypiFile,
|
||||
IPypiError,
|
||||
IPypiUploadResponse,
|
||||
} from './interfaces.pypi.js';
|
||||
import * as helpers from './helpers.pypi.js';
|
||||
import { PypiUpstream } from './classes.pypiupstream.js';
|
||||
|
||||
/**
|
||||
* PyPI registry implementation
|
||||
* Implements PEP 503 (Simple API), PEP 691 (JSON API), and legacy upload API
|
||||
*/
|
||||
export class PypiRegistry extends BaseRegistry {
|
||||
private storage: RegistryStorage;
|
||||
private authManager: AuthManager;
|
||||
private basePath: string = '/pypi';
|
||||
private registryUrl: string;
|
||||
private logger: Smartlog;
|
||||
private upstreamProvider: IUpstreamProvider | null = null;
|
||||
|
||||
constructor(
|
||||
storage: RegistryStorage,
|
||||
authManager: AuthManager,
|
||||
basePath: string = '/pypi',
|
||||
registryUrl: string = 'http://localhost:5000',
|
||||
upstreamProvider?: IUpstreamProvider
|
||||
) {
|
||||
super();
|
||||
this.storage = storage;
|
||||
this.authManager = authManager;
|
||||
this.basePath = basePath;
|
||||
this.registryUrl = registryUrl;
|
||||
this.upstreamProvider = upstreamProvider || null;
|
||||
|
||||
this.logger = this.createProtocolLogger('pypi-registry', 'pypi');
|
||||
}
|
||||
|
||||
/**
|
||||
* Get upstream for a specific request.
|
||||
* Calls the provider to resolve upstream config dynamically.
|
||||
*/
|
||||
private async getUpstreamForRequest(
|
||||
resource: string,
|
||||
resourceType: string,
|
||||
method: string,
|
||||
actor?: IRequestActor
|
||||
): Promise<PypiUpstream | null> {
|
||||
if (!this.upstreamProvider) return null;
|
||||
|
||||
const config = await this.upstreamProvider.resolveUpstreamConfig({
|
||||
protocol: 'pypi',
|
||||
resource,
|
||||
scope: resource, // For PyPI, package name is the scope
|
||||
actor,
|
||||
method,
|
||||
resourceType,
|
||||
});
|
||||
|
||||
if (!config?.enabled) return null;
|
||||
return new PypiUpstream(config, this.registryUrl, this.logger);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up resources (timers, connections, etc.)
|
||||
*/
|
||||
public destroy(): void {
|
||||
// No persistent upstream to clean up with dynamic provider
|
||||
}
|
||||
|
||||
public async init(): Promise<void> {
|
||||
// Initialize root Simple API index if not exists
|
||||
const existingIndex = await this.storage.getPypiSimpleRootIndex();
|
||||
if (!existingIndex) {
|
||||
const html = helpers.generateSimpleRootHtml([]);
|
||||
await this.storage.putPypiSimpleRootIndex(html);
|
||||
this.logger.log('info', 'Initialized PyPI root index');
|
||||
}
|
||||
}
|
||||
|
||||
public getBasePath(): string {
|
||||
return this.basePath;
|
||||
}
|
||||
|
||||
public async handleRequest(context: IRequestContext): Promise<IResponse> {
|
||||
let path = context.path.replace(this.basePath, '');
|
||||
|
||||
// Extract token (Basic Auth or Bearer)
|
||||
const token = await this.extractToken(context);
|
||||
|
||||
const actor: IRequestActor = this.buildRequestActor(context, token);
|
||||
|
||||
return this.storage.withContext({ protocol: 'pypi', actor }, async () => {
|
||||
// Also handle /simple path prefix
|
||||
if (path.startsWith('/simple')) {
|
||||
path = path.replace('/simple', '');
|
||||
return this.handleSimpleRequest(path, context, actor);
|
||||
}
|
||||
|
||||
this.logger.log('debug', `handleRequest: ${context.method} ${path}`, {
|
||||
method: context.method,
|
||||
path,
|
||||
hasAuth: !!token
|
||||
});
|
||||
|
||||
// Root upload endpoint (POST /)
|
||||
if ((path === '/' || path === '') && context.method === 'POST') {
|
||||
return this.handleUpload(context, token);
|
||||
}
|
||||
|
||||
// Package metadata JSON API: GET /{package}/json
|
||||
const jsonMatch = path.match(/^\/([^\/]+)\/json$/);
|
||||
if (jsonMatch && context.method === 'GET') {
|
||||
return this.handlePackageJson(jsonMatch[1]);
|
||||
}
|
||||
|
||||
// Version-specific JSON API: GET /{package}/{version}/json
|
||||
const versionJsonMatch = path.match(/^\/([^\/]+)\/([^\/]+)\/json$/);
|
||||
if (versionJsonMatch && context.method === 'GET') {
|
||||
return this.handleVersionJson(versionJsonMatch[1], versionJsonMatch[2]);
|
||||
}
|
||||
|
||||
// Package file download: GET /packages/{package}/{filename}
|
||||
const downloadMatch = path.match(/^\/packages\/([^\/]+)\/(.+)$/);
|
||||
if (downloadMatch && context.method === 'GET') {
|
||||
return this.handleDownload(downloadMatch[1], downloadMatch[2], actor);
|
||||
}
|
||||
|
||||
// Delete package: DELETE /packages/{package}
|
||||
if (path.match(/^\/packages\/([^\/]+)$/) && context.method === 'DELETE') {
|
||||
const packageName = path.match(/^\/packages\/([^\/]+)$/)?.[1];
|
||||
return this.handleDeletePackage(packageName!, token);
|
||||
}
|
||||
|
||||
// Delete version: DELETE /packages/{package}/{version}
|
||||
const deleteVersionMatch = path.match(/^\/packages\/([^\/]+)\/([^\/]+)$/);
|
||||
if (deleteVersionMatch && context.method === 'DELETE') {
|
||||
return this.handleDeleteVersion(deleteVersionMatch[1], deleteVersionMatch[2], token);
|
||||
}
|
||||
|
||||
return {
|
||||
status: 404,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: { error: 'Not Found' },
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if token has permission for resource
|
||||
*/
|
||||
protected async checkPermission(
|
||||
token: IAuthToken | null,
|
||||
resource: string,
|
||||
action: string
|
||||
): Promise<boolean> {
|
||||
if (!token) return false;
|
||||
return this.authManager.authorize(token, `pypi:package:${resource}`, action);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle Simple API requests (PEP 503 HTML or PEP 691 JSON)
|
||||
*/
|
||||
private async handleSimpleRequest(path: string, context: IRequestContext, actor?: IRequestActor): Promise<IResponse> {
|
||||
// Ensure path ends with / (PEP 503 requirement)
|
||||
if (!path.endsWith('/') && !path.includes('.')) {
|
||||
return {
|
||||
status: 301,
|
||||
headers: { 'Location': `${this.basePath}/simple${path}/` },
|
||||
body: Buffer.from(''),
|
||||
};
|
||||
}
|
||||
|
||||
// Root index: /simple/
|
||||
if (path === '/' || path === '') {
|
||||
return this.handleSimpleRoot(context);
|
||||
}
|
||||
|
||||
// Package index: /simple/{package}/
|
||||
const packageMatch = path.match(/^\/([^\/]+)\/$/);
|
||||
if (packageMatch) {
|
||||
return this.handleSimplePackage(packageMatch[1], context, actor);
|
||||
}
|
||||
|
||||
return {
|
||||
status: 404,
|
||||
headers: { 'Content-Type': 'text/html; charset=utf-8' },
|
||||
body: Buffer.from('<html><body><h1>404 Not Found</h1></body></html>'),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle Simple API root index
|
||||
* Returns HTML (PEP 503) or JSON (PEP 691) based on Accept header
|
||||
*/
|
||||
private async handleSimpleRoot(context: IRequestContext): Promise<IResponse> {
|
||||
const acceptHeader = context.headers['accept'] || context.headers['Accept'] || '';
|
||||
const preferJson = acceptHeader.includes('application/vnd.pypi.simple') &&
|
||||
acceptHeader.includes('json');
|
||||
|
||||
const packages = await this.storage.listPypiPackages();
|
||||
|
||||
if (preferJson) {
|
||||
// PEP 691: JSON response
|
||||
const response = helpers.generateJsonRootResponse(packages);
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/vnd.pypi.simple.v1+json',
|
||||
'Cache-Control': 'public, max-age=600'
|
||||
},
|
||||
body: response,
|
||||
};
|
||||
} else {
|
||||
// PEP 503: HTML response
|
||||
const html = helpers.generateSimpleRootHtml(packages);
|
||||
|
||||
// Update stored index
|
||||
await this.storage.putPypiSimpleRootIndex(html);
|
||||
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'text/html; charset=utf-8',
|
||||
'Cache-Control': 'public, max-age=600'
|
||||
},
|
||||
body: html,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle Simple API package index
|
||||
* Returns HTML (PEP 503) or JSON (PEP 691) based on Accept header
|
||||
*/
|
||||
private async handleSimplePackage(packageName: string, context: IRequestContext, actor?: IRequestActor): Promise<IResponse> {
|
||||
const normalized = helpers.normalizePypiPackageName(packageName);
|
||||
|
||||
// Get package metadata
|
||||
let metadata = await this.storage.getPypiPackageMetadata(normalized);
|
||||
|
||||
// Try upstream if not found locally
|
||||
if (!metadata) {
|
||||
const upstream = await this.getUpstreamForRequest(normalized, 'simple', 'GET', actor);
|
||||
if (upstream) {
|
||||
const upstreamHtml = await upstream.fetchSimplePackage(normalized);
|
||||
if (upstreamHtml) {
|
||||
// Parse the HTML to extract file information and cache it
|
||||
// For now, just return the upstream HTML directly (caching can be improved later)
|
||||
const acceptHeader = context.headers['accept'] || context.headers['Accept'] || '';
|
||||
const preferJson = acceptHeader.includes('application/vnd.pypi.simple') &&
|
||||
acceptHeader.includes('json');
|
||||
|
||||
if (preferJson) {
|
||||
// Try to get JSON format from upstream
|
||||
const upstreamJson = await upstream.fetchPackageJson(normalized);
|
||||
if (upstreamJson) {
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/vnd.pypi.simple.v1+json',
|
||||
'Cache-Control': 'public, max-age=300'
|
||||
},
|
||||
body: upstreamJson,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Return HTML format
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'text/html; charset=utf-8',
|
||||
'Cache-Control': 'public, max-age=300'
|
||||
},
|
||||
body: upstreamHtml,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!metadata) {
|
||||
return this.errorResponse(404, 'Package not found');
|
||||
}
|
||||
|
||||
// Build file list from all versions
|
||||
const files: IPypiFile[] = [];
|
||||
for (const [version, versionMeta] of Object.entries(metadata.versions || {})) {
|
||||
for (const file of (versionMeta as any).files || []) {
|
||||
files.push({
|
||||
filename: file.filename,
|
||||
url: `${this.registryUrl}/pypi/packages/${normalized}/${file.filename}`,
|
||||
hashes: file.hashes,
|
||||
'requires-python': file['requires-python'],
|
||||
yanked: file.yanked || (versionMeta as any).yanked,
|
||||
size: file.size,
|
||||
'upload-time': file['upload-time'],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const acceptHeader = context.headers['accept'] || context.headers['Accept'] || '';
|
||||
const preferJson = acceptHeader.includes('application/vnd.pypi.simple') &&
|
||||
acceptHeader.includes('json');
|
||||
|
||||
if (preferJson) {
|
||||
// PEP 691: JSON response
|
||||
const response = helpers.generateJsonPackageResponse(normalized, files);
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/vnd.pypi.simple.v1+json',
|
||||
'Cache-Control': 'public, max-age=300'
|
||||
},
|
||||
body: response,
|
||||
};
|
||||
} else {
|
||||
// PEP 503: HTML response
|
||||
const html = helpers.generateSimplePackageHtml(normalized, files, this.registryUrl);
|
||||
|
||||
// Update stored index
|
||||
await this.storage.putPypiSimpleIndex(normalized, html);
|
||||
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'text/html; charset=utf-8',
|
||||
'Cache-Control': 'public, max-age=300'
|
||||
},
|
||||
body: html,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract authentication token from request
|
||||
*/
|
||||
private async extractToken(context: IRequestContext): Promise<IAuthToken | null> {
|
||||
const authHeader = this.getAuthorizationHeader(context);
|
||||
if (!authHeader) return null;
|
||||
|
||||
// Handle Basic Auth (username:password or __token__:token)
|
||||
const basicCredentials = this.parseBasicAuthHeader(authHeader);
|
||||
if (basicCredentials) {
|
||||
const { username, password } = basicCredentials;
|
||||
|
||||
// PyPI token authentication: username = __token__
|
||||
if (username === '__token__') {
|
||||
return this.authManager.validateToken(password, 'pypi');
|
||||
}
|
||||
|
||||
// Username/password authentication (would need user lookup)
|
||||
// For now, not implemented
|
||||
return null;
|
||||
}
|
||||
|
||||
// Handle Bearer token
|
||||
const token = this.extractBearerToken(authHeader);
|
||||
if (token) {
|
||||
return this.authManager.validateToken(token, 'pypi');
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle package upload (multipart/form-data)
|
||||
* POST / with :action=file_upload
|
||||
*/
|
||||
private async handleUpload(context: IRequestContext, token: IAuthToken | null): Promise<IResponse> {
|
||||
if (!token) {
|
||||
return {
|
||||
status: 401,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'WWW-Authenticate': 'Basic realm="PyPI"'
|
||||
},
|
||||
body: { error: 'Authentication required' },
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
// Parse multipart form data (context.body should be parsed by server)
|
||||
const formData = context.body as any; // Assuming parsed multipart data
|
||||
|
||||
if (!formData || formData[':action'] !== 'file_upload') {
|
||||
return this.errorResponse(400, 'Invalid upload request');
|
||||
}
|
||||
|
||||
// Extract required fields - support both nested and flat body formats
|
||||
const packageName = formData.name;
|
||||
const version = formData.version;
|
||||
// Support both: formData.content.filename (multipart parsed) and formData.filename (flat)
|
||||
const filename = formData.content?.filename || formData.filename;
|
||||
// Support both: formData.content.data (multipart parsed) and formData.content (Buffer/Uint8Array directly)
|
||||
const rawContent = formData.content?.data || (isBinaryData(formData.content) ? formData.content : null);
|
||||
const fileData = rawContent ? toBuffer(rawContent) : null;
|
||||
const filetype = formData.filetype; // 'bdist_wheel' or 'sdist'
|
||||
const pyversion = formData.pyversion;
|
||||
|
||||
if (!packageName || !version || !filename || !fileData) {
|
||||
return this.errorResponse(400, 'Missing required fields');
|
||||
}
|
||||
|
||||
// Validate package name
|
||||
if (!helpers.isValidPackageName(packageName)) {
|
||||
return this.errorResponse(400, 'Invalid package name');
|
||||
}
|
||||
|
||||
const normalized = helpers.normalizePypiPackageName(packageName);
|
||||
|
||||
// Check permission
|
||||
if (!(await this.checkPermission(token, normalized, 'write'))) {
|
||||
return this.errorResponse(403, 'Insufficient permissions');
|
||||
}
|
||||
|
||||
// Calculate and verify hashes
|
||||
const hashes: Record<string, string> = {};
|
||||
|
||||
// Always calculate SHA256
|
||||
const actualSha256 = await helpers.calculateHash(fileData, 'sha256');
|
||||
hashes.sha256 = actualSha256;
|
||||
|
||||
// Verify client-provided SHA256 if present
|
||||
if (formData.sha256_digest && formData.sha256_digest !== actualSha256) {
|
||||
return this.errorResponse(400, 'SHA256 hash mismatch');
|
||||
}
|
||||
|
||||
// Calculate MD5 if requested
|
||||
if (formData.md5_digest) {
|
||||
const actualMd5 = await helpers.calculateHash(fileData, 'md5');
|
||||
hashes.md5 = actualMd5;
|
||||
|
||||
// Verify if client provided MD5
|
||||
if (formData.md5_digest !== actualMd5) {
|
||||
return this.errorResponse(400, 'MD5 hash mismatch');
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate Blake2b if requested
|
||||
if (formData.blake2_256_digest) {
|
||||
const actualBlake2b = await helpers.calculateHash(fileData, 'blake2b');
|
||||
hashes.blake2b = actualBlake2b;
|
||||
|
||||
// Verify if client provided Blake2b
|
||||
if (formData.blake2_256_digest !== actualBlake2b) {
|
||||
return this.errorResponse(400, 'Blake2b hash mismatch');
|
||||
}
|
||||
}
|
||||
|
||||
// Store file
|
||||
await this.storage.putPypiPackageFile(normalized, filename, fileData);
|
||||
|
||||
// Update metadata
|
||||
let metadata = await this.storage.getPypiPackageMetadata(normalized);
|
||||
if (!metadata) {
|
||||
metadata = {
|
||||
name: normalized,
|
||||
versions: {},
|
||||
};
|
||||
}
|
||||
|
||||
if (!metadata.versions[version]) {
|
||||
metadata.versions[version] = {
|
||||
version,
|
||||
files: [],
|
||||
};
|
||||
}
|
||||
|
||||
// Add file to version
|
||||
metadata.versions[version].files.push({
|
||||
filename,
|
||||
path: `pypi/packages/${normalized}/${filename}`,
|
||||
filetype,
|
||||
python_version: pyversion,
|
||||
hashes,
|
||||
size: fileData.length,
|
||||
'requires-python': formData.requires_python,
|
||||
'upload-time': new Date().toISOString(),
|
||||
'uploaded-by': token.userId,
|
||||
});
|
||||
|
||||
// Store core metadata if provided
|
||||
if (formData.summary || formData.description) {
|
||||
metadata.versions[version].metadata = helpers.extractCoreMetadata(formData);
|
||||
}
|
||||
|
||||
metadata['last-modified'] = new Date().toISOString();
|
||||
await this.storage.putPypiPackageMetadata(normalized, metadata);
|
||||
|
||||
this.logger.log('info', `Package uploaded: ${normalized} ${version}`, {
|
||||
filename,
|
||||
size: fileData.length
|
||||
});
|
||||
|
||||
return {
|
||||
status: 201,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: {
|
||||
message: 'Package uploaded successfully',
|
||||
url: `${this.registryUrl}/pypi/packages/${normalized}/${filename}`
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
this.logger.log('error', 'Upload failed', { error: (error as Error).message });
|
||||
return this.errorResponse(500, 'Upload failed: ' + (error as Error).message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle package download
|
||||
*/
|
||||
private async handleDownload(packageName: string, filename: string, actor?: IRequestActor): Promise<IResponse> {
|
||||
const normalized = helpers.normalizePypiPackageName(packageName);
|
||||
|
||||
// Try streaming from local storage first
|
||||
const streamResult = await this.storage.getPypiPackageFileStream(normalized, filename);
|
||||
|
||||
if (streamResult) {
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/octet-stream',
|
||||
'Content-Disposition': `attachment; filename="${filename}"`,
|
||||
'Content-Length': streamResult.size.toString()
|
||||
},
|
||||
body: streamResult.stream,
|
||||
};
|
||||
}
|
||||
|
||||
// Try upstream if not found locally
|
||||
let fileData: Buffer | null = null;
|
||||
const upstream = await this.getUpstreamForRequest(normalized, 'file', 'GET', actor);
|
||||
if (upstream) {
|
||||
fileData = await upstream.fetchPackageFile(normalized, filename);
|
||||
if (fileData) {
|
||||
// Cache locally
|
||||
await this.storage.putPypiPackageFile(normalized, filename, fileData);
|
||||
}
|
||||
}
|
||||
|
||||
if (!fileData) {
|
||||
return {
|
||||
status: 404,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: { error: 'File not found' },
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/octet-stream',
|
||||
'Content-Disposition': `attachment; filename="${filename}"`,
|
||||
'Content-Length': fileData.length.toString()
|
||||
},
|
||||
body: fileData,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle package JSON API (all versions)
|
||||
* Returns format compatible with official PyPI JSON API
|
||||
*/
|
||||
private async handlePackageJson(packageName: string): Promise<IResponse> {
|
||||
const normalized = helpers.normalizePypiPackageName(packageName);
|
||||
const metadata = await this.storage.getPypiPackageMetadata(normalized);
|
||||
|
||||
if (!metadata) {
|
||||
return this.errorResponse(404, 'Package not found');
|
||||
}
|
||||
|
||||
// Find latest version for info
|
||||
const versions = Object.keys(metadata.versions || {});
|
||||
const latestVersion = versions.length > 0 ? versions[versions.length - 1] : null;
|
||||
const latestMeta = latestVersion ? metadata.versions[latestVersion] : null;
|
||||
|
||||
// Build URLs array from latest version files
|
||||
const urls = latestMeta?.files?.map((file: any) => ({
|
||||
filename: file.filename,
|
||||
url: `${this.registryUrl}/pypi/packages/${normalized}/${file.filename}`,
|
||||
digests: file.hashes,
|
||||
requires_python: file['requires-python'],
|
||||
size: file.size,
|
||||
upload_time: file['upload-time'],
|
||||
packagetype: file.filetype,
|
||||
python_version: file.python_version,
|
||||
})) || [];
|
||||
|
||||
// Build releases object
|
||||
const releases: Record<string, any[]> = {};
|
||||
for (const [ver, verMeta] of Object.entries(metadata.versions || {})) {
|
||||
releases[ver] = (verMeta as any).files?.map((file: any) => ({
|
||||
filename: file.filename,
|
||||
url: `${this.registryUrl}/pypi/packages/${normalized}/${file.filename}`,
|
||||
digests: file.hashes,
|
||||
requires_python: file['requires-python'],
|
||||
size: file.size,
|
||||
upload_time: file['upload-time'],
|
||||
packagetype: file.filetype,
|
||||
python_version: file.python_version,
|
||||
})) || [];
|
||||
}
|
||||
|
||||
const response = {
|
||||
info: {
|
||||
name: normalized,
|
||||
version: latestVersion,
|
||||
summary: latestMeta?.metadata?.summary,
|
||||
description: latestMeta?.metadata?.description,
|
||||
author: latestMeta?.metadata?.author,
|
||||
author_email: latestMeta?.metadata?.['author-email'],
|
||||
license: latestMeta?.metadata?.license,
|
||||
requires_python: latestMeta?.files?.[0]?.['requires-python'],
|
||||
...latestMeta?.metadata,
|
||||
},
|
||||
urls,
|
||||
releases,
|
||||
};
|
||||
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'public, max-age=300'
|
||||
},
|
||||
body: response,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle version-specific JSON API
|
||||
* Returns format compatible with official PyPI JSON API
|
||||
*/
|
||||
private async handleVersionJson(packageName: string, version: string): Promise<IResponse> {
|
||||
const normalized = helpers.normalizePypiPackageName(packageName);
|
||||
const metadata = await this.storage.getPypiPackageMetadata(normalized);
|
||||
|
||||
if (!metadata || !metadata.versions[version]) {
|
||||
return this.errorResponse(404, 'Version not found');
|
||||
}
|
||||
|
||||
const verMeta = metadata.versions[version];
|
||||
|
||||
// Build URLs array from version files
|
||||
const urls = verMeta.files?.map((file: any) => ({
|
||||
filename: file.filename,
|
||||
url: `${this.registryUrl}/pypi/packages/${normalized}/${file.filename}`,
|
||||
digests: file.hashes,
|
||||
requires_python: file['requires-python'],
|
||||
size: file.size,
|
||||
upload_time: file['upload-time'],
|
||||
packagetype: file.filetype,
|
||||
python_version: file.python_version,
|
||||
})) || [];
|
||||
|
||||
const response = {
|
||||
info: {
|
||||
name: normalized,
|
||||
version,
|
||||
summary: verMeta.metadata?.summary,
|
||||
description: verMeta.metadata?.description,
|
||||
author: verMeta.metadata?.author,
|
||||
author_email: verMeta.metadata?.['author-email'],
|
||||
license: verMeta.metadata?.license,
|
||||
requires_python: verMeta.files?.[0]?.['requires-python'],
|
||||
...verMeta.metadata,
|
||||
},
|
||||
urls,
|
||||
};
|
||||
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'public, max-age=300'
|
||||
},
|
||||
body: response,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle package deletion
|
||||
*/
|
||||
private async handleDeletePackage(packageName: string, token: IAuthToken | null): Promise<IResponse> {
|
||||
if (!token) {
|
||||
return this.errorResponse(401, 'Authentication required');
|
||||
}
|
||||
|
||||
const normalized = helpers.normalizePypiPackageName(packageName);
|
||||
|
||||
if (!(await this.checkPermission(token, normalized, 'delete'))) {
|
||||
return this.errorResponse(403, 'Insufficient permissions');
|
||||
}
|
||||
|
||||
await this.storage.deletePypiPackage(normalized);
|
||||
|
||||
this.logger.log('info', `Package deleted: ${normalized}`);
|
||||
|
||||
return {
|
||||
status: 204,
|
||||
headers: {},
|
||||
body: Buffer.from(''),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle version deletion
|
||||
*/
|
||||
private async handleDeleteVersion(
|
||||
packageName: string,
|
||||
version: string,
|
||||
token: IAuthToken | null
|
||||
): Promise<IResponse> {
|
||||
if (!token) {
|
||||
return this.errorResponse(401, 'Authentication required');
|
||||
}
|
||||
|
||||
const normalized = helpers.normalizePypiPackageName(packageName);
|
||||
|
||||
if (!(await this.checkPermission(token, normalized, 'delete'))) {
|
||||
return this.errorResponse(403, 'Insufficient permissions');
|
||||
}
|
||||
|
||||
await this.storage.deletePypiPackageVersion(normalized, version);
|
||||
|
||||
this.logger.log('info', `Version deleted: ${normalized} ${version}`);
|
||||
|
||||
return {
|
||||
status: 204,
|
||||
headers: {},
|
||||
body: Buffer.from(''),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper: Create error response
|
||||
*/
|
||||
private errorResponse(status: number, message: string): IResponse {
|
||||
const error: IPypiError = { error: message, status };
|
||||
return {
|
||||
status,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: error,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,211 @@
|
||||
import * as plugins from '../plugins.js';
|
||||
import { BaseUpstream } from '../upstream/classes.baseupstream.js';
|
||||
import type {
|
||||
IProtocolUpstreamConfig,
|
||||
IUpstreamFetchContext,
|
||||
IUpstreamRegistryConfig,
|
||||
} from '../upstream/interfaces.upstream.js';
|
||||
|
||||
/**
|
||||
* PyPI-specific upstream implementation.
|
||||
*
|
||||
* Handles:
|
||||
* - Simple API (HTML) - PEP 503
|
||||
* - JSON API - PEP 691
|
||||
* - Package file downloads (wheels, sdists)
|
||||
* - Package name normalization
|
||||
*/
|
||||
export class PypiUpstream extends BaseUpstream {
|
||||
protected readonly protocolName = 'pypi';
|
||||
|
||||
/** Local registry URL for rewriting download URLs */
|
||||
private readonly localRegistryUrl: string;
|
||||
|
||||
constructor(
|
||||
config: IProtocolUpstreamConfig,
|
||||
localRegistryUrl: string,
|
||||
logger?: plugins.smartlog.Smartlog,
|
||||
) {
|
||||
super(config, logger);
|
||||
this.localRegistryUrl = localRegistryUrl;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch Simple API index (list of all packages) in HTML format.
|
||||
*/
|
||||
public async fetchSimpleIndex(): Promise<string | null> {
|
||||
const context: IUpstreamFetchContext = {
|
||||
protocol: 'pypi',
|
||||
resource: '*',
|
||||
resourceType: 'index',
|
||||
path: '/simple/',
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'accept': 'text/html',
|
||||
},
|
||||
query: {},
|
||||
};
|
||||
|
||||
const result = await this.fetch(context);
|
||||
|
||||
if (!result || !result.success) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (Buffer.isBuffer(result.body)) {
|
||||
return result.body.toString('utf8');
|
||||
}
|
||||
|
||||
return typeof result.body === 'string' ? result.body : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch Simple API package page (list of files) in HTML format.
|
||||
*/
|
||||
public async fetchSimplePackage(packageName: string): Promise<string | null> {
|
||||
const normalizedName = this.normalizePackageName(packageName);
|
||||
const path = `/simple/${normalizedName}/`;
|
||||
|
||||
const context: IUpstreamFetchContext = {
|
||||
protocol: 'pypi',
|
||||
resource: packageName,
|
||||
resourceType: 'simple',
|
||||
path,
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'accept': 'text/html',
|
||||
},
|
||||
query: {},
|
||||
};
|
||||
|
||||
const result = await this.fetch(context);
|
||||
|
||||
if (!result || !result.success) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (Buffer.isBuffer(result.body)) {
|
||||
return result.body.toString('utf8');
|
||||
}
|
||||
|
||||
return typeof result.body === 'string' ? result.body : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch package metadata using JSON API (PEP 691).
|
||||
*/
|
||||
public async fetchPackageJson(packageName: string): Promise<any | null> {
|
||||
const normalizedName = this.normalizePackageName(packageName);
|
||||
const path = `/simple/${normalizedName}/`;
|
||||
|
||||
const context: IUpstreamFetchContext = {
|
||||
protocol: 'pypi',
|
||||
resource: packageName,
|
||||
resourceType: 'metadata',
|
||||
path,
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'accept': 'application/vnd.pypi.simple.v1+json',
|
||||
},
|
||||
query: {},
|
||||
};
|
||||
|
||||
const result = await this.fetch(context);
|
||||
|
||||
if (!result || !result.success) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (Buffer.isBuffer(result.body)) {
|
||||
return JSON.parse(result.body.toString('utf8'));
|
||||
}
|
||||
|
||||
return result.body;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch full package info from PyPI JSON API (/pypi/{package}/json).
|
||||
*/
|
||||
public async fetchPypiJson(packageName: string): Promise<any | null> {
|
||||
const normalizedName = this.normalizePackageName(packageName);
|
||||
const path = `/pypi/${normalizedName}/json`;
|
||||
|
||||
const context: IUpstreamFetchContext = {
|
||||
protocol: 'pypi',
|
||||
resource: packageName,
|
||||
resourceType: 'pypi-json',
|
||||
path,
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'accept': 'application/json',
|
||||
},
|
||||
query: {},
|
||||
};
|
||||
|
||||
const result = await this.fetch(context);
|
||||
|
||||
if (!result || !result.success) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (Buffer.isBuffer(result.body)) {
|
||||
return JSON.parse(result.body.toString('utf8'));
|
||||
}
|
||||
|
||||
return result.body;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a package file (wheel or sdist) from upstream.
|
||||
*/
|
||||
public async fetchPackageFile(packageName: string, filename: string): Promise<Buffer | null> {
|
||||
const normalizedName = this.normalizePackageName(packageName);
|
||||
const path = `/packages/${normalizedName}/${filename}`;
|
||||
|
||||
const context: IUpstreamFetchContext = {
|
||||
protocol: 'pypi',
|
||||
resource: packageName,
|
||||
resourceType: 'package',
|
||||
path,
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'accept': 'application/octet-stream',
|
||||
},
|
||||
query: {},
|
||||
};
|
||||
|
||||
const result = await this.fetch(context);
|
||||
|
||||
if (!result || !result.success) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return Buffer.isBuffer(result.body) ? result.body : Buffer.from(result.body);
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a PyPI package name according to PEP 503.
|
||||
* - Lowercase all characters
|
||||
* - Replace runs of ., -, _ with single -
|
||||
*/
|
||||
private normalizePackageName(name: string): string {
|
||||
return name.toLowerCase().replace(/[-_.]+/g, '-');
|
||||
}
|
||||
|
||||
/**
|
||||
* Override URL building for PyPI-specific handling.
|
||||
*/
|
||||
protected buildUpstreamUrl(
|
||||
upstream: IUpstreamRegistryConfig,
|
||||
context: IUpstreamFetchContext,
|
||||
): string {
|
||||
let baseUrl = upstream.url;
|
||||
|
||||
// Remove trailing slash
|
||||
if (baseUrl.endsWith('/')) {
|
||||
baseUrl = baseUrl.slice(0, -1);
|
||||
}
|
||||
|
||||
return `${baseUrl}${context.path}`;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,299 @@
|
||||
/**
|
||||
* Helper functions for PyPI registry
|
||||
* Package name normalization, HTML generation, etc.
|
||||
*/
|
||||
|
||||
import type { IPypiFile, IPypiPackageMetadata } from './interfaces.pypi.js';
|
||||
|
||||
/**
|
||||
* Normalize package name according to PEP 503
|
||||
* Lowercase and replace runs of [._-] with a single dash
|
||||
* @param name - Package name
|
||||
* @returns Normalized name
|
||||
*/
|
||||
export function normalizePypiPackageName(name: string): string {
|
||||
return name
|
||||
.toLowerCase()
|
||||
.replace(/[-_.]+/g, '-');
|
||||
}
|
||||
|
||||
/**
|
||||
* Escape HTML special characters to prevent XSS
|
||||
* @param str - String to escape
|
||||
* @returns Escaped string
|
||||
*/
|
||||
export function escapeHtml(str: string): string {
|
||||
return str
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, ''');
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate PEP 503 compliant HTML for root index (all packages)
|
||||
* @param packages - List of package names
|
||||
* @returns HTML string
|
||||
*/
|
||||
export function generateSimpleRootHtml(packages: string[]): string {
|
||||
const links = packages
|
||||
.map(pkg => {
|
||||
const normalized = normalizePypiPackageName(pkg);
|
||||
return ` <a href="${escapeHtml(normalized)}/">${escapeHtml(pkg)}</a>`;
|
||||
})
|
||||
.join('\n');
|
||||
|
||||
return `<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<meta name="pypi:repository-version" content="1.0">
|
||||
<title>Simple Index</title>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Simple Index</h1>
|
||||
${links}
|
||||
</body>
|
||||
</html>`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate PEP 503 compliant HTML for package index (file list)
|
||||
* @param packageName - Package name (normalized)
|
||||
* @param files - List of files
|
||||
* @param baseUrl - Base URL for downloads
|
||||
* @returns HTML string
|
||||
*/
|
||||
export function generateSimplePackageHtml(
|
||||
packageName: string,
|
||||
files: IPypiFile[],
|
||||
baseUrl: string
|
||||
): string {
|
||||
const links = files
|
||||
.map(file => {
|
||||
// Build URL
|
||||
let url = file.url;
|
||||
if (!url.startsWith('http://') && !url.startsWith('https://')) {
|
||||
// Relative URL - make it absolute
|
||||
url = `${baseUrl}/packages/${packageName}/${file.filename}`;
|
||||
}
|
||||
|
||||
// Add hash fragment
|
||||
const hashName = Object.keys(file.hashes)[0];
|
||||
const hashValue = file.hashes[hashName];
|
||||
const fragment = hashName && hashValue ? `#${hashName}=${hashValue}` : '';
|
||||
|
||||
// Build data attributes
|
||||
const dataAttrs: string[] = [];
|
||||
|
||||
if (file['requires-python']) {
|
||||
const escaped = escapeHtml(file['requires-python']);
|
||||
dataAttrs.push(`data-requires-python="${escaped}"`);
|
||||
}
|
||||
|
||||
if (file['gpg-sig'] !== undefined) {
|
||||
dataAttrs.push(`data-gpg-sig="${file['gpg-sig'] ? 'true' : 'false'}"`);
|
||||
}
|
||||
|
||||
if (file.yanked) {
|
||||
const reason = typeof file.yanked === 'string' ? file.yanked : '';
|
||||
if (reason) {
|
||||
dataAttrs.push(`data-yanked="${escapeHtml(reason)}"`);
|
||||
} else {
|
||||
dataAttrs.push(`data-yanked=""`);
|
||||
}
|
||||
}
|
||||
|
||||
const dataAttrStr = dataAttrs.length > 0 ? ' ' + dataAttrs.join(' ') : '';
|
||||
|
||||
return ` <a href="${escapeHtml(url)}${fragment}"${dataAttrStr}>${escapeHtml(file.filename)}</a>`;
|
||||
})
|
||||
.join('\n');
|
||||
|
||||
return `<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<meta name="pypi:repository-version" content="1.0">
|
||||
<title>Links for ${escapeHtml(packageName)}</title>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Links for ${escapeHtml(packageName)}</h1>
|
||||
${links}
|
||||
</body>
|
||||
</html>`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse filename to extract package info
|
||||
* Supports wheel and sdist formats
|
||||
* @param filename - Package filename
|
||||
* @returns Parsed info or null
|
||||
*/
|
||||
export function parsePackageFilename(filename: string): {
|
||||
name: string;
|
||||
version: string;
|
||||
filetype: 'bdist_wheel' | 'sdist';
|
||||
pythonVersion?: string;
|
||||
} | null {
|
||||
// Wheel format: {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl
|
||||
const wheelMatch = filename.match(/^([a-zA-Z0-9_.-]+?)-([a-zA-Z0-9_.]+?)(?:-(\d+))?-([^-]+)-([^-]+)-([^-]+)\.whl$/);
|
||||
if (wheelMatch) {
|
||||
return {
|
||||
name: wheelMatch[1],
|
||||
version: wheelMatch[2],
|
||||
filetype: 'bdist_wheel',
|
||||
pythonVersion: wheelMatch[4],
|
||||
};
|
||||
}
|
||||
|
||||
// Sdist tar.gz format: {name}-{version}.tar.gz
|
||||
const sdistTarMatch = filename.match(/^([a-zA-Z0-9_.-]+?)-([a-zA-Z0-9_.]+)\.tar\.gz$/);
|
||||
if (sdistTarMatch) {
|
||||
return {
|
||||
name: sdistTarMatch[1],
|
||||
version: sdistTarMatch[2],
|
||||
filetype: 'sdist',
|
||||
pythonVersion: 'source',
|
||||
};
|
||||
}
|
||||
|
||||
// Sdist zip format: {name}-{version}.zip
|
||||
const sdistZipMatch = filename.match(/^([a-zA-Z0-9_.-]+?)-([a-zA-Z0-9_.]+)\.zip$/);
|
||||
if (sdistZipMatch) {
|
||||
return {
|
||||
name: sdistZipMatch[1],
|
||||
version: sdistZipMatch[2],
|
||||
filetype: 'sdist',
|
||||
pythonVersion: 'source',
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate hash digest for a buffer
|
||||
* @param data - Data to hash
|
||||
* @param algorithm - Hash algorithm (sha256, md5, blake2b)
|
||||
* @returns Hex-encoded hash
|
||||
*/
|
||||
export async function calculateHash(data: Buffer, algorithm: 'sha256' | 'md5' | 'blake2b'): Promise<string> {
|
||||
const crypto = await import('crypto');
|
||||
|
||||
let hash: any;
|
||||
if (algorithm === 'blake2b') {
|
||||
// Node.js uses 'blake2b512' for blake2b
|
||||
hash = crypto.createHash('blake2b512');
|
||||
} else {
|
||||
hash = crypto.createHash(algorithm);
|
||||
}
|
||||
|
||||
hash.update(data);
|
||||
return hash.digest('hex');
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate package name
|
||||
* Must contain only ASCII letters, numbers, ., -, and _
|
||||
* @param name - Package name
|
||||
* @returns true if valid
|
||||
*/
|
||||
export function isValidPackageName(name: string): boolean {
|
||||
return /^[a-zA-Z0-9._-]+$/.test(name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate version string (basic check)
|
||||
* @param version - Version string
|
||||
* @returns true if valid
|
||||
*/
|
||||
export function isValidVersion(version: string): boolean {
|
||||
// Basic check - allows numbers, letters, dots, hyphens, underscores
|
||||
// More strict validation would follow PEP 440
|
||||
return /^[a-zA-Z0-9._-]+$/.test(version);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract metadata from package metadata
|
||||
* Filters and normalizes metadata fields
|
||||
* @param metadata - Raw metadata object
|
||||
* @returns Filtered metadata
|
||||
*/
|
||||
export function extractCoreMetadata(metadata: Record<string, any>): Record<string, any> {
|
||||
const coreFields = [
|
||||
'metadata-version',
|
||||
'name',
|
||||
'version',
|
||||
'platform',
|
||||
'supported-platform',
|
||||
'summary',
|
||||
'description',
|
||||
'description-content-type',
|
||||
'keywords',
|
||||
'home-page',
|
||||
'download-url',
|
||||
'author',
|
||||
'author-email',
|
||||
'maintainer',
|
||||
'maintainer-email',
|
||||
'license',
|
||||
'classifier',
|
||||
'requires-python',
|
||||
'requires-dist',
|
||||
'requires-external',
|
||||
'provides-dist',
|
||||
'project-url',
|
||||
'provides-extra',
|
||||
];
|
||||
|
||||
const result: Record<string, any> = {};
|
||||
|
||||
for (const [key, value] of Object.entries(metadata)) {
|
||||
const normalizedKey = key.toLowerCase().replace(/_/g, '-');
|
||||
if (coreFields.includes(normalizedKey)) {
|
||||
result[normalizedKey] = value;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate JSON API response for package list (PEP 691)
|
||||
* @param packages - List of package names
|
||||
* @returns JSON object
|
||||
*/
|
||||
export function generateJsonRootResponse(packages: string[]): any {
|
||||
return {
|
||||
meta: {
|
||||
'api-version': '1.0',
|
||||
},
|
||||
projects: packages.map(name => ({ name })),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate JSON API response for package files (PEP 691)
|
||||
* @param packageName - Package name (normalized)
|
||||
* @param files - List of files
|
||||
* @returns JSON object
|
||||
*/
|
||||
export function generateJsonPackageResponse(packageName: string, files: IPypiFile[]): any {
|
||||
return {
|
||||
meta: {
|
||||
'api-version': '1.0',
|
||||
},
|
||||
name: packageName,
|
||||
files: files.map(file => ({
|
||||
filename: file.filename,
|
||||
url: file.url,
|
||||
hashes: file.hashes,
|
||||
'requires-python': file['requires-python'],
|
||||
'dist-info-metadata': file['dist-info-metadata'],
|
||||
'gpg-sig': file['gpg-sig'],
|
||||
yanked: file.yanked,
|
||||
size: file.size,
|
||||
'upload-time': file['upload-time'],
|
||||
})),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
/**
|
||||
* PyPI Registry Module
|
||||
* Python Package Index implementation
|
||||
*/
|
||||
|
||||
export * from './interfaces.pypi.js';
|
||||
export * from './classes.pypiregistry.js';
|
||||
export { PypiUpstream } from './classes.pypiupstream.js';
|
||||
export * as pypiHelpers from './helpers.pypi.js';
|
||||
@@ -0,0 +1,316 @@
|
||||
/**
|
||||
* PyPI Registry Type Definitions
|
||||
* Compliant with PEP 503 (Simple API), PEP 691 (JSON API), and PyPI upload API
|
||||
*/
|
||||
|
||||
/**
|
||||
* File information for a package distribution
|
||||
* Used in both PEP 503 HTML and PEP 691 JSON responses
|
||||
*/
|
||||
export interface IPypiFile {
|
||||
/** Filename (e.g., "package-1.0.0-py3-none-any.whl") */
|
||||
filename: string;
|
||||
/** Download URL (absolute or relative) */
|
||||
url: string;
|
||||
/** Hash digests (multiple algorithms supported in JSON) */
|
||||
hashes: Record<string, string>;
|
||||
/** Python version requirement (PEP 345 format) */
|
||||
'requires-python'?: string;
|
||||
/** Whether distribution info metadata is available (PEP 658) */
|
||||
'dist-info-metadata'?: boolean | { sha256: string };
|
||||
/** Whether GPG signature is available */
|
||||
'gpg-sig'?: boolean;
|
||||
/** Yank status: false or reason string */
|
||||
yanked?: boolean | string;
|
||||
/** File size in bytes */
|
||||
size?: number;
|
||||
/** Upload timestamp */
|
||||
'upload-time'?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Package metadata stored internally
|
||||
* Consolidated from multiple file uploads
|
||||
*/
|
||||
export interface IPypiPackageMetadata {
|
||||
/** Normalized package name */
|
||||
name: string;
|
||||
/** Map of version to file list */
|
||||
versions: Record<string, IPypiVersionMetadata>;
|
||||
/** Timestamp of last update */
|
||||
'last-modified'?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Metadata for a specific version
|
||||
*/
|
||||
export interface IPypiVersionMetadata {
|
||||
/** Version string */
|
||||
version: string;
|
||||
/** Files for this version (wheels, sdists) */
|
||||
files: IPypiFileMetadata[];
|
||||
/** Core metadata fields */
|
||||
metadata?: IPypiCoreMetadata;
|
||||
/** Whether entire version is yanked */
|
||||
yanked?: boolean | string;
|
||||
/** Upload timestamp */
|
||||
'upload-time'?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Internal file metadata
|
||||
*/
|
||||
export interface IPypiFileMetadata {
|
||||
filename: string;
|
||||
/** Storage key/path */
|
||||
path: string;
|
||||
/** File type: bdist_wheel or sdist */
|
||||
filetype: 'bdist_wheel' | 'sdist';
|
||||
/** Python version tag */
|
||||
python_version: string;
|
||||
/** Hash digests */
|
||||
hashes: Record<string, string>;
|
||||
/** File size in bytes */
|
||||
size: number;
|
||||
/** Python version requirement */
|
||||
'requires-python'?: string;
|
||||
/** Whether this file is yanked */
|
||||
yanked?: boolean | string;
|
||||
/** Upload timestamp */
|
||||
'upload-time': string;
|
||||
/** Uploader user ID */
|
||||
'uploaded-by': string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Core metadata fields (subset of PEP 566)
|
||||
* These are extracted from package uploads
|
||||
*/
|
||||
export interface IPypiCoreMetadata {
|
||||
/** Metadata version */
|
||||
'metadata-version': string;
|
||||
/** Package name */
|
||||
name: string;
|
||||
/** Version string */
|
||||
version: string;
|
||||
/** Platform compatibility */
|
||||
platform?: string;
|
||||
/** Supported platforms */
|
||||
'supported-platform'?: string;
|
||||
/** Summary/description */
|
||||
summary?: string;
|
||||
/** Long description */
|
||||
description?: string;
|
||||
/** Description content type (text/plain, text/markdown, text/x-rst) */
|
||||
'description-content-type'?: string;
|
||||
/** Keywords */
|
||||
keywords?: string;
|
||||
/** Homepage URL */
|
||||
'home-page'?: string;
|
||||
/** Download URL */
|
||||
'download-url'?: string;
|
||||
/** Author name */
|
||||
author?: string;
|
||||
/** Author email */
|
||||
'author-email'?: string;
|
||||
/** Maintainer name */
|
||||
maintainer?: string;
|
||||
/** Maintainer email */
|
||||
'maintainer-email'?: string;
|
||||
/** License */
|
||||
license?: string;
|
||||
/** Classifiers (Trove classifiers) */
|
||||
classifier?: string[];
|
||||
/** Python version requirement */
|
||||
'requires-python'?: string;
|
||||
/** Dist name requirement */
|
||||
'requires-dist'?: string[];
|
||||
/** External requirement */
|
||||
'requires-external'?: string[];
|
||||
/** Provides dist */
|
||||
'provides-dist'?: string[];
|
||||
/** Project URLs */
|
||||
'project-url'?: string[];
|
||||
/** Provides extra */
|
||||
'provides-extra'?: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* PEP 503: Simple API root response (project list)
|
||||
*/
|
||||
export interface IPypiSimpleRootHtml {
|
||||
/** List of project names */
|
||||
projects: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* PEP 503: Simple API project response (file list)
|
||||
*/
|
||||
export interface IPypiSimpleProjectHtml {
|
||||
/** Normalized project name */
|
||||
name: string;
|
||||
/** List of files */
|
||||
files: IPypiFile[];
|
||||
}
|
||||
|
||||
/**
|
||||
* PEP 691: JSON API root response
|
||||
*/
|
||||
export interface IPypiJsonRoot {
|
||||
/** API metadata */
|
||||
meta: {
|
||||
/** API version (e.g., "1.0") */
|
||||
'api-version': string;
|
||||
};
|
||||
/** List of projects */
|
||||
projects: Array<{
|
||||
/** Project name */
|
||||
name: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
/**
|
||||
* PEP 691: JSON API project response
|
||||
*/
|
||||
export interface IPypiJsonProject {
|
||||
/** Normalized project name */
|
||||
name: string;
|
||||
/** API metadata */
|
||||
meta: {
|
||||
/** API version (e.g., "1.0") */
|
||||
'api-version': string;
|
||||
};
|
||||
/** List of files */
|
||||
files: IPypiFile[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload form data (multipart/form-data fields)
|
||||
* Based on PyPI legacy upload API
|
||||
*/
|
||||
export interface IPypiUploadForm {
|
||||
/** Action type (always "file_upload") */
|
||||
':action': 'file_upload';
|
||||
/** Protocol version (always "1") */
|
||||
protocol_version: '1';
|
||||
/** File content (binary) */
|
||||
content: Buffer;
|
||||
/** File type */
|
||||
filetype: 'bdist_wheel' | 'sdist';
|
||||
/** Python version tag */
|
||||
pyversion: string;
|
||||
/** Package name */
|
||||
name: string;
|
||||
/** Version string */
|
||||
version: string;
|
||||
/** Metadata version */
|
||||
metadata_version: string;
|
||||
/** Hash digests (at least one required) */
|
||||
md5_digest?: string;
|
||||
sha256_digest?: string;
|
||||
blake2_256_digest?: string;
|
||||
/** Optional attestations */
|
||||
attestations?: string; // JSON array
|
||||
/** Optional core metadata fields */
|
||||
summary?: string;
|
||||
description?: string;
|
||||
description_content_type?: string;
|
||||
author?: string;
|
||||
author_email?: string;
|
||||
maintainer?: string;
|
||||
maintainer_email?: string;
|
||||
license?: string;
|
||||
keywords?: string;
|
||||
home_page?: string;
|
||||
download_url?: string;
|
||||
requires_python?: string;
|
||||
classifiers?: string[];
|
||||
platform?: string;
|
||||
[key: string]: any; // Allow additional metadata fields
|
||||
}
|
||||
|
||||
/**
|
||||
* JSON API upload response
|
||||
*/
|
||||
export interface IPypiUploadResponse {
|
||||
/** Success message */
|
||||
message?: string;
|
||||
/** URL of uploaded file */
|
||||
url?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Error response structure
|
||||
*/
|
||||
export interface IPypiError {
|
||||
/** Error message */
|
||||
error: string;
|
||||
/** HTTP status code */
|
||||
status?: number;
|
||||
/** Additional error details */
|
||||
details?: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Search query parameters
|
||||
*/
|
||||
export interface IPypiSearchQuery {
|
||||
/** Search term */
|
||||
q?: string;
|
||||
/** Page number */
|
||||
page?: number;
|
||||
/** Results per page */
|
||||
per_page?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Search result for a single package
|
||||
*/
|
||||
export interface IPypiSearchResult {
|
||||
/** Package name */
|
||||
name: string;
|
||||
/** Latest version */
|
||||
version: string;
|
||||
/** Summary */
|
||||
summary: string;
|
||||
/** Description */
|
||||
description?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Search response structure
|
||||
*/
|
||||
export interface IPypiSearchResponse {
|
||||
/** Search results */
|
||||
results: IPypiSearchResult[];
|
||||
/** Result count */
|
||||
count: number;
|
||||
/** Current page */
|
||||
page: number;
|
||||
/** Total pages */
|
||||
pages: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Yank request
|
||||
*/
|
||||
export interface IPypiYankRequest {
|
||||
/** Package name */
|
||||
name: string;
|
||||
/** Version to yank */
|
||||
version: string;
|
||||
/** Optional filename (specific file) */
|
||||
filename?: string;
|
||||
/** Reason for yanking */
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Yank response
|
||||
*/
|
||||
export interface IPypiYankResponse {
|
||||
/** Success indicator */
|
||||
success: boolean;
|
||||
/** Message */
|
||||
message?: string;
|
||||
}
|
||||
@@ -0,0 +1,799 @@
|
||||
import { Smartlog } from '@push.rocks/smartlog';
|
||||
import { BaseRegistry } from '../core/classes.baseregistry.js';
|
||||
import { RegistryStorage } from '../core/classes.registrystorage.js';
|
||||
import { AuthManager } from '../core/classes.authmanager.js';
|
||||
import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from '../core/interfaces.core.js';
|
||||
import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
|
||||
import type {
|
||||
IRubyGemsMetadata,
|
||||
IRubyGemsVersionMetadata,
|
||||
IRubyGemsUploadResponse,
|
||||
IRubyGemsYankResponse,
|
||||
IRubyGemsError,
|
||||
ICompactIndexInfoEntry,
|
||||
} from './interfaces.rubygems.js';
|
||||
import * as helpers from './helpers.rubygems.js';
|
||||
import { RubygemsUpstream } from './classes.rubygemsupstream.js';
|
||||
|
||||
/**
|
||||
* RubyGems registry implementation
|
||||
* Implements Compact Index API and RubyGems protocol
|
||||
*/
|
||||
export class RubyGemsRegistry extends BaseRegistry {
|
||||
private storage: RegistryStorage;
|
||||
private authManager: AuthManager;
|
||||
private basePath: string = '/rubygems';
|
||||
private registryUrl: string;
|
||||
private logger: Smartlog;
|
||||
private upstreamProvider: IUpstreamProvider | null = null;
|
||||
|
||||
constructor(
|
||||
storage: RegistryStorage,
|
||||
authManager: AuthManager,
|
||||
basePath: string = '/rubygems',
|
||||
registryUrl: string = 'http://localhost:5000/rubygems',
|
||||
upstreamProvider?: IUpstreamProvider
|
||||
) {
|
||||
super();
|
||||
this.storage = storage;
|
||||
this.authManager = authManager;
|
||||
this.basePath = basePath;
|
||||
this.registryUrl = registryUrl;
|
||||
this.upstreamProvider = upstreamProvider || null;
|
||||
|
||||
this.logger = this.createProtocolLogger('rubygems-registry', 'rubygems');
|
||||
}
|
||||
|
||||
/**
|
||||
* Get upstream for a specific request.
|
||||
* Calls the provider to resolve upstream config dynamically.
|
||||
*/
|
||||
private async getUpstreamForRequest(
|
||||
resource: string,
|
||||
resourceType: string,
|
||||
method: string,
|
||||
actor?: IRequestActor
|
||||
): Promise<RubygemsUpstream | null> {
|
||||
if (!this.upstreamProvider) return null;
|
||||
|
||||
const config = await this.upstreamProvider.resolveUpstreamConfig({
|
||||
protocol: 'rubygems',
|
||||
resource,
|
||||
scope: resource, // gem name is the scope
|
||||
actor,
|
||||
method,
|
||||
resourceType,
|
||||
});
|
||||
|
||||
if (!config?.enabled) return null;
|
||||
return new RubygemsUpstream(config, this.logger);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up resources (timers, connections, etc.)
|
||||
*/
|
||||
public destroy(): void {
|
||||
// No persistent upstream to clean up with dynamic provider
|
||||
}
|
||||
|
||||
public async init(): Promise<void> {
|
||||
// Initialize Compact Index files if not exist
|
||||
const existingVersions = await this.storage.getRubyGemsVersions();
|
||||
if (!existingVersions) {
|
||||
const versions = helpers.generateCompactIndexVersions([]);
|
||||
await this.storage.putRubyGemsVersions(versions);
|
||||
this.logger.log('info', 'Initialized RubyGems Compact Index');
|
||||
}
|
||||
|
||||
const existingNames = await this.storage.getRubyGemsNames();
|
||||
if (!existingNames) {
|
||||
const names = helpers.generateNamesFile([]);
|
||||
await this.storage.putRubyGemsNames(names);
|
||||
this.logger.log('info', 'Initialized RubyGems names file');
|
||||
}
|
||||
}
|
||||
|
||||
public getBasePath(): string {
|
||||
return this.basePath;
|
||||
}
|
||||
|
||||
public async handleRequest(context: IRequestContext): Promise<IResponse> {
|
||||
let path = context.path.replace(this.basePath, '');
|
||||
|
||||
// Extract token (Authorization header)
|
||||
const token = await this.extractToken(context);
|
||||
|
||||
const actor: IRequestActor = this.buildRequestActor(context, token);
|
||||
|
||||
this.logger.log('debug', `handleRequest: ${context.method} ${path}`, {
|
||||
method: context.method,
|
||||
path,
|
||||
hasAuth: !!token
|
||||
});
|
||||
|
||||
return this.storage.withContext({ protocol: 'rubygems', actor }, async () => {
|
||||
// Compact Index endpoints
|
||||
if (path === '/versions' && context.method === 'GET') {
|
||||
return this.handleVersionsFile(context);
|
||||
}
|
||||
|
||||
if (path === '/names' && context.method === 'GET') {
|
||||
return this.handleNamesFile();
|
||||
}
|
||||
|
||||
// Info file: GET /info/{gem}
|
||||
const infoMatch = path.match(/^\/info\/([^\/]+)$/);
|
||||
if (infoMatch && context.method === 'GET') {
|
||||
return this.handleInfoFile(infoMatch[1], actor);
|
||||
}
|
||||
|
||||
// Gem download: GET /gems/{gem}-{version}[-{platform}].gem
|
||||
const downloadMatch = path.match(/^\/gems\/(.+\.gem)$/);
|
||||
if (downloadMatch && context.method === 'GET') {
|
||||
return this.handleDownload(downloadMatch[1], actor);
|
||||
}
|
||||
|
||||
// Legacy specs endpoints (Marshal format)
|
||||
if (path === '/specs.4.8.gz' && context.method === 'GET') {
|
||||
return this.handleSpecs(false);
|
||||
}
|
||||
|
||||
if (path === '/latest_specs.4.8.gz' && context.method === 'GET') {
|
||||
return this.handleSpecs(true);
|
||||
}
|
||||
|
||||
// Quick gemspec endpoint: GET /quick/Marshal.4.8/{gem}-{version}.gemspec.rz
|
||||
const quickMatch = path.match(/^\/quick\/Marshal\.4\.8\/(.+)\.gemspec\.rz$/);
|
||||
if (quickMatch && context.method === 'GET') {
|
||||
return this.handleQuickGemspec(quickMatch[1]);
|
||||
}
|
||||
|
||||
// API v1 endpoints
|
||||
if (path.startsWith('/api/v1/')) {
|
||||
return this.handleApiRequest(path.substring(7), context, token);
|
||||
}
|
||||
|
||||
return {
|
||||
status: 404,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: { error: 'Not Found' },
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if token has permission for resource
|
||||
*/
|
||||
protected async checkPermission(
|
||||
token: IAuthToken | null,
|
||||
resource: string,
|
||||
action: string
|
||||
): Promise<boolean> {
|
||||
if (!token) return false;
|
||||
return this.authManager.authorize(token, `rubygems:gem:${resource}`, action);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract authentication token from request
|
||||
*/
|
||||
private async extractToken(context: IRequestContext): Promise<IAuthToken | null> {
|
||||
const authHeader = this.getAuthorizationHeader(context);
|
||||
if (!authHeader) return null;
|
||||
|
||||
// RubyGems typically uses plain API key in Authorization header
|
||||
return this.authManager.validateToken(authHeader, 'rubygems');
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle /versions endpoint (Compact Index)
|
||||
* Supports conditional GET with If-None-Match header
|
||||
*/
|
||||
private async handleVersionsFile(context: IRequestContext): Promise<IResponse> {
|
||||
const content = await this.storage.getRubyGemsVersions();
|
||||
|
||||
if (!content) {
|
||||
return this.errorResponse(500, 'Versions file not initialized');
|
||||
}
|
||||
|
||||
const etag = `"${await helpers.calculateMD5(content)}"`;
|
||||
|
||||
// Handle conditional GET with If-None-Match
|
||||
const ifNoneMatch = context.headers['if-none-match'] || context.headers['If-None-Match'];
|
||||
if (ifNoneMatch && ifNoneMatch === etag) {
|
||||
return {
|
||||
status: 304,
|
||||
headers: {
|
||||
'ETag': etag,
|
||||
'Cache-Control': 'public, max-age=60',
|
||||
},
|
||||
body: null,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'text/plain; charset=utf-8',
|
||||
'Cache-Control': 'public, max-age=60',
|
||||
'ETag': etag
|
||||
},
|
||||
body: Buffer.from(content),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle /names endpoint (Compact Index)
|
||||
*/
|
||||
private async handleNamesFile(): Promise<IResponse> {
|
||||
const content = await this.storage.getRubyGemsNames();
|
||||
|
||||
if (!content) {
|
||||
return this.errorResponse(500, 'Names file not initialized');
|
||||
}
|
||||
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'text/plain; charset=utf-8',
|
||||
'Cache-Control': 'public, max-age=300'
|
||||
},
|
||||
body: Buffer.from(content),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle /info/{gem} endpoint (Compact Index)
|
||||
*/
|
||||
private async handleInfoFile(gemName: string, actor?: IRequestActor): Promise<IResponse> {
|
||||
let content = await this.storage.getRubyGemsInfo(gemName);
|
||||
|
||||
// Try upstream if not found locally
|
||||
if (!content) {
|
||||
const upstream = await this.getUpstreamForRequest(gemName, 'info', 'GET', actor);
|
||||
if (upstream) {
|
||||
const upstreamInfo = await upstream.fetchInfo(gemName);
|
||||
if (upstreamInfo) {
|
||||
// Cache locally
|
||||
await this.storage.putRubyGemsInfo(gemName, upstreamInfo);
|
||||
content = upstreamInfo;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!content) {
|
||||
return {
|
||||
status: 404,
|
||||
headers: { 'Content-Type': 'text/plain' },
|
||||
body: Buffer.from('Not Found'),
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'text/plain; charset=utf-8',
|
||||
'Cache-Control': 'public, max-age=300',
|
||||
'ETag': `"${await helpers.calculateMD5(content)}"`
|
||||
},
|
||||
body: Buffer.from(content),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle gem file download
|
||||
*/
|
||||
private async handleDownload(filename: string, actor?: IRequestActor): Promise<IResponse> {
|
||||
const parsed = helpers.parseGemFilename(filename);
|
||||
if (!parsed) {
|
||||
return this.errorResponse(400, 'Invalid gem filename');
|
||||
}
|
||||
|
||||
// Try streaming from local storage first
|
||||
const streamResult = await this.storage.getRubyGemsGemStream(
|
||||
parsed.name,
|
||||
parsed.version,
|
||||
parsed.platform
|
||||
);
|
||||
|
||||
if (streamResult) {
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/octet-stream',
|
||||
'Content-Disposition': `attachment; filename="${filename}"`,
|
||||
'Content-Length': streamResult.size.toString()
|
||||
},
|
||||
body: streamResult.stream,
|
||||
};
|
||||
}
|
||||
|
||||
// Try upstream if not found locally
|
||||
let gemData: Buffer | null = null;
|
||||
const upstream = await this.getUpstreamForRequest(parsed.name, 'gem', 'GET', actor);
|
||||
if (upstream) {
|
||||
gemData = await upstream.fetchGem(parsed.name, parsed.version);
|
||||
if (gemData) {
|
||||
// Cache locally
|
||||
await this.storage.putRubyGemsGem(parsed.name, parsed.version, gemData, parsed.platform);
|
||||
}
|
||||
}
|
||||
|
||||
if (!gemData) {
|
||||
return this.errorResponse(404, 'Gem not found');
|
||||
}
|
||||
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/octet-stream',
|
||||
'Content-Disposition': `attachment; filename="${filename}"`,
|
||||
'Content-Length': gemData.length.toString()
|
||||
},
|
||||
body: gemData,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle API v1 requests
|
||||
*/
|
||||
private async handleApiRequest(
|
||||
path: string,
|
||||
context: IRequestContext,
|
||||
token: IAuthToken | null
|
||||
): Promise<IResponse> {
|
||||
// Upload gem: POST /gems
|
||||
if (path === '/gems' && context.method === 'POST') {
|
||||
return this.handleUpload(context, token);
|
||||
}
|
||||
|
||||
// Yank gem: DELETE /gems/yank
|
||||
if (path === '/gems/yank' && context.method === 'DELETE') {
|
||||
return this.handleYank(context, token);
|
||||
}
|
||||
|
||||
// Unyank gem: PUT /gems/unyank
|
||||
if (path === '/gems/unyank' && context.method === 'PUT') {
|
||||
return this.handleUnyank(context, token);
|
||||
}
|
||||
|
||||
// Version list: GET /versions/{gem}.json
|
||||
const versionsMatch = path.match(/^\/versions\/([^\/]+)\.json$/);
|
||||
if (versionsMatch && context.method === 'GET') {
|
||||
return this.handleVersionsJson(versionsMatch[1]);
|
||||
}
|
||||
|
||||
// Dependencies: GET /dependencies?gems={list}
|
||||
if (path.startsWith('/dependencies') && context.method === 'GET') {
|
||||
const gemsParam = context.query?.gems || '';
|
||||
return this.handleDependencies(gemsParam);
|
||||
}
|
||||
|
||||
return this.errorResponse(404, 'API endpoint not found');
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle gem upload
|
||||
* POST /api/v1/gems
|
||||
*/
|
||||
private async handleUpload(context: IRequestContext, token: IAuthToken | null): Promise<IResponse> {
|
||||
if (!token) {
|
||||
return this.errorResponse(401, 'Authentication required');
|
||||
}
|
||||
|
||||
try {
|
||||
// Extract gem data from request body
|
||||
const gemData = context.body as Buffer;
|
||||
if (!gemData || gemData.length === 0) {
|
||||
return this.errorResponse(400, 'No gem file provided');
|
||||
}
|
||||
|
||||
// Try to get metadata from query params or headers first
|
||||
let gemName = context.query?.name || context.headers['x-gem-name'] as string | undefined;
|
||||
let version = context.query?.version || context.headers['x-gem-version'] as string | undefined;
|
||||
let platform = context.query?.platform || context.headers['x-gem-platform'] as string | undefined;
|
||||
|
||||
// If not provided, try to extract from gem binary
|
||||
if (!gemName || !version || !platform) {
|
||||
const extracted = await helpers.extractGemMetadata(gemData);
|
||||
if (extracted) {
|
||||
gemName = gemName || extracted.name;
|
||||
version = version || extracted.version;
|
||||
platform = platform || extracted.platform;
|
||||
}
|
||||
}
|
||||
|
||||
if (!gemName || !version) {
|
||||
return this.errorResponse(400, 'Gem name and version required (provide in query, headers, or valid gem format)');
|
||||
}
|
||||
|
||||
// Validate gem name
|
||||
if (!helpers.isValidGemName(gemName)) {
|
||||
return this.errorResponse(400, 'Invalid gem name');
|
||||
}
|
||||
|
||||
// Check permission
|
||||
if (!(await this.checkPermission(token, gemName, 'write'))) {
|
||||
return this.errorResponse(403, 'Insufficient permissions');
|
||||
}
|
||||
|
||||
// Calculate checksum
|
||||
const checksum = await helpers.calculateSHA256(gemData);
|
||||
|
||||
// Store gem file
|
||||
await this.storage.putRubyGemsGem(gemName, version, gemData, platform);
|
||||
|
||||
// Update metadata
|
||||
let metadata: IRubyGemsMetadata = await this.storage.getRubyGemsMetadata(gemName) || {
|
||||
name: gemName,
|
||||
versions: {},
|
||||
};
|
||||
|
||||
const versionKey = platform ? `${version}-${platform}` : version;
|
||||
metadata.versions[versionKey] = {
|
||||
version,
|
||||
platform,
|
||||
checksum,
|
||||
size: gemData.length,
|
||||
'upload-time': new Date().toISOString(),
|
||||
'uploaded-by': token.userId,
|
||||
dependencies: [], // Would extract from gem spec
|
||||
requirements: [],
|
||||
};
|
||||
|
||||
metadata['last-modified'] = new Date().toISOString();
|
||||
await this.storage.putRubyGemsMetadata(gemName, metadata);
|
||||
|
||||
// Update Compact Index info file
|
||||
await this.updateCompactIndexForGem(gemName, metadata);
|
||||
|
||||
// Update versions file
|
||||
await this.updateVersionsFile(gemName, version, platform || 'ruby', false);
|
||||
|
||||
// Update names file
|
||||
await this.updateNamesFile(gemName);
|
||||
|
||||
this.logger.log('info', `Gem uploaded: ${gemName} ${version}`, {
|
||||
platform,
|
||||
size: gemData.length
|
||||
});
|
||||
|
||||
return {
|
||||
status: 201,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: {
|
||||
message: 'Gem uploaded successfully',
|
||||
name: gemName,
|
||||
version,
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
this.logger.log('error', 'Upload failed', { error: (error as Error).message });
|
||||
return this.errorResponse(500, 'Upload failed: ' + (error as Error).message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle gem yanking
|
||||
* DELETE /api/v1/gems/yank
|
||||
*/
|
||||
private async handleYank(context: IRequestContext, token: IAuthToken | null): Promise<IResponse> {
|
||||
if (!token) {
|
||||
return this.errorResponse(401, 'Authentication required');
|
||||
}
|
||||
|
||||
const gemName = context.query?.gem_name;
|
||||
const version = context.query?.version;
|
||||
const platform = context.query?.platform;
|
||||
|
||||
if (!gemName || !version) {
|
||||
return this.errorResponse(400, 'Gem name and version required');
|
||||
}
|
||||
|
||||
if (!(await this.checkPermission(token, gemName, 'yank'))) {
|
||||
return this.errorResponse(403, 'Insufficient permissions');
|
||||
}
|
||||
|
||||
// Update metadata to mark as yanked
|
||||
const metadata = await this.storage.getRubyGemsMetadata(gemName);
|
||||
if (!metadata) {
|
||||
return this.errorResponse(404, 'Gem not found');
|
||||
}
|
||||
|
||||
const versionKey = platform ? `${version}-${platform}` : version;
|
||||
if (!metadata.versions[versionKey]) {
|
||||
return this.errorResponse(404, 'Version not found');
|
||||
}
|
||||
|
||||
metadata.versions[versionKey].yanked = true;
|
||||
await this.storage.putRubyGemsMetadata(gemName, metadata);
|
||||
|
||||
// Update Compact Index
|
||||
await this.updateCompactIndexForGem(gemName, metadata);
|
||||
await this.updateVersionsFile(gemName, version, platform || 'ruby', true);
|
||||
|
||||
this.logger.log('info', `Gem yanked: ${gemName} ${version}`);
|
||||
|
||||
return {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: {
|
||||
success: true,
|
||||
message: 'Gem yanked successfully'
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle gem unyanking
|
||||
* PUT /api/v1/gems/unyank
|
||||
*/
|
||||
private async handleUnyank(context: IRequestContext, token: IAuthToken | null): Promise<IResponse> {
|
||||
if (!token) {
|
||||
return this.errorResponse(401, 'Authentication required');
|
||||
}
|
||||
|
||||
const gemName = context.query?.gem_name;
|
||||
const version = context.query?.version;
|
||||
const platform = context.query?.platform;
|
||||
|
||||
if (!gemName || !version) {
|
||||
return this.errorResponse(400, 'Gem name and version required');
|
||||
}
|
||||
|
||||
if (!(await this.checkPermission(token, gemName, 'write'))) {
|
||||
return this.errorResponse(403, 'Insufficient permissions');
|
||||
}
|
||||
|
||||
const metadata = await this.storage.getRubyGemsMetadata(gemName);
|
||||
if (!metadata) {
|
||||
return this.errorResponse(404, 'Gem not found');
|
||||
}
|
||||
|
||||
const versionKey = platform ? `${version}-${platform}` : version;
|
||||
if (!metadata.versions[versionKey]) {
|
||||
return this.errorResponse(404, 'Version not found');
|
||||
}
|
||||
|
||||
metadata.versions[versionKey].yanked = false;
|
||||
await this.storage.putRubyGemsMetadata(gemName, metadata);
|
||||
|
||||
// Update Compact Index
|
||||
await this.updateCompactIndexForGem(gemName, metadata);
|
||||
await this.updateVersionsFile(gemName, version, platform || 'ruby', false);
|
||||
|
||||
this.logger.log('info', `Gem unyanked: ${gemName} ${version}`);
|
||||
|
||||
return {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: {
|
||||
success: true,
|
||||
message: 'Gem unyanked successfully'
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle versions JSON API
|
||||
*/
|
||||
private async handleVersionsJson(gemName: string): Promise<IResponse> {
|
||||
const metadata = await this.storage.getRubyGemsMetadata(gemName);
|
||||
if (!metadata) {
|
||||
return this.errorResponse(404, 'Gem not found');
|
||||
}
|
||||
|
||||
const versions = Object.values(metadata.versions).map((v: any) => ({
|
||||
version: v.version,
|
||||
platform: v.platform,
|
||||
uploadTime: v['upload-time'],
|
||||
}));
|
||||
|
||||
const response = helpers.generateVersionsJson(gemName, versions);
|
||||
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'public, max-age=300'
|
||||
},
|
||||
body: response,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle dependencies query
|
||||
*/
|
||||
private async handleDependencies(gemsParam: string): Promise<IResponse> {
|
||||
const gemNames = gemsParam.split(',').filter(n => n.trim());
|
||||
const result = new Map();
|
||||
|
||||
for (const gemName of gemNames) {
|
||||
const metadata = await this.storage.getRubyGemsMetadata(gemName);
|
||||
if (metadata) {
|
||||
const versions = Object.values(metadata.versions).map((v: any) => ({
|
||||
version: v.version,
|
||||
platform: v.platform,
|
||||
dependencies: v.dependencies || [],
|
||||
}));
|
||||
result.set(gemName, versions);
|
||||
}
|
||||
}
|
||||
|
||||
const response = helpers.generateDependenciesJson(result);
|
||||
|
||||
return {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: response,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Update Compact Index info file for a gem
|
||||
*/
|
||||
private async updateCompactIndexForGem(
|
||||
gemName: string,
|
||||
metadata: IRubyGemsMetadata
|
||||
): Promise<void> {
|
||||
const entries: ICompactIndexInfoEntry[] = Object.values(metadata.versions)
|
||||
.filter(v => !v.yanked) // Exclude yanked from info file
|
||||
.map(v => ({
|
||||
version: v.version,
|
||||
platform: v.platform,
|
||||
dependencies: v.dependencies || [],
|
||||
requirements: v.requirements || [],
|
||||
checksum: v.checksum,
|
||||
}));
|
||||
|
||||
const content = helpers.generateCompactIndexInfo(entries);
|
||||
await this.storage.putRubyGemsInfo(gemName, content);
|
||||
}
|
||||
|
||||
/**
|
||||
* Update versions file with new/updated gem
|
||||
*/
|
||||
private async updateVersionsFile(
|
||||
gemName: string,
|
||||
version: string,
|
||||
platform: string,
|
||||
yanked: boolean
|
||||
): Promise<void> {
|
||||
const existingVersions = await this.storage.getRubyGemsVersions();
|
||||
if (!existingVersions) return;
|
||||
|
||||
// Calculate info file checksum
|
||||
const infoContent = await this.storage.getRubyGemsInfo(gemName) || '';
|
||||
const infoChecksum = await helpers.calculateMD5(infoContent);
|
||||
|
||||
const updated = helpers.updateCompactIndexVersions(
|
||||
existingVersions,
|
||||
gemName,
|
||||
{ version, platform: platform !== 'ruby' ? platform : undefined, yanked },
|
||||
infoChecksum
|
||||
);
|
||||
|
||||
await this.storage.putRubyGemsVersions(updated);
|
||||
}
|
||||
|
||||
/**
|
||||
* Update names file with new gem
|
||||
*/
|
||||
private async updateNamesFile(gemName: string): Promise<void> {
|
||||
const existingNames = await this.storage.getRubyGemsNames();
|
||||
if (!existingNames) return;
|
||||
|
||||
const lines = existingNames.split('\n').filter(l => l !== '---');
|
||||
if (!lines.includes(gemName)) {
|
||||
lines.push(gemName);
|
||||
lines.sort();
|
||||
const updated = helpers.generateNamesFile(lines);
|
||||
await this.storage.putRubyGemsNames(updated);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle /specs.4.8.gz and /latest_specs.4.8.gz endpoints
|
||||
* Returns gzipped Marshal array of [name, version, platform] tuples
|
||||
* @param latestOnly - If true, only return latest version of each gem
|
||||
*/
|
||||
private async handleSpecs(latestOnly: boolean): Promise<IResponse> {
|
||||
try {
|
||||
const names = await this.storage.getRubyGemsNames();
|
||||
if (!names) {
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/octet-stream',
|
||||
},
|
||||
body: await helpers.generateSpecsGz([]),
|
||||
};
|
||||
}
|
||||
|
||||
const gemNames = names.split('\n').filter(l => l && l !== '---');
|
||||
const specs: Array<[string, string, string]> = [];
|
||||
|
||||
for (const gemName of gemNames) {
|
||||
const metadata = await this.storage.getRubyGemsMetadata(gemName);
|
||||
if (!metadata) continue;
|
||||
|
||||
const versions = (Object.values(metadata.versions) as IRubyGemsVersionMetadata[])
|
||||
.filter(v => !v.yanked)
|
||||
.sort((a, b) => {
|
||||
// Sort by version descending
|
||||
return b.version.localeCompare(a.version, undefined, { numeric: true });
|
||||
});
|
||||
|
||||
if (latestOnly && versions.length > 0) {
|
||||
// Only include latest version
|
||||
const latest = versions[0];
|
||||
specs.push([gemName, latest.version, latest.platform || 'ruby']);
|
||||
} else {
|
||||
// Include all versions
|
||||
for (const v of versions) {
|
||||
specs.push([gemName, v.version, v.platform || 'ruby']);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const gzippedSpecs = await helpers.generateSpecsGz(specs);
|
||||
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/octet-stream',
|
||||
},
|
||||
body: gzippedSpecs,
|
||||
};
|
||||
} catch (error) {
|
||||
this.logger.log('error', 'Failed to generate specs', { error: (error as Error).message });
|
||||
return this.errorResponse(500, 'Failed to generate specs');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle /quick/Marshal.4.8/{gem}-{version}.gemspec.rz endpoint
|
||||
* Returns compressed gemspec for a specific gem version
|
||||
* @param gemVersionStr - Gem name and version string (e.g., "rails-7.0.0" or "rails-7.0.0-x86_64-linux")
|
||||
*/
|
||||
private async handleQuickGemspec(gemVersionStr: string): Promise<IResponse> {
|
||||
// Parse the gem-version string
|
||||
const parsed = helpers.parseGemFilename(gemVersionStr + '.gem');
|
||||
if (!parsed) {
|
||||
return this.errorResponse(400, 'Invalid gemspec path');
|
||||
}
|
||||
|
||||
const metadata = await this.storage.getRubyGemsMetadata(parsed.name);
|
||||
if (!metadata) {
|
||||
return this.errorResponse(404, 'Gem not found');
|
||||
}
|
||||
|
||||
const versionKey = parsed.platform ? `${parsed.version}-${parsed.platform}` : parsed.version;
|
||||
const versionMeta = metadata.versions[versionKey];
|
||||
if (!versionMeta) {
|
||||
return this.errorResponse(404, 'Version not found');
|
||||
}
|
||||
|
||||
// Generate a minimal gemspec representation
|
||||
const gemspecData = await helpers.generateGemspecRz(parsed.name, versionMeta);
|
||||
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/octet-stream',
|
||||
},
|
||||
body: gemspecData,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper: Create error response
|
||||
*/
|
||||
private errorResponse(status: number, message: string): IResponse {
|
||||
const error: IRubyGemsError = { error: message, status };
|
||||
return {
|
||||
status,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: error,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
import * as plugins from '../plugins.js';
|
||||
import { BaseUpstream } from '../upstream/classes.baseupstream.js';
|
||||
import type {
|
||||
IProtocolUpstreamConfig,
|
||||
IUpstreamFetchContext,
|
||||
IUpstreamRegistryConfig,
|
||||
} from '../upstream/interfaces.upstream.js';
|
||||
|
||||
/**
|
||||
* RubyGems-specific upstream implementation.
|
||||
*
|
||||
* Handles:
|
||||
* - Compact Index format (/versions, /info/{gem}, /names)
|
||||
* - Gem file (.gem) downloading
|
||||
* - Gem spec fetching
|
||||
* - HTTP Range requests for incremental updates
|
||||
*/
|
||||
export class RubygemsUpstream extends BaseUpstream {
|
||||
protected readonly protocolName = 'rubygems';
|
||||
|
||||
constructor(
|
||||
config: IProtocolUpstreamConfig,
|
||||
logger?: plugins.smartlog.Smartlog,
|
||||
) {
|
||||
super(config, logger);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch the /versions file (master list of all gems).
|
||||
*/
|
||||
public async fetchVersions(etag?: string): Promise<{ data: string; etag?: string } | null> {
|
||||
const headers: Record<string, string> = {
|
||||
'accept': 'text/plain',
|
||||
};
|
||||
|
||||
if (etag) {
|
||||
headers['if-none-match'] = etag;
|
||||
}
|
||||
|
||||
const context: IUpstreamFetchContext = {
|
||||
protocol: 'rubygems',
|
||||
resource: '*',
|
||||
resourceType: 'versions',
|
||||
path: '/versions',
|
||||
method: 'GET',
|
||||
headers,
|
||||
query: {},
|
||||
};
|
||||
|
||||
const result = await this.fetch(context);
|
||||
|
||||
if (!result || !result.success) {
|
||||
return null;
|
||||
}
|
||||
|
||||
let data: string;
|
||||
if (Buffer.isBuffer(result.body)) {
|
||||
data = result.body.toString('utf8');
|
||||
} else if (typeof result.body === 'string') {
|
||||
data = result.body;
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
data,
|
||||
etag: result.headers['etag'],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch gem info file (/info/{gemname}).
|
||||
*/
|
||||
public async fetchInfo(gemName: string): Promise<string | null> {
|
||||
const context: IUpstreamFetchContext = {
|
||||
protocol: 'rubygems',
|
||||
resource: gemName,
|
||||
resourceType: 'info',
|
||||
path: `/info/${gemName}`,
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'accept': 'text/plain',
|
||||
},
|
||||
query: {},
|
||||
};
|
||||
|
||||
const result = await this.fetch(context);
|
||||
|
||||
if (!result || !result.success) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (Buffer.isBuffer(result.body)) {
|
||||
return result.body.toString('utf8');
|
||||
}
|
||||
|
||||
return typeof result.body === 'string' ? result.body : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch the /names file (list of all gem names).
|
||||
*/
|
||||
public async fetchNames(): Promise<string | null> {
|
||||
const context: IUpstreamFetchContext = {
|
||||
protocol: 'rubygems',
|
||||
resource: '*',
|
||||
resourceType: 'names',
|
||||
path: '/names',
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'accept': 'text/plain',
|
||||
},
|
||||
query: {},
|
||||
};
|
||||
|
||||
const result = await this.fetch(context);
|
||||
|
||||
if (!result || !result.success) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (Buffer.isBuffer(result.body)) {
|
||||
return result.body.toString('utf8');
|
||||
}
|
||||
|
||||
return typeof result.body === 'string' ? result.body : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a gem file.
|
||||
*/
|
||||
public async fetchGem(gemName: string, version: string): Promise<Buffer | null> {
|
||||
const path = `/gems/${gemName}-${version}.gem`;
|
||||
|
||||
const context: IUpstreamFetchContext = {
|
||||
protocol: 'rubygems',
|
||||
resource: gemName,
|
||||
resourceType: 'gem',
|
||||
path,
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'accept': 'application/octet-stream',
|
||||
},
|
||||
query: {},
|
||||
};
|
||||
|
||||
const result = await this.fetch(context);
|
||||
|
||||
if (!result || !result.success) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return Buffer.isBuffer(result.body) ? result.body : Buffer.from(result.body);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch gem spec (quick spec).
|
||||
*/
|
||||
public async fetchQuickSpec(gemName: string, version: string): Promise<Buffer | null> {
|
||||
const path = `/quick/Marshal.4.8/${gemName}-${version}.gemspec.rz`;
|
||||
|
||||
const context: IUpstreamFetchContext = {
|
||||
protocol: 'rubygems',
|
||||
resource: gemName,
|
||||
resourceType: 'spec',
|
||||
path,
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'accept': 'application/octet-stream',
|
||||
},
|
||||
query: {},
|
||||
};
|
||||
|
||||
const result = await this.fetch(context);
|
||||
|
||||
if (!result || !result.success) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return Buffer.isBuffer(result.body) ? result.body : Buffer.from(result.body);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch gem versions JSON from API.
|
||||
*/
|
||||
public async fetchVersionsJson(gemName: string): Promise<any[] | null> {
|
||||
const path = `/api/v1/versions/${gemName}.json`;
|
||||
|
||||
const context: IUpstreamFetchContext = {
|
||||
protocol: 'rubygems',
|
||||
resource: gemName,
|
||||
resourceType: 'versions-json',
|
||||
path,
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'accept': 'application/json',
|
||||
},
|
||||
query: {},
|
||||
};
|
||||
|
||||
const result = await this.fetch(context);
|
||||
|
||||
if (!result || !result.success) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (Buffer.isBuffer(result.body)) {
|
||||
return JSON.parse(result.body.toString('utf8'));
|
||||
}
|
||||
|
||||
return Array.isArray(result.body) ? result.body : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Override URL building for RubyGems-specific handling.
|
||||
*/
|
||||
protected buildUpstreamUrl(
|
||||
upstream: IUpstreamRegistryConfig,
|
||||
context: IUpstreamFetchContext,
|
||||
): string {
|
||||
let baseUrl = upstream.url;
|
||||
|
||||
// Remove trailing slash
|
||||
if (baseUrl.endsWith('/')) {
|
||||
baseUrl = baseUrl.slice(0, -1);
|
||||
}
|
||||
|
||||
return `${baseUrl}${context.path}`;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,570 @@
|
||||
/**
|
||||
* Helper functions for RubyGems registry
|
||||
* Compact Index generation, dependency formatting, etc.
|
||||
*/
|
||||
|
||||
import * as plugins from '../plugins.js';
|
||||
|
||||
import type {
|
||||
IRubyGemsVersion,
|
||||
IRubyGemsDependency,
|
||||
IRubyGemsRequirement,
|
||||
ICompactIndexVersionsEntry,
|
||||
ICompactIndexInfoEntry,
|
||||
IRubyGemsMetadata,
|
||||
} from './interfaces.rubygems.js';
|
||||
|
||||
/**
|
||||
* Generate Compact Index versions file
|
||||
* Format: GEMNAME [-]VERSION_PLATFORM[,VERSION_PLATFORM,...] MD5
|
||||
* @param entries - Version entries for all gems
|
||||
* @returns Compact Index versions file content
|
||||
*/
|
||||
export function generateCompactIndexVersions(entries: ICompactIndexVersionsEntry[]): string {
|
||||
const lines: string[] = [];
|
||||
|
||||
// Add metadata header
|
||||
lines.push(`created_at: ${new Date().toISOString()}`);
|
||||
lines.push('---');
|
||||
|
||||
// Add gem entries
|
||||
for (const entry of entries) {
|
||||
const versions = entry.versions
|
||||
.map(v => {
|
||||
const yanked = v.yanked ? '-' : '';
|
||||
const platform = v.platform && v.platform !== 'ruby' ? `_${v.platform}` : '';
|
||||
return `${yanked}${v.version}${platform}`;
|
||||
})
|
||||
.join(',');
|
||||
|
||||
lines.push(`${entry.name} ${versions} ${entry.infoChecksum}`);
|
||||
}
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate Compact Index info file for a gem
|
||||
* Format: VERSION[-PLATFORM] [DEP[,DEP,...]]|REQ[,REQ,...]
|
||||
* @param entries - Info entries for gem versions
|
||||
* @returns Compact Index info file content
|
||||
*/
|
||||
export function generateCompactIndexInfo(entries: ICompactIndexInfoEntry[]): string {
|
||||
const lines: string[] = ['---']; // Info files start with ---
|
||||
|
||||
for (const entry of entries) {
|
||||
// Build version string with optional platform
|
||||
const versionStr = entry.platform && entry.platform !== 'ruby'
|
||||
? `${entry.version}-${entry.platform}`
|
||||
: entry.version;
|
||||
|
||||
// Build dependencies string
|
||||
const depsStr = entry.dependencies.length > 0
|
||||
? entry.dependencies.map(formatDependency).join(',')
|
||||
: '';
|
||||
|
||||
// Build requirements string (checksum is always required)
|
||||
const reqParts: string[] = [`checksum:${entry.checksum}`];
|
||||
|
||||
for (const req of entry.requirements) {
|
||||
reqParts.push(`${req.type}:${req.requirement}`);
|
||||
}
|
||||
|
||||
const reqStr = reqParts.join(',');
|
||||
|
||||
// Combine: VERSION[-PLATFORM] [DEPS]|REQS
|
||||
const depPart = depsStr ? ` ${depsStr}` : '';
|
||||
lines.push(`${versionStr}${depPart}|${reqStr}`);
|
||||
}
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Format a dependency for Compact Index
|
||||
* Format: GEM:CONSTRAINT[&CONSTRAINT]
|
||||
* @param dep - Dependency object
|
||||
* @returns Formatted dependency string
|
||||
*/
|
||||
export function formatDependency(dep: IRubyGemsDependency): string {
|
||||
return `${dep.name}:${dep.requirement}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse dependency string from Compact Index
|
||||
* @param depStr - Dependency string
|
||||
* @returns Dependency object
|
||||
*/
|
||||
export function parseDependency(depStr: string): IRubyGemsDependency {
|
||||
const [name, ...reqParts] = depStr.split(':');
|
||||
const requirement = reqParts.join(':'); // Handle :: in gem names
|
||||
|
||||
return { name, requirement };
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate names file (newline-separated gem names)
|
||||
* @param names - List of gem names
|
||||
* @returns Names file content
|
||||
*/
|
||||
export function generateNamesFile(names: string[]): string {
|
||||
return `---\n${names.sort().join('\n')}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate MD5 hash for Compact Index checksum
|
||||
* @param content - Content to hash
|
||||
* @returns MD5 hash (hex)
|
||||
*/
|
||||
export async function calculateMD5(content: string): Promise<string> {
|
||||
const crypto = await import('crypto');
|
||||
return crypto.createHash('md5').update(content).digest('hex');
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate SHA256 hash for gem files
|
||||
* @param data - Data to hash
|
||||
* @returns SHA256 hash (hex)
|
||||
*/
|
||||
export async function calculateSHA256(data: Buffer): Promise<string> {
|
||||
const crypto = await import('crypto');
|
||||
return crypto.createHash('sha256').update(data).digest('hex');
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse gem filename to extract name, version, and platform
|
||||
* @param filename - Gem filename (e.g., "rails-7.0.0-x86_64-linux.gem")
|
||||
* @returns Parsed info or null
|
||||
*/
|
||||
export function parseGemFilename(filename: string): {
|
||||
name: string;
|
||||
version: string;
|
||||
platform?: string;
|
||||
} | null {
|
||||
if (!filename.endsWith('.gem')) return null;
|
||||
|
||||
const withoutExt = filename.slice(0, -4); // Remove .gem
|
||||
|
||||
// Try to match: name-version-platform
|
||||
// Platform can contain hyphens (e.g., x86_64-linux)
|
||||
const parts = withoutExt.split('-');
|
||||
if (parts.length < 2) return null;
|
||||
|
||||
// Find version (first part that starts with a digit)
|
||||
let versionIndex = -1;
|
||||
for (let i = 1; i < parts.length; i++) {
|
||||
if (/^\d/.test(parts[i])) {
|
||||
versionIndex = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (versionIndex === -1) return null;
|
||||
|
||||
const name = parts.slice(0, versionIndex).join('-');
|
||||
const version = parts[versionIndex];
|
||||
const platform = versionIndex + 1 < parts.length
|
||||
? parts.slice(versionIndex + 1).join('-')
|
||||
: undefined;
|
||||
|
||||
return {
|
||||
name,
|
||||
version,
|
||||
platform: platform && platform !== 'ruby' ? platform : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate gem name
|
||||
* Must contain only ASCII letters, numbers, _, and -
|
||||
* @param name - Gem name
|
||||
* @returns true if valid
|
||||
*/
|
||||
export function isValidGemName(name: string): boolean {
|
||||
return /^[a-zA-Z0-9_-]+$/.test(name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate version string
|
||||
* Basic semantic versioning check
|
||||
* @param version - Version string
|
||||
* @returns true if valid
|
||||
*/
|
||||
export function isValidVersion(version: string): boolean {
|
||||
// Allow semver and other common Ruby version formats
|
||||
return /^[\d.a-zA-Z_-]+$/.test(version);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build version list entry for Compact Index
|
||||
* @param versions - Version info
|
||||
* @returns Version list string
|
||||
*/
|
||||
export function buildVersionList(versions: Array<{
|
||||
version: string;
|
||||
platform?: string;
|
||||
yanked: boolean;
|
||||
}>): string {
|
||||
return versions
|
||||
.map(v => {
|
||||
const yanked = v.yanked ? '-' : '';
|
||||
const platform = v.platform && v.platform !== 'ruby' ? `_${v.platform}` : '';
|
||||
return `${yanked}${v.version}${platform}`;
|
||||
})
|
||||
.join(',');
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse version list from Compact Index
|
||||
* @param versionStr - Version list string
|
||||
* @returns Parsed versions
|
||||
*/
|
||||
export function parseVersionList(versionStr: string): Array<{
|
||||
version: string;
|
||||
platform?: string;
|
||||
yanked: boolean;
|
||||
}> {
|
||||
return versionStr.split(',').map(v => {
|
||||
const yanked = v.startsWith('-');
|
||||
const withoutYank = yanked ? v.substring(1) : v;
|
||||
|
||||
// Split on _ to separate version from platform
|
||||
const [version, ...platformParts] = withoutYank.split('_');
|
||||
const platform = platformParts.length > 0 ? platformParts.join('_') : undefined;
|
||||
|
||||
return {
|
||||
version,
|
||||
platform: platform && platform !== 'ruby' ? platform : undefined,
|
||||
yanked,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate JSON response for /api/v1/versions/{gem}.json
|
||||
* @param gemName - Gem name
|
||||
* @param versions - Version list
|
||||
* @returns JSON response object
|
||||
*/
|
||||
export function generateVersionsJson(
|
||||
gemName: string,
|
||||
versions: Array<{
|
||||
version: string;
|
||||
platform?: string;
|
||||
uploadTime?: string;
|
||||
}>
|
||||
): any {
|
||||
// RubyGems.org API returns a flat array at /api/v1/versions/{gem}.json
|
||||
return versions.map(v => ({
|
||||
number: v.version,
|
||||
platform: v.platform || 'ruby',
|
||||
built_at: v.uploadTime,
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate JSON response for /api/v1/dependencies
|
||||
* @param gems - Map of gem names to version dependencies
|
||||
* @returns JSON response array
|
||||
*/
|
||||
export function generateDependenciesJson(gems: Map<string, Array<{
|
||||
version: string;
|
||||
platform?: string;
|
||||
dependencies: IRubyGemsDependency[];
|
||||
}>>): any {
|
||||
const result: any[] = [];
|
||||
|
||||
for (const [name, versions] of gems) {
|
||||
for (const v of versions) {
|
||||
result.push({
|
||||
name,
|
||||
number: v.version,
|
||||
platform: v.platform || 'ruby',
|
||||
dependencies: v.dependencies.map(d => ({
|
||||
name: d.name,
|
||||
requirements: d.requirement,
|
||||
})),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update Compact Index versions file with new gem version
|
||||
* Handles append-only semantics for the current month
|
||||
* @param existingContent - Current versions file content
|
||||
* @param gemName - Gem name
|
||||
* @param newVersion - New version info
|
||||
* @param infoChecksum - MD5 of info file
|
||||
* @returns Updated versions file content
|
||||
*/
|
||||
export function updateCompactIndexVersions(
|
||||
existingContent: string,
|
||||
gemName: string,
|
||||
newVersion: { version: string; platform?: string; yanked: boolean },
|
||||
infoChecksum: string
|
||||
): string {
|
||||
const lines = existingContent.split('\n');
|
||||
const headerEndIndex = lines.findIndex(l => l === '---');
|
||||
|
||||
if (headerEndIndex === -1) {
|
||||
throw new Error('Invalid Compact Index versions file');
|
||||
}
|
||||
|
||||
const header = lines.slice(0, headerEndIndex + 1);
|
||||
const entries = lines.slice(headerEndIndex + 1).filter(l => l.trim());
|
||||
|
||||
// Find existing entry for gem
|
||||
const gemLineIndex = entries.findIndex(l => l.startsWith(`${gemName} `));
|
||||
|
||||
const versionStr = buildVersionList([newVersion]);
|
||||
|
||||
if (gemLineIndex >= 0) {
|
||||
// Append to existing entry
|
||||
const parts = entries[gemLineIndex].split(' ');
|
||||
const existingVersions = parts[1];
|
||||
const updatedVersions = `${existingVersions},${versionStr}`;
|
||||
entries[gemLineIndex] = `${gemName} ${updatedVersions} ${infoChecksum}`;
|
||||
} else {
|
||||
// Add new entry
|
||||
entries.push(`${gemName} ${versionStr} ${infoChecksum}`);
|
||||
entries.sort(); // Keep alphabetical
|
||||
}
|
||||
|
||||
return [...header, ...entries].join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Update Compact Index info file with new version
|
||||
* @param existingContent - Current info file content
|
||||
* @param newEntry - New version entry
|
||||
* @returns Updated info file content
|
||||
*/
|
||||
export function updateCompactIndexInfo(
|
||||
existingContent: string,
|
||||
newEntry: ICompactIndexInfoEntry
|
||||
): string {
|
||||
const lines = existingContent ? existingContent.split('\n').filter(l => l !== '---') : [];
|
||||
|
||||
// Build version string
|
||||
const versionStr = newEntry.platform && newEntry.platform !== 'ruby'
|
||||
? `${newEntry.version}-${newEntry.platform}`
|
||||
: newEntry.version;
|
||||
|
||||
// Build dependencies string
|
||||
const depsStr = newEntry.dependencies.length > 0
|
||||
? newEntry.dependencies.map(formatDependency).join(',')
|
||||
: '';
|
||||
|
||||
// Build requirements string
|
||||
const reqParts: string[] = [`checksum:${newEntry.checksum}`];
|
||||
for (const req of newEntry.requirements) {
|
||||
reqParts.push(`${req.type}:${req.requirement}`);
|
||||
}
|
||||
const reqStr = reqParts.join(',');
|
||||
|
||||
// Combine
|
||||
const depPart = depsStr ? ` ${depsStr}` : '';
|
||||
const newLine = `${versionStr}${depPart}|${reqStr}`;
|
||||
|
||||
lines.push(newLine);
|
||||
|
||||
return `---\n${lines.join('\n')}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract gem specification from .gem file
|
||||
* Note: This is a simplified version. Full implementation would use tar + gzip + Marshal
|
||||
* @param gemData - Gem file data
|
||||
* @returns Extracted spec or null
|
||||
*/
|
||||
export async function extractGemSpec(gemData: Buffer): Promise<any | null> {
|
||||
try {
|
||||
// .gem files are gzipped tar archives
|
||||
// They contain metadata.gz which has Marshal-encoded spec
|
||||
// This is a placeholder - full implementation would need:
|
||||
// 1. Unzip outer gzip
|
||||
// 2. Untar to find metadata.gz
|
||||
// 3. Unzip metadata.gz
|
||||
// 4. Parse Ruby Marshal format
|
||||
|
||||
// For now, return null and expect metadata to be provided
|
||||
return null;
|
||||
} catch (error) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract basic metadata from a gem file
|
||||
* Gem files are plain tar archives (NOT gzipped) containing:
|
||||
* - metadata.gz: gzipped YAML with gem specification
|
||||
* - data.tar.gz: gzipped tar with actual gem files
|
||||
* This function extracts and parses the metadata.gz to get name/version/platform
|
||||
* @param gemData - Gem file data
|
||||
* @returns Extracted metadata or null
|
||||
*/
|
||||
export async function extractGemMetadata(gemData: Buffer): Promise<{
|
||||
name: string;
|
||||
version: string;
|
||||
platform?: string;
|
||||
} | null> {
|
||||
try {
|
||||
// Step 1: Extract the plain tar archive to get metadata.gz
|
||||
const smartArchive = plugins.smartarchive.SmartArchive.create();
|
||||
const files = await smartArchive.buffer(gemData).toSmartFiles();
|
||||
|
||||
// Find metadata.gz
|
||||
const metadataFile = files.find(f => f.path === 'metadata.gz' || f.relative === 'metadata.gz');
|
||||
if (!metadataFile) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Step 2: Decompress the gzipped metadata
|
||||
const gzipTools = new plugins.smartarchive.GzipTools();
|
||||
const metadataYaml = await gzipTools.decompress(metadataFile.contentBuffer);
|
||||
const yamlContent = Buffer.from(metadataYaml).toString('utf-8');
|
||||
|
||||
// Step 3: Parse the YAML to extract name, version, platform
|
||||
// Look for name: field in YAML
|
||||
const nameMatch = yamlContent.match(/name:\s*([^\n\r]+)/);
|
||||
|
||||
// Look for version in Ruby YAML format: version: !ruby/object:Gem::Version\n version: X.X.X
|
||||
const versionMatch = yamlContent.match(/version:\s*!ruby\/object:Gem::Version[\s\S]*?version:\s*['"]?([^'"\n\r]+)/);
|
||||
|
||||
// Also try simpler version format
|
||||
const simpleVersionMatch = !versionMatch ? yamlContent.match(/^version:\s*['"]?(\d[^'"\n\r]*)/m) : null;
|
||||
|
||||
// Look for platform
|
||||
const platformMatch = yamlContent.match(/platform:\s*([^\n\r]+)/);
|
||||
|
||||
const name = nameMatch?.[1]?.trim();
|
||||
const version = versionMatch?.[1]?.trim() || simpleVersionMatch?.[1]?.trim();
|
||||
const platform = platformMatch?.[1]?.trim();
|
||||
|
||||
if (name && version) {
|
||||
return {
|
||||
name,
|
||||
version,
|
||||
platform: platform && platform !== 'ruby' ? platform : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
} catch (_error) {
|
||||
// Error handled gracefully - return null and let caller handle missing metadata
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate gzipped specs array for /specs.4.8.gz and /latest_specs.4.8.gz
|
||||
* The format is a gzipped Ruby Marshal array of [name, version, platform] tuples
|
||||
* Since we can't easily generate Ruby Marshal format, we'll use a simple format
|
||||
* that represents the same data structure as a gzipped binary blob
|
||||
* @param specs - Array of [name, version, platform] tuples
|
||||
* @returns Gzipped specs data
|
||||
*/
|
||||
export async function generateSpecsGz(specs: Array<[string, string, string]>): Promise<Buffer> {
|
||||
const gzipTools = new plugins.smartarchive.GzipTools();
|
||||
|
||||
// Create a simplified binary representation
|
||||
// Real RubyGems uses Ruby Marshal format, but for compatibility we'll create
|
||||
// a gzipped representation that tools can recognize as valid
|
||||
|
||||
// Format: Simple binary encoding of specs array
|
||||
// Each spec: name_length(2 bytes) + name + version_length(2 bytes) + version + platform_length(2 bytes) + platform
|
||||
const parts: Buffer[] = [];
|
||||
|
||||
// Header: number of specs (4 bytes)
|
||||
const headerBuf = Buffer.alloc(4);
|
||||
headerBuf.writeUInt32LE(specs.length, 0);
|
||||
parts.push(headerBuf);
|
||||
|
||||
for (const [name, version, platform] of specs) {
|
||||
const nameBuf = Buffer.from(name, 'utf-8');
|
||||
const versionBuf = Buffer.from(version, 'utf-8');
|
||||
const platformBuf = Buffer.from(platform, 'utf-8');
|
||||
|
||||
const nameLenBuf = Buffer.alloc(2);
|
||||
nameLenBuf.writeUInt16LE(nameBuf.length, 0);
|
||||
|
||||
const versionLenBuf = Buffer.alloc(2);
|
||||
versionLenBuf.writeUInt16LE(versionBuf.length, 0);
|
||||
|
||||
const platformLenBuf = Buffer.alloc(2);
|
||||
platformLenBuf.writeUInt16LE(platformBuf.length, 0);
|
||||
|
||||
parts.push(nameLenBuf, nameBuf, versionLenBuf, versionBuf, platformLenBuf, platformBuf);
|
||||
}
|
||||
|
||||
const uncompressed = Buffer.concat(parts);
|
||||
return Buffer.from(await gzipTools.compress(uncompressed));
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate compressed gemspec for /quick/Marshal.4.8/{gem}-{version}.gemspec.rz
|
||||
* The format is a zlib-compressed Ruby Marshal representation of the gemspec
|
||||
* Since we can't easily generate Ruby Marshal, we'll create a simplified format
|
||||
* @param name - Gem name
|
||||
* @param versionMeta - Version metadata
|
||||
* @returns Zlib-compressed gemspec data
|
||||
*/
|
||||
export async function generateGemspecRz(
|
||||
name: string,
|
||||
versionMeta: {
|
||||
version: string;
|
||||
platform?: string;
|
||||
checksum: string;
|
||||
dependencies?: Array<{ name: string; requirement: string }>;
|
||||
}
|
||||
): Promise<Buffer> {
|
||||
const zlib = await import('zlib');
|
||||
const { promisify } = await import('util');
|
||||
const deflate = promisify(zlib.deflate);
|
||||
|
||||
// Create a YAML-like representation that can be parsed
|
||||
const gemspecYaml = `--- !ruby/object:Gem::Specification
|
||||
name: ${name}
|
||||
version: !ruby/object:Gem::Version
|
||||
version: ${versionMeta.version}
|
||||
platform: ${versionMeta.platform || 'ruby'}
|
||||
authors: []
|
||||
date: ${new Date().toISOString().split('T')[0]}
|
||||
dependencies: []
|
||||
description:
|
||||
email:
|
||||
executables: []
|
||||
extensions: []
|
||||
extra_rdoc_files: []
|
||||
files: []
|
||||
homepage:
|
||||
licenses: []
|
||||
metadata: {}
|
||||
post_install_message:
|
||||
rdoc_options: []
|
||||
require_paths:
|
||||
- lib
|
||||
required_ruby_version: !ruby/object:Gem::Requirement
|
||||
requirements:
|
||||
- - ">="
|
||||
- !ruby/object:Gem::Version
|
||||
version: '0'
|
||||
required_rubygems_version: !ruby/object:Gem::Requirement
|
||||
requirements:
|
||||
- - ">="
|
||||
- !ruby/object:Gem::Version
|
||||
version: '0'
|
||||
requirements: []
|
||||
rubygems_version: 3.0.0
|
||||
signing_key:
|
||||
specification_version: 4
|
||||
summary:
|
||||
test_files: []
|
||||
`;
|
||||
|
||||
// Use zlib deflate (not gzip) for .rz files
|
||||
return deflate(Buffer.from(gemspecYaml, 'utf-8'));
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
/**
|
||||
* RubyGems Registry Module
|
||||
* RubyGems/Bundler Compact Index implementation
|
||||
*/
|
||||
|
||||
export * from './interfaces.rubygems.js';
|
||||
export * from './classes.rubygemsregistry.js';
|
||||
export { RubygemsUpstream } from './classes.rubygemsupstream.js';
|
||||
export * as rubygemsHelpers from './helpers.rubygems.js';
|
||||
@@ -0,0 +1,251 @@
|
||||
/**
|
||||
* RubyGems Registry Type Definitions
|
||||
* Compliant with Compact Index API and RubyGems protocol
|
||||
*/
|
||||
|
||||
/**
|
||||
* Gem version entry in compact index
|
||||
*/
|
||||
export interface IRubyGemsVersion {
|
||||
/** Version number */
|
||||
version: string;
|
||||
/** Platform (e.g., ruby, x86_64-linux) */
|
||||
platform?: string;
|
||||
/** Dependencies */
|
||||
dependencies?: IRubyGemsDependency[];
|
||||
/** Requirements */
|
||||
requirements?: IRubyGemsRequirement[];
|
||||
/** Whether this version is yanked */
|
||||
yanked?: boolean;
|
||||
/** SHA256 checksum of .gem file */
|
||||
checksum?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gem dependency specification
|
||||
*/
|
||||
export interface IRubyGemsDependency {
|
||||
/** Gem name */
|
||||
name: string;
|
||||
/** Version requirement (e.g., ">= 1.0", "~> 2.0") */
|
||||
requirement: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gem requirements (ruby version, rubygems version, etc.)
|
||||
*/
|
||||
export interface IRubyGemsRequirement {
|
||||
/** Requirement type (ruby, rubygems) */
|
||||
type: 'ruby' | 'rubygems';
|
||||
/** Version requirement */
|
||||
requirement: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Complete gem metadata
|
||||
*/
|
||||
export interface IRubyGemsMetadata {
|
||||
/** Gem name */
|
||||
name: string;
|
||||
/** All versions */
|
||||
versions: Record<string, IRubyGemsVersionMetadata>;
|
||||
/** Last modified timestamp */
|
||||
'last-modified'?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Version-specific metadata
|
||||
*/
|
||||
export interface IRubyGemsVersionMetadata {
|
||||
/** Version number */
|
||||
version: string;
|
||||
/** Platform */
|
||||
platform?: string;
|
||||
/** Authors */
|
||||
authors?: string[];
|
||||
/** Description */
|
||||
description?: string;
|
||||
/** Summary */
|
||||
summary?: string;
|
||||
/** Homepage */
|
||||
homepage?: string;
|
||||
/** License */
|
||||
license?: string;
|
||||
/** Dependencies */
|
||||
dependencies?: IRubyGemsDependency[];
|
||||
/** Requirements */
|
||||
requirements?: IRubyGemsRequirement[];
|
||||
/** SHA256 checksum */
|
||||
checksum: string;
|
||||
/** File size */
|
||||
size: number;
|
||||
/** Upload timestamp */
|
||||
'upload-time': string;
|
||||
/** Uploader */
|
||||
'uploaded-by': string;
|
||||
/** Yanked status */
|
||||
yanked?: boolean;
|
||||
/** Yank reason */
|
||||
'yank-reason'?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compact index versions file entry
|
||||
* Format: GEMNAME [-]VERSION_PLATFORM[,VERSION_PLATFORM,...] MD5
|
||||
*/
|
||||
export interface ICompactIndexVersionsEntry {
|
||||
/** Gem name */
|
||||
name: string;
|
||||
/** Versions (with optional platform and yank flag) */
|
||||
versions: Array<{
|
||||
version: string;
|
||||
platform?: string;
|
||||
yanked: boolean;
|
||||
}>;
|
||||
/** MD5 checksum of info file */
|
||||
infoChecksum: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compact index info file entry
|
||||
* Format: VERSION[-PLATFORM] [DEP[,DEP,...]]|REQ[,REQ,...]
|
||||
*/
|
||||
export interface ICompactIndexInfoEntry {
|
||||
/** Version number */
|
||||
version: string;
|
||||
/** Platform (optional) */
|
||||
platform?: string;
|
||||
/** Dependencies */
|
||||
dependencies: IRubyGemsDependency[];
|
||||
/** Requirements */
|
||||
requirements: IRubyGemsRequirement[];
|
||||
/** SHA256 checksum */
|
||||
checksum: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gem upload request
|
||||
*/
|
||||
export interface IRubyGemsUploadRequest {
|
||||
/** Gem file data */
|
||||
gemData: Buffer;
|
||||
/** Gem filename */
|
||||
filename: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gem upload response
|
||||
*/
|
||||
export interface IRubyGemsUploadResponse {
|
||||
/** Success message */
|
||||
message?: string;
|
||||
/** Gem name */
|
||||
name?: string;
|
||||
/** Version */
|
||||
version?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Yank request
|
||||
*/
|
||||
export interface IRubyGemsYankRequest {
|
||||
/** Gem name */
|
||||
gem_name: string;
|
||||
/** Version to yank */
|
||||
version: string;
|
||||
/** Platform (optional) */
|
||||
platform?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Yank response
|
||||
*/
|
||||
export interface IRubyGemsYankResponse {
|
||||
/** Success indicator */
|
||||
success: boolean;
|
||||
/** Message */
|
||||
message?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Version info response (JSON)
|
||||
*/
|
||||
export interface IRubyGemsVersionInfo {
|
||||
/** Gem name */
|
||||
name: string;
|
||||
/** Versions list */
|
||||
versions: Array<{
|
||||
/** Version number */
|
||||
number: string;
|
||||
/** Platform */
|
||||
platform?: string;
|
||||
/** Build date */
|
||||
built_at?: string;
|
||||
/** Download count */
|
||||
downloads_count?: number;
|
||||
}>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Dependencies query response
|
||||
*/
|
||||
export interface IRubyGemsDependenciesResponse {
|
||||
/** Dependencies for requested gems */
|
||||
dependencies: Array<{
|
||||
/** Gem name */
|
||||
name: string;
|
||||
/** Version */
|
||||
number: string;
|
||||
/** Platform */
|
||||
platform?: string;
|
||||
/** Dependencies */
|
||||
dependencies: Array<{
|
||||
name: string;
|
||||
requirements: string;
|
||||
}>;
|
||||
}>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Error response structure
|
||||
*/
|
||||
export interface IRubyGemsError {
|
||||
/** Error message */
|
||||
error: string;
|
||||
/** HTTP status code */
|
||||
status?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gem specification (extracted from .gem file)
|
||||
*/
|
||||
export interface IRubyGemsSpec {
|
||||
/** Gem name */
|
||||
name: string;
|
||||
/** Version */
|
||||
version: string;
|
||||
/** Platform */
|
||||
platform?: string;
|
||||
/** Authors */
|
||||
authors?: string[];
|
||||
/** Email */
|
||||
email?: string;
|
||||
/** Homepage */
|
||||
homepage?: string;
|
||||
/** Summary */
|
||||
summary?: string;
|
||||
/** Description */
|
||||
description?: string;
|
||||
/** License */
|
||||
license?: string;
|
||||
/** Dependencies */
|
||||
dependencies?: IRubyGemsDependency[];
|
||||
/** Required Ruby version */
|
||||
required_ruby_version?: string;
|
||||
/** Required RubyGems version */
|
||||
required_rubygems_version?: string;
|
||||
/** Files */
|
||||
files?: string[];
|
||||
/** Requirements */
|
||||
requirements?: string[];
|
||||
}
|
||||
@@ -0,0 +1,526 @@
|
||||
import * as plugins from '../plugins.js';
|
||||
import type {
|
||||
IUpstreamRegistryConfig,
|
||||
IUpstreamAuthConfig,
|
||||
IUpstreamCacheConfig,
|
||||
IUpstreamResilienceConfig,
|
||||
IUpstreamResult,
|
||||
IUpstreamFetchContext,
|
||||
IProtocolUpstreamConfig,
|
||||
IUpstreamScopeRule,
|
||||
TCircuitState,
|
||||
} from './interfaces.upstream.js';
|
||||
import {
|
||||
DEFAULT_CACHE_CONFIG,
|
||||
DEFAULT_RESILIENCE_CONFIG,
|
||||
} from './interfaces.upstream.js';
|
||||
import { CircuitBreaker, CircuitOpenError, withCircuitBreaker } from './classes.circuitbreaker.js';
|
||||
import { UpstreamCache } from './classes.upstreamcache.js';
|
||||
|
||||
/**
|
||||
* Base class for protocol-specific upstream implementations.
|
||||
*
|
||||
* Provides:
|
||||
* - Multi-upstream routing with priority
|
||||
* - Scope-based filtering (glob patterns)
|
||||
* - Authentication handling
|
||||
* - Circuit breaker per upstream
|
||||
* - Caching with TTL
|
||||
* - Retry with exponential backoff
|
||||
* - 429 rate limit handling
|
||||
*/
|
||||
export abstract class BaseUpstream {
|
||||
/** Protocol name for logging */
|
||||
protected abstract readonly protocolName: string;
|
||||
|
||||
/** Upstream configuration */
|
||||
protected readonly config: IProtocolUpstreamConfig;
|
||||
|
||||
/** Resolved cache configuration */
|
||||
protected readonly cacheConfig: IUpstreamCacheConfig;
|
||||
|
||||
/** Resolved resilience configuration */
|
||||
protected readonly resilienceConfig: IUpstreamResilienceConfig;
|
||||
|
||||
/** Circuit breakers per upstream */
|
||||
protected readonly circuitBreakers: Map<string, CircuitBreaker> = new Map();
|
||||
|
||||
/** Upstream cache */
|
||||
protected readonly cache: UpstreamCache;
|
||||
|
||||
/** Logger instance */
|
||||
protected readonly logger: plugins.smartlog.Smartlog;
|
||||
|
||||
constructor(config: IProtocolUpstreamConfig, logger?: plugins.smartlog.Smartlog) {
|
||||
this.config = config;
|
||||
this.cacheConfig = { ...DEFAULT_CACHE_CONFIG, ...config.cache };
|
||||
this.resilienceConfig = { ...DEFAULT_RESILIENCE_CONFIG, ...config.resilience };
|
||||
this.cache = new UpstreamCache(this.cacheConfig);
|
||||
this.logger = logger || new plugins.smartlog.Smartlog({
|
||||
logContext: {
|
||||
company: 'smartregistry',
|
||||
companyunit: 'upstream',
|
||||
environment: 'production',
|
||||
runtime: 'node',
|
||||
}
|
||||
});
|
||||
|
||||
// Initialize circuit breakers for each upstream
|
||||
for (const upstream of config.upstreams) {
|
||||
const upstreamResilience = { ...this.resilienceConfig, ...upstream.resilience };
|
||||
this.circuitBreakers.set(upstream.id, new CircuitBreaker(upstream.id, upstreamResilience));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if upstream is enabled.
|
||||
*/
|
||||
public isEnabled(): boolean {
|
||||
return this.config.enabled;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all configured upstreams.
|
||||
*/
|
||||
public getUpstreams(): IUpstreamRegistryConfig[] {
|
||||
return this.config.upstreams;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get circuit breaker state for an upstream.
|
||||
*/
|
||||
public getCircuitState(upstreamId: string): TCircuitState | null {
|
||||
const breaker = this.circuitBreakers.get(upstreamId);
|
||||
return breaker ? breaker.getState() : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get cache statistics.
|
||||
*/
|
||||
public getCacheStats() {
|
||||
return this.cache.getStats();
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a resource from upstreams.
|
||||
* Tries upstreams in priority order, respecting circuit breakers and scope rules.
|
||||
*/
|
||||
public async fetch(context: IUpstreamFetchContext): Promise<IUpstreamResult | null> {
|
||||
if (!this.config.enabled) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Get applicable upstreams sorted by priority
|
||||
const applicableUpstreams = this.getApplicableUpstreams(context.resource);
|
||||
|
||||
if (applicableUpstreams.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Use the first applicable upstream's URL for cache key
|
||||
const primaryUpstreamUrl = applicableUpstreams[0]?.url;
|
||||
|
||||
// Check cache first
|
||||
const cached = await this.cache.get(context, primaryUpstreamUrl);
|
||||
if (cached && !cached.stale) {
|
||||
return {
|
||||
success: true,
|
||||
status: 200,
|
||||
headers: cached.headers,
|
||||
body: cached.data,
|
||||
upstreamId: cached.upstreamId,
|
||||
fromCache: true,
|
||||
latencyMs: 0,
|
||||
};
|
||||
}
|
||||
|
||||
// Check for negative cache (recent 404)
|
||||
if (await this.cache.hasNegative(context, primaryUpstreamUrl)) {
|
||||
return {
|
||||
success: false,
|
||||
status: 404,
|
||||
headers: {},
|
||||
upstreamId: 'cache',
|
||||
fromCache: true,
|
||||
latencyMs: 0,
|
||||
};
|
||||
}
|
||||
|
||||
// If we have stale cache, return it immediately and revalidate in background
|
||||
if (cached?.stale && this.cacheConfig.staleWhileRevalidate) {
|
||||
// Fire and forget revalidation
|
||||
this.revalidateInBackground(context, applicableUpstreams);
|
||||
return {
|
||||
success: true,
|
||||
status: 200,
|
||||
headers: cached.headers,
|
||||
body: cached.data,
|
||||
upstreamId: cached.upstreamId,
|
||||
fromCache: true,
|
||||
latencyMs: 0,
|
||||
};
|
||||
}
|
||||
|
||||
// Try each upstream in order
|
||||
let lastError: Error | null = null;
|
||||
|
||||
for (const upstream of applicableUpstreams) {
|
||||
const breaker = this.circuitBreakers.get(upstream.id);
|
||||
if (!breaker) continue;
|
||||
|
||||
try {
|
||||
const result = await withCircuitBreaker(
|
||||
breaker,
|
||||
() => this.fetchFromUpstream(upstream, context),
|
||||
);
|
||||
|
||||
// Cache successful responses
|
||||
if (result.success && result.body) {
|
||||
await this.cache.set(
|
||||
context,
|
||||
Buffer.isBuffer(result.body) ? result.body : Buffer.from(JSON.stringify(result.body)),
|
||||
result.headers['content-type'] || 'application/octet-stream',
|
||||
result.headers,
|
||||
upstream.id,
|
||||
upstream.url,
|
||||
);
|
||||
}
|
||||
|
||||
// Cache 404 responses
|
||||
if (result.status === 404) {
|
||||
await this.cache.setNegative(context, upstream.id, upstream.url);
|
||||
}
|
||||
|
||||
return result;
|
||||
} catch (error) {
|
||||
if (error instanceof CircuitOpenError) {
|
||||
this.logger.log('debug', `Circuit open for upstream ${upstream.id}, trying next`);
|
||||
} else {
|
||||
this.logger.log('warn', `Upstream ${upstream.id} failed: ${(error as Error).message}`);
|
||||
}
|
||||
lastError = error as Error;
|
||||
// Continue to next upstream
|
||||
}
|
||||
}
|
||||
|
||||
// All upstreams failed
|
||||
if (lastError) {
|
||||
this.logger.log('error', `All upstreams failed for ${context.resource}: ${lastError.message}`);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Invalidate cache for a resource pattern.
|
||||
*/
|
||||
public async invalidateCache(pattern: RegExp): Promise<number> {
|
||||
return this.cache.invalidatePattern(pattern);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all cache entries.
|
||||
*/
|
||||
public async clearCache(): Promise<void> {
|
||||
await this.cache.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the upstream (cleanup resources).
|
||||
*/
|
||||
public stop(): void {
|
||||
this.cache.stop();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get upstreams that apply to a resource, sorted by priority.
|
||||
*/
|
||||
protected getApplicableUpstreams(resource: string): IUpstreamRegistryConfig[] {
|
||||
return this.config.upstreams
|
||||
.filter(upstream => {
|
||||
if (!upstream.enabled) return false;
|
||||
|
||||
// Check circuit breaker
|
||||
const breaker = this.circuitBreakers.get(upstream.id);
|
||||
if (breaker && !breaker.canRequest()) return false;
|
||||
|
||||
// Check scope rules
|
||||
return this.matchesScopeRules(resource, upstream.scopeRules);
|
||||
})
|
||||
.sort((a, b) => a.priority - b.priority);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a resource matches scope rules.
|
||||
* Empty rules = match all.
|
||||
*/
|
||||
protected matchesScopeRules(resource: string, rules?: IUpstreamScopeRule[]): boolean {
|
||||
if (!rules || rules.length === 0) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Process rules in order
|
||||
// Start with default exclude (nothing matches)
|
||||
let matched = false;
|
||||
|
||||
for (const rule of rules) {
|
||||
const isMatch = plugins.minimatch(resource, rule.pattern);
|
||||
if (isMatch) {
|
||||
matched = rule.action === 'include';
|
||||
}
|
||||
}
|
||||
|
||||
return matched;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch from a specific upstream with retry logic.
|
||||
*/
|
||||
protected async fetchFromUpstream(
|
||||
upstream: IUpstreamRegistryConfig,
|
||||
context: IUpstreamFetchContext,
|
||||
): Promise<IUpstreamResult> {
|
||||
const upstreamResilience = { ...this.resilienceConfig, ...upstream.resilience };
|
||||
const startTime = Date.now();
|
||||
|
||||
let lastError: Error | null = null;
|
||||
|
||||
for (let attempt = 0; attempt <= upstreamResilience.maxRetries; attempt++) {
|
||||
try {
|
||||
const result = await this.executeRequest(upstream, context, upstreamResilience.timeoutMs);
|
||||
return {
|
||||
...result,
|
||||
upstreamId: upstream.id,
|
||||
fromCache: false,
|
||||
latencyMs: Date.now() - startTime,
|
||||
};
|
||||
} catch (error) {
|
||||
lastError = error as Error;
|
||||
|
||||
// Don't retry on 4xx errors (except 429)
|
||||
if (this.isNonRetryableError(error)) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Calculate delay with exponential backoff and jitter
|
||||
if (attempt < upstreamResilience.maxRetries) {
|
||||
const delay = this.calculateBackoffDelay(
|
||||
attempt,
|
||||
upstreamResilience.retryDelayMs,
|
||||
upstreamResilience.retryMaxDelayMs,
|
||||
);
|
||||
await this.sleep(delay);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError || new Error('Request failed');
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a single HTTP request to an upstream.
|
||||
*/
|
||||
protected async executeRequest(
|
||||
upstream: IUpstreamRegistryConfig,
|
||||
context: IUpstreamFetchContext,
|
||||
timeoutMs: number,
|
||||
): Promise<Omit<IUpstreamResult, 'upstreamId' | 'fromCache' | 'latencyMs'>> {
|
||||
// Build the full URL
|
||||
const url = this.buildUpstreamUrl(upstream, context);
|
||||
|
||||
// Build headers with auth
|
||||
const headers = this.buildHeaders(upstream, context);
|
||||
|
||||
// Make the request using SmartRequest
|
||||
const request = plugins.smartrequest.SmartRequest.create()
|
||||
.url(url)
|
||||
.method(context.method as any)
|
||||
.headers(headers)
|
||||
.timeout(timeoutMs)
|
||||
.handle429Backoff({ maxRetries: 3, fallbackDelay: 1000, maxWaitTime: 30000 });
|
||||
|
||||
// Add query params if present
|
||||
if (Object.keys(context.query).length > 0) {
|
||||
request.query(context.query);
|
||||
}
|
||||
|
||||
let response: plugins.smartrequest.ICoreResponse;
|
||||
|
||||
switch (context.method.toUpperCase()) {
|
||||
case 'GET':
|
||||
response = await request.get();
|
||||
break;
|
||||
case 'HEAD':
|
||||
// SmartRequest doesn't have head(), use options
|
||||
response = await request.method('HEAD').get();
|
||||
break;
|
||||
default:
|
||||
response = await request.get();
|
||||
}
|
||||
|
||||
// Parse response
|
||||
const responseHeaders: Record<string, string> = {};
|
||||
for (const [key, value] of Object.entries(response.headers)) {
|
||||
responseHeaders[key.toLowerCase()] = Array.isArray(value) ? value[0] : value;
|
||||
}
|
||||
|
||||
let body: Buffer | any;
|
||||
const contentType = responseHeaders['content-type'] || '';
|
||||
|
||||
if (response.ok) {
|
||||
if (contentType.includes('application/json')) {
|
||||
body = await response.json();
|
||||
} else {
|
||||
const arrayBuffer = await response.arrayBuffer();
|
||||
body = Buffer.from(arrayBuffer);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
success: response.ok,
|
||||
status: response.status,
|
||||
headers: responseHeaders,
|
||||
body,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the full URL for an upstream request.
|
||||
* Subclasses can override for protocol-specific URL building.
|
||||
*/
|
||||
protected buildUpstreamUrl(upstream: IUpstreamRegistryConfig, context: IUpstreamFetchContext): string {
|
||||
// Remove leading slash if URL already has trailing slash
|
||||
let path = context.path;
|
||||
if (upstream.url.endsWith('/') && path.startsWith('/')) {
|
||||
path = path.slice(1);
|
||||
}
|
||||
return `${upstream.url}${path}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build headers including authentication.
|
||||
*/
|
||||
protected buildHeaders(
|
||||
upstream: IUpstreamRegistryConfig,
|
||||
context: IUpstreamFetchContext,
|
||||
): Record<string, string> {
|
||||
const headers: Record<string, string> = { ...context.headers };
|
||||
|
||||
// Remove host header (will be set by HTTP client)
|
||||
delete headers['host'];
|
||||
|
||||
// Add authentication
|
||||
this.addAuthHeaders(headers, upstream.auth);
|
||||
|
||||
return headers;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add authentication headers based on auth config.
|
||||
*/
|
||||
protected addAuthHeaders(headers: Record<string, string>, auth: IUpstreamAuthConfig): void {
|
||||
switch (auth.type) {
|
||||
case 'basic':
|
||||
if (auth.username && auth.password) {
|
||||
const credentials = Buffer.from(`${auth.username}:${auth.password}`).toString('base64');
|
||||
headers['authorization'] = `Basic ${credentials}`;
|
||||
}
|
||||
break;
|
||||
case 'bearer':
|
||||
if (auth.token) {
|
||||
headers['authorization'] = `Bearer ${auth.token}`;
|
||||
}
|
||||
break;
|
||||
case 'api-key':
|
||||
if (auth.token) {
|
||||
const headerName = auth.headerName || 'authorization';
|
||||
headers[headerName.toLowerCase()] = auth.token;
|
||||
}
|
||||
break;
|
||||
case 'none':
|
||||
default:
|
||||
// No authentication
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if an error should not be retried.
|
||||
*/
|
||||
protected isNonRetryableError(error: unknown): boolean {
|
||||
// Check for HTTP status errors
|
||||
if (error && typeof error === 'object' && 'status' in error) {
|
||||
const status = (error as { status: number }).status;
|
||||
// Don't retry 4xx errors except 429 (rate limited)
|
||||
if (status >= 400 && status < 500 && status !== 429) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate backoff delay with exponential backoff and jitter.
|
||||
*/
|
||||
protected calculateBackoffDelay(
|
||||
attempt: number,
|
||||
baseDelayMs: number,
|
||||
maxDelayMs: number,
|
||||
): number {
|
||||
// Exponential backoff: delay = base * 2^attempt
|
||||
const exponentialDelay = baseDelayMs * Math.pow(2, attempt);
|
||||
|
||||
// Cap at max delay
|
||||
const cappedDelay = Math.min(exponentialDelay, maxDelayMs);
|
||||
|
||||
// Add jitter (±25%)
|
||||
const jitter = cappedDelay * 0.25 * (Math.random() * 2 - 1);
|
||||
|
||||
return Math.floor(cappedDelay + jitter);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sleep for a specified duration.
|
||||
*/
|
||||
protected sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
/**
|
||||
* Revalidate cache in background.
|
||||
*/
|
||||
protected async revalidateInBackground(
|
||||
context: IUpstreamFetchContext,
|
||||
upstreams: IUpstreamRegistryConfig[],
|
||||
): Promise<void> {
|
||||
try {
|
||||
for (const upstream of upstreams) {
|
||||
const breaker = this.circuitBreakers.get(upstream.id);
|
||||
if (!breaker || !breaker.canRequest()) continue;
|
||||
|
||||
try {
|
||||
const result = await withCircuitBreaker(
|
||||
breaker,
|
||||
() => this.fetchFromUpstream(upstream, context),
|
||||
);
|
||||
|
||||
if (result.success && result.body) {
|
||||
await this.cache.set(
|
||||
context,
|
||||
Buffer.isBuffer(result.body) ? result.body : Buffer.from(JSON.stringify(result.body)),
|
||||
result.headers['content-type'] || 'application/octet-stream',
|
||||
result.headers,
|
||||
upstream.id,
|
||||
upstream.url,
|
||||
);
|
||||
return; // Successfully revalidated
|
||||
}
|
||||
} catch {
|
||||
// Continue to next upstream
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.log('debug', `Background revalidation failed: ${(error as Error).message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,238 @@
|
||||
import type { TCircuitState, IUpstreamResilienceConfig } from './interfaces.upstream.js';
|
||||
import { DEFAULT_RESILIENCE_CONFIG } from './interfaces.upstream.js';
|
||||
|
||||
/**
|
||||
* Circuit breaker implementation for upstream resilience.
|
||||
*
|
||||
* States:
|
||||
* - CLOSED: Normal operation, requests pass through
|
||||
* - OPEN: Circuit is tripped, requests fail fast
|
||||
* - HALF_OPEN: Testing if upstream has recovered
|
||||
*
|
||||
* Transitions:
|
||||
* - CLOSED → OPEN: When failure count exceeds threshold
|
||||
* - OPEN → HALF_OPEN: After reset timeout expires
|
||||
* - HALF_OPEN → CLOSED: On successful request
|
||||
* - HALF_OPEN → OPEN: On failed request
|
||||
*/
|
||||
export class CircuitBreaker {
|
||||
/** Unique identifier for logging and metrics */
|
||||
public readonly id: string;
|
||||
|
||||
/** Current circuit state */
|
||||
private state: TCircuitState = 'CLOSED';
|
||||
|
||||
/** Count of consecutive failures */
|
||||
private failureCount: number = 0;
|
||||
|
||||
/** Timestamp when circuit was opened */
|
||||
private openedAt: number = 0;
|
||||
|
||||
/** Number of successful requests in half-open state */
|
||||
private halfOpenSuccesses: number = 0;
|
||||
|
||||
/** Configuration */
|
||||
private readonly config: IUpstreamResilienceConfig;
|
||||
|
||||
/** Number of successes required to close circuit from half-open */
|
||||
private readonly halfOpenThreshold: number = 2;
|
||||
|
||||
constructor(id: string, config?: Partial<IUpstreamResilienceConfig>) {
|
||||
this.id = id;
|
||||
this.config = { ...DEFAULT_RESILIENCE_CONFIG, ...config };
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current circuit state.
|
||||
*/
|
||||
public getState(): TCircuitState {
|
||||
// Check if we should transition from OPEN to HALF_OPEN
|
||||
if (this.state === 'OPEN') {
|
||||
const elapsed = Date.now() - this.openedAt;
|
||||
if (elapsed >= this.config.circuitBreakerResetMs) {
|
||||
this.transitionTo('HALF_OPEN');
|
||||
}
|
||||
}
|
||||
return this.state;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if circuit allows requests.
|
||||
* Returns true if requests should be allowed.
|
||||
*/
|
||||
public canRequest(): boolean {
|
||||
const currentState = this.getState();
|
||||
return currentState !== 'OPEN';
|
||||
}
|
||||
|
||||
/**
|
||||
* Record a successful request.
|
||||
* May transition circuit from HALF_OPEN to CLOSED.
|
||||
*/
|
||||
public recordSuccess(): void {
|
||||
if (this.state === 'HALF_OPEN') {
|
||||
this.halfOpenSuccesses++;
|
||||
if (this.halfOpenSuccesses >= this.halfOpenThreshold) {
|
||||
this.transitionTo('CLOSED');
|
||||
}
|
||||
} else if (this.state === 'CLOSED') {
|
||||
// Reset failure count on success
|
||||
this.failureCount = 0;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Record a failed request.
|
||||
* May transition circuit from CLOSED/HALF_OPEN to OPEN.
|
||||
*/
|
||||
public recordFailure(): void {
|
||||
if (this.state === 'HALF_OPEN') {
|
||||
// Any failure in half-open immediately opens circuit
|
||||
this.transitionTo('OPEN');
|
||||
} else if (this.state === 'CLOSED') {
|
||||
this.failureCount++;
|
||||
if (this.failureCount >= this.config.circuitBreakerThreshold) {
|
||||
this.transitionTo('OPEN');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Force circuit to open state.
|
||||
* Useful for manual intervention or external health checks.
|
||||
*/
|
||||
public forceOpen(): void {
|
||||
this.transitionTo('OPEN');
|
||||
}
|
||||
|
||||
/**
|
||||
* Force circuit to closed state.
|
||||
* Useful for manual intervention after fixing upstream issues.
|
||||
*/
|
||||
public forceClose(): void {
|
||||
this.transitionTo('CLOSED');
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset circuit to initial state.
|
||||
*/
|
||||
public reset(): void {
|
||||
this.state = 'CLOSED';
|
||||
this.failureCount = 0;
|
||||
this.openedAt = 0;
|
||||
this.halfOpenSuccesses = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get circuit metrics for monitoring.
|
||||
*/
|
||||
public getMetrics(): ICircuitBreakerMetrics {
|
||||
return {
|
||||
id: this.id,
|
||||
state: this.getState(),
|
||||
failureCount: this.failureCount,
|
||||
openedAt: this.openedAt > 0 ? new Date(this.openedAt) : null,
|
||||
timeUntilHalfOpen: this.state === 'OPEN'
|
||||
? Math.max(0, this.config.circuitBreakerResetMs - (Date.now() - this.openedAt))
|
||||
: 0,
|
||||
halfOpenSuccesses: this.halfOpenSuccesses,
|
||||
threshold: this.config.circuitBreakerThreshold,
|
||||
resetMs: this.config.circuitBreakerResetMs,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Transition to a new state with proper cleanup.
|
||||
*/
|
||||
private transitionTo(newState: TCircuitState): void {
|
||||
const previousState = this.state;
|
||||
this.state = newState;
|
||||
|
||||
switch (newState) {
|
||||
case 'OPEN':
|
||||
this.openedAt = Date.now();
|
||||
this.halfOpenSuccesses = 0;
|
||||
break;
|
||||
case 'HALF_OPEN':
|
||||
this.halfOpenSuccesses = 0;
|
||||
break;
|
||||
case 'CLOSED':
|
||||
this.failureCount = 0;
|
||||
this.openedAt = 0;
|
||||
this.halfOpenSuccesses = 0;
|
||||
break;
|
||||
}
|
||||
|
||||
// Log state transition (useful for debugging and monitoring)
|
||||
// In production, this would emit events or metrics
|
||||
if (previousState !== newState) {
|
||||
// State changed - could emit event here
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Metrics for circuit breaker monitoring.
|
||||
*/
|
||||
export interface ICircuitBreakerMetrics {
|
||||
/** Circuit breaker identifier */
|
||||
id: string;
|
||||
/** Current state */
|
||||
state: TCircuitState;
|
||||
/** Number of consecutive failures */
|
||||
failureCount: number;
|
||||
/** When circuit was opened (null if never opened) */
|
||||
openedAt: Date | null;
|
||||
/** Milliseconds until circuit transitions to half-open (0 if not open) */
|
||||
timeUntilHalfOpen: number;
|
||||
/** Number of successes in half-open state */
|
||||
halfOpenSuccesses: number;
|
||||
/** Failure threshold for opening circuit */
|
||||
threshold: number;
|
||||
/** Reset timeout in milliseconds */
|
||||
resetMs: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a function with circuit breaker protection.
|
||||
*
|
||||
* @param breaker The circuit breaker to use
|
||||
* @param fn The async function to execute
|
||||
* @param fallback Optional fallback function when circuit is open
|
||||
* @returns The result of fn or fallback
|
||||
* @throws CircuitOpenError if circuit is open and no fallback provided
|
||||
*/
|
||||
export async function withCircuitBreaker<T>(
|
||||
breaker: CircuitBreaker,
|
||||
fn: () => Promise<T>,
|
||||
fallback?: () => Promise<T>,
|
||||
): Promise<T> {
|
||||
if (!breaker.canRequest()) {
|
||||
if (fallback) {
|
||||
return fallback();
|
||||
}
|
||||
throw new CircuitOpenError(breaker.id);
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await fn();
|
||||
breaker.recordSuccess();
|
||||
return result;
|
||||
} catch (error) {
|
||||
breaker.recordFailure();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Error thrown when circuit is open and no fallback is provided.
|
||||
*/
|
||||
export class CircuitOpenError extends Error {
|
||||
public readonly circuitId: string;
|
||||
|
||||
constructor(circuitId: string) {
|
||||
super(`Circuit breaker '${circuitId}' is open`);
|
||||
this.name = 'CircuitOpenError';
|
||||
this.circuitId = circuitId;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,626 @@
|
||||
import type {
|
||||
ICacheEntry,
|
||||
IUpstreamCacheConfig,
|
||||
IUpstreamFetchContext,
|
||||
} from './interfaces.upstream.js';
|
||||
import { DEFAULT_CACHE_CONFIG } from './interfaces.upstream.js';
|
||||
import type { IStorageBackend } from '../core/interfaces.core.js';
|
||||
|
||||
/**
|
||||
* Cache metadata stored alongside cache entries.
|
||||
*/
|
||||
interface ICacheMetadata {
|
||||
contentType: string;
|
||||
headers: Record<string, string>;
|
||||
cachedAt: string;
|
||||
expiresAt?: string;
|
||||
etag?: string;
|
||||
upstreamId: string;
|
||||
upstreamUrl: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* S3-backed upstream cache with in-memory hot layer.
|
||||
*
|
||||
* Features:
|
||||
* - TTL-based expiration
|
||||
* - Stale-while-revalidate support
|
||||
* - Negative caching (404s)
|
||||
* - Content-type aware caching
|
||||
* - ETag support for conditional requests
|
||||
* - Multi-upstream support via URL-based cache paths
|
||||
* - Persistent S3 storage with in-memory hot layer
|
||||
*
|
||||
* Cache paths are structured as:
|
||||
* cache/{escaped-upstream-url}/{protocol}:{method}:{path}
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* // In-memory only (default)
|
||||
* const cache = new UpstreamCache(config);
|
||||
*
|
||||
* // With S3 persistence
|
||||
* const cache = new UpstreamCache(config, 10000, storage);
|
||||
* ```
|
||||
*/
|
||||
export class UpstreamCache {
|
||||
/** In-memory hot cache */
|
||||
private readonly memoryCache: Map<string, ICacheEntry> = new Map();
|
||||
|
||||
/** Configuration */
|
||||
private readonly config: IUpstreamCacheConfig;
|
||||
|
||||
/** Maximum in-memory cache entries */
|
||||
private readonly maxMemoryEntries: number;
|
||||
|
||||
/** S3 storage backend (optional) */
|
||||
private readonly storage?: IStorageBackend;
|
||||
|
||||
/** Cleanup interval handle */
|
||||
private cleanupInterval: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
constructor(
|
||||
config?: Partial<IUpstreamCacheConfig>,
|
||||
maxMemoryEntries: number = 10000,
|
||||
storage?: IStorageBackend
|
||||
) {
|
||||
this.config = { ...DEFAULT_CACHE_CONFIG, ...config };
|
||||
this.maxMemoryEntries = maxMemoryEntries;
|
||||
this.storage = storage;
|
||||
|
||||
// Start periodic cleanup if caching is enabled
|
||||
if (this.config.enabled) {
|
||||
this.startCleanup();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if caching is enabled.
|
||||
*/
|
||||
public isEnabled(): boolean {
|
||||
return this.config.enabled;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if S3 storage is configured.
|
||||
*/
|
||||
public hasStorage(): boolean {
|
||||
return !!this.storage;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get cached entry for a request context.
|
||||
* Checks memory first, then falls back to S3.
|
||||
* Returns null if not found or expired (unless stale-while-revalidate).
|
||||
*/
|
||||
public async get(context: IUpstreamFetchContext, upstreamUrl?: string): Promise<ICacheEntry | null> {
|
||||
if (!this.config.enabled) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const key = this.buildCacheKey(context, upstreamUrl);
|
||||
|
||||
// Check memory cache first
|
||||
let entry = this.memoryCache.get(key);
|
||||
|
||||
// If not in memory and we have storage, check S3
|
||||
if (!entry && this.storage) {
|
||||
entry = (await this.loadFromStorage(key)) ?? undefined;
|
||||
if (entry) {
|
||||
// Promote to memory cache
|
||||
this.memoryCache.set(key, entry);
|
||||
}
|
||||
}
|
||||
|
||||
if (!entry) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const now = new Date();
|
||||
|
||||
// Check if entry is expired
|
||||
if (entry.expiresAt && entry.expiresAt < now) {
|
||||
// Check if we can serve stale content
|
||||
if (this.config.staleWhileRevalidate && !entry.stale) {
|
||||
const staleAge = (now.getTime() - entry.expiresAt.getTime()) / 1000;
|
||||
if (staleAge <= this.config.staleMaxAgeSeconds) {
|
||||
// Mark as stale and return
|
||||
entry.stale = true;
|
||||
return entry;
|
||||
}
|
||||
}
|
||||
// Entry is too old, remove it
|
||||
this.memoryCache.delete(key);
|
||||
if (this.storage) {
|
||||
await this.deleteFromStorage(key).catch(() => {});
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
return entry;
|
||||
}
|
||||
|
||||
/**
|
||||
* Store a response in the cache (memory and optionally S3).
|
||||
*/
|
||||
public async set(
|
||||
context: IUpstreamFetchContext,
|
||||
data: Buffer,
|
||||
contentType: string,
|
||||
headers: Record<string, string>,
|
||||
upstreamId: string,
|
||||
upstreamUrl: string,
|
||||
options?: ICacheSetOptions,
|
||||
): Promise<void> {
|
||||
if (!this.config.enabled) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Enforce max memory entries limit
|
||||
if (this.memoryCache.size >= this.maxMemoryEntries) {
|
||||
this.evictOldest();
|
||||
}
|
||||
|
||||
const key = this.buildCacheKey(context, upstreamUrl);
|
||||
const now = new Date();
|
||||
|
||||
// Determine TTL based on content type
|
||||
const ttlSeconds = options?.ttlSeconds ?? this.determineTtl(context, contentType, headers);
|
||||
|
||||
const entry: ICacheEntry = {
|
||||
data,
|
||||
contentType,
|
||||
headers,
|
||||
cachedAt: now,
|
||||
expiresAt: ttlSeconds > 0 ? new Date(now.getTime() + ttlSeconds * 1000) : undefined,
|
||||
etag: headers['etag'] || options?.etag,
|
||||
upstreamId,
|
||||
stale: false,
|
||||
};
|
||||
|
||||
// Store in memory
|
||||
this.memoryCache.set(key, entry);
|
||||
|
||||
// Store in S3 if available
|
||||
if (this.storage) {
|
||||
await this.saveToStorage(key, entry, upstreamUrl).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Store a negative cache entry (404 response).
|
||||
*/
|
||||
public async setNegative(context: IUpstreamFetchContext, upstreamId: string, upstreamUrl: string): Promise<void> {
|
||||
if (!this.config.enabled || this.config.negativeCacheTtlSeconds <= 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const key = this.buildCacheKey(context, upstreamUrl);
|
||||
const now = new Date();
|
||||
|
||||
const entry: ICacheEntry = {
|
||||
data: Buffer.from(''),
|
||||
contentType: 'application/octet-stream',
|
||||
headers: {},
|
||||
cachedAt: now,
|
||||
expiresAt: new Date(now.getTime() + this.config.negativeCacheTtlSeconds * 1000),
|
||||
upstreamId,
|
||||
stale: false,
|
||||
};
|
||||
|
||||
this.memoryCache.set(key, entry);
|
||||
|
||||
if (this.storage) {
|
||||
await this.saveToStorage(key, entry, upstreamUrl).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if there's a negative cache entry for this context.
|
||||
*/
|
||||
public async hasNegative(context: IUpstreamFetchContext, upstreamUrl?: string): Promise<boolean> {
|
||||
const entry = await this.get(context, upstreamUrl);
|
||||
return entry !== null && entry.data.length === 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Invalidate a specific cache entry.
|
||||
*/
|
||||
public async invalidate(context: IUpstreamFetchContext, upstreamUrl?: string): Promise<boolean> {
|
||||
const key = this.buildCacheKey(context, upstreamUrl);
|
||||
const deleted = this.memoryCache.delete(key);
|
||||
|
||||
if (this.storage) {
|
||||
await this.deleteFromStorage(key).catch(() => {});
|
||||
}
|
||||
|
||||
return deleted;
|
||||
}
|
||||
|
||||
/**
|
||||
* Invalidate all entries matching a pattern.
|
||||
* Useful for invalidating all versions of a package.
|
||||
*/
|
||||
public async invalidatePattern(pattern: RegExp): Promise<number> {
|
||||
let count = 0;
|
||||
for (const key of this.memoryCache.keys()) {
|
||||
if (pattern.test(key)) {
|
||||
this.memoryCache.delete(key);
|
||||
if (this.storage) {
|
||||
await this.deleteFromStorage(key).catch(() => {});
|
||||
}
|
||||
count++;
|
||||
}
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
/**
|
||||
* Invalidate all entries from a specific upstream.
|
||||
*/
|
||||
public async invalidateUpstream(upstreamId: string): Promise<number> {
|
||||
let count = 0;
|
||||
for (const [key, entry] of this.memoryCache.entries()) {
|
||||
if (entry.upstreamId === upstreamId) {
|
||||
this.memoryCache.delete(key);
|
||||
if (this.storage) {
|
||||
await this.deleteFromStorage(key).catch(() => {});
|
||||
}
|
||||
count++;
|
||||
}
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all cache entries (memory and S3).
|
||||
*/
|
||||
public async clear(): Promise<void> {
|
||||
this.memoryCache.clear();
|
||||
|
||||
// Note: S3 cleanup would require listing and deleting all cache/* objects
|
||||
// This is left as a future enhancement for bulk cleanup
|
||||
}
|
||||
|
||||
/**
|
||||
* Get cache statistics.
|
||||
*/
|
||||
public getStats(): ICacheStats {
|
||||
let freshCount = 0;
|
||||
let staleCount = 0;
|
||||
let negativeCount = 0;
|
||||
let totalSize = 0;
|
||||
const now = new Date();
|
||||
|
||||
for (const entry of this.memoryCache.values()) {
|
||||
totalSize += entry.data.length;
|
||||
|
||||
if (entry.data.length === 0) {
|
||||
negativeCount++;
|
||||
} else if (entry.stale || (entry.expiresAt && entry.expiresAt < now)) {
|
||||
staleCount++;
|
||||
} else {
|
||||
freshCount++;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
totalEntries: this.memoryCache.size,
|
||||
freshEntries: freshCount,
|
||||
staleEntries: staleCount,
|
||||
negativeEntries: negativeCount,
|
||||
totalSizeBytes: totalSize,
|
||||
maxEntries: this.maxMemoryEntries,
|
||||
enabled: this.config.enabled,
|
||||
hasStorage: !!this.storage,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the cache and cleanup.
|
||||
*/
|
||||
public stop(): void {
|
||||
if (this.cleanupInterval) {
|
||||
clearInterval(this.cleanupInterval);
|
||||
this.cleanupInterval = null;
|
||||
}
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// Storage Methods
|
||||
// ========================================================================
|
||||
|
||||
/**
|
||||
* Build storage path for a cache key.
|
||||
* Escapes upstream URL for safe use in S3 paths.
|
||||
*/
|
||||
private buildStoragePath(key: string): string {
|
||||
return `cache/${key}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build storage path for cache metadata.
|
||||
*/
|
||||
private buildMetadataPath(key: string): string {
|
||||
return `cache/${key}.meta`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load a cache entry from S3 storage.
|
||||
*/
|
||||
private async loadFromStorage(key: string): Promise<ICacheEntry | null> {
|
||||
if (!this.storage) return null;
|
||||
|
||||
try {
|
||||
const dataPath = this.buildStoragePath(key);
|
||||
const metaPath = this.buildMetadataPath(key);
|
||||
|
||||
// Load data and metadata in parallel
|
||||
const [data, metaBuffer] = await Promise.all([
|
||||
this.storage.getObject(dataPath),
|
||||
this.storage.getObject(metaPath),
|
||||
]);
|
||||
|
||||
if (!data || !metaBuffer) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const meta: ICacheMetadata = JSON.parse(metaBuffer.toString('utf-8'));
|
||||
|
||||
return {
|
||||
data,
|
||||
contentType: meta.contentType,
|
||||
headers: meta.headers,
|
||||
cachedAt: new Date(meta.cachedAt),
|
||||
expiresAt: meta.expiresAt ? new Date(meta.expiresAt) : undefined,
|
||||
etag: meta.etag,
|
||||
upstreamId: meta.upstreamId,
|
||||
stale: false,
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save a cache entry to S3 storage.
|
||||
*/
|
||||
private async saveToStorage(key: string, entry: ICacheEntry, upstreamUrl: string): Promise<void> {
|
||||
if (!this.storage) return;
|
||||
|
||||
const dataPath = this.buildStoragePath(key);
|
||||
const metaPath = this.buildMetadataPath(key);
|
||||
|
||||
const meta: ICacheMetadata = {
|
||||
contentType: entry.contentType,
|
||||
headers: entry.headers,
|
||||
cachedAt: entry.cachedAt.toISOString(),
|
||||
expiresAt: entry.expiresAt?.toISOString(),
|
||||
etag: entry.etag,
|
||||
upstreamId: entry.upstreamId,
|
||||
upstreamUrl,
|
||||
};
|
||||
|
||||
// Save data and metadata in parallel
|
||||
await Promise.all([
|
||||
this.storage.putObject(dataPath, entry.data),
|
||||
this.storage.putObject(metaPath, Buffer.from(JSON.stringify(meta), 'utf-8')),
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a cache entry from S3 storage.
|
||||
*/
|
||||
private async deleteFromStorage(key: string): Promise<void> {
|
||||
if (!this.storage) return;
|
||||
|
||||
const dataPath = this.buildStoragePath(key);
|
||||
const metaPath = this.buildMetadataPath(key);
|
||||
|
||||
await Promise.all([
|
||||
this.storage.deleteObject(dataPath).catch(() => {}),
|
||||
this.storage.deleteObject(metaPath).catch(() => {}),
|
||||
]);
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// Helper Methods
|
||||
// ========================================================================
|
||||
|
||||
/**
|
||||
* Escape a URL for safe use in storage paths.
|
||||
*/
|
||||
private escapeUrl(url: string): string {
|
||||
// Remove protocol prefix and escape special characters
|
||||
return url
|
||||
.replace(/^https?:\/\//, '')
|
||||
.replace(/[\/\\:*?"<>|]/g, '_')
|
||||
.replace(/__+/g, '_');
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a unique cache key for a request context.
|
||||
* Includes escaped upstream URL for multi-upstream support.
|
||||
*/
|
||||
private buildCacheKey(context: IUpstreamFetchContext, upstreamUrl?: string): string {
|
||||
// Include method, protocol, path, and sorted query params
|
||||
const queryString = Object.keys(context.query)
|
||||
.sort()
|
||||
.map(k => `${k}=${context.query[k]}`)
|
||||
.join('&');
|
||||
|
||||
const baseKey = `${context.protocol}:${context.method}:${context.path}${queryString ? '?' + queryString : ''}`;
|
||||
|
||||
if (upstreamUrl) {
|
||||
return `${this.escapeUrl(upstreamUrl)}/${baseKey}`;
|
||||
}
|
||||
|
||||
return baseKey;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine TTL based on content characteristics.
|
||||
*/
|
||||
private determineTtl(
|
||||
context: IUpstreamFetchContext,
|
||||
contentType: string,
|
||||
headers: Record<string, string>,
|
||||
): number {
|
||||
// Check for Cache-Control header
|
||||
const cacheControl = headers['cache-control'];
|
||||
if (cacheControl) {
|
||||
const maxAgeMatch = cacheControl.match(/max-age=(\d+)/);
|
||||
if (maxAgeMatch) {
|
||||
return parseInt(maxAgeMatch[1], 10);
|
||||
}
|
||||
if (cacheControl.includes('no-store') || cacheControl.includes('no-cache')) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
// Check if content is immutable (content-addressable)
|
||||
if (this.isImmutableContent(context, contentType)) {
|
||||
return this.config.immutableTtlSeconds;
|
||||
}
|
||||
|
||||
// Default TTL for mutable content
|
||||
return this.config.defaultTtlSeconds;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if content is immutable (content-addressable).
|
||||
*/
|
||||
private isImmutableContent(context: IUpstreamFetchContext, contentType: string): boolean {
|
||||
// OCI blobs with digest are immutable
|
||||
if (context.protocol === 'oci' && context.resourceType === 'blob') {
|
||||
return true;
|
||||
}
|
||||
|
||||
// NPM tarballs are immutable (versioned)
|
||||
if (context.protocol === 'npm' && context.resourceType === 'tarball') {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Maven artifacts with version are immutable
|
||||
if (context.protocol === 'maven' && context.resourceType === 'artifact') {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Cargo crate files are immutable
|
||||
if (context.protocol === 'cargo' && context.resourceType === 'crate') {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Composer dist files are immutable
|
||||
if (context.protocol === 'composer' && context.resourceType === 'dist') {
|
||||
return true;
|
||||
}
|
||||
|
||||
// PyPI package files are immutable
|
||||
if (context.protocol === 'pypi' && context.resourceType === 'package') {
|
||||
return true;
|
||||
}
|
||||
|
||||
// RubyGems .gem files are immutable
|
||||
if (context.protocol === 'rubygems' && context.resourceType === 'gem') {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Evict oldest entries to make room for new ones.
|
||||
*/
|
||||
private evictOldest(): void {
|
||||
// Evict 10% of max entries
|
||||
const evictCount = Math.ceil(this.maxMemoryEntries * 0.1);
|
||||
let evicted = 0;
|
||||
|
||||
// First, try to evict stale entries
|
||||
const now = new Date();
|
||||
for (const [key, entry] of this.memoryCache.entries()) {
|
||||
if (evicted >= evictCount) break;
|
||||
if (entry.stale || (entry.expiresAt && entry.expiresAt < now)) {
|
||||
this.memoryCache.delete(key);
|
||||
evicted++;
|
||||
}
|
||||
}
|
||||
|
||||
// If not enough evicted, evict oldest by cachedAt
|
||||
if (evicted < evictCount) {
|
||||
const entries = Array.from(this.memoryCache.entries())
|
||||
.sort((a, b) => a[1].cachedAt.getTime() - b[1].cachedAt.getTime());
|
||||
|
||||
for (const [key] of entries) {
|
||||
if (evicted >= evictCount) break;
|
||||
this.memoryCache.delete(key);
|
||||
evicted++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start periodic cleanup of expired entries.
|
||||
*/
|
||||
private startCleanup(): void {
|
||||
// Run cleanup every minute
|
||||
this.cleanupInterval = setInterval(() => {
|
||||
this.cleanup();
|
||||
}, 60000);
|
||||
|
||||
// Don't keep the process alive just for cleanup
|
||||
if (this.cleanupInterval.unref) {
|
||||
this.cleanupInterval.unref();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove all expired entries from memory cache.
|
||||
*/
|
||||
private cleanup(): void {
|
||||
const now = new Date();
|
||||
const staleDeadline = new Date(now.getTime() - this.config.staleMaxAgeSeconds * 1000);
|
||||
|
||||
for (const [key, entry] of this.memoryCache.entries()) {
|
||||
if (entry.expiresAt) {
|
||||
// Remove if past stale deadline
|
||||
if (entry.expiresAt < staleDeadline) {
|
||||
this.memoryCache.delete(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Options for cache set operation.
|
||||
*/
|
||||
export interface ICacheSetOptions {
|
||||
/** Override TTL in seconds */
|
||||
ttlSeconds?: number;
|
||||
/** ETag for conditional requests */
|
||||
etag?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Cache statistics.
|
||||
*/
|
||||
export interface ICacheStats {
|
||||
/** Total number of cached entries in memory */
|
||||
totalEntries: number;
|
||||
/** Number of fresh (non-expired) entries */
|
||||
freshEntries: number;
|
||||
/** Number of stale entries (expired but still usable) */
|
||||
staleEntries: number;
|
||||
/** Number of negative cache entries */
|
||||
negativeEntries: number;
|
||||
/** Total size of cached data in bytes (memory only) */
|
||||
totalSizeBytes: number;
|
||||
/** Maximum allowed memory entries */
|
||||
maxEntries: number;
|
||||
/** Whether caching is enabled */
|
||||
enabled: boolean;
|
||||
/** Whether S3 storage is configured */
|
||||
hasStorage: boolean;
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
// Interfaces and types
|
||||
export * from './interfaces.upstream.js';
|
||||
|
||||
// Classes
|
||||
export { CircuitBreaker, CircuitOpenError, withCircuitBreaker } from './classes.circuitbreaker.js';
|
||||
export type { ICircuitBreakerMetrics } from './classes.circuitbreaker.js';
|
||||
|
||||
export { UpstreamCache } from './classes.upstreamcache.js';
|
||||
export type { ICacheSetOptions, ICacheStats } from './classes.upstreamcache.js';
|
||||
|
||||
export { BaseUpstream } from './classes.baseupstream.js';
|
||||
@@ -0,0 +1,276 @@
|
||||
import type { TRegistryProtocol, IRequestActor } from '../core/interfaces.core.js';
|
||||
|
||||
/**
|
||||
* Scope rule for routing requests to specific upstreams.
|
||||
* Uses glob patterns for flexible matching.
|
||||
*/
|
||||
export interface IUpstreamScopeRule {
|
||||
/** Glob pattern (e.g., "@company/*", "com.example.*", "library/*") */
|
||||
pattern: string;
|
||||
/** Whether matching resources should be included or excluded */
|
||||
action: 'include' | 'exclude';
|
||||
}
|
||||
|
||||
/**
|
||||
* Authentication configuration for an upstream registry.
|
||||
* Supports multiple auth strategies.
|
||||
*/
|
||||
export interface IUpstreamAuthConfig {
|
||||
/** Authentication type */
|
||||
type: 'none' | 'basic' | 'bearer' | 'api-key';
|
||||
/** Username for basic auth */
|
||||
username?: string;
|
||||
/** Password for basic auth */
|
||||
password?: string;
|
||||
/** Token for bearer or api-key auth */
|
||||
token?: string;
|
||||
/** Custom header name for api-key auth (default: 'Authorization') */
|
||||
headerName?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Cache configuration for upstream content.
|
||||
*/
|
||||
export interface IUpstreamCacheConfig {
|
||||
/** Whether caching is enabled */
|
||||
enabled: boolean;
|
||||
/** Default TTL in seconds for mutable content (default: 300 = 5 min) */
|
||||
defaultTtlSeconds: number;
|
||||
/** TTL in seconds for immutable/content-addressable content (default: 2592000 = 30 days) */
|
||||
immutableTtlSeconds: number;
|
||||
/** Whether to serve stale content while revalidating in background */
|
||||
staleWhileRevalidate: boolean;
|
||||
/** Maximum age in seconds for stale content (default: 3600 = 1 hour) */
|
||||
staleMaxAgeSeconds: number;
|
||||
/** TTL in seconds for negative cache entries (404s) (default: 60 = 1 min) */
|
||||
negativeCacheTtlSeconds: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resilience configuration for upstream requests.
|
||||
*/
|
||||
export interface IUpstreamResilienceConfig {
|
||||
/** Request timeout in milliseconds (default: 30000) */
|
||||
timeoutMs: number;
|
||||
/** Maximum number of retry attempts (default: 3) */
|
||||
maxRetries: number;
|
||||
/** Initial retry delay in milliseconds (default: 1000) */
|
||||
retryDelayMs: number;
|
||||
/** Maximum retry delay in milliseconds (default: 30000) */
|
||||
retryMaxDelayMs: number;
|
||||
/** Number of failures before circuit breaker opens (default: 5) */
|
||||
circuitBreakerThreshold: number;
|
||||
/** Time in milliseconds before circuit breaker attempts reset (default: 30000) */
|
||||
circuitBreakerResetMs: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Configuration for a single upstream registry.
|
||||
*/
|
||||
export interface IUpstreamRegistryConfig {
|
||||
/** Unique identifier for this upstream */
|
||||
id: string;
|
||||
/** Human-readable name */
|
||||
name: string;
|
||||
/** Base URL of the upstream registry (e.g., "https://registry.npmjs.org") */
|
||||
url: string;
|
||||
/** Priority for routing (lower = higher priority, 1 = first) */
|
||||
priority: number;
|
||||
/** Whether this upstream is enabled */
|
||||
enabled: boolean;
|
||||
/** Scope rules for routing (empty = match all) */
|
||||
scopeRules?: IUpstreamScopeRule[];
|
||||
/** Authentication configuration */
|
||||
auth: IUpstreamAuthConfig;
|
||||
/** Cache configuration overrides */
|
||||
cache?: Partial<IUpstreamCacheConfig>;
|
||||
/** Resilience configuration overrides */
|
||||
resilience?: Partial<IUpstreamResilienceConfig>;
|
||||
/** API path prefix for OCI registries (default: /v2). Useful for registries behind reverse proxies. */
|
||||
apiPrefix?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Protocol-level upstream configuration.
|
||||
* Configures upstream behavior for a specific protocol (npm, oci, etc.)
|
||||
*/
|
||||
export interface IProtocolUpstreamConfig {
|
||||
/** Whether upstream is enabled for this protocol */
|
||||
enabled: boolean;
|
||||
/** List of upstream registries, ordered by priority */
|
||||
upstreams: IUpstreamRegistryConfig[];
|
||||
/** Protocol-level cache configuration defaults */
|
||||
cache?: Partial<IUpstreamCacheConfig>;
|
||||
/** Protocol-level resilience configuration defaults */
|
||||
resilience?: Partial<IUpstreamResilienceConfig>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Result of an upstream fetch operation.
|
||||
*/
|
||||
export interface IUpstreamResult {
|
||||
/** Whether the fetch was successful (2xx status) */
|
||||
success: boolean;
|
||||
/** HTTP status code */
|
||||
status: number;
|
||||
/** Response headers */
|
||||
headers: Record<string, string>;
|
||||
/** Response body (Buffer for binary, object for JSON) */
|
||||
body?: Buffer | any;
|
||||
/** ID of the upstream that served the request */
|
||||
upstreamId: string;
|
||||
/** Whether the response was served from cache */
|
||||
fromCache: boolean;
|
||||
/** Request latency in milliseconds */
|
||||
latencyMs: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Circuit breaker state.
|
||||
*/
|
||||
export type TCircuitState = 'CLOSED' | 'OPEN' | 'HALF_OPEN';
|
||||
|
||||
/**
|
||||
* Context for an upstream fetch request.
|
||||
*/
|
||||
export interface IUpstreamFetchContext {
|
||||
/** Protocol type */
|
||||
protocol: TRegistryProtocol;
|
||||
/** Resource identifier (package name, artifact name, etc.) */
|
||||
resource: string;
|
||||
/** Type of resource being fetched (packument, tarball, manifest, blob, etc.) */
|
||||
resourceType: string;
|
||||
/** Original request path */
|
||||
path: string;
|
||||
/** HTTP method */
|
||||
method: string;
|
||||
/** Request headers */
|
||||
headers: Record<string, string>;
|
||||
/** Query parameters */
|
||||
query: Record<string, string>;
|
||||
/** Actor performing the request (for cache key isolation) */
|
||||
actor?: IRequestActor;
|
||||
}
|
||||
|
||||
/**
|
||||
* Cache entry stored in the upstream cache.
|
||||
*/
|
||||
export interface ICacheEntry {
|
||||
/** Cached data */
|
||||
data: Buffer;
|
||||
/** Content type of the cached data */
|
||||
contentType: string;
|
||||
/** Original response headers */
|
||||
headers: Record<string, string>;
|
||||
/** When the entry was cached */
|
||||
cachedAt: Date;
|
||||
/** When the entry expires */
|
||||
expiresAt?: Date;
|
||||
/** ETag for conditional requests */
|
||||
etag?: string;
|
||||
/** ID of the upstream that provided the data */
|
||||
upstreamId: string;
|
||||
/** Whether the entry is stale but still usable */
|
||||
stale?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Default cache configuration values.
|
||||
*/
|
||||
export const DEFAULT_CACHE_CONFIG: IUpstreamCacheConfig = {
|
||||
enabled: true,
|
||||
defaultTtlSeconds: 300, // 5 minutes
|
||||
immutableTtlSeconds: 2592000, // 30 days
|
||||
staleWhileRevalidate: true,
|
||||
staleMaxAgeSeconds: 3600, // 1 hour
|
||||
negativeCacheTtlSeconds: 60, // 1 minute
|
||||
};
|
||||
|
||||
/**
|
||||
* Default resilience configuration values.
|
||||
*/
|
||||
export const DEFAULT_RESILIENCE_CONFIG: IUpstreamResilienceConfig = {
|
||||
timeoutMs: 30000,
|
||||
maxRetries: 3,
|
||||
retryDelayMs: 1000,
|
||||
retryMaxDelayMs: 30000,
|
||||
circuitBreakerThreshold: 5,
|
||||
circuitBreakerResetMs: 30000,
|
||||
};
|
||||
|
||||
// ============================================================================
|
||||
// Upstream Provider Interfaces
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Context for resolving upstream configuration.
|
||||
* Passed to IUpstreamProvider per-request to enable dynamic upstream routing.
|
||||
*/
|
||||
export interface IUpstreamResolutionContext {
|
||||
/** Protocol being accessed */
|
||||
protocol: TRegistryProtocol;
|
||||
/** Resource identifier (package name, repository, coordinates, etc.) */
|
||||
resource: string;
|
||||
/** Extracted scope (e.g., "company" from "@company/pkg", "myorg" from "myorg/image") */
|
||||
scope: string | null;
|
||||
/** Actor performing the request */
|
||||
actor?: IRequestActor;
|
||||
/** HTTP method */
|
||||
method: string;
|
||||
/** Resource type (packument, tarball, manifest, blob, etc.) */
|
||||
resourceType: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Dynamic upstream configuration provider.
|
||||
* Implement this interface to provide per-request upstream routing
|
||||
* based on actor context (user, organization, etc.)
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* class OrgUpstreamProvider implements IUpstreamProvider {
|
||||
* constructor(private db: Database) {}
|
||||
*
|
||||
* async resolveUpstreamConfig(ctx: IUpstreamResolutionContext) {
|
||||
* if (ctx.actor?.orgId) {
|
||||
* const orgConfig = await this.db.getOrgUpstream(ctx.actor.orgId, ctx.protocol);
|
||||
* if (orgConfig) return orgConfig;
|
||||
* }
|
||||
* return this.db.getDefaultUpstream(ctx.protocol);
|
||||
* }
|
||||
* }
|
||||
* ```
|
||||
*/
|
||||
export interface IUpstreamProvider {
|
||||
/** Optional initialization */
|
||||
init?(): Promise<void>;
|
||||
|
||||
/**
|
||||
* Resolve upstream configuration for a request.
|
||||
* @param context - Information about the current request
|
||||
* @returns Upstream config to use, or null to skip upstream lookup
|
||||
*/
|
||||
resolveUpstreamConfig(context: IUpstreamResolutionContext): Promise<IProtocolUpstreamConfig | null>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Static upstream provider for simple configurations.
|
||||
* Use this when you have fixed upstream registries that don't change per-request.
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const provider = new StaticUpstreamProvider({
|
||||
* npm: {
|
||||
* enabled: true,
|
||||
* upstreams: [{ id: 'npmjs', url: 'https://registry.npmjs.org', priority: 1, enabled: true, auth: { type: 'none' } }],
|
||||
* },
|
||||
* });
|
||||
* ```
|
||||
*/
|
||||
export class StaticUpstreamProvider implements IUpstreamProvider {
|
||||
constructor(private configs: Partial<Record<TRegistryProtocol, IProtocolUpstreamConfig>>) {}
|
||||
|
||||
async resolveUpstreamConfig(ctx: IUpstreamResolutionContext): Promise<IProtocolUpstreamConfig | null> {
|
||||
return this.configs[ctx.protocol] ?? null;
|
||||
}
|
||||
}
|
||||
+1
-3
@@ -4,9 +4,7 @@
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"esModuleInterop": true,
|
||||
"verbatimModuleSyntax": true,
|
||||
"baseUrl": ".",
|
||||
"paths": {}
|
||||
"verbatimModuleSyntax": true
|
||||
},
|
||||
"exclude": ["dist_*/**/*.d.ts"]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user