26 Commits

Author SHA1 Message Date
e81fa41b18 v2.2.0
Some checks failed
Default (tags) / security (push) Successful in 39s
Default (tags) / test (push) Failing after 36s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-25 22:10:06 +00:00
41405eb40a feat(core/registrystorage): Persist OCI manifest content-type in sidecar and normalize manifest body handling 2025-11-25 22:10:06 +00:00
67188a4e9f v2.1.2
Some checks failed
Default (tags) / security (push) Successful in 39s
Default (tags) / test (push) Failing after 36s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-25 17:15:47 +00:00
a2f7f43027 fix(oci): Prefer raw request body for content-addressable OCI operations and expose rawBody on request context 2025-11-25 17:15:47 +00:00
37a89239d9 v2.1.1
Some checks failed
Default (tags) / security (push) Successful in 35s
Default (tags) / test (push) Failing after 36s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-25 16:59:37 +00:00
93fee289e7 fix(oci): Preserve raw manifest bytes for digest calculation and handle string/JSON manifest bodies in OCI registry 2025-11-25 16:59:37 +00:00
30fd9a4238 v2.1.0
Some checks failed
Default (tags) / security (push) Successful in 47s
Default (tags) / test (push) Failing after 48s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-25 16:48:08 +00:00
3b5bf5e789 feat(oci): Support configurable OCI token realm/service and centralize unauthorized responses 2025-11-25 16:48:08 +00:00
9b92e1c0d2 v2.0.0
Some checks failed
Default (tags) / security (push) Successful in 49s
Default (tags) / test (push) Failing after 50s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-25 15:07:59 +00:00
6291ebf79b BREAKING CHANGE(pypi,rubygems): Revise PyPI and RubyGems handling: normalize error payloads, fix .gem parsing/packing, adjust PyPI JSON API and tests, and export smartarchive plugin 2025-11-25 15:07:59 +00:00
fcd95677a0 v1.9.0
Some checks failed
Default (tags) / security (push) Successful in 51s
Default (tags) / test (push) Failing after 52s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-25 14:28:19 +00:00
547c262578 feat(auth): Implement HMAC-SHA256 OCI JWTs; enhance PyPI & RubyGems uploads and normalize responses 2025-11-25 14:28:19 +00:00
2d6059ba7f v1.8.0
Some checks failed
Default (tags) / security (push) Successful in 40s
Default (tags) / test (push) Failing after 37s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-24 00:15:29 +00:00
284329c191 feat(smarts3): Add local smarts3 testing support and documentation 2025-11-24 00:15:29 +00:00
4f662ff611 v1.7.0
Some checks failed
Default (tags) / security (push) Successful in 40s
Default (tags) / test (push) Failing after 36s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-23 23:54:42 +00:00
b3da95e6c1 feat(core): Standardize S3 storage config using @tsclass/tsclass IS3Descriptor and wire it into RegistryStorage and plugins exports; update README and package dependencies. 2025-11-23 23:54:41 +00:00
b1bb6af312 v1.6.0
Some checks failed
Default (tags) / security (push) Successful in 27s
Default (tags) / test (push) Failing after 36s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-21 17:13:06 +00:00
0d73230d5a feat(core): Add PyPI and RubyGems registries, integrate into SmartRegistry, extend storage and auth 2025-11-21 17:13:06 +00:00
ac51a94c8b v1.5.0
Some checks failed
Default (tags) / security (push) Successful in 40s
Default (tags) / test (push) Failing after 37s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-21 14:23:18 +00:00
9ca1e670ef feat(core): Add PyPI and RubyGems protocol support, Cargo token management, and storage helpers 2025-11-21 14:23:18 +00:00
fb8d6897e3 v1.4.1
Some checks failed
Default (tags) / security (push) Successful in 35s
Default (tags) / test (push) Failing after 35s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-21 09:36:02 +00:00
81ae4f2d59 fix(devcontainer): Simplify devcontainer configuration and rename container image 2025-11-21 09:36:02 +00:00
374469e37e v1.4.0
Some checks failed
Default (tags) / security (push) Successful in 39s
Default (tags) / test (push) Failing after 36s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-21 09:25:19 +00:00
9039613f7a feat(registrystorage): Add deleteMavenMetadata to RegistryStorage and update Maven DELETE test to expect 204 No Content 2025-11-21 09:25:19 +00:00
4d13fac9f1 v1.3.1
Some checks failed
Default (tags) / security (push) Successful in 25s
Default (tags) / test (push) Failing after 44s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-21 09:17:35 +00:00
42209d235d fix(maven): Pass request path to Maven checksum handler so checksum files are resolved correctly 2025-11-21 09:17:35 +00:00
34 changed files with 8406 additions and 213 deletions

View File

@@ -1,35 +1,5 @@
// The Dev Container format allows you to configure your environment. At the heart of it
// is a Docker image or Dockerfile which controls the tools available in your environment.
//
// See https://aka.ms/devcontainer.json for more information.
{
"name": "Ona",
// This universal image (~10GB) includes many development tools and languages,
// providing a convenient all-in-one development environment.
//
// This image is already available on remote runners for fast startup. On desktop
// and linux runners, it will need to be downloaded, which may take longer.
//
// For faster startup on desktop/linux, consider a smaller, language-specific image:
// • For Python: mcr.microsoft.com/devcontainers/python:3.13
// • For Node.js: mcr.microsoft.com/devcontainers/javascript-node:24
// • For Go: mcr.microsoft.com/devcontainers/go:1.24
// • For Java: mcr.microsoft.com/devcontainers/java:21
//
// Browse more options at: https://hub.docker.com/r/microsoft/devcontainers
// or build your own using the Dockerfile option below.
"name": "gitzone.universal",
"image": "mcr.microsoft.com/devcontainers/universal:4.0.1-noble"
// Use "build":
// instead of the image to use a Dockerfile to build an image.
// "build": {
// "context": ".",
// "dockerfile": "Dockerfile"
// }
// Features add additional features to your environment. See https://containers.dev/features
// Beware: features are not supported on all platforms and may have unintended side-effects.
// "features": {
// "ghcr.io/devcontainers/features/docker-in-docker": {
// "moby": false
// }
// }
}

View File

@@ -1,5 +1,116 @@
# Changelog
## 2025-11-25 - 2.2.0 - feat(core/registrystorage)
Persist OCI manifest content-type in sidecar and normalize manifest body handling
- Add getOciManifestContentType(repository, digest) to read stored manifest Content-Type
- Store manifest Content-Type in a .type sidecar file when putOciManifest is called
- Update putOciManifest to persist both manifest data and its content type
- OciRegistry now retrieves stored content type (with fallback to detectManifestContentType) when serving manifests
- Add toBuffer helper in OciRegistry to consistently convert various request body forms to Buffer for digest calculation and uploads
## 2025-11-25 - 2.1.2 - fix(oci)
Prefer raw request body for content-addressable OCI operations and expose rawBody on request context
- Add rawBody?: Buffer to IRequestContext to allow callers to provide the exact raw request bytes for digest calculation (falls back to body if absent).
- OCI registry handlers now prefer context.rawBody over context.body for content-addressable operations (manifests, blobs, and blob uploads) to preserve exact bytes and ensure digest calculation matches client expectations.
- Upload flow updates: upload init, PATCH (upload chunk) and PUT (complete upload) now pass rawBody when available.
## 2025-11-25 - 2.1.1 - fix(oci)
Preserve raw manifest bytes for digest calculation and handle string/JSON manifest bodies in OCI registry
- Preserve the exact bytes of the manifest payload when computing the sha256 digest to comply with the OCI spec and avoid mismatches caused by re-serialization.
- Accept string request bodies (converted using UTF-8) and treat already-parsed JSON objects by re-serializing as a fallback.
- Keep existing content-type fallback logic while ensuring accurate digest calculation prior to storing manifests.
## 2025-11-25 - 2.1.0 - feat(oci)
Support configurable OCI token realm/service and centralize unauthorized responses
- SmartRegistry now forwards optional ociTokens (realm and service) from auth configuration to OciRegistry when OCI is enabled
- OciRegistry constructor accepts an optional ociTokens parameter and stores it for use in auth headers
- Replaced repeated construction of WWW-Authenticate headers with createUnauthorizedResponse and createUnauthorizedHeadResponse helpers that use configured realm/service
- Behavior is backwards-compatible: when ociTokens are not configured the registry falls back to the previous defaults (realm: <basePath>/v2/token, service: "registry")
## 2025-11-25 - 2.0.0 - BREAKING CHANGE(pypi,rubygems)
Revise PyPI and RubyGems handling: normalize error payloads, fix .gem parsing/packing, adjust PyPI JSON API and tests, and export smartarchive plugin
- Rename error payload property from 'message' to 'error' in PyPI and RubyGems interfaces and responses; error responses are now returned as JSON objects (body: { error: ... }) instead of Buffer(JSON.stringify(...)).
- RubyGems: treat .gem files as plain tar archives (not gzipped). Use metadata.gz and data.tar.gz correctly, switch packing helper to pack plain tar, and use zlib deflate for .rz gemspec data.
- RubyGems registry: add legacy Marshal specs endpoint (specs.4.8.gz) and adjust versions handler invocation to accept request context.
- PyPI: adopt PEP 691 style (files is an array of file objects) in tests and metadata; include requires_python in test package metadata; update JSON API path matching to the package-level '/{package}/json' style used by the handler.
- Fix HTML escaping expectations in tests (requires_python values are HTML-escaped in attributes, e.g. '&gt;=3.8').
- Export smartarchive from plugins to enable archive helpers in core modules and helpers.
- Update tests and internal code to match the new error shape and API/format behaviour.
## 2025-11-25 - 1.9.0 - feat(auth)
Implement HMAC-SHA256 OCI JWTs; enhance PyPI & RubyGems uploads and normalize responses
- AuthManager: create and validate OCI JWTs signed with HMAC-SHA256 (header.payload.signature). Signature verification, exp/nbf checks and payload decoding implemented.
- PyPI: improved Simple API handling (PEP-691 JSON responses returned as objects), Simple HTML responses updated, upload handling enhanced to support nested/flat multipart fields, verify hashes (sha256/md5/blake2b), store files and return 201 on success.
- RubyGems: upload flow now attempts to extract gem metadata from the .gem binary when name/version are not provided, improved validation, and upload returns 201. Added extractGemMetadata helper.
- OCI: centralized 401 response creation (including proper WWW-Authenticate header) and HEAD behavior fixed to return no body per HTTP spec.
- SmartRegistry: use nullish coalescing for protocol basePath defaults to avoid falsy-value bugs when basePath is an empty string.
- Tests and helpers: test expectations adjusted (Content-Type startsWith check for HTML, PEP-691 projects is an array), test helper switched to smartarchive for packaging.
- Package.json: added devDependency @push.rocks/smartarchive and updated dev deps.
- Various response normalization: avoid unnecessary Buffer.from() for already-serialized objects/strings and standardize status codes for create/upload endpoints (201).
## 2025-11-24 - 1.8.0 - feat(smarts3)
Add local smarts3 testing support and documentation
- Added @push.rocks/smarts3 ^5.1.0 to devDependencies to enable a local S3-compatible test server.
- Updated README with a new "Testing with smarts3" section including a Quick Start example and integration test commands.
- Documented benefits and CI-friendly usage for running registry integration tests locally without cloud credentials.
## 2025-11-23 - 1.7.0 - feat(core)
Standardize S3 storage config using @tsclass/tsclass IS3Descriptor and wire it into RegistryStorage and plugins exports; update README and package dependencies.
- Add @tsclass/tsclass dependency to package.json to provide a standardized IS3Descriptor for S3 configuration.
- Export tsclass from ts/plugins.ts so plugin types are available to core modules.
- Update IStorageConfig to extend plugins.tsclass.storage.IS3Descriptor, consolidating storage configuration typing.
- Change RegistryStorage.init() to pass the storage config directly as an IS3Descriptor to SmartBucket (bucketName remains part of IStorageConfig).
- Update README storage section with example config and mention IS3Descriptor integration.
## 2025-11-21 - 1.6.0 - feat(core)
Add PyPI and RubyGems registries, integrate into SmartRegistry, extend storage and auth
- Introduce PyPI registry implementation with PEP 503 (Simple API) and PEP 691 (JSON API), legacy upload support, content negotiation and HTML/JSON generators (ts/pypi/*).
- Introduce RubyGems registry implementation with Compact Index support, API v1 endpoints (upload, yank/unyank), versions/names files and helpers (ts/rubygems/*).
- Wire PyPI and RubyGems into the main orchestrator: SmartRegistry now initializes, exposes and routes requests to pypi and rubygems handlers.
- Extend RegistryStorage with PyPI and RubyGems storage helpers (metadata, simple index, package files, compact index files, gem files).
- Extend AuthManager to support PyPI and RubyGems UUID token creation, validation and revocation and include them in unified token validation.
- Add verification of client-provided hashes during PyPI uploads (SHA256 always calculated and verified; MD5 and Blake2b verified when provided) to prevent corrupted uploads.
- Export new modules from library entry point (ts/index.ts) and add lightweight rubygems index file export.
- Add helper utilities for PyPI and RubyGems (name normalization, HTML generation, hash calculations, compact index generation/parsing).
- Update documentation hints/readme to reflect implementation status and configuration examples for pypi and rubygems.
## 2025-11-21 - 1.5.0 - feat(core)
Add PyPI and RubyGems protocol support, Cargo token management, and storage helpers
- Extend core protocol types to include 'pypi' and 'rubygems' and add protocol config entries for pypi and rubygems.
- Add PyPI storage methods for metadata, Simple API HTML/JSON indexes, package files, version listing and deletion in RegistryStorage.
- Add Cargo-specific storage helpers (index paths, crate storage) and ensure Cargo registry initialization and endpoints are wired into SmartRegistry.
- Extend AuthManager with Cargo, PyPI and RubyGems token creation, validation and revocation methods; update unified validateToken to check these token types.
- Update test helpers to create Cargo tokens and return cargoToken from registry setup.
## 2025-11-21 - 1.4.1 - fix(devcontainer)
Simplify devcontainer configuration and rename container image
- Rename Dev Container name to 'gitzone.universal' and set image to mcr.microsoft.com/devcontainers/universal:4.0.1-noble
- Remove large inline comments and example 'build'/'features' blocks to simplify the devcontainer.json
## 2025-11-21 - 1.4.0 - feat(registrystorage)
Add deleteMavenMetadata to RegistryStorage and update Maven DELETE test to expect 204 No Content
- Add deleteMavenMetadata(groupId, artifactId) to RegistryStorage to remove maven-metadata.xml.
- Update Maven test to assert 204 No Content for DELETE responses (previously expected 200).
## 2025-11-21 - 1.3.1 - fix(maven)
Pass request path to Maven checksum handler so checksum files are resolved correctly
- Call handleChecksumRequest with the full request path from MavenRegistry.handleRequest
- Allows getChecksum to extract the checksum filename from the URL and fetch the correct checksum file from storage
- Fixes 404s when requesting artifact checksum files (md5, sha1, sha256, sha512)
## 2025-11-21 - 1.3.0 - feat(core)
Add Cargo and Composer registries with storage, auth and helpers

View File

@@ -1,8 +1,8 @@
{
"name": "@push.rocks/smartregistry",
"version": "1.3.0",
"version": "2.2.0",
"private": false,
"description": "a registry for npm modules and oci images",
"description": "A composable TypeScript library implementing OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems registries for building unified container and package registries",
"main": "dist_ts/index.js",
"typings": "dist_ts/index.d.ts",
"type": "module",
@@ -18,6 +18,8 @@
"@git.zone/tsbundle": "^2.0.5",
"@git.zone/tsrun": "^2.0.0",
"@git.zone/tstest": "^3.1.0",
"@push.rocks/smartarchive": "^5.0.1",
"@push.rocks/smarts3": "^5.1.0",
"@types/node": "^24.10.1"
},
"repository": {
@@ -48,6 +50,7 @@
"@push.rocks/smartbucket": "^4.3.0",
"@push.rocks/smartlog": "^3.1.10",
"@push.rocks/smartpath": "^6.0.0",
"@tsclass/tsclass": "^9.3.0",
"adm-zip": "^0.5.10"
},
"packageManager": "pnpm@10.18.1+sha512.77a884a165cbba2d8d1c19e3b4880eee6d2fcabd0d879121e282196b80042351d5eb3ca0935fa599da1dc51265cc68816ad2bddd2a2de5ea9fdf92adbec7cd34"

114
pnpm-lock.yaml generated
View File

@@ -20,6 +20,9 @@ importers:
'@push.rocks/smartpath':
specifier: ^6.0.0
version: 6.0.0
'@tsclass/tsclass':
specifier: ^9.3.0
version: 9.3.0
adm-zip:
specifier: ^0.5.10
version: 0.5.16
@@ -36,6 +39,12 @@ importers:
'@git.zone/tstest':
specifier: ^3.1.0
version: 3.1.0(socks@2.8.7)(typescript@5.9.3)
'@push.rocks/smartarchive':
specifier: ^5.0.1
version: 5.0.1(@push.rocks/smartfs@1.1.0)
'@push.rocks/smarts3':
specifier: ^5.1.0
version: 5.1.0
'@types/node':
specifier: ^24.10.1
version: 24.10.1
@@ -573,7 +582,6 @@ packages:
'@koa/router@9.4.0':
resolution: {integrity: sha512-dOOXgzqaDoHu5qqMEPLKEgLz5CeIA7q8+1W62mCvFVCOqeC71UoTGJ4u1xUSOpIl2J1x2pqrNULkFteUeZW3/A==}
engines: {node: '>= 8.0.0'}
deprecated: '**IMPORTANT 10x+ PERFORMANCE UPGRADE**: Please upgrade to v12.0.1+ as we have fixed an issue with debuglog causing 10x slower router benchmark performance, see https://github.com/koajs/router/pull/173'
'@leichtgewicht/ip-codec@2.0.5':
resolution: {integrity: sha512-Vo+PSpZG2/fmgmiNzYK9qWRh8h/CHrwD0mo1h1DzL4yzHNSfWYujGTYsWGreD000gcgmZ7K4Ys6Tx9TxtsKdDw==}
@@ -700,6 +708,9 @@ packages:
'@push.rocks/smartarchive@4.2.2':
resolution: {integrity: sha512-6EpqbKU32D6Gcqsc9+Tn1dOCU5HoTlrqqs/7IdUr9Tirp9Ngtptkapca1Fw/D0kVJ7SSw3kG/miAYnuPMZLEoA==}
'@push.rocks/smartarchive@5.0.1':
resolution: {integrity: sha512-x4bie9IIdL9BZqBZLc8Pemp8xZOJGa6mXSVgKJRL4/Rw+E5N4rVHjQOYGRV75nC2mAMJh9GIbixuxLnWjj77ag==}
'@push.rocks/smartbrowser@2.0.8':
resolution: {integrity: sha512-0KWRZj3TuKo/sNwgPbiSE6WL+TMeR19t1JmXBZWh9n8iA2mpc4HhMrQAndEUdRCkx5ofSaHWojIRVFzGChj0Dg==}
@@ -760,6 +771,17 @@ packages:
'@push.rocks/smartfile@11.2.7':
resolution: {integrity: sha512-8Yp7/sAgPpWJBHohV92ogHWKzRomI5MEbSG6b5W2n18tqwfAmjMed0rQvsvGrSBlnEWCKgoOrYIIZbLO61+J0Q==}
'@push.rocks/smartfile@13.0.1':
resolution: {integrity: sha512-phtryDFtBYHo7R2H9V3Y7VeiYQU9YzKL140gKD3bTicBgXoIYrJ6+b3mbZunSO2yQt1Vy1AxCxYXrFE/K+4grw==}
peerDependencies:
'@push.rocks/smartfs': ^1.0.0
peerDependenciesMeta:
'@push.rocks/smartfs':
optional: true
'@push.rocks/smartfs@1.1.0':
resolution: {integrity: sha512-fg8JIjFUPPX5laRoBpTaGwhMfZ3Y8mFT4fUaW54Y4J/BfOBa/y0+rIFgvgvqcOZgkQlyZU+FIfL8Z6zezqxyTg==}
'@push.rocks/smartguard@3.1.0':
resolution: {integrity: sha512-J23q84f1O+TwFGmd4lrO9XLHUh2DaLXo9PN/9VmTWYzTkQDv5JehmifXVI0esophXcCIfbdIu6hbt7/aHlDF4A==}
@@ -847,6 +869,9 @@ packages:
'@push.rocks/smarts3@2.2.7':
resolution: {integrity: sha512-9ZXGMlmUL2Wd+YJO0xOB8KyqPf4V++fWJvTq4s76bnqEuaCr9OLfq6czhban+i4cD3ZdIjehfuHqctzjuLw8Jw==}
'@push.rocks/smarts3@5.1.0':
resolution: {integrity: sha512-jmoSaJkdWOWxiS5aiTXvE6+zS7n6+OZe1jxIOq3weX54tPmDCjpLLTl12rdgvvpDE1ai5ayftirWhLGk96hkaw==}
'@push.rocks/smartshell@3.3.0':
resolution: {integrity: sha512-m0w618H6YBs+vXGz1CgS4nPi5CUAnqRtckcS9/koGwfcIx1IpjqmiP47BoCTbdgcv0IPUxQVBG1IXTHPuZ8Z5g==}
@@ -1751,7 +1776,7 @@ packages:
engines: {node: '>=12'}
co@4.6.0:
resolution: {integrity: sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==}
resolution: {integrity: sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=}
engines: {iojs: '>= 1.0.0', node: '>= 0.12.0'}
color-convert@1.9.3:
@@ -1766,7 +1791,7 @@ packages:
engines: {node: '>=14.6'}
color-name@1.1.3:
resolution: {integrity: sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==}
resolution: {integrity: sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=}
color-name@1.1.4:
resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==}
@@ -1891,7 +1916,7 @@ packages:
engines: {node: '>=10'}
deep-equal@1.0.1:
resolution: {integrity: sha512-bHtC0iYvWhyaTzvV3CZgPeZQqCOBGyGsVV7v4eevpdkLHfiSrXUdBG+qAuSz4RI70sszvjQ1QSZ98An1yNwpSw==}
resolution: {integrity: sha1-9dJgKStmDghO/0zbyfCK0yR0SLU=}
deep-extend@0.6.0:
resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==}
@@ -1922,10 +1947,10 @@ packages:
engines: {node: '>=0.4.0'}
delegates@1.0.0:
resolution: {integrity: sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==}
resolution: {integrity: sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=}
depd@1.1.2:
resolution: {integrity: sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==}
resolution: {integrity: sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=}
engines: {node: '>= 0.6'}
depd@2.0.0:
@@ -1977,7 +2002,7 @@ packages:
resolution: {integrity: sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==}
encodeurl@1.0.2:
resolution: {integrity: sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==}
resolution: {integrity: sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=}
engines: {node: '>= 0.8'}
encodeurl@2.0.0:
@@ -2038,7 +2063,7 @@ packages:
resolution: {integrity: sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==}
escape-string-regexp@1.0.5:
resolution: {integrity: sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==}
resolution: {integrity: sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=}
engines: {node: '>=0.8.0'}
escape-string-regexp@5.0.0:
@@ -2199,7 +2224,7 @@ packages:
engines: {node: '>= 0.6'}
fresh@0.5.2:
resolution: {integrity: sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==}
resolution: {integrity: sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=}
engines: {node: '>= 0.6'}
fresh@2.0.0:
@@ -2288,7 +2313,7 @@ packages:
engines: {node: '>=18.0.0'}
has-flag@3.0.0:
resolution: {integrity: sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==}
resolution: {integrity: sha1-tdRU3CGZriJWmfNGfloH87lVuv0=}
engines: {node: '>=4'}
has-property-descriptors@1.0.2:
@@ -2364,7 +2389,7 @@ packages:
resolution: {integrity: sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==}
humanize-number@0.0.2:
resolution: {integrity: sha512-un3ZAcNQGI7RzaWGZzQDH47HETM4Wrj6z6E4TId8Yeq9w5ZKUVB1nrT2jwFheTUjEmqcgTjXDc959jum+ai1kQ==}
resolution: {integrity: sha1-EcCvakcWQ2M1iFiASPF5lUFInBg=}
iconv-lite@0.6.3:
resolution: {integrity: sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==}
@@ -2493,7 +2518,7 @@ packages:
resolution: {integrity: sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==}
jsonfile@4.0.0:
resolution: {integrity: sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg==}
resolution: {integrity: sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=}
jsonfile@6.2.0:
resolution: {integrity: sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg==}
@@ -2501,7 +2526,6 @@ packages:
keygrip@1.1.0:
resolution: {integrity: sha512-iYSchDJ+liQ8iwbSI2QqsQOvqv58eJCEanyJPJi+Khyu8smkcKSFUCbPwzFcL7YVtZ6eONjqRX/38caJ7QjRAQ==}
engines: {node: '>= 0.6'}
deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.
keyv@4.5.4:
resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==}
@@ -2683,7 +2707,7 @@ packages:
resolution: {integrity: sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg==}
media-typer@0.3.0:
resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==}
resolution: {integrity: sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=}
engines: {node: '>= 0.6'}
media-typer@1.1.0:
@@ -2698,7 +2722,7 @@ packages:
engines: {node: '>=18'}
methods@1.1.2:
resolution: {integrity: sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==}
resolution: {integrity: sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=}
engines: {node: '>= 0.6'}
micromark-core-commonmark@2.0.3:
@@ -2951,7 +2975,7 @@ packages:
resolution: {integrity: sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g==}
only@0.0.2:
resolution: {integrity: sha512-Fvw+Jemq5fjjyWz6CpKx6w9s7xxqo3+JCyM0WXWeCSOboZ8ABkyvP8ID4CZuChA/wxSx+XSJmdOm8rGVyJ1hdQ==}
resolution: {integrity: sha1-Kv3oTQPlC5qO3EROMGEKcCle37Q=}
open@8.4.2:
resolution: {integrity: sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ==}
@@ -3023,7 +3047,7 @@ packages:
engines: {node: '>= 0.8'}
passthrough-counter@1.0.0:
resolution: {integrity: sha512-Wy8PXTLqPAN0oEgBrlnsXPMww3SYJ44tQ8aVrGAI4h4JZYCS0oYqsPqtPR8OhJpv6qFbpbB7XAn0liKV7EXubA==}
resolution: {integrity: sha1-GWfZ5m2lcrXAI8eH2xEqOHqxZvo=}
path-exists@4.0.0:
resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==}
@@ -3349,10 +3373,10 @@ packages:
resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==}
stack-trace@0.0.10:
resolution: {integrity: sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==}
resolution: {integrity: sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA=}
statuses@1.5.0:
resolution: {integrity: sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==}
resolution: {integrity: sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=}
engines: {node: '>= 0.6'}
statuses@2.0.1:
@@ -3364,7 +3388,7 @@ packages:
engines: {node: '>= 0.8'}
streamsearch@0.1.2:
resolution: {integrity: sha512-jos8u++JKm0ARcSUTAZXOVC0mSox7Bhn6sBgty73P1f3JGf7yG2clTbBNHUdde/kdvP2FESam+vM6l8jBrNxHA==}
resolution: {integrity: sha1-gIudDlb8Jz2Am6VzOOkpkZoanxo=}
engines: {node: '>=0.8.0'}
streamx@2.23.0:
@@ -5255,6 +5279,27 @@ snapshots:
- react-native-b4a
- supports-color
'@push.rocks/smartarchive@5.0.1(@push.rocks/smartfs@1.1.0)':
dependencies:
'@push.rocks/smartdelay': 3.0.5
'@push.rocks/smartfile': 13.0.1(@push.rocks/smartfs@1.1.0)
'@push.rocks/smartpath': 6.0.0
'@push.rocks/smartpromise': 4.2.3
'@push.rocks/smartrequest': 4.4.2
'@push.rocks/smartrx': 3.0.10
'@push.rocks/smartstream': 3.2.5
'@push.rocks/smartunique': 3.0.9
'@push.rocks/smarturl': 3.1.0
'@types/tar-stream': 3.1.4
fflate: 0.8.2
file-type: 21.1.0
tar-stream: 3.1.7
transitivePeerDependencies:
- '@push.rocks/smartfs'
- bare-abort-controller
- react-native-b4a
- supports-color
'@push.rocks/smartbrowser@2.0.8(typescript@5.9.3)':
dependencies:
'@push.rocks/smartdelay': 3.0.5
@@ -5443,6 +5488,28 @@ snapshots:
glob: 11.1.0
js-yaml: 4.1.1
'@push.rocks/smartfile@13.0.1(@push.rocks/smartfs@1.1.0)':
dependencies:
'@push.rocks/lik': 6.2.2
'@push.rocks/smartdelay': 3.0.5
'@push.rocks/smartfile-interfaces': 1.0.7
'@push.rocks/smarthash': 3.2.6
'@push.rocks/smartjson': 5.2.0
'@push.rocks/smartmime': 2.0.4
'@push.rocks/smartpath': 6.0.0
'@push.rocks/smartpromise': 4.2.3
'@push.rocks/smartrequest': 4.4.2
'@push.rocks/smartstream': 3.2.5
'@types/js-yaml': 4.0.9
glob: 11.1.0
js-yaml: 4.1.1
optionalDependencies:
'@push.rocks/smartfs': 1.1.0
'@push.rocks/smartfs@1.1.0':
dependencies:
'@push.rocks/smartpath': 6.0.0
'@push.rocks/smartguard@3.1.0':
dependencies:
'@push.rocks/smartpromise': 4.2.3
@@ -5691,6 +5758,13 @@ snapshots:
- aws-crt
- supports-color
'@push.rocks/smarts3@5.1.0':
dependencies:
'@push.rocks/smartfs': 1.1.0
'@push.rocks/smartpath': 6.0.0
'@push.rocks/smartxml': 2.0.0
'@tsclass/tsclass': 9.3.0
'@push.rocks/smartshell@3.3.0':
dependencies:
'@push.rocks/smartdelay': 3.0.5

View File

@@ -1,3 +1,439 @@
# Project Readme Hints
# Project Implementation Notes
This is the initial readme hints file.
This file contains technical implementation details for PyPI and RubyGems protocols.
## Python (PyPI) Protocol Implementation ✅
### PEP 503: Simple Repository API (HTML-based)
**URL Structure:**
- Root: `/<base>/` - Lists all projects
- Project: `/<base>/<project>/` - Lists all files for a project
- All URLs MUST end with `/` (redirect if missing)
**Package Name Normalization:**
- Lowercase all characters
- Replace runs of `.`, `-`, `_` with single `-`
- Implementation: `re.sub(r"[-_.]+", "-", name).lower()`
**HTML Format:**
- Root: One anchor per project
- Project: One anchor per file
- Anchor text must match final filename
- Anchor href links to download URL
**Hash Fragments:**
Format: `#<hashname>=<hashvalue>`
- hashname: lowercase hash function name (recommend `sha256`)
- hashvalue: hex-encoded digest
**Data Attributes:**
- `data-gpg-sig`: `true`/`false` for GPG signature presence
- `data-requires-python`: PEP 345 requirement string (HTML-encode `<` as `&lt;`, `>` as `&gt;`)
### PEP 691: JSON-based Simple API
**Content Types:**
- `application/vnd.pypi.simple.v1+json` - JSON format
- `application/vnd.pypi.simple.v1+html` - HTML format
- `text/html` - Alias for HTML (backwards compat)
**Root Endpoint JSON:**
```json
{
"meta": {"api-version": "1.0"},
"projects": [{"name": "ProjectName"}]
}
```
**Project Endpoint JSON:**
```json
{
"name": "normalized-name",
"meta": {"api-version": "1.0"},
"files": [
{
"filename": "package-1.0-py3-none-any.whl",
"url": "https://example.com/path/to/file",
"hashes": {"sha256": "..."},
"requires-python": ">=3.7",
"dist-info-metadata": true | {"sha256": "..."},
"gpg-sig": true,
"yanked": false | "reason string"
}
]
}
```
**Content Negotiation:**
- Use `Accept` header for format selection
- Server responds with `Content-Type` header
- Support both JSON and HTML formats
### PyPI Upload API (Legacy /legacy/)
**Endpoint:**
- URL: `https://upload.pypi.org/legacy/`
- Method: `POST`
- Content-Type: `multipart/form-data`
**Required Form Fields:**
- `:action` = `file_upload`
- `protocol_version` = `1`
- `content` = Binary file data with filename
- `filetype` = `bdist_wheel` | `sdist`
- `pyversion` = Python tag (e.g., `py3`, `py2.py3`) or `source` for sdist
- `metadata_version` = Metadata standard version
- `name` = Package name
- `version` = Version string
**Hash Digest (one required):**
- `md5_digest`: urlsafe base64 without padding
- `sha256_digest`: hexadecimal
- `blake2_256_digest`: hexadecimal
**Optional Fields:**
- `attestations`: JSON array of attestation objects
- Any Core Metadata fields (lowercase, hyphens → underscores)
- Example: `Description-Content-Type``description_content_type`
**Authentication:**
- Username/password or API token in HTTP Basic Auth
- API tokens: username = `__token__`, password = token value
**Behavior:**
- First file uploaded creates the release
- Multiple files uploaded sequentially for same version
### PEP 694: Upload 2.0 API
**Status:** Draft (not yet required, legacy API still supported)
- Multi-step workflow with sessions
- Async upload support with resumption
- JSON-based API
- Standard HTTP auth (RFC 7235)
- Not implementing initially (legacy API sufficient)
---
## Ruby (RubyGems) Protocol Implementation ✅
### Compact Index Format
**Endpoints:**
- `/versions` - Master list of all gems and versions
- `/info/<RUBYGEM>` - Detailed info for specific gem
- `/names` - Simple list of gem names
**Authentication:**
- UUID tokens similar to NPM pattern
- API key in `Authorization` header
- Scope format: `rubygems:gem:{name}:{read|write|yank}`
### `/versions` File Format
**Structure:**
```
created_at: 2024-04-01T00:00:05Z
---
RUBYGEM [-]VERSION_PLATFORM[,VERSION_PLATFORM,...] MD5
```
**Details:**
- Metadata lines before `---` delimiter
- One line per gem with comma-separated versions
- `[-]` prefix indicates yanked version
- `MD5`: Checksum of corresponding `/info/<RUBYGEM>` file
- Append-only during month, recalculated monthly
### `/info/<RUBYGEM>` File Format
**Structure:**
```
---
VERSION[-PLATFORM] [DEPENDENCY[,DEPENDENCY,...]]|REQUIREMENT[,REQUIREMENT,...]
```
**Dependency Format:**
```
GEM:CONSTRAINT[&CONSTRAINT]
```
- Examples: `actionmailer:= 2.2.2`, `parser:>= 3.2.2.3`
- Operators: `=`, `>`, `<`, `>=`, `<=`, `~>`, `!=`
- Multiple constraints: `unicode-display_width:< 3.0&>= 2.4.0`
**Requirement Format:**
```
checksum:SHA256_HEX
ruby:CONSTRAINT
rubygems:CONSTRAINT
```
**Platform:**
- Default platform is `ruby`
- Non-default platforms: `VERSION-PLATFORM` (e.g., `3.2.1-arm64-darwin`)
**Yanked Gems:**
- Listed with `-` prefix in `/versions`
- Excluded entirely from `/info/<RUBYGEM>` file
### `/names` File Format
```
---
gemname1
gemname2
gemname3
```
### HTTP Range Support
**Headers:**
- `Range: bytes=#{start}-`: Request from byte position
- `If-None-Match`: ETag conditional request
- `Repr-Digest`: SHA256 checksum in response
**Caching Strategy:**
1. Store file with last byte position
2. Request range from last position
3. Append response to existing file
4. Verify SHA256 against `Repr-Digest`
### RubyGems Upload/Management API
**Upload Gem:**
- `POST /api/v1/gems`
- Binary `.gem` file in request body
- `Authorization` header with API key
**Yank Version:**
- `DELETE /api/v1/gems/yank`
- Parameters: `gem_name`, `version`
**Unyank Version:**
- `PUT /api/v1/gems/unyank`
- Parameters: `gem_name`, `version`
**Version Metadata:**
- `GET /api/v1/versions/<gem>.json`
- Returns JSON array of versions
**Dependencies:**
- `GET /api/v1/dependencies?gems=<comma-list>`
- Returns dependency information for resolution
---
## Implementation Details
### Completed Protocols
- ✅ OCI Distribution Spec v1.1
- ✅ NPM Registry API
- ✅ Maven Repository
- ✅ Cargo/crates.io Registry
- ✅ Composer/Packagist
- ✅ PyPI (Python Package Index) - PEP 503/691
- ✅ RubyGems - Compact Index
### Storage Paths
**PyPI:**
```
pypi/
├── simple/ # PEP 503 HTML files
│ ├── index.html # All packages list
│ └── {package}/index.html # Package versions list
├── packages/
│ └── {package}/{filename} # .whl and .tar.gz files
└── metadata/
└── {package}/metadata.json # Package metadata
```
**RubyGems:**
```
rubygems/
├── versions # Master versions file
├── info/{gemname} # Per-gem info files
├── names # All gem names
└── gems/{gemname}-{version}.gem # .gem files
```
### Authentication Pattern
Both protocols should follow the existing UUID token pattern used by NPM, Maven, Cargo, Composer:
```typescript
// AuthManager additions
createPypiToken(userId: string, readonly: boolean): string
validatePypiToken(token: string): ITokenInfo | null
revokePypiToken(token: string): boolean
createRubyGemsToken(userId: string, readonly: boolean): string
validateRubyGemsToken(token: string): ITokenInfo | null
revokeRubyGemsToken(token: string): boolean
```
### Scope Format
```
pypi:package:{name}:{read|write}
rubygems:gem:{name}:{read|write|yank}
```
### Common Patterns
1. **Package name normalization** - Critical for PyPI
2. **Checksum calculation** - SHA256 for both protocols
3. **Append-only files** - RubyGems compact index
4. **Content negotiation** - PyPI JSON vs HTML
5. **Multipart upload parsing** - PyPI file uploads
6. **Binary file handling** - Both protocols (.whl, .tar.gz, .gem)
---
## Key Differences from Existing Protocols
**PyPI vs NPM:**
- PyPI uses Simple API (HTML) + JSON API
- PyPI requires package name normalization
- PyPI uses multipart form data for uploads (not JSON)
- PyPI supports multiple file types per release (wheel + sdist)
**RubyGems vs Cargo:**
- RubyGems uses compact index (append-only text files)
- RubyGems uses checksums in index files (not just filenames)
- RubyGems has HTTP Range support for incremental updates
- RubyGems uses MD5 for index checksums, SHA256 for .gem files
---
## Testing Requirements
### PyPI Tests Must Cover:
- Package upload (wheel and sdist)
- Package name normalization
- Simple API HTML generation (PEP 503)
- JSON API responses (PEP 691)
- Content negotiation
- Hash calculation and verification
- Authentication (tokens)
- Multi-file releases
- Yanked packages
### RubyGems Tests Must Cover:
- Gem upload
- Compact index generation
- `/versions` file updates (append-only)
- `/info/<gem>` file generation
- `/names` file generation
- Checksum calculations (MD5 and SHA256)
- Platform-specific gems
- Yanking/unyanking
- HTTP Range requests
- Authentication (API keys)
---
## Security Considerations
1. **Package name validation** - Prevent path traversal
2. **File size limits** - Prevent DoS via large uploads
3. **Content-Type validation** - Verify file types
4. **Checksum verification** - Ensure file integrity
5. **Token scope enforcement** - Read vs write permissions
6. **HTML escaping** - Prevent XSS in generated HTML
7. **Metadata sanitization** - Clean user-provided strings
8. **Rate limiting** - Consider upload frequency limits
---
## Implementation Status (Completed)
### PyPI Implementation ✅
- **Files Created:**
- `ts/pypi/interfaces.pypi.ts` - Type definitions (354 lines)
- `ts/pypi/helpers.pypi.ts` - Helper functions (280 lines)
- `ts/pypi/classes.pypiregistry.ts` - Main registry (650 lines)
- `ts/pypi/index.ts` - Module exports
- **Features Implemented:**
- ✅ PEP 503 Simple API (HTML)
- ✅ PEP 691 JSON API
- ✅ Content negotiation (Accept header)
- ✅ Package name normalization
- ✅ File upload with multipart/form-data
- ✅ Hash verification (SHA256, MD5, Blake2b)
- ✅ Package metadata management
- ✅ JSON API endpoints (/pypi/{package}/json)
- ✅ Token-based authentication
- ✅ Scope-based permissions (read/write/delete)
- **Security Enhancements:**
- ✅ Hash verification on upload (validates client-provided hashes)
- ✅ Package name validation (regex check)
- ✅ HTML escaping in generated pages
- ✅ Permission checks on all mutating operations
### RubyGems Implementation ✅
- **Files Created:**
- `ts/rubygems/interfaces.rubygems.ts` - Type definitions (215 lines)
- `ts/rubygems/helpers.rubygems.ts` - Helper functions (350 lines)
- `ts/rubygems/classes.rubygemsregistry.ts` - Main registry (580 lines)
- `ts/rubygems/index.ts` - Module exports
- **Features Implemented:**
- ✅ Compact Index format (modern Bundler)
- ✅ /versions endpoint (all gems list)
- ✅ /info/{gem} endpoint (gem-specific metadata)
- ✅ /names endpoint (gem names list)
- ✅ Gem upload API
- ✅ Yank/unyank functionality
- ✅ Platform-specific gems support
- ✅ JSON API endpoints
- ✅ Legacy endpoints (specs.4.8.gz, Marshal.4.8)
- ✅ Token-based authentication
- ✅ Scope-based permissions
### Integration ✅
- **Core Updates:**
- ✅ Updated `IRegistryConfig` interface
- ✅ Updated `TRegistryProtocol` type
- ✅ Added authentication methods to `AuthManager`
- ✅ Added 30+ storage methods to `RegistryStorage`
- ✅ Updated `SmartRegistry` initialization and routing
- ✅ Module exports from `ts/index.ts`
- **Test Coverage:**
-`test/test.pypi.ts` - 25+ tests covering all PyPI endpoints
-`test/test.rubygems.ts` - 30+ tests covering all RubyGems endpoints
-`test/test.integration.pypi-rubygems.ts` - Integration tests
- ✅ Updated test helpers with PyPI and RubyGems support
### Known Limitations
1. **PyPI:**
- Does not implement legacy XML-RPC API
- No support for PGP signatures (data-gpg-sig always false)
- Metadata extraction from wheel files not implemented
2. **RubyGems:**
- Gem spec extraction from .gem files returns placeholder (Ruby Marshal parsing not implemented)
- Legacy Marshal endpoints return basic data only
- No support for gem dependencies resolution
### Configuration Example
```typescript
{
pypi: {
enabled: true,
basePath: '/pypi', // Also handles /simple
},
rubygems: {
enabled: true,
basePath: '/rubygems',
},
auth: {
pypiTokens: { enabled: true },
rubygemsTokens: { enabled: true },
}
}
```

394
readme.md
View File

@@ -1,6 +1,10 @@
# @push.rocks/smartregistry
> 🚀 A composable TypeScript library implementing **OCI Distribution Specification v1.1**, **NPM Registry API**, **Maven Repository**, **Cargo/crates.io Registry**, and **Composer/Packagist** for building unified container and package registries.
> 🚀 A composable TypeScript library implementing **OCI Distribution Specification v1.1**, **NPM Registry API**, **Maven Repository**, **Cargo/crates.io Registry**, **Composer/Packagist**, **PyPI (Python Package Index)**, and **RubyGems Registry** for building unified container and package registries.
## Issue Reporting and Security
For reporting bugs, issues, or security vulnerabilities, please visit [community.foss.global/](https://community.foss.global/). This is the central community hub for all issue reporting. Developers who want to sign a contribution agreement and go through identification can also get a [code.foss.global/](https://code.foss.global/) account to submit Pull Requests directly.
## ✨ Features
@@ -10,12 +14,14 @@
- **Maven Repository**: Java/JVM artifact management with POM support
- **Cargo/crates.io Registry**: Rust crate registry with sparse HTTP protocol
- **Composer/Packagist**: PHP package registry with Composer v2 protocol
- **PyPI (Python Package Index)**: Python package registry with PEP 503/691 support
- **RubyGems Registry**: Ruby gem registry with compact index protocol
### 🏗️ Unified Architecture
- **Composable Design**: Core infrastructure with protocol plugins
- **Shared Storage**: Cloud-agnostic S3-compatible backend ([@push.rocks/smartbucket](https://www.npmjs.com/package/@push.rocks/smartbucket))
- **Shared Storage**: Cloud-agnostic S3-compatible backend using [@push.rocks/smartbucket](https://www.npmjs.com/package/@push.rocks/smartbucket) with standardized `IS3Descriptor` from [@tsclass/tsclass](https://www.npmjs.com/package/@tsclass/tsclass)
- **Unified Authentication**: Scope-based permissions across all protocols
- **Path-based Routing**: `/oci/*` for containers, `/npm/*` for packages, `/maven/*` for Java artifacts, `/cargo/*` for Rust crates, `/composer/*` for PHP packages
- **Path-based Routing**: `/oci/*` for containers, `/npm/*` for packages, `/maven/*` for Java artifacts, `/cargo/*` for Rust crates, `/composer/*` for PHP packages, `/pypi/*` for Python packages, `/rubygems/*` for Ruby gems
### 🔐 Authentication & Authorization
- NPM UUID tokens for package operations
@@ -59,6 +65,23 @@
- ✅ Dependency resolution
- ✅ PSR-4/PSR-0 autoloading support
**PyPI Features:**
- ✅ PEP 503 Simple Repository API (HTML)
- ✅ PEP 691 JSON-based Simple API
- ✅ Package upload (wheel and sdist)
- ✅ Package name normalization
- ✅ Hash verification (SHA256, MD5, Blake2b)
- ✅ Content negotiation (JSON/HTML)
- ✅ Metadata API (JSON endpoints)
**RubyGems Features:**
- ✅ Compact Index protocol (modern Bundler)
- ✅ Gem publish/download (.gem files)
- ✅ Version yank/unyank
- ✅ Platform-specific gems
- ✅ Dependency resolution
- ✅ Legacy API compatibility
## 📥 Installation
```bash
@@ -114,6 +137,14 @@ const config: IRegistryConfig = {
enabled: true,
basePath: '/composer',
},
pypi: {
enabled: true,
basePath: '/pypi',
},
rubygems: {
enabled: true,
basePath: '/rubygems',
},
};
const registry = new SmartRegistry(config);
@@ -145,6 +176,11 @@ ts/
├── npm/ # NPM implementation
│ ├── classes.npmregistry.ts
│ └── interfaces.npm.ts
├── maven/ # Maven implementation
├── cargo/ # Cargo implementation
├── composer/ # Composer implementation
├── pypi/ # PyPI implementation
├── rubygems/ # RubyGems implementation
└── classes.smartregistry.ts # Main orchestrator
```
@@ -157,7 +193,12 @@ SmartRegistry (orchestrator)
Path-based routing
├─→ /oci/* → OciRegistry
─→ /npm/* → NpmRegistry
─→ /npm/* → NpmRegistry
├─→ /maven/* → MavenRegistry
├─→ /cargo/* → CargoRegistry
├─→ /composer/* → ComposerRegistry
├─→ /pypi/* → PypiRegistry
└─→ /rubygems/* → RubyGemsRegistry
Shared Storage & Auth
@@ -409,6 +450,171 @@ composer require vendor/package
composer update
```
### 🐍 PyPI Registry (Python Packages)
```typescript
// Get package index (PEP 503 HTML format)
const htmlIndex = await registry.handleRequest({
method: 'GET',
path: '/simple/requests/',
headers: { 'Accept': 'text/html' },
query: {},
});
// Get package index (PEP 691 JSON format)
const jsonIndex = await registry.handleRequest({
method: 'GET',
path: '/simple/requests/',
headers: { 'Accept': 'application/vnd.pypi.simple.v1+json' },
query: {},
});
// Upload a Python package (wheel or sdist)
const formData = new FormData();
formData.append(':action', 'file_upload');
formData.append('protocol_version', '1');
formData.append('name', 'my-package');
formData.append('version', '1.0.0');
formData.append('filetype', 'bdist_wheel');
formData.append('pyversion', 'py3');
formData.append('metadata_version', '2.1');
formData.append('sha256_digest', 'abc123...');
formData.append('content', packageFile, { filename: 'my_package-1.0.0-py3-none-any.whl' });
const upload = await registry.handleRequest({
method: 'POST',
path: '/pypi/legacy/',
headers: {
'Authorization': `Bearer <pypi-token>`,
'Content-Type': 'multipart/form-data',
},
query: {},
body: formData,
});
// Get package metadata (PyPI JSON API)
const metadata = await registry.handleRequest({
method: 'GET',
path: '/pypi/my-package/json',
headers: {},
query: {},
});
// Download a specific version
const download = await registry.handleRequest({
method: 'GET',
path: '/packages/my-package/my_package-1.0.0-py3-none-any.whl',
headers: {},
query: {},
});
```
**Using with pip:**
```bash
# Install from custom registry
pip install --index-url https://registry.example.com/simple/ my-package
# Upload to custom registry
python -m twine upload --repository-url https://registry.example.com/pypi/legacy/ dist/*
# Configure in pip.conf or pip.ini
[global]
index-url = https://registry.example.com/simple/
```
### 💎 RubyGems Registry (Ruby Gems)
```typescript
// Get versions file (compact index)
const versions = await registry.handleRequest({
method: 'GET',
path: '/rubygems/versions',
headers: {},
query: {},
});
// Get gem-specific info
const gemInfo = await registry.handleRequest({
method: 'GET',
path: '/rubygems/info/rails',
headers: {},
query: {},
});
// Get list of all gem names
const names = await registry.handleRequest({
method: 'GET',
path: '/rubygems/names',
headers: {},
query: {},
});
// Upload a gem file
const gemBuffer = await readFile('my-gem-1.0.0.gem');
const uploadGem = await registry.handleRequest({
method: 'POST',
path: '/rubygems/api/v1/gems',
headers: { 'Authorization': '<rubygems-api-key>' },
query: {},
body: gemBuffer,
});
// Yank a version (make unavailable for install)
const yank = await registry.handleRequest({
method: 'DELETE',
path: '/rubygems/api/v1/gems/yank',
headers: { 'Authorization': '<rubygems-api-key>' },
query: { gem_name: 'my-gem', version: '1.0.0' },
});
// Unyank a version
const unyank = await registry.handleRequest({
method: 'PUT',
path: '/rubygems/api/v1/gems/unyank',
headers: { 'Authorization': '<rubygems-api-key>' },
query: { gem_name: 'my-gem', version: '1.0.0' },
});
// Get gem version metadata
const versionMeta = await registry.handleRequest({
method: 'GET',
path: '/rubygems/api/v1/versions/rails.json',
headers: {},
query: {},
});
// Download gem file
const gemDownload = await registry.handleRequest({
method: 'GET',
path: '/rubygems/gems/rails-7.0.0.gem',
headers: {},
query: {},
});
```
**Using with Bundler:**
```ruby
# Gemfile
source 'https://registry.example.com/rubygems' do
gem 'my-gem'
gem 'rails'
end
```
```bash
# Install gems
bundle install
# Push gem to custom registry
gem push my-gem-1.0.0.gem --host https://registry.example.com/rubygems
# Configure gem source
gem sources --add https://registry.example.com/rubygems/
gem sources --remove https://rubygems.org/
```
### 🔐 Authentication
```typescript
@@ -446,15 +652,24 @@ const canWrite = await authManager.authorize(
### Storage Configuration
The storage configuration extends `IS3Descriptor` from `@tsclass/tsclass` for standardized S3 configuration:
```typescript
import type { IS3Descriptor } from '@tsclass/tsclass';
storage: IS3Descriptor & {
bucketName: string; // Bucket name for registry storage
}
// Example:
storage: {
accessKey: string; // S3 access key
accessSecret: string; // S3 secret key
endpoint: string; // S3 endpoint
endpoint: string; // S3 endpoint (e.g., 's3.amazonaws.com')
port?: number; // Default: 443
useSsl?: boolean; // Default: true
region?: string; // Default: 'us-east-1'
bucketName: string; // Bucket name
region?: string; // AWS region (e.g., 'us-east-1')
bucketName: string; // Bucket name for this registry
}
```
@@ -530,6 +745,20 @@ Unified storage abstraction for both OCI and NPM content.
- `getNpmTarball(name, version)` - Get tarball
- `putNpmTarball(name, version, data)` - Store tarball
**PyPI Methods:**
- `getPypiPackageMetadata(name)` - Get package metadata
- `putPypiPackageMetadata(name, data)` - Store package metadata
- `getPypiPackageFile(name, filename)` - Get package file
- `putPypiPackageFile(name, filename, data)` - Store package file
**RubyGems Methods:**
- `getRubyGemsVersions()` - Get versions index
- `putRubyGemsVersions(data)` - Store versions index
- `getRubyGemsInfo(gemName)` - Get gem info
- `putRubyGemsInfo(gemName, data)` - Store gem info
- `getRubyGem(gemName, version)` - Get .gem file
- `putRubyGem(gemName, version, data)` - Store .gem file
#### AuthManager
Unified authentication manager supporting both NPM and OCI authentication schemes.
@@ -607,11 +836,45 @@ Composer v2 repository API compliant implementation.
- `DELETE /packages/{vendor}/{package}` - Delete entire package
- `DELETE /packages/{vendor}/{package}/{version}` - Delete specific version
**Package Format:**
- ZIP archives with composer.json in root
- SHA-1 checksums for verification
- Version normalization (1.0.0 → 1.0.0.0)
- PSR-4/PSR-0 autoloading configuration
#### PypiRegistry
PyPI (Python Package Index) registry implementing PEP 503 and PEP 691.
**Endpoints:**
- `GET /simple/` - List all packages (HTML or JSON)
- `GET /simple/{package}/` - List package files (HTML or JSON)
- `POST /legacy/` - Upload package (multipart/form-data)
- `GET /pypi/{package}/json` - Package metadata API
- `GET /pypi/{package}/{version}/json` - Version-specific metadata
- `GET /packages/{package}/{filename}` - Download package file
**Features:**
- PEP 503 Simple Repository API (HTML)
- PEP 691 JSON-based Simple API
- Content negotiation via Accept header
- Package name normalization
- Hash verification (SHA256, MD5, Blake2b)
#### RubyGemsRegistry
RubyGems registry with compact index protocol for modern Bundler.
**Endpoints:**
- `GET /versions` - Master versions file (all gems)
- `GET /info/{gem}` - Gem-specific info file
- `GET /names` - List of all gem names
- `POST /api/v1/gems` - Upload gem file
- `DELETE /api/v1/gems/yank` - Yank (deprecate) version
- `PUT /api/v1/gems/unyank` - Unyank version
- `GET /api/v1/versions/{gem}.json` - Version metadata
- `GET /gems/{gem}-{version}.gem` - Download gem file
**Features:**
- Compact Index format (append-only text files)
- Platform-specific gems support
- Yank/unyank functionality
- Checksum calculations (MD5 for index, SHA256 for gems)
- Legacy Marshal API compatibility
## 🗄️ Storage Structure
@@ -651,11 +914,24 @@ bucket/
│ │ └── {p1}/{p2}/{name} # 4+ char (e.g., "se/rd/serde")
│ └── crates/
│ └── {name}/{name}-{version}.crate # Gzipped tar archives
── composer/
└── packages/
└── {vendor}/{package}/
├── metadata.json # All versions metadata
└── {reference}.zip # Package ZIP files
── composer/
└── packages/
└── {vendor}/{package}/
├── metadata.json # All versions metadata
└── {reference}.zip # Package ZIP files
├── pypi/
│ ├── simple/ # PEP 503 HTML files
│ │ ├── index.html # All packages list
│ │ └── {package}/index.html # Package versions list
│ ├── packages/
│ │ └── {package}/{filename} # .whl and .tar.gz files
│ └── metadata/
│ └── {package}/metadata.json # Package metadata
└── rubygems/
├── versions # Master versions file
├── info/{gemname} # Per-gem info files
├── names # All gem names
└── gems/{gemname}-{version}.gem # .gem files
```
## 🎯 Scope Format
@@ -685,6 +961,14 @@ Examples:
composer:package:vendor/package:read # Read Composer package
composer:package:*:write # Write any package
composer:*:*:* # Full Composer access
pypi:package:my-package:read # Read PyPI package
pypi:package:*:write # Write any package
pypi:*:*:* # Full PyPI access
rubygems:gem:rails:read # Read RubyGems gem
rubygems:gem:*:write # Write any gem
rubygems:*:*:* # Full RubyGems access
```
## 🔌 Integration Examples
@@ -740,6 +1024,82 @@ pnpm run build
pnpm test
```
## 🧪 Testing with smarts3
smartregistry works seamlessly with [@push.rocks/smarts3](https://code.foss.global/push.rocks/smarts3), a local S3-compatible server for testing. This allows you to test the registry without needing cloud credentials or external services.
### Quick Start with smarts3
```typescript
import { Smarts3 } from '@push.rocks/smarts3';
import { SmartRegistry } from '@push.rocks/smartregistry';
// Start local S3 server
const s3Server = await Smarts3.createAndStart({
server: { port: 3456 },
storage: { cleanSlate: true },
});
// Manually create IS3Descriptor matching smarts3 configuration
// Note: smarts3 v5.1.0 doesn't properly expose getS3Descriptor() yet
const s3Descriptor = {
endpoint: 'localhost',
port: 3456,
accessKey: 'test',
accessSecret: 'test',
useSsl: false,
region: 'us-east-1',
};
// Create registry with smarts3 configuration
const registry = new SmartRegistry({
storage: {
...s3Descriptor,
bucketName: 'my-test-registry',
},
auth: {
jwtSecret: 'test-secret',
tokenStore: 'memory',
npmTokens: { enabled: true },
ociTokens: {
enabled: true,
realm: 'https://auth.example.com/token',
service: 'my-registry',
},
},
npm: { enabled: true, basePath: '/npm' },
oci: { enabled: true, basePath: '/oci' },
pypi: { enabled: true, basePath: '/pypi' },
cargo: { enabled: true, basePath: '/cargo' },
});
await registry.init();
// Use registry...
// Your tests here
// Cleanup
await s3Server.stop();
```
### Benefits of Testing with smarts3
-**Zero Setup** - No cloud credentials or external services needed
-**Fast** - Local filesystem storage, no network latency
-**Isolated** - Clean slate per test run, no shared state
-**CI/CD Ready** - Works in automated pipelines without configuration
-**Full Compatibility** - Implements S3 API, works with IS3Descriptor
### Running Integration Tests
```bash
# Run smarts3 integration test
pnpm exec tstest test/test.integration.smarts3.node.ts --verbose
# Run all tests (includes smarts3)
pnpm test
```
## License and Legal Information
This repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository.

View File

@@ -1,12 +1,13 @@
import * as qenv from '@push.rocks/qenv';
import * as crypto from 'crypto';
import * as smartarchive from '@push.rocks/smartarchive';
import { SmartRegistry } from '../../ts/classes.smartregistry.js';
import type { IRegistryConfig } from '../../ts/core/interfaces.core.js';
const testQenv = new qenv.Qenv('./', './.nogit');
/**
* Create a test SmartRegistry instance with OCI, NPM, Maven, and Composer enabled
* Create a test SmartRegistry instance with all protocols enabled
*/
export async function createTestRegistry(): Promise<SmartRegistry> {
// Read S3 config from env.json
@@ -36,6 +37,12 @@ export async function createTestRegistry(): Promise<SmartRegistry> {
realm: 'https://auth.example.com/token',
service: 'test-registry',
},
pypiTokens: {
enabled: true,
},
rubygemsTokens: {
enabled: true,
},
},
oci: {
enabled: true,
@@ -53,6 +60,18 @@ export async function createTestRegistry(): Promise<SmartRegistry> {
enabled: true,
basePath: '/composer',
},
cargo: {
enabled: true,
basePath: '/cargo',
},
pypi: {
enabled: true,
basePath: '/pypi',
},
rubygems: {
enabled: true,
basePath: '/rubygems',
},
};
const registry = new SmartRegistry(config);
@@ -93,7 +112,16 @@ export async function createTestTokens(registry: SmartRegistry) {
// Create Composer token with full access
const composerToken = await authManager.createComposerToken(userId, false);
return { npmToken, ociToken, mavenToken, composerToken, userId };
// Create Cargo token with full access
const cargoToken = await authManager.createCargoToken(userId, false);
// Create PyPI token with full access
const pypiToken = await authManager.createPypiToken(userId, false);
// Create RubyGems token with full access
const rubygemsToken = await authManager.createRubyGemsToken(userId, false);
return { npmToken, ociToken, mavenToken, composerToken, cargoToken, pypiToken, rubygemsToken, userId };
}
/**
@@ -214,7 +242,7 @@ export function calculateMavenChecksums(data: Buffer) {
}
/**
* Helper to create a Composer package ZIP
* Helper to create a Composer package ZIP using smartarchive
*/
export async function createComposerZip(
vendorPackage: string,
@@ -225,8 +253,7 @@ export async function createComposerZip(
authors?: Array<{ name: string; email?: string }>;
}
): Promise<Buffer> {
const AdmZip = (await import('adm-zip')).default;
const zip = new AdmZip();
const zipTools = new smartarchive.ZipTools();
const composerJson = {
name: vendorPackage,
@@ -245,9 +272,6 @@ export async function createComposerZip(
},
};
// Add composer.json
zip.addFile('composer.json', Buffer.from(JSON.stringify(composerJson, null, 2), 'utf-8'));
// Add a test PHP file
const [vendor, pkg] = vendorPackage.split('/');
const namespace = `${vendor.charAt(0).toUpperCase() + vendor.slice(1)}\\${pkg.charAt(0).toUpperCase() + pkg.slice(1).replace(/-/g, '')}`;
@@ -263,10 +287,272 @@ class TestClass
}
`;
zip.addFile('src/TestClass.php', Buffer.from(testPhpContent, 'utf-8'));
const entries: smartarchive.IArchiveEntry[] = [
{
archivePath: 'composer.json',
content: Buffer.from(JSON.stringify(composerJson, null, 2), 'utf-8'),
},
{
archivePath: 'src/TestClass.php',
content: Buffer.from(testPhpContent, 'utf-8'),
},
{
archivePath: 'README.md',
content: Buffer.from(`# ${vendorPackage}\n\nTest package`, 'utf-8'),
},
];
// Add README
zip.addFile('README.md', Buffer.from(`# ${vendorPackage}\n\nTest package`, 'utf-8'));
return zip.toBuffer();
return zipTools.createZip(entries);
}
/**
* Helper to create a test Python wheel file (minimal ZIP structure) using smartarchive
*/
export async function createPythonWheel(
packageName: string,
version: string,
pyVersion: string = 'py3'
): Promise<Buffer> {
const zipTools = new smartarchive.ZipTools();
const normalizedName = packageName.replace(/-/g, '_');
const distInfoDir = `${normalizedName}-${version}.dist-info`;
// Create METADATA file
const metadata = `Metadata-Version: 2.1
Name: ${packageName}
Version: ${version}
Summary: Test Python package
Home-page: https://example.com
Author: Test Author
Author-email: test@example.com
License: MIT
Platform: UNKNOWN
Classifier: Programming Language :: Python :: 3
Requires-Python: >=3.7
Description-Content-Type: text/markdown
# ${packageName}
Test package for SmartRegistry
`;
// Create WHEEL file
const wheelContent = `Wheel-Version: 1.0
Generator: test 1.0.0
Root-Is-Purelib: true
Tag: ${pyVersion}-none-any
`;
// Create a simple Python module
const moduleContent = `"""${packageName} module"""
__version__ = "${version}"
def hello():
return "Hello from ${packageName}!"
`;
const entries: smartarchive.IArchiveEntry[] = [
{
archivePath: `${distInfoDir}/METADATA`,
content: Buffer.from(metadata, 'utf-8'),
},
{
archivePath: `${distInfoDir}/WHEEL`,
content: Buffer.from(wheelContent, 'utf-8'),
},
{
archivePath: `${distInfoDir}/RECORD`,
content: Buffer.from('', 'utf-8'),
},
{
archivePath: `${distInfoDir}/top_level.txt`,
content: Buffer.from(normalizedName, 'utf-8'),
},
{
archivePath: `${normalizedName}/__init__.py`,
content: Buffer.from(moduleContent, 'utf-8'),
},
];
return zipTools.createZip(entries);
}
/**
* Helper to create a test Python source distribution (sdist) using smartarchive
*/
export async function createPythonSdist(
packageName: string,
version: string
): Promise<Buffer> {
const tarTools = new smartarchive.TarTools();
const normalizedName = packageName.replace(/-/g, '_');
const dirPrefix = `${packageName}-${version}`;
// PKG-INFO
const pkgInfo = `Metadata-Version: 2.1
Name: ${packageName}
Version: ${version}
Summary: Test Python package
Home-page: https://example.com
Author: Test Author
Author-email: test@example.com
License: MIT
`;
// setup.py
const setupPy = `from setuptools import setup, find_packages
setup(
name="${packageName}",
version="${version}",
packages=find_packages(),
python_requires=">=3.7",
)
`;
// Module file
const moduleContent = `"""${packageName} module"""
__version__ = "${version}"
def hello():
return "Hello from ${packageName}!"
`;
const entries: smartarchive.IArchiveEntry[] = [
{
archivePath: `${dirPrefix}/PKG-INFO`,
content: Buffer.from(pkgInfo, 'utf-8'),
},
{
archivePath: `${dirPrefix}/setup.py`,
content: Buffer.from(setupPy, 'utf-8'),
},
{
archivePath: `${dirPrefix}/${normalizedName}/__init__.py`,
content: Buffer.from(moduleContent, 'utf-8'),
},
];
return tarTools.packFilesToTarGz(entries);
}
/**
* Helper to calculate PyPI file hashes
*/
export function calculatePypiHashes(data: Buffer) {
return {
md5: crypto.createHash('md5').update(data).digest('hex'),
sha256: crypto.createHash('sha256').update(data).digest('hex'),
blake2b: crypto.createHash('blake2b512').update(data).digest('hex'),
};
}
/**
* Helper to create a test RubyGem file (minimal tar.gz structure) using smartarchive
*/
export async function createRubyGem(
gemName: string,
version: string,
platform: string = 'ruby'
): Promise<Buffer> {
const tarTools = new smartarchive.TarTools();
const gzipTools = new smartarchive.GzipTools();
// Create metadata.gz (simplified)
const metadataYaml = `--- !ruby/object:Gem::Specification
name: ${gemName}
version: !ruby/object:Gem::Version
version: ${version}
platform: ${platform}
authors:
- Test Author
autorequire:
bindir: bin
cert_chain: []
date: ${new Date().toISOString().split('T')[0]}
dependencies: []
description: Test RubyGem
email: test@example.com
executables: []
extensions: []
extra_rdoc_files: []
files:
- lib/${gemName}.rb
homepage: https://example.com
licenses:
- MIT
metadata: {}
post_install_message:
rdoc_options: []
require_paths:
- lib
required_ruby_version: !ruby/object:Gem::Requirement
requirements:
- - ">="
- !ruby/object:Gem::Version
version: '2.7'
required_rubygems_version: !ruby/object:Gem::Requirement
requirements:
- - ">="
- !ruby/object:Gem::Version
version: '0'
requirements: []
rubygems_version: 3.0.0
signing_key:
specification_version: 4
summary: Test gem for SmartRegistry
test_files: []
`;
const metadataGz = await gzipTools.compress(Buffer.from(metadataYaml, 'utf-8'));
// Create data.tar.gz content
const libContent = `# ${gemName}
module ${gemName.charAt(0).toUpperCase() + gemName.slice(1).replace(/-/g, '')}
VERSION = "${version}"
def self.hello
"Hello from #{gemName}!"
end
end
`;
const dataEntries: smartarchive.IArchiveEntry[] = [
{
archivePath: `lib/${gemName}.rb`,
content: Buffer.from(libContent, 'utf-8'),
},
];
const dataTarGz = await tarTools.packFilesToTarGz(dataEntries);
// Create the outer gem (tar.gz containing metadata.gz and data.tar.gz)
const gemEntries: smartarchive.IArchiveEntry[] = [
{
archivePath: 'metadata.gz',
content: metadataGz,
},
{
archivePath: 'data.tar.gz',
content: dataTarGz,
},
];
// RubyGems .gem files are plain tar archives (NOT gzipped), containing metadata.gz and data.tar.gz
return tarTools.packFiles(gemEntries);
}
/**
* Helper to calculate RubyGems checksums
*/
export function calculateRubyGemsChecksums(data: Buffer) {
return {
md5: crypto.createHash('md5').update(data).digest('hex'),
sha256: crypto.createHash('sha256').update(data).digest('hex'),
};
}

View File

@@ -0,0 +1,475 @@
/**
* Native cargo CLI Testing
* Tests the Cargo registry implementation using the actual cargo CLI
*/
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { tapNodeTools } from '@git.zone/tstest/tapbundle_serverside';
import { SmartRegistry } from '../ts/index.js';
import { createTestRegistry, createTestTokens } from './helpers/registry.js';
import type { IRequestContext, IResponse } from '../ts/core/interfaces.core.js';
import * as http from 'http';
import * as url from 'url';
import * as fs from 'fs';
import * as path from 'path';
// Test context
let registry: SmartRegistry;
let server: http.Server;
let registryUrl: string;
let registryPort: number;
let cargoToken: string;
let testDir: string;
let cargoHome: string;
/**
* Create HTTP server wrapper around SmartRegistry
*/
async function createHttpServer(
registryInstance: SmartRegistry,
port: number
): Promise<{ server: http.Server; url: string }> {
return new Promise((resolve, reject) => {
const httpServer = http.createServer(async (req, res) => {
try {
// Parse request
const parsedUrl = url.parse(req.url || '', true);
const pathname = parsedUrl.pathname || '/';
const query = parsedUrl.query;
// Read body
const chunks: Buffer[] = [];
for await (const chunk of req) {
chunks.push(chunk);
}
const bodyBuffer = Buffer.concat(chunks);
// Parse body based on content type
let body: any;
if (bodyBuffer.length > 0) {
const contentType = req.headers['content-type'] || '';
if (contentType.includes('application/json')) {
try {
body = JSON.parse(bodyBuffer.toString('utf-8'));
} catch (error) {
body = bodyBuffer;
}
} else {
body = bodyBuffer;
}
}
// Convert to IRequestContext
const context: IRequestContext = {
method: req.method || 'GET',
path: pathname,
headers: req.headers as Record<string, string>,
query: query as Record<string, string>,
body: body,
};
// Handle request
const response: IResponse = await registryInstance.handleRequest(context);
// Convert IResponse to HTTP response
res.statusCode = response.status;
// Set headers
for (const [key, value] of Object.entries(response.headers || {})) {
res.setHeader(key, value);
}
// Send body
if (response.body) {
if (Buffer.isBuffer(response.body)) {
res.end(response.body);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
} else {
res.end();
}
} catch (error) {
console.error('Server error:', error);
res.statusCode = 500;
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify({ error: 'INTERNAL_ERROR', message: String(error) }));
}
});
httpServer.listen(port, () => {
const serverUrl = `http://localhost:${port}`;
resolve({ server: httpServer, url: serverUrl });
});
httpServer.on('error', reject);
});
}
/**
* Setup Cargo configuration
*/
function setupCargoConfig(registryUrlArg: string, token: string, cargoHomeArg: string): void {
const cargoConfigDir = path.join(cargoHomeArg, '.cargo');
fs.mkdirSync(cargoConfigDir, { recursive: true });
// Create config.toml with sparse protocol
const configContent = `[registries.test-registry]
index = "sparse+${registryUrlArg}/cargo/"
[source.crates-io]
replace-with = "test-registry"
[net]
retry = 0
`;
fs.writeFileSync(path.join(cargoConfigDir, 'config.toml'), configContent, 'utf-8');
// Create credentials.toml (Cargo uses plain token, no "Bearer" prefix)
const credentialsContent = `[registries.test-registry]
token = "${token}"
`;
fs.writeFileSync(path.join(cargoConfigDir, 'credentials.toml'), credentialsContent, 'utf-8');
}
/**
* Create a test Cargo crate
*/
function createTestCrate(
crateName: string,
version: string,
targetDir: string
): string {
const crateDir = path.join(targetDir, crateName);
fs.mkdirSync(crateDir, { recursive: true });
// Create Cargo.toml
const cargoToml = `[package]
name = "${crateName}"
version = "${version}"
edition = "2021"
description = "Test crate ${crateName}"
license = "MIT"
authors = ["Test Author <test@example.com>"]
[dependencies]
`;
fs.writeFileSync(path.join(crateDir, 'Cargo.toml'), cargoToml, 'utf-8');
// Create src directory
const srcDir = path.join(crateDir, 'src');
fs.mkdirSync(srcDir, { recursive: true });
// Create lib.rs
const libRs = `//! Test crate ${crateName}
/// Returns a greeting message
pub fn greet() -> String {
format!("Hello from {}@{}", "${crateName}", "${version}")
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_greet() {
let greeting = greet();
assert!(greeting.contains("${crateName}"));
}
}
`;
fs.writeFileSync(path.join(srcDir, 'lib.rs'), libRs, 'utf-8');
// Create README.md
const readme = `# ${crateName}
Test crate for SmartRegistry.
Version: ${version}
`;
fs.writeFileSync(path.join(crateDir, 'README.md'), readme, 'utf-8');
return crateDir;
}
/**
* Run cargo command with proper environment
*/
async function runCargoCommand(
command: string,
cwd: string,
includeToken: boolean = true
): Promise<{ stdout: string; stderr: string; exitCode: number }> {
// Prepare environment variables
// NOTE: Cargo converts registry name "test-registry" to "TEST_REGISTRY" for env vars
const envVars = [
`CARGO_HOME="${cargoHome}"`,
`CARGO_REGISTRIES_TEST_REGISTRY_INDEX="sparse+${registryUrl}/cargo/"`,
includeToken ? `CARGO_REGISTRIES_TEST_REGISTRY_TOKEN="${cargoToken}"` : '',
`CARGO_NET_RETRY="0"`,
].filter(Boolean).join(' ');
// Build command with cd to correct directory and environment variables
const fullCommand = `cd "${cwd}" && ${envVars} ${command}`;
try {
const result = await tapNodeTools.runCommand(fullCommand);
return {
stdout: result.stdout || '',
stderr: result.stderr || '',
exitCode: result.exitCode || 0,
};
} catch (error: any) {
return {
stdout: error.stdout || '',
stderr: error.stderr || String(error),
exitCode: error.exitCode || 1,
};
}
}
/**
* Cleanup test directory
*/
function cleanupTestDir(dir: string): void {
if (fs.existsSync(dir)) {
fs.rmSync(dir, { recursive: true, force: true });
}
}
// ========================================================================
// TESTS
// ========================================================================
tap.test('Cargo CLI: should setup registry and HTTP server', async () => {
// Create registry
registry = await createTestRegistry();
const tokens = await createTestTokens(registry);
cargoToken = tokens.cargoToken;
expect(registry).toBeInstanceOf(SmartRegistry);
expect(cargoToken).toBeTypeOf('string');
// Clean up any existing index from previous test runs
const storage = registry.getStorage();
try {
await storage.putCargoIndex('test-crate-cli', []);
} catch (error) {
// Ignore error if operation fails
}
// Use port 5000 (hardcoded in CargoRegistry default config)
// TODO: Once registryUrl is configurable, use dynamic port like npm test (35001)
registryPort = 5000;
const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server;
registryUrl = serverSetup.url;
expect(server).toBeDefined();
expect(registryUrl).toEqual(`http://localhost:${registryPort}`);
// Setup test directory
testDir = path.join(process.cwd(), '.nogit', 'test-cargo-cli');
cleanupTestDir(testDir);
fs.mkdirSync(testDir, { recursive: true });
// Setup CARGO_HOME
cargoHome = path.join(testDir, '.cargo-home');
fs.mkdirSync(cargoHome, { recursive: true });
// Setup Cargo config
setupCargoConfig(registryUrl, cargoToken, cargoHome);
expect(fs.existsSync(path.join(cargoHome, '.cargo', 'config.toml'))).toEqual(true);
expect(fs.existsSync(path.join(cargoHome, '.cargo', 'credentials.toml'))).toEqual(true);
});
tap.test('Cargo CLI: should verify server is responding', async () => {
// Check server is up by doing a direct HTTP request to the cargo index
const response = await fetch(`${registryUrl}/cargo/`);
expect(response.status).toBeGreaterThanOrEqual(200);
expect(response.status).toBeLessThan(500);
});
tap.test('Cargo CLI: should publish a crate', async () => {
const crateName = 'test-crate-cli';
const version = '0.1.0';
const crateDir = createTestCrate(crateName, version, testDir);
const result = await runCargoCommand('cargo publish --registry test-registry --allow-dirty', crateDir);
console.log('cargo publish output:', result.stdout);
console.log('cargo publish stderr:', result.stderr);
expect(result.exitCode).toEqual(0);
expect(result.stdout || result.stderr).toContain(crateName);
});
tap.test('Cargo CLI: should verify crate in index', async () => {
const crateName = 'test-crate-cli';
// Cargo uses a specific index structure
// For crate "test-crate-cli", the index path is based on the first characters
// 1 char: <name>
// 2 char: 2/<name>
// 3 char: 3/<first-char>/<name>
// 4+ char: <first-2-chars>/<second-2-chars>/<name>
// "test-crate-cli" is 14 chars, so it should be at: te/st/test-crate-cli
const indexPath = `/cargo/te/st/${crateName}`;
const response = await fetch(`${registryUrl}${indexPath}`);
expect(response.status).toEqual(200);
const indexData = await response.text();
console.log('Index data:', indexData);
// Index should contain JSON line with crate info
expect(indexData).toContain(crateName);
expect(indexData).toContain('0.1.0');
});
tap.test('Cargo CLI: should download published crate', async () => {
const crateName = 'test-crate-cli';
const version = '0.1.0';
// Cargo downloads crates from /cargo/api/v1/crates/{name}/{version}/download
const downloadPath = `/cargo/api/v1/crates/${crateName}/${version}/download`;
const response = await fetch(`${registryUrl}${downloadPath}`);
expect(response.status).toEqual(200);
const crateData = await response.arrayBuffer();
expect(crateData.byteLength).toBeGreaterThan(0);
});
tap.test('Cargo CLI: should publish second version', async () => {
const crateName = 'test-crate-cli';
const version = '0.2.0';
const crateDir = createTestCrate(crateName, version, testDir);
const result = await runCargoCommand('cargo publish --registry test-registry --allow-dirty', crateDir);
console.log('cargo publish v0.2.0 output:', result.stdout);
expect(result.exitCode).toEqual(0);
});
tap.test('Cargo CLI: should list versions in index', async () => {
const crateName = 'test-crate-cli';
const indexPath = `/cargo/te/st/${crateName}`;
const response = await fetch(`${registryUrl}${indexPath}`);
expect(response.status).toEqual(200);
const indexData = await response.text();
const lines = indexData.trim().split('\n');
// Should have 2 lines (2 versions)
expect(lines.length).toEqual(2);
// Parse JSON lines
const version1 = JSON.parse(lines[0]);
const version2 = JSON.parse(lines[1]);
expect(version1.vers).toEqual('0.1.0');
expect(version2.vers).toEqual('0.2.0');
});
tap.test('Cargo CLI: should search for crate', async () => {
const crateName = 'test-crate-cli';
// Cargo search endpoint: /cargo/api/v1/crates?q={query}
const response = await fetch(`${registryUrl}/cargo/api/v1/crates?q=${crateName}`);
expect(response.status).toEqual(200);
const searchResults = await response.json();
console.log('Search results:', searchResults);
expect(searchResults).toHaveProperty('crates');
expect(searchResults.crates).toBeInstanceOf(Array);
expect(searchResults.crates.length).toBeGreaterThan(0);
expect(searchResults.crates[0].name).toEqual(crateName);
});
tap.test('Cargo CLI: should yank a version', async () => {
const crateName = 'test-crate-cli';
const crateDir = path.join(testDir, crateName);
const result = await runCargoCommand('cargo yank --registry test-registry --vers 0.1.0', crateDir);
console.log('cargo yank output:', result.stdout);
console.log('cargo yank stderr:', result.stderr);
expect(result.exitCode).toEqual(0);
// Verify version is yanked in index
const indexPath = `/cargo/te/st/${crateName}`;
const response = await fetch(`${registryUrl}${indexPath}`);
const indexData = await response.text();
const lines = indexData.trim().split('\n');
const version1 = JSON.parse(lines[0]);
expect(version1.yanked).toEqual(true);
});
tap.test('Cargo CLI: should unyank a version', async () => {
const crateName = 'test-crate-cli';
const crateDir = path.join(testDir, crateName);
const result = await runCargoCommand('cargo yank --registry test-registry --vers 0.1.0 --undo', crateDir);
console.log('cargo unyank output:', result.stdout);
console.log('cargo unyank stderr:', result.stderr);
expect(result.exitCode).toEqual(0);
// Verify version is not yanked in index
const indexPath = `/cargo/te/st/${crateName}`;
const response = await fetch(`${registryUrl}${indexPath}`);
const indexData = await response.text();
const lines = indexData.trim().split('\n');
const version1 = JSON.parse(lines[0]);
expect(version1.yanked).toEqual(false);
});
tap.test('Cargo CLI: should fail to publish without auth', async () => {
const crateName = 'unauth-crate';
const version = '0.1.0';
const crateDir = createTestCrate(crateName, version, testDir);
// Run without token (includeToken: false)
const result = await runCargoCommand('cargo publish --registry test-registry --allow-dirty', crateDir, false);
console.log('cargo publish unauth output:', result.stdout);
console.log('cargo publish unauth stderr:', result.stderr);
// Should fail with auth error
expect(result.exitCode).not.toEqual(0);
expect(result.stderr).toContain('token');
});
tap.postTask('cleanup cargo cli tests', async () => {
// Stop server
if (server) {
await new Promise<void>((resolve) => {
server.close(() => resolve());
});
}
// Cleanup test directory
if (testDir) {
cleanupTestDir(testDir);
}
// Destroy registry
if (registry) {
registry.destroy();
}
});
export default tap.start();

View File

@@ -0,0 +1,288 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js';
import {
createTestRegistry,
createTestTokens,
createPythonWheel,
createRubyGem,
} from './helpers/registry.js';
let registry: SmartRegistry;
let pypiToken: string;
let rubygemsToken: string;
tap.test('Integration: should initialize registry with all protocols', async () => {
registry = await createTestRegistry();
const tokens = await createTestTokens(registry);
pypiToken = tokens.pypiToken;
rubygemsToken = tokens.rubygemsToken;
expect(registry).toBeInstanceOf(SmartRegistry);
expect(registry.isInitialized()).toEqual(true);
expect(pypiToken).toBeTypeOf('string');
expect(rubygemsToken).toBeTypeOf('string');
});
tap.test('Integration: should correctly route PyPI requests', async () => {
const wheelData = await createPythonWheel('integration-test-py', '1.0.0');
const response = await registry.handleRequest({
method: 'POST',
path: '/pypi/',
headers: {
Authorization: `Bearer ${pypiToken}`,
'Content-Type': 'multipart/form-data',
},
query: {},
body: {
':action': 'file_upload',
protocol_version: '1',
name: 'integration-test-py',
version: '1.0.0',
filetype: 'bdist_wheel',
pyversion: 'py3',
metadata_version: '2.1',
content: wheelData,
filename: 'integration_test_py-1.0.0-py3-none-any.whl',
},
});
expect(response.status).toEqual(201);
});
tap.test('Integration: should correctly route RubyGems requests', async () => {
const gemData = await createRubyGem('integration-test-gem', '1.0.0');
const response = await registry.handleRequest({
method: 'POST',
path: '/rubygems/api/v1/gems',
headers: {
Authorization: rubygemsToken,
'Content-Type': 'application/octet-stream',
},
query: {},
body: gemData,
});
expect(response.status).toEqual(201);
});
tap.test('Integration: should handle /simple path for PyPI', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/simple/',
headers: {
Accept: 'text/html',
},
query: {},
});
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toStartWith('text/html');
expect(response.body).toContain('integration-test-py');
});
tap.test('Integration: should reject PyPI token for RubyGems endpoint', async () => {
const gemData = await createRubyGem('unauthorized-gem', '1.0.0');
const response = await registry.handleRequest({
method: 'POST',
path: '/rubygems/api/v1/gems',
headers: {
Authorization: pypiToken, // Using PyPI token for RubyGems endpoint
'Content-Type': 'application/octet-stream',
},
query: {},
body: gemData,
});
expect(response.status).toEqual(401);
});
tap.test('Integration: should reject RubyGems token for PyPI endpoint', async () => {
const wheelData = await createPythonWheel('unauthorized-py', '1.0.0');
const response = await registry.handleRequest({
method: 'POST',
path: '/pypi/',
headers: {
Authorization: `Bearer ${rubygemsToken}`, // Using RubyGems token for PyPI endpoint
'Content-Type': 'multipart/form-data',
},
query: {},
body: {
':action': 'file_upload',
protocol_version: '1',
name: 'unauthorized-py',
version: '1.0.0',
filetype: 'bdist_wheel',
pyversion: 'py3',
metadata_version: '2.1',
content: wheelData,
filename: 'unauthorized_py-1.0.0-py3-none-any.whl',
},
});
expect(response.status).toEqual(401);
});
tap.test('Integration: should return 404 for unknown paths', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/unknown-protocol/endpoint',
headers: {},
query: {},
});
expect(response.status).toEqual(404);
expect(response.body).toHaveProperty('error');
expect((response.body as any).error).toEqual('NOT_FOUND');
});
tap.test('Integration: should retrieve PyPI registry instance', async () => {
const pypiRegistry = registry.getRegistry('pypi');
expect(pypiRegistry).toBeDefined();
expect(pypiRegistry).not.toBeNull();
});
tap.test('Integration: should retrieve RubyGems registry instance', async () => {
const rubygemsRegistry = registry.getRegistry('rubygems');
expect(rubygemsRegistry).toBeDefined();
expect(rubygemsRegistry).not.toBeNull();
});
tap.test('Integration: should retrieve all other protocol instances', async () => {
const ociRegistry = registry.getRegistry('oci');
const npmRegistry = registry.getRegistry('npm');
const mavenRegistry = registry.getRegistry('maven');
const composerRegistry = registry.getRegistry('composer');
const cargoRegistry = registry.getRegistry('cargo');
expect(ociRegistry).toBeDefined();
expect(npmRegistry).toBeDefined();
expect(mavenRegistry).toBeDefined();
expect(composerRegistry).toBeDefined();
expect(cargoRegistry).toBeDefined();
});
tap.test('Integration: should share storage across protocols', async () => {
const storage = registry.getStorage();
expect(storage).toBeDefined();
// Verify storage has methods for all protocols
expect(typeof storage.getPypiPackageMetadata).toEqual('function');
expect(typeof storage.getRubyGemsVersions).toEqual('function');
expect(typeof storage.getNpmPackument).toEqual('function');
expect(typeof storage.getOciBlob).toEqual('function');
});
tap.test('Integration: should share auth manager across protocols', async () => {
const authManager = registry.getAuthManager();
expect(authManager).toBeDefined();
// Verify auth manager has methods for all protocols
expect(typeof authManager.createPypiToken).toEqual('function');
expect(typeof authManager.createRubyGemsToken).toEqual('function');
expect(typeof authManager.createNpmToken).toEqual('function');
expect(typeof authManager.createOciToken).toEqual('function');
});
tap.test('Integration: should handle concurrent requests to different protocols', async () => {
const pypiRequest = registry.handleRequest({
method: 'GET',
path: '/simple/',
headers: {},
query: {},
});
const rubygemsRequest = registry.handleRequest({
method: 'GET',
path: '/rubygems/versions',
headers: {},
query: {},
});
const [pypiResponse, rubygemsResponse] = await Promise.all([pypiRequest, rubygemsRequest]);
expect(pypiResponse.status).toEqual(200);
expect(rubygemsResponse.status).toEqual(200);
});
tap.test('Integration: should handle package name conflicts across protocols', async () => {
const packageName = 'conflict-test';
// Upload PyPI package
const wheelData = await createPythonWheel(packageName, '1.0.0');
const pypiResponse = await registry.handleRequest({
method: 'POST',
path: '/pypi/',
headers: {
Authorization: `Bearer ${pypiToken}`,
'Content-Type': 'multipart/form-data',
},
query: {},
body: {
':action': 'file_upload',
protocol_version: '1',
name: packageName,
version: '1.0.0',
filetype: 'bdist_wheel',
pyversion: 'py3',
metadata_version: '2.1',
content: wheelData,
filename: `${packageName.replace(/-/g, '_')}-1.0.0-py3-none-any.whl`,
},
});
expect(pypiResponse.status).toEqual(201);
// Upload RubyGems package with same name
const gemData = await createRubyGem(packageName, '1.0.0');
const rubygemsResponse = await registry.handleRequest({
method: 'POST',
path: '/rubygems/api/v1/gems',
headers: {
Authorization: rubygemsToken,
'Content-Type': 'application/octet-stream',
},
query: {},
body: gemData,
});
expect(rubygemsResponse.status).toEqual(201);
// Both should exist independently
const pypiGetResponse = await registry.handleRequest({
method: 'GET',
path: `/simple/${packageName}/`,
headers: {},
query: {},
});
const rubygemsGetResponse = await registry.handleRequest({
method: 'GET',
path: `/rubygems/gems/${packageName}-1.0.0.gem`,
headers: {},
query: {},
});
expect(pypiGetResponse.status).toEqual(200);
expect(rubygemsGetResponse.status).toEqual(200);
});
tap.test('Integration: should properly clean up resources on destroy', async () => {
// Destroy should clean up all registries
expect(() => registry.destroy()).not.toThrow();
});
tap.postTask('cleanup registry', async () => {
if (registry && registry.isInitialized()) {
registry.destroy();
}
});
export default tap.start();

View File

@@ -0,0 +1,291 @@
/**
* Integration test for smartregistry with smarts3
* Verifies that smartregistry works with a local S3-compatible server
*/
import { expect, tap } from '@git.zone/tstest/tapbundle';
import * as smarts3Module from '@push.rocks/smarts3';
import { SmartRegistry } from '../ts/classes.smartregistry.js';
import type { IRegistryConfig } from '../ts/core/interfaces.core.js';
import * as crypto from 'crypto';
let s3Server: smarts3Module.Smarts3;
let registry: SmartRegistry;
/**
* Setup: Start smarts3 server
*/
tap.test('should start smarts3 server', async () => {
s3Server = await smarts3Module.Smarts3.createAndStart({
server: {
port: 3456, // Use different port to avoid conflicts with other tests
host: '0.0.0.0',
},
storage: {
cleanSlate: true, // Fresh storage for each test run
bucketsDir: './.nogit/smarts3-test-buckets',
},
logging: {
silent: true, // Reduce test output noise
},
});
expect(s3Server).toBeDefined();
});
/**
* Setup: Create SmartRegistry with smarts3 configuration
*/
tap.test('should create SmartRegistry instance with smarts3 IS3Descriptor', async () => {
// Manually construct IS3Descriptor based on smarts3 configuration
// Note: smarts3.getS3Descriptor() returns empty object as of v5.1.0
// This is a known limitation - smarts3 doesn't expose its config properly
const s3Descriptor = {
endpoint: 'localhost',
port: 3456,
accessKey: 'test', // smarts3 doesn't require real credentials
accessSecret: 'test',
useSsl: false,
region: 'us-east-1',
};
const config: IRegistryConfig = {
storage: {
...s3Descriptor,
bucketName: 'test-registry-smarts3',
},
auth: {
jwtSecret: 'test-secret-key',
tokenStore: 'memory',
npmTokens: {
enabled: true,
},
ociTokens: {
enabled: true,
realm: 'https://auth.example.com/token',
service: 'test-registry-smarts3',
},
pypiTokens: {
enabled: true,
},
rubygemsTokens: {
enabled: true,
},
},
npm: {
enabled: true,
basePath: '/npm',
},
oci: {
enabled: true,
basePath: '/oci',
},
pypi: {
enabled: true,
basePath: '/pypi',
},
cargo: {
enabled: true,
basePath: '/cargo',
},
};
registry = new SmartRegistry(config);
await registry.init();
expect(registry).toBeDefined();
});
/**
* Test NPM protocol with smarts3
*/
tap.test('NPM: should publish package to smarts3', async () => {
const authManager = registry.getAuthManager();
const userId = await authManager.authenticate({
username: 'testuser',
password: 'testpass',
});
const token = await authManager.createNpmToken(userId, false);
const packageData = {
name: 'test-package-smarts3',
'dist-tags': {
latest: '1.0.0',
},
versions: {
'1.0.0': {
name: 'test-package-smarts3',
version: '1.0.0',
description: 'Test package for smarts3 integration',
},
},
_attachments: {
'test-package-smarts3-1.0.0.tgz': {
content_type: 'application/octet-stream',
data: Buffer.from('test tarball content').toString('base64'),
length: 20,
},
},
};
const response = await registry.handleRequest({
method: 'PUT',
path: '/npm/test-package-smarts3',
headers: {
'Authorization': `Bearer ${token}`,
'Content-Type': 'application/json',
},
query: {},
body: packageData,
});
expect(response.status).toEqual(201); // 201 Created is correct for publishing
});
tap.test('NPM: should retrieve package from smarts3', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/npm/test-package-smarts3',
headers: {},
query: {},
});
expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('name');
expect(response.body.name).toEqual('test-package-smarts3');
});
/**
* Test OCI protocol with smarts3
*/
tap.test('OCI: should store blob in smarts3', async () => {
const authManager = registry.getAuthManager();
const userId = await authManager.authenticate({
username: 'testuser',
password: 'testpass',
});
const token = await authManager.createOciToken(
userId,
['oci:repository:test-image:push'],
3600
);
// Initiate blob upload
const initiateResponse = await registry.handleRequest({
method: 'POST',
path: '/oci/v2/test-image/blobs/uploads/',
headers: {
'Authorization': `Bearer ${token}`,
},
query: {},
});
expect(initiateResponse.status).toEqual(202);
expect(initiateResponse.headers).toHaveProperty('Location');
// Extract upload ID from location
const location = initiateResponse.headers['Location'];
const uploadId = location.split('/').pop();
// Upload blob data
const blobData = Buffer.from('test blob content');
const digest = 'sha256:' + crypto
.createHash('sha256')
.update(blobData)
.digest('hex');
const uploadResponse = await registry.handleRequest({
method: 'PUT',
path: `/oci/v2/test-image/blobs/uploads/${uploadId}`,
headers: {
'Authorization': `Bearer ${token}`,
'Content-Type': 'application/octet-stream',
},
query: { digest },
body: blobData,
});
expect(uploadResponse.status).toEqual(201);
});
/**
* Test PyPI protocol with smarts3
*/
tap.test('PyPI: should upload package to smarts3', async () => {
const authManager = registry.getAuthManager();
const userId = await authManager.authenticate({
username: 'testuser',
password: 'testpass',
});
const token = await authManager.createPypiToken(userId, false);
// Note: In a real test, this would be multipart/form-data
// For simplicity, we're testing the storage layer
const storage = registry.getStorage();
// Store a test package file
const packageContent = Buffer.from('test wheel content');
await storage.putPypiPackageFile(
'test-package',
'test_package-1.0.0-py3-none-any.whl',
packageContent
);
// Store metadata
const metadata = {
name: 'test-package',
version: '1.0.0',
files: [
{
filename: 'test_package-1.0.0-py3-none-any.whl',
url: '/packages/test-package/test_package-1.0.0-py3-none-any.whl',
hashes: { sha256: 'abc123' },
},
],
};
await storage.putPypiPackageMetadata('test-package', metadata);
// Verify stored
const retrievedMetadata = await storage.getPypiPackageMetadata('test-package');
expect(retrievedMetadata).toBeDefined();
expect(retrievedMetadata.name).toEqual('test-package');
});
/**
* Test Cargo protocol with smarts3
*/
tap.test('Cargo: should store crate in smarts3', async () => {
const storage = registry.getStorage();
// Store a test crate index entry
const indexEntry = {
name: 'test-crate',
vers: '1.0.0',
deps: [],
cksum: 'abc123',
features: {},
yanked: false,
};
await storage.putCargoIndex('test-crate', [indexEntry]);
// Store the actual .crate file
const crateContent = Buffer.from('test crate tarball');
await storage.putCargoCrate('test-crate', '1.0.0', crateContent);
// Verify stored
const retrievedIndex = await storage.getCargoIndex('test-crate');
expect(retrievedIndex).toBeDefined();
expect(retrievedIndex.length).toEqual(1);
expect(retrievedIndex[0].name).toEqual('test-crate');
});
/**
* Cleanup: Stop smarts3 server
*/
tap.test('should stop smarts3 server', async () => {
await s3Server.stop();
expect(true).toEqual(true); // Just verify it completes without error
});
export default tap.start();

View File

@@ -30,6 +30,14 @@ tap.test('Maven: should create registry instance', async () => {
expect(registry).toBeInstanceOf(SmartRegistry);
expect(mavenToken).toBeTypeOf('string');
// Clean up any existing metadata from previous test runs
const storage = registry.getStorage();
try {
await storage.deleteMavenMetadata(testGroupId, testArtifactId);
} catch (error) {
// Ignore error if metadata doesn't exist
}
});
tap.test('Maven: should upload POM file (PUT /{groupPath}/{artifactId}/{version}/*.pom)', async () => {
@@ -336,7 +344,7 @@ tap.test('Maven: should delete an artifact (DELETE)', async () => {
query: {},
});
expect(response.status).toEqual(200);
expect(response.status).toEqual(204); // 204 No Content is correct for DELETE
// Verify artifact was deleted
const getResponse = await registry.handleRequest({

View File

@@ -0,0 +1,412 @@
/**
* Native npm CLI Testing
* Tests the NPM registry implementation using the actual npm CLI
*/
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { tapNodeTools } from '@git.zone/tstest/tapbundle_serverside';
import { SmartRegistry } from '../ts/index.js';
import { createTestRegistry, createTestTokens } from './helpers/registry.js';
import type { IRequestContext, IResponse } from '../ts/core/interfaces.core.js';
import * as http from 'http';
import * as url from 'url';
import * as fs from 'fs';
import * as path from 'path';
// Test context
let registry: SmartRegistry;
let server: http.Server;
let registryUrl: string;
let registryPort: number;
let npmToken: string;
let testDir: string;
let npmrcPath: string;
/**
* Create HTTP server wrapper around SmartRegistry
*/
async function createHttpServer(
registryInstance: SmartRegistry,
port: number
): Promise<{ server: http.Server; url: string }> {
return new Promise((resolve, reject) => {
const httpServer = http.createServer(async (req, res) => {
try {
// Parse request
const parsedUrl = url.parse(req.url || '', true);
const pathname = parsedUrl.pathname || '/';
const query = parsedUrl.query;
// Read body
const chunks: Buffer[] = [];
for await (const chunk of req) {
chunks.push(chunk);
}
const bodyBuffer = Buffer.concat(chunks);
// Parse body based on content type
let body: any;
if (bodyBuffer.length > 0) {
const contentType = req.headers['content-type'] || '';
if (contentType.includes('application/json')) {
try {
body = JSON.parse(bodyBuffer.toString('utf-8'));
} catch (error) {
body = bodyBuffer;
}
} else {
body = bodyBuffer;
}
}
// Convert to IRequestContext
const context: IRequestContext = {
method: req.method || 'GET',
path: pathname,
headers: req.headers as Record<string, string>,
query: query as Record<string, string>,
body: body,
};
// Handle request
const response: IResponse = await registryInstance.handleRequest(context);
// Convert IResponse to HTTP response
res.statusCode = response.status;
// Set headers
for (const [key, value] of Object.entries(response.headers || {})) {
res.setHeader(key, value);
}
// Send body
if (response.body) {
if (Buffer.isBuffer(response.body)) {
res.end(response.body);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
} else {
res.end();
}
} catch (error) {
console.error('Server error:', error);
res.statusCode = 500;
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify({ error: 'INTERNAL_ERROR', message: String(error) }));
}
});
httpServer.listen(port, () => {
const serverUrl = `http://localhost:${port}`;
resolve({ server: httpServer, url: serverUrl });
});
httpServer.on('error', reject);
});
}
/**
* Setup .npmrc configuration
*/
function setupNpmrc(registryUrlArg: string, token: string, testDirArg: string): string {
const npmrcContent = `registry=${registryUrlArg}/npm/
//localhost:${registryPort}/npm/:_authToken=${token}
`;
const npmrcFilePath = path.join(testDirArg, '.npmrc');
fs.writeFileSync(npmrcFilePath, npmrcContent, 'utf-8');
return npmrcFilePath;
}
/**
* Create a test package
*/
function createTestPackage(
packageName: string,
version: string,
targetDir: string
): string {
const packageDir = path.join(targetDir, packageName);
fs.mkdirSync(packageDir, { recursive: true });
// Create package.json
const packageJson = {
name: packageName,
version: version,
description: `Test package ${packageName}`,
main: 'index.js',
scripts: {
test: 'echo "Test passed"',
},
keywords: ['test'],
author: 'Test Author',
license: 'MIT',
};
fs.writeFileSync(
path.join(packageDir, 'package.json'),
JSON.stringify(packageJson, null, 2),
'utf-8'
);
// Create index.js
const indexJs = `module.exports = {
name: '${packageName}',
version: '${version}',
message: 'Hello from ${packageName}@${version}'
};
`;
fs.writeFileSync(path.join(packageDir, 'index.js'), indexJs, 'utf-8');
// Create README.md
const readme = `# ${packageName}
Test package for SmartRegistry.
Version: ${version}
`;
fs.writeFileSync(path.join(packageDir, 'README.md'), readme, 'utf-8');
return packageDir;
}
/**
* Run npm command with proper environment
*/
async function runNpmCommand(
command: string,
cwd: string
): Promise<{ stdout: string; stderr: string; exitCode: number }> {
// Prepare environment variables
const envVars = [
`NPM_CONFIG_USERCONFIG="${npmrcPath}"`,
`NPM_CONFIG_CACHE="${path.join(testDir, '.npm-cache')}"`,
`NPM_CONFIG_PREFIX="${path.join(testDir, '.npm-global')}"`,
`NPM_CONFIG_REGISTRY="${registryUrl}/npm/"`,
].join(' ');
// Build command with cd to correct directory and environment variables
const fullCommand = `cd "${cwd}" && ${envVars} ${command}`;
try {
const result = await tapNodeTools.runCommand(fullCommand);
return {
stdout: result.stdout || '',
stderr: result.stderr || '',
exitCode: result.exitCode || 0,
};
} catch (error: any) {
return {
stdout: error.stdout || '',
stderr: error.stderr || String(error),
exitCode: error.exitCode || 1,
};
}
}
/**
* Cleanup test directory
*/
function cleanupTestDir(dir: string): void {
if (fs.existsSync(dir)) {
fs.rmSync(dir, { recursive: true, force: true });
}
}
// ========================================================================
// TESTS
// ========================================================================
tap.test('NPM CLI: should setup registry and HTTP server', async () => {
// Create registry
registry = await createTestRegistry();
const tokens = await createTestTokens(registry);
npmToken = tokens.npmToken;
expect(registry).toBeInstanceOf(SmartRegistry);
expect(npmToken).toBeTypeOf('string');
// Find available port
registryPort = 35000;
const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server;
registryUrl = serverSetup.url;
expect(server).toBeDefined();
expect(registryUrl).toEqual(`http://localhost:${registryPort}`);
// Setup test directory
testDir = path.join(process.cwd(), '.nogit', 'test-npm-cli');
cleanupTestDir(testDir);
fs.mkdirSync(testDir, { recursive: true });
// Setup .npmrc
npmrcPath = setupNpmrc(registryUrl, npmToken, testDir);
expect(fs.existsSync(npmrcPath)).toEqual(true);
});
tap.test('NPM CLI: should verify server is responding', async () => {
const result = await runNpmCommand('npm ping', testDir);
console.log('npm ping output:', result.stdout, result.stderr);
// npm ping may not work with custom registries, so just check server is up
// by doing a direct HTTP request
const response = await fetch(`${registryUrl}/npm/`);
expect(response.status).toBeGreaterThanOrEqual(200);
expect(response.status).toBeLessThan(500);
});
tap.test('NPM CLI: should publish a package', async () => {
const packageName = 'test-package-cli';
const version = '1.0.0';
const packageDir = createTestPackage(packageName, version, testDir);
const result = await runNpmCommand('npm publish', packageDir);
console.log('npm publish output:', result.stdout);
console.log('npm publish stderr:', result.stderr);
expect(result.exitCode).toEqual(0);
expect(result.stdout || result.stderr).toContain(packageName);
});
tap.test('NPM CLI: should view published package', async () => {
const packageName = 'test-package-cli';
const result = await runNpmCommand(`npm view ${packageName}`, testDir);
console.log('npm view output:', result.stdout);
expect(result.exitCode).toEqual(0);
expect(result.stdout).toContain(packageName);
expect(result.stdout).toContain('1.0.0');
});
tap.test('NPM CLI: should install published package', async () => {
const packageName = 'test-package-cli';
const installDir = path.join(testDir, 'install-test');
fs.mkdirSync(installDir, { recursive: true });
// Create package.json for installation
const packageJson = {
name: 'install-test',
version: '1.0.0',
dependencies: {
[packageName]: '1.0.0',
},
};
fs.writeFileSync(
path.join(installDir, 'package.json'),
JSON.stringify(packageJson, null, 2),
'utf-8'
);
const result = await runNpmCommand('npm install', installDir);
console.log('npm install output:', result.stdout);
console.log('npm install stderr:', result.stderr);
expect(result.exitCode).toEqual(0);
// Verify package was installed
const nodeModulesPath = path.join(installDir, 'node_modules', packageName);
expect(fs.existsSync(nodeModulesPath)).toEqual(true);
expect(fs.existsSync(path.join(nodeModulesPath, 'package.json'))).toEqual(true);
expect(fs.existsSync(path.join(nodeModulesPath, 'index.js'))).toEqual(true);
// Verify package contents
const installedPackageJson = JSON.parse(
fs.readFileSync(path.join(nodeModulesPath, 'package.json'), 'utf-8')
);
expect(installedPackageJson.name).toEqual(packageName);
expect(installedPackageJson.version).toEqual('1.0.0');
});
tap.test('NPM CLI: should publish second version', async () => {
const packageName = 'test-package-cli';
const version = '1.1.0';
const packageDir = createTestPackage(packageName, version, testDir);
const result = await runNpmCommand('npm publish', packageDir);
console.log('npm publish v1.1.0 output:', result.stdout);
expect(result.exitCode).toEqual(0);
});
tap.test('NPM CLI: should list versions', async () => {
const packageName = 'test-package-cli';
const result = await runNpmCommand(`npm view ${packageName} versions`, testDir);
console.log('npm view versions output:', result.stdout);
expect(result.exitCode).toEqual(0);
expect(result.stdout).toContain('1.0.0');
expect(result.stdout).toContain('1.1.0');
});
tap.test('NPM CLI: should publish scoped package', async () => {
const packageName = '@testscope/scoped-package';
const version = '1.0.0';
const packageDir = createTestPackage(packageName, version, testDir);
const result = await runNpmCommand('npm publish --access public', packageDir);
console.log('npm publish scoped output:', result.stdout);
console.log('npm publish scoped stderr:', result.stderr);
expect(result.exitCode).toEqual(0);
});
tap.test('NPM CLI: should view scoped package', async () => {
const packageName = '@testscope/scoped-package';
const result = await runNpmCommand(`npm view ${packageName}`, testDir);
console.log('npm view scoped output:', result.stdout);
expect(result.exitCode).toEqual(0);
expect(result.stdout).toContain('scoped-package');
});
tap.test('NPM CLI: should fail to publish without auth', async () => {
const packageName = 'unauth-package';
const version = '1.0.0';
const packageDir = createTestPackage(packageName, version, testDir);
// Temporarily remove .npmrc
const npmrcBackup = fs.readFileSync(npmrcPath, 'utf-8');
fs.writeFileSync(npmrcPath, 'registry=' + registryUrl + '/npm/\n', 'utf-8');
const result = await runNpmCommand('npm publish', packageDir);
console.log('npm publish unauth output:', result.stdout);
console.log('npm publish unauth stderr:', result.stderr);
// Restore .npmrc
fs.writeFileSync(npmrcPath, npmrcBackup, 'utf-8');
// Should fail with auth error
expect(result.exitCode).not.toEqual(0);
});
tap.postTask('cleanup npm cli tests', async () => {
// Stop server
if (server) {
await new Promise<void>((resolve) => {
server.close(() => resolve());
});
}
// Cleanup test directory
if (testDir) {
cleanupTestDir(testDir);
}
// Destroy registry
if (registry) {
registry.destroy();
}
});
export default tap.start();

View File

@@ -0,0 +1,406 @@
/**
* Native Docker CLI Testing
* Tests the OCI registry implementation using the actual Docker CLI
*/
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { tapNodeTools } from '@git.zone/tstest/tapbundle_serverside';
import { SmartRegistry } from '../ts/index.js';
import type { IRequestContext, IResponse, IRegistryConfig } from '../ts/core/interfaces.core.js';
import * as qenv from '@push.rocks/qenv';
import * as http from 'http';
import * as url from 'url';
import * as fs from 'fs';
import * as path from 'path';
const testQenv = new qenv.Qenv('./', './.nogit');
/**
* Create a test registry with local token endpoint realm
*/
async function createDockerTestRegistry(port: number): Promise<SmartRegistry> {
const s3AccessKey = await testQenv.getEnvVarOnDemand('S3_ACCESSKEY');
const s3SecretKey = await testQenv.getEnvVarOnDemand('S3_SECRETKEY');
const s3Endpoint = await testQenv.getEnvVarOnDemand('S3_ENDPOINT');
const s3Port = await testQenv.getEnvVarOnDemand('S3_PORT');
const config: IRegistryConfig = {
storage: {
accessKey: s3AccessKey || 'minioadmin',
accessSecret: s3SecretKey || 'minioadmin',
endpoint: s3Endpoint || 'localhost',
port: parseInt(s3Port || '9000', 10),
useSsl: false,
region: 'us-east-1',
bucketName: 'test-registry',
},
auth: {
jwtSecret: 'test-secret-key',
tokenStore: 'memory',
npmTokens: {
enabled: true,
},
ociTokens: {
enabled: true,
realm: `http://localhost:${port}/v2/token`,
service: 'test-registry',
},
},
oci: {
enabled: true,
basePath: '/oci',
},
};
const reg = new SmartRegistry(config);
await reg.init();
return reg;
}
/**
* Create test tokens for the registry
*/
async function createDockerTestTokens(reg: SmartRegistry) {
const authManager = reg.getAuthManager();
const userId = await authManager.authenticate({
username: 'testuser',
password: 'testpass',
});
if (!userId) {
throw new Error('Failed to authenticate test user');
}
// Create OCI token with full access
const ociToken = await authManager.createOciToken(
userId,
['oci:repository:*:*'],
3600
);
return { ociToken, userId };
}
// Test context
let registry: SmartRegistry;
let server: http.Server;
let registryUrl: string;
let registryPort: number;
let ociToken: string;
let testDir: string;
let testImageName: string;
/**
* Create HTTP server wrapper around SmartRegistry
* CRITICAL: Always passes rawBody for content-addressable operations (OCI manifests/blobs)
*
* Docker expects registry at /v2/ but SmartRegistry serves at /oci/v2/
* This wrapper rewrites paths for Docker compatibility
*
* Also implements a simple /v2/token endpoint for Docker Bearer auth flow
*/
async function createHttpServer(
registryInstance: SmartRegistry,
port: number,
tokenForAuth: string
): Promise<{ server: http.Server; url: string }> {
return new Promise((resolve, reject) => {
const httpServer = http.createServer(async (req, res) => {
try {
// Parse request
const parsedUrl = url.parse(req.url || '', true);
let pathname = parsedUrl.pathname || '/';
const query = parsedUrl.query;
// Handle token endpoint for Docker Bearer auth
if (pathname === '/v2/token' || pathname === '/token') {
console.log(`[Token Request] ${req.method} ${req.url}`);
res.statusCode = 200;
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify({
token: tokenForAuth,
access_token: tokenForAuth,
expires_in: 3600,
issued_at: new Date().toISOString(),
}));
return;
}
// Log all requests for debugging
console.log(`[Registry] ${req.method} ${pathname}`);
// Docker expects /v2/ but SmartRegistry serves at /oci/v2/
if (pathname.startsWith('/v2')) {
pathname = '/oci' + pathname;
}
// Read raw body - ALWAYS preserve exact bytes for OCI
const chunks: Buffer[] = [];
for await (const chunk of req) {
chunks.push(chunk);
}
const bodyBuffer = Buffer.concat(chunks);
// Parse body based on content type (for non-OCI protocols that need it)
let parsedBody: any;
if (bodyBuffer.length > 0) {
const contentType = req.headers['content-type'] || '';
if (contentType.includes('application/json')) {
try {
parsedBody = JSON.parse(bodyBuffer.toString('utf-8'));
} catch (error) {
parsedBody = bodyBuffer;
}
} else {
parsedBody = bodyBuffer;
}
}
// Convert to IRequestContext
const context: IRequestContext = {
method: req.method || 'GET',
path: pathname,
headers: req.headers as Record<string, string>,
query: query as Record<string, string>,
body: parsedBody,
rawBody: bodyBuffer,
};
// Handle request
const response: IResponse = await registryInstance.handleRequest(context);
console.log(`[Registry] Response: ${response.status} for ${pathname}`);
// Convert IResponse to HTTP response
res.statusCode = response.status;
// Set headers
for (const [key, value] of Object.entries(response.headers || {})) {
res.setHeader(key, value);
}
// Send body
if (response.body) {
if (Buffer.isBuffer(response.body)) {
res.end(response.body);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
} else {
res.end();
}
} catch (error) {
console.error('Server error:', error);
res.statusCode = 500;
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify({ error: 'INTERNAL_ERROR', message: String(error) }));
}
});
httpServer.listen(port, '0.0.0.0', () => {
const serverUrl = `http://localhost:${port}`;
resolve({ server: httpServer, url: serverUrl });
});
httpServer.on('error', reject);
});
}
/**
* Create a test Dockerfile
*/
function createTestDockerfile(targetDir: string, content?: string): string {
const dockerfilePath = path.join(targetDir, 'Dockerfile');
const dockerfileContent = content || `FROM alpine:latest
RUN echo "Hello from SmartRegistry test" > /hello.txt
CMD ["cat", "/hello.txt"]
`;
fs.writeFileSync(dockerfilePath, dockerfileContent, 'utf-8');
return dockerfilePath;
}
/**
* Run Docker command using the main Docker daemon (not rootless)
* Rootless Docker runs in its own network namespace and can't access host localhost
*
* IMPORTANT: DOCKER_HOST env var overrides --context flag, so we must unset it
* and explicitly set the socket path to use the main Docker daemon.
*/
async function runDockerCommand(
command: string,
cwd?: string
): Promise<{ stdout: string; stderr: string; exitCode: number }> {
// First unset DOCKER_HOST then set it to main Docker daemon socket
// Using both unset and export ensures we override any inherited env var
const dockerCommand = `unset DOCKER_HOST && export DOCKER_HOST=unix:///var/run/docker.sock && ${command}`;
const fullCommand = cwd ? `cd "${cwd}" && ${dockerCommand}` : dockerCommand;
try {
const result = await tapNodeTools.runCommand(fullCommand);
return {
stdout: result.stdout || '',
stderr: result.stderr || '',
exitCode: result.exitCode || 0,
};
} catch (error: any) {
return {
stdout: error.stdout || '',
stderr: error.stderr || String(error),
exitCode: error.exitCode || 1,
};
}
}
/**
* Cleanup test directory
*/
function cleanupTestDir(dir: string): void {
if (fs.existsSync(dir)) {
fs.rmSync(dir, { recursive: true, force: true });
}
}
/**
* Cleanup Docker resources
*/
async function cleanupDocker(imageName: string): Promise<void> {
await runDockerCommand(`docker rmi ${imageName} 2>/dev/null || true`);
await runDockerCommand(`docker rmi ${imageName}:v1 2>/dev/null || true`);
await runDockerCommand(`docker rmi ${imageName}:v2 2>/dev/null || true`);
}
// ========================================================================
// TESTS
// ========================================================================
tap.test('Docker CLI: should verify Docker is installed', async () => {
const result = await runDockerCommand('docker version');
console.log('Docker version output:', result.stdout.substring(0, 200));
expect(result.exitCode).toEqual(0);
});
tap.test('Docker CLI: should setup registry and HTTP server', async () => {
// Use localhost - Docker allows HTTP for localhost without any special config
registryPort = 15000 + Math.floor(Math.random() * 1000);
console.log(`Using port: ${registryPort}`);
registry = await createDockerTestRegistry(registryPort);
const tokens = await createDockerTestTokens(registry);
ociToken = tokens.ociToken;
expect(registry).toBeInstanceOf(SmartRegistry);
expect(ociToken).toBeTypeOf('string');
const serverSetup = await createHttpServer(registry, registryPort, ociToken);
server = serverSetup.server;
registryUrl = serverSetup.url;
expect(server).toBeDefined();
console.log(`Registry server started at ${registryUrl}`);
// Setup test directory
testDir = path.join(process.cwd(), '.nogit', 'test-docker-cli');
cleanupTestDir(testDir);
fs.mkdirSync(testDir, { recursive: true });
testImageName = `localhost:${registryPort}/test-image`;
});
tap.test('Docker CLI: should verify server is responding', async () => {
// Give the server a moment to fully initialize
await new Promise(resolve => setTimeout(resolve, 500));
const response = await fetch(`${registryUrl}/oci/v2/`);
expect(response.status).toEqual(200);
console.log('OCI v2 response:', await response.json());
});
tap.test('Docker CLI: should login to registry', async () => {
const result = await runDockerCommand(
`echo "${ociToken}" | docker login localhost:${registryPort} -u testuser --password-stdin`
);
console.log('docker login output:', result.stdout);
console.log('docker login stderr:', result.stderr);
const combinedOutput = result.stdout + result.stderr;
expect(combinedOutput).toContain('Login Succeeded');
});
tap.test('Docker CLI: should build test image', async () => {
createTestDockerfile(testDir);
const result = await runDockerCommand(
`docker build -t ${testImageName}:v1 .`,
testDir
);
console.log('docker build output:', result.stdout.substring(0, 500));
expect(result.exitCode).toEqual(0);
});
tap.test('Docker CLI: should push image to registry', async () => {
// This is the critical test - if the digest mismatch bug is fixed,
// this should succeed. The manifest bytes must be preserved exactly.
const result = await runDockerCommand(`docker push ${testImageName}:v1`);
console.log('docker push output:', result.stdout);
console.log('docker push stderr:', result.stderr);
expect(result.exitCode).toEqual(0);
});
tap.test('Docker CLI: should verify manifest in registry via API', async () => {
const response = await fetch(`${registryUrl}/oci/v2/test-image/tags/list`, {
headers: { Authorization: `Bearer ${ociToken}` },
});
expect(response.status).toEqual(200);
const tagList = await response.json();
console.log('Tags list:', tagList);
expect(tagList.name).toEqual('test-image');
expect(tagList.tags).toContain('v1');
});
tap.test('Docker CLI: should pull pushed image', async () => {
// First remove the local image
await runDockerCommand(`docker rmi ${testImageName}:v1 || true`);
const result = await runDockerCommand(`docker pull ${testImageName}:v1`);
console.log('docker pull output:', result.stdout);
expect(result.exitCode).toEqual(0);
});
tap.test('Docker CLI: should run pulled image', async () => {
const result = await runDockerCommand(`docker run --rm ${testImageName}:v1`);
console.log('docker run output:', result.stdout);
expect(result.exitCode).toEqual(0);
expect(result.stdout).toContain('Hello from SmartRegistry test');
});
tap.postTask('cleanup docker cli tests', async () => {
if (testImageName) {
await cleanupDocker(testImageName);
}
if (server) {
await new Promise<void>((resolve) => {
server.close(() => resolve());
});
}
if (testDir) {
cleanupTestDir(testDir);
}
if (registry) {
registry.destroy();
}
});
export default tap.start();

477
test/test.pypi.ts Normal file
View File

@@ -0,0 +1,477 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js';
import {
createTestRegistry,
createTestTokens,
createPythonWheel,
createPythonSdist,
calculatePypiHashes,
} from './helpers/registry.js';
import { normalizePypiPackageName } from '../ts/pypi/helpers.pypi.js';
let registry: SmartRegistry;
let pypiToken: string;
let userId: string;
// Test data
const testPackageName = 'test-package';
const normalizedPackageName = normalizePypiPackageName(testPackageName);
const testVersion = '1.0.0';
let testWheelData: Buffer;
let testSdistData: Buffer;
tap.test('PyPI: should create registry instance', async () => {
registry = await createTestRegistry();
const tokens = await createTestTokens(registry);
pypiToken = tokens.pypiToken;
userId = tokens.userId;
expect(registry).toBeInstanceOf(SmartRegistry);
expect(pypiToken).toBeTypeOf('string');
// Clean up any existing metadata from previous test runs
const storage = registry.getStorage();
try {
await storage.deletePypiPackage(normalizedPackageName);
} catch (error) {
// Ignore error if package doesn't exist
}
});
tap.test('PyPI: should create test package files', async () => {
testWheelData = await createPythonWheel(testPackageName, testVersion);
testSdistData = await createPythonSdist(testPackageName, testVersion);
expect(testWheelData).toBeInstanceOf(Buffer);
expect(testWheelData.length).toBeGreaterThan(0);
expect(testSdistData).toBeInstanceOf(Buffer);
expect(testSdistData.length).toBeGreaterThan(0);
});
tap.test('PyPI: should upload wheel file (POST /pypi/)', async () => {
const hashes = calculatePypiHashes(testWheelData);
const filename = `${testPackageName.replace(/-/g, '_')}-${testVersion}-py3-none-any.whl`;
const formData = new FormData();
formData.append(':action', 'file_upload');
formData.append('protocol_version', '1');
formData.append('name', testPackageName);
formData.append('version', testVersion);
formData.append('filetype', 'bdist_wheel');
formData.append('pyversion', 'py3');
formData.append('metadata_version', '2.1');
formData.append('sha256_digest', hashes.sha256);
formData.append('content', new Blob([testWheelData]), filename);
const response = await registry.handleRequest({
method: 'POST',
path: '/pypi/',
headers: {
Authorization: `Bearer ${pypiToken}`,
'Content-Type': 'multipart/form-data',
},
query: {},
body: {
':action': 'file_upload',
protocol_version: '1',
name: testPackageName,
version: testVersion,
filetype: 'bdist_wheel',
pyversion: 'py3',
metadata_version: '2.1',
sha256_digest: hashes.sha256,
requires_python: '>=3.7',
content: testWheelData,
filename: filename,
},
});
expect(response.status).toEqual(201);
});
tap.test('PyPI: should retrieve Simple API root index HTML (GET /simple/)', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/simple/',
headers: {
Accept: 'text/html',
},
query: {},
});
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toStartWith('text/html');
expect(response.body).toBeTypeOf('string');
const html = response.body as string;
expect(html).toContain('<!DOCTYPE html>');
expect(html).toContain('<title>Simple Index</title>');
expect(html).toContain(normalizedPackageName);
});
tap.test('PyPI: should retrieve Simple API root index JSON (GET /simple/ with Accept: application/vnd.pypi.simple.v1+json)', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/simple/',
headers: {
Accept: 'application/vnd.pypi.simple.v1+json',
},
query: {},
});
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/vnd.pypi.simple.v1+json');
expect(response.body).toBeTypeOf('object');
const json = response.body as any;
expect(json).toHaveProperty('meta');
expect(json).toHaveProperty('projects');
expect(json.projects).toBeInstanceOf(Array);
// Check that the package is in the projects list (PEP 691 format: array of { name } objects)
const packageNames = json.projects.map((p: any) => p.name);
expect(packageNames).toContain(normalizedPackageName);
});
tap.test('PyPI: should retrieve Simple API package HTML (GET /simple/{package}/)', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: `/simple/${normalizedPackageName}/`,
headers: {
Accept: 'text/html',
},
query: {},
});
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toStartWith('text/html');
expect(response.body).toBeTypeOf('string');
const html = response.body as string;
expect(html).toContain('<!DOCTYPE html>');
expect(html).toContain(`<title>Links for ${normalizedPackageName}</title>`);
expect(html).toContain('.whl');
expect(html).toContain('data-requires-python');
});
tap.test('PyPI: should retrieve Simple API package JSON (GET /simple/{package}/ with Accept: application/vnd.pypi.simple.v1+json)', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: `/simple/${normalizedPackageName}/`,
headers: {
Accept: 'application/vnd.pypi.simple.v1+json',
},
query: {},
});
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/vnd.pypi.simple.v1+json');
expect(response.body).toBeTypeOf('object');
const json = response.body as any;
expect(json).toHaveProperty('meta');
expect(json).toHaveProperty('name');
expect(json.name).toEqual(normalizedPackageName);
expect(json).toHaveProperty('files');
expect(json.files).toBeTypeOf('object');
expect(Object.keys(json.files).length).toBeGreaterThan(0);
});
tap.test('PyPI: should download wheel file (GET /pypi/packages/{package}/{filename})', async () => {
const filename = `${testPackageName.replace(/-/g, '_')}-${testVersion}-py3-none-any.whl`;
const response = await registry.handleRequest({
method: 'GET',
path: `/pypi/packages/${normalizedPackageName}/${filename}`,
headers: {},
query: {},
});
expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer);
expect((response.body as Buffer).length).toEqual(testWheelData.length);
expect(response.headers['Content-Type']).toEqual('application/octet-stream');
});
tap.test('PyPI: should upload sdist file (POST /pypi/)', async () => {
const hashes = calculatePypiHashes(testSdistData);
const filename = `${testPackageName}-${testVersion}.tar.gz`;
const response = await registry.handleRequest({
method: 'POST',
path: '/pypi/',
headers: {
Authorization: `Bearer ${pypiToken}`,
'Content-Type': 'multipart/form-data',
},
query: {},
body: {
':action': 'file_upload',
protocol_version: '1',
name: testPackageName,
version: testVersion,
filetype: 'sdist',
pyversion: 'source',
metadata_version: '2.1',
sha256_digest: hashes.sha256,
requires_python: '>=3.7',
content: testSdistData,
filename: filename,
},
});
expect(response.status).toEqual(201);
});
tap.test('PyPI: should list both wheel and sdist in Simple API', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: `/simple/${normalizedPackageName}/`,
headers: {
Accept: 'application/vnd.pypi.simple.v1+json',
},
query: {},
});
expect(response.status).toEqual(200);
const json = response.body as any;
// PEP 691: files is an array of file objects
expect(json.files.length).toEqual(2);
const hasWheel = json.files.some((f: any) => f.filename.endsWith('.whl'));
const hasSdist = json.files.some((f: any) => f.filename.endsWith('.tar.gz'));
expect(hasWheel).toEqual(true);
expect(hasSdist).toEqual(true);
});
tap.test('PyPI: should upload a second version', async () => {
const newVersion = '2.0.0';
const newWheelData = await createPythonWheel(testPackageName, newVersion);
const hashes = calculatePypiHashes(newWheelData);
const filename = `${testPackageName.replace(/-/g, '_')}-${newVersion}-py3-none-any.whl`;
const response = await registry.handleRequest({
method: 'POST',
path: '/pypi/',
headers: {
Authorization: `Bearer ${pypiToken}`,
'Content-Type': 'multipart/form-data',
},
query: {},
body: {
':action': 'file_upload',
protocol_version: '1',
name: testPackageName,
version: newVersion,
filetype: 'bdist_wheel',
pyversion: 'py3',
metadata_version: '2.1',
sha256_digest: hashes.sha256,
requires_python: '>=3.7',
content: newWheelData,
filename: filename,
},
});
expect(response.status).toEqual(201);
});
tap.test('PyPI: should list multiple versions in Simple API', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: `/simple/${normalizedPackageName}/`,
headers: {
Accept: 'application/vnd.pypi.simple.v1+json',
},
query: {},
});
expect(response.status).toEqual(200);
const json = response.body as any;
// PEP 691: files is an array of file objects
expect(json.files.length).toBeGreaterThan(2);
const hasVersion1 = json.files.some((f: any) => f.filename.includes('1.0.0'));
const hasVersion2 = json.files.some((f: any) => f.filename.includes('2.0.0'));
expect(hasVersion1).toEqual(true);
expect(hasVersion2).toEqual(true);
});
tap.test('PyPI: should normalize package names correctly', async () => {
const testNames = [
{ input: 'Test-Package', expected: 'test-package' },
{ input: 'Test_Package', expected: 'test-package' },
{ input: 'Test..Package', expected: 'test-package' },
{ input: 'Test---Package', expected: 'test-package' },
];
for (const { input, expected } of testNames) {
const normalized = normalizePypiPackageName(input);
expect(normalized).toEqual(expected);
}
});
tap.test('PyPI: should return 404 for non-existent package', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/simple/nonexistent-package/',
headers: {},
query: {},
});
expect(response.status).toEqual(404);
expect(response.body).toHaveProperty('error');
});
tap.test('PyPI: should return 401 for unauthorized upload', async () => {
const wheelData = await createPythonWheel('unauthorized-test', '1.0.0');
const hashes = calculatePypiHashes(wheelData);
const response = await registry.handleRequest({
method: 'POST',
path: '/pypi/',
headers: {
// No authorization header
'Content-Type': 'multipart/form-data',
},
query: {},
body: {
':action': 'file_upload',
protocol_version: '1',
name: 'unauthorized-test',
version: '1.0.0',
filetype: 'bdist_wheel',
pyversion: 'py3',
metadata_version: '2.1',
sha256_digest: hashes.sha256,
content: wheelData,
filename: 'unauthorized_test-1.0.0-py3-none-any.whl',
},
});
expect(response.status).toEqual(401);
expect(response.body).toHaveProperty('error');
});
tap.test('PyPI: should reject upload with mismatched hash', async () => {
const wheelData = await createPythonWheel('hash-test', '1.0.0');
const response = await registry.handleRequest({
method: 'POST',
path: '/pypi/',
headers: {
Authorization: `Bearer ${pypiToken}`,
'Content-Type': 'multipart/form-data',
},
query: {},
body: {
':action': 'file_upload',
protocol_version: '1',
name: 'hash-test',
version: '1.0.0',
filetype: 'bdist_wheel',
pyversion: 'py3',
metadata_version: '2.1',
sha256_digest: 'wrong_hash_value',
content: wheelData,
filename: 'hash_test-1.0.0-py3-none-any.whl',
},
});
expect(response.status).toEqual(400);
expect(response.body).toHaveProperty('error');
});
tap.test('PyPI: should handle package with requires-python metadata', async () => {
const packageName = 'python-version-test';
const wheelData = await createPythonWheel(packageName, '1.0.0');
const hashes = calculatePypiHashes(wheelData);
const response = await registry.handleRequest({
method: 'POST',
path: '/pypi/',
headers: {
Authorization: `Bearer ${pypiToken}`,
'Content-Type': 'multipart/form-data',
},
query: {},
body: {
':action': 'file_upload',
protocol_version: '1',
name: packageName,
version: '1.0.0',
filetype: 'bdist_wheel',
pyversion: 'py3',
metadata_version: '2.1',
sha256_digest: hashes.sha256,
'requires_python': '>=3.8',
content: wheelData,
filename: `${packageName.replace(/-/g, '_')}-1.0.0-py3-none-any.whl`,
},
});
expect(response.status).toEqual(201);
// Verify requires-python is in Simple API
const getResponse = await registry.handleRequest({
method: 'GET',
path: `/simple/${normalizePypiPackageName(packageName)}/`,
headers: {
Accept: 'text/html',
},
query: {},
});
const html = getResponse.body as string;
expect(html).toContain('data-requires-python');
// Note: >= gets HTML-escaped to &gt;= in attribute values
expect(html).toContain('&gt;=3.8');
});
tap.test('PyPI: should support JSON API for package metadata', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: `/pypi/${normalizedPackageName}/json`,
headers: {},
query: {},
});
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/json');
expect(response.body).toBeTypeOf('object');
const json = response.body as any;
expect(json).toHaveProperty('info');
expect(json.info).toHaveProperty('name');
expect(json.info.name).toEqual(normalizedPackageName);
expect(json).toHaveProperty('urls');
});
tap.test('PyPI: should support JSON API for specific version', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: `/pypi/${normalizedPackageName}/${testVersion}/json`,
headers: {},
query: {},
});
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/json');
expect(response.body).toBeTypeOf('object');
const json = response.body as any;
expect(json).toHaveProperty('info');
expect(json.info.version).toEqual(testVersion);
expect(json).toHaveProperty('urls');
});
tap.postTask('cleanup registry', async () => {
if (registry) {
registry.destroy();
}
});
export default tap.start();

View File

@@ -0,0 +1,448 @@
/**
* Native gem CLI Testing
* Tests the RubyGems registry implementation using the actual gem CLI
*/
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { tapNodeTools } from '@git.zone/tstest/tapbundle_serverside';
import { SmartRegistry } from '../ts/index.js';
import { createTestRegistry, createTestTokens, createRubyGem } from './helpers/registry.js';
import type { IRequestContext, IResponse } from '../ts/core/interfaces.core.js';
import * as http from 'http';
import * as url from 'url';
import * as fs from 'fs';
import * as path from 'path';
// Test context
let registry: SmartRegistry;
let server: http.Server;
let registryUrl: string;
let registryPort: number;
let rubygemsToken: string;
let testDir: string;
let gemHome: string;
/**
* Create HTTP server wrapper around SmartRegistry
*/
async function createHttpServer(
registryInstance: SmartRegistry,
port: number
): Promise<{ server: http.Server; url: string }> {
return new Promise((resolve, reject) => {
const httpServer = http.createServer(async (req, res) => {
try {
// Parse request
const parsedUrl = url.parse(req.url || '', true);
const pathname = parsedUrl.pathname || '/';
const query = parsedUrl.query;
// Read body
const chunks: Buffer[] = [];
for await (const chunk of req) {
chunks.push(chunk);
}
const bodyBuffer = Buffer.concat(chunks);
// Parse body based on content type
let body: any;
if (bodyBuffer.length > 0) {
const contentType = req.headers['content-type'] || '';
if (contentType.includes('application/json')) {
try {
body = JSON.parse(bodyBuffer.toString('utf-8'));
} catch (error) {
body = bodyBuffer;
}
} else {
body = bodyBuffer;
}
}
// Convert to IRequestContext
const context: IRequestContext = {
method: req.method || 'GET',
path: pathname,
headers: req.headers as Record<string, string>,
query: query as Record<string, string>,
body: body,
};
// Handle request
const response: IResponse = await registryInstance.handleRequest(context);
// Convert IResponse to HTTP response
res.statusCode = response.status;
// Set headers
for (const [key, value] of Object.entries(response.headers || {})) {
res.setHeader(key, value);
}
// Send body
if (response.body) {
if (Buffer.isBuffer(response.body)) {
res.end(response.body);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
} else {
res.end();
}
} catch (error) {
console.error('Server error:', error);
res.statusCode = 500;
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify({ error: 'INTERNAL_ERROR', message: String(error) }));
}
});
httpServer.listen(port, () => {
const serverUrl = `http://localhost:${port}`;
resolve({ server: httpServer, url: serverUrl });
});
httpServer.on('error', reject);
});
}
/**
* Setup gem credentials file
* Format: YAML with :rubygems_api_key: TOKEN
*/
function setupGemCredentials(token: string, gemHomeArg: string): string {
const gemDir = path.join(gemHomeArg, '.gem');
fs.mkdirSync(gemDir, { recursive: true });
// Create credentials file in YAML format
const credentialsContent = `:rubygems_api_key: ${token}\n`;
const credentialsPath = path.join(gemDir, 'credentials');
fs.writeFileSync(credentialsPath, credentialsContent, 'utf-8');
// Set restrictive permissions (gem requires 0600)
fs.chmodSync(credentialsPath, 0o600);
return credentialsPath;
}
/**
* Create a test gem file
*/
async function createTestGemFile(
gemName: string,
version: string,
targetDir: string
): Promise<string> {
const gemData = await createRubyGem(gemName, version);
const gemFilename = `${gemName}-${version}.gem`;
const gemPath = path.join(targetDir, gemFilename);
fs.writeFileSync(gemPath, gemData);
return gemPath;
}
/**
* Run gem command with proper environment
*/
async function runGemCommand(
command: string,
cwd: string,
includeAuth: boolean = true
): Promise<{ stdout: string; stderr: string; exitCode: number }> {
// Prepare environment variables
const envVars = [
`HOME="${gemHome}"`,
`GEM_HOME="${gemHome}"`,
includeAuth ? '' : 'RUBYGEMS_API_KEY=""',
].filter(Boolean).join(' ');
// Build command with cd to correct directory and environment variables
const fullCommand = `cd "${cwd}" && ${envVars} ${command}`;
try {
const result = await tapNodeTools.runCommand(fullCommand);
return {
stdout: result.stdout || '',
stderr: result.stderr || '',
exitCode: result.exitCode || 0,
};
} catch (error: any) {
return {
stdout: error.stdout || '',
stderr: error.stderr || String(error),
exitCode: error.exitCode || 1,
};
}
}
/**
* Cleanup test directory
*/
function cleanupTestDir(dir: string): void {
if (fs.existsSync(dir)) {
fs.rmSync(dir, { recursive: true, force: true });
}
}
// ========================================================================
// TESTS
// ========================================================================
tap.test('RubyGems CLI: should setup registry and HTTP server', async () => {
// Create registry
registry = await createTestRegistry();
const tokens = await createTestTokens(registry);
rubygemsToken = tokens.rubygemsToken;
expect(registry).toBeInstanceOf(SmartRegistry);
expect(rubygemsToken).toBeTypeOf('string');
// Use port 36000 (avoids npm:35000, cargo:5000 conflicts)
registryPort = 36000;
const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server;
registryUrl = serverSetup.url;
expect(server).toBeDefined();
expect(registryUrl).toEqual(`http://localhost:${registryPort}`);
// Setup test directory
testDir = path.join(process.cwd(), '.nogit', 'test-rubygems-cli');
cleanupTestDir(testDir);
fs.mkdirSync(testDir, { recursive: true });
// Setup GEM_HOME
gemHome = path.join(testDir, '.gem-home');
fs.mkdirSync(gemHome, { recursive: true });
// Setup gem credentials
const credentialsPath = setupGemCredentials(rubygemsToken, gemHome);
expect(fs.existsSync(credentialsPath)).toEqual(true);
// Verify credentials file has correct permissions
const stats = fs.statSync(credentialsPath);
const mode = stats.mode & 0o777;
expect(mode).toEqual(0o600);
});
tap.test('RubyGems CLI: should verify server is responding', async () => {
// Check server is up by doing a direct HTTP request to the Compact Index
const response = await fetch(`${registryUrl}/rubygems/versions`);
expect(response.status).toBeGreaterThanOrEqual(200);
expect(response.status).toBeLessThan(500);
});
tap.test('RubyGems CLI: should build and push a gem', async () => {
const gemName = 'test-gem-cli';
const version = '1.0.0';
const gemPath = await createTestGemFile(gemName, version, testDir);
expect(fs.existsSync(gemPath)).toEqual(true);
const result = await runGemCommand(
`gem push ${gemPath} --host ${registryUrl}/rubygems`,
testDir
);
console.log('gem push output:', result.stdout);
console.log('gem push stderr:', result.stderr);
expect(result.exitCode).toEqual(0);
expect(result.stdout || result.stderr).toContain(gemName);
});
tap.test('RubyGems CLI: should verify gem in Compact Index /versions', async () => {
const gemName = 'test-gem-cli';
const response = await fetch(`${registryUrl}/rubygems/versions`);
expect(response.status).toEqual(200);
const versionsData = await response.text();
console.log('Versions data:', versionsData);
// Format: GEMNAME VERSION[,VERSION...] MD5
expect(versionsData).toContain(gemName);
expect(versionsData).toContain('1.0.0');
});
tap.test('RubyGems CLI: should verify gem in Compact Index /info file', async () => {
const gemName = 'test-gem-cli';
const response = await fetch(`${registryUrl}/rubygems/info/${gemName}`);
expect(response.status).toEqual(200);
const infoData = await response.text();
console.log('Info data:', infoData);
// Format: VERSION [DEPS]|REQS
expect(infoData).toContain('1.0.0');
});
tap.test('RubyGems CLI: should download gem file', async () => {
const gemName = 'test-gem-cli';
const version = '1.0.0';
const response = await fetch(`${registryUrl}/rubygems/gems/${gemName}-${version}.gem`);
expect(response.status).toEqual(200);
const gemData = await response.arrayBuffer();
expect(gemData.byteLength).toBeGreaterThan(0);
// Verify content type
expect(response.headers.get('content-type')).toContain('application/octet-stream');
});
tap.test('RubyGems CLI: should fetch gem metadata JSON', async () => {
const gemName = 'test-gem-cli';
const response = await fetch(`${registryUrl}/rubygems/api/v1/versions/${gemName}.json`);
expect(response.status).toEqual(200);
const metadata = await response.json();
console.log('Metadata:', metadata);
expect(metadata).toBeInstanceOf(Array);
expect(metadata.length).toBeGreaterThan(0);
expect(metadata[0].number).toEqual('1.0.0');
});
tap.test('RubyGems CLI: should push second version', async () => {
const gemName = 'test-gem-cli';
const version = '2.0.0';
const gemPath = await createTestGemFile(gemName, version, testDir);
const result = await runGemCommand(
`gem push ${gemPath} --host ${registryUrl}/rubygems`,
testDir
);
console.log('gem push v2.0.0 output:', result.stdout);
expect(result.exitCode).toEqual(0);
});
tap.test('RubyGems CLI: should list all versions in /versions file', async () => {
const gemName = 'test-gem-cli';
const response = await fetch(`${registryUrl}/rubygems/versions`);
expect(response.status).toEqual(200);
const versionsData = await response.text();
console.log('All versions data:', versionsData);
// Should contain both versions
expect(versionsData).toContain(gemName);
expect(versionsData).toContain('1.0.0');
expect(versionsData).toContain('2.0.0');
});
tap.test('RubyGems CLI: should yank a version', async () => {
const gemName = 'test-gem-cli';
const version = '1.0.0';
const result = await runGemCommand(
`gem yank ${gemName} -v ${version} --host ${registryUrl}/rubygems`,
testDir
);
console.log('gem yank output:', result.stdout);
console.log('gem yank stderr:', result.stderr);
expect(result.exitCode).toEqual(0);
// Verify version is yanked in /versions file
// Yanked versions are prefixed with '-'
const response = await fetch(`${registryUrl}/rubygems/versions`);
const versionsData = await response.text();
console.log('Versions after yank:', versionsData);
// Yanked version should have '-' prefix
expect(versionsData).toContain('-1.0.0');
});
tap.test('RubyGems CLI: should unyank a version', async () => {
const gemName = 'test-gem-cli';
const version = '1.0.0';
const result = await runGemCommand(
`gem yank ${gemName} -v ${version} --undo --host ${registryUrl}/rubygems`,
testDir
);
console.log('gem unyank output:', result.stdout);
console.log('gem unyank stderr:', result.stderr);
expect(result.exitCode).toEqual(0);
// Verify version is not yanked in /versions file
const response = await fetch(`${registryUrl}/rubygems/versions`);
const versionsData = await response.text();
console.log('Versions after unyank:', versionsData);
// Should not have '-' prefix anymore (or have both without prefix)
// Check that we have the version without yank marker
const lines = versionsData.trim().split('\n');
const gemLine = lines.find(line => line.startsWith(gemName));
if (gemLine) {
// Parse format: "gemname version[,version...] md5"
const parts = gemLine.split(' ');
const versions = parts[1];
// Should have 1.0.0 without '-' prefix
expect(versions).toContain('1.0.0');
expect(versions).not.toContain('-1.0.0');
}
});
tap.test('RubyGems CLI: should fetch dependencies', async () => {
const gemName = 'test-gem-cli';
const response = await fetch(`${registryUrl}/rubygems/api/v1/dependencies?gems=${gemName}`);
expect(response.status).toEqual(200);
const dependencies = await response.json();
console.log('Dependencies:', dependencies);
expect(dependencies).toBeInstanceOf(Array);
});
tap.test('RubyGems CLI: should fail to push without auth', async () => {
const gemName = 'unauth-gem';
const version = '1.0.0';
const gemPath = await createTestGemFile(gemName, version, testDir);
// Run without auth
const result = await runGemCommand(
`gem push ${gemPath} --host ${registryUrl}/rubygems`,
testDir,
false
);
console.log('gem push unauth output:', result.stdout);
console.log('gem push unauth stderr:', result.stderr);
// Should fail with auth error
expect(result.exitCode).not.toEqual(0);
});
tap.postTask('cleanup rubygems cli tests', async () => {
// Stop server
if (server) {
await new Promise<void>((resolve) => {
server.close(() => resolve());
});
}
// Cleanup test directory
if (testDir) {
cleanupTestDir(testDir);
}
// Destroy registry
if (registry) {
registry.destroy();
}
});
export default tap.start();

506
test/test.rubygems.ts Normal file
View File

@@ -0,0 +1,506 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { SmartRegistry } from '../ts/index.js';
import {
createTestRegistry,
createTestTokens,
createRubyGem,
calculateRubyGemsChecksums,
} from './helpers/registry.js';
let registry: SmartRegistry;
let rubygemsToken: string;
let userId: string;
// Test data
const testGemName = 'test-gem';
const testVersion = '1.0.0';
let testGemData: Buffer;
tap.test('RubyGems: should create registry instance', async () => {
registry = await createTestRegistry();
const tokens = await createTestTokens(registry);
rubygemsToken = tokens.rubygemsToken;
userId = tokens.userId;
expect(registry).toBeInstanceOf(SmartRegistry);
expect(rubygemsToken).toBeTypeOf('string');
// Clean up any existing metadata from previous test runs
const storage = registry.getStorage();
try {
await storage.deleteRubyGem(testGemName);
} catch (error) {
// Ignore error if gem doesn't exist
}
});
tap.test('RubyGems: should create test gem file', async () => {
testGemData = await createRubyGem(testGemName, testVersion);
expect(testGemData).toBeInstanceOf(Buffer);
expect(testGemData.length).toBeGreaterThan(0);
});
tap.test('RubyGems: should upload gem file (POST /rubygems/api/v1/gems)', async () => {
const response = await registry.handleRequest({
method: 'POST',
path: '/rubygems/api/v1/gems',
headers: {
Authorization: rubygemsToken,
'Content-Type': 'application/octet-stream',
},
query: {},
body: testGemData,
});
expect(response.status).toEqual(201);
expect(response.body).toHaveProperty('message');
});
tap.test('RubyGems: should retrieve Compact Index versions file (GET /rubygems/versions)', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/rubygems/versions',
headers: {},
query: {},
});
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('text/plain; charset=utf-8');
expect(response.body).toBeInstanceOf(Buffer);
const content = (response.body as Buffer).toString('utf-8');
expect(content).toContain('created_at:');
expect(content).toContain('---');
expect(content).toContain(testGemName);
expect(content).toContain(testVersion);
});
tap.test('RubyGems: should retrieve Compact Index info file (GET /rubygems/info/{gem})', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: `/rubygems/info/${testGemName}`,
headers: {},
query: {},
});
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('text/plain; charset=utf-8');
expect(response.body).toBeInstanceOf(Buffer);
const content = (response.body as Buffer).toString('utf-8');
expect(content).toContain('---');
expect(content).toContain(testVersion);
expect(content).toContain('checksum:');
});
tap.test('RubyGems: should retrieve Compact Index names file (GET /rubygems/names)', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/rubygems/names',
headers: {},
query: {},
});
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('text/plain; charset=utf-8');
expect(response.body).toBeInstanceOf(Buffer);
const content = (response.body as Buffer).toString('utf-8');
expect(content).toContain('---');
expect(content).toContain(testGemName);
});
tap.test('RubyGems: should download gem file (GET /rubygems/gems/{gem}-{version}.gem)', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: `/rubygems/gems/${testGemName}-${testVersion}.gem`,
headers: {},
query: {},
});
expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer);
expect((response.body as Buffer).length).toEqual(testGemData.length);
expect(response.headers['Content-Type']).toEqual('application/octet-stream');
});
tap.test('RubyGems: should upload a second version', async () => {
const newVersion = '2.0.0';
const newGemData = await createRubyGem(testGemName, newVersion);
const response = await registry.handleRequest({
method: 'POST',
path: '/rubygems/api/v1/gems',
headers: {
Authorization: rubygemsToken,
'Content-Type': 'application/octet-stream',
},
query: {},
body: newGemData,
});
expect(response.status).toEqual(201);
});
tap.test('RubyGems: should list multiple versions in Compact Index', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/rubygems/versions',
headers: {},
query: {},
});
expect(response.status).toEqual(200);
const content = (response.body as Buffer).toString('utf-8');
const lines = content.split('\n');
const gemLine = lines.find(l => l.startsWith(`${testGemName} `));
expect(gemLine).toBeDefined();
expect(gemLine).toContain('1.0.0');
expect(gemLine).toContain('2.0.0');
});
tap.test('RubyGems: should list multiple versions in info file', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: `/rubygems/info/${testGemName}`,
headers: {},
query: {},
});
expect(response.status).toEqual(200);
const content = (response.body as Buffer).toString('utf-8');
expect(content).toContain('1.0.0');
expect(content).toContain('2.0.0');
});
tap.test('RubyGems: should support platform-specific gems', async () => {
const platformVersion = '1.5.0';
const platform = 'x86_64-linux';
const platformGemData = await createRubyGem(testGemName, platformVersion, platform);
const response = await registry.handleRequest({
method: 'POST',
path: '/rubygems/api/v1/gems',
headers: {
Authorization: rubygemsToken,
'Content-Type': 'application/octet-stream',
},
query: {},
body: platformGemData,
});
expect(response.status).toEqual(201);
// Verify platform is listed in versions
const versionsResponse = await registry.handleRequest({
method: 'GET',
path: '/rubygems/versions',
headers: {},
query: {},
});
const content = (versionsResponse.body as Buffer).toString('utf-8');
const lines = content.split('\n');
const gemLine = lines.find(l => l.startsWith(`${testGemName} `));
expect(gemLine).toContain(`${platformVersion}_${platform}`);
});
tap.test('RubyGems: should yank a gem version (DELETE /rubygems/api/v1/gems/yank)', async () => {
const response = await registry.handleRequest({
method: 'DELETE',
path: '/rubygems/api/v1/gems/yank',
headers: {
Authorization: rubygemsToken,
},
query: {
gem_name: testGemName,
version: testVersion,
},
});
expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('message');
expect((response.body as any).message).toContain('yanked');
});
tap.test('RubyGems: should mark yanked version in Compact Index', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/rubygems/versions',
headers: {},
query: {},
});
expect(response.status).toEqual(200);
const content = (response.body as Buffer).toString('utf-8');
const lines = content.split('\n');
const gemLine = lines.find(l => l.startsWith(`${testGemName} `));
// Yanked versions are prefixed with '-'
expect(gemLine).toContain(`-${testVersion}`);
});
tap.test('RubyGems: should still allow downloading yanked gem', async () => {
// Yanked gems can still be downloaded if explicitly requested
const response = await registry.handleRequest({
method: 'GET',
path: `/rubygems/gems/${testGemName}-${testVersion}.gem`,
headers: {},
query: {},
});
expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer);
});
tap.test('RubyGems: should unyank a gem version (PUT /rubygems/api/v1/gems/unyank)', async () => {
const response = await registry.handleRequest({
method: 'PUT',
path: '/rubygems/api/v1/gems/unyank',
headers: {
Authorization: rubygemsToken,
},
query: {
gem_name: testGemName,
version: testVersion,
},
});
expect(response.status).toEqual(200);
expect(response.body).toHaveProperty('message');
expect((response.body as any).message).toContain('unyanked');
});
tap.test('RubyGems: should remove yank marker after unyank', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/rubygems/versions',
headers: {},
query: {},
});
expect(response.status).toEqual(200);
const content = (response.body as Buffer).toString('utf-8');
const lines = content.split('\n');
const gemLine = lines.find(l => l.startsWith(`${testGemName} `));
// After unyank, version should not have '-' prefix
const versions = gemLine!.split(' ')[1].split(',');
const version1 = versions.find(v => v.includes('1.0.0'));
expect(version1).not.toStartWith('-');
expect(version1).toContain('1.0.0');
});
tap.test('RubyGems: should retrieve versions JSON (GET /rubygems/api/v1/versions/{gem}.json)', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: `/rubygems/api/v1/versions/${testGemName}.json`,
headers: {},
query: {},
});
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/json');
expect(response.body).toBeTypeOf('object');
const json = response.body as any;
expect(json).toHaveProperty('name');
expect(json.name).toEqual(testGemName);
expect(json).toHaveProperty('versions');
expect(json.versions).toBeTypeOf('object');
expect(json.versions.length).toBeGreaterThan(0);
});
tap.test('RubyGems: should retrieve dependencies JSON (GET /rubygems/api/v1/dependencies)', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/rubygems/api/v1/dependencies',
headers: {},
query: {
gems: `${testGemName}`,
},
});
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/json');
expect(response.body).toBeTypeOf('object');
const json = response.body as any;
expect(Array.isArray(json)).toEqual(true);
});
tap.test('RubyGems: should retrieve gem spec (GET /rubygems/quick/Marshal.4.8/{gem}-{version}.gemspec.rz)', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: `/rubygems/quick/Marshal.4.8/${testGemName}-${testVersion}.gemspec.rz`,
headers: {},
query: {},
});
expect(response.status).toEqual(200);
expect(response.body).toBeInstanceOf(Buffer);
});
tap.test('RubyGems: should support latest specs endpoint (GET /rubygems/latest_specs.4.8.gz)', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/rubygems/latest_specs.4.8.gz',
headers: {},
query: {},
});
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/octet-stream');
expect(response.body).toBeInstanceOf(Buffer);
});
tap.test('RubyGems: should support specs endpoint (GET /rubygems/specs.4.8.gz)', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/rubygems/specs.4.8.gz',
headers: {},
query: {},
});
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('application/octet-stream');
expect(response.body).toBeInstanceOf(Buffer);
});
tap.test('RubyGems: should return 404 for non-existent gem', async () => {
const response = await registry.handleRequest({
method: 'GET',
path: '/rubygems/gems/nonexistent-gem-1.0.0.gem',
headers: {},
query: {},
});
expect(response.status).toEqual(404);
expect(response.body).toHaveProperty('error');
});
tap.test('RubyGems: should return 401 for unauthorized upload', async () => {
const gemData = await createRubyGem('unauthorized-gem', '1.0.0');
const response = await registry.handleRequest({
method: 'POST',
path: '/rubygems/api/v1/gems',
headers: {
// No authorization header
'Content-Type': 'application/octet-stream',
},
query: {},
body: gemData,
});
expect(response.status).toEqual(401);
expect(response.body).toHaveProperty('error');
});
tap.test('RubyGems: should return 401 for unauthorized yank', async () => {
const response = await registry.handleRequest({
method: 'DELETE',
path: '/rubygems/api/v1/gems/yank',
headers: {
// No authorization header
},
query: {
gem_name: testGemName,
version: '2.0.0',
},
});
expect(response.status).toEqual(401);
expect(response.body).toHaveProperty('error');
});
tap.test('RubyGems: should handle gem with dependencies', async () => {
const gemWithDeps = 'gem-with-deps';
const version = '1.0.0';
const gemData = await createRubyGem(gemWithDeps, version);
const response = await registry.handleRequest({
method: 'POST',
path: '/rubygems/api/v1/gems',
headers: {
Authorization: rubygemsToken,
'Content-Type': 'application/octet-stream',
},
query: {},
body: gemData,
});
expect(response.status).toEqual(201);
// Check info file contains dependency info
const infoResponse = await registry.handleRequest({
method: 'GET',
path: `/rubygems/info/${gemWithDeps}`,
headers: {},
query: {},
});
expect(infoResponse.status).toEqual(200);
const content = (infoResponse.body as Buffer).toString('utf-8');
expect(content).toContain('checksum:');
});
tap.test('RubyGems: should validate gem filename format', async () => {
const invalidGemData = Buffer.from('invalid gem data');
const response = await registry.handleRequest({
method: 'POST',
path: '/rubygems/api/v1/gems',
headers: {
Authorization: rubygemsToken,
'Content-Type': 'application/octet-stream',
},
query: {},
body: invalidGemData,
});
// Should fail validation
expect(response.status).toBeGreaterThanOrEqual(400);
});
tap.test('RubyGems: should support conditional GET with ETag', async () => {
// First request to get ETag
const response1 = await registry.handleRequest({
method: 'GET',
path: '/rubygems/versions',
headers: {},
query: {},
});
const etag = response1.headers['ETag'];
expect(etag).toBeDefined();
// Second request with If-None-Match
const response2 = await registry.handleRequest({
method: 'GET',
path: '/rubygems/versions',
headers: {
'If-None-Match': etag as string,
},
query: {},
});
expect(response2.status).toEqual(304);
});
tap.postTask('cleanup registry', async () => {
if (registry) {
registry.destroy();
}
});
export default tap.start();

View File

@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@push.rocks/smartregistry',
version: '1.3.0',
description: 'a registry for npm modules and oci images'
version: '2.2.0',
description: 'A composable TypeScript library implementing OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems registries for building unified container and package registries'
}

View File

@@ -7,10 +7,12 @@ import { NpmRegistry } from './npm/classes.npmregistry.js';
import { MavenRegistry } from './maven/classes.mavenregistry.js';
import { CargoRegistry } from './cargo/classes.cargoregistry.js';
import { ComposerRegistry } from './composer/classes.composerregistry.js';
import { PypiRegistry } from './pypi/classes.pypiregistry.js';
import { RubyGemsRegistry } from './rubygems/classes.rubygemsregistry.js';
/**
* Main registry orchestrator
* Routes requests to appropriate protocol handlers (OCI, NPM, Maven, Cargo, or Composer)
* Routes requests to appropriate protocol handlers (OCI, NPM, Maven, Cargo, Composer, PyPI, or RubyGems)
*/
export class SmartRegistry {
private storage: RegistryStorage;
@@ -39,15 +41,19 @@ export class SmartRegistry {
// Initialize OCI registry if enabled
if (this.config.oci?.enabled) {
const ociBasePath = this.config.oci.basePath || '/oci';
const ociRegistry = new OciRegistry(this.storage, this.authManager, ociBasePath);
const ociBasePath = this.config.oci.basePath ?? '/oci';
const ociTokens = this.config.auth.ociTokens?.enabled ? {
realm: this.config.auth.ociTokens.realm,
service: this.config.auth.ociTokens.service,
} : undefined;
const ociRegistry = new OciRegistry(this.storage, this.authManager, ociBasePath, ociTokens);
await ociRegistry.init();
this.registries.set('oci', ociRegistry);
}
// Initialize NPM registry if enabled
if (this.config.npm?.enabled) {
const npmBasePath = this.config.npm.basePath || '/npm';
const npmBasePath = this.config.npm.basePath ?? '/npm';
const registryUrl = `http://localhost:5000${npmBasePath}`; // TODO: Make configurable
const npmRegistry = new NpmRegistry(this.storage, this.authManager, npmBasePath, registryUrl);
await npmRegistry.init();
@@ -56,7 +62,7 @@ export class SmartRegistry {
// Initialize Maven registry if enabled
if (this.config.maven?.enabled) {
const mavenBasePath = this.config.maven.basePath || '/maven';
const mavenBasePath = this.config.maven.basePath ?? '/maven';
const registryUrl = `http://localhost:5000${mavenBasePath}`; // TODO: Make configurable
const mavenRegistry = new MavenRegistry(this.storage, this.authManager, mavenBasePath, registryUrl);
await mavenRegistry.init();
@@ -65,7 +71,7 @@ export class SmartRegistry {
// Initialize Cargo registry if enabled
if (this.config.cargo?.enabled) {
const cargoBasePath = this.config.cargo.basePath || '/cargo';
const cargoBasePath = this.config.cargo.basePath ?? '/cargo';
const registryUrl = `http://localhost:5000${cargoBasePath}`; // TODO: Make configurable
const cargoRegistry = new CargoRegistry(this.storage, this.authManager, cargoBasePath, registryUrl);
await cargoRegistry.init();
@@ -74,13 +80,31 @@ export class SmartRegistry {
// Initialize Composer registry if enabled
if (this.config.composer?.enabled) {
const composerBasePath = this.config.composer.basePath || '/composer';
const composerBasePath = this.config.composer.basePath ?? '/composer';
const registryUrl = `http://localhost:5000${composerBasePath}`; // TODO: Make configurable
const composerRegistry = new ComposerRegistry(this.storage, this.authManager, composerBasePath, registryUrl);
await composerRegistry.init();
this.registries.set('composer', composerRegistry);
}
// Initialize PyPI registry if enabled
if (this.config.pypi?.enabled) {
const pypiBasePath = this.config.pypi.basePath ?? '/pypi';
const registryUrl = `http://localhost:5000`; // TODO: Make configurable
const pypiRegistry = new PypiRegistry(this.storage, this.authManager, pypiBasePath, registryUrl);
await pypiRegistry.init();
this.registries.set('pypi', pypiRegistry);
}
// Initialize RubyGems registry if enabled
if (this.config.rubygems?.enabled) {
const rubygemsBasePath = this.config.rubygems.basePath ?? '/rubygems';
const registryUrl = `http://localhost:5000${rubygemsBasePath}`; // TODO: Make configurable
const rubygemsRegistry = new RubyGemsRegistry(this.storage, this.authManager, rubygemsBasePath, registryUrl);
await rubygemsRegistry.init();
this.registries.set('rubygems', rubygemsRegistry);
}
this.initialized = true;
}
@@ -131,6 +155,25 @@ export class SmartRegistry {
}
}
// Route to PyPI registry (also handles /simple prefix)
if (this.config.pypi?.enabled) {
const pypiBasePath = this.config.pypi.basePath ?? '/pypi';
if (path.startsWith(pypiBasePath) || path.startsWith('/simple')) {
const pypiRegistry = this.registries.get('pypi');
if (pypiRegistry) {
return pypiRegistry.handleRequest(context);
}
}
}
// Route to RubyGems registry
if (this.config.rubygems?.enabled && path.startsWith(this.config.rubygems.basePath)) {
const rubygemsRegistry = this.registries.get('rubygems');
if (rubygemsRegistry) {
return rubygemsRegistry.handleRequest(context);
}
}
// No matching registry
return {
status: 404,
@@ -159,7 +202,7 @@ export class SmartRegistry {
/**
* Get a specific registry handler
*/
public getRegistry(protocol: 'oci' | 'npm' | 'maven' | 'cargo' | 'composer'): BaseRegistry | undefined {
public getRegistry(protocol: 'oci' | 'npm' | 'maven' | 'cargo' | 'composer' | 'pypi' | 'rubygems'): BaseRegistry | undefined {
return this.registries.get(protocol);
}

View File

@@ -159,15 +159,7 @@ export class ComposerRegistry extends BaseRegistry {
includeDev: boolean,
token: IAuthToken | null
): Promise<IResponse> {
// Check read permission
if (!await this.checkPermission(token, vendorPackage, 'read')) {
return {
status: 401,
headers: { 'WWW-Authenticate': 'Bearer realm="composer"' },
body: { status: 'error', message: 'Authentication required' },
};
}
// Read operations are public, no authentication required
const metadata = await this.storage.getComposerPackageMetadata(vendorPackage);
if (!metadata) {
@@ -227,15 +219,7 @@ export class ComposerRegistry extends BaseRegistry {
reference: string,
token: IAuthToken | null
): Promise<IResponse> {
// Check read permission
if (!await this.checkPermission(token, vendorPackage, 'read')) {
return {
status: 401,
headers: { 'WWW-Authenticate': 'Bearer realm="composer"' },
body: { status: 'error', message: 'Authentication required' },
};
}
// Read operations are public, no authentication required
const zipData = await this.storage.getComposerPackageZip(vendorPackage, reference);
if (!zipData) {

View File

@@ -1,4 +1,5 @@
import type { IAuthConfig, IAuthToken, ICredentials, TRegistryProtocol } from './interfaces.core.js';
import * as crypto from 'crypto';
/**
* Unified authentication manager for all registry protocols
@@ -136,7 +137,7 @@ export class AuthManager {
* @param userId - User ID
* @param scopes - Permission scopes
* @param expiresIn - Expiration time in seconds
* @returns JWT token string
* @returns JWT token string (HMAC-SHA256 signed)
*/
public async createOciToken(
userId: string,
@@ -158,9 +159,17 @@ export class AuthManager {
access: this.scopesToOciAccess(scopes),
};
// In production, use proper JWT library with signing
// For now, return JSON string (mock JWT)
return JSON.stringify(payload);
// Create JWT with HMAC-SHA256 signature
const header = { alg: 'HS256', typ: 'JWT' };
const headerB64 = Buffer.from(JSON.stringify(header)).toString('base64url');
const payloadB64 = Buffer.from(JSON.stringify(payload)).toString('base64url');
const signature = crypto
.createHmac('sha256', this.config.jwtSecret)
.update(`${headerB64}.${payloadB64}`)
.digest('base64url');
return `${headerB64}.${payloadB64}.${signature}`;
}
/**
@@ -170,8 +179,25 @@ export class AuthManager {
*/
public async validateOciToken(jwt: string): Promise<IAuthToken | null> {
try {
// In production, verify JWT signature
const payload = JSON.parse(jwt);
const parts = jwt.split('.');
if (parts.length !== 3) {
return null;
}
const [headerB64, payloadB64, signatureB64] = parts;
// Verify signature
const expectedSignature = crypto
.createHmac('sha256', this.config.jwtSecret)
.update(`${headerB64}.${payloadB64}`)
.digest('base64url');
if (signatureB64 !== expectedSignature) {
return null;
}
// Decode and parse payload
const payload = JSON.parse(Buffer.from(payloadB64, 'base64url').toString('utf-8'));
// Check expiration
const now = Math.floor(Date.now() / 1000);
@@ -179,6 +205,11 @@ export class AuthManager {
return null;
}
// Check not-before time
if (payload.nbf && payload.nbf > now) {
return null;
}
// Convert to unified token format
const scopes = this.ociAccessToScopes(payload.access || []);
@@ -317,12 +348,153 @@ export class AuthManager {
this.tokenStore.delete(token);
}
// ========================================================================
// CARGO TOKEN MANAGEMENT
// ========================================================================
/**
* Create a Cargo token
* @param userId - User ID
* @param readonly - Whether the token is readonly
* @returns Cargo UUID token
*/
public async createCargoToken(userId: string, readonly: boolean = false): Promise<string> {
const scopes = readonly ? ['cargo:*:*:read'] : ['cargo:*:*:*'];
return this.createUuidToken(userId, 'cargo', scopes, readonly);
}
/**
* Validate a Cargo token
* @param token - Cargo UUID token
* @returns Auth token object or null
*/
public async validateCargoToken(token: string): Promise<IAuthToken | null> {
if (!this.isValidUuid(token)) {
return null;
}
const authToken = this.tokenStore.get(token);
if (!authToken || authToken.type !== 'cargo') {
return null;
}
// Check expiration if set
if (authToken.expiresAt && authToken.expiresAt < new Date()) {
this.tokenStore.delete(token);
return null;
}
return authToken;
}
/**
* Revoke a Cargo token
* @param token - Cargo UUID token
*/
public async revokeCargoToken(token: string): Promise<void> {
this.tokenStore.delete(token);
}
// ========================================================================
// PYPI AUTHENTICATION
// ========================================================================
/**
* Create a PyPI token
* @param userId - User ID
* @param readonly - Whether the token is readonly
* @returns PyPI UUID token
*/
public async createPypiToken(userId: string, readonly: boolean = false): Promise<string> {
const scopes = readonly ? ['pypi:*:*:read'] : ['pypi:*:*:*'];
return this.createUuidToken(userId, 'pypi', scopes, readonly);
}
/**
* Validate a PyPI token
* @param token - PyPI UUID token
* @returns Auth token object or null
*/
public async validatePypiToken(token: string): Promise<IAuthToken | null> {
if (!this.isValidUuid(token)) {
return null;
}
const authToken = this.tokenStore.get(token);
if (!authToken || authToken.type !== 'pypi') {
return null;
}
// Check expiration if set
if (authToken.expiresAt && authToken.expiresAt < new Date()) {
this.tokenStore.delete(token);
return null;
}
return authToken;
}
/**
* Revoke a PyPI token
* @param token - PyPI UUID token
*/
public async revokePypiToken(token: string): Promise<void> {
this.tokenStore.delete(token);
}
// ========================================================================
// RUBYGEMS AUTHENTICATION
// ========================================================================
/**
* Create a RubyGems token
* @param userId - User ID
* @param readonly - Whether the token is readonly
* @returns RubyGems UUID token
*/
public async createRubyGemsToken(userId: string, readonly: boolean = false): Promise<string> {
const scopes = readonly ? ['rubygems:*:*:read'] : ['rubygems:*:*:*'];
return this.createUuidToken(userId, 'rubygems', scopes, readonly);
}
/**
* Validate a RubyGems token
* @param token - RubyGems UUID token
* @returns Auth token object or null
*/
public async validateRubyGemsToken(token: string): Promise<IAuthToken | null> {
if (!this.isValidUuid(token)) {
return null;
}
const authToken = this.tokenStore.get(token);
if (!authToken || authToken.type !== 'rubygems') {
return null;
}
// Check expiration if set
if (authToken.expiresAt && authToken.expiresAt < new Date()) {
this.tokenStore.delete(token);
return null;
}
return authToken;
}
/**
* Revoke a RubyGems token
* @param token - RubyGems UUID token
*/
public async revokeRubyGemsToken(token: string): Promise<void> {
this.tokenStore.delete(token);
}
// ========================================================================
// UNIFIED AUTHENTICATION
// ========================================================================
/**
* Validate any token (NPM, Maven, or OCI)
* Validate any token (NPM, Maven, OCI, PyPI, RubyGems, Composer, Cargo)
* @param tokenString - Token string (UUID or JWT)
* @param protocol - Expected protocol type
* @returns Auth token object or null
@@ -331,7 +503,7 @@ export class AuthManager {
tokenString: string,
protocol?: TRegistryProtocol
): Promise<IAuthToken | null> {
// Try UUID-based tokens (NPM, Maven, Composer)
// Try UUID-based tokens (NPM, Maven, Composer, Cargo, PyPI, RubyGems)
if (this.isValidUuid(tokenString)) {
// Try NPM token
const npmToken = await this.validateNpmToken(tokenString);
@@ -350,6 +522,24 @@ export class AuthManager {
if (composerToken && (!protocol || protocol === 'composer')) {
return composerToken;
}
// Try Cargo token
const cargoToken = await this.validateCargoToken(tokenString);
if (cargoToken && (!protocol || protocol === 'cargo')) {
return cargoToken;
}
// Try PyPI token
const pypiToken = await this.validatePypiToken(tokenString);
if (pypiToken && (!protocol || protocol === 'pypi')) {
return pypiToken;
}
// Try RubyGems token
const rubygemsToken = await this.validateRubyGemsToken(tokenString);
if (rubygemsToken && (!protocol || protocol === 'rubygems')) {
return rubygemsToken;
}
}
// Try OCI JWT

View File

@@ -18,14 +18,8 @@ export class RegistryStorage implements IStorageBackend {
* Initialize the storage backend
*/
public async init(): Promise<void> {
this.smartBucket = new plugins.smartbucket.SmartBucket({
accessKey: this.config.accessKey,
accessSecret: this.config.accessSecret,
endpoint: this.config.endpoint,
port: this.config.port || 443,
useSsl: this.config.useSsl !== false,
region: this.config.region || 'us-east-1',
});
// Pass config as IS3Descriptor to SmartBucket (bucketName is extra, SmartBucket ignores it)
this.smartBucket = new plugins.smartbucket.SmartBucket(this.config as plugins.tsclass.storage.IS3Descriptor);
// Ensure bucket exists
await this.smartBucket.createBucket(this.bucketName).catch(() => {
@@ -135,7 +129,7 @@ export class RegistryStorage implements IStorageBackend {
}
/**
* Get OCI manifest
* Get OCI manifest and its content type
*/
public async getOciManifest(repository: string, digest: string): Promise<Buffer | null> {
const path = this.getOciManifestPath(repository, digest);
@@ -143,7 +137,17 @@ export class RegistryStorage implements IStorageBackend {
}
/**
* Store OCI manifest
* Get OCI manifest content type
* Returns the stored content type or null if not found
*/
public async getOciManifestContentType(repository: string, digest: string): Promise<string | null> {
const typePath = this.getOciManifestPath(repository, digest) + '.type';
const data = await this.getObject(typePath);
return data ? data.toString('utf-8') : null;
}
/**
* Store OCI manifest with its content type
*/
public async putOciManifest(
repository: string,
@@ -152,7 +156,11 @@ export class RegistryStorage implements IStorageBackend {
contentType: string
): Promise<void> {
const path = this.getOciManifestPath(repository, digest);
return this.putObject(path, data, { 'Content-Type': contentType });
// Store manifest data
await this.putObject(path, data, { 'Content-Type': contentType });
// Store content type in sidecar file for later retrieval
const typePath = path + '.type';
await this.putObject(typePath, Buffer.from(contentType, 'utf-8'));
}
/**
@@ -348,6 +356,17 @@ export class RegistryStorage implements IStorageBackend {
return this.putObject(path, data);
}
/**
* Delete Maven metadata (maven-metadata.xml)
*/
public async deleteMavenMetadata(
groupId: string,
artifactId: string
): Promise<void> {
const path = this.getMavenMetadataPath(groupId, artifactId);
return this.deleteObject(path);
}
/**
* List Maven versions for an artifact
* Returns all version directories under the artifact path
@@ -590,4 +609,467 @@ export class RegistryStorage implements IStorageBackend {
private getComposerZipPath(vendorPackage: string, reference: string): string {
return `composer/packages/${vendorPackage}/${reference}.zip`;
}
// ========================================================================
// PYPI STORAGE METHODS
// ========================================================================
/**
* Get PyPI package metadata
*/
public async getPypiPackageMetadata(packageName: string): Promise<any | null> {
const path = this.getPypiMetadataPath(packageName);
const data = await this.getObject(path);
return data ? JSON.parse(data.toString('utf-8')) : null;
}
/**
* Store PyPI package metadata
*/
public async putPypiPackageMetadata(packageName: string, metadata: any): Promise<void> {
const path = this.getPypiMetadataPath(packageName);
const data = Buffer.from(JSON.stringify(metadata, null, 2), 'utf-8');
return this.putObject(path, data, { 'Content-Type': 'application/json' });
}
/**
* Check if PyPI package metadata exists
*/
public async pypiPackageMetadataExists(packageName: string): Promise<boolean> {
const path = this.getPypiMetadataPath(packageName);
return this.objectExists(path);
}
/**
* Delete PyPI package metadata
*/
public async deletePypiPackageMetadata(packageName: string): Promise<void> {
const path = this.getPypiMetadataPath(packageName);
return this.deleteObject(path);
}
/**
* Get PyPI Simple API index (HTML)
*/
public async getPypiSimpleIndex(packageName: string): Promise<string | null> {
const path = this.getPypiSimpleIndexPath(packageName);
const data = await this.getObject(path);
return data ? data.toString('utf-8') : null;
}
/**
* Store PyPI Simple API index (HTML)
*/
public async putPypiSimpleIndex(packageName: string, html: string): Promise<void> {
const path = this.getPypiSimpleIndexPath(packageName);
const data = Buffer.from(html, 'utf-8');
return this.putObject(path, data, { 'Content-Type': 'text/html; charset=utf-8' });
}
/**
* Get PyPI root Simple API index (HTML)
*/
public async getPypiSimpleRootIndex(): Promise<string | null> {
const path = this.getPypiSimpleRootIndexPath();
const data = await this.getObject(path);
return data ? data.toString('utf-8') : null;
}
/**
* Store PyPI root Simple API index (HTML)
*/
public async putPypiSimpleRootIndex(html: string): Promise<void> {
const path = this.getPypiSimpleRootIndexPath();
const data = Buffer.from(html, 'utf-8');
return this.putObject(path, data, { 'Content-Type': 'text/html; charset=utf-8' });
}
/**
* Get PyPI package file (wheel, sdist)
*/
public async getPypiPackageFile(packageName: string, filename: string): Promise<Buffer | null> {
const path = this.getPypiPackageFilePath(packageName, filename);
return this.getObject(path);
}
/**
* Store PyPI package file (wheel, sdist)
*/
public async putPypiPackageFile(
packageName: string,
filename: string,
data: Buffer
): Promise<void> {
const path = this.getPypiPackageFilePath(packageName, filename);
return this.putObject(path, data, { 'Content-Type': 'application/octet-stream' });
}
/**
* Check if PyPI package file exists
*/
public async pypiPackageFileExists(packageName: string, filename: string): Promise<boolean> {
const path = this.getPypiPackageFilePath(packageName, filename);
return this.objectExists(path);
}
/**
* Delete PyPI package file
*/
public async deletePypiPackageFile(packageName: string, filename: string): Promise<void> {
const path = this.getPypiPackageFilePath(packageName, filename);
return this.deleteObject(path);
}
/**
* List all PyPI packages
*/
public async listPypiPackages(): Promise<string[]> {
const prefix = 'pypi/metadata/';
const objects = await this.listObjects(prefix);
const packages = new Set<string>();
// Extract package names from paths like: pypi/metadata/package-name/metadata.json
for (const obj of objects) {
const match = obj.match(/^pypi\/metadata\/([^\/]+)\/metadata\.json$/);
if (match) {
packages.add(match[1]);
}
}
return Array.from(packages).sort();
}
/**
* List all versions of a PyPI package
*/
public async listPypiPackageVersions(packageName: string): Promise<string[]> {
const prefix = `pypi/packages/${packageName}/`;
const objects = await this.listObjects(prefix);
const versions = new Set<string>();
// Extract versions from filenames
for (const obj of objects) {
const filename = obj.split('/').pop();
if (!filename) continue;
// Extract version from wheel filename: package-1.0.0-py3-none-any.whl
// or sdist filename: package-1.0.0.tar.gz
const wheelMatch = filename.match(/^[^-]+-([^-]+)-.*\.whl$/);
const sdistMatch = filename.match(/^[^-]+-([^.]+)\.(tar\.gz|zip)$/);
if (wheelMatch) versions.add(wheelMatch[1]);
else if (sdistMatch) versions.add(sdistMatch[1]);
}
return Array.from(versions).sort();
}
/**
* Delete entire PyPI package (all versions and files)
*/
public async deletePypiPackage(packageName: string): Promise<void> {
// Delete metadata
await this.deletePypiPackageMetadata(packageName);
// Delete Simple API index
const simpleIndexPath = this.getPypiSimpleIndexPath(packageName);
try {
await this.deleteObject(simpleIndexPath);
} catch (error) {
// Ignore if doesn't exist
}
// Delete all package files
const prefix = `pypi/packages/${packageName}/`;
const objects = await this.listObjects(prefix);
for (const obj of objects) {
await this.deleteObject(obj);
}
}
/**
* Delete specific version of a PyPI package
*/
public async deletePypiPackageVersion(packageName: string, version: string): Promise<void> {
const prefix = `pypi/packages/${packageName}/`;
const objects = await this.listObjects(prefix);
// Delete all files matching this version
for (const obj of objects) {
const filename = obj.split('/').pop();
if (!filename) continue;
// Check if filename contains this version
const wheelMatch = filename.match(/^[^-]+-([^-]+)-.*\.whl$/);
const sdistMatch = filename.match(/^[^-]+-([^.]+)\.(tar\.gz|zip)$/);
const fileVersion = wheelMatch?.[1] || sdistMatch?.[1];
if (fileVersion === version) {
await this.deleteObject(obj);
}
}
// Update metadata to remove this version
const metadata = await this.getPypiPackageMetadata(packageName);
if (metadata && metadata.versions) {
delete metadata.versions[version];
await this.putPypiPackageMetadata(packageName, metadata);
}
}
// ========================================================================
// PYPI PATH HELPERS
// ========================================================================
private getPypiMetadataPath(packageName: string): string {
return `pypi/metadata/${packageName}/metadata.json`;
}
private getPypiSimpleIndexPath(packageName: string): string {
return `pypi/simple/${packageName}/index.html`;
}
private getPypiSimpleRootIndexPath(): string {
return `pypi/simple/index.html`;
}
private getPypiPackageFilePath(packageName: string, filename: string): string {
return `pypi/packages/${packageName}/${filename}`;
}
// ========================================================================
// RUBYGEMS STORAGE METHODS
// ========================================================================
/**
* Get RubyGems versions file (compact index)
*/
public async getRubyGemsVersions(): Promise<string | null> {
const path = this.getRubyGemsVersionsPath();
const data = await this.getObject(path);
return data ? data.toString('utf-8') : null;
}
/**
* Store RubyGems versions file (compact index)
*/
public async putRubyGemsVersions(content: string): Promise<void> {
const path = this.getRubyGemsVersionsPath();
const data = Buffer.from(content, 'utf-8');
return this.putObject(path, data, { 'Content-Type': 'text/plain; charset=utf-8' });
}
/**
* Get RubyGems info file for a gem (compact index)
*/
public async getRubyGemsInfo(gemName: string): Promise<string | null> {
const path = this.getRubyGemsInfoPath(gemName);
const data = await this.getObject(path);
return data ? data.toString('utf-8') : null;
}
/**
* Store RubyGems info file for a gem (compact index)
*/
public async putRubyGemsInfo(gemName: string, content: string): Promise<void> {
const path = this.getRubyGemsInfoPath(gemName);
const data = Buffer.from(content, 'utf-8');
return this.putObject(path, data, { 'Content-Type': 'text/plain; charset=utf-8' });
}
/**
* Get RubyGems names file
*/
public async getRubyGemsNames(): Promise<string | null> {
const path = this.getRubyGemsNamesPath();
const data = await this.getObject(path);
return data ? data.toString('utf-8') : null;
}
/**
* Store RubyGems names file
*/
public async putRubyGemsNames(content: string): Promise<void> {
const path = this.getRubyGemsNamesPath();
const data = Buffer.from(content, 'utf-8');
return this.putObject(path, data, { 'Content-Type': 'text/plain; charset=utf-8' });
}
/**
* Get RubyGems .gem file
*/
public async getRubyGemsGem(gemName: string, version: string, platform?: string): Promise<Buffer | null> {
const path = this.getRubyGemsGemPath(gemName, version, platform);
return this.getObject(path);
}
/**
* Store RubyGems .gem file
*/
public async putRubyGemsGem(
gemName: string,
version: string,
data: Buffer,
platform?: string
): Promise<void> {
const path = this.getRubyGemsGemPath(gemName, version, platform);
return this.putObject(path, data, { 'Content-Type': 'application/octet-stream' });
}
/**
* Check if RubyGems .gem file exists
*/
public async rubyGemsGemExists(gemName: string, version: string, platform?: string): Promise<boolean> {
const path = this.getRubyGemsGemPath(gemName, version, platform);
return this.objectExists(path);
}
/**
* Delete RubyGems .gem file
*/
public async deleteRubyGemsGem(gemName: string, version: string, platform?: string): Promise<void> {
const path = this.getRubyGemsGemPath(gemName, version, platform);
return this.deleteObject(path);
}
/**
* Get RubyGems metadata
*/
public async getRubyGemsMetadata(gemName: string): Promise<any | null> {
const path = this.getRubyGemsMetadataPath(gemName);
const data = await this.getObject(path);
return data ? JSON.parse(data.toString('utf-8')) : null;
}
/**
* Store RubyGems metadata
*/
public async putRubyGemsMetadata(gemName: string, metadata: any): Promise<void> {
const path = this.getRubyGemsMetadataPath(gemName);
const data = Buffer.from(JSON.stringify(metadata, null, 2), 'utf-8');
return this.putObject(path, data, { 'Content-Type': 'application/json' });
}
/**
* Check if RubyGems metadata exists
*/
public async rubyGemsMetadataExists(gemName: string): Promise<boolean> {
const path = this.getRubyGemsMetadataPath(gemName);
return this.objectExists(path);
}
/**
* Delete RubyGems metadata
*/
public async deleteRubyGemsMetadata(gemName: string): Promise<void> {
const path = this.getRubyGemsMetadataPath(gemName);
return this.deleteObject(path);
}
/**
* List all RubyGems
*/
public async listRubyGems(): Promise<string[]> {
const prefix = 'rubygems/metadata/';
const objects = await this.listObjects(prefix);
const gems = new Set<string>();
// Extract gem names from paths like: rubygems/metadata/gem-name/metadata.json
for (const obj of objects) {
const match = obj.match(/^rubygems\/metadata\/([^\/]+)\/metadata\.json$/);
if (match) {
gems.add(match[1]);
}
}
return Array.from(gems).sort();
}
/**
* List all versions of a RubyGem
*/
public async listRubyGemsVersions(gemName: string): Promise<string[]> {
const prefix = `rubygems/gems/`;
const objects = await this.listObjects(prefix);
const versions = new Set<string>();
// Extract versions from filenames: gem-name-version[-platform].gem
const gemPrefix = `${gemName}-`;
for (const obj of objects) {
const filename = obj.split('/').pop();
if (!filename || !filename.startsWith(gemPrefix) || !filename.endsWith('.gem')) continue;
// Remove gem name prefix and .gem suffix
const versionPart = filename.substring(gemPrefix.length, filename.length - 4);
// Split on last hyphen to separate version from platform
const lastHyphen = versionPart.lastIndexOf('-');
const version = lastHyphen > 0 ? versionPart.substring(0, lastHyphen) : versionPart;
versions.add(version);
}
return Array.from(versions).sort();
}
/**
* Delete entire RubyGem (all versions and files)
*/
public async deleteRubyGem(gemName: string): Promise<void> {
// Delete metadata
await this.deleteRubyGemsMetadata(gemName);
// Delete all gem files
const prefix = `rubygems/gems/`;
const objects = await this.listObjects(prefix);
const gemPrefix = `${gemName}-`;
for (const obj of objects) {
const filename = obj.split('/').pop();
if (filename && filename.startsWith(gemPrefix) && filename.endsWith('.gem')) {
await this.deleteObject(obj);
}
}
}
/**
* Delete specific version of a RubyGem
*/
public async deleteRubyGemsVersion(gemName: string, version: string, platform?: string): Promise<void> {
// Delete gem file
await this.deleteRubyGemsGem(gemName, version, platform);
// Update metadata to remove this version
const metadata = await this.getRubyGemsMetadata(gemName);
if (metadata && metadata.versions) {
const versionKey = platform ? `${version}-${platform}` : version;
delete metadata.versions[versionKey];
await this.putRubyGemsMetadata(gemName, metadata);
}
}
// ========================================================================
// RUBYGEMS PATH HELPERS
// ========================================================================
private getRubyGemsVersionsPath(): string {
return 'rubygems/versions';
}
private getRubyGemsInfoPath(gemName: string): string {
return `rubygems/info/${gemName}`;
}
private getRubyGemsNamesPath(): string {
return 'rubygems/names';
}
private getRubyGemsGemPath(gemName: string, version: string, platform?: string): string {
const filename = platform ? `${gemName}-${version}-${platform}.gem` : `${gemName}-${version}.gem`;
return `rubygems/gems/${filename}`;
}
private getRubyGemsMetadataPath(gemName: string): string {
return `rubygems/metadata/${gemName}/metadata.json`;
}
}

View File

@@ -2,10 +2,12 @@
* Core interfaces for the composable registry system
*/
import type * as plugins from '../plugins.js';
/**
* Registry protocol types
*/
export type TRegistryProtocol = 'oci' | 'npm' | 'maven' | 'cargo' | 'composer';
export type TRegistryProtocol = 'oci' | 'npm' | 'maven' | 'cargo' | 'composer' | 'pypi' | 'rubygems';
/**
* Unified action types across protocols
@@ -40,14 +42,9 @@ export interface ICredentials {
/**
* Storage backend configuration
* Extends IS3Descriptor from @tsclass/tsclass with bucketName
*/
export interface IStorageConfig {
accessKey: string;
accessSecret: string;
endpoint: string;
port?: number;
useSsl?: boolean;
region?: string;
export interface IStorageConfig extends plugins.tsclass.storage.IS3Descriptor {
bucketName: string;
}
@@ -70,6 +67,16 @@ export interface IAuthConfig {
realm: string;
service: string;
};
/** PyPI token settings */
pypiTokens?: {
enabled: boolean;
defaultReadonly?: boolean;
};
/** RubyGems token settings */
rubygemsTokens?: {
enabled: boolean;
defaultReadonly?: boolean;
};
}
/**
@@ -92,6 +99,8 @@ export interface IRegistryConfig {
maven?: IProtocolConfig;
cargo?: IProtocolConfig;
composer?: IProtocolConfig;
pypi?: IProtocolConfig;
rubygems?: IProtocolConfig;
}
/**
@@ -149,6 +158,12 @@ export interface IRequestContext {
headers: Record<string, string>;
query: Record<string, string>;
body?: any;
/**
* Raw request body as bytes. MUST be provided for content-addressable operations
* (OCI manifests, blobs) to ensure digest calculation matches client expectations.
* If not provided, falls back to 'body' field.
*/
rawBody?: Buffer;
token?: string;
}

View File

@@ -1,6 +1,6 @@
/**
* @push.rocks/smartregistry
* Composable registry supporting OCI, NPM, Maven, Cargo, and Composer protocols
* Composable registry supporting OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems protocols
*/
// Main orchestrator
@@ -23,3 +23,9 @@ export * from './cargo/index.js';
// Composer Registry
export * from './composer/index.js';
// PyPI Registry
export * from './pypi/index.js';
// RubyGems Registry
export * from './rubygems/index.js';

View File

@@ -85,7 +85,7 @@ export class MavenRegistry extends BaseRegistry {
// Check if it's a checksum file
if (coordinate.extension === 'md5' || coordinate.extension === 'sha1' ||
coordinate.extension === 'sha256' || coordinate.extension === 'sha512') {
return this.handleChecksumRequest(context.method, coordinate, token);
return this.handleChecksumRequest(context.method, coordinate, token, path);
}
// Handle artifact requests (JAR, POM, WAR, etc.)

View File

@@ -20,12 +20,19 @@ export class OciRegistry extends BaseRegistry {
private uploadSessions: Map<string, IUploadSession> = new Map();
private basePath: string = '/oci';
private cleanupInterval?: NodeJS.Timeout;
private ociTokens?: { realm: string; service: string };
constructor(storage: RegistryStorage, authManager: AuthManager, basePath: string = '/oci') {
constructor(
storage: RegistryStorage,
authManager: AuthManager,
basePath: string = '/oci',
ociTokens?: { realm: string; service: string }
) {
super();
this.storage = storage;
this.authManager = authManager;
this.basePath = basePath;
this.ociTokens = ociTokens;
}
public async init(): Promise<void> {
@@ -55,7 +62,9 @@ export class OciRegistry extends BaseRegistry {
const manifestMatch = path.match(/^\/v2\/([^\/]+(?:\/[^\/]+)*)\/manifests\/([^\/]+)$/);
if (manifestMatch) {
const [, name, reference] = manifestMatch;
return this.handleManifestRequest(context.method, name, reference, token, context.body, context.headers);
// Prefer rawBody for content-addressable operations to preserve exact bytes
const bodyData = context.rawBody || context.body;
return this.handleManifestRequest(context.method, name, reference, token, bodyData, context.headers);
}
// Blob operations: /v2/{name}/blobs/{digest}
@@ -69,7 +78,9 @@ export class OciRegistry extends BaseRegistry {
const uploadInitMatch = path.match(/^\/v2\/([^\/]+(?:\/[^\/]+)*)\/blobs\/uploads\/?$/);
if (uploadInitMatch && context.method === 'POST') {
const [, name] = uploadInitMatch;
return this.handleUploadInit(name, token, context.query, context.body);
// Prefer rawBody for content-addressable operations to preserve exact bytes
const bodyData = context.rawBody || context.body;
return this.handleUploadInit(name, token, context.query, bodyData);
}
// Blob upload operations: /v2/{name}/blobs/uploads/{uuid}
@@ -180,18 +191,14 @@ export class OciRegistry extends BaseRegistry {
body?: Buffer | any
): Promise<IResponse> {
if (!await this.checkPermission(token, repository, 'push')) {
return {
status: 401,
headers: {},
body: this.createError('DENIED', 'Insufficient permissions'),
};
return this.createUnauthorizedResponse(repository, 'push');
}
// Check for monolithic upload (digest + body provided)
const digest = query.digest;
if (digest && body) {
// Monolithic upload: complete upload in single POST
const blobData = Buffer.isBuffer(body) ? body : Buffer.from(JSON.stringify(body));
const blobData = this.toBuffer(body);
// Verify digest
const calculatedDigest = await this.calculateDigest(blobData);
@@ -255,18 +262,17 @@ export class OciRegistry extends BaseRegistry {
}
if (!await this.checkPermission(token, session.repository, 'push')) {
return {
status: 401,
headers: {},
body: this.createError('DENIED', 'Insufficient permissions'),
};
return this.createUnauthorizedResponse(session.repository, 'push');
}
// Prefer rawBody for content-addressable operations to preserve exact bytes
const bodyData = context.rawBody || context.body;
switch (method) {
case 'PATCH':
return this.uploadChunk(uploadId, context.body, context.headers['content-range']);
return this.uploadChunk(uploadId, bodyData, context.headers['content-range']);
case 'PUT':
return this.completeUpload(uploadId, context.query['digest'], context.body);
return this.completeUpload(uploadId, context.query['digest'], bodyData);
case 'GET':
return this.getUploadStatus(uploadId);
default:
@@ -288,13 +294,7 @@ export class OciRegistry extends BaseRegistry {
headers?: Record<string, string>
): Promise<IResponse> {
if (!await this.checkPermission(token, repository, 'pull')) {
return {
status: 401,
headers: {
'WWW-Authenticate': `Bearer realm="${this.basePath}/v2/token",service="registry",scope="repository:${repository}:pull"`,
},
body: this.createError('DENIED', 'Insufficient permissions'),
};
return this.createUnauthorizedResponse(repository, 'pull');
}
// Resolve tag to digest if needed
@@ -320,10 +320,17 @@ export class OciRegistry extends BaseRegistry {
};
}
// Get stored content type, falling back to detecting from manifest content
let contentType = await this.storage.getOciManifestContentType(repository, digest);
if (!contentType) {
// Fallback: detect content type from manifest content
contentType = this.detectManifestContentType(manifestData);
}
return {
status: 200,
headers: {
'Content-Type': 'application/vnd.oci.image.manifest.v1+json',
'Content-Type': contentType,
'Docker-Content-Digest': digest,
},
body: manifestData,
@@ -336,11 +343,7 @@ export class OciRegistry extends BaseRegistry {
token: IAuthToken | null
): Promise<IResponse> {
if (!await this.checkPermission(token, repository, 'pull')) {
return {
status: 401,
headers: {},
body: null,
};
return this.createUnauthorizedHeadResponse(repository, 'pull');
}
// Similar logic as getManifest but return headers only
@@ -360,10 +363,18 @@ export class OciRegistry extends BaseRegistry {
const manifestData = await this.storage.getOciManifest(repository, digest);
// Get stored content type, falling back to detecting from manifest content
let contentType = await this.storage.getOciManifestContentType(repository, digest);
if (!contentType && manifestData) {
// Fallback: detect content type from manifest content
contentType = this.detectManifestContentType(manifestData);
}
contentType = contentType || 'application/vnd.oci.image.manifest.v1+json';
return {
status: 200,
headers: {
'Content-Type': 'application/vnd.oci.image.manifest.v1+json',
'Content-Type': contentType,
'Docker-Content-Digest': digest,
'Content-Length': manifestData ? manifestData.length.toString() : '0',
},
@@ -379,13 +390,7 @@ export class OciRegistry extends BaseRegistry {
headers?: Record<string, string>
): Promise<IResponse> {
if (!await this.checkPermission(token, repository, 'push')) {
return {
status: 401,
headers: {
'WWW-Authenticate': `Bearer realm="${this.basePath}/v2/token",service="registry",scope="repository:${repository}:push"`,
},
body: this.createError('DENIED', 'Insufficient permissions'),
};
return this.createUnauthorizedResponse(repository, 'push');
}
if (!body) {
@@ -396,7 +401,9 @@ export class OciRegistry extends BaseRegistry {
};
}
const manifestData = Buffer.isBuffer(body) ? body : Buffer.from(JSON.stringify(body));
// Preserve raw bytes for accurate digest calculation
// Per OCI spec, digest must match the exact bytes sent by client
const manifestData = this.toBuffer(body);
const contentType = headers?.['content-type'] || headers?.['Content-Type'] || 'application/vnd.oci.image.manifest.v1+json';
// Calculate manifest digest
@@ -437,11 +444,7 @@ export class OciRegistry extends BaseRegistry {
}
if (!await this.checkPermission(token, repository, 'delete')) {
return {
status: 401,
headers: {},
body: this.createError('DENIED', 'Insufficient permissions'),
};
return this.createUnauthorizedResponse(repository, 'delete');
}
await this.storage.deleteOciManifest(repository, digest);
@@ -460,11 +463,7 @@ export class OciRegistry extends BaseRegistry {
range?: string
): Promise<IResponse> {
if (!await this.checkPermission(token, repository, 'pull')) {
return {
status: 401,
headers: {},
body: this.createError('DENIED', 'Insufficient permissions'),
};
return this.createUnauthorizedResponse(repository, 'pull');
}
const data = await this.storage.getOciBlob(digest);
@@ -492,7 +491,7 @@ export class OciRegistry extends BaseRegistry {
token: IAuthToken | null
): Promise<IResponse> {
if (!await this.checkPermission(token, repository, 'pull')) {
return { status: 401, headers: {}, body: null };
return this.createUnauthorizedHeadResponse(repository, 'pull');
}
const exists = await this.storage.ociBlobExists(digest);
@@ -518,11 +517,7 @@ export class OciRegistry extends BaseRegistry {
token: IAuthToken | null
): Promise<IResponse> {
if (!await this.checkPermission(token, repository, 'delete')) {
return {
status: 401,
headers: {},
body: this.createError('DENIED', 'Insufficient permissions'),
};
return this.createUnauthorizedResponse(repository, 'delete');
}
await this.storage.deleteOciBlob(digest);
@@ -536,7 +531,7 @@ export class OciRegistry extends BaseRegistry {
private async uploadChunk(
uploadId: string,
data: Buffer,
data: Buffer | Uint8Array | unknown,
contentRange: string
): Promise<IResponse> {
const session = this.uploadSessions.get(uploadId);
@@ -548,8 +543,9 @@ export class OciRegistry extends BaseRegistry {
};
}
session.chunks.push(data);
session.totalSize += data.length;
const chunkData = this.toBuffer(data);
session.chunks.push(chunkData);
session.totalSize += chunkData.length;
session.lastActivity = new Date();
return {
@@ -566,7 +562,7 @@ export class OciRegistry extends BaseRegistry {
private async completeUpload(
uploadId: string,
digest: string,
finalData?: Buffer
finalData?: Buffer | Uint8Array | unknown
): Promise<IResponse> {
const session = this.uploadSessions.get(uploadId);
if (!session) {
@@ -578,7 +574,7 @@ export class OciRegistry extends BaseRegistry {
}
const chunks = [...session.chunks];
if (finalData) chunks.push(finalData);
if (finalData) chunks.push(this.toBuffer(finalData));
const blobData = Buffer.concat(chunks);
// Verify digest
@@ -631,11 +627,7 @@ export class OciRegistry extends BaseRegistry {
query: Record<string, string>
): Promise<IResponse> {
if (!await this.checkPermission(token, repository, 'pull')) {
return {
status: 401,
headers: {},
body: this.createError('DENIED', 'Insufficient permissions'),
};
return this.createUnauthorizedResponse(repository, 'pull');
}
const tags = await this.getTagsData(repository);
@@ -660,11 +652,7 @@ export class OciRegistry extends BaseRegistry {
query: Record<string, string>
): Promise<IResponse> {
if (!await this.checkPermission(token, repository, 'pull')) {
return {
status: 401,
headers: {},
body: this.createError('DENIED', 'Insufficient permissions'),
};
return this.createUnauthorizedResponse(repository, 'pull');
}
const response: IReferrersResponse = {
@@ -684,6 +672,59 @@ export class OciRegistry extends BaseRegistry {
// HELPER METHODS
// ========================================================================
/**
* Detect manifest content type from manifest content.
* OCI Image Index has "manifests" array, OCI Image Manifest has "config" object.
* Also checks the mediaType field if present.
*/
private detectManifestContentType(manifestData: Buffer): string {
try {
const manifest = JSON.parse(manifestData.toString('utf-8'));
// First check if manifest has explicit mediaType field
if (manifest.mediaType) {
return manifest.mediaType;
}
// Otherwise detect from structure
if (Array.isArray(manifest.manifests)) {
// OCI Image Index (multi-arch manifest list)
return 'application/vnd.oci.image.index.v1+json';
} else if (manifest.config) {
// OCI Image Manifest
return 'application/vnd.oci.image.manifest.v1+json';
}
// Fallback to standard manifest type
return 'application/vnd.oci.image.manifest.v1+json';
} catch (e) {
// If parsing fails, return default
return 'application/vnd.oci.image.manifest.v1+json';
}
}
/**
* Convert any binary-like data to Buffer.
* Handles Buffer, Uint8Array (modern cross-platform), string, and objects.
*
* Note: Buffer.isBuffer(Uint8Array) returns false even though Buffer extends Uint8Array.
* This is because Uint8Array is the modern, cross-platform standard while Buffer is Node.js-specific.
* Many HTTP frameworks pass request bodies as Uint8Array for better compatibility.
*/
private toBuffer(data: unknown): Buffer {
if (Buffer.isBuffer(data)) {
return data;
}
if (data instanceof Uint8Array) {
return Buffer.from(data);
}
if (typeof data === 'string') {
return Buffer.from(data, 'utf-8');
}
// Fallback: serialize object to JSON (may cause digest mismatch for manifests)
return Buffer.from(JSON.stringify(data));
}
private async getTagsData(repository: string): Promise<Record<string, string>> {
const path = `oci/tags/${repository}/tags.json`;
const data = await this.storage.getObject(path);
@@ -712,6 +753,37 @@ export class OciRegistry extends BaseRegistry {
};
}
/**
* Create an unauthorized response with proper WWW-Authenticate header.
* Per OCI Distribution Spec, 401 responses MUST include WWW-Authenticate header.
*/
private createUnauthorizedResponse(repository: string, action: string): IResponse {
const realm = this.ociTokens?.realm || `${this.basePath}/v2/token`;
const service = this.ociTokens?.service || 'registry';
return {
status: 401,
headers: {
'WWW-Authenticate': `Bearer realm="${realm}",service="${service}",scope="repository:${repository}:${action}"`,
},
body: this.createError('DENIED', 'Insufficient permissions'),
};
}
/**
* Create an unauthorized HEAD response (no body per HTTP spec).
*/
private createUnauthorizedHeadResponse(repository: string, action: string): IResponse {
const realm = this.ociTokens?.realm || `${this.basePath}/v2/token`;
const service = this.ociTokens?.service || 'registry';
return {
status: 401,
headers: {
'WWW-Authenticate': `Bearer realm="${realm}",service="${service}",scope="repository:${repository}:${action}"`,
},
body: null,
};
}
private startUploadSessionCleanup(): void {
this.cleanupInterval = setInterval(() => {
const now = new Date();

View File

@@ -4,8 +4,14 @@ import * as path from 'path';
export { path };
// @push.rocks scope
import * as smartarchive from '@push.rocks/smartarchive';
import * as smartbucket from '@push.rocks/smartbucket';
import * as smartlog from '@push.rocks/smartlog';
import * as smartpath from '@push.rocks/smartpath';
export { smartbucket, smartlog, smartpath };
export { smartarchive, smartbucket, smartlog, smartpath };
// @tsclass scope
import * as tsclass from '@tsclass/tsclass';
export { tsclass };

View File

@@ -0,0 +1,657 @@
import { Smartlog } from '@push.rocks/smartlog';
import { BaseRegistry } from '../core/classes.baseregistry.js';
import { RegistryStorage } from '../core/classes.registrystorage.js';
import { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js';
import type {
IPypiPackageMetadata,
IPypiFile,
IPypiError,
IPypiUploadResponse,
} from './interfaces.pypi.js';
import * as helpers from './helpers.pypi.js';
/**
* PyPI registry implementation
* Implements PEP 503 (Simple API), PEP 691 (JSON API), and legacy upload API
*/
export class PypiRegistry extends BaseRegistry {
private storage: RegistryStorage;
private authManager: AuthManager;
private basePath: string = '/pypi';
private registryUrl: string;
private logger: Smartlog;
constructor(
storage: RegistryStorage,
authManager: AuthManager,
basePath: string = '/pypi',
registryUrl: string = 'http://localhost:5000'
) {
super();
this.storage = storage;
this.authManager = authManager;
this.basePath = basePath;
this.registryUrl = registryUrl;
// Initialize logger
this.logger = new Smartlog({
logContext: {
company: 'push.rocks',
companyunit: 'smartregistry',
containerName: 'pypi-registry',
environment: (process.env.NODE_ENV as any) || 'development',
runtime: 'node',
zone: 'pypi'
}
});
this.logger.enableConsole();
}
public async init(): Promise<void> {
// Initialize root Simple API index if not exists
const existingIndex = await this.storage.getPypiSimpleRootIndex();
if (!existingIndex) {
const html = helpers.generateSimpleRootHtml([]);
await this.storage.putPypiSimpleRootIndex(html);
this.logger.log('info', 'Initialized PyPI root index');
}
}
public getBasePath(): string {
return this.basePath;
}
public async handleRequest(context: IRequestContext): Promise<IResponse> {
let path = context.path.replace(this.basePath, '');
// Also handle /simple path prefix
if (path.startsWith('/simple')) {
path = path.replace('/simple', '');
return this.handleSimpleRequest(path, context);
}
// Extract token (Basic Auth or Bearer)
const token = await this.extractToken(context);
this.logger.log('debug', `handleRequest: ${context.method} ${path}`, {
method: context.method,
path,
hasAuth: !!token
});
// Root upload endpoint (POST /)
if ((path === '/' || path === '') && context.method === 'POST') {
return this.handleUpload(context, token);
}
// Package metadata JSON API: GET /{package}/json
const jsonMatch = path.match(/^\/([^\/]+)\/json$/);
if (jsonMatch && context.method === 'GET') {
return this.handlePackageJson(jsonMatch[1]);
}
// Version-specific JSON API: GET /{package}/{version}/json
const versionJsonMatch = path.match(/^\/([^\/]+)\/([^\/]+)\/json$/);
if (versionJsonMatch && context.method === 'GET') {
return this.handleVersionJson(versionJsonMatch[1], versionJsonMatch[2]);
}
// Package file download: GET /packages/{package}/{filename}
const downloadMatch = path.match(/^\/packages\/([^\/]+)\/(.+)$/);
if (downloadMatch && context.method === 'GET') {
return this.handleDownload(downloadMatch[1], downloadMatch[2]);
}
// Delete package: DELETE /packages/{package}
if (path.match(/^\/packages\/([^\/]+)$/) && context.method === 'DELETE') {
const packageName = path.match(/^\/packages\/([^\/]+)$/)?.[1];
return this.handleDeletePackage(packageName!, token);
}
// Delete version: DELETE /packages/{package}/{version}
const deleteVersionMatch = path.match(/^\/packages\/([^\/]+)\/([^\/]+)$/);
if (deleteVersionMatch && context.method === 'DELETE') {
return this.handleDeleteVersion(deleteVersionMatch[1], deleteVersionMatch[2], token);
}
return {
status: 404,
headers: { 'Content-Type': 'application/json' },
body: { error: 'Not Found' },
};
}
/**
* Check if token has permission for resource
*/
protected async checkPermission(
token: IAuthToken | null,
resource: string,
action: string
): Promise<boolean> {
if (!token) return false;
return this.authManager.authorize(token, `pypi:package:${resource}`, action);
}
/**
* Handle Simple API requests (PEP 503 HTML or PEP 691 JSON)
*/
private async handleSimpleRequest(path: string, context: IRequestContext): Promise<IResponse> {
// Ensure path ends with / (PEP 503 requirement)
if (!path.endsWith('/') && !path.includes('.')) {
return {
status: 301,
headers: { 'Location': `${this.basePath}/simple${path}/` },
body: Buffer.from(''),
};
}
// Root index: /simple/
if (path === '/' || path === '') {
return this.handleSimpleRoot(context);
}
// Package index: /simple/{package}/
const packageMatch = path.match(/^\/([^\/]+)\/$/);
if (packageMatch) {
return this.handleSimplePackage(packageMatch[1], context);
}
return {
status: 404,
headers: { 'Content-Type': 'text/html; charset=utf-8' },
body: Buffer.from('<html><body><h1>404 Not Found</h1></body></html>'),
};
}
/**
* Handle Simple API root index
* Returns HTML (PEP 503) or JSON (PEP 691) based on Accept header
*/
private async handleSimpleRoot(context: IRequestContext): Promise<IResponse> {
const acceptHeader = context.headers['accept'] || context.headers['Accept'] || '';
const preferJson = acceptHeader.includes('application/vnd.pypi.simple') &&
acceptHeader.includes('json');
const packages = await this.storage.listPypiPackages();
if (preferJson) {
// PEP 691: JSON response
const response = helpers.generateJsonRootResponse(packages);
return {
status: 200,
headers: {
'Content-Type': 'application/vnd.pypi.simple.v1+json',
'Cache-Control': 'public, max-age=600'
},
body: response,
};
} else {
// PEP 503: HTML response
const html = helpers.generateSimpleRootHtml(packages);
// Update stored index
await this.storage.putPypiSimpleRootIndex(html);
return {
status: 200,
headers: {
'Content-Type': 'text/html; charset=utf-8',
'Cache-Control': 'public, max-age=600'
},
body: html,
};
}
}
/**
* Handle Simple API package index
* Returns HTML (PEP 503) or JSON (PEP 691) based on Accept header
*/
private async handleSimplePackage(packageName: string, context: IRequestContext): Promise<IResponse> {
const normalized = helpers.normalizePypiPackageName(packageName);
// Get package metadata
const metadata = await this.storage.getPypiPackageMetadata(normalized);
if (!metadata) {
return this.errorResponse(404, 'Package not found');
}
// Build file list from all versions
const files: IPypiFile[] = [];
for (const [version, versionMeta] of Object.entries(metadata.versions || {})) {
for (const file of (versionMeta as any).files || []) {
files.push({
filename: file.filename,
url: `${this.registryUrl}/pypi/packages/${normalized}/${file.filename}`,
hashes: file.hashes,
'requires-python': file['requires-python'],
yanked: file.yanked || (versionMeta as any).yanked,
size: file.size,
'upload-time': file['upload-time'],
});
}
}
const acceptHeader = context.headers['accept'] || context.headers['Accept'] || '';
const preferJson = acceptHeader.includes('application/vnd.pypi.simple') &&
acceptHeader.includes('json');
if (preferJson) {
// PEP 691: JSON response
const response = helpers.generateJsonPackageResponse(normalized, files);
return {
status: 200,
headers: {
'Content-Type': 'application/vnd.pypi.simple.v1+json',
'Cache-Control': 'public, max-age=300'
},
body: response,
};
} else {
// PEP 503: HTML response
const html = helpers.generateSimplePackageHtml(normalized, files, this.registryUrl);
// Update stored index
await this.storage.putPypiSimpleIndex(normalized, html);
return {
status: 200,
headers: {
'Content-Type': 'text/html; charset=utf-8',
'Cache-Control': 'public, max-age=300'
},
body: html,
};
}
}
/**
* Extract authentication token from request
*/
private async extractToken(context: IRequestContext): Promise<IAuthToken | null> {
const authHeader = context.headers['authorization'] || context.headers['Authorization'];
if (!authHeader) return null;
// Handle Basic Auth (username:password or __token__:token)
if (authHeader.startsWith('Basic ')) {
const base64 = authHeader.substring(6);
const decoded = Buffer.from(base64, 'base64').toString('utf-8');
const [username, password] = decoded.split(':');
// PyPI token authentication: username = __token__
if (username === '__token__') {
return this.authManager.validateToken(password, 'pypi');
}
// Username/password authentication (would need user lookup)
// For now, not implemented
return null;
}
// Handle Bearer token
if (authHeader.startsWith('Bearer ')) {
const token = authHeader.substring(7);
return this.authManager.validateToken(token, 'pypi');
}
return null;
}
/**
* Handle package upload (multipart/form-data)
* POST / with :action=file_upload
*/
private async handleUpload(context: IRequestContext, token: IAuthToken | null): Promise<IResponse> {
if (!token) {
return {
status: 401,
headers: {
'Content-Type': 'application/json',
'WWW-Authenticate': 'Basic realm="PyPI"'
},
body: { error: 'Authentication required' },
};
}
try {
// Parse multipart form data (context.body should be parsed by server)
const formData = context.body as any; // Assuming parsed multipart data
if (!formData || formData[':action'] !== 'file_upload') {
return this.errorResponse(400, 'Invalid upload request');
}
// Extract required fields - support both nested and flat body formats
const packageName = formData.name;
const version = formData.version;
// Support both: formData.content.filename (multipart parsed) and formData.filename (flat)
const filename = formData.content?.filename || formData.filename;
// Support both: formData.content.data (multipart parsed) and formData.content (Buffer directly)
const fileData = (formData.content?.data || (Buffer.isBuffer(formData.content) ? formData.content : null)) as Buffer;
const filetype = formData.filetype; // 'bdist_wheel' or 'sdist'
const pyversion = formData.pyversion;
if (!packageName || !version || !filename || !fileData) {
return this.errorResponse(400, 'Missing required fields');
}
// Validate package name
if (!helpers.isValidPackageName(packageName)) {
return this.errorResponse(400, 'Invalid package name');
}
const normalized = helpers.normalizePypiPackageName(packageName);
// Check permission
if (!(await this.checkPermission(token, normalized, 'write'))) {
return this.errorResponse(403, 'Insufficient permissions');
}
// Calculate and verify hashes
const hashes: Record<string, string> = {};
// Always calculate SHA256
const actualSha256 = await helpers.calculateHash(fileData, 'sha256');
hashes.sha256 = actualSha256;
// Verify client-provided SHA256 if present
if (formData.sha256_digest && formData.sha256_digest !== actualSha256) {
return this.errorResponse(400, 'SHA256 hash mismatch');
}
// Calculate MD5 if requested
if (formData.md5_digest) {
const actualMd5 = await helpers.calculateHash(fileData, 'md5');
hashes.md5 = actualMd5;
// Verify if client provided MD5
if (formData.md5_digest !== actualMd5) {
return this.errorResponse(400, 'MD5 hash mismatch');
}
}
// Calculate Blake2b if requested
if (formData.blake2_256_digest) {
const actualBlake2b = await helpers.calculateHash(fileData, 'blake2b');
hashes.blake2b = actualBlake2b;
// Verify if client provided Blake2b
if (formData.blake2_256_digest !== actualBlake2b) {
return this.errorResponse(400, 'Blake2b hash mismatch');
}
}
// Store file
await this.storage.putPypiPackageFile(normalized, filename, fileData);
// Update metadata
let metadata = await this.storage.getPypiPackageMetadata(normalized);
if (!metadata) {
metadata = {
name: normalized,
versions: {},
};
}
if (!metadata.versions[version]) {
metadata.versions[version] = {
version,
files: [],
};
}
// Add file to version
metadata.versions[version].files.push({
filename,
path: `pypi/packages/${normalized}/${filename}`,
filetype,
python_version: pyversion,
hashes,
size: fileData.length,
'requires-python': formData.requires_python,
'upload-time': new Date().toISOString(),
'uploaded-by': token.userId,
});
// Store core metadata if provided
if (formData.summary || formData.description) {
metadata.versions[version].metadata = helpers.extractCoreMetadata(formData);
}
metadata['last-modified'] = new Date().toISOString();
await this.storage.putPypiPackageMetadata(normalized, metadata);
this.logger.log('info', `Package uploaded: ${normalized} ${version}`, {
filename,
size: fileData.length
});
return {
status: 201,
headers: { 'Content-Type': 'application/json' },
body: {
message: 'Package uploaded successfully',
url: `${this.registryUrl}/pypi/packages/${normalized}/${filename}`
},
};
} catch (error) {
this.logger.log('error', 'Upload failed', { error: (error as Error).message });
return this.errorResponse(500, 'Upload failed: ' + (error as Error).message);
}
}
/**
* Handle package download
*/
private async handleDownload(packageName: string, filename: string): Promise<IResponse> {
const normalized = helpers.normalizePypiPackageName(packageName);
const fileData = await this.storage.getPypiPackageFile(normalized, filename);
if (!fileData) {
return {
status: 404,
headers: { 'Content-Type': 'application/json' },
body: { error: 'File not found' },
};
}
return {
status: 200,
headers: {
'Content-Type': 'application/octet-stream',
'Content-Disposition': `attachment; filename="${filename}"`,
'Content-Length': fileData.length.toString()
},
body: fileData,
};
}
/**
* Handle package JSON API (all versions)
* Returns format compatible with official PyPI JSON API
*/
private async handlePackageJson(packageName: string): Promise<IResponse> {
const normalized = helpers.normalizePypiPackageName(packageName);
const metadata = await this.storage.getPypiPackageMetadata(normalized);
if (!metadata) {
return this.errorResponse(404, 'Package not found');
}
// Find latest version for info
const versions = Object.keys(metadata.versions || {});
const latestVersion = versions.length > 0 ? versions[versions.length - 1] : null;
const latestMeta = latestVersion ? metadata.versions[latestVersion] : null;
// Build URLs array from latest version files
const urls = latestMeta?.files?.map((file: any) => ({
filename: file.filename,
url: `${this.registryUrl}/pypi/packages/${normalized}/${file.filename}`,
digests: file.hashes,
requires_python: file['requires-python'],
size: file.size,
upload_time: file['upload-time'],
packagetype: file.filetype,
python_version: file.python_version,
})) || [];
// Build releases object
const releases: Record<string, any[]> = {};
for (const [ver, verMeta] of Object.entries(metadata.versions || {})) {
releases[ver] = (verMeta as any).files?.map((file: any) => ({
filename: file.filename,
url: `${this.registryUrl}/pypi/packages/${normalized}/${file.filename}`,
digests: file.hashes,
requires_python: file['requires-python'],
size: file.size,
upload_time: file['upload-time'],
packagetype: file.filetype,
python_version: file.python_version,
})) || [];
}
const response = {
info: {
name: normalized,
version: latestVersion,
summary: latestMeta?.metadata?.summary,
description: latestMeta?.metadata?.description,
author: latestMeta?.metadata?.author,
author_email: latestMeta?.metadata?.['author-email'],
license: latestMeta?.metadata?.license,
requires_python: latestMeta?.files?.[0]?.['requires-python'],
...latestMeta?.metadata,
},
urls,
releases,
};
return {
status: 200,
headers: {
'Content-Type': 'application/json',
'Cache-Control': 'public, max-age=300'
},
body: response,
};
}
/**
* Handle version-specific JSON API
* Returns format compatible with official PyPI JSON API
*/
private async handleVersionJson(packageName: string, version: string): Promise<IResponse> {
const normalized = helpers.normalizePypiPackageName(packageName);
const metadata = await this.storage.getPypiPackageMetadata(normalized);
if (!metadata || !metadata.versions[version]) {
return this.errorResponse(404, 'Version not found');
}
const verMeta = metadata.versions[version];
// Build URLs array from version files
const urls = verMeta.files?.map((file: any) => ({
filename: file.filename,
url: `${this.registryUrl}/pypi/packages/${normalized}/${file.filename}`,
digests: file.hashes,
requires_python: file['requires-python'],
size: file.size,
upload_time: file['upload-time'],
packagetype: file.filetype,
python_version: file.python_version,
})) || [];
const response = {
info: {
name: normalized,
version,
summary: verMeta.metadata?.summary,
description: verMeta.metadata?.description,
author: verMeta.metadata?.author,
author_email: verMeta.metadata?.['author-email'],
license: verMeta.metadata?.license,
requires_python: verMeta.files?.[0]?.['requires-python'],
...verMeta.metadata,
},
urls,
};
return {
status: 200,
headers: {
'Content-Type': 'application/json',
'Cache-Control': 'public, max-age=300'
},
body: response,
};
}
/**
* Handle package deletion
*/
private async handleDeletePackage(packageName: string, token: IAuthToken | null): Promise<IResponse> {
if (!token) {
return this.errorResponse(401, 'Authentication required');
}
const normalized = helpers.normalizePypiPackageName(packageName);
if (!(await this.checkPermission(token, normalized, 'delete'))) {
return this.errorResponse(403, 'Insufficient permissions');
}
await this.storage.deletePypiPackage(normalized);
this.logger.log('info', `Package deleted: ${normalized}`);
return {
status: 204,
headers: {},
body: Buffer.from(''),
};
}
/**
* Handle version deletion
*/
private async handleDeleteVersion(
packageName: string,
version: string,
token: IAuthToken | null
): Promise<IResponse> {
if (!token) {
return this.errorResponse(401, 'Authentication required');
}
const normalized = helpers.normalizePypiPackageName(packageName);
if (!(await this.checkPermission(token, normalized, 'delete'))) {
return this.errorResponse(403, 'Insufficient permissions');
}
await this.storage.deletePypiPackageVersion(normalized, version);
this.logger.log('info', `Version deleted: ${normalized} ${version}`);
return {
status: 204,
headers: {},
body: Buffer.from(''),
};
}
/**
* Helper: Create error response
*/
private errorResponse(status: number, message: string): IResponse {
const error: IPypiError = { error: message, status };
return {
status,
headers: { 'Content-Type': 'application/json' },
body: error,
};
}
}

299
ts/pypi/helpers.pypi.ts Normal file
View File

@@ -0,0 +1,299 @@
/**
* Helper functions for PyPI registry
* Package name normalization, HTML generation, etc.
*/
import type { IPypiFile, IPypiPackageMetadata } from './interfaces.pypi.js';
/**
* Normalize package name according to PEP 503
* Lowercase and replace runs of [._-] with a single dash
* @param name - Package name
* @returns Normalized name
*/
export function normalizePypiPackageName(name: string): string {
return name
.toLowerCase()
.replace(/[-_.]+/g, '-');
}
/**
* Escape HTML special characters to prevent XSS
* @param str - String to escape
* @returns Escaped string
*/
export function escapeHtml(str: string): string {
return str
.replace(/&/g, '&amp;')
.replace(/</g, '&lt;')
.replace(/>/g, '&gt;')
.replace(/"/g, '&quot;')
.replace(/'/g, '&#039;');
}
/**
* Generate PEP 503 compliant HTML for root index (all packages)
* @param packages - List of package names
* @returns HTML string
*/
export function generateSimpleRootHtml(packages: string[]): string {
const links = packages
.map(pkg => {
const normalized = normalizePypiPackageName(pkg);
return ` <a href="${escapeHtml(normalized)}/">${escapeHtml(pkg)}</a>`;
})
.join('\n');
return `<!DOCTYPE html>
<html>
<head>
<meta name="pypi:repository-version" content="1.0">
<title>Simple Index</title>
</head>
<body>
<h1>Simple Index</h1>
${links}
</body>
</html>`;
}
/**
* Generate PEP 503 compliant HTML for package index (file list)
* @param packageName - Package name (normalized)
* @param files - List of files
* @param baseUrl - Base URL for downloads
* @returns HTML string
*/
export function generateSimplePackageHtml(
packageName: string,
files: IPypiFile[],
baseUrl: string
): string {
const links = files
.map(file => {
// Build URL
let url = file.url;
if (!url.startsWith('http://') && !url.startsWith('https://')) {
// Relative URL - make it absolute
url = `${baseUrl}/packages/${packageName}/${file.filename}`;
}
// Add hash fragment
const hashName = Object.keys(file.hashes)[0];
const hashValue = file.hashes[hashName];
const fragment = hashName && hashValue ? `#${hashName}=${hashValue}` : '';
// Build data attributes
const dataAttrs: string[] = [];
if (file['requires-python']) {
const escaped = escapeHtml(file['requires-python']);
dataAttrs.push(`data-requires-python="${escaped}"`);
}
if (file['gpg-sig'] !== undefined) {
dataAttrs.push(`data-gpg-sig="${file['gpg-sig'] ? 'true' : 'false'}"`);
}
if (file.yanked) {
const reason = typeof file.yanked === 'string' ? file.yanked : '';
if (reason) {
dataAttrs.push(`data-yanked="${escapeHtml(reason)}"`);
} else {
dataAttrs.push(`data-yanked=""`);
}
}
const dataAttrStr = dataAttrs.length > 0 ? ' ' + dataAttrs.join(' ') : '';
return ` <a href="${escapeHtml(url)}${fragment}"${dataAttrStr}>${escapeHtml(file.filename)}</a>`;
})
.join('\n');
return `<!DOCTYPE html>
<html>
<head>
<meta name="pypi:repository-version" content="1.0">
<title>Links for ${escapeHtml(packageName)}</title>
</head>
<body>
<h1>Links for ${escapeHtml(packageName)}</h1>
${links}
</body>
</html>`;
}
/**
* Parse filename to extract package info
* Supports wheel and sdist formats
* @param filename - Package filename
* @returns Parsed info or null
*/
export function parsePackageFilename(filename: string): {
name: string;
version: string;
filetype: 'bdist_wheel' | 'sdist';
pythonVersion?: string;
} | null {
// Wheel format: {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl
const wheelMatch = filename.match(/^([a-zA-Z0-9_.-]+?)-([a-zA-Z0-9_.]+?)(?:-(\d+))?-([^-]+)-([^-]+)-([^-]+)\.whl$/);
if (wheelMatch) {
return {
name: wheelMatch[1],
version: wheelMatch[2],
filetype: 'bdist_wheel',
pythonVersion: wheelMatch[4],
};
}
// Sdist tar.gz format: {name}-{version}.tar.gz
const sdistTarMatch = filename.match(/^([a-zA-Z0-9_.-]+?)-([a-zA-Z0-9_.]+)\.tar\.gz$/);
if (sdistTarMatch) {
return {
name: sdistTarMatch[1],
version: sdistTarMatch[2],
filetype: 'sdist',
pythonVersion: 'source',
};
}
// Sdist zip format: {name}-{version}.zip
const sdistZipMatch = filename.match(/^([a-zA-Z0-9_.-]+?)-([a-zA-Z0-9_.]+)\.zip$/);
if (sdistZipMatch) {
return {
name: sdistZipMatch[1],
version: sdistZipMatch[2],
filetype: 'sdist',
pythonVersion: 'source',
};
}
return null;
}
/**
* Calculate hash digest for a buffer
* @param data - Data to hash
* @param algorithm - Hash algorithm (sha256, md5, blake2b)
* @returns Hex-encoded hash
*/
export async function calculateHash(data: Buffer, algorithm: 'sha256' | 'md5' | 'blake2b'): Promise<string> {
const crypto = await import('crypto');
let hash: any;
if (algorithm === 'blake2b') {
// Node.js uses 'blake2b512' for blake2b
hash = crypto.createHash('blake2b512');
} else {
hash = crypto.createHash(algorithm);
}
hash.update(data);
return hash.digest('hex');
}
/**
* Validate package name
* Must contain only ASCII letters, numbers, ., -, and _
* @param name - Package name
* @returns true if valid
*/
export function isValidPackageName(name: string): boolean {
return /^[a-zA-Z0-9._-]+$/.test(name);
}
/**
* Validate version string (basic check)
* @param version - Version string
* @returns true if valid
*/
export function isValidVersion(version: string): boolean {
// Basic check - allows numbers, letters, dots, hyphens, underscores
// More strict validation would follow PEP 440
return /^[a-zA-Z0-9._-]+$/.test(version);
}
/**
* Extract metadata from package metadata
* Filters and normalizes metadata fields
* @param metadata - Raw metadata object
* @returns Filtered metadata
*/
export function extractCoreMetadata(metadata: Record<string, any>): Record<string, any> {
const coreFields = [
'metadata-version',
'name',
'version',
'platform',
'supported-platform',
'summary',
'description',
'description-content-type',
'keywords',
'home-page',
'download-url',
'author',
'author-email',
'maintainer',
'maintainer-email',
'license',
'classifier',
'requires-python',
'requires-dist',
'requires-external',
'provides-dist',
'project-url',
'provides-extra',
];
const result: Record<string, any> = {};
for (const [key, value] of Object.entries(metadata)) {
const normalizedKey = key.toLowerCase().replace(/_/g, '-');
if (coreFields.includes(normalizedKey)) {
result[normalizedKey] = value;
}
}
return result;
}
/**
* Generate JSON API response for package list (PEP 691)
* @param packages - List of package names
* @returns JSON object
*/
export function generateJsonRootResponse(packages: string[]): any {
return {
meta: {
'api-version': '1.0',
},
projects: packages.map(name => ({ name })),
};
}
/**
* Generate JSON API response for package files (PEP 691)
* @param packageName - Package name (normalized)
* @param files - List of files
* @returns JSON object
*/
export function generateJsonPackageResponse(packageName: string, files: IPypiFile[]): any {
return {
meta: {
'api-version': '1.0',
},
name: packageName,
files: files.map(file => ({
filename: file.filename,
url: file.url,
hashes: file.hashes,
'requires-python': file['requires-python'],
'dist-info-metadata': file['dist-info-metadata'],
'gpg-sig': file['gpg-sig'],
yanked: file.yanked,
size: file.size,
'upload-time': file['upload-time'],
})),
};
}

8
ts/pypi/index.ts Normal file
View File

@@ -0,0 +1,8 @@
/**
* PyPI Registry Module
* Python Package Index implementation
*/
export * from './interfaces.pypi.js';
export * from './classes.pypiregistry.js';
export * as pypiHelpers from './helpers.pypi.js';

316
ts/pypi/interfaces.pypi.ts Normal file
View File

@@ -0,0 +1,316 @@
/**
* PyPI Registry Type Definitions
* Compliant with PEP 503 (Simple API), PEP 691 (JSON API), and PyPI upload API
*/
/**
* File information for a package distribution
* Used in both PEP 503 HTML and PEP 691 JSON responses
*/
export interface IPypiFile {
/** Filename (e.g., "package-1.0.0-py3-none-any.whl") */
filename: string;
/** Download URL (absolute or relative) */
url: string;
/** Hash digests (multiple algorithms supported in JSON) */
hashes: Record<string, string>;
/** Python version requirement (PEP 345 format) */
'requires-python'?: string;
/** Whether distribution info metadata is available (PEP 658) */
'dist-info-metadata'?: boolean | { sha256: string };
/** Whether GPG signature is available */
'gpg-sig'?: boolean;
/** Yank status: false or reason string */
yanked?: boolean | string;
/** File size in bytes */
size?: number;
/** Upload timestamp */
'upload-time'?: string;
}
/**
* Package metadata stored internally
* Consolidated from multiple file uploads
*/
export interface IPypiPackageMetadata {
/** Normalized package name */
name: string;
/** Map of version to file list */
versions: Record<string, IPypiVersionMetadata>;
/** Timestamp of last update */
'last-modified'?: string;
}
/**
* Metadata for a specific version
*/
export interface IPypiVersionMetadata {
/** Version string */
version: string;
/** Files for this version (wheels, sdists) */
files: IPypiFileMetadata[];
/** Core metadata fields */
metadata?: IPypiCoreMetadata;
/** Whether entire version is yanked */
yanked?: boolean | string;
/** Upload timestamp */
'upload-time'?: string;
}
/**
* Internal file metadata
*/
export interface IPypiFileMetadata {
filename: string;
/** Storage key/path */
path: string;
/** File type: bdist_wheel or sdist */
filetype: 'bdist_wheel' | 'sdist';
/** Python version tag */
python_version: string;
/** Hash digests */
hashes: Record<string, string>;
/** File size in bytes */
size: number;
/** Python version requirement */
'requires-python'?: string;
/** Whether this file is yanked */
yanked?: boolean | string;
/** Upload timestamp */
'upload-time': string;
/** Uploader user ID */
'uploaded-by': string;
}
/**
* Core metadata fields (subset of PEP 566)
* These are extracted from package uploads
*/
export interface IPypiCoreMetadata {
/** Metadata version */
'metadata-version': string;
/** Package name */
name: string;
/** Version string */
version: string;
/** Platform compatibility */
platform?: string;
/** Supported platforms */
'supported-platform'?: string;
/** Summary/description */
summary?: string;
/** Long description */
description?: string;
/** Description content type (text/plain, text/markdown, text/x-rst) */
'description-content-type'?: string;
/** Keywords */
keywords?: string;
/** Homepage URL */
'home-page'?: string;
/** Download URL */
'download-url'?: string;
/** Author name */
author?: string;
/** Author email */
'author-email'?: string;
/** Maintainer name */
maintainer?: string;
/** Maintainer email */
'maintainer-email'?: string;
/** License */
license?: string;
/** Classifiers (Trove classifiers) */
classifier?: string[];
/** Python version requirement */
'requires-python'?: string;
/** Dist name requirement */
'requires-dist'?: string[];
/** External requirement */
'requires-external'?: string[];
/** Provides dist */
'provides-dist'?: string[];
/** Project URLs */
'project-url'?: string[];
/** Provides extra */
'provides-extra'?: string[];
}
/**
* PEP 503: Simple API root response (project list)
*/
export interface IPypiSimpleRootHtml {
/** List of project names */
projects: string[];
}
/**
* PEP 503: Simple API project response (file list)
*/
export interface IPypiSimpleProjectHtml {
/** Normalized project name */
name: string;
/** List of files */
files: IPypiFile[];
}
/**
* PEP 691: JSON API root response
*/
export interface IPypiJsonRoot {
/** API metadata */
meta: {
/** API version (e.g., "1.0") */
'api-version': string;
};
/** List of projects */
projects: Array<{
/** Project name */
name: string;
}>;
}
/**
* PEP 691: JSON API project response
*/
export interface IPypiJsonProject {
/** Normalized project name */
name: string;
/** API metadata */
meta: {
/** API version (e.g., "1.0") */
'api-version': string;
};
/** List of files */
files: IPypiFile[];
}
/**
* Upload form data (multipart/form-data fields)
* Based on PyPI legacy upload API
*/
export interface IPypiUploadForm {
/** Action type (always "file_upload") */
':action': 'file_upload';
/** Protocol version (always "1") */
protocol_version: '1';
/** File content (binary) */
content: Buffer;
/** File type */
filetype: 'bdist_wheel' | 'sdist';
/** Python version tag */
pyversion: string;
/** Package name */
name: string;
/** Version string */
version: string;
/** Metadata version */
metadata_version: string;
/** Hash digests (at least one required) */
md5_digest?: string;
sha256_digest?: string;
blake2_256_digest?: string;
/** Optional attestations */
attestations?: string; // JSON array
/** Optional core metadata fields */
summary?: string;
description?: string;
description_content_type?: string;
author?: string;
author_email?: string;
maintainer?: string;
maintainer_email?: string;
license?: string;
keywords?: string;
home_page?: string;
download_url?: string;
requires_python?: string;
classifiers?: string[];
platform?: string;
[key: string]: any; // Allow additional metadata fields
}
/**
* JSON API upload response
*/
export interface IPypiUploadResponse {
/** Success message */
message?: string;
/** URL of uploaded file */
url?: string;
}
/**
* Error response structure
*/
export interface IPypiError {
/** Error message */
error: string;
/** HTTP status code */
status?: number;
/** Additional error details */
details?: string[];
}
/**
* Search query parameters
*/
export interface IPypiSearchQuery {
/** Search term */
q?: string;
/** Page number */
page?: number;
/** Results per page */
per_page?: number;
}
/**
* Search result for a single package
*/
export interface IPypiSearchResult {
/** Package name */
name: string;
/** Latest version */
version: string;
/** Summary */
summary: string;
/** Description */
description?: string;
}
/**
* Search response structure
*/
export interface IPypiSearchResponse {
/** Search results */
results: IPypiSearchResult[];
/** Result count */
count: number;
/** Current page */
page: number;
/** Total pages */
pages: number;
}
/**
* Yank request
*/
export interface IPypiYankRequest {
/** Package name */
name: string;
/** Version to yank */
version: string;
/** Optional filename (specific file) */
filename?: string;
/** Reason for yanking */
reason?: string;
}
/**
* Yank response
*/
export interface IPypiYankResponse {
/** Success indicator */
success: boolean;
/** Message */
message?: string;
}

View File

@@ -0,0 +1,732 @@
import { Smartlog } from '@push.rocks/smartlog';
import { BaseRegistry } from '../core/classes.baseregistry.js';
import { RegistryStorage } from '../core/classes.registrystorage.js';
import { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js';
import type {
IRubyGemsMetadata,
IRubyGemsVersionMetadata,
IRubyGemsUploadResponse,
IRubyGemsYankResponse,
IRubyGemsError,
ICompactIndexInfoEntry,
} from './interfaces.rubygems.js';
import * as helpers from './helpers.rubygems.js';
/**
* RubyGems registry implementation
* Implements Compact Index API and RubyGems protocol
*/
export class RubyGemsRegistry extends BaseRegistry {
private storage: RegistryStorage;
private authManager: AuthManager;
private basePath: string = '/rubygems';
private registryUrl: string;
private logger: Smartlog;
constructor(
storage: RegistryStorage,
authManager: AuthManager,
basePath: string = '/rubygems',
registryUrl: string = 'http://localhost:5000/rubygems'
) {
super();
this.storage = storage;
this.authManager = authManager;
this.basePath = basePath;
this.registryUrl = registryUrl;
// Initialize logger
this.logger = new Smartlog({
logContext: {
company: 'push.rocks',
companyunit: 'smartregistry',
containerName: 'rubygems-registry',
environment: (process.env.NODE_ENV as any) || 'development',
runtime: 'node',
zone: 'rubygems'
}
});
this.logger.enableConsole();
}
public async init(): Promise<void> {
// Initialize Compact Index files if not exist
const existingVersions = await this.storage.getRubyGemsVersions();
if (!existingVersions) {
const versions = helpers.generateCompactIndexVersions([]);
await this.storage.putRubyGemsVersions(versions);
this.logger.log('info', 'Initialized RubyGems Compact Index');
}
const existingNames = await this.storage.getRubyGemsNames();
if (!existingNames) {
const names = helpers.generateNamesFile([]);
await this.storage.putRubyGemsNames(names);
this.logger.log('info', 'Initialized RubyGems names file');
}
}
public getBasePath(): string {
return this.basePath;
}
public async handleRequest(context: IRequestContext): Promise<IResponse> {
let path = context.path.replace(this.basePath, '');
// Extract token (Authorization header)
const token = await this.extractToken(context);
this.logger.log('debug', `handleRequest: ${context.method} ${path}`, {
method: context.method,
path,
hasAuth: !!token
});
// Compact Index endpoints
if (path === '/versions' && context.method === 'GET') {
return this.handleVersionsFile(context);
}
if (path === '/names' && context.method === 'GET') {
return this.handleNamesFile();
}
// Info file: GET /info/{gem}
const infoMatch = path.match(/^\/info\/([^\/]+)$/);
if (infoMatch && context.method === 'GET') {
return this.handleInfoFile(infoMatch[1]);
}
// Gem download: GET /gems/{gem}-{version}[-{platform}].gem
const downloadMatch = path.match(/^\/gems\/(.+\.gem)$/);
if (downloadMatch && context.method === 'GET') {
return this.handleDownload(downloadMatch[1]);
}
// Legacy specs endpoints (Marshal format)
if (path === '/specs.4.8.gz' && context.method === 'GET') {
return this.handleSpecs(false);
}
if (path === '/latest_specs.4.8.gz' && context.method === 'GET') {
return this.handleSpecs(true);
}
// Quick gemspec endpoint: GET /quick/Marshal.4.8/{gem}-{version}.gemspec.rz
const quickMatch = path.match(/^\/quick\/Marshal\.4\.8\/(.+)\.gemspec\.rz$/);
if (quickMatch && context.method === 'GET') {
return this.handleQuickGemspec(quickMatch[1]);
}
// API v1 endpoints
if (path.startsWith('/api/v1/')) {
return this.handleApiRequest(path.substring(7), context, token);
}
return {
status: 404,
headers: { 'Content-Type': 'application/json' },
body: { error: 'Not Found' },
};
}
/**
* Check if token has permission for resource
*/
protected async checkPermission(
token: IAuthToken | null,
resource: string,
action: string
): Promise<boolean> {
if (!token) return false;
return this.authManager.authorize(token, `rubygems:gem:${resource}`, action);
}
/**
* Extract authentication token from request
*/
private async extractToken(context: IRequestContext): Promise<IAuthToken | null> {
const authHeader = context.headers['authorization'] || context.headers['Authorization'];
if (!authHeader) return null;
// RubyGems typically uses plain API key in Authorization header
return this.authManager.validateToken(authHeader, 'rubygems');
}
/**
* Handle /versions endpoint (Compact Index)
* Supports conditional GET with If-None-Match header
*/
private async handleVersionsFile(context: IRequestContext): Promise<IResponse> {
const content = await this.storage.getRubyGemsVersions();
if (!content) {
return this.errorResponse(500, 'Versions file not initialized');
}
const etag = `"${await helpers.calculateMD5(content)}"`;
// Handle conditional GET with If-None-Match
const ifNoneMatch = context.headers['if-none-match'] || context.headers['If-None-Match'];
if (ifNoneMatch && ifNoneMatch === etag) {
return {
status: 304,
headers: {
'ETag': etag,
'Cache-Control': 'public, max-age=60',
},
body: null,
};
}
return {
status: 200,
headers: {
'Content-Type': 'text/plain; charset=utf-8',
'Cache-Control': 'public, max-age=60',
'ETag': etag
},
body: Buffer.from(content),
};
}
/**
* Handle /names endpoint (Compact Index)
*/
private async handleNamesFile(): Promise<IResponse> {
const content = await this.storage.getRubyGemsNames();
if (!content) {
return this.errorResponse(500, 'Names file not initialized');
}
return {
status: 200,
headers: {
'Content-Type': 'text/plain; charset=utf-8',
'Cache-Control': 'public, max-age=300'
},
body: Buffer.from(content),
};
}
/**
* Handle /info/{gem} endpoint (Compact Index)
*/
private async handleInfoFile(gemName: string): Promise<IResponse> {
const content = await this.storage.getRubyGemsInfo(gemName);
if (!content) {
return {
status: 404,
headers: { 'Content-Type': 'text/plain' },
body: Buffer.from('Not Found'),
};
}
return {
status: 200,
headers: {
'Content-Type': 'text/plain; charset=utf-8',
'Cache-Control': 'public, max-age=300',
'ETag': `"${await helpers.calculateMD5(content)}"`
},
body: Buffer.from(content),
};
}
/**
* Handle gem file download
*/
private async handleDownload(filename: string): Promise<IResponse> {
const parsed = helpers.parseGemFilename(filename);
if (!parsed) {
return this.errorResponse(400, 'Invalid gem filename');
}
const gemData = await this.storage.getRubyGemsGem(
parsed.name,
parsed.version,
parsed.platform
);
if (!gemData) {
return this.errorResponse(404, 'Gem not found');
}
return {
status: 200,
headers: {
'Content-Type': 'application/octet-stream',
'Content-Disposition': `attachment; filename="${filename}"`,
'Content-Length': gemData.length.toString()
},
body: gemData,
};
}
/**
* Handle API v1 requests
*/
private async handleApiRequest(
path: string,
context: IRequestContext,
token: IAuthToken | null
): Promise<IResponse> {
// Upload gem: POST /gems
if (path === '/gems' && context.method === 'POST') {
return this.handleUpload(context, token);
}
// Yank gem: DELETE /gems/yank
if (path === '/gems/yank' && context.method === 'DELETE') {
return this.handleYank(context, token);
}
// Unyank gem: PUT /gems/unyank
if (path === '/gems/unyank' && context.method === 'PUT') {
return this.handleUnyank(context, token);
}
// Version list: GET /versions/{gem}.json
const versionsMatch = path.match(/^\/versions\/([^\/]+)\.json$/);
if (versionsMatch && context.method === 'GET') {
return this.handleVersionsJson(versionsMatch[1]);
}
// Dependencies: GET /dependencies?gems={list}
if (path.startsWith('/dependencies') && context.method === 'GET') {
const gemsParam = context.query?.gems || '';
return this.handleDependencies(gemsParam);
}
return this.errorResponse(404, 'API endpoint not found');
}
/**
* Handle gem upload
* POST /api/v1/gems
*/
private async handleUpload(context: IRequestContext, token: IAuthToken | null): Promise<IResponse> {
if (!token) {
return this.errorResponse(401, 'Authentication required');
}
try {
// Extract gem data from request body
const gemData = context.body as Buffer;
if (!gemData || gemData.length === 0) {
return this.errorResponse(400, 'No gem file provided');
}
// Try to get metadata from query params or headers first
let gemName = context.query?.name || context.headers['x-gem-name'] as string | undefined;
let version = context.query?.version || context.headers['x-gem-version'] as string | undefined;
let platform = context.query?.platform || context.headers['x-gem-platform'] as string | undefined;
// If not provided, try to extract from gem binary
if (!gemName || !version || !platform) {
const extracted = await helpers.extractGemMetadata(gemData);
if (extracted) {
gemName = gemName || extracted.name;
version = version || extracted.version;
platform = platform || extracted.platform;
}
}
if (!gemName || !version) {
return this.errorResponse(400, 'Gem name and version required (provide in query, headers, or valid gem format)');
}
// Validate gem name
if (!helpers.isValidGemName(gemName)) {
return this.errorResponse(400, 'Invalid gem name');
}
// Check permission
if (!(await this.checkPermission(token, gemName, 'write'))) {
return this.errorResponse(403, 'Insufficient permissions');
}
// Calculate checksum
const checksum = await helpers.calculateSHA256(gemData);
// Store gem file
await this.storage.putRubyGemsGem(gemName, version, gemData, platform);
// Update metadata
let metadata: IRubyGemsMetadata = await this.storage.getRubyGemsMetadata(gemName) || {
name: gemName,
versions: {},
};
const versionKey = platform ? `${version}-${platform}` : version;
metadata.versions[versionKey] = {
version,
platform,
checksum,
size: gemData.length,
'upload-time': new Date().toISOString(),
'uploaded-by': token.userId,
dependencies: [], // Would extract from gem spec
requirements: [],
};
metadata['last-modified'] = new Date().toISOString();
await this.storage.putRubyGemsMetadata(gemName, metadata);
// Update Compact Index info file
await this.updateCompactIndexForGem(gemName, metadata);
// Update versions file
await this.updateVersionsFile(gemName, version, platform || 'ruby', false);
// Update names file
await this.updateNamesFile(gemName);
this.logger.log('info', `Gem uploaded: ${gemName} ${version}`, {
platform,
size: gemData.length
});
return {
status: 201,
headers: { 'Content-Type': 'application/json' },
body: {
message: 'Gem uploaded successfully',
name: gemName,
version,
},
};
} catch (error) {
this.logger.log('error', 'Upload failed', { error: (error as Error).message });
return this.errorResponse(500, 'Upload failed: ' + (error as Error).message);
}
}
/**
* Handle gem yanking
* DELETE /api/v1/gems/yank
*/
private async handleYank(context: IRequestContext, token: IAuthToken | null): Promise<IResponse> {
if (!token) {
return this.errorResponse(401, 'Authentication required');
}
const gemName = context.query?.gem_name;
const version = context.query?.version;
const platform = context.query?.platform;
if (!gemName || !version) {
return this.errorResponse(400, 'Gem name and version required');
}
if (!(await this.checkPermission(token, gemName, 'yank'))) {
return this.errorResponse(403, 'Insufficient permissions');
}
// Update metadata to mark as yanked
const metadata = await this.storage.getRubyGemsMetadata(gemName);
if (!metadata) {
return this.errorResponse(404, 'Gem not found');
}
const versionKey = platform ? `${version}-${platform}` : version;
if (!metadata.versions[versionKey]) {
return this.errorResponse(404, 'Version not found');
}
metadata.versions[versionKey].yanked = true;
await this.storage.putRubyGemsMetadata(gemName, metadata);
// Update Compact Index
await this.updateCompactIndexForGem(gemName, metadata);
await this.updateVersionsFile(gemName, version, platform || 'ruby', true);
this.logger.log('info', `Gem yanked: ${gemName} ${version}`);
return {
status: 200,
headers: { 'Content-Type': 'application/json' },
body: {
success: true,
message: 'Gem yanked successfully'
},
};
}
/**
* Handle gem unyanking
* PUT /api/v1/gems/unyank
*/
private async handleUnyank(context: IRequestContext, token: IAuthToken | null): Promise<IResponse> {
if (!token) {
return this.errorResponse(401, 'Authentication required');
}
const gemName = context.query?.gem_name;
const version = context.query?.version;
const platform = context.query?.platform;
if (!gemName || !version) {
return this.errorResponse(400, 'Gem name and version required');
}
if (!(await this.checkPermission(token, gemName, 'write'))) {
return this.errorResponse(403, 'Insufficient permissions');
}
const metadata = await this.storage.getRubyGemsMetadata(gemName);
if (!metadata) {
return this.errorResponse(404, 'Gem not found');
}
const versionKey = platform ? `${version}-${platform}` : version;
if (!metadata.versions[versionKey]) {
return this.errorResponse(404, 'Version not found');
}
metadata.versions[versionKey].yanked = false;
await this.storage.putRubyGemsMetadata(gemName, metadata);
// Update Compact Index
await this.updateCompactIndexForGem(gemName, metadata);
await this.updateVersionsFile(gemName, version, platform || 'ruby', false);
this.logger.log('info', `Gem unyanked: ${gemName} ${version}`);
return {
status: 200,
headers: { 'Content-Type': 'application/json' },
body: {
success: true,
message: 'Gem unyanked successfully'
},
};
}
/**
* Handle versions JSON API
*/
private async handleVersionsJson(gemName: string): Promise<IResponse> {
const metadata = await this.storage.getRubyGemsMetadata(gemName);
if (!metadata) {
return this.errorResponse(404, 'Gem not found');
}
const versions = Object.values(metadata.versions).map((v: any) => ({
version: v.version,
platform: v.platform,
uploadTime: v['upload-time'],
}));
const response = helpers.generateVersionsJson(gemName, versions);
return {
status: 200,
headers: {
'Content-Type': 'application/json',
'Cache-Control': 'public, max-age=300'
},
body: response,
};
}
/**
* Handle dependencies query
*/
private async handleDependencies(gemsParam: string): Promise<IResponse> {
const gemNames = gemsParam.split(',').filter(n => n.trim());
const result = new Map();
for (const gemName of gemNames) {
const metadata = await this.storage.getRubyGemsMetadata(gemName);
if (metadata) {
const versions = Object.values(metadata.versions).map((v: any) => ({
version: v.version,
platform: v.platform,
dependencies: v.dependencies || [],
}));
result.set(gemName, versions);
}
}
const response = helpers.generateDependenciesJson(result);
return {
status: 200,
headers: { 'Content-Type': 'application/json' },
body: response,
};
}
/**
* Update Compact Index info file for a gem
*/
private async updateCompactIndexForGem(
gemName: string,
metadata: IRubyGemsMetadata
): Promise<void> {
const entries: ICompactIndexInfoEntry[] = Object.values(metadata.versions)
.filter(v => !v.yanked) // Exclude yanked from info file
.map(v => ({
version: v.version,
platform: v.platform,
dependencies: v.dependencies || [],
requirements: v.requirements || [],
checksum: v.checksum,
}));
const content = helpers.generateCompactIndexInfo(entries);
await this.storage.putRubyGemsInfo(gemName, content);
}
/**
* Update versions file with new/updated gem
*/
private async updateVersionsFile(
gemName: string,
version: string,
platform: string,
yanked: boolean
): Promise<void> {
const existingVersions = await this.storage.getRubyGemsVersions();
if (!existingVersions) return;
// Calculate info file checksum
const infoContent = await this.storage.getRubyGemsInfo(gemName) || '';
const infoChecksum = await helpers.calculateMD5(infoContent);
const updated = helpers.updateCompactIndexVersions(
existingVersions,
gemName,
{ version, platform: platform !== 'ruby' ? platform : undefined, yanked },
infoChecksum
);
await this.storage.putRubyGemsVersions(updated);
}
/**
* Update names file with new gem
*/
private async updateNamesFile(gemName: string): Promise<void> {
const existingNames = await this.storage.getRubyGemsNames();
if (!existingNames) return;
const lines = existingNames.split('\n').filter(l => l !== '---');
if (!lines.includes(gemName)) {
lines.push(gemName);
lines.sort();
const updated = helpers.generateNamesFile(lines);
await this.storage.putRubyGemsNames(updated);
}
}
/**
* Handle /specs.4.8.gz and /latest_specs.4.8.gz endpoints
* Returns gzipped Marshal array of [name, version, platform] tuples
* @param latestOnly - If true, only return latest version of each gem
*/
private async handleSpecs(latestOnly: boolean): Promise<IResponse> {
try {
const names = await this.storage.getRubyGemsNames();
if (!names) {
return {
status: 200,
headers: {
'Content-Type': 'application/octet-stream',
},
body: await helpers.generateSpecsGz([]),
};
}
const gemNames = names.split('\n').filter(l => l && l !== '---');
const specs: Array<[string, string, string]> = [];
for (const gemName of gemNames) {
const metadata = await this.storage.getRubyGemsMetadata(gemName);
if (!metadata) continue;
const versions = (Object.values(metadata.versions) as IRubyGemsVersionMetadata[])
.filter(v => !v.yanked)
.sort((a, b) => {
// Sort by version descending
return b.version.localeCompare(a.version, undefined, { numeric: true });
});
if (latestOnly && versions.length > 0) {
// Only include latest version
const latest = versions[0];
specs.push([gemName, latest.version, latest.platform || 'ruby']);
} else {
// Include all versions
for (const v of versions) {
specs.push([gemName, v.version, v.platform || 'ruby']);
}
}
}
const gzippedSpecs = await helpers.generateSpecsGz(specs);
return {
status: 200,
headers: {
'Content-Type': 'application/octet-stream',
},
body: gzippedSpecs,
};
} catch (error) {
this.logger.log('error', 'Failed to generate specs', { error: (error as Error).message });
return this.errorResponse(500, 'Failed to generate specs');
}
}
/**
* Handle /quick/Marshal.4.8/{gem}-{version}.gemspec.rz endpoint
* Returns compressed gemspec for a specific gem version
* @param gemVersionStr - Gem name and version string (e.g., "rails-7.0.0" or "rails-7.0.0-x86_64-linux")
*/
private async handleQuickGemspec(gemVersionStr: string): Promise<IResponse> {
// Parse the gem-version string
const parsed = helpers.parseGemFilename(gemVersionStr + '.gem');
if (!parsed) {
return this.errorResponse(400, 'Invalid gemspec path');
}
const metadata = await this.storage.getRubyGemsMetadata(parsed.name);
if (!metadata) {
return this.errorResponse(404, 'Gem not found');
}
const versionKey = parsed.platform ? `${parsed.version}-${parsed.platform}` : parsed.version;
const versionMeta = metadata.versions[versionKey];
if (!versionMeta) {
return this.errorResponse(404, 'Version not found');
}
// Generate a minimal gemspec representation
const gemspecData = await helpers.generateGemspecRz(parsed.name, versionMeta);
return {
status: 200,
headers: {
'Content-Type': 'application/octet-stream',
},
body: gemspecData,
};
}
/**
* Helper: Create error response
*/
private errorResponse(status: number, message: string): IResponse {
const error: IRubyGemsError = { error: message, status };
return {
status,
headers: { 'Content-Type': 'application/json' },
body: error,
};
}
}

View File

@@ -0,0 +1,573 @@
/**
* Helper functions for RubyGems registry
* Compact Index generation, dependency formatting, etc.
*/
import * as plugins from '../plugins.js';
import type {
IRubyGemsVersion,
IRubyGemsDependency,
IRubyGemsRequirement,
ICompactIndexVersionsEntry,
ICompactIndexInfoEntry,
IRubyGemsMetadata,
} from './interfaces.rubygems.js';
/**
* Generate Compact Index versions file
* Format: GEMNAME [-]VERSION_PLATFORM[,VERSION_PLATFORM,...] MD5
* @param entries - Version entries for all gems
* @returns Compact Index versions file content
*/
export function generateCompactIndexVersions(entries: ICompactIndexVersionsEntry[]): string {
const lines: string[] = [];
// Add metadata header
lines.push(`created_at: ${new Date().toISOString()}`);
lines.push('---');
// Add gem entries
for (const entry of entries) {
const versions = entry.versions
.map(v => {
const yanked = v.yanked ? '-' : '';
const platform = v.platform && v.platform !== 'ruby' ? `_${v.platform}` : '';
return `${yanked}${v.version}${platform}`;
})
.join(',');
lines.push(`${entry.name} ${versions} ${entry.infoChecksum}`);
}
return lines.join('\n');
}
/**
* Generate Compact Index info file for a gem
* Format: VERSION[-PLATFORM] [DEP[,DEP,...]]|REQ[,REQ,...]
* @param entries - Info entries for gem versions
* @returns Compact Index info file content
*/
export function generateCompactIndexInfo(entries: ICompactIndexInfoEntry[]): string {
const lines: string[] = ['---']; // Info files start with ---
for (const entry of entries) {
// Build version string with optional platform
const versionStr = entry.platform && entry.platform !== 'ruby'
? `${entry.version}-${entry.platform}`
: entry.version;
// Build dependencies string
const depsStr = entry.dependencies.length > 0
? entry.dependencies.map(formatDependency).join(',')
: '';
// Build requirements string (checksum is always required)
const reqParts: string[] = [`checksum:${entry.checksum}`];
for (const req of entry.requirements) {
reqParts.push(`${req.type}:${req.requirement}`);
}
const reqStr = reqParts.join(',');
// Combine: VERSION[-PLATFORM] [DEPS]|REQS
const depPart = depsStr ? ` ${depsStr}` : '';
lines.push(`${versionStr}${depPart}|${reqStr}`);
}
return lines.join('\n');
}
/**
* Format a dependency for Compact Index
* Format: GEM:CONSTRAINT[&CONSTRAINT]
* @param dep - Dependency object
* @returns Formatted dependency string
*/
export function formatDependency(dep: IRubyGemsDependency): string {
return `${dep.name}:${dep.requirement}`;
}
/**
* Parse dependency string from Compact Index
* @param depStr - Dependency string
* @returns Dependency object
*/
export function parseDependency(depStr: string): IRubyGemsDependency {
const [name, ...reqParts] = depStr.split(':');
const requirement = reqParts.join(':'); // Handle :: in gem names
return { name, requirement };
}
/**
* Generate names file (newline-separated gem names)
* @param names - List of gem names
* @returns Names file content
*/
export function generateNamesFile(names: string[]): string {
return `---\n${names.sort().join('\n')}`;
}
/**
* Calculate MD5 hash for Compact Index checksum
* @param content - Content to hash
* @returns MD5 hash (hex)
*/
export async function calculateMD5(content: string): Promise<string> {
const crypto = await import('crypto');
return crypto.createHash('md5').update(content).digest('hex');
}
/**
* Calculate SHA256 hash for gem files
* @param data - Data to hash
* @returns SHA256 hash (hex)
*/
export async function calculateSHA256(data: Buffer): Promise<string> {
const crypto = await import('crypto');
return crypto.createHash('sha256').update(data).digest('hex');
}
/**
* Parse gem filename to extract name, version, and platform
* @param filename - Gem filename (e.g., "rails-7.0.0-x86_64-linux.gem")
* @returns Parsed info or null
*/
export function parseGemFilename(filename: string): {
name: string;
version: string;
platform?: string;
} | null {
if (!filename.endsWith('.gem')) return null;
const withoutExt = filename.slice(0, -4); // Remove .gem
// Try to match: name-version-platform
// Platform can contain hyphens (e.g., x86_64-linux)
const parts = withoutExt.split('-');
if (parts.length < 2) return null;
// Find version (first part that starts with a digit)
let versionIndex = -1;
for (let i = 1; i < parts.length; i++) {
if (/^\d/.test(parts[i])) {
versionIndex = i;
break;
}
}
if (versionIndex === -1) return null;
const name = parts.slice(0, versionIndex).join('-');
const version = parts[versionIndex];
const platform = versionIndex + 1 < parts.length
? parts.slice(versionIndex + 1).join('-')
: undefined;
return {
name,
version,
platform: platform && platform !== 'ruby' ? platform : undefined,
};
}
/**
* Validate gem name
* Must contain only ASCII letters, numbers, _, and -
* @param name - Gem name
* @returns true if valid
*/
export function isValidGemName(name: string): boolean {
return /^[a-zA-Z0-9_-]+$/.test(name);
}
/**
* Validate version string
* Basic semantic versioning check
* @param version - Version string
* @returns true if valid
*/
export function isValidVersion(version: string): boolean {
// Allow semver and other common Ruby version formats
return /^[\d.a-zA-Z_-]+$/.test(version);
}
/**
* Build version list entry for Compact Index
* @param versions - Version info
* @returns Version list string
*/
export function buildVersionList(versions: Array<{
version: string;
platform?: string;
yanked: boolean;
}>): string {
return versions
.map(v => {
const yanked = v.yanked ? '-' : '';
const platform = v.platform && v.platform !== 'ruby' ? `_${v.platform}` : '';
return `${yanked}${v.version}${platform}`;
})
.join(',');
}
/**
* Parse version list from Compact Index
* @param versionStr - Version list string
* @returns Parsed versions
*/
export function parseVersionList(versionStr: string): Array<{
version: string;
platform?: string;
yanked: boolean;
}> {
return versionStr.split(',').map(v => {
const yanked = v.startsWith('-');
const withoutYank = yanked ? v.substring(1) : v;
// Split on _ to separate version from platform
const [version, ...platformParts] = withoutYank.split('_');
const platform = platformParts.length > 0 ? platformParts.join('_') : undefined;
return {
version,
platform: platform && platform !== 'ruby' ? platform : undefined,
yanked,
};
});
}
/**
* Generate JSON response for /api/v1/versions/{gem}.json
* @param gemName - Gem name
* @param versions - Version list
* @returns JSON response object
*/
export function generateVersionsJson(
gemName: string,
versions: Array<{
version: string;
platform?: string;
uploadTime?: string;
}>
): any {
return {
name: gemName,
versions: versions.map(v => ({
number: v.version,
platform: v.platform || 'ruby',
built_at: v.uploadTime,
})),
};
}
/**
* Generate JSON response for /api/v1/dependencies
* @param gems - Map of gem names to version dependencies
* @returns JSON response array
*/
export function generateDependenciesJson(gems: Map<string, Array<{
version: string;
platform?: string;
dependencies: IRubyGemsDependency[];
}>>): any {
const result: any[] = [];
for (const [name, versions] of gems) {
for (const v of versions) {
result.push({
name,
number: v.version,
platform: v.platform || 'ruby',
dependencies: v.dependencies.map(d => ({
name: d.name,
requirements: d.requirement,
})),
});
}
}
return result;
}
/**
* Update Compact Index versions file with new gem version
* Handles append-only semantics for the current month
* @param existingContent - Current versions file content
* @param gemName - Gem name
* @param newVersion - New version info
* @param infoChecksum - MD5 of info file
* @returns Updated versions file content
*/
export function updateCompactIndexVersions(
existingContent: string,
gemName: string,
newVersion: { version: string; platform?: string; yanked: boolean },
infoChecksum: string
): string {
const lines = existingContent.split('\n');
const headerEndIndex = lines.findIndex(l => l === '---');
if (headerEndIndex === -1) {
throw new Error('Invalid Compact Index versions file');
}
const header = lines.slice(0, headerEndIndex + 1);
const entries = lines.slice(headerEndIndex + 1).filter(l => l.trim());
// Find existing entry for gem
const gemLineIndex = entries.findIndex(l => l.startsWith(`${gemName} `));
const versionStr = buildVersionList([newVersion]);
if (gemLineIndex >= 0) {
// Append to existing entry
const parts = entries[gemLineIndex].split(' ');
const existingVersions = parts[1];
const updatedVersions = `${existingVersions},${versionStr}`;
entries[gemLineIndex] = `${gemName} ${updatedVersions} ${infoChecksum}`;
} else {
// Add new entry
entries.push(`${gemName} ${versionStr} ${infoChecksum}`);
entries.sort(); // Keep alphabetical
}
return [...header, ...entries].join('\n');
}
/**
* Update Compact Index info file with new version
* @param existingContent - Current info file content
* @param newEntry - New version entry
* @returns Updated info file content
*/
export function updateCompactIndexInfo(
existingContent: string,
newEntry: ICompactIndexInfoEntry
): string {
const lines = existingContent ? existingContent.split('\n').filter(l => l !== '---') : [];
// Build version string
const versionStr = newEntry.platform && newEntry.platform !== 'ruby'
? `${newEntry.version}-${newEntry.platform}`
: newEntry.version;
// Build dependencies string
const depsStr = newEntry.dependencies.length > 0
? newEntry.dependencies.map(formatDependency).join(',')
: '';
// Build requirements string
const reqParts: string[] = [`checksum:${newEntry.checksum}`];
for (const req of newEntry.requirements) {
reqParts.push(`${req.type}:${req.requirement}`);
}
const reqStr = reqParts.join(',');
// Combine
const depPart = depsStr ? ` ${depsStr}` : '';
const newLine = `${versionStr}${depPart}|${reqStr}`;
lines.push(newLine);
return `---\n${lines.join('\n')}`;
}
/**
* Extract gem specification from .gem file
* Note: This is a simplified version. Full implementation would use tar + gzip + Marshal
* @param gemData - Gem file data
* @returns Extracted spec or null
*/
export async function extractGemSpec(gemData: Buffer): Promise<any | null> {
try {
// .gem files are gzipped tar archives
// They contain metadata.gz which has Marshal-encoded spec
// This is a placeholder - full implementation would need:
// 1. Unzip outer gzip
// 2. Untar to find metadata.gz
// 3. Unzip metadata.gz
// 4. Parse Ruby Marshal format
// For now, return null and expect metadata to be provided
return null;
} catch (error) {
return null;
}
}
/**
* Extract basic metadata from a gem file
* Gem files are plain tar archives (NOT gzipped) containing:
* - metadata.gz: gzipped YAML with gem specification
* - data.tar.gz: gzipped tar with actual gem files
* This function extracts and parses the metadata.gz to get name/version/platform
* @param gemData - Gem file data
* @returns Extracted metadata or null
*/
export async function extractGemMetadata(gemData: Buffer): Promise<{
name: string;
version: string;
platform?: string;
} | null> {
try {
// Step 1: Extract the plain tar archive to get metadata.gz
const smartArchive = plugins.smartarchive.SmartArchive.create();
const files = await smartArchive.buffer(gemData).toSmartFiles();
// Find metadata.gz
const metadataFile = files.find(f => f.path === 'metadata.gz' || f.relative === 'metadata.gz');
if (!metadataFile) {
return null;
}
// Step 2: Decompress the gzipped metadata
const gzipTools = new plugins.smartarchive.GzipTools();
const metadataYaml = await gzipTools.decompress(metadataFile.contentBuffer);
const yamlContent = metadataYaml.toString('utf-8');
// Step 3: Parse the YAML to extract name, version, platform
// Look for name: field in YAML
const nameMatch = yamlContent.match(/name:\s*([^\n\r]+)/);
// Look for version in Ruby YAML format: version: !ruby/object:Gem::Version\n version: X.X.X
const versionMatch = yamlContent.match(/version:\s*!ruby\/object:Gem::Version[\s\S]*?version:\s*['"]?([^'"\n\r]+)/);
// Also try simpler version format
const simpleVersionMatch = !versionMatch ? yamlContent.match(/^version:\s*['"]?(\d[^'"\n\r]*)/m) : null;
// Look for platform
const platformMatch = yamlContent.match(/platform:\s*([^\n\r]+)/);
const name = nameMatch?.[1]?.trim();
const version = versionMatch?.[1]?.trim() || simpleVersionMatch?.[1]?.trim();
const platform = platformMatch?.[1]?.trim();
if (name && version) {
return {
name,
version,
platform: platform && platform !== 'ruby' ? platform : undefined,
};
}
return null;
} catch (error) {
// Log error for debugging but return null gracefully
console.error('Failed to extract gem metadata:', error);
return null;
}
}
/**
* Generate gzipped specs array for /specs.4.8.gz and /latest_specs.4.8.gz
* The format is a gzipped Ruby Marshal array of [name, version, platform] tuples
* Since we can't easily generate Ruby Marshal format, we'll use a simple format
* that represents the same data structure as a gzipped binary blob
* @param specs - Array of [name, version, platform] tuples
* @returns Gzipped specs data
*/
export async function generateSpecsGz(specs: Array<[string, string, string]>): Promise<Buffer> {
const gzipTools = new plugins.smartarchive.GzipTools();
// Create a simplified binary representation
// Real RubyGems uses Ruby Marshal format, but for compatibility we'll create
// a gzipped representation that tools can recognize as valid
// Format: Simple binary encoding of specs array
// Each spec: name_length(2 bytes) + name + version_length(2 bytes) + version + platform_length(2 bytes) + platform
const parts: Buffer[] = [];
// Header: number of specs (4 bytes)
const headerBuf = Buffer.alloc(4);
headerBuf.writeUInt32LE(specs.length, 0);
parts.push(headerBuf);
for (const [name, version, platform] of specs) {
const nameBuf = Buffer.from(name, 'utf-8');
const versionBuf = Buffer.from(version, 'utf-8');
const platformBuf = Buffer.from(platform, 'utf-8');
const nameLenBuf = Buffer.alloc(2);
nameLenBuf.writeUInt16LE(nameBuf.length, 0);
const versionLenBuf = Buffer.alloc(2);
versionLenBuf.writeUInt16LE(versionBuf.length, 0);
const platformLenBuf = Buffer.alloc(2);
platformLenBuf.writeUInt16LE(platformBuf.length, 0);
parts.push(nameLenBuf, nameBuf, versionLenBuf, versionBuf, platformLenBuf, platformBuf);
}
const uncompressed = Buffer.concat(parts);
return gzipTools.compress(uncompressed);
}
/**
* Generate compressed gemspec for /quick/Marshal.4.8/{gem}-{version}.gemspec.rz
* The format is a zlib-compressed Ruby Marshal representation of the gemspec
* Since we can't easily generate Ruby Marshal, we'll create a simplified format
* @param name - Gem name
* @param versionMeta - Version metadata
* @returns Zlib-compressed gemspec data
*/
export async function generateGemspecRz(
name: string,
versionMeta: {
version: string;
platform?: string;
checksum: string;
dependencies?: Array<{ name: string; requirement: string }>;
}
): Promise<Buffer> {
const zlib = await import('zlib');
const { promisify } = await import('util');
const deflate = promisify(zlib.deflate);
// Create a YAML-like representation that can be parsed
const gemspecYaml = `--- !ruby/object:Gem::Specification
name: ${name}
version: !ruby/object:Gem::Version
version: ${versionMeta.version}
platform: ${versionMeta.platform || 'ruby'}
authors: []
date: ${new Date().toISOString().split('T')[0]}
dependencies: []
description:
email:
executables: []
extensions: []
extra_rdoc_files: []
files: []
homepage:
licenses: []
metadata: {}
post_install_message:
rdoc_options: []
require_paths:
- lib
required_ruby_version: !ruby/object:Gem::Requirement
requirements:
- - ">="
- !ruby/object:Gem::Version
version: '0'
required_rubygems_version: !ruby/object:Gem::Requirement
requirements:
- - ">="
- !ruby/object:Gem::Version
version: '0'
requirements: []
rubygems_version: 3.0.0
signing_key:
specification_version: 4
summary:
test_files: []
`;
// Use zlib deflate (not gzip) for .rz files
return deflate(Buffer.from(gemspecYaml, 'utf-8'));
}

8
ts/rubygems/index.ts Normal file
View File

@@ -0,0 +1,8 @@
/**
* RubyGems Registry Module
* RubyGems/Bundler Compact Index implementation
*/
export * from './interfaces.rubygems.js';
export * from './classes.rubygemsregistry.js';
export * as rubygemsHelpers from './helpers.rubygems.js';

View File

@@ -0,0 +1,251 @@
/**
* RubyGems Registry Type Definitions
* Compliant with Compact Index API and RubyGems protocol
*/
/**
* Gem version entry in compact index
*/
export interface IRubyGemsVersion {
/** Version number */
version: string;
/** Platform (e.g., ruby, x86_64-linux) */
platform?: string;
/** Dependencies */
dependencies?: IRubyGemsDependency[];
/** Requirements */
requirements?: IRubyGemsRequirement[];
/** Whether this version is yanked */
yanked?: boolean;
/** SHA256 checksum of .gem file */
checksum?: string;
}
/**
* Gem dependency specification
*/
export interface IRubyGemsDependency {
/** Gem name */
name: string;
/** Version requirement (e.g., ">= 1.0", "~> 2.0") */
requirement: string;
}
/**
* Gem requirements (ruby version, rubygems version, etc.)
*/
export interface IRubyGemsRequirement {
/** Requirement type (ruby, rubygems) */
type: 'ruby' | 'rubygems';
/** Version requirement */
requirement: string;
}
/**
* Complete gem metadata
*/
export interface IRubyGemsMetadata {
/** Gem name */
name: string;
/** All versions */
versions: Record<string, IRubyGemsVersionMetadata>;
/** Last modified timestamp */
'last-modified'?: string;
}
/**
* Version-specific metadata
*/
export interface IRubyGemsVersionMetadata {
/** Version number */
version: string;
/** Platform */
platform?: string;
/** Authors */
authors?: string[];
/** Description */
description?: string;
/** Summary */
summary?: string;
/** Homepage */
homepage?: string;
/** License */
license?: string;
/** Dependencies */
dependencies?: IRubyGemsDependency[];
/** Requirements */
requirements?: IRubyGemsRequirement[];
/** SHA256 checksum */
checksum: string;
/** File size */
size: number;
/** Upload timestamp */
'upload-time': string;
/** Uploader */
'uploaded-by': string;
/** Yanked status */
yanked?: boolean;
/** Yank reason */
'yank-reason'?: string;
}
/**
* Compact index versions file entry
* Format: GEMNAME [-]VERSION_PLATFORM[,VERSION_PLATFORM,...] MD5
*/
export interface ICompactIndexVersionsEntry {
/** Gem name */
name: string;
/** Versions (with optional platform and yank flag) */
versions: Array<{
version: string;
platform?: string;
yanked: boolean;
}>;
/** MD5 checksum of info file */
infoChecksum: string;
}
/**
* Compact index info file entry
* Format: VERSION[-PLATFORM] [DEP[,DEP,...]]|REQ[,REQ,...]
*/
export interface ICompactIndexInfoEntry {
/** Version number */
version: string;
/** Platform (optional) */
platform?: string;
/** Dependencies */
dependencies: IRubyGemsDependency[];
/** Requirements */
requirements: IRubyGemsRequirement[];
/** SHA256 checksum */
checksum: string;
}
/**
* Gem upload request
*/
export interface IRubyGemsUploadRequest {
/** Gem file data */
gemData: Buffer;
/** Gem filename */
filename: string;
}
/**
* Gem upload response
*/
export interface IRubyGemsUploadResponse {
/** Success message */
message?: string;
/** Gem name */
name?: string;
/** Version */
version?: string;
}
/**
* Yank request
*/
export interface IRubyGemsYankRequest {
/** Gem name */
gem_name: string;
/** Version to yank */
version: string;
/** Platform (optional) */
platform?: string;
}
/**
* Yank response
*/
export interface IRubyGemsYankResponse {
/** Success indicator */
success: boolean;
/** Message */
message?: string;
}
/**
* Version info response (JSON)
*/
export interface IRubyGemsVersionInfo {
/** Gem name */
name: string;
/** Versions list */
versions: Array<{
/** Version number */
number: string;
/** Platform */
platform?: string;
/** Build date */
built_at?: string;
/** Download count */
downloads_count?: number;
}>;
}
/**
* Dependencies query response
*/
export interface IRubyGemsDependenciesResponse {
/** Dependencies for requested gems */
dependencies: Array<{
/** Gem name */
name: string;
/** Version */
number: string;
/** Platform */
platform?: string;
/** Dependencies */
dependencies: Array<{
name: string;
requirements: string;
}>;
}>;
}
/**
* Error response structure
*/
export interface IRubyGemsError {
/** Error message */
error: string;
/** HTTP status code */
status?: number;
}
/**
* Gem specification (extracted from .gem file)
*/
export interface IRubyGemsSpec {
/** Gem name */
name: string;
/** Version */
version: string;
/** Platform */
platform?: string;
/** Authors */
authors?: string[];
/** Email */
email?: string;
/** Homepage */
homepage?: string;
/** Summary */
summary?: string;
/** Description */
description?: string;
/** License */
license?: string;
/** Dependencies */
dependencies?: IRubyGemsDependency[];
/** Required Ruby version */
required_ruby_version?: string;
/** Required RubyGems version */
required_rubygems_version?: string;
/** Files */
files?: string[];
/** Requirements */
requirements?: string[];
}