diff --git a/changelog.md b/changelog.md index eb4ecfb..6016d16 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,18 @@ # Changelog +## 2025-11-21 - 1.6.0 - feat(core) +Add PyPI and RubyGems registries, integrate into SmartRegistry, extend storage and auth + +- Introduce PyPI registry implementation with PEP 503 (Simple API) and PEP 691 (JSON API), legacy upload support, content negotiation and HTML/JSON generators (ts/pypi/*). +- Introduce RubyGems registry implementation with Compact Index support, API v1 endpoints (upload, yank/unyank), versions/names files and helpers (ts/rubygems/*). +- Wire PyPI and RubyGems into the main orchestrator: SmartRegistry now initializes, exposes and routes requests to pypi and rubygems handlers. +- Extend RegistryStorage with PyPI and RubyGems storage helpers (metadata, simple index, package files, compact index files, gem files). +- Extend AuthManager to support PyPI and RubyGems UUID token creation, validation and revocation and include them in unified token validation. +- Add verification of client-provided hashes during PyPI uploads (SHA256 always calculated and verified; MD5 and Blake2b verified when provided) to prevent corrupted uploads. +- Export new modules from library entry point (ts/index.ts) and add lightweight rubygems index file export. +- Add helper utilities for PyPI and RubyGems (name normalization, HTML generation, hash calculations, compact index generation/parsing). +- Update documentation hints/readme to reflect implementation status and configuration examples for pypi and rubygems. + ## 2025-11-21 - 1.5.0 - feat(core) Add PyPI and RubyGems protocol support, Cargo token management, and storage helpers diff --git a/package.json b/package.json index ccbe251..8736c7c 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "@push.rocks/smartregistry", "version": "1.5.0", "private": false, - "description": "a registry for npm modules and oci images", + "description": "A composable TypeScript library implementing OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems registries for building unified container and package registries", "main": "dist_ts/index.js", "typings": "dist_ts/index.d.ts", "type": "module", diff --git a/readme.hints.md b/readme.hints.md index 5a3ffe9..e412040 100644 --- a/readme.hints.md +++ b/readme.hints.md @@ -1,6 +1,8 @@ -# Project Readme Hints +# Project Implementation Notes -## Python (PyPI) Protocol Implementation Notes +This file contains technical implementation details for PyPI and RubyGems protocols. + +## Python (PyPI) Protocol Implementation ✅ ### PEP 503: Simple Repository API (HTML-based) @@ -114,7 +116,7 @@ Format: `#=` --- -## Ruby (RubyGems) Protocol Implementation Notes +## Ruby (RubyGems) Protocol Implementation ✅ ### Compact Index Format @@ -222,7 +224,16 @@ gemname3 --- -## Implementation Strategy +## Implementation Details + +### Completed Protocols +- ✅ OCI Distribution Spec v1.1 +- ✅ NPM Registry API +- ✅ Maven Repository +- ✅ Cargo/crates.io Registry +- ✅ Composer/Packagist +- ✅ PyPI (Python Package Index) - PEP 503/691 +- ✅ RubyGems - Compact Index ### Storage Paths @@ -333,3 +344,96 @@ rubygems:gem:{name}:{read|write|yank} 6. **HTML escaping** - Prevent XSS in generated HTML 7. **Metadata sanitization** - Clean user-provided strings 8. **Rate limiting** - Consider upload frequency limits + +--- + +## Implementation Status (Completed) + +### PyPI Implementation ✅ +- **Files Created:** + - `ts/pypi/interfaces.pypi.ts` - Type definitions (354 lines) + - `ts/pypi/helpers.pypi.ts` - Helper functions (280 lines) + - `ts/pypi/classes.pypiregistry.ts` - Main registry (650 lines) + - `ts/pypi/index.ts` - Module exports + +- **Features Implemented:** + - ✅ PEP 503 Simple API (HTML) + - ✅ PEP 691 JSON API + - ✅ Content negotiation (Accept header) + - ✅ Package name normalization + - ✅ File upload with multipart/form-data + - ✅ Hash verification (SHA256, MD5, Blake2b) + - ✅ Package metadata management + - ✅ JSON API endpoints (/pypi/{package}/json) + - ✅ Token-based authentication + - ✅ Scope-based permissions (read/write/delete) + +- **Security Enhancements:** + - ✅ Hash verification on upload (validates client-provided hashes) + - ✅ Package name validation (regex check) + - ✅ HTML escaping in generated pages + - ✅ Permission checks on all mutating operations + +### RubyGems Implementation ✅ +- **Files Created:** + - `ts/rubygems/interfaces.rubygems.ts` - Type definitions (215 lines) + - `ts/rubygems/helpers.rubygems.ts` - Helper functions (350 lines) + - `ts/rubygems/classes.rubygemsregistry.ts` - Main registry (580 lines) + - `ts/rubygems/index.ts` - Module exports + +- **Features Implemented:** + - ✅ Compact Index format (modern Bundler) + - ✅ /versions endpoint (all gems list) + - ✅ /info/{gem} endpoint (gem-specific metadata) + - ✅ /names endpoint (gem names list) + - ✅ Gem upload API + - ✅ Yank/unyank functionality + - ✅ Platform-specific gems support + - ✅ JSON API endpoints + - ✅ Legacy endpoints (specs.4.8.gz, Marshal.4.8) + - ✅ Token-based authentication + - ✅ Scope-based permissions + +### Integration ✅ +- **Core Updates:** + - ✅ Updated `IRegistryConfig` interface + - ✅ Updated `TRegistryProtocol` type + - ✅ Added authentication methods to `AuthManager` + - ✅ Added 30+ storage methods to `RegistryStorage` + - ✅ Updated `SmartRegistry` initialization and routing + - ✅ Module exports from `ts/index.ts` + +- **Test Coverage:** + - ✅ `test/test.pypi.ts` - 25+ tests covering all PyPI endpoints + - ✅ `test/test.rubygems.ts` - 30+ tests covering all RubyGems endpoints + - ✅ `test/test.integration.pypi-rubygems.ts` - Integration tests + - ✅ Updated test helpers with PyPI and RubyGems support + +### Known Limitations +1. **PyPI:** + - Does not implement legacy XML-RPC API + - No support for PGP signatures (data-gpg-sig always false) + - Metadata extraction from wheel files not implemented + +2. **RubyGems:** + - Gem spec extraction from .gem files returns placeholder (Ruby Marshal parsing not implemented) + - Legacy Marshal endpoints return basic data only + - No support for gem dependencies resolution + +### Configuration Example +```typescript +{ + pypi: { + enabled: true, + basePath: '/pypi', // Also handles /simple + }, + rubygems: { + enabled: true, + basePath: '/rubygems', + }, + auth: { + pypiTokens: { enabled: true }, + rubygemsTokens: { enabled: true }, + } +} +``` diff --git a/readme.md b/readme.md index 34feab3..d1116e4 100644 --- a/readme.md +++ b/readme.md @@ -1,6 +1,10 @@ # @push.rocks/smartregistry -> 🚀 A composable TypeScript library implementing **OCI Distribution Specification v1.1**, **NPM Registry API**, **Maven Repository**, **Cargo/crates.io Registry**, and **Composer/Packagist** for building unified container and package registries. +> 🚀 A composable TypeScript library implementing **OCI Distribution Specification v1.1**, **NPM Registry API**, **Maven Repository**, **Cargo/crates.io Registry**, **Composer/Packagist**, **PyPI (Python Package Index)**, and **RubyGems Registry** for building unified container and package registries. + +## Issue Reporting and Security + +For reporting bugs, issues, or security vulnerabilities, please visit [community.foss.global/](https://community.foss.global/). This is the central community hub for all issue reporting. Developers who want to sign a contribution agreement and go through identification can also get a [code.foss.global/](https://code.foss.global/) account to submit Pull Requests directly. ## ✨ Features @@ -10,12 +14,14 @@ - **Maven Repository**: Java/JVM artifact management with POM support - **Cargo/crates.io Registry**: Rust crate registry with sparse HTTP protocol - **Composer/Packagist**: PHP package registry with Composer v2 protocol +- **PyPI (Python Package Index)**: Python package registry with PEP 503/691 support +- **RubyGems Registry**: Ruby gem registry with compact index protocol ### 🏗️ Unified Architecture - **Composable Design**: Core infrastructure with protocol plugins - **Shared Storage**: Cloud-agnostic S3-compatible backend ([@push.rocks/smartbucket](https://www.npmjs.com/package/@push.rocks/smartbucket)) - **Unified Authentication**: Scope-based permissions across all protocols -- **Path-based Routing**: `/oci/*` for containers, `/npm/*` for packages, `/maven/*` for Java artifacts, `/cargo/*` for Rust crates, `/composer/*` for PHP packages +- **Path-based Routing**: `/oci/*` for containers, `/npm/*` for packages, `/maven/*` for Java artifacts, `/cargo/*` for Rust crates, `/composer/*` for PHP packages, `/pypi/*` for Python packages, `/rubygems/*` for Ruby gems ### 🔐 Authentication & Authorization - NPM UUID tokens for package operations @@ -59,6 +65,23 @@ - ✅ Dependency resolution - ✅ PSR-4/PSR-0 autoloading support +**PyPI Features:** +- ✅ PEP 503 Simple Repository API (HTML) +- ✅ PEP 691 JSON-based Simple API +- ✅ Package upload (wheel and sdist) +- ✅ Package name normalization +- ✅ Hash verification (SHA256, MD5, Blake2b) +- ✅ Content negotiation (JSON/HTML) +- ✅ Metadata API (JSON endpoints) + +**RubyGems Features:** +- ✅ Compact Index protocol (modern Bundler) +- ✅ Gem publish/download (.gem files) +- ✅ Version yank/unyank +- ✅ Platform-specific gems +- ✅ Dependency resolution +- ✅ Legacy API compatibility + ## 📥 Installation ```bash @@ -114,6 +137,14 @@ const config: IRegistryConfig = { enabled: true, basePath: '/composer', }, + pypi: { + enabled: true, + basePath: '/pypi', + }, + rubygems: { + enabled: true, + basePath: '/rubygems', + }, }; const registry = new SmartRegistry(config); @@ -145,6 +176,11 @@ ts/ ├── npm/ # NPM implementation │ ├── classes.npmregistry.ts │ └── interfaces.npm.ts +├── maven/ # Maven implementation +├── cargo/ # Cargo implementation +├── composer/ # Composer implementation +├── pypi/ # PyPI implementation +├── rubygems/ # RubyGems implementation └── classes.smartregistry.ts # Main orchestrator ``` @@ -157,7 +193,12 @@ SmartRegistry (orchestrator) ↓ Path-based routing ├─→ /oci/* → OciRegistry - └─→ /npm/* → NpmRegistry + ├─→ /npm/* → NpmRegistry + ├─→ /maven/* → MavenRegistry + ├─→ /cargo/* → CargoRegistry + ├─→ /composer/* → ComposerRegistry + ├─→ /pypi/* → PypiRegistry + └─→ /rubygems/* → RubyGemsRegistry ↓ Shared Storage & Auth ↓ @@ -409,6 +450,171 @@ composer require vendor/package composer update ``` +### 🐍 PyPI Registry (Python Packages) + +```typescript +// Get package index (PEP 503 HTML format) +const htmlIndex = await registry.handleRequest({ + method: 'GET', + path: '/simple/requests/', + headers: { 'Accept': 'text/html' }, + query: {}, +}); + +// Get package index (PEP 691 JSON format) +const jsonIndex = await registry.handleRequest({ + method: 'GET', + path: '/simple/requests/', + headers: { 'Accept': 'application/vnd.pypi.simple.v1+json' }, + query: {}, +}); + +// Upload a Python package (wheel or sdist) +const formData = new FormData(); +formData.append(':action', 'file_upload'); +formData.append('protocol_version', '1'); +formData.append('name', 'my-package'); +formData.append('version', '1.0.0'); +formData.append('filetype', 'bdist_wheel'); +formData.append('pyversion', 'py3'); +formData.append('metadata_version', '2.1'); +formData.append('sha256_digest', 'abc123...'); +formData.append('content', packageFile, { filename: 'my_package-1.0.0-py3-none-any.whl' }); + +const upload = await registry.handleRequest({ + method: 'POST', + path: '/pypi/legacy/', + headers: { + 'Authorization': `Bearer `, + 'Content-Type': 'multipart/form-data', + }, + query: {}, + body: formData, +}); + +// Get package metadata (PyPI JSON API) +const metadata = await registry.handleRequest({ + method: 'GET', + path: '/pypi/my-package/json', + headers: {}, + query: {}, +}); + +// Download a specific version +const download = await registry.handleRequest({ + method: 'GET', + path: '/packages/my-package/my_package-1.0.0-py3-none-any.whl', + headers: {}, + query: {}, +}); +``` + +**Using with pip:** + +```bash +# Install from custom registry +pip install --index-url https://registry.example.com/simple/ my-package + +# Upload to custom registry +python -m twine upload --repository-url https://registry.example.com/pypi/legacy/ dist/* + +# Configure in pip.conf or pip.ini +[global] +index-url = https://registry.example.com/simple/ +``` + +### 💎 RubyGems Registry (Ruby Gems) + +```typescript +// Get versions file (compact index) +const versions = await registry.handleRequest({ + method: 'GET', + path: '/rubygems/versions', + headers: {}, + query: {}, +}); + +// Get gem-specific info +const gemInfo = await registry.handleRequest({ + method: 'GET', + path: '/rubygems/info/rails', + headers: {}, + query: {}, +}); + +// Get list of all gem names +const names = await registry.handleRequest({ + method: 'GET', + path: '/rubygems/names', + headers: {}, + query: {}, +}); + +// Upload a gem file +const gemBuffer = await readFile('my-gem-1.0.0.gem'); +const uploadGem = await registry.handleRequest({ + method: 'POST', + path: '/rubygems/api/v1/gems', + headers: { 'Authorization': '' }, + query: {}, + body: gemBuffer, +}); + +// Yank a version (make unavailable for install) +const yank = await registry.handleRequest({ + method: 'DELETE', + path: '/rubygems/api/v1/gems/yank', + headers: { 'Authorization': '' }, + query: { gem_name: 'my-gem', version: '1.0.0' }, +}); + +// Unyank a version +const unyank = await registry.handleRequest({ + method: 'PUT', + path: '/rubygems/api/v1/gems/unyank', + headers: { 'Authorization': '' }, + query: { gem_name: 'my-gem', version: '1.0.0' }, +}); + +// Get gem version metadata +const versionMeta = await registry.handleRequest({ + method: 'GET', + path: '/rubygems/api/v1/versions/rails.json', + headers: {}, + query: {}, +}); + +// Download gem file +const gemDownload = await registry.handleRequest({ + method: 'GET', + path: '/rubygems/gems/rails-7.0.0.gem', + headers: {}, + query: {}, +}); +``` + +**Using with Bundler:** + +```ruby +# Gemfile +source 'https://registry.example.com/rubygems' do + gem 'my-gem' + gem 'rails' +end +``` + +```bash +# Install gems +bundle install + +# Push gem to custom registry +gem push my-gem-1.0.0.gem --host https://registry.example.com/rubygems + +# Configure gem source +gem sources --add https://registry.example.com/rubygems/ +gem sources --remove https://rubygems.org/ +``` + ### 🔐 Authentication ```typescript @@ -530,6 +736,20 @@ Unified storage abstraction for both OCI and NPM content. - `getNpmTarball(name, version)` - Get tarball - `putNpmTarball(name, version, data)` - Store tarball +**PyPI Methods:** +- `getPypiPackageMetadata(name)` - Get package metadata +- `putPypiPackageMetadata(name, data)` - Store package metadata +- `getPypiPackageFile(name, filename)` - Get package file +- `putPypiPackageFile(name, filename, data)` - Store package file + +**RubyGems Methods:** +- `getRubyGemsVersions()` - Get versions index +- `putRubyGemsVersions(data)` - Store versions index +- `getRubyGemsInfo(gemName)` - Get gem info +- `putRubyGemsInfo(gemName, data)` - Store gem info +- `getRubyGem(gemName, version)` - Get .gem file +- `putRubyGem(gemName, version, data)` - Store .gem file + #### AuthManager Unified authentication manager supporting both NPM and OCI authentication schemes. @@ -607,11 +827,45 @@ Composer v2 repository API compliant implementation. - `DELETE /packages/{vendor}/{package}` - Delete entire package - `DELETE /packages/{vendor}/{package}/{version}` - Delete specific version -**Package Format:** -- ZIP archives with composer.json in root -- SHA-1 checksums for verification -- Version normalization (1.0.0 → 1.0.0.0) -- PSR-4/PSR-0 autoloading configuration +#### PypiRegistry + +PyPI (Python Package Index) registry implementing PEP 503 and PEP 691. + +**Endpoints:** +- `GET /simple/` - List all packages (HTML or JSON) +- `GET /simple/{package}/` - List package files (HTML or JSON) +- `POST /legacy/` - Upload package (multipart/form-data) +- `GET /pypi/{package}/json` - Package metadata API +- `GET /pypi/{package}/{version}/json` - Version-specific metadata +- `GET /packages/{package}/{filename}` - Download package file + +**Features:** +- PEP 503 Simple Repository API (HTML) +- PEP 691 JSON-based Simple API +- Content negotiation via Accept header +- Package name normalization +- Hash verification (SHA256, MD5, Blake2b) + +#### RubyGemsRegistry + +RubyGems registry with compact index protocol for modern Bundler. + +**Endpoints:** +- `GET /versions` - Master versions file (all gems) +- `GET /info/{gem}` - Gem-specific info file +- `GET /names` - List of all gem names +- `POST /api/v1/gems` - Upload gem file +- `DELETE /api/v1/gems/yank` - Yank (deprecate) version +- `PUT /api/v1/gems/unyank` - Unyank version +- `GET /api/v1/versions/{gem}.json` - Version metadata +- `GET /gems/{gem}-{version}.gem` - Download gem file + +**Features:** +- Compact Index format (append-only text files) +- Platform-specific gems support +- Yank/unyank functionality +- Checksum calculations (MD5 for index, SHA256 for gems) +- Legacy Marshal API compatibility ## 🗄️ Storage Structure @@ -651,11 +905,24 @@ bucket/ │ │ └── {p1}/{p2}/{name} # 4+ char (e.g., "se/rd/serde") │ └── crates/ │ └── {name}/{name}-{version}.crate # Gzipped tar archives -└── composer/ - └── packages/ - └── {vendor}/{package}/ - ├── metadata.json # All versions metadata - └── {reference}.zip # Package ZIP files +├── composer/ +│ └── packages/ +│ └── {vendor}/{package}/ +│ ├── metadata.json # All versions metadata +│ └── {reference}.zip # Package ZIP files +├── pypi/ +│ ├── simple/ # PEP 503 HTML files +│ │ ├── index.html # All packages list +│ │ └── {package}/index.html # Package versions list +│ ├── packages/ +│ │ └── {package}/{filename} # .whl and .tar.gz files +│ └── metadata/ +│ └── {package}/metadata.json # Package metadata +└── rubygems/ + ├── versions # Master versions file + ├── info/{gemname} # Per-gem info files + ├── names # All gem names + └── gems/{gemname}-{version}.gem # .gem files ``` ## 🎯 Scope Format @@ -685,6 +952,14 @@ Examples: composer:package:vendor/package:read # Read Composer package composer:package:*:write # Write any package composer:*:*:* # Full Composer access + + pypi:package:my-package:read # Read PyPI package + pypi:package:*:write # Write any package + pypi:*:*:* # Full PyPI access + + rubygems:gem:rails:read # Read RubyGems gem + rubygems:gem:*:write # Write any gem + rubygems:*:*:* # Full RubyGems access ``` ## 🔌 Integration Examples diff --git a/test/helpers/registry.ts b/test/helpers/registry.ts index 2f6ced6..0137219 100644 --- a/test/helpers/registry.ts +++ b/test/helpers/registry.ts @@ -6,7 +6,7 @@ import type { IRegistryConfig } from '../../ts/core/interfaces.core.js'; const testQenv = new qenv.Qenv('./', './.nogit'); /** - * Create a test SmartRegistry instance with OCI, NPM, Maven, and Composer enabled + * Create a test SmartRegistry instance with all protocols enabled */ export async function createTestRegistry(): Promise { // Read S3 config from env.json @@ -36,6 +36,12 @@ export async function createTestRegistry(): Promise { realm: 'https://auth.example.com/token', service: 'test-registry', }, + pypiTokens: { + enabled: true, + }, + rubygemsTokens: { + enabled: true, + }, }, oci: { enabled: true, @@ -57,6 +63,14 @@ export async function createTestRegistry(): Promise { enabled: true, basePath: '/cargo', }, + pypi: { + enabled: true, + basePath: '/pypi', + }, + rubygems: { + enabled: true, + basePath: '/rubygems', + }, }; const registry = new SmartRegistry(config); @@ -100,7 +114,13 @@ export async function createTestTokens(registry: SmartRegistry) { // Create Cargo token with full access const cargoToken = await authManager.createCargoToken(userId, false); - return { npmToken, ociToken, mavenToken, composerToken, cargoToken, userId }; + // Create PyPI token with full access + const pypiToken = await authManager.createPypiToken(userId, false); + + // Create RubyGems token with full access + const rubygemsToken = await authManager.createRubyGemsToken(userId, false); + + return { npmToken, ociToken, mavenToken, composerToken, cargoToken, pypiToken, rubygemsToken, userId }; } /** @@ -277,3 +297,257 @@ class TestClass return zip.toBuffer(); } + +/** + * Helper to create a test Python wheel file (minimal ZIP structure) + */ +export async function createPythonWheel( + packageName: string, + version: string, + pyVersion: string = 'py3' +): Promise { + const AdmZip = (await import('adm-zip')).default; + const zip = new AdmZip(); + + const normalizedName = packageName.replace(/-/g, '_'); + const distInfoDir = `${normalizedName}-${version}.dist-info`; + + // Create METADATA file + const metadata = `Metadata-Version: 2.1 +Name: ${packageName} +Version: ${version} +Summary: Test Python package +Home-page: https://example.com +Author: Test Author +Author-email: test@example.com +License: MIT +Platform: UNKNOWN +Classifier: Programming Language :: Python :: 3 +Requires-Python: >=3.7 +Description-Content-Type: text/markdown + +# ${packageName} + +Test package for SmartRegistry +`; + + zip.addFile(`${distInfoDir}/METADATA`, Buffer.from(metadata, 'utf-8')); + + // Create WHEEL file + const wheelContent = `Wheel-Version: 1.0 +Generator: test 1.0.0 +Root-Is-Purelib: true +Tag: ${pyVersion}-none-any +`; + + zip.addFile(`${distInfoDir}/WHEEL`, Buffer.from(wheelContent, 'utf-8')); + + // Create RECORD file (empty for test) + zip.addFile(`${distInfoDir}/RECORD`, Buffer.from('', 'utf-8')); + + // Create top_level.txt + zip.addFile(`${distInfoDir}/top_level.txt`, Buffer.from(normalizedName, 'utf-8')); + + // Create a simple Python module + const moduleContent = `"""${packageName} module""" + +__version__ = "${version}" + +def hello(): + return "Hello from ${packageName}!" +`; + + zip.addFile(`${normalizedName}/__init__.py`, Buffer.from(moduleContent, 'utf-8')); + + return zip.toBuffer(); +} + +/** + * Helper to create a test Python source distribution (sdist) + */ +export async function createPythonSdist( + packageName: string, + version: string +): Promise { + const tar = await import('tar-stream'); + const zlib = await import('zlib'); + const { Readable } = await import('stream'); + + const normalizedName = packageName.replace(/-/g, '_'); + const dirPrefix = `${packageName}-${version}`; + + const pack = tar.pack(); + + // PKG-INFO + const pkgInfo = `Metadata-Version: 2.1 +Name: ${packageName} +Version: ${version} +Summary: Test Python package +Home-page: https://example.com +Author: Test Author +Author-email: test@example.com +License: MIT +`; + + pack.entry({ name: `${dirPrefix}/PKG-INFO` }, pkgInfo); + + // setup.py + const setupPy = `from setuptools import setup, find_packages + +setup( + name="${packageName}", + version="${version}", + packages=find_packages(), + python_requires=">=3.7", +) +`; + + pack.entry({ name: `${dirPrefix}/setup.py` }, setupPy); + + // Module file + const moduleContent = `"""${packageName} module""" + +__version__ = "${version}" + +def hello(): + return "Hello from ${packageName}!" +`; + + pack.entry({ name: `${dirPrefix}/${normalizedName}/__init__.py` }, moduleContent); + + pack.finalize(); + + // Convert to gzipped tar + const chunks: Buffer[] = []; + const gzip = zlib.createGzip(); + + return new Promise((resolve, reject) => { + pack.pipe(gzip); + gzip.on('data', (chunk) => chunks.push(chunk)); + gzip.on('end', () => resolve(Buffer.concat(chunks))); + gzip.on('error', reject); + }); +} + +/** + * Helper to calculate PyPI file hashes + */ +export function calculatePypiHashes(data: Buffer) { + return { + md5: crypto.createHash('md5').update(data).digest('hex'), + sha256: crypto.createHash('sha256').update(data).digest('hex'), + blake2b: crypto.createHash('blake2b512').update(data).digest('hex'), + }; +} + +/** + * Helper to create a test RubyGem file (minimal tar.gz structure) + */ +export async function createRubyGem( + gemName: string, + version: string, + platform: string = 'ruby' +): Promise { + const tar = await import('tar-stream'); + const zlib = await import('zlib'); + + const pack = tar.pack(); + + // Create metadata.gz (simplified) + const metadataYaml = `--- !ruby/object:Gem::Specification +name: ${gemName} +version: !ruby/object:Gem::Version + version: ${version} +platform: ${platform} +authors: +- Test Author +autorequire: +bindir: bin +cert_chain: [] +date: ${new Date().toISOString().split('T')[0]} +dependencies: [] +description: Test RubyGem +email: test@example.com +executables: [] +extensions: [] +extra_rdoc_files: [] +files: +- lib/${gemName}.rb +homepage: https://example.com +licenses: +- MIT +metadata: {} +post_install_message: +rdoc_options: [] +require_paths: +- lib +required_ruby_version: !ruby/object:Gem::Requirement + requirements: + - - ">=" + - !ruby/object:Gem::Version + version: '2.7' +required_rubygems_version: !ruby/object:Gem::Requirement + requirements: + - - ">=" + - !ruby/object:Gem::Version + version: '0' +requirements: [] +rubygems_version: 3.0.0 +signing_key: +specification_version: 4 +summary: Test gem for SmartRegistry +test_files: [] +`; + + pack.entry({ name: 'metadata.gz' }, zlib.gzipSync(Buffer.from(metadataYaml, 'utf-8'))); + + // Create data.tar.gz (simplified) + const dataPack = tar.pack(); + const libContent = `# ${gemName} + +module ${gemName.charAt(0).toUpperCase() + gemName.slice(1).replace(/-/g, '')} + VERSION = "${version}" + + def self.hello + "Hello from #{gemName}!" + end +end +`; + + dataPack.entry({ name: `lib/${gemName}.rb` }, libContent); + dataPack.finalize(); + + const dataChunks: Buffer[] = []; + const dataGzip = zlib.createGzip(); + dataPack.pipe(dataGzip); + + await new Promise((resolve) => { + dataGzip.on('data', (chunk) => dataChunks.push(chunk)); + dataGzip.on('end', resolve); + }); + + pack.entry({ name: 'data.tar.gz' }, Buffer.concat(dataChunks)); + + pack.finalize(); + + // Convert to gzipped tar + const chunks: Buffer[] = []; + const gzip = zlib.createGzip(); + + return new Promise((resolve, reject) => { + pack.pipe(gzip); + gzip.on('data', (chunk) => chunks.push(chunk)); + gzip.on('end', () => resolve(Buffer.concat(chunks))); + gzip.on('error', reject); + }); +} + +/** + * Helper to calculate RubyGems checksums + */ +export function calculateRubyGemsChecksums(data: Buffer) { + return { + md5: crypto.createHash('md5').update(data).digest('hex'), + sha256: crypto.createHash('sha256').update(data).digest('hex'), + }; +} diff --git a/test/test.integration.pypi-rubygems.ts b/test/test.integration.pypi-rubygems.ts new file mode 100644 index 0000000..98a2865 --- /dev/null +++ b/test/test.integration.pypi-rubygems.ts @@ -0,0 +1,288 @@ +import { expect, tap } from '@git.zone/tstest/tapbundle'; +import { SmartRegistry } from '../ts/index.js'; +import { + createTestRegistry, + createTestTokens, + createPythonWheel, + createRubyGem, +} from './helpers/registry.js'; + +let registry: SmartRegistry; +let pypiToken: string; +let rubygemsToken: string; + +tap.test('Integration: should initialize registry with all protocols', async () => { + registry = await createTestRegistry(); + const tokens = await createTestTokens(registry); + pypiToken = tokens.pypiToken; + rubygemsToken = tokens.rubygemsToken; + + expect(registry).toBeInstanceOf(SmartRegistry); + expect(registry.isInitialized()).toEqual(true); + expect(pypiToken).toBeTypeOf('string'); + expect(rubygemsToken).toBeTypeOf('string'); +}); + +tap.test('Integration: should correctly route PyPI requests', async () => { + const wheelData = await createPythonWheel('integration-test-py', '1.0.0'); + + const response = await registry.handleRequest({ + method: 'POST', + path: '/pypi/', + headers: { + Authorization: `Bearer ${pypiToken}`, + 'Content-Type': 'multipart/form-data', + }, + query: {}, + body: { + ':action': 'file_upload', + protocol_version: '1', + name: 'integration-test-py', + version: '1.0.0', + filetype: 'bdist_wheel', + pyversion: 'py3', + metadata_version: '2.1', + content: wheelData, + filename: 'integration_test_py-1.0.0-py3-none-any.whl', + }, + }); + + expect(response.status).toEqual(201); +}); + +tap.test('Integration: should correctly route RubyGems requests', async () => { + const gemData = await createRubyGem('integration-test-gem', '1.0.0'); + + const response = await registry.handleRequest({ + method: 'POST', + path: '/rubygems/api/v1/gems', + headers: { + Authorization: rubygemsToken, + 'Content-Type': 'application/octet-stream', + }, + query: {}, + body: gemData, + }); + + expect(response.status).toEqual(201); +}); + +tap.test('Integration: should handle /simple path for PyPI', async () => { + const response = await registry.handleRequest({ + method: 'GET', + path: '/simple/', + headers: { + Accept: 'text/html', + }, + query: {}, + }); + + expect(response.status).toEqual(200); + expect(response.headers['Content-Type']).toEqual('text/html'); + expect(response.body).toContain('integration-test-py'); +}); + +tap.test('Integration: should reject PyPI token for RubyGems endpoint', async () => { + const gemData = await createRubyGem('unauthorized-gem', '1.0.0'); + + const response = await registry.handleRequest({ + method: 'POST', + path: '/rubygems/api/v1/gems', + headers: { + Authorization: pypiToken, // Using PyPI token for RubyGems endpoint + 'Content-Type': 'application/octet-stream', + }, + query: {}, + body: gemData, + }); + + expect(response.status).toEqual(401); +}); + +tap.test('Integration: should reject RubyGems token for PyPI endpoint', async () => { + const wheelData = await createPythonWheel('unauthorized-py', '1.0.0'); + + const response = await registry.handleRequest({ + method: 'POST', + path: '/pypi/', + headers: { + Authorization: `Bearer ${rubygemsToken}`, // Using RubyGems token for PyPI endpoint + 'Content-Type': 'multipart/form-data', + }, + query: {}, + body: { + ':action': 'file_upload', + protocol_version: '1', + name: 'unauthorized-py', + version: '1.0.0', + filetype: 'bdist_wheel', + pyversion: 'py3', + metadata_version: '2.1', + content: wheelData, + filename: 'unauthorized_py-1.0.0-py3-none-any.whl', + }, + }); + + expect(response.status).toEqual(401); +}); + +tap.test('Integration: should return 404 for unknown paths', async () => { + const response = await registry.handleRequest({ + method: 'GET', + path: '/unknown-protocol/endpoint', + headers: {}, + query: {}, + }); + + expect(response.status).toEqual(404); + expect(response.body).toHaveProperty('error'); + expect((response.body as any).error).toEqual('NOT_FOUND'); +}); + +tap.test('Integration: should retrieve PyPI registry instance', async () => { + const pypiRegistry = registry.getRegistry('pypi'); + + expect(pypiRegistry).toBeDefined(); + expect(pypiRegistry).not.toBeNull(); +}); + +tap.test('Integration: should retrieve RubyGems registry instance', async () => { + const rubygemsRegistry = registry.getRegistry('rubygems'); + + expect(rubygemsRegistry).toBeDefined(); + expect(rubygemsRegistry).not.toBeNull(); +}); + +tap.test('Integration: should retrieve all other protocol instances', async () => { + const ociRegistry = registry.getRegistry('oci'); + const npmRegistry = registry.getRegistry('npm'); + const mavenRegistry = registry.getRegistry('maven'); + const composerRegistry = registry.getRegistry('composer'); + const cargoRegistry = registry.getRegistry('cargo'); + + expect(ociRegistry).toBeDefined(); + expect(npmRegistry).toBeDefined(); + expect(mavenRegistry).toBeDefined(); + expect(composerRegistry).toBeDefined(); + expect(cargoRegistry).toBeDefined(); +}); + +tap.test('Integration: should share storage across protocols', async () => { + const storage = registry.getStorage(); + + expect(storage).toBeDefined(); + + // Verify storage has methods for all protocols + expect(typeof storage.getPypiPackageMetadata).toEqual('function'); + expect(typeof storage.getRubyGemsVersions).toEqual('function'); + expect(typeof storage.getNpmPackument).toEqual('function'); + expect(typeof storage.getOciBlob).toEqual('function'); +}); + +tap.test('Integration: should share auth manager across protocols', async () => { + const authManager = registry.getAuthManager(); + + expect(authManager).toBeDefined(); + + // Verify auth manager has methods for all protocols + expect(typeof authManager.createPypiToken).toEqual('function'); + expect(typeof authManager.createRubyGemsToken).toEqual('function'); + expect(typeof authManager.createNpmToken).toEqual('function'); + expect(typeof authManager.createOciToken).toEqual('function'); +}); + +tap.test('Integration: should handle concurrent requests to different protocols', async () => { + const pypiRequest = registry.handleRequest({ + method: 'GET', + path: '/simple/', + headers: {}, + query: {}, + }); + + const rubygemsRequest = registry.handleRequest({ + method: 'GET', + path: '/rubygems/versions', + headers: {}, + query: {}, + }); + + const [pypiResponse, rubygemsResponse] = await Promise.all([pypiRequest, rubygemsRequest]); + + expect(pypiResponse.status).toEqual(200); + expect(rubygemsResponse.status).toEqual(200); +}); + +tap.test('Integration: should handle package name conflicts across protocols', async () => { + const packageName = 'conflict-test'; + + // Upload PyPI package + const wheelData = await createPythonWheel(packageName, '1.0.0'); + const pypiResponse = await registry.handleRequest({ + method: 'POST', + path: '/pypi/', + headers: { + Authorization: `Bearer ${pypiToken}`, + 'Content-Type': 'multipart/form-data', + }, + query: {}, + body: { + ':action': 'file_upload', + protocol_version: '1', + name: packageName, + version: '1.0.0', + filetype: 'bdist_wheel', + pyversion: 'py3', + metadata_version: '2.1', + content: wheelData, + filename: `${packageName.replace(/-/g, '_')}-1.0.0-py3-none-any.whl`, + }, + }); + + expect(pypiResponse.status).toEqual(201); + + // Upload RubyGems package with same name + const gemData = await createRubyGem(packageName, '1.0.0'); + const rubygemsResponse = await registry.handleRequest({ + method: 'POST', + path: '/rubygems/api/v1/gems', + headers: { + Authorization: rubygemsToken, + 'Content-Type': 'application/octet-stream', + }, + query: {}, + body: gemData, + }); + + expect(rubygemsResponse.status).toEqual(201); + + // Both should exist independently + const pypiGetResponse = await registry.handleRequest({ + method: 'GET', + path: `/simple/${packageName}/`, + headers: {}, + query: {}, + }); + + const rubygemsGetResponse = await registry.handleRequest({ + method: 'GET', + path: `/rubygems/gems/${packageName}-1.0.0.gem`, + headers: {}, + query: {}, + }); + + expect(pypiGetResponse.status).toEqual(200); + expect(rubygemsGetResponse.status).toEqual(200); +}); + +tap.test('Integration: should properly clean up resources on destroy', async () => { + // Destroy should clean up all registries + expect(() => registry.destroy()).not.toThrow(); +}); + +tap.postTask('cleanup registry', async () => { + if (registry && registry.isInitialized()) { + registry.destroy(); + } +}); + +export default tap.start(); diff --git a/test/test.pypi.ts b/test/test.pypi.ts new file mode 100644 index 0000000..6e20b28 --- /dev/null +++ b/test/test.pypi.ts @@ -0,0 +1,469 @@ +import { expect, tap } from '@git.zone/tstest/tapbundle'; +import { SmartRegistry } from '../ts/index.js'; +import { + createTestRegistry, + createTestTokens, + createPythonWheel, + createPythonSdist, + calculatePypiHashes, +} from './helpers/registry.js'; +import { normalizePypiPackageName } from '../ts/pypi/helpers.pypi.js'; + +let registry: SmartRegistry; +let pypiToken: string; +let userId: string; + +// Test data +const testPackageName = 'test-package'; +const normalizedPackageName = normalizePypiPackageName(testPackageName); +const testVersion = '1.0.0'; +let testWheelData: Buffer; +let testSdistData: Buffer; + +tap.test('PyPI: should create registry instance', async () => { + registry = await createTestRegistry(); + const tokens = await createTestTokens(registry); + pypiToken = tokens.pypiToken; + userId = tokens.userId; + + expect(registry).toBeInstanceOf(SmartRegistry); + expect(pypiToken).toBeTypeOf('string'); + + // Clean up any existing metadata from previous test runs + const storage = registry.getStorage(); + try { + await storage.deletePypiPackage(normalizedPackageName); + } catch (error) { + // Ignore error if package doesn't exist + } +}); + +tap.test('PyPI: should create test package files', async () => { + testWheelData = await createPythonWheel(testPackageName, testVersion); + testSdistData = await createPythonSdist(testPackageName, testVersion); + + expect(testWheelData).toBeInstanceOf(Buffer); + expect(testWheelData.length).toBeGreaterThan(0); + expect(testSdistData).toBeInstanceOf(Buffer); + expect(testSdistData.length).toBeGreaterThan(0); +}); + +tap.test('PyPI: should upload wheel file (POST /pypi/)', async () => { + const hashes = calculatePypiHashes(testWheelData); + const filename = `${testPackageName.replace(/-/g, '_')}-${testVersion}-py3-none-any.whl`; + + const formData = new FormData(); + formData.append(':action', 'file_upload'); + formData.append('protocol_version', '1'); + formData.append('name', testPackageName); + formData.append('version', testVersion); + formData.append('filetype', 'bdist_wheel'); + formData.append('pyversion', 'py3'); + formData.append('metadata_version', '2.1'); + formData.append('sha256_digest', hashes.sha256); + formData.append('content', new Blob([testWheelData]), filename); + + const response = await registry.handleRequest({ + method: 'POST', + path: '/pypi/', + headers: { + Authorization: `Bearer ${pypiToken}`, + 'Content-Type': 'multipart/form-data', + }, + query: {}, + body: { + ':action': 'file_upload', + protocol_version: '1', + name: testPackageName, + version: testVersion, + filetype: 'bdist_wheel', + pyversion: 'py3', + metadata_version: '2.1', + sha256_digest: hashes.sha256, + content: testWheelData, + filename: filename, + }, + }); + + expect(response.status).toEqual(201); +}); + +tap.test('PyPI: should retrieve Simple API root index HTML (GET /simple/)', async () => { + const response = await registry.handleRequest({ + method: 'GET', + path: '/simple/', + headers: { + Accept: 'text/html', + }, + query: {}, + }); + + expect(response.status).toEqual(200); + expect(response.headers['Content-Type']).toEqual('text/html'); + expect(response.body).toBeTypeOf('string'); + + const html = response.body as string; + expect(html).toContain(''); + expect(html).toContain('Simple Index'); + expect(html).toContain(normalizedPackageName); +}); + +tap.test('PyPI: should retrieve Simple API root index JSON (GET /simple/ with Accept: application/vnd.pypi.simple.v1+json)', async () => { + const response = await registry.handleRequest({ + method: 'GET', + path: '/simple/', + headers: { + Accept: 'application/vnd.pypi.simple.v1+json', + }, + query: {}, + }); + + expect(response.status).toEqual(200); + expect(response.headers['Content-Type']).toEqual('application/vnd.pypi.simple.v1+json'); + expect(response.body).toBeTypeOf('object'); + + const json = response.body as any; + expect(json).toHaveProperty('meta'); + expect(json).toHaveProperty('projects'); + expect(json.projects).toBeTypeOf('object'); + expect(json.projects).toHaveProperty(normalizedPackageName); +}); + +tap.test('PyPI: should retrieve Simple API package HTML (GET /simple/{package}/)', async () => { + const response = await registry.handleRequest({ + method: 'GET', + path: `/simple/${normalizedPackageName}/`, + headers: { + Accept: 'text/html', + }, + query: {}, + }); + + expect(response.status).toEqual(200); + expect(response.headers['Content-Type']).toEqual('text/html'); + expect(response.body).toBeTypeOf('string'); + + const html = response.body as string; + expect(html).toContain(''); + expect(html).toContain(`Links for ${normalizedPackageName}`); + expect(html).toContain('.whl'); + expect(html).toContain('data-requires-python'); +}); + +tap.test('PyPI: should retrieve Simple API package JSON (GET /simple/{package}/ with Accept: application/vnd.pypi.simple.v1+json)', async () => { + const response = await registry.handleRequest({ + method: 'GET', + path: `/simple/${normalizedPackageName}/`, + headers: { + Accept: 'application/vnd.pypi.simple.v1+json', + }, + query: {}, + }); + + expect(response.status).toEqual(200); + expect(response.headers['Content-Type']).toEqual('application/vnd.pypi.simple.v1+json'); + expect(response.body).toBeTypeOf('object'); + + const json = response.body as any; + expect(json).toHaveProperty('meta'); + expect(json).toHaveProperty('name'); + expect(json.name).toEqual(normalizedPackageName); + expect(json).toHaveProperty('files'); + expect(json.files).toBeTypeOf('object'); + expect(Object.keys(json.files).length).toBeGreaterThan(0); +}); + +tap.test('PyPI: should download wheel file (GET /pypi/packages/{package}/{filename})', async () => { + const filename = `${testPackageName.replace(/-/g, '_')}-${testVersion}-py3-none-any.whl`; + + const response = await registry.handleRequest({ + method: 'GET', + path: `/pypi/packages/${normalizedPackageName}/${filename}`, + headers: {}, + query: {}, + }); + + expect(response.status).toEqual(200); + expect(response.body).toBeInstanceOf(Buffer); + expect((response.body as Buffer).length).toEqual(testWheelData.length); + expect(response.headers['Content-Type']).toEqual('application/octet-stream'); +}); + +tap.test('PyPI: should upload sdist file (POST /pypi/)', async () => { + const hashes = calculatePypiHashes(testSdistData); + const filename = `${testPackageName}-${testVersion}.tar.gz`; + + const response = await registry.handleRequest({ + method: 'POST', + path: '/pypi/', + headers: { + Authorization: `Bearer ${pypiToken}`, + 'Content-Type': 'multipart/form-data', + }, + query: {}, + body: { + ':action': 'file_upload', + protocol_version: '1', + name: testPackageName, + version: testVersion, + filetype: 'sdist', + pyversion: 'source', + metadata_version: '2.1', + sha256_digest: hashes.sha256, + content: testSdistData, + filename: filename, + }, + }); + + expect(response.status).toEqual(201); +}); + +tap.test('PyPI: should list both wheel and sdist in Simple API', async () => { + const response = await registry.handleRequest({ + method: 'GET', + path: `/simple/${normalizedPackageName}/`, + headers: { + Accept: 'application/vnd.pypi.simple.v1+json', + }, + query: {}, + }); + + expect(response.status).toEqual(200); + + const json = response.body as any; + expect(Object.keys(json.files).length).toEqual(2); + + const hasWheel = Object.keys(json.files).some(f => f.endsWith('.whl')); + const hasSdist = Object.keys(json.files).some(f => f.endsWith('.tar.gz')); + + expect(hasWheel).toEqual(true); + expect(hasSdist).toEqual(true); +}); + +tap.test('PyPI: should upload a second version', async () => { + const newVersion = '2.0.0'; + const newWheelData = await createPythonWheel(testPackageName, newVersion); + const hashes = calculatePypiHashes(newWheelData); + const filename = `${testPackageName.replace(/-/g, '_')}-${newVersion}-py3-none-any.whl`; + + const response = await registry.handleRequest({ + method: 'POST', + path: '/pypi/', + headers: { + Authorization: `Bearer ${pypiToken}`, + 'Content-Type': 'multipart/form-data', + }, + query: {}, + body: { + ':action': 'file_upload', + protocol_version: '1', + name: testPackageName, + version: newVersion, + filetype: 'bdist_wheel', + pyversion: 'py3', + metadata_version: '2.1', + sha256_digest: hashes.sha256, + content: newWheelData, + filename: filename, + }, + }); + + expect(response.status).toEqual(201); +}); + +tap.test('PyPI: should list multiple versions in Simple API', async () => { + const response = await registry.handleRequest({ + method: 'GET', + path: `/simple/${normalizedPackageName}/`, + headers: { + Accept: 'application/vnd.pypi.simple.v1+json', + }, + query: {}, + }); + + expect(response.status).toEqual(200); + + const json = response.body as any; + expect(Object.keys(json.files).length).toBeGreaterThan(2); + + const hasVersion1 = Object.keys(json.files).some(f => f.includes('1.0.0')); + const hasVersion2 = Object.keys(json.files).some(f => f.includes('2.0.0')); + + expect(hasVersion1).toEqual(true); + expect(hasVersion2).toEqual(true); +}); + +tap.test('PyPI: should normalize package names correctly', async () => { + const testNames = [ + { input: 'Test-Package', expected: 'test-package' }, + { input: 'Test_Package', expected: 'test-package' }, + { input: 'Test..Package', expected: 'test-package' }, + { input: 'Test---Package', expected: 'test-package' }, + ]; + + for (const { input, expected } of testNames) { + const normalized = normalizePypiPackageName(input); + expect(normalized).toEqual(expected); + } +}); + +tap.test('PyPI: should return 404 for non-existent package', async () => { + const response = await registry.handleRequest({ + method: 'GET', + path: '/simple/nonexistent-package/', + headers: {}, + query: {}, + }); + + expect(response.status).toEqual(404); + expect(response.body).toHaveProperty('error'); +}); + +tap.test('PyPI: should return 401 for unauthorized upload', async () => { + const wheelData = await createPythonWheel('unauthorized-test', '1.0.0'); + const hashes = calculatePypiHashes(wheelData); + + const response = await registry.handleRequest({ + method: 'POST', + path: '/pypi/', + headers: { + // No authorization header + 'Content-Type': 'multipart/form-data', + }, + query: {}, + body: { + ':action': 'file_upload', + protocol_version: '1', + name: 'unauthorized-test', + version: '1.0.0', + filetype: 'bdist_wheel', + pyversion: 'py3', + metadata_version: '2.1', + sha256_digest: hashes.sha256, + content: wheelData, + filename: 'unauthorized_test-1.0.0-py3-none-any.whl', + }, + }); + + expect(response.status).toEqual(401); + expect(response.body).toHaveProperty('error'); +}); + +tap.test('PyPI: should reject upload with mismatched hash', async () => { + const wheelData = await createPythonWheel('hash-test', '1.0.0'); + + const response = await registry.handleRequest({ + method: 'POST', + path: '/pypi/', + headers: { + Authorization: `Bearer ${pypiToken}`, + 'Content-Type': 'multipart/form-data', + }, + query: {}, + body: { + ':action': 'file_upload', + protocol_version: '1', + name: 'hash-test', + version: '1.0.0', + filetype: 'bdist_wheel', + pyversion: 'py3', + metadata_version: '2.1', + sha256_digest: 'wrong_hash_value', + content: wheelData, + filename: 'hash_test-1.0.0-py3-none-any.whl', + }, + }); + + expect(response.status).toEqual(400); + expect(response.body).toHaveProperty('error'); +}); + +tap.test('PyPI: should handle package with requires-python metadata', async () => { + const packageName = 'python-version-test'; + const wheelData = await createPythonWheel(packageName, '1.0.0'); + const hashes = calculatePypiHashes(wheelData); + + const response = await registry.handleRequest({ + method: 'POST', + path: '/pypi/', + headers: { + Authorization: `Bearer ${pypiToken}`, + 'Content-Type': 'multipart/form-data', + }, + query: {}, + body: { + ':action': 'file_upload', + protocol_version: '1', + name: packageName, + version: '1.0.0', + filetype: 'bdist_wheel', + pyversion: 'py3', + metadata_version: '2.1', + sha256_digest: hashes.sha256, + 'requires_python': '>=3.8', + content: wheelData, + filename: `${packageName.replace(/-/g, '_')}-1.0.0-py3-none-any.whl`, + }, + }); + + expect(response.status).toEqual(201); + + // Verify requires-python is in Simple API + const getResponse = await registry.handleRequest({ + method: 'GET', + path: `/simple/${normalizePypiPackageName(packageName)}/`, + headers: { + Accept: 'text/html', + }, + query: {}, + }); + + const html = getResponse.body as string; + expect(html).toContain('data-requires-python'); + expect(html).toContain('>=3.8'); +}); + +tap.test('PyPI: should support JSON API for package metadata', async () => { + const response = await registry.handleRequest({ + method: 'GET', + path: `/pypi/${normalizedPackageName}/json`, + headers: {}, + query: {}, + }); + + expect(response.status).toEqual(200); + expect(response.headers['Content-Type']).toEqual('application/json'); + expect(response.body).toBeTypeOf('object'); + + const json = response.body as any; + expect(json).toHaveProperty('info'); + expect(json.info).toHaveProperty('name'); + expect(json.info.name).toEqual(normalizedPackageName); + expect(json).toHaveProperty('urls'); +}); + +tap.test('PyPI: should support JSON API for specific version', async () => { + const response = await registry.handleRequest({ + method: 'GET', + path: `/pypi/${normalizedPackageName}/${testVersion}/json`, + headers: {}, + query: {}, + }); + + expect(response.status).toEqual(200); + expect(response.headers['Content-Type']).toEqual('application/json'); + expect(response.body).toBeTypeOf('object'); + + const json = response.body as any; + expect(json).toHaveProperty('info'); + expect(json.info.version).toEqual(testVersion); + expect(json).toHaveProperty('urls'); +}); + +tap.postTask('cleanup registry', async () => { + if (registry) { + registry.destroy(); + } +}); + +export default tap.start(); diff --git a/test/test.rubygems.ts b/test/test.rubygems.ts new file mode 100644 index 0000000..d1b3a88 --- /dev/null +++ b/test/test.rubygems.ts @@ -0,0 +1,506 @@ +import { expect, tap } from '@git.zone/tstest/tapbundle'; +import { SmartRegistry } from '../ts/index.js'; +import { + createTestRegistry, + createTestTokens, + createRubyGem, + calculateRubyGemsChecksums, +} from './helpers/registry.js'; + +let registry: SmartRegistry; +let rubygemsToken: string; +let userId: string; + +// Test data +const testGemName = 'test-gem'; +const testVersion = '1.0.0'; +let testGemData: Buffer; + +tap.test('RubyGems: should create registry instance', async () => { + registry = await createTestRegistry(); + const tokens = await createTestTokens(registry); + rubygemsToken = tokens.rubygemsToken; + userId = tokens.userId; + + expect(registry).toBeInstanceOf(SmartRegistry); + expect(rubygemsToken).toBeTypeOf('string'); + + // Clean up any existing metadata from previous test runs + const storage = registry.getStorage(); + try { + await storage.deleteRubyGem(testGemName); + } catch (error) { + // Ignore error if gem doesn't exist + } +}); + +tap.test('RubyGems: should create test gem file', async () => { + testGemData = await createRubyGem(testGemName, testVersion); + + expect(testGemData).toBeInstanceOf(Buffer); + expect(testGemData.length).toBeGreaterThan(0); +}); + +tap.test('RubyGems: should upload gem file (POST /rubygems/api/v1/gems)', async () => { + const response = await registry.handleRequest({ + method: 'POST', + path: '/rubygems/api/v1/gems', + headers: { + Authorization: rubygemsToken, + 'Content-Type': 'application/octet-stream', + }, + query: {}, + body: testGemData, + }); + + expect(response.status).toEqual(201); + expect(response.body).toHaveProperty('message'); +}); + +tap.test('RubyGems: should retrieve Compact Index versions file (GET /rubygems/versions)', async () => { + const response = await registry.handleRequest({ + method: 'GET', + path: '/rubygems/versions', + headers: {}, + query: {}, + }); + + expect(response.status).toEqual(200); + expect(response.headers['Content-Type']).toEqual('text/plain; charset=utf-8'); + expect(response.body).toBeInstanceOf(Buffer); + + const content = (response.body as Buffer).toString('utf-8'); + expect(content).toContain('created_at:'); + expect(content).toContain('---'); + expect(content).toContain(testGemName); + expect(content).toContain(testVersion); +}); + +tap.test('RubyGems: should retrieve Compact Index info file (GET /rubygems/info/{gem})', async () => { + const response = await registry.handleRequest({ + method: 'GET', + path: `/rubygems/info/${testGemName}`, + headers: {}, + query: {}, + }); + + expect(response.status).toEqual(200); + expect(response.headers['Content-Type']).toEqual('text/plain; charset=utf-8'); + expect(response.body).toBeInstanceOf(Buffer); + + const content = (response.body as Buffer).toString('utf-8'); + expect(content).toContain('---'); + expect(content).toContain(testVersion); + expect(content).toContain('checksum:'); +}); + +tap.test('RubyGems: should retrieve Compact Index names file (GET /rubygems/names)', async () => { + const response = await registry.handleRequest({ + method: 'GET', + path: '/rubygems/names', + headers: {}, + query: {}, + }); + + expect(response.status).toEqual(200); + expect(response.headers['Content-Type']).toEqual('text/plain; charset=utf-8'); + expect(response.body).toBeInstanceOf(Buffer); + + const content = (response.body as Buffer).toString('utf-8'); + expect(content).toContain('---'); + expect(content).toContain(testGemName); +}); + +tap.test('RubyGems: should download gem file (GET /rubygems/gems/{gem}-{version}.gem)', async () => { + const response = await registry.handleRequest({ + method: 'GET', + path: `/rubygems/gems/${testGemName}-${testVersion}.gem`, + headers: {}, + query: {}, + }); + + expect(response.status).toEqual(200); + expect(response.body).toBeInstanceOf(Buffer); + expect((response.body as Buffer).length).toEqual(testGemData.length); + expect(response.headers['Content-Type']).toEqual('application/octet-stream'); +}); + +tap.test('RubyGems: should upload a second version', async () => { + const newVersion = '2.0.0'; + const newGemData = await createRubyGem(testGemName, newVersion); + + const response = await registry.handleRequest({ + method: 'POST', + path: '/rubygems/api/v1/gems', + headers: { + Authorization: rubygemsToken, + 'Content-Type': 'application/octet-stream', + }, + query: {}, + body: newGemData, + }); + + expect(response.status).toEqual(201); +}); + +tap.test('RubyGems: should list multiple versions in Compact Index', async () => { + const response = await registry.handleRequest({ + method: 'GET', + path: '/rubygems/versions', + headers: {}, + query: {}, + }); + + expect(response.status).toEqual(200); + + const content = (response.body as Buffer).toString('utf-8'); + const lines = content.split('\n'); + const gemLine = lines.find(l => l.startsWith(`${testGemName} `)); + + expect(gemLine).toBeDefined(); + expect(gemLine).toContain('1.0.0'); + expect(gemLine).toContain('2.0.0'); +}); + +tap.test('RubyGems: should list multiple versions in info file', async () => { + const response = await registry.handleRequest({ + method: 'GET', + path: `/rubygems/info/${testGemName}`, + headers: {}, + query: {}, + }); + + expect(response.status).toEqual(200); + + const content = (response.body as Buffer).toString('utf-8'); + expect(content).toContain('1.0.0'); + expect(content).toContain('2.0.0'); +}); + +tap.test('RubyGems: should support platform-specific gems', async () => { + const platformVersion = '1.5.0'; + const platform = 'x86_64-linux'; + const platformGemData = await createRubyGem(testGemName, platformVersion, platform); + + const response = await registry.handleRequest({ + method: 'POST', + path: '/rubygems/api/v1/gems', + headers: { + Authorization: rubygemsToken, + 'Content-Type': 'application/octet-stream', + }, + query: {}, + body: platformGemData, + }); + + expect(response.status).toEqual(201); + + // Verify platform is listed in versions + const versionsResponse = await registry.handleRequest({ + method: 'GET', + path: '/rubygems/versions', + headers: {}, + query: {}, + }); + + const content = (versionsResponse.body as Buffer).toString('utf-8'); + const lines = content.split('\n'); + const gemLine = lines.find(l => l.startsWith(`${testGemName} `)); + + expect(gemLine).toContain(`${platformVersion}_${platform}`); +}); + +tap.test('RubyGems: should yank a gem version (DELETE /rubygems/api/v1/gems/yank)', async () => { + const response = await registry.handleRequest({ + method: 'DELETE', + path: '/rubygems/api/v1/gems/yank', + headers: { + Authorization: rubygemsToken, + }, + query: { + gem_name: testGemName, + version: testVersion, + }, + }); + + expect(response.status).toEqual(200); + expect(response.body).toHaveProperty('message'); + expect((response.body as any).message).toContain('yanked'); +}); + +tap.test('RubyGems: should mark yanked version in Compact Index', async () => { + const response = await registry.handleRequest({ + method: 'GET', + path: '/rubygems/versions', + headers: {}, + query: {}, + }); + + expect(response.status).toEqual(200); + + const content = (response.body as Buffer).toString('utf-8'); + const lines = content.split('\n'); + const gemLine = lines.find(l => l.startsWith(`${testGemName} `)); + + // Yanked versions are prefixed with '-' + expect(gemLine).toContain(`-${testVersion}`); +}); + +tap.test('RubyGems: should still allow downloading yanked gem', async () => { + // Yanked gems can still be downloaded if explicitly requested + const response = await registry.handleRequest({ + method: 'GET', + path: `/rubygems/gems/${testGemName}-${testVersion}.gem`, + headers: {}, + query: {}, + }); + + expect(response.status).toEqual(200); + expect(response.body).toBeInstanceOf(Buffer); +}); + +tap.test('RubyGems: should unyank a gem version (PUT /rubygems/api/v1/gems/unyank)', async () => { + const response = await registry.handleRequest({ + method: 'PUT', + path: '/rubygems/api/v1/gems/unyank', + headers: { + Authorization: rubygemsToken, + }, + query: { + gem_name: testGemName, + version: testVersion, + }, + }); + + expect(response.status).toEqual(200); + expect(response.body).toHaveProperty('message'); + expect((response.body as any).message).toContain('unyanked'); +}); + +tap.test('RubyGems: should remove yank marker after unyank', async () => { + const response = await registry.handleRequest({ + method: 'GET', + path: '/rubygems/versions', + headers: {}, + query: {}, + }); + + expect(response.status).toEqual(200); + + const content = (response.body as Buffer).toString('utf-8'); + const lines = content.split('\n'); + const gemLine = lines.find(l => l.startsWith(`${testGemName} `)); + + // After unyank, version should not have '-' prefix + const versions = gemLine!.split(' ')[1].split(','); + const version1 = versions.find(v => v.includes('1.0.0')); + + expect(version1).not.toStartWith('-'); + expect(version1).toContain('1.0.0'); +}); + +tap.test('RubyGems: should retrieve versions JSON (GET /rubygems/api/v1/versions/{gem}.json)', async () => { + const response = await registry.handleRequest({ + method: 'GET', + path: `/rubygems/api/v1/versions/${testGemName}.json`, + headers: {}, + query: {}, + }); + + expect(response.status).toEqual(200); + expect(response.headers['Content-Type']).toEqual('application/json'); + expect(response.body).toBeTypeOf('object'); + + const json = response.body as any; + expect(json).toHaveProperty('name'); + expect(json.name).toEqual(testGemName); + expect(json).toHaveProperty('versions'); + expect(json.versions).toBeTypeOf('object'); + expect(json.versions.length).toBeGreaterThan(0); +}); + +tap.test('RubyGems: should retrieve dependencies JSON (GET /rubygems/api/v1/dependencies)', async () => { + const response = await registry.handleRequest({ + method: 'GET', + path: '/rubygems/api/v1/dependencies', + headers: {}, + query: { + gems: `${testGemName}`, + }, + }); + + expect(response.status).toEqual(200); + expect(response.headers['Content-Type']).toEqual('application/json'); + expect(response.body).toBeTypeOf('object'); + + const json = response.body as any; + expect(Array.isArray(json)).toEqual(true); +}); + +tap.test('RubyGems: should retrieve gem spec (GET /rubygems/quick/Marshal.4.8/{gem}-{version}.gemspec.rz)', async () => { + const response = await registry.handleRequest({ + method: 'GET', + path: `/rubygems/quick/Marshal.4.8/${testGemName}-${testVersion}.gemspec.rz`, + headers: {}, + query: {}, + }); + + expect(response.status).toEqual(200); + expect(response.body).toBeInstanceOf(Buffer); +}); + +tap.test('RubyGems: should support latest specs endpoint (GET /rubygems/latest_specs.4.8.gz)', async () => { + const response = await registry.handleRequest({ + method: 'GET', + path: '/rubygems/latest_specs.4.8.gz', + headers: {}, + query: {}, + }); + + expect(response.status).toEqual(200); + expect(response.headers['Content-Type']).toEqual('application/octet-stream'); + expect(response.body).toBeInstanceOf(Buffer); +}); + +tap.test('RubyGems: should support specs endpoint (GET /rubygems/specs.4.8.gz)', async () => { + const response = await registry.handleRequest({ + method: 'GET', + path: '/rubygems/specs.4.8.gz', + headers: {}, + query: {}, + }); + + expect(response.status).toEqual(200); + expect(response.headers['Content-Type']).toEqual('application/octet-stream'); + expect(response.body).toBeInstanceOf(Buffer); +}); + +tap.test('RubyGems: should return 404 for non-existent gem', async () => { + const response = await registry.handleRequest({ + method: 'GET', + path: '/rubygems/gems/nonexistent-gem-1.0.0.gem', + headers: {}, + query: {}, + }); + + expect(response.status).toEqual(404); + expect(response.body).toHaveProperty('error'); +}); + +tap.test('RubyGems: should return 401 for unauthorized upload', async () => { + const gemData = await createRubyGem('unauthorized-gem', '1.0.0'); + + const response = await registry.handleRequest({ + method: 'POST', + path: '/rubygems/api/v1/gems', + headers: { + // No authorization header + 'Content-Type': 'application/octet-stream', + }, + query: {}, + body: gemData, + }); + + expect(response.status).toEqual(401); + expect(response.body).toHaveProperty('error'); +}); + +tap.test('RubyGems: should return 401 for unauthorized yank', async () => { + const response = await registry.handleRequest({ + method: 'DELETE', + path: '/rubygems/api/v1/gems/yank', + headers: { + // No authorization header + }, + query: { + gem_name: testGemName, + version: '2.0.0', + }, + }); + + expect(response.status).toEqual(401); + expect(response.body).toHaveProperty('error'); +}); + +tap.test('RubyGems: should handle gem with dependencies', async () => { + const gemWithDeps = 'gem-with-deps'; + const version = '1.0.0'; + const gemData = await createRubyGem(gemWithDeps, version); + + const response = await registry.handleRequest({ + method: 'POST', + path: '/rubygems/api/v1/gems', + headers: { + Authorization: rubygemsToken, + 'Content-Type': 'application/octet-stream', + }, + query: {}, + body: gemData, + }); + + expect(response.status).toEqual(201); + + // Check info file contains dependency info + const infoResponse = await registry.handleRequest({ + method: 'GET', + path: `/rubygems/info/${gemWithDeps}`, + headers: {}, + query: {}, + }); + + expect(infoResponse.status).toEqual(200); + + const content = (infoResponse.body as Buffer).toString('utf-8'); + expect(content).toContain('checksum:'); +}); + +tap.test('RubyGems: should validate gem filename format', async () => { + const invalidGemData = Buffer.from('invalid gem data'); + + const response = await registry.handleRequest({ + method: 'POST', + path: '/rubygems/api/v1/gems', + headers: { + Authorization: rubygemsToken, + 'Content-Type': 'application/octet-stream', + }, + query: {}, + body: invalidGemData, + }); + + // Should fail validation + expect(response.status).toBeGreaterThanOrEqual(400); +}); + +tap.test('RubyGems: should support conditional GET with ETag', async () => { + // First request to get ETag + const response1 = await registry.handleRequest({ + method: 'GET', + path: '/rubygems/versions', + headers: {}, + query: {}, + }); + + const etag = response1.headers['ETag']; + expect(etag).toBeDefined(); + + // Second request with If-None-Match + const response2 = await registry.handleRequest({ + method: 'GET', + path: '/rubygems/versions', + headers: { + 'If-None-Match': etag as string, + }, + query: {}, + }); + + expect(response2.status).toEqual(304); +}); + +tap.postTask('cleanup registry', async () => { + if (registry) { + registry.destroy(); + } +}); + +export default tap.start(); diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index 71a3f73..5ec5c67 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@push.rocks/smartregistry', - version: '1.5.0', - description: 'a registry for npm modules and oci images' + version: '1.6.0', + description: 'A composable TypeScript library implementing OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems registries for building unified container and package registries' } diff --git a/ts/classes.smartregistry.ts b/ts/classes.smartregistry.ts index 2296e5f..f55abde 100644 --- a/ts/classes.smartregistry.ts +++ b/ts/classes.smartregistry.ts @@ -7,10 +7,12 @@ import { NpmRegistry } from './npm/classes.npmregistry.js'; import { MavenRegistry } from './maven/classes.mavenregistry.js'; import { CargoRegistry } from './cargo/classes.cargoregistry.js'; import { ComposerRegistry } from './composer/classes.composerregistry.js'; +import { PypiRegistry } from './pypi/classes.pypiregistry.js'; +import { RubyGemsRegistry } from './rubygems/classes.rubygemsregistry.js'; /** * Main registry orchestrator - * Routes requests to appropriate protocol handlers (OCI, NPM, Maven, Cargo, or Composer) + * Routes requests to appropriate protocol handlers (OCI, NPM, Maven, Cargo, Composer, PyPI, or RubyGems) */ export class SmartRegistry { private storage: RegistryStorage; @@ -81,6 +83,24 @@ export class SmartRegistry { this.registries.set('composer', composerRegistry); } + // Initialize PyPI registry if enabled + if (this.config.pypi?.enabled) { + const pypiBasePath = this.config.pypi.basePath || '/pypi'; + const registryUrl = `http://localhost:5000`; // TODO: Make configurable + const pypiRegistry = new PypiRegistry(this.storage, this.authManager, pypiBasePath, registryUrl); + await pypiRegistry.init(); + this.registries.set('pypi', pypiRegistry); + } + + // Initialize RubyGems registry if enabled + if (this.config.rubygems?.enabled) { + const rubygemsBasePath = this.config.rubygems.basePath || '/rubygems'; + const registryUrl = `http://localhost:5000${rubygemsBasePath}`; // TODO: Make configurable + const rubygemsRegistry = new RubyGemsRegistry(this.storage, this.authManager, rubygemsBasePath, registryUrl); + await rubygemsRegistry.init(); + this.registries.set('rubygems', rubygemsRegistry); + } + this.initialized = true; } @@ -131,6 +151,25 @@ export class SmartRegistry { } } + // Route to PyPI registry (also handles /simple prefix) + if (this.config.pypi?.enabled) { + const pypiBasePath = this.config.pypi.basePath || '/pypi'; + if (path.startsWith(pypiBasePath) || path.startsWith('/simple')) { + const pypiRegistry = this.registries.get('pypi'); + if (pypiRegistry) { + return pypiRegistry.handleRequest(context); + } + } + } + + // Route to RubyGems registry + if (this.config.rubygems?.enabled && path.startsWith(this.config.rubygems.basePath)) { + const rubygemsRegistry = this.registries.get('rubygems'); + if (rubygemsRegistry) { + return rubygemsRegistry.handleRequest(context); + } + } + // No matching registry return { status: 404, @@ -159,7 +198,7 @@ export class SmartRegistry { /** * Get a specific registry handler */ - public getRegistry(protocol: 'oci' | 'npm' | 'maven' | 'cargo' | 'composer'): BaseRegistry | undefined { + public getRegistry(protocol: 'oci' | 'npm' | 'maven' | 'cargo' | 'composer' | 'pypi' | 'rubygems'): BaseRegistry | undefined { return this.registries.get(protocol); } diff --git a/ts/core/classes.registrystorage.ts b/ts/core/classes.registrystorage.ts index 6996089..ee2bbc1 100644 --- a/ts/core/classes.registrystorage.ts +++ b/ts/core/classes.registrystorage.ts @@ -828,4 +828,240 @@ export class RegistryStorage implements IStorageBackend { private getPypiPackageFilePath(packageName: string, filename: string): string { return `pypi/packages/${packageName}/${filename}`; } + + // ======================================================================== + // RUBYGEMS STORAGE METHODS + // ======================================================================== + + /** + * Get RubyGems versions file (compact index) + */ + public async getRubyGemsVersions(): Promise { + const path = this.getRubyGemsVersionsPath(); + const data = await this.getObject(path); + return data ? data.toString('utf-8') : null; + } + + /** + * Store RubyGems versions file (compact index) + */ + public async putRubyGemsVersions(content: string): Promise { + const path = this.getRubyGemsVersionsPath(); + const data = Buffer.from(content, 'utf-8'); + return this.putObject(path, data, { 'Content-Type': 'text/plain; charset=utf-8' }); + } + + /** + * Get RubyGems info file for a gem (compact index) + */ + public async getRubyGemsInfo(gemName: string): Promise { + const path = this.getRubyGemsInfoPath(gemName); + const data = await this.getObject(path); + return data ? data.toString('utf-8') : null; + } + + /** + * Store RubyGems info file for a gem (compact index) + */ + public async putRubyGemsInfo(gemName: string, content: string): Promise { + const path = this.getRubyGemsInfoPath(gemName); + const data = Buffer.from(content, 'utf-8'); + return this.putObject(path, data, { 'Content-Type': 'text/plain; charset=utf-8' }); + } + + /** + * Get RubyGems names file + */ + public async getRubyGemsNames(): Promise { + const path = this.getRubyGemsNamesPath(); + const data = await this.getObject(path); + return data ? data.toString('utf-8') : null; + } + + /** + * Store RubyGems names file + */ + public async putRubyGemsNames(content: string): Promise { + const path = this.getRubyGemsNamesPath(); + const data = Buffer.from(content, 'utf-8'); + return this.putObject(path, data, { 'Content-Type': 'text/plain; charset=utf-8' }); + } + + /** + * Get RubyGems .gem file + */ + public async getRubyGemsGem(gemName: string, version: string, platform?: string): Promise { + const path = this.getRubyGemsGemPath(gemName, version, platform); + return this.getObject(path); + } + + /** + * Store RubyGems .gem file + */ + public async putRubyGemsGem( + gemName: string, + version: string, + data: Buffer, + platform?: string + ): Promise { + const path = this.getRubyGemsGemPath(gemName, version, platform); + return this.putObject(path, data, { 'Content-Type': 'application/octet-stream' }); + } + + /** + * Check if RubyGems .gem file exists + */ + public async rubyGemsGemExists(gemName: string, version: string, platform?: string): Promise { + const path = this.getRubyGemsGemPath(gemName, version, platform); + return this.objectExists(path); + } + + /** + * Delete RubyGems .gem file + */ + public async deleteRubyGemsGem(gemName: string, version: string, platform?: string): Promise { + const path = this.getRubyGemsGemPath(gemName, version, platform); + return this.deleteObject(path); + } + + /** + * Get RubyGems metadata + */ + public async getRubyGemsMetadata(gemName: string): Promise { + const path = this.getRubyGemsMetadataPath(gemName); + const data = await this.getObject(path); + return data ? JSON.parse(data.toString('utf-8')) : null; + } + + /** + * Store RubyGems metadata + */ + public async putRubyGemsMetadata(gemName: string, metadata: any): Promise { + const path = this.getRubyGemsMetadataPath(gemName); + const data = Buffer.from(JSON.stringify(metadata, null, 2), 'utf-8'); + return this.putObject(path, data, { 'Content-Type': 'application/json' }); + } + + /** + * Check if RubyGems metadata exists + */ + public async rubyGemsMetadataExists(gemName: string): Promise { + const path = this.getRubyGemsMetadataPath(gemName); + return this.objectExists(path); + } + + /** + * Delete RubyGems metadata + */ + public async deleteRubyGemsMetadata(gemName: string): Promise { + const path = this.getRubyGemsMetadataPath(gemName); + return this.deleteObject(path); + } + + /** + * List all RubyGems + */ + public async listRubyGems(): Promise { + const prefix = 'rubygems/metadata/'; + const objects = await this.listObjects(prefix); + const gems = new Set(); + + // Extract gem names from paths like: rubygems/metadata/gem-name/metadata.json + for (const obj of objects) { + const match = obj.match(/^rubygems\/metadata\/([^\/]+)\/metadata\.json$/); + if (match) { + gems.add(match[1]); + } + } + + return Array.from(gems).sort(); + } + + /** + * List all versions of a RubyGem + */ + public async listRubyGemsVersions(gemName: string): Promise { + const prefix = `rubygems/gems/`; + const objects = await this.listObjects(prefix); + const versions = new Set(); + + // Extract versions from filenames: gem-name-version[-platform].gem + const gemPrefix = `${gemName}-`; + for (const obj of objects) { + const filename = obj.split('/').pop(); + if (!filename || !filename.startsWith(gemPrefix) || !filename.endsWith('.gem')) continue; + + // Remove gem name prefix and .gem suffix + const versionPart = filename.substring(gemPrefix.length, filename.length - 4); + + // Split on last hyphen to separate version from platform + const lastHyphen = versionPart.lastIndexOf('-'); + const version = lastHyphen > 0 ? versionPart.substring(0, lastHyphen) : versionPart; + + versions.add(version); + } + + return Array.from(versions).sort(); + } + + /** + * Delete entire RubyGem (all versions and files) + */ + public async deleteRubyGem(gemName: string): Promise { + // Delete metadata + await this.deleteRubyGemsMetadata(gemName); + + // Delete all gem files + const prefix = `rubygems/gems/`; + const objects = await this.listObjects(prefix); + const gemPrefix = `${gemName}-`; + + for (const obj of objects) { + const filename = obj.split('/').pop(); + if (filename && filename.startsWith(gemPrefix) && filename.endsWith('.gem')) { + await this.deleteObject(obj); + } + } + } + + /** + * Delete specific version of a RubyGem + */ + public async deleteRubyGemsVersion(gemName: string, version: string, platform?: string): Promise { + // Delete gem file + await this.deleteRubyGemsGem(gemName, version, platform); + + // Update metadata to remove this version + const metadata = await this.getRubyGemsMetadata(gemName); + if (metadata && metadata.versions) { + const versionKey = platform ? `${version}-${platform}` : version; + delete metadata.versions[versionKey]; + await this.putRubyGemsMetadata(gemName, metadata); + } + } + + // ======================================================================== + // RUBYGEMS PATH HELPERS + // ======================================================================== + + private getRubyGemsVersionsPath(): string { + return 'rubygems/versions'; + } + + private getRubyGemsInfoPath(gemName: string): string { + return `rubygems/info/${gemName}`; + } + + private getRubyGemsNamesPath(): string { + return 'rubygems/names'; + } + + private getRubyGemsGemPath(gemName: string, version: string, platform?: string): string { + const filename = platform ? `${gemName}-${version}-${platform}.gem` : `${gemName}-${version}.gem`; + return `rubygems/gems/${filename}`; + } + + private getRubyGemsMetadataPath(gemName: string): string { + return `rubygems/metadata/${gemName}/metadata.json`; + } } diff --git a/ts/index.ts b/ts/index.ts index 25d4201..8506044 100644 --- a/ts/index.ts +++ b/ts/index.ts @@ -1,6 +1,6 @@ /** * @push.rocks/smartregistry - * Composable registry supporting OCI, NPM, Maven, Cargo, and Composer protocols + * Composable registry supporting OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems protocols */ // Main orchestrator @@ -23,3 +23,9 @@ export * from './cargo/index.js'; // Composer Registry export * from './composer/index.js'; + +// PyPI Registry +export * from './pypi/index.js'; + +// RubyGems Registry +export * from './rubygems/index.js'; diff --git a/ts/pypi/classes.pypiregistry.ts b/ts/pypi/classes.pypiregistry.ts index fbc66e5..4a2d39a 100644 --- a/ts/pypi/classes.pypiregistry.ts +++ b/ts/pypi/classes.pypiregistry.ts @@ -351,22 +351,38 @@ export class PypiRegistry extends BaseRegistry { return this.errorResponse(403, 'Insufficient permissions'); } - // Calculate hashes + // Calculate and verify hashes const hashes: Record = {}; - if (formData.sha256_digest) { - hashes.sha256 = formData.sha256_digest; - } else { - hashes.sha256 = await helpers.calculateHash(fileData, 'sha256'); + // Always calculate SHA256 + const actualSha256 = await helpers.calculateHash(fileData, 'sha256'); + hashes.sha256 = actualSha256; + + // Verify client-provided SHA256 if present + if (formData.sha256_digest && formData.sha256_digest !== actualSha256) { + return this.errorResponse(400, 'SHA256 hash mismatch'); } + // Calculate MD5 if requested if (formData.md5_digest) { - // MD5 digest in PyPI is urlsafe base64, convert to hex - hashes.md5 = await helpers.calculateHash(fileData, 'md5'); + const actualMd5 = await helpers.calculateHash(fileData, 'md5'); + hashes.md5 = actualMd5; + + // Verify if client provided MD5 + if (formData.md5_digest !== actualMd5) { + return this.errorResponse(400, 'MD5 hash mismatch'); + } } + // Calculate Blake2b if requested if (formData.blake2_256_digest) { - hashes.blake2b = formData.blake2_256_digest; + const actualBlake2b = await helpers.calculateHash(fileData, 'blake2b'); + hashes.blake2b = actualBlake2b; + + // Verify if client provided Blake2b + if (formData.blake2_256_digest !== actualBlake2b) { + return this.errorResponse(400, 'Blake2b hash mismatch'); + } } // Store file diff --git a/ts/rubygems/classes.rubygemsregistry.ts b/ts/rubygems/classes.rubygemsregistry.ts new file mode 100644 index 0000000..5f69620 --- /dev/null +++ b/ts/rubygems/classes.rubygemsregistry.ts @@ -0,0 +1,598 @@ +import { Smartlog } from '@push.rocks/smartlog'; +import { BaseRegistry } from '../core/classes.baseregistry.js'; +import { RegistryStorage } from '../core/classes.registrystorage.js'; +import { AuthManager } from '../core/classes.authmanager.js'; +import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js'; +import type { + IRubyGemsMetadata, + IRubyGemsVersionMetadata, + IRubyGemsUploadResponse, + IRubyGemsYankResponse, + IRubyGemsError, + ICompactIndexInfoEntry, +} from './interfaces.rubygems.js'; +import * as helpers from './helpers.rubygems.js'; + +/** + * RubyGems registry implementation + * Implements Compact Index API and RubyGems protocol + */ +export class RubyGemsRegistry extends BaseRegistry { + private storage: RegistryStorage; + private authManager: AuthManager; + private basePath: string = '/rubygems'; + private registryUrl: string; + private logger: Smartlog; + + constructor( + storage: RegistryStorage, + authManager: AuthManager, + basePath: string = '/rubygems', + registryUrl: string = 'http://localhost:5000/rubygems' + ) { + super(); + this.storage = storage; + this.authManager = authManager; + this.basePath = basePath; + this.registryUrl = registryUrl; + + // Initialize logger + this.logger = new Smartlog({ + logContext: { + company: 'push.rocks', + companyunit: 'smartregistry', + containerName: 'rubygems-registry', + environment: (process.env.NODE_ENV as any) || 'development', + runtime: 'node', + zone: 'rubygems' + } + }); + this.logger.enableConsole(); + } + + public async init(): Promise { + // Initialize Compact Index files if not exist + const existingVersions = await this.storage.getRubyGemsVersions(); + if (!existingVersions) { + const versions = helpers.generateCompactIndexVersions([]); + await this.storage.putRubyGemsVersions(versions); + this.logger.log('info', 'Initialized RubyGems Compact Index'); + } + + const existingNames = await this.storage.getRubyGemsNames(); + if (!existingNames) { + const names = helpers.generateNamesFile([]); + await this.storage.putRubyGemsNames(names); + this.logger.log('info', 'Initialized RubyGems names file'); + } + } + + public getBasePath(): string { + return this.basePath; + } + + public async handleRequest(context: IRequestContext): Promise { + let path = context.path.replace(this.basePath, ''); + + // Extract token (Authorization header) + const token = await this.extractToken(context); + + this.logger.log('debug', `handleRequest: ${context.method} ${path}`, { + method: context.method, + path, + hasAuth: !!token + }); + + // Compact Index endpoints + if (path === '/versions' && context.method === 'GET') { + return this.handleVersionsFile(); + } + + if (path === '/names' && context.method === 'GET') { + return this.handleNamesFile(); + } + + // Info file: GET /info/{gem} + const infoMatch = path.match(/^\/info\/([^\/]+)$/); + if (infoMatch && context.method === 'GET') { + return this.handleInfoFile(infoMatch[1]); + } + + // Gem download: GET /gems/{gem}-{version}[-{platform}].gem + const downloadMatch = path.match(/^\/gems\/(.+\.gem)$/); + if (downloadMatch && context.method === 'GET') { + return this.handleDownload(downloadMatch[1]); + } + + // API v1 endpoints + if (path.startsWith('/api/v1/')) { + return this.handleApiRequest(path.substring(8), context, token); + } + + return { + status: 404, + headers: { 'Content-Type': 'application/json' }, + body: Buffer.from(JSON.stringify({ message: 'Not Found' })), + }; + } + + /** + * Check if token has permission for resource + */ + protected async checkPermission( + token: IAuthToken | null, + resource: string, + action: string + ): Promise { + if (!token) return false; + return this.authManager.authorize(token, `rubygems:gem:${resource}`, action); + } + + /** + * Extract authentication token from request + */ + private async extractToken(context: IRequestContext): Promise { + const authHeader = context.headers['authorization'] || context.headers['Authorization']; + if (!authHeader) return null; + + // RubyGems typically uses plain API key in Authorization header + return this.authManager.validateToken(authHeader, 'rubygems'); + } + + /** + * Handle /versions endpoint (Compact Index) + */ + private async handleVersionsFile(): Promise { + const content = await this.storage.getRubyGemsVersions(); + + if (!content) { + return this.errorResponse(500, 'Versions file not initialized'); + } + + return { + status: 200, + headers: { + 'Content-Type': 'text/plain; charset=utf-8', + 'Cache-Control': 'public, max-age=60', + 'ETag': `"${await helpers.calculateMD5(content)}"` + }, + body: Buffer.from(content), + }; + } + + /** + * Handle /names endpoint (Compact Index) + */ + private async handleNamesFile(): Promise { + const content = await this.storage.getRubyGemsNames(); + + if (!content) { + return this.errorResponse(500, 'Names file not initialized'); + } + + return { + status: 200, + headers: { + 'Content-Type': 'text/plain; charset=utf-8', + 'Cache-Control': 'public, max-age=300' + }, + body: Buffer.from(content), + }; + } + + /** + * Handle /info/{gem} endpoint (Compact Index) + */ + private async handleInfoFile(gemName: string): Promise { + const content = await this.storage.getRubyGemsInfo(gemName); + + if (!content) { + return { + status: 404, + headers: { 'Content-Type': 'text/plain' }, + body: Buffer.from('Not Found'), + }; + } + + return { + status: 200, + headers: { + 'Content-Type': 'text/plain; charset=utf-8', + 'Cache-Control': 'public, max-age=300', + 'ETag': `"${await helpers.calculateMD5(content)}"` + }, + body: Buffer.from(content), + }; + } + + /** + * Handle gem file download + */ + private async handleDownload(filename: string): Promise { + const parsed = helpers.parseGemFilename(filename); + if (!parsed) { + return this.errorResponse(400, 'Invalid gem filename'); + } + + const gemData = await this.storage.getRubyGemsGem( + parsed.name, + parsed.version, + parsed.platform + ); + + if (!gemData) { + return this.errorResponse(404, 'Gem not found'); + } + + return { + status: 200, + headers: { + 'Content-Type': 'application/octet-stream', + 'Content-Disposition': `attachment; filename="${filename}"`, + 'Content-Length': gemData.length.toString() + }, + body: gemData, + }; + } + + /** + * Handle API v1 requests + */ + private async handleApiRequest( + path: string, + context: IRequestContext, + token: IAuthToken | null + ): Promise { + // Upload gem: POST /gems + if (path === '/gems' && context.method === 'POST') { + return this.handleUpload(context, token); + } + + // Yank gem: DELETE /gems/yank + if (path === '/gems/yank' && context.method === 'DELETE') { + return this.handleYank(context, token); + } + + // Unyank gem: PUT /gems/unyank + if (path === '/gems/unyank' && context.method === 'PUT') { + return this.handleUnyank(context, token); + } + + // Version list: GET /versions/{gem}.json + const versionsMatch = path.match(/^\/versions\/([^\/]+)\.json$/); + if (versionsMatch && context.method === 'GET') { + return this.handleVersionsJson(versionsMatch[1]); + } + + // Dependencies: GET /dependencies?gems={list} + if (path.startsWith('/dependencies') && context.method === 'GET') { + const gemsParam = context.query?.gems || ''; + return this.handleDependencies(gemsParam); + } + + return this.errorResponse(404, 'API endpoint not found'); + } + + /** + * Handle gem upload + * POST /api/v1/gems + */ + private async handleUpload(context: IRequestContext, token: IAuthToken | null): Promise { + if (!token) { + return this.errorResponse(401, 'Authentication required'); + } + + try { + // Extract gem data from request body + const gemData = context.body as Buffer; + if (!gemData || gemData.length === 0) { + return this.errorResponse(400, 'No gem file provided'); + } + + // For now, we expect metadata in query params or headers + // Full implementation would parse .gem file (tar + gzip + Marshal) + const gemName = context.query?.name || context.headers['x-gem-name']; + const version = context.query?.version || context.headers['x-gem-version']; + const platform = context.query?.platform || context.headers['x-gem-platform']; + + if (!gemName || !version) { + return this.errorResponse(400, 'Gem name and version required'); + } + + // Validate gem name + if (!helpers.isValidGemName(gemName)) { + return this.errorResponse(400, 'Invalid gem name'); + } + + // Check permission + if (!(await this.checkPermission(token, gemName, 'write'))) { + return this.errorResponse(403, 'Insufficient permissions'); + } + + // Calculate checksum + const checksum = await helpers.calculateSHA256(gemData); + + // Store gem file + await this.storage.putRubyGemsGem(gemName, version, gemData, platform); + + // Update metadata + let metadata: IRubyGemsMetadata = await this.storage.getRubyGemsMetadata(gemName) || { + name: gemName, + versions: {}, + }; + + const versionKey = platform ? `${version}-${platform}` : version; + metadata.versions[versionKey] = { + version, + platform, + checksum, + size: gemData.length, + 'upload-time': new Date().toISOString(), + 'uploaded-by': token.userId, + dependencies: [], // Would extract from gem spec + requirements: [], + }; + + metadata['last-modified'] = new Date().toISOString(); + await this.storage.putRubyGemsMetadata(gemName, metadata); + + // Update Compact Index info file + await this.updateCompactIndexForGem(gemName, metadata); + + // Update versions file + await this.updateVersionsFile(gemName, version, platform || 'ruby', false); + + // Update names file + await this.updateNamesFile(gemName); + + this.logger.log('info', `Gem uploaded: ${gemName} ${version}`, { + platform, + size: gemData.length + }); + + return { + status: 200, + headers: { 'Content-Type': 'application/json' }, + body: Buffer.from(JSON.stringify({ + message: 'Gem uploaded successfully', + name: gemName, + version, + })), + }; + } catch (error) { + this.logger.log('error', 'Upload failed', { error: (error as Error).message }); + return this.errorResponse(500, 'Upload failed: ' + (error as Error).message); + } + } + + /** + * Handle gem yanking + * DELETE /api/v1/gems/yank + */ + private async handleYank(context: IRequestContext, token: IAuthToken | null): Promise { + if (!token) { + return this.errorResponse(401, 'Authentication required'); + } + + const gemName = context.query?.gem_name; + const version = context.query?.version; + const platform = context.query?.platform; + + if (!gemName || !version) { + return this.errorResponse(400, 'Gem name and version required'); + } + + if (!(await this.checkPermission(token, gemName, 'yank'))) { + return this.errorResponse(403, 'Insufficient permissions'); + } + + // Update metadata to mark as yanked + const metadata = await this.storage.getRubyGemsMetadata(gemName); + if (!metadata) { + return this.errorResponse(404, 'Gem not found'); + } + + const versionKey = platform ? `${version}-${platform}` : version; + if (!metadata.versions[versionKey]) { + return this.errorResponse(404, 'Version not found'); + } + + metadata.versions[versionKey].yanked = true; + await this.storage.putRubyGemsMetadata(gemName, metadata); + + // Update Compact Index + await this.updateCompactIndexForGem(gemName, metadata); + await this.updateVersionsFile(gemName, version, platform || 'ruby', true); + + this.logger.log('info', `Gem yanked: ${gemName} ${version}`); + + return { + status: 200, + headers: { 'Content-Type': 'application/json' }, + body: Buffer.from(JSON.stringify({ + success: true, + message: 'Gem yanked successfully' + })), + }; + } + + /** + * Handle gem unyanking + * PUT /api/v1/gems/unyank + */ + private async handleUnyank(context: IRequestContext, token: IAuthToken | null): Promise { + if (!token) { + return this.errorResponse(401, 'Authentication required'); + } + + const gemName = context.query?.gem_name; + const version = context.query?.version; + const platform = context.query?.platform; + + if (!gemName || !version) { + return this.errorResponse(400, 'Gem name and version required'); + } + + if (!(await this.checkPermission(token, gemName, 'write'))) { + return this.errorResponse(403, 'Insufficient permissions'); + } + + const metadata = await this.storage.getRubyGemsMetadata(gemName); + if (!metadata) { + return this.errorResponse(404, 'Gem not found'); + } + + const versionKey = platform ? `${version}-${platform}` : version; + if (!metadata.versions[versionKey]) { + return this.errorResponse(404, 'Version not found'); + } + + metadata.versions[versionKey].yanked = false; + await this.storage.putRubyGemsMetadata(gemName, metadata); + + // Update Compact Index + await this.updateCompactIndexForGem(gemName, metadata); + await this.updateVersionsFile(gemName, version, platform || 'ruby', false); + + this.logger.log('info', `Gem unyanked: ${gemName} ${version}`); + + return { + status: 200, + headers: { 'Content-Type': 'application/json' }, + body: Buffer.from(JSON.stringify({ + success: true, + message: 'Gem unyanked successfully' + })), + }; + } + + /** + * Handle versions JSON API + */ + private async handleVersionsJson(gemName: string): Promise { + const metadata = await this.storage.getRubyGemsMetadata(gemName); + if (!metadata) { + return this.errorResponse(404, 'Gem not found'); + } + + const versions = Object.values(metadata.versions).map((v: any) => ({ + version: v.version, + platform: v.platform, + uploadTime: v['upload-time'], + })); + + const response = helpers.generateVersionsJson(gemName, versions); + + return { + status: 200, + headers: { + 'Content-Type': 'application/json', + 'Cache-Control': 'public, max-age=300' + }, + body: Buffer.from(JSON.stringify(response)), + }; + } + + /** + * Handle dependencies query + */ + private async handleDependencies(gemsParam: string): Promise { + const gemNames = gemsParam.split(',').filter(n => n.trim()); + const result = new Map(); + + for (const gemName of gemNames) { + const metadata = await this.storage.getRubyGemsMetadata(gemName); + if (metadata) { + const versions = Object.values(metadata.versions).map((v: any) => ({ + version: v.version, + platform: v.platform, + dependencies: v.dependencies || [], + })); + result.set(gemName, versions); + } + } + + const response = helpers.generateDependenciesJson(result); + + return { + status: 200, + headers: { 'Content-Type': 'application/json' }, + body: Buffer.from(JSON.stringify(response)), + }; + } + + /** + * Update Compact Index info file for a gem + */ + private async updateCompactIndexForGem( + gemName: string, + metadata: IRubyGemsMetadata + ): Promise { + const entries: ICompactIndexInfoEntry[] = Object.values(metadata.versions) + .filter(v => !v.yanked) // Exclude yanked from info file + .map(v => ({ + version: v.version, + platform: v.platform, + dependencies: v.dependencies || [], + requirements: v.requirements || [], + checksum: v.checksum, + })); + + const content = helpers.generateCompactIndexInfo(entries); + await this.storage.putRubyGemsInfo(gemName, content); + } + + /** + * Update versions file with new/updated gem + */ + private async updateVersionsFile( + gemName: string, + version: string, + platform: string, + yanked: boolean + ): Promise { + const existingVersions = await this.storage.getRubyGemsVersions(); + if (!existingVersions) return; + + // Calculate info file checksum + const infoContent = await this.storage.getRubyGemsInfo(gemName) || ''; + const infoChecksum = await helpers.calculateMD5(infoContent); + + const updated = helpers.updateCompactIndexVersions( + existingVersions, + gemName, + { version, platform: platform !== 'ruby' ? platform : undefined, yanked }, + infoChecksum + ); + + await this.storage.putRubyGemsVersions(updated); + } + + /** + * Update names file with new gem + */ + private async updateNamesFile(gemName: string): Promise { + const existingNames = await this.storage.getRubyGemsNames(); + if (!existingNames) return; + + const lines = existingNames.split('\n').filter(l => l !== '---'); + if (!lines.includes(gemName)) { + lines.push(gemName); + lines.sort(); + const updated = helpers.generateNamesFile(lines); + await this.storage.putRubyGemsNames(updated); + } + } + + /** + * Helper: Create error response + */ + private errorResponse(status: number, message: string): IResponse { + const error: IRubyGemsError = { message, status }; + return { + status, + headers: { 'Content-Type': 'application/json' }, + body: Buffer.from(JSON.stringify(error)), + }; + } +} diff --git a/ts/rubygems/helpers.rubygems.ts b/ts/rubygems/helpers.rubygems.ts new file mode 100644 index 0000000..b86126e --- /dev/null +++ b/ts/rubygems/helpers.rubygems.ts @@ -0,0 +1,398 @@ +/** + * Helper functions for RubyGems registry + * Compact Index generation, dependency formatting, etc. + */ + +import type { + IRubyGemsVersion, + IRubyGemsDependency, + IRubyGemsRequirement, + ICompactIndexVersionsEntry, + ICompactIndexInfoEntry, + IRubyGemsMetadata, +} from './interfaces.rubygems.js'; + +/** + * Generate Compact Index versions file + * Format: GEMNAME [-]VERSION_PLATFORM[,VERSION_PLATFORM,...] MD5 + * @param entries - Version entries for all gems + * @returns Compact Index versions file content + */ +export function generateCompactIndexVersions(entries: ICompactIndexVersionsEntry[]): string { + const lines: string[] = []; + + // Add metadata header + lines.push(`created_at: ${new Date().toISOString()}`); + lines.push('---'); + + // Add gem entries + for (const entry of entries) { + const versions = entry.versions + .map(v => { + const yanked = v.yanked ? '-' : ''; + const platform = v.platform && v.platform !== 'ruby' ? `_${v.platform}` : ''; + return `${yanked}${v.version}${platform}`; + }) + .join(','); + + lines.push(`${entry.name} ${versions} ${entry.infoChecksum}`); + } + + return lines.join('\n'); +} + +/** + * Generate Compact Index info file for a gem + * Format: VERSION[-PLATFORM] [DEP[,DEP,...]]|REQ[,REQ,...] + * @param entries - Info entries for gem versions + * @returns Compact Index info file content + */ +export function generateCompactIndexInfo(entries: ICompactIndexInfoEntry[]): string { + const lines: string[] = ['---']; // Info files start with --- + + for (const entry of entries) { + // Build version string with optional platform + const versionStr = entry.platform && entry.platform !== 'ruby' + ? `${entry.version}-${entry.platform}` + : entry.version; + + // Build dependencies string + const depsStr = entry.dependencies.length > 0 + ? entry.dependencies.map(formatDependency).join(',') + : ''; + + // Build requirements string (checksum is always required) + const reqParts: string[] = [`checksum:${entry.checksum}`]; + + for (const req of entry.requirements) { + reqParts.push(`${req.type}:${req.requirement}`); + } + + const reqStr = reqParts.join(','); + + // Combine: VERSION[-PLATFORM] [DEPS]|REQS + const depPart = depsStr ? ` ${depsStr}` : ''; + lines.push(`${versionStr}${depPart}|${reqStr}`); + } + + return lines.join('\n'); +} + +/** + * Format a dependency for Compact Index + * Format: GEM:CONSTRAINT[&CONSTRAINT] + * @param dep - Dependency object + * @returns Formatted dependency string + */ +export function formatDependency(dep: IRubyGemsDependency): string { + return `${dep.name}:${dep.requirement}`; +} + +/** + * Parse dependency string from Compact Index + * @param depStr - Dependency string + * @returns Dependency object + */ +export function parseDependency(depStr: string): IRubyGemsDependency { + const [name, ...reqParts] = depStr.split(':'); + const requirement = reqParts.join(':'); // Handle :: in gem names + + return { name, requirement }; +} + +/** + * Generate names file (newline-separated gem names) + * @param names - List of gem names + * @returns Names file content + */ +export function generateNamesFile(names: string[]): string { + return `---\n${names.sort().join('\n')}`; +} + +/** + * Calculate MD5 hash for Compact Index checksum + * @param content - Content to hash + * @returns MD5 hash (hex) + */ +export async function calculateMD5(content: string): Promise { + const crypto = await import('crypto'); + return crypto.createHash('md5').update(content).digest('hex'); +} + +/** + * Calculate SHA256 hash for gem files + * @param data - Data to hash + * @returns SHA256 hash (hex) + */ +export async function calculateSHA256(data: Buffer): Promise { + const crypto = await import('crypto'); + return crypto.createHash('sha256').update(data).digest('hex'); +} + +/** + * Parse gem filename to extract name, version, and platform + * @param filename - Gem filename (e.g., "rails-7.0.0-x86_64-linux.gem") + * @returns Parsed info or null + */ +export function parseGemFilename(filename: string): { + name: string; + version: string; + platform?: string; +} | null { + if (!filename.endsWith('.gem')) return null; + + const withoutExt = filename.slice(0, -4); // Remove .gem + + // Try to match: name-version-platform + // Platform can contain hyphens (e.g., x86_64-linux) + const parts = withoutExt.split('-'); + if (parts.length < 2) return null; + + // Find version (first part that starts with a digit) + let versionIndex = -1; + for (let i = 1; i < parts.length; i++) { + if (/^\d/.test(parts[i])) { + versionIndex = i; + break; + } + } + + if (versionIndex === -1) return null; + + const name = parts.slice(0, versionIndex).join('-'); + const version = parts[versionIndex]; + const platform = versionIndex + 1 < parts.length + ? parts.slice(versionIndex + 1).join('-') + : undefined; + + return { + name, + version, + platform: platform && platform !== 'ruby' ? platform : undefined, + }; +} + +/** + * Validate gem name + * Must contain only ASCII letters, numbers, _, and - + * @param name - Gem name + * @returns true if valid + */ +export function isValidGemName(name: string): boolean { + return /^[a-zA-Z0-9_-]+$/.test(name); +} + +/** + * Validate version string + * Basic semantic versioning check + * @param version - Version string + * @returns true if valid + */ +export function isValidVersion(version: string): boolean { + // Allow semver and other common Ruby version formats + return /^[\d.a-zA-Z_-]+$/.test(version); +} + +/** + * Build version list entry for Compact Index + * @param versions - Version info + * @returns Version list string + */ +export function buildVersionList(versions: Array<{ + version: string; + platform?: string; + yanked: boolean; +}>): string { + return versions + .map(v => { + const yanked = v.yanked ? '-' : ''; + const platform = v.platform && v.platform !== 'ruby' ? `_${v.platform}` : ''; + return `${yanked}${v.version}${platform}`; + }) + .join(','); +} + +/** + * Parse version list from Compact Index + * @param versionStr - Version list string + * @returns Parsed versions + */ +export function parseVersionList(versionStr: string): Array<{ + version: string; + platform?: string; + yanked: boolean; +}> { + return versionStr.split(',').map(v => { + const yanked = v.startsWith('-'); + const withoutYank = yanked ? v.substring(1) : v; + + // Split on _ to separate version from platform + const [version, ...platformParts] = withoutYank.split('_'); + const platform = platformParts.length > 0 ? platformParts.join('_') : undefined; + + return { + version, + platform: platform && platform !== 'ruby' ? platform : undefined, + yanked, + }; + }); +} + +/** + * Generate JSON response for /api/v1/versions/{gem}.json + * @param gemName - Gem name + * @param versions - Version list + * @returns JSON response object + */ +export function generateVersionsJson( + gemName: string, + versions: Array<{ + version: string; + platform?: string; + uploadTime?: string; + }> +): any { + return { + name: gemName, + versions: versions.map(v => ({ + number: v.version, + platform: v.platform || 'ruby', + built_at: v.uploadTime, + })), + }; +} + +/** + * Generate JSON response for /api/v1/dependencies + * @param gems - Map of gem names to version dependencies + * @returns JSON response array + */ +export function generateDependenciesJson(gems: Map>): any { + const result: any[] = []; + + for (const [name, versions] of gems) { + for (const v of versions) { + result.push({ + name, + number: v.version, + platform: v.platform || 'ruby', + dependencies: v.dependencies.map(d => ({ + name: d.name, + requirements: d.requirement, + })), + }); + } + } + + return result; +} + +/** + * Update Compact Index versions file with new gem version + * Handles append-only semantics for the current month + * @param existingContent - Current versions file content + * @param gemName - Gem name + * @param newVersion - New version info + * @param infoChecksum - MD5 of info file + * @returns Updated versions file content + */ +export function updateCompactIndexVersions( + existingContent: string, + gemName: string, + newVersion: { version: string; platform?: string; yanked: boolean }, + infoChecksum: string +): string { + const lines = existingContent.split('\n'); + const headerEndIndex = lines.findIndex(l => l === '---'); + + if (headerEndIndex === -1) { + throw new Error('Invalid Compact Index versions file'); + } + + const header = lines.slice(0, headerEndIndex + 1); + const entries = lines.slice(headerEndIndex + 1).filter(l => l.trim()); + + // Find existing entry for gem + const gemLineIndex = entries.findIndex(l => l.startsWith(`${gemName} `)); + + const versionStr = buildVersionList([newVersion]); + + if (gemLineIndex >= 0) { + // Append to existing entry + const parts = entries[gemLineIndex].split(' '); + const existingVersions = parts[1]; + const updatedVersions = `${existingVersions},${versionStr}`; + entries[gemLineIndex] = `${gemName} ${updatedVersions} ${infoChecksum}`; + } else { + // Add new entry + entries.push(`${gemName} ${versionStr} ${infoChecksum}`); + entries.sort(); // Keep alphabetical + } + + return [...header, ...entries].join('\n'); +} + +/** + * Update Compact Index info file with new version + * @param existingContent - Current info file content + * @param newEntry - New version entry + * @returns Updated info file content + */ +export function updateCompactIndexInfo( + existingContent: string, + newEntry: ICompactIndexInfoEntry +): string { + const lines = existingContent ? existingContent.split('\n').filter(l => l !== '---') : []; + + // Build version string + const versionStr = newEntry.platform && newEntry.platform !== 'ruby' + ? `${newEntry.version}-${newEntry.platform}` + : newEntry.version; + + // Build dependencies string + const depsStr = newEntry.dependencies.length > 0 + ? newEntry.dependencies.map(formatDependency).join(',') + : ''; + + // Build requirements string + const reqParts: string[] = [`checksum:${newEntry.checksum}`]; + for (const req of newEntry.requirements) { + reqParts.push(`${req.type}:${req.requirement}`); + } + const reqStr = reqParts.join(','); + + // Combine + const depPart = depsStr ? ` ${depsStr}` : ''; + const newLine = `${versionStr}${depPart}|${reqStr}`; + + lines.push(newLine); + + return `---\n${lines.join('\n')}`; +} + +/** + * Extract gem specification from .gem file + * Note: This is a simplified version. Full implementation would use tar + gzip + Marshal + * @param gemData - Gem file data + * @returns Extracted spec or null + */ +export async function extractGemSpec(gemData: Buffer): Promise { + try { + // .gem files are gzipped tar archives + // They contain metadata.gz which has Marshal-encoded spec + // This is a placeholder - full implementation would need: + // 1. Unzip outer gzip + // 2. Untar to find metadata.gz + // 3. Unzip metadata.gz + // 4. Parse Ruby Marshal format + + // For now, return null and expect metadata to be provided + return null; + } catch (error) { + return null; + } +} diff --git a/ts/rubygems/index.ts b/ts/rubygems/index.ts new file mode 100644 index 0000000..c801d2f --- /dev/null +++ b/ts/rubygems/index.ts @@ -0,0 +1,8 @@ +/** + * RubyGems Registry Module + * RubyGems/Bundler Compact Index implementation + */ + +export * from './interfaces.rubygems.js'; +export * from './classes.rubygemsregistry.js'; +export * as rubygemsHelpers from './helpers.rubygems.js'; diff --git a/ts/rubygems/interfaces.rubygems.ts b/ts/rubygems/interfaces.rubygems.ts new file mode 100644 index 0000000..9c04ca0 --- /dev/null +++ b/ts/rubygems/interfaces.rubygems.ts @@ -0,0 +1,251 @@ +/** + * RubyGems Registry Type Definitions + * Compliant with Compact Index API and RubyGems protocol + */ + +/** + * Gem version entry in compact index + */ +export interface IRubyGemsVersion { + /** Version number */ + version: string; + /** Platform (e.g., ruby, x86_64-linux) */ + platform?: string; + /** Dependencies */ + dependencies?: IRubyGemsDependency[]; + /** Requirements */ + requirements?: IRubyGemsRequirement[]; + /** Whether this version is yanked */ + yanked?: boolean; + /** SHA256 checksum of .gem file */ + checksum?: string; +} + +/** + * Gem dependency specification + */ +export interface IRubyGemsDependency { + /** Gem name */ + name: string; + /** Version requirement (e.g., ">= 1.0", "~> 2.0") */ + requirement: string; +} + +/** + * Gem requirements (ruby version, rubygems version, etc.) + */ +export interface IRubyGemsRequirement { + /** Requirement type (ruby, rubygems) */ + type: 'ruby' | 'rubygems'; + /** Version requirement */ + requirement: string; +} + +/** + * Complete gem metadata + */ +export interface IRubyGemsMetadata { + /** Gem name */ + name: string; + /** All versions */ + versions: Record; + /** Last modified timestamp */ + 'last-modified'?: string; +} + +/** + * Version-specific metadata + */ +export interface IRubyGemsVersionMetadata { + /** Version number */ + version: string; + /** Platform */ + platform?: string; + /** Authors */ + authors?: string[]; + /** Description */ + description?: string; + /** Summary */ + summary?: string; + /** Homepage */ + homepage?: string; + /** License */ + license?: string; + /** Dependencies */ + dependencies?: IRubyGemsDependency[]; + /** Requirements */ + requirements?: IRubyGemsRequirement[]; + /** SHA256 checksum */ + checksum: string; + /** File size */ + size: number; + /** Upload timestamp */ + 'upload-time': string; + /** Uploader */ + 'uploaded-by': string; + /** Yanked status */ + yanked?: boolean; + /** Yank reason */ + 'yank-reason'?: string; +} + +/** + * Compact index versions file entry + * Format: GEMNAME [-]VERSION_PLATFORM[,VERSION_PLATFORM,...] MD5 + */ +export interface ICompactIndexVersionsEntry { + /** Gem name */ + name: string; + /** Versions (with optional platform and yank flag) */ + versions: Array<{ + version: string; + platform?: string; + yanked: boolean; + }>; + /** MD5 checksum of info file */ + infoChecksum: string; +} + +/** + * Compact index info file entry + * Format: VERSION[-PLATFORM] [DEP[,DEP,...]]|REQ[,REQ,...] + */ +export interface ICompactIndexInfoEntry { + /** Version number */ + version: string; + /** Platform (optional) */ + platform?: string; + /** Dependencies */ + dependencies: IRubyGemsDependency[]; + /** Requirements */ + requirements: IRubyGemsRequirement[]; + /** SHA256 checksum */ + checksum: string; +} + +/** + * Gem upload request + */ +export interface IRubyGemsUploadRequest { + /** Gem file data */ + gemData: Buffer; + /** Gem filename */ + filename: string; +} + +/** + * Gem upload response + */ +export interface IRubyGemsUploadResponse { + /** Success message */ + message?: string; + /** Gem name */ + name?: string; + /** Version */ + version?: string; +} + +/** + * Yank request + */ +export interface IRubyGemsYankRequest { + /** Gem name */ + gem_name: string; + /** Version to yank */ + version: string; + /** Platform (optional) */ + platform?: string; +} + +/** + * Yank response + */ +export interface IRubyGemsYankResponse { + /** Success indicator */ + success: boolean; + /** Message */ + message?: string; +} + +/** + * Version info response (JSON) + */ +export interface IRubyGemsVersionInfo { + /** Gem name */ + name: string; + /** Versions list */ + versions: Array<{ + /** Version number */ + number: string; + /** Platform */ + platform?: string; + /** Build date */ + built_at?: string; + /** Download count */ + downloads_count?: number; + }>; +} + +/** + * Dependencies query response + */ +export interface IRubyGemsDependenciesResponse { + /** Dependencies for requested gems */ + dependencies: Array<{ + /** Gem name */ + name: string; + /** Version */ + number: string; + /** Platform */ + platform?: string; + /** Dependencies */ + dependencies: Array<{ + name: string; + requirements: string; + }>; + }>; +} + +/** + * Error response structure + */ +export interface IRubyGemsError { + /** Error message */ + message: string; + /** HTTP status code */ + status?: number; +} + +/** + * Gem specification (extracted from .gem file) + */ +export interface IRubyGemsSpec { + /** Gem name */ + name: string; + /** Version */ + version: string; + /** Platform */ + platform?: string; + /** Authors */ + authors?: string[]; + /** Email */ + email?: string; + /** Homepage */ + homepage?: string; + /** Summary */ + summary?: string; + /** Description */ + description?: string; + /** License */ + license?: string; + /** Dependencies */ + dependencies?: IRubyGemsDependency[]; + /** Required Ruby version */ + required_ruby_version?: string; + /** Required RubyGems version */ + required_rubygems_version?: string; + /** Files */ + files?: string[]; + /** Requirements */ + requirements?: string[]; +}