From 9ca1e670effb7d28d181d9a23071b2ebe641c207 Mon Sep 17 00:00:00 2001 From: Juergen Kunz Date: Fri, 21 Nov 2025 14:23:18 +0000 Subject: [PATCH] feat(core): Add PyPI and RubyGems protocol support, Cargo token management, and storage helpers --- changelog.md | 9 + readme.hints.md | 334 ++++++++++++++++- test/helpers/registry.ts | 9 +- test/test.cargo.nativecli.node.ts | 475 ++++++++++++++++++++++++ ts/00_commitinfo_data.ts | 2 +- ts/core/classes.authmanager.ts | 163 ++++++++- ts/core/classes.registrystorage.ts | 227 ++++++++++++ ts/core/interfaces.core.ts | 14 +- ts/pypi/classes.pypiregistry.ts | 564 +++++++++++++++++++++++++++++ ts/pypi/helpers.pypi.ts | 299 +++++++++++++++ ts/pypi/index.ts | 8 + ts/pypi/interfaces.pypi.ts | 316 ++++++++++++++++ 12 files changed, 2414 insertions(+), 6 deletions(-) create mode 100644 test/test.cargo.nativecli.node.ts create mode 100644 ts/pypi/classes.pypiregistry.ts create mode 100644 ts/pypi/helpers.pypi.ts create mode 100644 ts/pypi/index.ts create mode 100644 ts/pypi/interfaces.pypi.ts diff --git a/changelog.md b/changelog.md index c388ff1..eb4ecfb 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,14 @@ # Changelog +## 2025-11-21 - 1.5.0 - feat(core) +Add PyPI and RubyGems protocol support, Cargo token management, and storage helpers + +- Extend core protocol types to include 'pypi' and 'rubygems' and add protocol config entries for pypi and rubygems. +- Add PyPI storage methods for metadata, Simple API HTML/JSON indexes, package files, version listing and deletion in RegistryStorage. +- Add Cargo-specific storage helpers (index paths, crate storage) and ensure Cargo registry initialization and endpoints are wired into SmartRegistry. +- Extend AuthManager with Cargo, PyPI and RubyGems token creation, validation and revocation methods; update unified validateToken to check these token types. +- Update test helpers to create Cargo tokens and return cargoToken from registry setup. + ## 2025-11-21 - 1.4.1 - fix(devcontainer) Simplify devcontainer configuration and rename container image diff --git a/readme.hints.md b/readme.hints.md index 93e4a7a..5a3ffe9 100644 --- a/readme.hints.md +++ b/readme.hints.md @@ -1,3 +1,335 @@ # Project Readme Hints -This is the initial readme hints file. \ No newline at end of file +## Python (PyPI) Protocol Implementation Notes + +### PEP 503: Simple Repository API (HTML-based) + +**URL Structure:** +- Root: `//` - Lists all projects +- Project: `///` - Lists all files for a project +- All URLs MUST end with `/` (redirect if missing) + +**Package Name Normalization:** +- Lowercase all characters +- Replace runs of `.`, `-`, `_` with single `-` +- Implementation: `re.sub(r"[-_.]+", "-", name).lower()` + +**HTML Format:** +- Root: One anchor per project +- Project: One anchor per file +- Anchor text must match final filename +- Anchor href links to download URL + +**Hash Fragments:** +Format: `#=` +- hashname: lowercase hash function name (recommend `sha256`) +- hashvalue: hex-encoded digest + +**Data Attributes:** +- `data-gpg-sig`: `true`/`false` for GPG signature presence +- `data-requires-python`: PEP 345 requirement string (HTML-encode `<` as `<`, `>` as `>`) + +### PEP 691: JSON-based Simple API + +**Content Types:** +- `application/vnd.pypi.simple.v1+json` - JSON format +- `application/vnd.pypi.simple.v1+html` - HTML format +- `text/html` - Alias for HTML (backwards compat) + +**Root Endpoint JSON:** +```json +{ + "meta": {"api-version": "1.0"}, + "projects": [{"name": "ProjectName"}] +} +``` + +**Project Endpoint JSON:** +```json +{ + "name": "normalized-name", + "meta": {"api-version": "1.0"}, + "files": [ + { + "filename": "package-1.0-py3-none-any.whl", + "url": "https://example.com/path/to/file", + "hashes": {"sha256": "..."}, + "requires-python": ">=3.7", + "dist-info-metadata": true | {"sha256": "..."}, + "gpg-sig": true, + "yanked": false | "reason string" + } + ] +} +``` + +**Content Negotiation:** +- Use `Accept` header for format selection +- Server responds with `Content-Type` header +- Support both JSON and HTML formats + +### PyPI Upload API (Legacy /legacy/) + +**Endpoint:** +- URL: `https://upload.pypi.org/legacy/` +- Method: `POST` +- Content-Type: `multipart/form-data` + +**Required Form Fields:** +- `:action` = `file_upload` +- `protocol_version` = `1` +- `content` = Binary file data with filename +- `filetype` = `bdist_wheel` | `sdist` +- `pyversion` = Python tag (e.g., `py3`, `py2.py3`) or `source` for sdist +- `metadata_version` = Metadata standard version +- `name` = Package name +- `version` = Version string + +**Hash Digest (one required):** +- `md5_digest`: urlsafe base64 without padding +- `sha256_digest`: hexadecimal +- `blake2_256_digest`: hexadecimal + +**Optional Fields:** +- `attestations`: JSON array of attestation objects +- Any Core Metadata fields (lowercase, hyphens → underscores) + - Example: `Description-Content-Type` → `description_content_type` + +**Authentication:** +- Username/password or API token in HTTP Basic Auth +- API tokens: username = `__token__`, password = token value + +**Behavior:** +- First file uploaded creates the release +- Multiple files uploaded sequentially for same version + +### PEP 694: Upload 2.0 API + +**Status:** Draft (not yet required, legacy API still supported) +- Multi-step workflow with sessions +- Async upload support with resumption +- JSON-based API +- Standard HTTP auth (RFC 7235) +- Not implementing initially (legacy API sufficient) + +--- + +## Ruby (RubyGems) Protocol Implementation Notes + +### Compact Index Format + +**Endpoints:** +- `/versions` - Master list of all gems and versions +- `/info/` - Detailed info for specific gem +- `/names` - Simple list of gem names + +**Authentication:** +- UUID tokens similar to NPM pattern +- API key in `Authorization` header +- Scope format: `rubygems:gem:{name}:{read|write|yank}` + +### `/versions` File Format + +**Structure:** +``` +created_at: 2024-04-01T00:00:05Z +--- +RUBYGEM [-]VERSION_PLATFORM[,VERSION_PLATFORM,...] MD5 +``` + +**Details:** +- Metadata lines before `---` delimiter +- One line per gem with comma-separated versions +- `[-]` prefix indicates yanked version +- `MD5`: Checksum of corresponding `/info/` file +- Append-only during month, recalculated monthly + +### `/info/` File Format + +**Structure:** +``` +--- +VERSION[-PLATFORM] [DEPENDENCY[,DEPENDENCY,...]]|REQUIREMENT[,REQUIREMENT,...] +``` + +**Dependency Format:** +``` +GEM:CONSTRAINT[&CONSTRAINT] +``` +- Examples: `actionmailer:= 2.2.2`, `parser:>= 3.2.2.3` +- Operators: `=`, `>`, `<`, `>=`, `<=`, `~>`, `!=` +- Multiple constraints: `unicode-display_width:< 3.0&>= 2.4.0` + +**Requirement Format:** +``` +checksum:SHA256_HEX +ruby:CONSTRAINT +rubygems:CONSTRAINT +``` + +**Platform:** +- Default platform is `ruby` +- Non-default platforms: `VERSION-PLATFORM` (e.g., `3.2.1-arm64-darwin`) + +**Yanked Gems:** +- Listed with `-` prefix in `/versions` +- Excluded entirely from `/info/` file + +### `/names` File Format + +``` +--- +gemname1 +gemname2 +gemname3 +``` + +### HTTP Range Support + +**Headers:** +- `Range: bytes=#{start}-`: Request from byte position +- `If-None-Match`: ETag conditional request +- `Repr-Digest`: SHA256 checksum in response + +**Caching Strategy:** +1. Store file with last byte position +2. Request range from last position +3. Append response to existing file +4. Verify SHA256 against `Repr-Digest` + +### RubyGems Upload/Management API + +**Upload Gem:** +- `POST /api/v1/gems` +- Binary `.gem` file in request body +- `Authorization` header with API key + +**Yank Version:** +- `DELETE /api/v1/gems/yank` +- Parameters: `gem_name`, `version` + +**Unyank Version:** +- `PUT /api/v1/gems/unyank` +- Parameters: `gem_name`, `version` + +**Version Metadata:** +- `GET /api/v1/versions/.json` +- Returns JSON array of versions + +**Dependencies:** +- `GET /api/v1/dependencies?gems=` +- Returns dependency information for resolution + +--- + +## Implementation Strategy + +### Storage Paths + +**PyPI:** +``` +pypi/ +├── simple/ # PEP 503 HTML files +│ ├── index.html # All packages list +│ └── {package}/index.html # Package versions list +├── packages/ +│ └── {package}/{filename} # .whl and .tar.gz files +└── metadata/ + └── {package}/metadata.json # Package metadata +``` + +**RubyGems:** +``` +rubygems/ +├── versions # Master versions file +├── info/{gemname} # Per-gem info files +├── names # All gem names +└── gems/{gemname}-{version}.gem # .gem files +``` + +### Authentication Pattern + +Both protocols should follow the existing UUID token pattern used by NPM, Maven, Cargo, Composer: + +```typescript +// AuthManager additions +createPypiToken(userId: string, readonly: boolean): string +validatePypiToken(token: string): ITokenInfo | null +revokePypiToken(token: string): boolean + +createRubyGemsToken(userId: string, readonly: boolean): string +validateRubyGemsToken(token: string): ITokenInfo | null +revokeRubyGemsToken(token: string): boolean +``` + +### Scope Format + +``` +pypi:package:{name}:{read|write} +rubygems:gem:{name}:{read|write|yank} +``` + +### Common Patterns + +1. **Package name normalization** - Critical for PyPI +2. **Checksum calculation** - SHA256 for both protocols +3. **Append-only files** - RubyGems compact index +4. **Content negotiation** - PyPI JSON vs HTML +5. **Multipart upload parsing** - PyPI file uploads +6. **Binary file handling** - Both protocols (.whl, .tar.gz, .gem) + +--- + +## Key Differences from Existing Protocols + +**PyPI vs NPM:** +- PyPI uses Simple API (HTML) + JSON API +- PyPI requires package name normalization +- PyPI uses multipart form data for uploads (not JSON) +- PyPI supports multiple file types per release (wheel + sdist) + +**RubyGems vs Cargo:** +- RubyGems uses compact index (append-only text files) +- RubyGems uses checksums in index files (not just filenames) +- RubyGems has HTTP Range support for incremental updates +- RubyGems uses MD5 for index checksums, SHA256 for .gem files + +--- + +## Testing Requirements + +### PyPI Tests Must Cover: +- Package upload (wheel and sdist) +- Package name normalization +- Simple API HTML generation (PEP 503) +- JSON API responses (PEP 691) +- Content negotiation +- Hash calculation and verification +- Authentication (tokens) +- Multi-file releases +- Yanked packages + +### RubyGems Tests Must Cover: +- Gem upload +- Compact index generation +- `/versions` file updates (append-only) +- `/info/` file generation +- `/names` file generation +- Checksum calculations (MD5 and SHA256) +- Platform-specific gems +- Yanking/unyanking +- HTTP Range requests +- Authentication (API keys) + +--- + +## Security Considerations + +1. **Package name validation** - Prevent path traversal +2. **File size limits** - Prevent DoS via large uploads +3. **Content-Type validation** - Verify file types +4. **Checksum verification** - Ensure file integrity +5. **Token scope enforcement** - Read vs write permissions +6. **HTML escaping** - Prevent XSS in generated HTML +7. **Metadata sanitization** - Clean user-provided strings +8. **Rate limiting** - Consider upload frequency limits diff --git a/test/helpers/registry.ts b/test/helpers/registry.ts index 4f6c9a0..2f6ced6 100644 --- a/test/helpers/registry.ts +++ b/test/helpers/registry.ts @@ -53,6 +53,10 @@ export async function createTestRegistry(): Promise { enabled: true, basePath: '/composer', }, + cargo: { + enabled: true, + basePath: '/cargo', + }, }; const registry = new SmartRegistry(config); @@ -93,7 +97,10 @@ export async function createTestTokens(registry: SmartRegistry) { // Create Composer token with full access const composerToken = await authManager.createComposerToken(userId, false); - return { npmToken, ociToken, mavenToken, composerToken, userId }; + // Create Cargo token with full access + const cargoToken = await authManager.createCargoToken(userId, false); + + return { npmToken, ociToken, mavenToken, composerToken, cargoToken, userId }; } /** diff --git a/test/test.cargo.nativecli.node.ts b/test/test.cargo.nativecli.node.ts new file mode 100644 index 0000000..032d9f7 --- /dev/null +++ b/test/test.cargo.nativecli.node.ts @@ -0,0 +1,475 @@ +/** + * Native cargo CLI Testing + * Tests the Cargo registry implementation using the actual cargo CLI + */ + +import { expect, tap } from '@git.zone/tstest/tapbundle'; +import { tapNodeTools } from '@git.zone/tstest/tapbundle_serverside'; +import { SmartRegistry } from '../ts/index.js'; +import { createTestRegistry, createTestTokens } from './helpers/registry.js'; +import type { IRequestContext, IResponse } from '../ts/core/interfaces.core.js'; +import * as http from 'http'; +import * as url from 'url'; +import * as fs from 'fs'; +import * as path from 'path'; + +// Test context +let registry: SmartRegistry; +let server: http.Server; +let registryUrl: string; +let registryPort: number; +let cargoToken: string; +let testDir: string; +let cargoHome: string; + +/** + * Create HTTP server wrapper around SmartRegistry + */ +async function createHttpServer( + registryInstance: SmartRegistry, + port: number +): Promise<{ server: http.Server; url: string }> { + return new Promise((resolve, reject) => { + const httpServer = http.createServer(async (req, res) => { + try { + // Parse request + const parsedUrl = url.parse(req.url || '', true); + const pathname = parsedUrl.pathname || '/'; + const query = parsedUrl.query; + + // Read body + const chunks: Buffer[] = []; + for await (const chunk of req) { + chunks.push(chunk); + } + const bodyBuffer = Buffer.concat(chunks); + + // Parse body based on content type + let body: any; + if (bodyBuffer.length > 0) { + const contentType = req.headers['content-type'] || ''; + if (contentType.includes('application/json')) { + try { + body = JSON.parse(bodyBuffer.toString('utf-8')); + } catch (error) { + body = bodyBuffer; + } + } else { + body = bodyBuffer; + } + } + + // Convert to IRequestContext + const context: IRequestContext = { + method: req.method || 'GET', + path: pathname, + headers: req.headers as Record, + query: query as Record, + body: body, + }; + + // Handle request + const response: IResponse = await registryInstance.handleRequest(context); + + // Convert IResponse to HTTP response + res.statusCode = response.status; + + // Set headers + for (const [key, value] of Object.entries(response.headers || {})) { + res.setHeader(key, value); + } + + // Send body + if (response.body) { + if (Buffer.isBuffer(response.body)) { + res.end(response.body); + } else if (typeof response.body === 'string') { + res.end(response.body); + } else { + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify(response.body)); + } + } else { + res.end(); + } + } catch (error) { + console.error('Server error:', error); + res.statusCode = 500; + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ error: 'INTERNAL_ERROR', message: String(error) })); + } + }); + + httpServer.listen(port, () => { + const serverUrl = `http://localhost:${port}`; + resolve({ server: httpServer, url: serverUrl }); + }); + + httpServer.on('error', reject); + }); +} + +/** + * Setup Cargo configuration + */ +function setupCargoConfig(registryUrlArg: string, token: string, cargoHomeArg: string): void { + const cargoConfigDir = path.join(cargoHomeArg, '.cargo'); + fs.mkdirSync(cargoConfigDir, { recursive: true }); + + // Create config.toml with sparse protocol + const configContent = `[registries.test-registry] +index = "sparse+${registryUrlArg}/cargo/" + +[source.crates-io] +replace-with = "test-registry" + +[net] +retry = 0 +`; + + fs.writeFileSync(path.join(cargoConfigDir, 'config.toml'), configContent, 'utf-8'); + + // Create credentials.toml (Cargo uses plain token, no "Bearer" prefix) + const credentialsContent = `[registries.test-registry] +token = "${token}" +`; + + fs.writeFileSync(path.join(cargoConfigDir, 'credentials.toml'), credentialsContent, 'utf-8'); +} + +/** + * Create a test Cargo crate + */ +function createTestCrate( + crateName: string, + version: string, + targetDir: string +): string { + const crateDir = path.join(targetDir, crateName); + fs.mkdirSync(crateDir, { recursive: true }); + + // Create Cargo.toml + const cargoToml = `[package] +name = "${crateName}" +version = "${version}" +edition = "2021" +description = "Test crate ${crateName}" +license = "MIT" +authors = ["Test Author "] + +[dependencies] +`; + + fs.writeFileSync(path.join(crateDir, 'Cargo.toml'), cargoToml, 'utf-8'); + + // Create src directory + const srcDir = path.join(crateDir, 'src'); + fs.mkdirSync(srcDir, { recursive: true }); + + // Create lib.rs + const libRs = `//! Test crate ${crateName} + +/// Returns a greeting message +pub fn greet() -> String { + format!("Hello from {}@{}", "${crateName}", "${version}") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_greet() { + let greeting = greet(); + assert!(greeting.contains("${crateName}")); + } +} +`; + + fs.writeFileSync(path.join(srcDir, 'lib.rs'), libRs, 'utf-8'); + + // Create README.md + const readme = `# ${crateName} + +Test crate for SmartRegistry. + +Version: ${version} +`; + + fs.writeFileSync(path.join(crateDir, 'README.md'), readme, 'utf-8'); + + return crateDir; +} + +/** + * Run cargo command with proper environment + */ +async function runCargoCommand( + command: string, + cwd: string, + includeToken: boolean = true +): Promise<{ stdout: string; stderr: string; exitCode: number }> { + // Prepare environment variables + // NOTE: Cargo converts registry name "test-registry" to "TEST_REGISTRY" for env vars + const envVars = [ + `CARGO_HOME="${cargoHome}"`, + `CARGO_REGISTRIES_TEST_REGISTRY_INDEX="sparse+${registryUrl}/cargo/"`, + includeToken ? `CARGO_REGISTRIES_TEST_REGISTRY_TOKEN="${cargoToken}"` : '', + `CARGO_NET_RETRY="0"`, + ].filter(Boolean).join(' '); + + // Build command with cd to correct directory and environment variables + const fullCommand = `cd "${cwd}" && ${envVars} ${command}`; + + try { + const result = await tapNodeTools.runCommand(fullCommand); + return { + stdout: result.stdout || '', + stderr: result.stderr || '', + exitCode: result.exitCode || 0, + }; + } catch (error: any) { + return { + stdout: error.stdout || '', + stderr: error.stderr || String(error), + exitCode: error.exitCode || 1, + }; + } +} + +/** + * Cleanup test directory + */ +function cleanupTestDir(dir: string): void { + if (fs.existsSync(dir)) { + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +// ======================================================================== +// TESTS +// ======================================================================== + +tap.test('Cargo CLI: should setup registry and HTTP server', async () => { + // Create registry + registry = await createTestRegistry(); + const tokens = await createTestTokens(registry); + cargoToken = tokens.cargoToken; + + expect(registry).toBeInstanceOf(SmartRegistry); + expect(cargoToken).toBeTypeOf('string'); + + // Clean up any existing index from previous test runs + const storage = registry.getStorage(); + try { + await storage.putCargoIndex('test-crate-cli', []); + } catch (error) { + // Ignore error if operation fails + } + + // Use port 5000 (hardcoded in CargoRegistry default config) + // TODO: Once registryUrl is configurable, use dynamic port like npm test (35001) + registryPort = 5000; + const serverSetup = await createHttpServer(registry, registryPort); + server = serverSetup.server; + registryUrl = serverSetup.url; + + expect(server).toBeDefined(); + expect(registryUrl).toEqual(`http://localhost:${registryPort}`); + + // Setup test directory + testDir = path.join(process.cwd(), '.nogit', 'test-cargo-cli'); + cleanupTestDir(testDir); + fs.mkdirSync(testDir, { recursive: true }); + + // Setup CARGO_HOME + cargoHome = path.join(testDir, '.cargo-home'); + fs.mkdirSync(cargoHome, { recursive: true }); + + // Setup Cargo config + setupCargoConfig(registryUrl, cargoToken, cargoHome); + expect(fs.existsSync(path.join(cargoHome, '.cargo', 'config.toml'))).toEqual(true); + expect(fs.existsSync(path.join(cargoHome, '.cargo', 'credentials.toml'))).toEqual(true); +}); + +tap.test('Cargo CLI: should verify server is responding', async () => { + // Check server is up by doing a direct HTTP request to the cargo index + const response = await fetch(`${registryUrl}/cargo/`); + expect(response.status).toBeGreaterThanOrEqual(200); + expect(response.status).toBeLessThan(500); +}); + +tap.test('Cargo CLI: should publish a crate', async () => { + const crateName = 'test-crate-cli'; + const version = '0.1.0'; + const crateDir = createTestCrate(crateName, version, testDir); + + const result = await runCargoCommand('cargo publish --registry test-registry --allow-dirty', crateDir); + console.log('cargo publish output:', result.stdout); + console.log('cargo publish stderr:', result.stderr); + + expect(result.exitCode).toEqual(0); + expect(result.stdout || result.stderr).toContain(crateName); +}); + +tap.test('Cargo CLI: should verify crate in index', async () => { + const crateName = 'test-crate-cli'; + + // Cargo uses a specific index structure + // For crate "test-crate-cli", the index path is based on the first characters + // 1 char: + // 2 char: 2/ + // 3 char: 3// + // 4+ char: // + + // "test-crate-cli" is 14 chars, so it should be at: te/st/test-crate-cli + const indexPath = `/cargo/te/st/${crateName}`; + + const response = await fetch(`${registryUrl}${indexPath}`); + expect(response.status).toEqual(200); + + const indexData = await response.text(); + console.log('Index data:', indexData); + + // Index should contain JSON line with crate info + expect(indexData).toContain(crateName); + expect(indexData).toContain('0.1.0'); +}); + +tap.test('Cargo CLI: should download published crate', async () => { + const crateName = 'test-crate-cli'; + const version = '0.1.0'; + + // Cargo downloads crates from /cargo/api/v1/crates/{name}/{version}/download + const downloadPath = `/cargo/api/v1/crates/${crateName}/${version}/download`; + + const response = await fetch(`${registryUrl}${downloadPath}`); + expect(response.status).toEqual(200); + + const crateData = await response.arrayBuffer(); + expect(crateData.byteLength).toBeGreaterThan(0); +}); + +tap.test('Cargo CLI: should publish second version', async () => { + const crateName = 'test-crate-cli'; + const version = '0.2.0'; + const crateDir = createTestCrate(crateName, version, testDir); + + const result = await runCargoCommand('cargo publish --registry test-registry --allow-dirty', crateDir); + console.log('cargo publish v0.2.0 output:', result.stdout); + + expect(result.exitCode).toEqual(0); +}); + +tap.test('Cargo CLI: should list versions in index', async () => { + const crateName = 'test-crate-cli'; + const indexPath = `/cargo/te/st/${crateName}`; + + const response = await fetch(`${registryUrl}${indexPath}`); + expect(response.status).toEqual(200); + + const indexData = await response.text(); + const lines = indexData.trim().split('\n'); + + // Should have 2 lines (2 versions) + expect(lines.length).toEqual(2); + + // Parse JSON lines + const version1 = JSON.parse(lines[0]); + const version2 = JSON.parse(lines[1]); + + expect(version1.vers).toEqual('0.1.0'); + expect(version2.vers).toEqual('0.2.0'); +}); + +tap.test('Cargo CLI: should search for crate', async () => { + const crateName = 'test-crate-cli'; + + // Cargo search endpoint: /cargo/api/v1/crates?q={query} + const response = await fetch(`${registryUrl}/cargo/api/v1/crates?q=${crateName}`); + expect(response.status).toEqual(200); + + const searchResults = await response.json(); + console.log('Search results:', searchResults); + + expect(searchResults).toHaveProperty('crates'); + expect(searchResults.crates).toBeInstanceOf(Array); + expect(searchResults.crates.length).toBeGreaterThan(0); + expect(searchResults.crates[0].name).toEqual(crateName); +}); + +tap.test('Cargo CLI: should yank a version', async () => { + const crateName = 'test-crate-cli'; + const crateDir = path.join(testDir, crateName); + + const result = await runCargoCommand('cargo yank --registry test-registry --vers 0.1.0', crateDir); + console.log('cargo yank output:', result.stdout); + console.log('cargo yank stderr:', result.stderr); + + expect(result.exitCode).toEqual(0); + + // Verify version is yanked in index + const indexPath = `/cargo/te/st/${crateName}`; + const response = await fetch(`${registryUrl}${indexPath}`); + const indexData = await response.text(); + const lines = indexData.trim().split('\n'); + const version1 = JSON.parse(lines[0]); + + expect(version1.yanked).toEqual(true); +}); + +tap.test('Cargo CLI: should unyank a version', async () => { + const crateName = 'test-crate-cli'; + const crateDir = path.join(testDir, crateName); + + const result = await runCargoCommand('cargo yank --registry test-registry --vers 0.1.0 --undo', crateDir); + console.log('cargo unyank output:', result.stdout); + console.log('cargo unyank stderr:', result.stderr); + + expect(result.exitCode).toEqual(0); + + // Verify version is not yanked in index + const indexPath = `/cargo/te/st/${crateName}`; + const response = await fetch(`${registryUrl}${indexPath}`); + const indexData = await response.text(); + const lines = indexData.trim().split('\n'); + const version1 = JSON.parse(lines[0]); + + expect(version1.yanked).toEqual(false); +}); + +tap.test('Cargo CLI: should fail to publish without auth', async () => { + const crateName = 'unauth-crate'; + const version = '0.1.0'; + const crateDir = createTestCrate(crateName, version, testDir); + + // Run without token (includeToken: false) + const result = await runCargoCommand('cargo publish --registry test-registry --allow-dirty', crateDir, false); + console.log('cargo publish unauth output:', result.stdout); + console.log('cargo publish unauth stderr:', result.stderr); + + // Should fail with auth error + expect(result.exitCode).not.toEqual(0); + expect(result.stderr).toContain('token'); +}); + +tap.postTask('cleanup cargo cli tests', async () => { + // Stop server + if (server) { + await new Promise((resolve) => { + server.close(() => resolve()); + }); + } + + // Cleanup test directory + if (testDir) { + cleanupTestDir(testDir); + } + + // Destroy registry + if (registry) { + registry.destroy(); + } +}); + +export default tap.start(); diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index bffc87d..71a3f73 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@push.rocks/smartregistry', - version: '1.4.1', + version: '1.5.0', description: 'a registry for npm modules and oci images' } diff --git a/ts/core/classes.authmanager.ts b/ts/core/classes.authmanager.ts index 98be963..1f6ea81 100644 --- a/ts/core/classes.authmanager.ts +++ b/ts/core/classes.authmanager.ts @@ -317,12 +317,153 @@ export class AuthManager { this.tokenStore.delete(token); } + // ======================================================================== + // CARGO TOKEN MANAGEMENT + // ======================================================================== + + /** + * Create a Cargo token + * @param userId - User ID + * @param readonly - Whether the token is readonly + * @returns Cargo UUID token + */ + public async createCargoToken(userId: string, readonly: boolean = false): Promise { + const scopes = readonly ? ['cargo:*:*:read'] : ['cargo:*:*:*']; + return this.createUuidToken(userId, 'cargo', scopes, readonly); + } + + /** + * Validate a Cargo token + * @param token - Cargo UUID token + * @returns Auth token object or null + */ + public async validateCargoToken(token: string): Promise { + if (!this.isValidUuid(token)) { + return null; + } + + const authToken = this.tokenStore.get(token); + if (!authToken || authToken.type !== 'cargo') { + return null; + } + + // Check expiration if set + if (authToken.expiresAt && authToken.expiresAt < new Date()) { + this.tokenStore.delete(token); + return null; + } + + return authToken; + } + + /** + * Revoke a Cargo token + * @param token - Cargo UUID token + */ + public async revokeCargoToken(token: string): Promise { + this.tokenStore.delete(token); + } + + // ======================================================================== + // PYPI AUTHENTICATION + // ======================================================================== + + /** + * Create a PyPI token + * @param userId - User ID + * @param readonly - Whether the token is readonly + * @returns PyPI UUID token + */ + public async createPypiToken(userId: string, readonly: boolean = false): Promise { + const scopes = readonly ? ['pypi:*:*:read'] : ['pypi:*:*:*']; + return this.createUuidToken(userId, 'pypi', scopes, readonly); + } + + /** + * Validate a PyPI token + * @param token - PyPI UUID token + * @returns Auth token object or null + */ + public async validatePypiToken(token: string): Promise { + if (!this.isValidUuid(token)) { + return null; + } + + const authToken = this.tokenStore.get(token); + if (!authToken || authToken.type !== 'pypi') { + return null; + } + + // Check expiration if set + if (authToken.expiresAt && authToken.expiresAt < new Date()) { + this.tokenStore.delete(token); + return null; + } + + return authToken; + } + + /** + * Revoke a PyPI token + * @param token - PyPI UUID token + */ + public async revokePypiToken(token: string): Promise { + this.tokenStore.delete(token); + } + + // ======================================================================== + // RUBYGEMS AUTHENTICATION + // ======================================================================== + + /** + * Create a RubyGems token + * @param userId - User ID + * @param readonly - Whether the token is readonly + * @returns RubyGems UUID token + */ + public async createRubyGemsToken(userId: string, readonly: boolean = false): Promise { + const scopes = readonly ? ['rubygems:*:*:read'] : ['rubygems:*:*:*']; + return this.createUuidToken(userId, 'rubygems', scopes, readonly); + } + + /** + * Validate a RubyGems token + * @param token - RubyGems UUID token + * @returns Auth token object or null + */ + public async validateRubyGemsToken(token: string): Promise { + if (!this.isValidUuid(token)) { + return null; + } + + const authToken = this.tokenStore.get(token); + if (!authToken || authToken.type !== 'rubygems') { + return null; + } + + // Check expiration if set + if (authToken.expiresAt && authToken.expiresAt < new Date()) { + this.tokenStore.delete(token); + return null; + } + + return authToken; + } + + /** + * Revoke a RubyGems token + * @param token - RubyGems UUID token + */ + public async revokeRubyGemsToken(token: string): Promise { + this.tokenStore.delete(token); + } + // ======================================================================== // UNIFIED AUTHENTICATION // ======================================================================== /** - * Validate any token (NPM, Maven, or OCI) + * Validate any token (NPM, Maven, OCI, PyPI, RubyGems, Composer, Cargo) * @param tokenString - Token string (UUID or JWT) * @param protocol - Expected protocol type * @returns Auth token object or null @@ -331,7 +472,7 @@ export class AuthManager { tokenString: string, protocol?: TRegistryProtocol ): Promise { - // Try UUID-based tokens (NPM, Maven, Composer) + // Try UUID-based tokens (NPM, Maven, Composer, Cargo, PyPI, RubyGems) if (this.isValidUuid(tokenString)) { // Try NPM token const npmToken = await this.validateNpmToken(tokenString); @@ -350,6 +491,24 @@ export class AuthManager { if (composerToken && (!protocol || protocol === 'composer')) { return composerToken; } + + // Try Cargo token + const cargoToken = await this.validateCargoToken(tokenString); + if (cargoToken && (!protocol || protocol === 'cargo')) { + return cargoToken; + } + + // Try PyPI token + const pypiToken = await this.validatePypiToken(tokenString); + if (pypiToken && (!protocol || protocol === 'pypi')) { + return pypiToken; + } + + // Try RubyGems token + const rubygemsToken = await this.validateRubyGemsToken(tokenString); + if (rubygemsToken && (!protocol || protocol === 'rubygems')) { + return rubygemsToken; + } } // Try OCI JWT diff --git a/ts/core/classes.registrystorage.ts b/ts/core/classes.registrystorage.ts index fb4f38a..6996089 100644 --- a/ts/core/classes.registrystorage.ts +++ b/ts/core/classes.registrystorage.ts @@ -601,4 +601,231 @@ export class RegistryStorage implements IStorageBackend { private getComposerZipPath(vendorPackage: string, reference: string): string { return `composer/packages/${vendorPackage}/${reference}.zip`; } + + // ======================================================================== + // PYPI STORAGE METHODS + // ======================================================================== + + /** + * Get PyPI package metadata + */ + public async getPypiPackageMetadata(packageName: string): Promise { + const path = this.getPypiMetadataPath(packageName); + const data = await this.getObject(path); + return data ? JSON.parse(data.toString('utf-8')) : null; + } + + /** + * Store PyPI package metadata + */ + public async putPypiPackageMetadata(packageName: string, metadata: any): Promise { + const path = this.getPypiMetadataPath(packageName); + const data = Buffer.from(JSON.stringify(metadata, null, 2), 'utf-8'); + return this.putObject(path, data, { 'Content-Type': 'application/json' }); + } + + /** + * Check if PyPI package metadata exists + */ + public async pypiPackageMetadataExists(packageName: string): Promise { + const path = this.getPypiMetadataPath(packageName); + return this.objectExists(path); + } + + /** + * Delete PyPI package metadata + */ + public async deletePypiPackageMetadata(packageName: string): Promise { + const path = this.getPypiMetadataPath(packageName); + return this.deleteObject(path); + } + + /** + * Get PyPI Simple API index (HTML) + */ + public async getPypiSimpleIndex(packageName: string): Promise { + const path = this.getPypiSimpleIndexPath(packageName); + const data = await this.getObject(path); + return data ? data.toString('utf-8') : null; + } + + /** + * Store PyPI Simple API index (HTML) + */ + public async putPypiSimpleIndex(packageName: string, html: string): Promise { + const path = this.getPypiSimpleIndexPath(packageName); + const data = Buffer.from(html, 'utf-8'); + return this.putObject(path, data, { 'Content-Type': 'text/html; charset=utf-8' }); + } + + /** + * Get PyPI root Simple API index (HTML) + */ + public async getPypiSimpleRootIndex(): Promise { + const path = this.getPypiSimpleRootIndexPath(); + const data = await this.getObject(path); + return data ? data.toString('utf-8') : null; + } + + /** + * Store PyPI root Simple API index (HTML) + */ + public async putPypiSimpleRootIndex(html: string): Promise { + const path = this.getPypiSimpleRootIndexPath(); + const data = Buffer.from(html, 'utf-8'); + return this.putObject(path, data, { 'Content-Type': 'text/html; charset=utf-8' }); + } + + /** + * Get PyPI package file (wheel, sdist) + */ + public async getPypiPackageFile(packageName: string, filename: string): Promise { + const path = this.getPypiPackageFilePath(packageName, filename); + return this.getObject(path); + } + + /** + * Store PyPI package file (wheel, sdist) + */ + public async putPypiPackageFile( + packageName: string, + filename: string, + data: Buffer + ): Promise { + const path = this.getPypiPackageFilePath(packageName, filename); + return this.putObject(path, data, { 'Content-Type': 'application/octet-stream' }); + } + + /** + * Check if PyPI package file exists + */ + public async pypiPackageFileExists(packageName: string, filename: string): Promise { + const path = this.getPypiPackageFilePath(packageName, filename); + return this.objectExists(path); + } + + /** + * Delete PyPI package file + */ + public async deletePypiPackageFile(packageName: string, filename: string): Promise { + const path = this.getPypiPackageFilePath(packageName, filename); + return this.deleteObject(path); + } + + /** + * List all PyPI packages + */ + public async listPypiPackages(): Promise { + const prefix = 'pypi/metadata/'; + const objects = await this.listObjects(prefix); + const packages = new Set(); + + // Extract package names from paths like: pypi/metadata/package-name/metadata.json + for (const obj of objects) { + const match = obj.match(/^pypi\/metadata\/([^\/]+)\/metadata\.json$/); + if (match) { + packages.add(match[1]); + } + } + + return Array.from(packages).sort(); + } + + /** + * List all versions of a PyPI package + */ + public async listPypiPackageVersions(packageName: string): Promise { + const prefix = `pypi/packages/${packageName}/`; + const objects = await this.listObjects(prefix); + const versions = new Set(); + + // Extract versions from filenames + for (const obj of objects) { + const filename = obj.split('/').pop(); + if (!filename) continue; + + // Extract version from wheel filename: package-1.0.0-py3-none-any.whl + // or sdist filename: package-1.0.0.tar.gz + const wheelMatch = filename.match(/^[^-]+-([^-]+)-.*\.whl$/); + const sdistMatch = filename.match(/^[^-]+-([^.]+)\.(tar\.gz|zip)$/); + + if (wheelMatch) versions.add(wheelMatch[1]); + else if (sdistMatch) versions.add(sdistMatch[1]); + } + + return Array.from(versions).sort(); + } + + /** + * Delete entire PyPI package (all versions and files) + */ + public async deletePypiPackage(packageName: string): Promise { + // Delete metadata + await this.deletePypiPackageMetadata(packageName); + + // Delete Simple API index + const simpleIndexPath = this.getPypiSimpleIndexPath(packageName); + try { + await this.deleteObject(simpleIndexPath); + } catch (error) { + // Ignore if doesn't exist + } + + // Delete all package files + const prefix = `pypi/packages/${packageName}/`; + const objects = await this.listObjects(prefix); + for (const obj of objects) { + await this.deleteObject(obj); + } + } + + /** + * Delete specific version of a PyPI package + */ + public async deletePypiPackageVersion(packageName: string, version: string): Promise { + const prefix = `pypi/packages/${packageName}/`; + const objects = await this.listObjects(prefix); + + // Delete all files matching this version + for (const obj of objects) { + const filename = obj.split('/').pop(); + if (!filename) continue; + + // Check if filename contains this version + const wheelMatch = filename.match(/^[^-]+-([^-]+)-.*\.whl$/); + const sdistMatch = filename.match(/^[^-]+-([^.]+)\.(tar\.gz|zip)$/); + + const fileVersion = wheelMatch?.[1] || sdistMatch?.[1]; + if (fileVersion === version) { + await this.deleteObject(obj); + } + } + + // Update metadata to remove this version + const metadata = await this.getPypiPackageMetadata(packageName); + if (metadata && metadata.versions) { + delete metadata.versions[version]; + await this.putPypiPackageMetadata(packageName, metadata); + } + } + + // ======================================================================== + // PYPI PATH HELPERS + // ======================================================================== + + private getPypiMetadataPath(packageName: string): string { + return `pypi/metadata/${packageName}/metadata.json`; + } + + private getPypiSimpleIndexPath(packageName: string): string { + return `pypi/simple/${packageName}/index.html`; + } + + private getPypiSimpleRootIndexPath(): string { + return `pypi/simple/index.html`; + } + + private getPypiPackageFilePath(packageName: string, filename: string): string { + return `pypi/packages/${packageName}/${filename}`; + } } diff --git a/ts/core/interfaces.core.ts b/ts/core/interfaces.core.ts index f119848..a0fc535 100644 --- a/ts/core/interfaces.core.ts +++ b/ts/core/interfaces.core.ts @@ -5,7 +5,7 @@ /** * Registry protocol types */ -export type TRegistryProtocol = 'oci' | 'npm' | 'maven' | 'cargo' | 'composer'; +export type TRegistryProtocol = 'oci' | 'npm' | 'maven' | 'cargo' | 'composer' | 'pypi' | 'rubygems'; /** * Unified action types across protocols @@ -70,6 +70,16 @@ export interface IAuthConfig { realm: string; service: string; }; + /** PyPI token settings */ + pypiTokens?: { + enabled: boolean; + defaultReadonly?: boolean; + }; + /** RubyGems token settings */ + rubygemsTokens?: { + enabled: boolean; + defaultReadonly?: boolean; + }; } /** @@ -92,6 +102,8 @@ export interface IRegistryConfig { maven?: IProtocolConfig; cargo?: IProtocolConfig; composer?: IProtocolConfig; + pypi?: IProtocolConfig; + rubygems?: IProtocolConfig; } /** diff --git a/ts/pypi/classes.pypiregistry.ts b/ts/pypi/classes.pypiregistry.ts new file mode 100644 index 0000000..fbc66e5 --- /dev/null +++ b/ts/pypi/classes.pypiregistry.ts @@ -0,0 +1,564 @@ +import { Smartlog } from '@push.rocks/smartlog'; +import { BaseRegistry } from '../core/classes.baseregistry.js'; +import { RegistryStorage } from '../core/classes.registrystorage.js'; +import { AuthManager } from '../core/classes.authmanager.js'; +import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js'; +import type { + IPypiPackageMetadata, + IPypiFile, + IPypiError, + IPypiUploadResponse, +} from './interfaces.pypi.js'; +import * as helpers from './helpers.pypi.js'; + +/** + * PyPI registry implementation + * Implements PEP 503 (Simple API), PEP 691 (JSON API), and legacy upload API + */ +export class PypiRegistry extends BaseRegistry { + private storage: RegistryStorage; + private authManager: AuthManager; + private basePath: string = '/pypi'; + private registryUrl: string; + private logger: Smartlog; + + constructor( + storage: RegistryStorage, + authManager: AuthManager, + basePath: string = '/pypi', + registryUrl: string = 'http://localhost:5000' + ) { + super(); + this.storage = storage; + this.authManager = authManager; + this.basePath = basePath; + this.registryUrl = registryUrl; + + // Initialize logger + this.logger = new Smartlog({ + logContext: { + company: 'push.rocks', + companyunit: 'smartregistry', + containerName: 'pypi-registry', + environment: (process.env.NODE_ENV as any) || 'development', + runtime: 'node', + zone: 'pypi' + } + }); + this.logger.enableConsole(); + } + + public async init(): Promise { + // Initialize root Simple API index if not exists + const existingIndex = await this.storage.getPypiSimpleRootIndex(); + if (!existingIndex) { + const html = helpers.generateSimpleRootHtml([]); + await this.storage.putPypiSimpleRootIndex(html); + this.logger.log('info', 'Initialized PyPI root index'); + } + } + + public getBasePath(): string { + return this.basePath; + } + + public async handleRequest(context: IRequestContext): Promise { + let path = context.path.replace(this.basePath, ''); + + // Also handle /simple path prefix + if (path.startsWith('/simple')) { + path = path.replace('/simple', ''); + return this.handleSimpleRequest(path, context); + } + + // Extract token (Basic Auth or Bearer) + const token = await this.extractToken(context); + + this.logger.log('debug', `handleRequest: ${context.method} ${path}`, { + method: context.method, + path, + hasAuth: !!token + }); + + // Root upload endpoint (POST /) + if ((path === '/' || path === '') && context.method === 'POST') { + return this.handleUpload(context, token); + } + + // Package metadata JSON API: GET /pypi/{package}/json + const jsonMatch = path.match(/^\/pypi\/([^\/]+)\/json$/); + if (jsonMatch && context.method === 'GET') { + return this.handlePackageJson(jsonMatch[1]); + } + + // Version-specific JSON API: GET /pypi/{package}/{version}/json + const versionJsonMatch = path.match(/^\/pypi\/([^\/]+)\/([^\/]+)\/json$/); + if (versionJsonMatch && context.method === 'GET') { + return this.handleVersionJson(versionJsonMatch[1], versionJsonMatch[2]); + } + + // Package file download: GET /packages/{package}/{filename} + const downloadMatch = path.match(/^\/packages\/([^\/]+)\/(.+)$/); + if (downloadMatch && context.method === 'GET') { + return this.handleDownload(downloadMatch[1], downloadMatch[2]); + } + + // Delete package: DELETE /packages/{package} + if (path.match(/^\/packages\/([^\/]+)$/) && context.method === 'DELETE') { + const packageName = path.match(/^\/packages\/([^\/]+)$/)?.[1]; + return this.handleDeletePackage(packageName!, token); + } + + // Delete version: DELETE /packages/{package}/{version} + const deleteVersionMatch = path.match(/^\/packages\/([^\/]+)\/([^\/]+)$/); + if (deleteVersionMatch && context.method === 'DELETE') { + return this.handleDeleteVersion(deleteVersionMatch[1], deleteVersionMatch[2], token); + } + + return { + status: 404, + headers: { 'Content-Type': 'application/json' }, + body: Buffer.from(JSON.stringify({ message: 'Not Found' })), + }; + } + + /** + * Check if token has permission for resource + */ + protected async checkPermission( + token: IAuthToken | null, + resource: string, + action: string + ): Promise { + if (!token) return false; + return this.authManager.authorize(token, `pypi:package:${resource}`, action); + } + + /** + * Handle Simple API requests (PEP 503 HTML or PEP 691 JSON) + */ + private async handleSimpleRequest(path: string, context: IRequestContext): Promise { + // Ensure path ends with / (PEP 503 requirement) + if (!path.endsWith('/') && !path.includes('.')) { + return { + status: 301, + headers: { 'Location': `${this.basePath}/simple${path}/` }, + body: Buffer.from(''), + }; + } + + // Root index: /simple/ + if (path === '/' || path === '') { + return this.handleSimpleRoot(context); + } + + // Package index: /simple/{package}/ + const packageMatch = path.match(/^\/([^\/]+)\/$/); + if (packageMatch) { + return this.handleSimplePackage(packageMatch[1], context); + } + + return { + status: 404, + headers: { 'Content-Type': 'text/html; charset=utf-8' }, + body: Buffer.from('

404 Not Found

'), + }; + } + + /** + * Handle Simple API root index + * Returns HTML (PEP 503) or JSON (PEP 691) based on Accept header + */ + private async handleSimpleRoot(context: IRequestContext): Promise { + const acceptHeader = context.headers['accept'] || context.headers['Accept'] || ''; + const preferJson = acceptHeader.includes('application/vnd.pypi.simple') && + acceptHeader.includes('json'); + + const packages = await this.storage.listPypiPackages(); + + if (preferJson) { + // PEP 691: JSON response + const response = helpers.generateJsonRootResponse(packages); + return { + status: 200, + headers: { + 'Content-Type': 'application/vnd.pypi.simple.v1+json', + 'Cache-Control': 'public, max-age=600' + }, + body: Buffer.from(JSON.stringify(response)), + }; + } else { + // PEP 503: HTML response + const html = helpers.generateSimpleRootHtml(packages); + + // Update stored index + await this.storage.putPypiSimpleRootIndex(html); + + return { + status: 200, + headers: { + 'Content-Type': 'text/html; charset=utf-8', + 'Cache-Control': 'public, max-age=600' + }, + body: Buffer.from(html), + }; + } + } + + /** + * Handle Simple API package index + * Returns HTML (PEP 503) or JSON (PEP 691) based on Accept header + */ + private async handleSimplePackage(packageName: string, context: IRequestContext): Promise { + const normalized = helpers.normalizePypiPackageName(packageName); + + // Get package metadata + const metadata = await this.storage.getPypiPackageMetadata(normalized); + if (!metadata) { + return { + status: 404, + headers: { 'Content-Type': 'text/html; charset=utf-8' }, + body: Buffer.from('

404 Not Found

'), + }; + } + + // Build file list from all versions + const files: IPypiFile[] = []; + for (const [version, versionMeta] of Object.entries(metadata.versions || {})) { + for (const file of (versionMeta as any).files || []) { + files.push({ + filename: file.filename, + url: `${this.registryUrl}/pypi/packages/${normalized}/${file.filename}`, + hashes: file.hashes, + 'requires-python': file['requires-python'], + yanked: file.yanked || (versionMeta as any).yanked, + size: file.size, + 'upload-time': file['upload-time'], + }); + } + } + + const acceptHeader = context.headers['accept'] || context.headers['Accept'] || ''; + const preferJson = acceptHeader.includes('application/vnd.pypi.simple') && + acceptHeader.includes('json'); + + if (preferJson) { + // PEP 691: JSON response + const response = helpers.generateJsonPackageResponse(normalized, files); + return { + status: 200, + headers: { + 'Content-Type': 'application/vnd.pypi.simple.v1+json', + 'Cache-Control': 'public, max-age=300' + }, + body: Buffer.from(JSON.stringify(response)), + }; + } else { + // PEP 503: HTML response + const html = helpers.generateSimplePackageHtml(normalized, files, this.registryUrl); + + // Update stored index + await this.storage.putPypiSimpleIndex(normalized, html); + + return { + status: 200, + headers: { + 'Content-Type': 'text/html; charset=utf-8', + 'Cache-Control': 'public, max-age=300' + }, + body: Buffer.from(html), + }; + } + } + + /** + * Extract authentication token from request + */ + private async extractToken(context: IRequestContext): Promise { + const authHeader = context.headers['authorization'] || context.headers['Authorization']; + if (!authHeader) return null; + + // Handle Basic Auth (username:password or __token__:token) + if (authHeader.startsWith('Basic ')) { + const base64 = authHeader.substring(6); + const decoded = Buffer.from(base64, 'base64').toString('utf-8'); + const [username, password] = decoded.split(':'); + + // PyPI token authentication: username = __token__ + if (username === '__token__') { + return this.authManager.validateToken(password, 'pypi'); + } + + // Username/password authentication (would need user lookup) + // For now, not implemented + return null; + } + + // Handle Bearer token + if (authHeader.startsWith('Bearer ')) { + const token = authHeader.substring(7); + return this.authManager.validateToken(token, 'pypi'); + } + + return null; + } + + /** + * Handle package upload (multipart/form-data) + * POST / with :action=file_upload + */ + private async handleUpload(context: IRequestContext, token: IAuthToken | null): Promise { + if (!token) { + return { + status: 401, + headers: { + 'Content-Type': 'application/json', + 'WWW-Authenticate': 'Basic realm="PyPI"' + }, + body: Buffer.from(JSON.stringify({ message: 'Authentication required' })), + }; + } + + try { + // Parse multipart form data (context.body should be parsed by server) + const formData = context.body as any; // Assuming parsed multipart data + + if (!formData || formData[':action'] !== 'file_upload') { + return this.errorResponse(400, 'Invalid upload request'); + } + + // Extract required fields + const packageName = formData.name; + const version = formData.version; + const filename = formData.content?.filename; + const fileData = formData.content?.data as Buffer; + const filetype = formData.filetype; // 'bdist_wheel' or 'sdist' + const pyversion = formData.pyversion; + + if (!packageName || !version || !filename || !fileData) { + return this.errorResponse(400, 'Missing required fields'); + } + + // Validate package name + if (!helpers.isValidPackageName(packageName)) { + return this.errorResponse(400, 'Invalid package name'); + } + + const normalized = helpers.normalizePypiPackageName(packageName); + + // Check permission + if (!(await this.checkPermission(token, normalized, 'write'))) { + return this.errorResponse(403, 'Insufficient permissions'); + } + + // Calculate hashes + const hashes: Record = {}; + + if (formData.sha256_digest) { + hashes.sha256 = formData.sha256_digest; + } else { + hashes.sha256 = await helpers.calculateHash(fileData, 'sha256'); + } + + if (formData.md5_digest) { + // MD5 digest in PyPI is urlsafe base64, convert to hex + hashes.md5 = await helpers.calculateHash(fileData, 'md5'); + } + + if (formData.blake2_256_digest) { + hashes.blake2b = formData.blake2_256_digest; + } + + // Store file + await this.storage.putPypiPackageFile(normalized, filename, fileData); + + // Update metadata + let metadata = await this.storage.getPypiPackageMetadata(normalized); + if (!metadata) { + metadata = { + name: normalized, + versions: {}, + }; + } + + if (!metadata.versions[version]) { + metadata.versions[version] = { + version, + files: [], + }; + } + + // Add file to version + metadata.versions[version].files.push({ + filename, + path: `pypi/packages/${normalized}/${filename}`, + filetype, + python_version: pyversion, + hashes, + size: fileData.length, + 'requires-python': formData.requires_python, + 'upload-time': new Date().toISOString(), + 'uploaded-by': token.userId, + }); + + // Store core metadata if provided + if (formData.summary || formData.description) { + metadata.versions[version].metadata = helpers.extractCoreMetadata(formData); + } + + metadata['last-modified'] = new Date().toISOString(); + await this.storage.putPypiPackageMetadata(normalized, metadata); + + this.logger.log('info', `Package uploaded: ${normalized} ${version}`, { + filename, + size: fileData.length + }); + + return { + status: 200, + headers: { 'Content-Type': 'application/json' }, + body: Buffer.from(JSON.stringify({ + message: 'Package uploaded successfully', + url: `${this.registryUrl}/pypi/packages/${normalized}/${filename}` + })), + }; + } catch (error) { + this.logger.log('error', 'Upload failed', { error: (error as Error).message }); + return this.errorResponse(500, 'Upload failed: ' + (error as Error).message); + } + } + + /** + * Handle package download + */ + private async handleDownload(packageName: string, filename: string): Promise { + const normalized = helpers.normalizePypiPackageName(packageName); + const fileData = await this.storage.getPypiPackageFile(normalized, filename); + + if (!fileData) { + return { + status: 404, + headers: { 'Content-Type': 'application/json' }, + body: Buffer.from(JSON.stringify({ message: 'File not found' })), + }; + } + + return { + status: 200, + headers: { + 'Content-Type': 'application/octet-stream', + 'Content-Disposition': `attachment; filename="${filename}"`, + 'Content-Length': fileData.length.toString() + }, + body: fileData, + }; + } + + /** + * Handle package JSON API (all versions) + */ + private async handlePackageJson(packageName: string): Promise { + const normalized = helpers.normalizePypiPackageName(packageName); + const metadata = await this.storage.getPypiPackageMetadata(normalized); + + if (!metadata) { + return this.errorResponse(404, 'Package not found'); + } + + return { + status: 200, + headers: { + 'Content-Type': 'application/json', + 'Cache-Control': 'public, max-age=300' + }, + body: Buffer.from(JSON.stringify(metadata)), + }; + } + + /** + * Handle version-specific JSON API + */ + private async handleVersionJson(packageName: string, version: string): Promise { + const normalized = helpers.normalizePypiPackageName(packageName); + const metadata = await this.storage.getPypiPackageMetadata(normalized); + + if (!metadata || !metadata.versions[version]) { + return this.errorResponse(404, 'Version not found'); + } + + return { + status: 200, + headers: { + 'Content-Type': 'application/json', + 'Cache-Control': 'public, max-age=300' + }, + body: Buffer.from(JSON.stringify(metadata.versions[version])), + }; + } + + /** + * Handle package deletion + */ + private async handleDeletePackage(packageName: string, token: IAuthToken | null): Promise { + if (!token) { + return this.errorResponse(401, 'Authentication required'); + } + + const normalized = helpers.normalizePypiPackageName(packageName); + + if (!(await this.checkPermission(token, normalized, 'delete'))) { + return this.errorResponse(403, 'Insufficient permissions'); + } + + await this.storage.deletePypiPackage(normalized); + + this.logger.log('info', `Package deleted: ${normalized}`); + + return { + status: 204, + headers: {}, + body: Buffer.from(''), + }; + } + + /** + * Handle version deletion + */ + private async handleDeleteVersion( + packageName: string, + version: string, + token: IAuthToken | null + ): Promise { + if (!token) { + return this.errorResponse(401, 'Authentication required'); + } + + const normalized = helpers.normalizePypiPackageName(packageName); + + if (!(await this.checkPermission(token, normalized, 'delete'))) { + return this.errorResponse(403, 'Insufficient permissions'); + } + + await this.storage.deletePypiPackageVersion(normalized, version); + + this.logger.log('info', `Version deleted: ${normalized} ${version}`); + + return { + status: 204, + headers: {}, + body: Buffer.from(''), + }; + } + + /** + * Helper: Create error response + */ + private errorResponse(status: number, message: string): IResponse { + const error: IPypiError = { message, status }; + return { + status, + headers: { 'Content-Type': 'application/json' }, + body: Buffer.from(JSON.stringify(error)), + }; + } +} diff --git a/ts/pypi/helpers.pypi.ts b/ts/pypi/helpers.pypi.ts new file mode 100644 index 0000000..d663169 --- /dev/null +++ b/ts/pypi/helpers.pypi.ts @@ -0,0 +1,299 @@ +/** + * Helper functions for PyPI registry + * Package name normalization, HTML generation, etc. + */ + +import type { IPypiFile, IPypiPackageMetadata } from './interfaces.pypi.js'; + +/** + * Normalize package name according to PEP 503 + * Lowercase and replace runs of [._-] with a single dash + * @param name - Package name + * @returns Normalized name + */ +export function normalizePypiPackageName(name: string): string { + return name + .toLowerCase() + .replace(/[-_.]+/g, '-'); +} + +/** + * Escape HTML special characters to prevent XSS + * @param str - String to escape + * @returns Escaped string + */ +export function escapeHtml(str: string): string { + return str + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); +} + +/** + * Generate PEP 503 compliant HTML for root index (all packages) + * @param packages - List of package names + * @returns HTML string + */ +export function generateSimpleRootHtml(packages: string[]): string { + const links = packages + .map(pkg => { + const normalized = normalizePypiPackageName(pkg); + return ` ${escapeHtml(pkg)}`; + }) + .join('\n'); + + return ` + + + + Simple Index + + +

Simple Index

+${links} + +`; +} + +/** + * Generate PEP 503 compliant HTML for package index (file list) + * @param packageName - Package name (normalized) + * @param files - List of files + * @param baseUrl - Base URL for downloads + * @returns HTML string + */ +export function generateSimplePackageHtml( + packageName: string, + files: IPypiFile[], + baseUrl: string +): string { + const links = files + .map(file => { + // Build URL + let url = file.url; + if (!url.startsWith('http://') && !url.startsWith('https://')) { + // Relative URL - make it absolute + url = `${baseUrl}/packages/${packageName}/${file.filename}`; + } + + // Add hash fragment + const hashName = Object.keys(file.hashes)[0]; + const hashValue = file.hashes[hashName]; + const fragment = hashName && hashValue ? `#${hashName}=${hashValue}` : ''; + + // Build data attributes + const dataAttrs: string[] = []; + + if (file['requires-python']) { + const escaped = escapeHtml(file['requires-python']); + dataAttrs.push(`data-requires-python="${escaped}"`); + } + + if (file['gpg-sig'] !== undefined) { + dataAttrs.push(`data-gpg-sig="${file['gpg-sig'] ? 'true' : 'false'}"`); + } + + if (file.yanked) { + const reason = typeof file.yanked === 'string' ? file.yanked : ''; + if (reason) { + dataAttrs.push(`data-yanked="${escapeHtml(reason)}"`); + } else { + dataAttrs.push(`data-yanked=""`); + } + } + + const dataAttrStr = dataAttrs.length > 0 ? ' ' + dataAttrs.join(' ') : ''; + + return ` ${escapeHtml(file.filename)}`; + }) + .join('\n'); + + return ` + + + + Links for ${escapeHtml(packageName)} + + +

Links for ${escapeHtml(packageName)}

+${links} + +`; +} + +/** + * Parse filename to extract package info + * Supports wheel and sdist formats + * @param filename - Package filename + * @returns Parsed info or null + */ +export function parsePackageFilename(filename: string): { + name: string; + version: string; + filetype: 'bdist_wheel' | 'sdist'; + pythonVersion?: string; +} | null { + // Wheel format: {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl + const wheelMatch = filename.match(/^([a-zA-Z0-9_.-]+?)-([a-zA-Z0-9_.]+?)(?:-(\d+))?-([^-]+)-([^-]+)-([^-]+)\.whl$/); + if (wheelMatch) { + return { + name: wheelMatch[1], + version: wheelMatch[2], + filetype: 'bdist_wheel', + pythonVersion: wheelMatch[4], + }; + } + + // Sdist tar.gz format: {name}-{version}.tar.gz + const sdistTarMatch = filename.match(/^([a-zA-Z0-9_.-]+?)-([a-zA-Z0-9_.]+)\.tar\.gz$/); + if (sdistTarMatch) { + return { + name: sdistTarMatch[1], + version: sdistTarMatch[2], + filetype: 'sdist', + pythonVersion: 'source', + }; + } + + // Sdist zip format: {name}-{version}.zip + const sdistZipMatch = filename.match(/^([a-zA-Z0-9_.-]+?)-([a-zA-Z0-9_.]+)\.zip$/); + if (sdistZipMatch) { + return { + name: sdistZipMatch[1], + version: sdistZipMatch[2], + filetype: 'sdist', + pythonVersion: 'source', + }; + } + + return null; +} + +/** + * Calculate hash digest for a buffer + * @param data - Data to hash + * @param algorithm - Hash algorithm (sha256, md5, blake2b) + * @returns Hex-encoded hash + */ +export async function calculateHash(data: Buffer, algorithm: 'sha256' | 'md5' | 'blake2b'): Promise { + const crypto = await import('crypto'); + + let hash: any; + if (algorithm === 'blake2b') { + // Node.js uses 'blake2b512' for blake2b + hash = crypto.createHash('blake2b512'); + } else { + hash = crypto.createHash(algorithm); + } + + hash.update(data); + return hash.digest('hex'); +} + +/** + * Validate package name + * Must contain only ASCII letters, numbers, ., -, and _ + * @param name - Package name + * @returns true if valid + */ +export function isValidPackageName(name: string): boolean { + return /^[a-zA-Z0-9._-]+$/.test(name); +} + +/** + * Validate version string (basic check) + * @param version - Version string + * @returns true if valid + */ +export function isValidVersion(version: string): boolean { + // Basic check - allows numbers, letters, dots, hyphens, underscores + // More strict validation would follow PEP 440 + return /^[a-zA-Z0-9._-]+$/.test(version); +} + +/** + * Extract metadata from package metadata + * Filters and normalizes metadata fields + * @param metadata - Raw metadata object + * @returns Filtered metadata + */ +export function extractCoreMetadata(metadata: Record): Record { + const coreFields = [ + 'metadata-version', + 'name', + 'version', + 'platform', + 'supported-platform', + 'summary', + 'description', + 'description-content-type', + 'keywords', + 'home-page', + 'download-url', + 'author', + 'author-email', + 'maintainer', + 'maintainer-email', + 'license', + 'classifier', + 'requires-python', + 'requires-dist', + 'requires-external', + 'provides-dist', + 'project-url', + 'provides-extra', + ]; + + const result: Record = {}; + + for (const [key, value] of Object.entries(metadata)) { + const normalizedKey = key.toLowerCase().replace(/_/g, '-'); + if (coreFields.includes(normalizedKey)) { + result[normalizedKey] = value; + } + } + + return result; +} + +/** + * Generate JSON API response for package list (PEP 691) + * @param packages - List of package names + * @returns JSON object + */ +export function generateJsonRootResponse(packages: string[]): any { + return { + meta: { + 'api-version': '1.0', + }, + projects: packages.map(name => ({ name })), + }; +} + +/** + * Generate JSON API response for package files (PEP 691) + * @param packageName - Package name (normalized) + * @param files - List of files + * @returns JSON object + */ +export function generateJsonPackageResponse(packageName: string, files: IPypiFile[]): any { + return { + meta: { + 'api-version': '1.0', + }, + name: packageName, + files: files.map(file => ({ + filename: file.filename, + url: file.url, + hashes: file.hashes, + 'requires-python': file['requires-python'], + 'dist-info-metadata': file['dist-info-metadata'], + 'gpg-sig': file['gpg-sig'], + yanked: file.yanked, + size: file.size, + 'upload-time': file['upload-time'], + })), + }; +} diff --git a/ts/pypi/index.ts b/ts/pypi/index.ts new file mode 100644 index 0000000..a2361e8 --- /dev/null +++ b/ts/pypi/index.ts @@ -0,0 +1,8 @@ +/** + * PyPI Registry Module + * Python Package Index implementation + */ + +export * from './interfaces.pypi.js'; +export * from './classes.pypiregistry.js'; +export * as pypiHelpers from './helpers.pypi.js'; diff --git a/ts/pypi/interfaces.pypi.ts b/ts/pypi/interfaces.pypi.ts new file mode 100644 index 0000000..91a47af --- /dev/null +++ b/ts/pypi/interfaces.pypi.ts @@ -0,0 +1,316 @@ +/** + * PyPI Registry Type Definitions + * Compliant with PEP 503 (Simple API), PEP 691 (JSON API), and PyPI upload API + */ + +/** + * File information for a package distribution + * Used in both PEP 503 HTML and PEP 691 JSON responses + */ +export interface IPypiFile { + /** Filename (e.g., "package-1.0.0-py3-none-any.whl") */ + filename: string; + /** Download URL (absolute or relative) */ + url: string; + /** Hash digests (multiple algorithms supported in JSON) */ + hashes: Record; + /** Python version requirement (PEP 345 format) */ + 'requires-python'?: string; + /** Whether distribution info metadata is available (PEP 658) */ + 'dist-info-metadata'?: boolean | { sha256: string }; + /** Whether GPG signature is available */ + 'gpg-sig'?: boolean; + /** Yank status: false or reason string */ + yanked?: boolean | string; + /** File size in bytes */ + size?: number; + /** Upload timestamp */ + 'upload-time'?: string; +} + +/** + * Package metadata stored internally + * Consolidated from multiple file uploads + */ +export interface IPypiPackageMetadata { + /** Normalized package name */ + name: string; + /** Map of version to file list */ + versions: Record; + /** Timestamp of last update */ + 'last-modified'?: string; +} + +/** + * Metadata for a specific version + */ +export interface IPypiVersionMetadata { + /** Version string */ + version: string; + /** Files for this version (wheels, sdists) */ + files: IPypiFileMetadata[]; + /** Core metadata fields */ + metadata?: IPypiCoreMetadata; + /** Whether entire version is yanked */ + yanked?: boolean | string; + /** Upload timestamp */ + 'upload-time'?: string; +} + +/** + * Internal file metadata + */ +export interface IPypiFileMetadata { + filename: string; + /** Storage key/path */ + path: string; + /** File type: bdist_wheel or sdist */ + filetype: 'bdist_wheel' | 'sdist'; + /** Python version tag */ + python_version: string; + /** Hash digests */ + hashes: Record; + /** File size in bytes */ + size: number; + /** Python version requirement */ + 'requires-python'?: string; + /** Whether this file is yanked */ + yanked?: boolean | string; + /** Upload timestamp */ + 'upload-time': string; + /** Uploader user ID */ + 'uploaded-by': string; +} + +/** + * Core metadata fields (subset of PEP 566) + * These are extracted from package uploads + */ +export interface IPypiCoreMetadata { + /** Metadata version */ + 'metadata-version': string; + /** Package name */ + name: string; + /** Version string */ + version: string; + /** Platform compatibility */ + platform?: string; + /** Supported platforms */ + 'supported-platform'?: string; + /** Summary/description */ + summary?: string; + /** Long description */ + description?: string; + /** Description content type (text/plain, text/markdown, text/x-rst) */ + 'description-content-type'?: string; + /** Keywords */ + keywords?: string; + /** Homepage URL */ + 'home-page'?: string; + /** Download URL */ + 'download-url'?: string; + /** Author name */ + author?: string; + /** Author email */ + 'author-email'?: string; + /** Maintainer name */ + maintainer?: string; + /** Maintainer email */ + 'maintainer-email'?: string; + /** License */ + license?: string; + /** Classifiers (Trove classifiers) */ + classifier?: string[]; + /** Python version requirement */ + 'requires-python'?: string; + /** Dist name requirement */ + 'requires-dist'?: string[]; + /** External requirement */ + 'requires-external'?: string[]; + /** Provides dist */ + 'provides-dist'?: string[]; + /** Project URLs */ + 'project-url'?: string[]; + /** Provides extra */ + 'provides-extra'?: string[]; +} + +/** + * PEP 503: Simple API root response (project list) + */ +export interface IPypiSimpleRootHtml { + /** List of project names */ + projects: string[]; +} + +/** + * PEP 503: Simple API project response (file list) + */ +export interface IPypiSimpleProjectHtml { + /** Normalized project name */ + name: string; + /** List of files */ + files: IPypiFile[]; +} + +/** + * PEP 691: JSON API root response + */ +export interface IPypiJsonRoot { + /** API metadata */ + meta: { + /** API version (e.g., "1.0") */ + 'api-version': string; + }; + /** List of projects */ + projects: Array<{ + /** Project name */ + name: string; + }>; +} + +/** + * PEP 691: JSON API project response + */ +export interface IPypiJsonProject { + /** Normalized project name */ + name: string; + /** API metadata */ + meta: { + /** API version (e.g., "1.0") */ + 'api-version': string; + }; + /** List of files */ + files: IPypiFile[]; +} + +/** + * Upload form data (multipart/form-data fields) + * Based on PyPI legacy upload API + */ +export interface IPypiUploadForm { + /** Action type (always "file_upload") */ + ':action': 'file_upload'; + /** Protocol version (always "1") */ + protocol_version: '1'; + /** File content (binary) */ + content: Buffer; + /** File type */ + filetype: 'bdist_wheel' | 'sdist'; + /** Python version tag */ + pyversion: string; + /** Package name */ + name: string; + /** Version string */ + version: string; + /** Metadata version */ + metadata_version: string; + /** Hash digests (at least one required) */ + md5_digest?: string; + sha256_digest?: string; + blake2_256_digest?: string; + /** Optional attestations */ + attestations?: string; // JSON array + /** Optional core metadata fields */ + summary?: string; + description?: string; + description_content_type?: string; + author?: string; + author_email?: string; + maintainer?: string; + maintainer_email?: string; + license?: string; + keywords?: string; + home_page?: string; + download_url?: string; + requires_python?: string; + classifiers?: string[]; + platform?: string; + [key: string]: any; // Allow additional metadata fields +} + +/** + * JSON API upload response + */ +export interface IPypiUploadResponse { + /** Success message */ + message?: string; + /** URL of uploaded file */ + url?: string; +} + +/** + * Error response structure + */ +export interface IPypiError { + /** Error message */ + message: string; + /** HTTP status code */ + status?: number; + /** Additional error details */ + details?: string[]; +} + +/** + * Search query parameters + */ +export interface IPypiSearchQuery { + /** Search term */ + q?: string; + /** Page number */ + page?: number; + /** Results per page */ + per_page?: number; +} + +/** + * Search result for a single package + */ +export interface IPypiSearchResult { + /** Package name */ + name: string; + /** Latest version */ + version: string; + /** Summary */ + summary: string; + /** Description */ + description?: string; +} + +/** + * Search response structure + */ +export interface IPypiSearchResponse { + /** Search results */ + results: IPypiSearchResult[]; + /** Result count */ + count: number; + /** Current page */ + page: number; + /** Total pages */ + pages: number; +} + +/** + * Yank request + */ +export interface IPypiYankRequest { + /** Package name */ + name: string; + /** Version to yank */ + version: string; + /** Optional filename (specific file) */ + filename?: string; + /** Reason for yanking */ + reason?: string; +} + +/** + * Yank response + */ +export interface IPypiYankResponse { + /** Success indicator */ + success: boolean; + /** Message */ + message?: string; +}