From bd64a7b14042f04c5b5f1ec60862adbc6bdafa03 Mon Sep 17 00:00:00 2001 From: Juergen Kunz Date: Thu, 27 Nov 2025 21:11:04 +0000 Subject: [PATCH] feat(pypi,rubygems): Add PyPI and RubyGems protocol implementations, upstream caching, and auth/storage improvements --- changelog.md | 10 ++ readme.md | 226 ++++++++++++++++++++++++++++++++++++++- ts/00_commitinfo_data.ts | 2 +- 3 files changed, 236 insertions(+), 2 deletions(-) diff --git a/changelog.md b/changelog.md index fbe2673..0d75995 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,15 @@ # Changelog +## 2025-11-27 - 2.5.0 - feat(pypi,rubygems) +Add PyPI and RubyGems protocol implementations, upstream caching, and auth/storage improvements + +- Implemented full PyPI support (PEP 503 Simple API HTML, PEP 691 JSON API, legacy upload handling, name normalization, hash verification, content negotiation, package/file storage and metadata management). +- Implemented RubyGems support (compact index, /versions, /info, /names endpoints, gem upload, yank/unyank, platform handling and file storage). +- Expanded RegistryStorage with protocol-specific helpers for OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems (get/put/delete/list helpers, metadata handling, context-aware hooks). +- Added AuthManager and DefaultAuthProvider improvements: unified token creation/validation for multiple protocols (npm, oci, maven, composer, cargo, pypi, rubygems) and OCI JWT support. +- Added upstream infrastructure: BaseUpstream, UpstreamCache (S3-backed optional, stale-while-revalidate, negative caching), circuit breaker with retries/backoff and resilience defaults. +- Added various protocol registries (NPM, Maven, Cargo, OCI, PyPI) with request routing, permission checks, and optional upstream proxying/caching. + ## 2025-11-27 - 2.4.0 - feat(core) Add pluggable auth providers, storage hooks, multi-upstream cache awareness, and PyPI/RubyGems protocol implementations diff --git a/readme.md b/readme.md index c21ca52..3e0f551 100644 --- a/readme.md +++ b/readme.md @@ -4,7 +4,7 @@ ## Issue Reporting and Security -For reporting bugs, issues, or security vulnerabilities, please visit [community.foss.global/](https://community.foss.global/). This is the central community hub for all issue reporting. Developers who want to sign a contribution agreement and go through identification can also get a [code.foss.global/](https://code.foss.global/) account to submit Pull Requests directly. +For reporting bugs, issues, or security vulnerabilities, please visit [community.foss.global/](https://community.foss.global/). This is the central community hub for all issue reporting. Developers who sign and comply with our contribution agreement and go through identification can also get a [code.foss.global/](https://code.foss.global/) account to submit Pull Requests directly. ## ✨ Features @@ -82,6 +82,19 @@ For reporting bugs, issues, or security vulnerabilities, please visit [community - ✅ Dependency resolution - ✅ Legacy API compatibility +### 🌐 Upstream Proxy & Caching +- **Multi-Upstream Support**: Configure multiple upstream registries per protocol with priority ordering +- **Scope-Based Routing**: Route specific packages/scopes to different upstreams (e.g., `@company/*` → private registry) +- **S3-Backed Cache**: Persistent caching using existing S3 storage with URL-based cache paths +- **Circuit Breaker**: Automatic failover with configurable thresholds +- **Stale-While-Revalidate**: Serve cached content while refreshing in background +- **Content-Aware TTLs**: Different TTLs for immutable (tarballs) vs mutable (metadata) content + +### 🔌 Enterprise Extensibility +- **Pluggable Auth Provider** (`IAuthProvider`): Integrate LDAP, OAuth, SSO, or custom auth systems +- **Storage Event Hooks** (`IStorageHooks`): Quota tracking, audit logging, virus scanning, cache invalidation +- **Request Actor Context**: Pass user/org info through requests for audit trails and rate limiting + ## 📥 Installation ```bash @@ -648,6 +661,217 @@ const canWrite = await authManager.authorize( ); ``` +### 🌐 Upstream Proxy Configuration + +```typescript +import { SmartRegistry, IRegistryConfig } from '@push.rocks/smartregistry'; + +const config: IRegistryConfig = { + storage: { /* S3 config */ }, + auth: { /* Auth config */ }, + npm: { + enabled: true, + basePath: '/npm', + upstream: { + enabled: true, + upstreams: [ + { + id: 'company-private', + name: 'Company Private NPM', + url: 'https://npm.internal.company.com', + priority: 1, // Lower = higher priority + enabled: true, + scopeRules: [ + { pattern: '@company/*', action: 'include' }, + { pattern: '@internal/*', action: 'include' }, + ], + auth: { type: 'bearer', token: process.env.NPM_PRIVATE_TOKEN }, + }, + { + id: 'npmjs', + name: 'NPM Public Registry', + url: 'https://registry.npmjs.org', + priority: 10, + enabled: true, + scopeRules: [ + { pattern: '@company/*', action: 'exclude' }, + { pattern: '@internal/*', action: 'exclude' }, + ], + auth: { type: 'none' }, + cache: { defaultTtlSeconds: 300 }, + resilience: { timeoutMs: 30000, maxRetries: 3 }, + }, + ], + cache: { enabled: true, staleWhileRevalidate: true }, + }, + }, + oci: { + enabled: true, + basePath: '/oci', + upstream: { + enabled: true, + upstreams: [ + { + id: 'dockerhub', + name: 'Docker Hub', + url: 'https://registry-1.docker.io', + priority: 1, + enabled: true, + auth: { type: 'none' }, + }, + { + id: 'ghcr', + name: 'GitHub Container Registry', + url: 'https://ghcr.io', + priority: 2, + enabled: true, + scopeRules: [{ pattern: 'ghcr.io/*', action: 'include' }], + auth: { type: 'bearer', token: process.env.GHCR_TOKEN }, + }, + ], + }, + }, +}; + +const registry = new SmartRegistry(config); +await registry.init(); + +// Requests for @company/* packages go to private registry +// Other packages proxy through to npmjs.org with caching +``` + +### 🔌 Custom Auth Provider + +```typescript +import { SmartRegistry, IAuthProvider, IAuthToken, ICredentials, TRegistryProtocol } from '@push.rocks/smartregistry'; + +// Implement custom auth (e.g., LDAP, OAuth) +class LdapAuthProvider implements IAuthProvider { + constructor(private ldapClient: LdapClient) {} + + async authenticate(credentials: ICredentials): Promise { + const result = await this.ldapClient.bind(credentials.username, credentials.password); + return result.success ? credentials.username : null; + } + + async validateToken(token: string, protocol?: TRegistryProtocol): Promise { + const session = await this.sessionStore.get(token); + if (!session) return null; + return { + userId: session.userId, + scopes: session.scopes, + readonly: session.readonly, + created: session.created, + }; + } + + async createToken(userId: string, protocol: TRegistryProtocol, options?: ITokenOptions): Promise { + const token = crypto.randomUUID(); + await this.sessionStore.set(token, { userId, protocol, ...options }); + return token; + } + + async revokeToken(token: string): Promise { + await this.sessionStore.delete(token); + } + + async authorize(token: IAuthToken | null, resource: string, action: string): Promise { + if (!token) return action === 'read'; // Anonymous read-only + // Check LDAP groups, roles, etc. + return this.checkPermissions(token.userId, resource, action); + } +} + +// Use custom provider +const registry = new SmartRegistry({ + ...config, + authProvider: new LdapAuthProvider(ldapClient), +}); +``` + +### 📊 Storage Hooks (Quota & Audit) + +```typescript +import { SmartRegistry, IStorageHooks, IStorageHookContext } from '@push.rocks/smartregistry'; + +const storageHooks: IStorageHooks = { + // Block uploads that exceed quota + async beforePut(ctx: IStorageHookContext) { + if (ctx.actor?.orgId) { + const usage = await getStorageUsage(ctx.actor.orgId); + const quota = await getQuota(ctx.actor.orgId); + + if (usage + (ctx.metadata?.size || 0) > quota) { + return { allowed: false, reason: 'Storage quota exceeded' }; + } + } + return { allowed: true }; + }, + + // Update usage tracking after successful upload + async afterPut(ctx: IStorageHookContext) { + if (ctx.actor?.orgId && ctx.metadata?.size) { + await incrementUsage(ctx.actor.orgId, ctx.metadata.size); + } + + // Audit log + await auditLog.write({ + action: 'storage.put', + key: ctx.key, + protocol: ctx.protocol, + actor: ctx.actor, + timestamp: ctx.timestamp, + }); + }, + + // Prevent deletion of protected packages + async beforeDelete(ctx: IStorageHookContext) { + if (await isProtectedPackage(ctx.key)) { + return { allowed: false, reason: 'Cannot delete protected package' }; + } + return { allowed: true }; + }, + + // Log all access for compliance + async afterGet(ctx: IStorageHookContext) { + await accessLog.write({ + action: 'storage.get', + key: ctx.key, + actor: ctx.actor, + timestamp: ctx.timestamp, + }); + }, +}; + +const registry = new SmartRegistry({ + ...config, + storageHooks, +}); +``` + +### 👤 Request Actor Context + +```typescript +// Pass actor information through requests for audit/quota tracking +const response = await registry.handleRequest({ + method: 'PUT', + path: '/npm/my-package', + headers: { 'Authorization': 'Bearer ' }, + query: {}, + body: packageData, + actor: { + userId: 'user123', + tokenId: 'token-abc', + ip: req.ip, + userAgent: req.headers['user-agent'], + orgId: 'org-456', + sessionId: 'session-xyz', + }, +}); + +// Actor info is available in storage hooks for quota/audit +``` + ## ⚙️ Configuration ### Storage Configuration diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index 3f9b220..3cc6f6c 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@push.rocks/smartregistry', - version: '2.4.0', + version: '2.5.0', description: 'A composable TypeScript library implementing OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems registries for building unified container and package registries' }