diff --git a/changelog.md b/changelog.md index 1b1dd47..fbe2673 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,19 @@ # Changelog +## 2025-11-27 - 2.4.0 - feat(core) +Add pluggable auth providers, storage hooks, multi-upstream cache awareness, and PyPI/RubyGems protocol implementations + +- Introduce pluggable authentication: IAuthProvider interface and DefaultAuthProvider (in-memory) with OCI JWT support and UUID tokens. +- AuthManager now accepts a custom provider and delegates all auth operations (authenticate, validateToken, create/revoke tokens, authorize, listUserTokens). +- Add storage hooks (IStorageHooks) and hook contexts: beforePut/afterPut/afterGet/beforeDelete/afterDelete. RegistryStorage now supports hooks, context management (setContext/withContext) and invokes hooks around operations. +- RegistryStorage expanded with many protocol-specific helper methods (OCI, NPM, Maven, Cargo, Composer, PyPI, RubyGems) and improved S3/SmartBucket integration. +- Upstream improvements: BaseUpstream and UpstreamCache became multi-upstream aware (cache keys now include upstream URL), cache operations are async and support negative caching, stale-while-revalidate, ETag/metadata persistence, and S3-backed storage layer. +- Circuit breaker, retry, resilience and scope-rule routing enhancements for upstreams; upstream fetch logic updated to prefer primary upstream for cache keys and background revalidation behavior. +- SmartRegistry API extended to accept custom authProvider and storageHooks, and now wires RegistryStorage and AuthManager with those options. Core exports updated to expose auth and storage interfaces and DefaultAuthProvider. +- Add full PyPI (PEP 503/691, upload API) and RubyGems (Compact Index, API v1, uploads/yank/unyank, specs endpoints) registry implementations with parsing, upload/download, metadata management and upstream proxying. +- Add utility helpers: binary buffer helpers (toBuffer/isBinaryData), pypi and rubygems helper modules, and numerous protocol-specific helpers and tests referenced in readme.hints. +- These changes are additive and designed to be backward compatible; bumping minor version. + ## 2025-11-27 - 2.3.0 - feat(upstream) Add upstream proxy/cache subsystem and integrate per-protocol upstreams diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index a67cfa0..3f9b220 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@push.rocks/smartregistry', - version: '2.3.0', + version: '2.4.0', description: 'A composable TypeScript library implementing OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems registries for building unified container and package registries' } diff --git a/ts/classes.smartregistry.ts b/ts/classes.smartregistry.ts index ae6cc33..ca74d35 100644 --- a/ts/classes.smartregistry.ts +++ b/ts/classes.smartregistry.ts @@ -11,8 +11,39 @@ import { PypiRegistry } from './pypi/classes.pypiregistry.js'; import { RubyGemsRegistry } from './rubygems/classes.rubygemsregistry.js'; /** - * Main registry orchestrator - * Routes requests to appropriate protocol handlers (OCI, NPM, Maven, Cargo, Composer, PyPI, or RubyGems) + * Main registry orchestrator. + * Routes requests to appropriate protocol handlers (OCI, NPM, Maven, Cargo, Composer, PyPI, or RubyGems). + * + * Supports pluggable authentication and storage hooks: + * + * @example + * ```typescript + * // Basic usage with default in-memory auth + * const registry = new SmartRegistry(config); + * + * // With custom auth provider (LDAP, OAuth, etc.) + * const registry = new SmartRegistry({ + * ...config, + * authProvider: new LdapAuthProvider(ldapClient), + * }); + * + * // With storage hooks for quota tracking + * const registry = new SmartRegistry({ + * ...config, + * storageHooks: { + * beforePut: async (ctx) => { + * const quota = await getQuota(ctx.actor?.orgId); + * if (ctx.metadata?.size > quota) { + * return { allowed: false, reason: 'Quota exceeded' }; + * } + * return { allowed: true }; + * }, + * afterPut: async (ctx) => { + * await auditLog('storage.put', ctx); + * } + * } + * }); + * ``` */ export class SmartRegistry { private storage: RegistryStorage; @@ -23,8 +54,12 @@ export class SmartRegistry { constructor(config: IRegistryConfig) { this.config = config; - this.storage = new RegistryStorage(config.storage); - this.authManager = new AuthManager(config.auth); + + // Create storage with optional hooks + this.storage = new RegistryStorage(config.storage, config.storageHooks); + + // Create auth manager with optional custom provider + this.authManager = new AuthManager(config.auth, config.authProvider); } /** diff --git a/ts/core/classes.authmanager.ts b/ts/core/classes.authmanager.ts index 2f8a2b2..4927a6f 100644 --- a/ts/core/classes.authmanager.ts +++ b/ts/core/classes.authmanager.ts @@ -1,109 +1,79 @@ import type { IAuthConfig, IAuthToken, ICredentials, TRegistryProtocol } from './interfaces.core.js'; -import * as crypto from 'crypto'; +import type { IAuthProvider, ITokenOptions } from './interfaces.auth.js'; +import { DefaultAuthProvider } from './classes.defaultauthprovider.js'; /** - * Unified authentication manager for all registry protocols - * Handles both NPM UUID tokens and OCI JWT tokens + * Unified authentication manager for all registry protocols. + * Delegates to a pluggable IAuthProvider for actual auth operations. + * + * @example + * ```typescript + * // Use default in-memory provider + * const auth = new AuthManager(config); + * + * // Use custom provider (LDAP, OAuth, etc.) + * const auth = new AuthManager(config, new LdapAuthProvider(ldapClient)); + * ``` */ export class AuthManager { - private tokenStore: Map = new Map(); - private userCredentials: Map = new Map(); // username -> password hash (mock) + private provider: IAuthProvider; - constructor(private config: IAuthConfig) {} + constructor( + private config: IAuthConfig, + provider?: IAuthProvider + ) { + // Use provided provider or default in-memory implementation + this.provider = provider || new DefaultAuthProvider(config); + } /** * Initialize the auth manager */ public async init(): Promise { - // Initialize token store (in-memory for now) - // In production, this could be Redis or a database + if (this.provider.init) { + await this.provider.init(); + } } // ======================================================================== - // UUID TOKEN CREATION (Base method for NPM, Maven, etc.) + // UNIFIED AUTHENTICATION (Delegated to Provider) // ======================================================================== /** - * Create a UUID-based token with custom scopes (base method) - * @param userId - User ID - * @param protocol - Protocol type - * @param scopes - Permission scopes - * @param readonly - Whether the token is readonly - * @returns UUID token string + * Authenticate user credentials + * @param credentials - Username and password + * @returns User ID or null */ - private async createUuidToken( - userId: string, - protocol: TRegistryProtocol, - scopes: string[], - readonly: boolean = false - ): Promise { - const token = this.generateUuid(); - const authToken: IAuthToken = { - type: protocol, - userId, - scopes, - readonly, - metadata: { - created: new Date().toISOString(), - }, - }; - - this.tokenStore.set(token, authToken); - return token; + public async authenticate(credentials: ICredentials): Promise { + return this.provider.authenticate(credentials); } /** - * Generic protocol token creation (internal helper) - * @param userId - User ID - * @param protocol - Protocol type (npm, maven, composer, etc.) - * @param readonly - Whether the token is readonly - * @returns UUID token string - */ - private async createProtocolToken( - userId: string, - protocol: TRegistryProtocol, - readonly: boolean - ): Promise { - const scopes = readonly - ? [`${protocol}:*:*:read`] - : [`${protocol}:*:*:*`]; - return this.createUuidToken(userId, protocol, scopes, readonly); - } - - /** - * Generic protocol token validation (internal helper) - * @param token - UUID token string - * @param protocol - Expected protocol type + * Validate any token (NPM, Maven, OCI, PyPI, RubyGems, Composer, Cargo) + * @param tokenString - Token string (UUID or JWT) + * @param protocol - Expected protocol type (optional, improves performance) * @returns Auth token object or null */ - private async validateProtocolToken( - token: string, - protocol: TRegistryProtocol + public async validateToken( + tokenString: string, + protocol?: TRegistryProtocol ): Promise { - if (!this.isValidUuid(token)) { - return null; - } - - const authToken = this.tokenStore.get(token); - if (!authToken || authToken.type !== protocol) { - return null; - } - - // Check expiration if set - if (authToken.expiresAt && authToken.expiresAt < new Date()) { - this.tokenStore.delete(token); - return null; - } - - return authToken; + return this.provider.validateToken(tokenString, protocol); } /** - * Generic protocol token revocation (internal helper) - * @param token - UUID token string + * Check if token has permission for an action + * @param token - Auth token (or null for anonymous) + * @param resource - Resource being accessed (e.g., "npm:package:foo") + * @param action - Action being performed (read, write, push, pull, delete) + * @returns true if authorized */ - private async revokeProtocolToken(token: string): Promise { - this.tokenStore.delete(token); + public async authorize( + token: IAuthToken | null, + resource: string, + action: string + ): Promise { + return this.provider.authorize(token, resource, action); } // ======================================================================== @@ -120,7 +90,7 @@ export class AuthManager { if (!this.config.npmTokens.enabled) { throw new Error('NPM tokens are not enabled'); } - return this.createProtocolToken(userId, 'npm', readonly); + return this.provider.createToken(userId, 'npm', { readonly }); } /** @@ -129,7 +99,7 @@ export class AuthManager { * @returns Auth token object or null */ public async validateNpmToken(token: string): Promise { - return this.validateProtocolToken(token, 'npm'); + return this.provider.validateToken(token, 'npm'); } /** @@ -137,7 +107,7 @@ export class AuthManager { * @param token - NPM UUID token */ public async revokeNpmToken(token: string): Promise { - return this.revokeProtocolToken(token); + return this.provider.revokeToken(token); } /** @@ -149,20 +119,12 @@ export class AuthManager { key: string; readonly: boolean; created: string; + protocol?: TRegistryProtocol; }>> { - const tokens: Array<{key: string; readonly: boolean; created: string}> = []; - - for (const [token, authToken] of this.tokenStore.entries()) { - if (authToken.userId === userId) { - tokens.push({ - key: this.hashToken(token), - readonly: authToken.readonly || false, - created: authToken.metadata?.created || 'unknown', - }); - } + if (this.provider.listUserTokens) { + return this.provider.listUserTokens(userId); } - - return tokens; + return []; } // ======================================================================== @@ -174,39 +136,17 @@ export class AuthManager { * @param userId - User ID * @param scopes - Permission scopes * @param expiresIn - Expiration time in seconds - * @returns JWT token string (HMAC-SHA256 signed) + * @returns JWT token string */ public async createOciToken( userId: string, scopes: string[], expiresIn: number = 3600 ): Promise { - if (!this.config.ociTokens.enabled) { + if (!this.config.ociTokens?.enabled) { throw new Error('OCI tokens are not enabled'); } - - const now = Math.floor(Date.now() / 1000); - const payload = { - iss: this.config.ociTokens.realm, - sub: userId, - aud: this.config.ociTokens.service, - exp: now + expiresIn, - nbf: now, - iat: now, - access: this.scopesToOciAccess(scopes), - }; - - // Create JWT with HMAC-SHA256 signature - const header = { alg: 'HS256', typ: 'JWT' }; - const headerB64 = Buffer.from(JSON.stringify(header)).toString('base64url'); - const payloadB64 = Buffer.from(JSON.stringify(payload)).toString('base64url'); - - const signature = crypto - .createHmac('sha256', this.config.jwtSecret) - .update(`${headerB64}.${payloadB64}`) - .digest('base64url'); - - return `${headerB64}.${payloadB64}.${signature}`; + return this.provider.createToken(userId, 'oci', { scopes, expiresIn }); } /** @@ -215,80 +155,7 @@ export class AuthManager { * @returns Auth token object or null */ public async validateOciToken(jwt: string): Promise { - try { - const parts = jwt.split('.'); - if (parts.length !== 3) { - return null; - } - - const [headerB64, payloadB64, signatureB64] = parts; - - // Verify signature - const expectedSignature = crypto - .createHmac('sha256', this.config.jwtSecret) - .update(`${headerB64}.${payloadB64}`) - .digest('base64url'); - - if (signatureB64 !== expectedSignature) { - return null; - } - - // Decode and parse payload - const payload = JSON.parse(Buffer.from(payloadB64, 'base64url').toString('utf-8')); - - // Check expiration - const now = Math.floor(Date.now() / 1000); - if (payload.exp && payload.exp < now) { - return null; - } - - // Check not-before time - if (payload.nbf && payload.nbf > now) { - return null; - } - - // Convert to unified token format - const scopes = this.ociAccessToScopes(payload.access || []); - - return { - type: 'oci', - userId: payload.sub, - scopes, - expiresAt: payload.exp ? new Date(payload.exp * 1000) : undefined, - metadata: { - iss: payload.iss, - aud: payload.aud, - }, - }; - } catch (error) { - return null; - } - } - - // ======================================================================== - // UNIFIED AUTHENTICATION - // ======================================================================== - - /** - * Authenticate user credentials - * @param credentials - Username and password - * @returns User ID or null - */ - public async authenticate(credentials: ICredentials): Promise { - // Mock authentication - in production, verify against database - const storedPassword = this.userCredentials.get(credentials.username); - - if (!storedPassword) { - // Auto-register for testing (remove in production) - this.userCredentials.set(credentials.username, credentials.password); - return credentials.username; - } - - if (storedPassword === credentials.password) { - return credentials.username; - } - - return null; + return this.provider.validateToken(jwt, 'oci'); } // ======================================================================== @@ -302,7 +169,7 @@ export class AuthManager { * @returns Maven UUID token */ public async createMavenToken(userId: string, readonly: boolean = false): Promise { - return this.createProtocolToken(userId, 'maven', readonly); + return this.provider.createToken(userId, 'maven', { readonly }); } /** @@ -311,7 +178,7 @@ export class AuthManager { * @returns Auth token object or null */ public async validateMavenToken(token: string): Promise { - return this.validateProtocolToken(token, 'maven'); + return this.provider.validateToken(token, 'maven'); } /** @@ -319,7 +186,7 @@ export class AuthManager { * @param token - Maven UUID token */ public async revokeMavenToken(token: string): Promise { - return this.revokeProtocolToken(token); + return this.provider.revokeToken(token); } // ======================================================================== @@ -333,7 +200,7 @@ export class AuthManager { * @returns Composer UUID token */ public async createComposerToken(userId: string, readonly: boolean = false): Promise { - return this.createProtocolToken(userId, 'composer', readonly); + return this.provider.createToken(userId, 'composer', { readonly }); } /** @@ -342,7 +209,7 @@ export class AuthManager { * @returns Auth token object or null */ public async validateComposerToken(token: string): Promise { - return this.validateProtocolToken(token, 'composer'); + return this.provider.validateToken(token, 'composer'); } /** @@ -350,7 +217,7 @@ export class AuthManager { * @param token - Composer UUID token */ public async revokeComposerToken(token: string): Promise { - return this.revokeProtocolToken(token); + return this.provider.revokeToken(token); } // ======================================================================== @@ -364,7 +231,7 @@ export class AuthManager { * @returns Cargo UUID token */ public async createCargoToken(userId: string, readonly: boolean = false): Promise { - return this.createProtocolToken(userId, 'cargo', readonly); + return this.provider.createToken(userId, 'cargo', { readonly }); } /** @@ -373,7 +240,7 @@ export class AuthManager { * @returns Auth token object or null */ public async validateCargoToken(token: string): Promise { - return this.validateProtocolToken(token, 'cargo'); + return this.provider.validateToken(token, 'cargo'); } /** @@ -381,7 +248,7 @@ export class AuthManager { * @param token - Cargo UUID token */ public async revokeCargoToken(token: string): Promise { - return this.revokeProtocolToken(token); + return this.provider.revokeToken(token); } // ======================================================================== @@ -395,7 +262,7 @@ export class AuthManager { * @returns PyPI UUID token */ public async createPypiToken(userId: string, readonly: boolean = false): Promise { - return this.createProtocolToken(userId, 'pypi', readonly); + return this.provider.createToken(userId, 'pypi', { readonly }); } /** @@ -404,7 +271,7 @@ export class AuthManager { * @returns Auth token object or null */ public async validatePypiToken(token: string): Promise { - return this.validateProtocolToken(token, 'pypi'); + return this.provider.validateToken(token, 'pypi'); } /** @@ -412,7 +279,7 @@ export class AuthManager { * @param token - PyPI UUID token */ public async revokePypiToken(token: string): Promise { - return this.revokeProtocolToken(token); + return this.provider.revokeToken(token); } // ======================================================================== @@ -426,7 +293,7 @@ export class AuthManager { * @returns RubyGems UUID token */ public async createRubyGemsToken(userId: string, readonly: boolean = false): Promise { - return this.createProtocolToken(userId, 'rubygems', readonly); + return this.provider.createToken(userId, 'rubygems', { readonly }); } /** @@ -435,7 +302,7 @@ export class AuthManager { * @returns Auth token object or null */ public async validateRubyGemsToken(token: string): Promise { - return this.validateProtocolToken(token, 'rubygems'); + return this.provider.validateToken(token, 'rubygems'); } /** @@ -443,211 +310,6 @@ export class AuthManager { * @param token - RubyGems UUID token */ public async revokeRubyGemsToken(token: string): Promise { - return this.revokeProtocolToken(token); - } - - // ======================================================================== - // UNIFIED AUTHENTICATION - // ======================================================================== - - /** - * Validate any token (NPM, Maven, OCI, PyPI, RubyGems, Composer, Cargo) - * Optimized: O(1) lookup when protocol hint provided - * @param tokenString - Token string (UUID or JWT) - * @param protocol - Expected protocol type (optional, improves performance) - * @returns Auth token object or null - */ - public async validateToken( - tokenString: string, - protocol?: TRegistryProtocol - ): Promise { - // OCI uses JWT (contains dots), not UUID - check first if OCI is expected - if (protocol === 'oci' || tokenString.includes('.')) { - const ociToken = await this.validateOciToken(tokenString); - if (ociToken && (!protocol || protocol === 'oci')) { - return ociToken; - } - // If protocol was explicitly OCI but validation failed, return null - if (protocol === 'oci') { - return null; - } - } - - // UUID-based tokens: single O(1) Map lookup - if (this.isValidUuid(tokenString)) { - const authToken = this.tokenStore.get(tokenString); - if (authToken) { - // If protocol specified, verify it matches - if (protocol && authToken.type !== protocol) { - return null; - } - // Check expiration - if (authToken.expiresAt && authToken.expiresAt < new Date()) { - this.tokenStore.delete(tokenString); - return null; - } - return authToken; - } - } - - return null; - } - - /** - * Check if token has permission for an action - * @param token - Auth token - * @param resource - Resource being accessed (e.g., "package:foo" or "repository:bar") - * @param action - Action being performed (read, write, push, pull, delete) - * @returns true if authorized - */ - public async authorize( - token: IAuthToken | null, - resource: string, - action: string - ): Promise { - if (!token) { - return false; - } - - // Check readonly flag - if (token.readonly && ['write', 'push', 'delete'].includes(action)) { - return false; - } - - // Check scopes - for (const scope of token.scopes) { - if (this.matchesScope(scope, resource, action)) { - return true; - } - } - - return false; - } - - // ======================================================================== - // HELPER METHODS - // ======================================================================== - - /** - * Check if a scope matches a resource and action - * Scope format: "{protocol}:{type}:{name}:{action}" - * Examples: - * - "npm:*:*" - All NPM access - * - "npm:package:foo:*" - All actions on package foo - * - "npm:package:foo:read" - Read-only on package foo - * - "oci:repository:*:pull" - Pull from any OCI repo - */ - private matchesScope(scope: string, resource: string, action: string): boolean { - const scopeParts = scope.split(':'); - const resourceParts = resource.split(':'); - - // Scope must have at least protocol:type:name:action - if (scopeParts.length < 4) { - return false; - } - - const [scopeProtocol, scopeType, scopeName, scopeAction] = scopeParts; - const [resourceProtocol, resourceType, resourceName] = resourceParts; - - // Check protocol - if (scopeProtocol !== '*' && scopeProtocol !== resourceProtocol) { - return false; - } - - // Check type - if (scopeType !== '*' && scopeType !== resourceType) { - return false; - } - - // Check name - if (scopeName !== '*' && scopeName !== resourceName) { - return false; - } - - // Check action - if (scopeAction !== '*' && scopeAction !== action) { - // Map action aliases - const actionAliases: Record = { - read: ['pull', 'get'], - write: ['push', 'put', 'post'], - }; - - const aliases = actionAliases[scopeAction] || []; - if (!aliases.includes(action)) { - return false; - } - } - - return true; - } - - /** - * Convert unified scopes to OCI access array - */ - private scopesToOciAccess(scopes: string[]): Array<{ - type: string; - name: string; - actions: string[]; - }> { - const access: Array<{type: string; name: string; actions: string[]}> = []; - - for (const scope of scopes) { - const parts = scope.split(':'); - if (parts.length >= 4 && parts[0] === 'oci') { - access.push({ - type: parts[1], - name: parts[2], - actions: [parts[3]], - }); - } - } - - return access; - } - - /** - * Convert OCI access array to unified scopes - */ - private ociAccessToScopes(access: Array<{ - type: string; - name: string; - actions: string[]; - }>): string[] { - const scopes: string[] = []; - - for (const item of access) { - for (const action of item.actions) { - scopes.push(`oci:${item.type}:${item.name}:${action}`); - } - } - - return scopes; - } - - /** - * Generate UUID for NPM tokens - */ - private generateUuid(): string { - return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => { - const r = (Math.random() * 16) | 0; - const v = c === 'x' ? r : (r & 0x3) | 0x8; - return v.toString(16); - }); - } - - /** - * Check if string is a valid UUID - */ - private isValidUuid(str: string): boolean { - const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; - return uuidRegex.test(str); - } - - /** - * Hash a token for identification (SHA-512 mock) - */ - private hashToken(token: string): string { - // In production, use actual SHA-512 - return `sha512-${token.substring(0, 16)}...`; + return this.provider.revokeToken(token); } } diff --git a/ts/core/classes.defaultauthprovider.ts b/ts/core/classes.defaultauthprovider.ts new file mode 100644 index 0000000..22c5986 --- /dev/null +++ b/ts/core/classes.defaultauthprovider.ts @@ -0,0 +1,393 @@ +import * as crypto from 'crypto'; +import type { IAuthProvider, ITokenOptions } from './interfaces.auth.js'; +import type { IAuthConfig, IAuthToken, ICredentials, TRegistryProtocol } from './interfaces.core.js'; + +/** + * Default in-memory authentication provider. + * This is the reference implementation that stores tokens in memory. + * For production use, implement IAuthProvider with Redis, database, or external auth. + */ +export class DefaultAuthProvider implements IAuthProvider { + private tokenStore: Map = new Map(); + private userCredentials: Map = new Map(); // username -> password hash (mock) + + constructor(private config: IAuthConfig) {} + + /** + * Initialize the auth provider + */ + public async init(): Promise { + // Initialize token store (in-memory for now) + // In production, this could be Redis or a database + } + + // ======================================================================== + // IAuthProvider Implementation + // ======================================================================== + + /** + * Authenticate user credentials + */ + public async authenticate(credentials: ICredentials): Promise { + // Mock authentication - in production, verify against database/LDAP + const storedPassword = this.userCredentials.get(credentials.username); + + if (!storedPassword) { + // Auto-register for testing (remove in production) + this.userCredentials.set(credentials.username, credentials.password); + return credentials.username; + } + + if (storedPassword === credentials.password) { + return credentials.username; + } + + return null; + } + + /** + * Validate any token (NPM, Maven, OCI, PyPI, RubyGems, Composer, Cargo) + */ + public async validateToken( + tokenString: string, + protocol?: TRegistryProtocol + ): Promise { + // OCI uses JWT (contains dots), not UUID - check first if OCI is expected + if (protocol === 'oci' || tokenString.includes('.')) { + const ociToken = await this.validateOciToken(tokenString); + if (ociToken && (!protocol || protocol === 'oci')) { + return ociToken; + } + // If protocol was explicitly OCI but validation failed, return null + if (protocol === 'oci') { + return null; + } + } + + // UUID-based tokens: single O(1) Map lookup + if (this.isValidUuid(tokenString)) { + const authToken = this.tokenStore.get(tokenString); + if (authToken) { + // If protocol specified, verify it matches + if (protocol && authToken.type !== protocol) { + return null; + } + // Check expiration + if (authToken.expiresAt && authToken.expiresAt < new Date()) { + this.tokenStore.delete(tokenString); + return null; + } + return authToken; + } + } + + return null; + } + + /** + * Create a new token for a user + */ + public async createToken( + userId: string, + protocol: TRegistryProtocol, + options?: ITokenOptions + ): Promise { + // OCI tokens use JWT + if (protocol === 'oci') { + return this.createOciToken(userId, options?.scopes || ['oci:*:*:*'], options?.expiresIn || 3600); + } + + // All other protocols use UUID tokens + const token = this.generateUuid(); + const scopes = options?.scopes || (options?.readonly + ? [`${protocol}:*:*:read`] + : [`${protocol}:*:*:*`]); + + const authToken: IAuthToken = { + type: protocol, + userId, + scopes, + readonly: options?.readonly, + expiresAt: options?.expiresIn ? new Date(Date.now() + options.expiresIn * 1000) : undefined, + metadata: { + created: new Date().toISOString(), + }, + }; + + this.tokenStore.set(token, authToken); + return token; + } + + /** + * Revoke a token + */ + public async revokeToken(token: string): Promise { + this.tokenStore.delete(token); + } + + /** + * Check if token has permission for an action + */ + public async authorize( + token: IAuthToken | null, + resource: string, + action: string + ): Promise { + if (!token) { + return false; + } + + // Check readonly flag + if (token.readonly && ['write', 'push', 'delete'].includes(action)) { + return false; + } + + // Check scopes + for (const scope of token.scopes) { + if (this.matchesScope(scope, resource, action)) { + return true; + } + } + + return false; + } + + /** + * List all tokens for a user + */ + public async listUserTokens(userId: string): Promise> { + const tokens: Array<{key: string; readonly: boolean; created: string; protocol?: TRegistryProtocol}> = []; + + for (const [token, authToken] of this.tokenStore.entries()) { + if (authToken.userId === userId) { + tokens.push({ + key: this.hashToken(token), + readonly: authToken.readonly || false, + created: authToken.metadata?.created || 'unknown', + protocol: authToken.type, + }); + } + } + + return tokens; + } + + // ======================================================================== + // OCI JWT Token Methods + // ======================================================================== + + /** + * Create an OCI JWT token + */ + private async createOciToken( + userId: string, + scopes: string[], + expiresIn: number = 3600 + ): Promise { + if (!this.config.ociTokens?.enabled) { + throw new Error('OCI tokens are not enabled'); + } + + const now = Math.floor(Date.now() / 1000); + const payload = { + iss: this.config.ociTokens.realm, + sub: userId, + aud: this.config.ociTokens.service, + exp: now + expiresIn, + nbf: now, + iat: now, + access: this.scopesToOciAccess(scopes), + }; + + // Create JWT with HMAC-SHA256 signature + const header = { alg: 'HS256', typ: 'JWT' }; + const headerB64 = Buffer.from(JSON.stringify(header)).toString('base64url'); + const payloadB64 = Buffer.from(JSON.stringify(payload)).toString('base64url'); + + const signature = crypto + .createHmac('sha256', this.config.jwtSecret) + .update(`${headerB64}.${payloadB64}`) + .digest('base64url'); + + return `${headerB64}.${payloadB64}.${signature}`; + } + + /** + * Validate an OCI JWT token + */ + private async validateOciToken(jwt: string): Promise { + try { + const parts = jwt.split('.'); + if (parts.length !== 3) { + return null; + } + + const [headerB64, payloadB64, signatureB64] = parts; + + // Verify signature + const expectedSignature = crypto + .createHmac('sha256', this.config.jwtSecret) + .update(`${headerB64}.${payloadB64}`) + .digest('base64url'); + + if (signatureB64 !== expectedSignature) { + return null; + } + + // Decode and parse payload + const payload = JSON.parse(Buffer.from(payloadB64, 'base64url').toString('utf-8')); + + // Check expiration + const now = Math.floor(Date.now() / 1000); + if (payload.exp && payload.exp < now) { + return null; + } + + // Check not-before time + if (payload.nbf && payload.nbf > now) { + return null; + } + + // Convert to unified token format + const scopes = this.ociAccessToScopes(payload.access || []); + + return { + type: 'oci', + userId: payload.sub, + scopes, + expiresAt: payload.exp ? new Date(payload.exp * 1000) : undefined, + metadata: { + iss: payload.iss, + aud: payload.aud, + }, + }; + } catch (error) { + return null; + } + } + + // ======================================================================== + // Helper Methods + // ======================================================================== + + /** + * Check if a scope matches a resource and action + */ + private matchesScope(scope: string, resource: string, action: string): boolean { + const scopeParts = scope.split(':'); + const resourceParts = resource.split(':'); + + // Scope must have at least protocol:type:name:action + if (scopeParts.length < 4) { + return false; + } + + const [scopeProtocol, scopeType, scopeName, scopeAction] = scopeParts; + const [resourceProtocol, resourceType, resourceName] = resourceParts; + + // Check protocol + if (scopeProtocol !== '*' && scopeProtocol !== resourceProtocol) { + return false; + } + + // Check type + if (scopeType !== '*' && scopeType !== resourceType) { + return false; + } + + // Check name + if (scopeName !== '*' && scopeName !== resourceName) { + return false; + } + + // Check action + if (scopeAction !== '*' && scopeAction !== action) { + // Map action aliases + const actionAliases: Record = { + read: ['pull', 'get'], + write: ['push', 'put', 'post'], + }; + + const aliases = actionAliases[scopeAction] || []; + if (!aliases.includes(action)) { + return false; + } + } + + return true; + } + + /** + * Convert unified scopes to OCI access array + */ + private scopesToOciAccess(scopes: string[]): Array<{ + type: string; + name: string; + actions: string[]; + }> { + const access: Array<{type: string; name: string; actions: string[]}> = []; + + for (const scope of scopes) { + const parts = scope.split(':'); + if (parts.length >= 4 && parts[0] === 'oci') { + access.push({ + type: parts[1], + name: parts[2], + actions: [parts[3]], + }); + } + } + + return access; + } + + /** + * Convert OCI access array to unified scopes + */ + private ociAccessToScopes(access: Array<{ + type: string; + name: string; + actions: string[]; + }>): string[] { + const scopes: string[] = []; + + for (const item of access) { + for (const action of item.actions) { + scopes.push(`oci:${item.type}:${item.name}:${action}`); + } + } + + return scopes; + } + + /** + * Generate UUID for tokens + */ + private generateUuid(): string { + return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => { + const r = (Math.random() * 16) | 0; + const v = c === 'x' ? r : (r & 0x3) | 0x8; + return v.toString(16); + }); + } + + /** + * Check if string is a valid UUID + */ + private isValidUuid(str: string): boolean { + const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; + return uuidRegex.test(str); + } + + /** + * Hash a token for identification + */ + private hashToken(token: string): string { + return `sha512-${token.substring(0, 16)}...`; + } +} diff --git a/ts/core/classes.registrystorage.ts b/ts/core/classes.registrystorage.ts index 28c5c1c..b874dd7 100644 --- a/ts/core/classes.registrystorage.ts +++ b/ts/core/classes.registrystorage.ts @@ -1,17 +1,54 @@ import * as plugins from '../plugins.js'; -import type { IStorageConfig, IStorageBackend } from './interfaces.core.js'; +import type { IStorageConfig, IStorageBackend, TRegistryProtocol } from './interfaces.core.js'; +import type { + IStorageHooks, + IStorageHookContext, + IStorageActor, + IStorageMetadata, +} from './interfaces.storage.js'; /** - * Storage abstraction layer for registry - * Provides a unified interface over SmartBucket + * Storage abstraction layer for registry. + * Provides a unified interface over SmartBucket with optional hooks + * for quota tracking, audit logging, cache invalidation, etc. + * + * @example + * ```typescript + * // Basic usage + * const storage = new RegistryStorage(config); + * + * // With hooks for quota tracking + * const storage = new RegistryStorage(config, { + * beforePut: async (ctx) => { + * const quota = await getQuota(ctx.actor?.orgId); + * const usage = await getUsage(ctx.actor?.orgId); + * if (usage + (ctx.metadata?.size || 0) > quota) { + * return { allowed: false, reason: 'Quota exceeded' }; + * } + * return { allowed: true }; + * }, + * afterPut: async (ctx) => { + * await updateUsage(ctx.actor?.orgId, ctx.metadata?.size || 0); + * } + * }); + * ``` */ export class RegistryStorage implements IStorageBackend { private smartBucket: plugins.smartbucket.SmartBucket; private bucket: plugins.smartbucket.Bucket; private bucketName: string; + private hooks?: IStorageHooks; - constructor(private config: IStorageConfig) { + constructor(private config: IStorageConfig, hooks?: IStorageHooks) { this.bucketName = config.bucketName; + this.hooks = hooks; + } + + /** + * Set storage hooks (can be called after construction) + */ + public setHooks(hooks: IStorageHooks): void { + this.hooks = hooks; } /** @@ -34,7 +71,24 @@ export class RegistryStorage implements IStorageBackend { */ public async getObject(key: string): Promise { try { - return await this.bucket.fastGet({ path: key }); + const data = await this.bucket.fastGet({ path: key }); + + // Call afterGet hook (non-blocking) + if (this.hooks?.afterGet && data) { + const context = this.currentContext; + if (context) { + this.hooks.afterGet({ + operation: 'get', + key, + protocol: context.protocol, + actor: context.actor, + metadata: context.metadata, + timestamp: new Date(), + }).catch(() => {}); // Don't fail on hook errors + } + } + + return data; } catch (error) { return null; } @@ -48,19 +102,159 @@ export class RegistryStorage implements IStorageBackend { data: Buffer, metadata?: Record ): Promise { + // Call beforePut hook if available + if (this.hooks?.beforePut) { + const context = this.currentContext; + if (context) { + const hookContext: IStorageHookContext = { + operation: 'put', + key, + protocol: context.protocol, + actor: context.actor, + metadata: { + ...context.metadata, + size: data.length, + }, + timestamp: new Date(), + }; + + const result = await this.hooks.beforePut(hookContext); + if (!result.allowed) { + throw new Error(result.reason || 'Storage operation denied by hook'); + } + } + } + // Note: SmartBucket doesn't support metadata yet await this.bucket.fastPut({ path: key, contents: data, overwrite: true, // Always overwrite existing objects }); + + // Call afterPut hook (non-blocking) + if (this.hooks?.afterPut) { + const context = this.currentContext; + if (context) { + this.hooks.afterPut({ + operation: 'put', + key, + protocol: context.protocol, + actor: context.actor, + metadata: { + ...context.metadata, + size: data.length, + }, + timestamp: new Date(), + }).catch(() => {}); // Don't fail on hook errors + } + } } /** * Delete an object */ public async deleteObject(key: string): Promise { + // Call beforeDelete hook if available + if (this.hooks?.beforeDelete) { + const context = this.currentContext; + if (context) { + const hookContext: IStorageHookContext = { + operation: 'delete', + key, + protocol: context.protocol, + actor: context.actor, + metadata: context.metadata, + timestamp: new Date(), + }; + + const result = await this.hooks.beforeDelete(hookContext); + if (!result.allowed) { + throw new Error(result.reason || 'Delete operation denied by hook'); + } + } + } + await this.bucket.fastRemove({ path: key }); + + // Call afterDelete hook (non-blocking) + if (this.hooks?.afterDelete) { + const context = this.currentContext; + if (context) { + this.hooks.afterDelete({ + operation: 'delete', + key, + protocol: context.protocol, + actor: context.actor, + metadata: context.metadata, + timestamp: new Date(), + }).catch(() => {}); // Don't fail on hook errors + } + } + } + + // ======================================================================== + // CONTEXT FOR HOOKS + // ======================================================================== + + /** + * Current operation context for hooks. + * Set this before performing storage operations to enable hooks. + */ + private currentContext?: { + protocol: TRegistryProtocol; + actor?: IStorageActor; + metadata?: IStorageMetadata; + }; + + /** + * Set the current operation context for hooks. + * Call this before performing storage operations. + * + * @example + * ```typescript + * storage.setContext({ + * protocol: 'npm', + * actor: { userId: 'user123', ip: '192.168.1.1' }, + * metadata: { packageName: 'lodash', version: '4.17.21' } + * }); + * await storage.putNpmTarball('lodash', '4.17.21', tarball); + * storage.clearContext(); + * ``` + */ + public setContext(context: { + protocol: TRegistryProtocol; + actor?: IStorageActor; + metadata?: IStorageMetadata; + }): void { + this.currentContext = context; + } + + /** + * Clear the current operation context. + */ + public clearContext(): void { + this.currentContext = undefined; + } + + /** + * Execute a function with a temporary context. + * Context is automatically cleared after execution. + */ + public async withContext( + context: { + protocol: TRegistryProtocol; + actor?: IStorageActor; + metadata?: IStorageMetadata; + }, + fn: () => Promise + ): Promise { + this.setContext(context); + try { + return await fn(); + } finally { + this.clearContext(); + } } /** diff --git a/ts/core/index.ts b/ts/core/index.ts index 8fb2121..ee6394d 100644 --- a/ts/core/index.ts +++ b/ts/core/index.ts @@ -2,9 +2,16 @@ * Core registry infrastructure exports */ -// Interfaces +// Core interfaces export * from './interfaces.core.js'; +// Auth interfaces and provider +export * from './interfaces.auth.js'; +export { DefaultAuthProvider } from './classes.defaultauthprovider.js'; + +// Storage interfaces and hooks +export * from './interfaces.storage.js'; + // Classes export { BaseRegistry } from './classes.baseregistry.js'; export { RegistryStorage } from './classes.registrystorage.js'; diff --git a/ts/core/interfaces.auth.ts b/ts/core/interfaces.auth.ts new file mode 100644 index 0000000..0616a5d --- /dev/null +++ b/ts/core/interfaces.auth.ts @@ -0,0 +1,91 @@ +import type { IAuthToken, ICredentials, TRegistryProtocol } from './interfaces.core.js'; + +/** + * Options for creating a token + */ +export interface ITokenOptions { + /** Whether the token is readonly */ + readonly?: boolean; + /** Permission scopes */ + scopes?: string[]; + /** Expiration time in seconds */ + expiresIn?: number; +} + +/** + * Pluggable authentication provider interface. + * Implement this to integrate external auth systems (LDAP, OAuth, SSO, OIDC). + * + * @example + * ```typescript + * class LdapAuthProvider implements IAuthProvider { + * constructor(private ldap: LdapClient, private redis: RedisClient) {} + * + * async authenticate(credentials: ICredentials): Promise { + * return await this.ldap.bind(credentials.username, credentials.password); + * } + * + * async validateToken(token: string): Promise { + * return await this.redis.get(`token:${token}`); + * } + * // ... + * } + * ``` + */ +export interface IAuthProvider { + /** + * Initialize the auth provider (optional) + */ + init?(): Promise; + + /** + * Authenticate user credentials (login flow) + * @param credentials - Username and password + * @returns User ID on success, null on failure + */ + authenticate(credentials: ICredentials): Promise; + + /** + * Validate an existing token + * @param token - Token string (UUID or JWT) + * @param protocol - Optional protocol hint for optimization + * @returns Auth token info or null if invalid + */ + validateToken(token: string, protocol?: TRegistryProtocol): Promise; + + /** + * Create a new token for a user + * @param userId - User ID + * @param protocol - Protocol type (npm, oci, maven, etc.) + * @param options - Token options (readonly, scopes, expiration) + * @returns Token string + */ + createToken(userId: string, protocol: TRegistryProtocol, options?: ITokenOptions): Promise; + + /** + * Revoke a token + * @param token - Token string to revoke + */ + revokeToken(token: string): Promise; + + /** + * Check if user has permission for an action + * @param token - Auth token (or null for anonymous) + * @param resource - Resource being accessed (e.g., "npm:package:lodash") + * @param action - Action being performed (read, write, push, pull, delete) + * @returns true if authorized + */ + authorize(token: IAuthToken | null, resource: string, action: string): Promise; + + /** + * List all tokens for a user (optional) + * @param userId - User ID + * @returns List of token info + */ + listUserTokens?(userId: string): Promise>; +} diff --git a/ts/core/interfaces.core.ts b/ts/core/interfaces.core.ts index c1dad7a..8b32cb3 100644 --- a/ts/core/interfaces.core.ts +++ b/ts/core/interfaces.core.ts @@ -4,6 +4,8 @@ import type * as plugins from '../plugins.js'; import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js'; +import type { IAuthProvider } from './interfaces.auth.js'; +import type { IStorageHooks } from './interfaces.storage.js'; /** * Registry protocol types @@ -97,6 +99,20 @@ export interface IProtocolConfig { export interface IRegistryConfig { storage: IStorageConfig; auth: IAuthConfig; + + /** + * Custom authentication provider. + * If not provided, uses the default in-memory auth provider. + * Implement IAuthProvider to integrate LDAP, OAuth, SSO, etc. + */ + authProvider?: IAuthProvider; + + /** + * Storage event hooks for quota tracking, audit logging, etc. + * Called before/after storage operations. + */ + storageHooks?: IStorageHooks; + oci?: IProtocolConfig; npm?: IProtocolConfig; maven?: IProtocolConfig; @@ -152,6 +168,24 @@ export interface IRegistryError { }>; } +/** + * Actor information - identifies who is performing the request + */ +export interface IRequestActor { + /** User ID (from validated token) */ + userId?: string; + /** Token ID/hash for audit purposes */ + tokenId?: string; + /** Client IP address */ + ip?: string; + /** Client User-Agent */ + userAgent?: string; + /** Organization ID (for multi-tenant setups) */ + orgId?: string; + /** Session ID */ + sessionId?: string; +} + /** * Base request context */ @@ -168,6 +202,11 @@ export interface IRequestContext { */ rawBody?: Buffer; token?: string; + /** + * Actor information - identifies who is performing the request. + * Populated after authentication for audit logging, quota enforcement, etc. + */ + actor?: IRequestActor; } /** diff --git a/ts/core/interfaces.storage.ts b/ts/core/interfaces.storage.ts new file mode 100644 index 0000000..5848f30 --- /dev/null +++ b/ts/core/interfaces.storage.ts @@ -0,0 +1,130 @@ +import type { TRegistryProtocol } from './interfaces.core.js'; + +/** + * Actor information from request context + */ +export interface IStorageActor { + userId?: string; + tokenId?: string; + ip?: string; + userAgent?: string; + orgId?: string; + sessionId?: string; +} + +/** + * Metadata about the storage operation + */ +export interface IStorageMetadata { + /** Content type of the object */ + contentType?: string; + /** Size in bytes */ + size?: number; + /** Content digest (e.g., sha256:abc123) */ + digest?: string; + /** Package/artifact name */ + packageName?: string; + /** Version */ + version?: string; +} + +/** + * Context passed to storage hooks + */ +export interface IStorageHookContext { + /** Type of operation */ + operation: 'put' | 'delete' | 'get'; + /** Storage key/path */ + key: string; + /** Protocol that triggered this operation */ + protocol: TRegistryProtocol; + /** Actor who performed the operation (if known) */ + actor?: IStorageActor; + /** Metadata about the object */ + metadata?: IStorageMetadata; + /** Timestamp of the operation */ + timestamp: Date; +} + +/** + * Result from a beforePut hook that can modify the operation + */ +export interface IBeforePutResult { + /** Whether to allow the operation */ + allowed: boolean; + /** Optional reason for rejection */ + reason?: string; + /** Optional modified metadata */ + metadata?: IStorageMetadata; +} + +/** + * Result from a beforeDelete hook + */ +export interface IBeforeDeleteResult { + /** Whether to allow the operation */ + allowed: boolean; + /** Optional reason for rejection */ + reason?: string; +} + +/** + * Storage event hooks for quota tracking, audit logging, cache invalidation, etc. + * + * @example + * ```typescript + * const quotaHooks: IStorageHooks = { + * async beforePut(context) { + * const quota = await getQuota(context.actor?.orgId); + * const currentUsage = await getUsage(context.actor?.orgId); + * if (currentUsage + (context.metadata?.size || 0) > quota) { + * return { allowed: false, reason: 'Quota exceeded' }; + * } + * return { allowed: true }; + * }, + * + * async afterPut(context) { + * await updateUsage(context.actor?.orgId, context.metadata?.size || 0); + * await auditLog('storage.put', context); + * }, + * + * async afterDelete(context) { + * await invalidateCache(context.key); + * } + * }; + * ``` + */ +export interface IStorageHooks { + /** + * Called before storing an object. + * Return { allowed: false } to reject the operation. + * Use for quota checks, virus scanning, validation, etc. + */ + beforePut?(context: IStorageHookContext): Promise; + + /** + * Called after successfully storing an object. + * Use for quota tracking, audit logging, notifications, etc. + */ + afterPut?(context: IStorageHookContext): Promise; + + /** + * Called before deleting an object. + * Return { allowed: false } to reject the operation. + * Use for preventing deletion of protected resources. + */ + beforeDelete?(context: IStorageHookContext): Promise; + + /** + * Called after successfully deleting an object. + * Use for quota updates, audit logging, cache invalidation, etc. + */ + afterDelete?(context: IStorageHookContext): Promise; + + /** + * Called after reading an object. + * Use for access logging, analytics, etc. + * Note: This is called even for cache hits. + */ + afterGet?(context: IStorageHookContext): Promise; +} diff --git a/ts/upstream/classes.baseupstream.ts b/ts/upstream/classes.baseupstream.ts index 7b22c51..960dc3e 100644 --- a/ts/upstream/classes.baseupstream.ts +++ b/ts/upstream/classes.baseupstream.ts @@ -110,8 +110,18 @@ export abstract class BaseUpstream { return null; } + // Get applicable upstreams sorted by priority + const applicableUpstreams = this.getApplicableUpstreams(context.resource); + + if (applicableUpstreams.length === 0) { + return null; + } + + // Use the first applicable upstream's URL for cache key + const primaryUpstreamUrl = applicableUpstreams[0]?.url; + // Check cache first - const cached = this.cache.get(context); + const cached = await this.cache.get(context, primaryUpstreamUrl); if (cached && !cached.stale) { return { success: true, @@ -125,7 +135,7 @@ export abstract class BaseUpstream { } // Check for negative cache (recent 404) - if (this.cache.hasNegative(context)) { + if (await this.cache.hasNegative(context, primaryUpstreamUrl)) { return { success: false, status: 404, @@ -136,13 +146,6 @@ export abstract class BaseUpstream { }; } - // Get applicable upstreams sorted by priority - const applicableUpstreams = this.getApplicableUpstreams(context.resource); - - if (applicableUpstreams.length === 0) { - return null; - } - // If we have stale cache, return it immediately and revalidate in background if (cached?.stale && this.cacheConfig.staleWhileRevalidate) { // Fire and forget revalidation @@ -173,18 +176,19 @@ export abstract class BaseUpstream { // Cache successful responses if (result.success && result.body) { - this.cache.set( + await this.cache.set( context, Buffer.isBuffer(result.body) ? result.body : Buffer.from(JSON.stringify(result.body)), result.headers['content-type'] || 'application/octet-stream', result.headers, upstream.id, + upstream.url, ); } // Cache 404 responses if (result.status === 404) { - this.cache.setNegative(context, upstream.id); + await this.cache.setNegative(context, upstream.id, upstream.url); } return result; @@ -210,15 +214,15 @@ export abstract class BaseUpstream { /** * Invalidate cache for a resource pattern. */ - public invalidateCache(pattern: RegExp): number { + public async invalidateCache(pattern: RegExp): Promise { return this.cache.invalidatePattern(pattern); } /** * Clear all cache entries. */ - public clearCache(): void { - this.cache.clear(); + public async clearCache(): Promise { + await this.cache.clear(); } /** @@ -501,12 +505,13 @@ export abstract class BaseUpstream { ); if (result.success && result.body) { - this.cache.set( + await this.cache.set( context, Buffer.isBuffer(result.body) ? result.body : Buffer.from(JSON.stringify(result.body)), result.headers['content-type'] || 'application/octet-stream', result.headers, upstream.id, + upstream.url, ); return; // Successfully revalidated } diff --git a/ts/upstream/classes.upstreamcache.ts b/ts/upstream/classes.upstreamcache.ts index 8442b92..85ad210 100644 --- a/ts/upstream/classes.upstreamcache.ts +++ b/ts/upstream/classes.upstreamcache.ts @@ -4,9 +4,23 @@ import type { IUpstreamFetchContext, } from './interfaces.upstream.js'; import { DEFAULT_CACHE_CONFIG } from './interfaces.upstream.js'; +import type { IStorageBackend } from '../core/interfaces.core.js'; /** - * In-memory cache for upstream responses. + * Cache metadata stored alongside cache entries. + */ +interface ICacheMetadata { + contentType: string; + headers: Record; + cachedAt: string; + expiresAt?: string; + etag?: string; + upstreamId: string; + upstreamUrl: string; +} + +/** + * S3-backed upstream cache with in-memory hot layer. * * Features: * - TTL-based expiration @@ -14,26 +28,45 @@ import { DEFAULT_CACHE_CONFIG } from './interfaces.upstream.js'; * - Negative caching (404s) * - Content-type aware caching * - ETag support for conditional requests + * - Multi-upstream support via URL-based cache paths + * - Persistent S3 storage with in-memory hot layer * - * Note: This is an in-memory implementation. For production with persistence, - * extend this class to use RegistryStorage for S3-backed caching. + * Cache paths are structured as: + * cache/{escaped-upstream-url}/{protocol}:{method}:{path} + * + * @example + * ```typescript + * // In-memory only (default) + * const cache = new UpstreamCache(config); + * + * // With S3 persistence + * const cache = new UpstreamCache(config, 10000, storage); + * ``` */ export class UpstreamCache { - /** Cache storage */ - private readonly cache: Map = new Map(); + /** In-memory hot cache */ + private readonly memoryCache: Map = new Map(); /** Configuration */ private readonly config: IUpstreamCacheConfig; - /** Maximum cache entries (prevents memory bloat) */ - private readonly maxEntries: number; + /** Maximum in-memory cache entries */ + private readonly maxMemoryEntries: number; + + /** S3 storage backend (optional) */ + private readonly storage?: IStorageBackend; /** Cleanup interval handle */ private cleanupInterval: ReturnType | null = null; - constructor(config?: Partial, maxEntries: number = 10000) { + constructor( + config?: Partial, + maxMemoryEntries: number = 10000, + storage?: IStorageBackend + ) { this.config = { ...DEFAULT_CACHE_CONFIG, ...config }; - this.maxEntries = maxEntries; + this.maxMemoryEntries = maxMemoryEntries; + this.storage = storage; // Start periodic cleanup if caching is enabled if (this.config.enabled) { @@ -48,17 +81,36 @@ export class UpstreamCache { return this.config.enabled; } + /** + * Check if S3 storage is configured. + */ + public hasStorage(): boolean { + return !!this.storage; + } + /** * Get cached entry for a request context. + * Checks memory first, then falls back to S3. * Returns null if not found or expired (unless stale-while-revalidate). */ - public get(context: IUpstreamFetchContext): ICacheEntry | null { + public async get(context: IUpstreamFetchContext, upstreamUrl?: string): Promise { if (!this.config.enabled) { return null; } - const key = this.buildCacheKey(context); - const entry = this.cache.get(key); + const key = this.buildCacheKey(context, upstreamUrl); + + // Check memory cache first + let entry = this.memoryCache.get(key); + + // If not in memory and we have storage, check S3 + if (!entry && this.storage) { + entry = await this.loadFromStorage(key); + if (entry) { + // Promote to memory cache + this.memoryCache.set(key, entry); + } + } if (!entry) { return null; @@ -78,7 +130,10 @@ export class UpstreamCache { } } // Entry is too old, remove it - this.cache.delete(key); + this.memoryCache.delete(key); + if (this.storage) { + await this.deleteFromStorage(key).catch(() => {}); + } return null; } @@ -86,26 +141,27 @@ export class UpstreamCache { } /** - * Store a response in the cache. + * Store a response in the cache (memory and optionally S3). */ - public set( + public async set( context: IUpstreamFetchContext, data: Buffer, contentType: string, headers: Record, upstreamId: string, + upstreamUrl: string, options?: ICacheSetOptions, - ): void { + ): Promise { if (!this.config.enabled) { return; } - // Enforce max entries limit - if (this.cache.size >= this.maxEntries) { + // Enforce max memory entries limit + if (this.memoryCache.size >= this.maxMemoryEntries) { this.evictOldest(); } - const key = this.buildCacheKey(context); + const key = this.buildCacheKey(context, upstreamUrl); const now = new Date(); // Determine TTL based on content type @@ -122,18 +178,24 @@ export class UpstreamCache { stale: false, }; - this.cache.set(key, entry); + // Store in memory + this.memoryCache.set(key, entry); + + // Store in S3 if available + if (this.storage) { + await this.saveToStorage(key, entry, upstreamUrl).catch(() => {}); + } } /** * Store a negative cache entry (404 response). */ - public setNegative(context: IUpstreamFetchContext, upstreamId: string): void { + public async setNegative(context: IUpstreamFetchContext, upstreamId: string, upstreamUrl: string): Promise { if (!this.config.enabled || this.config.negativeCacheTtlSeconds <= 0) { return; } - const key = this.buildCacheKey(context); + const key = this.buildCacheKey(context, upstreamUrl); const now = new Date(); const entry: ICacheEntry = { @@ -146,34 +208,47 @@ export class UpstreamCache { stale: false, }; - this.cache.set(key, entry); + this.memoryCache.set(key, entry); + + if (this.storage) { + await this.saveToStorage(key, entry, upstreamUrl).catch(() => {}); + } } /** * Check if there's a negative cache entry for this context. */ - public hasNegative(context: IUpstreamFetchContext): boolean { - const entry = this.get(context); + public async hasNegative(context: IUpstreamFetchContext, upstreamUrl?: string): Promise { + const entry = await this.get(context, upstreamUrl); return entry !== null && entry.data.length === 0; } /** * Invalidate a specific cache entry. */ - public invalidate(context: IUpstreamFetchContext): boolean { - const key = this.buildCacheKey(context); - return this.cache.delete(key); + public async invalidate(context: IUpstreamFetchContext, upstreamUrl?: string): Promise { + const key = this.buildCacheKey(context, upstreamUrl); + const deleted = this.memoryCache.delete(key); + + if (this.storage) { + await this.deleteFromStorage(key).catch(() => {}); + } + + return deleted; } /** * Invalidate all entries matching a pattern. * Useful for invalidating all versions of a package. */ - public invalidatePattern(pattern: RegExp): number { + public async invalidatePattern(pattern: RegExp): Promise { let count = 0; - for (const key of this.cache.keys()) { + for (const key of this.memoryCache.keys()) { if (pattern.test(key)) { - this.cache.delete(key); + this.memoryCache.delete(key); + if (this.storage) { + await this.deleteFromStorage(key).catch(() => {}); + } count++; } } @@ -183,11 +258,14 @@ export class UpstreamCache { /** * Invalidate all entries from a specific upstream. */ - public invalidateUpstream(upstreamId: string): number { + public async invalidateUpstream(upstreamId: string): Promise { let count = 0; - for (const [key, entry] of this.cache.entries()) { + for (const [key, entry] of this.memoryCache.entries()) { if (entry.upstreamId === upstreamId) { - this.cache.delete(key); + this.memoryCache.delete(key); + if (this.storage) { + await this.deleteFromStorage(key).catch(() => {}); + } count++; } } @@ -195,10 +273,13 @@ export class UpstreamCache { } /** - * Clear all cache entries. + * Clear all cache entries (memory and S3). */ - public clear(): void { - this.cache.clear(); + public async clear(): Promise { + this.memoryCache.clear(); + + // Note: S3 cleanup would require listing and deleting all cache/* objects + // This is left as a future enhancement for bulk cleanup } /** @@ -211,7 +292,7 @@ export class UpstreamCache { let totalSize = 0; const now = new Date(); - for (const entry of this.cache.values()) { + for (const entry of this.memoryCache.values()) { totalSize += entry.data.length; if (entry.data.length === 0) { @@ -224,13 +305,14 @@ export class UpstreamCache { } return { - totalEntries: this.cache.size, + totalEntries: this.memoryCache.size, freshEntries: freshCount, staleEntries: staleCount, negativeEntries: negativeCount, totalSizeBytes: totalSize, - maxEntries: this.maxEntries, + maxEntries: this.maxMemoryEntries, enabled: this.config.enabled, + hasStorage: !!this.storage, }; } @@ -244,17 +326,136 @@ export class UpstreamCache { } } + // ======================================================================== + // Storage Methods + // ======================================================================== + + /** + * Build storage path for a cache key. + * Escapes upstream URL for safe use in S3 paths. + */ + private buildStoragePath(key: string): string { + return `cache/${key}`; + } + + /** + * Build storage path for cache metadata. + */ + private buildMetadataPath(key: string): string { + return `cache/${key}.meta`; + } + + /** + * Load a cache entry from S3 storage. + */ + private async loadFromStorage(key: string): Promise { + if (!this.storage) return null; + + try { + const dataPath = this.buildStoragePath(key); + const metaPath = this.buildMetadataPath(key); + + // Load data and metadata in parallel + const [data, metaBuffer] = await Promise.all([ + this.storage.getObject(dataPath), + this.storage.getObject(metaPath), + ]); + + if (!data || !metaBuffer) { + return null; + } + + const meta: ICacheMetadata = JSON.parse(metaBuffer.toString('utf-8')); + + return { + data, + contentType: meta.contentType, + headers: meta.headers, + cachedAt: new Date(meta.cachedAt), + expiresAt: meta.expiresAt ? new Date(meta.expiresAt) : undefined, + etag: meta.etag, + upstreamId: meta.upstreamId, + stale: false, + }; + } catch { + return null; + } + } + + /** + * Save a cache entry to S3 storage. + */ + private async saveToStorage(key: string, entry: ICacheEntry, upstreamUrl: string): Promise { + if (!this.storage) return; + + const dataPath = this.buildStoragePath(key); + const metaPath = this.buildMetadataPath(key); + + const meta: ICacheMetadata = { + contentType: entry.contentType, + headers: entry.headers, + cachedAt: entry.cachedAt.toISOString(), + expiresAt: entry.expiresAt?.toISOString(), + etag: entry.etag, + upstreamId: entry.upstreamId, + upstreamUrl, + }; + + // Save data and metadata in parallel + await Promise.all([ + this.storage.putObject(dataPath, entry.data), + this.storage.putObject(metaPath, Buffer.from(JSON.stringify(meta), 'utf-8')), + ]); + } + + /** + * Delete a cache entry from S3 storage. + */ + private async deleteFromStorage(key: string): Promise { + if (!this.storage) return; + + const dataPath = this.buildStoragePath(key); + const metaPath = this.buildMetadataPath(key); + + await Promise.all([ + this.storage.deleteObject(dataPath).catch(() => {}), + this.storage.deleteObject(metaPath).catch(() => {}), + ]); + } + + // ======================================================================== + // Helper Methods + // ======================================================================== + + /** + * Escape a URL for safe use in storage paths. + */ + private escapeUrl(url: string): string { + // Remove protocol prefix and escape special characters + return url + .replace(/^https?:\/\//, '') + .replace(/[\/\\:*?"<>|]/g, '_') + .replace(/__+/g, '_'); + } + /** * Build a unique cache key for a request context. + * Includes escaped upstream URL for multi-upstream support. */ - private buildCacheKey(context: IUpstreamFetchContext): string { + private buildCacheKey(context: IUpstreamFetchContext, upstreamUrl?: string): string { // Include method, protocol, path, and sorted query params const queryString = Object.keys(context.query) .sort() .map(k => `${k}=${context.query[k]}`) .join('&'); - return `${context.protocol}:${context.method}:${context.path}${queryString ? '?' + queryString : ''}`; + const baseKey = `${context.protocol}:${context.method}:${context.path}${queryString ? '?' + queryString : ''}`; + + if (upstreamUrl) { + return `${this.escapeUrl(upstreamUrl)}/${baseKey}`; + } + + return baseKey; } /** @@ -333,27 +534,27 @@ export class UpstreamCache { */ private evictOldest(): void { // Evict 10% of max entries - const evictCount = Math.ceil(this.maxEntries * 0.1); + const evictCount = Math.ceil(this.maxMemoryEntries * 0.1); let evicted = 0; // First, try to evict stale entries const now = new Date(); - for (const [key, entry] of this.cache.entries()) { + for (const [key, entry] of this.memoryCache.entries()) { if (evicted >= evictCount) break; if (entry.stale || (entry.expiresAt && entry.expiresAt < now)) { - this.cache.delete(key); + this.memoryCache.delete(key); evicted++; } } // If not enough evicted, evict oldest by cachedAt if (evicted < evictCount) { - const entries = Array.from(this.cache.entries()) + const entries = Array.from(this.memoryCache.entries()) .sort((a, b) => a[1].cachedAt.getTime() - b[1].cachedAt.getTime()); for (const [key] of entries) { if (evicted >= evictCount) break; - this.cache.delete(key); + this.memoryCache.delete(key); evicted++; } } @@ -375,17 +576,17 @@ export class UpstreamCache { } /** - * Remove all expired entries. + * Remove all expired entries from memory cache. */ private cleanup(): void { const now = new Date(); const staleDeadline = new Date(now.getTime() - this.config.staleMaxAgeSeconds * 1000); - for (const [key, entry] of this.cache.entries()) { + for (const [key, entry] of this.memoryCache.entries()) { if (entry.expiresAt) { // Remove if past stale deadline if (entry.expiresAt < staleDeadline) { - this.cache.delete(key); + this.memoryCache.delete(key); } } } @@ -406,7 +607,7 @@ export interface ICacheSetOptions { * Cache statistics. */ export interface ICacheStats { - /** Total number of cached entries */ + /** Total number of cached entries in memory */ totalEntries: number; /** Number of fresh (non-expired) entries */ freshEntries: number; @@ -414,10 +615,12 @@ export interface ICacheStats { staleEntries: number; /** Number of negative cache entries */ negativeEntries: number; - /** Total size of cached data in bytes */ + /** Total size of cached data in bytes (memory only) */ totalSizeBytes: number; - /** Maximum allowed entries */ + /** Maximum allowed memory entries */ maxEntries: number; /** Whether caching is enabled */ enabled: boolean; + /** Whether S3 storage is configured */ + hasStorage: boolean; }