push.rocks/smartregistry
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(core): Add pluggable auth providers, storage hooks, multi-upstream cache awareness, and PyPI/RubyGems protocol implementations

Browse Source
This commit is contained in:
Jürgen Kunz
2025-11-27 20:59:49 +00:00
parent 99b01733e7
commit 19da87a9df
12 changed files with 1264 additions and 491 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
changelog.md
View File

@@ -1,5 +1,19 @@
# Changelog
## 2025-11-27 - 2.4.0 - feat(core)
Add pluggable auth providers, storage hooks, multi-upstream cache awareness, and PyPI/RubyGems protocol implementations
- Introduce pluggable authentication: IAuthProvider interface and DefaultAuthProvider (in-memory) with OCI JWT support and UUID tokens.
- AuthManager now accepts a custom provider and delegates all auth operations (authenticate, validateToken, create/revoke tokens, authorize, listUserTokens).
- Add storage hooks (IStorageHooks) and hook contexts: beforePut/afterPut/afterGet/beforeDelete/afterDelete. RegistryStorage now supports hooks, context management (setContext/withContext) and invokes hooks around operations.
- RegistryStorage expanded with many protocol-specific helper methods (OCI, NPM, Maven, Cargo, Composer, PyPI, RubyGems) and improved S3/SmartBucket integration.
- Upstream improvements: BaseUpstream and UpstreamCache became multi-upstream aware (cache keys now include upstream URL), cache operations are async and support negative caching, stale-while-revalidate, ETag/metadata persistence, and S3-backed storage layer.
- Circuit breaker, retry, resilience and scope-rule routing enhancements for upstreams; upstream fetch logic updated to prefer primary upstream for cache keys and background revalidation behavior.
- SmartRegistry API extended to accept custom authProvider and storageHooks, and now wires RegistryStorage and AuthManager with those options. Core exports updated to expose auth and storage interfaces and DefaultAuthProvider.
- Add full PyPI (PEP 503/691, upload API) and RubyGems (Compact Index, API v1, uploads/yank/unyank, specs endpoints) registry implementations with parsing, upload/download, metadata management and upstream proxying.
- Add utility helpers: binary buffer helpers (toBuffer/isBinaryData), pypi and rubygems helper modules, and numerous protocol-specific helpers and tests referenced in readme.hints.
- These changes are additive and designed to be backward compatible; bumping minor version.
## 2025-11-27 - 2.3.0 - feat(upstream)
Add upstream proxy/cache subsystem and integrate per-protocol upstreams

2
ts/00_commitinfo_data.ts
View File

@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@push.rocks/smartregistry',
version: '2.3.0',
version: '2.4.0',
description: 'A composable TypeScript library implementing OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems registries for building unified container and package registries'
}

43
ts/classes.smartregistry.ts
View File

@@ -11,8 +11,39 @@ import { PypiRegistry } from './pypi/classes.pypiregistry.js';
import { RubyGemsRegistry } from './rubygems/classes.rubygemsregistry.js';
/**
* Main registry orchestrator
* Routes requests to appropriate protocol handlers (OCI, NPM, Maven, Cargo, Composer, PyPI, or RubyGems)
* Main registry orchestrator.
* Routes requests to appropriate protocol handlers (OCI, NPM, Maven, Cargo, Composer, PyPI, or RubyGems).
*
* Supports pluggable authentication and storage hooks:
*
* @example
* ```typescript
* // Basic usage with default in-memory auth
* const registry = new SmartRegistry(config);
*
* // With custom auth provider (LDAP, OAuth, etc.)
* const registry = new SmartRegistry({
* ...config,
* authProvider: new LdapAuthProvider(ldapClient),
* });
*
* // With storage hooks for quota tracking
* const registry = new SmartRegistry({
* ...config,
* storageHooks: {
* beforePut: async (ctx) => {
* const quota = await getQuota(ctx.actor?.orgId);
* if (ctx.metadata?.size > quota) {
* return { allowed: false, reason: 'Quota exceeded' };
* }
* return { allowed: true };
* },
* afterPut: async (ctx) => {
* await auditLog('storage.put', ctx);
* }
* }
* });
* ```
*/
export class SmartRegistry {
private storage: RegistryStorage;
@@ -23,8 +54,12 @@ export class SmartRegistry {
constructor(config: IRegistryConfig) {
this.config = config;
this.storage = new RegistryStorage(config.storage);
this.authManager = new AuthManager(config.auth);
// Create storage with optional hooks
this.storage = new RegistryStorage(config.storage, config.storageHooks);
// Create auth manager with optional custom provider
this.authManager = new AuthManager(config.auth, config.authProvider);
}
/**

486
ts/core/classes.authmanager.ts
View File

@@ -1,109 +1,79 @@
import type { IAuthConfig, IAuthToken, ICredentials, TRegistryProtocol } from './interfaces.core.js';
import * as crypto from 'crypto';
import type { IAuthProvider, ITokenOptions } from './interfaces.auth.js';
import { DefaultAuthProvider } from './classes.defaultauthprovider.js';
/**
* Unified authentication manager for all registry protocols
* Handles both NPM UUID tokens and OCI JWT tokens
* Unified authentication manager for all registry protocols.
* Delegates to a pluggable IAuthProvider for actual auth operations.
*
* @example
* ```typescript
* // Use default in-memory provider
* const auth = new AuthManager(config);
*
* // Use custom provider (LDAP, OAuth, etc.)
* const auth = new AuthManager(config, new LdapAuthProvider(ldapClient));
* ```
*/
export class AuthManager {
private tokenStore: Map<string, IAuthToken> = new Map();
private userCredentials: Map<string, string> = new Map(); // username -> password hash (mock)
private provider: IAuthProvider;
constructor(private config: IAuthConfig) {}
constructor(
private config: IAuthConfig,
provider?: IAuthProvider
) {
// Use provided provider or default in-memory implementation
this.provider = provider || new DefaultAuthProvider(config);
}
/**
* Initialize the auth manager
*/
public async init(): Promise<void> {
// Initialize token store (in-memory for now)
// In production, this could be Redis or a database
if (this.provider.init) {
await this.provider.init();
}
}
// ========================================================================
// UUID TOKEN CREATION (Base method for NPM, Maven, etc.)
// UNIFIED AUTHENTICATION (Delegated to Provider)
// ========================================================================
/**
* Create a UUID-based token with custom scopes (base method)
* @param userId - User ID
* @param protocol - Protocol type
* @param scopes - Permission scopes
* @param readonly - Whether the token is readonly
* @returns UUID token string
* Authenticate user credentials
* @param credentials - Username and password
* @returns User ID or null
*/
private async createUuidToken(
userId: string,
protocol: TRegistryProtocol,
scopes: string[],
readonly: boolean = false
): Promise<string> {
const token = this.generateUuid();
const authToken: IAuthToken = {
type: protocol,
userId,
scopes,
readonly,
metadata: {
created: new Date().toISOString(),
},
};
this.tokenStore.set(token, authToken);
return token;
public async authenticate(credentials: ICredentials): Promise<string | null> {
return this.provider.authenticate(credentials);
}
/**
* Generic protocol token creation (internal helper)
* @param userId - User ID
* @param protocol - Protocol type (npm, maven, composer, etc.)
* @param readonly - Whether the token is readonly
* @returns UUID token string
*/
private async createProtocolToken(
userId: string,
protocol: TRegistryProtocol,
readonly: boolean
): Promise<string> {
const scopes = readonly
? [`${protocol}:*:*:read`]
: [`${protocol}:*:*:*`];
return this.createUuidToken(userId, protocol, scopes, readonly);
}
/**
* Generic protocol token validation (internal helper)
* @param token - UUID token string
* @param protocol - Expected protocol type
* Validate any token (NPM, Maven, OCI, PyPI, RubyGems, Composer, Cargo)
* @param tokenString - Token string (UUID or JWT)
* @param protocol - Expected protocol type (optional, improves performance)
* @returns Auth token object or null
*/
private async validateProtocolToken(
token: string,
protocol: TRegistryProtocol
public async validateToken(
tokenString: string,
protocol?: TRegistryProtocol
): Promise<IAuthToken | null> {
if (!this.isValidUuid(token)) {
return null;
}
const authToken = this.tokenStore.get(token);
if (!authToken || authToken.type !== protocol) {
return null;
}
// Check expiration if set
if (authToken.expiresAt && authToken.expiresAt < new Date()) {
this.tokenStore.delete(token);
return null;
}
return authToken;
return this.provider.validateToken(tokenString, protocol);
}
/**
* Generic protocol token revocation (internal helper)
* @param token - UUID token string
* Check if token has permission for an action
* @param token - Auth token (or null for anonymous)
* @param resource - Resource being accessed (e.g., "npm:package:foo")
* @param action - Action being performed (read, write, push, pull, delete)
* @returns true if authorized
*/
private async revokeProtocolToken(token: string): Promise<void> {
this.tokenStore.delete(token);
public async authorize(
token: IAuthToken | null,
resource: string,
action: string
): Promise<boolean> {
return this.provider.authorize(token, resource, action);
}
// ========================================================================
@@ -120,7 +90,7 @@ export class AuthManager {
if (!this.config.npmTokens.enabled) {
throw new Error('NPM tokens are not enabled');
}
return this.createProtocolToken(userId, 'npm', readonly);
return this.provider.createToken(userId, 'npm', { readonly });
}
/**
@@ -129,7 +99,7 @@ export class AuthManager {
* @returns Auth token object or null
*/
public async validateNpmToken(token: string): Promise<IAuthToken | null> {
return this.validateProtocolToken(token, 'npm');
return this.provider.validateToken(token, 'npm');
}
/**
@@ -137,7 +107,7 @@ export class AuthManager {
* @param token - NPM UUID token
*/
public async revokeNpmToken(token: string): Promise<void> {
return this.revokeProtocolToken(token);
return this.provider.revokeToken(token);
}
/**
@@ -149,20 +119,12 @@ export class AuthManager {
key: string;
readonly: boolean;
created: string;
protocol?: TRegistryProtocol;
}>> {
const tokens: Array<{key: string; readonly: boolean; created: string}> = [];
for (const [token, authToken] of this.tokenStore.entries()) {
if (authToken.userId === userId) {
tokens.push({
key: this.hashToken(token),
readonly: authToken.readonly || false,
created: authToken.metadata?.created || 'unknown',
});
if (this.provider.listUserTokens) {
return this.provider.listUserTokens(userId);
}
}
return tokens;
return [];
}
// ========================================================================
@@ -174,39 +136,17 @@ export class AuthManager {
* @param userId - User ID
* @param scopes - Permission scopes
* @param expiresIn - Expiration time in seconds
* @returns JWT token string (HMAC-SHA256 signed)
* @returns JWT token string
*/
public async createOciToken(
userId: string,
scopes: string[],
expiresIn: number = 3600
): Promise<string> {
if (!this.config.ociTokens.enabled) {
if (!this.config.ociTokens?.enabled) {
throw new Error('OCI tokens are not enabled');
}
const now = Math.floor(Date.now() / 1000);
const payload = {
iss: this.config.ociTokens.realm,
sub: userId,
aud: this.config.ociTokens.service,
exp: now + expiresIn,
nbf: now,
iat: now,
access: this.scopesToOciAccess(scopes),
};
// Create JWT with HMAC-SHA256 signature
const header = { alg: 'HS256', typ: 'JWT' };
const headerB64 = Buffer.from(JSON.stringify(header)).toString('base64url');
const payloadB64 = Buffer.from(JSON.stringify(payload)).toString('base64url');
const signature = crypto
.createHmac('sha256', this.config.jwtSecret)
.update(`${headerB64}.${payloadB64}`)
.digest('base64url');
return `${headerB64}.${payloadB64}.${signature}`;
return this.provider.createToken(userId, 'oci', { scopes, expiresIn });
}
/**
@@ -215,80 +155,7 @@ export class AuthManager {
* @returns Auth token object or null
*/
public async validateOciToken(jwt: string): Promise<IAuthToken | null> {
try {
const parts = jwt.split('.');
if (parts.length !== 3) {
return null;
}
const [headerB64, payloadB64, signatureB64] = parts;
// Verify signature
const expectedSignature = crypto
.createHmac('sha256', this.config.jwtSecret)
.update(`${headerB64}.${payloadB64}`)
.digest('base64url');
if (signatureB64 !== expectedSignature) {
return null;
}
// Decode and parse payload
const payload = JSON.parse(Buffer.from(payloadB64, 'base64url').toString('utf-8'));
// Check expiration
const now = Math.floor(Date.now() / 1000);
if (payload.exp && payload.exp < now) {
return null;
}
// Check not-before time
if (payload.nbf && payload.nbf > now) {
return null;
}
// Convert to unified token format
const scopes = this.ociAccessToScopes(payload.access || []);
return {
type: 'oci',
userId: payload.sub,
scopes,
expiresAt: payload.exp ? new Date(payload.exp * 1000) : undefined,
metadata: {
iss: payload.iss,
aud: payload.aud,
},
};
} catch (error) {
return null;
}
}
// ========================================================================
// UNIFIED AUTHENTICATION
// ========================================================================
/**
* Authenticate user credentials
* @param credentials - Username and password
* @returns User ID or null
*/
public async authenticate(credentials: ICredentials): Promise<string | null> {
// Mock authentication - in production, verify against database
const storedPassword = this.userCredentials.get(credentials.username);
if (!storedPassword) {
// Auto-register for testing (remove in production)
this.userCredentials.set(credentials.username, credentials.password);
return credentials.username;
}
if (storedPassword === credentials.password) {
return credentials.username;
}
return null;
return this.provider.validateToken(jwt, 'oci');
}
// ========================================================================
@@ -302,7 +169,7 @@ export class AuthManager {
* @returns Maven UUID token
*/
public async createMavenToken(userId: string, readonly: boolean = false): Promise<string> {
return this.createProtocolToken(userId, 'maven', readonly);
return this.provider.createToken(userId, 'maven', { readonly });
}
/**
@@ -311,7 +178,7 @@ export class AuthManager {
* @returns Auth token object or null
*/
public async validateMavenToken(token: string): Promise<IAuthToken | null> {
return this.validateProtocolToken(token, 'maven');
return this.provider.validateToken(token, 'maven');
}
/**
@@ -319,7 +186,7 @@ export class AuthManager {
* @param token - Maven UUID token
*/
public async revokeMavenToken(token: string): Promise<void> {
return this.revokeProtocolToken(token);
return this.provider.revokeToken(token);
}
// ========================================================================
@@ -333,7 +200,7 @@ export class AuthManager {
* @returns Composer UUID token
*/
public async createComposerToken(userId: string, readonly: boolean = false): Promise<string> {
return this.createProtocolToken(userId, 'composer', readonly);
return this.provider.createToken(userId, 'composer', { readonly });
}
/**
@@ -342,7 +209,7 @@ export class AuthManager {
* @returns Auth token object or null
*/
public async validateComposerToken(token: string): Promise<IAuthToken | null> {
return this.validateProtocolToken(token, 'composer');
return this.provider.validateToken(token, 'composer');
}
/**
@@ -350,7 +217,7 @@ export class AuthManager {
* @param token - Composer UUID token
*/
public async revokeComposerToken(token: string): Promise<void> {
return this.revokeProtocolToken(token);
return this.provider.revokeToken(token);
}
// ========================================================================
@@ -364,7 +231,7 @@ export class AuthManager {
* @returns Cargo UUID token
*/
public async createCargoToken(userId: string, readonly: boolean = false): Promise<string> {
return this.createProtocolToken(userId, 'cargo', readonly);
return this.provider.createToken(userId, 'cargo', { readonly });
}
/**
@@ -373,7 +240,7 @@ export class AuthManager {
* @returns Auth token object or null
*/
public async validateCargoToken(token: string): Promise<IAuthToken | null> {
return this.validateProtocolToken(token, 'cargo');
return this.provider.validateToken(token, 'cargo');
}
/**
@@ -381,7 +248,7 @@ export class AuthManager {
* @param token - Cargo UUID token
*/
public async revokeCargoToken(token: string): Promise<void> {
return this.revokeProtocolToken(token);
return this.provider.revokeToken(token);
}
// ========================================================================
@@ -395,7 +262,7 @@ export class AuthManager {
* @returns PyPI UUID token
*/
public async createPypiToken(userId: string, readonly: boolean = false): Promise<string> {
return this.createProtocolToken(userId, 'pypi', readonly);
return this.provider.createToken(userId, 'pypi', { readonly });
}
/**
@@ -404,7 +271,7 @@ export class AuthManager {
* @returns Auth token object or null
*/
public async validatePypiToken(token: string): Promise<IAuthToken | null> {
return this.validateProtocolToken(token, 'pypi');
return this.provider.validateToken(token, 'pypi');
}
/**
@@ -412,7 +279,7 @@ export class AuthManager {
* @param token - PyPI UUID token
*/
public async revokePypiToken(token: string): Promise<void> {
return this.revokeProtocolToken(token);
return this.provider.revokeToken(token);
}
// ========================================================================
@@ -426,7 +293,7 @@ export class AuthManager {
* @returns RubyGems UUID token
*/
public async createRubyGemsToken(userId: string, readonly: boolean = false): Promise<string> {
return this.createProtocolToken(userId, 'rubygems', readonly);
return this.provider.createToken(userId, 'rubygems', { readonly });
}
/**
@@ -435,7 +302,7 @@ export class AuthManager {
* @returns Auth token object or null
*/
public async validateRubyGemsToken(token: string): Promise<IAuthToken | null> {
return this.validateProtocolToken(token, 'rubygems');
return this.provider.validateToken(token, 'rubygems');
}
/**
@@ -443,211 +310,6 @@ export class AuthManager {
* @param token - RubyGems UUID token
*/
public async revokeRubyGemsToken(token: string): Promise<void> {
return this.revokeProtocolToken(token);
}
// ========================================================================
// UNIFIED AUTHENTICATION
// ========================================================================
/**
* Validate any token (NPM, Maven, OCI, PyPI, RubyGems, Composer, Cargo)
* Optimized: O(1) lookup when protocol hint provided
* @param tokenString - Token string (UUID or JWT)
* @param protocol - Expected protocol type (optional, improves performance)
* @returns Auth token object or null
*/
public async validateToken(
tokenString: string,
protocol?: TRegistryProtocol
): Promise<IAuthToken | null> {
// OCI uses JWT (contains dots), not UUID - check first if OCI is expected
if (protocol === 'oci' || tokenString.includes('.')) {
const ociToken = await this.validateOciToken(tokenString);
if (ociToken && (!protocol || protocol === 'oci')) {
return ociToken;
}
// If protocol was explicitly OCI but validation failed, return null
if (protocol === 'oci') {
return null;
}
}
// UUID-based tokens: single O(1) Map lookup
if (this.isValidUuid(tokenString)) {
const authToken = this.tokenStore.get(tokenString);
if (authToken) {
// If protocol specified, verify it matches
if (protocol && authToken.type !== protocol) {
return null;
}
// Check expiration
if (authToken.expiresAt && authToken.expiresAt < new Date()) {
this.tokenStore.delete(tokenString);
return null;
}
return authToken;
}
}
return null;
}
/**
* Check if token has permission for an action
* @param token - Auth token
* @param resource - Resource being accessed (e.g., "package:foo" or "repository:bar")
* @param action - Action being performed (read, write, push, pull, delete)
* @returns true if authorized
*/
public async authorize(
token: IAuthToken | null,
resource: string,
action: string
): Promise<boolean> {
if (!token) {
return false;
}
// Check readonly flag
if (token.readonly && ['write', 'push', 'delete'].includes(action)) {
return false;
}
// Check scopes
for (const scope of token.scopes) {
if (this.matchesScope(scope, resource, action)) {
return true;
}
}
return false;
}
// ========================================================================
// HELPER METHODS
// ========================================================================
/**
* Check if a scope matches a resource and action
* Scope format: "{protocol}:{type}:{name}:{action}"
* Examples:
* - "npm:*:*" - All NPM access
* - "npm:package:foo:*" - All actions on package foo
* - "npm:package:foo:read" - Read-only on package foo
* - "oci:repository:*:pull" - Pull from any OCI repo
*/
private matchesScope(scope: string, resource: string, action: string): boolean {
const scopeParts = scope.split(':');
const resourceParts = resource.split(':');
// Scope must have at least protocol:type:name:action
if (scopeParts.length < 4) {
return false;
}
const [scopeProtocol, scopeType, scopeName, scopeAction] = scopeParts;
const [resourceProtocol, resourceType, resourceName] = resourceParts;
// Check protocol
if (scopeProtocol !== '*' && scopeProtocol !== resourceProtocol) {
return false;
}
// Check type
if (scopeType !== '*' && scopeType !== resourceType) {
return false;
}
// Check name
if (scopeName !== '*' && scopeName !== resourceName) {
return false;
}
// Check action
if (scopeAction !== '*' && scopeAction !== action) {
// Map action aliases
const actionAliases: Record<string, string[]> = {
read: ['pull', 'get'],
write: ['push', 'put', 'post'],
};
const aliases = actionAliases[scopeAction] || [];
if (!aliases.includes(action)) {
return false;
}
}
return true;
}
/**
* Convert unified scopes to OCI access array
*/
private scopesToOciAccess(scopes: string[]): Array<{
type: string;
name: string;
actions: string[];
}> {
const access: Array<{type: string; name: string; actions: string[]}> = [];
for (const scope of scopes) {
const parts = scope.split(':');
if (parts.length >= 4 && parts[0] === 'oci') {
access.push({
type: parts[1],
name: parts[2],
actions: [parts[3]],
});
}
}
return access;
}
/**
* Convert OCI access array to unified scopes
*/
private ociAccessToScopes(access: Array<{
type: string;
name: string;
actions: string[];
}>): string[] {
const scopes: string[] = [];
for (const item of access) {
for (const action of item.actions) {
scopes.push(`oci:${item.type}:${item.name}:${action}`);
}
}
return scopes;
}
/**
* Generate UUID for NPM tokens
*/
private generateUuid(): string {
return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => {
const r = (Math.random() * 16) | 0;
const v = c === 'x' ? r : (r & 0x3) | 0x8;
return v.toString(16);
});
}
/**
* Check if string is a valid UUID
*/
private isValidUuid(str: string): boolean {
const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
return uuidRegex.test(str);
}
/**
* Hash a token for identification (SHA-512 mock)
*/
private hashToken(token: string): string {
// In production, use actual SHA-512
return `sha512-${token.substring(0, 16)}...`;
return this.provider.revokeToken(token);
}
}

393
ts/core/classes.defaultauthprovider.ts Normal file
View File

@@ -0,0 +1,393 @@
import * as crypto from 'crypto';
import type { IAuthProvider, ITokenOptions } from './interfaces.auth.js';
import type { IAuthConfig, IAuthToken, ICredentials, TRegistryProtocol } from './interfaces.core.js';
/**
* Default in-memory authentication provider.
* This is the reference implementation that stores tokens in memory.
* For production use, implement IAuthProvider with Redis, database, or external auth.
*/
export class DefaultAuthProvider implements IAuthProvider {
private tokenStore: Map<string, IAuthToken> = new Map();
private userCredentials: Map<string, string> = new Map(); // username -> password hash (mock)
constructor(private config: IAuthConfig) {}
/**
* Initialize the auth provider
*/
public async init(): Promise<void> {
// Initialize token store (in-memory for now)
// In production, this could be Redis or a database
}
// ========================================================================
// IAuthProvider Implementation
// ========================================================================
/**
* Authenticate user credentials
*/
public async authenticate(credentials: ICredentials): Promise<string | null> {
// Mock authentication - in production, verify against database/LDAP
const storedPassword = this.userCredentials.get(credentials.username);
if (!storedPassword) {
// Auto-register for testing (remove in production)
this.userCredentials.set(credentials.username, credentials.password);
return credentials.username;
}
if (storedPassword === credentials.password) {
return credentials.username;
}
return null;
}
/**
* Validate any token (NPM, Maven, OCI, PyPI, RubyGems, Composer, Cargo)
*/
public async validateToken(
tokenString: string,
protocol?: TRegistryProtocol
): Promise<IAuthToken | null> {
// OCI uses JWT (contains dots), not UUID - check first if OCI is expected
if (protocol === 'oci' || tokenString.includes('.')) {
const ociToken = await this.validateOciToken(tokenString);
if (ociToken && (!protocol || protocol === 'oci')) {
return ociToken;
}
// If protocol was explicitly OCI but validation failed, return null
if (protocol === 'oci') {
return null;
}
}
// UUID-based tokens: single O(1) Map lookup
if (this.isValidUuid(tokenString)) {
const authToken = this.tokenStore.get(tokenString);
if (authToken) {
// If protocol specified, verify it matches
if (protocol && authToken.type !== protocol) {
return null;
}
// Check expiration
if (authToken.expiresAt && authToken.expiresAt < new Date()) {
this.tokenStore.delete(tokenString);
return null;
}
return authToken;
}
}
return null;
}
/**
* Create a new token for a user
*/
public async createToken(
userId: string,
protocol: TRegistryProtocol,
options?: ITokenOptions
): Promise<string> {
// OCI tokens use JWT
if (protocol === 'oci') {
return this.createOciToken(userId, options?.scopes || ['oci:*:*:*'], options?.expiresIn || 3600);
}
// All other protocols use UUID tokens
const token = this.generateUuid();
const scopes = options?.scopes || (options?.readonly
? [`${protocol}:*:*:read`]
: [`${protocol}:*:*:*`]);
const authToken: IAuthToken = {
type: protocol,
userId,
scopes,
readonly: options?.readonly,
expiresAt: options?.expiresIn ? new Date(Date.now() + options.expiresIn * 1000) : undefined,
metadata: {
created: new Date().toISOString(),
},
};
this.tokenStore.set(token, authToken);
return token;
}
/**
* Revoke a token
*/
public async revokeToken(token: string): Promise<void> {
this.tokenStore.delete(token);
}
/**
* Check if token has permission for an action
*/
public async authorize(
token: IAuthToken | null,
resource: string,
action: string
): Promise<boolean> {
if (!token) {
return false;
}
// Check readonly flag
if (token.readonly && ['write', 'push', 'delete'].includes(action)) {
return false;
}
// Check scopes
for (const scope of token.scopes) {
if (this.matchesScope(scope, resource, action)) {
return true;
}
}
return false;
}
/**
* List all tokens for a user
*/
public async listUserTokens(userId: string): Promise<Array<{
key: string;
readonly: boolean;
created: string;
protocol?: TRegistryProtocol;
}>> {
const tokens: Array<{key: string; readonly: boolean; created: string; protocol?: TRegistryProtocol}> = [];
for (const [token, authToken] of this.tokenStore.entries()) {
if (authToken.userId === userId) {
tokens.push({
key: this.hashToken(token),
readonly: authToken.readonly || false,
created: authToken.metadata?.created || 'unknown',
protocol: authToken.type,
});
}
}
return tokens;
}
// ========================================================================
// OCI JWT Token Methods
// ========================================================================
/**
* Create an OCI JWT token
*/
private async createOciToken(
userId: string,
scopes: string[],
expiresIn: number = 3600
): Promise<string> {
if (!this.config.ociTokens?.enabled) {
throw new Error('OCI tokens are not enabled');
}
const now = Math.floor(Date.now() / 1000);
const payload = {
iss: this.config.ociTokens.realm,
sub: userId,
aud: this.config.ociTokens.service,
exp: now + expiresIn,
nbf: now,
iat: now,
access: this.scopesToOciAccess(scopes),
};
// Create JWT with HMAC-SHA256 signature
const header = { alg: 'HS256', typ: 'JWT' };
const headerB64 = Buffer.from(JSON.stringify(header)).toString('base64url');
const payloadB64 = Buffer.from(JSON.stringify(payload)).toString('base64url');
const signature = crypto
.createHmac('sha256', this.config.jwtSecret)
.update(`${headerB64}.${payloadB64}`)
.digest('base64url');
return `${headerB64}.${payloadB64}.${signature}`;
}
/**
* Validate an OCI JWT token
*/
private async validateOciToken(jwt: string): Promise<IAuthToken | null> {
try {
const parts = jwt.split('.');
if (parts.length !== 3) {
return null;
}
const [headerB64, payloadB64, signatureB64] = parts;
// Verify signature
const expectedSignature = crypto
.createHmac('sha256', this.config.jwtSecret)
.update(`${headerB64}.${payloadB64}`)
.digest('base64url');
if (signatureB64 !== expectedSignature) {
return null;
}
// Decode and parse payload
const payload = JSON.parse(Buffer.from(payloadB64, 'base64url').toString('utf-8'));
// Check expiration
const now = Math.floor(Date.now() / 1000);
if (payload.exp && payload.exp < now) {
return null;
}
// Check not-before time
if (payload.nbf && payload.nbf > now) {
return null;
}
// Convert to unified token format
const scopes = this.ociAccessToScopes(payload.access || []);
return {
type: 'oci',
userId: payload.sub,
scopes,
expiresAt: payload.exp ? new Date(payload.exp * 1000) : undefined,
metadata: {
iss: payload.iss,
aud: payload.aud,
},
};
} catch (error) {
return null;
}
}
// ========================================================================
// Helper Methods
// ========================================================================
/**
* Check if a scope matches a resource and action
*/
private matchesScope(scope: string, resource: string, action: string): boolean {
const scopeParts = scope.split(':');
const resourceParts = resource.split(':');
// Scope must have at least protocol:type:name:action
if (scopeParts.length < 4) {
return false;
}
const [scopeProtocol, scopeType, scopeName, scopeAction] = scopeParts;
const [resourceProtocol, resourceType, resourceName] = resourceParts;
// Check protocol
if (scopeProtocol !== '*' && scopeProtocol !== resourceProtocol) {
return false;
}
// Check type
if (scopeType !== '*' && scopeType !== resourceType) {
return false;
}
// Check name
if (scopeName !== '*' && scopeName !== resourceName) {
return false;
}
// Check action
if (scopeAction !== '*' && scopeAction !== action) {
// Map action aliases
const actionAliases: Record<string, string[]> = {
read: ['pull', 'get'],
write: ['push', 'put', 'post'],
};
const aliases = actionAliases[scopeAction] || [];
if (!aliases.includes(action)) {
return false;
}
}
return true;
}
/**
* Convert unified scopes to OCI access array
*/
private scopesToOciAccess(scopes: string[]): Array<{
type: string;
name: string;
actions: string[];
}> {
const access: Array<{type: string; name: string; actions: string[]}> = [];
for (const scope of scopes) {
const parts = scope.split(':');
if (parts.length >= 4 && parts[0] === 'oci') {
access.push({
type: parts[1],
name: parts[2],
actions: [parts[3]],
});
}
}
return access;
}
/**
* Convert OCI access array to unified scopes
*/
private ociAccessToScopes(access: Array<{
type: string;
name: string;
actions: string[];
}>): string[] {
const scopes: string[] = [];
for (const item of access) {
for (const action of item.actions) {
scopes.push(`oci:${item.type}:${item.name}:${action}`);
}
}
return scopes;
}
/**
* Generate UUID for tokens
*/
private generateUuid(): string {
return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => {
const r = (Math.random() * 16) | 0;
const v = c === 'x' ? r : (r & 0x3) | 0x8;
return v.toString(16);
});
}
/**
* Check if string is a valid UUID
*/
private isValidUuid(str: string): boolean {
const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
return uuidRegex.test(str);
}
/**
* Hash a token for identification
*/
private hashToken(token: string): string {
return `sha512-${token.substring(0, 16)}...`;
}
}

204
ts/core/classes.registrystorage.ts
View File

@@ -1,17 +1,54 @@
import * as plugins from '../plugins.js';
import type { IStorageConfig, IStorageBackend } from './interfaces.core.js';
import type { IStorageConfig, IStorageBackend, TRegistryProtocol } from './interfaces.core.js';
import type {
IStorageHooks,
IStorageHookContext,
IStorageActor,
IStorageMetadata,
} from './interfaces.storage.js';
/**
* Storage abstraction layer for registry
* Provides a unified interface over SmartBucket
* Storage abstraction layer for registry.
* Provides a unified interface over SmartBucket with optional hooks
* for quota tracking, audit logging, cache invalidation, etc.
*
* @example
* ```typescript
* // Basic usage
* const storage = new RegistryStorage(config);
*
* // With hooks for quota tracking
* const storage = new RegistryStorage(config, {
* beforePut: async (ctx) => {
* const quota = await getQuota(ctx.actor?.orgId);
* const usage = await getUsage(ctx.actor?.orgId);
* if (usage + (ctx.metadata?.size || 0) > quota) {
* return { allowed: false, reason: 'Quota exceeded' };
* }
* return { allowed: true };
* },
* afterPut: async (ctx) => {
* await updateUsage(ctx.actor?.orgId, ctx.metadata?.size || 0);
* }
* });
* ```
*/
export class RegistryStorage implements IStorageBackend {
private smartBucket: plugins.smartbucket.SmartBucket;
private bucket: plugins.smartbucket.Bucket;
private bucketName: string;
private hooks?: IStorageHooks;
constructor(private config: IStorageConfig) {
constructor(private config: IStorageConfig, hooks?: IStorageHooks) {
this.bucketName = config.bucketName;
this.hooks = hooks;
}
/**
* Set storage hooks (can be called after construction)
*/
public setHooks(hooks: IStorageHooks): void {
this.hooks = hooks;
}
/**
@@ -34,7 +71,24 @@ export class RegistryStorage implements IStorageBackend {
*/
public async getObject(key: string): Promise<Buffer | null> {
try {
return await this.bucket.fastGet({ path: key });
const data = await this.bucket.fastGet({ path: key });
// Call afterGet hook (non-blocking)
if (this.hooks?.afterGet && data) {
const context = this.currentContext;
if (context) {
this.hooks.afterGet({
operation: 'get',
key,
protocol: context.protocol,
actor: context.actor,
metadata: context.metadata,
timestamp: new Date(),
}).catch(() => {}); // Don't fail on hook errors
}
}
return data;
} catch (error) {
return null;
}
@@ -48,19 +102,159 @@ export class RegistryStorage implements IStorageBackend {
data: Buffer,
metadata?: Record<string, string>
): Promise<void> {
// Call beforePut hook if available
if (this.hooks?.beforePut) {
const context = this.currentContext;
if (context) {
const hookContext: IStorageHookContext = {
operation: 'put',
key,
protocol: context.protocol,
actor: context.actor,
metadata: {
...context.metadata,
size: data.length,
},
timestamp: new Date(),
};
const result = await this.hooks.beforePut(hookContext);
if (!result.allowed) {
throw new Error(result.reason || 'Storage operation denied by hook');
}
}
}
// Note: SmartBucket doesn't support metadata yet
await this.bucket.fastPut({
path: key,
contents: data,
overwrite: true, // Always overwrite existing objects
});
// Call afterPut hook (non-blocking)
if (this.hooks?.afterPut) {
const context = this.currentContext;
if (context) {
this.hooks.afterPut({
operation: 'put',
key,
protocol: context.protocol,
actor: context.actor,
metadata: {
...context.metadata,
size: data.length,
},
timestamp: new Date(),
}).catch(() => {}); // Don't fail on hook errors
}
}
}
/**
* Delete an object
*/
public async deleteObject(key: string): Promise<void> {
// Call beforeDelete hook if available
if (this.hooks?.beforeDelete) {
const context = this.currentContext;
if (context) {
const hookContext: IStorageHookContext = {
operation: 'delete',
key,
protocol: context.protocol,
actor: context.actor,
metadata: context.metadata,
timestamp: new Date(),
};
const result = await this.hooks.beforeDelete(hookContext);
if (!result.allowed) {
throw new Error(result.reason || 'Delete operation denied by hook');
}
}
}
await this.bucket.fastRemove({ path: key });
// Call afterDelete hook (non-blocking)
if (this.hooks?.afterDelete) {
const context = this.currentContext;
if (context) {
this.hooks.afterDelete({
operation: 'delete',
key,
protocol: context.protocol,
actor: context.actor,
metadata: context.metadata,
timestamp: new Date(),
}).catch(() => {}); // Don't fail on hook errors
}
}
}
// ========================================================================
// CONTEXT FOR HOOKS
// ========================================================================
/**
* Current operation context for hooks.
* Set this before performing storage operations to enable hooks.
*/
private currentContext?: {
protocol: TRegistryProtocol;
actor?: IStorageActor;
metadata?: IStorageMetadata;
};
/**
* Set the current operation context for hooks.
* Call this before performing storage operations.
*
* @example
* ```typescript
* storage.setContext({
* protocol: 'npm',
* actor: { userId: 'user123', ip: '192.168.1.1' },
* metadata: { packageName: 'lodash', version: '4.17.21' }
* });
* await storage.putNpmTarball('lodash', '4.17.21', tarball);
* storage.clearContext();
* ```
*/
public setContext(context: {
protocol: TRegistryProtocol;
actor?: IStorageActor;
metadata?: IStorageMetadata;
}): void {
this.currentContext = context;
}
/**
* Clear the current operation context.
*/
public clearContext(): void {
this.currentContext = undefined;
}
/**
* Execute a function with a temporary context.
* Context is automatically cleared after execution.
*/
public async withContext<T>(
context: {
protocol: TRegistryProtocol;
actor?: IStorageActor;
metadata?: IStorageMetadata;
},
fn: () => Promise<T>
): Promise<T> {
this.setContext(context);
try {
return await fn();
} finally {
this.clearContext();
}
}
/**

9
ts/core/index.ts
View File

@@ -2,9 +2,16 @@
* Core registry infrastructure exports
*/
// Interfaces
// Core interfaces
export * from './interfaces.core.js';
// Auth interfaces and provider
export * from './interfaces.auth.js';
export { DefaultAuthProvider } from './classes.defaultauthprovider.js';
// Storage interfaces and hooks
export * from './interfaces.storage.js';
// Classes
export { BaseRegistry } from './classes.baseregistry.js';
export { RegistryStorage } from './classes.registrystorage.js';

91
ts/core/interfaces.auth.ts Normal file
View File

@@ -0,0 +1,91 @@
import type { IAuthToken, ICredentials, TRegistryProtocol } from './interfaces.core.js';
/**
* Options for creating a token
*/
export interface ITokenOptions {
/** Whether the token is readonly */
readonly?: boolean;
/** Permission scopes */
scopes?: string[];
/** Expiration time in seconds */
expiresIn?: number;
}
/**
* Pluggable authentication provider interface.
* Implement this to integrate external auth systems (LDAP, OAuth, SSO, OIDC).
*
* @example
* ```typescript
* class LdapAuthProvider implements IAuthProvider {
* constructor(private ldap: LdapClient, private redis: RedisClient) {}
*
* async authenticate(credentials: ICredentials): Promise<string | null> {
* return await this.ldap.bind(credentials.username, credentials.password);
* }
*
* async validateToken(token: string): Promise<IAuthToken | null> {
* return await this.redis.get(`token:${token}`);
* }
* // ...
* }
* ```
*/
export interface IAuthProvider {
/**
* Initialize the auth provider (optional)
*/
init?(): Promise<void>;
/**
* Authenticate user credentials (login flow)
* @param credentials - Username and password
* @returns User ID on success, null on failure
*/
authenticate(credentials: ICredentials): Promise<string | null>;
/**
* Validate an existing token
* @param token - Token string (UUID or JWT)
* @param protocol - Optional protocol hint for optimization
* @returns Auth token info or null if invalid
*/
validateToken(token: string, protocol?: TRegistryProtocol): Promise<IAuthToken | null>;
/**
* Create a new token for a user
* @param userId - User ID
* @param protocol - Protocol type (npm, oci, maven, etc.)
* @param options - Token options (readonly, scopes, expiration)
* @returns Token string
*/
createToken(userId: string, protocol: TRegistryProtocol, options?: ITokenOptions): Promise<string>;
/**
* Revoke a token
* @param token - Token string to revoke
*/
revokeToken(token: string): Promise<void>;
/**
* Check if user has permission for an action
* @param token - Auth token (or null for anonymous)
* @param resource - Resource being accessed (e.g., "npm:package:lodash")
* @param action - Action being performed (read, write, push, pull, delete)
* @returns true if authorized
*/
authorize(token: IAuthToken | null, resource: string, action: string): Promise<boolean>;
/**
* List all tokens for a user (optional)
* @param userId - User ID
* @returns List of token info
*/
listUserTokens?(userId: string): Promise<Array<{
key: string;
readonly: boolean;
created: string;
protocol?: TRegistryProtocol;
}>>;
}

39
ts/core/interfaces.core.ts
View File

@@ -4,6 +4,8 @@
import type * as plugins from '../plugins.js';
import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js';
import type { IAuthProvider } from './interfaces.auth.js';
import type { IStorageHooks } from './interfaces.storage.js';
/**
* Registry protocol types
@@ -97,6 +99,20 @@ export interface IProtocolConfig {
export interface IRegistryConfig {
storage: IStorageConfig;
auth: IAuthConfig;
/**
* Custom authentication provider.
* If not provided, uses the default in-memory auth provider.
* Implement IAuthProvider to integrate LDAP, OAuth, SSO, etc.
*/
authProvider?: IAuthProvider;
/**
* Storage event hooks for quota tracking, audit logging, etc.
* Called before/after storage operations.
*/
storageHooks?: IStorageHooks;
oci?: IProtocolConfig;
npm?: IProtocolConfig;
maven?: IProtocolConfig;
@@ -152,6 +168,24 @@ export interface IRegistryError {
}>;
}
/**
* Actor information - identifies who is performing the request
*/
export interface IRequestActor {
/** User ID (from validated token) */
userId?: string;
/** Token ID/hash for audit purposes */
tokenId?: string;
/** Client IP address */
ip?: string;
/** Client User-Agent */
userAgent?: string;
/** Organization ID (for multi-tenant setups) */
orgId?: string;
/** Session ID */
sessionId?: string;
}
/**
* Base request context
*/
@@ -168,6 +202,11 @@ export interface IRequestContext {
*/
rawBody?: Buffer;
token?: string;
/**
* Actor information - identifies who is performing the request.
* Populated after authentication for audit logging, quota enforcement, etc.
*/
actor?: IRequestActor;
}
/**

130
ts/core/interfaces.storage.ts Normal file
View File

@@ -0,0 +1,130 @@
import type { TRegistryProtocol } from './interfaces.core.js';
/**
* Actor information from request context
*/
export interface IStorageActor {
userId?: string;
tokenId?: string;
ip?: string;
userAgent?: string;
orgId?: string;
sessionId?: string;
}
/**
* Metadata about the storage operation
*/
export interface IStorageMetadata {
/** Content type of the object */
contentType?: string;
/** Size in bytes */
size?: number;
/** Content digest (e.g., sha256:abc123) */
digest?: string;
/** Package/artifact name */
packageName?: string;
/** Version */
version?: string;
}
/**
* Context passed to storage hooks
*/
export interface IStorageHookContext {
/** Type of operation */
operation: 'put' | 'delete' | 'get';
/** Storage key/path */
key: string;
/** Protocol that triggered this operation */
protocol: TRegistryProtocol;
/** Actor who performed the operation (if known) */
actor?: IStorageActor;
/** Metadata about the object */
metadata?: IStorageMetadata;
/** Timestamp of the operation */
timestamp: Date;
}
/**
* Result from a beforePut hook that can modify the operation
*/
export interface IBeforePutResult {
/** Whether to allow the operation */
allowed: boolean;
/** Optional reason for rejection */
reason?: string;
/** Optional modified metadata */
metadata?: IStorageMetadata;
}
/**
* Result from a beforeDelete hook
*/
export interface IBeforeDeleteResult {
/** Whether to allow the operation */
allowed: boolean;
/** Optional reason for rejection */
reason?: string;
}
/**
* Storage event hooks for quota tracking, audit logging, cache invalidation, etc.
*
* @example
* ```typescript
* const quotaHooks: IStorageHooks = {
* async beforePut(context) {
* const quota = await getQuota(context.actor?.orgId);
* const currentUsage = await getUsage(context.actor?.orgId);
* if (currentUsage + (context.metadata?.size || 0) > quota) {
* return { allowed: false, reason: 'Quota exceeded' };
* }
* return { allowed: true };
* },
*
* async afterPut(context) {
* await updateUsage(context.actor?.orgId, context.metadata?.size || 0);
* await auditLog('storage.put', context);
* },
*
* async afterDelete(context) {
* await invalidateCache(context.key);
* }
* };
* ```
*/
export interface IStorageHooks {
/**
* Called before storing an object.
* Return { allowed: false } to reject the operation.
* Use for quota checks, virus scanning, validation, etc.
*/
beforePut?(context: IStorageHookContext): Promise<IBeforePutResult>;
/**
* Called after successfully storing an object.
* Use for quota tracking, audit logging, notifications, etc.
*/
afterPut?(context: IStorageHookContext): Promise<void>;
/**
* Called before deleting an object.
* Return { allowed: false } to reject the operation.
* Use for preventing deletion of protected resources.
*/
beforeDelete?(context: IStorageHookContext): Promise<IBeforeDeleteResult>;
/**
* Called after successfully deleting an object.
* Use for quota updates, audit logging, cache invalidation, etc.
*/
afterDelete?(context: IStorageHookContext): Promise<void>;
/**
* Called after reading an object.
* Use for access logging, analytics, etc.
* Note: This is called even for cache hits.
*/
afterGet?(context: IStorageHookContext): Promise<void>;
}

35
ts/upstream/classes.baseupstream.ts
View File

@@ -110,8 +110,18 @@ export abstract class BaseUpstream {
return null;
}
// Get applicable upstreams sorted by priority
const applicableUpstreams = this.getApplicableUpstreams(context.resource);
if (applicableUpstreams.length === 0) {
return null;
}
// Use the first applicable upstream's URL for cache key
const primaryUpstreamUrl = applicableUpstreams[0]?.url;
// Check cache first
const cached = this.cache.get(context);
const cached = await this.cache.get(context, primaryUpstreamUrl);
if (cached && !cached.stale) {
return {
success: true,
@@ -125,7 +135,7 @@ export abstract class BaseUpstream {
}
// Check for negative cache (recent 404)
if (this.cache.hasNegative(context)) {
if (await this.cache.hasNegative(context, primaryUpstreamUrl)) {
return {
success: false,
status: 404,
@@ -136,13 +146,6 @@ export abstract class BaseUpstream {
};
}
// Get applicable upstreams sorted by priority
const applicableUpstreams = this.getApplicableUpstreams(context.resource);
if (applicableUpstreams.length === 0) {
return null;
}
// If we have stale cache, return it immediately and revalidate in background
if (cached?.stale && this.cacheConfig.staleWhileRevalidate) {
// Fire and forget revalidation
@@ -173,18 +176,19 @@ export abstract class BaseUpstream {
// Cache successful responses
if (result.success && result.body) {
this.cache.set(
await this.cache.set(
context,
Buffer.isBuffer(result.body) ? result.body : Buffer.from(JSON.stringify(result.body)),
result.headers['content-type'] || 'application/octet-stream',
result.headers,
upstream.id,
upstream.url,
);
}
// Cache 404 responses
if (result.status === 404) {
this.cache.setNegative(context, upstream.id);
await this.cache.setNegative(context, upstream.id, upstream.url);
}
return result;
@@ -210,15 +214,15 @@ export abstract class BaseUpstream {
/**
* Invalidate cache for a resource pattern.
*/
public invalidateCache(pattern: RegExp): number {
public async invalidateCache(pattern: RegExp): Promise<number> {
return this.cache.invalidatePattern(pattern);
}
/**
* Clear all cache entries.
*/
public clearCache(): void {
this.cache.clear();
public async clearCache(): Promise<void> {
await this.cache.clear();
}
/**
@@ -501,12 +505,13 @@ export abstract class BaseUpstream {
);
if (result.success && result.body) {
this.cache.set(
await this.cache.set(
context,
Buffer.isBuffer(result.body) ? result.body : Buffer.from(JSON.stringify(result.body)),
result.headers['content-type'] || 'application/octet-stream',
result.headers,
upstream.id,
upstream.url,
);
return; // Successfully revalidated
}

309
ts/upstream/classes.upstreamcache.ts
View File

@@ -4,9 +4,23 @@ import type {
IUpstreamFetchContext,
} from './interfaces.upstream.js';
import { DEFAULT_CACHE_CONFIG } from './interfaces.upstream.js';
import type { IStorageBackend } from '../core/interfaces.core.js';
/**
* In-memory cache for upstream responses.
* Cache metadata stored alongside cache entries.
*/
interface ICacheMetadata {
contentType: string;
headers: Record<string, string>;
cachedAt: string;
expiresAt?: string;
etag?: string;
upstreamId: string;
upstreamUrl: string;
}
/**
* S3-backed upstream cache with in-memory hot layer.
*
* Features:
* - TTL-based expiration
@@ -14,26 +28,45 @@ import { DEFAULT_CACHE_CONFIG } from './interfaces.upstream.js';
* - Negative caching (404s)
* - Content-type aware caching
* - ETag support for conditional requests
* - Multi-upstream support via URL-based cache paths
* - Persistent S3 storage with in-memory hot layer
*
* Note: This is an in-memory implementation. For production with persistence,
* extend this class to use RegistryStorage for S3-backed caching.
* Cache paths are structured as:
* cache/{escaped-upstream-url}/{protocol}:{method}:{path}
*
* @example
* ```typescript
* // In-memory only (default)
* const cache = new UpstreamCache(config);
*
* // With S3 persistence
* const cache = new UpstreamCache(config, 10000, storage);
* ```
*/
export class UpstreamCache {
/** Cache storage */
private readonly cache: Map<string, ICacheEntry> = new Map();
/** In-memory hot cache */
private readonly memoryCache: Map<string, ICacheEntry> = new Map();
/** Configuration */
private readonly config: IUpstreamCacheConfig;
/** Maximum cache entries (prevents memory bloat) */
private readonly maxEntries: number;
/** Maximum in-memory cache entries */
private readonly maxMemoryEntries: number;
/** S3 storage backend (optional) */
private readonly storage?: IStorageBackend;
/** Cleanup interval handle */
private cleanupInterval: ReturnType<typeof setInterval> | null = null;
constructor(config?: Partial<IUpstreamCacheConfig>, maxEntries: number = 10000) {
constructor(
config?: Partial<IUpstreamCacheConfig>,
maxMemoryEntries: number = 10000,
storage?: IStorageBackend
) {
this.config = { ...DEFAULT_CACHE_CONFIG, ...config };
this.maxEntries = maxEntries;
this.maxMemoryEntries = maxMemoryEntries;
this.storage = storage;
// Start periodic cleanup if caching is enabled
if (this.config.enabled) {
@@ -48,17 +81,36 @@ export class UpstreamCache {
return this.config.enabled;
}
/**
* Check if S3 storage is configured.
*/
public hasStorage(): boolean {
return !!this.storage;
}
/**
* Get cached entry for a request context.
* Checks memory first, then falls back to S3.
* Returns null if not found or expired (unless stale-while-revalidate).
*/
public get(context: IUpstreamFetchContext): ICacheEntry | null {
public async get(context: IUpstreamFetchContext, upstreamUrl?: string): Promise<ICacheEntry | null> {
if (!this.config.enabled) {
return null;
}
const key = this.buildCacheKey(context);
const entry = this.cache.get(key);
const key = this.buildCacheKey(context, upstreamUrl);
// Check memory cache first
let entry = this.memoryCache.get(key);
// If not in memory and we have storage, check S3
if (!entry && this.storage) {
entry = await this.loadFromStorage(key);
if (entry) {
// Promote to memory cache
this.memoryCache.set(key, entry);
}
}
if (!entry) {
return null;
@@ -78,7 +130,10 @@ export class UpstreamCache {
}
}
// Entry is too old, remove it
this.cache.delete(key);
this.memoryCache.delete(key);
if (this.storage) {
await this.deleteFromStorage(key).catch(() => {});
}
return null;
}
@@ -86,26 +141,27 @@ export class UpstreamCache {
}
/**
* Store a response in the cache.
* Store a response in the cache (memory and optionally S3).
*/
public set(
public async set(
context: IUpstreamFetchContext,
data: Buffer,
contentType: string,
headers: Record<string, string>,
upstreamId: string,
upstreamUrl: string,
options?: ICacheSetOptions,
): void {
): Promise<void> {
if (!this.config.enabled) {
return;
}
// Enforce max entries limit
if (this.cache.size >= this.maxEntries) {
// Enforce max memory entries limit
if (this.memoryCache.size >= this.maxMemoryEntries) {
this.evictOldest();
}
const key = this.buildCacheKey(context);
const key = this.buildCacheKey(context, upstreamUrl);
const now = new Date();
// Determine TTL based on content type
@@ -122,18 +178,24 @@ export class UpstreamCache {
stale: false,
};
this.cache.set(key, entry);
// Store in memory
this.memoryCache.set(key, entry);
// Store in S3 if available
if (this.storage) {
await this.saveToStorage(key, entry, upstreamUrl).catch(() => {});
}
}
/**
* Store a negative cache entry (404 response).
*/
public setNegative(context: IUpstreamFetchContext, upstreamId: string): void {
public async setNegative(context: IUpstreamFetchContext, upstreamId: string, upstreamUrl: string): Promise<void> {
if (!this.config.enabled || this.config.negativeCacheTtlSeconds <= 0) {
return;
}
const key = this.buildCacheKey(context);
const key = this.buildCacheKey(context, upstreamUrl);
const now = new Date();
const entry: ICacheEntry = {
@@ -146,34 +208,47 @@ export class UpstreamCache {
stale: false,
};
this.cache.set(key, entry);
this.memoryCache.set(key, entry);
if (this.storage) {
await this.saveToStorage(key, entry, upstreamUrl).catch(() => {});
}
}
/**
* Check if there's a negative cache entry for this context.
*/
public hasNegative(context: IUpstreamFetchContext): boolean {
const entry = this.get(context);
public async hasNegative(context: IUpstreamFetchContext, upstreamUrl?: string): Promise<boolean> {
const entry = await this.get(context, upstreamUrl);
return entry !== null && entry.data.length === 0;
}
/**
* Invalidate a specific cache entry.
*/
public invalidate(context: IUpstreamFetchContext): boolean {
const key = this.buildCacheKey(context);
return this.cache.delete(key);
public async invalidate(context: IUpstreamFetchContext, upstreamUrl?: string): Promise<boolean> {
const key = this.buildCacheKey(context, upstreamUrl);
const deleted = this.memoryCache.delete(key);
if (this.storage) {
await this.deleteFromStorage(key).catch(() => {});
}
return deleted;
}
/**
* Invalidate all entries matching a pattern.
* Useful for invalidating all versions of a package.
*/
public invalidatePattern(pattern: RegExp): number {
public async invalidatePattern(pattern: RegExp): Promise<number> {
let count = 0;
for (const key of this.cache.keys()) {
for (const key of this.memoryCache.keys()) {
if (pattern.test(key)) {
this.cache.delete(key);
this.memoryCache.delete(key);
if (this.storage) {
await this.deleteFromStorage(key).catch(() => {});
}
count++;
}
}
@@ -183,11 +258,14 @@ export class UpstreamCache {
/**
* Invalidate all entries from a specific upstream.
*/
public invalidateUpstream(upstreamId: string): number {
public async invalidateUpstream(upstreamId: string): Promise<number> {
let count = 0;
for (const [key, entry] of this.cache.entries()) {
for (const [key, entry] of this.memoryCache.entries()) {
if (entry.upstreamId === upstreamId) {
this.cache.delete(key);
this.memoryCache.delete(key);
if (this.storage) {
await this.deleteFromStorage(key).catch(() => {});
}
count++;
}
}
@@ -195,10 +273,13 @@ export class UpstreamCache {
}
/**
* Clear all cache entries.
* Clear all cache entries (memory and S3).
*/
public clear(): void {
this.cache.clear();
public async clear(): Promise<void> {
this.memoryCache.clear();
// Note: S3 cleanup would require listing and deleting all cache/* objects
// This is left as a future enhancement for bulk cleanup
}
/**
@@ -211,7 +292,7 @@ export class UpstreamCache {
let totalSize = 0;
const now = new Date();
for (const entry of this.cache.values()) {
for (const entry of this.memoryCache.values()) {
totalSize += entry.data.length;
if (entry.data.length === 0) {
@@ -224,13 +305,14 @@ export class UpstreamCache {
}
return {
totalEntries: this.cache.size,
totalEntries: this.memoryCache.size,
freshEntries: freshCount,
staleEntries: staleCount,
negativeEntries: negativeCount,
totalSizeBytes: totalSize,
maxEntries: this.maxEntries,
maxEntries: this.maxMemoryEntries,
enabled: this.config.enabled,
hasStorage: !!this.storage,
};
}
@@ -244,17 +326,136 @@ export class UpstreamCache {
}
}
// ========================================================================
// Storage Methods
// ========================================================================
/**
* Build storage path for a cache key.
* Escapes upstream URL for safe use in S3 paths.
*/
private buildStoragePath(key: string): string {
return `cache/${key}`;
}
/**
* Build storage path for cache metadata.
*/
private buildMetadataPath(key: string): string {
return `cache/${key}.meta`;
}
/**
* Load a cache entry from S3 storage.
*/
private async loadFromStorage(key: string): Promise<ICacheEntry | null> {
if (!this.storage) return null;
try {
const dataPath = this.buildStoragePath(key);
const metaPath = this.buildMetadataPath(key);
// Load data and metadata in parallel
const [data, metaBuffer] = await Promise.all([
this.storage.getObject(dataPath),
this.storage.getObject(metaPath),
]);
if (!data || !metaBuffer) {
return null;
}
const meta: ICacheMetadata = JSON.parse(metaBuffer.toString('utf-8'));
return {
data,
contentType: meta.contentType,
headers: meta.headers,
cachedAt: new Date(meta.cachedAt),
expiresAt: meta.expiresAt ? new Date(meta.expiresAt) : undefined,
etag: meta.etag,
upstreamId: meta.upstreamId,
stale: false,
};
} catch {
return null;
}
}
/**
* Save a cache entry to S3 storage.
*/
private async saveToStorage(key: string, entry: ICacheEntry, upstreamUrl: string): Promise<void> {
if (!this.storage) return;
const dataPath = this.buildStoragePath(key);
const metaPath = this.buildMetadataPath(key);
const meta: ICacheMetadata = {
contentType: entry.contentType,
headers: entry.headers,
cachedAt: entry.cachedAt.toISOString(),
expiresAt: entry.expiresAt?.toISOString(),
etag: entry.etag,
upstreamId: entry.upstreamId,
upstreamUrl,
};
// Save data and metadata in parallel
await Promise.all([
this.storage.putObject(dataPath, entry.data),
this.storage.putObject(metaPath, Buffer.from(JSON.stringify(meta), 'utf-8')),
]);
}
/**
* Delete a cache entry from S3 storage.
*/
private async deleteFromStorage(key: string): Promise<void> {
if (!this.storage) return;
const dataPath = this.buildStoragePath(key);
const metaPath = this.buildMetadataPath(key);
await Promise.all([
this.storage.deleteObject(dataPath).catch(() => {}),
this.storage.deleteObject(metaPath).catch(() => {}),
]);
}
// ========================================================================
// Helper Methods
// ========================================================================
/**
* Escape a URL for safe use in storage paths.
*/
private escapeUrl(url: string): string {
// Remove protocol prefix and escape special characters
return url
.replace(/^https?:\/\//, '')
.replace(/[\/\\:*?"<>|]/g, '_')
.replace(/__+/g, '_');
}
/**
* Build a unique cache key for a request context.
* Includes escaped upstream URL for multi-upstream support.
*/
private buildCacheKey(context: IUpstreamFetchContext): string {
private buildCacheKey(context: IUpstreamFetchContext, upstreamUrl?: string): string {
// Include method, protocol, path, and sorted query params
const queryString = Object.keys(context.query)
.sort()
.map(k => `${k}=${context.query[k]}`)
.join('&');
return `${context.protocol}:${context.method}:${context.path}${queryString ? '?' + queryString : ''}`;
const baseKey = `${context.protocol}:${context.method}:${context.path}${queryString ? '?' + queryString : ''}`;
if (upstreamUrl) {
return `${this.escapeUrl(upstreamUrl)}/${baseKey}`;
}
return baseKey;
}
/**
@@ -333,27 +534,27 @@ export class UpstreamCache {
*/
private evictOldest(): void {
// Evict 10% of max entries
const evictCount = Math.ceil(this.maxEntries * 0.1);
const evictCount = Math.ceil(this.maxMemoryEntries * 0.1);
let evicted = 0;
// First, try to evict stale entries
const now = new Date();
for (const [key, entry] of this.cache.entries()) {
for (const [key, entry] of this.memoryCache.entries()) {
if (evicted >= evictCount) break;
if (entry.stale || (entry.expiresAt && entry.expiresAt < now)) {
this.cache.delete(key);
this.memoryCache.delete(key);
evicted++;
}
}
// If not enough evicted, evict oldest by cachedAt
if (evicted < evictCount) {
const entries = Array.from(this.cache.entries())
const entries = Array.from(this.memoryCache.entries())
.sort((a, b) => a[1].cachedAt.getTime() - b[1].cachedAt.getTime());
for (const [key] of entries) {
if (evicted >= evictCount) break;
this.cache.delete(key);
this.memoryCache.delete(key);
evicted++;
}
}
@@ -375,17 +576,17 @@ export class UpstreamCache {
}
/**
* Remove all expired entries.
* Remove all expired entries from memory cache.
*/
private cleanup(): void {
const now = new Date();
const staleDeadline = new Date(now.getTime() - this.config.staleMaxAgeSeconds * 1000);
for (const [key, entry] of this.cache.entries()) {
for (const [key, entry] of this.memoryCache.entries()) {
if (entry.expiresAt) {
// Remove if past stale deadline
if (entry.expiresAt < staleDeadline) {
this.cache.delete(key);
this.memoryCache.delete(key);
}
}
}
@@ -406,7 +607,7 @@ export interface ICacheSetOptions {
* Cache statistics.
*/
export interface ICacheStats {
/** Total number of cached entries */
/** Total number of cached entries in memory */
totalEntries: number;
/** Number of fresh (non-expired) entries */
freshEntries: number;
@@ -414,10 +615,12 @@ export interface ICacheStats {
staleEntries: number;
/** Number of negative cache entries */
negativeEntries: number;
/** Total size of cached data in bytes */
/** Total size of cached data in bytes (memory only) */
totalSizeBytes: number;
/** Maximum allowed entries */
/** Maximum allowed memory entries */
maxEntries: number;
/** Whether caching is enabled */
enabled: boolean;
/** Whether S3 storage is configured */
hasStorage: boolean;
}