push.rocks/smartregistry
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(upstream): Add dynamic per-request upstream provider and integrate into registries

Browse Source
This commit is contained in:
Jürgen Kunz
2025-12-03 22:16:40 +00:00
parent 351680159b
commit e9af3f8328
14 changed files with 1117 additions and 287 deletions
Download Patch File Download Diff File Expand all files Collapse all files

143
ts/pypi/classes.pypiregistry.ts
View File

@@ -2,8 +2,8 @@ import { Smartlog } from '@push.rocks/smartlog';
import { BaseRegistry } from '../core/classes.baseregistry.js';
import { RegistryStorage } from '../core/classes.registrystorage.js';
import { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js';
import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js';
import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from '../core/interfaces.core.js';
import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
import { isBinaryData, toBuffer } from '../core/helpers.buffer.js';
import type {
IPypiPackageMetadata,
@@ -24,20 +24,21 @@ export class PypiRegistry extends BaseRegistry {
private basePath: string = '/pypi';
private registryUrl: string;
private logger: Smartlog;
private upstream: PypiUpstream | null = null;
private upstreamProvider: IUpstreamProvider | null = null;
constructor(
storage: RegistryStorage,
authManager: AuthManager,
basePath: string = '/pypi',
registryUrl: string = 'http://localhost:5000',
upstreamConfig?: IProtocolUpstreamConfig
upstreamProvider?: IUpstreamProvider
) {
super();
this.storage = storage;
this.authManager = authManager;
this.basePath = basePath;
this.registryUrl = registryUrl;
this.upstreamProvider = upstreamProvider || null;
// Initialize logger
this.logger = new Smartlog({
@@ -51,20 +52,38 @@ export class PypiRegistry extends BaseRegistry {
}
});
this.logger.enableConsole();
}
// Initialize upstream if configured
if (upstreamConfig?.enabled) {
this.upstream = new PypiUpstream(upstreamConfig, registryUrl, this.logger);
}
/**
* Get upstream for a specific request.
* Calls the provider to resolve upstream config dynamically.
*/
private async getUpstreamForRequest(
resource: string,
resourceType: string,
method: string,
actor?: IRequestActor
): Promise<PypiUpstream | null> {
if (!this.upstreamProvider) return null;
const config = await this.upstreamProvider.resolveUpstreamConfig({
protocol: 'pypi',
resource,
scope: resource, // For PyPI, package name is the scope
actor,
method,
resourceType,
});
if (!config?.enabled) return null;
return new PypiUpstream(config, this.registryUrl, this.logger);
}
/**
* Clean up resources (timers, connections, etc.)
*/
public destroy(): void {
if (this.upstream) {
this.upstream.stop();
}
// No persistent upstream to clean up with dynamic provider
}
public async init(): Promise<void> {
@@ -84,15 +103,23 @@ export class PypiRegistry extends BaseRegistry {
public async handleRequest(context: IRequestContext): Promise<IResponse> {
let path = context.path.replace(this.basePath, '');
// Extract token (Basic Auth or Bearer)
const token = await this.extractToken(context);
// Build actor from context and validated token
const actor: IRequestActor = {
...context.actor,
userId: token?.userId,
ip: context.headers['x-forwarded-for'] || context.headers['X-Forwarded-For'],
userAgent: context.headers['user-agent'] || context.headers['User-Agent'],
};
// Also handle /simple path prefix
if (path.startsWith('/simple')) {
path = path.replace('/simple', '');
return this.handleSimpleRequest(path, context);
return this.handleSimpleRequest(path, context, actor);
}
// Extract token (Basic Auth or Bearer)
const token = await this.extractToken(context);
this.logger.log('debug', `handleRequest: ${context.method} ${path}`, {
method: context.method,
path,
@@ -119,7 +146,7 @@ export class PypiRegistry extends BaseRegistry {
// Package file download: GET /packages/{package}/{filename}
const downloadMatch = path.match(/^\/packages\/([^\/]+)\/(.+)$/);
if (downloadMatch && context.method === 'GET') {
return this.handleDownload(downloadMatch[1], downloadMatch[2]);
return this.handleDownload(downloadMatch[1], downloadMatch[2], actor);
}
// Delete package: DELETE /packages/{package}
@@ -156,7 +183,7 @@ export class PypiRegistry extends BaseRegistry {
/**
* Handle Simple API requests (PEP 503 HTML or PEP 691 JSON)
*/
private async handleSimpleRequest(path: string, context: IRequestContext): Promise<IResponse> {
private async handleSimpleRequest(path: string, context: IRequestContext, actor?: IRequestActor): Promise<IResponse> {
// Ensure path ends with / (PEP 503 requirement)
if (!path.endsWith('/') && !path.includes('.')) {
return {
@@ -174,7 +201,7 @@ export class PypiRegistry extends BaseRegistry {
// Package index: /simple/{package}/
const packageMatch = path.match(/^\/([^\/]+)\/$/);
if (packageMatch) {
return this.handleSimplePackage(packageMatch[1], context);
return this.handleSimplePackage(packageMatch[1], context, actor);
}
return {
@@ -228,46 +255,49 @@ export class PypiRegistry extends BaseRegistry {
* Handle Simple API package index
* Returns HTML (PEP 503) or JSON (PEP 691) based on Accept header
*/
private async handleSimplePackage(packageName: string, context: IRequestContext): Promise<IResponse> {
private async handleSimplePackage(packageName: string, context: IRequestContext, actor?: IRequestActor): Promise<IResponse> {
const normalized = helpers.normalizePypiPackageName(packageName);
// Get package metadata
let metadata = await this.storage.getPypiPackageMetadata(normalized);
// Try upstream if not found locally
if (!metadata && this.upstream) {
const upstreamHtml = await this.upstream.fetchSimplePackage(normalized);
if (upstreamHtml) {
// Parse the HTML to extract file information and cache it
// For now, just return the upstream HTML directly (caching can be improved later)
const acceptHeader = context.headers['accept'] || context.headers['Accept'] || '';
const preferJson = acceptHeader.includes('application/vnd.pypi.simple') &&
acceptHeader.includes('json');
if (!metadata) {
const upstream = await this.getUpstreamForRequest(normalized, 'simple', 'GET', actor);
if (upstream) {
const upstreamHtml = await upstream.fetchSimplePackage(normalized);
if (upstreamHtml) {
// Parse the HTML to extract file information and cache it
// For now, just return the upstream HTML directly (caching can be improved later)
const acceptHeader = context.headers['accept'] || context.headers['Accept'] || '';
const preferJson = acceptHeader.includes('application/vnd.pypi.simple') &&
acceptHeader.includes('json');
if (preferJson) {
// Try to get JSON format from upstream
const upstreamJson = await this.upstream.fetchPackageJson(normalized);
if (upstreamJson) {
return {
status: 200,
headers: {
'Content-Type': 'application/vnd.pypi.simple.v1+json',
'Cache-Control': 'public, max-age=300'
},
body: upstreamJson,
};
if (preferJson) {
// Try to get JSON format from upstream
const upstreamJson = await upstream.fetchPackageJson(normalized);
if (upstreamJson) {
return {
status: 200,
headers: {
'Content-Type': 'application/vnd.pypi.simple.v1+json',
'Cache-Control': 'public, max-age=300'
},
body: upstreamJson,
};
}
}
}
// Return HTML format
return {
status: 200,
headers: {
'Content-Type': 'text/html; charset=utf-8',
'Cache-Control': 'public, max-age=300'
},
body: upstreamHtml,
};
// Return HTML format
return {
status: 200,
headers: {
'Content-Type': 'text/html; charset=utf-8',
'Cache-Control': 'public, max-age=300'
},
body: upstreamHtml,
};
}
}
}
@@ -503,16 +533,19 @@ export class PypiRegistry extends BaseRegistry {
/**
* Handle package download
*/
private async handleDownload(packageName: string, filename: string): Promise<IResponse> {
private async handleDownload(packageName: string, filename: string, actor?: IRequestActor): Promise<IResponse> {
const normalized = helpers.normalizePypiPackageName(packageName);
let fileData = await this.storage.getPypiPackageFile(normalized, filename);
// Try upstream if not found locally
if (!fileData && this.upstream) {
fileData = await this.upstream.fetchPackageFile(normalized, filename);
if (fileData) {
// Cache locally
await this.storage.putPypiPackageFile(normalized, filename, fileData);
if (!fileData) {
const upstream = await this.getUpstreamForRequest(normalized, 'file', 'GET', actor);
if (upstream) {
fileData = await upstream.fetchPackageFile(normalized, filename);
if (fileData) {
// Cache locally
await this.storage.putPypiPackageFile(normalized, filename, fileData);
}
}
}