feat(core): Add PyPI and RubyGems protocol support, Cargo token management, and storage helpers
This commit is contained in:
564
ts/pypi/classes.pypiregistry.ts
Normal file
564
ts/pypi/classes.pypiregistry.ts
Normal file
@@ -0,0 +1,564 @@
|
||||
import { Smartlog } from '@push.rocks/smartlog';
|
||||
import { BaseRegistry } from '../core/classes.baseregistry.js';
|
||||
import { RegistryStorage } from '../core/classes.registrystorage.js';
|
||||
import { AuthManager } from '../core/classes.authmanager.js';
|
||||
import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js';
|
||||
import type {
|
||||
IPypiPackageMetadata,
|
||||
IPypiFile,
|
||||
IPypiError,
|
||||
IPypiUploadResponse,
|
||||
} from './interfaces.pypi.js';
|
||||
import * as helpers from './helpers.pypi.js';
|
||||
|
||||
/**
|
||||
* PyPI registry implementation
|
||||
* Implements PEP 503 (Simple API), PEP 691 (JSON API), and legacy upload API
|
||||
*/
|
||||
export class PypiRegistry extends BaseRegistry {
|
||||
private storage: RegistryStorage;
|
||||
private authManager: AuthManager;
|
||||
private basePath: string = '/pypi';
|
||||
private registryUrl: string;
|
||||
private logger: Smartlog;
|
||||
|
||||
constructor(
|
||||
storage: RegistryStorage,
|
||||
authManager: AuthManager,
|
||||
basePath: string = '/pypi',
|
||||
registryUrl: string = 'http://localhost:5000'
|
||||
) {
|
||||
super();
|
||||
this.storage = storage;
|
||||
this.authManager = authManager;
|
||||
this.basePath = basePath;
|
||||
this.registryUrl = registryUrl;
|
||||
|
||||
// Initialize logger
|
||||
this.logger = new Smartlog({
|
||||
logContext: {
|
||||
company: 'push.rocks',
|
||||
companyunit: 'smartregistry',
|
||||
containerName: 'pypi-registry',
|
||||
environment: (process.env.NODE_ENV as any) || 'development',
|
||||
runtime: 'node',
|
||||
zone: 'pypi'
|
||||
}
|
||||
});
|
||||
this.logger.enableConsole();
|
||||
}
|
||||
|
||||
public async init(): Promise<void> {
|
||||
// Initialize root Simple API index if not exists
|
||||
const existingIndex = await this.storage.getPypiSimpleRootIndex();
|
||||
if (!existingIndex) {
|
||||
const html = helpers.generateSimpleRootHtml([]);
|
||||
await this.storage.putPypiSimpleRootIndex(html);
|
||||
this.logger.log('info', 'Initialized PyPI root index');
|
||||
}
|
||||
}
|
||||
|
||||
public getBasePath(): string {
|
||||
return this.basePath;
|
||||
}
|
||||
|
||||
public async handleRequest(context: IRequestContext): Promise<IResponse> {
|
||||
let path = context.path.replace(this.basePath, '');
|
||||
|
||||
// Also handle /simple path prefix
|
||||
if (path.startsWith('/simple')) {
|
||||
path = path.replace('/simple', '');
|
||||
return this.handleSimpleRequest(path, context);
|
||||
}
|
||||
|
||||
// Extract token (Basic Auth or Bearer)
|
||||
const token = await this.extractToken(context);
|
||||
|
||||
this.logger.log('debug', `handleRequest: ${context.method} ${path}`, {
|
||||
method: context.method,
|
||||
path,
|
||||
hasAuth: !!token
|
||||
});
|
||||
|
||||
// Root upload endpoint (POST /)
|
||||
if ((path === '/' || path === '') && context.method === 'POST') {
|
||||
return this.handleUpload(context, token);
|
||||
}
|
||||
|
||||
// Package metadata JSON API: GET /pypi/{package}/json
|
||||
const jsonMatch = path.match(/^\/pypi\/([^\/]+)\/json$/);
|
||||
if (jsonMatch && context.method === 'GET') {
|
||||
return this.handlePackageJson(jsonMatch[1]);
|
||||
}
|
||||
|
||||
// Version-specific JSON API: GET /pypi/{package}/{version}/json
|
||||
const versionJsonMatch = path.match(/^\/pypi\/([^\/]+)\/([^\/]+)\/json$/);
|
||||
if (versionJsonMatch && context.method === 'GET') {
|
||||
return this.handleVersionJson(versionJsonMatch[1], versionJsonMatch[2]);
|
||||
}
|
||||
|
||||
// Package file download: GET /packages/{package}/{filename}
|
||||
const downloadMatch = path.match(/^\/packages\/([^\/]+)\/(.+)$/);
|
||||
if (downloadMatch && context.method === 'GET') {
|
||||
return this.handleDownload(downloadMatch[1], downloadMatch[2]);
|
||||
}
|
||||
|
||||
// Delete package: DELETE /packages/{package}
|
||||
if (path.match(/^\/packages\/([^\/]+)$/) && context.method === 'DELETE') {
|
||||
const packageName = path.match(/^\/packages\/([^\/]+)$/)?.[1];
|
||||
return this.handleDeletePackage(packageName!, token);
|
||||
}
|
||||
|
||||
// Delete version: DELETE /packages/{package}/{version}
|
||||
const deleteVersionMatch = path.match(/^\/packages\/([^\/]+)\/([^\/]+)$/);
|
||||
if (deleteVersionMatch && context.method === 'DELETE') {
|
||||
return this.handleDeleteVersion(deleteVersionMatch[1], deleteVersionMatch[2], token);
|
||||
}
|
||||
|
||||
return {
|
||||
status: 404,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: Buffer.from(JSON.stringify({ message: 'Not Found' })),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if token has permission for resource
|
||||
*/
|
||||
protected async checkPermission(
|
||||
token: IAuthToken | null,
|
||||
resource: string,
|
||||
action: string
|
||||
): Promise<boolean> {
|
||||
if (!token) return false;
|
||||
return this.authManager.authorize(token, `pypi:package:${resource}`, action);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle Simple API requests (PEP 503 HTML or PEP 691 JSON)
|
||||
*/
|
||||
private async handleSimpleRequest(path: string, context: IRequestContext): Promise<IResponse> {
|
||||
// Ensure path ends with / (PEP 503 requirement)
|
||||
if (!path.endsWith('/') && !path.includes('.')) {
|
||||
return {
|
||||
status: 301,
|
||||
headers: { 'Location': `${this.basePath}/simple${path}/` },
|
||||
body: Buffer.from(''),
|
||||
};
|
||||
}
|
||||
|
||||
// Root index: /simple/
|
||||
if (path === '/' || path === '') {
|
||||
return this.handleSimpleRoot(context);
|
||||
}
|
||||
|
||||
// Package index: /simple/{package}/
|
||||
const packageMatch = path.match(/^\/([^\/]+)\/$/);
|
||||
if (packageMatch) {
|
||||
return this.handleSimplePackage(packageMatch[1], context);
|
||||
}
|
||||
|
||||
return {
|
||||
status: 404,
|
||||
headers: { 'Content-Type': 'text/html; charset=utf-8' },
|
||||
body: Buffer.from('<html><body><h1>404 Not Found</h1></body></html>'),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle Simple API root index
|
||||
* Returns HTML (PEP 503) or JSON (PEP 691) based on Accept header
|
||||
*/
|
||||
private async handleSimpleRoot(context: IRequestContext): Promise<IResponse> {
|
||||
const acceptHeader = context.headers['accept'] || context.headers['Accept'] || '';
|
||||
const preferJson = acceptHeader.includes('application/vnd.pypi.simple') &&
|
||||
acceptHeader.includes('json');
|
||||
|
||||
const packages = await this.storage.listPypiPackages();
|
||||
|
||||
if (preferJson) {
|
||||
// PEP 691: JSON response
|
||||
const response = helpers.generateJsonRootResponse(packages);
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/vnd.pypi.simple.v1+json',
|
||||
'Cache-Control': 'public, max-age=600'
|
||||
},
|
||||
body: Buffer.from(JSON.stringify(response)),
|
||||
};
|
||||
} else {
|
||||
// PEP 503: HTML response
|
||||
const html = helpers.generateSimpleRootHtml(packages);
|
||||
|
||||
// Update stored index
|
||||
await this.storage.putPypiSimpleRootIndex(html);
|
||||
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'text/html; charset=utf-8',
|
||||
'Cache-Control': 'public, max-age=600'
|
||||
},
|
||||
body: Buffer.from(html),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle Simple API package index
|
||||
* Returns HTML (PEP 503) or JSON (PEP 691) based on Accept header
|
||||
*/
|
||||
private async handleSimplePackage(packageName: string, context: IRequestContext): Promise<IResponse> {
|
||||
const normalized = helpers.normalizePypiPackageName(packageName);
|
||||
|
||||
// Get package metadata
|
||||
const metadata = await this.storage.getPypiPackageMetadata(normalized);
|
||||
if (!metadata) {
|
||||
return {
|
||||
status: 404,
|
||||
headers: { 'Content-Type': 'text/html; charset=utf-8' },
|
||||
body: Buffer.from('<html><body><h1>404 Not Found</h1></body></html>'),
|
||||
};
|
||||
}
|
||||
|
||||
// Build file list from all versions
|
||||
const files: IPypiFile[] = [];
|
||||
for (const [version, versionMeta] of Object.entries(metadata.versions || {})) {
|
||||
for (const file of (versionMeta as any).files || []) {
|
||||
files.push({
|
||||
filename: file.filename,
|
||||
url: `${this.registryUrl}/pypi/packages/${normalized}/${file.filename}`,
|
||||
hashes: file.hashes,
|
||||
'requires-python': file['requires-python'],
|
||||
yanked: file.yanked || (versionMeta as any).yanked,
|
||||
size: file.size,
|
||||
'upload-time': file['upload-time'],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const acceptHeader = context.headers['accept'] || context.headers['Accept'] || '';
|
||||
const preferJson = acceptHeader.includes('application/vnd.pypi.simple') &&
|
||||
acceptHeader.includes('json');
|
||||
|
||||
if (preferJson) {
|
||||
// PEP 691: JSON response
|
||||
const response = helpers.generateJsonPackageResponse(normalized, files);
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/vnd.pypi.simple.v1+json',
|
||||
'Cache-Control': 'public, max-age=300'
|
||||
},
|
||||
body: Buffer.from(JSON.stringify(response)),
|
||||
};
|
||||
} else {
|
||||
// PEP 503: HTML response
|
||||
const html = helpers.generateSimplePackageHtml(normalized, files, this.registryUrl);
|
||||
|
||||
// Update stored index
|
||||
await this.storage.putPypiSimpleIndex(normalized, html);
|
||||
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'text/html; charset=utf-8',
|
||||
'Cache-Control': 'public, max-age=300'
|
||||
},
|
||||
body: Buffer.from(html),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract authentication token from request
|
||||
*/
|
||||
private async extractToken(context: IRequestContext): Promise<IAuthToken | null> {
|
||||
const authHeader = context.headers['authorization'] || context.headers['Authorization'];
|
||||
if (!authHeader) return null;
|
||||
|
||||
// Handle Basic Auth (username:password or __token__:token)
|
||||
if (authHeader.startsWith('Basic ')) {
|
||||
const base64 = authHeader.substring(6);
|
||||
const decoded = Buffer.from(base64, 'base64').toString('utf-8');
|
||||
const [username, password] = decoded.split(':');
|
||||
|
||||
// PyPI token authentication: username = __token__
|
||||
if (username === '__token__') {
|
||||
return this.authManager.validateToken(password, 'pypi');
|
||||
}
|
||||
|
||||
// Username/password authentication (would need user lookup)
|
||||
// For now, not implemented
|
||||
return null;
|
||||
}
|
||||
|
||||
// Handle Bearer token
|
||||
if (authHeader.startsWith('Bearer ')) {
|
||||
const token = authHeader.substring(7);
|
||||
return this.authManager.validateToken(token, 'pypi');
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle package upload (multipart/form-data)
|
||||
* POST / with :action=file_upload
|
||||
*/
|
||||
private async handleUpload(context: IRequestContext, token: IAuthToken | null): Promise<IResponse> {
|
||||
if (!token) {
|
||||
return {
|
||||
status: 401,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'WWW-Authenticate': 'Basic realm="PyPI"'
|
||||
},
|
||||
body: Buffer.from(JSON.stringify({ message: 'Authentication required' })),
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
// Parse multipart form data (context.body should be parsed by server)
|
||||
const formData = context.body as any; // Assuming parsed multipart data
|
||||
|
||||
if (!formData || formData[':action'] !== 'file_upload') {
|
||||
return this.errorResponse(400, 'Invalid upload request');
|
||||
}
|
||||
|
||||
// Extract required fields
|
||||
const packageName = formData.name;
|
||||
const version = formData.version;
|
||||
const filename = formData.content?.filename;
|
||||
const fileData = formData.content?.data as Buffer;
|
||||
const filetype = formData.filetype; // 'bdist_wheel' or 'sdist'
|
||||
const pyversion = formData.pyversion;
|
||||
|
||||
if (!packageName || !version || !filename || !fileData) {
|
||||
return this.errorResponse(400, 'Missing required fields');
|
||||
}
|
||||
|
||||
// Validate package name
|
||||
if (!helpers.isValidPackageName(packageName)) {
|
||||
return this.errorResponse(400, 'Invalid package name');
|
||||
}
|
||||
|
||||
const normalized = helpers.normalizePypiPackageName(packageName);
|
||||
|
||||
// Check permission
|
||||
if (!(await this.checkPermission(token, normalized, 'write'))) {
|
||||
return this.errorResponse(403, 'Insufficient permissions');
|
||||
}
|
||||
|
||||
// Calculate hashes
|
||||
const hashes: Record<string, string> = {};
|
||||
|
||||
if (formData.sha256_digest) {
|
||||
hashes.sha256 = formData.sha256_digest;
|
||||
} else {
|
||||
hashes.sha256 = await helpers.calculateHash(fileData, 'sha256');
|
||||
}
|
||||
|
||||
if (formData.md5_digest) {
|
||||
// MD5 digest in PyPI is urlsafe base64, convert to hex
|
||||
hashes.md5 = await helpers.calculateHash(fileData, 'md5');
|
||||
}
|
||||
|
||||
if (formData.blake2_256_digest) {
|
||||
hashes.blake2b = formData.blake2_256_digest;
|
||||
}
|
||||
|
||||
// Store file
|
||||
await this.storage.putPypiPackageFile(normalized, filename, fileData);
|
||||
|
||||
// Update metadata
|
||||
let metadata = await this.storage.getPypiPackageMetadata(normalized);
|
||||
if (!metadata) {
|
||||
metadata = {
|
||||
name: normalized,
|
||||
versions: {},
|
||||
};
|
||||
}
|
||||
|
||||
if (!metadata.versions[version]) {
|
||||
metadata.versions[version] = {
|
||||
version,
|
||||
files: [],
|
||||
};
|
||||
}
|
||||
|
||||
// Add file to version
|
||||
metadata.versions[version].files.push({
|
||||
filename,
|
||||
path: `pypi/packages/${normalized}/${filename}`,
|
||||
filetype,
|
||||
python_version: pyversion,
|
||||
hashes,
|
||||
size: fileData.length,
|
||||
'requires-python': formData.requires_python,
|
||||
'upload-time': new Date().toISOString(),
|
||||
'uploaded-by': token.userId,
|
||||
});
|
||||
|
||||
// Store core metadata if provided
|
||||
if (formData.summary || formData.description) {
|
||||
metadata.versions[version].metadata = helpers.extractCoreMetadata(formData);
|
||||
}
|
||||
|
||||
metadata['last-modified'] = new Date().toISOString();
|
||||
await this.storage.putPypiPackageMetadata(normalized, metadata);
|
||||
|
||||
this.logger.log('info', `Package uploaded: ${normalized} ${version}`, {
|
||||
filename,
|
||||
size: fileData.length
|
||||
});
|
||||
|
||||
return {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: Buffer.from(JSON.stringify({
|
||||
message: 'Package uploaded successfully',
|
||||
url: `${this.registryUrl}/pypi/packages/${normalized}/${filename}`
|
||||
})),
|
||||
};
|
||||
} catch (error) {
|
||||
this.logger.log('error', 'Upload failed', { error: (error as Error).message });
|
||||
return this.errorResponse(500, 'Upload failed: ' + (error as Error).message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle package download
|
||||
*/
|
||||
private async handleDownload(packageName: string, filename: string): Promise<IResponse> {
|
||||
const normalized = helpers.normalizePypiPackageName(packageName);
|
||||
const fileData = await this.storage.getPypiPackageFile(normalized, filename);
|
||||
|
||||
if (!fileData) {
|
||||
return {
|
||||
status: 404,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: Buffer.from(JSON.stringify({ message: 'File not found' })),
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/octet-stream',
|
||||
'Content-Disposition': `attachment; filename="${filename}"`,
|
||||
'Content-Length': fileData.length.toString()
|
||||
},
|
||||
body: fileData,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle package JSON API (all versions)
|
||||
*/
|
||||
private async handlePackageJson(packageName: string): Promise<IResponse> {
|
||||
const normalized = helpers.normalizePypiPackageName(packageName);
|
||||
const metadata = await this.storage.getPypiPackageMetadata(normalized);
|
||||
|
||||
if (!metadata) {
|
||||
return this.errorResponse(404, 'Package not found');
|
||||
}
|
||||
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'public, max-age=300'
|
||||
},
|
||||
body: Buffer.from(JSON.stringify(metadata)),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle version-specific JSON API
|
||||
*/
|
||||
private async handleVersionJson(packageName: string, version: string): Promise<IResponse> {
|
||||
const normalized = helpers.normalizePypiPackageName(packageName);
|
||||
const metadata = await this.storage.getPypiPackageMetadata(normalized);
|
||||
|
||||
if (!metadata || !metadata.versions[version]) {
|
||||
return this.errorResponse(404, 'Version not found');
|
||||
}
|
||||
|
||||
return {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'public, max-age=300'
|
||||
},
|
||||
body: Buffer.from(JSON.stringify(metadata.versions[version])),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle package deletion
|
||||
*/
|
||||
private async handleDeletePackage(packageName: string, token: IAuthToken | null): Promise<IResponse> {
|
||||
if (!token) {
|
||||
return this.errorResponse(401, 'Authentication required');
|
||||
}
|
||||
|
||||
const normalized = helpers.normalizePypiPackageName(packageName);
|
||||
|
||||
if (!(await this.checkPermission(token, normalized, 'delete'))) {
|
||||
return this.errorResponse(403, 'Insufficient permissions');
|
||||
}
|
||||
|
||||
await this.storage.deletePypiPackage(normalized);
|
||||
|
||||
this.logger.log('info', `Package deleted: ${normalized}`);
|
||||
|
||||
return {
|
||||
status: 204,
|
||||
headers: {},
|
||||
body: Buffer.from(''),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle version deletion
|
||||
*/
|
||||
private async handleDeleteVersion(
|
||||
packageName: string,
|
||||
version: string,
|
||||
token: IAuthToken | null
|
||||
): Promise<IResponse> {
|
||||
if (!token) {
|
||||
return this.errorResponse(401, 'Authentication required');
|
||||
}
|
||||
|
||||
const normalized = helpers.normalizePypiPackageName(packageName);
|
||||
|
||||
if (!(await this.checkPermission(token, normalized, 'delete'))) {
|
||||
return this.errorResponse(403, 'Insufficient permissions');
|
||||
}
|
||||
|
||||
await this.storage.deletePypiPackageVersion(normalized, version);
|
||||
|
||||
this.logger.log('info', `Version deleted: ${normalized} ${version}`);
|
||||
|
||||
return {
|
||||
status: 204,
|
||||
headers: {},
|
||||
body: Buffer.from(''),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper: Create error response
|
||||
*/
|
||||
private errorResponse(status: number, message: string): IResponse {
|
||||
const error: IPypiError = { message, status };
|
||||
return {
|
||||
status,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: Buffer.from(JSON.stringify(error)),
|
||||
};
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user