push.rocks/smartregistry
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(core,storage,oci,registry-config): add streaming response support and configurable registry URLs across protocols

Browse Source
This commit is contained in:
Jürgen Kunz
2026-03-24 22:59:37 +00:00
parent 1f0acf2825
commit 7da1a35efe
42 changed files with 4179 additions and 5396 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ts/00_commitinfo_data.ts
View File

@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@push.rocks/smartregistry',
version: '2.7.0',
version: '2.8.0',
description: 'A composable TypeScript library implementing OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems registries for building unified container and package registries'
}

34
ts/cargo/classes.cargoregistry.ts
View File

@@ -370,7 +370,7 @@ export class CargoRegistry extends BaseRegistry {
const parsed = this.parsePublishRequest(body);
metadata = parsed.metadata;
crateFile = parsed.crateFile;
} catch (error) {
} catch (error: any) {
this.logger.log('error', 'handlePublish: parse error', { error: error.message });
return {
status: 400,
@@ -467,17 +467,29 @@ export class CargoRegistry extends BaseRegistry {
): Promise<IResponse> {
this.logger.log('debug', 'handleDownload', { crate: crateName, version });
let crateFile = await this.storage.getCargoCrate(crateName, version);
// Try streaming from local storage first
const streamResult = await this.storage.getCargoCrateStream(crateName, version);
if (streamResult) {
return {
status: 200,
headers: {
'Content-Type': 'application/gzip',
'Content-Length': streamResult.size.toString(),
'Content-Disposition': `attachment; filename="${crateName}-${version}.crate"`,
},
body: streamResult.stream,
};
}
// Try upstream if not found locally
if (!crateFile) {
const upstream = await this.getUpstreamForRequest(crateName, 'crate', 'GET', actor);
if (upstream) {
crateFile = await upstream.fetchCrate(crateName, version);
if (crateFile) {
// Cache locally
await this.storage.putCargoCrate(crateName, version, crateFile);
}
let crateFile: Buffer | null = null;
const upstream = await this.getUpstreamForRequest(crateName, 'crate', 'GET', actor);
if (upstream) {
crateFile = await upstream.fetchCrate(crateName, version);
if (crateFile) {
// Cache locally
await this.storage.putCargoCrate(crateName, version, crateFile);
}
}
@@ -647,7 +659,7 @@ export class CargoRegistry extends BaseRegistry {
}
}
}
} catch (error) {
} catch (error: any) {
this.logger.log('error', 'handleSearch: error', { error: error.message });
}

70
ts/classes.smartregistry.ts
View File

@@ -2,6 +2,7 @@ import { RegistryStorage } from './core/classes.registrystorage.js';
import { AuthManager } from './core/classes.authmanager.js';
import { BaseRegistry } from './core/classes.baseregistry.js';
import type { IRegistryConfig, IRequestContext, IResponse } from './core/interfaces.core.js';
import { toReadableStream } from './core/helpers.stream.js';
import { OciRegistry } from './oci/classes.ociregistry.js';
import { NpmRegistry } from './npm/classes.npmregistry.js';
import { MavenRegistry } from './maven/classes.mavenregistry.js';
@@ -95,7 +96,7 @@ export class SmartRegistry {
// Initialize NPM registry if enabled
if (this.config.npm?.enabled) {
const npmBasePath = this.config.npm.basePath ?? '/npm';
const registryUrl = `http://localhost:5000${npmBasePath}`; // TODO: Make configurable
const registryUrl = this.config.npm.registryUrl ?? `http://localhost:5000${npmBasePath}`;
const npmRegistry = new NpmRegistry(
this.storage,
this.authManager,
@@ -110,7 +111,7 @@ export class SmartRegistry {
// Initialize Maven registry if enabled
if (this.config.maven?.enabled) {
const mavenBasePath = this.config.maven.basePath ?? '/maven';
const registryUrl = `http://localhost:5000${mavenBasePath}`; // TODO: Make configurable
const registryUrl = this.config.maven.registryUrl ?? `http://localhost:5000${mavenBasePath}`;
const mavenRegistry = new MavenRegistry(
this.storage,
this.authManager,
@@ -125,7 +126,7 @@ export class SmartRegistry {
// Initialize Cargo registry if enabled
if (this.config.cargo?.enabled) {
const cargoBasePath = this.config.cargo.basePath ?? '/cargo';
const registryUrl = `http://localhost:5000${cargoBasePath}`; // TODO: Make configurable
const registryUrl = this.config.cargo.registryUrl ?? `http://localhost:5000${cargoBasePath}`;
const cargoRegistry = new CargoRegistry(
this.storage,
this.authManager,
@@ -140,7 +141,7 @@ export class SmartRegistry {
// Initialize Composer registry if enabled
if (this.config.composer?.enabled) {
const composerBasePath = this.config.composer.basePath ?? '/composer';
const registryUrl = `http://localhost:5000${composerBasePath}`; // TODO: Make configurable
const registryUrl = this.config.composer.registryUrl ?? `http://localhost:5000${composerBasePath}`;
const composerRegistry = new ComposerRegistry(
this.storage,
this.authManager,
@@ -155,7 +156,7 @@ export class SmartRegistry {
// Initialize PyPI registry if enabled
if (this.config.pypi?.enabled) {
const pypiBasePath = this.config.pypi.basePath ?? '/pypi';
const registryUrl = `http://localhost:5000`; // TODO: Make configurable
const registryUrl = this.config.pypi.registryUrl ?? `http://localhost:5000`;
const pypiRegistry = new PypiRegistry(
this.storage,
this.authManager,
@@ -170,7 +171,7 @@ export class SmartRegistry {
// Initialize RubyGems registry if enabled
if (this.config.rubygems?.enabled) {
const rubygemsBasePath = this.config.rubygems.basePath ?? '/rubygems';
const registryUrl = `http://localhost:5000${rubygemsBasePath}`; // TODO: Make configurable
const registryUrl = this.config.rubygems.registryUrl ?? `http://localhost:5000${rubygemsBasePath}`;
const rubygemsRegistry = new RubyGemsRegistry(
this.storage,
this.authManager,
@@ -191,75 +192,88 @@ export class SmartRegistry {
*/
public async handleRequest(context: IRequestContext): Promise<IResponse> {
const path = context.path;
let response: IResponse | undefined;
// Route to OCI registry
if (this.config.oci?.enabled && path.startsWith(this.config.oci.basePath)) {
if (!response && this.config.oci?.enabled && path.startsWith(this.config.oci.basePath)) {
const ociRegistry = this.registries.get('oci');
if (ociRegistry) {
return ociRegistry.handleRequest(context);
response = await ociRegistry.handleRequest(context);
}
}
// Route to NPM registry
if (this.config.npm?.enabled && path.startsWith(this.config.npm.basePath)) {
if (!response && this.config.npm?.enabled && path.startsWith(this.config.npm.basePath)) {
const npmRegistry = this.registries.get('npm');
if (npmRegistry) {
return npmRegistry.handleRequest(context);
response = await npmRegistry.handleRequest(context);
}
}
// Route to Maven registry
if (this.config.maven?.enabled && path.startsWith(this.config.maven.basePath)) {
if (!response && this.config.maven?.enabled && path.startsWith(this.config.maven.basePath)) {
const mavenRegistry = this.registries.get('maven');
if (mavenRegistry) {
return mavenRegistry.handleRequest(context);
response = await mavenRegistry.handleRequest(context);
}
}
// Route to Cargo registry
if (this.config.cargo?.enabled && path.startsWith(this.config.cargo.basePath)) {
if (!response && this.config.cargo?.enabled && path.startsWith(this.config.cargo.basePath)) {
const cargoRegistry = this.registries.get('cargo');
if (cargoRegistry) {
return cargoRegistry.handleRequest(context);
response = await cargoRegistry.handleRequest(context);
}
}
// Route to Composer registry
if (this.config.composer?.enabled && path.startsWith(this.config.composer.basePath)) {
if (!response && this.config.composer?.enabled && path.startsWith(this.config.composer.basePath)) {
const composerRegistry = this.registries.get('composer');
if (composerRegistry) {
return composerRegistry.handleRequest(context);
response = await composerRegistry.handleRequest(context);
}
}
// Route to PyPI registry (also handles /simple prefix)
if (this.config.pypi?.enabled) {
if (!response && this.config.pypi?.enabled) {
const pypiBasePath = this.config.pypi.basePath ?? '/pypi';
if (path.startsWith(pypiBasePath) || path.startsWith('/simple')) {
const pypiRegistry = this.registries.get('pypi');
if (pypiRegistry) {
return pypiRegistry.handleRequest(context);
response = await pypiRegistry.handleRequest(context);
}
}
}
// Route to RubyGems registry
if (this.config.rubygems?.enabled && path.startsWith(this.config.rubygems.basePath)) {
if (!response && this.config.rubygems?.enabled && path.startsWith(this.config.rubygems.basePath)) {
const rubygemsRegistry = this.registries.get('rubygems');
if (rubygemsRegistry) {
return rubygemsRegistry.handleRequest(context);
response = await rubygemsRegistry.handleRequest(context);
}
}
// No matching registry
return {
status: 404,
headers: { 'Content-Type': 'application/json' },
body: {
error: 'NOT_FOUND',
message: 'No registry handler for this path',
},
};
if (!response) {
response = {
status: 404,
headers: { 'Content-Type': 'application/json' },
body: {
error: 'NOT_FOUND',
message: 'No registry handler for this path',
},
};
}
// Normalize body to ReadableStream<Uint8Array> at the API boundary
if (response.body != null && !(response.body instanceof ReadableStream)) {
if (!Buffer.isBuffer(response.body) && typeof response.body === 'object' && !(response.body instanceof Uint8Array)) {
response.headers['Content-Type'] ??= 'application/json';
}
response.body = toReadableStream(response.body);
}
return response;
}
/**

8
ts/composer/classes.composerregistry.ts
View File

@@ -302,9 +302,9 @@ export class ComposerRegistry extends BaseRegistry {
token: IAuthToken | null
): Promise<IResponse> {
// Read operations are public, no authentication required
const zipData = await this.storage.getComposerPackageZip(vendorPackage, reference);
const streamResult = await this.storage.getComposerPackageZipStream(vendorPackage, reference);
if (!zipData) {
if (!streamResult) {
return {
status: 404,
headers: {},
@@ -316,10 +316,10 @@ export class ComposerRegistry extends BaseRegistry {
status: 200,
headers: {
'Content-Type': 'application/zip',
'Content-Length': zipData.length.toString(),
'Content-Length': streamResult.size.toString(),
'Content-Disposition': `attachment; filename="${reference}.zip"`,
},
body: zipData,
body: streamResult.stream,
};
}

135
ts/core/classes.registrystorage.ts
View File

@@ -34,8 +34,8 @@ import type {
* ```
*/
export class RegistryStorage implements IStorageBackend {
private smartBucket: plugins.smartbucket.SmartBucket;
private bucket: plugins.smartbucket.Bucket;
private smartBucket!: plugins.smartbucket.SmartBucket;
private bucket!: plugins.smartbucket.Bucket;
private bucketName: string;
private hooks?: IStorageHooks;
@@ -1266,4 +1266,135 @@ export class RegistryStorage implements IStorageBackend {
private getRubyGemsMetadataPath(gemName: string): string {
return `rubygems/metadata/${gemName}/metadata.json`;
}
// ========================================================================
// STREAMING METHODS (Web Streams API)
// ========================================================================
/**
* Get an object as a ReadableStream. Returns null if not found.
*/
public async getObjectStream(key: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null> {
try {
const stat = await this.bucket.fastStat({ path: key });
const size = stat.ContentLength ?? 0;
const stream = await this.bucket.fastGetStream({ path: key }, 'webstream');
// Call afterGet hook (non-blocking)
if (this.hooks?.afterGet) {
const context = this.currentContext;
if (context) {
this.hooks.afterGet({
operation: 'get',
key,
protocol: context.protocol,
actor: context.actor,
metadata: context.metadata,
timestamp: new Date(),
}).catch(() => {});
}
}
return { stream: stream as ReadableStream<Uint8Array>, size };
} catch {
return null;
}
}
/**
* Store an object from a ReadableStream.
*/
public async putObjectStream(key: string, stream: ReadableStream<Uint8Array>): Promise<void> {
if (this.hooks?.beforePut) {
const context = this.currentContext;
if (context) {
const hookContext: IStorageHookContext = {
operation: 'put',
key,
protocol: context.protocol,
actor: context.actor,
metadata: context.metadata,
timestamp: new Date(),
};
const result = await this.hooks.beforePut(hookContext);
if (!result.allowed) {
throw new Error(result.reason || 'Storage operation denied by hook');
}
}
}
// Convert WebStream to Node Readable at the S3 SDK boundary
// AWS SDK v3 PutObjectCommand requires a Node.js Readable (not WebStream)
const { Readable } = await import('stream');
const nodeStream = Readable.fromWeb(stream as any);
await this.bucket.fastPutStream({
path: key,
readableStream: nodeStream,
overwrite: true,
});
if (this.hooks?.afterPut) {
const context = this.currentContext;
if (context) {
this.hooks.afterPut({
operation: 'put',
key,
protocol: context.protocol,
actor: context.actor,
metadata: context.metadata,
timestamp: new Date(),
}).catch(() => {});
}
}
}
/**
* Get object size without reading data (S3 HEAD request).
*/
public async getObjectSize(key: string): Promise<number | null> {
try {
const stat = await this.bucket.fastStat({ path: key });
return stat.ContentLength ?? null;
} catch {
return null;
}
}
// ---- Protocol-specific streaming wrappers ----
public async getOciBlobStream(digest: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null> {
return this.getObjectStream(this.getOciBlobPath(digest));
}
public async putOciBlobStream(digest: string, stream: ReadableStream<Uint8Array>): Promise<void> {
return this.putObjectStream(this.getOciBlobPath(digest), stream);
}
public async getOciBlobSize(digest: string): Promise<number | null> {
return this.getObjectSize(this.getOciBlobPath(digest));
}
public async getNpmTarballStream(packageName: string, version: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null> {
return this.getObjectStream(this.getNpmTarballPath(packageName, version));
}
public async getMavenArtifactStream(groupId: string, artifactId: string, version: string, filename: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null> {
return this.getObjectStream(this.getMavenArtifactPath(groupId, artifactId, version, filename));
}
public async getCargoCrateStream(crateName: string, version: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null> {
return this.getObjectStream(this.getCargoCratePath(crateName, version));
}
public async getComposerPackageZipStream(vendorPackage: string, reference: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null> {
return this.getObjectStream(this.getComposerZipPath(vendorPackage, reference));
}
public async getPypiPackageFileStream(packageName: string, filename: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null> {
return this.getObjectStream(this.getPypiPackageFilePath(packageName, filename));
}
public async getRubyGemsGemStream(gemName: string, version: string, platform?: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null> {
return this.getObjectStream(this.getRubyGemsGemPath(gemName, version, platform));
}
}

63
ts/core/helpers.stream.ts Normal file
View File

@@ -0,0 +1,63 @@
import * as crypto from 'crypto';
/**
* Convert Buffer, Uint8Array, string, or JSON object to a ReadableStream<Uint8Array>.
*/
export function toReadableStream(data: Buffer | Uint8Array | string | object): ReadableStream<Uint8Array> {
const buf = Buffer.isBuffer(data)
? data
: data instanceof Uint8Array
? Buffer.from(data)
: typeof data === 'string'
? Buffer.from(data, 'utf-8')
: Buffer.from(JSON.stringify(data), 'utf-8');
return new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(new Uint8Array(buf));
controller.close();
},
});
}
/**
* Consume a ReadableStream into a Buffer.
*/
export async function streamToBuffer(stream: ReadableStream<Uint8Array>): Promise<Buffer> {
const reader = stream.getReader();
const chunks: Uint8Array[] = [];
while (true) {
const { done, value } = await reader.read();
if (done) break;
if (value) chunks.push(value);
}
return Buffer.concat(chunks);
}
/**
* Consume a ReadableStream into a parsed JSON object.
*/
export async function streamToJson<T = any>(stream: ReadableStream<Uint8Array>): Promise<T> {
const buf = await streamToBuffer(stream);
return JSON.parse(buf.toString('utf-8'));
}
/**
* Create a TransformStream that incrementally hashes data passing through.
* Data flows through unchanged; the digest is available after the stream completes.
*/
export function createHashTransform(algorithm: string = 'sha256'): {
transform: TransformStream<Uint8Array, Uint8Array>;
getDigest: () => string;
} {
const hash = crypto.createHash(algorithm);
const transform = new TransformStream<Uint8Array, Uint8Array>({
transform(chunk, controller) {
hash.update(chunk);
controller.enqueue(chunk);
},
});
return {
transform,
getDigest: () => hash.digest('hex'),
};
}

3
ts/core/index.ts
View File

@@ -12,6 +12,9 @@ export { DefaultAuthProvider } from './classes.defaultauthprovider.js';
// Storage interfaces and hooks
export * from './interfaces.storage.js';
// Stream helpers
export { toReadableStream, streamToBuffer, streamToJson, createHashTransform } from './helpers.stream.js';
// Classes
export { BaseRegistry } from './classes.baseregistry.js';
export { RegistryStorage } from './classes.registrystorage.js';

23
ts/core/interfaces.core.ts
View File

@@ -88,6 +88,7 @@ export interface IAuthConfig {
export interface IProtocolConfig {
enabled: boolean;
basePath: string;
registryUrl?: string;
features?: Record<string, boolean>;
}
@@ -160,6 +161,21 @@ export interface IStorageBackend {
* Get object metadata
*/
getMetadata(key: string): Promise<Record<string, string> | null>;
/**
* Get an object as a ReadableStream. Returns null if not found.
*/
getObjectStream?(key: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null>;
/**
* Store an object from a ReadableStream.
*/
putObjectStream?(key: string, stream: ReadableStream<Uint8Array>): Promise<void>;
/**
* Get object size without reading data (S3 HEAD request).
*/
getObjectSize?(key: string): Promise<number | null>;
}
/**
@@ -215,10 +231,13 @@ export interface IRequestContext {
}
/**
* Base response structure
* Base response structure.
* `body` is always a `ReadableStream<Uint8Array>` at the public API boundary.
* Internal handlers may return Buffer/string/object — the SmartRegistry orchestrator
* auto-wraps them via `toReadableStream()` before returning to the caller.
*/
export interface IResponse {
status: number;
headers: Record<string, string>;
body?: any;
body?: ReadableStream<Uint8Array> | any;
}

44
ts/maven/classes.mavenregistry.ts
View File

@@ -293,24 +293,36 @@ export class MavenRegistry extends BaseRegistry {
filename: string,
actor?: IRequestActor
): Promise<IResponse> {
let data = await this.storage.getMavenArtifact(groupId, artifactId, version, filename);
// Try local storage first (streaming)
const streamResult = await this.storage.getMavenArtifactStream(groupId, artifactId, version, filename);
if (streamResult) {
const ext = filename.split('.').pop() || '';
const contentType = this.getContentType(ext);
return {
status: 200,
headers: {
'Content-Type': contentType,
'Content-Length': streamResult.size.toString(),
},
body: streamResult.stream,
};
}
// Try upstream if not found locally
if (!data) {
const resource = `${groupId}:${artifactId}`;
const upstream = await this.getUpstreamForRequest(resource, 'artifact', 'GET', actor);
if (upstream) {
// Parse the filename to extract extension and classifier
const { extension, classifier } = this.parseFilename(filename, artifactId, version);
if (extension) {
data = await upstream.fetchArtifact(groupId, artifactId, version, extension, classifier);
if (data) {
// Cache the artifact locally
await this.storage.putMavenArtifact(groupId, artifactId, version, filename, data);
// Generate and store checksums
const checksums = await calculateChecksums(data);
await this.storeChecksums(groupId, artifactId, version, filename, checksums);
}
let data: Buffer | null = null;
const resource = `${groupId}:${artifactId}`;
const upstream = await this.getUpstreamForRequest(resource, 'artifact', 'GET', actor);
if (upstream) {
// Parse the filename to extract extension and classifier
const { extension, classifier } = this.parseFilename(filename, artifactId, version);
if (extension) {
data = await upstream.fetchArtifact(groupId, artifactId, version, extension, classifier);
if (data) {
// Cache the artifact locally
await this.storage.putMavenArtifact(groupId, artifactId, version, filename, data);
// Generate and store checksums
const checksums = await calculateChecksums(data);
await this.storeChecksums(groupId, artifactId, version, filename, checksums);
}
}
}

43
ts/npm/classes.npmregistry.ts
View File

@@ -631,27 +631,38 @@ export class NpmRegistry extends BaseRegistry {
}
const version = versionMatch[1];
let tarball = await this.storage.getNpmTarball(packageName, version);
// Try local storage first (streaming)
const streamResult = await this.storage.getNpmTarballStream(packageName, version);
if (streamResult) {
return {
status: 200,
headers: {
'Content-Type': 'application/octet-stream',
'Content-Length': streamResult.size.toString(),
},
body: streamResult.stream,
};
}
// If not found locally, try upstream
if (!tarball) {
const upstream = await this.getUpstreamForRequest(packageName, 'tarball', 'GET', actor);
if (upstream) {
this.logger.log('debug', 'handleTarballDownload: fetching from upstream', {
let tarball: Buffer | null = null;
const upstream = await this.getUpstreamForRequest(packageName, 'tarball', 'GET', actor);
if (upstream) {
this.logger.log('debug', 'handleTarballDownload: fetching from upstream', {
packageName,
version,
});
const upstreamTarball = await upstream.fetchTarball(packageName, version);
if (upstreamTarball) {
tarball = upstreamTarball;
// Cache the tarball locally for future requests
await this.storage.putNpmTarball(packageName, version, tarball);
this.logger.log('debug', 'handleTarballDownload: cached tarball locally', {
packageName,
version,
size: tarball.length,
});
const upstreamTarball = await upstream.fetchTarball(packageName, version);
if (upstreamTarball) {
tarball = upstreamTarball;
// Cache the tarball locally for future requests
await this.storage.putNpmTarball(packageName, version, tarball);
this.logger.log('debug', 'handleTarballDownload: cached tarball locally', {
packageName,
version,
size: tarball.length,
});
}
}
}

130
ts/oci/classes.ociregistry.ts
View File

@@ -3,6 +3,7 @@ import { BaseRegistry } from '../core/classes.baseregistry.js';
import { RegistryStorage } from '../core/classes.registrystorage.js';
import { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken, IRegistryError, IRequestActor } from '../core/interfaces.core.js';
import { createHashTransform, streamToBuffer } from '../core/helpers.stream.js';
import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
import { OciUpstream } from './classes.ociupstream.js';
import type {
@@ -302,6 +303,8 @@ export class OciRegistry extends BaseRegistry {
uploadId,
repository,
chunks: [],
chunkPaths: [],
chunkIndex: 0,
totalSize: 0,
createdAt: new Date(),
lastActivity: new Date(),
@@ -571,25 +574,35 @@ export class OciRegistry extends BaseRegistry {
return this.createUnauthorizedResponse(repository, 'pull');
}
// Try local storage first
let data = await this.storage.getOciBlob(digest);
// Try local storage first (streaming)
const streamResult = await this.storage.getOciBlobStream(digest);
if (streamResult) {
return {
status: 200,
headers: {
'Content-Type': 'application/octet-stream',
'Content-Length': streamResult.size.toString(),
'Docker-Content-Digest': digest,
},
body: streamResult.stream,
};
}
// If not found locally, try upstream
if (!data) {
const upstream = await this.getUpstreamForRequest(repository, 'blob', 'GET', actor);
if (upstream) {
this.logger.log('debug', 'getBlob: fetching from upstream', { repository, digest });
const upstreamBlob = await upstream.fetchBlob(repository, digest);
if (upstreamBlob) {
data = upstreamBlob;
// Cache the blob locally (blobs are content-addressable and immutable)
await this.storage.putOciBlob(digest, data);
this.logger.log('debug', 'getBlob: cached blob locally', {
repository,
digest,
size: data.length,
});
}
let data: Buffer | null = null;
const upstream = await this.getUpstreamForRequest(repository, 'blob', 'GET', actor);
if (upstream) {
this.logger.log('debug', 'getBlob: fetching from upstream', { repository, digest });
const upstreamBlob = await upstream.fetchBlob(repository, digest);
if (upstreamBlob) {
data = upstreamBlob;
// Cache the blob locally (blobs are content-addressable and immutable)
await this.storage.putOciBlob(digest, data);
this.logger.log('debug', 'getBlob: cached blob locally', {
repository,
digest,
size: data.length,
});
}
}
@@ -620,17 +633,15 @@ export class OciRegistry extends BaseRegistry {
return this.createUnauthorizedHeadResponse(repository, 'pull');
}
const exists = await this.storage.ociBlobExists(digest);
if (!exists) {
const blobSize = await this.storage.getOciBlobSize(digest);
if (blobSize === null) {
return { status: 404, headers: {}, body: null };
}
const blob = await this.storage.getOciBlob(digest);
return {
status: 200,
headers: {
'Content-Length': blob ? blob.length.toString() : '0',
'Content-Length': blobSize.toString(),
'Docker-Content-Digest': digest,
},
body: null,
@@ -670,7 +681,12 @@ export class OciRegistry extends BaseRegistry {
}
const chunkData = this.toBuffer(data);
session.chunks.push(chunkData);
// Write chunk to temp S3 object instead of accumulating in memory
const chunkPath = `oci/uploads/${uploadId}/chunk-${session.chunkIndex}`;
await this.storage.putObject(chunkPath, chunkData);
session.chunkPaths.push(chunkPath);
session.chunkIndex++;
session.totalSize += chunkData.length;
session.lastActivity = new Date();
@@ -699,13 +715,52 @@ export class OciRegistry extends BaseRegistry {
};
}
const chunks = [...session.chunks];
if (finalData) chunks.push(this.toBuffer(finalData));
const blobData = Buffer.concat(chunks);
// If there's final data in the PUT body, write it as the last chunk
if (finalData) {
const buf = this.toBuffer(finalData);
const chunkPath = `oci/uploads/${uploadId}/chunk-${session.chunkIndex}`;
await this.storage.putObject(chunkPath, buf);
session.chunkPaths.push(chunkPath);
session.chunkIndex++;
session.totalSize += buf.length;
}
// Verify digest
const calculatedDigest = await this.calculateDigest(blobData);
// Create a ReadableStream that assembles all chunks from S3 sequentially
const chunkPaths = [...session.chunkPaths];
const storage = this.storage;
let chunkIdx = 0;
const assembledStream = new ReadableStream<Uint8Array>({
async pull(controller) {
if (chunkIdx >= chunkPaths.length) {
controller.close();
return;
}
const result = await storage.getObjectStream(chunkPaths[chunkIdx++]);
if (result) {
const reader = result.stream.getReader();
while (true) {
const { done, value } = await reader.read();
if (done) break;
if (value) controller.enqueue(value);
}
}
},
});
// Pipe through hash transform for incremental digest verification
const { transform: hashTransform, getDigest } = createHashTransform('sha256');
const hashedStream = assembledStream.pipeThrough(hashTransform);
// Consume stream to buffer for S3 upload
// (AWS SDK PutObjectCommand requires known content-length for streams;
// the key win is chunks are NOT accumulated in memory during PATCH — they live in S3)
const blobData = await streamToBuffer(hashedStream);
// Verify digest before storing
const calculatedDigest = `sha256:${getDigest()}`;
if (calculatedDigest !== digest) {
await this.cleanupUploadChunks(session);
this.uploadSessions.delete(uploadId);
return {
status: 400,
headers: {},
@@ -713,7 +768,11 @@ export class OciRegistry extends BaseRegistry {
};
}
// Store verified blob
await this.storage.putOciBlob(digest, blobData);
// Cleanup temp chunks and session
await this.cleanupUploadChunks(session);
this.uploadSessions.delete(uploadId);
return {
@@ -726,6 +785,19 @@ export class OciRegistry extends BaseRegistry {
};
}
/**
* Delete all temp S3 chunk objects for an upload session.
*/
private async cleanupUploadChunks(session: IUploadSession): Promise<void> {
for (const chunkPath of session.chunkPaths) {
try {
await this.storage.deleteObject(chunkPath);
} catch {
// Best-effort cleanup
}
}
}
private async getUploadStatus(uploadId: string): Promise<IResponse> {
const session = this.uploadSessions.get(uploadId);
if (!session) {
@@ -917,6 +989,8 @@ export class OciRegistry extends BaseRegistry {
for (const [uploadId, session] of this.uploadSessions.entries()) {
if (now.getTime() - session.lastActivity.getTime() > maxAge) {
// Clean up temp S3 chunks for stale sessions
this.cleanupUploadChunks(session).catch(() => {});
this.uploadSessions.delete(uploadId);
}
}

4
ts/oci/interfaces.oci.ts
View File

@@ -62,6 +62,10 @@ export interface IUploadSession {
uploadId: string;
repository: string;
chunks: Buffer[];
/** S3 paths to temp chunk objects (streaming mode) */
chunkPaths: string[];
/** Index counter for naming temp chunk objects */
chunkIndex: number;
totalSize: number;
createdAt: Date;
lastActivity: Date;

31
ts/pypi/classes.pypiregistry.ts
View File

@@ -535,17 +535,30 @@ export class PypiRegistry extends BaseRegistry {
*/
private async handleDownload(packageName: string, filename: string, actor?: IRequestActor): Promise<IResponse> {
const normalized = helpers.normalizePypiPackageName(packageName);
let fileData = await this.storage.getPypiPackageFile(normalized, filename);
// Try streaming from local storage first
const streamResult = await this.storage.getPypiPackageFileStream(normalized, filename);
if (streamResult) {
return {
status: 200,
headers: {
'Content-Type': 'application/octet-stream',
'Content-Disposition': `attachment; filename="${filename}"`,
'Content-Length': streamResult.size.toString()
},
body: streamResult.stream,
};
}
// Try upstream if not found locally
if (!fileData) {
const upstream = await this.getUpstreamForRequest(normalized, 'file', 'GET', actor);
if (upstream) {
fileData = await upstream.fetchPackageFile(normalized, filename);
if (fileData) {
// Cache locally
await this.storage.putPypiPackageFile(normalized, filename, fileData);
}
let fileData: Buffer | null = null;
const upstream = await this.getUpstreamForRequest(normalized, 'file', 'GET', actor);
if (upstream) {
fileData = await upstream.fetchPackageFile(normalized, filename);
if (fileData) {
// Cache locally
await this.storage.putPypiPackageFile(normalized, filename, fileData);
}
}

30
ts/rubygems/classes.rubygemsregistry.ts
View File

@@ -303,21 +303,33 @@ export class RubyGemsRegistry extends BaseRegistry {
return this.errorResponse(400, 'Invalid gem filename');
}
let gemData = await this.storage.getRubyGemsGem(
// Try streaming from local storage first
const streamResult = await this.storage.getRubyGemsGemStream(
parsed.name,
parsed.version,
parsed.platform
);
if (streamResult) {
return {
status: 200,
headers: {
'Content-Type': 'application/octet-stream',
'Content-Disposition': `attachment; filename="${filename}"`,
'Content-Length': streamResult.size.toString()
},
body: streamResult.stream,
};
}
// Try upstream if not found locally
if (!gemData) {
const upstream = await this.getUpstreamForRequest(parsed.name, 'gem', 'GET', actor);
if (upstream) {
gemData = await upstream.fetchGem(parsed.name, parsed.version);
if (gemData) {
// Cache locally
await this.storage.putRubyGemsGem(parsed.name, parsed.version, gemData, parsed.platform);
}
let gemData: Buffer | null = null;
const upstream = await this.getUpstreamForRequest(parsed.name, 'gem', 'GET', actor);
if (upstream) {
gemData = await upstream.fetchGem(parsed.name, parsed.version);
if (gemData) {
// Cache locally
await this.storage.putRubyGemsGem(parsed.name, parsed.version, gemData, parsed.platform);
}
}

4
ts/rubygems/helpers.rubygems.ts
View File

@@ -427,7 +427,7 @@ export async function extractGemMetadata(gemData: Buffer): Promise<{
// Step 2: Decompress the gzipped metadata
const gzipTools = new plugins.smartarchive.GzipTools();
const metadataYaml = await gzipTools.decompress(metadataFile.contentBuffer);
const yamlContent = metadataYaml.toString('utf-8');
const yamlContent = Buffer.from(metadataYaml).toString('utf-8');
// Step 3: Parse the YAML to extract name, version, platform
// Look for name: field in YAML
@@ -503,7 +503,7 @@ export async function generateSpecsGz(specs: Array<[string, string, string]>): P
}
const uncompressed = Buffer.concat(parts);
return gzipTools.compress(uncompressed);
return Buffer.from(await gzipTools.compress(uncompressed));
}
/**

2
ts/upstream/classes.upstreamcache.ts
View File

@@ -105,7 +105,7 @@ export class UpstreamCache {
// If not in memory and we have storage, check S3
if (!entry && this.storage) {
entry = await this.loadFromStorage(key);
entry = (await this.loadFromStorage(key)) ?? undefined;
if (entry) {
// Promote to memory cache
this.memoryCache.set(key, entry);