push.rocks/smartregistry
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(upstream): Add upstream proxy/cache subsystem and integrate per-protocol upstreams

Browse Source
This commit is contained in:
Jürgen Kunz
2025-11-27 14:20:01 +00:00
parent cfadc89b5a
commit 0610077eec
34 changed files with 3450 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

106
ts/oci/classes.ociregistry.ts
View File

@@ -1,7 +1,10 @@
import { Smartlog } from '@push.rocks/smartlog';
import { BaseRegistry } from '../core/classes.baseregistry.js';
import { RegistryStorage } from '../core/classes.registrystorage.js';
import { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken, IRegistryError } from '../core/interfaces.core.js';
import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js';
import { OciUpstream } from './classes.ociupstream.js';
import type {
IUploadSession,
IOciManifest,
@@ -21,18 +24,42 @@ export class OciRegistry extends BaseRegistry {
private basePath: string = '/oci';
private cleanupInterval?: NodeJS.Timeout;
private ociTokens?: { realm: string; service: string };
private upstream: OciUpstream | null = null;
private logger: Smartlog;
constructor(
storage: RegistryStorage,
authManager: AuthManager,
basePath: string = '/oci',
ociTokens?: { realm: string; service: string }
ociTokens?: { realm: string; service: string },
upstreamConfig?: IProtocolUpstreamConfig
) {
super();
this.storage = storage;
this.authManager = authManager;
this.basePath = basePath;
this.ociTokens = ociTokens;
// Initialize logger
this.logger = new Smartlog({
logContext: {
company: 'push.rocks',
companyunit: 'smartregistry',
containerName: 'oci-registry',
environment: (process.env.NODE_ENV as any) || 'development',
runtime: 'node',
zone: 'oci'
}
});
this.logger.enableConsole();
// Initialize upstream if configured
if (upstreamConfig?.enabled) {
this.upstream = new OciUpstream(upstreamConfig, basePath, this.logger);
this.logger.log('info', 'OCI upstream initialized', {
upstreams: upstreamConfig.upstreams.map(u => u.name),
});
}
}
public async init(): Promise<void> {
@@ -302,16 +329,50 @@ export class OciRegistry extends BaseRegistry {
if (!reference.startsWith('sha256:')) {
const tags = await this.getTagsData(repository);
digest = tags[reference];
if (!digest) {
return {
status: 404,
headers: {},
body: this.createError('MANIFEST_UNKNOWN', 'Manifest not found'),
};
}
// Try local storage first (if we have a digest)
let manifestData: Buffer | null = null;
let contentType: string | null = null;
if (digest) {
manifestData = await this.storage.getOciManifest(repository, digest);
if (manifestData) {
contentType = await this.storage.getOciManifestContentType(repository, digest);
if (!contentType) {
contentType = this.detectManifestContentType(manifestData);
}
}
}
// If not found locally, try upstream
if (!manifestData && this.upstream) {
this.logger.log('debug', 'getManifest: fetching from upstream', { repository, reference });
const upstreamResult = await this.upstream.fetchManifest(repository, reference);
if (upstreamResult) {
manifestData = Buffer.from(JSON.stringify(upstreamResult.manifest), 'utf8');
contentType = upstreamResult.contentType;
digest = upstreamResult.digest;
// Cache the manifest locally
await this.storage.putOciManifest(repository, digest, manifestData, contentType);
// If reference is a tag, update tags mapping
if (!reference.startsWith('sha256:')) {
const tags = await this.getTagsData(repository);
tags[reference] = digest;
const tagsPath = `oci/tags/${repository}/tags.json`;
await this.storage.putObject(tagsPath, Buffer.from(JSON.stringify(tags), 'utf-8'));
}
this.logger.log('debug', 'getManifest: cached manifest locally', {
repository,
reference,
digest,
});
}
}
const manifestData = await this.storage.getOciManifest(repository, digest);
if (!manifestData) {
return {
status: 404,
@@ -320,17 +381,10 @@ export class OciRegistry extends BaseRegistry {
};
}
// Get stored content type, falling back to detecting from manifest content
let contentType = await this.storage.getOciManifestContentType(repository, digest);
if (!contentType) {
// Fallback: detect content type from manifest content
contentType = this.detectManifestContentType(manifestData);
}
return {
status: 200,
headers: {
'Content-Type': contentType,
'Content-Type': contentType || 'application/vnd.oci.image.manifest.v1+json',
'Docker-Content-Digest': digest,
},
body: manifestData,
@@ -466,7 +520,25 @@ export class OciRegistry extends BaseRegistry {
return this.createUnauthorizedResponse(repository, 'pull');
}
const data = await this.storage.getOciBlob(digest);
// Try local storage first
let data = await this.storage.getOciBlob(digest);
// If not found locally, try upstream
if (!data && this.upstream) {
this.logger.log('debug', 'getBlob: fetching from upstream', { repository, digest });
const upstreamBlob = await this.upstream.fetchBlob(repository, digest);
if (upstreamBlob) {
data = upstreamBlob;
// Cache the blob locally (blobs are content-addressable and immutable)
await this.storage.putOciBlob(digest, data);
this.logger.log('debug', 'getBlob: cached blob locally', {
repository,
digest,
size: data.length,
});
}
}
if (!data) {
return {
status: 404,

263
ts/oci/classes.ociupstream.ts Normal file
View File

@@ -0,0 +1,263 @@
import * as plugins from '../plugins.js';
import { BaseUpstream } from '../upstream/classes.baseupstream.js';
import type {
IProtocolUpstreamConfig,
IUpstreamFetchContext,
IUpstreamResult,
IUpstreamRegistryConfig,
} from '../upstream/interfaces.upstream.js';
import type { IOciManifest, IOciImageIndex, ITagList } from './interfaces.oci.js';
/**
* OCI-specific upstream implementation.
*
* Handles:
* - Manifest fetching (image manifests and index manifests)
* - Blob proxying (layers, configs)
* - Tag list fetching
* - Content-addressable caching (blobs are immutable)
* - Docker Hub authentication flow
*/
export class OciUpstream extends BaseUpstream {
protected readonly protocolName = 'oci';
/** Local registry base path for URL building */
private readonly localBasePath: string;
constructor(
config: IProtocolUpstreamConfig,
localBasePath: string = '/oci',
logger?: plugins.smartlog.Smartlog,
) {
super(config, logger);
this.localBasePath = localBasePath;
}
/**
* Fetch a manifest from upstream registries.
*/
public async fetchManifest(
repository: string,
reference: string,
): Promise<{ manifest: IOciManifest | IOciImageIndex; contentType: string; digest: string } | null> {
const context: IUpstreamFetchContext = {
protocol: 'oci',
resource: repository,
resourceType: 'manifest',
path: `/v2/${repository}/manifests/${reference}`,
method: 'GET',
headers: {
'accept': [
'application/vnd.oci.image.manifest.v1+json',
'application/vnd.oci.image.index.v1+json',
'application/vnd.docker.distribution.manifest.v2+json',
'application/vnd.docker.distribution.manifest.list.v2+json',
'application/vnd.docker.distribution.manifest.v1+json',
].join(', '),
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
let manifest: IOciManifest | IOciImageIndex;
if (Buffer.isBuffer(result.body)) {
manifest = JSON.parse(result.body.toString('utf8'));
} else {
manifest = result.body;
}
const contentType = result.headers['content-type'] || 'application/vnd.oci.image.manifest.v1+json';
const digest = result.headers['docker-content-digest'] || '';
return { manifest, contentType, digest };
}
/**
* Check if a manifest exists in upstream (HEAD request).
*/
public async headManifest(
repository: string,
reference: string,
): Promise<{ exists: boolean; contentType?: string; digest?: string; size?: number } | null> {
const context: IUpstreamFetchContext = {
protocol: 'oci',
resource: repository,
resourceType: 'manifest',
path: `/v2/${repository}/manifests/${reference}`,
method: 'HEAD',
headers: {
'accept': [
'application/vnd.oci.image.manifest.v1+json',
'application/vnd.oci.image.index.v1+json',
'application/vnd.docker.distribution.manifest.v2+json',
'application/vnd.docker.distribution.manifest.list.v2+json',
].join(', '),
},
query: {},
};
const result = await this.fetch(context);
if (!result) {
return null;
}
if (!result.success) {
return { exists: false };
}
return {
exists: true,
contentType: result.headers['content-type'],
digest: result.headers['docker-content-digest'],
size: result.headers['content-length'] ? parseInt(result.headers['content-length'], 10) : undefined,
};
}
/**
* Fetch a blob from upstream registries.
*/
public async fetchBlob(repository: string, digest: string): Promise<Buffer | null> {
const context: IUpstreamFetchContext = {
protocol: 'oci',
resource: repository,
resourceType: 'blob',
path: `/v2/${repository}/blobs/${digest}`,
method: 'GET',
headers: {
'accept': 'application/octet-stream',
},
query: {},
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
return Buffer.isBuffer(result.body) ? result.body : Buffer.from(result.body);
}
/**
* Check if a blob exists in upstream (HEAD request).
*/
public async headBlob(
repository: string,
digest: string,
): Promise<{ exists: boolean; size?: number } | null> {
const context: IUpstreamFetchContext = {
protocol: 'oci',
resource: repository,
resourceType: 'blob',
path: `/v2/${repository}/blobs/${digest}`,
method: 'HEAD',
headers: {},
query: {},
};
const result = await this.fetch(context);
if (!result) {
return null;
}
if (!result.success) {
return { exists: false };
}
return {
exists: true,
size: result.headers['content-length'] ? parseInt(result.headers['content-length'], 10) : undefined,
};
}
/**
* Fetch the tag list for a repository.
*/
public async fetchTags(repository: string, n?: number, last?: string): Promise<ITagList | null> {
const query: Record<string, string> = {};
if (n) query.n = n.toString();
if (last) query.last = last;
const context: IUpstreamFetchContext = {
protocol: 'oci',
resource: repository,
resourceType: 'tags',
path: `/v2/${repository}/tags/list`,
method: 'GET',
headers: {
'accept': 'application/json',
},
query,
};
const result = await this.fetch(context);
if (!result || !result.success) {
return null;
}
let tagList: ITagList;
if (Buffer.isBuffer(result.body)) {
tagList = JSON.parse(result.body.toString('utf8'));
} else {
tagList = result.body;
}
return tagList;
}
/**
* Override URL building for OCI-specific handling.
* OCI registries use /v2/ prefix and may require special handling for Docker Hub.
*/
protected buildUpstreamUrl(
upstream: IUpstreamRegistryConfig,
context: IUpstreamFetchContext,
): string {
let baseUrl = upstream.url;
// Remove trailing slash
if (baseUrl.endsWith('/')) {
baseUrl = baseUrl.slice(0, -1);
}
// Handle Docker Hub special case
// Docker Hub uses registry-1.docker.io but library images need special handling
if (baseUrl.includes('docker.io') || baseUrl.includes('registry-1.docker.io')) {
// For library images (e.g., "nginx" -> "library/nginx")
const pathParts = context.path.match(/^\/v2\/([^\/]+)\/(.+)$/);
if (pathParts) {
const [, repository, rest] = pathParts;
// If repository doesn't contain a slash, it's a library image
if (!repository.includes('/')) {
return `${baseUrl}/v2/library/${repository}/${rest}`;
}
}
}
return `${baseUrl}${context.path}`;
}
/**
* Override header building for OCI-specific authentication.
* OCI registries may require token-based auth obtained from a separate endpoint.
*/
protected buildHeaders(
upstream: IUpstreamRegistryConfig,
context: IUpstreamFetchContext,
): Record<string, string> {
const headers = super.buildHeaders(upstream, context);
// OCI registries typically use Docker-Distribution-API-Version header
headers['docker-distribution-api-version'] = 'registry/2.0';
return headers;
}
}

1
ts/oci/index.ts
View File

@@ -3,4 +3,5 @@
*/
export { OciRegistry } from './classes.ociregistry.js';
export { OciUpstream } from './classes.ociupstream.js';
export * from './interfaces.oci.js';