From 41405eb40a28e24a4b3d28cef894467ed522e57c Mon Sep 17 00:00:00 2001 From: Juergen Kunz Date: Tue, 25 Nov 2025 22:10:06 +0000 Subject: [PATCH] feat(core/registrystorage): Persist OCI manifest content-type in sidecar and normalize manifest body handling --- changelog.md | 9 + test/test.oci.nativecli.node.ts | 406 +++++++++++++++++++++++++++++ ts/00_commitinfo_data.ts | 2 +- ts/core/classes.registrystorage.ts | 20 +- ts/oci/classes.ociregistry.ts | 96 +++++-- 5 files changed, 511 insertions(+), 22 deletions(-) create mode 100644 test/test.oci.nativecli.node.ts diff --git a/changelog.md b/changelog.md index 7fc6220..ca4c570 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,14 @@ # Changelog +## 2025-11-25 - 2.2.0 - feat(core/registrystorage) +Persist OCI manifest content-type in sidecar and normalize manifest body handling + +- Add getOciManifestContentType(repository, digest) to read stored manifest Content-Type +- Store manifest Content-Type in a .type sidecar file when putOciManifest is called +- Update putOciManifest to persist both manifest data and its content type +- OciRegistry now retrieves stored content type (with fallback to detectManifestContentType) when serving manifests +- Add toBuffer helper in OciRegistry to consistently convert various request body forms to Buffer for digest calculation and uploads + ## 2025-11-25 - 2.1.2 - fix(oci) Prefer raw request body for content-addressable OCI operations and expose rawBody on request context diff --git a/test/test.oci.nativecli.node.ts b/test/test.oci.nativecli.node.ts new file mode 100644 index 0000000..f572a72 --- /dev/null +++ b/test/test.oci.nativecli.node.ts @@ -0,0 +1,406 @@ +/** + * Native Docker CLI Testing + * Tests the OCI registry implementation using the actual Docker CLI + */ + +import { expect, tap } from '@git.zone/tstest/tapbundle'; +import { tapNodeTools } from '@git.zone/tstest/tapbundle_serverside'; +import { SmartRegistry } from '../ts/index.js'; +import type { IRequestContext, IResponse, IRegistryConfig } from '../ts/core/interfaces.core.js'; +import * as qenv from '@push.rocks/qenv'; +import * as http from 'http'; +import * as url from 'url'; +import * as fs from 'fs'; +import * as path from 'path'; + +const testQenv = new qenv.Qenv('./', './.nogit'); + +/** + * Create a test registry with local token endpoint realm + */ +async function createDockerTestRegistry(port: number): Promise { + const s3AccessKey = await testQenv.getEnvVarOnDemand('S3_ACCESSKEY'); + const s3SecretKey = await testQenv.getEnvVarOnDemand('S3_SECRETKEY'); + const s3Endpoint = await testQenv.getEnvVarOnDemand('S3_ENDPOINT'); + const s3Port = await testQenv.getEnvVarOnDemand('S3_PORT'); + + const config: IRegistryConfig = { + storage: { + accessKey: s3AccessKey || 'minioadmin', + accessSecret: s3SecretKey || 'minioadmin', + endpoint: s3Endpoint || 'localhost', + port: parseInt(s3Port || '9000', 10), + useSsl: false, + region: 'us-east-1', + bucketName: 'test-registry', + }, + auth: { + jwtSecret: 'test-secret-key', + tokenStore: 'memory', + npmTokens: { + enabled: true, + }, + ociTokens: { + enabled: true, + realm: `http://localhost:${port}/v2/token`, + service: 'test-registry', + }, + }, + oci: { + enabled: true, + basePath: '/oci', + }, + }; + + const reg = new SmartRegistry(config); + await reg.init(); + return reg; +} + +/** + * Create test tokens for the registry + */ +async function createDockerTestTokens(reg: SmartRegistry) { + const authManager = reg.getAuthManager(); + + const userId = await authManager.authenticate({ + username: 'testuser', + password: 'testpass', + }); + + if (!userId) { + throw new Error('Failed to authenticate test user'); + } + + // Create OCI token with full access + const ociToken = await authManager.createOciToken( + userId, + ['oci:repository:*:*'], + 3600 + ); + + return { ociToken, userId }; +} + +// Test context +let registry: SmartRegistry; +let server: http.Server; +let registryUrl: string; +let registryPort: number; +let ociToken: string; +let testDir: string; +let testImageName: string; + +/** + * Create HTTP server wrapper around SmartRegistry + * CRITICAL: Always passes rawBody for content-addressable operations (OCI manifests/blobs) + * + * Docker expects registry at /v2/ but SmartRegistry serves at /oci/v2/ + * This wrapper rewrites paths for Docker compatibility + * + * Also implements a simple /v2/token endpoint for Docker Bearer auth flow + */ +async function createHttpServer( + registryInstance: SmartRegistry, + port: number, + tokenForAuth: string +): Promise<{ server: http.Server; url: string }> { + return new Promise((resolve, reject) => { + const httpServer = http.createServer(async (req, res) => { + try { + // Parse request + const parsedUrl = url.parse(req.url || '', true); + let pathname = parsedUrl.pathname || '/'; + const query = parsedUrl.query; + + // Handle token endpoint for Docker Bearer auth + if (pathname === '/v2/token' || pathname === '/token') { + console.log(`[Token Request] ${req.method} ${req.url}`); + res.statusCode = 200; + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ + token: tokenForAuth, + access_token: tokenForAuth, + expires_in: 3600, + issued_at: new Date().toISOString(), + })); + return; + } + + // Log all requests for debugging + console.log(`[Registry] ${req.method} ${pathname}`); + + // Docker expects /v2/ but SmartRegistry serves at /oci/v2/ + if (pathname.startsWith('/v2')) { + pathname = '/oci' + pathname; + } + + // Read raw body - ALWAYS preserve exact bytes for OCI + const chunks: Buffer[] = []; + for await (const chunk of req) { + chunks.push(chunk); + } + const bodyBuffer = Buffer.concat(chunks); + + // Parse body based on content type (for non-OCI protocols that need it) + let parsedBody: any; + if (bodyBuffer.length > 0) { + const contentType = req.headers['content-type'] || ''; + if (contentType.includes('application/json')) { + try { + parsedBody = JSON.parse(bodyBuffer.toString('utf-8')); + } catch (error) { + parsedBody = bodyBuffer; + } + } else { + parsedBody = bodyBuffer; + } + } + + // Convert to IRequestContext + const context: IRequestContext = { + method: req.method || 'GET', + path: pathname, + headers: req.headers as Record, + query: query as Record, + body: parsedBody, + rawBody: bodyBuffer, + }; + + // Handle request + const response: IResponse = await registryInstance.handleRequest(context); + console.log(`[Registry] Response: ${response.status} for ${pathname}`); + + // Convert IResponse to HTTP response + res.statusCode = response.status; + + // Set headers + for (const [key, value] of Object.entries(response.headers || {})) { + res.setHeader(key, value); + } + + // Send body + if (response.body) { + if (Buffer.isBuffer(response.body)) { + res.end(response.body); + } else if (typeof response.body === 'string') { + res.end(response.body); + } else { + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify(response.body)); + } + } else { + res.end(); + } + } catch (error) { + console.error('Server error:', error); + res.statusCode = 500; + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ error: 'INTERNAL_ERROR', message: String(error) })); + } + }); + + httpServer.listen(port, '0.0.0.0', () => { + const serverUrl = `http://localhost:${port}`; + resolve({ server: httpServer, url: serverUrl }); + }); + + httpServer.on('error', reject); + }); +} + +/** + * Create a test Dockerfile + */ +function createTestDockerfile(targetDir: string, content?: string): string { + const dockerfilePath = path.join(targetDir, 'Dockerfile'); + const dockerfileContent = content || `FROM alpine:latest +RUN echo "Hello from SmartRegistry test" > /hello.txt +CMD ["cat", "/hello.txt"] +`; + fs.writeFileSync(dockerfilePath, dockerfileContent, 'utf-8'); + return dockerfilePath; +} + +/** + * Run Docker command using the main Docker daemon (not rootless) + * Rootless Docker runs in its own network namespace and can't access host localhost + * + * IMPORTANT: DOCKER_HOST env var overrides --context flag, so we must unset it + * and explicitly set the socket path to use the main Docker daemon. + */ +async function runDockerCommand( + command: string, + cwd?: string +): Promise<{ stdout: string; stderr: string; exitCode: number }> { + // First unset DOCKER_HOST then set it to main Docker daemon socket + // Using both unset and export ensures we override any inherited env var + const dockerCommand = `unset DOCKER_HOST && export DOCKER_HOST=unix:///var/run/docker.sock && ${command}`; + const fullCommand = cwd ? `cd "${cwd}" && ${dockerCommand}` : dockerCommand; + + try { + const result = await tapNodeTools.runCommand(fullCommand); + return { + stdout: result.stdout || '', + stderr: result.stderr || '', + exitCode: result.exitCode || 0, + }; + } catch (error: any) { + return { + stdout: error.stdout || '', + stderr: error.stderr || String(error), + exitCode: error.exitCode || 1, + }; + } +} + +/** + * Cleanup test directory + */ +function cleanupTestDir(dir: string): void { + if (fs.existsSync(dir)) { + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +/** + * Cleanup Docker resources + */ +async function cleanupDocker(imageName: string): Promise { + await runDockerCommand(`docker rmi ${imageName} 2>/dev/null || true`); + await runDockerCommand(`docker rmi ${imageName}:v1 2>/dev/null || true`); + await runDockerCommand(`docker rmi ${imageName}:v2 2>/dev/null || true`); +} + +// ======================================================================== +// TESTS +// ======================================================================== + +tap.test('Docker CLI: should verify Docker is installed', async () => { + const result = await runDockerCommand('docker version'); + console.log('Docker version output:', result.stdout.substring(0, 200)); + expect(result.exitCode).toEqual(0); +}); + +tap.test('Docker CLI: should setup registry and HTTP server', async () => { + // Use localhost - Docker allows HTTP for localhost without any special config + registryPort = 15000 + Math.floor(Math.random() * 1000); + console.log(`Using port: ${registryPort}`); + + registry = await createDockerTestRegistry(registryPort); + const tokens = await createDockerTestTokens(registry); + ociToken = tokens.ociToken; + + expect(registry).toBeInstanceOf(SmartRegistry); + expect(ociToken).toBeTypeOf('string'); + + const serverSetup = await createHttpServer(registry, registryPort, ociToken); + server = serverSetup.server; + registryUrl = serverSetup.url; + + expect(server).toBeDefined(); + console.log(`Registry server started at ${registryUrl}`); + + // Setup test directory + testDir = path.join(process.cwd(), '.nogit', 'test-docker-cli'); + cleanupTestDir(testDir); + fs.mkdirSync(testDir, { recursive: true }); + + testImageName = `localhost:${registryPort}/test-image`; +}); + +tap.test('Docker CLI: should verify server is responding', async () => { + // Give the server a moment to fully initialize + await new Promise(resolve => setTimeout(resolve, 500)); + + const response = await fetch(`${registryUrl}/oci/v2/`); + expect(response.status).toEqual(200); + console.log('OCI v2 response:', await response.json()); +}); + +tap.test('Docker CLI: should login to registry', async () => { + const result = await runDockerCommand( + `echo "${ociToken}" | docker login localhost:${registryPort} -u testuser --password-stdin` + ); + console.log('docker login output:', result.stdout); + console.log('docker login stderr:', result.stderr); + + const combinedOutput = result.stdout + result.stderr; + expect(combinedOutput).toContain('Login Succeeded'); +}); + +tap.test('Docker CLI: should build test image', async () => { + createTestDockerfile(testDir); + + const result = await runDockerCommand( + `docker build -t ${testImageName}:v1 .`, + testDir + ); + console.log('docker build output:', result.stdout.substring(0, 500)); + + expect(result.exitCode).toEqual(0); +}); + +tap.test('Docker CLI: should push image to registry', async () => { + // This is the critical test - if the digest mismatch bug is fixed, + // this should succeed. The manifest bytes must be preserved exactly. + const result = await runDockerCommand(`docker push ${testImageName}:v1`); + console.log('docker push output:', result.stdout); + console.log('docker push stderr:', result.stderr); + + expect(result.exitCode).toEqual(0); +}); + +tap.test('Docker CLI: should verify manifest in registry via API', async () => { + const response = await fetch(`${registryUrl}/oci/v2/test-image/tags/list`, { + headers: { Authorization: `Bearer ${ociToken}` }, + }); + + expect(response.status).toEqual(200); + + const tagList = await response.json(); + console.log('Tags list:', tagList); + + expect(tagList.name).toEqual('test-image'); + expect(tagList.tags).toContain('v1'); +}); + +tap.test('Docker CLI: should pull pushed image', async () => { + // First remove the local image + await runDockerCommand(`docker rmi ${testImageName}:v1 || true`); + + const result = await runDockerCommand(`docker pull ${testImageName}:v1`); + console.log('docker pull output:', result.stdout); + + expect(result.exitCode).toEqual(0); +}); + +tap.test('Docker CLI: should run pulled image', async () => { + const result = await runDockerCommand(`docker run --rm ${testImageName}:v1`); + console.log('docker run output:', result.stdout); + + expect(result.exitCode).toEqual(0); + expect(result.stdout).toContain('Hello from SmartRegistry test'); +}); + +tap.postTask('cleanup docker cli tests', async () => { + if (testImageName) { + await cleanupDocker(testImageName); + } + + if (server) { + await new Promise((resolve) => { + server.close(() => resolve()); + }); + } + + if (testDir) { + cleanupTestDir(testDir); + } + + if (registry) { + registry.destroy(); + } +}); + +export default tap.start(); diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index 29cf398..597d64f 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@push.rocks/smartregistry', - version: '2.1.2', + version: '2.2.0', description: 'A composable TypeScript library implementing OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems registries for building unified container and package registries' } diff --git a/ts/core/classes.registrystorage.ts b/ts/core/classes.registrystorage.ts index a35aa5e..28c5c1c 100644 --- a/ts/core/classes.registrystorage.ts +++ b/ts/core/classes.registrystorage.ts @@ -129,7 +129,7 @@ export class RegistryStorage implements IStorageBackend { } /** - * Get OCI manifest + * Get OCI manifest and its content type */ public async getOciManifest(repository: string, digest: string): Promise { const path = this.getOciManifestPath(repository, digest); @@ -137,7 +137,17 @@ export class RegistryStorage implements IStorageBackend { } /** - * Store OCI manifest + * Get OCI manifest content type + * Returns the stored content type or null if not found + */ + public async getOciManifestContentType(repository: string, digest: string): Promise { + const typePath = this.getOciManifestPath(repository, digest) + '.type'; + const data = await this.getObject(typePath); + return data ? data.toString('utf-8') : null; + } + + /** + * Store OCI manifest with its content type */ public async putOciManifest( repository: string, @@ -146,7 +156,11 @@ export class RegistryStorage implements IStorageBackend { contentType: string ): Promise { const path = this.getOciManifestPath(repository, digest); - return this.putObject(path, data, { 'Content-Type': contentType }); + // Store manifest data + await this.putObject(path, data, { 'Content-Type': contentType }); + // Store content type in sidecar file for later retrieval + const typePath = path + '.type'; + await this.putObject(typePath, Buffer.from(contentType, 'utf-8')); } /** diff --git a/ts/oci/classes.ociregistry.ts b/ts/oci/classes.ociregistry.ts index 1a49634..2ed5676 100644 --- a/ts/oci/classes.ociregistry.ts +++ b/ts/oci/classes.ociregistry.ts @@ -198,7 +198,7 @@ export class OciRegistry extends BaseRegistry { const digest = query.digest; if (digest && body) { // Monolithic upload: complete upload in single POST - const blobData = Buffer.isBuffer(body) ? body : Buffer.from(JSON.stringify(body)); + const blobData = this.toBuffer(body); // Verify digest const calculatedDigest = await this.calculateDigest(blobData); @@ -320,10 +320,17 @@ export class OciRegistry extends BaseRegistry { }; } + // Get stored content type, falling back to detecting from manifest content + let contentType = await this.storage.getOciManifestContentType(repository, digest); + if (!contentType) { + // Fallback: detect content type from manifest content + contentType = this.detectManifestContentType(manifestData); + } + return { status: 200, headers: { - 'Content-Type': 'application/vnd.oci.image.manifest.v1+json', + 'Content-Type': contentType, 'Docker-Content-Digest': digest, }, body: manifestData, @@ -356,10 +363,18 @@ export class OciRegistry extends BaseRegistry { const manifestData = await this.storage.getOciManifest(repository, digest); + // Get stored content type, falling back to detecting from manifest content + let contentType = await this.storage.getOciManifestContentType(repository, digest); + if (!contentType && manifestData) { + // Fallback: detect content type from manifest content + contentType = this.detectManifestContentType(manifestData); + } + contentType = contentType || 'application/vnd.oci.image.manifest.v1+json'; + return { status: 200, headers: { - 'Content-Type': 'application/vnd.oci.image.manifest.v1+json', + 'Content-Type': contentType, 'Docker-Content-Digest': digest, 'Content-Length': manifestData ? manifestData.length.toString() : '0', }, @@ -388,16 +403,7 @@ export class OciRegistry extends BaseRegistry { // Preserve raw bytes for accurate digest calculation // Per OCI spec, digest must match the exact bytes sent by client - let manifestData: Buffer; - if (Buffer.isBuffer(body)) { - manifestData = body; - } else if (typeof body === 'string') { - // String body - convert directly without JSON transformation - manifestData = Buffer.from(body, 'utf-8'); - } else { - // Body was already parsed as JSON object - re-serialize as fallback - manifestData = Buffer.from(JSON.stringify(body)); - } + const manifestData = this.toBuffer(body); const contentType = headers?.['content-type'] || headers?.['Content-Type'] || 'application/vnd.oci.image.manifest.v1+json'; // Calculate manifest digest @@ -525,7 +531,7 @@ export class OciRegistry extends BaseRegistry { private async uploadChunk( uploadId: string, - data: Buffer, + data: Buffer | Uint8Array | unknown, contentRange: string ): Promise { const session = this.uploadSessions.get(uploadId); @@ -537,8 +543,9 @@ export class OciRegistry extends BaseRegistry { }; } - session.chunks.push(data); - session.totalSize += data.length; + const chunkData = this.toBuffer(data); + session.chunks.push(chunkData); + session.totalSize += chunkData.length; session.lastActivity = new Date(); return { @@ -555,7 +562,7 @@ export class OciRegistry extends BaseRegistry { private async completeUpload( uploadId: string, digest: string, - finalData?: Buffer + finalData?: Buffer | Uint8Array | unknown ): Promise { const session = this.uploadSessions.get(uploadId); if (!session) { @@ -567,7 +574,7 @@ export class OciRegistry extends BaseRegistry { } const chunks = [...session.chunks]; - if (finalData) chunks.push(finalData); + if (finalData) chunks.push(this.toBuffer(finalData)); const blobData = Buffer.concat(chunks); // Verify digest @@ -665,6 +672,59 @@ export class OciRegistry extends BaseRegistry { // HELPER METHODS // ======================================================================== + /** + * Detect manifest content type from manifest content. + * OCI Image Index has "manifests" array, OCI Image Manifest has "config" object. + * Also checks the mediaType field if present. + */ + private detectManifestContentType(manifestData: Buffer): string { + try { + const manifest = JSON.parse(manifestData.toString('utf-8')); + + // First check if manifest has explicit mediaType field + if (manifest.mediaType) { + return manifest.mediaType; + } + + // Otherwise detect from structure + if (Array.isArray(manifest.manifests)) { + // OCI Image Index (multi-arch manifest list) + return 'application/vnd.oci.image.index.v1+json'; + } else if (manifest.config) { + // OCI Image Manifest + return 'application/vnd.oci.image.manifest.v1+json'; + } + + // Fallback to standard manifest type + return 'application/vnd.oci.image.manifest.v1+json'; + } catch (e) { + // If parsing fails, return default + return 'application/vnd.oci.image.manifest.v1+json'; + } + } + + /** + * Convert any binary-like data to Buffer. + * Handles Buffer, Uint8Array (modern cross-platform), string, and objects. + * + * Note: Buffer.isBuffer(Uint8Array) returns false even though Buffer extends Uint8Array. + * This is because Uint8Array is the modern, cross-platform standard while Buffer is Node.js-specific. + * Many HTTP frameworks pass request bodies as Uint8Array for better compatibility. + */ + private toBuffer(data: unknown): Buffer { + if (Buffer.isBuffer(data)) { + return data; + } + if (data instanceof Uint8Array) { + return Buffer.from(data); + } + if (typeof data === 'string') { + return Buffer.from(data, 'utf-8'); + } + // Fallback: serialize object to JSON (may cause digest mismatch for manifests) + return Buffer.from(JSON.stringify(data)); + } + private async getTagsData(repository: string): Promise> { const path = `oci/tags/${repository}/tags.json`; const data = await this.storage.getObject(path);