From 334a036d027408a702e6d79ca14f266cf35541f8 Mon Sep 17 00:00:00 2001 From: Juergen Kunz Date: Tue, 2 Jun 2026 15:19:03 +0000 Subject: [PATCH] fix(registrycopy): send Content-Range for chunked blob upload PATCH requests --- changelog.md | 6 + package.json | 2 +- test.registrycopy.node.ts | 254 +++++++++++++++++++++++++++++++++++++ ts/classes.registrycopy.ts | 2 + 4 files changed, 263 insertions(+), 1 deletion(-) create mode 100644 test.registrycopy.node.ts diff --git a/changelog.md b/changelog.md index b7b7bb0..0858e41 100644 --- a/changelog.md +++ b/changelog.md @@ -4,6 +4,12 @@ +### Fixes + +- send Content-Range for chunked blob upload PATCH requests (registrycopy) + - Adds Content-Range values based on the current upload offset and chunk length for each PATCH upload chunk. + - Adds a regression test covering multi-patch registry uploads that reject stream-style continuation. + ## 2026-06-02 - 2.4.1 ### Fixes diff --git a/package.json b/package.json index 03acf6d..1c61c7d 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "tsdocker": "./cli.js" }, "scripts": { - "test": "(pnpm run build)", + "test": "(pnpm run build) && tstest test.registrycopy.node.ts --verbose", "build": "(tsbuild)", "buildDocs": "tsdoc" }, diff --git a/test.registrycopy.node.ts b/test.registrycopy.node.ts new file mode 100644 index 0000000..d36d596 --- /dev/null +++ b/test.registrycopy.node.ts @@ -0,0 +1,254 @@ +import * as crypto from 'node:crypto'; +import * as http from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { tap, expect } from '@git.zone/tstest/tapbundle'; + +import { RegistryCopy } from './ts/classes.registrycopy.js'; + +const chunkSize = 32 * 1024 * 1024; + +const createDigest = (data: Buffer): string => { + return `sha256:${crypto.createHash('sha256').update(data).digest('hex')}`; +}; + +const readRequestBody = async (request: http.IncomingMessage): Promise => { + const chunks: Buffer[] = []; + for await (const chunk of request) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + return Buffer.concat(chunks); +}; + +const startServer = async ( + handler: http.RequestListener, +): Promise<{ server: http.Server; registry: string }> => { + const server = http.createServer(handler); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const address = server.address() as AddressInfo; + return { server, registry: `127.0.0.1:${address.port}` }; +}; + +const closeServer = async (server: http.Server): Promise => { + await new Promise((resolve, reject) => { + server.close((error) => error ? reject(error) : resolve()); + }); +}; + +tap.test('should include Content-Range for multi-patch Gitea uploads', async () => { + const configBlob = Buffer.from(JSON.stringify({ created: '2026-06-02T00:00:00.000Z' })); + const configDigest = createDigest(configBlob); + const layerBlob = Buffer.alloc(chunkSize + 7, 42); + const layerDigest = createDigest(layerBlob); + const manifest = { + schemaVersion: 2, + mediaType: 'application/vnd.oci.image.manifest.v1+json', + config: { + mediaType: 'application/vnd.oci.image.config.v1+json', + digest: configDigest, + size: configBlob.length, + }, + layers: [ + { + mediaType: 'application/vnd.oci.image.layer.v1.tar+gzip', + digest: layerDigest, + size: layerBlob.length, + }, + ], + }; + const manifestRaw = Buffer.from(JSON.stringify(manifest)); + const manifestDigest = createDigest(manifestRaw); + + const source = await startServer(async (request, response) => { + const url = new URL(request.url || '/', 'http://127.0.0.1'); + if (url.pathname === '/v2/') { + response.writeHead(200, { 'Docker-Distribution-Api-Version': 'registry/2.0' }); + response.end(); + return; + } + + if (request.method === 'GET' && url.pathname === '/v2/src/manifests/latest') { + response.writeHead(200, { + 'Content-Type': 'application/vnd.oci.image.manifest.v1+json', + 'Docker-Content-Digest': manifestDigest, + }); + response.end(manifestRaw); + return; + } + + if (request.method === 'GET' && url.pathname.startsWith('/v2/src/blobs/')) { + const digest = url.pathname.substring('/v2/src/blobs/'.length); + const body = digest === configDigest + ? configBlob + : digest === layerDigest + ? layerBlob + : null; + if (!body) { + response.writeHead(404); + response.end(); + return; + } + response.writeHead(200, { + 'Content-Type': 'application/octet-stream', + 'Content-Length': String(body.length), + 'Docker-Content-Digest': digest, + }); + response.end(body); + return; + } + + response.writeHead(404); + response.end(); + }); + + const uploadSessions = new Map(); + const blobRanges = new Map(); + const storedBlobs = new Set(); + let uploadIndex = 0; + + const destination = await startServer(async (request, response) => { + const url = new URL(request.url || '/', 'http://127.0.0.1'); + if (url.pathname === '/v2/') { + response.writeHead(200, { 'Docker-Distribution-Api-Version': 'registry/2.0' }); + response.end(); + return; + } + + if (request.method === 'HEAD' && url.pathname.startsWith('/v2/dest/blobs/')) { + const digest = url.pathname.substring('/v2/dest/blobs/'.length); + response.writeHead(storedBlobs.has(digest) ? 200 : 404, storedBlobs.has(digest) ? { + 'Docker-Content-Digest': digest, + } : {}); + response.end(); + return; + } + + if (request.method === 'POST' && url.pathname === '/v2/dest/blobs/uploads/') { + const uploadId = `upload-${++uploadIndex}`; + uploadSessions.set(uploadId, { size: 0, chunks: [], ranges: [] }); + response.writeHead(202, { + Location: `/v2/dest/blobs/uploads/${uploadId}`, + 'Docker-Upload-UUID': uploadId, + }); + response.end(); + return; + } + + if (url.pathname.startsWith('/v2/dest/blobs/uploads/')) { + const uploadId = url.pathname.substring('/v2/dest/blobs/uploads/'.length); + const upload = uploadSessions.get(uploadId); + if (!upload) { + response.writeHead(404); + response.end(); + return; + } + + if (request.method === 'GET') { + response.writeHead(204, { + Location: `/v2/dest/blobs/uploads/${uploadId}`, + Range: upload.size > 0 ? `0-${upload.size - 1}` : '0-0', + 'Docker-Upload-UUID': uploadId, + }); + response.end(); + return; + } + + if (request.method === 'PATCH') { + const body = await readRequestBody(request); + const contentRange = request.headers['content-range']; + if (!contentRange && upload.size !== 0) { + response.writeHead(400, { 'Content-Type': 'application/json' }); + response.end(JSON.stringify({ + errors: [ + { + code: 'BLOB_UPLOAD_INVALID', + message: 'Stream uploads after first write are not allowed', + }, + ], + })); + return; + } + + if (typeof contentRange === 'string') { + const rangeMatch = contentRange.match(/^(\d+)-(\d+)$/); + const start = rangeMatch ? Number(rangeMatch[1]) : NaN; + const end = rangeMatch ? Number(rangeMatch[2]) : NaN; + if (start !== upload.size || end !== upload.size + body.length - 1) { + response.writeHead(416, { + Location: `/v2/dest/blobs/uploads/${uploadId}`, + Range: upload.size > 0 ? `0-${upload.size - 1}` : '0-0', + }); + response.end(); + return; + } + upload.ranges.push(contentRange); + } + + upload.chunks.push(body); + upload.size += body.length; + response.writeHead(202, { + Location: `/v2/dest/blobs/uploads/${uploadId}`, + Range: `0-${upload.size - 1}`, + 'Docker-Upload-UUID': uploadId, + }); + response.end(); + return; + } + + if (request.method === 'PUT') { + const digest = url.searchParams.get('digest') || ''; + const finalBody = await readRequestBody(request); + if (finalBody.length > 0) { + upload.chunks.push(finalBody); + upload.size += finalBody.length; + } + const blob = Buffer.concat(upload.chunks); + if (createDigest(blob) !== digest) { + response.writeHead(400, { 'Content-Type': 'application/json' }); + response.end(JSON.stringify({ errors: [{ code: 'DIGEST_INVALID', message: 'Digest mismatch' }] })); + return; + } + storedBlobs.add(digest); + blobRanges.set(digest, [...upload.ranges]); + uploadSessions.delete(uploadId); + response.writeHead(201, { + Location: `/v2/dest/blobs/${digest}`, + 'Docker-Content-Digest': digest, + }); + response.end(); + return; + } + + if (request.method === 'DELETE') { + uploadSessions.delete(uploadId); + response.writeHead(204); + response.end(); + return; + } + } + + if (request.method === 'PUT' && url.pathname === '/v2/dest/manifests/latest') { + await readRequestBody(request); + response.writeHead(201, { 'Docker-Content-Digest': manifestDigest }); + response.end(); + return; + } + + response.writeHead(404); + response.end(); + }); + + try { + const registryCopy = new RegistryCopy(); + await registryCopy.copyImage(source.registry, 'src', 'latest', destination.registry, 'dest', 'latest'); + + expect(blobRanges.get(layerDigest)).toEqual([ + `0-${chunkSize - 1}`, + `${chunkSize}-${chunkSize + 6}`, + ]); + } finally { + await closeServer(source.server); + await closeServer(destination.server); + } +}); + +export default tap.start(); diff --git a/ts/classes.registrycopy.ts b/ts/classes.registrycopy.ts index 42789c8..002d94a 100644 --- a/ts/classes.registrycopy.ts +++ b/ts/classes.registrycopy.ts @@ -317,11 +317,13 @@ export class RegistryCopy { while (remainingChunk.length > 0) { try { + const chunkEnd = currentUploadedBytes + remainingChunk.length - 1; const resp = await this.fetchUploadUrl(registry, repo, currentUploadUrl, { method: 'PATCH', headers: { 'Content-Type': 'application/octet-stream', 'Content-Length': String(remainingChunk.length), + 'Content-Range': `${currentUploadedBytes}-${chunkEnd}`, }, body: remainingChunk as any, }, credentials, 300_000, 1);