fix(registry): use persistent local registry and OCI Distribution API image copy for pushes
This commit is contained in:
11
changelog.md
11
changelog.md
@@ -1,5 +1,16 @@
|
||||
# Changelog
|
||||
|
||||
## 2026-02-07 - 1.15.1 - fix(registry)
|
||||
use persistent local registry and OCI Distribution API image copy for pushes
|
||||
|
||||
- Adds RegistryCopy class implementing the OCI Distribution API to copy images (including multi-arch manifest lists) from the local registry to remote registries.
|
||||
- All builds now go through a persistent local registry at localhost:5234 with volume storage at .nogit/docker-registry/; Dockerfile.startLocalRegistry mounts this directory.
|
||||
- Dockerfile.push now delegates to RegistryCopy.copyImage; Dockerfile.needsLocalRegistry() always returns true and config.push is now a no-op (kept for backward compat).
|
||||
- Multi-platform buildx builds are pushed to the local registry (this.localRegistryTag) during buildx --push; code avoids redundant pushes when images are already pushed by buildx.
|
||||
- Build, cached build, test, push and pull flows now start/stop the local registry automatically to support multi-platform/image resolution.
|
||||
- Introduces Dockerfile.getDestRepo and support for config.registryRepoMap to control destination repository mapping.
|
||||
- Breaking change: registry usage and push behavior changed (config.push ignored and local registry mandatory) — bump major version.
|
||||
|
||||
## 2026-02-07 - 1.15.0 - feat(clean)
|
||||
Make the `clean` command interactive: add smartinteract prompts, docker context detection, and selective resource removal with support for --all and -y auto-confirm
|
||||
|
||||
|
||||
@@ -108,6 +108,18 @@ tsdocker build --parallel --cached # works with both modes
|
||||
|
||||
Implementation: `Dockerfile.computeLevels()` groups topologically sorted Dockerfiles into dependency levels. `Dockerfile.runWithConcurrency()` provides a worker-pool pattern for bounded concurrency. Both are public static methods on the `Dockerfile` class. The parallel logic exists in both `Dockerfile.buildDockerfiles()` (standard mode) and `TsDockerManager.build()` (cached mode).
|
||||
|
||||
## OCI Distribution API Push (v1.16+)
|
||||
|
||||
All builds now go through a persistent local registry (`localhost:5234`) with volume storage at `.nogit/docker-registry/`. Pushes use the `RegistryCopy` class (`ts/classes.registrycopy.ts`) which implements the OCI Distribution API to copy images (including multi-arch manifest lists) from the local registry to remote registries. This replaces the old `docker tag + docker push` approach that only worked for single-platform images.
|
||||
|
||||
Key classes:
|
||||
- `RegistryCopy` — HTTP-based OCI image copy (auth, blob transfer, manifest handling)
|
||||
- `Dockerfile.push()` — Now delegates to `RegistryCopy.copyImage()`
|
||||
- `Dockerfile.needsLocalRegistry()` — Always returns true
|
||||
- `Dockerfile.startLocalRegistry()` — Uses persistent volume mount
|
||||
|
||||
The `config.push` field is now a no-op (kept for backward compat).
|
||||
|
||||
## Build Status
|
||||
|
||||
- Build: ✅ Passes
|
||||
|
||||
@@ -3,6 +3,6 @@
|
||||
*/
|
||||
export const commitinfo = {
|
||||
name: '@git.zone/tsdocker',
|
||||
version: '1.15.0',
|
||||
version: '1.15.1',
|
||||
description: 'develop npm modules cross platform with docker'
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ import * as plugins from './tsdocker.plugins.js';
|
||||
import * as paths from './tsdocker.paths.js';
|
||||
import { logger, formatDuration } from './tsdocker.logging.js';
|
||||
import { DockerRegistry } from './classes.dockerregistry.js';
|
||||
import { RegistryCopy } from './classes.registrycopy.js';
|
||||
import type { IDockerfileOptions, ITsDockerConfig, IBuildCommandOptions } from './interfaces/index.js';
|
||||
import type { TsDockerManager } from './classes.tsdockermanager.js';
|
||||
import * as fs from 'fs';
|
||||
@@ -139,31 +140,32 @@ export class Dockerfile {
|
||||
return sortedDockerfileArray;
|
||||
}
|
||||
|
||||
/** Determines if a local registry is needed for buildx dependency resolution. */
|
||||
/** Local registry is always needed — it's the canonical store for all built images. */
|
||||
public static needsLocalRegistry(
|
||||
dockerfiles: Dockerfile[],
|
||||
options?: { platform?: string },
|
||||
_dockerfiles?: Dockerfile[],
|
||||
_options?: { platform?: string },
|
||||
): boolean {
|
||||
const hasLocalDeps = dockerfiles.some(df => df.localBaseImageDependent);
|
||||
if (!hasLocalDeps) return false;
|
||||
const config = dockerfiles[0]?.managerRef?.config;
|
||||
return !!options?.platform || !!(config?.platforms && config.platforms.length > 1);
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Starts a temporary registry:2 container on port 5234. */
|
||||
/** Starts a persistent registry:2 container on port 5234 with volume storage. */
|
||||
public static async startLocalRegistry(isRootless?: boolean): Promise<void> {
|
||||
// Ensure persistent storage directory exists
|
||||
const registryDataDir = plugins.path.join(paths.cwd, '.nogit', 'docker-registry');
|
||||
fs.mkdirSync(registryDataDir, { recursive: true });
|
||||
|
||||
await smartshellInstance.execSilent(
|
||||
`docker rm -f ${LOCAL_REGISTRY_CONTAINER} 2>/dev/null || true`
|
||||
);
|
||||
const result = await smartshellInstance.execSilent(
|
||||
`docker run -d --name ${LOCAL_REGISTRY_CONTAINER} -p ${LOCAL_REGISTRY_PORT}:5000 registry:2`
|
||||
`docker run -d --name ${LOCAL_REGISTRY_CONTAINER} -p ${LOCAL_REGISTRY_PORT}:5000 -v "${registryDataDir}:/var/lib/registry" registry:2`
|
||||
);
|
||||
if (result.exitCode !== 0) {
|
||||
throw new Error(`Failed to start local registry: ${result.stderr || result.stdout}`);
|
||||
}
|
||||
// registry:2 starts near-instantly; brief wait for readiness
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
logger.log('info', `Started local registry at ${LOCAL_REGISTRY_HOST} (buildx dependency bridge)`);
|
||||
logger.log('info', `Started local registry at ${LOCAL_REGISTRY_HOST} (persistent storage at .nogit/docker-registry/)`);
|
||||
if (isRootless) {
|
||||
logger.log('warn', `[rootless] Registry on port ${LOCAL_REGISTRY_PORT} — if buildx cannot reach localhost:${LOCAL_REGISTRY_PORT}, try 127.0.0.1:${LOCAL_REGISTRY_PORT}`);
|
||||
}
|
||||
@@ -246,11 +248,8 @@ export class Dockerfile {
|
||||
): Promise<Dockerfile[]> {
|
||||
const total = sortedArrayArg.length;
|
||||
const overallStart = Date.now();
|
||||
const useRegistry = Dockerfile.needsLocalRegistry(sortedArrayArg, options);
|
||||
|
||||
if (useRegistry) {
|
||||
await Dockerfile.startLocalRegistry(options?.isRootless);
|
||||
}
|
||||
|
||||
try {
|
||||
if (options?.parallel) {
|
||||
@@ -282,8 +281,9 @@ export class Dockerfile {
|
||||
|
||||
await Dockerfile.runWithConcurrency(tasks, concurrency);
|
||||
|
||||
// After the entire level completes, tag + push for dependency resolution
|
||||
// After the entire level completes, push all to local registry + tag for deps
|
||||
for (const df of level) {
|
||||
// Tag in host daemon for dependency resolution
|
||||
const dependentBaseImages = new Set<string>();
|
||||
for (const other of sortedArrayArg) {
|
||||
if (other.localBaseDockerfile === df && other.baseImage !== df.buildTag) {
|
||||
@@ -294,7 +294,8 @@ export class Dockerfile {
|
||||
logger.log('info', `Tagging ${df.buildTag} as ${fullTag} for local dependency resolution`);
|
||||
await smartshellInstance.exec(`docker tag ${df.buildTag} ${fullTag}`);
|
||||
}
|
||||
if (useRegistry && sortedArrayArg.some(other => other.localBaseDockerfile === df)) {
|
||||
// Push ALL images to local registry (skip if already pushed via buildx)
|
||||
if (!df.localRegistryTag) {
|
||||
await Dockerfile.pushToLocalRegistry(df);
|
||||
}
|
||||
}
|
||||
@@ -321,17 +322,15 @@ export class Dockerfile {
|
||||
await smartshellInstance.exec(`docker tag ${dockerfileArg.buildTag} ${fullTag}`);
|
||||
}
|
||||
|
||||
// Push to local registry for buildx dependency resolution
|
||||
if (useRegistry && sortedArrayArg.some(other => other.localBaseDockerfile === dockerfileArg)) {
|
||||
// Push ALL images to local registry (skip if already pushed via buildx)
|
||||
if (!dockerfileArg.localRegistryTag) {
|
||||
await Dockerfile.pushToLocalRegistry(dockerfileArg);
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (useRegistry) {
|
||||
await Dockerfile.stopLocalRegistry();
|
||||
}
|
||||
}
|
||||
|
||||
logger.log('info', `Total build time: ${formatDuration(Date.now() - overallStart)}`);
|
||||
return sortedArrayArg;
|
||||
@@ -594,17 +593,12 @@ export class Dockerfile {
|
||||
buildCommand = `docker buildx build --platform ${platformOverride}${noCacheFlag}${buildContextFlag} --load -t ${this.buildTag} -f ${this.filePath} ${buildArgsString} .`;
|
||||
logger.log('info', `Build: buildx --platform ${platformOverride} --load`);
|
||||
} else if (config.platforms && config.platforms.length > 1) {
|
||||
// Multi-platform build using buildx
|
||||
// Multi-platform build using buildx — always push to local registry
|
||||
const platformString = config.platforms.join(',');
|
||||
buildCommand = `docker buildx build --platform ${platformString}${noCacheFlag}${buildContextFlag} -t ${this.buildTag} -f ${this.filePath} ${buildArgsString} .`;
|
||||
|
||||
if (config.push) {
|
||||
buildCommand += ' --push';
|
||||
logger.log('info', `Build: buildx --platform ${platformString} --push`);
|
||||
} else {
|
||||
buildCommand += ' --load';
|
||||
logger.log('info', `Build: buildx --platform ${platformString} --load`);
|
||||
}
|
||||
const localTag = `${LOCAL_REGISTRY_HOST}/${this.buildTag}`;
|
||||
buildCommand = `docker buildx build --platform ${platformString}${noCacheFlag}${buildContextFlag} -t ${localTag} -f ${this.filePath} ${buildArgsString} --push .`;
|
||||
this.localRegistryTag = localTag;
|
||||
logger.log('info', `Build: buildx --platform ${platformString} --push to local registry`);
|
||||
} else {
|
||||
// Standard build
|
||||
const versionLabel = this.managerRef.projectInfo?.npm?.version || 'unknown';
|
||||
@@ -645,38 +639,39 @@ export class Dockerfile {
|
||||
}
|
||||
|
||||
/**
|
||||
* Pushes the Dockerfile to a registry
|
||||
* Pushes the Dockerfile to a registry using OCI Distribution API copy
|
||||
* from the local registry to the remote registry.
|
||||
*/
|
||||
public async push(dockerRegistryArg: DockerRegistry, versionSuffix?: string): Promise<void> {
|
||||
this.pushTag = Dockerfile.getDockerTagString(
|
||||
this.managerRef,
|
||||
dockerRegistryArg.registryUrl,
|
||||
const destRepo = this.getDestRepo(dockerRegistryArg.registryUrl);
|
||||
const destTag = versionSuffix ? `${this.version}_${versionSuffix}` : this.version;
|
||||
const registryCopy = new RegistryCopy();
|
||||
|
||||
this.pushTag = `${dockerRegistryArg.registryUrl}/${destRepo}:${destTag}`;
|
||||
logger.log('info', `Pushing ${this.pushTag} via OCI copy from local registry...`);
|
||||
|
||||
await registryCopy.copyImage(
|
||||
LOCAL_REGISTRY_HOST,
|
||||
this.repo,
|
||||
this.version,
|
||||
versionSuffix
|
||||
dockerRegistryArg.registryUrl,
|
||||
destRepo,
|
||||
destTag,
|
||||
{ username: dockerRegistryArg.username, password: dockerRegistryArg.password },
|
||||
);
|
||||
|
||||
await smartshellInstance.exec(`docker tag ${this.buildTag} ${this.pushTag}`);
|
||||
const pushResult = await smartshellInstance.exec(`docker push ${this.pushTag}`);
|
||||
|
||||
if (pushResult.exitCode !== 0) {
|
||||
logger.log('error', `Push failed for ${this.pushTag}`);
|
||||
throw new Error(`Push failed for ${this.pushTag}`);
|
||||
}
|
||||
|
||||
// Get image digest
|
||||
const inspectResult = await smartshellInstance.exec(
|
||||
`docker inspect --format="{{index .RepoDigests 0}}" ${this.pushTag}`
|
||||
);
|
||||
|
||||
if (inspectResult.exitCode === 0 && inspectResult.stdout.includes('@')) {
|
||||
const imageDigest = inspectResult.stdout.split('@')[1]?.trim();
|
||||
logger.log('info', `The image ${this.pushTag} has digest ${imageDigest}`);
|
||||
}
|
||||
|
||||
logger.log('ok', `Pushed ${this.pushTag}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the destination repository for a given registry URL,
|
||||
* using registryRepoMap if configured, otherwise the default repo.
|
||||
*/
|
||||
private getDestRepo(registryUrl: string): string {
|
||||
const config = this.managerRef.config;
|
||||
return config.registryRepoMap?.[registryUrl] || this.repo;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pulls the Dockerfile from a registry
|
||||
*/
|
||||
@@ -696,19 +691,22 @@ export class Dockerfile {
|
||||
}
|
||||
|
||||
/**
|
||||
* Tests the Dockerfile by running a test script if it exists
|
||||
* Tests the Dockerfile by running a test script if it exists.
|
||||
* For multi-platform builds, uses the local registry tag so Docker can auto-pull.
|
||||
*/
|
||||
public async test(): Promise<number> {
|
||||
const startTime = Date.now();
|
||||
const testDir = this.managerRef.config.testDir || plugins.path.join(paths.cwd, 'test');
|
||||
const testFile = plugins.path.join(testDir, 'test_' + this.version + '.sh');
|
||||
// Use local registry tag for multi-platform images (not in daemon), otherwise buildTag
|
||||
const imageRef = this.localRegistryTag || this.buildTag;
|
||||
|
||||
const testFileExists = fs.existsSync(testFile);
|
||||
|
||||
if (testFileExists) {
|
||||
// Run tests in container
|
||||
await smartshellInstance.exec(
|
||||
`docker run --name tsdocker_test_container --entrypoint="bash" ${this.buildTag} -c "mkdir /tsdocker_test"`
|
||||
`docker run --name tsdocker_test_container --entrypoint="bash" ${imageRef} -c "mkdir /tsdocker_test"`
|
||||
);
|
||||
await smartshellInstance.exec(`docker cp ${testFile} tsdocker_test_container:/tsdocker_test/test.sh`);
|
||||
await smartshellInstance.exec(`docker commit tsdocker_test_container tsdocker_test_image`);
|
||||
|
||||
511
ts/classes.registrycopy.ts
Normal file
511
ts/classes.registrycopy.ts
Normal file
@@ -0,0 +1,511 @@
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { logger } from './tsdocker.logging.js';
|
||||
|
||||
interface IRegistryCredentials {
|
||||
username: string;
|
||||
password: string;
|
||||
}
|
||||
|
||||
interface ITokenCache {
|
||||
[scope: string]: { token: string; expiry: number };
|
||||
}
|
||||
|
||||
/**
|
||||
* OCI Distribution API client for copying images between registries.
|
||||
* Supports manifest lists (multi-arch) and single-platform manifests.
|
||||
* Uses native fetch (Node 18+).
|
||||
*/
|
||||
export class RegistryCopy {
|
||||
private tokenCache: ITokenCache = {};
|
||||
|
||||
/**
|
||||
* Reads Docker credentials from ~/.docker/config.json for a given registry.
|
||||
* Supports base64-encoded "auth" field in the config.
|
||||
*/
|
||||
public static getDockerConfigCredentials(registryUrl: string): IRegistryCredentials | null {
|
||||
try {
|
||||
const configPath = path.join(os.homedir(), '.docker', 'config.json');
|
||||
if (!fs.existsSync(configPath)) return null;
|
||||
|
||||
const config = JSON.parse(fs.readFileSync(configPath, 'utf-8'));
|
||||
const auths = config.auths || {};
|
||||
|
||||
// Try exact match first, then common variations
|
||||
const keys = [
|
||||
registryUrl,
|
||||
`https://${registryUrl}`,
|
||||
`http://${registryUrl}`,
|
||||
];
|
||||
|
||||
// Docker Hub special cases
|
||||
if (registryUrl === 'docker.io' || registryUrl === 'registry-1.docker.io') {
|
||||
keys.push(
|
||||
'https://index.docker.io/v1/',
|
||||
'https://index.docker.io/v2/',
|
||||
'index.docker.io',
|
||||
'docker.io',
|
||||
'registry-1.docker.io',
|
||||
);
|
||||
}
|
||||
|
||||
for (const key of keys) {
|
||||
if (auths[key]?.auth) {
|
||||
const decoded = Buffer.from(auths[key].auth, 'base64').toString('utf-8');
|
||||
const colonIndex = decoded.indexOf(':');
|
||||
if (colonIndex > 0) {
|
||||
return {
|
||||
username: decoded.substring(0, colonIndex),
|
||||
password: decoded.substring(colonIndex + 1),
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the API base URL for a registry.
|
||||
* Docker Hub uses registry-1.docker.io as API endpoint.
|
||||
*/
|
||||
private getRegistryApiBase(registry: string): string {
|
||||
if (registry === 'docker.io' || registry === 'index.docker.io') {
|
||||
return 'https://registry-1.docker.io';
|
||||
}
|
||||
// Local registries (localhost) use HTTP
|
||||
if (registry.startsWith('localhost') || registry.startsWith('127.0.0.1')) {
|
||||
return `http://${registry}`;
|
||||
}
|
||||
return `https://${registry}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Obtains a Bearer token for registry operations.
|
||||
* Follows the standard Docker auth flow:
|
||||
* GET /v2/ → 401 with Www-Authenticate → request token
|
||||
*/
|
||||
private async getToken(
|
||||
registry: string,
|
||||
repo: string,
|
||||
actions: string,
|
||||
credentials?: IRegistryCredentials | null,
|
||||
): Promise<string | null> {
|
||||
const scope = `repository:${repo}:${actions}`;
|
||||
const cached = this.tokenCache[`${registry}/${scope}`];
|
||||
if (cached && cached.expiry > Date.now()) {
|
||||
return cached.token;
|
||||
}
|
||||
|
||||
const apiBase = this.getRegistryApiBase(registry);
|
||||
|
||||
// Local registries typically don't need auth
|
||||
if (registry.startsWith('localhost') || registry.startsWith('127.0.0.1')) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const checkResp = await fetch(`${apiBase}/v2/`, { method: 'GET' });
|
||||
if (checkResp.ok) return null; // No auth needed
|
||||
|
||||
const wwwAuth = checkResp.headers.get('www-authenticate') || '';
|
||||
const realmMatch = wwwAuth.match(/realm="([^"]+)"/);
|
||||
const serviceMatch = wwwAuth.match(/service="([^"]+)"/);
|
||||
|
||||
if (!realmMatch) return null;
|
||||
|
||||
const realm = realmMatch[1];
|
||||
const service = serviceMatch ? serviceMatch[1] : '';
|
||||
|
||||
const tokenUrl = new URL(realm);
|
||||
tokenUrl.searchParams.set('scope', scope);
|
||||
if (service) tokenUrl.searchParams.set('service', service);
|
||||
|
||||
const headers: Record<string, string> = {};
|
||||
const creds = credentials || RegistryCopy.getDockerConfigCredentials(registry);
|
||||
if (creds) {
|
||||
headers['Authorization'] = 'Basic ' + Buffer.from(`${creds.username}:${creds.password}`).toString('base64');
|
||||
}
|
||||
|
||||
const tokenResp = await fetch(tokenUrl.toString(), { headers });
|
||||
if (!tokenResp.ok) {
|
||||
const body = await tokenResp.text();
|
||||
throw new Error(`Token request failed (${tokenResp.status}): ${body}`);
|
||||
}
|
||||
|
||||
const tokenData = await tokenResp.json() as any;
|
||||
const token = tokenData.token || tokenData.access_token;
|
||||
|
||||
if (token) {
|
||||
// Cache for 5 minutes (conservative)
|
||||
this.tokenCache[`${registry}/${scope}`] = {
|
||||
token,
|
||||
expiry: Date.now() + 5 * 60 * 1000,
|
||||
};
|
||||
}
|
||||
|
||||
return token;
|
||||
} catch (err) {
|
||||
logger.log('warn', `Auth for ${registry}: ${(err as Error).message}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Makes an authenticated request to a registry.
|
||||
*/
|
||||
private async registryFetch(
|
||||
registry: string,
|
||||
path: string,
|
||||
options: {
|
||||
method?: string;
|
||||
headers?: Record<string, string>;
|
||||
body?: Buffer | ReadableStream | null;
|
||||
repo?: string;
|
||||
actions?: string;
|
||||
credentials?: IRegistryCredentials | null;
|
||||
} = {},
|
||||
): Promise<Response> {
|
||||
const apiBase = this.getRegistryApiBase(registry);
|
||||
const method = options.method || 'GET';
|
||||
const headers: Record<string, string> = { ...(options.headers || {}) };
|
||||
|
||||
const repo = options.repo || '';
|
||||
const actions = options.actions || 'pull';
|
||||
const token = await this.getToken(registry, repo, actions, options.credentials);
|
||||
|
||||
if (token) {
|
||||
headers['Authorization'] = `Bearer ${token}`;
|
||||
}
|
||||
|
||||
const url = `${apiBase}${path}`;
|
||||
const fetchOptions: any = { method, headers };
|
||||
if (options.body) {
|
||||
fetchOptions.body = options.body;
|
||||
fetchOptions.duplex = 'half'; // Required for streaming body in Node
|
||||
}
|
||||
|
||||
return fetch(url, fetchOptions);
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets a manifest from a registry (supports both manifest lists and single manifests).
|
||||
*/
|
||||
private async getManifest(
|
||||
registry: string,
|
||||
repo: string,
|
||||
reference: string,
|
||||
credentials?: IRegistryCredentials | null,
|
||||
): Promise<{ contentType: string; body: any; digest: string; raw: Buffer }> {
|
||||
const accept = [
|
||||
'application/vnd.oci.image.index.v1+json',
|
||||
'application/vnd.docker.distribution.manifest.list.v2+json',
|
||||
'application/vnd.oci.image.manifest.v1+json',
|
||||
'application/vnd.docker.distribution.manifest.v2+json',
|
||||
].join(', ');
|
||||
|
||||
const resp = await this.registryFetch(registry, `/v2/${repo}/manifests/${reference}`, {
|
||||
headers: { 'Accept': accept },
|
||||
repo,
|
||||
actions: 'pull',
|
||||
credentials,
|
||||
});
|
||||
|
||||
if (!resp.ok) {
|
||||
const body = await resp.text();
|
||||
throw new Error(`Failed to get manifest ${registry}/${repo}:${reference} (${resp.status}): ${body}`);
|
||||
}
|
||||
|
||||
const raw = Buffer.from(await resp.arrayBuffer());
|
||||
const contentType = resp.headers.get('content-type') || '';
|
||||
const digest = resp.headers.get('docker-content-digest') || this.computeDigest(raw);
|
||||
const body = JSON.parse(raw.toString('utf-8'));
|
||||
|
||||
return { contentType, body, digest, raw };
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if a blob exists in the destination registry.
|
||||
*/
|
||||
private async blobExists(
|
||||
registry: string,
|
||||
repo: string,
|
||||
digest: string,
|
||||
credentials?: IRegistryCredentials | null,
|
||||
): Promise<boolean> {
|
||||
const resp = await this.registryFetch(registry, `/v2/${repo}/blobs/${digest}`, {
|
||||
method: 'HEAD',
|
||||
repo,
|
||||
actions: 'pull,push',
|
||||
credentials,
|
||||
});
|
||||
return resp.ok;
|
||||
}
|
||||
|
||||
/**
|
||||
* Copies a single blob from source to destination registry.
|
||||
* Uses monolithic upload (POST initiate + PUT complete).
|
||||
*/
|
||||
private async copyBlob(
|
||||
srcRegistry: string,
|
||||
srcRepo: string,
|
||||
destRegistry: string,
|
||||
destRepo: string,
|
||||
digest: string,
|
||||
srcCredentials?: IRegistryCredentials | null,
|
||||
destCredentials?: IRegistryCredentials | null,
|
||||
): Promise<void> {
|
||||
// Check if blob already exists at destination
|
||||
const exists = await this.blobExists(destRegistry, destRepo, digest, destCredentials);
|
||||
if (exists) {
|
||||
logger.log('info', ` Blob ${digest.substring(0, 19)}... already exists, skipping`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Download blob from source
|
||||
const getResp = await this.registryFetch(srcRegistry, `/v2/${srcRepo}/blobs/${digest}`, {
|
||||
repo: srcRepo,
|
||||
actions: 'pull',
|
||||
credentials: srcCredentials,
|
||||
});
|
||||
|
||||
if (!getResp.ok) {
|
||||
throw new Error(`Failed to get blob ${digest} from ${srcRegistry}/${srcRepo}: ${getResp.status}`);
|
||||
}
|
||||
|
||||
const blobData = Buffer.from(await getResp.arrayBuffer());
|
||||
const blobSize = blobData.length;
|
||||
|
||||
// Initiate upload at destination
|
||||
const postResp = await this.registryFetch(destRegistry, `/v2/${destRepo}/blobs/uploads/`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Length': '0' },
|
||||
repo: destRepo,
|
||||
actions: 'pull,push',
|
||||
credentials: destCredentials,
|
||||
});
|
||||
|
||||
if (!postResp.ok && postResp.status !== 202) {
|
||||
const body = await postResp.text();
|
||||
throw new Error(`Failed to initiate upload at ${destRegistry}/${destRepo}: ${postResp.status} ${body}`);
|
||||
}
|
||||
|
||||
// Get upload URL from Location header
|
||||
let uploadUrl = postResp.headers.get('location') || '';
|
||||
if (!uploadUrl) {
|
||||
throw new Error(`No upload location returned from ${destRegistry}/${destRepo}`);
|
||||
}
|
||||
|
||||
// Make upload URL absolute if relative
|
||||
if (uploadUrl.startsWith('/')) {
|
||||
const apiBase = this.getRegistryApiBase(destRegistry);
|
||||
uploadUrl = `${apiBase}${uploadUrl}`;
|
||||
}
|
||||
|
||||
// Complete upload with PUT (monolithic)
|
||||
const separator = uploadUrl.includes('?') ? '&' : '?';
|
||||
const putUrl = `${uploadUrl}${separator}digest=${encodeURIComponent(digest)}`;
|
||||
|
||||
// For PUT to the upload URL, we need auth
|
||||
const token = await this.getToken(destRegistry, destRepo, 'pull,push', destCredentials);
|
||||
const putHeaders: Record<string, string> = {
|
||||
'Content-Type': 'application/octet-stream',
|
||||
'Content-Length': String(blobSize),
|
||||
};
|
||||
if (token) {
|
||||
putHeaders['Authorization'] = `Bearer ${token}`;
|
||||
}
|
||||
|
||||
const putResp = await fetch(putUrl, {
|
||||
method: 'PUT',
|
||||
headers: putHeaders,
|
||||
body: blobData,
|
||||
});
|
||||
|
||||
if (!putResp.ok) {
|
||||
const body = await putResp.text();
|
||||
throw new Error(`Failed to upload blob ${digest} to ${destRegistry}/${destRepo}: ${putResp.status} ${body}`);
|
||||
}
|
||||
|
||||
const sizeStr = blobSize > 1048576
|
||||
? `${(blobSize / 1048576).toFixed(1)} MB`
|
||||
: `${(blobSize / 1024).toFixed(1)} KB`;
|
||||
logger.log('info', ` Copied blob ${digest.substring(0, 19)}... (${sizeStr})`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Pushes a manifest to a registry.
|
||||
*/
|
||||
private async putManifest(
|
||||
registry: string,
|
||||
repo: string,
|
||||
reference: string,
|
||||
manifest: Buffer,
|
||||
contentType: string,
|
||||
credentials?: IRegistryCredentials | null,
|
||||
): Promise<string> {
|
||||
const resp = await this.registryFetch(registry, `/v2/${repo}/manifests/${reference}`, {
|
||||
method: 'PUT',
|
||||
headers: {
|
||||
'Content-Type': contentType,
|
||||
'Content-Length': String(manifest.length),
|
||||
},
|
||||
body: manifest,
|
||||
repo,
|
||||
actions: 'pull,push',
|
||||
credentials,
|
||||
});
|
||||
|
||||
if (!resp.ok) {
|
||||
const body = await resp.text();
|
||||
throw new Error(`Failed to put manifest ${registry}/${repo}:${reference} (${resp.status}): ${body}`);
|
||||
}
|
||||
|
||||
const digest = resp.headers.get('docker-content-digest') || this.computeDigest(manifest);
|
||||
return digest;
|
||||
}
|
||||
|
||||
/**
|
||||
* Copies a single-platform manifest and all its blobs from source to destination.
|
||||
*/
|
||||
private async copySingleManifest(
|
||||
srcRegistry: string,
|
||||
srcRepo: string,
|
||||
destRegistry: string,
|
||||
destRepo: string,
|
||||
manifestDigest: string,
|
||||
srcCredentials?: IRegistryCredentials | null,
|
||||
destCredentials?: IRegistryCredentials | null,
|
||||
): Promise<void> {
|
||||
// Get the platform manifest
|
||||
const { body: manifest, contentType, raw } = await this.getManifest(
|
||||
srcRegistry, srcRepo, manifestDigest, srcCredentials,
|
||||
);
|
||||
|
||||
// Copy config blob
|
||||
if (manifest.config?.digest) {
|
||||
logger.log('info', ` Copying config blob...`);
|
||||
await this.copyBlob(
|
||||
srcRegistry, srcRepo, destRegistry, destRepo,
|
||||
manifest.config.digest, srcCredentials, destCredentials,
|
||||
);
|
||||
}
|
||||
|
||||
// Copy layer blobs
|
||||
const layers = manifest.layers || [];
|
||||
for (let i = 0; i < layers.length; i++) {
|
||||
const layer = layers[i];
|
||||
logger.log('info', ` Copying layer ${i + 1}/${layers.length}...`);
|
||||
await this.copyBlob(
|
||||
srcRegistry, srcRepo, destRegistry, destRepo,
|
||||
layer.digest, srcCredentials, destCredentials,
|
||||
);
|
||||
}
|
||||
|
||||
// Push the platform manifest by digest
|
||||
await this.putManifest(
|
||||
destRegistry, destRepo, manifestDigest, raw, contentType, destCredentials,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Copies a complete image (single or multi-arch) from source to destination registry.
|
||||
*
|
||||
* @param srcRegistry - Source registry host (e.g., "localhost:5234")
|
||||
* @param srcRepo - Source repository (e.g., "myapp")
|
||||
* @param srcTag - Source tag (e.g., "v1.0.0")
|
||||
* @param destRegistry - Destination registry host (e.g., "registry.gitlab.com")
|
||||
* @param destRepo - Destination repository (e.g., "org/myapp")
|
||||
* @param destTag - Destination tag (e.g., "v1.0.0" or "v1.0.0_arm64")
|
||||
* @param credentials - Optional credentials for destination registry
|
||||
*/
|
||||
public async copyImage(
|
||||
srcRegistry: string,
|
||||
srcRepo: string,
|
||||
srcTag: string,
|
||||
destRegistry: string,
|
||||
destRepo: string,
|
||||
destTag: string,
|
||||
credentials?: IRegistryCredentials | null,
|
||||
): Promise<void> {
|
||||
logger.log('info', `Copying ${srcRegistry}/${srcRepo}:${srcTag} -> ${destRegistry}/${destRepo}:${destTag}`);
|
||||
|
||||
// Source is always the local registry (no credentials needed)
|
||||
const srcCredentials: IRegistryCredentials | null = null;
|
||||
const destCredentials = credentials || RegistryCopy.getDockerConfigCredentials(destRegistry);
|
||||
|
||||
// Get the top-level manifest
|
||||
const topManifest = await this.getManifest(srcRegistry, srcRepo, srcTag, srcCredentials);
|
||||
const { body, contentType, raw } = topManifest;
|
||||
|
||||
const isManifestList =
|
||||
contentType.includes('manifest.list') ||
|
||||
contentType.includes('image.index') ||
|
||||
body.manifests !== undefined;
|
||||
|
||||
if (isManifestList) {
|
||||
// Multi-arch: copy each platform manifest + blobs, then push the manifest list
|
||||
const platforms = (body.manifests || []) as any[];
|
||||
logger.log('info', `Multi-arch manifest with ${platforms.length} platform(s)`);
|
||||
|
||||
for (const platformEntry of platforms) {
|
||||
const platDesc = platformEntry.platform
|
||||
? `${platformEntry.platform.os}/${platformEntry.platform.architecture}`
|
||||
: platformEntry.digest;
|
||||
logger.log('info', `Copying platform: ${platDesc}`);
|
||||
|
||||
await this.copySingleManifest(
|
||||
srcRegistry, srcRepo, destRegistry, destRepo,
|
||||
platformEntry.digest, srcCredentials, destCredentials,
|
||||
);
|
||||
}
|
||||
|
||||
// Push the manifest list/index with the destination tag
|
||||
const digest = await this.putManifest(
|
||||
destRegistry, destRepo, destTag, raw, contentType, destCredentials,
|
||||
);
|
||||
logger.log('ok', `Pushed manifest list to ${destRegistry}/${destRepo}:${destTag} (${digest.substring(0, 19)}...)`);
|
||||
} else {
|
||||
// Single-platform manifest: copy blobs + push manifest
|
||||
logger.log('info', 'Single-platform manifest');
|
||||
|
||||
// Copy config blob
|
||||
if (body.config?.digest) {
|
||||
logger.log('info', ' Copying config blob...');
|
||||
await this.copyBlob(
|
||||
srcRegistry, srcRepo, destRegistry, destRepo,
|
||||
body.config.digest, srcCredentials, destCredentials,
|
||||
);
|
||||
}
|
||||
|
||||
// Copy layer blobs
|
||||
const layers = body.layers || [];
|
||||
for (let i = 0; i < layers.length; i++) {
|
||||
logger.log('info', ` Copying layer ${i + 1}/${layers.length}...`);
|
||||
await this.copyBlob(
|
||||
srcRegistry, srcRepo, destRegistry, destRepo,
|
||||
layers[i].digest, srcCredentials, destCredentials,
|
||||
);
|
||||
}
|
||||
|
||||
// Push the manifest with the destination tag
|
||||
const digest = await this.putManifest(
|
||||
destRegistry, destRepo, destTag, raw, contentType, destCredentials,
|
||||
);
|
||||
logger.log('ok', `Pushed manifest to ${destRegistry}/${destRepo}:${destTag} (${digest.substring(0, 19)}...)`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes sha256 digest of a buffer.
|
||||
*/
|
||||
private computeDigest(data: Buffer): string {
|
||||
const crypto = require('crypto');
|
||||
const hash = crypto.createHash('sha256').update(data).digest('hex');
|
||||
return `sha256:${hash}`;
|
||||
}
|
||||
}
|
||||
@@ -187,11 +187,7 @@ export class TsDockerManager {
|
||||
|
||||
const total = toBuild.length;
|
||||
const overallStart = Date.now();
|
||||
const useRegistry = Dockerfile.needsLocalRegistry(toBuild, options);
|
||||
|
||||
if (useRegistry) {
|
||||
await Dockerfile.startLocalRegistry(this.dockerContext.contextInfo?.isRootless);
|
||||
}
|
||||
|
||||
try {
|
||||
if (options?.parallel) {
|
||||
@@ -230,7 +226,7 @@ export class TsDockerManager {
|
||||
|
||||
await Dockerfile.runWithConcurrency(tasks, concurrency);
|
||||
|
||||
// After the entire level completes, tag + push for dependency resolution
|
||||
// After the entire level completes, push all to local registry + tag for deps
|
||||
for (const df of level) {
|
||||
const dependentBaseImages = new Set<string>();
|
||||
for (const other of toBuild) {
|
||||
@@ -242,7 +238,8 @@ export class TsDockerManager {
|
||||
logger.log('info', `Tagging ${df.buildTag} as ${fullTag} for local dependency resolution`);
|
||||
await smartshellInstance.exec(`docker tag ${df.buildTag} ${fullTag}`);
|
||||
}
|
||||
if (useRegistry && toBuild.some(other => other.localBaseDockerfile === df)) {
|
||||
// Push ALL images to local registry (skip if already pushed via buildx)
|
||||
if (!df.localRegistryTag) {
|
||||
await Dockerfile.pushToLocalRegistry(df);
|
||||
}
|
||||
}
|
||||
@@ -281,17 +278,15 @@ export class TsDockerManager {
|
||||
await smartshellInstance.exec(`docker tag ${dockerfileArg.buildTag} ${fullTag}`);
|
||||
}
|
||||
|
||||
// Push to local registry for buildx (even for cache hits — image exists but registry doesn't)
|
||||
if (useRegistry && toBuild.some(other => other.localBaseDockerfile === dockerfileArg)) {
|
||||
// Push ALL images to local registry (skip if already pushed via buildx)
|
||||
if (!dockerfileArg.localRegistryTag) {
|
||||
await Dockerfile.pushToLocalRegistry(dockerfileArg);
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (useRegistry) {
|
||||
await Dockerfile.stopLocalRegistry();
|
||||
}
|
||||
}
|
||||
|
||||
logger.log('info', `Total build time: ${formatDuration(Date.now() - overallStart)}`);
|
||||
cache.save();
|
||||
@@ -398,12 +393,18 @@ export class TsDockerManager {
|
||||
return;
|
||||
}
|
||||
|
||||
// Push each Dockerfile to each registry
|
||||
// Start local registry (reads from persistent .nogit/docker-registry/)
|
||||
await Dockerfile.startLocalRegistry(this.dockerContext.contextInfo?.isRootless);
|
||||
try {
|
||||
// Push each Dockerfile to each registry via OCI copy
|
||||
for (const dockerfile of this.dockerfiles) {
|
||||
for (const registry of registriesToPush) {
|
||||
await dockerfile.push(registry);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await Dockerfile.stopLocalRegistry();
|
||||
}
|
||||
|
||||
logger.log('success', 'All images pushed successfully');
|
||||
}
|
||||
@@ -429,7 +430,8 @@ export class TsDockerManager {
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs tests for all Dockerfiles
|
||||
* Runs tests for all Dockerfiles.
|
||||
* Starts the local registry so multi-platform images can be auto-pulled.
|
||||
*/
|
||||
public async test(): Promise<void> {
|
||||
if (this.dockerfiles.length === 0) {
|
||||
@@ -443,7 +445,14 @@ export class TsDockerManager {
|
||||
|
||||
logger.log('info', '');
|
||||
logger.log('info', '=== TEST PHASE ===');
|
||||
|
||||
await Dockerfile.startLocalRegistry(this.dockerContext.contextInfo?.isRootless);
|
||||
try {
|
||||
await Dockerfile.testDockerfiles(this.dockerfiles);
|
||||
} finally {
|
||||
await Dockerfile.stopLocalRegistry();
|
||||
}
|
||||
|
||||
logger.log('success', 'All tests completed');
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user