From 04b75b42f36fc1dfab3376fd2c00e17376c05d08 Mon Sep 17 00:00:00 2001 From: Juergen Kunz Date: Sat, 7 Feb 2026 04:46:06 +0000 Subject: [PATCH] feat(build): add level-based parallel builds with --parallel and configurable concurrency --- changelog.md | 10 +++ readme.hints.md | 12 +++ ts/00_commitinfo_data.ts | 2 +- ts/classes.dockerfile.ts | 135 +++++++++++++++++++++++++++++----- ts/classes.tsdockermanager.ts | 126 +++++++++++++++++++++++-------- ts/interfaces/index.ts | 2 + ts/tsdocker.cli.ts | 18 +++++ 7 files changed, 256 insertions(+), 49 deletions(-) diff --git a/changelog.md b/changelog.md index c1a731c..0a32c23 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,15 @@ # Changelog +## 2026-02-07 - 1.14.0 - feat(build) +add level-based parallel builds with --parallel and configurable concurrency + +- Introduces --parallel and --parallel= CLI flags to enable level-based parallel Docker builds (default concurrency 4). +- Adds Dockerfile.computeLevels() to group topologically-sorted Dockerfiles into dependency levels. +- Adds Dockerfile.runWithConcurrency() implementing a bounded-concurrency worker-pool (fast-fail via Promise.all). +- Integrates parallel build mode into Dockerfile.buildDockerfiles() and TsDockerManager.build() for both cached and non-cached flows, including tagging and pushing for dependency resolution after each level. +- Adds options.parallel and options.parallelConcurrency to the build interface and wires them through the CLI and manager. +- Updates documentation (readme.hints.md) with usage examples and implementation notes. + ## 2026-02-07 - 1.13.0 - feat(docker) add Docker context detection, rootless support, and context-aware buildx registry handling diff --git a/readme.hints.md b/readme.hints.md index 0284638..c4ac35e 100644 --- a/readme.hints.md +++ b/readme.hints.md @@ -96,6 +96,18 @@ ts/ - `@push.rocks/smartcli`: CLI framework - `@push.rocks/projectinfo`: Project metadata +## Parallel Builds + +`--parallel` flag enables level-based parallel Docker builds: + +```bash +tsdocker build --parallel # parallel, default concurrency (4) +tsdocker build --parallel=8 # parallel, concurrency 8 +tsdocker build --parallel --cached # works with both modes +``` + +Implementation: `Dockerfile.computeLevels()` groups topologically sorted Dockerfiles into dependency levels. `Dockerfile.runWithConcurrency()` provides a worker-pool pattern for bounded concurrency. Both are public static methods on the `Dockerfile` class. The parallel logic exists in both `Dockerfile.buildDockerfiles()` (standard mode) and `TsDockerManager.build()` (cached mode). + ## Build Status - Build: ✅ Passes diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index 0299981..c05ded4 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@git.zone/tsdocker', - version: '1.13.0', + version: '1.14.0', description: 'develop npm modules cross platform with docker' } diff --git a/ts/classes.dockerfile.ts b/ts/classes.dockerfile.ts index 7d3f137..92b54da 100644 --- a/ts/classes.dockerfile.ts +++ b/ts/classes.dockerfile.ts @@ -189,12 +189,60 @@ export class Dockerfile { logger.log('info', `Pushed ${dockerfile.buildTag} to local registry as ${registryTag}`); } + /** + * Groups topologically sorted Dockerfiles into dependency levels. + * Level 0 = no local dependencies; level N = depends on something in level N-1. + * Images within the same level are independent and can build in parallel. + */ + public static computeLevels(sortedDockerfiles: Dockerfile[]): Dockerfile[][] { + const levelMap = new Map(); + for (const df of sortedDockerfiles) { + if (!df.localBaseImageDependent || !df.localBaseDockerfile) { + levelMap.set(df, 0); + } else { + const depLevel = levelMap.get(df.localBaseDockerfile) ?? 0; + levelMap.set(df, depLevel + 1); + } + } + const maxLevel = Math.max(...Array.from(levelMap.values()), 0); + const levels: Dockerfile[][] = []; + for (let l = 0; l <= maxLevel; l++) { + levels.push(sortedDockerfiles.filter(df => levelMap.get(df) === l)); + } + return levels; + } + + /** + * Runs async tasks with bounded concurrency (worker-pool pattern). + * Fast-fail: if any task throws, Promise.all rejects immediately. + */ + public static async runWithConcurrency( + tasks: (() => Promise)[], + concurrency: number, + ): Promise { + const results: T[] = new Array(tasks.length); + let nextIndex = 0; + async function worker(): Promise { + while (true) { + const idx = nextIndex++; + if (idx >= tasks.length) break; + results[idx] = await tasks[idx](); + } + } + const workers = Array.from( + { length: Math.min(concurrency, tasks.length) }, + () => worker(), + ); + await Promise.all(workers); + return results; + } + /** * Builds the corresponding real docker image for each Dockerfile class instance */ public static async buildDockerfiles( sortedArrayArg: Dockerfile[], - options?: { platform?: string; timeout?: number; noCache?: boolean; verbose?: boolean; isRootless?: boolean }, + options?: { platform?: string; timeout?: number; noCache?: boolean; verbose?: boolean; isRootless?: boolean; parallel?: boolean; parallelConcurrency?: number }, ): Promise { const total = sortedArrayArg.length; const overallStart = Date.now(); @@ -205,29 +253,78 @@ export class Dockerfile { } try { - for (let i = 0; i < total; i++) { - const dockerfileArg = sortedArrayArg[i]; - const progress = `(${i + 1}/${total})`; - logger.log('info', `${progress} Building ${dockerfileArg.cleanTag}...`); + if (options?.parallel) { + // === PARALLEL MODE: build independent images concurrently within each level === + const concurrency = options.parallelConcurrency ?? 4; + const levels = Dockerfile.computeLevels(sortedArrayArg); - const elapsed = await dockerfileArg.build(options); - logger.log('ok', `${progress} Built ${dockerfileArg.cleanTag} in ${formatDuration(elapsed)}`); + logger.log('info', `Parallel build: ${levels.length} level(s), concurrency ${concurrency}`); + for (let l = 0; l < levels.length; l++) { + const level = levels[l]; + logger.log('info', ` Level ${l} (${level.length}): ${level.map(df => df.cleanTag).join(', ')}`); + } - // Tag in host daemon for standard docker build compatibility - const dependentBaseImages = new Set(); - for (const other of sortedArrayArg) { - if (other.localBaseDockerfile === dockerfileArg && other.baseImage !== dockerfileArg.buildTag) { - dependentBaseImages.add(other.baseImage); + let built = 0; + for (let l = 0; l < levels.length; l++) { + const level = levels[l]; + logger.log('info', `--- Level ${l}: building ${level.length} image(s) in parallel ---`); + + const tasks = level.map((df) => { + const myIndex = ++built; + return async () => { + const progress = `(${myIndex}/${total})`; + logger.log('info', `${progress} Building ${df.cleanTag}...`); + const elapsed = await df.build(options); + logger.log('ok', `${progress} Built ${df.cleanTag} in ${formatDuration(elapsed)}`); + return df; + }; + }); + + await Dockerfile.runWithConcurrency(tasks, concurrency); + + // After the entire level completes, tag + push for dependency resolution + for (const df of level) { + const dependentBaseImages = new Set(); + for (const other of sortedArrayArg) { + if (other.localBaseDockerfile === df && other.baseImage !== df.buildTag) { + dependentBaseImages.add(other.baseImage); + } + } + for (const fullTag of dependentBaseImages) { + logger.log('info', `Tagging ${df.buildTag} as ${fullTag} for local dependency resolution`); + await smartshellInstance.exec(`docker tag ${df.buildTag} ${fullTag}`); + } + if (useRegistry && sortedArrayArg.some(other => other.localBaseDockerfile === df)) { + await Dockerfile.pushToLocalRegistry(df); + } } } - for (const fullTag of dependentBaseImages) { - logger.log('info', `Tagging ${dockerfileArg.buildTag} as ${fullTag} for local dependency resolution`); - await smartshellInstance.exec(`docker tag ${dockerfileArg.buildTag} ${fullTag}`); - } + } else { + // === SEQUENTIAL MODE: build one at a time === + for (let i = 0; i < total; i++) { + const dockerfileArg = sortedArrayArg[i]; + const progress = `(${i + 1}/${total})`; + logger.log('info', `${progress} Building ${dockerfileArg.cleanTag}...`); - // Push to local registry for buildx dependency resolution - if (useRegistry && sortedArrayArg.some(other => other.localBaseDockerfile === dockerfileArg)) { - await Dockerfile.pushToLocalRegistry(dockerfileArg); + const elapsed = await dockerfileArg.build(options); + logger.log('ok', `${progress} Built ${dockerfileArg.cleanTag} in ${formatDuration(elapsed)}`); + + // Tag in host daemon for standard docker build compatibility + const dependentBaseImages = new Set(); + for (const other of sortedArrayArg) { + if (other.localBaseDockerfile === dockerfileArg && other.baseImage !== dockerfileArg.buildTag) { + dependentBaseImages.add(other.baseImage); + } + } + for (const fullTag of dependentBaseImages) { + logger.log('info', `Tagging ${dockerfileArg.buildTag} as ${fullTag} for local dependency resolution`); + await smartshellInstance.exec(`docker tag ${dockerfileArg.buildTag} ${fullTag}`); + } + + // Push to local registry for buildx dependency resolution + if (useRegistry && sortedArrayArg.some(other => other.localBaseDockerfile === dockerfileArg)) { + await Dockerfile.pushToLocalRegistry(dockerfileArg); + } } } } finally { diff --git a/ts/classes.tsdockermanager.ts b/ts/classes.tsdockermanager.ts index 4a74970..6a75094 100644 --- a/ts/classes.tsdockermanager.ts +++ b/ts/classes.tsdockermanager.ts @@ -167,6 +167,16 @@ export class TsDockerManager { logger.log('info', 'Cache: disabled (--no-cache)'); } + if (options?.parallel) { + const concurrency = options.parallelConcurrency ?? 4; + const levels = Dockerfile.computeLevels(toBuild); + logger.log('info', `Parallel build: ${levels.length} level(s), concurrency ${concurrency}`); + for (let l = 0; l < levels.length; l++) { + const level = levels[l]; + logger.log('info', ` Level ${l} (${level.length}): ${level.map(df => df.cleanTag).join(', ')}`); + } + } + logger.log('info', `Building ${toBuild.length} Dockerfile(s)...`); if (options?.cached) { @@ -184,41 +194,97 @@ export class TsDockerManager { } try { - for (let i = 0; i < total; i++) { - const dockerfileArg = toBuild[i]; - const progress = `(${i + 1}/${total})`; - const skip = await cache.shouldSkipBuild(dockerfileArg.cleanTag, dockerfileArg.content); + if (options?.parallel) { + // === PARALLEL CACHED MODE === + const concurrency = options.parallelConcurrency ?? 4; + const levels = Dockerfile.computeLevels(toBuild); - if (skip) { - logger.log('ok', `${progress} Skipped ${dockerfileArg.cleanTag} (cached)`); - } else { - logger.log('info', `${progress} Building ${dockerfileArg.cleanTag}...`); - const elapsed = await dockerfileArg.build({ - platform: options?.platform, - timeout: options?.timeout, - noCache: options?.noCache, - verbose: options?.verbose, + let built = 0; + for (let l = 0; l < levels.length; l++) { + const level = levels[l]; + logger.log('info', `--- Level ${l}: building ${level.length} image(s) in parallel ---`); + + const tasks = level.map((df) => { + const myIndex = ++built; + return async () => { + const progress = `(${myIndex}/${total})`; + const skip = await cache.shouldSkipBuild(df.cleanTag, df.content); + + if (skip) { + logger.log('ok', `${progress} Skipped ${df.cleanTag} (cached)`); + } else { + logger.log('info', `${progress} Building ${df.cleanTag}...`); + const elapsed = await df.build({ + platform: options?.platform, + timeout: options?.timeout, + noCache: options?.noCache, + verbose: options?.verbose, + }); + logger.log('ok', `${progress} Built ${df.cleanTag} in ${formatDuration(elapsed)}`); + const imageId = await df.getId(); + cache.recordBuild(df.cleanTag, df.content, imageId, df.buildTag); + } + return df; + }; }); - logger.log('ok', `${progress} Built ${dockerfileArg.cleanTag} in ${formatDuration(elapsed)}`); - const imageId = await dockerfileArg.getId(); - cache.recordBuild(dockerfileArg.cleanTag, dockerfileArg.content, imageId, dockerfileArg.buildTag); - } - // Tag for dependents IMMEDIATELY (not after all builds) - const dependentBaseImages = new Set(); - for (const other of toBuild) { - if (other.localBaseDockerfile === dockerfileArg && other.baseImage !== dockerfileArg.buildTag) { - dependentBaseImages.add(other.baseImage); + await Dockerfile.runWithConcurrency(tasks, concurrency); + + // After the entire level completes, tag + push for dependency resolution + for (const df of level) { + const dependentBaseImages = new Set(); + for (const other of toBuild) { + if (other.localBaseDockerfile === df && other.baseImage !== df.buildTag) { + dependentBaseImages.add(other.baseImage); + } + } + for (const fullTag of dependentBaseImages) { + logger.log('info', `Tagging ${df.buildTag} as ${fullTag} for local dependency resolution`); + await smartshellInstance.exec(`docker tag ${df.buildTag} ${fullTag}`); + } + if (useRegistry && toBuild.some(other => other.localBaseDockerfile === df)) { + await Dockerfile.pushToLocalRegistry(df); + } } } - for (const fullTag of dependentBaseImages) { - logger.log('info', `Tagging ${dockerfileArg.buildTag} as ${fullTag} for local dependency resolution`); - await smartshellInstance.exec(`docker tag ${dockerfileArg.buildTag} ${fullTag}`); - } + } else { + // === SEQUENTIAL CACHED MODE === + for (let i = 0; i < total; i++) { + const dockerfileArg = toBuild[i]; + const progress = `(${i + 1}/${total})`; + const skip = await cache.shouldSkipBuild(dockerfileArg.cleanTag, dockerfileArg.content); - // Push to local registry for buildx (even for cache hits — image exists but registry doesn't) - if (useRegistry && toBuild.some(other => other.localBaseDockerfile === dockerfileArg)) { - await Dockerfile.pushToLocalRegistry(dockerfileArg); + if (skip) { + logger.log('ok', `${progress} Skipped ${dockerfileArg.cleanTag} (cached)`); + } else { + logger.log('info', `${progress} Building ${dockerfileArg.cleanTag}...`); + const elapsed = await dockerfileArg.build({ + platform: options?.platform, + timeout: options?.timeout, + noCache: options?.noCache, + verbose: options?.verbose, + }); + logger.log('ok', `${progress} Built ${dockerfileArg.cleanTag} in ${formatDuration(elapsed)}`); + const imageId = await dockerfileArg.getId(); + cache.recordBuild(dockerfileArg.cleanTag, dockerfileArg.content, imageId, dockerfileArg.buildTag); + } + + // Tag for dependents IMMEDIATELY (not after all builds) + const dependentBaseImages = new Set(); + for (const other of toBuild) { + if (other.localBaseDockerfile === dockerfileArg && other.baseImage !== dockerfileArg.buildTag) { + dependentBaseImages.add(other.baseImage); + } + } + for (const fullTag of dependentBaseImages) { + logger.log('info', `Tagging ${dockerfileArg.buildTag} as ${fullTag} for local dependency resolution`); + await smartshellInstance.exec(`docker tag ${dockerfileArg.buildTag} ${fullTag}`); + } + + // Push to local registry for buildx (even for cache hits — image exists but registry doesn't) + if (useRegistry && toBuild.some(other => other.localBaseDockerfile === dockerfileArg)) { + await Dockerfile.pushToLocalRegistry(dockerfileArg); + } } } } finally { @@ -237,6 +303,8 @@ export class TsDockerManager { noCache: options?.noCache, verbose: options?.verbose, isRootless: this.dockerContext.contextInfo?.isRootless, + parallel: options?.parallel, + parallelConcurrency: options?.parallelConcurrency, }); } diff --git a/ts/interfaces/index.ts b/ts/interfaces/index.ts index e2a571a..9dfd5a2 100644 --- a/ts/interfaces/index.ts +++ b/ts/interfaces/index.ts @@ -80,6 +80,8 @@ export interface IBuildCommandOptions { cached?: boolean; // Skip builds when Dockerfile content hasn't changed verbose?: boolean; // Stream raw docker build output (default: silent) context?: string; // Explicit Docker context name (--context flag) + parallel?: boolean; // Enable parallel builds within dependency levels + parallelConcurrency?: number; // Max concurrent builds per level (default 4) } export interface ICacheEntry { diff --git a/ts/tsdocker.cli.ts b/ts/tsdocker.cli.ts index dea831f..8987d16 100644 --- a/ts/tsdocker.cli.ts +++ b/ts/tsdocker.cli.ts @@ -55,6 +55,12 @@ export let run = () => { if (argvArg.verbose) { buildOptions.verbose = true; } + if (argvArg.parallel) { + buildOptions.parallel = true; + if (typeof argvArg.parallel === 'number') { + buildOptions.parallelConcurrency = argvArg.parallel; + } + } await manager.build(buildOptions); logger.log('success', 'Build completed successfully'); @@ -95,6 +101,12 @@ export let run = () => { if (argvArg.verbose) { buildOptions.verbose = true; } + if (argvArg.parallel) { + buildOptions.parallel = true; + if (typeof argvArg.parallel === 'number') { + buildOptions.parallelConcurrency = argvArg.parallel; + } + } // Build images first (if not already built) await manager.build(buildOptions); @@ -157,6 +169,12 @@ export let run = () => { if (argvArg.verbose) { buildOptions.verbose = true; } + if (argvArg.parallel) { + buildOptions.parallel = true; + if (typeof argvArg.parallel === 'number') { + buildOptions.parallelConcurrency = argvArg.parallel; + } + } await manager.build(buildOptions); // Run tests