git.zone/tsdocker
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(build): add level-based parallel builds with --parallel and configurable concurrency

Browse Source
This commit is contained in:
Jürgen Kunz
2026-02-07 04:46:06 +00:00
parent b04b8c9033
commit 04b75b42f3
7 changed files with 256 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ts/00_commitinfo_data.ts
View File

@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@git.zone/tsdocker',
version: '1.13.0',
version: '1.14.0',
description: 'develop npm modules cross platform with docker'
}

135
ts/classes.dockerfile.ts
View File

@@ -189,12 +189,60 @@ export class Dockerfile {
logger.log('info', `Pushed ${dockerfile.buildTag} to local registry as ${registryTag}`);
}
/**
* Groups topologically sorted Dockerfiles into dependency levels.
* Level 0 = no local dependencies; level N = depends on something in level N-1.
* Images within the same level are independent and can build in parallel.
*/
public static computeLevels(sortedDockerfiles: Dockerfile[]): Dockerfile[][] {
const levelMap = new Map<Dockerfile, number>();
for (const df of sortedDockerfiles) {
if (!df.localBaseImageDependent || !df.localBaseDockerfile) {
levelMap.set(df, 0);
} else {
const depLevel = levelMap.get(df.localBaseDockerfile) ?? 0;
levelMap.set(df, depLevel + 1);
}
}
const maxLevel = Math.max(...Array.from(levelMap.values()), 0);
const levels: Dockerfile[][] = [];
for (let l = 0; l <= maxLevel; l++) {
levels.push(sortedDockerfiles.filter(df => levelMap.get(df) === l));
}
return levels;
}
/**
* Runs async tasks with bounded concurrency (worker-pool pattern).
* Fast-fail: if any task throws, Promise.all rejects immediately.
*/
public static async runWithConcurrency<T>(
tasks: (() => Promise<T>)[],
concurrency: number,
): Promise<T[]> {
const results: T[] = new Array(tasks.length);
let nextIndex = 0;
async function worker(): Promise<void> {
while (true) {
const idx = nextIndex++;
if (idx >= tasks.length) break;
results[idx] = await tasks[idx]();
}
}
const workers = Array.from(
{ length: Math.min(concurrency, tasks.length) },
() => worker(),
);
await Promise.all(workers);
return results;
}
/**
* Builds the corresponding real docker image for each Dockerfile class instance
*/
public static async buildDockerfiles(
sortedArrayArg: Dockerfile[],
options?: { platform?: string; timeout?: number; noCache?: boolean; verbose?: boolean; isRootless?: boolean },
options?: { platform?: string; timeout?: number; noCache?: boolean; verbose?: boolean; isRootless?: boolean; parallel?: boolean; parallelConcurrency?: number },
): Promise<Dockerfile[]> {
const total = sortedArrayArg.length;
const overallStart = Date.now();
@@ -205,29 +253,78 @@ export class Dockerfile {
}
try {
for (let i = 0; i < total; i++) {
const dockerfileArg = sortedArrayArg[i];
const progress = `(${i + 1}/${total})`;
logger.log('info', `${progress} Building ${dockerfileArg.cleanTag}...`);
if (options?.parallel) {
// === PARALLEL MODE: build independent images concurrently within each level ===
const concurrency = options.parallelConcurrency ?? 4;
const levels = Dockerfile.computeLevels(sortedArrayArg);
const elapsed = await dockerfileArg.build(options);
logger.log('ok', `${progress} Built ${dockerfileArg.cleanTag} in ${formatDuration(elapsed)}`);
logger.log('info', `Parallel build: ${levels.length} level(s), concurrency ${concurrency}`);
for (let l = 0; l < levels.length; l++) {
const level = levels[l];
logger.log('info', ` Level ${l} (${level.length}): ${level.map(df => df.cleanTag).join(', ')}`);
}
// Tag in host daemon for standard docker build compatibility
const dependentBaseImages = new Set<string>();
for (const other of sortedArrayArg) {
if (other.localBaseDockerfile === dockerfileArg && other.baseImage !== dockerfileArg.buildTag) {
dependentBaseImages.add(other.baseImage);
let built = 0;
for (let l = 0; l < levels.length; l++) {
const level = levels[l];
logger.log('info', `--- Level ${l}: building ${level.length} image(s) in parallel ---`);
const tasks = level.map((df) => {
const myIndex = ++built;
return async () => {
const progress = `(${myIndex}/${total})`;
logger.log('info', `${progress} Building ${df.cleanTag}...`);
const elapsed = await df.build(options);
logger.log('ok', `${progress} Built ${df.cleanTag} in ${formatDuration(elapsed)}`);
return df;
};
});
await Dockerfile.runWithConcurrency(tasks, concurrency);
// After the entire level completes, tag + push for dependency resolution
for (const df of level) {
const dependentBaseImages = new Set<string>();
for (const other of sortedArrayArg) {
if (other.localBaseDockerfile === df && other.baseImage !== df.buildTag) {
dependentBaseImages.add(other.baseImage);
}
}
for (const fullTag of dependentBaseImages) {
logger.log('info', `Tagging ${df.buildTag} as ${fullTag} for local dependency resolution`);
await smartshellInstance.exec(`docker tag ${df.buildTag} ${fullTag}`);
}
if (useRegistry && sortedArrayArg.some(other => other.localBaseDockerfile === df)) {
await Dockerfile.pushToLocalRegistry(df);
}
}
}
for (const fullTag of dependentBaseImages) {
logger.log('info', `Tagging ${dockerfileArg.buildTag} as ${fullTag} for local dependency resolution`);
await smartshellInstance.exec(`docker tag ${dockerfileArg.buildTag} ${fullTag}`);
}
} else {
// === SEQUENTIAL MODE: build one at a time ===
for (let i = 0; i < total; i++) {
const dockerfileArg = sortedArrayArg[i];
const progress = `(${i + 1}/${total})`;
logger.log('info', `${progress} Building ${dockerfileArg.cleanTag}...`);
// Push to local registry for buildx dependency resolution
if (useRegistry && sortedArrayArg.some(other => other.localBaseDockerfile === dockerfileArg)) {
await Dockerfile.pushToLocalRegistry(dockerfileArg);
const elapsed = await dockerfileArg.build(options);
logger.log('ok', `${progress} Built ${dockerfileArg.cleanTag} in ${formatDuration(elapsed)}`);
// Tag in host daemon for standard docker build compatibility
const dependentBaseImages = new Set<string>();
for (const other of sortedArrayArg) {
if (other.localBaseDockerfile === dockerfileArg && other.baseImage !== dockerfileArg.buildTag) {
dependentBaseImages.add(other.baseImage);
}
}
for (const fullTag of dependentBaseImages) {
logger.log('info', `Tagging ${dockerfileArg.buildTag} as ${fullTag} for local dependency resolution`);
await smartshellInstance.exec(`docker tag ${dockerfileArg.buildTag} ${fullTag}`);
}
// Push to local registry for buildx dependency resolution
if (useRegistry && sortedArrayArg.some(other => other.localBaseDockerfile === dockerfileArg)) {
await Dockerfile.pushToLocalRegistry(dockerfileArg);
}
}
}
} finally {

126
ts/classes.tsdockermanager.ts
View File

@@ -167,6 +167,16 @@ export class TsDockerManager {
logger.log('info', 'Cache: disabled (--no-cache)');
}
if (options?.parallel) {
const concurrency = options.parallelConcurrency ?? 4;
const levels = Dockerfile.computeLevels(toBuild);
logger.log('info', `Parallel build: ${levels.length} level(s), concurrency ${concurrency}`);
for (let l = 0; l < levels.length; l++) {
const level = levels[l];
logger.log('info', ` Level ${l} (${level.length}): ${level.map(df => df.cleanTag).join(', ')}`);
}
}
logger.log('info', `Building ${toBuild.length} Dockerfile(s)...`);
if (options?.cached) {
@@ -184,41 +194,97 @@ export class TsDockerManager {
}
try {
for (let i = 0; i < total; i++) {
const dockerfileArg = toBuild[i];
const progress = `(${i + 1}/${total})`;
const skip = await cache.shouldSkipBuild(dockerfileArg.cleanTag, dockerfileArg.content);
if (options?.parallel) {
// === PARALLEL CACHED MODE ===
const concurrency = options.parallelConcurrency ?? 4;
const levels = Dockerfile.computeLevels(toBuild);
if (skip) {
logger.log('ok', `${progress} Skipped ${dockerfileArg.cleanTag} (cached)`);
} else {
logger.log('info', `${progress} Building ${dockerfileArg.cleanTag}...`);
const elapsed = await dockerfileArg.build({
platform: options?.platform,
timeout: options?.timeout,
noCache: options?.noCache,
verbose: options?.verbose,
let built = 0;
for (let l = 0; l < levels.length; l++) {
const level = levels[l];
logger.log('info', `--- Level ${l}: building ${level.length} image(s) in parallel ---`);
const tasks = level.map((df) => {
const myIndex = ++built;
return async () => {
const progress = `(${myIndex}/${total})`;
const skip = await cache.shouldSkipBuild(df.cleanTag, df.content);
if (skip) {
logger.log('ok', `${progress} Skipped ${df.cleanTag} (cached)`);
} else {
logger.log('info', `${progress} Building ${df.cleanTag}...`);
const elapsed = await df.build({
platform: options?.platform,
timeout: options?.timeout,
noCache: options?.noCache,
verbose: options?.verbose,
});
logger.log('ok', `${progress} Built ${df.cleanTag} in ${formatDuration(elapsed)}`);
const imageId = await df.getId();
cache.recordBuild(df.cleanTag, df.content, imageId, df.buildTag);
}
return df;
};
});
logger.log('ok', `${progress} Built ${dockerfileArg.cleanTag} in ${formatDuration(elapsed)}`);
const imageId = await dockerfileArg.getId();
cache.recordBuild(dockerfileArg.cleanTag, dockerfileArg.content, imageId, dockerfileArg.buildTag);
}
// Tag for dependents IMMEDIATELY (not after all builds)
const dependentBaseImages = new Set<string>();
for (const other of toBuild) {
if (other.localBaseDockerfile === dockerfileArg && other.baseImage !== dockerfileArg.buildTag) {
dependentBaseImages.add(other.baseImage);
await Dockerfile.runWithConcurrency(tasks, concurrency);
// After the entire level completes, tag + push for dependency resolution
for (const df of level) {
const dependentBaseImages = new Set<string>();
for (const other of toBuild) {
if (other.localBaseDockerfile === df && other.baseImage !== df.buildTag) {
dependentBaseImages.add(other.baseImage);
}
}
for (const fullTag of dependentBaseImages) {
logger.log('info', `Tagging ${df.buildTag} as ${fullTag} for local dependency resolution`);
await smartshellInstance.exec(`docker tag ${df.buildTag} ${fullTag}`);
}
if (useRegistry && toBuild.some(other => other.localBaseDockerfile === df)) {
await Dockerfile.pushToLocalRegistry(df);
}
}
}
for (const fullTag of dependentBaseImages) {
logger.log('info', `Tagging ${dockerfileArg.buildTag} as ${fullTag} for local dependency resolution`);
await smartshellInstance.exec(`docker tag ${dockerfileArg.buildTag} ${fullTag}`);
}
} else {
// === SEQUENTIAL CACHED MODE ===
for (let i = 0; i < total; i++) {
const dockerfileArg = toBuild[i];
const progress = `(${i + 1}/${total})`;
const skip = await cache.shouldSkipBuild(dockerfileArg.cleanTag, dockerfileArg.content);
// Push to local registry for buildx (even for cache hits — image exists but registry doesn't)
if (useRegistry && toBuild.some(other => other.localBaseDockerfile === dockerfileArg)) {
await Dockerfile.pushToLocalRegistry(dockerfileArg);
if (skip) {
logger.log('ok', `${progress} Skipped ${dockerfileArg.cleanTag} (cached)`);
} else {
logger.log('info', `${progress} Building ${dockerfileArg.cleanTag}...`);
const elapsed = await dockerfileArg.build({
platform: options?.platform,
timeout: options?.timeout,
noCache: options?.noCache,
verbose: options?.verbose,
});
logger.log('ok', `${progress} Built ${dockerfileArg.cleanTag} in ${formatDuration(elapsed)}`);
const imageId = await dockerfileArg.getId();
cache.recordBuild(dockerfileArg.cleanTag, dockerfileArg.content, imageId, dockerfileArg.buildTag);
}
// Tag for dependents IMMEDIATELY (not after all builds)
const dependentBaseImages = new Set<string>();
for (const other of toBuild) {
if (other.localBaseDockerfile === dockerfileArg && other.baseImage !== dockerfileArg.buildTag) {
dependentBaseImages.add(other.baseImage);
}
}
for (const fullTag of dependentBaseImages) {
logger.log('info', `Tagging ${dockerfileArg.buildTag} as ${fullTag} for local dependency resolution`);
await smartshellInstance.exec(`docker tag ${dockerfileArg.buildTag} ${fullTag}`);
}
// Push to local registry for buildx (even for cache hits — image exists but registry doesn't)
if (useRegistry && toBuild.some(other => other.localBaseDockerfile === dockerfileArg)) {
await Dockerfile.pushToLocalRegistry(dockerfileArg);
}
}
}
} finally {
@@ -237,6 +303,8 @@ export class TsDockerManager {
noCache: options?.noCache,
verbose: options?.verbose,
isRootless: this.dockerContext.contextInfo?.isRootless,
parallel: options?.parallel,
parallelConcurrency: options?.parallelConcurrency,
});
}

2
ts/interfaces/index.ts
View File

@@ -80,6 +80,8 @@ export interface IBuildCommandOptions {
cached?: boolean; // Skip builds when Dockerfile content hasn't changed
verbose?: boolean; // Stream raw docker build output (default: silent)
context?: string; // Explicit Docker context name (--context flag)
parallel?: boolean; // Enable parallel builds within dependency levels
parallelConcurrency?: number; // Max concurrent builds per level (default 4)
}
export interface ICacheEntry {

18
ts/tsdocker.cli.ts
View File

@@ -55,6 +55,12 @@ export let run = () => {
if (argvArg.verbose) {
buildOptions.verbose = true;
}
if (argvArg.parallel) {
buildOptions.parallel = true;
if (typeof argvArg.parallel === 'number') {
buildOptions.parallelConcurrency = argvArg.parallel;
}
}
await manager.build(buildOptions);
logger.log('success', 'Build completed successfully');
@@ -95,6 +101,12 @@ export let run = () => {
if (argvArg.verbose) {
buildOptions.verbose = true;
}
if (argvArg.parallel) {
buildOptions.parallel = true;
if (typeof argvArg.parallel === 'number') {
buildOptions.parallelConcurrency = argvArg.parallel;
}
}
// Build images first (if not already built)
await manager.build(buildOptions);
@@ -157,6 +169,12 @@ export let run = () => {
if (argvArg.verbose) {
buildOptions.verbose = true;
}
if (argvArg.parallel) {
buildOptions.parallel = true;
if (typeof argvArg.parallel === 'number') {
buildOptions.parallelConcurrency = argvArg.parallel;
}
}
await manager.build(buildOptions);
// Run tests