feat(build): add level-based parallel builds with --parallel and configurable concurrency
This commit is contained in:
@@ -189,12 +189,60 @@ export class Dockerfile {
|
||||
logger.log('info', `Pushed ${dockerfile.buildTag} to local registry as ${registryTag}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Groups topologically sorted Dockerfiles into dependency levels.
|
||||
* Level 0 = no local dependencies; level N = depends on something in level N-1.
|
||||
* Images within the same level are independent and can build in parallel.
|
||||
*/
|
||||
public static computeLevels(sortedDockerfiles: Dockerfile[]): Dockerfile[][] {
|
||||
const levelMap = new Map<Dockerfile, number>();
|
||||
for (const df of sortedDockerfiles) {
|
||||
if (!df.localBaseImageDependent || !df.localBaseDockerfile) {
|
||||
levelMap.set(df, 0);
|
||||
} else {
|
||||
const depLevel = levelMap.get(df.localBaseDockerfile) ?? 0;
|
||||
levelMap.set(df, depLevel + 1);
|
||||
}
|
||||
}
|
||||
const maxLevel = Math.max(...Array.from(levelMap.values()), 0);
|
||||
const levels: Dockerfile[][] = [];
|
||||
for (let l = 0; l <= maxLevel; l++) {
|
||||
levels.push(sortedDockerfiles.filter(df => levelMap.get(df) === l));
|
||||
}
|
||||
return levels;
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs async tasks with bounded concurrency (worker-pool pattern).
|
||||
* Fast-fail: if any task throws, Promise.all rejects immediately.
|
||||
*/
|
||||
public static async runWithConcurrency<T>(
|
||||
tasks: (() => Promise<T>)[],
|
||||
concurrency: number,
|
||||
): Promise<T[]> {
|
||||
const results: T[] = new Array(tasks.length);
|
||||
let nextIndex = 0;
|
||||
async function worker(): Promise<void> {
|
||||
while (true) {
|
||||
const idx = nextIndex++;
|
||||
if (idx >= tasks.length) break;
|
||||
results[idx] = await tasks[idx]();
|
||||
}
|
||||
}
|
||||
const workers = Array.from(
|
||||
{ length: Math.min(concurrency, tasks.length) },
|
||||
() => worker(),
|
||||
);
|
||||
await Promise.all(workers);
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the corresponding real docker image for each Dockerfile class instance
|
||||
*/
|
||||
public static async buildDockerfiles(
|
||||
sortedArrayArg: Dockerfile[],
|
||||
options?: { platform?: string; timeout?: number; noCache?: boolean; verbose?: boolean; isRootless?: boolean },
|
||||
options?: { platform?: string; timeout?: number; noCache?: boolean; verbose?: boolean; isRootless?: boolean; parallel?: boolean; parallelConcurrency?: number },
|
||||
): Promise<Dockerfile[]> {
|
||||
const total = sortedArrayArg.length;
|
||||
const overallStart = Date.now();
|
||||
@@ -205,29 +253,78 @@ export class Dockerfile {
|
||||
}
|
||||
|
||||
try {
|
||||
for (let i = 0; i < total; i++) {
|
||||
const dockerfileArg = sortedArrayArg[i];
|
||||
const progress = `(${i + 1}/${total})`;
|
||||
logger.log('info', `${progress} Building ${dockerfileArg.cleanTag}...`);
|
||||
if (options?.parallel) {
|
||||
// === PARALLEL MODE: build independent images concurrently within each level ===
|
||||
const concurrency = options.parallelConcurrency ?? 4;
|
||||
const levels = Dockerfile.computeLevels(sortedArrayArg);
|
||||
|
||||
const elapsed = await dockerfileArg.build(options);
|
||||
logger.log('ok', `${progress} Built ${dockerfileArg.cleanTag} in ${formatDuration(elapsed)}`);
|
||||
logger.log('info', `Parallel build: ${levels.length} level(s), concurrency ${concurrency}`);
|
||||
for (let l = 0; l < levels.length; l++) {
|
||||
const level = levels[l];
|
||||
logger.log('info', ` Level ${l} (${level.length}): ${level.map(df => df.cleanTag).join(', ')}`);
|
||||
}
|
||||
|
||||
// Tag in host daemon for standard docker build compatibility
|
||||
const dependentBaseImages = new Set<string>();
|
||||
for (const other of sortedArrayArg) {
|
||||
if (other.localBaseDockerfile === dockerfileArg && other.baseImage !== dockerfileArg.buildTag) {
|
||||
dependentBaseImages.add(other.baseImage);
|
||||
let built = 0;
|
||||
for (let l = 0; l < levels.length; l++) {
|
||||
const level = levels[l];
|
||||
logger.log('info', `--- Level ${l}: building ${level.length} image(s) in parallel ---`);
|
||||
|
||||
const tasks = level.map((df) => {
|
||||
const myIndex = ++built;
|
||||
return async () => {
|
||||
const progress = `(${myIndex}/${total})`;
|
||||
logger.log('info', `${progress} Building ${df.cleanTag}...`);
|
||||
const elapsed = await df.build(options);
|
||||
logger.log('ok', `${progress} Built ${df.cleanTag} in ${formatDuration(elapsed)}`);
|
||||
return df;
|
||||
};
|
||||
});
|
||||
|
||||
await Dockerfile.runWithConcurrency(tasks, concurrency);
|
||||
|
||||
// After the entire level completes, tag + push for dependency resolution
|
||||
for (const df of level) {
|
||||
const dependentBaseImages = new Set<string>();
|
||||
for (const other of sortedArrayArg) {
|
||||
if (other.localBaseDockerfile === df && other.baseImage !== df.buildTag) {
|
||||
dependentBaseImages.add(other.baseImage);
|
||||
}
|
||||
}
|
||||
for (const fullTag of dependentBaseImages) {
|
||||
logger.log('info', `Tagging ${df.buildTag} as ${fullTag} for local dependency resolution`);
|
||||
await smartshellInstance.exec(`docker tag ${df.buildTag} ${fullTag}`);
|
||||
}
|
||||
if (useRegistry && sortedArrayArg.some(other => other.localBaseDockerfile === df)) {
|
||||
await Dockerfile.pushToLocalRegistry(df);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const fullTag of dependentBaseImages) {
|
||||
logger.log('info', `Tagging ${dockerfileArg.buildTag} as ${fullTag} for local dependency resolution`);
|
||||
await smartshellInstance.exec(`docker tag ${dockerfileArg.buildTag} ${fullTag}`);
|
||||
}
|
||||
} else {
|
||||
// === SEQUENTIAL MODE: build one at a time ===
|
||||
for (let i = 0; i < total; i++) {
|
||||
const dockerfileArg = sortedArrayArg[i];
|
||||
const progress = `(${i + 1}/${total})`;
|
||||
logger.log('info', `${progress} Building ${dockerfileArg.cleanTag}...`);
|
||||
|
||||
// Push to local registry for buildx dependency resolution
|
||||
if (useRegistry && sortedArrayArg.some(other => other.localBaseDockerfile === dockerfileArg)) {
|
||||
await Dockerfile.pushToLocalRegistry(dockerfileArg);
|
||||
const elapsed = await dockerfileArg.build(options);
|
||||
logger.log('ok', `${progress} Built ${dockerfileArg.cleanTag} in ${formatDuration(elapsed)}`);
|
||||
|
||||
// Tag in host daemon for standard docker build compatibility
|
||||
const dependentBaseImages = new Set<string>();
|
||||
for (const other of sortedArrayArg) {
|
||||
if (other.localBaseDockerfile === dockerfileArg && other.baseImage !== dockerfileArg.buildTag) {
|
||||
dependentBaseImages.add(other.baseImage);
|
||||
}
|
||||
}
|
||||
for (const fullTag of dependentBaseImages) {
|
||||
logger.log('info', `Tagging ${dockerfileArg.buildTag} as ${fullTag} for local dependency resolution`);
|
||||
await smartshellInstance.exec(`docker tag ${dockerfileArg.buildTag} ${fullTag}`);
|
||||
}
|
||||
|
||||
// Push to local registry for buildx dependency resolution
|
||||
if (useRegistry && sortedArrayArg.some(other => other.localBaseDockerfile === dockerfileArg)) {
|
||||
await Dockerfile.pushToLocalRegistry(dockerfileArg);
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
|
||||
Reference in New Issue
Block a user