Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 3085eb590f | |||
| 04b75b42f3 |
10
changelog.md
10
changelog.md
@@ -1,5 +1,15 @@
|
|||||||
# Changelog
|
# Changelog
|
||||||
|
|
||||||
|
## 2026-02-07 - 1.14.0 - feat(build)
|
||||||
|
add level-based parallel builds with --parallel and configurable concurrency
|
||||||
|
|
||||||
|
- Introduces --parallel and --parallel=<n> CLI flags to enable level-based parallel Docker builds (default concurrency 4).
|
||||||
|
- Adds Dockerfile.computeLevels() to group topologically-sorted Dockerfiles into dependency levels.
|
||||||
|
- Adds Dockerfile.runWithConcurrency() implementing a bounded-concurrency worker-pool (fast-fail via Promise.all).
|
||||||
|
- Integrates parallel build mode into Dockerfile.buildDockerfiles() and TsDockerManager.build() for both cached and non-cached flows, including tagging and pushing for dependency resolution after each level.
|
||||||
|
- Adds options.parallel and options.parallelConcurrency to the build interface and wires them through the CLI and manager.
|
||||||
|
- Updates documentation (readme.hints.md) with usage examples and implementation notes.
|
||||||
|
|
||||||
## 2026-02-07 - 1.13.0 - feat(docker)
|
## 2026-02-07 - 1.13.0 - feat(docker)
|
||||||
add Docker context detection, rootless support, and context-aware buildx registry handling
|
add Docker context detection, rootless support, and context-aware buildx registry handling
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@git.zone/tsdocker",
|
"name": "@git.zone/tsdocker",
|
||||||
"version": "1.13.0",
|
"version": "1.14.0",
|
||||||
"private": false,
|
"private": false,
|
||||||
"description": "develop npm modules cross platform with docker",
|
"description": "develop npm modules cross platform with docker",
|
||||||
"main": "dist_ts/index.js",
|
"main": "dist_ts/index.js",
|
||||||
|
|||||||
@@ -96,6 +96,18 @@ ts/
|
|||||||
- `@push.rocks/smartcli`: CLI framework
|
- `@push.rocks/smartcli`: CLI framework
|
||||||
- `@push.rocks/projectinfo`: Project metadata
|
- `@push.rocks/projectinfo`: Project metadata
|
||||||
|
|
||||||
|
## Parallel Builds
|
||||||
|
|
||||||
|
`--parallel` flag enables level-based parallel Docker builds:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
tsdocker build --parallel # parallel, default concurrency (4)
|
||||||
|
tsdocker build --parallel=8 # parallel, concurrency 8
|
||||||
|
tsdocker build --parallel --cached # works with both modes
|
||||||
|
```
|
||||||
|
|
||||||
|
Implementation: `Dockerfile.computeLevels()` groups topologically sorted Dockerfiles into dependency levels. `Dockerfile.runWithConcurrency()` provides a worker-pool pattern for bounded concurrency. Both are public static methods on the `Dockerfile` class. The parallel logic exists in both `Dockerfile.buildDockerfiles()` (standard mode) and `TsDockerManager.build()` (cached mode).
|
||||||
|
|
||||||
## Build Status
|
## Build Status
|
||||||
|
|
||||||
- Build: ✅ Passes
|
- Build: ✅ Passes
|
||||||
|
|||||||
@@ -3,6 +3,6 @@
|
|||||||
*/
|
*/
|
||||||
export const commitinfo = {
|
export const commitinfo = {
|
||||||
name: '@git.zone/tsdocker',
|
name: '@git.zone/tsdocker',
|
||||||
version: '1.13.0',
|
version: '1.14.0',
|
||||||
description: 'develop npm modules cross platform with docker'
|
description: 'develop npm modules cross platform with docker'
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -189,12 +189,60 @@ export class Dockerfile {
|
|||||||
logger.log('info', `Pushed ${dockerfile.buildTag} to local registry as ${registryTag}`);
|
logger.log('info', `Pushed ${dockerfile.buildTag} to local registry as ${registryTag}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Groups topologically sorted Dockerfiles into dependency levels.
|
||||||
|
* Level 0 = no local dependencies; level N = depends on something in level N-1.
|
||||||
|
* Images within the same level are independent and can build in parallel.
|
||||||
|
*/
|
||||||
|
public static computeLevels(sortedDockerfiles: Dockerfile[]): Dockerfile[][] {
|
||||||
|
const levelMap = new Map<Dockerfile, number>();
|
||||||
|
for (const df of sortedDockerfiles) {
|
||||||
|
if (!df.localBaseImageDependent || !df.localBaseDockerfile) {
|
||||||
|
levelMap.set(df, 0);
|
||||||
|
} else {
|
||||||
|
const depLevel = levelMap.get(df.localBaseDockerfile) ?? 0;
|
||||||
|
levelMap.set(df, depLevel + 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const maxLevel = Math.max(...Array.from(levelMap.values()), 0);
|
||||||
|
const levels: Dockerfile[][] = [];
|
||||||
|
for (let l = 0; l <= maxLevel; l++) {
|
||||||
|
levels.push(sortedDockerfiles.filter(df => levelMap.get(df) === l));
|
||||||
|
}
|
||||||
|
return levels;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Runs async tasks with bounded concurrency (worker-pool pattern).
|
||||||
|
* Fast-fail: if any task throws, Promise.all rejects immediately.
|
||||||
|
*/
|
||||||
|
public static async runWithConcurrency<T>(
|
||||||
|
tasks: (() => Promise<T>)[],
|
||||||
|
concurrency: number,
|
||||||
|
): Promise<T[]> {
|
||||||
|
const results: T[] = new Array(tasks.length);
|
||||||
|
let nextIndex = 0;
|
||||||
|
async function worker(): Promise<void> {
|
||||||
|
while (true) {
|
||||||
|
const idx = nextIndex++;
|
||||||
|
if (idx >= tasks.length) break;
|
||||||
|
results[idx] = await tasks[idx]();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const workers = Array.from(
|
||||||
|
{ length: Math.min(concurrency, tasks.length) },
|
||||||
|
() => worker(),
|
||||||
|
);
|
||||||
|
await Promise.all(workers);
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Builds the corresponding real docker image for each Dockerfile class instance
|
* Builds the corresponding real docker image for each Dockerfile class instance
|
||||||
*/
|
*/
|
||||||
public static async buildDockerfiles(
|
public static async buildDockerfiles(
|
||||||
sortedArrayArg: Dockerfile[],
|
sortedArrayArg: Dockerfile[],
|
||||||
options?: { platform?: string; timeout?: number; noCache?: boolean; verbose?: boolean; isRootless?: boolean },
|
options?: { platform?: string; timeout?: number; noCache?: boolean; verbose?: boolean; isRootless?: boolean; parallel?: boolean; parallelConcurrency?: number },
|
||||||
): Promise<Dockerfile[]> {
|
): Promise<Dockerfile[]> {
|
||||||
const total = sortedArrayArg.length;
|
const total = sortedArrayArg.length;
|
||||||
const overallStart = Date.now();
|
const overallStart = Date.now();
|
||||||
@@ -205,6 +253,54 @@ export class Dockerfile {
|
|||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
if (options?.parallel) {
|
||||||
|
// === PARALLEL MODE: build independent images concurrently within each level ===
|
||||||
|
const concurrency = options.parallelConcurrency ?? 4;
|
||||||
|
const levels = Dockerfile.computeLevels(sortedArrayArg);
|
||||||
|
|
||||||
|
logger.log('info', `Parallel build: ${levels.length} level(s), concurrency ${concurrency}`);
|
||||||
|
for (let l = 0; l < levels.length; l++) {
|
||||||
|
const level = levels[l];
|
||||||
|
logger.log('info', ` Level ${l} (${level.length}): ${level.map(df => df.cleanTag).join(', ')}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
let built = 0;
|
||||||
|
for (let l = 0; l < levels.length; l++) {
|
||||||
|
const level = levels[l];
|
||||||
|
logger.log('info', `--- Level ${l}: building ${level.length} image(s) in parallel ---`);
|
||||||
|
|
||||||
|
const tasks = level.map((df) => {
|
||||||
|
const myIndex = ++built;
|
||||||
|
return async () => {
|
||||||
|
const progress = `(${myIndex}/${total})`;
|
||||||
|
logger.log('info', `${progress} Building ${df.cleanTag}...`);
|
||||||
|
const elapsed = await df.build(options);
|
||||||
|
logger.log('ok', `${progress} Built ${df.cleanTag} in ${formatDuration(elapsed)}`);
|
||||||
|
return df;
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
await Dockerfile.runWithConcurrency(tasks, concurrency);
|
||||||
|
|
||||||
|
// After the entire level completes, tag + push for dependency resolution
|
||||||
|
for (const df of level) {
|
||||||
|
const dependentBaseImages = new Set<string>();
|
||||||
|
for (const other of sortedArrayArg) {
|
||||||
|
if (other.localBaseDockerfile === df && other.baseImage !== df.buildTag) {
|
||||||
|
dependentBaseImages.add(other.baseImage);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (const fullTag of dependentBaseImages) {
|
||||||
|
logger.log('info', `Tagging ${df.buildTag} as ${fullTag} for local dependency resolution`);
|
||||||
|
await smartshellInstance.exec(`docker tag ${df.buildTag} ${fullTag}`);
|
||||||
|
}
|
||||||
|
if (useRegistry && sortedArrayArg.some(other => other.localBaseDockerfile === df)) {
|
||||||
|
await Dockerfile.pushToLocalRegistry(df);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// === SEQUENTIAL MODE: build one at a time ===
|
||||||
for (let i = 0; i < total; i++) {
|
for (let i = 0; i < total; i++) {
|
||||||
const dockerfileArg = sortedArrayArg[i];
|
const dockerfileArg = sortedArrayArg[i];
|
||||||
const progress = `(${i + 1}/${total})`;
|
const progress = `(${i + 1}/${total})`;
|
||||||
@@ -230,6 +326,7 @@ export class Dockerfile {
|
|||||||
await Dockerfile.pushToLocalRegistry(dockerfileArg);
|
await Dockerfile.pushToLocalRegistry(dockerfileArg);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
} finally {
|
} finally {
|
||||||
if (useRegistry) {
|
if (useRegistry) {
|
||||||
await Dockerfile.stopLocalRegistry();
|
await Dockerfile.stopLocalRegistry();
|
||||||
|
|||||||
@@ -167,6 +167,16 @@ export class TsDockerManager {
|
|||||||
logger.log('info', 'Cache: disabled (--no-cache)');
|
logger.log('info', 'Cache: disabled (--no-cache)');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (options?.parallel) {
|
||||||
|
const concurrency = options.parallelConcurrency ?? 4;
|
||||||
|
const levels = Dockerfile.computeLevels(toBuild);
|
||||||
|
logger.log('info', `Parallel build: ${levels.length} level(s), concurrency ${concurrency}`);
|
||||||
|
for (let l = 0; l < levels.length; l++) {
|
||||||
|
const level = levels[l];
|
||||||
|
logger.log('info', ` Level ${l} (${level.length}): ${level.map(df => df.cleanTag).join(', ')}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
logger.log('info', `Building ${toBuild.length} Dockerfile(s)...`);
|
logger.log('info', `Building ${toBuild.length} Dockerfile(s)...`);
|
||||||
|
|
||||||
if (options?.cached) {
|
if (options?.cached) {
|
||||||
@@ -184,6 +194,61 @@ export class TsDockerManager {
|
|||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
if (options?.parallel) {
|
||||||
|
// === PARALLEL CACHED MODE ===
|
||||||
|
const concurrency = options.parallelConcurrency ?? 4;
|
||||||
|
const levels = Dockerfile.computeLevels(toBuild);
|
||||||
|
|
||||||
|
let built = 0;
|
||||||
|
for (let l = 0; l < levels.length; l++) {
|
||||||
|
const level = levels[l];
|
||||||
|
logger.log('info', `--- Level ${l}: building ${level.length} image(s) in parallel ---`);
|
||||||
|
|
||||||
|
const tasks = level.map((df) => {
|
||||||
|
const myIndex = ++built;
|
||||||
|
return async () => {
|
||||||
|
const progress = `(${myIndex}/${total})`;
|
||||||
|
const skip = await cache.shouldSkipBuild(df.cleanTag, df.content);
|
||||||
|
|
||||||
|
if (skip) {
|
||||||
|
logger.log('ok', `${progress} Skipped ${df.cleanTag} (cached)`);
|
||||||
|
} else {
|
||||||
|
logger.log('info', `${progress} Building ${df.cleanTag}...`);
|
||||||
|
const elapsed = await df.build({
|
||||||
|
platform: options?.platform,
|
||||||
|
timeout: options?.timeout,
|
||||||
|
noCache: options?.noCache,
|
||||||
|
verbose: options?.verbose,
|
||||||
|
});
|
||||||
|
logger.log('ok', `${progress} Built ${df.cleanTag} in ${formatDuration(elapsed)}`);
|
||||||
|
const imageId = await df.getId();
|
||||||
|
cache.recordBuild(df.cleanTag, df.content, imageId, df.buildTag);
|
||||||
|
}
|
||||||
|
return df;
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
await Dockerfile.runWithConcurrency(tasks, concurrency);
|
||||||
|
|
||||||
|
// After the entire level completes, tag + push for dependency resolution
|
||||||
|
for (const df of level) {
|
||||||
|
const dependentBaseImages = new Set<string>();
|
||||||
|
for (const other of toBuild) {
|
||||||
|
if (other.localBaseDockerfile === df && other.baseImage !== df.buildTag) {
|
||||||
|
dependentBaseImages.add(other.baseImage);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (const fullTag of dependentBaseImages) {
|
||||||
|
logger.log('info', `Tagging ${df.buildTag} as ${fullTag} for local dependency resolution`);
|
||||||
|
await smartshellInstance.exec(`docker tag ${df.buildTag} ${fullTag}`);
|
||||||
|
}
|
||||||
|
if (useRegistry && toBuild.some(other => other.localBaseDockerfile === df)) {
|
||||||
|
await Dockerfile.pushToLocalRegistry(df);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// === SEQUENTIAL CACHED MODE ===
|
||||||
for (let i = 0; i < total; i++) {
|
for (let i = 0; i < total; i++) {
|
||||||
const dockerfileArg = toBuild[i];
|
const dockerfileArg = toBuild[i];
|
||||||
const progress = `(${i + 1}/${total})`;
|
const progress = `(${i + 1}/${total})`;
|
||||||
@@ -221,6 +286,7 @@ export class TsDockerManager {
|
|||||||
await Dockerfile.pushToLocalRegistry(dockerfileArg);
|
await Dockerfile.pushToLocalRegistry(dockerfileArg);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
} finally {
|
} finally {
|
||||||
if (useRegistry) {
|
if (useRegistry) {
|
||||||
await Dockerfile.stopLocalRegistry();
|
await Dockerfile.stopLocalRegistry();
|
||||||
@@ -237,6 +303,8 @@ export class TsDockerManager {
|
|||||||
noCache: options?.noCache,
|
noCache: options?.noCache,
|
||||||
verbose: options?.verbose,
|
verbose: options?.verbose,
|
||||||
isRootless: this.dockerContext.contextInfo?.isRootless,
|
isRootless: this.dockerContext.contextInfo?.isRootless,
|
||||||
|
parallel: options?.parallel,
|
||||||
|
parallelConcurrency: options?.parallelConcurrency,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -80,6 +80,8 @@ export interface IBuildCommandOptions {
|
|||||||
cached?: boolean; // Skip builds when Dockerfile content hasn't changed
|
cached?: boolean; // Skip builds when Dockerfile content hasn't changed
|
||||||
verbose?: boolean; // Stream raw docker build output (default: silent)
|
verbose?: boolean; // Stream raw docker build output (default: silent)
|
||||||
context?: string; // Explicit Docker context name (--context flag)
|
context?: string; // Explicit Docker context name (--context flag)
|
||||||
|
parallel?: boolean; // Enable parallel builds within dependency levels
|
||||||
|
parallelConcurrency?: number; // Max concurrent builds per level (default 4)
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ICacheEntry {
|
export interface ICacheEntry {
|
||||||
|
|||||||
@@ -55,6 +55,12 @@ export let run = () => {
|
|||||||
if (argvArg.verbose) {
|
if (argvArg.verbose) {
|
||||||
buildOptions.verbose = true;
|
buildOptions.verbose = true;
|
||||||
}
|
}
|
||||||
|
if (argvArg.parallel) {
|
||||||
|
buildOptions.parallel = true;
|
||||||
|
if (typeof argvArg.parallel === 'number') {
|
||||||
|
buildOptions.parallelConcurrency = argvArg.parallel;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
await manager.build(buildOptions);
|
await manager.build(buildOptions);
|
||||||
logger.log('success', 'Build completed successfully');
|
logger.log('success', 'Build completed successfully');
|
||||||
@@ -95,6 +101,12 @@ export let run = () => {
|
|||||||
if (argvArg.verbose) {
|
if (argvArg.verbose) {
|
||||||
buildOptions.verbose = true;
|
buildOptions.verbose = true;
|
||||||
}
|
}
|
||||||
|
if (argvArg.parallel) {
|
||||||
|
buildOptions.parallel = true;
|
||||||
|
if (typeof argvArg.parallel === 'number') {
|
||||||
|
buildOptions.parallelConcurrency = argvArg.parallel;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Build images first (if not already built)
|
// Build images first (if not already built)
|
||||||
await manager.build(buildOptions);
|
await manager.build(buildOptions);
|
||||||
@@ -157,6 +169,12 @@ export let run = () => {
|
|||||||
if (argvArg.verbose) {
|
if (argvArg.verbose) {
|
||||||
buildOptions.verbose = true;
|
buildOptions.verbose = true;
|
||||||
}
|
}
|
||||||
|
if (argvArg.parallel) {
|
||||||
|
buildOptions.parallel = true;
|
||||||
|
if (typeof argvArg.parallel === 'number') {
|
||||||
|
buildOptions.parallelConcurrency = argvArg.parallel;
|
||||||
|
}
|
||||||
|
}
|
||||||
await manager.build(buildOptions);
|
await manager.build(buildOptions);
|
||||||
|
|
||||||
// Run tests
|
// Run tests
|
||||||
|
|||||||
Reference in New Issue
Block a user