diff --git a/changelog.md b/changelog.md index da91147..34d7ef9 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,17 @@ # Changelog +## 2026-02-07 - 1.16.0 - feat(core) +Introduce per-invocation TsDockerSession and session-aware local registry and build orchestration; stream and parse buildx output for improved logging and visibility; detect Docker topology and add CI-safe cleanup; update README with multi-arch, parallel-build, caching, and local registry usage and new CLI flags. + +- Add TsDockerSession to allocate unique ports, container names and builder suffixes for concurrent runs (especially in CI). +- Make local registry session-aware: start/stop/use registry container and persistent storage per session; retry on port conflicts. +- Inject session into Dockerfile instances and TsDockerManager; use session.config.registryHost for tagging/pushing and test container naming. +- Stream and parse buildx/docker build output via createBuildOutputHandler for clearer step/platform/CACHED/DONE logging and --progress=plain usage. +- Detect Docker topology (socket-mount, dind, local) in DockerContext and expose it in context info. +- Add manager.cleanup to remove CI-scoped buildx builders and ensure CLI calls cleanup after build/push/test. +- Update interfaces to include topology and adjust many Dockerfile/manager methods to be session-aware. +- Large README improvements: multi-arch flow, persistent local registry, parallel builds, caching, new CLI and clean flags, and examples for CI integration. + ## 2026-02-07 - 1.15.1 - fix(registry) use persistent local registry and OCI Distribution API image copy for pushes diff --git a/readme.md b/readme.md index d52495e..0248345 100644 --- a/readme.md +++ b/readme.md @@ -1,6 +1,6 @@ # @git.zone/tsdocker -> 🐳 The ultimate Docker development toolkit for TypeScript projects β€” build, test, and ship containerized applications with ease. +> 🐳 The ultimate Docker development toolkit for TypeScript projects β€” build, test, and ship multi-arch containerized applications with zero friction. ## Issue Reporting and Security @@ -8,15 +8,18 @@ For reporting bugs, issues, or security vulnerabilities, please visit [community ## What is tsdocker? -**tsdocker** is a comprehensive Docker development and building tool that handles everything from testing npm packages in clean environments to building and pushing multi-architecture Docker images across multiple registries. +**tsdocker** is a comprehensive Docker development and build tool that handles everything from testing npm packages in clean environments to building and pushing multi-architecture Docker images across multiple registries β€” all from a single CLI. ### 🎯 Key Capabilities - πŸ§ͺ **Containerized Testing** β€” Run your tests in pristine Docker environments - πŸ—οΈ **Smart Docker Builds** β€” Automatically discover, sort, and build Dockerfiles by dependency -- πŸš€ **Multi-Registry Push** β€” Ship to Docker Hub, GitLab, GitHub Container Registry, and more -- πŸ”§ **Multi-Architecture** β€” Build for `amd64` and `arm64` with Docker Buildx -- ⚑ **Zero Config Start** β€” Works out of the box, scales with your needs +- 🌍 **True Multi-Architecture** β€” Build for `amd64` and `arm64` simultaneously with Docker Buildx +- πŸš€ **Multi-Registry Push** β€” Ship to Docker Hub, GitLab, GitHub Container Registry, and more via OCI Distribution API +- ⚑ **Parallel Builds** β€” Level-based parallel builds with configurable concurrency +- πŸ—„οΈ **Persistent Local Registry** β€” All images flow through a local OCI registry with persistent storage +- πŸ“¦ **Build Caching** β€” Skip unchanged Dockerfiles with content-hash caching +- πŸ”§ **Zero Config Start** β€” Works out of the box, scales with your needs ## Installation @@ -53,6 +56,7 @@ tsdocker will: 2. πŸ“Š Analyze `FROM` dependencies between them 3. πŸ”„ Sort them topologically 4. πŸ—οΈ Build each image in the correct order +5. πŸ“¦ Push every image to a persistent local registry (`.nogit/docker-registry/`) ### πŸ“€ Push to Registries @@ -63,33 +67,52 @@ Ship your images to one or all configured registries: tsdocker push # Push to a specific registry -tsdocker push registry.gitlab.com +tsdocker push --registry=registry.gitlab.com ``` +Under the hood, `tsdocker push` uses the **OCI Distribution API** to copy images directly from the local registry to remote registries. This means multi-arch manifest lists are preserved end-to-end β€” no more single-platform-only pushes. + ## CLI Commands | Command | Description | |---------|-------------| -| `tsdocker` | Run tests in a fresh Docker container | +| `tsdocker` | Run tests in a fresh Docker container (legacy mode) | | `tsdocker build` | Build all Dockerfiles with dependency ordering | -| `tsdocker push [registry]` | Push images to configured registries | +| `tsdocker push` | Build + push images to configured registries | | `tsdocker pull ` | Pull images from a specific registry | -| `tsdocker test` | Run container test scripts (test_*.sh) | +| `tsdocker test` | Build + run container test scripts (`test_*.sh`) | | `tsdocker login` | Authenticate with configured registries | | `tsdocker list` | Display discovered Dockerfiles and their dependencies | -| `tsdocker clean --all` | ⚠️ Aggressively clean Docker environment | +| `tsdocker clean` | Interactively clean Docker environment | | `tsdocker vscode` | Launch containerized VS Code in browser | +### Build Flags + +| Flag | Description | +|------|-------------| +| `--platform=linux/arm64` | Override build platform for a single architecture | +| `--timeout=600` | Build timeout in seconds | +| `--no-cache` | Force rebuild without Docker layer cache | +| `--cached` | Skip unchanged Dockerfiles (content-hash based) | +| `--verbose` | Stream raw `docker build` output | +| `--parallel` | Enable level-based parallel builds (default concurrency: 4) | +| `--parallel=8` | Parallel builds with custom concurrency | +| `--context=mycontext` | Use a specific Docker context | + +### Clean Flags + +| Flag | Description | +|------|-------------| +| `--all` | Include all images and volumes (not just dangling) | +| `-y` | Auto-confirm all prompts | + ## Configuration -Configure tsdocker in your `package.json` or `npmextra.json`: +Configure tsdocker in your `package.json` or `npmextra.json` under the `@git.zone/tsdocker` key: ```json { "@git.zone/tsdocker": { - "baseImage": "node:20", - "command": "npm test", - "dockerSock": false, "registries": ["registry.gitlab.com", "docker.io"], "registryRepoMap": { "registry.gitlab.com": "myorg/myproject" @@ -98,7 +121,6 @@ Configure tsdocker in your `package.json` or `npmextra.json`: "NODE_VERSION": "NODE_VERSION" }, "platforms": ["linux/amd64", "linux/arm64"], - "push": false, "testDir": "./test" } } @@ -106,24 +128,73 @@ Configure tsdocker in your `package.json` or `npmextra.json`: ### Configuration Options -#### Testing Options (Legacy) - -| Option | Type | Description | -|--------|------|-------------| -| `baseImage` | `string` | Docker image for test environment (default: `hosttoday/ht-docker-node:npmdocker`) | -| `command` | `string` | Command to run inside container (default: `npmci npm test`) | -| `dockerSock` | `boolean` | Mount Docker socket for DinD scenarios (default: `false`) | - #### Build & Push Options -| Option | Type | Description | -|--------|------|-------------| -| `registries` | `string[]` | Registry URLs to push to | -| `registryRepoMap` | `object` | Map registries to different repository paths | -| `buildArgEnvMap` | `object` | Map Docker build ARGs to environment variables | -| `platforms` | `string[]` | Target architectures (default: `["linux/amd64"]`) | -| `push` | `boolean` | Auto-push after build (default: `false`) | -| `testDir` | `string` | Directory containing test scripts | +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `registries` | `string[]` | `[]` | Registry URLs to push to | +| `registryRepoMap` | `object` | `{}` | Map registries to different repository paths | +| `buildArgEnvMap` | `object` | `{}` | Map Docker build ARGs to environment variables | +| `platforms` | `string[]` | `["linux/amd64"]` | Target architectures for multi-arch builds | +| `testDir` | `string` | `./test` | Directory containing test scripts | + +#### Legacy Testing Options + +These options configure the `tsdocker` default command (containerized test runner): + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `baseImage` | `string` | `hosttoday/ht-docker-node:npmdocker` | Docker image for test environment | +| `command` | `string` | `npmci npm test` | Command to run inside the container | +| `dockerSock` | `boolean` | `false` | Mount Docker socket for DinD scenarios | + +## Architecture: How tsdocker Works + +tsdocker uses a **local OCI registry** as the canonical store for all built images. This design solves fundamental problems with Docker's local daemon, which cannot hold multi-architecture manifest lists. + +### πŸ“ Build Flow + +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ tsdocker build β”‚ +β”‚ β”‚ +β”‚ 1. Start local registry (localhost:5234) β”‚ +β”‚ └── Persistent volume: .nogit/docker-registry/ +β”‚ β”‚ +β”‚ 2. For each Dockerfile (topological order): β”‚ +β”‚ β”œβ”€β”€ Multi-platform: buildx --push β†’ registry β”‚ +β”‚ └── Single-platform: docker build β†’ registry β”‚ +β”‚ β”‚ +β”‚ 3. Stop local registry (data persists on disk) β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +### πŸ“€ Push Flow + +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ tsdocker push β”‚ +β”‚ β”‚ +β”‚ 1. Start local registry (loads persisted data) β”‚ +β”‚ β”‚ +β”‚ 2. For each image Γ— each remote registry: β”‚ +β”‚ └── OCI Distribution API copy: β”‚ +β”‚ β”œβ”€β”€ Fetch manifest (single or multi-arch) β”‚ +β”‚ β”œβ”€β”€ Copy blobs (skip if already exist) β”‚ +β”‚ └── Push manifest with destination tag β”‚ +β”‚ β”‚ +β”‚ 3. Stop local registry β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +### πŸ”‘ Why a Local Registry? + +| Problem | Solution | +|---------|----------| +| `docker buildx --load` fails for multi-arch images | `buildx --push` to local registry works for any number of platforms | +| `docker push` only pushes single-platform manifests | OCI API copy preserves full manifest lists (multi-arch) | +| Images lost between build and push phases | Persistent storage at `.nogit/docker-registry/` survives restarts | +| Redundant blob uploads on incremental pushes | HEAD checks skip blobs that already exist on the remote | ## Registry Authentication @@ -140,13 +211,17 @@ export DOCKER_REGISTRY_USER="username" export DOCKER_REGISTRY_PASSWORD="password" ``` +### Docker Config Fallback + +When pushing, tsdocker will also read credentials from `~/.docker/config.json` if no explicit credentials are provided via environment variables. This means `docker login` credentials work automatically. + ### Login Command ```bash tsdocker login ``` -Authenticates with all configured registries. +Authenticates with all configured registries using the provided environment variables. ## Advanced Usage @@ -162,7 +237,27 @@ Build for multiple platforms using Docker Buildx: } ``` -tsdocker automatically sets up a Buildx builder when multiple platforms are specified. +tsdocker automatically: +- Sets up a Buildx builder with `--driver-opt network=host` (so buildx can reach the local registry) +- Pushes multi-platform images to the local registry via `buildx --push` +- Copies the full manifest list (including all platform variants) to remote registries on `tsdocker push` + +### ⚑ Parallel Builds + +Speed up builds by building independent images concurrently: + +```bash +# Default concurrency (4 workers) +tsdocker build --parallel + +# Custom concurrency +tsdocker build --parallel=8 + +# Works with caching too +tsdocker build --parallel --cached +``` + +tsdocker groups Dockerfiles into **dependency levels** using topological analysis. Images within the same level have no dependencies on each other and build in parallel. Each level completes before the next begins. ### πŸ“¦ Dockerfile Naming Conventions @@ -190,7 +285,7 @@ COPY . . RUN npm run build ``` -tsdocker automatically detects that `Dockerfile_app` depends on `Dockerfile_base` and builds them in the correct order. +tsdocker automatically detects that `Dockerfile_app` depends on `Dockerfile_base`, builds them in the correct order, and makes the base image available to dependent builds via the local registry (using `--build-context` for buildx). ### πŸ§ͺ Container Test Scripts @@ -210,6 +305,8 @@ Run with: tsdocker test ``` +This builds all images, starts the local registry (so multi-arch images can be pulled), and runs each matching test script inside a container. + ### πŸ”§ Build Args from Environment Pass environment variables as Docker build arguments: @@ -232,6 +329,24 @@ FROM node:${NODE_VERSION} RUN echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > ~/.npmrc ``` +### πŸ—ΊοΈ Registry Repo Mapping + +Use different repository names for different registries: + +```json +{ + "@git.zone/tsdocker": { + "registries": ["registry.gitlab.com", "docker.io"], + "registryRepoMap": { + "registry.gitlab.com": "mygroup/myproject", + "docker.io": "myuser/myproject" + } + } +} +``` + +When pushing, tsdocker maps the local repo name to the registry-specific path. For example, a locally built `myproject:latest` becomes `registry.gitlab.com/mygroup/myproject:latest` and `docker.io/myuser/myproject:latest`. + ### 🐳 Docker-in-Docker Testing Test Docker-related tools by mounting the Docker socket: @@ -259,68 +374,40 @@ Output: Discovered Dockerfiles: ======================== -1. Dockerfile_base +1. /path/to/Dockerfile_base Tag: myproject:base Base Image: node:20-alpine Version: base -2. Dockerfile_app +2. /path/to/Dockerfile_app Tag: myproject:app Base Image: myproject:base Version: app Depends on: myproject:base ``` -### πŸ—ΊοΈ Registry Repo Mapping - -Use different repository names for different registries: - -```json -{ - "@git.zone/tsdocker": { - "registries": ["registry.gitlab.com", "docker.io"], - "registryRepoMap": { - "registry.gitlab.com": "mygroup/myproject", - "docker.io": "myuser/myproject" - } - } -} -``` - -## Environment Variables - -### qenv Integration - -tsdocker automatically loads environment variables from `qenv.yml`: - -```yaml -# qenv.yml -API_KEY: your-api-key -DATABASE_URL: postgres://localhost/test -``` - -These are injected into your test container automatically. - ## Examples -### Basic Test Configuration +### Minimal Build & Push ```json { "@git.zone/tsdocker": { - "baseImage": "node:20", - "command": "npm test" + "registries": ["docker.io"], + "platforms": ["linux/amd64"] } } ``` +```bash +tsdocker push +``` + ### Full Production Setup ```json { "@git.zone/tsdocker": { - "baseImage": "node:20-alpine", - "command": "pnpm test", "registries": ["registry.gitlab.com", "ghcr.io", "docker.io"], "registryRepoMap": { "registry.gitlab.com": "myorg/myapp", @@ -338,57 +425,37 @@ These are injected into your test container automatically. ### CI/CD Integration +**GitLab CI:** + ```yaml -# .gitlab-ci.yml -build: +build-and-push: stage: build script: - npm install -g @git.zone/tsdocker - - tsdocker build - tsdocker push + variables: + DOCKER_REGISTRY_1: "registry.gitlab.com|$CI_REGISTRY_USER|$CI_REGISTRY_PASSWORD" +``` -# GitHub Actions +**GitHub Actions:** + +```yaml - name: Build and Push run: | npm install -g @git.zone/tsdocker tsdocker login - tsdocker build tsdocker push env: DOCKER_REGISTRY_1: "ghcr.io|${{ github.actor }}|${{ secrets.GITHUB_TOKEN }}" ``` -## Requirements - -- **Docker** β€” Docker Engine or Docker Desktop must be installed -- **Node.js** β€” Version 18 or higher (ESM support required) -- **Docker Buildx** β€” Required for multi-architecture builds (included in Docker Desktop) - -## Why tsdocker? - -### 🎯 The Problem - -Managing Docker workflows manually is tedious: -- Remembering build order for dependent images -- Pushing to multiple registries with different credentials -- Setting up Buildx for multi-arch builds -- Ensuring consistent test environments - -### ✨ The Solution - -tsdocker automates the entire workflow: -- **One command** to build all images in dependency order -- **One command** to push to all registries -- **Automatic** Buildx setup for multi-platform builds -- **Consistent** containerized test environments - ## TypeScript API -tsdocker exposes its types for programmatic use: +tsdocker can also be used programmatically: ```typescript -import type { ITsDockerConfig } from '@git.zone/tsdocker/dist_ts/interfaces/index.js'; import { TsDockerManager } from '@git.zone/tsdocker/dist_ts/classes.tsdockermanager.js'; +import type { ITsDockerConfig } from '@git.zone/tsdocker/dist_ts/interfaces/index.js'; const config: ITsDockerConfig = { baseImage: 'node:20', @@ -396,15 +463,21 @@ const config: ITsDockerConfig = { dockerSock: false, keyValueObject: {}, registries: ['docker.io'], - platforms: ['linux/amd64'], + platforms: ['linux/amd64', 'linux/arm64'], }; const manager = new TsDockerManager(config); await manager.prepare(); -await manager.build(); +await manager.build({ parallel: true }); await manager.push(); ``` +## Requirements + +- **Docker** β€” Docker Engine 20+ or Docker Desktop +- **Node.js** β€” Version 18 or higher (for native `fetch` and ESM support) +- **Docker Buildx** β€” Required for multi-architecture builds (included in Docker Desktop) + ## Troubleshooting ### "docker not found" @@ -417,11 +490,10 @@ docker --version ### Multi-arch build fails -Make sure Docker Buildx is available: +Make sure Docker Buildx is available. tsdocker will set up the builder automatically, but you can verify: ```bash docker buildx version -docker buildx create --use ``` ### Registry authentication fails @@ -433,19 +505,22 @@ echo $DOCKER_REGISTRY_1 tsdocker login ``` +tsdocker also falls back to `~/.docker/config.json` β€” ensure you've run `docker login` for your target registries. + ### Circular dependency detected Review your Dockerfiles' `FROM` statements β€” you have images depending on each other in a loop. -## Performance Tips +### Build context too large -πŸš€ **Use specific tags**: `node:20-alpine` is smaller and faster than `node:latest` +Use a `.dockerignore` file to exclude `node_modules`, `.git`, `.nogit`, and other large directories: -πŸš€ **Leverage caching**: Docker layers are cached β€” your builds get faster over time - -πŸš€ **Prune regularly**: `docker system prune` reclaims disk space - -πŸš€ **Use .dockerignore**: Exclude `node_modules`, `.git`, etc. from build context +``` +node_modules +.git +.nogit +dist_ts +``` ## Migration from Legacy diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index e95f2d2..d0f1de3 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@git.zone/tsdocker', - version: '1.15.1', + version: '1.16.0', description: 'develop npm modules cross platform with docker' } diff --git a/ts/classes.dockercontext.ts b/ts/classes.dockercontext.ts index 1a8b1a7..bc3df4e 100644 --- a/ts/classes.dockercontext.ts +++ b/ts/classes.dockercontext.ts @@ -1,4 +1,5 @@ import * as plugins from './tsdocker.plugins.js'; +import * as fs from 'fs'; import { logger } from './tsdocker.logging.js'; import type { IDockerContextInfo } from './interfaces/index.js'; @@ -38,19 +39,28 @@ export class DockerContext { isRootless = infoResult.stdout.includes('name=rootless'); } - this.contextInfo = { name, endpoint, isRootless, dockerHost: process.env.DOCKER_HOST }; + // Detect topology + let topology: 'socket-mount' | 'dind' | 'local' = 'local'; + if (process.env.DOCKER_HOST && process.env.DOCKER_HOST.startsWith('tcp://')) { + topology = 'dind'; + } else if (fs.existsSync('/.dockerenv')) { + topology = 'socket-mount'; + } + + this.contextInfo = { name, endpoint, isRootless, dockerHost: process.env.DOCKER_HOST, topology }; return this.contextInfo; } /** Logs context info prominently. */ public logContextInfo(): void { if (!this.contextInfo) return; - const { name, endpoint, isRootless, dockerHost } = this.contextInfo; + const { name, endpoint, isRootless, dockerHost, topology } = this.contextInfo; logger.log('info', '=== DOCKER CONTEXT ==='); logger.log('info', `Context: ${name}`); logger.log('info', `Endpoint: ${endpoint}`); if (dockerHost) logger.log('info', `DOCKER_HOST: ${dockerHost}`); logger.log('info', `Rootless: ${isRootless ? 'yes' : 'no'}`); + logger.log('info', `Topology: ${topology || 'local'}`); } /** Emits rootless-specific warnings. */ diff --git a/ts/classes.dockerfile.ts b/ts/classes.dockerfile.ts index 642dbc0..00582e1 100644 --- a/ts/classes.dockerfile.ts +++ b/ts/classes.dockerfile.ts @@ -3,6 +3,7 @@ import * as paths from './tsdocker.paths.js'; import { logger, formatDuration } from './tsdocker.logging.js'; import { DockerRegistry } from './classes.dockerregistry.js'; import { RegistryCopy } from './classes.registrycopy.js'; +import { TsDockerSession } from './classes.tsdockersession.js'; import type { IDockerfileOptions, ITsDockerConfig, IBuildCommandOptions } from './interfaces/index.js'; import type { TsDockerManager } from './classes.tsdockermanager.js'; import * as fs from 'fs'; @@ -11,9 +12,14 @@ const smartshellInstance = new plugins.smartshell.Smartshell({ executor: 'bash', }); -const LOCAL_REGISTRY_PORT = 5234; -const LOCAL_REGISTRY_HOST = `localhost:${LOCAL_REGISTRY_PORT}`; -const LOCAL_REGISTRY_CONTAINER = 'tsdocker-local-registry'; +/** + * Extracts a platform string (e.g. "linux/amd64") from a buildx bracket prefix. + * The prefix may be like "linux/amd64 ", "linux/amd64 stage-1 ", "stage-1 ", or "". + */ +function extractPlatform(prefix: string): string | null { + const match = prefix.match(/linux\/\w+/); + return match ? match[0] : null; +} /** * Class Dockerfile represents a Dockerfile on disk @@ -148,40 +154,55 @@ export class Dockerfile { return true; } - /** Starts a persistent registry:2 container on port 5234 with volume storage. */ - public static async startLocalRegistry(isRootless?: boolean): Promise { - // Ensure persistent storage directory exists - const registryDataDir = plugins.path.join(paths.cwd, '.nogit', 'docker-registry'); + /** Starts a persistent registry:2 container with session-unique port and name. */ + public static async startLocalRegistry(session: TsDockerSession, isRootless?: boolean): Promise { + const { registryPort, registryHost, registryContainerName, isCI, sessionId } = session.config; + + // Ensure persistent storage directory exists β€” isolate per session in CI + const registryDataDir = isCI + ? plugins.path.join(paths.cwd, '.nogit', 'docker-registry', sessionId) + : plugins.path.join(paths.cwd, '.nogit', 'docker-registry'); fs.mkdirSync(registryDataDir, { recursive: true }); await smartshellInstance.execSilent( - `docker rm -f ${LOCAL_REGISTRY_CONTAINER} 2>/dev/null || true` - ); - const result = await smartshellInstance.execSilent( - `docker run -d --name ${LOCAL_REGISTRY_CONTAINER} -p ${LOCAL_REGISTRY_PORT}:5000 -v "${registryDataDir}:/var/lib/registry" registry:2` + `docker rm -f ${registryContainerName} 2>/dev/null || true` ); + + const runCmd = `docker run -d --name ${registryContainerName} -p ${registryPort}:5000 -v "${registryDataDir}:/var/lib/registry" registry:2`; + let result = await smartshellInstance.execSilent(runCmd); + + // Port retry: if port was stolen between allocation and docker run, reallocate once + if (result.exitCode !== 0 && (result.stderr || result.stdout || '').includes('port is already allocated')) { + const newPort = await TsDockerSession.allocatePort(); + logger.log('warn', `Port ${registryPort} taken, retrying with ${newPort}`); + session.config.registryPort = newPort; + session.config.registryHost = `localhost:${newPort}`; + const retryCmd = `docker run -d --name ${registryContainerName} -p ${newPort}:5000 -v "${registryDataDir}:/var/lib/registry" registry:2`; + result = await smartshellInstance.execSilent(retryCmd); + } + if (result.exitCode !== 0) { throw new Error(`Failed to start local registry: ${result.stderr || result.stdout}`); } // registry:2 starts near-instantly; brief wait for readiness await new Promise(resolve => setTimeout(resolve, 1000)); - logger.log('info', `Started local registry at ${LOCAL_REGISTRY_HOST} (persistent storage at .nogit/docker-registry/)`); + logger.log('info', `Started local registry at ${session.config.registryHost} (container: ${registryContainerName})`); if (isRootless) { - logger.log('warn', `[rootless] Registry on port ${LOCAL_REGISTRY_PORT} β€” if buildx cannot reach localhost:${LOCAL_REGISTRY_PORT}, try 127.0.0.1:${LOCAL_REGISTRY_PORT}`); + logger.log('warn', `[rootless] Registry on port ${session.config.registryPort} β€” if buildx cannot reach localhost, try 127.0.0.1`); } } - /** Stops and removes the temporary local registry container. */ - public static async stopLocalRegistry(): Promise { + /** Stops and removes the session-specific local registry container. */ + public static async stopLocalRegistry(session: TsDockerSession): Promise { await smartshellInstance.execSilent( - `docker rm -f ${LOCAL_REGISTRY_CONTAINER} 2>/dev/null || true` + `docker rm -f ${session.config.registryContainerName} 2>/dev/null || true` ); - logger.log('info', 'Stopped local registry'); + logger.log('info', `Stopped local registry (${session.config.registryContainerName})`); } /** Pushes a built image to the local registry for buildx consumption. */ - public static async pushToLocalRegistry(dockerfile: Dockerfile): Promise { - const registryTag = `${LOCAL_REGISTRY_HOST}/${dockerfile.buildTag}`; + public static async pushToLocalRegistry(session: TsDockerSession, dockerfile: Dockerfile): Promise { + const registryTag = `${session.config.registryHost}/${dockerfile.buildTag}`; await smartshellInstance.execSilent(`docker tag ${dockerfile.buildTag} ${registryTag}`); const result = await smartshellInstance.execSilent(`docker push ${registryTag}`); if (result.exitCode !== 0) { @@ -244,12 +265,13 @@ export class Dockerfile { */ public static async buildDockerfiles( sortedArrayArg: Dockerfile[], + session: TsDockerSession, options?: { platform?: string; timeout?: number; noCache?: boolean; verbose?: boolean; isRootless?: boolean; parallel?: boolean; parallelConcurrency?: number }, ): Promise { const total = sortedArrayArg.length; const overallStart = Date.now(); - await Dockerfile.startLocalRegistry(options?.isRootless); + await Dockerfile.startLocalRegistry(session, options?.isRootless); try { if (options?.parallel) { @@ -296,7 +318,7 @@ export class Dockerfile { } // Push ALL images to local registry (skip if already pushed via buildx) if (!df.localRegistryTag) { - await Dockerfile.pushToLocalRegistry(df); + await Dockerfile.pushToLocalRegistry(session, df); } } } @@ -324,12 +346,12 @@ export class Dockerfile { // Push ALL images to local registry (skip if already pushed via buildx) if (!dockerfileArg.localRegistryTag) { - await Dockerfile.pushToLocalRegistry(dockerfileArg); + await Dockerfile.pushToLocalRegistry(session, dockerfileArg); } } } } finally { - await Dockerfile.stopLocalRegistry(); + await Dockerfile.stopLocalRegistry(session); } logger.log('info', `Total build time: ${formatDuration(Date.now() - overallStart)}`); @@ -520,6 +542,7 @@ export class Dockerfile { // INSTANCE PROPERTIES public managerRef: TsDockerManager; + public session?: TsDockerSession; public filePath!: string; public repo: string; public version: string; @@ -563,6 +586,79 @@ export class Dockerfile { this.localBaseImageDependent = false; } + /** + * Creates a line-by-line handler for Docker build output that logs + * recognized layer/step lines in an emphasized format. + */ + private createBuildOutputHandler(verbose: boolean): { + handleChunk: (chunk: Buffer | string) => void; + } { + let buffer = ''; + const tag = this.cleanTag; + + const handleLine = (line: string) => { + // In verbose mode, write raw output prefixed with tag for identification + if (verbose) { + process.stdout.write(`[${tag}] ${line}\n`); + } + + // Buildx step: #N [platform step/total] INSTRUCTION + const bxStep = line.match(/^#\d+ \[([^\]]+?)(\d+\/\d+)\] (.+)/); + if (bxStep) { + const prefix = bxStep[1].trim(); + const step = bxStep[2]; + const instruction = bxStep[3]; + const platform = extractPlatform(prefix); + const platStr = platform ? `${platform} β–Έ ` : ''; + logger.log('note', `[${tag}] ${platStr}[${step}] ${instruction}`); + return; + } + + // Buildx CACHED: #N CACHED + const bxCached = line.match(/^#(\d+) CACHED/); + if (bxCached) { + logger.log('note', `[${tag}] CACHED`); + return; + } + + // Buildx DONE: #N DONE 12.3s + const bxDone = line.match(/^#\d+ DONE (.+)/); + if (bxDone) { + const timing = bxDone[1]; + if (!timing.startsWith('0.0')) { + logger.log('note', `[${tag}] DONE ${timing}`); + } + return; + } + + // Buildx export phase: #N exporting ... + const bxExport = line.match(/^#\d+ exporting (.+)/); + if (bxExport) { + logger.log('note', `[${tag}] exporting ${bxExport[1]}`); + return; + } + + // Standard docker build: Step N/M : INSTRUCTION + const stdStep = line.match(/^Step (\d+\/\d+) : (.+)/); + if (stdStep) { + logger.log('note', `[${tag}] Step ${stdStep[1]}: ${stdStep[2]}`); + return; + } + }; + + return { + handleChunk: (chunk: Buffer | string) => { + buffer += chunk.toString(); + const lines = buffer.split('\n'); + buffer = lines.pop() || ''; + for (const line of lines) { + const trimmed = line.replace(/\r$/, '').trim(); + if (trimmed) handleLine(trimmed); + } + }, + }; + } + /** * Builds the Dockerfile */ @@ -590,27 +686,32 @@ export class Dockerfile { if (platformOverride) { // Single platform override via buildx - buildCommand = `docker buildx build --platform ${platformOverride}${noCacheFlag}${buildContextFlag} --load -t ${this.buildTag} -f ${this.filePath} ${buildArgsString} .`; + buildCommand = `docker buildx build --progress=plain --platform ${platformOverride}${noCacheFlag}${buildContextFlag} --load -t ${this.buildTag} -f ${this.filePath} ${buildArgsString} .`; logger.log('info', `Build: buildx --platform ${platformOverride} --load`); } else if (config.platforms && config.platforms.length > 1) { // Multi-platform build using buildx β€” always push to local registry const platformString = config.platforms.join(','); - const localTag = `${LOCAL_REGISTRY_HOST}/${this.buildTag}`; - buildCommand = `docker buildx build --platform ${platformString}${noCacheFlag}${buildContextFlag} -t ${localTag} -f ${this.filePath} ${buildArgsString} --push .`; + const registryHost = this.session?.config.registryHost || 'localhost:5234'; + const localTag = `${registryHost}/${this.buildTag}`; + buildCommand = `docker buildx build --progress=plain --platform ${platformString}${noCacheFlag}${buildContextFlag} -t ${localTag} -f ${this.filePath} ${buildArgsString} --push .`; this.localRegistryTag = localTag; logger.log('info', `Build: buildx --platform ${platformString} --push to local registry`); } else { // Standard build const versionLabel = this.managerRef.projectInfo?.npm?.version || 'unknown'; - buildCommand = `docker build --label="version=${versionLabel}"${noCacheFlag} -t ${this.buildTag} -f ${this.filePath} ${buildArgsString} .`; + buildCommand = `docker build --progress=plain --label="version=${versionLabel}"${noCacheFlag} -t ${this.buildTag} -f ${this.filePath} ${buildArgsString} .`; logger.log('info', 'Build: docker build (standard)'); } + // Execute build with real-time layer logging + const handler = this.createBuildOutputHandler(verbose); + const streaming = await smartshellInstance.execStreamingSilent(buildCommand); + + // Intercept output for layer logging + streaming.childProcess.stdout?.on('data', handler.handleChunk); + streaming.childProcess.stderr?.on('data', handler.handleChunk); + if (timeout) { - // Use streaming execution with timeout - const streaming = verbose - ? await smartshellInstance.execStreaming(buildCommand) - : await smartshellInstance.execStreamingSilent(buildCommand); const timeoutPromise = new Promise((_, reject) => { setTimeout(() => { streaming.childProcess.kill(); @@ -623,9 +724,7 @@ export class Dockerfile { throw new Error(`Build failed for ${this.cleanTag}`); } } else { - const result = verbose - ? await smartshellInstance.exec(buildCommand) - : await smartshellInstance.execSilent(buildCommand); + const result = await streaming.finalPromise; if (result.exitCode !== 0) { logger.log('error', `Build failed for ${this.cleanTag}`); if (!verbose && result.stdout) { @@ -646,12 +745,13 @@ export class Dockerfile { const destRepo = this.getDestRepo(dockerRegistryArg.registryUrl); const destTag = versionSuffix ? `${this.version}_${versionSuffix}` : this.version; const registryCopy = new RegistryCopy(); + const registryHost = this.session?.config.registryHost || 'localhost:5234'; this.pushTag = `${dockerRegistryArg.registryUrl}/${destRepo}:${destTag}`; logger.log('info', `Pushing ${this.pushTag} via OCI copy from local registry...`); await registryCopy.copyImage( - LOCAL_REGISTRY_HOST, + registryHost, this.repo, this.version, dockerRegistryArg.registryUrl, @@ -701,23 +801,27 @@ export class Dockerfile { // Use local registry tag for multi-platform images (not in daemon), otherwise buildTag const imageRef = this.localRegistryTag || this.buildTag; + const sessionId = this.session?.config.sessionId || 'default'; + const testContainerName = `tsdocker_test_${sessionId}`; + const testImageName = `tsdocker_test_image_${sessionId}`; + const testFileExists = fs.existsSync(testFile); if (testFileExists) { // Run tests in container await smartshellInstance.exec( - `docker run --name tsdocker_test_container --entrypoint="bash" ${imageRef} -c "mkdir /tsdocker_test"` + `docker run --name ${testContainerName} --entrypoint="bash" ${imageRef} -c "mkdir /tsdocker_test"` ); - await smartshellInstance.exec(`docker cp ${testFile} tsdocker_test_container:/tsdocker_test/test.sh`); - await smartshellInstance.exec(`docker commit tsdocker_test_container tsdocker_test_image`); + await smartshellInstance.exec(`docker cp ${testFile} ${testContainerName}:/tsdocker_test/test.sh`); + await smartshellInstance.exec(`docker commit ${testContainerName} ${testImageName}`); const testResult = await smartshellInstance.exec( - `docker run --entrypoint="bash" tsdocker_test_image -x /tsdocker_test/test.sh` + `docker run --entrypoint="bash" ${testImageName} -x /tsdocker_test/test.sh` ); // Cleanup - await smartshellInstance.exec(`docker rm tsdocker_test_container`); - await smartshellInstance.exec(`docker rmi --force tsdocker_test_image`); + await smartshellInstance.exec(`docker rm ${testContainerName}`); + await smartshellInstance.exec(`docker rmi --force ${testImageName}`); if (testResult.exitCode !== 0) { throw new Error(`Tests failed for ${this.cleanTag}`); diff --git a/ts/classes.tsdockermanager.ts b/ts/classes.tsdockermanager.ts index 97ff51b..c22b686 100644 --- a/ts/classes.tsdockermanager.ts +++ b/ts/classes.tsdockermanager.ts @@ -6,6 +6,7 @@ import { DockerRegistry } from './classes.dockerregistry.js'; import { RegistryStorage } from './classes.registrystorage.js'; import { TsDockerCache } from './classes.tsdockercache.js'; import { DockerContext } from './classes.dockercontext.js'; +import { TsDockerSession } from './classes.tsdockersession.js'; import type { ITsDockerConfig, IBuildCommandOptions } from './interfaces/index.js'; const smartshellInstance = new plugins.smartshell.Smartshell({ @@ -20,6 +21,7 @@ export class TsDockerManager { public config: ITsDockerConfig; public projectInfo: any; public dockerContext: DockerContext; + public session!: TsDockerSession; private dockerfiles: Dockerfile[] = []; constructor(config: ITsDockerConfig) { @@ -77,6 +79,9 @@ export class TsDockerManager { } } + // Create session identity (unique ports, names for CI concurrency) + this.session = await TsDockerSession.create(); + logger.log('info', `Prepared TsDockerManager with ${this.registryStorage.getAllRegistries().length} registries`); } @@ -98,6 +103,10 @@ export class TsDockerManager { this.dockerfiles = await Dockerfile.readDockerfiles(this); this.dockerfiles = await Dockerfile.sortDockerfiles(this.dockerfiles); this.dockerfiles = await Dockerfile.mapDockerfiles(this.dockerfiles); + // Inject session into each Dockerfile + for (const df of this.dockerfiles) { + df.session = this.session; + } return this.dockerfiles; } @@ -187,7 +196,7 @@ export class TsDockerManager { const total = toBuild.length; const overallStart = Date.now(); - await Dockerfile.startLocalRegistry(this.dockerContext.contextInfo?.isRootless); + await Dockerfile.startLocalRegistry(this.session, this.dockerContext.contextInfo?.isRootless); try { if (options?.parallel) { @@ -240,7 +249,7 @@ export class TsDockerManager { } // Push ALL images to local registry (skip if already pushed via buildx) if (!df.localRegistryTag) { - await Dockerfile.pushToLocalRegistry(df); + await Dockerfile.pushToLocalRegistry(this.session, df); } } } @@ -280,19 +289,19 @@ export class TsDockerManager { // Push ALL images to local registry (skip if already pushed via buildx) if (!dockerfileArg.localRegistryTag) { - await Dockerfile.pushToLocalRegistry(dockerfileArg); + await Dockerfile.pushToLocalRegistry(this.session, dockerfileArg); } } } } finally { - await Dockerfile.stopLocalRegistry(); + await Dockerfile.stopLocalRegistry(this.session); } logger.log('info', `Total build time: ${formatDuration(Date.now() - overallStart)}`); cache.save(); } else { // === STANDARD MODE: build all via static helper === - await Dockerfile.buildDockerfiles(toBuild, { + await Dockerfile.buildDockerfiles(toBuild, this.session, { platform: options?.platform, timeout: options?.timeout, noCache: options?.noCache, @@ -329,7 +338,7 @@ export class TsDockerManager { * Ensures Docker buildx is set up for multi-architecture builds */ private async ensureBuildx(): Promise { - const builderName = this.dockerContext.getBuilderName(); + const builderName = this.dockerContext.getBuilderName() + (this.session?.config.builderSuffix || ''); const platforms = this.config.platforms?.join(', ') || 'default'; logger.log('info', `Setting up Docker buildx [${platforms}]...`); logger.log('info', `Builder: ${builderName}`); @@ -394,7 +403,7 @@ export class TsDockerManager { } // Start local registry (reads from persistent .nogit/docker-registry/) - await Dockerfile.startLocalRegistry(this.dockerContext.contextInfo?.isRootless); + await Dockerfile.startLocalRegistry(this.session, this.dockerContext.contextInfo?.isRootless); try { // Push each Dockerfile to each registry via OCI copy for (const dockerfile of this.dockerfiles) { @@ -403,7 +412,7 @@ export class TsDockerManager { } } } finally { - await Dockerfile.stopLocalRegistry(); + await Dockerfile.stopLocalRegistry(this.session); } logger.log('success', 'All images pushed successfully'); @@ -446,11 +455,11 @@ export class TsDockerManager { logger.log('info', ''); logger.log('info', '=== TEST PHASE ==='); - await Dockerfile.startLocalRegistry(this.dockerContext.contextInfo?.isRootless); + await Dockerfile.startLocalRegistry(this.session, this.dockerContext.contextInfo?.isRootless); try { await Dockerfile.testDockerfiles(this.dockerfiles); } finally { - await Dockerfile.stopLocalRegistry(); + await Dockerfile.stopLocalRegistry(this.session); } logger.log('success', 'All tests completed'); @@ -490,4 +499,16 @@ export class TsDockerManager { public getDockerfiles(): Dockerfile[] { return this.dockerfiles; } + + /** + * Cleans up session-specific resources. + * In CI, removes the session-specific buildx builder to avoid accumulation. + */ + public async cleanup(): Promise { + if (this.session?.config.isCI && this.session.config.builderSuffix) { + const builderName = this.dockerContext.getBuilderName() + this.session.config.builderSuffix; + logger.log('info', `CI cleanup: removing buildx builder ${builderName}`); + await smartshellInstance.execSilent(`docker buildx rm ${builderName} 2>/dev/null || true`); + } + } } diff --git a/ts/classes.tsdockersession.ts b/ts/classes.tsdockersession.ts new file mode 100644 index 0000000..69e1fbc --- /dev/null +++ b/ts/classes.tsdockersession.ts @@ -0,0 +1,107 @@ +import * as crypto from 'crypto'; +import * as net from 'net'; +import { logger } from './tsdocker.logging.js'; + +export interface ISessionConfig { + sessionId: string; + registryPort: number; + registryHost: string; + registryContainerName: string; + isCI: boolean; + ciSystem: string | null; + builderSuffix: string; +} + +/** + * Per-invocation session identity for tsdocker. + * Generates unique ports, container names, and builder names so that + * concurrent CI jobs on the same Docker host don't collide. + * + * In local (non-CI) dev the builder suffix is empty, preserving the + * persistent builder behavior. + */ +export class TsDockerSession { + public config: ISessionConfig; + + private constructor(config: ISessionConfig) { + this.config = config; + } + + /** + * Creates a new session. Allocates a dynamic port unless overridden + * via `TSDOCKER_REGISTRY_PORT`. + */ + public static async create(): Promise { + const sessionId = + process.env.TSDOCKER_SESSION_ID || crypto.randomBytes(4).toString('hex'); + + const registryPort = await TsDockerSession.allocatePort(); + const registryHost = `localhost:${registryPort}`; + const registryContainerName = `tsdocker-registry-${sessionId}`; + + const { isCI, ciSystem } = TsDockerSession.detectCI(); + const builderSuffix = isCI ? `-${sessionId}` : ''; + + const config: ISessionConfig = { + sessionId, + registryPort, + registryHost, + registryContainerName, + isCI, + ciSystem, + builderSuffix, + }; + + const session = new TsDockerSession(config); + session.logInfo(); + return session; + } + + /** + * Allocates a free TCP port. Respects `TSDOCKER_REGISTRY_PORT` override. + */ + public static async allocatePort(): Promise { + const envPort = process.env.TSDOCKER_REGISTRY_PORT; + if (envPort) { + const parsed = parseInt(envPort, 10); + if (!isNaN(parsed) && parsed > 0) { + return parsed; + } + } + + return new Promise((resolve, reject) => { + const srv = net.createServer(); + srv.listen(0, '127.0.0.1', () => { + const addr = srv.address() as net.AddressInfo; + const port = addr.port; + srv.close((err) => { + if (err) reject(err); + else resolve(port); + }); + }); + srv.on('error', reject); + }); + } + + /** + * Detects whether we're running inside a CI system. + */ + private static detectCI(): { isCI: boolean; ciSystem: string | null } { + if (process.env.GITEA_ACTIONS) return { isCI: true, ciSystem: 'gitea-actions' }; + if (process.env.GITHUB_ACTIONS) return { isCI: true, ciSystem: 'github-actions' }; + if (process.env.GITLAB_CI) return { isCI: true, ciSystem: 'gitlab-ci' }; + if (process.env.CI) return { isCI: true, ciSystem: 'generic' }; + return { isCI: false, ciSystem: null }; + } + + private logInfo(): void { + const c = this.config; + logger.log('info', '=== TSDOCKER SESSION ==='); + logger.log('info', `Session ID: ${c.sessionId}`); + logger.log('info', `Registry: ${c.registryHost} (container: ${c.registryContainerName})`); + if (c.isCI) { + logger.log('info', `CI detected: ${c.ciSystem}`); + logger.log('info', `Builder suffix: ${c.builderSuffix}`); + } + } +} diff --git a/ts/interfaces/index.ts b/ts/interfaces/index.ts index 9dfd5a2..284e101 100644 --- a/ts/interfaces/index.ts +++ b/ts/interfaces/index.ts @@ -101,4 +101,5 @@ export interface IDockerContextInfo { endpoint: string; // 'unix:///var/run/docker.sock' isRootless: boolean; dockerHost?: string; // value of DOCKER_HOST env var, if set + topology?: 'socket-mount' | 'dind' | 'local'; } diff --git a/ts/tsdocker.cli.ts b/ts/tsdocker.cli.ts index 1d1e2b4..8f5d131 100644 --- a/ts/tsdocker.cli.ts +++ b/ts/tsdocker.cli.ts @@ -64,6 +64,7 @@ export let run = () => { } await manager.build(buildOptions); + await manager.cleanup(); logger.log('success', 'Build completed successfully'); } catch (err) { logger.log('error', `Build failed: ${(err as Error).message}`); @@ -117,6 +118,7 @@ export let run = () => { const registries = registryArg ? [registryArg] : undefined; await manager.push(registries); + await manager.cleanup(); logger.log('success', 'Push completed successfully'); } catch (err) { logger.log('error', `Push failed: ${(err as Error).message}`); @@ -180,6 +182,7 @@ export let run = () => { // Run tests await manager.test(); + await manager.cleanup(); logger.log('success', 'Tests completed successfully'); } catch (err) { logger.log('error', `Tests failed: ${(err as Error).message}`);