4.1.37

.gitea/workflows/default_tags.yaml

@@ -103,6 +103,6 @@ jobs:
       run: |
         npmci node install stable
         npmci npm install
-        pnpm install -g @gitzone/tsdoc
+        pnpm install -g @git.zone/tsdoc
         npmci command tsdoc
       continue-on-error: true

changelog.md

@@ -0,0 +1,93 @@
+# Changelog
+
+## 2024-11-17 - 4.1.37 - fix(docker)
+Enhanced base image extraction logic from Dockerfile
+
+- Improved dockerBaseImage to accurately extract base images considering ARG variables.
+- Added support for parsing Dockerfile content without external libraries.
+- Enhanced error handling for missing FROM instructions.
+
+## 2024-11-17 - 4.1.36 - fix(docker)
+Improve logging for Dockerfile build order with base image details.
+
+- Enhance logging in Dockerfile sorting process to include base image information.
+
+## 2024-11-17 - 4.1.35 - fix(docker)
+Fix Dockerfile dependency sorting and enhance environment variable handling for GitHub repos
+
+- Refined the algorithm for sorting Dockerfiles based on dependencies to ensure proper build order.
+- Enhanced environment variable handling in the NpmciEnv class to support conditional assignments.
+- Updated various dependencies in package.json for improved performance and compatibility.
+- Added error handling to circular dependency detection in Dockerfile sorting.
+
+## 2024-11-05 - 4.1.34 - fix(connector)
+Remove unused typedrequest implementation in cloudlyconnector
+
+- Removed commented out code that initialized typedrequest in CloudlyConnector.
+
+## 2024-11-05 - 4.1.33 - fix(core)
+Updated dependencies and improved npm preparation logic.
+
+- Updated @git.zone/tsbuild from ^2.1.84 to ^2.2.0.
+- Updated @git.zone/tsrun from ^1.2.49 to ^1.3.3.
+- Updated @types/node from ^22.7.9 to ^22.8.7.
+- Updated @serve.zone/api from ^1.2.1 to ^4.3.1.
+- Improved npm preparation logic to handle empty tokens gracefully.
+
+## 2024-10-23 - 4.1.32 - fix(dependencies)
+Update project dependencies to latest versions
+
+- Updated development dependencies, including @git.zone/tsbuild and @git.zone/tsrun.
+- Updated production dependencies such as @api.global/typedrequest and @push.rocks/smartfile.
+
+## 2022-10-24 - 4.0.11 - prerelease
+now includes a precheck for more generic runner execution
+
+- Implemented a precheck feature for runners.
+
+## 2022-10-09 to 2022-10-11 - 4.0.0 to 4.0.10 - migration
+internal migrations and fixes
+
+- Major switch to ESM style module: **BREAKING CHANGE**.
+- Multiple fixes in core functionalities and module updates.
+
+## 2019-11-26 - 3.1.73 - fixes
+correctly setting npm cache and other updates
+
+- Ensured correct npm cache setting during preparation.
+- Various core updates.
+
+## 2018-12-23 - 3.1.19 - privacy updates
+enhanced mirroring controls for private code
+
+- Now refusing to mirror private code.
+
+## 2018-11-24 - 3.1.2 - ci improvement
+removed unnecessary build dependency
+
+- Removed npmts build dependency in CI pipeline.
+
+## 2018-09-22 - 3.0.59 - enhancement
+integrated smartlog for improved logging
+
+- Logs now utilize smartlog for better management.
+
+## 2017-09-08 - 3.0.14 - analytics
+added analytics features
+
+- Enabled analytics throughout the system.
+
+## 2017-08-29 - 3.0.9 - docker enhancements
+docker improvements and build args implementation
+
+- Implemented working `dockerBuildArgEnvMap`.
+
+## 2017-07-27 - 2.4.0 - stability improvements
+various updates to stabilize the environment
+
+- Fixed npmci versioning issues.
+
+## 2016-11-25 - 2.3.24 - global tools
+improved handling for global tool installations
+
+- Improved install handling for needed global tools.

cli.ts.js

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
 process.env.CLI_CALL = 'true';
 
-import * as tsrun from '@gitzone/tsrun';
+import * as tsrun from '@git.zone/tsrun';
 tsrun.runPath('./cli.child.js', import.meta.url);

npmextra.json

@@ -14,9 +14,26 @@
       "githost": "gitlab.com",
       "gitscope": "ship.zone",
       "gitrepo": "npmci",
-      "description": "node and docker in gitlab ci on steroids",
+      "description": "A tool to streamline Node.js and Docker workflows within CI environments, particularly GitLab CI, providing various CI/CD utilities.",
       "npmPackagename": "@ship.zone/npmci",
-      "license": "MIT"
+      "license": "MIT",
+      "keywords": [
+        "Node.js",
+        "Docker",
+        "GitLab CI",
+        "GitHub CI",
+        "Gitea CI",
+        "CI/CD",
+        "automation",
+        "npm",
+        "TypeScript",
+        "cloud",
+        "SSH",
+        "registry",
+        "container management",
+        "continuous integration",
+        "continuous deployment"
+      ]
     }
   }
 }

package.json

@@ -1,8 +1,8 @@
 {
-  "name": "@shipzone/npmci",
-  "version": "4.1.24",
+  "name": "@ship.zone/npmci",
+  "version": "4.1.37",
   "private": false,
-  "description": "node and docker in gitlab ci on steroids",
+  "description": "A tool to streamline Node.js and Docker workflows within CI environments, particularly GitLab CI, providing various CI/CD utilities.",
   "main": "dist_ts/index.js",
   "typings": "dist_ts/index.d.ts",
   "type": "module",
@@ -26,36 +26,36 @@
   },
   "homepage": "https://gitlab.com/gitzone/npmci#README",
   "devDependencies": {
-    "@gitzone/tsbuild": "^2.1.66",
-    "@gitzone/tsrun": "^1.2.44",
-    "@gitzone/tstest": "^1.0.77",
-    "@push.rocks/tapbundle": "^5.0.12",
-    "@types/node": "^20.4.2"
+    "@git.zone/tsbuild": "^2.2.0",
+    "@git.zone/tsrun": "^1.3.3",
+    "@git.zone/tstest": "^1.0.77",
+    "@push.rocks/tapbundle": "^5.5.0",
+    "@types/node": "^22.9.0"
   },
   "dependencies": {
-    "@api.global/typedrequest": "^3.0.1",
-    "@push.rocks/lik": "^6.0.3",
-    "@push.rocks/npmextra": "^3.0.9",
-    "@push.rocks/projectinfo": "^5.0.1",
-    "@push.rocks/qenv": "^5.0.2",
+    "@api.global/typedrequest": "^3.1.10",
+    "@push.rocks/lik": "^6.1.0",
+    "@push.rocks/npmextra": "^5.1.2",
+    "@push.rocks/projectinfo": "^5.0.2",
+    "@push.rocks/qenv": "^6.0.2",
     "@push.rocks/smartanalytics": "^2.0.15",
-    "@push.rocks/smartcli": "^4.0.6",
+    "@push.rocks/smartcli": "^4.0.11",
     "@push.rocks/smartdelay": "^3.0.5",
     "@push.rocks/smartenv": "^5.0.5",
-    "@push.rocks/smartfile": "^10.0.28",
-    "@push.rocks/smartgit": "^3.0.0",
-    "@push.rocks/smartlog": "^3.0.3",
+    "@push.rocks/smartfile": "^11.0.21",
+    "@push.rocks/smartgit": "^3.1.1",
+    "@push.rocks/smartlog": "^3.0.7",
     "@push.rocks/smartlog-destination-local": "^9.0.0",
     "@push.rocks/smartobject": "^1.0.12",
     "@push.rocks/smartpath": "^5.0.11",
-    "@push.rocks/smartpromise": "^4.0.2",
-    "@push.rocks/smartrequest": "^2.0.18",
-    "@push.rocks/smartshell": "^3.0.3",
-    "@push.rocks/smartsocket": "^2.0.19",
-    "@push.rocks/smartssh": "^2.0.0",
-    "@push.rocks/smartstring": "^4.0.7",
-    "@servezone/interfaces": "^1.0.3",
-    "@tsclass/tsclass": "^4.0.42",
+    "@push.rocks/smartpromise": "^4.0.4",
+    "@push.rocks/smartrequest": "^2.0.23",
+    "@push.rocks/smartshell": "^3.0.6",
+    "@push.rocks/smartsocket": "^2.0.22",
+    "@push.rocks/smartssh": "^2.0.1",
+    "@push.rocks/smartstring": "^4.0.8",
+    "@serve.zone/api": "^4.3.11",
+    "@tsclass/tsclass": "^4.1.2",
     "@types/through2": "^2.0.38",
     "through2": "^4.0.2"
   },
@@ -73,5 +73,22 @@
   ],
   "browserslist": [
     "last 1 chrome versions"
+  ],
+  "keywords": [
+    "Node.js",
+    "Docker",
+    "GitLab CI",
+    "GitHub CI",
+    "Gitea CI",
+    "CI/CD",
+    "automation",
+    "npm",
+    "TypeScript",
+    "cloud",
+    "SSH",
+    "registry",
+    "container management",
+    "continuous integration",
+    "continuous deployment"
   ]
 }

readme.hints.md

@@ -0,0 +1,2 @@
+- focus on cli usage in CI environments.
+- show Gitlab CI, GitHub CI and Gitea CI examples.

readme.md

@@ -1,110 +1,494 @@
-# @shipzone/npmci
-node and docker in gitlab ci on steroids
+# @ship.zone/npmci
+A tool to enhance Node.js and Docker workflows within GitLab CI, providing various CI/CD utilities.
 
-## Availabililty and Links
-* [npmjs.org (npm package)](https://www.npmjs.com/package/@shipzone/npmci)
-* [gitlab.com (source)](https://gitlab.com/shipzone/npmci)
-* [github.com (source mirror)](https://github.com/shipzone/npmci)
-* [docs (typedoc)](https://shipzone.gitlab.io/npmci/)
+## Install
 
-## Status for master
+To install `@ship.zone/npmci`, you can use npm or yarn:
 
-Status Category | Status Badge
--- | --
-GitLab Pipelines | [![pipeline status](https://gitlab.com/shipzone/npmci/badges/master/pipeline.svg)](https://lossless.cloud)
-GitLab Pipline Test Coverage | [![coverage report](https://gitlab.com/shipzone/npmci/badges/master/coverage.svg)](https://lossless.cloud)
-npm | [![npm downloads per month](https://badgen.net/npm/dy/@shipzone/npmci)](https://lossless.cloud)
-Snyk | [![Known Vulnerabilities](https://badgen.net/snyk/shipzone/npmci)](https://lossless.cloud)
-TypeScript Support | [![TypeScript](https://badgen.net/badge/TypeScript/>=%203.x/blue?icon=typescript)](https://lossless.cloud)
-node Support | [![node](https://img.shields.io/badge/node->=%2010.x.x-blue.svg)](https://nodejs.org/dist/latest-v10.x/docs/api/)
-Code Style | [![Code Style](https://badgen.net/badge/style/prettier/purple)](https://lossless.cloud)
-PackagePhobia (total standalone install weight) | [![PackagePhobia](https://badgen.net/packagephobia/install/@shipzone/npmci)](https://lossless.cloud)
-PackagePhobia (package size on registry) | [![PackagePhobia](https://badgen.net/packagephobia/publish/@shipzone/npmci)](https://lossless.cloud)
-BundlePhobia (total size when bundled) | [![BundlePhobia](https://badgen.net/bundlephobia/minzip/@shipzone/npmci)](https://lossless.cloud)
+```sh
+# Using npm
+npm install @ship.zone/npmci
+
+# Using yarn
+yarn add @ship.zone/npmci
+```
 
 ## Usage
 
-Use TypeScript for best in class instellisense.
+`npmci` is designed to streamline CI/CD processes, particularly in Docker and Node.js environments. The following sections illustrate its usage in various scenarios, from handling Node versions to building Docker images and more.
 
-npmci is designed to work in docker CI environments. The following docker images come with npmci presinstalled:
+### 1. Integration with GitLab CI, GitHub CI, and Gitea CI
 
-Docker Hub:
+#### GitLab CI
 
-- [hosttoday/ht-docker-node:npmci](https://hub.docker.com/r/hosttoday/ht-docker-node/)  
-  has LTS node version and npmci preinstalled.
-- [hosttoday/ht-docker-dbase](https://hub.docker.com/r/hosttoday/ht-docker-dbase/)  
-  based on docker:git, can be used to build docker images in conjunction with docker:dind
+An example of integrating `npmci` into a GitLab CI configuration could look like this:
 
-npmci can be called from commandline and handle a lot of tasks durug ci:
+```yaml
+image: hosttoday/ht-docker-node:npmci
 
-```shell
-# Handle node versions
-npmci node install stable # will install latest stable node version and update PATH for node and npm
-npmci node install lts # will install latest LTS node version and update PATH for node and npm versions
-npmci node install legacy # will install latest legacy node version and update PATH for node and npm
-npmci node install x.x.x #  will install any specific node version.
+stages:
+  - prepare
+  - build
+  - test
+  - deploy
 
-# Handle npm and yarn tasks
-npmcu npm login # logs in npm using the auth key provided at env var "NPMCI_TOKEN_NPM"
-npmci npm install  # installs dependencies using npm or yarn dependending on availablity
-npmci npm test # tests the package
-npmci npm publish # builds a package and publishes it
+default:
+  before_script:
+    - npmci node install stable
+    - npmci npm install
 
-# handle docker tasks
-npmci docker prepare
-## npmci test docker will look at all Dockerfiles and look for according tags on GitLab container registry
+prepare:
+  stage: prepare
+  script:
+    - npmci prepare npm
+    - npmci prepare docker
 
+build:
+  stage: build
+  script:
+    - npmci docker build
 
-# prepare tools
-npmci prepare npm # will look for $NPMCI_TOKEN_NPM env var and create .npmrc, so npm is authenticated
-npmci prepare docker # will look for $NPMCI_LOGIN_DOCKER in form username|password and authenticate docker
-npmci prepare docker-gitlab # will authenticate docker for gitlab container registry
+test:
+  stage: test
+  script:
+    - npmci npm test
 
-# build containers
-npmci docker build # will build containers
-## all Dockerfiles named Dockerfile* are picked up.
-## specify tags like this Dockerfile_[tag]
-## uploads all built images as [username]/[reponame]:[tag]_test to GitLab
-## then test in next step with "npmci test docker"
+deploy:
+  stage: deploy
+  script:
+    - npmci publish npm
+    - npmci docker push
 
-# publish npm module
-npmci publish npm # will look vor $NPMCI_TOKEN_NPM env var and push any module in cwd to npm
-npmci publish docker
-
-# trigger webhooks
-npmci trigger # will look for NPMCI_TRIGGER_1 to NPMCI_TRIGGER_100 in form domain|id|token|ref|name
+  environment:
+    name: production
+    url: http://example.com
 ```
 
-## Configuration
+#### GitHub Actions
 
-npmci supports the use of npmextra.
+Similarly, you can set up `npmci` in GitHub Actions:
 
-To configure npmci create a `npmextra.json` file at the root of your project
+```yaml
+name: CI Pipeline
 
-```json
-{
-  "npmci": {
-    "globalNpmTools": ["npm-check-updates", "protractor", "npmts", "gitzone"]
-  }
-}
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  prepare:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Node.js
+        uses: actions/setup-node@v2
+        with:
+          node-version: '14'
+      - run: npm install -g @ship.zone/npmci
+      - run: npmci node install stable
+      - run: npmci npm install
+
+  build:
+    runs-on: ubuntu-latest
+    needs: prepare
+    steps:
+      - run: npmci docker build
+
+  test:
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - run: npmci npm test
+
+  deploy:
+    runs-on: ubuntu-latest
+    needs: test
+    steps:
+      - run: npmci publish npm
+      - run: npmci docker push
 ```
 
-**Available options**
+#### Gitea CI
 
-| setting        | example                       | description                                                                                       |
-| -------------- | ----------------------------- | ------------------------------------------------------------------------------------------------- |
-| globalNpmTools | "globalNpmTools": ["gitbook"] | Will look for the specified package names locally and (if not yet present) install them from npm. |
+Lastly, for Gitea CI:
 
-For further information read the linked docs at the top of this README.
+```yaml
+image: hosttoday/ht-docker-node:npmci
 
-Use TypeScript for best in class instellisense.
+pipelines:
+  default:
+    - step:
+        name: Prepare
+        image: hosttoday/ht-docker-node:npmci
+        commands:
+          - npmci node install stable
+          - npmci npm install
+          - npmci prepare npm
+          - npmci prepare docker
 
-## Contribution
+    - step:
+        name: Build
+        image: hosttoday/ht-docker-node:npmci
+        commands:
+          - npmci docker build
 
-We are always happy for code contributions. If you are not the code contributing type that is ok. Still, maintaining Open Source repositories takes considerable time and thought. If you like the quality of what we do and our modules are useful to you we would appreciate a little monthly contribution: You can [contribute one time](https://lossless.link/contribute-onetime) or [contribute monthly](https://lossless.link/contribute). :)
+    - step:
+        name: Test
+        image: hosttoday/ht-docker-node:npmci
+        commands:
+          - npmci npm test
 
-For further information read the linked docs at the top of this readme.
+    - step:
+        name: Deploy
+        image: hosttoday/ht-docker-node:npmci
+        commands:
+          - npmci publish npm
+          - npmci docker push
+```
 
-## Legal
-> MIT licensed | **©** [Task Venture Capital GmbH](https://task.vc)
-| By using this npm module you agree to our [privacy policy](https://lossless.gmbH/privacy)
+### 2. Handle Node Versions
+
+One of the core features of `npmci` is managing Node versions in your CI environment. You can specify which version of Node to install:
+
+```typescript
+import { Npmci } from '@ship.zone/npmci';
+
+async function manageNodeVersions() {
+  const npmciInstance = new Npmci();
+  await npmciInstance.start();
+
+  await npmciInstance.nodejsManager.handleCli({
+    _: ['node', 'install', 'stable'] // Installs the latest stable version
+  });
+
+  await npmciInstance.nodejsManager.handleCli({
+    _: ['node', 'install', 'lts'] // Installs the Long-Term Support (LTS) version
+  });
+
+  await npmciInstance.nodejsManager.handleCli({
+    _: ['node', 'install', 'legacy'] // Installs a legacy version
+  });
+
+  await npmciInstance.nodejsManager.handleCli({
+    _: ['node', 'install', '14.17.0'] // Install a specific version of Node
+  });
+}
+
+manageNodeVersions().then(() => console.log('Node versions managed successfully.'));
+```
+
+### 3. Handling npm and Yarn Tasks
+
+`npmci` provides numerous utilities to streamline npm and yarn workflow tasks within a CI/CD pipeline.
+
+```typescript
+import { Npmci } from '@ship.zone/npmci';
+
+async function manageNpmTasks() {
+  const npmciInstance = new Npmci();
+  await npmciInstance.start();
+
+  await npmciInstance.npmManager.handleCli({ _: ['npm', 'install'] }); // Installs dependencies
+  await npmciInstance.npmManager.handleCli({ _: ['npm', 'test'] }); // Runs tests
+  await npmciInstance.npmManager.handleCli({ _: ['npm', 'publish'] }); // Publishes the package
+}
+
+manageNpmTasks().then(() => console.log('Npm tasks handled successfully.'));
+```
+
+### 4. Docker Task Handling
+
+`npmci` simplifies Docker operations, particularly in building, testing, and publishing Docker images.
+
+**Prepare Docker Environment:**
+
+```typescript
+import { Npmci } from '@ship.zone/npmci';
+
+async function prepareDocker() {
+  const npmciInstance = new Npmci();
+  await npmciInstance.start();
+
+  await npmciInstance.dockerManager.handleCli({ _: ['docker', 'prepare'] }); // Prepares Docker environment
+}
+
+prepareDocker().then(() => console.log('Docker environment prepared successfully.'));
+```
+
+**Building Docker Images:**
+
+```typescript
+import { Npmci } from '@ship.zone/npmci';
+
+async function buildDockerImages() {
+  const npmciInstance = new Npmci();
+  await npmciInstance.start();
+
+  await npmciInstance.dockerManager.handleCli({ _: ['docker', 'build'] }); // Builds Docker images
+}
+
+buildDockerImages().then(() => console.log('Docker images built successfully.'));
+```
+
+**Testing Docker Images:**
+
+```typescript
+import { Npmci } from '@ship.zone/npmci';
+
+async function testDockerImages() {
+  const npmciInstance = new Npmci();
+  await npmciInstance.start();
+
+  await npmciInstance.dockerManager.handleCli({ _: ['docker', 'test'] }); // Tests Docker images
+}
+
+testDockerImages().then(() => console.log('Docker images tested successfully.'));
+```
+
+**Publishing Docker Images:**
+
+```typescript
+import { Npmci } from '@ship.zone/npmci';
+
+async function pushDockerImages() {
+  const npmciInstance = new Npmci();
+  await npmciInstance.start();
+
+  await npmciInstance.dockerManager.handleCli({ _: ['docker', 'push'] }); // Pushes Docker images to registry
+}
+
+pushDockerImages().then(() => console.log('Docker images pushed successfully.'));
+```
+
+### 5. Managing Docker Registries
+
+`npmci` can handle multiple Docker registries and allows for easy integration within your CI pipeline.
+
+**Logging in to Docker Registries:**
+
+```typescript
+import { Npmci } from '@ship.zone/npmci';
+
+async function loginToDockerRegistries() {
+  const npmciInstance = new Npmci();
+  await npmciInstance.start();
+
+  await npmciInstance.dockerManager.handleCli({ _: ['docker', 'login'] }); // Logs into all configured Docker registries
+}
+
+loginToDockerRegistries().then(() => console.log('Logged into Docker registries.'));
+```
+
+**Pulling Docker Images:**
+
+```typescript
+import { Npmci } from '@ship.zone/npmci';
+
+async function pullDockerImages() {
+  const npmciInstance = new Npmci();
+  await npmciInstance.start();
+
+  await npmciInstance.dockerManager.handleCli({ _: ['docker', 'pull', 'registry.gitlab.com/mygroup/myrepo'] }); // Pulls Docker images from a registry
+}
+
+pullDockerImages().then(() => console.log('Docker images pulled successfully.'));
+```
+
+### 6. SSH Key Management
+
+`npmci` also simplifies the management of SSH keys, which is crucial for accessing private repositories and servers.
+
+**Preparing SSH Keys:**
+
+```typescript
+import { Npmci } from '@ship.zone/npmci';
+
+async function prepareSshKeys() {
+  const npmciInstance = new Npmci();
+  await npmciInstance.start();
+
+  await npmciInstance.sshManager.handleCli({ _: ['ssh', 'prepare'] }); // Prepares SSH keys from environment variables
+}
+
+prepareSshKeys().then(() => console.log('SSH keys prepared successfully.'));
+```
+
+### 7. Cloudron Integration
+
+For users deploying applications on Cloudron, `npmci` provides a set of utilities for automating Cloudron tasks.
+
+**Deploying to Cloudron:**
+
+```typescript
+import { Npmci } from '@ship.zone/npmci';
+
+async function deployToCloudron() {
+  const npmciInstance = new Npmci();
+  await npmciInstance.start();
+
+  await npmciInstance.cloudronManager.handleCli({
+    _: ['cloudron', 'deploy']
+  }); // Deploys application to Cloudron platform
+}
+
+deployToCloudron().then(() => console.log('Deployment to Cloudron completed.'));
+```
+
+**Preparing Cloudron Manifest:**
+Before deployment, replace version placeholders in the Cloudron Manifest:
+
+```typescript
+import { Npmci } from '@ship.zone/npmci';
+import * as fs from 'fs';
+import * as path from 'path';
+
+async function prepareCloudronManifest(version: string) {
+  const manifestPath = path.join(process.cwd(), "CloudronManifest.json");
+  let manifestFile = fs.readFileSync(manifestPath, { encoding: 'utf-8' });
+  manifestFile = manifestFile.replace(/##version##/g, version);
+  fs.writeFileSync(manifestPath, manifestFile);
+  console.log('CloudronManifest prepared');
+}
+
+async function deployWithPreparedManifest() {
+  const npmciInstance = new Npmci();
+  await npmciInstance.start();
+  
+  await prepareCloudronManifest('1.0.0');
+  await npmciInstance.cloudronManager.handleCli({
+    _: ['cloudron', 'deploy']
+  }); // Deploys application to Cloudron platform
+}
+
+deployWithPreparedManifest().then(() => console.log('Deployment to Cloudron with manifest preparation completed.'));
+```
+
+### 8. Webhook Triggers
+
+`npmci` supports webhook triggers, allowing you to trigger builds and other activities based on various conditions.
+
+**Triggering Webhooks:**
+
+```typescript
+import { Npmci } from '@ship.zone/npmci';
+
+async function triggerWebhooks() {
+  const npmciInstance = new Npmci();
+  await npmciInstance.start();
+
+  await npmciInstance.triggerManager.handleCli({
+    _: ['trigger']
+  }); // Triggers webhooks based on environment variables
+}
+
+triggerWebhooks().then(() => console.log('Webhooks triggered successfully.'));
+```
+
+### 9. Using the bash Helper
+
+`npmci` includes a bash helper for executing commands within a bash shell, useful for various custom tasks.
+
+**Using bash to Execute Commands:**
+
+```typescript
+import { bash } from '@ship.zone/npmci';
+
+async function runCustomBashCommand(command: string) {
+  const output = await bash(command);
+  console.log('Command output:', output);
+}
+
+runCustomBashCommand('echo Hello World').then(() => console.log('Custom command executed successfully.'));
+```
+
+### Full Features and Use Cases
+
+Below is a comprehensive set of features and use cases supported by `npmci`. This section ensures you can take full advantage of the library's capabilities in multiple scenarios.
+
+### Comprehensive Docker Workflow
+
+**Step-by-step Docker Image Handling:**
+
+1. **Detect and Build All Dockerfiles:**
+
+   ```typescript
+   import { Npmci } from '@ship.zone/npmci';
+
+   async function detectAndBuildDockerfiles() {
+     const npmciInstance = new Npmci();
+     await npmciInstance.start();
+     const dockerfiles = await npmciInstance.dockerManager.getDockerfiles();
+     console.log('Dockerfiles detected:', dockerfiles.map(d => d.filePath));
+     
+     await npmciInstance.dockerManager.handleCli({ _: ['docker', 'build'] });
+     console.log('Dockerfiles built successfully.');
+   }
+
+   detectAndBuildDockerfiles().then(() => console.log('Docker detection and build process completed.'));
+   ```
+
+2. **Test All Dockerfiles:**
+
+   ```typescript
+   import { Npmci } from '@ship.zone/npmci';
+
+   async function testAllDockerfiles() {
+     const npmciInstance = new Npmci();
+     await npmciInstance.start();
+     await npmciInstance.dockerManager.handleCli({ _: ['docker', 'test'] });
+     console.log('Dockerfiles tested successfully.');
+   }
+
+   testAllDockerfiles().then(() => console.log('Docker testing process completed.'));
+   ```
+
+3. **Push Dockerfiles to a Registry:**
+
+   ```typescript
+   import { Npmci } from '@ship.zone/npmci';
+
+   async function pushDockerfilesToRegistry() {
+     const npmciInstance = new Npmci();
+     await npmciInstance.start();
+     
+     await npmciInstance.dockerManager.handleCli({ _: ['docker', 'push'] });
+     console.log('Dockerfiles pushed to registry successfully.');
+   }
+
+   pushDockerfilesToRegistry().then(() => console.log('Docker push process completed.'));
+   ```
+
+**Dockerfile Class Example:**
+
+Here's a snippet showcasing how the `Dockerfile` class can be used to handle Dockerfile-specific operations:
+
+```typescript
+import { Dockerfile } from '@ship.zone/npmci';
+
+async function handleDockerfileOperations() {
+  // Initialize Dockerfile instances
+  const dockerfile1 = new Dockerfile(/* required parameters */);
+  const dockerfile2 = new Dockerfile(/* required parameters */);
+
+  // Read and sort Dockerfiles
+  const dockerfiles = await Dockerfile.readDockerfiles(/* required parameters */);
+  const sortedDockerfiles = await Dockerfile.sortDockerfiles(dockerfiles);
+
+  // Build and Test Dockerfiles
+  await Dockerfile.buildDockerfiles(sortedDockerfiles);
+  await Dockerfile.testDockerfiles(sortedDockerfiles);
+
+  // Push Dockerfile images to a registry
+  for (const dockerfile of sortedDockerfiles) {
+    await dockerfile.push(/* registry and tag parameters */);
+  }
+
+  console.log('Dockerfile operations completed successfully.');
+}
+
+handleDockerfileOperations().then(() => console.log('Dockerfile processing flow completed.'));
+```
+
+This completes the comprehensive guide to `@ship.zone/npmci`. With the examples and explanations provided, you should be able to harness the full power and flexibility of the library to streamline your CI/CD processes effectively.
+undefined

test/assets/Dockerfile_hello_##version##

@@ -0,0 +1,2 @@
+FROM mygroup/myrepo:sometag2
+RUN apt-get update

test/assets/npmextra.json

@@ -1 +1,9 @@
-{}
+{
+  "gitzone": {
+    "module": {
+      "githost": "code.foss.global",
+      "gitscope": "mygroup",
+      "gitrepo": "myrepo"
+    }
+  }
+}

test/test.ts

@@ -19,24 +19,20 @@ process.cwd = () => {
   return path.join(smartpath.get.dirnameFromImportMetaUrl(import.meta.url), 'assets/');
 };
 
-let npmci: typeof import('../ts/index.js');
-
-type TNpmciTypes = typeof import('../ts/index.js');
-
-tap.preTask('should import npmci', async () => {
-  npmci = await import('../ts/index.js');
-});
+import type * as npmciTypes from '../ts/index.js';
+const npmci = await import('../ts/index.js');
 
 // ======
 // Docker
 // ======
 
-let dockerfile1: npmci.Dockerfile;
-let dockerfile2: npmci.Dockerfile;
-let sortableArray: npmci.Dockerfile[];
+let dockerfile1: npmciTypes.Dockerfile;
+let dockerfile2: npmciTypes.Dockerfile;
+let sortableArray: npmciTypes.Dockerfile[];
 
 tap.test('should return valid Dockerfiles', async () => {
   const npmciInstance = new npmci.Npmci();
+  await npmciInstance.start();
   dockerfile1 = new npmci.Dockerfile(npmciInstance.dockerManager, {
     filePath: './Dockerfile',
     read: true,
@@ -51,8 +47,9 @@ tap.test('should return valid Dockerfiles', async () => {
 
 tap.test('should read a directory of Dockerfiles', async () => {
   const npmciInstance = new npmci.Npmci();
+  await npmciInstance.start();
   return npmci.Dockerfile.readDockerfiles(npmciInstance.dockerManager).then(
-    async (readDockerfilesArrayArg: npmci.Dockerfile[]) => {
+    async (readDockerfilesArrayArg: npmciTypes.Dockerfile[]) => {
       sortableArray = readDockerfilesArrayArg;
       return expect(readDockerfilesArrayArg[1].version).toEqual('sometag1');
     }
@@ -61,7 +58,7 @@ tap.test('should read a directory of Dockerfiles', async () => {
 
 tap.test('should sort an array of Dockerfiles', async () => {
   return npmci.Dockerfile.sortDockerfiles(sortableArray).then(
-    async (sortedArrayArg: npmci.Dockerfile[]) => {
+    async (sortedArrayArg: npmciTypes.Dockerfile[]) => {
       console.log(sortedArrayArg);
     }
   );
@@ -69,6 +66,7 @@ tap.test('should sort an array of Dockerfiles', async () => {
 
 tap.test('should build all Dockerfiles', async () => {
   const npmciInstance = new npmci.Npmci();
+  await npmciInstance.start();
   return npmciInstance.dockerManager.handleCli({
     _: ['docker', 'build'],
   });
@@ -76,6 +74,7 @@ tap.test('should build all Dockerfiles', async () => {
 
 tap.test('should test all Dockerfiles', async () => {
   const npmciInstance = new npmci.Npmci();
+  await npmciInstance.start();
   return npmciInstance.dockerManager.handleCli({
     _: ['docker', 'test'],
   });
@@ -83,6 +82,7 @@ tap.test('should test all Dockerfiles', async () => {
 
 tap.test('should test dockerfiles', async () => {
   const npmciInstance = new npmci.Npmci();
+  await npmciInstance.start();
   return npmciInstance.dockerManager.handleCli({
     _: ['docker', 'test'],
   });
@@ -90,6 +90,7 @@ tap.test('should test dockerfiles', async () => {
 
 tap.test('should login docker daemon', async () => {
   const npmciInstance = new npmci.Npmci();
+  await npmciInstance.start();
   return npmciInstance.dockerManager.handleCli({
     _: ['docker', 'login'],
   });
@@ -110,6 +111,7 @@ tap.test('should prepare SSH keys', async () => {
 // ====
 tap.test('should install a certain version of node', async () => {
   const npmciInstance = new npmci.Npmci();
+  await npmciInstance.start();
   await npmciInstance.nodejsManager.handleCli({
     _: ['node', 'install', 'stable'],
   });

ts/00_commitinfo_data.ts

@@ -1,8 +1,8 @@
 /**
- * autocreated commitinfo by @pushrocks/commitinfo
+ * autocreated commitinfo by @push.rocks/commitinfo
  */
 export const commitinfo = {
-  name: '@shipzone/npmci',
-  version: '4.1.24',
-  description: 'node and docker in gitlab ci on steroids'
+  name: '@ship.zone/npmci',
+  version: '4.1.37',
+  description: 'A tool to streamline Node.js and Docker workflows within CI environments, particularly GitLab CI, providing various CI/CD utilities.'
 }

ts/connector.cloudly/cloudlyconnector.ts

@@ -26,14 +26,6 @@ export class CloudlyConnector {
       return;
     }
 
-    const typedrequest =
-      new plugins.typedrequest.TypedRequest<plugins.servezoneInterfaces.requests.IRequest_InformAboutNewContainerImage>(
-        `https://${cloudlyUrl}/typedrequest`,
-        'servezonestandard_InformAboutNewContainerVersion'
-      );
-
-    const response = await typedrequest.fire({
-      containerImageInfo: optionsArg,
-    });
+    // lets push to cloudly here
   }
 }

ts/index.ts

@@ -6,5 +6,5 @@ export const npmciInstance = new Npmci();
 export { Dockerfile, Npmci };
 
 export const runCli = async () => {
-  npmciInstance.start();
+  await npmciInstance.start();
 };

ts/manager.docker/mod.classes.dockerfile.ts

@@ -40,43 +40,76 @@ export class Dockerfile {
   }
 
   /**
-   * sorts Dockerfiles into a dependency chain
-   * @param sortableArrayArg an array of instances of class Dockerfile
-   * @returns Promise<Dockerfile[]>
+   * Sorts Dockerfiles into a build order based on dependencies.
+   * @param dockerfiles An array of Dockerfile instances.
+   * @returns A Promise that resolves to a sorted array of Dockerfiles.
    */
-  public static async sortDockerfiles(sortableArrayArg: Dockerfile[]): Promise<Dockerfile[]> {
-    const done = plugins.smartpromise.defer<Dockerfile[]>();
-    logger.log('info', 'sorting Dockerfiles:');
-    const sortedArray: Dockerfile[] = [];
-    const cleanTagsOriginal = Dockerfile.cleanTagsArrayFunction(sortableArrayArg, sortedArray);
-    let sorterFunctionCounter: number = 0;
-    const sorterFunction = () => {
-      sortableArrayArg.forEach((dockerfileArg) => {
-        const cleanTags = Dockerfile.cleanTagsArrayFunction(sortableArrayArg, sortedArray);
-        if (
-          cleanTags.indexOf(dockerfileArg.baseImage) === -1 &&
-          sortedArray.indexOf(dockerfileArg) === -1
-        ) {
-          sortedArray.push(dockerfileArg);
-        }
-        if (cleanTagsOriginal.indexOf(dockerfileArg.baseImage) !== -1) {
-          dockerfileArg.localBaseImageDependent = true;
-        }
+  public static async sortDockerfiles(dockerfiles: Dockerfile[]): Promise<Dockerfile[]> {
+    logger.log('info', 'Sorting Dockerfiles based on dependencies...');
+
+    // Map from cleanTag to Dockerfile instance for quick lookup
+    const tagToDockerfile = new Map<string, Dockerfile>();
+    dockerfiles.forEach((dockerfile) => {
+      tagToDockerfile.set(dockerfile.cleanTag, dockerfile);
     });
-      if (sortableArrayArg.length === sortedArray.length) {
-        let counter = 1;
-        for (const dockerfile of sortedArray) {
-          logger.log('info', `tag ${counter}: -> ${dockerfile.cleanTag}`);
-          counter++;
+
+    // Build the dependency graph
+    const graph = new Map<Dockerfile, Dockerfile[]>();
+    dockerfiles.forEach((dockerfile) => {
+      const dependencies: Dockerfile[] = [];
+      const baseImage = dockerfile.baseImage;
+
+      // Check if the baseImage is among the local Dockerfiles
+      if (tagToDockerfile.has(baseImage)) {
+        const baseDockerfile = tagToDockerfile.get(baseImage);
+        dependencies.push(baseDockerfile);
+        dockerfile.localBaseImageDependent = true;
+        dockerfile.localBaseDockerfile = baseDockerfile;
       }
-        done.resolve(sortedArray);
-      } else if (sorterFunctionCounter < 10) {
-        sorterFunctionCounter++;
-        sorterFunction();
+
+      graph.set(dockerfile, dependencies);
+    });
+
+    // Perform topological sort
+    const sortedDockerfiles: Dockerfile[] = [];
+    const visited = new Set<Dockerfile>();
+    const tempMarked = new Set<Dockerfile>();
+
+    const visit = (dockerfile: Dockerfile) => {
+      if (tempMarked.has(dockerfile)) {
+        throw new Error(`Circular dependency detected involving ${dockerfile.cleanTag}`);
+      }
+      if (!visited.has(dockerfile)) {
+        tempMarked.add(dockerfile);
+        const dependencies = graph.get(dockerfile) || [];
+        dependencies.forEach((dep) => visit(dep));
+        tempMarked.delete(dockerfile);
+        visited.add(dockerfile);
+        sortedDockerfiles.push(dockerfile);
       }
     };
-    sorterFunction();
-    return done.promise;
+
+    try {
+      dockerfiles.forEach((dockerfile) => {
+        if (!visited.has(dockerfile)) {
+          visit(dockerfile);
+        }
+      });
+    } catch (error) {
+      logger.log('error', error.message);
+      throw error;
+    }
+
+    // Log the sorted order
+    sortedDockerfiles.forEach((dockerfile, index) => {
+      logger.log(
+        'info',
+        `Build order ${index + 1}: ${dockerfile.cleanTag}
+      with base image ${dockerfile.baseImage}`
+      );
+    });
+
+    return sortedDockerfiles;
   }
 
   /**
@@ -120,9 +153,12 @@ export class Dockerfile {
    * returns a version for a docker file
    * @execution SYNC
    */
-  public static dockerFileVersion(dockerfileInstanceArg: Dockerfile, dockerfileNameArg: string): string {
+  public static dockerFileVersion(
+    dockerfileInstanceArg: Dockerfile,
+    dockerfileNameArg: string
+  ): string {
     let versionString: string;
-    const versionRegex = /Dockerfile_([^:_]*)$/;
+    const versionRegex = /Dockerfile_(.+)$/;
     const regexResultArray = versionRegex.exec(dockerfileNameArg);
     if (regexResultArray && regexResultArray.length === 2) {
       versionString = regexResultArray[1];
@@ -131,18 +167,69 @@ export class Dockerfile {
     }
     versionString = versionString.replace(
       '##version##',
-      dockerfileInstanceArg.npmciDockerManagerRef.npmciRef.npmciConfig.getConfig().projectInfo.npm.version
+      dockerfileInstanceArg.npmciDockerManagerRef.npmciRef.npmciConfig.getConfig().projectInfo.npm
+        .version
     );
     return versionString;
   }
 
   /**
-   * returns the docker base image for a Dockerfile
+   * Extracts the base image from a Dockerfile content without using external libraries.
+   * @param dockerfileContentArg The content of the Dockerfile as a string.
+   * @returns The base image specified in the first FROM instruction.
    */
   public static dockerBaseImage(dockerfileContentArg: string): string {
-    const baseImageRegex = /FROM\s([a-zA-z0-9\/\-\:]*)\n?/;
-    const regexResultArray = baseImageRegex.exec(dockerfileContentArg);
-    return regexResultArray[1];
+    const lines = dockerfileContentArg.split(/\r?\n/);
+    const args: { [key: string]: string } = {};
+
+    for (const line of lines) {
+      const trimmedLine = line.trim();
+
+      // Skip empty lines and comments
+      if (trimmedLine === '' || trimmedLine.startsWith('#')) {
+        continue;
+      }
+
+      // Match ARG instructions
+      const argMatch = trimmedLine.match(/^ARG\s+([^\s=]+)(?:=(.*))?$/i);
+      if (argMatch) {
+        const argName = argMatch[1];
+        const argValue = argMatch[2] !== undefined ? argMatch[2] : process.env[argName] || '';
+        args[argName] = argValue;
+        continue;
+      }
+
+      // Match FROM instructions
+      const fromMatch = trimmedLine.match(/^FROM\s+(.+?)(?:\s+AS\s+[^\s]+)?$/i);
+      if (fromMatch) {
+        let baseImage = fromMatch[1].trim();
+
+        // Substitute variables in the base image name
+        baseImage = Dockerfile.substituteVariables(baseImage, args);
+
+        return baseImage;
+      }
+    }
+
+    throw new Error('No FROM instruction found in Dockerfile');
+  }
+
+  /**
+   * Substitutes variables in a string, supporting default values like ${VAR:-default}.
+   * @param str The string containing variables.
+   * @param vars The object containing variable values.
+   * @returns The string with variables substituted.
+   */
+  private static substituteVariables(str: string, vars: { [key: string]: string }): string {
+    return str.replace(/\${([^}:]+)(:-([^}]+))?}/g, (_, varName, __, defaultValue) => {
+      if (vars[varName] !== undefined) {
+        return vars[varName];
+      } else if (defaultValue !== undefined) {
+        return defaultValue;
+      } else {
+        return '';
+      }
+    });
   }
 
   /**
@@ -196,22 +283,6 @@ export class Dockerfile {
     return buildArgsString;
   }
 
-  /**
-   *
-   */
-  public static cleanTagsArrayFunction(
-    dockerfileArrayArg: Dockerfile[],
-    trackingArrayArg: Dockerfile[]
-  ): string[] {
-    const cleanTagsArray: string[] = [];
-    dockerfileArrayArg.forEach((dockerfileArg) => {
-      if (trackingArrayArg.indexOf(dockerfileArg) === -1) {
-        cleanTagsArray.push(dockerfileArg.cleanTag);
-      }
-    });
-    return cleanTagsArray;
-  }
-
   // INSTANCE
   public npmciDockerManagerRef: NpmciDockerManager;
 
@@ -285,7 +356,10 @@ export class Dockerfile {
       labels: [],
       version: this.npmciDockerManagerRef.npmciRef.npmciConfig.getConfig().projectInfo.npm.version,
     });
-    await this.npmciDockerManagerRef.npmciRef.npmciConfig.kvStorage.writeKey('latestPushedDockerTag', this.pushTag)
+    await this.npmciDockerManagerRef.npmciRef.npmciConfig.kvStorage.writeKey(
+      'latestPushedDockerTag',
+      this.pushTag
+    );
   }
 
   /**

ts/manager.npm/index.ts

@@ -52,12 +52,17 @@ export class NpmciNpmManager {
    * authenticates npm with token from env var
    */
   public async prepare() {
+    logger.log('info', 'running >>npm prepare<<');
     const config = this.npmciRef.npmciConfig.getConfig();
     let npmrcFileString: string = '';
     await plugins.smartobject.forEachMinimatch(
       process.env,
       'NPMCI_TOKEN_NPM*',
       (npmEnvArg: string) => {
+        if (!npmEnvArg) {
+          logger.log('note','found empty token...');
+          return;
+        }
         const npmRegistryUrl = npmEnvArg.split('|')[0];
         logger.log('ok', `found token for ${npmRegistryUrl}`);
         let npmToken = npmEnvArg.split('|')[1];

ts/npmci.classes.npmci.ts

@@ -39,11 +39,16 @@ export class Npmci {
       projectId: 'gitzone',
       appName: 'npmci',
     });
+  }
+
+  public async start() {
     this.cloudlyConnector = new CloudlyConnector(this);
     this.npmciEnv = new NpmciEnv(this);
     this.npmciInfo = new NpmciInfo(this);
+    await this.npmciInfo.printToConsole();
     this.npmciCli = new NpmciCli(this);
     this.npmciConfig = new NpmciConfig(this);
+    await this.npmciConfig.init();
 
     // managers
     this.cloudronManager = new NpmciCloudronManager(this);
@@ -51,11 +56,6 @@ export class Npmci {
     this.gitManager = new NpmciGitManager(this);
     this.nodejsManager = new NpmciNodeJsManager(this);
     this.npmManager = new NpmciNpmManager(this);
-  }
-
-  public async start() {
-    await this.npmciInfo.printToConsole();
-    await this.npmciConfig.init();
     this.npmciCli.startParse();
   }
 }

ts/npmci.classes.npmciconfig.ts

@@ -44,12 +44,14 @@ export class NpmciConfig {
 
   constructor(npmciRefArg: Npmci) {
     this.npmciRef = npmciRefArg;
+  }
 
+  public async init() {
     this.npmciNpmextra = new plugins.npmextra.Npmextra(paths.cwd);
-    this.kvStorage = new plugins.npmextra.KeyValueStore(
-      'custom',
-      `${this.npmciRef.npmciEnv.repo.user}_${this.npmciRef.npmciEnv.repo.repo}`
-    );
+    this.kvStorage = new plugins.npmextra.KeyValueStore({
+      typeArg: 'userHomeDir',
+      identityArg: `.npmci_${this.npmciRef.npmciEnv.repo.user}_${this.npmciRef.npmciEnv.repo.repo}`,
+    });
     this.npmciQenv = new plugins.qenv.Qenv(
       paths.NpmciProjectDir,
       paths.NpmciProjectNogitDir,
@@ -65,11 +67,8 @@ export class NpmciConfig {
       npmRegistryUrl: 'registry.npmjs.org',
       gitlabRunnerTags: [],
       dockerBuildargEnvMap: {},
-      urlCloudly: this.npmciQenv.getEnvVarOnDemand('NPMCI_URL_CLOUDLY'),
+      urlCloudly: await this.npmciQenv.getEnvVarOnDemand('NPMCI_URL_CLOUDLY'),
     };
-  }
-
-  public async init() {
     this.configObject = this.npmciNpmextra.dataFor<INpmciOptions>('npmci', this.configObject);
   }
 

ts/npmci.classes.npmcienv.ts

@@ -9,10 +9,10 @@ export class NpmciEnv {
 
   constructor(npmciRefArg: Npmci) {
     this.npmciRef = npmciRefArg;
-    if (process.env.GITLAB_CI) {
+    if (!this.repoString && process.env.GITLAB_CI) {
       this.repoString = process.env.CI_REPOSITORY_URL;
     }
-    if (process.env.NPMCI_COMPUTED_REPOURL) {
+    if (!this.repoString && process.env.NPMCI_COMPUTED_REPOURL) {
       this.repoString = process.env.NPMCI_COMPUTED_REPOURL;
     }
     if (!this.repoString) {

ts/npmci.classes.npmciinfo.ts

@@ -11,7 +11,7 @@ export class NpmciInfo {
     this.npmciRef = npmciArg;
   }
 
-  public printToConsole() {
-    logger.log('info', `npmci version: ${this.projectInfo.version}`);
+  public async printToConsole() {
+    await logger.log('info', `npmci version: ${this.projectInfo.version}`);
   }
 }

ts/npmci.plugins.ts

@@ -4,14 +4,14 @@ import * as path from 'path';
 export { path };
 
 // @apiglobal
-import * as typedrequest from '@apiglobal/typedrequest';
+import * as typedrequest from '@api.global/typedrequest';
 
 export { typedrequest };
 
 // @servezone
-import * as servezoneInterfaces from '@servezone/interfaces';
+import * as servezoneApi from '@serve.zone/api';
 
-export { servezoneInterfaces };
+export { servezoneApi };
 
 // @push.rocks
 import * as lik from '@push.rocks/lik';

tsconfig.json

@@ -3,8 +3,12 @@
     "experimentalDecorators": true,
     "useDefineForClassFields": false,
     "target": "ES2022",
-    "module": "ES2022",
-    "moduleResolution": "nodenext",
-    "esModuleInterop": true
-  }
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "esModuleInterop": true,
+    "verbatimModuleSyntax": true
+  },
+  "exclude": [
+    "dist_*/**/*.d.ts"
+  ]
 }