Compare commits

...

10 Commits

Author SHA1 Message Date
00f324e151 4.1.35
Some checks failed
Default (tags) / security (push) Failing after 16s
Default (tags) / test (push) Successful in 1m32s
Default (tags) / metadata (push) Failing after 10s
Default (tags) / release (push) Failing after 21s
2024-11-17 00:32:56 +01:00
e38cc40f11 fix(docker): Fix Dockerfile dependency sorting and enhance environment variable handling for GitHub repos 2024-11-17 00:32:56 +01:00
e9e8acafe4 4.1.34 2024-11-05 02:39:53 +01:00
c763db40bb fix(connector): Remove unused typedrequest implementation in cloudlyconnector 2024-11-05 02:39:53 +01:00
01256480c4 4.1.33 2024-11-05 02:38:21 +01:00
c6918399bf fix(core): Updated dependencies and improved npm preparation logic. 2024-11-05 02:38:21 +01:00
66d28e5081 4.1.32 2024-10-23 21:50:30 +02:00
9de77139ea fix(dependencies): Update project dependencies to latest versions 2024-10-23 21:50:30 +02:00
7b4bf10cc0 4.1.31 2024-05-24 15:54:33 +02:00
9eaa6347c1 fix(core): update 2024-05-24 15:54:32 +02:00
14 changed files with 4368 additions and 1350 deletions

81
changelog.md Normal file
View File

@@ -0,0 +1,81 @@
# Changelog
## 2024-11-17 - 4.1.35 - fix(docker)
Fix Dockerfile dependency sorting and enhance environment variable handling for GitHub repos
- Refined the algorithm for sorting Dockerfiles based on dependencies to ensure proper build order.
- Enhanced environment variable handling in the NpmciEnv class to support conditional assignments.
- Updated various dependencies in package.json for improved performance and compatibility.
- Added error handling to circular dependency detection in Dockerfile sorting.
## 2024-11-05 - 4.1.34 - fix(connector)
Remove unused typedrequest implementation in cloudlyconnector
- Removed commented out code that initialized typedrequest in CloudlyConnector.
## 2024-11-05 - 4.1.33 - fix(core)
Updated dependencies and improved npm preparation logic.
- Updated @git.zone/tsbuild from ^2.1.84 to ^2.2.0.
- Updated @git.zone/tsrun from ^1.2.49 to ^1.3.3.
- Updated @types/node from ^22.7.9 to ^22.8.7.
- Updated @serve.zone/api from ^1.2.1 to ^4.3.1.
- Improved npm preparation logic to handle empty tokens gracefully.
## 2024-10-23 - 4.1.32 - fix(dependencies)
Update project dependencies to latest versions
- Updated development dependencies, including @git.zone/tsbuild and @git.zone/tsrun.
- Updated production dependencies such as @api.global/typedrequest and @push.rocks/smartfile.
## 2022-10-24 - 4.0.11 - prerelease
now includes a precheck for more generic runner execution
- Implemented a precheck feature for runners.
## 2022-10-09 to 2022-10-11 - 4.0.0 to 4.0.10 - migration
internal migrations and fixes
- Major switch to ESM style module: **BREAKING CHANGE**.
- Multiple fixes in core functionalities and module updates.
## 2019-11-26 - 3.1.73 - fixes
correctly setting npm cache and other updates
- Ensured correct npm cache setting during preparation.
- Various core updates.
## 2018-12-23 - 3.1.19 - privacy updates
enhanced mirroring controls for private code
- Now refusing to mirror private code.
## 2018-11-24 - 3.1.2 - ci improvement
removed unnecessary build dependency
- Removed npmts build dependency in CI pipeline.
## 2018-09-22 - 3.0.59 - enhancement
integrated smartlog for improved logging
- Logs now utilize smartlog for better management.
## 2017-09-08 - 3.0.14 - analytics
added analytics features
- Enabled analytics throughout the system.
## 2017-08-29 - 3.0.9 - docker enhancements
docker improvements and build args implementation
- Implemented working `dockerBuildArgEnvMap`.
## 2017-07-27 - 2.4.0 - stability improvements
various updates to stabilize the environment
- Fixed npmci versioning issues.
## 2016-11-25 - 2.3.24 - global tools
improved handling for global tool installations
- Improved install handling for needed global tools.

View File

@@ -14,15 +14,15 @@
"githost": "gitlab.com",
"gitscope": "ship.zone",
"gitrepo": "npmci",
"description": "A tool to enhance Node.js and Docker workflows within GitLab CI, providing various CI/CD utilities.",
"description": "A tool to streamline Node.js and Docker workflows within CI environments, particularly GitLab CI, providing various CI/CD utilities.",
"npmPackagename": "@ship.zone/npmci",
"license": "MIT",
"keywords": [
"Node.js",
"Docker",
"GitLab CI",
"continuous integration",
"continuous deployment",
"GitHub CI",
"Gitea CI",
"CI/CD",
"automation",
"npm",
@@ -30,7 +30,9 @@
"cloud",
"SSH",
"registry",
"container management"
"container management",
"continuous integration",
"continuous deployment"
]
}
}

View File

@@ -1,8 +1,8 @@
{
"name": "@ship.zone/npmci",
"version": "4.1.30",
"version": "4.1.35",
"private": false,
"description": "A tool to enhance Node.js and Docker workflows within GitLab CI, providing various CI/CD utilities.",
"description": "A tool to streamline Node.js and Docker workflows within CI environments, particularly GitLab CI, providing various CI/CD utilities.",
"main": "dist_ts/index.js",
"typings": "dist_ts/index.d.ts",
"type": "module",
@@ -26,36 +26,36 @@
},
"homepage": "https://gitlab.com/gitzone/npmci#README",
"devDependencies": {
"@git.zone/tsbuild": "^2.1.66",
"@git.zone/tsrun": "^1.2.44",
"@git.zone/tsbuild": "^2.2.0",
"@git.zone/tsrun": "^1.3.3",
"@git.zone/tstest": "^1.0.77",
"@push.rocks/tapbundle": "^5.0.15",
"@types/node": "^20.5.4"
"@push.rocks/tapbundle": "^5.5.0",
"@types/node": "^22.9.0"
},
"dependencies": {
"@api.global/typedrequest": "^3.0.1",
"@push.rocks/lik": "^6.0.5",
"@push.rocks/npmextra": "^4.0.0",
"@api.global/typedrequest": "^3.1.10",
"@push.rocks/lik": "^6.1.0",
"@push.rocks/npmextra": "^5.1.2",
"@push.rocks/projectinfo": "^5.0.2",
"@push.rocks/qenv": "^6.0.2",
"@push.rocks/smartanalytics": "^2.0.15",
"@push.rocks/smartcli": "^4.0.8",
"@push.rocks/smartcli": "^4.0.11",
"@push.rocks/smartdelay": "^3.0.5",
"@push.rocks/smartenv": "^5.0.5",
"@push.rocks/smartfile": "^10.0.30",
"@push.rocks/smartgit": "^3.0.1",
"@push.rocks/smartlog": "^3.0.3",
"@push.rocks/smartfile": "^11.0.21",
"@push.rocks/smartgit": "^3.1.1",
"@push.rocks/smartlog": "^3.0.7",
"@push.rocks/smartlog-destination-local": "^9.0.0",
"@push.rocks/smartobject": "^1.0.12",
"@push.rocks/smartpath": "^5.0.11",
"@push.rocks/smartpromise": "^4.0.2",
"@push.rocks/smartrequest": "^2.0.18",
"@push.rocks/smartshell": "^3.0.3",
"@push.rocks/smartpromise": "^4.0.4",
"@push.rocks/smartrequest": "^2.0.23",
"@push.rocks/smartshell": "^3.0.6",
"@push.rocks/smartsocket": "^2.0.22",
"@push.rocks/smartssh": "^2.0.1",
"@push.rocks/smartstring": "^4.0.8",
"@serve.zone/interfaces": "^1.0.3",
"@tsclass/tsclass": "^4.0.42",
"@serve.zone/api": "^4.3.11",
"@tsclass/tsclass": "^4.1.2",
"@types/through2": "^2.0.38",
"through2": "^4.0.2"
},
@@ -78,8 +78,8 @@
"Node.js",
"Docker",
"GitLab CI",
"continuous integration",
"continuous deployment",
"GitHub CI",
"Gitea CI",
"CI/CD",
"automation",
"npm",
@@ -87,6 +87,8 @@
"cloud",
"SSH",
"registry",
"container management"
"container management",
"continuous integration",
"continuous deployment"
]
}

5178
pnpm-lock.yaml generated

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,2 @@
- focus on cli usage in CI environments.
- show Gitlab CI, GitHub CI and Gitea CI examples.

200
readme.md
View File

@@ -1,6 +1,5 @@
```markdown
# @ship.zone/npmci
node and docker in gitlab ci on steroids
A tool to enhance Node.js and Docker workflows within GitLab CI, providing various CI/CD utilities.
## Install
@@ -18,7 +17,137 @@ yarn add @ship.zone/npmci
`npmci` is designed to streamline CI/CD processes, particularly in Docker and Node.js environments. The following sections illustrate its usage in various scenarios, from handling Node versions to building Docker images and more.
### 1. Handle Node Versions
### 1. Integration with GitLab CI, GitHub CI, and Gitea CI
#### GitLab CI
An example of integrating `npmci` into a GitLab CI configuration could look like this:
```yaml
image: hosttoday/ht-docker-node:npmci
stages:
- prepare
- build
- test
- deploy
default:
before_script:
- npmci node install stable
- npmci npm install
prepare:
stage: prepare
script:
- npmci prepare npm
- npmci prepare docker
build:
stage: build
script:
- npmci docker build
test:
stage: test
script:
- npmci npm test
deploy:
stage: deploy
script:
- npmci publish npm
- npmci docker push
environment:
name: production
url: http://example.com
```
#### GitHub Actions
Similarly, you can set up `npmci` in GitHub Actions:
```yaml
name: CI Pipeline
on:
push:
branches:
- main
jobs:
prepare:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up Node.js
uses: actions/setup-node@v2
with:
node-version: '14'
- run: npm install -g @ship.zone/npmci
- run: npmci node install stable
- run: npmci npm install
build:
runs-on: ubuntu-latest
needs: prepare
steps:
- run: npmci docker build
test:
runs-on: ubuntu-latest
needs: build
steps:
- run: npmci npm test
deploy:
runs-on: ubuntu-latest
needs: test
steps:
- run: npmci publish npm
- run: npmci docker push
```
#### Gitea CI
Lastly, for Gitea CI:
```yaml
image: hosttoday/ht-docker-node:npmci
pipelines:
default:
- step:
name: Prepare
image: hosttoday/ht-docker-node:npmci
commands:
- npmci node install stable
- npmci npm install
- npmci prepare npm
- npmci prepare docker
- step:
name: Build
image: hosttoday/ht-docker-node:npmci
commands:
- npmci docker build
- step:
name: Test
image: hosttoday/ht-docker-node:npmci
commands:
- npmci npm test
- step:
name: Deploy
image: hosttoday/ht-docker-node:npmci
commands:
- npmci publish npm
- npmci docker push
```
### 2. Handle Node Versions
One of the core features of `npmci` is managing Node versions in your CI environment. You can specify which version of Node to install:
@@ -49,7 +178,7 @@ async function manageNodeVersions() {
manageNodeVersions().then(() => console.log('Node versions managed successfully.'));
```
### 2. Handling npm and Yarn Tasks
### 3. Handling npm and Yarn Tasks
`npmci` provides numerous utilities to streamline npm and yarn workflow tasks within a CI/CD pipeline.
@@ -68,7 +197,7 @@ async function manageNpmTasks() {
manageNpmTasks().then(() => console.log('Npm tasks handled successfully.'));
```
### 3. Docker Task Handling
### 4. Docker Task Handling
`npmci` simplifies Docker operations, particularly in building, testing, and publishing Docker images.
@@ -132,7 +261,7 @@ async function pushDockerImages() {
pushDockerImages().then(() => console.log('Docker images pushed successfully.'));
```
### 4. Managing Docker Registries
### 5. Managing Docker Registries
`npmci` can handle multiple Docker registries and allows for easy integration within your CI pipeline.
@@ -166,7 +295,7 @@ async function pullDockerImages() {
pullDockerImages().then(() => console.log('Docker images pulled successfully.'));
```
### 5. SSH Key Management
### 6. SSH Key Management
`npmci` also simplifies the management of SSH keys, which is crucial for accessing private repositories and servers.
@@ -185,7 +314,7 @@ async function prepareSshKeys() {
prepareSshKeys().then(() => console.log('SSH keys prepared successfully.'));
```
### 6. Cloudron Integration
### 7. Cloudron Integration
For users deploying applications on Cloudron, `npmci` provides a set of utilities for automating Cloudron tasks.
@@ -235,7 +364,7 @@ async function deployWithPreparedManifest() {
deployWithPreparedManifest().then(() => console.log('Deployment to Cloudron with manifest preparation completed.'));
```
### 7. Webhook Triggers
### 8. Webhook Triggers
`npmci` supports webhook triggers, allowing you to trigger builds and other activities based on various conditions.
@@ -256,7 +385,7 @@ async function triggerWebhooks() {
triggerWebhooks().then(() => console.log('Webhooks triggered successfully.'));
```
### 8. Using the bash Helper
### 9. Using the bash Helper
`npmci` includes a bash helper for executing commands within a bash shell, useful for various custom tasks.
@@ -273,55 +402,7 @@ async function runCustomBashCommand(command: string) {
runCustomBashCommand('echo Hello World').then(() => console.log('Custom command executed successfully.'));
```
### 9. Example CI Configuration
An example of integrating `npmci` into a GitLab CI configuration might look like this:
```yaml
image: hosttoday/ht-docker-node:npmci
stages:
- prepare
- build
- test
- deploy
default:
before_script:
- npmci node install stable
- npmci npm install
prepare:
stage: prepare
script:
- npmci prepare npm
- npmci prepare docker
build:
stage: build
script:
- npmci docker build
test:
stage: test
script:
- npmci npm test
deploy:
stage: deploy
script:
- npmci publish npm
- npmci docker push
- npmci cloudron deploy
environment:
name: production
url: http://example.com
```
This example YAML file configures stages for preparing the environment, building the project, running tests, and deploying the project. `npmci` commands are used throughout to handle various tasks.
## Full Features and Use Cases
### Full Features and Use Cases
Below is a comprehensive set of features and use cases supported by `npmci`. This section ensures you can take full advantage of the library's capabilities in multiple scenarios.
@@ -410,5 +491,4 @@ handleDockerfileOperations().then(() => console.log('Dockerfile processing flow
```
This completes the comprehensive guide to `@ship.zone/npmci`. With the examples and explanations provided, you should be able to harness the full power and flexibility of the library to streamline your CI/CD processes effectively.
```
undefined

View File

@@ -1 +1,9 @@
{}
{
"gitzone": {
"module": {
"githost": "code.foss.global",
"gitscope": "mygroup",
"gitrepo": "myrepo"
}
}
}

View File

@@ -1,8 +1,8 @@
/**
* autocreated commitinfo by @pushrocks/commitinfo
* autocreated commitinfo by @push.rocks/commitinfo
*/
export const commitinfo = {
name: '@ship.zone/npmci',
version: '4.1.30',
description: 'A tool to enhance Node.js and Docker workflows within GitLab CI, providing various CI/CD utilities.'
version: '4.1.35',
description: 'A tool to streamline Node.js and Docker workflows within CI environments, particularly GitLab CI, providing various CI/CD utilities.'
}

View File

@@ -26,14 +26,6 @@ export class CloudlyConnector {
return;
}
const typedrequest =
new plugins.typedrequest.TypedRequest<plugins.servezoneInterfaces.requests.IRequest_InformAboutNewContainerImage>(
`https://${cloudlyUrl}/typedrequest`,
'servezonestandard_InformAboutNewContainerVersion'
);
const response = await typedrequest.fire({
containerImageInfo: optionsArg,
});
// lets push to cloudly here
}
}

View File

@@ -40,43 +40,72 @@ export class Dockerfile {
}
/**
* sorts Dockerfiles into a dependency chain
* @param sortableArrayArg an array of instances of class Dockerfile
* @returns Promise<Dockerfile[]>
* Sorts Dockerfiles into a build order based on dependencies.
* @param dockerfiles An array of Dockerfile instances.
* @returns A Promise that resolves to a sorted array of Dockerfiles.
*/
public static async sortDockerfiles(sortableArrayArg: Dockerfile[]): Promise<Dockerfile[]> {
const done = plugins.smartpromise.defer<Dockerfile[]>();
logger.log('info', 'sorting Dockerfiles:');
const sortedArray: Dockerfile[] = [];
const cleanTagsOriginal = Dockerfile.cleanTagsArrayFunction(sortableArrayArg, sortedArray);
let sorterFunctionCounter: number = 0;
const sorterFunction = () => {
sortableArrayArg.forEach((dockerfileArg) => {
const cleanTags = Dockerfile.cleanTagsArrayFunction(sortableArrayArg, sortedArray);
if (
cleanTags.indexOf(dockerfileArg.baseImage) === -1 &&
sortedArray.indexOf(dockerfileArg) === -1
) {
sortedArray.push(dockerfileArg);
}
if (cleanTagsOriginal.indexOf(dockerfileArg.baseImage) !== -1) {
dockerfileArg.localBaseImageDependent = true;
}
public static async sortDockerfiles(dockerfiles: Dockerfile[]): Promise<Dockerfile[]> {
logger.log('info', 'Sorting Dockerfiles based on dependencies...');
// Map from cleanTag to Dockerfile instance for quick lookup
const tagToDockerfile = new Map<string, Dockerfile>();
dockerfiles.forEach((dockerfile) => {
tagToDockerfile.set(dockerfile.cleanTag, dockerfile);
});
if (sortableArrayArg.length === sortedArray.length) {
let counter = 1;
for (const dockerfile of sortedArray) {
logger.log('info', `tag ${counter}: -> ${dockerfile.cleanTag}`);
counter++;
// Build the dependency graph
const graph = new Map<Dockerfile, Dockerfile[]>();
dockerfiles.forEach((dockerfile) => {
const dependencies: Dockerfile[] = [];
const baseImage = dockerfile.baseImage;
// Check if the baseImage is among the local Dockerfiles
if (tagToDockerfile.has(baseImage)) {
const baseDockerfile = tagToDockerfile.get(baseImage);
dependencies.push(baseDockerfile);
dockerfile.localBaseImageDependent = true;
dockerfile.localBaseDockerfile = baseDockerfile;
}
done.resolve(sortedArray);
} else if (sorterFunctionCounter < 10) {
sorterFunctionCounter++;
sorterFunction();
graph.set(dockerfile, dependencies);
});
// Perform topological sort
const sortedDockerfiles: Dockerfile[] = [];
const visited = new Set<Dockerfile>();
const tempMarked = new Set<Dockerfile>();
const visit = (dockerfile: Dockerfile) => {
if (tempMarked.has(dockerfile)) {
throw new Error(`Circular dependency detected involving ${dockerfile.cleanTag}`);
}
if (!visited.has(dockerfile)) {
tempMarked.add(dockerfile);
const dependencies = graph.get(dockerfile) || [];
dependencies.forEach((dep) => visit(dep));
tempMarked.delete(dockerfile);
visited.add(dockerfile);
sortedDockerfiles.push(dockerfile);
}
};
sorterFunction();
return done.promise;
try {
dockerfiles.forEach((dockerfile) => {
if (!visited.has(dockerfile)) {
visit(dockerfile);
}
});
} catch (error) {
logger.log('error', error.message);
throw error;
}
// Log the sorted order
sortedDockerfiles.forEach((dockerfile, index) => {
logger.log('info', `Build order ${index + 1}: ${dockerfile.cleanTag}`);
});
return sortedDockerfiles;
}
/**
@@ -120,7 +149,10 @@ export class Dockerfile {
* returns a version for a docker file
* @execution SYNC
*/
public static dockerFileVersion(dockerfileInstanceArg: Dockerfile, dockerfileNameArg: string): string {
public static dockerFileVersion(
dockerfileInstanceArg: Dockerfile,
dockerfileNameArg: string
): string {
let versionString: string;
const versionRegex = /Dockerfile_(.+)$/;
const regexResultArray = versionRegex.exec(dockerfileNameArg);
@@ -131,10 +163,11 @@ public static dockerFileVersion(dockerfileInstanceArg: Dockerfile, dockerfileNam
}
versionString = versionString.replace(
'##version##',
dockerfileInstanceArg.npmciDockerManagerRef.npmciRef.npmciConfig.getConfig().projectInfo.npm.version
dockerfileInstanceArg.npmciDockerManagerRef.npmciRef.npmciConfig.getConfig().projectInfo.npm
.version
);
return versionString;
}
}
/**
* returns the docker base image for a Dockerfile
@@ -196,22 +229,6 @@ public static dockerFileVersion(dockerfileInstanceArg: Dockerfile, dockerfileNam
return buildArgsString;
}
/**
*
*/
public static cleanTagsArrayFunction(
dockerfileArrayArg: Dockerfile[],
trackingArrayArg: Dockerfile[]
): string[] {
const cleanTagsArray: string[] = [];
dockerfileArrayArg.forEach((dockerfileArg) => {
if (trackingArrayArg.indexOf(dockerfileArg) === -1) {
cleanTagsArray.push(dockerfileArg.cleanTag);
}
});
return cleanTagsArray;
}
// INSTANCE
public npmciDockerManagerRef: NpmciDockerManager;
@@ -285,7 +302,10 @@ public static dockerFileVersion(dockerfileInstanceArg: Dockerfile, dockerfileNam
labels: [],
version: this.npmciDockerManagerRef.npmciRef.npmciConfig.getConfig().projectInfo.npm.version,
});
await this.npmciDockerManagerRef.npmciRef.npmciConfig.kvStorage.writeKey('latestPushedDockerTag', this.pushTag)
await this.npmciDockerManagerRef.npmciRef.npmciConfig.kvStorage.writeKey(
'latestPushedDockerTag',
this.pushTag
);
}
/**

View File

@@ -52,12 +52,17 @@ export class NpmciNpmManager {
* authenticates npm with token from env var
*/
public async prepare() {
logger.log('info', 'running >>npm prepare<<');
const config = this.npmciRef.npmciConfig.getConfig();
let npmrcFileString: string = '';
await plugins.smartobject.forEachMinimatch(
process.env,
'NPMCI_TOKEN_NPM*',
(npmEnvArg: string) => {
if (!npmEnvArg) {
logger.log('note','found empty token...');
return;
}
const npmRegistryUrl = npmEnvArg.split('|')[0];
logger.log('ok', `found token for ${npmRegistryUrl}`);
let npmToken = npmEnvArg.split('|')[1];

View File

@@ -48,10 +48,10 @@ export class NpmciConfig {
public async init() {
this.npmciNpmextra = new plugins.npmextra.Npmextra(paths.cwd);
this.kvStorage = new plugins.npmextra.KeyValueStore(
'userHomeDir',
`${this.npmciRef.npmciEnv.repo.user}_${this.npmciRef.npmciEnv.repo.repo}`
);
this.kvStorage = new plugins.npmextra.KeyValueStore({
typeArg: 'userHomeDir',
identityArg: `.npmci_${this.npmciRef.npmciEnv.repo.user}_${this.npmciRef.npmciEnv.repo.repo}`,
});
this.npmciQenv = new plugins.qenv.Qenv(
paths.NpmciProjectDir,
paths.NpmciProjectNogitDir,

View File

@@ -9,10 +9,10 @@ export class NpmciEnv {
constructor(npmciRefArg: Npmci) {
this.npmciRef = npmciRefArg;
if (process.env.GITLAB_CI) {
if (!this.repoString && process.env.GITLAB_CI) {
this.repoString = process.env.CI_REPOSITORY_URL;
}
if (process.env.NPMCI_COMPUTED_REPOURL) {
if (!this.repoString && process.env.NPMCI_COMPUTED_REPOURL) {
this.repoString = process.env.NPMCI_COMPUTED_REPOURL;
}
if (!this.repoString) {

View File

@@ -9,9 +9,9 @@ import * as typedrequest from '@api.global/typedrequest';
export { typedrequest };
// @servezone
import * as servezoneInterfaces from '@servezone/interfaces';
import * as servezoneApi from '@serve.zone/api';
export { servezoneInterfaces };
export { servezoneApi };
// @push.rocks
import * as lik from '@push.rocks/lik';