1.1.0

.gitea/workflows/docker_nottags.yaml

@@ -0,0 +1,71 @@
+name: Docker (tags)
+
+on:
+  push:
+    tags-ignore:
+      - '**'
+
+env:
+  IMAGE: code.foss.global/hosttoday/ht-docker-node:npmci
+  NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@/${{gitea.repository}}.git
+  NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
+  NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
+  NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
+  NPMCI_LOGIN_DOCKER_GITEA: ${{ github.server_url }}|${{ gitea.repository_owner }}|${{ secrets.GITEA_TOKEN }}
+  NPMCI_LOGIN_DOCKER_DOCKERREGISTRY: ${{ secrets.NPMCI_LOGIN_DOCKER_DOCKERREGISTRY }}
+
+jobs:
+  security:
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+    continue-on-error: true
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install pnpm and npmci
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @ship.zone/npmci
+          npmci npm prepare
+
+      - name: Audit production dependencies
+        run: |
+          npmci command npm config set registry https://registry.npmjs.org
+          npmci command pnpm audit --audit-level=high --prod
+        continue-on-error: true
+
+      - name: Audit development dependencies
+        run: |
+          npmci command npm config set registry https://registry.npmjs.org
+          npmci command pnpm audit --audit-level=high --dev
+        continue-on-error: true
+
+  test:
+    needs: security
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Prepare
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @ship.zone/npmci
+          npmci npm prepare
+
+      - name: Test stable
+        run: |
+          npmci node install stable
+          npmci npm install
+          npmci npm test
+
+      - name: Test build
+        run: |
+          npmci npm prepare
+          npmci node install stable
+          npmci npm install
+          npmci command npm run build

.gitea/workflows/docker_tags.yaml

@@ -0,0 +1,106 @@
+name: Docker (tags)
+
+on:
+  push:
+    tags:
+      - '*'
+
+env:
+  IMAGE: code.foss.global/hosttoday/ht-docker-node:npmci
+  NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@/${{gitea.repository}}.git
+  NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
+  NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
+  NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
+  NPMCI_LOGIN_DOCKER_GITEA: ${{ github.server_url }}|${{ gitea.repository_owner }}|${{ secrets.GITEA_TOKEN }}
+  NPMCI_LOGIN_DOCKER_DOCKERREGISTRY: ${{ secrets.NPMCI_LOGIN_DOCKER_DOCKERREGISTRY }}
+
+jobs:
+  security:
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+    continue-on-error: true
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Prepare
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @ship.zone/npmci
+          npmci npm prepare
+
+      - name: Audit production dependencies
+        run: |
+          npmci command npm config set registry https://registry.npmjs.org
+          npmci command pnpm audit --audit-level=high --prod
+        continue-on-error: true
+
+      - name: Audit development dependencies
+        run: |
+          npmci command npm config set registry https://registry.npmjs.org
+          npmci command pnpm audit --audit-level=high --dev
+        continue-on-error: true
+
+  test:
+    needs: security
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Prepare
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @ship.zone/npmci
+          npmci npm prepare
+
+      - name: Test stable
+        run: |
+          npmci node install stable
+          npmci npm install
+          npmci npm test
+
+      - name: Test build
+        run: |
+          npmci node install stable
+          npmci npm install
+          npmci command npm run build
+
+  release:
+    needs: test
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-latest
+    container:
+      image: code.foss.global/hosttoday/ht-docker-dbase:npmci
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Prepare
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @ship.zone/npmci
+
+      - name: Release
+        run: |
+          npmci docker login
+          npmci docker build
+          npmci docker test
+          # npmci docker push 
+          npmci docker push 
+
+  metadata:
+    needs: test
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Trigger
+        run: npmci trigger

.gitignore

@@ -3,7 +3,6 @@
 # artifacts
 coverage/
 public/
-pages/
 
 # installs
 node_modules/
@@ -17,4 +16,4 @@ node_modules/
 dist/
 dist_*/
 
-# custom
+#------# custom

Dockerfile

@@ -24,7 +24,7 @@ RUN rm -rf node_modules/ && pnpm install --prod
 
 
 ## STAGE 3 // rebuild dependencies for alpine
-FROM code.foss.global/host.today/ht-docker-node:alpine_npmci as node3
+FROM code.foss.global/host.today/ht-docker-node:alpinenpmci as node3
 WORKDIR /app
 COPY --from=node2 /app /app
 ARG NPMCI_TOKEN_NPM2

changelog.md

@@ -1,5 +1,20 @@
 # Changelog
 
+## 2024-12-29 - 1.1.0 - feat(.gitea/workflows)
+Add GitHub Actions workflows for Docker build and test
+
+- Added .gitea/workflows/docker_nottags.yaml for handling Docker builds on non-tagged commits.
+- Added .gitea/workflows/docker_tags.yaml for handling Docker builds on tagged commits.
+- Both workflows include steps for security audits, testing, and build preparation.
+
+## 2024-12-29 - 1.0.135 - fix(core)
+Fix image retrieval and service deployment process for workload services
+
+- Updated the handling of internal and external registry images when provisioning workload services.
+- Refined the image pulling logic to accommodate both internal and external sources.
+- Ensured proper secret management during service deployment.
+- Corrected and optimized key function parameters and API calls for clarity and functionality.
+
 ## 2024-12-20 - 1.0.134 - fix(core)
 Updated dependencies and refactored ClusterManager and CloudlyConnector for improved Docker management.
 

npmextra.json

@@ -13,12 +13,12 @@
   "gitzone": {
     "projectType": "service",
     "module": {
-      "githost": "gitlab.com",
-      "gitscope": "losslessone/services/servezone",
+      "githost": "code.foss.global",
+      "gitscope": "serve.zone",
       "gitrepo": "coreflow",
       "description": "A comprehensive tool for managing Docker-based applications and services, enabling efficient scaling, network management, and integration with cloud services.",
       "npmPackagename": "@serve.zone/coreflow",
-      "license": "UNLICENSED",
+      "license": "MIT",
       "keywords": [
         "Docker",
         "Service scaling",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@serve.zone/coreflow",
-  "version": "1.0.134",
+  "version": "1.1.0",
   "description": "A comprehensive tool for managing Docker-based applications and services, enabling efficient scaling, network management, and integration with cloud services.",
   "main": "dist_ts/index.js",
   "typings": "dist_ts/index.d.ts",
@@ -18,7 +18,7 @@
   },
   "repository": {
     "type": "git",
-    "url": "git+https://gitlab.com/pushrocks/coreflow.git"
+    "url": "https://code.foss.global/serve.zone/coreflow.git"
   },
   "keywords": [
     "Docker",
@@ -49,9 +49,9 @@
   "author": "Lossless GmbH",
   "license": "MIT",
   "bugs": {
-    "url": "https://gitlab.com/pushrocks/coreflow/issues"
+    "url": "https://gitlab.com/losslessone/services/servezone/coreflow/issues"
   },
-  "homepage": "https://gitlab.com/pushrocks/coreflow#readme",
+  "homepage": "https://gitlab.com/losslessone/services/servezone/coreflow#readme",
   "devDependencies": {
     "@git.zone/tsbuild": "^2.2.0",
     "@git.zone/tsrun": "^1.3.3",
@@ -62,7 +62,7 @@
   "dependencies": {
     "@api.global/typedrequest": "^3.1.10",
     "@api.global/typedsocket": "^3.0.1",
-    "@apiclient.xyz/docker": "^1.2.7",
+    "@apiclient.xyz/docker": "^1.3.0",
     "@push.rocks/early": "^4.0.3",
     "@push.rocks/lik": "^6.1.0",
     "@push.rocks/projectinfo": "^5.0.1",
@@ -79,8 +79,8 @@
     "@push.rocks/smartstream": "^3.2.5",
     "@push.rocks/smartstring": "^4.0.15",
     "@push.rocks/taskbuffer": "^3.0.10",
-    "@serve.zone/api": "^4.5.1",
-    "@serve.zone/interfaces": "^4.5.1",
+    "@serve.zone/api": "^4.10.0",
+    "@serve.zone/interfaces": "^4.10.0",
     "@tsclass/tsclass": "^4.2.0",
     "@types/node": "22.10.2"
   },

readme.md

@@ -1,4 +1,5 @@
 # @serve.zone/coreflow
+
 A comprehensive solution for managing Docker and scaling applications across servers, handling tasks from service provisioning to network traffic management.
 
 ## Install
@@ -42,6 +43,7 @@ await coreflowInstance.stop();
 ```
 
 In the above example:
+
 - The Coreflow instance is initialized.
 - Coreflow is started, which internally initializes various managers and connectors.
 - The method `handleDockerEvents` is used to handle Docker events.
@@ -54,8 +56,8 @@ Coreflow manages applications and services, often requiring direct interactions
 ```typescript
 // Assuming coreflowInstance is already started as per previous examples
 const serviceConnection = coreflowInstance.createServiceConnection({
-  serviceName: "myDatabaseService",
-  servicePort: 3306
+  serviceName: 'myDatabaseService',
+  servicePort: 3306,
 });
 
 serviceConnection.connect().then(() => {
@@ -69,7 +71,7 @@ Coreflow excels in scaling applications across multiple servers. This involves n
 
 ```typescript
 const scalingPolicy = {
-  serviceName: "apiService",
+  serviceName: 'apiService',
   replicaCount: 5, // Target number of replicas
   maxReplicaCount: 10, // Maximum number of replicas
   minReplicaCount: 2, // Minimum number of replicas
@@ -81,6 +83,7 @@ coreflowInstance.applyScalingPolicy(scalingPolicy).then(() => {
 ```
 
 In the above example:
+
 - A scaling policy is defined with target, maximum, and minimum replica counts for the `apiService`.
 - The `applyScalingPolicy` method of the Coreflow instance is used to apply this scaling policy.
 
@@ -92,10 +95,10 @@ One of Coreflow's key features is its ability to manage network traffic, ensurin
 import { TrafficRule } from '@serve.zone/coreflow';
 
 const rule: TrafficRule = {
-  serviceName: "webService",
+  serviceName: 'webService',
   externalPort: 80,
   internalPort: 3000,
-  protocol: "http",
+  protocol: 'http',
 };
 
 coreflowInstance.applyTrafficRule(rule).then(() => {
@@ -104,6 +107,7 @@ coreflowInstance.applyTrafficRule(rule).then(() => {
 ```
 
 In the above example:
+
 - A traffic rule is defined for the `webService`, redirecting external traffic from port 80 to the service's internal port 3000.
 - The `applyTrafficRule` method is used to enforce this rule.
 
@@ -113,9 +117,9 @@ Coreflow integrates continuous integration and deployment processes, allowing se
 
 ```typescript
 const deploymentConfig = {
-  serviceName: "userAuthService",
-  image: "myregistry.com/userauthservice:latest",
-  updatePolicy: "rolling" // or "recreate"
+  serviceName: 'userAuthService',
+  image: 'myregistry.com/userauthservice:latest',
+  updatePolicy: 'rolling', // or "recreate"
 };
 
 coreflowInstance.deployService(deploymentConfig).then(() => {
@@ -124,6 +128,7 @@ coreflowInstance.deployService(deploymentConfig).then(() => {
 ```
 
 In the above example:
+
 - A deployment configuration is created for the `userAuthService` using the latest image from the specified registry.
 - The `deployService` method is then used to deploy the service using the specified update policy (e.g., rolling updates or recreating the service).
 
@@ -132,12 +137,13 @@ In the above example:
 To keep track of your applications' health and performance, Coreflow provides tools for logging, monitoring, and alerting.
 
 ```typescript
-coreflowInstance.monitorService("webService").on('serviceHealthUpdate', (healthStatus) => {
+coreflowInstance.monitorService('webService').on('serviceHealthUpdate', (healthStatus) => {
   console.log(`Received health update for webService: ${healthStatus}`);
 });
 ```
 
 In the above example:
+
 - The `monitorService` method is used to monitor the health status of the `webService`.
 - When a health update event is received, it is logged to the console.
 
@@ -175,8 +181,8 @@ coreflowInstance.handleDockerEvents().then(() => {
 
 ```typescript
 const serviceConnection = coreflowInstance.createServiceConnection({
-  serviceName: "databaseService",
-  servicePort: 5432
+  serviceName: 'databaseService',
+  servicePort: 5432,
 });
 
 serviceConnection.connect().then(() => {
@@ -188,7 +194,7 @@ serviceConnection.connect().then(() => {
 
 ```typescript
 const scalingPolicy = {
-  serviceName: "microserviceA",
+  serviceName: 'microserviceA',
   replicaCount: 3, // Starting with 3 replicas
   maxReplicaCount: 10, // Allowing up to 10 replicas
   minReplicaCount: 2, // Ensuring at least 2 replicas
@@ -206,17 +212,17 @@ import { TrafficRule } from '@serve.zone/coreflow';
 
 const trafficRules: TrafficRule[] = [
   {
-    serviceName: "frontendService",
+    serviceName: 'frontendService',
     externalPort: 80,
     internalPort: 3000,
-    protocol: "http",
+    protocol: 'http',
   },
   {
-    serviceName: "apiService",
+    serviceName: 'apiService',
     externalPort: 443,
     internalPort: 4000,
-    protocol: "https",
-  }
+    protocol: 'https',
+  },
 ];
 
 Promise.all(trafficRules.map((rule) => coreflowInstance.applyTrafficRule(rule))).then(() => {
@@ -228,9 +234,9 @@ Promise.all(trafficRules.map((rule) => coreflowInstance.applyTrafficRule(rule)))
 
 ```typescript
 const deploymentConfig = {
-  serviceName: "authService",
-  image: "myregistry.com/authservice:latest",
-  updatePolicy: "rolling", // Performing rolling updates
+  serviceName: 'authService',
+  image: 'myregistry.com/authservice:latest',
+  updatePolicy: 'rolling', // Performing rolling updates
 };
 
 coreflowInstance.deployService(deploymentConfig).then(() => {
@@ -241,7 +247,7 @@ coreflowInstance.deployService(deploymentConfig).then(() => {
 #### Step 7: Monitoring a Service
 
 ```typescript
-coreflowInstance.monitorService("frontendService").on('serviceHealthUpdate', (healthStatus) => {
+coreflowInstance.monitorService('frontendService').on('serviceHealthUpdate', (healthStatus) => {
   console.log(`Health update for frontendService: ${healthStatus}`);
 });
 ```
@@ -262,7 +268,7 @@ const checkinTask = new Task({
   buffered: true,
   taskFunction: async () => {
     console.log('Running checkin task...');
-  }
+  },
 });
 
 const taskManager = coreflowInstance.taskManager;
@@ -284,18 +290,18 @@ const coretrafficConnector = new CoretrafficConnector(coreflowInstance);
 
 const reverseProxyConfigs = [
   {
-    hostName: "example.com",
-    destinationIp: "192.168.1.100",
-    destinationPort: "3000",
-    privateKey: "<your-private-key>",
-    publicKey: "<your-public-key>",
+    hostName: 'example.com',
+    destinationIp: '192.168.1.100',
+    destinationPort: '3000',
+    privateKey: '<your-private-key>',
+    publicKey: '<your-public-key>',
   },
   {
-    hostName: "api.example.com",
-    destinationIp: "192.168.1.101",
-    destinationPort: "4000",
-    privateKey: "<your-private-key>",
-    publicKey: "<your-public-key>",
+    hostName: 'api.example.com',
+    destinationIp: '192.168.1.101',
+    destinationPort: '4000',
+    privateKey: '<your-private-key>',
+    publicKey: '<your-public-key>',
   },
 ];
 

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@serve.zone/coreflow',
-  version: '1.0.134',
+  version: '1.1.0',
   description: 'A comprehensive tool for managing Docker-based applications and services, enabling efficient scaling, network management, and integration with cloud services.'
 }

ts/coreflow.classes.clustermanager.ts

@@ -34,7 +34,7 @@ export class ClusterManager {
       this.coreflowRef.cloudlyConnector.cloudlyApiClient.configUpdateSubject.subscribe(
         async (dataArg) => {
           this.coreflowRef.taskManager.updateBaseServicesTask.trigger();
-        }
+        },
       );
   }
 
@@ -60,14 +60,17 @@ export class ClusterManager {
     // make sure there is a network for the webgateway
     let sznWebgatewayNetwork = await plugins.docker.DockerNetwork.getNetworkByName(
       this.coreflowRef.dockerHost,
-      this.commonDockerData.networkNames.sznWebgateway
+      this.commonDockerData.networkNames.sznWebgateway,
     );
 
     if (!sznWebgatewayNetwork) {
       logger.log('info', 'Creating network: ' + this.commonDockerData.networkNames.sznWebgateway);
-      sznWebgatewayNetwork = await plugins.docker.DockerNetwork.createNetwork(this.coreflowRef.dockerHost, {
-        Name: this.commonDockerData.networkNames.sznWebgateway,
-      });
+      sznWebgatewayNetwork = await plugins.docker.DockerNetwork.createNetwork(
+        this.coreflowRef.dockerHost,
+        {
+          Name: this.commonDockerData.networkNames.sznWebgateway,
+        },
+      );
     } else {
       logger.log('ok', 'sznWebgateway is already present');
     }
@@ -75,12 +78,15 @@ export class ClusterManager {
     // corechat network so base services can talk to each other
     let sznCorechatNetwork = await plugins.docker.DockerNetwork.getNetworkByName(
       this.coreflowRef.dockerHost,
-      this.commonDockerData.networkNames.sznCorechat
+      this.commonDockerData.networkNames.sznCorechat,
     );
     if (!sznCorechatNetwork) {
-      sznCorechatNetwork = await plugins.docker.DockerNetwork.createNetwork(this.coreflowRef.dockerHost, {
-        Name: this.commonDockerData.networkNames.sznCorechat,
-      });
+      sznCorechatNetwork = await plugins.docker.DockerNetwork.createNetwork(
+        this.coreflowRef.dockerHost,
+        {
+          Name: this.commonDockerData.networkNames.sznCorechat,
+        },
+      );
     } else {
       logger.log('ok', 'sznCorechat is already present');
     }
@@ -93,17 +99,23 @@ export class ClusterManager {
     // Images
     logger.log('info', `now updating docker images of base services...`);
 
-    const coretrafficImage = await plugins.docker.DockerImage.createFromRegistry(this.coreflowRef.dockerHost, {
-      creationObject: {
-        imageUrl: 'code.foss.global/serve.zone/coretraffic',
-      }
-    });
+    const coretrafficImage = await plugins.docker.DockerImage.createFromRegistry(
+      this.coreflowRef.dockerHost,
+      {
+        creationObject: {
+          imageUrl: 'code.foss.global/serve.zone/coretraffic',
+        },
+      },
+    );
 
-    const corelogImage = await plugins.docker.DockerImage.createFromRegistry(this.coreflowRef.dockerHost, {
-      creationObject: {
-        imageUrl: 'code.foss.global/serve.zone/corelog',
-      }
-    });
+    const corelogImage = await plugins.docker.DockerImage.createFromRegistry(
+      this.coreflowRef.dockerHost,
+      {
+        creationObject: {
+          imageUrl: 'code.foss.global/serve.zone/corelog',
+        },
+      },
+    );
 
     // SERVICES
     // lets deploy the base services
@@ -111,7 +123,7 @@ export class ClusterManager {
     let coretrafficService: plugins.docker.DockerService;
     coretrafficService = await plugins.docker.DockerService.getServiceByName(
       this.coreflowRef.dockerHost,
-      'coretraffic'
+      'coretraffic',
     );
 
     if (coretrafficService && (await coretrafficService.needsUpdate())) {
@@ -123,19 +135,22 @@ export class ClusterManager {
     }
 
     if (!coretrafficService) {
-      coretrafficService = await plugins.docker.DockerService.createService(this.coreflowRef.dockerHost, {
-        image: coretrafficImage,
-        labels: {},
-        name: 'coretraffic',
-        networks: [sznCorechatNetwork, sznWebgatewayNetwork],
-        networkAlias: 'coretraffic',
-        ports: ['80:7999', '443:8000'],
-        secrets: [],
-        resources: {
-          memorySizeMB: 1100,
-          volumeMounts: [],
+      coretrafficService = await plugins.docker.DockerService.createService(
+        this.coreflowRef.dockerHost,
+        {
+          image: coretrafficImage,
+          labels: {},
+          name: 'coretraffic',
+          networks: [sznCorechatNetwork, sznWebgatewayNetwork],
+          networkAlias: 'coretraffic',
+          ports: ['80:7999', '443:8000'],
+          secrets: [],
+          resources: {
+            memorySizeMB: 1100,
+            volumeMounts: [],
+          },
         },
-      });
+      );
     } else {
       logger.log('ok', 'coretraffic service is already present');
     }
@@ -146,7 +161,7 @@ export class ClusterManager {
     let corelogService: plugins.docker.DockerService;
     corelogService = await plugins.docker.DockerService.getServiceByName(
       this.coreflowRef.dockerHost,
-      'corelog'
+      'corelog',
     );
 
     if (corelogService && (await corelogService.needsUpdate())) {
@@ -157,19 +172,22 @@ export class ClusterManager {
     }
 
     if (!corelogService) {
-      corelogService = await plugins.docker.DockerService.createService(this.coreflowRef.dockerHost, {
-        image: corelogImage,
-        labels: {},
-        name: 'corelog',
-        networks: [sznCorechatNetwork],
-        networkAlias: 'corelog',
-        ports: [],
-        secrets: [],
-        resources: {
-          memorySizeMB: 120,
-          volumeMounts: [],
+      corelogService = await plugins.docker.DockerService.createService(
+        this.coreflowRef.dockerHost,
+        {
+          image: corelogImage,
+          labels: {},
+          name: 'corelog',
+          networks: [sznCorechatNetwork],
+          networkAlias: 'corelog',
+          ports: [],
+          secrets: [],
+          resources: {
+            memorySizeMB: 120,
+            volumeMounts: [],
+          },
         },
-      });
+      );
     } else {
       logger.log('ok', 'corelog service is already present');
     }
@@ -178,39 +196,85 @@ export class ClusterManager {
   }
 
   public async provisionWorkloadService(
-    serviceArg: plugins.servezoneInterfaces.data.IService
+    serviceArgFromCloudly: plugins.servezoneInterfaces.data.IService,
   ) {
-    logger.log('info', `deploying service ${serviceArg.data.name}@${serviceArg.data.imageVersion}...`);
+    logger.log(
+      'info',
+      `deploying service ${serviceArgFromCloudly.data.name}@${serviceArgFromCloudly.data.imageVersion}...`,
+    );
 
     // get the image from cloudly
-    logger.log('info', `getting image for ${serviceArg.data.name}@${serviceArg.data.imageVersion}`);
-    const containerImage = await this.coreflowRef.cloudlyConnector.cloudlyApiClient.images.getImageById(serviceArg.data.imageId);
-    const imageStream = await containerImage.pullImageVersion(serviceArg.data.imageVersion);
-    await plugins.docker.DockerImage.createFromTarStream(this.coreflowRef.dockerHost, {
-      creationObject: {
-        imageUrl: containerImage.id,
-        imageTag: serviceArg.data.imageVersion,
-      },
-      tarStream: plugins.smartstream.nodewebhelpers.convertWebReadableToNodeReadable(imageStream)
-    });
+    logger.log(
+      'info',
+      `getting image for ${serviceArgFromCloudly.data.name}@${serviceArgFromCloudly.data.imageVersion}`,
+    );
+    const containerImageFromCloudly =
+      await this.coreflowRef.cloudlyConnector.cloudlyApiClient.image.getImageById(
+        serviceArgFromCloudly.data.imageId,
+      );
+    let localDockerImage: plugins.docker.DockerImage;
+
+    // lets get the docker image for the service
+    if (containerImageFromCloudly.data.location.internal) {
+      const imageStream = await containerImageFromCloudly.pullImageVersion(
+        serviceArgFromCloudly.data.imageVersion,
+      );
+      localDockerImage = await plugins.docker.DockerImage.createFromTarStream(
+        this.coreflowRef.dockerHost,
+        {
+          creationObject: {
+            imageUrl: containerImageFromCloudly.id,
+            imageTag: serviceArgFromCloudly.data.imageVersion,
+          },
+          tarStream:
+            plugins.smartstream.nodewebhelpers.convertWebReadableToNodeReadable(imageStream),
+        },
+      );
+    } else if (
+      containerImageFromCloudly.data.location.externalRegistryId &&
+      containerImageFromCloudly.data.location.externalImageTag
+    ) {
+      const externalRegistry =
+        await this.coreflowRef.cloudlyConnector.cloudlyApiClient.externalRegistry.getRegistryById(
+          containerImageFromCloudly.data.location.externalRegistryId,
+        );
+      // Lets authenticate against the external registry
+      // TODO: deduplicate this, check wether we are already authenticated
+      await this.coreflowRef.dockerHost.auth({
+        username: externalRegistry.data.username,
+        password: externalRegistry.data.password,
+        serveraddress: externalRegistry.data.url,
+      });
+      localDockerImage = await plugins.docker.DockerImage.createFromRegistry(
+        this.coreflowRef.dockerHost,
+        {
+          creationObject: {
+            imageUrl: containerImageFromCloudly.id,
+            imageTag: serviceArgFromCloudly.data.imageVersion,
+          },
+        },
+      );
+      await localDockerImage.pullLatestImageFromRegistry();
+    } else {
+      throw new Error('Invalid image location');
+    }
 
     let containerService = await plugins.docker.DockerService.getServiceByName(
       this.coreflowRef.dockerHost,
-      serviceArg.data.name
+      serviceArgFromCloudly.data.name,
     );
-    const secretBundleId = serviceArg.data.secretBundleId;
     this.coreflowRef.cloudlyConnector.cloudlyApiClient;
 
-    const dockerSecretName = `${serviceArg.id}_${serviceArg.data.name}_Secret`;
+    const dockerSecretName = `${serviceArgFromCloudly.id}_${serviceArgFromCloudly.data.name}_Secret`;
     let containerSecret = await plugins.docker.DockerSecret.getSecretByName(
       this.coreflowRef.dockerHost,
-      dockerSecretName
+      dockerSecretName,
     );
 
     // existing network to connect to
     const webGatewayNetwork = await plugins.docker.DockerNetwork.getNetworkByName(
       this.coreflowRef.dockerHost,
-      this.commonDockerData.networkNames.sznWebgateway
+      this.commonDockerData.networkNames.sznWebgateway,
     );
 
     if (containerService && (await containerService.needsUpdate())) {
@@ -225,41 +289,55 @@ export class ClusterManager {
     if (!containerService) {
       containerSecret = await plugins.docker.DockerSecret.getSecretByName(
         this.coreflowRef.dockerHost,
-        dockerSecretName
+        dockerSecretName,
       );
       if (containerSecret) {
         await containerSecret.remove();
       }
 
-      const secretBundle: plugins.servezoneInterfaces.data.ISecretBundle =this.coreflowRef.cloudlyConnector.cloudlyApiClient.;
+      const secretBundle =
+        await this.coreflowRef.cloudlyConnector.cloudlyApiClient.secretbundle.getSecretBundleById(
+          serviceArgFromCloudly.data.secretBundleId,
+        );
 
-      containerSecret = await plugins.docker.DockerSecret.createSecret(this.coreflowRef.dockerHost, {
-        name: dockerSecretName,
-        contentArg: JSON.stringify(secretBundle.data.),
-        labels: {},
-        version: await containerImage.getVersion(),
-      });
-      containerService = await plugins.docker.DockerService.createService(this.coreflowRef.dockerHost, {
-        name: deploymentDirectiveArg.name,
-        image: containerImage,
-        networks: [webGatewayNetwork],
-        secrets: [containerSecret],
-        ports: [],
-        labels: {},
-        resources: deploymentDirectiveArg.resources,
-        networkAlias: deploymentDirectiveArg.name,
-      });
+      // lets create the relevant stuff on the docker side
+      containerSecret = await plugins.docker.DockerSecret.createSecret(
+        this.coreflowRef.dockerHost,
+        {
+          name: dockerSecretName,
+          contentArg: JSON.stringify(await secretBundle.getFlatKeyValueObjectForEnvironment()),
+          labels: {},
+          version:
+            await containerImageFromCloudly.data.versions[serviceArgFromCloudly.data.imageVersion],
+        },
+      );
+      containerService = await plugins.docker.DockerService.createService(
+        this.coreflowRef.dockerHost,
+        {
+          name: serviceArgFromCloudly.data.name,
+          image: localDockerImage,
+          networks: [webGatewayNetwork],
+          secrets: [containerSecret],
+          ports: [],
+          labels: {},
+          resources: serviceArgFromCloudly.data.resources,
+          // TODO: introduce a clean name here, that is guaranteed to work with APIs.
+          networkAlias: serviceArgFromCloudly.data.name,
+        },
+      );
     }
   }
 
   /**
    * update traffic routing
    */
-  public async updateTrafficRouting(clusterConfigArg: plugins.servezoneInterfaces.data.IClusterConfig) {
+  public async updateTrafficRouting(
+    clusterConfigArg: plugins.servezoneInterfaces.data.IClusterConfig,
+  ) {
     const services = await this.coreflowRef.dockerHost.getServices();
     const webGatewayNetwork = await plugins.docker.DockerNetwork.getNetworkByName(
       this.coreflowRef.dockerHost,
-      this.commonDockerData.networkNames.sznWebgateway
+      this.commonDockerData.networkNames.sznWebgateway,
     );
     const reverseProxyConfigs: plugins.servezoneInterfaces.data.IReverseProxyConfig[] = [];
 
@@ -267,7 +345,7 @@ export class ClusterManager {
       serviceNameArg: string,
       hostNameArg: string,
       containerDestinationIp: string,
-      webDestinationPort: string
+      webDestinationPort: string,
     ) => {
       logger.log('ok', `trying to obtain a certificate for ${hostNameArg}`);
       const certificate =
@@ -281,7 +359,7 @@ export class ClusterManager {
       });
       logger.log(
         'success',
-        `pushed routing config for ${hostNameArg} on workload service ${serviceNameArg}`
+        `pushed routing config for ${hostNameArg} on workload service ${serviceNameArg}`,
       );
     };
 
@@ -303,7 +381,7 @@ export class ClusterManager {
         if (!containersOfServicesOnNetwork[0]) {
           logger.log(
             'error',
-            `There seems to be no container available for service ${service.Spec.Name}`
+            `There seems to be no container available for service ${service.Spec.Name}`,
           );
           continue;
         }
@@ -317,7 +395,7 @@ export class ClusterManager {
             workloadConfig.name,
             hostName,
             containerDestinationIp,
-            webDestinationPort
+            webDestinationPort,
           );
         }
 
@@ -329,14 +407,14 @@ export class ClusterManager {
               workloadConfig.name,
               customDomainKey,
               containerDestinationIp,
-              workloadConfig.ports.custom[customDomainKey]
+              workloadConfig.ports.custom[customDomainKey],
             );
           }
         }
       } else {
         logger.log(
           'ok',
-          `service ${service.Spec.Name} is not a workload service and won't receive traffic`
+          `service ${service.Spec.Name} is not a workload service and won't receive traffic`,
         );
       }
     }

ts/coreflow.classes.internalserver.ts

@@ -11,7 +11,7 @@ export class InternalServer {
 
   public async start() {
     this.typedsocketServer = await plugins.typedsocket.TypedSocket.createServer(
-      this.coreflowRef.typedrouter
+      this.coreflowRef.typedrouter,
     );
   }
 

ts/coreflow.classes.taskmanager.ts

@@ -77,7 +77,7 @@ export class CoreflowTaskmanager {
         await this.updateBaseServicesTask.trigger();
         logger.log(
           'success',
-          'initial tasks successfully executed! Now handing over to longterm taskmanager!'
+          'initial tasks successfully executed! Now handing over to longterm taskmanager!',
         );
       } catch (e) {
         console.log(e);

ts/coreflow.connector.cloudlyconnector.ts

@@ -18,7 +18,7 @@ export class CloudlyConnector {
   public async start() {
     this.cloudlyApiClient = new plugins.servezoneApi.CloudlyApiClient({
       registerAs: 'coreflow',
-      cloudlyUrl: await this.coreflowRef.serviceQenv.getEnvVarOnDemand('CLOUDLY_URL')
+      cloudlyUrl: await this.coreflowRef.serviceQenv.getEnvVarOnDemand('CLOUDLY_URL'),
     });
     await this.cloudlyApiClient.start();
     this.coreflowJumpCode = await this.coreflowRef.serviceQenv.getEnvVarOnDemand('JUMPCODE');
@@ -35,14 +35,12 @@ export class CloudlyConnector {
   }
 
   public async getConfigFromCloudly(): Promise<plugins.servezoneInterfaces.data.ICluster> {
-    const config = await this.cloudlyApiClient.getClusterConfigFromCloudlyByIdentity(
-      this.identity
-    );
+    const config = await this.cloudlyApiClient.getClusterConfigFromCloudlyByIdentity(this.identity);
     return config;
   }
 
   public async getCertificateForDomainFromCloudly(
-    domainNameArg: string
+    domainNameArg: string,
   ): Promise<plugins.tsclass.network.ICert> {
     const certificate = await this.cloudlyApiClient.getCertificateForDomain({
       identity: this.identity,

ts/coreflow.connector.coretrafficconnector.ts

@@ -16,15 +16,15 @@ export class CoretrafficConnector {
   }
 
   public async setReverseConfigs(
-    reverseConfigsArg: plugins.servezoneInterfaces.data.IReverseProxyConfig[]
+    reverseConfigsArg: plugins.servezoneInterfaces.data.IReverseProxyConfig[],
   ) {
     await this.start();
     const reactionRequest =
       this.coreflowRef.internalServer.typedsocketServer.createTypedRequest<plugins.servezoneInterfaces.requests.routing.IRequest_Coreflow_Coretraffic_RoutingUpdate>(
         'updateRouting',
         await this.coreflowRef.internalServer.typedsocketServer.findTargetConnection(
-          async (targetConnection) => targetConnection.alias === 'coretraffic'
-        )
+          async (targetConnection) => targetConnection.alias === 'coretraffic',
+        ),
       );
     const response = await reactionRequest.fire({
       reverseConfigs: reverseConfigsArg,

ts/coreflow.paths.ts

@@ -2,5 +2,5 @@ import * as plugins from './coreflow.plugins.js';
 
 export const packageDir = plugins.path.join(
   plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url),
-  '../'
+  '../',
 );

ts/index.ts

@@ -18,7 +18,7 @@ export const runCli = async () => {
     'info',
     `trying to start coreflow@v${projectInfoNpm.version} on ${
       (await smartnetworkInstance.getPublicIps()).v4
-    }`
+    }`,
   );
   coreflowInstance = new Coreflow();
   await coreflowInstance.start();

tsconfig.json

@@ -6,7 +6,9 @@
     "module": "NodeNext",
     "moduleResolution": "NodeNext",
     "esModuleInterop": true,
-    "verbatimModuleSyntax": true
+    "verbatimModuleSyntax": true,
+    "baseUrl": ".",
+    "paths": {}
   },
   "exclude": [
     "dist_*/**/*.d.ts"