1.1.0

.gitea/workflows/docker_nottags.yaml

@@ -0,0 +1,71 @@
+name: Docker (tags)
+
+on:
+  push:
+    tags-ignore:
+      - '**'
+
+env:
+  IMAGE: code.foss.global/hosttoday/ht-docker-node:npmci
+  NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@/${{gitea.repository}}.git
+  NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
+  NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
+  NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
+  NPMCI_LOGIN_DOCKER_GITEA: ${{ github.server_url }}|${{ gitea.repository_owner }}|${{ secrets.GITEA_TOKEN }}
+  NPMCI_LOGIN_DOCKER_DOCKERREGISTRY: ${{ secrets.NPMCI_LOGIN_DOCKER_DOCKERREGISTRY }}
+
+jobs:
+  security:
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+    continue-on-error: true
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install pnpm and npmci
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @ship.zone/npmci
+          npmci npm prepare
+
+      - name: Audit production dependencies
+        run: |
+          npmci command npm config set registry https://registry.npmjs.org
+          npmci command pnpm audit --audit-level=high --prod
+        continue-on-error: true
+
+      - name: Audit development dependencies
+        run: |
+          npmci command npm config set registry https://registry.npmjs.org
+          npmci command pnpm audit --audit-level=high --dev
+        continue-on-error: true
+
+  test:
+    needs: security
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Prepare
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @ship.zone/npmci
+          npmci npm prepare
+
+      - name: Test stable
+        run: |
+          npmci node install stable
+          npmci npm install
+          npmci npm test
+
+      - name: Test build
+        run: |
+          npmci npm prepare
+          npmci node install stable
+          npmci npm install
+          npmci command npm run build

.gitea/workflows/docker_tags.yaml

@@ -0,0 +1,106 @@
+name: Docker (tags)
+
+on:
+  push:
+    tags:
+      - '*'
+
+env:
+  IMAGE: code.foss.global/hosttoday/ht-docker-node:npmci
+  NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@/${{gitea.repository}}.git
+  NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
+  NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
+  NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
+  NPMCI_LOGIN_DOCKER_GITEA: ${{ github.server_url }}|${{ gitea.repository_owner }}|${{ secrets.GITEA_TOKEN }}
+  NPMCI_LOGIN_DOCKER_DOCKERREGISTRY: ${{ secrets.NPMCI_LOGIN_DOCKER_DOCKERREGISTRY }}
+
+jobs:
+  security:
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+    continue-on-error: true
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Prepare
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @ship.zone/npmci
+          npmci npm prepare
+
+      - name: Audit production dependencies
+        run: |
+          npmci command npm config set registry https://registry.npmjs.org
+          npmci command pnpm audit --audit-level=high --prod
+        continue-on-error: true
+
+      - name: Audit development dependencies
+        run: |
+          npmci command npm config set registry https://registry.npmjs.org
+          npmci command pnpm audit --audit-level=high --dev
+        continue-on-error: true
+
+  test:
+    needs: security
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Prepare
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @ship.zone/npmci
+          npmci npm prepare
+
+      - name: Test stable
+        run: |
+          npmci node install stable
+          npmci npm install
+          npmci npm test
+
+      - name: Test build
+        run: |
+          npmci node install stable
+          npmci npm install
+          npmci command npm run build
+
+  release:
+    needs: test
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-latest
+    container:
+      image: code.foss.global/hosttoday/ht-docker-dbase:npmci
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Prepare
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @ship.zone/npmci
+
+      - name: Release
+        run: |
+          npmci docker login
+          npmci docker build
+          npmci docker test
+          # npmci docker push 
+          npmci docker push 
+
+  metadata:
+    needs: test
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Trigger
+        run: npmci trigger

.gitignore

@@ -3,7 +3,6 @@
 # artifacts
 coverage/
 public/
-pages/
 
 # installs
 node_modules/
@@ -17,4 +16,4 @@ node_modules/
 dist/
 dist_*/
 
-# custom
+#------# custom

Dockerfile

@@ -24,7 +24,7 @@ RUN rm -rf node_modules/ && pnpm install --prod
 
 
 ## STAGE 3 // rebuild dependencies for alpine
-FROM code.foss.global/host.today/ht-docker-node:alpine_npmci as node3
+FROM code.foss.global/host.today/ht-docker-node:alpinenpmci as node3
 WORKDIR /app
 COPY --from=node2 /app /app
 ARG NPMCI_TOKEN_NPM2

changelog.md

@@ -1,5 +1,28 @@
 # Changelog
 
+## 2024-12-29 - 1.1.0 - feat(.gitea/workflows)
+Add GitHub Actions workflows for Docker build and test
+
+- Added .gitea/workflows/docker_nottags.yaml for handling Docker builds on non-tagged commits.
+- Added .gitea/workflows/docker_tags.yaml for handling Docker builds on tagged commits.
+- Both workflows include steps for security audits, testing, and build preparation.
+
+## 2024-12-29 - 1.0.135 - fix(core)
+Fix image retrieval and service deployment process for workload services
+
+- Updated the handling of internal and external registry images when provisioning workload services.
+- Refined the image pulling logic to accommodate both internal and external sources.
+- Ensured proper secret management during service deployment.
+- Corrected and optimized key function parameters and API calls for clarity and functionality.
+
+## 2024-12-20 - 1.0.134 - fix(core)
+Updated dependencies and refactored ClusterManager and CloudlyConnector for improved Docker management.
+
+- Updated `@push.rocks/tswatch` and `@push.rocks/tapbundle` to latest versions in devDependencies.
+- Refactored `ClusterManager` to remove unused subscriptions and improve network and service management.
+- Refactored `CloudlyConnector` to use `CloudlyApiClient` for improved cloud configuration management.
+- Added logging for network and Docker service configurations.
+
 ## 2024-11-18 - 1.0.133 - fix(dockerfile)
 Update Docker base images to use code.foss.global instead of registry.gitlab.com
 

npmextra.json

@@ -13,12 +13,12 @@
   "gitzone": {
     "projectType": "service",
     "module": {
-      "githost": "gitlab.com",
+      "githost": "code.foss.global",
-      "gitscope": "losslessone/services/servezone",
+      "gitscope": "serve.zone",
       "gitrepo": "coreflow",
       "description": "A comprehensive tool for managing Docker-based applications and services, enabling efficient scaling, network management, and integration with cloud services.",
       "npmPackagename": "@serve.zone/coreflow",
-      "license": "UNLICENSED",
+      "license": "MIT",
       "keywords": [
         "Docker",
         "Service scaling",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@serve.zone/coreflow",
-  "version": "1.0.133",
+  "version": "1.1.0",
   "description": "A comprehensive tool for managing Docker-based applications and services, enabling efficient scaling, network management, and integration with cloud services.",
   "main": "dist_ts/index.js",
   "typings": "dist_ts/index.d.ts",
@@ -9,6 +9,7 @@
     "test": "(tstest test/ --web)",
     "start": "(node --max_old_space_size=500 ./cli.js)",
     "startTs": "(node cli.ts.js)",
+    "watch": "(tswatch service)",
     "build": "(tsbuild --web --allowimplicitany)",
     "buildDocs": "tsdoc"
   },
@@ -17,7 +18,7 @@
   },
   "repository": {
     "type": "git",
-    "url": "git+https://gitlab.com/pushrocks/coreflow.git"
+    "url": "https://code.foss.global/serve.zone/coreflow.git"
   },
   "keywords": [
     "Docker",
@@ -48,24 +49,24 @@
   "author": "Lossless GmbH",
   "license": "MIT",
   "bugs": {
-    "url": "https://gitlab.com/pushrocks/coreflow/issues"
+    "url": "https://gitlab.com/losslessone/services/servezone/coreflow/issues"
   },
-  "homepage": "https://gitlab.com/pushrocks/coreflow#readme",
+  "homepage": "https://gitlab.com/losslessone/services/servezone/coreflow#readme",
   "devDependencies": {
     "@git.zone/tsbuild": "^2.2.0",
     "@git.zone/tsrun": "^1.3.3",
     "@git.zone/tstest": "^1.0.90",
-    "@git.zone/tswatch": "^2.0.25",
+    "@git.zone/tswatch": "^2.0.37",
-    "@push.rocks/tapbundle": "^5.5.0"
+    "@push.rocks/tapbundle": "^5.5.3"
   },
   "dependencies": {
     "@api.global/typedrequest": "^3.1.10",
     "@api.global/typedsocket": "^3.0.1",
-    "@apiclient.xyz/docker": "^1.2.7",
+    "@apiclient.xyz/docker": "^1.3.0",
     "@push.rocks/early": "^4.0.3",
     "@push.rocks/lik": "^6.1.0",
     "@push.rocks/projectinfo": "^5.0.1",
-    "@push.rocks/qenv": "^6.0.5",
+    "@push.rocks/qenv": "^6.1.0",
     "@push.rocks/smartcli": "^4.0.11",
     "@push.rocks/smartdelay": "^3.0.1",
     "@push.rocks/smartlog": "^3.0.7",
@@ -75,12 +76,13 @@
     "@push.rocks/smartrequest": "^2.0.23",
     "@push.rocks/smartrx": "^3.0.2",
     "@push.rocks/smartstate": "^2.0.19",
+    "@push.rocks/smartstream": "^3.2.5",
     "@push.rocks/smartstring": "^4.0.15",
     "@push.rocks/taskbuffer": "^3.0.10",
-    "@serve.zone/api": "^4.3.11",
+    "@serve.zone/api": "^4.10.0",
-    "@serve.zone/interfaces": "^4.3.11",
+    "@serve.zone/interfaces": "^4.10.0",
-    "@tsclass/tsclass": "^4.1.2",
+    "@tsclass/tsclass": "^4.2.0",
-    "@types/node": "22.9.0"
+    "@types/node": "22.10.2"
   },
   "private": true,
   "files": [

readme.md

@@ -1,4 +1,5 @@
 # @serve.zone/coreflow
+
 A comprehensive solution for managing Docker and scaling applications across servers, handling tasks from service provisioning to network traffic management.
 
 ## Install
@@ -42,6 +43,7 @@ await coreflowInstance.stop();
 ```
 
 In the above example:
+
 - The Coreflow instance is initialized.
 - Coreflow is started, which internally initializes various managers and connectors.
 - The method `handleDockerEvents` is used to handle Docker events.
@@ -54,8 +56,8 @@ Coreflow manages applications and services, often requiring direct interactions
 ```typescript
 // Assuming coreflowInstance is already started as per previous examples
 const serviceConnection = coreflowInstance.createServiceConnection({
-  serviceName: "myDatabaseService",
+  serviceName: 'myDatabaseService',
-  servicePort: 3306
+  servicePort: 3306,
 });
 
 serviceConnection.connect().then(() => {
@@ -69,7 +71,7 @@ Coreflow excels in scaling applications across multiple servers. This involves n
 
 ```typescript
 const scalingPolicy = {
-  serviceName: "apiService",
+  serviceName: 'apiService',
   replicaCount: 5, // Target number of replicas
   maxReplicaCount: 10, // Maximum number of replicas
   minReplicaCount: 2, // Minimum number of replicas
@@ -81,6 +83,7 @@ coreflowInstance.applyScalingPolicy(scalingPolicy).then(() => {
 ```
 
 In the above example:
+
 - A scaling policy is defined with target, maximum, and minimum replica counts for the `apiService`.
 - The `applyScalingPolicy` method of the Coreflow instance is used to apply this scaling policy.
 
@@ -92,10 +95,10 @@ One of Coreflow's key features is its ability to manage network traffic, ensurin
 import { TrafficRule } from '@serve.zone/coreflow';
 
 const rule: TrafficRule = {
-  serviceName: "webService",
+  serviceName: 'webService',
   externalPort: 80,
   internalPort: 3000,
-  protocol: "http",
+  protocol: 'http',
 };
 
 coreflowInstance.applyTrafficRule(rule).then(() => {
@@ -104,6 +107,7 @@ coreflowInstance.applyTrafficRule(rule).then(() => {
 ```
 
 In the above example:
+
 - A traffic rule is defined for the `webService`, redirecting external traffic from port 80 to the service's internal port 3000.
 - The `applyTrafficRule` method is used to enforce this rule.
 
@@ -113,9 +117,9 @@ Coreflow integrates continuous integration and deployment processes, allowing se
 
 ```typescript
 const deploymentConfig = {
-  serviceName: "userAuthService",
+  serviceName: 'userAuthService',
-  image: "myregistry.com/userauthservice:latest",
+  image: 'myregistry.com/userauthservice:latest',
-  updatePolicy: "rolling" // or "recreate"
+  updatePolicy: 'rolling', // or "recreate"
 };
 
 coreflowInstance.deployService(deploymentConfig).then(() => {
@@ -124,6 +128,7 @@ coreflowInstance.deployService(deploymentConfig).then(() => {
 ```
 
 In the above example:
+
 - A deployment configuration is created for the `userAuthService` using the latest image from the specified registry.
 - The `deployService` method is then used to deploy the service using the specified update policy (e.g., rolling updates or recreating the service).
 
@@ -132,12 +137,13 @@ In the above example:
 To keep track of your applications' health and performance, Coreflow provides tools for logging, monitoring, and alerting.
 
 ```typescript
-coreflowInstance.monitorService("webService").on('serviceHealthUpdate', (healthStatus) => {
+coreflowInstance.monitorService('webService').on('serviceHealthUpdate', (healthStatus) => {
   console.log(`Received health update for webService: ${healthStatus}`);
 });
 ```
 
 In the above example:
+
 - The `monitorService` method is used to monitor the health status of the `webService`.
 - When a health update event is received, it is logged to the console.
 
@@ -175,8 +181,8 @@ coreflowInstance.handleDockerEvents().then(() => {
 
 ```typescript
 const serviceConnection = coreflowInstance.createServiceConnection({
-  serviceName: "databaseService",
+  serviceName: 'databaseService',
-  servicePort: 5432
+  servicePort: 5432,
 });
 
 serviceConnection.connect().then(() => {
@@ -188,7 +194,7 @@ serviceConnection.connect().then(() => {
 
 ```typescript
 const scalingPolicy = {
-  serviceName: "microserviceA",
+  serviceName: 'microserviceA',
   replicaCount: 3, // Starting with 3 replicas
   maxReplicaCount: 10, // Allowing up to 10 replicas
   minReplicaCount: 2, // Ensuring at least 2 replicas
@@ -206,17 +212,17 @@ import { TrafficRule } from '@serve.zone/coreflow';
 
 const trafficRules: TrafficRule[] = [
   {
-    serviceName: "frontendService",
+    serviceName: 'frontendService',
     externalPort: 80,
     internalPort: 3000,
-    protocol: "http",
+    protocol: 'http',
   },
   {
-    serviceName: "apiService",
+    serviceName: 'apiService',
     externalPort: 443,
     internalPort: 4000,
-    protocol: "https",
+    protocol: 'https',
-  }
+  },
 ];
 
 Promise.all(trafficRules.map((rule) => coreflowInstance.applyTrafficRule(rule))).then(() => {
@@ -228,9 +234,9 @@ Promise.all(trafficRules.map((rule) => coreflowInstance.applyTrafficRule(rule)))
 
 ```typescript
 const deploymentConfig = {
-  serviceName: "authService",
+  serviceName: 'authService',
-  image: "myregistry.com/authservice:latest",
+  image: 'myregistry.com/authservice:latest',
-  updatePolicy: "rolling", // Performing rolling updates
+  updatePolicy: 'rolling', // Performing rolling updates
 };
 
 coreflowInstance.deployService(deploymentConfig).then(() => {
@@ -241,7 +247,7 @@ coreflowInstance.deployService(deploymentConfig).then(() => {
 #### Step 7: Monitoring a Service
 
 ```typescript
-coreflowInstance.monitorService("frontendService").on('serviceHealthUpdate', (healthStatus) => {
+coreflowInstance.monitorService('frontendService').on('serviceHealthUpdate', (healthStatus) => {
   console.log(`Health update for frontendService: ${healthStatus}`);
 });
 ```
@@ -262,7 +268,7 @@ const checkinTask = new Task({
   buffered: true,
   taskFunction: async () => {
     console.log('Running checkin task...');
-  }
+  },
 });
 
 const taskManager = coreflowInstance.taskManager;
@@ -284,18 +290,18 @@ const coretrafficConnector = new CoretrafficConnector(coreflowInstance);
 
 const reverseProxyConfigs = [
   {
-    hostName: "example.com",
+    hostName: 'example.com',
-    destinationIp: "192.168.1.100",
+    destinationIp: '192.168.1.100',
-    destinationPort: "3000",
+    destinationPort: '3000',
-    privateKey: "<your-private-key>",
+    privateKey: '<your-private-key>',
-    publicKey: "<your-public-key>",
+    publicKey: '<your-public-key>',
   },
   {
-    hostName: "api.example.com",
+    hostName: 'api.example.com',
-    destinationIp: "192.168.1.101",
+    destinationIp: '192.168.1.101',
-    destinationPort: "4000",
+    destinationPort: '4000',
-    privateKey: "<your-private-key>",
+    privateKey: '<your-private-key>',
-    publicKey: "<your-public-key>",
+    publicKey: '<your-public-key>',
   },
 ];
 

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@serve.zone/coreflow',
-  version: '1.0.133',
+  version: '1.1.0',
   description: 'A comprehensive tool for managing Docker-based applications and services, enabling efficient scaling, network management, and integration with cloud services.'
 }

ts/coreflow.classes.clustermanager.ts

@@ -4,7 +4,6 @@ import { Coreflow } from './coreflow.classes.coreflow.js';
 
 export class ClusterManager {
   public coreflowRef: Coreflow;
-  public dockerHost: plugins.docker.DockerHost;
   public configSubscription: plugins.smartrx.rxjs.Subscription;
   public containerSubscription: plugins.smartrx.rxjs.Subscription;
   public containerVersionSubscription: plugins.smartrx.rxjs.Subscription;
@@ -20,7 +19,6 @@ export class ClusterManager {
 
   constructor(coreflowRefArg: Coreflow) {
     this.coreflowRef = coreflowRefArg;
-    this.dockerHost = new plugins.docker.DockerHost();
   }
 
   /**
@@ -31,26 +29,12 @@ export class ClusterManager {
     this.readyDeferred.resolve();
 
     // subscriptions
+    // this subscription is the start point for most updates on the cluster
     this.configSubscription =
-      this.coreflowRef.cloudlyConnector.cloudlyClient.configUpdateSubject.subscribe(
+      this.coreflowRef.cloudlyConnector.cloudlyApiClient.configUpdateSubject.subscribe(
         async (dataArg) => {
           this.coreflowRef.taskManager.updateBaseServicesTask.trigger();
-        }
+        },
-      );
-    this.containerSubscription =
-      this.coreflowRef.cloudlyConnector.cloudlyClient.containerUpdateSubject.subscribe(
-        async (dataArg) => {
-          this.coreflowRef.taskManager.updateBaseServicesTask.trigger();
-        }
-      );
-    this.containerVersionSubscription =
-      this.coreflowRef.cloudlyConnector.cloudlyClient.containerVersionUpdateSubject.subscribe(
-        async (dataArg) => {
-          console.log(
-            `Got a container version update trigger for ${dataArg.dockerImageUrl}@${dataArg.dockerImageVersion}`
-          );
-          this.coreflowRef.taskManager.updateBaseServicesTask.trigger();
-        }
       );
   }
 
@@ -67,29 +51,42 @@ export class ClusterManager {
   public async provisionBaseServices() {
     // swarm should be enabled by lower level serverconfig package
     // get current situation
-    const networks = await this.dockerHost.getNetworks();
+    const networks = await this.coreflowRef.dockerHost.getNetworks();
+    logger.log('info', 'There are currently ' + networks.length + ' networks');
+    for (const network of networks) {
+      logger.log('info', 'Network: ' + network.Name);
+    }
+
+    // make sure there is a network for the webgateway
     let sznWebgatewayNetwork = await plugins.docker.DockerNetwork.getNetworkByName(
-      this.dockerHost,
+      this.coreflowRef.dockerHost,
-      this.commonDockerData.networkNames.sznWebgateway
+      this.commonDockerData.networkNames.sznWebgateway,
     );
 
     if (!sznWebgatewayNetwork) {
-      sznWebgatewayNetwork = await plugins.docker.DockerNetwork.createNetwork(this.dockerHost, {
+      logger.log('info', 'Creating network: ' + this.commonDockerData.networkNames.sznWebgateway);
+      sznWebgatewayNetwork = await plugins.docker.DockerNetwork.createNetwork(
+        this.coreflowRef.dockerHost,
+        {
           Name: this.commonDockerData.networkNames.sznWebgateway,
-      });
+        },
+      );
     } else {
       logger.log('ok', 'sznWebgateway is already present');
     }
 
-    // corechat network
+    // corechat network so base services can talk to each other
     let sznCorechatNetwork = await plugins.docker.DockerNetwork.getNetworkByName(
-      this.dockerHost,
+      this.coreflowRef.dockerHost,
-      this.commonDockerData.networkNames.sznCorechat
+      this.commonDockerData.networkNames.sznCorechat,
     );
     if (!sznCorechatNetwork) {
-      sznCorechatNetwork = await plugins.docker.DockerNetwork.createNetwork(this.dockerHost, {
+      sznCorechatNetwork = await plugins.docker.DockerNetwork.createNetwork(
+        this.coreflowRef.dockerHost,
+        {
           Name: this.commonDockerData.networkNames.sznCorechat,
-      });
+        },
+      );
     } else {
       logger.log('ok', 'sznCorechat is already present');
     }
@@ -102,21 +99,31 @@ export class ClusterManager {
     // Images
     logger.log('info', `now updating docker images of base services...`);
 
-    const coretrafficImage = await plugins.docker.DockerImage.createFromRegistry(this.dockerHost, {
+    const coretrafficImage = await plugins.docker.DockerImage.createFromRegistry(
-      imageUrl: 'registry.gitlab.com/losslessone/services/servezone/coretraffic',
+      this.coreflowRef.dockerHost,
-    });
+      {
+        creationObject: {
+          imageUrl: 'code.foss.global/serve.zone/coretraffic',
+        },
+      },
+    );
 
-    const corelogImage = await plugins.docker.DockerImage.createFromRegistry(this.dockerHost, {
+    const corelogImage = await plugins.docker.DockerImage.createFromRegistry(
-      imageUrl: 'registry.gitlab.com/losslessone/services/servezone/corelog',
+      this.coreflowRef.dockerHost,
-    });
+      {
+        creationObject: {
+          imageUrl: 'code.foss.global/serve.zone/corelog',
+        },
+      },
+    );
 
     // SERVICES
     // lets deploy the base services
     // coretraffic
     let coretrafficService: plugins.docker.DockerService;
     coretrafficService = await plugins.docker.DockerService.getServiceByName(
-      this.dockerHost,
+      this.coreflowRef.dockerHost,
-      'coretraffic'
+      'coretraffic',
     );
 
     if (coretrafficService && (await coretrafficService.needsUpdate())) {
@@ -128,7 +135,9 @@ export class ClusterManager {
     }
 
     if (!coretrafficService) {
-      coretrafficService = await plugins.docker.DockerService.createService(this.dockerHost, {
+      coretrafficService = await plugins.docker.DockerService.createService(
+        this.coreflowRef.dockerHost,
+        {
           image: coretrafficImage,
           labels: {},
           name: 'coretraffic',
@@ -140,7 +149,8 @@ export class ClusterManager {
             memorySizeMB: 1100,
             volumeMounts: [],
           },
-      });
+        },
+      );
     } else {
       logger.log('ok', 'coretraffic service is already present');
     }
@@ -150,8 +160,8 @@ export class ClusterManager {
     // corelog
     let corelogService: plugins.docker.DockerService;
     corelogService = await plugins.docker.DockerService.getServiceByName(
-      this.dockerHost,
+      this.coreflowRef.dockerHost,
-      'corelog'
+      'corelog',
     );
 
     if (corelogService && (await corelogService.needsUpdate())) {
@@ -162,7 +172,9 @@ export class ClusterManager {
     }
 
     if (!corelogService) {
-      corelogService = await plugins.docker.DockerService.createService(this.dockerHost, {
+      corelogService = await plugins.docker.DockerService.createService(
+        this.coreflowRef.dockerHost,
+        {
           image: corelogImage,
           labels: {},
           name: 'corelog',
@@ -174,7 +186,8 @@ export class ClusterManager {
             memorySizeMB: 120,
             volumeMounts: [],
           },
-      });
+        },
+      );
     } else {
       logger.log('ok', 'corelog service is already present');
     }
@@ -182,41 +195,86 @@ export class ClusterManager {
     await plugins.smartdelay.delayFor(10000);
   }
 
-  /**
+  public async provisionWorkloadService(
-   * provision services obtained from cloudly
+    serviceArgFromCloudly: plugins.servezoneInterfaces.data.IService,
-   */
-  public async provisionWorkloadServices(configData: plugins.servezoneInterfaces.data.ICluster) {
-    // lets get the config + deploymentDirectives
-    
-
-    
-    for (const containerConfig of configData.data.containers) {
-      await this.provisionSpecificWorkloadService(containerConfig);
-    }
-    logger.log('ok', 'Waiting for scheduled workload services to settle');
-    await plugins.smartdelay.delayFor(10000);
-  }
-
-  public async provisionSpecificWorkloadService(
-    containerConfigArg: plugins.servezoneInterfaces.data.IClusterConfigContainer
   ) {
-    const containerImage = await plugins.docker.DockerImage.createFromRegistry(this.dockerHost, {
+    logger.log(
-      imageUrl: containerConfigArg.image,
+      'info',
+      `deploying service ${serviceArgFromCloudly.data.name}@${serviceArgFromCloudly.data.imageVersion}...`,
+    );
+
+    // get the image from cloudly
+    logger.log(
+      'info',
+      `getting image for ${serviceArgFromCloudly.data.name}@${serviceArgFromCloudly.data.imageVersion}`,
+    );
+    const containerImageFromCloudly =
+      await this.coreflowRef.cloudlyConnector.cloudlyApiClient.image.getImageById(
+        serviceArgFromCloudly.data.imageId,
+      );
+    let localDockerImage: plugins.docker.DockerImage;
+
+    // lets get the docker image for the service
+    if (containerImageFromCloudly.data.location.internal) {
+      const imageStream = await containerImageFromCloudly.pullImageVersion(
+        serviceArgFromCloudly.data.imageVersion,
+      );
+      localDockerImage = await plugins.docker.DockerImage.createFromTarStream(
+        this.coreflowRef.dockerHost,
+        {
+          creationObject: {
+            imageUrl: containerImageFromCloudly.id,
+            imageTag: serviceArgFromCloudly.data.imageVersion,
+          },
+          tarStream:
+            plugins.smartstream.nodewebhelpers.convertWebReadableToNodeReadable(imageStream),
+        },
+      );
+    } else if (
+      containerImageFromCloudly.data.location.externalRegistryId &&
+      containerImageFromCloudly.data.location.externalImageTag
+    ) {
+      const externalRegistry =
+        await this.coreflowRef.cloudlyConnector.cloudlyApiClient.externalRegistry.getRegistryById(
+          containerImageFromCloudly.data.location.externalRegistryId,
+        );
+      // Lets authenticate against the external registry
+      // TODO: deduplicate this, check wether we are already authenticated
+      await this.coreflowRef.dockerHost.auth({
+        username: externalRegistry.data.username,
+        password: externalRegistry.data.password,
+        serveraddress: externalRegistry.data.url,
       });
+      localDockerImage = await plugins.docker.DockerImage.createFromRegistry(
+        this.coreflowRef.dockerHost,
+        {
+          creationObject: {
+            imageUrl: containerImageFromCloudly.id,
+            imageTag: serviceArgFromCloudly.data.imageVersion,
+          },
+        },
+      );
+      await localDockerImage.pullLatestImageFromRegistry();
+    } else {
+      throw new Error('Invalid image location');
+    }
 
     let containerService = await plugins.docker.DockerService.getServiceByName(
-      this.dockerHost,
+      this.coreflowRef.dockerHost,
-      containerConfigArg.name
+      serviceArgFromCloudly.data.name,
     );
+    this.coreflowRef.cloudlyConnector.cloudlyApiClient;
+
+    const dockerSecretName = `${serviceArgFromCloudly.id}_${serviceArgFromCloudly.data.name}_Secret`;
     let containerSecret = await plugins.docker.DockerSecret.getSecretByName(
-      this.dockerHost,
+      this.coreflowRef.dockerHost,
-      `${containerConfigArg.name}Secret`
+      dockerSecretName,
     );
 
     // existing network to connect to
     const webGatewayNetwork = await plugins.docker.DockerNetwork.getNetworkByName(
-      this.dockerHost,
+      this.coreflowRef.dockerHost,
-      this.commonDockerData.networkNames.sznWebgateway
+      this.commonDockerData.networkNames.sznWebgateway,
     );
 
     if (containerService && (await containerService.needsUpdate())) {
@@ -230,39 +288,56 @@ export class ClusterManager {
 
     if (!containerService) {
       containerSecret = await plugins.docker.DockerSecret.getSecretByName(
-        this.dockerHost,
+        this.coreflowRef.dockerHost,
-        `${containerConfigArg.name}Secret`
+        dockerSecretName,
       );
       if (containerSecret) {
         await containerSecret.remove();
       }
-      containerSecret = await plugins.docker.DockerSecret.createSecret(this.dockerHost, {
+
-        name: `${containerConfigArg.name}Secret`,
+      const secretBundle =
-        contentArg: JSON.stringify(containerConfigArg.secrets),
+        await this.coreflowRef.cloudlyConnector.cloudlyApiClient.secretbundle.getSecretBundleById(
+          serviceArgFromCloudly.data.secretBundleId,
+        );
+
+      // lets create the relevant stuff on the docker side
+      containerSecret = await plugins.docker.DockerSecret.createSecret(
+        this.coreflowRef.dockerHost,
+        {
+          name: dockerSecretName,
+          contentArg: JSON.stringify(await secretBundle.getFlatKeyValueObjectForEnvironment()),
           labels: {},
-        version: await containerImage.getVersion(),
+          version:
-      });
+            await containerImageFromCloudly.data.versions[serviceArgFromCloudly.data.imageVersion],
-      containerService = await plugins.docker.DockerService.createService(this.dockerHost, {
+        },
-        name: containerConfigArg.name,
+      );
-        image: containerImage,
+      containerService = await plugins.docker.DockerService.createService(
+        this.coreflowRef.dockerHost,
+        {
+          name: serviceArgFromCloudly.data.name,
+          image: localDockerImage,
           networks: [webGatewayNetwork],
           secrets: [containerSecret],
           ports: [],
           labels: {},
-        resources: containerConfigArg.resources,
+          resources: serviceArgFromCloudly.data.resources,
-        networkAlias: containerConfigArg.name,
+          // TODO: introduce a clean name here, that is guaranteed to work with APIs.
-      });
+          networkAlias: serviceArgFromCloudly.data.name,
+        },
+      );
     }
   }
 
   /**
    * update traffic routing
    */
-  public async updateTrafficRouting(clusterConfigArg: plugins.servezoneInterfaces.data.IClusterConfig) {
+  public async updateTrafficRouting(
-    const services = await this.dockerHost.getServices();
+    clusterConfigArg: plugins.servezoneInterfaces.data.IClusterConfig,
+  ) {
+    const services = await this.coreflowRef.dockerHost.getServices();
     const webGatewayNetwork = await plugins.docker.DockerNetwork.getNetworkByName(
-      this.dockerHost,
+      this.coreflowRef.dockerHost,
-      this.commonDockerData.networkNames.sznWebgateway
+      this.commonDockerData.networkNames.sznWebgateway,
     );
     const reverseProxyConfigs: plugins.servezoneInterfaces.data.IReverseProxyConfig[] = [];
 
@@ -270,7 +345,7 @@ export class ClusterManager {
       serviceNameArg: string,
       hostNameArg: string,
       containerDestinationIp: string,
-      webDestinationPort: string
+      webDestinationPort: string,
     ) => {
       logger.log('ok', `trying to obtain a certificate for ${hostNameArg}`);
       const certificate =
@@ -284,7 +359,7 @@ export class ClusterManager {
       });
       logger.log(
         'success',
-        `pushed routing config for ${hostNameArg} on workload service ${serviceNameArg}`
+        `pushed routing config for ${hostNameArg} on workload service ${serviceNameArg}`,
       );
     };
 
@@ -306,7 +381,7 @@ export class ClusterManager {
         if (!containersOfServicesOnNetwork[0]) {
           logger.log(
             'error',
-            `There seems to be no container available for service ${service.Spec.Name}`
+            `There seems to be no container available for service ${service.Spec.Name}`,
           );
           continue;
         }
@@ -320,7 +395,7 @@ export class ClusterManager {
             workloadConfig.name,
             hostName,
             containerDestinationIp,
-            webDestinationPort
+            webDestinationPort,
           );
         }
 
@@ -332,14 +407,14 @@ export class ClusterManager {
               workloadConfig.name,
               customDomainKey,
               containerDestinationIp,
-              workloadConfig.ports.custom[customDomainKey]
+              workloadConfig.ports.custom[customDomainKey],
             );
           }
         }
       } else {
         logger.log(
           'ok',
-          `service ${service.Spec.Name} is not a workload service and won't receive traffic`
+          `service ${service.Spec.Name} is not a workload service and won't receive traffic`,
         );
       }
     }

ts/coreflow.classes.coreflow.ts

@@ -22,7 +22,7 @@ export class Coreflow {
 
   constructor() {
     this.serviceQenv = new plugins.qenv.Qenv('./', './.nogit');
-    this.dockerHost = new plugins.docker.DockerHost(); // defaults to locally mounted docker sock
+    this.dockerHost = new plugins.docker.DockerHost({}); // defaults to locally mounted docker sock
     this.internalServer = new InternalServer(this);
     this.cloudlyConnector = new CloudlyConnector(this);
     this.corechatConnector = new CoretrafficConnector(this);

ts/coreflow.classes.internalserver.ts

@@ -11,7 +11,7 @@ export class InternalServer {
 
   public async start() {
     this.typedsocketServer = await plugins.typedsocket.TypedSocket.createServer(
-      this.coreflowRef.typedrouter
+      this.coreflowRef.typedrouter,
     );
   }
 

ts/coreflow.classes.taskmanager.ts

@@ -77,7 +77,7 @@ export class CoreflowTaskmanager {
         await this.updateBaseServicesTask.trigger();
         logger.log(
           'success',
-          'initial tasks successfully executed! Now handing over to longterm taskmanager!'
+          'initial tasks successfully executed! Now handing over to longterm taskmanager!',
         );
       } catch (e) {
         console.log(e);

ts/coreflow.connector.cloudlyconnector.ts

@@ -7,47 +7,46 @@ import { Coreflow } from './coreflow.classes.coreflow.js';
 
 export class CloudlyConnector {
   public coreflowRef: Coreflow;
-  public cloudlyClient: plugins.servezoneApi.CloudlyClient;
+  public cloudlyApiClient: plugins.servezoneApi.CloudlyApiClient;
   public coreflowJumpCode: string;
-  public identity: plugins.servezoneInterfaces.data.IClusterIdentifier;
+  public identity: plugins.servezoneInterfaces.data.IIdentity;
 
   constructor(coreflowRefArg: Coreflow) {
     this.coreflowRef = coreflowRefArg;
   }
 
   public async start() {
-    this.cloudlyClient = new plugins.servezoneApi.CloudlyClient('coreflow');
+    this.cloudlyApiClient = new plugins.servezoneApi.CloudlyApiClient({
-    await this.cloudlyClient.start();
+      registerAs: 'coreflow',
+      cloudlyUrl: await this.coreflowRef.serviceQenv.getEnvVarOnDemand('CLOUDLY_URL'),
+    });
+    await this.cloudlyApiClient.start();
     this.coreflowJumpCode = await this.coreflowRef.serviceQenv.getEnvVarOnDemand('JUMPCODE');
 
     // get identity and tag connection (second parameter is true -> tags the connection)
-    this.identity = await this.cloudlyClient.getIdentityByJumpCode(this.coreflowJumpCode, true);
+    this.identity = await this.cloudlyApiClient.getIdentityByToken(this.coreflowJumpCode, {
-  }
+      statefullIdentity: true,
-
+      tagConnection: true,
-  public async stop() {
-    await this.cloudlyClient.stop();
-  }
-
-  public async getConfigFromCloudly(): Promise<plugins.servezoneInterfaces.data.IClusterConfig> {
-    const config = await this.cloudlyClient.getClusterConfigFromCloudlyByIdentity(
-      this.identity
-    );
-    return config;
-  }
-
-  public async triggerConfigEvent() {
-    const config = await this.getConfigFromCloudly();
-    this.cloudlyClient.configUpdateSubject.next({
-      configData: config,
     });
   }
 
+  public async stop() {
+    await this.cloudlyApiClient.stop();
+  }
+
+  public async getConfigFromCloudly(): Promise<plugins.servezoneInterfaces.data.ICluster> {
+    const config = await this.cloudlyApiClient.getClusterConfigFromCloudlyByIdentity(this.identity);
+    return config;
+  }
+
   public async getCertificateForDomainFromCloudly(
-    domainNameArg: string
+    domainNameArg: string,
   ): Promise<plugins.tsclass.network.ICert> {
-    const certificate = await this.cloudlyClient.getCertificateForDomainOverHttps(
+    const certificate = await this.cloudlyApiClient.getCertificateForDomain({
-      domainNameArg
+      identity: this.identity,
-    );
+      domainName: domainNameArg,
+      type: 'ssl',
+    });
     return certificate;
   }
 }

ts/coreflow.connector.coretrafficconnector.ts

@@ -16,15 +16,15 @@ export class CoretrafficConnector {
   }
 
   public async setReverseConfigs(
-    reverseConfigsArg: plugins.servezoneInterfaces.data.IReverseProxyConfig[]
+    reverseConfigsArg: plugins.servezoneInterfaces.data.IReverseProxyConfig[],
   ) {
     await this.start();
     const reactionRequest =
       this.coreflowRef.internalServer.typedsocketServer.createTypedRequest<plugins.servezoneInterfaces.requests.routing.IRequest_Coreflow_Coretraffic_RoutingUpdate>(
         'updateRouting',
         await this.coreflowRef.internalServer.typedsocketServer.findTargetConnection(
-          async (targetConnection) => targetConnection.alias === 'coretraffic'
+          async (targetConnection) => targetConnection.alias === 'coretraffic',
-        )
+        ),
       );
     const response = await reactionRequest.fire({
       reverseConfigs: reverseConfigsArg,

ts/coreflow.paths.ts

@@ -2,5 +2,5 @@ import * as plugins from './coreflow.plugins.js';
 
 export const packageDir = plugins.path.join(
   plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url),
-  '../'
+  '../',
 );

ts/coreflow.plugins.ts

@@ -28,6 +28,7 @@ import * as smartpromise from '@push.rocks/smartpromise';
 import * as smartrequest from '@push.rocks/smartrequest';
 import * as smartrx from '@push.rocks/smartrx';
 import * as smartstate from '@push.rocks/smartstate';
+import * as smartstream from '@push.rocks/smartstream';
 import * as smartstring from '@push.rocks/smartstring';
 import * as taskbuffer from '@push.rocks/taskbuffer';
 
@@ -44,6 +45,7 @@ export {
   smartrequest,
   smartrx,
   smartstate,
+  smartstream,
   smartstring,
   taskbuffer,
 };

ts/index.ts

@@ -18,7 +18,7 @@ export const runCli = async () => {
     'info',
     `trying to start coreflow@v${projectInfoNpm.version} on ${
       (await smartnetworkInstance.getPublicIps()).v4
-    }`
+    }`,
   );
   coreflowInstance = new Coreflow();
   await coreflowInstance.start();

tsconfig.json

@@ -6,7 +6,9 @@
     "module": "NodeNext",
     "moduleResolution": "NodeNext",
     "esModuleInterop": true,
-    "verbatimModuleSyntax": true
+    "verbatimModuleSyntax": true,
+    "baseUrl": ".",
+    "paths": {}
   },
   "exclude": [
     "dist_*/**/*.d.ts"