1.0.135

changelog.md

@@ -1,5 +1,21 @@
 # Changelog
 
+## 2024-12-29 - 1.0.135 - fix(core)
+Fix image retrieval and service deployment process for workload services
+
+- Updated the handling of internal and external registry images when provisioning workload services.
+- Refined the image pulling logic to accommodate both internal and external sources.
+- Ensured proper secret management during service deployment.
+- Corrected and optimized key function parameters and API calls for clarity and functionality.
+
+## 2024-12-20 - 1.0.134 - fix(core)
+Updated dependencies and refactored ClusterManager and CloudlyConnector for improved Docker management.
+
+- Updated `@push.rocks/tswatch` and `@push.rocks/tapbundle` to latest versions in devDependencies.
+- Refactored `ClusterManager` to remove unused subscriptions and improve network and service management.
+- Refactored `CloudlyConnector` to use `CloudlyApiClient` for improved cloud configuration management.
+- Added logging for network and Docker service configurations.
+
 ## 2024-11-18 - 1.0.133 - fix(dockerfile)
 Update Docker base images to use code.foss.global instead of registry.gitlab.com
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@serve.zone/coreflow",
-  "version": "1.0.133",
+  "version": "1.0.135",
   "description": "A comprehensive tool for managing Docker-based applications and services, enabling efficient scaling, network management, and integration with cloud services.",
   "main": "dist_ts/index.js",
   "typings": "dist_ts/index.d.ts",
@@ -9,6 +9,7 @@
     "test": "(tstest test/ --web)",
     "start": "(node --max_old_space_size=500 ./cli.js)",
     "startTs": "(node cli.ts.js)",
+    "watch": "(tswatch service)",
     "build": "(tsbuild --web --allowimplicitany)",
     "buildDocs": "tsdoc"
   },
@@ -55,17 +56,17 @@
     "@git.zone/tsbuild": "^2.2.0",
     "@git.zone/tsrun": "^1.3.3",
     "@git.zone/tstest": "^1.0.90",
-    "@git.zone/tswatch": "^2.0.25",
-    "@push.rocks/tapbundle": "^5.5.0"
+    "@git.zone/tswatch": "^2.0.37",
+    "@push.rocks/tapbundle": "^5.5.3"
   },
   "dependencies": {
     "@api.global/typedrequest": "^3.1.10",
     "@api.global/typedsocket": "^3.0.1",
-    "@apiclient.xyz/docker": "^1.2.7",
+    "@apiclient.xyz/docker": "^1.3.0",
     "@push.rocks/early": "^4.0.3",
     "@push.rocks/lik": "^6.1.0",
     "@push.rocks/projectinfo": "^5.0.1",
-    "@push.rocks/qenv": "^6.0.5",
+    "@push.rocks/qenv": "^6.1.0",
     "@push.rocks/smartcli": "^4.0.11",
     "@push.rocks/smartdelay": "^3.0.1",
     "@push.rocks/smartlog": "^3.0.7",
@@ -75,12 +76,13 @@
     "@push.rocks/smartrequest": "^2.0.23",
     "@push.rocks/smartrx": "^3.0.2",
     "@push.rocks/smartstate": "^2.0.19",
+    "@push.rocks/smartstream": "^3.2.5",
     "@push.rocks/smartstring": "^4.0.15",
     "@push.rocks/taskbuffer": "^3.0.10",
-    "@serve.zone/api": "^4.3.11",
-    "@serve.zone/interfaces": "^4.3.11",
-    "@tsclass/tsclass": "^4.1.2",
-    "@types/node": "22.9.0"
+    "@serve.zone/api": "^4.10.0",
+    "@serve.zone/interfaces": "^4.10.0",
+    "@tsclass/tsclass": "^4.2.0",
+    "@types/node": "22.10.2"
   },
   "private": true,
   "files": [

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@serve.zone/coreflow',
-  version: '1.0.133',
+  version: '1.0.135',
   description: 'A comprehensive tool for managing Docker-based applications and services, enabling efficient scaling, network management, and integration with cloud services.'
 }

ts/coreflow.classes.clustermanager.ts

@@ -4,7 +4,6 @@ import { Coreflow } from './coreflow.classes.coreflow.js';
 
 export class ClusterManager {
   public coreflowRef: Coreflow;
-  public dockerHost: plugins.docker.DockerHost;
   public configSubscription: plugins.smartrx.rxjs.Subscription;
   public containerSubscription: plugins.smartrx.rxjs.Subscription;
   public containerVersionSubscription: plugins.smartrx.rxjs.Subscription;
@@ -20,7 +19,6 @@ export class ClusterManager {
 
   constructor(coreflowRefArg: Coreflow) {
     this.coreflowRef = coreflowRefArg;
-    this.dockerHost = new plugins.docker.DockerHost();
   }
 
   /**
@@ -31,27 +29,13 @@ export class ClusterManager {
     this.readyDeferred.resolve();
 
     // subscriptions
+    // this subscription is the start point for most updates on the cluster
     this.configSubscription =
-      this.coreflowRef.cloudlyConnector.cloudlyClient.configUpdateSubject.subscribe(
+      this.coreflowRef.cloudlyConnector.cloudlyApiClient.configUpdateSubject.subscribe(
         async (dataArg) => {
           this.coreflowRef.taskManager.updateBaseServicesTask.trigger();
         }
       );
-    this.containerSubscription =
-      this.coreflowRef.cloudlyConnector.cloudlyClient.containerUpdateSubject.subscribe(
-        async (dataArg) => {
-          this.coreflowRef.taskManager.updateBaseServicesTask.trigger();
-        }
-      );
-    this.containerVersionSubscription =
-      this.coreflowRef.cloudlyConnector.cloudlyClient.containerVersionUpdateSubject.subscribe(
-        async (dataArg) => {
-          console.log(
-            `Got a container version update trigger for ${dataArg.dockerImageUrl}@${dataArg.dockerImageVersion}`
-          );
-          this.coreflowRef.taskManager.updateBaseServicesTask.trigger();
-        }
-      );
   }
 
   /**
@@ -67,27 +51,34 @@ export class ClusterManager {
   public async provisionBaseServices() {
     // swarm should be enabled by lower level serverconfig package
     // get current situation
-    const networks = await this.dockerHost.getNetworks();
+    const networks = await this.coreflowRef.dockerHost.getNetworks();
+    logger.log('info', 'There are currently ' + networks.length + ' networks');
+    for (const network of networks) {
+      logger.log('info', 'Network: ' + network.Name);
+    }
+
+    // make sure there is a network for the webgateway
     let sznWebgatewayNetwork = await plugins.docker.DockerNetwork.getNetworkByName(
-      this.dockerHost,
+      this.coreflowRef.dockerHost,
       this.commonDockerData.networkNames.sznWebgateway
     );
 
     if (!sznWebgatewayNetwork) {
-      sznWebgatewayNetwork = await plugins.docker.DockerNetwork.createNetwork(this.dockerHost, {
+      logger.log('info', 'Creating network: ' + this.commonDockerData.networkNames.sznWebgateway);
+      sznWebgatewayNetwork = await plugins.docker.DockerNetwork.createNetwork(this.coreflowRef.dockerHost, {
         Name: this.commonDockerData.networkNames.sznWebgateway,
       });
     } else {
       logger.log('ok', 'sznWebgateway is already present');
     }
 
-    // corechat network
+    // corechat network so base services can talk to each other
     let sznCorechatNetwork = await plugins.docker.DockerNetwork.getNetworkByName(
-      this.dockerHost,
+      this.coreflowRef.dockerHost,
       this.commonDockerData.networkNames.sznCorechat
     );
     if (!sznCorechatNetwork) {
-      sznCorechatNetwork = await plugins.docker.DockerNetwork.createNetwork(this.dockerHost, {
+      sznCorechatNetwork = await plugins.docker.DockerNetwork.createNetwork(this.coreflowRef.dockerHost, {
         Name: this.commonDockerData.networkNames.sznCorechat,
       });
     } else {
@@ -102,12 +93,16 @@ export class ClusterManager {
     // Images
     logger.log('info', `now updating docker images of base services...`);
 
-    const coretrafficImage = await plugins.docker.DockerImage.createFromRegistry(this.dockerHost, {
-      imageUrl: 'registry.gitlab.com/losslessone/services/servezone/coretraffic',
+    const coretrafficImage = await plugins.docker.DockerImage.createFromRegistry(this.coreflowRef.dockerHost, {
+      creationObject: {
+        imageUrl: 'code.foss.global/serve.zone/coretraffic',
+      }
     });
 
-    const corelogImage = await plugins.docker.DockerImage.createFromRegistry(this.dockerHost, {
-      imageUrl: 'registry.gitlab.com/losslessone/services/servezone/corelog',
+    const corelogImage = await plugins.docker.DockerImage.createFromRegistry(this.coreflowRef.dockerHost, {
+      creationObject: {
+        imageUrl: 'code.foss.global/serve.zone/corelog',
+      }
     });
 
     // SERVICES
@@ -115,7 +110,7 @@ export class ClusterManager {
     // coretraffic
     let coretrafficService: plugins.docker.DockerService;
     coretrafficService = await plugins.docker.DockerService.getServiceByName(
-      this.dockerHost,
+      this.coreflowRef.dockerHost,
       'coretraffic'
     );
 
@@ -128,7 +123,7 @@ export class ClusterManager {
     }
 
     if (!coretrafficService) {
-      coretrafficService = await plugins.docker.DockerService.createService(this.dockerHost, {
+      coretrafficService = await plugins.docker.DockerService.createService(this.coreflowRef.dockerHost, {
         image: coretrafficImage,
         labels: {},
         name: 'coretraffic',
@@ -150,7 +145,7 @@ export class ClusterManager {
     // corelog
     let corelogService: plugins.docker.DockerService;
     corelogService = await plugins.docker.DockerService.getServiceByName(
-      this.dockerHost,
+      this.coreflowRef.dockerHost,
       'corelog'
     );
 
@@ -162,7 +157,7 @@ export class ClusterManager {
     }
 
     if (!corelogService) {
-      corelogService = await plugins.docker.DockerService.createService(this.dockerHost, {
+      corelogService = await plugins.docker.DockerService.createService(this.coreflowRef.dockerHost, {
         image: corelogImage,
         labels: {},
         name: 'corelog',
@@ -182,40 +177,64 @@ export class ClusterManager {
     await plugins.smartdelay.delayFor(10000);
   }
 
-  /**
-   * provision services obtained from cloudly
-   */
-  public async provisionWorkloadServices(configData: plugins.servezoneInterfaces.data.ICluster) {
-    // lets get the config + deploymentDirectives
-    
-
-    
-    for (const containerConfig of configData.data.containers) {
-      await this.provisionSpecificWorkloadService(containerConfig);
-    }
-    logger.log('ok', 'Waiting for scheduled workload services to settle');
-    await plugins.smartdelay.delayFor(10000);
-  }
-
-  public async provisionSpecificWorkloadService(
-    containerConfigArg: plugins.servezoneInterfaces.data.IClusterConfigContainer
+  public async provisionWorkloadService(
+    serviceArgFromCloudly: plugins.servezoneInterfaces.data.IService
   ) {
-    const containerImage = await plugins.docker.DockerImage.createFromRegistry(this.dockerHost, {
-      imageUrl: containerConfigArg.image,
+    logger.log('info', `deploying service ${serviceArgFromCloudly.data.name}@${serviceArgFromCloudly.data.imageVersion}...`);
+
+    // get the image from cloudly
+    logger.log('info', `getting image for ${serviceArgFromCloudly.data.name}@${serviceArgFromCloudly.data.imageVersion}`);
+    const containerImageFromCloudly = await this.coreflowRef.cloudlyConnector.cloudlyApiClient.image.getImageById(serviceArgFromCloudly.data.imageId);
+    let localDockerImage: plugins.docker.DockerImage
+    
+    // lets get the docker image for the service
+    if (containerImageFromCloudly.data.location.internal) {
+      const imageStream = await containerImageFromCloudly.pullImageVersion(serviceArgFromCloudly.data.imageVersion);
+      localDockerImage= await plugins.docker.DockerImage.createFromTarStream(this.coreflowRef.dockerHost, {
+        creationObject: {
+          imageUrl: containerImageFromCloudly.id,
+          imageTag: serviceArgFromCloudly.data.imageVersion,
+        },
+        tarStream: plugins.smartstream.nodewebhelpers.convertWebReadableToNodeReadable(imageStream)
       });
+    } else if (
+      containerImageFromCloudly.data.location.externalRegistryId
+      && containerImageFromCloudly.data.location.externalImageTag
+    ) {
+      const externalRegistry = await this.coreflowRef.cloudlyConnector.cloudlyApiClient.externalRegistry.getRegistryById(containerImageFromCloudly.data.location.externalRegistryId);
+      // Lets authenticate against the external registry
+      // TODO: deduplicate this, check wether we are already authenticated
+      await this.coreflowRef.dockerHost.auth({
+        username: externalRegistry.data.username,
+        password: externalRegistry.data.password,
+        serveraddress: externalRegistry.data.url,
+      });
+      localDockerImage = await plugins.docker.DockerImage.createFromRegistry(this.coreflowRef.dockerHost, {
+        creationObject: {
+          imageUrl: containerImageFromCloudly.id,
+          imageTag: serviceArgFromCloudly.data.imageVersion,
+        },
+      });
+      await localDockerImage.pullLatestImageFromRegistry();
+    } else {
+      throw new Error('Invalid image location');
+    }
 
     let containerService = await plugins.docker.DockerService.getServiceByName(
-      this.dockerHost,
-      containerConfigArg.name
+      this.coreflowRef.dockerHost,
+      serviceArgFromCloudly.data.name
     );
+    this.coreflowRef.cloudlyConnector.cloudlyApiClient;
+    
+    const dockerSecretName = `${serviceArgFromCloudly.id}_${serviceArgFromCloudly.data.name}_Secret`;
     let containerSecret = await plugins.docker.DockerSecret.getSecretByName(
-      this.dockerHost,
-      `${containerConfigArg.name}Secret`
+      this.coreflowRef.dockerHost,
+      dockerSecretName
     );
 
     // existing network to connect to
     const webGatewayNetwork = await plugins.docker.DockerNetwork.getNetworkByName(
-      this.dockerHost,
+      this.coreflowRef.dockerHost,
       this.commonDockerData.networkNames.sznWebgateway
     );
 
@@ -230,27 +249,32 @@ export class ClusterManager {
 
     if (!containerService) {
       containerSecret = await plugins.docker.DockerSecret.getSecretByName(
-        this.dockerHost,
-        `${containerConfigArg.name}Secret`
+        this.coreflowRef.dockerHost,
+        dockerSecretName
       );
       if (containerSecret) {
         await containerSecret.remove();
       }
-      containerSecret = await plugins.docker.DockerSecret.createSecret(this.dockerHost, {
-        name: `${containerConfigArg.name}Secret`,
-        contentArg: JSON.stringify(containerConfigArg.secrets),
+
+      const secretBundle = await this.coreflowRef.cloudlyConnector.cloudlyApiClient.secretbundle.getSecretBundleById(serviceArgFromCloudly.data.secretBundleId);
+
+      // lets create the relevant stuff on the docker side
+      containerSecret = await plugins.docker.DockerSecret.createSecret(this.coreflowRef.dockerHost, {
+        name: dockerSecretName,
+        contentArg: JSON.stringify(await secretBundle.getFlatKeyValueObjectForEnvironment()),
         labels: {},
-        version: await containerImage.getVersion(),
+        version: await containerImageFromCloudly.data.versions[serviceArgFromCloudly.data.imageVersion],
       });
-      containerService = await plugins.docker.DockerService.createService(this.dockerHost, {
-        name: containerConfigArg.name,
-        image: containerImage,
+      containerService = await plugins.docker.DockerService.createService(this.coreflowRef.dockerHost, {
+        name: serviceArgFromCloudly.data.name,
+        image: localDockerImage,
         networks: [webGatewayNetwork],
         secrets: [containerSecret],
         ports: [],
         labels: {},
-        resources: containerConfigArg.resources,
-        networkAlias: containerConfigArg.name,
+        resources: serviceArgFromCloudly.data.resources,
+        // TODO: introduce a clean name here, that is guaranteed to work with APIs.
+        networkAlias: serviceArgFromCloudly.data.name,
       });
     }
   }
@@ -259,9 +283,9 @@ export class ClusterManager {
    * update traffic routing
    */
   public async updateTrafficRouting(clusterConfigArg: plugins.servezoneInterfaces.data.IClusterConfig) {
-    const services = await this.dockerHost.getServices();
+    const services = await this.coreflowRef.dockerHost.getServices();
     const webGatewayNetwork = await plugins.docker.DockerNetwork.getNetworkByName(
-      this.dockerHost,
+      this.coreflowRef.dockerHost,
       this.commonDockerData.networkNames.sznWebgateway
     );
     const reverseProxyConfigs: plugins.servezoneInterfaces.data.IReverseProxyConfig[] = [];

ts/coreflow.classes.coreflow.ts

@@ -22,7 +22,7 @@ export class Coreflow {
 
   constructor() {
     this.serviceQenv = new plugins.qenv.Qenv('./', './.nogit');
-    this.dockerHost = new plugins.docker.DockerHost(); // defaults to locally mounted docker sock
+    this.dockerHost = new plugins.docker.DockerHost({}); // defaults to locally mounted docker sock
     this.internalServer = new InternalServer(this);
     this.cloudlyConnector = new CloudlyConnector(this);
     this.corechatConnector = new CoretrafficConnector(this);

ts/coreflow.connector.cloudlyconnector.ts

@@ -7,47 +7,48 @@ import { Coreflow } from './coreflow.classes.coreflow.js';
 
 export class CloudlyConnector {
   public coreflowRef: Coreflow;
-  public cloudlyClient: plugins.servezoneApi.CloudlyClient;
+  public cloudlyApiClient: plugins.servezoneApi.CloudlyApiClient;
   public coreflowJumpCode: string;
-  public identity: plugins.servezoneInterfaces.data.IClusterIdentifier;
+  public identity: plugins.servezoneInterfaces.data.IIdentity;
 
   constructor(coreflowRefArg: Coreflow) {
     this.coreflowRef = coreflowRefArg;
   }
 
   public async start() {
-    this.cloudlyClient = new plugins.servezoneApi.CloudlyClient('coreflow');
-    await this.cloudlyClient.start();
+    this.cloudlyApiClient = new plugins.servezoneApi.CloudlyApiClient({
+      registerAs: 'coreflow',
+      cloudlyUrl: await this.coreflowRef.serviceQenv.getEnvVarOnDemand('CLOUDLY_URL')
+    });
+    await this.cloudlyApiClient.start();
     this.coreflowJumpCode = await this.coreflowRef.serviceQenv.getEnvVarOnDemand('JUMPCODE');
 
     // get identity and tag connection (second parameter is true -> tags the connection)
-    this.identity = await this.cloudlyClient.getIdentityByJumpCode(this.coreflowJumpCode, true);
+    this.identity = await this.cloudlyApiClient.getIdentityByToken(this.coreflowJumpCode, {
+      statefullIdentity: true,
+      tagConnection: true,
+    });
   }
 
   public async stop() {
-    await this.cloudlyClient.stop();
+    await this.cloudlyApiClient.stop();
   }
 
-  public async getConfigFromCloudly(): Promise<plugins.servezoneInterfaces.data.IClusterConfig> {
-    const config = await this.cloudlyClient.getClusterConfigFromCloudlyByIdentity(
+  public async getConfigFromCloudly(): Promise<plugins.servezoneInterfaces.data.ICluster> {
+    const config = await this.cloudlyApiClient.getClusterConfigFromCloudlyByIdentity(
       this.identity
     );
     return config;
   }
 
-  public async triggerConfigEvent() {
-    const config = await this.getConfigFromCloudly();
-    this.cloudlyClient.configUpdateSubject.next({
-      configData: config,
-    });
-  }
-
   public async getCertificateForDomainFromCloudly(
     domainNameArg: string
   ): Promise<plugins.tsclass.network.ICert> {
-    const certificate = await this.cloudlyClient.getCertificateForDomainOverHttps(
-      domainNameArg
-    );
+    const certificate = await this.cloudlyApiClient.getCertificateForDomain({
+      identity: this.identity,
+      domainName: domainNameArg,
+      type: 'ssl',
+    });
     return certificate;
   }
 }

ts/coreflow.plugins.ts

@@ -28,6 +28,7 @@ import * as smartpromise from '@push.rocks/smartpromise';
 import * as smartrequest from '@push.rocks/smartrequest';
 import * as smartrx from '@push.rocks/smartrx';
 import * as smartstate from '@push.rocks/smartstate';
+import * as smartstream from '@push.rocks/smartstream';
 import * as smartstring from '@push.rocks/smartstring';
 import * as taskbuffer from '@push.rocks/taskbuffer';
 
@@ -44,6 +45,7 @@ export {
   smartrequest,
   smartrx,
   smartstate,
+  smartstream,
   smartstring,
   taskbuffer,
 };