fix(core): Updated dependencies and refactored ClusterManager and CloudlyConnector for improved Docker management.

ts/coreflow.classes.clustermanager.ts

@@ -4,7 +4,6 @@ import { Coreflow } from './coreflow.classes.coreflow.js';
 
 export class ClusterManager {
   public coreflowRef: Coreflow;
-  public dockerHost: plugins.docker.DockerHost;
   public configSubscription: plugins.smartrx.rxjs.Subscription;
   public containerSubscription: plugins.smartrx.rxjs.Subscription;
   public containerVersionSubscription: plugins.smartrx.rxjs.Subscription;
@@ -20,7 +19,6 @@ export class ClusterManager {
 
   constructor(coreflowRefArg: Coreflow) {
     this.coreflowRef = coreflowRefArg;
-    this.dockerHost = new plugins.docker.DockerHost();
   }
 
   /**
@@ -31,27 +29,13 @@ export class ClusterManager {
     this.readyDeferred.resolve();
 
     // subscriptions
+    // this subscription is the start point for most updates on the cluster
     this.configSubscription =
-      this.coreflowRef.cloudlyConnector.cloudlyClient.configUpdateSubject.subscribe(
+      this.coreflowRef.cloudlyConnector.cloudlyApiClient.configUpdateSubject.subscribe(
         async (dataArg) => {
           this.coreflowRef.taskManager.updateBaseServicesTask.trigger();
         }
       );
-    this.containerSubscription =
-      this.coreflowRef.cloudlyConnector.cloudlyClient.containerUpdateSubject.subscribe(
-        async (dataArg) => {
-          this.coreflowRef.taskManager.updateBaseServicesTask.trigger();
-        }
-      );
-    this.containerVersionSubscription =
-      this.coreflowRef.cloudlyConnector.cloudlyClient.containerVersionUpdateSubject.subscribe(
-        async (dataArg) => {
-          console.log(
-            `Got a container version update trigger for ${dataArg.dockerImageUrl}@${dataArg.dockerImageVersion}`
-          );
-          this.coreflowRef.taskManager.updateBaseServicesTask.trigger();
-        }
-      );
   }
 
   /**
@@ -67,27 +51,34 @@ export class ClusterManager {
   public async provisionBaseServices() {
     // swarm should be enabled by lower level serverconfig package
     // get current situation
-    const networks = await this.dockerHost.getNetworks();
+    const networks = await this.coreflowRef.dockerHost.getNetworks();
+    logger.log('info', 'There are currently ' + networks.length + ' networks');
+    for (const network of networks) {
+      logger.log('info', 'Network: ' + network.Name);
+    }
+
+    // make sure there is a network for the webgateway
     let sznWebgatewayNetwork = await plugins.docker.DockerNetwork.getNetworkByName(
-      this.dockerHost,
+      this.coreflowRef.dockerHost,
       this.commonDockerData.networkNames.sznWebgateway
     );
 
     if (!sznWebgatewayNetwork) {
-      sznWebgatewayNetwork = await plugins.docker.DockerNetwork.createNetwork(this.dockerHost, {
+      logger.log('info', 'Creating network: ' + this.commonDockerData.networkNames.sznWebgateway);
+      sznWebgatewayNetwork = await plugins.docker.DockerNetwork.createNetwork(this.coreflowRef.dockerHost, {
         Name: this.commonDockerData.networkNames.sznWebgateway,
       });
     } else {
       logger.log('ok', 'sznWebgateway is already present');
     }
 
-    // corechat network
+    // corechat network so base services can talk to each other
     let sznCorechatNetwork = await plugins.docker.DockerNetwork.getNetworkByName(
-      this.dockerHost,
+      this.coreflowRef.dockerHost,
       this.commonDockerData.networkNames.sznCorechat
     );
     if (!sznCorechatNetwork) {
-      sznCorechatNetwork = await plugins.docker.DockerNetwork.createNetwork(this.dockerHost, {
+      sznCorechatNetwork = await plugins.docker.DockerNetwork.createNetwork(this.coreflowRef.dockerHost, {
         Name: this.commonDockerData.networkNames.sznCorechat,
       });
     } else {
@@ -102,12 +93,16 @@ export class ClusterManager {
     // Images
     logger.log('info', `now updating docker images of base services...`);
 
-    const coretrafficImage = await plugins.docker.DockerImage.createFromRegistry(this.dockerHost, {
-      imageUrl: 'registry.gitlab.com/losslessone/services/servezone/coretraffic',
+    const coretrafficImage = await plugins.docker.DockerImage.createFromRegistry(this.coreflowRef.dockerHost, {
+      creationObject: {
+        imageUrl: 'code.foss.global/serve.zone/coretraffic',
+      }
     });
 
-    const corelogImage = await plugins.docker.DockerImage.createFromRegistry(this.dockerHost, {
-      imageUrl: 'registry.gitlab.com/losslessone/services/servezone/corelog',
+    const corelogImage = await plugins.docker.DockerImage.createFromRegistry(this.coreflowRef.dockerHost, {
+      creationObject: {
+        imageUrl: 'code.foss.global/serve.zone/corelog',
+      }
     });
 
     // SERVICES
@@ -115,7 +110,7 @@ export class ClusterManager {
     // coretraffic
     let coretrafficService: plugins.docker.DockerService;
     coretrafficService = await plugins.docker.DockerService.getServiceByName(
-      this.dockerHost,
+      this.coreflowRef.dockerHost,
       'coretraffic'
     );
 
@@ -128,7 +123,7 @@ export class ClusterManager {
     }
 
     if (!coretrafficService) {
-      coretrafficService = await plugins.docker.DockerService.createService(this.dockerHost, {
+      coretrafficService = await plugins.docker.DockerService.createService(this.coreflowRef.dockerHost, {
         image: coretrafficImage,
         labels: {},
         name: 'coretraffic',
@@ -150,7 +145,7 @@ export class ClusterManager {
     // corelog
     let corelogService: plugins.docker.DockerService;
     corelogService = await plugins.docker.DockerService.getServiceByName(
-      this.dockerHost,
+      this.coreflowRef.dockerHost,
       'corelog'
     );
 
@@ -162,7 +157,7 @@ export class ClusterManager {
     }
 
     if (!corelogService) {
-      corelogService = await plugins.docker.DockerService.createService(this.dockerHost, {
+      corelogService = await plugins.docker.DockerService.createService(this.coreflowRef.dockerHost, {
         image: corelogImage,
         labels: {},
         name: 'corelog',
@@ -182,40 +177,39 @@ export class ClusterManager {
     await plugins.smartdelay.delayFor(10000);
   }
 
-  /**
-   * provision services obtained from cloudly
-   */
-  public async provisionWorkloadServices(configData: plugins.servezoneInterfaces.data.ICluster) {
-    // lets get the config + deploymentDirectives
-    
-
-    
-    for (const containerConfig of configData.data.containers) {
-      await this.provisionSpecificWorkloadService(containerConfig);
-    }
-    logger.log('ok', 'Waiting for scheduled workload services to settle');
-    await plugins.smartdelay.delayFor(10000);
-  }
-
-  public async provisionSpecificWorkloadService(
-    containerConfigArg: plugins.servezoneInterfaces.data.IClusterConfigContainer
+  public async provisionWorkloadService(
+    serviceArg: plugins.servezoneInterfaces.data.IService
   ) {
-    const containerImage = await plugins.docker.DockerImage.createFromRegistry(this.dockerHost, {
-      imageUrl: containerConfigArg.image,
+    logger.log('info', `deploying service ${serviceArg.data.name}@${serviceArg.data.imageVersion}...`);
+
+    // get the image from cloudly
+    logger.log('info', `getting image for ${serviceArg.data.name}@${serviceArg.data.imageVersion}`);
+    const containerImage = await this.coreflowRef.cloudlyConnector.cloudlyApiClient.images.getImageById(serviceArg.data.imageId);
+    const imageStream = await containerImage.pullImageVersion(serviceArg.data.imageVersion);
+    await plugins.docker.DockerImage.createFromTarStream(this.coreflowRef.dockerHost, {
+      creationObject: {
+        imageUrl: containerImage.id,
+        imageTag: serviceArg.data.imageVersion,
+      },
+      tarStream: plugins.smartstream.nodewebhelpers.convertWebReadableToNodeReadable(imageStream)
     });
 
     let containerService = await plugins.docker.DockerService.getServiceByName(
-      this.dockerHost,
-      containerConfigArg.name
+      this.coreflowRef.dockerHost,
+      serviceArg.data.name
     );
+    const secretBundleId = serviceArg.data.secretBundleId;
+    this.coreflowRef.cloudlyConnector.cloudlyApiClient;
+    
+    const dockerSecretName = `${serviceArg.id}_${serviceArg.data.name}_Secret`;
     let containerSecret = await plugins.docker.DockerSecret.getSecretByName(
-      this.dockerHost,
-      `${containerConfigArg.name}Secret`
+      this.coreflowRef.dockerHost,
+      dockerSecretName
     );
 
     // existing network to connect to
     const webGatewayNetwork = await plugins.docker.DockerNetwork.getNetworkByName(
-      this.dockerHost,
+      this.coreflowRef.dockerHost,
       this.commonDockerData.networkNames.sznWebgateway
     );
 
@@ -230,27 +224,30 @@ export class ClusterManager {
 
     if (!containerService) {
       containerSecret = await plugins.docker.DockerSecret.getSecretByName(
-        this.dockerHost,
-        `${containerConfigArg.name}Secret`
+        this.coreflowRef.dockerHost,
+        dockerSecretName
       );
       if (containerSecret) {
         await containerSecret.remove();
       }
-      containerSecret = await plugins.docker.DockerSecret.createSecret(this.dockerHost, {
-        name: `${containerConfigArg.name}Secret`,
-        contentArg: JSON.stringify(containerConfigArg.secrets),
+
+      const secretBundle: plugins.servezoneInterfaces.data.ISecretBundle =this.coreflowRef.cloudlyConnector.cloudlyApiClient.;
+
+      containerSecret = await plugins.docker.DockerSecret.createSecret(this.coreflowRef.dockerHost, {
+        name: dockerSecretName,
+        contentArg: JSON.stringify(secretBundle.data.),
         labels: {},
         version: await containerImage.getVersion(),
       });
-      containerService = await plugins.docker.DockerService.createService(this.dockerHost, {
-        name: containerConfigArg.name,
+      containerService = await plugins.docker.DockerService.createService(this.coreflowRef.dockerHost, {
+        name: deploymentDirectiveArg.name,
         image: containerImage,
         networks: [webGatewayNetwork],
         secrets: [containerSecret],
         ports: [],
         labels: {},
-        resources: containerConfigArg.resources,
-        networkAlias: containerConfigArg.name,
+        resources: deploymentDirectiveArg.resources,
+        networkAlias: deploymentDirectiveArg.name,
       });
     }
   }
@@ -259,9 +256,9 @@ export class ClusterManager {
    * update traffic routing
    */
   public async updateTrafficRouting(clusterConfigArg: plugins.servezoneInterfaces.data.IClusterConfig) {
-    const services = await this.dockerHost.getServices();
+    const services = await this.coreflowRef.dockerHost.getServices();
     const webGatewayNetwork = await plugins.docker.DockerNetwork.getNetworkByName(
-      this.dockerHost,
+      this.coreflowRef.dockerHost,
       this.commonDockerData.networkNames.sznWebgateway
     );
     const reverseProxyConfigs: plugins.servezoneInterfaces.data.IReverseProxyConfig[] = [];