fix(core): Fix image retrieval and service deployment process for workload services

changelog.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## 2024-12-29 - 1.0.135 - fix(core)
+Fix image retrieval and service deployment process for workload services
+
+- Updated the handling of internal and external registry images when provisioning workload services.
+- Refined the image pulling logic to accommodate both internal and external sources.
+- Ensured proper secret management during service deployment.
+- Corrected and optimized key function parameters and API calls for clarity and functionality.
+
 ## 2024-12-20 - 1.0.134 - fix(core)
 Updated dependencies and refactored ClusterManager and CloudlyConnector for improved Docker management.
 

package.json

@@ -62,7 +62,7 @@
   "dependencies": {
     "@api.global/typedrequest": "^3.1.10",
     "@api.global/typedsocket": "^3.0.1",
-    "@apiclient.xyz/docker": "^1.2.7",
+    "@apiclient.xyz/docker": "^1.3.0",
     "@push.rocks/early": "^4.0.3",
     "@push.rocks/lik": "^6.1.0",
     "@push.rocks/projectinfo": "^5.0.1",
@@ -79,8 +79,8 @@
     "@push.rocks/smartstream": "^3.2.5",
     "@push.rocks/smartstring": "^4.0.15",
     "@push.rocks/taskbuffer": "^3.0.10",
-    "@serve.zone/api": "^4.5.1",
+    "@serve.zone/api": "^4.10.0",
-    "@serve.zone/interfaces": "^4.5.1",
+    "@serve.zone/interfaces": "^4.10.0",
     "@tsclass/tsclass": "^4.2.0",
     "@types/node": "22.10.2"
   },

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@serve.zone/coreflow',
-  version: '1.0.134',
+  version: '1.0.135',
   description: 'A comprehensive tool for managing Docker-based applications and services, enabling efficient scaling, network management, and integration with cloud services.'
 }

ts/coreflow.classes.clustermanager.ts

@@ -178,30 +178,55 @@ export class ClusterManager {
   }
 
   public async provisionWorkloadService(
-    serviceArg: plugins.servezoneInterfaces.data.IService
+    serviceArgFromCloudly: plugins.servezoneInterfaces.data.IService
   ) {
-    logger.log('info', `deploying service ${serviceArg.data.name}@${serviceArg.data.imageVersion}...`);
+    logger.log('info', `deploying service ${serviceArgFromCloudly.data.name}@${serviceArgFromCloudly.data.imageVersion}...`);
 
     // get the image from cloudly
-    logger.log('info', `getting image for ${serviceArg.data.name}@${serviceArg.data.imageVersion}`);
+    logger.log('info', `getting image for ${serviceArgFromCloudly.data.name}@${serviceArgFromCloudly.data.imageVersion}`);
-    const containerImage = await this.coreflowRef.cloudlyConnector.cloudlyApiClient.images.getImageById(serviceArg.data.imageId);
+    const containerImageFromCloudly = await this.coreflowRef.cloudlyConnector.cloudlyApiClient.image.getImageById(serviceArgFromCloudly.data.imageId);
-    const imageStream = await containerImage.pullImageVersion(serviceArg.data.imageVersion);
+    let localDockerImage: plugins.docker.DockerImage
-    await plugins.docker.DockerImage.createFromTarStream(this.coreflowRef.dockerHost, {
+    
-      creationObject: {
+    // lets get the docker image for the service
-        imageUrl: containerImage.id,
+    if (containerImageFromCloudly.data.location.internal) {
-        imageTag: serviceArg.data.imageVersion,
+      const imageStream = await containerImageFromCloudly.pullImageVersion(serviceArgFromCloudly.data.imageVersion);
-      },
+      localDockerImage= await plugins.docker.DockerImage.createFromTarStream(this.coreflowRef.dockerHost, {
-      tarStream: plugins.smartstream.nodewebhelpers.convertWebReadableToNodeReadable(imageStream)
+        creationObject: {
-    });
+          imageUrl: containerImageFromCloudly.id,
+          imageTag: serviceArgFromCloudly.data.imageVersion,
+        },
+        tarStream: plugins.smartstream.nodewebhelpers.convertWebReadableToNodeReadable(imageStream)
+      });
+    } else if (
+      containerImageFromCloudly.data.location.externalRegistryId
+      && containerImageFromCloudly.data.location.externalImageTag
+    ) {
+      const externalRegistry = await this.coreflowRef.cloudlyConnector.cloudlyApiClient.externalRegistry.getRegistryById(containerImageFromCloudly.data.location.externalRegistryId);
+      // Lets authenticate against the external registry
+      // TODO: deduplicate this, check wether we are already authenticated
+      await this.coreflowRef.dockerHost.auth({
+        username: externalRegistry.data.username,
+        password: externalRegistry.data.password,
+        serveraddress: externalRegistry.data.url,
+      });
+      localDockerImage = await plugins.docker.DockerImage.createFromRegistry(this.coreflowRef.dockerHost, {
+        creationObject: {
+          imageUrl: containerImageFromCloudly.id,
+          imageTag: serviceArgFromCloudly.data.imageVersion,
+        },
+      });
+      await localDockerImage.pullLatestImageFromRegistry();
+    } else {
+      throw new Error('Invalid image location');
+    }
 
     let containerService = await plugins.docker.DockerService.getServiceByName(
       this.coreflowRef.dockerHost,
-      serviceArg.data.name
+      serviceArgFromCloudly.data.name
     );
-    const secretBundleId = serviceArg.data.secretBundleId;
     this.coreflowRef.cloudlyConnector.cloudlyApiClient;
     
-    const dockerSecretName = `${serviceArg.id}_${serviceArg.data.name}_Secret`;
+    const dockerSecretName = `${serviceArgFromCloudly.id}_${serviceArgFromCloudly.data.name}_Secret`;
     let containerSecret = await plugins.docker.DockerSecret.getSecretByName(
       this.coreflowRef.dockerHost,
       dockerSecretName
@@ -231,23 +256,25 @@ export class ClusterManager {
         await containerSecret.remove();
       }
 
-      const secretBundle: plugins.servezoneInterfaces.data.ISecretBundle =this.coreflowRef.cloudlyConnector.cloudlyApiClient.;
+      const secretBundle = await this.coreflowRef.cloudlyConnector.cloudlyApiClient.secretbundle.getSecretBundleById(serviceArgFromCloudly.data.secretBundleId);
 
+      // lets create the relevant stuff on the docker side
       containerSecret = await plugins.docker.DockerSecret.createSecret(this.coreflowRef.dockerHost, {
         name: dockerSecretName,
-        contentArg: JSON.stringify(secretBundle.data.),
+        contentArg: JSON.stringify(await secretBundle.getFlatKeyValueObjectForEnvironment()),
         labels: {},
-        version: await containerImage.getVersion(),
+        version: await containerImageFromCloudly.data.versions[serviceArgFromCloudly.data.imageVersion],
       });
       containerService = await plugins.docker.DockerService.createService(this.coreflowRef.dockerHost, {
-        name: deploymentDirectiveArg.name,
+        name: serviceArgFromCloudly.data.name,
-        image: containerImage,
+        image: localDockerImage,
         networks: [webGatewayNetwork],
         secrets: [containerSecret],
         ports: [],
         labels: {},
-        resources: deploymentDirectiveArg.resources,
+        resources: serviceArgFromCloudly.data.resources,
-        networkAlias: deploymentDirectiveArg.name,
+        // TODO: introduce a clean name here, that is guaranteed to work with APIs.
+        networkAlias: serviceArgFromCloudly.data.name,
       });
     }
   }