feat: mount corestore volumes

2026-05-02 18:58:21 +00:00
parent 8eea6c36ea
commit b747f07abd
2 changed files with 230 additions and 12 deletions
@@ -134,11 +134,29 @@ After connection, Coreflow authenticates with `JUMPCODE` and requests a stateful
 Coreflow depends on these Cloudly-side resources being present and valid:

 - Cluster configuration for the authenticated identity.
- Service records with image, resource, domain, port, and secret bundle references.
+- Service records with image, volume, resource, domain, port, and secret bundle references.
 - Image records pointing either to internal Cloudly image storage or an external registry.
 - Secret bundles that can be flattened into environment key/value data.
 - SSL certificates for all routed domains.

+## Corestore Volumes
+
+Coreflow deploys `corestore` as a global base service and bind mounts `/run/docker/plugins` so Docker can discover the `corestore` VolumeDriver socket on each node.
+
+Workload services can declare first-class volumes:
+
+```ts
+volumes: [
+  {
+    mountPath: '/data',
+    driver: 'corestore',
+    backup: true,
+  },
+]
+```
+
+If `name` is omitted, Coreflow derives a stable Docker volume name from the service id and mount path. During service creation it sends a Docker volume mount with `DriverConfig.Name = 'corestore'`, plus service metadata as driver options and volume labels.
+
 ## Coretraffic Integration

 Coreflow starts an internal SmartServe/TypedSocket server on port `3000`. Coretraffic is expected to connect to that server and tag its connection as `coretraffic`.
@@ -190,7 +208,7 @@ Project layout:
 - Reconciliation removes and recreates services when the Docker service reports that it needs an update.
 - Workload services must be attached to `sznwebgateway` for routing to be generated.
 - The current routing logic uses the first available container IP for a service.
- `PlatformManager` currently provides lifecycle hooks but does not reconcile platform services yet.
+- `PlatformManager` provisions `database` and `objectstorage` bindings through corestore.

 ## License and Legal Information

@@ -4,6 +4,18 @@ import { Coreflow } from './coreflow.classes.coreflow.js';
 import type { IExternalGatewayConfig } from './coreflow.connector.externalgateway.js';
 import * as crypto from 'node:crypto';

+type TServiceVolumeConfig = {
+  name?: string;
+  source?: string;
+  mountPath?: string;
+  target?: string;
+  containerFsPath?: string;
+  driver?: string;
+  readOnly?: boolean;
+  backup?: boolean;
+  options?: Record<string, string>;
+};
+
 export class ClusterManager {
  public coreflowRef: Coreflow;
  public configSubscription?: plugins.smartrx.rxjs.Subscription;
@@ -51,6 +63,7 @@ export class ClusterManager {
    serviceArgFromCloudly: plugins.servezoneInterfaces.data.IService,
    containerImageFromCloudly: plugins.servezoneInterfaces.data.IImage,
    secretHashArg = '',
+    volumeHashArg = '',
  ) {
    const desiredImageVersion =
      serviceArgFromCloudly.data.imageVersion ||
@@ -72,6 +85,7 @@ export class ClusterManager {
      'serve.zone.registryImageUrl': serviceArgFromCloudly.data.registryTarget?.imageUrl || '',
      'serve.zone.registryDigest': desiredRegistryDigest || '',
      'serve.zone.secretHash': secretHashArg,
+      ...(volumeHashArg ? { 'serve.zone.volumeHash': volumeHashArg } : {}),
    };
  }

@@ -92,6 +106,10 @@ export class ClusterManager {
    return crypto.createHash('sha256').update(this.stableStringify(secretObjectArg)).digest('hex');
  }

+  private hashStableValue(valueArg: unknown) {
+    return crypto.createHash('sha256').update(this.stableStringify(valueArg)).digest('hex');
+  }
+
  private async pullRegistryTargetImage(
    registryTargetArg: plugins.servezoneInterfaces.data.IRegistryTarget,
  ): Promise<plugins.docker.DockerImage> {
@@ -123,6 +141,182 @@ export class ClusterManager {
    return localDockerImage;
  }

+  private getServiceVolumeConfigs(serviceArgFromCloudly: plugins.servezoneInterfaces.data.IService) {
+    const serviceData = serviceArgFromCloudly.data as plugins.servezoneInterfaces.data.IService['data'] & {
+      volumes?: TServiceVolumeConfig[];
+    };
+    return (serviceData.volumes || []).filter((volumeArg) => {
+      return Boolean(volumeArg.mountPath || volumeArg.target || volumeArg.containerFsPath);
+    });
+  }
+
+  private getCoreStoreVolumeName(
+    serviceArgFromCloudly: plugins.servezoneInterfaces.data.IService,
+    volumeArg: TServiceVolumeConfig,
+  ) {
+    const requestedName = volumeArg.source || volumeArg.name;
+    if (requestedName) {
+      return this.getDockerSafeName(requestedName, 120);
+    }
+    const mountPath = volumeArg.mountPath || volumeArg.target || volumeArg.containerFsPath || 'data';
+    const serviceName = this.getDockerSafeName(serviceArgFromCloudly.data.name, 36);
+    const mountName = this.getDockerSafeName(mountPath.replace(/^\/+/, '').replace(/\/+$/g, ''), 28);
+    const hash = crypto.createHash('sha1').update(`${serviceArgFromCloudly.id}:${mountPath}`).digest('hex').slice(0, 12);
+    return this.getDockerSafeName(`sz-${serviceName}-${mountName}-${hash}`, 120);
+  }
+
+  private getServiceDockerMounts(serviceArgFromCloudly: plugins.servezoneInterfaces.data.IService) {
+    const mounts: Array<Record<string, unknown>> = [];
+    const resources = serviceArgFromCloudly.data.resources as (plugins.servezoneInterfaces.data.IService['data']['resources'] & {
+      volumeMounts?: Array<{ hostFsPath: string; containerFsPath: string }>;
+    }) | undefined;
+
+    for (const volumeMount of resources?.volumeMounts || []) {
+      mounts.push({
+        Target: volumeMount.containerFsPath,
+        Source: volumeMount.hostFsPath,
+        Consistency: 'default',
+        ReadOnly: false,
+        Type: 'bind',
+      });
+    }
+
+    for (const volume of this.getServiceVolumeConfigs(serviceArgFromCloudly)) {
+      const target = volume.mountPath || volume.target || volume.containerFsPath;
+      if (!target) {
+        continue;
+      }
+      const driver = volume.driver || 'corestore';
+      const source = this.getCoreStoreVolumeName(serviceArgFromCloudly, volume);
+      const backup = volume.backup !== false;
+      const driverOptions: Record<string, string> = {
+        ...(volume.options || {}),
+        serviceId: serviceArgFromCloudly.id,
+        serviceName: serviceArgFromCloudly.data.name,
+        mountPath: target,
+        backup: String(backup),
+      };
+
+      mounts.push({
+        Target: target,
+        Source: source,
+        Type: 'volume',
+        ReadOnly: Boolean(volume.readOnly),
+        VolumeOptions: {
+          DriverConfig: {
+            Name: driver,
+            Options: driverOptions,
+          },
+          Labels: {
+            'serve.zone.serviceId': serviceArgFromCloudly.id,
+            'serve.zone.serviceName': serviceArgFromCloudly.data.name,
+            'serve.zone.mountPath': target,
+            'serve.zone.backup': String(backup),
+          },
+        },
+      });
+    }
+
+    return mounts;
+  }
+
+  private getServiceVolumeHash(serviceArgFromCloudly: plugins.servezoneInterfaces.data.IService) {
+    const volumeConfigs = this.getServiceVolumeConfigs(serviceArgFromCloudly);
+    if (volumeConfigs.length === 0) {
+      return '';
+    }
+    const volumeSpecs = volumeConfigs.map((volumeArg) => ({
+      ...volumeArg,
+      source: this.getCoreStoreVolumeName(serviceArgFromCloudly, volumeArg),
+      driver: volumeArg.driver || 'corestore',
+      backup: volumeArg.backup !== false,
+    }));
+    return this.hashStableValue(volumeSpecs);
+  }
+
+  private async createWorkloadDockerService(argsArg: {
+    service: plugins.servezoneInterfaces.data.IService;
+    image: plugins.docker.DockerImage;
+    network: plugins.docker.DockerNetwork;
+    secret: plugins.docker.DockerSecret;
+    labels: Record<string, string>;
+  }) {
+    const image = argsArg.image as unknown as { RepoTags?: string[] };
+    const imageRef = image.RepoTags?.[0];
+    if (!imageRef) {
+      throw new Error(`Docker image for ${argsArg.service.data.name} has no tag`);
+    }
+
+    const ports: Array<{ Protocol: string; PublishedPort: number; TargetPort: number }> = [];
+    const resources = argsArg.service.data.resources as (plugins.servezoneInterfaces.data.IService['data']['resources'] & {
+      memorySizeMB?: number;
+    }) | undefined;
+    const memoryLimitMB = resources?.memorySizeMB || resources?.memorySizeLimitMB || 1000;
+    const replicas = Math.max(1, Number(argsArg.service.data.scaleFactor || 1));
+
+    const response = await this.coreflowRef.dockerHost.request('POST', '/services/create', {
+      Name: argsArg.service.data.name,
+      Labels: argsArg.labels,
+      TaskTemplate: {
+        ContainerSpec: {
+          Image: imageRef,
+          Labels: argsArg.labels,
+          Secrets: [
+            {
+              File: {
+                Name: 'secret.json',
+                UID: '33',
+                GID: '33',
+                Mode: 384,
+              },
+              SecretID: argsArg.secret.ID,
+              SecretName: argsArg.secret.Spec.Name,
+            },
+          ],
+          Mounts: this.getServiceDockerMounts(argsArg.service),
+        },
+        UpdateConfig: {
+          Parallelism: 0,
+          Delay: 0,
+          FailureAction: 'pause',
+          Monitor: 15000000000,
+          MaxFailureRatio: 0.15,
+        },
+        ForceUpdate: 1,
+        Resources: {
+          Limits: {
+            MemoryBytes: memoryLimitMB * 1000000,
+          },
+        },
+        Networks: [
+          {
+            Target: argsArg.network.Name,
+            Aliases: [argsArg.service.data.name],
+          },
+        ],
+        LogDriver: {
+          Name: 'json-file',
+          Options: {
+            'max-file': '3',
+            'max-size': '10M',
+          },
+        },
+      },
+      Mode: {
+        Replicated: {
+          Replicas: replicas,
+        },
+      },
+      EndpointSpec: {
+        Ports: ports,
+      },
+    });
+    if (response.statusCode >= 300) {
+      throw new Error(`Failed to create workload service ${argsArg.service.data.name}: ${JSON.stringify(response.body)}`);
+    }
+    return this.getDockerServiceByName(argsArg.service.data.name);
+  }
+
  private async createCorestoreGlobalService(
    corestoreImageArg: plugins.docker.DockerImage,
    networksArg: Array<plugins.docker.DockerNetwork>,
@@ -135,6 +329,7 @@ export class ClusterManager {
    const corestoreEnv = [
      'CORESTORE_DATA_DIR=/data/corestore',
      'CORESTORE_PUBLIC_HOST=corestore',
+      'CORESTORE_VOLUME_PLUGIN_SOCKET=/run/docker/plugins/corestore.sock',
      ...(process.env.CORESTORE_API_TOKEN
        ? [`CORESTORE_API_TOKEN=${process.env.CORESTORE_API_TOKEN}`]
        : []),
@@ -144,14 +339,14 @@ export class ClusterManager {
      Labels: {
        version: corestoreImage.Labels?.version || '',
        'serve.zone.serviceCategory': 'base',
-        'serve.zone.provides': 'database,objectstorage',
+        'serve.zone.provides': 'database,objectstorage,volume',
      },
      TaskTemplate: {
        ContainerSpec: {
          Image: imageRef,
          Labels: {
            'serve.zone.serviceCategory': 'base',
-            'serve.zone.provides': 'database,objectstorage',
+            'serve.zone.provides': 'database,objectstorage,volume',
          },
          Env: corestoreEnv,
          Mounts: [
@@ -162,6 +357,13 @@ export class ClusterManager {
              ReadOnly: false,
              Consistency: 'default',
            },
+            {
+              Target: '/run/docker/plugins',
+              Source: '/run/docker/plugins',
+              Type: 'bind',
+              ReadOnly: false,
+              Consistency: 'default',
+            },
          ],
        },
        Networks: networksArg.map((networkArg) => ({
@@ -452,10 +654,12 @@ export class ClusterManager {
      ...(await secretBundle.getFlatKeyValueObjectForEnvironment()),
    };
    const secretHash = this.hashSecretObject(secretObject);
+    const volumeHash = this.getServiceVolumeHash(serviceArgFromCloudly);
    const deploymentLabels = this.getWorkloadServiceDeploymentLabels(
      serviceArgFromCloudly,
      containerImageFromCloudly,
      secretHash,
+      volumeHash,
    );

    // existing network to connect to
@@ -501,16 +705,12 @@ export class ClusterManager {
        labels: {},
        version: serviceArgFromCloudly.data.imageVersion,
      });
-      containerService = await this.coreflowRef.dockerHost.createService({
-        name: serviceArgFromCloudly.data.name,
+      containerService = await this.createWorkloadDockerService({
+        service: serviceArgFromCloudly,
        image: localDockerImage,
-        networks: [webGatewayNetwork],
-        secrets: [containerSecret],
-        ports: [],
+        network: webGatewayNetwork,
+        secret: containerSecret,
        labels: deploymentLabels,
-        resources: serviceArgFromCloudly.data.resources,
-        // TODO: introduce a clean name here, that is guaranteed to work with APIs.
-        networkAlias: serviceArgFromCloudly.data.name,
      });
    }
  }