From b747f07abdb6c720128e165f74e7d78c0c5d99dd Mon Sep 17 00:00:00 2001 From: Juergen Kunz Date: Sat, 2 May 2026 18:58:21 +0000 Subject: [PATCH] feat: mount corestore volumes --- readme.md | 22 ++- ts/coreflow.classes.clustermanager.ts | 220 ++++++++++++++++++++++++-- 2 files changed, 230 insertions(+), 12 deletions(-) diff --git a/readme.md b/readme.md index 8559856..87202f1 100644 --- a/readme.md +++ b/readme.md @@ -134,11 +134,29 @@ After connection, Coreflow authenticates with `JUMPCODE` and requests a stateful Coreflow depends on these Cloudly-side resources being present and valid: - Cluster configuration for the authenticated identity. -- Service records with image, resource, domain, port, and secret bundle references. +- Service records with image, volume, resource, domain, port, and secret bundle references. - Image records pointing either to internal Cloudly image storage or an external registry. - Secret bundles that can be flattened into environment key/value data. - SSL certificates for all routed domains. +## Corestore Volumes + +Coreflow deploys `corestore` as a global base service and bind mounts `/run/docker/plugins` so Docker can discover the `corestore` VolumeDriver socket on each node. + +Workload services can declare first-class volumes: + +```ts +volumes: [ + { + mountPath: '/data', + driver: 'corestore', + backup: true, + }, +] +``` + +If `name` is omitted, Coreflow derives a stable Docker volume name from the service id and mount path. During service creation it sends a Docker volume mount with `DriverConfig.Name = 'corestore'`, plus service metadata as driver options and volume labels. + ## Coretraffic Integration Coreflow starts an internal SmartServe/TypedSocket server on port `3000`. Coretraffic is expected to connect to that server and tag its connection as `coretraffic`. @@ -190,7 +208,7 @@ Project layout: - Reconciliation removes and recreates services when the Docker service reports that it needs an update. - Workload services must be attached to `sznwebgateway` for routing to be generated. - The current routing logic uses the first available container IP for a service. -- `PlatformManager` currently provides lifecycle hooks but does not reconcile platform services yet. +- `PlatformManager` provisions `database` and `objectstorage` bindings through corestore. ## License and Legal Information diff --git a/ts/coreflow.classes.clustermanager.ts b/ts/coreflow.classes.clustermanager.ts index a6722de..3231e1f 100644 --- a/ts/coreflow.classes.clustermanager.ts +++ b/ts/coreflow.classes.clustermanager.ts @@ -4,6 +4,18 @@ import { Coreflow } from './coreflow.classes.coreflow.js'; import type { IExternalGatewayConfig } from './coreflow.connector.externalgateway.js'; import * as crypto from 'node:crypto'; +type TServiceVolumeConfig = { + name?: string; + source?: string; + mountPath?: string; + target?: string; + containerFsPath?: string; + driver?: string; + readOnly?: boolean; + backup?: boolean; + options?: Record; +}; + export class ClusterManager { public coreflowRef: Coreflow; public configSubscription?: plugins.smartrx.rxjs.Subscription; @@ -51,6 +63,7 @@ export class ClusterManager { serviceArgFromCloudly: plugins.servezoneInterfaces.data.IService, containerImageFromCloudly: plugins.servezoneInterfaces.data.IImage, secretHashArg = '', + volumeHashArg = '', ) { const desiredImageVersion = serviceArgFromCloudly.data.imageVersion || @@ -72,6 +85,7 @@ export class ClusterManager { 'serve.zone.registryImageUrl': serviceArgFromCloudly.data.registryTarget?.imageUrl || '', 'serve.zone.registryDigest': desiredRegistryDigest || '', 'serve.zone.secretHash': secretHashArg, + ...(volumeHashArg ? { 'serve.zone.volumeHash': volumeHashArg } : {}), }; } @@ -92,6 +106,10 @@ export class ClusterManager { return crypto.createHash('sha256').update(this.stableStringify(secretObjectArg)).digest('hex'); } + private hashStableValue(valueArg: unknown) { + return crypto.createHash('sha256').update(this.stableStringify(valueArg)).digest('hex'); + } + private async pullRegistryTargetImage( registryTargetArg: plugins.servezoneInterfaces.data.IRegistryTarget, ): Promise { @@ -123,6 +141,182 @@ export class ClusterManager { return localDockerImage; } + private getServiceVolumeConfigs(serviceArgFromCloudly: plugins.servezoneInterfaces.data.IService) { + const serviceData = serviceArgFromCloudly.data as plugins.servezoneInterfaces.data.IService['data'] & { + volumes?: TServiceVolumeConfig[]; + }; + return (serviceData.volumes || []).filter((volumeArg) => { + return Boolean(volumeArg.mountPath || volumeArg.target || volumeArg.containerFsPath); + }); + } + + private getCoreStoreVolumeName( + serviceArgFromCloudly: plugins.servezoneInterfaces.data.IService, + volumeArg: TServiceVolumeConfig, + ) { + const requestedName = volumeArg.source || volumeArg.name; + if (requestedName) { + return this.getDockerSafeName(requestedName, 120); + } + const mountPath = volumeArg.mountPath || volumeArg.target || volumeArg.containerFsPath || 'data'; + const serviceName = this.getDockerSafeName(serviceArgFromCloudly.data.name, 36); + const mountName = this.getDockerSafeName(mountPath.replace(/^\/+/, '').replace(/\/+$/g, ''), 28); + const hash = crypto.createHash('sha1').update(`${serviceArgFromCloudly.id}:${mountPath}`).digest('hex').slice(0, 12); + return this.getDockerSafeName(`sz-${serviceName}-${mountName}-${hash}`, 120); + } + + private getServiceDockerMounts(serviceArgFromCloudly: plugins.servezoneInterfaces.data.IService) { + const mounts: Array> = []; + const resources = serviceArgFromCloudly.data.resources as (plugins.servezoneInterfaces.data.IService['data']['resources'] & { + volumeMounts?: Array<{ hostFsPath: string; containerFsPath: string }>; + }) | undefined; + + for (const volumeMount of resources?.volumeMounts || []) { + mounts.push({ + Target: volumeMount.containerFsPath, + Source: volumeMount.hostFsPath, + Consistency: 'default', + ReadOnly: false, + Type: 'bind', + }); + } + + for (const volume of this.getServiceVolumeConfigs(serviceArgFromCloudly)) { + const target = volume.mountPath || volume.target || volume.containerFsPath; + if (!target) { + continue; + } + const driver = volume.driver || 'corestore'; + const source = this.getCoreStoreVolumeName(serviceArgFromCloudly, volume); + const backup = volume.backup !== false; + const driverOptions: Record = { + ...(volume.options || {}), + serviceId: serviceArgFromCloudly.id, + serviceName: serviceArgFromCloudly.data.name, + mountPath: target, + backup: String(backup), + }; + + mounts.push({ + Target: target, + Source: source, + Type: 'volume', + ReadOnly: Boolean(volume.readOnly), + VolumeOptions: { + DriverConfig: { + Name: driver, + Options: driverOptions, + }, + Labels: { + 'serve.zone.serviceId': serviceArgFromCloudly.id, + 'serve.zone.serviceName': serviceArgFromCloudly.data.name, + 'serve.zone.mountPath': target, + 'serve.zone.backup': String(backup), + }, + }, + }); + } + + return mounts; + } + + private getServiceVolumeHash(serviceArgFromCloudly: plugins.servezoneInterfaces.data.IService) { + const volumeConfigs = this.getServiceVolumeConfigs(serviceArgFromCloudly); + if (volumeConfigs.length === 0) { + return ''; + } + const volumeSpecs = volumeConfigs.map((volumeArg) => ({ + ...volumeArg, + source: this.getCoreStoreVolumeName(serviceArgFromCloudly, volumeArg), + driver: volumeArg.driver || 'corestore', + backup: volumeArg.backup !== false, + })); + return this.hashStableValue(volumeSpecs); + } + + private async createWorkloadDockerService(argsArg: { + service: plugins.servezoneInterfaces.data.IService; + image: plugins.docker.DockerImage; + network: plugins.docker.DockerNetwork; + secret: plugins.docker.DockerSecret; + labels: Record; + }) { + const image = argsArg.image as unknown as { RepoTags?: string[] }; + const imageRef = image.RepoTags?.[0]; + if (!imageRef) { + throw new Error(`Docker image for ${argsArg.service.data.name} has no tag`); + } + + const ports: Array<{ Protocol: string; PublishedPort: number; TargetPort: number }> = []; + const resources = argsArg.service.data.resources as (plugins.servezoneInterfaces.data.IService['data']['resources'] & { + memorySizeMB?: number; + }) | undefined; + const memoryLimitMB = resources?.memorySizeMB || resources?.memorySizeLimitMB || 1000; + const replicas = Math.max(1, Number(argsArg.service.data.scaleFactor || 1)); + + const response = await this.coreflowRef.dockerHost.request('POST', '/services/create', { + Name: argsArg.service.data.name, + Labels: argsArg.labels, + TaskTemplate: { + ContainerSpec: { + Image: imageRef, + Labels: argsArg.labels, + Secrets: [ + { + File: { + Name: 'secret.json', + UID: '33', + GID: '33', + Mode: 384, + }, + SecretID: argsArg.secret.ID, + SecretName: argsArg.secret.Spec.Name, + }, + ], + Mounts: this.getServiceDockerMounts(argsArg.service), + }, + UpdateConfig: { + Parallelism: 0, + Delay: 0, + FailureAction: 'pause', + Monitor: 15000000000, + MaxFailureRatio: 0.15, + }, + ForceUpdate: 1, + Resources: { + Limits: { + MemoryBytes: memoryLimitMB * 1000000, + }, + }, + Networks: [ + { + Target: argsArg.network.Name, + Aliases: [argsArg.service.data.name], + }, + ], + LogDriver: { + Name: 'json-file', + Options: { + 'max-file': '3', + 'max-size': '10M', + }, + }, + }, + Mode: { + Replicated: { + Replicas: replicas, + }, + }, + EndpointSpec: { + Ports: ports, + }, + }); + if (response.statusCode >= 300) { + throw new Error(`Failed to create workload service ${argsArg.service.data.name}: ${JSON.stringify(response.body)}`); + } + return this.getDockerServiceByName(argsArg.service.data.name); + } + private async createCorestoreGlobalService( corestoreImageArg: plugins.docker.DockerImage, networksArg: Array, @@ -135,6 +329,7 @@ export class ClusterManager { const corestoreEnv = [ 'CORESTORE_DATA_DIR=/data/corestore', 'CORESTORE_PUBLIC_HOST=corestore', + 'CORESTORE_VOLUME_PLUGIN_SOCKET=/run/docker/plugins/corestore.sock', ...(process.env.CORESTORE_API_TOKEN ? [`CORESTORE_API_TOKEN=${process.env.CORESTORE_API_TOKEN}`] : []), @@ -144,14 +339,14 @@ export class ClusterManager { Labels: { version: corestoreImage.Labels?.version || '', 'serve.zone.serviceCategory': 'base', - 'serve.zone.provides': 'database,objectstorage', + 'serve.zone.provides': 'database,objectstorage,volume', }, TaskTemplate: { ContainerSpec: { Image: imageRef, Labels: { 'serve.zone.serviceCategory': 'base', - 'serve.zone.provides': 'database,objectstorage', + 'serve.zone.provides': 'database,objectstorage,volume', }, Env: corestoreEnv, Mounts: [ @@ -162,6 +357,13 @@ export class ClusterManager { ReadOnly: false, Consistency: 'default', }, + { + Target: '/run/docker/plugins', + Source: '/run/docker/plugins', + Type: 'bind', + ReadOnly: false, + Consistency: 'default', + }, ], }, Networks: networksArg.map((networkArg) => ({ @@ -452,10 +654,12 @@ export class ClusterManager { ...(await secretBundle.getFlatKeyValueObjectForEnvironment()), }; const secretHash = this.hashSecretObject(secretObject); + const volumeHash = this.getServiceVolumeHash(serviceArgFromCloudly); const deploymentLabels = this.getWorkloadServiceDeploymentLabels( serviceArgFromCloudly, containerImageFromCloudly, secretHash, + volumeHash, ); // existing network to connect to @@ -501,16 +705,12 @@ export class ClusterManager { labels: {}, version: serviceArgFromCloudly.data.imageVersion, }); - containerService = await this.coreflowRef.dockerHost.createService({ - name: serviceArgFromCloudly.data.name, + containerService = await this.createWorkloadDockerService({ + service: serviceArgFromCloudly, image: localDockerImage, - networks: [webGatewayNetwork], - secrets: [containerSecret], - ports: [], + network: webGatewayNetwork, + secret: containerSecret, labels: deploymentLabels, - resources: serviceArgFromCloudly.data.resources, - // TODO: introduce a clean name here, that is guaranteed to work with APIs. - networkAlias: serviceArgFromCloudly.data.name, }); } }