import * as plugins from './coreflow.plugins.js'; import { logger } from './coreflow.logging.js'; import { Coreflow } from './coreflow.classes.coreflow.js'; export class ClusterManager { public coreflowRef: Coreflow; public configSubscription: plugins.smartrx.rxjs.Subscription; public containerSubscription: plugins.smartrx.rxjs.Subscription; public containerVersionSubscription: plugins.smartrx.rxjs.Subscription; public readyDeferred = plugins.smartpromise.defer(); public commonDockerData = { networkNames: { sznWebgateway: 'sznwebgateway', sznCorechat: 'szncorechat', }, }; constructor(coreflowRefArg: Coreflow) { this.coreflowRef = coreflowRefArg; } /** * starts the cluster manager */ public async start() { const config = await this.coreflowRef.cloudlyConnector.getConfigFromCloudly(); this.readyDeferred.resolve(); // subscriptions // this subscription is the start point for most updates on the cluster this.configSubscription = this.coreflowRef.cloudlyConnector.cloudlyApiClient.configUpdateSubject.subscribe( async (dataArg) => { this.coreflowRef.taskManager.updateBaseServicesTask.trigger(); }, ); } /** * stops the clustermanager */ public async stop() { this.configSubscription ? this.configSubscription.unsubscribe() : null; } /** * provisions base services */ public async provisionBaseServices() { // swarm should be enabled by lower level serverconfig package // get current situation const networks = await this.coreflowRef.dockerHost.getNetworks(); logger.log('info', 'There are currently ' + networks.length + ' networks'); for (const network of networks) { logger.log('info', 'Network: ' + network.Name); } // make sure there is a network for the webgateway let sznWebgatewayNetwork = await plugins.docker.DockerNetwork.getNetworkByName( this.coreflowRef.dockerHost, this.commonDockerData.networkNames.sznWebgateway, ); if (!sznWebgatewayNetwork) { logger.log('info', 'Creating network: ' + this.commonDockerData.networkNames.sznWebgateway); sznWebgatewayNetwork = await plugins.docker.DockerNetwork.createNetwork( this.coreflowRef.dockerHost, { Name: this.commonDockerData.networkNames.sznWebgateway, }, ); } else { logger.log('ok', 'sznWebgateway is already present'); } // corechat network so base services can talk to each other let sznCorechatNetwork = await plugins.docker.DockerNetwork.getNetworkByName( this.coreflowRef.dockerHost, this.commonDockerData.networkNames.sznCorechat, ); if (!sznCorechatNetwork) { sznCorechatNetwork = await plugins.docker.DockerNetwork.createNetwork( this.coreflowRef.dockerHost, { Name: this.commonDockerData.networkNames.sznCorechat, }, ); } else { logger.log('ok', 'sznCorechat is already present'); } logger.log('info', `Waiting for networks to be ready...`); await plugins.smartdelay.delayFor(10000); logger.log('success', `Successfully created networks for servezone`); // Images logger.log('info', `now updating docker images of base services...`); const coretrafficImage = await plugins.docker.DockerImage.createFromRegistry( this.coreflowRef.dockerHost, { creationObject: { imageUrl: 'code.foss.global/serve.zone/coretraffic', }, }, ); const corelogImage = await plugins.docker.DockerImage.createFromRegistry( this.coreflowRef.dockerHost, { creationObject: { imageUrl: 'code.foss.global/serve.zone/corelog', }, }, ); // SERVICES // lets deploy the base services // coretraffic let coretrafficService: plugins.docker.DockerService; coretrafficService = await plugins.docker.DockerService.getServiceByName( this.coreflowRef.dockerHost, 'coretraffic', ); if (coretrafficService && (await coretrafficService.needsUpdate())) { await coretrafficService.remove(); await plugins.smartdelay.delayFor(5000); coretrafficService = null; } else { logger.log('ok', `coretraffic service is up to date`); } if (!coretrafficService) { coretrafficService = await plugins.docker.DockerService.createService( this.coreflowRef.dockerHost, { image: coretrafficImage, labels: {}, name: 'coretraffic', networks: [sznCorechatNetwork, sznWebgatewayNetwork], networkAlias: 'coretraffic', ports: ['80:7999', '443:8000'], secrets: [], resources: { memorySizeMB: 1100, volumeMounts: [], }, }, ); } else { logger.log('ok', 'coretraffic service is already present'); } logger.log('info', 'wait for coretraffic to be up and running'); await plugins.smartdelay.delayFor(10000); // corelog let corelogService: plugins.docker.DockerService; corelogService = await plugins.docker.DockerService.getServiceByName( this.coreflowRef.dockerHost, 'corelog', ); if (corelogService && (await corelogService.needsUpdate())) { await corelogService.remove(); corelogService = null; } else { logger.log('ok', `corelog service is up to date`); } if (!corelogService) { corelogService = await plugins.docker.DockerService.createService( this.coreflowRef.dockerHost, { image: corelogImage, labels: {}, name: 'corelog', networks: [sznCorechatNetwork], networkAlias: 'corelog', ports: [], secrets: [], resources: { memorySizeMB: 120, volumeMounts: [], }, }, ); } else { logger.log('ok', 'corelog service is already present'); } logger.log('info', 'waiting for corelog to be up and running'); await plugins.smartdelay.delayFor(10000); } public async provisionWorkloadService( serviceArgFromCloudly: plugins.servezoneInterfaces.data.IService, ) { logger.log( 'info', `deploying service ${serviceArgFromCloudly.data.name}@${serviceArgFromCloudly.data.imageVersion}...`, ); // get the image from cloudly logger.log( 'info', `getting image for ${serviceArgFromCloudly.data.name}@${serviceArgFromCloudly.data.imageVersion}`, ); const containerImageFromCloudly = await this.coreflowRef.cloudlyConnector.cloudlyApiClient.image.getImageById( serviceArgFromCloudly.data.imageId, ); let localDockerImage: plugins.docker.DockerImage; // lets get the docker image for the service if (containerImageFromCloudly.data.location.internal) { const imageStream = await containerImageFromCloudly.pullImageVersion( serviceArgFromCloudly.data.imageVersion, ); localDockerImage = await plugins.docker.DockerImage.createFromTarStream( this.coreflowRef.dockerHost, { creationObject: { imageUrl: containerImageFromCloudly.id, imageTag: serviceArgFromCloudly.data.imageVersion, }, tarStream: plugins.smartstream.nodewebhelpers.convertWebReadableToNodeReadable(imageStream), }, ); } else if ( containerImageFromCloudly.data.location.externalRegistryId && containerImageFromCloudly.data.location.externalImageTag ) { const externalRegistry = await this.coreflowRef.cloudlyConnector.cloudlyApiClient.externalRegistry.getRegistryById( containerImageFromCloudly.data.location.externalRegistryId, ); // Lets authenticate against the external registry // TODO: deduplicate this, check wether we are already authenticated await this.coreflowRef.dockerHost.auth({ username: externalRegistry.data.username, password: externalRegistry.data.password, serveraddress: externalRegistry.data.url, }); localDockerImage = await plugins.docker.DockerImage.createFromRegistry( this.coreflowRef.dockerHost, { creationObject: { imageUrl: containerImageFromCloudly.id, imageTag: serviceArgFromCloudly.data.imageVersion, }, }, ); await localDockerImage.pullLatestImageFromRegistry(); } else { throw new Error('Invalid image location'); } let containerService = await plugins.docker.DockerService.getServiceByName( this.coreflowRef.dockerHost, serviceArgFromCloudly.data.name, ); this.coreflowRef.cloudlyConnector.cloudlyApiClient; const dockerSecretName = `${serviceArgFromCloudly.id}_${serviceArgFromCloudly.data.name}_Secret`; let containerSecret = await plugins.docker.DockerSecret.getSecretByName( this.coreflowRef.dockerHost, dockerSecretName, ); // existing network to connect to const webGatewayNetwork = await plugins.docker.DockerNetwork.getNetworkByName( this.coreflowRef.dockerHost, this.commonDockerData.networkNames.sznWebgateway, ); if (containerService && (await containerService.needsUpdate())) { await containerService.remove(); if (containerSecret) { await containerSecret.remove(); } containerService = null; containerSecret = null; } if (!containerService) { containerSecret = await plugins.docker.DockerSecret.getSecretByName( this.coreflowRef.dockerHost, dockerSecretName, ); if (containerSecret) { await containerSecret.remove(); } const secretBundle = await this.coreflowRef.cloudlyConnector.cloudlyApiClient.secretbundle.getSecretBundleById( serviceArgFromCloudly.data.secretBundleId, ); // lets create the relevant stuff on the docker side containerSecret = await plugins.docker.DockerSecret.createSecret( this.coreflowRef.dockerHost, { name: dockerSecretName, contentArg: JSON.stringify(await secretBundle.getFlatKeyValueObjectForEnvironment()), labels: {}, version: await containerImageFromCloudly.data.versions[serviceArgFromCloudly.data.imageVersion], }, ); containerService = await plugins.docker.DockerService.createService( this.coreflowRef.dockerHost, { name: serviceArgFromCloudly.data.name, image: localDockerImage, networks: [webGatewayNetwork], secrets: [containerSecret], ports: [], labels: {}, resources: serviceArgFromCloudly.data.resources, // TODO: introduce a clean name here, that is guaranteed to work with APIs. networkAlias: serviceArgFromCloudly.data.name, }, ); } } /** * update traffic routing */ public async updateTrafficRouting( clusterConfigArg: plugins.servezoneInterfaces.data.IClusterConfig, ) { const services = await this.coreflowRef.dockerHost.getServices(); const webGatewayNetwork = await plugins.docker.DockerNetwork.getNetworkByName( this.coreflowRef.dockerHost, this.commonDockerData.networkNames.sznWebgateway, ); const reverseProxyConfigs: plugins.servezoneInterfaces.data.IReverseProxyConfig[] = []; const pushProxyConfig = async ( serviceNameArg: string, hostNameArg: string, containerDestinationIp: string, webDestinationPort: string, ) => { logger.log('ok', `trying to obtain a certificate for ${hostNameArg}`); const certificate = await this.coreflowRef.cloudlyConnector.getCertificateForDomainFromCloudly(hostNameArg); reverseProxyConfigs.push({ destinationIp: containerDestinationIp, destinationPort: webDestinationPort, hostName: hostNameArg, privateKey: certificate.privateKey, publicKey: certificate.publicKey, }); logger.log( 'success', `pushed routing config for ${hostNameArg} on workload service ${serviceNameArg}`, ); }; logger.log('info', `Found ${services.length} services!`); for (const service of services) { let workloadConfig: plugins.servezoneInterfaces.data.IClusterConfigContainer; for (const workloadServiceConfig of clusterConfigArg.data.containers) { if (service.Spec.Name === workloadServiceConfig.name) { workloadConfig = workloadServiceConfig; break; } } if (workloadConfig) { logger.log('ok', `found workload service ${service.Spec.Name}`); const containersOfServicesOnNetwork = await webGatewayNetwork.getContainersOnNetworkForService(service); // TODO: make this multi container ready if (!containersOfServicesOnNetwork[0]) { logger.log( 'error', `There seems to be no container available for service ${service.Spec.Name}`, ); continue; } const containerDestinationIp = containersOfServicesOnNetwork[0].IPv4Address.split('/')[0]; const hostNames: string[] = workloadConfig.domains; // lets route the web port const webDestinationPort: string = workloadConfig.ports.web.toString(); for (const hostName of hostNames) { await pushProxyConfig( workloadConfig.name, hostName, containerDestinationIp, webDestinationPort, ); } // lets route custom ports if (workloadConfig.ports.custom) { const customDomainKeys = Object.keys(workloadConfig.ports.custom); for (const customDomainKey of customDomainKeys) { await pushProxyConfig( workloadConfig.name, customDomainKey, containerDestinationIp, workloadConfig.ports.custom[customDomainKey], ); } } } else { logger.log( 'ok', `service ${service.Spec.Name} is not a workload service and won't receive traffic`, ); } } console.log(reverseProxyConfigs); await this.coreflowRef.corechatConnector.setReverseConfigs(reverseProxyConfigs); } }