serve.zone/coreflow
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(core): Updated dependencies and refactored ClusterManager and CloudlyConnector for improved Docker management.

Browse Source
This commit is contained in:
Philipp Kunz
2024-12-20 02:58:26 +01:00
parent 51903eff5b
commit 4b7bf5fde4
8 changed files with 1696 additions and 1698 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ts/00_commitinfo_data.ts
View File

@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@serve.zone/coreflow',
version: '1.0.133',
version: '1.0.134',
description: 'A comprehensive tool for managing Docker-based applications and services, enabling efficient scaling, network management, and integration with cloud services.'
}

131
ts/coreflow.classes.clustermanager.ts
View File

@ -4,7 +4,6 @@ import { Coreflow } from './coreflow.classes.coreflow.js';
export class ClusterManager {
public coreflowRef: Coreflow;
public dockerHost: plugins.docker.DockerHost;
public configSubscription: plugins.smartrx.rxjs.Subscription;
public containerSubscription: plugins.smartrx.rxjs.Subscription;
public containerVersionSubscription: plugins.smartrx.rxjs.Subscription;
@ -20,7 +19,6 @@ export class ClusterManager {
constructor(coreflowRefArg: Coreflow) {
this.coreflowRef = coreflowRefArg;
this.dockerHost = new plugins.docker.DockerHost();
}
/**
@ -31,27 +29,13 @@ export class ClusterManager {
this.readyDeferred.resolve();
// subscriptions
// this subscription is the start point for most updates on the cluster
this.configSubscription =
this.coreflowRef.cloudlyConnector.cloudlyClient.configUpdateSubject.subscribe(
this.coreflowRef.cloudlyConnector.cloudlyApiClient.configUpdateSubject.subscribe(
async (dataArg) => {
this.coreflowRef.taskManager.updateBaseServicesTask.trigger();
}
);
this.containerSubscription =
this.coreflowRef.cloudlyConnector.cloudlyClient.containerUpdateSubject.subscribe(
async (dataArg) => {
this.coreflowRef.taskManager.updateBaseServicesTask.trigger();
}
);
this.containerVersionSubscription =
this.coreflowRef.cloudlyConnector.cloudlyClient.containerVersionUpdateSubject.subscribe(
async (dataArg) => {
console.log(
`Got a container version update trigger for ${dataArg.dockerImageUrl}@${dataArg.dockerImageVersion}`
);
this.coreflowRef.taskManager.updateBaseServicesTask.trigger();
}
);
}
/**
@ -67,27 +51,34 @@ export class ClusterManager {
public async provisionBaseServices() {
// swarm should be enabled by lower level serverconfig package
// get current situation
const networks = await this.dockerHost.getNetworks();
const networks = await this.coreflowRef.dockerHost.getNetworks();
logger.log('info', 'There are currently ' + networks.length + ' networks');
for (const network of networks) {
logger.log('info', 'Network: ' + network.Name);
}
// make sure there is a network for the webgateway
let sznWebgatewayNetwork = await plugins.docker.DockerNetwork.getNetworkByName(
this.dockerHost,
this.coreflowRef.dockerHost,
this.commonDockerData.networkNames.sznWebgateway
);
if (!sznWebgatewayNetwork) {
sznWebgatewayNetwork = await plugins.docker.DockerNetwork.createNetwork(this.dockerHost, {
logger.log('info', 'Creating network: ' + this.commonDockerData.networkNames.sznWebgateway);
sznWebgatewayNetwork = await plugins.docker.DockerNetwork.createNetwork(this.coreflowRef.dockerHost, {
Name: this.commonDockerData.networkNames.sznWebgateway,
});
} else {
logger.log('ok', 'sznWebgateway is already present');
}
// corechat network
// corechat network so base services can talk to each other
let sznCorechatNetwork = await plugins.docker.DockerNetwork.getNetworkByName(
this.dockerHost,
this.coreflowRef.dockerHost,
this.commonDockerData.networkNames.sznCorechat
);
if (!sznCorechatNetwork) {
sznCorechatNetwork = await plugins.docker.DockerNetwork.createNetwork(this.dockerHost, {
sznCorechatNetwork = await plugins.docker.DockerNetwork.createNetwork(this.coreflowRef.dockerHost, {
Name: this.commonDockerData.networkNames.sznCorechat,
});
} else {
@ -102,12 +93,16 @@ export class ClusterManager {
// Images
logger.log('info', `now updating docker images of base services...`);
const coretrafficImage = await plugins.docker.DockerImage.createFromRegistry(this.dockerHost, {
imageUrl: 'registry.gitlab.com/losslessone/services/servezone/coretraffic',
const coretrafficImage = await plugins.docker.DockerImage.createFromRegistry(this.coreflowRef.dockerHost, {
creationObject: {
imageUrl: 'code.foss.global/serve.zone/coretraffic',
}
});
const corelogImage = await plugins.docker.DockerImage.createFromRegistry(this.dockerHost, {
imageUrl: 'registry.gitlab.com/losslessone/services/servezone/corelog',
const corelogImage = await plugins.docker.DockerImage.createFromRegistry(this.coreflowRef.dockerHost, {
creationObject: {
imageUrl: 'code.foss.global/serve.zone/corelog',
}
});
// SERVICES
@ -115,7 +110,7 @@ export class ClusterManager {
// coretraffic
let coretrafficService: plugins.docker.DockerService;
coretrafficService = await plugins.docker.DockerService.getServiceByName(
this.dockerHost,
this.coreflowRef.dockerHost,
'coretraffic'
);
@ -128,7 +123,7 @@ export class ClusterManager {
}
if (!coretrafficService) {
coretrafficService = await plugins.docker.DockerService.createService(this.dockerHost, {
coretrafficService = await plugins.docker.DockerService.createService(this.coreflowRef.dockerHost, {
image: coretrafficImage,
labels: {},
name: 'coretraffic',
@ -150,7 +145,7 @@ export class ClusterManager {
// corelog
let corelogService: plugins.docker.DockerService;
corelogService = await plugins.docker.DockerService.getServiceByName(
this.dockerHost,
this.coreflowRef.dockerHost,
'corelog'
);
@ -162,7 +157,7 @@ export class ClusterManager {
}
if (!corelogService) {
corelogService = await plugins.docker.DockerService.createService(this.dockerHost, {
corelogService = await plugins.docker.DockerService.createService(this.coreflowRef.dockerHost, {
image: corelogImage,
labels: {},
name: 'corelog',
@ -182,40 +177,39 @@ export class ClusterManager {
await plugins.smartdelay.delayFor(10000);
}
/**
* provision services obtained from cloudly
*/
public async provisionWorkloadServices(configData: plugins.servezoneInterfaces.data.ICluster) {
// lets get the config + deploymentDirectives
for (const containerConfig of configData.data.containers) {
await this.provisionSpecificWorkloadService(containerConfig);
}
logger.log('ok', 'Waiting for scheduled workload services to settle');
await plugins.smartdelay.delayFor(10000);
}
public async provisionSpecificWorkloadService(
containerConfigArg: plugins.servezoneInterfaces.data.IClusterConfigContainer
public async provisionWorkloadService(
serviceArg: plugins.servezoneInterfaces.data.IService
) {
const containerImage = await plugins.docker.DockerImage.createFromRegistry(this.dockerHost, {
imageUrl: containerConfigArg.image,
logger.log('info', `deploying service ${serviceArg.data.name}@${serviceArg.data.imageVersion}...`);
// get the image from cloudly
logger.log('info', `getting image for ${serviceArg.data.name}@${serviceArg.data.imageVersion}`);
const containerImage = await this.coreflowRef.cloudlyConnector.cloudlyApiClient.images.getImageById(serviceArg.data.imageId);
const imageStream = await containerImage.pullImageVersion(serviceArg.data.imageVersion);
await plugins.docker.DockerImage.createFromTarStream(this.coreflowRef.dockerHost, {
creationObject: {
imageUrl: containerImage.id,
imageTag: serviceArg.data.imageVersion,
},
tarStream: plugins.smartstream.nodewebhelpers.convertWebReadableToNodeReadable(imageStream)
});
let containerService = await plugins.docker.DockerService.getServiceByName(
this.dockerHost,
containerConfigArg.name
this.coreflowRef.dockerHost,
serviceArg.data.name
);
const secretBundleId = serviceArg.data.secretBundleId;
this.coreflowRef.cloudlyConnector.cloudlyApiClient;
const dockerSecretName = `${serviceArg.id}_${serviceArg.data.name}_Secret`;
let containerSecret = await plugins.docker.DockerSecret.getSecretByName(
this.dockerHost,
`${containerConfigArg.name}Secret`
this.coreflowRef.dockerHost,
dockerSecretName
);
// existing network to connect to
const webGatewayNetwork = await plugins.docker.DockerNetwork.getNetworkByName(
this.dockerHost,
this.coreflowRef.dockerHost,
this.commonDockerData.networkNames.sznWebgateway
);
@ -230,27 +224,30 @@ export class ClusterManager {
if (!containerService) {
containerSecret = await plugins.docker.DockerSecret.getSecretByName(
this.dockerHost,
`${containerConfigArg.name}Secret`
this.coreflowRef.dockerHost,
dockerSecretName
);
if (containerSecret) {
await containerSecret.remove();
}
containerSecret = await plugins.docker.DockerSecret.createSecret(this.dockerHost, {
name: `${containerConfigArg.name}Secret`,
contentArg: JSON.stringify(containerConfigArg.secrets),
const secretBundle: plugins.servezoneInterfaces.data.ISecretBundle =this.coreflowRef.cloudlyConnector.cloudlyApiClient.;
containerSecret = await plugins.docker.DockerSecret.createSecret(this.coreflowRef.dockerHost, {
name: dockerSecretName,
contentArg: JSON.stringify(secretBundle.data.),
labels: {},
version: await containerImage.getVersion(),
});
containerService = await plugins.docker.DockerService.createService(this.dockerHost, {
name: containerConfigArg.name,
containerService = await plugins.docker.DockerService.createService(this.coreflowRef.dockerHost, {
name: deploymentDirectiveArg.name,
image: containerImage,
networks: [webGatewayNetwork],
secrets: [containerSecret],
ports: [],
labels: {},
resources: containerConfigArg.resources,
networkAlias: containerConfigArg.name,
resources: deploymentDirectiveArg.resources,
networkAlias: deploymentDirectiveArg.name,
});
}
}
@ -259,9 +256,9 @@ export class ClusterManager {
* update traffic routing
*/
public async updateTrafficRouting(clusterConfigArg: plugins.servezoneInterfaces.data.IClusterConfig) {
const services = await this.dockerHost.getServices();
const services = await this.coreflowRef.dockerHost.getServices();
const webGatewayNetwork = await plugins.docker.DockerNetwork.getNetworkByName(
this.dockerHost,
this.coreflowRef.dockerHost,
this.commonDockerData.networkNames.sznWebgateway
);
const reverseProxyConfigs: plugins.servezoneInterfaces.data.IReverseProxyConfig[] = [];

2
ts/coreflow.classes.coreflow.ts
View File

@ -22,7 +22,7 @@ export class Coreflow {
constructor() {
this.serviceQenv = new plugins.qenv.Qenv('./', './.nogit');
this.dockerHost = new plugins.docker.DockerHost(); // defaults to locally mounted docker sock
this.dockerHost = new plugins.docker.DockerHost({}); // defaults to locally mounted docker sock
this.internalServer = new InternalServer(this);
this.cloudlyConnector = new CloudlyConnector(this);
this.corechatConnector = new CoretrafficConnector(this);

37
ts/coreflow.connector.cloudlyconnector.ts
View File

@ -7,47 +7,48 @@ import { Coreflow } from './coreflow.classes.coreflow.js';
export class CloudlyConnector {
public coreflowRef: Coreflow;
public cloudlyClient: plugins.servezoneApi.CloudlyClient;
public cloudlyApiClient: plugins.servezoneApi.CloudlyApiClient;
public coreflowJumpCode: string;
public identity: plugins.servezoneInterfaces.data.IClusterIdentifier;
public identity: plugins.servezoneInterfaces.data.IIdentity;
constructor(coreflowRefArg: Coreflow) {
this.coreflowRef = coreflowRefArg;
}
public async start() {
this.cloudlyClient = new plugins.servezoneApi.CloudlyClient('coreflow');
await this.cloudlyClient.start();
this.cloudlyApiClient = new plugins.servezoneApi.CloudlyApiClient({
registerAs: 'coreflow',
cloudlyUrl: await this.coreflowRef.serviceQenv.getEnvVarOnDemand('CLOUDLY_URL')
});
await this.cloudlyApiClient.start();
this.coreflowJumpCode = await this.coreflowRef.serviceQenv.getEnvVarOnDemand('JUMPCODE');
// get identity and tag connection (second parameter is true -> tags the connection)
this.identity = await this.cloudlyClient.getIdentityByJumpCode(this.coreflowJumpCode, true);
this.identity = await this.cloudlyApiClient.getIdentityByToken(this.coreflowJumpCode, {
statefullIdentity: true,
tagConnection: true,
});
}
public async stop() {
await this.cloudlyClient.stop();
await this.cloudlyApiClient.stop();
}
public async getConfigFromCloudly(): Promise<plugins.servezoneInterfaces.data.IClusterConfig> {
const config = await this.cloudlyClient.getClusterConfigFromCloudlyByIdentity(
public async getConfigFromCloudly(): Promise<plugins.servezoneInterfaces.data.ICluster> {
const config = await this.cloudlyApiClient.getClusterConfigFromCloudlyByIdentity(
this.identity
);
return config;
}
public async triggerConfigEvent() {
const config = await this.getConfigFromCloudly();
this.cloudlyClient.configUpdateSubject.next({
configData: config,
});
}
public async getCertificateForDomainFromCloudly(
domainNameArg: string
): Promise<plugins.tsclass.network.ICert> {
const certificate = await this.cloudlyClient.getCertificateForDomainOverHttps(
domainNameArg
);
const certificate = await this.cloudlyApiClient.getCertificateForDomain({
identity: this.identity,
domainName: domainNameArg,
type: 'ssl',
});
return certificate;
}
}

2
ts/coreflow.plugins.ts
View File

@ -28,6 +28,7 @@ import * as smartpromise from '@push.rocks/smartpromise';
import * as smartrequest from '@push.rocks/smartrequest';
import * as smartrx from '@push.rocks/smartrx';
import * as smartstate from '@push.rocks/smartstate';
import * as smartstream from '@push.rocks/smartstream';
import * as smartstring from '@push.rocks/smartstring';
import * as taskbuffer from '@push.rocks/taskbuffer';
@ -44,6 +45,7 @@ export {
smartrequest,
smartrx,
smartstate,
smartstream,
smartstring,
taskbuffer,
};