fix(core): update

This commit is contained in:
2024-05-09 00:05:16 +02:00
commit 73187eb20e
30 changed files with 6666 additions and 0 deletions

8
ts/00_commitinfo_data.ts Normal file
View File

@ -0,0 +1,8 @@
/**
* autocreated commitinfo by @pushrocks/commitinfo
*/
export const commitinfo = {
name: '@servezone_private/coreflow',
version: '1.0.131',
description: 'A comprehensive solution for managing Docker and scaling applications across servers, handling tasks from service provisioning to network traffic management.'
}

View File

@ -0,0 +1,351 @@
import * as plugins from './coreflow.plugins.js';
import { logger } from './coreflow.logging.js';
import { Coreflow } from './coreflow.classes.coreflow.js';
export class ClusterManager {
public coreflowRef: Coreflow;
public dockerHost: plugins.docker.DockerHost;
public configSubscription: plugins.smartrx.rxjs.Subscription;
public containerSubscription: plugins.smartrx.rxjs.Subscription;
public containerVersionSubscription: plugins.smartrx.rxjs.Subscription;
public readyDeferred = plugins.smartpromise.defer();
public commonDockerData = {
networkNames: {
sznWebgateway: 'sznwebgateway',
sznCorechat: 'szncorechat',
},
};
constructor(coreflowRefArg: Coreflow) {
this.coreflowRef = coreflowRefArg;
this.dockerHost = new plugins.docker.DockerHost();
}
/**
* starts the cluster manager
*/
public async start() {
const config = await this.coreflowRef.cloudlyConnector.getConfigFromCloudly();
await this.setDockerAuth(config);
this.readyDeferred.resolve();
// subscriptions
this.configSubscription =
this.coreflowRef.cloudlyConnector.cloudlyClient.configUpdateSubject.subscribe(
async (dataArg) => {
await this.setDockerAuth(dataArg.configData);
this.coreflowRef.taskManager.updateBaseServicesTask.trigger();
}
);
this.containerSubscription =
this.coreflowRef.cloudlyConnector.cloudlyClient.containerUpdateSubject.subscribe(
async (dataArg) => {
this.coreflowRef.taskManager.updateBaseServicesTask.trigger();
}
);
this.containerVersionSubscription =
this.coreflowRef.cloudlyConnector.cloudlyClient.containerVersionUpdateSubject.subscribe(
async (dataArg) => {
console.log(
`Got a container version update trigger for ${dataArg.dockerImageUrl}@${dataArg.dockerImageVersion}`
);
this.coreflowRef.taskManager.updateBaseServicesTask.trigger();
}
);
}
/**
* stops the clustermanager
*/
public async stop() {
this.configSubscription ? this.configSubscription.unsubscribe() : null;
}
public async setDockerAuth(configArg: plugins.servezoneInterfaces.data.IClusterConfig) {
await this.dockerHost.auth(configArg.data.registryInfo);
}
/**
* provisions base services
*/
public async provisionBaseServices() {
// swarm should be enabled by lower level serverconfig package
// get current situation
const networks = await this.dockerHost.getNetworks();
let sznWebgatewayNetwork = await plugins.docker.DockerNetwork.getNetworkByName(
this.dockerHost,
this.commonDockerData.networkNames.sznWebgateway
);
if (!sznWebgatewayNetwork) {
sznWebgatewayNetwork = await plugins.docker.DockerNetwork.createNetwork(this.dockerHost, {
Name: this.commonDockerData.networkNames.sznWebgateway,
});
} else {
logger.log('ok', 'sznWebgateway is already present');
}
// corechat network
let sznCorechatNetwork = await plugins.docker.DockerNetwork.getNetworkByName(
this.dockerHost,
this.commonDockerData.networkNames.sznCorechat
);
if (!sznCorechatNetwork) {
sznCorechatNetwork = await plugins.docker.DockerNetwork.createNetwork(this.dockerHost, {
Name: this.commonDockerData.networkNames.sznCorechat,
});
} else {
logger.log('ok', 'sznCorechat is already present');
}
logger.log('info', `Waiting for networks to be ready...`);
await plugins.smartdelay.delayFor(10000);
logger.log('success', `Successfully created networks for servezone`);
// Images
logger.log('info', `now updating docker images of base services...`);
const coretrafficImage = await plugins.docker.DockerImage.createFromRegistry(this.dockerHost, {
imageUrl: 'registry.gitlab.com/losslessone/services/servezone/coretraffic',
});
const corelogImage = await plugins.docker.DockerImage.createFromRegistry(this.dockerHost, {
imageUrl: 'registry.gitlab.com/losslessone/services/servezone/corelog',
});
// SERVICES
// lets deploy the base services
// coretraffic
let coretrafficService: plugins.docker.DockerService;
coretrafficService = await plugins.docker.DockerService.getServiceByName(
this.dockerHost,
'coretraffic'
);
if (coretrafficService && (await coretrafficService.needsUpdate())) {
await coretrafficService.remove();
await plugins.smartdelay.delayFor(5000);
coretrafficService = null;
} else {
logger.log('ok', `coretraffic service is up to date`);
}
if (!coretrafficService) {
coretrafficService = await plugins.docker.DockerService.createService(this.dockerHost, {
image: coretrafficImage,
labels: {},
name: 'coretraffic',
networks: [sznCorechatNetwork, sznWebgatewayNetwork],
networkAlias: 'coretraffic',
ports: ['80:7999', '443:8000'],
secrets: [],
resources: {
memorySizeMB: 1100,
volumeMounts: [],
},
});
} else {
logger.log('ok', 'coretraffic service is already present');
}
logger.log('info', 'wait for coretraffic to be up and running');
await plugins.smartdelay.delayFor(10000);
// corelog
let corelogService: plugins.docker.DockerService;
corelogService = await plugins.docker.DockerService.getServiceByName(
this.dockerHost,
'corelog'
);
if (corelogService && (await corelogService.needsUpdate())) {
await corelogService.remove();
corelogService = null;
} else {
logger.log('ok', `corelog service is up to date`);
}
if (!corelogService) {
corelogService = await plugins.docker.DockerService.createService(this.dockerHost, {
image: corelogImage,
labels: {},
name: 'corelog',
networks: [sznCorechatNetwork],
networkAlias: 'corelog',
ports: [],
secrets: [],
resources: {
memorySizeMB: 120,
volumeMounts: [],
},
});
} else {
logger.log('ok', 'corelog service is already present');
}
logger.log('info', 'waiting for corelog to be up and running');
await plugins.smartdelay.delayFor(10000);
}
/**
* provision services obtained from cloudly
*/
public async provisionWorkloadServices(configData: plugins.servezoneInterfaces.data.IClusterConfig) {
for (const containerConfig of configData.data.containers) {
await this.provisionSpecificWorkloadService(containerConfig);
}
logger.log('ok', 'Waiting for scheduled workload services to settle');
await plugins.smartdelay.delayFor(10000);
}
public async provisionSpecificWorkloadService(
containerConfigArg: plugins.servezoneInterfaces.data.IClusterConfigContainer
) {
const containerImage = await plugins.docker.DockerImage.createFromRegistry(this.dockerHost, {
imageUrl: containerConfigArg.image,
});
let containerService = await plugins.docker.DockerService.getServiceByName(
this.dockerHost,
containerConfigArg.name
);
let containerSecret = await plugins.docker.DockerSecret.getSecretByName(
this.dockerHost,
`${containerConfigArg.name}Secret`
);
// existing network to connect to
const webGatewayNetwork = await plugins.docker.DockerNetwork.getNetworkByName(
this.dockerHost,
this.commonDockerData.networkNames.sznWebgateway
);
if (containerService && (await containerService.needsUpdate())) {
await containerService.remove();
if (containerSecret) {
await containerSecret.remove();
}
containerService = null;
containerSecret = null;
}
if (!containerService) {
containerSecret = await plugins.docker.DockerSecret.getSecretByName(
this.dockerHost,
`${containerConfigArg.name}Secret`
);
if (containerSecret) {
await containerSecret.remove();
}
containerSecret = await plugins.docker.DockerSecret.createSecret(this.dockerHost, {
name: `${containerConfigArg.name}Secret`,
contentArg: JSON.stringify(containerConfigArg.secrets),
labels: {},
version: await containerImage.getVersion(),
});
containerService = await plugins.docker.DockerService.createService(this.dockerHost, {
name: containerConfigArg.name,
image: containerImage,
networks: [webGatewayNetwork],
secrets: [containerSecret],
ports: [],
labels: {},
resources: containerConfigArg.resources,
networkAlias: containerConfigArg.name,
});
}
}
/**
* update traffic routing
*/
public async updateTrafficRouting(clusterConfigArg: plugins.servezoneInterfaces.data.IClusterConfig) {
const services = await this.dockerHost.getServices();
const webGatewayNetwork = await plugins.docker.DockerNetwork.getNetworkByName(
this.dockerHost,
this.commonDockerData.networkNames.sznWebgateway
);
const reverseProxyConfigs: plugins.servezoneInterfaces.data.IReverseProxyConfig[] = [];
const pushProxyConfig = async (
serviceNameArg: string,
hostNameArg: string,
containerDestinationIp: string,
webDestinationPort: string
) => {
logger.log('ok', `trying to obtain a certificate for ${hostNameArg}`);
const certificate =
await this.coreflowRef.cloudlyConnector.getCertificateForDomainFromCloudly(hostNameArg);
reverseProxyConfigs.push({
destinationIp: containerDestinationIp,
destinationPort: webDestinationPort,
hostName: hostNameArg,
privateKey: certificate.privateKey,
publicKey: certificate.publicKey,
});
logger.log(
'success',
`pushed routing config for ${hostNameArg} on workload service ${serviceNameArg}`
);
};
logger.log('info', `Found ${services.length} services!`);
for (const service of services) {
let workloadConfig: plugins.servezoneInterfaces.data.IClusterConfigContainer;
for (const workloadServiceConfig of clusterConfigArg.data.containers) {
if (service.Spec.Name === workloadServiceConfig.name) {
workloadConfig = workloadServiceConfig;
break;
}
}
if (workloadConfig) {
logger.log('ok', `found workload service ${service.Spec.Name}`);
const containersOfServicesOnNetwork =
await webGatewayNetwork.getContainersOnNetworkForService(service);
// TODO: make this multi container ready
if (!containersOfServicesOnNetwork[0]) {
logger.log(
'error',
`There seems to be no container available for service ${service.Spec.Name}`
);
continue;
}
const containerDestinationIp = containersOfServicesOnNetwork[0].IPv4Address.split('/')[0];
const hostNames: string[] = workloadConfig.domains;
// lets route the web port
const webDestinationPort: string = workloadConfig.ports.web.toString();
for (const hostName of hostNames) {
await pushProxyConfig(
workloadConfig.name,
hostName,
containerDestinationIp,
webDestinationPort
);
}
// lets route custom ports
if (workloadConfig.ports.custom) {
const customDomainKeys = Object.keys(workloadConfig.ports.custom);
for (const customDomainKey of customDomainKeys) {
await pushProxyConfig(
workloadConfig.name,
customDomainKey,
containerDestinationIp,
workloadConfig.ports.custom[customDomainKey]
);
}
}
} else {
logger.log(
'ok',
`service ${service.Spec.Name} is not a workload service and won't receive traffic`
);
}
}
console.log(reverseProxyConfigs);
await this.coreflowRef.corechatConnector.setReverseConfigs(reverseProxyConfigs);
}
}

View File

@ -0,0 +1,75 @@
import * as plugins from './coreflow.plugins.js';
import { logger } from './coreflow.logging.js';
import { CloudlyConnector } from './coreflow.connector.cloudlyconnector.js';
import { ClusterManager } from './coreflow.classes.clustermanager.js';
import { CoreflowTaskmanager } from './coreflow.classes.taskmanager.js';
import { CoretrafficConnector } from './coreflow.connector.coretrafficconnector.js';
import { InternalServer } from './coreflow.classes.internalserver.js';
/**
* the main Coreflow class
*/
export class Coreflow {
public typedrouter = new plugins.typedrequest.TypedRouter();
public internalServer: InternalServer;
public serviceQenv: plugins.qenv.Qenv;
public dockerHost: plugins.docker.DockerHost;
public cloudlyConnector: CloudlyConnector;
public corechatConnector: CoretrafficConnector;
public clusterManager: ClusterManager;
public taskManager: CoreflowTaskmanager;
constructor() {
this.serviceQenv = new plugins.qenv.Qenv('./', './.nogit');
this.dockerHost = new plugins.docker.DockerHost(); // defaults to locally mounted docker sock
this.internalServer = new InternalServer(this);
this.cloudlyConnector = new CloudlyConnector(this);
this.corechatConnector = new CoretrafficConnector(this);
this.clusterManager = new ClusterManager(this);
this.taskManager = new CoreflowTaskmanager(this);
}
/**
* will handle Docker Events
*/
public async handleDockerEvents() {
const eventObservable = await this.dockerHost.getEventObservable();
const eventSubscription = eventObservable.subscribe((event) => {
logger.log('info', `Docker event of type ${event.Type}`);
if (
event.Type === 'image' ||
event.Type === 'network' ||
event.Type === 'container' ||
event.Type === 'service' ||
event.Type === 'node'
) {
console.log('got docker event, but for now not doing anything');
}
});
}
/**
* starts the coreflow instance
*/
public async start() {
await this.internalServer.start();
console.log('internal server started!');
await this.cloudlyConnector.start();
console.log('cloudly connector started!');
await this.clusterManager.start();
console.log('cluster manager started!');
await this.taskManager.start();
console.log('task manager started!');
}
/**
* stops the coreflow instance
*/
public async stop() {
await this.cloudlyConnector.stop();
await this.clusterManager.stop();
await this.taskManager.stop();
await this.internalServer.stop();
}
}

View File

@ -0,0 +1,21 @@
import { Coreflow } from './coreflow.classes.coreflow.js';
import * as plugins from './coreflow.plugins.js';
export class InternalServer {
public coreflowRef: Coreflow;
public typedsocketServer: plugins.typedsocket.TypedSocket;
constructor(coreflowRefArg: Coreflow) {
this.coreflowRef = coreflowRefArg;
}
public async start() {
this.typedsocketServer = await plugins.typedsocket.TypedSocket.createServer(
this.coreflowRef.typedrouter
);
}
public async stop() {
await this.typedsocketServer.stop();
}
}

View File

@ -0,0 +1,94 @@
import * as plugins from './coreflow.plugins.js';
import { Coreflow } from './coreflow.classes.coreflow.js';
import { logger } from './coreflow.logging.js';
export class CoreflowTaskmanager {
public coreflowRef: Coreflow;
public taskmanager: plugins.taskbuffer.TaskManager;
// checkin tasks
public checkinTask: plugins.taskbuffer.Task;
// event based tasks
/**
* updates baseservices
* namely: coretraffic
*/
public updateBaseServicesTask: plugins.taskbuffer.Task;
public updateWorkloadServicesTask: plugins.taskbuffer.Task;
// timed
public updateTrafficRoutingTask: plugins.taskbuffer.Task;
public updateConfigTask: plugins.taskbuffer.Task;
constructor(coreflowRefArg: Coreflow) {
this.coreflowRef = coreflowRefArg;
this.taskmanager = new plugins.taskbuffer.TaskManager();
this.updateBaseServicesTask = new plugins.taskbuffer.Task({
name: 'updateBaseServices',
buffered: true,
bufferMax: 1,
taskFunction: async () => {
logger.log('info', 'running provisioning task for base services');
await this.coreflowRef.clusterManager.provisionBaseServices();
logger.log('success', 'provisioning task for base services completed!');
},
afterTask: () => {
return this.updateWorkloadServicesTask;
},
});
this.updateWorkloadServicesTask = new plugins.taskbuffer.Task({
name: 'updateWorkloadServices',
buffered: true,
bufferMax: 1,
taskFunction: async () => {
logger.log('info', 'now updating workloadServices');
const config = await this.coreflowRef.cloudlyConnector.getConfigFromCloudly();
await this.coreflowRef.clusterManager.provisionWorkloadServices(config);
logger.log('success', 'provisioning task for workload services completed!');
},
afterTask: () => {
return this.updateTrafficRoutingTask;
},
});
this.updateTrafficRoutingTask = new plugins.taskbuffer.Task({
name: 'updateTrafficRouting',
buffered: true,
bufferMax: 1,
taskFunction: async () => {
logger.log('info', 'now updating traffic routing');
const config = await this.coreflowRef.cloudlyConnector.getConfigFromCloudly();
await this.coreflowRef.clusterManager.updateTrafficRouting(config);
logger.log('success', 'traffic routing completed!');
},
});
}
/**
* starts the task manager
*/
public async start() {
logger.log('info', 'starting task manager async in 10 seconds (unrefed)');
plugins.smartdelay.delayFor(10000, null, true).then(async () => {
try {
await this.updateBaseServicesTask.trigger();
logger.log(
'success',
'initial tasks successfully executed! Now handing over to longterm taskmanager!'
);
} catch (e) {
console.log(e);
}
this.taskmanager.addAndScheduleTask(this.updateBaseServicesTask, '0 0 * * * *');
// note: workload services are updated after the fact
this.taskmanager.start();
});
}
public async stop() {
this.taskmanager.stop();
}
}

View File

@ -0,0 +1,53 @@
// This file contains logic to connect to a servezone cloudly instance
// The connection is done using websockets to have an always open channel
// that is bidirectional
import * as plugins from './coreflow.plugins.js';
import { Coreflow } from './coreflow.classes.coreflow.js';
export class CloudlyConnector {
public coreflowRef: Coreflow;
public cloudlyClient: plugins.servezoneApi.CloudlyClient;
public coreflowJumpCode: string;
public identity: plugins.servezoneInterfaces.data.IClusterIdentifier;
constructor(coreflowRefArg: Coreflow) {
this.coreflowRef = coreflowRefArg;
}
public async start() {
this.cloudlyClient = new plugins.servezoneApi.CloudlyClient('coreflow');
await this.cloudlyClient.start();
this.coreflowJumpCode = await this.coreflowRef.serviceQenv.getEnvVarOnDemand('JUMPCODE');
// get identity and tag connection (second parameter is true -> tags the connection)
this.identity = await this.cloudlyClient.getIdentityByJumpCode(this.coreflowJumpCode, true);
}
public async stop() {
await this.cloudlyClient.stop();
}
public async getConfigFromCloudly(): Promise<plugins.servezoneInterfaces.data.IClusterConfig> {
const config = await this.cloudlyClient.getClusterConfigFromCloudlyByIdentity(
this.identity
);
return config;
}
public async triggerConfigEvent() {
const config = await this.getConfigFromCloudly();
this.cloudlyClient.configUpdateSubject.next({
configData: config,
});
}
public async getCertificateForDomainFromCloudly(
domainNameArg: string
): Promise<plugins.tsclass.network.ICert> {
const certificate = await this.cloudlyClient.getCertificateForDomainOverHttps(
domainNameArg
);
return certificate;
}
}

View File

@ -0,0 +1,33 @@
import * as plugins from './coreflow.plugins.js';
import { Coreflow } from './coreflow.classes.coreflow.js';
import { logger } from './coreflow.logging.js';
export class CoretrafficConnector {
public coreflowRef: Coreflow;
public readyDeferred = plugins.smartpromise.defer();
constructor(coreflowRefArg: Coreflow) {
this.coreflowRef = coreflowRefArg;
}
public async start() {
// any work necessary to get connected
this.readyDeferred.resolve();
}
public async setReverseConfigs(
reverseConfigsArg: plugins.servezoneInterfaces.data.IReverseProxyConfig[]
) {
await this.start();
const reactionRequest =
this.coreflowRef.internalServer.typedsocketServer.createTypedRequest<plugins.servezoneInterfaces.requests.routing.IRequest_Coreflow_Coretraffic_RoutingUpdate>(
'updateRouting',
await this.coreflowRef.internalServer.typedsocketServer.findTargetConnection(
async (targetConnection) => targetConnection.alias === 'coretraffic'
)
);
const response = await reactionRequest.fire({
reverseConfigs: reverseConfigsArg,
});
}
}

4
ts/coreflow.info.ts Normal file
View File

@ -0,0 +1,4 @@
import * as plugins from './coreflow.plugins.js';
import * as paths from './coreflow.paths.js';
export const projectInfoNpm = new plugins.projectinfo.ProjectinfoNpm(paths.packageDir);

12
ts/coreflow.logging.ts Normal file
View File

@ -0,0 +1,12 @@
import * as loleLog from '@losslessone_private/lole-log';
import { projectInfoNpm } from './coreflow.info.js';
export const logger = loleLog.createLoleLogger({
companyUnit: 'Lossless Cloud',
containerName: 'coreflow',
containerVersion: projectInfoNpm.version,
sentryAppName: 'coreflow',
sentryDsn: 'https://2a449cd768b34a1b9b5ec34845b4bbd0@sentry.io/1762080',
zone: 'servezone',
});
logger.enableConsole();

3
ts/coreflow.network.ts Normal file
View File

@ -0,0 +1,3 @@
import * as plugins from './coreflow.plugins.js';
export const smartnetworkInstance = new plugins.smartnetwork.SmartNetwork();

6
ts/coreflow.paths.ts Normal file
View File

@ -0,0 +1,6 @@
import * as plugins from './coreflow.plugins.js';
export const packageDir = plugins.path.join(
plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url),
'../'
);

59
ts/coreflow.plugins.ts Normal file
View File

@ -0,0 +1,59 @@
// node native#
import * as path from 'path';
export { path };
// @losslessone_private scope
import * as servezoneInterfaces from '@serve.zone/interfaces';
import * as servezoneApi from '@serve.zone/api';
export { servezoneInterfaces, servezoneApi };
// @apiglobal scope
import * as typedrequest from '@api.global/typedrequest';
import * as typedsocket from '@api.global/typedsocket';
export { typedrequest, typedsocket };
// @pushrocks scope
import * as lik from '@push.rocks/lik';
import * as projectinfo from '@push.rocks/projectinfo';
import * as qenv from '@push.rocks/qenv';
import * as smartcli from '@push.rocks/smartcli';
import * as smartdelay from '@push.rocks/smartdelay';
import * as smartlog from '@push.rocks/smartlog';
import * as smartnetwork from '@push.rocks/smartnetwork';
import * as smartpath from '@push.rocks/smartpath';
import * as smartpromise from '@push.rocks/smartpromise';
import * as smartrequest from '@push.rocks/smartrequest';
import * as smartrx from '@push.rocks/smartrx';
import * as smartstate from '@push.rocks/smartstate';
import * as smartstring from '@push.rocks/smartstring';
import * as taskbuffer from '@push.rocks/taskbuffer';
export {
lik,
projectinfo,
qenv,
smartcli,
smartdelay,
smartlog,
smartnetwork,
smartpath,
smartpromise,
smartrequest,
smartrx,
smartstate,
smartstring,
taskbuffer,
};
// @mojoio scope
import * as docker from '@apiclient.xyz/docker';
export { docker };
// @tsclass scope
import * as tsclass from '@tsclass/tsclass';
export { tsclass };

30
ts/index.ts Normal file
View File

@ -0,0 +1,30 @@
import * as early from '@push.rocks/early';
// early
early.start('coreflow');
import * as plugins from './coreflow.plugins.js';
import * as paths from './coreflow.paths.js';
import { logger } from './coreflow.logging.js';
import { projectInfoNpm } from './coreflow.info.js';
import { smartnetworkInstance } from './coreflow.network.js';
import { Coreflow } from './coreflow.classes.coreflow.js';
early.stop();
// startup
let coreflowInstance: Coreflow;
export const runCli = async () => {
logger.log(
'info',
`trying to start coreflow@v${projectInfoNpm.version} on ${
(await smartnetworkInstance.getPublicIps()).v4
}`
);
coreflowInstance = new Coreflow();
await coreflowInstance.start();
logger.log('success', `coreflow@v${projectInfoNpm.version} successfully started!`);
};
export const stop = async () => {
coreflowInstance.stop();
};