serve.zone/coreflow
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(core): update

Browse Source
This commit is contained in:
Philipp Kunz 2024-05-09 00:05:16 +02:00
commit 73187eb20e
30 changed files with 6666 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.dockerignore Normal file
View File

@ -0,0 +1 @@
node_modules/

20
.gitignore vendored Normal file
View File

@ -0,0 +1,20 @@
.nogit/
# artifacts
coverage/
public/
pages/
# installs
node_modules/
# caches
.yarn/
.cache/
.rpt2_cache
# builds
dist/
dist_*/
# custom

11
.vscode/launch.json vendored Normal file
View File

@ -0,0 +1,11 @@
{
"version": "0.2.0",
"configurations": [
{
"command": "npm test",
"name": "Run npm test",
"request": "launch",
"type": "node-terminal"
}
]
}

26
.vscode/settings.json vendored Normal file
View File

@ -0,0 +1,26 @@
{
"json.schemas": [
{
"fileMatch": ["/npmextra.json"],
"schema": {
"type": "object",
"properties": {
"npmci": {
"type": "object",
"description": "settings for npmci"
},
"gitzone": {
"type": "object",
"description": "settings for gitzone",
"properties": {
"projectType": {
"type": "string",
"enum": ["website", "element", "service", "npm", "wcc"]
}
}
}
}
}
}
]
}

46
Dockerfile Normal file
View File

@ -0,0 +1,46 @@
# gitzone dockerfile_service
## STAGE 1 // BUILD
FROM registry.gitlab.com/hosttoday/ht-docker-node:npmci as node1
COPY ./ /app
WORKDIR /app
ARG NPMCI_TOKEN_NPM2
ENV NPMCI_TOKEN_NPM2 $NPMCI_TOKEN_NPM2
RUN npmci npm prepare
RUN pnpm config set store-dir .pnpm-store
RUN rm -rf node_modules && pnpm install
RUN pnpm run build
# gitzone dockerfile_service
## STAGE 2 // install production
FROM registry.gitlab.com/hosttoday/ht-docker-node:npmci as node2
WORKDIR /app
COPY --from=node1 /app /app
RUN rm -rf .pnpm-store
ARG NPMCI_TOKEN_NPM2
ENV NPMCI_TOKEN_NPM2 $NPMCI_TOKEN_NPM2
RUN npmci npm prepare
RUN pnpm config set store-dir .pnpm-store
RUN rm -rf node_modules/ && pnpm install --prod
## STAGE 3 // rebuild dependencies for alpine
FROM registry.gitlab.com/hosttoday/ht-docker-node:alpinenpmci as node3
WORKDIR /app
COPY --from=node2 /app /app
ARG NPMCI_TOKEN_NPM2
ENV NPMCI_TOKEN_NPM2 $NPMCI_TOKEN_NPM2
RUN npmci npm prepare
RUN pnpm config set store-dir .pnpm-store
RUN pnpm rebuild -r
## STAGE 4 // the final production image with all dependencies in place
FROM registry.gitlab.com/hosttoday/ht-docker-node:alpine as node4
WORKDIR /app
COPY --from=node3 /app /app
### Healthchecks
RUN pnpm install -g @servezone/healthy
HEALTHCHECK --interval=30s --timeout=30s --start-period=30s --retries=3 CMD [ "healthy" ]
EXPOSE 80
CMD ["npm", "start"]

4
cli.child.ts Normal file
View File

@ -0,0 +1,4 @@
#!/usr/bin/env node
process.env.CLI_CALL = 'true';
import * as cliTool from './ts/index.js';
cliTool.runCli();

4
cli.js Normal file
View File

@ -0,0 +1,4 @@
#!/usr/bin/env node
process.env.CLI_CALL = 'true';
const cliTool = await import('./dist_ts/index.js');
cliTool.runCli();

5
cli.ts.js Normal file
View File

@ -0,0 +1,5 @@
#!/usr/bin/env node
process.env.CLI_CALL = 'true';
import * as tsrun from '@git.zone/tsrun';
tsrun.runPath('./cli.child.js', import.meta.url);

21
license Normal file
View File

@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2014 Task Venture Capital GmbH (hello@task.vc)
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

44
npmextra.json Normal file
View File

@ -0,0 +1,44 @@
{
"npmci": {
"npmGlobalTools": [],
"npmAccessLevel": "private",
"npmRegistryUrl": "verdaccio.lossless.one",
"dockerRegistryRepoMap": {
"registry.gitlab.com": "losslessone/services/servezone/coreflow"
},
"dockerBuildargEnvMap": {
"NPMCI_TOKEN_NPM2": "NPMCI_TOKEN_NPM2"
}
},
"gitzone": {
"projectType": "service",
"module": {
"githost": "gitlab.com",
"gitscope": "losslessone/services/servezone",
"gitrepo": "coreflow",
"description": "A comprehensive solution for managing Docker and scaling applications across servers, handling tasks from service provisioning to network traffic management.",
"npmPackagename": "@servezone/coreflow",
"license": "UNLICENSED",
"keywords": [
"Docker",
"Service scaling",
"Networking",
"Server management",
"Continuous deployment",
"Microservices",
"Load balancing",
"Task scheduling",
"Web services",
"Container management",
"Cluster configuration",
"DevOps tools",
"Cloud services integration",
"Security management",
"High availability services",
"Multi-cloud orchestration",
"Service discovery",
"Distributed systems"
]
}
}
}

95
package.json Normal file
View File

@ -0,0 +1,95 @@
{
"name": "@servezone_private/coreflow",
"version": "1.0.130",
"description": "A comprehensive solution for managing Docker and scaling applications across servers, handling tasks from service provisioning to network traffic management.",
"main": "dist_ts/index.js",
"typings": "dist_ts/index.d.ts",
"type": "module",
"scripts": {
"test": "(tstest test/ --web)",
"start": "(node --max_old_space_size=500 ./cli.js)",
"startTs": "(node cli.ts.js)",
"build": "(tsbuild --web --allowimplicitany)",
"buildDocs": "tsdoc"
},
"bin": {
"coreflow": "dist/cli.js"
},
"repository": {
"type": "git",
"url": "git+https://gitlab.com/pushrocks/coreflow.git"
},
"keywords": [
"Docker",
"Service scaling",
"Networking",
"Server management",
"Continuous deployment",
"Microservices",
"Load balancing",
"Task scheduling",
"Web services",
"Container management",
"Cluster configuration",
"DevOps tools",
"Cloud services integration",
"Security management",
"High availability services",
"Multi-cloud orchestration",
"Service discovery",
"Distributed systems"
],
"author": "Lossless GmbH",
"license": "MIT",
"bugs": {
"url": "https://gitlab.com/pushrocks/coreflow/issues"
},
"homepage": "https://gitlab.com/pushrocks/coreflow#readme",
"devDependencies": {
"@git.zone/tsbuild": "^2.1.66",
"@git.zone/tsrun": "^1.2.46",
"@git.zone/tstest": "^1.0.77",
"@git.zone/tswatch": "^2.0.7",
"@push.rocks/tapbundle": "^5.0.8"
},
"dependencies": {
"@api.global/typedrequest": "^3.0.4",
"@api.global/typedsocket": "^3.0.0",
"@apiclient.xyz/docker": "^1.0.105",
"@push.rocks/early": "^4.0.3",
"@push.rocks/lik": "^6.0.2",
"@push.rocks/projectinfo": "^5.0.1",
"@push.rocks/qenv": "^6.0.4",
"@push.rocks/smartcli": "^4.0.6",
"@push.rocks/smartdelay": "^3.0.1",
"@push.rocks/smartlog": "^3.0.2",
"@push.rocks/smartnetwork": "3.0.2",
"@push.rocks/smartpath": "^5.0.5",
"@push.rocks/smartpromise": "^4.0.2",
"@push.rocks/smartrequest": "^2.0.15",
"@push.rocks/smartrx": "^3.0.2",
"@push.rocks/smartstate": "^2.0.6",
"@push.rocks/smartstring": "^4.0.7",
"@push.rocks/taskbuffer": "^3.0.10",
"@serve.zone/api": "^2.0.27",
"@serve.zone/interfaces": "^1.0.13",
"@tsclass/tsclass": "^4.0.46",
"@types/node": "20.11.16"
},
"private": true,
"files": [
"ts/**/*",
"ts_web/**/*",
"dist/**/*",
"dist_*/**/*",
"dist_ts/**/*",
"dist_ts_web/**/*",
"assets/**/*",
"cli.js",
"npmextra.json",
"readme.md"
],
"browserslist": [
"last 1 chrome versions"
]
}

5489
pnpm-lock.yaml generated Normal file
View File

File diff suppressed because it is too large Load Diff

2
qenv.yml Normal file
View File

@ -0,0 +1,2 @@
required:
- JUMPCODE

1
readme.hints.md Normal file
View File

@ -0,0 +1 @@
- this package runs as docker container on a docker swarm cluster and manages containers according to the desired state defined in cloudly.

112
readme.md Normal file
View File

@ -0,0 +1,112 @@
# @servezone_private/coreflow
A comprehensive solution for managing Docker and scaling applications across servers, handling tasks from service provisioning to network traffic management.
## Install
To install @servezone_private/coreflow, you can use npm with the following command:
```sh
npm install @servezone_private/coreflow --save
```
Given that this is a private package, make sure you have access to the required npm registry and that you are authenticated properly.
## Usage
Coreflow is designed as an advanced tool for managing Docker-based applications and services, enabling efficient scaling across servers, and handling multiple aspects of service provisioning and network traffic management. Below are examples and explanations to illustrate its capabilities and how you can leverage Coreflow in your infrastructure. Note that these examples are based on TypeScript and use ESM syntax.
### Prerequisites
Before you start, ensure you have Docker and Docker Swarm configured in your environment as Coreflow operates on top of these technologies. Additionally, verify that your environment variables are properly set up for accessing Coreflow's functionalities.
### Setting Up Coreflow
To get started, you need to import and initialize coreflow within your application. Here's an example of how to do this in a TypeScript module:
```typescript
import { Coreflow } from '@servezone_private/coreflow';
// Initialize Coreflow
const coreflowInstance = new Coreflow();
// Start Coreflow
await coreflowInstance.start();
// Example: Add your logic here for handling Docker events
coreflowInstance.handleDockerEvents().then(() => {
console.log('Docker events are being handled.');
});
// Stop Coreflow when done
await coreflowInstance.stop();
```
### Configuring Service Connections
Coreflow manages applications and services, often requiring direct interactions with other services like a database, message broker, or external API. Coreflow simplifies these connections through its configuration and service discovery layers.
```typescript
// Assuming coreflowInstance is already started as per previous examples
const serviceConnection = coreflowInstance.createServiceConnection({
serviceName: "myDatabaseService",
servicePort: 3306
});
serviceConnection.connect().then(() => {
console.log('Successfully connected to the service');
});
```
### Scaling Your Application
Coreflow excels in scaling applications across multiple servers. This involves not just replicating services, but also ensuring they are properly networked, balanced, and monitored.
```typescript
const scalingPolicy = {
serviceName: "apiService",
replicaCount: 5, // Target number of replicas
maxReplicaCount: 10, // Maximum number of replicas
minReplicaCount: 2, // Minimum number of replicas
};
coreflowInstance.applyScalingPolicy(scalingPolicy).then(() => {
console.log('Scaling policy applied successfully.');
});
```
### Managing Network Traffic
One of Coreflow's key features is its ability to manage network traffic, ensuring that it is efficiently distributed among various services based on load, priority, and other custom rules.
```typescript
import { TrafficRule } from '@servezone_private/coreflow';
const rule: TrafficRule = {
serviceName: "webService",
externalPort: 80,
internalPort: 3000,
protocol: "http",
};
coreflowInstance.applyTrafficRule(rule).then(() => {
console.log('Traffic rule applied successfully.');
});
```
### Continuous Deployment
Coreflow integrates continuous integration and deployment processes, allowing seamless updates and rollbacks for your services:
```typescript
const deploymentConfig = {
serviceName: "userAuthService",
image: "myregistry.com/userauthservice:latest",
updatePolicy: "rolling" // or "recreate"
};
coreflowInstance.deployService(deploymentConfig).then(() => {
console.log('Service deployed successfully.');
});
```
### Observability and Monitoring
To keep track of your applications' health and performances, Coreflow provides tools for logging, monitoring, and alerting.
```typescript
coreflowInstance.monitorService("webService").on('serviceHealthUpdate', (healthStatus) => {
console.log(`Received health update for webService: ${healthStatus}`);
});
```
By following these examples, you can utilize Coreflow to manage Docker-based applications, ensuring scalability, reliability, and efficiency across your infrastructure.
undefined

22
test/test.nonci.ts Normal file
View File

@ -0,0 +1,22 @@
import { tap, expect } from '@push.rocks/tapbundle';
delete process.env.CLI_CALL;
// process.env.CLOUDLY_TESTURL = 'http://localhost:3000';
import * as coreflow from '../ts/index.js';
if (process.env.CI) {
tap.start();
process.exit(0);
}
tap.test('should startup correctly', async () => {
await coreflow.runCli();
});
tap.test('should end correctly', async (tools) => {
await tools.delayFor(2000);
await coreflow.stop();
});
tap.start();

8
ts/00_commitinfo_data.ts Normal file
View File

@ -0,0 +1,8 @@
/**
* autocreated commitinfo by @pushrocks/commitinfo
*/
export const commitinfo = {
name: '@servezone_private/coreflow',
version: '1.0.131',
description: 'A comprehensive solution for managing Docker and scaling applications across servers, handling tasks from service provisioning to network traffic management.'
}

351
ts/coreflow.classes.clustermanager.ts Normal file
View File

@ -0,0 +1,351 @@
import * as plugins from './coreflow.plugins.js';
import { logger } from './coreflow.logging.js';
import { Coreflow } from './coreflow.classes.coreflow.js';
export class ClusterManager {
public coreflowRef: Coreflow;
public dockerHost: plugins.docker.DockerHost;
public configSubscription: plugins.smartrx.rxjs.Subscription;
public containerSubscription: plugins.smartrx.rxjs.Subscription;
public containerVersionSubscription: plugins.smartrx.rxjs.Subscription;
public readyDeferred = plugins.smartpromise.defer();
public commonDockerData = {
networkNames: {
sznWebgateway: 'sznwebgateway',
sznCorechat: 'szncorechat',
},
};
constructor(coreflowRefArg: Coreflow) {
this.coreflowRef = coreflowRefArg;
this.dockerHost = new plugins.docker.DockerHost();
}
/**
* starts the cluster manager
*/
public async start() {
const config = await this.coreflowRef.cloudlyConnector.getConfigFromCloudly();
await this.setDockerAuth(config);
this.readyDeferred.resolve();
// subscriptions
this.configSubscription =
this.coreflowRef.cloudlyConnector.cloudlyClient.configUpdateSubject.subscribe(
async (dataArg) => {
await this.setDockerAuth(dataArg.configData);
this.coreflowRef.taskManager.updateBaseServicesTask.trigger();
}
);
this.containerSubscription =
this.coreflowRef.cloudlyConnector.cloudlyClient.containerUpdateSubject.subscribe(
async (dataArg) => {
this.coreflowRef.taskManager.updateBaseServicesTask.trigger();
}
);
this.containerVersionSubscription =
this.coreflowRef.cloudlyConnector.cloudlyClient.containerVersionUpdateSubject.subscribe(
async (dataArg) => {
console.log(
`Got a container version update trigger for ${dataArg.dockerImageUrl}@${dataArg.dockerImageVersion}`
);
this.coreflowRef.taskManager.updateBaseServicesTask.trigger();
}
);
}
/**
* stops the clustermanager
*/
public async stop() {
this.configSubscription ? this.configSubscription.unsubscribe() : null;
}
public async setDockerAuth(configArg: plugins.servezoneInterfaces.data.IClusterConfig) {
await this.dockerHost.auth(configArg.data.registryInfo);
}
/**
* provisions base services
*/
public async provisionBaseServices() {
// swarm should be enabled by lower level serverconfig package
// get current situation
const networks = await this.dockerHost.getNetworks();
let sznWebgatewayNetwork = await plugins.docker.DockerNetwork.getNetworkByName(
this.dockerHost,
this.commonDockerData.networkNames.sznWebgateway
);
if (!sznWebgatewayNetwork) {
sznWebgatewayNetwork = await plugins.docker.DockerNetwork.createNetwork(this.dockerHost, {
Name: this.commonDockerData.networkNames.sznWebgateway,
});
} else {
logger.log('ok', 'sznWebgateway is already present');
}
// corechat network
let sznCorechatNetwork = await plugins.docker.DockerNetwork.getNetworkByName(
this.dockerHost,
this.commonDockerData.networkNames.sznCorechat
);
if (!sznCorechatNetwork) {
sznCorechatNetwork = await plugins.docker.DockerNetwork.createNetwork(this.dockerHost, {
Name: this.commonDockerData.networkNames.sznCorechat,
});
} else {
logger.log('ok', 'sznCorechat is already present');
}
logger.log('info', `Waiting for networks to be ready...`);
await plugins.smartdelay.delayFor(10000);
logger.log('success', `Successfully created networks for servezone`);
// Images
logger.log('info', `now updating docker images of base services...`);
const coretrafficImage = await plugins.docker.DockerImage.createFromRegistry(this.dockerHost, {
imageUrl: 'registry.gitlab.com/losslessone/services/servezone/coretraffic',
});
const corelogImage = await plugins.docker.DockerImage.createFromRegistry(this.dockerHost, {
imageUrl: 'registry.gitlab.com/losslessone/services/servezone/corelog',
});
// SERVICES
// lets deploy the base services
// coretraffic
let coretrafficService: plugins.docker.DockerService;
coretrafficService = await plugins.docker.DockerService.getServiceByName(
this.dockerHost,
'coretraffic'
);
if (coretrafficService && (await coretrafficService.needsUpdate())) {
await coretrafficService.remove();
await plugins.smartdelay.delayFor(5000);
coretrafficService = null;
} else {
logger.log('ok', `coretraffic service is up to date`);
}
if (!coretrafficService) {
coretrafficService = await plugins.docker.DockerService.createService(this.dockerHost, {
image: coretrafficImage,
labels: {},
name: 'coretraffic',
networks: [sznCorechatNetwork, sznWebgatewayNetwork],
networkAlias: 'coretraffic',
ports: ['80:7999', '443:8000'],
secrets: [],
resources: {
memorySizeMB: 1100,
volumeMounts: [],
},
});
} else {
logger.log('ok', 'coretraffic service is already present');
}
logger.log('info', 'wait for coretraffic to be up and running');
await plugins.smartdelay.delayFor(10000);
// corelog
let corelogService: plugins.docker.DockerService;
corelogService = await plugins.docker.DockerService.getServiceByName(
this.dockerHost,
'corelog'
);
if (corelogService && (await corelogService.needsUpdate())) {
await corelogService.remove();
corelogService = null;
} else {
logger.log('ok', `corelog service is up to date`);
}
if (!corelogService) {
corelogService = await plugins.docker.DockerService.createService(this.dockerHost, {
image: corelogImage,
labels: {},
name: 'corelog',
networks: [sznCorechatNetwork],
networkAlias: 'corelog',
ports: [],
secrets: [],
resources: {
memorySizeMB: 120,
volumeMounts: [],
},
});
} else {
logger.log('ok', 'corelog service is already present');
}
logger.log('info', 'waiting for corelog to be up and running');
await plugins.smartdelay.delayFor(10000);
}
/**
* provision services obtained from cloudly
*/
public async provisionWorkloadServices(configData: plugins.servezoneInterfaces.data.IClusterConfig) {
for (const containerConfig of configData.data.containers) {
await this.provisionSpecificWorkloadService(containerConfig);
}
logger.log('ok', 'Waiting for scheduled workload services to settle');
await plugins.smartdelay.delayFor(10000);
}
public async provisionSpecificWorkloadService(
containerConfigArg: plugins.servezoneInterfaces.data.IClusterConfigContainer
) {
const containerImage = await plugins.docker.DockerImage.createFromRegistry(this.dockerHost, {
imageUrl: containerConfigArg.image,
});
let containerService = await plugins.docker.DockerService.getServiceByName(
this.dockerHost,
containerConfigArg.name
);
let containerSecret = await plugins.docker.DockerSecret.getSecretByName(
this.dockerHost,
`${containerConfigArg.name}Secret`
);
// existing network to connect to
const webGatewayNetwork = await plugins.docker.DockerNetwork.getNetworkByName(
this.dockerHost,
this.commonDockerData.networkNames.sznWebgateway
);
if (containerService && (await containerService.needsUpdate())) {
await containerService.remove();
if (containerSecret) {
await containerSecret.remove();
}
containerService = null;
containerSecret = null;
}
if (!containerService) {
containerSecret = await plugins.docker.DockerSecret.getSecretByName(
this.dockerHost,
`${containerConfigArg.name}Secret`
);
if (containerSecret) {
await containerSecret.remove();
}
containerSecret = await plugins.docker.DockerSecret.createSecret(this.dockerHost, {
name: `${containerConfigArg.name}Secret`,
contentArg: JSON.stringify(containerConfigArg.secrets),
labels: {},
version: await containerImage.getVersion(),
});
containerService = await plugins.docker.DockerService.createService(this.dockerHost, {
name: containerConfigArg.name,
image: containerImage,
networks: [webGatewayNetwork],
secrets: [containerSecret],
ports: [],
labels: {},
resources: containerConfigArg.resources,
networkAlias: containerConfigArg.name,
});
}
}
/**
* update traffic routing
*/
public async updateTrafficRouting(clusterConfigArg: plugins.servezoneInterfaces.data.IClusterConfig) {
const services = await this.dockerHost.getServices();
const webGatewayNetwork = await plugins.docker.DockerNetwork.getNetworkByName(
this.dockerHost,
this.commonDockerData.networkNames.sznWebgateway
);
const reverseProxyConfigs: plugins.servezoneInterfaces.data.IReverseProxyConfig[] = [];
const pushProxyConfig = async (
serviceNameArg: string,
hostNameArg: string,
containerDestinationIp: string,
webDestinationPort: string
) => {
logger.log('ok', `trying to obtain a certificate for ${hostNameArg}`);
const certificate =
await this.coreflowRef.cloudlyConnector.getCertificateForDomainFromCloudly(hostNameArg);
reverseProxyConfigs.push({
destinationIp: containerDestinationIp,
destinationPort: webDestinationPort,
hostName: hostNameArg,
privateKey: certificate.privateKey,
publicKey: certificate.publicKey,
});
logger.log(
'success',
`pushed routing config for ${hostNameArg} on workload service ${serviceNameArg}`
);
};
logger.log('info', `Found ${services.length} services!`);
for (const service of services) {
let workloadConfig: plugins.servezoneInterfaces.data.IClusterConfigContainer;
for (const workloadServiceConfig of clusterConfigArg.data.containers) {
if (service.Spec.Name === workloadServiceConfig.name) {
workloadConfig = workloadServiceConfig;
break;
}
}
if (workloadConfig) {
logger.log('ok', `found workload service ${service.Spec.Name}`);
const containersOfServicesOnNetwork =
await webGatewayNetwork.getContainersOnNetworkForService(service);
// TODO: make this multi container ready
if (!containersOfServicesOnNetwork[0]) {
logger.log(
'error',
`There seems to be no container available for service ${service.Spec.Name}`
);
continue;
}
const containerDestinationIp = containersOfServicesOnNetwork[0].IPv4Address.split('/')[0];
const hostNames: string[] = workloadConfig.domains;
// lets route the web port
const webDestinationPort: string = workloadConfig.ports.web.toString();
for (const hostName of hostNames) {
await pushProxyConfig(
workloadConfig.name,
hostName,
containerDestinationIp,
webDestinationPort
);
}
// lets route custom ports
if (workloadConfig.ports.custom) {
const customDomainKeys = Object.keys(workloadConfig.ports.custom);
for (const customDomainKey of customDomainKeys) {
await pushProxyConfig(
workloadConfig.name,
customDomainKey,
containerDestinationIp,
workloadConfig.ports.custom[customDomainKey]
);
}
}
} else {
logger.log(
'ok',
`service ${service.Spec.Name} is not a workload service and won't receive traffic`
);
}
}
console.log(reverseProxyConfigs);
await this.coreflowRef.corechatConnector.setReverseConfigs(reverseProxyConfigs);
}
}

75
ts/coreflow.classes.coreflow.ts Normal file
View File

@ -0,0 +1,75 @@
import * as plugins from './coreflow.plugins.js';
import { logger } from './coreflow.logging.js';
import { CloudlyConnector } from './coreflow.connector.cloudlyconnector.js';
import { ClusterManager } from './coreflow.classes.clustermanager.js';
import { CoreflowTaskmanager } from './coreflow.classes.taskmanager.js';
import { CoretrafficConnector } from './coreflow.connector.coretrafficconnector.js';
import { InternalServer } from './coreflow.classes.internalserver.js';
/**
* the main Coreflow class
*/
export class Coreflow {
public typedrouter = new plugins.typedrequest.TypedRouter();
public internalServer: InternalServer;
public serviceQenv: plugins.qenv.Qenv;
public dockerHost: plugins.docker.DockerHost;
public cloudlyConnector: CloudlyConnector;
public corechatConnector: CoretrafficConnector;
public clusterManager: ClusterManager;
public taskManager: CoreflowTaskmanager;
constructor() {
this.serviceQenv = new plugins.qenv.Qenv('./', './.nogit');
this.dockerHost = new plugins.docker.DockerHost(); // defaults to locally mounted docker sock
this.internalServer = new InternalServer(this);
this.cloudlyConnector = new CloudlyConnector(this);
this.corechatConnector = new CoretrafficConnector(this);
this.clusterManager = new ClusterManager(this);
this.taskManager = new CoreflowTaskmanager(this);
}
/**
* will handle Docker Events
*/
public async handleDockerEvents() {
const eventObservable = await this.dockerHost.getEventObservable();
const eventSubscription = eventObservable.subscribe((event) => {
logger.log('info', `Docker event of type ${event.Type}`);
if (
event.Type === 'image' ||
event.Type === 'network' ||
event.Type === 'container' ||
event.Type === 'service' ||
event.Type === 'node'
) {
console.log('got docker event, but for now not doing anything');
}
});
}
/**
* starts the coreflow instance
*/
public async start() {
await this.internalServer.start();
console.log('internal server started!');
await this.cloudlyConnector.start();
console.log('cloudly connector started!');
await this.clusterManager.start();
console.log('cluster manager started!');
await this.taskManager.start();
console.log('task manager started!');
}
/**
* stops the coreflow instance
*/
public async stop() {
await this.cloudlyConnector.stop();
await this.clusterManager.stop();
await this.taskManager.stop();
await this.internalServer.stop();
}
}

21
ts/coreflow.classes.internalserver.ts Normal file
View File

@ -0,0 +1,21 @@
import { Coreflow } from './coreflow.classes.coreflow.js';
import * as plugins from './coreflow.plugins.js';
export class InternalServer {
public coreflowRef: Coreflow;
public typedsocketServer: plugins.typedsocket.TypedSocket;
constructor(coreflowRefArg: Coreflow) {
this.coreflowRef = coreflowRefArg;
}
public async start() {
this.typedsocketServer = await plugins.typedsocket.TypedSocket.createServer(
this.coreflowRef.typedrouter
);
}
public async stop() {
await this.typedsocketServer.stop();
}
}

94
ts/coreflow.classes.taskmanager.ts Normal file
View File

@ -0,0 +1,94 @@
import * as plugins from './coreflow.plugins.js';
import { Coreflow } from './coreflow.classes.coreflow.js';
import { logger } from './coreflow.logging.js';
export class CoreflowTaskmanager {
public coreflowRef: Coreflow;
public taskmanager: plugins.taskbuffer.TaskManager;
// checkin tasks
public checkinTask: plugins.taskbuffer.Task;
// event based tasks
/**
* updates baseservices
* namely: coretraffic
*/
public updateBaseServicesTask: plugins.taskbuffer.Task;
public updateWorkloadServicesTask: plugins.taskbuffer.Task;
// timed
public updateTrafficRoutingTask: plugins.taskbuffer.Task;
public updateConfigTask: plugins.taskbuffer.Task;
constructor(coreflowRefArg: Coreflow) {
this.coreflowRef = coreflowRefArg;
this.taskmanager = new plugins.taskbuffer.TaskManager();
this.updateBaseServicesTask = new plugins.taskbuffer.Task({
name: 'updateBaseServices',
buffered: true,
bufferMax: 1,
taskFunction: async () => {
logger.log('info', 'running provisioning task for base services');
await this.coreflowRef.clusterManager.provisionBaseServices();
logger.log('success', 'provisioning task for base services completed!');
},
afterTask: () => {
return this.updateWorkloadServicesTask;
},
});
this.updateWorkloadServicesTask = new plugins.taskbuffer.Task({
name: 'updateWorkloadServices',
buffered: true,
bufferMax: 1,
taskFunction: async () => {
logger.log('info', 'now updating workloadServices');
const config = await this.coreflowRef.cloudlyConnector.getConfigFromCloudly();
await this.coreflowRef.clusterManager.provisionWorkloadServices(config);
logger.log('success', 'provisioning task for workload services completed!');
},
afterTask: () => {
return this.updateTrafficRoutingTask;
},
});
this.updateTrafficRoutingTask = new plugins.taskbuffer.Task({
name: 'updateTrafficRouting',
buffered: true,
bufferMax: 1,
taskFunction: async () => {
logger.log('info', 'now updating traffic routing');
const config = await this.coreflowRef.cloudlyConnector.getConfigFromCloudly();
await this.coreflowRef.clusterManager.updateTrafficRouting(config);
logger.log('success', 'traffic routing completed!');
},
});
}
/**
* starts the task manager
*/
public async start() {
logger.log('info', 'starting task manager async in 10 seconds (unrefed)');
plugins.smartdelay.delayFor(10000, null, true).then(async () => {
try {
await this.updateBaseServicesTask.trigger();
logger.log(
'success',
'initial tasks successfully executed! Now handing over to longterm taskmanager!'
);
} catch (e) {
console.log(e);
}
this.taskmanager.addAndScheduleTask(this.updateBaseServicesTask, '0 0 * * * *');
// note: workload services are updated after the fact
this.taskmanager.start();
});
}
public async stop() {
this.taskmanager.stop();
}
}

53
ts/coreflow.connector.cloudlyconnector.ts Normal file
View File

@ -0,0 +1,53 @@
// This file contains logic to connect to a servezone cloudly instance
// The connection is done using websockets to have an always open channel
// that is bidirectional
import * as plugins from './coreflow.plugins.js';
import { Coreflow } from './coreflow.classes.coreflow.js';
export class CloudlyConnector {
public coreflowRef: Coreflow;
public cloudlyClient: plugins.servezoneApi.CloudlyClient;
public coreflowJumpCode: string;
public identity: plugins.servezoneInterfaces.data.IClusterIdentifier;
constructor(coreflowRefArg: Coreflow) {
this.coreflowRef = coreflowRefArg;
}
public async start() {
this.cloudlyClient = new plugins.servezoneApi.CloudlyClient('coreflow');
await this.cloudlyClient.start();
this.coreflowJumpCode = await this.coreflowRef.serviceQenv.getEnvVarOnDemand('JUMPCODE');
// get identity and tag connection (second parameter is true -> tags the connection)
this.identity = await this.cloudlyClient.getIdentityByJumpCode(this.coreflowJumpCode, true);
}
public async stop() {
await this.cloudlyClient.stop();
}
public async getConfigFromCloudly(): Promise<plugins.servezoneInterfaces.data.IClusterConfig> {
const config = await this.cloudlyClient.getClusterConfigFromCloudlyByIdentity(
this.identity
);
return config;
}
public async triggerConfigEvent() {
const config = await this.getConfigFromCloudly();
this.cloudlyClient.configUpdateSubject.next({
configData: config,
});
}
public async getCertificateForDomainFromCloudly(
domainNameArg: string
): Promise<plugins.tsclass.network.ICert> {
const certificate = await this.cloudlyClient.getCertificateForDomainOverHttps(
domainNameArg
);
return certificate;
}
}

33
ts/coreflow.connector.coretrafficconnector.ts Normal file
View File

@ -0,0 +1,33 @@
import * as plugins from './coreflow.plugins.js';
import { Coreflow } from './coreflow.classes.coreflow.js';
import { logger } from './coreflow.logging.js';
export class CoretrafficConnector {
public coreflowRef: Coreflow;
public readyDeferred = plugins.smartpromise.defer();
constructor(coreflowRefArg: Coreflow) {
this.coreflowRef = coreflowRefArg;
}
public async start() {
// any work necessary to get connected
this.readyDeferred.resolve();
}
public async setReverseConfigs(
reverseConfigsArg: plugins.servezoneInterfaces.data.IReverseProxyConfig[]
) {
await this.start();
const reactionRequest =
this.coreflowRef.internalServer.typedsocketServer.createTypedRequest<plugins.servezoneInterfaces.requests.routing.IRequest_Coreflow_Coretraffic_RoutingUpdate>(
'updateRouting',
await this.coreflowRef.internalServer.typedsocketServer.findTargetConnection(
async (targetConnection) => targetConnection.alias === 'coretraffic'
)
);
const response = await reactionRequest.fire({
reverseConfigs: reverseConfigsArg,
});
}
}

4
ts/coreflow.info.ts Normal file
View File

@ -0,0 +1,4 @@
import * as plugins from './coreflow.plugins.js';
import * as paths from './coreflow.paths.js';
export const projectInfoNpm = new plugins.projectinfo.ProjectinfoNpm(paths.packageDir);

12
ts/coreflow.logging.ts Normal file
View File

@ -0,0 +1,12 @@
import * as loleLog from '@losslessone_private/lole-log';
import { projectInfoNpm } from './coreflow.info.js';
export const logger = loleLog.createLoleLogger({
companyUnit: 'Lossless Cloud',
containerName: 'coreflow',
containerVersion: projectInfoNpm.version,
sentryAppName: 'coreflow',
sentryDsn: 'https://2a449cd768b34a1b9b5ec34845b4bbd0@sentry.io/1762080',
zone: 'servezone',
});
logger.enableConsole();

3
ts/coreflow.network.ts Normal file
View File

@ -0,0 +1,3 @@
import * as plugins from './coreflow.plugins.js';
export const smartnetworkInstance = new plugins.smartnetwork.SmartNetwork();

6
ts/coreflow.paths.ts Normal file
View File

@ -0,0 +1,6 @@
import * as plugins from './coreflow.plugins.js';
export const packageDir = plugins.path.join(
plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url),
'../'
);

59
ts/coreflow.plugins.ts Normal file
View File

@ -0,0 +1,59 @@
// node native#
import * as path from 'path';
export { path };
// @losslessone_private scope
import * as servezoneInterfaces from '@serve.zone/interfaces';
import * as servezoneApi from '@serve.zone/api';
export { servezoneInterfaces, servezoneApi };
// @apiglobal scope
import * as typedrequest from '@api.global/typedrequest';
import * as typedsocket from '@api.global/typedsocket';
export { typedrequest, typedsocket };
// @pushrocks scope
import * as lik from '@push.rocks/lik';
import * as projectinfo from '@push.rocks/projectinfo';
import * as qenv from '@push.rocks/qenv';
import * as smartcli from '@push.rocks/smartcli';
import * as smartdelay from '@push.rocks/smartdelay';
import * as smartlog from '@push.rocks/smartlog';
import * as smartnetwork from '@push.rocks/smartnetwork';
import * as smartpath from '@push.rocks/smartpath';
import * as smartpromise from '@push.rocks/smartpromise';
import * as smartrequest from '@push.rocks/smartrequest';
import * as smartrx from '@push.rocks/smartrx';
import * as smartstate from '@push.rocks/smartstate';
import * as smartstring from '@push.rocks/smartstring';
import * as taskbuffer from '@push.rocks/taskbuffer';
export {
lik,
projectinfo,
qenv,
smartcli,
smartdelay,
smartlog,
smartnetwork,
smartpath,
smartpromise,
smartrequest,
smartrx,
smartstate,
smartstring,
taskbuffer,
};
// @mojoio scope
import * as docker from '@apiclient.xyz/docker';
export { docker };
// @tsclass scope
import * as tsclass from '@tsclass/tsclass';
export { tsclass };

30
ts/index.ts Normal file
View File

@ -0,0 +1,30 @@
import * as early from '@push.rocks/early';
// early
early.start('coreflow');
import * as plugins from './coreflow.plugins.js';
import * as paths from './coreflow.paths.js';
import { logger } from './coreflow.logging.js';
import { projectInfoNpm } from './coreflow.info.js';
import { smartnetworkInstance } from './coreflow.network.js';
import { Coreflow } from './coreflow.classes.coreflow.js';
early.stop();
// startup
let coreflowInstance: Coreflow;
export const runCli = async () => {
logger.log(
'info',
`trying to start coreflow@v${projectInfoNpm.version} on ${
(await smartnetworkInstance.getPublicIps()).v4
}`
);
coreflowInstance = new Coreflow();
await coreflowInstance.start();
logger.log('success', `coreflow@v${projectInfoNpm.version} successfully started!`);
};
export const stop = async () => {
coreflowInstance.stop();
};

14
tsconfig.json Normal file
View File

@ -0,0 +1,14 @@
{
"compilerOptions": {
"experimentalDecorators": true,
"useDefineForClassFields": false,
"target": "ES2022",
"module": "NodeNext",
"moduleResolution": "NodeNext",
"esModuleInterop": true,
"verbatimModuleSyntax": true
},
"exclude": [
"dist_*/**/*.d.ts"
]
}