serve.zone/coreflow
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(.gitea/workflows): Add GitHub Actions workflows for Docker build and test

Browse Source
This commit is contained in:
Philipp Kunz
2024-12-29 14:14:46 +01:00
parent bec47150a3
commit 9de86bd382
17 changed files with 396 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ts/00_commitinfo_data.ts
View File

@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@serve.zone/coreflow',
version: '1.0.135',
version: '1.1.0',
description: 'A comprehensive tool for managing Docker-based applications and services, enabling efficient scaling, network management, and integration with cloud services.'
}

245
ts/coreflow.classes.clustermanager.ts
View File

@ -34,7 +34,7 @@ export class ClusterManager {
this.coreflowRef.cloudlyConnector.cloudlyApiClient.configUpdateSubject.subscribe(
async (dataArg) => {
this.coreflowRef.taskManager.updateBaseServicesTask.trigger();
}
},
);
}
@ -60,14 +60,17 @@ export class ClusterManager {
// make sure there is a network for the webgateway
let sznWebgatewayNetwork = await plugins.docker.DockerNetwork.getNetworkByName(
this.coreflowRef.dockerHost,
this.commonDockerData.networkNames.sznWebgateway
this.commonDockerData.networkNames.sznWebgateway,
);
if (!sznWebgatewayNetwork) {
logger.log('info', 'Creating network: ' + this.commonDockerData.networkNames.sznWebgateway);
sznWebgatewayNetwork = await plugins.docker.DockerNetwork.createNetwork(this.coreflowRef.dockerHost, {
Name: this.commonDockerData.networkNames.sznWebgateway,
});
sznWebgatewayNetwork = await plugins.docker.DockerNetwork.createNetwork(
this.coreflowRef.dockerHost,
{
Name: this.commonDockerData.networkNames.sznWebgateway,
},
);
} else {
logger.log('ok', 'sznWebgateway is already present');
}
@ -75,12 +78,15 @@ export class ClusterManager {
// corechat network so base services can talk to each other
let sznCorechatNetwork = await plugins.docker.DockerNetwork.getNetworkByName(
this.coreflowRef.dockerHost,
this.commonDockerData.networkNames.sznCorechat
this.commonDockerData.networkNames.sznCorechat,
);
if (!sznCorechatNetwork) {
sznCorechatNetwork = await plugins.docker.DockerNetwork.createNetwork(this.coreflowRef.dockerHost, {
Name: this.commonDockerData.networkNames.sznCorechat,
});
sznCorechatNetwork = await plugins.docker.DockerNetwork.createNetwork(
this.coreflowRef.dockerHost,
{
Name: this.commonDockerData.networkNames.sznCorechat,
},
);
} else {
logger.log('ok', 'sznCorechat is already present');
}
@ -93,17 +99,23 @@ export class ClusterManager {
// Images
logger.log('info', `now updating docker images of base services...`);
const coretrafficImage = await plugins.docker.DockerImage.createFromRegistry(this.coreflowRef.dockerHost, {
creationObject: {
imageUrl: 'code.foss.global/serve.zone/coretraffic',
}
});
const coretrafficImage = await plugins.docker.DockerImage.createFromRegistry(
this.coreflowRef.dockerHost,
{
creationObject: {
imageUrl: 'code.foss.global/serve.zone/coretraffic',
},
},
);
const corelogImage = await plugins.docker.DockerImage.createFromRegistry(this.coreflowRef.dockerHost, {
creationObject: {
imageUrl: 'code.foss.global/serve.zone/corelog',
}
});
const corelogImage = await plugins.docker.DockerImage.createFromRegistry(
this.coreflowRef.dockerHost,
{
creationObject: {
imageUrl: 'code.foss.global/serve.zone/corelog',
},
},
);
// SERVICES
// lets deploy the base services
@ -111,7 +123,7 @@ export class ClusterManager {
let coretrafficService: plugins.docker.DockerService;
coretrafficService = await plugins.docker.DockerService.getServiceByName(
this.coreflowRef.dockerHost,
'coretraffic'
'coretraffic',
);
if (coretrafficService && (await coretrafficService.needsUpdate())) {
@ -123,19 +135,22 @@ export class ClusterManager {
}
if (!coretrafficService) {
coretrafficService = await plugins.docker.DockerService.createService(this.coreflowRef.dockerHost, {
image: coretrafficImage,
labels: {},
name: 'coretraffic',
networks: [sznCorechatNetwork, sznWebgatewayNetwork],
networkAlias: 'coretraffic',
ports: ['80:7999', '443:8000'],
secrets: [],
resources: {
memorySizeMB: 1100,
volumeMounts: [],
coretrafficService = await plugins.docker.DockerService.createService(
this.coreflowRef.dockerHost,
{
image: coretrafficImage,
labels: {},
name: 'coretraffic',
networks: [sznCorechatNetwork, sznWebgatewayNetwork],
networkAlias: 'coretraffic',
ports: ['80:7999', '443:8000'],
secrets: [],
resources: {
memorySizeMB: 1100,
volumeMounts: [],
},
},
});
);
} else {
logger.log('ok', 'coretraffic service is already present');
}
@ -146,7 +161,7 @@ export class ClusterManager {
let corelogService: plugins.docker.DockerService;
corelogService = await plugins.docker.DockerService.getServiceByName(
this.coreflowRef.dockerHost,
'corelog'
'corelog',
);
if (corelogService && (await corelogService.needsUpdate())) {
@ -157,19 +172,22 @@ export class ClusterManager {
}
if (!corelogService) {
corelogService = await plugins.docker.DockerService.createService(this.coreflowRef.dockerHost, {
image: corelogImage,
labels: {},
name: 'corelog',
networks: [sznCorechatNetwork],
networkAlias: 'corelog',
ports: [],
secrets: [],
resources: {
memorySizeMB: 120,
volumeMounts: [],
corelogService = await plugins.docker.DockerService.createService(
this.coreflowRef.dockerHost,
{
image: corelogImage,
labels: {},
name: 'corelog',
networks: [sznCorechatNetwork],
networkAlias: 'corelog',
ports: [],
secrets: [],
resources: {
memorySizeMB: 120,
volumeMounts: [],
},
},
});
);
} else {
logger.log('ok', 'corelog service is already present');
}
@ -178,30 +196,48 @@ export class ClusterManager {
}
public async provisionWorkloadService(
serviceArgFromCloudly: plugins.servezoneInterfaces.data.IService
serviceArgFromCloudly: plugins.servezoneInterfaces.data.IService,
) {
logger.log('info', `deploying service ${serviceArgFromCloudly.data.name}@${serviceArgFromCloudly.data.imageVersion}...`);
logger.log(
'info',
`deploying service ${serviceArgFromCloudly.data.name}@${serviceArgFromCloudly.data.imageVersion}...`,
);
// get the image from cloudly
logger.log('info', `getting image for ${serviceArgFromCloudly.data.name}@${serviceArgFromCloudly.data.imageVersion}`);
const containerImageFromCloudly = await this.coreflowRef.cloudlyConnector.cloudlyApiClient.image.getImageById(serviceArgFromCloudly.data.imageId);
let localDockerImage: plugins.docker.DockerImage
logger.log(
'info',
`getting image for ${serviceArgFromCloudly.data.name}@${serviceArgFromCloudly.data.imageVersion}`,
);
const containerImageFromCloudly =
await this.coreflowRef.cloudlyConnector.cloudlyApiClient.image.getImageById(
serviceArgFromCloudly.data.imageId,
);
let localDockerImage: plugins.docker.DockerImage;
// lets get the docker image for the service
if (containerImageFromCloudly.data.location.internal) {
const imageStream = await containerImageFromCloudly.pullImageVersion(serviceArgFromCloudly.data.imageVersion);
localDockerImage= await plugins.docker.DockerImage.createFromTarStream(this.coreflowRef.dockerHost, {
creationObject: {
imageUrl: containerImageFromCloudly.id,
imageTag: serviceArgFromCloudly.data.imageVersion,
const imageStream = await containerImageFromCloudly.pullImageVersion(
serviceArgFromCloudly.data.imageVersion,
);
localDockerImage = await plugins.docker.DockerImage.createFromTarStream(
this.coreflowRef.dockerHost,
{
creationObject: {
imageUrl: containerImageFromCloudly.id,
imageTag: serviceArgFromCloudly.data.imageVersion,
},
tarStream:
plugins.smartstream.nodewebhelpers.convertWebReadableToNodeReadable(imageStream),
},
tarStream: plugins.smartstream.nodewebhelpers.convertWebReadableToNodeReadable(imageStream)
});
);
} else if (
containerImageFromCloudly.data.location.externalRegistryId
&& containerImageFromCloudly.data.location.externalImageTag
containerImageFromCloudly.data.location.externalRegistryId &&
containerImageFromCloudly.data.location.externalImageTag
) {
const externalRegistry = await this.coreflowRef.cloudlyConnector.cloudlyApiClient.externalRegistry.getRegistryById(containerImageFromCloudly.data.location.externalRegistryId);
const externalRegistry =
await this.coreflowRef.cloudlyConnector.cloudlyApiClient.externalRegistry.getRegistryById(
containerImageFromCloudly.data.location.externalRegistryId,
);
// Lets authenticate against the external registry
// TODO: deduplicate this, check wether we are already authenticated
await this.coreflowRef.dockerHost.auth({
@ -209,12 +245,15 @@ export class ClusterManager {
password: externalRegistry.data.password,
serveraddress: externalRegistry.data.url,
});
localDockerImage = await plugins.docker.DockerImage.createFromRegistry(this.coreflowRef.dockerHost, {
creationObject: {
imageUrl: containerImageFromCloudly.id,
imageTag: serviceArgFromCloudly.data.imageVersion,
localDockerImage = await plugins.docker.DockerImage.createFromRegistry(
this.coreflowRef.dockerHost,
{
creationObject: {
imageUrl: containerImageFromCloudly.id,
imageTag: serviceArgFromCloudly.data.imageVersion,
},
},
});
);
await localDockerImage.pullLatestImageFromRegistry();
} else {
throw new Error('Invalid image location');
@ -222,20 +261,20 @@ export class ClusterManager {
let containerService = await plugins.docker.DockerService.getServiceByName(
this.coreflowRef.dockerHost,
serviceArgFromCloudly.data.name
serviceArgFromCloudly.data.name,
);
this.coreflowRef.cloudlyConnector.cloudlyApiClient;
const dockerSecretName = `${serviceArgFromCloudly.id}_${serviceArgFromCloudly.data.name}_Secret`;
let containerSecret = await plugins.docker.DockerSecret.getSecretByName(
this.coreflowRef.dockerHost,
dockerSecretName
dockerSecretName,
);
// existing network to connect to
const webGatewayNetwork = await plugins.docker.DockerNetwork.getNetworkByName(
this.coreflowRef.dockerHost,
this.commonDockerData.networkNames.sznWebgateway
this.commonDockerData.networkNames.sznWebgateway,
);
if (containerService && (await containerService.needsUpdate())) {
@ -250,43 +289,55 @@ export class ClusterManager {
if (!containerService) {
containerSecret = await plugins.docker.DockerSecret.getSecretByName(
this.coreflowRef.dockerHost,
dockerSecretName
dockerSecretName,
);
if (containerSecret) {
await containerSecret.remove();
}
const secretBundle = await this.coreflowRef.cloudlyConnector.cloudlyApiClient.secretbundle.getSecretBundleById(serviceArgFromCloudly.data.secretBundleId);
const secretBundle =
await this.coreflowRef.cloudlyConnector.cloudlyApiClient.secretbundle.getSecretBundleById(
serviceArgFromCloudly.data.secretBundleId,
);
// lets create the relevant stuff on the docker side
containerSecret = await plugins.docker.DockerSecret.createSecret(this.coreflowRef.dockerHost, {
name: dockerSecretName,
contentArg: JSON.stringify(await secretBundle.getFlatKeyValueObjectForEnvironment()),
labels: {},
version: await containerImageFromCloudly.data.versions[serviceArgFromCloudly.data.imageVersion],
});
containerService = await plugins.docker.DockerService.createService(this.coreflowRef.dockerHost, {
name: serviceArgFromCloudly.data.name,
image: localDockerImage,
networks: [webGatewayNetwork],
secrets: [containerSecret],
ports: [],
labels: {},
resources: serviceArgFromCloudly.data.resources,
// TODO: introduce a clean name here, that is guaranteed to work with APIs.
networkAlias: serviceArgFromCloudly.data.name,
});
containerSecret = await plugins.docker.DockerSecret.createSecret(
this.coreflowRef.dockerHost,
{
name: dockerSecretName,
contentArg: JSON.stringify(await secretBundle.getFlatKeyValueObjectForEnvironment()),
labels: {},
version:
await containerImageFromCloudly.data.versions[serviceArgFromCloudly.data.imageVersion],
},
);
containerService = await plugins.docker.DockerService.createService(
this.coreflowRef.dockerHost,
{
name: serviceArgFromCloudly.data.name,
image: localDockerImage,
networks: [webGatewayNetwork],
secrets: [containerSecret],
ports: [],
labels: {},
resources: serviceArgFromCloudly.data.resources,
// TODO: introduce a clean name here, that is guaranteed to work with APIs.
networkAlias: serviceArgFromCloudly.data.name,
},
);
}
}
/**
* update traffic routing
*/
public async updateTrafficRouting(clusterConfigArg: plugins.servezoneInterfaces.data.IClusterConfig) {
public async updateTrafficRouting(
clusterConfigArg: plugins.servezoneInterfaces.data.IClusterConfig,
) {
const services = await this.coreflowRef.dockerHost.getServices();
const webGatewayNetwork = await plugins.docker.DockerNetwork.getNetworkByName(
this.coreflowRef.dockerHost,
this.commonDockerData.networkNames.sznWebgateway
this.commonDockerData.networkNames.sznWebgateway,
);
const reverseProxyConfigs: plugins.servezoneInterfaces.data.IReverseProxyConfig[] = [];
@ -294,7 +345,7 @@ export class ClusterManager {
serviceNameArg: string,
hostNameArg: string,
containerDestinationIp: string,
webDestinationPort: string
webDestinationPort: string,
) => {
logger.log('ok', `trying to obtain a certificate for ${hostNameArg}`);
const certificate =
@ -308,7 +359,7 @@ export class ClusterManager {
});
logger.log(
'success',
`pushed routing config for ${hostNameArg} on workload service ${serviceNameArg}`
`pushed routing config for ${hostNameArg} on workload service ${serviceNameArg}`,
);
};
@ -330,7 +381,7 @@ export class ClusterManager {
if (!containersOfServicesOnNetwork[0]) {
logger.log(
'error',
`There seems to be no container available for service ${service.Spec.Name}`
`There seems to be no container available for service ${service.Spec.Name}`,
);
continue;
}
@ -344,7 +395,7 @@ export class ClusterManager {
workloadConfig.name,
hostName,
containerDestinationIp,
webDestinationPort
webDestinationPort,
);
}
@ -356,14 +407,14 @@ export class ClusterManager {
workloadConfig.name,
customDomainKey,
containerDestinationIp,
workloadConfig.ports.custom[customDomainKey]
workloadConfig.ports.custom[customDomainKey],
);
}
}
} else {
logger.log(
'ok',
`service ${service.Spec.Name} is not a workload service and won't receive traffic`
`service ${service.Spec.Name} is not a workload service and won't receive traffic`,
);
}
}

2
ts/coreflow.classes.internalserver.ts
View File

@ -11,7 +11,7 @@ export class InternalServer {
public async start() {
this.typedsocketServer = await plugins.typedsocket.TypedSocket.createServer(
this.coreflowRef.typedrouter
this.coreflowRef.typedrouter,
);
}

2
ts/coreflow.classes.taskmanager.ts
View File

@ -77,7 +77,7 @@ export class CoreflowTaskmanager {
await this.updateBaseServicesTask.trigger();
logger.log(
'success',
'initial tasks successfully executed! Now handing over to longterm taskmanager!'
'initial tasks successfully executed! Now handing over to longterm taskmanager!',
);
} catch (e) {
console.log(e);

8
ts/coreflow.connector.cloudlyconnector.ts
View File

@ -18,7 +18,7 @@ export class CloudlyConnector {
public async start() {
this.cloudlyApiClient = new plugins.servezoneApi.CloudlyApiClient({
registerAs: 'coreflow',
cloudlyUrl: await this.coreflowRef.serviceQenv.getEnvVarOnDemand('CLOUDLY_URL')
cloudlyUrl: await this.coreflowRef.serviceQenv.getEnvVarOnDemand('CLOUDLY_URL'),
});
await this.cloudlyApiClient.start();
this.coreflowJumpCode = await this.coreflowRef.serviceQenv.getEnvVarOnDemand('JUMPCODE');
@ -35,14 +35,12 @@ export class CloudlyConnector {
}
public async getConfigFromCloudly(): Promise<plugins.servezoneInterfaces.data.ICluster> {
const config = await this.cloudlyApiClient.getClusterConfigFromCloudlyByIdentity(
this.identity
);
const config = await this.cloudlyApiClient.getClusterConfigFromCloudlyByIdentity(this.identity);
return config;
}
public async getCertificateForDomainFromCloudly(
domainNameArg: string
domainNameArg: string,
): Promise<plugins.tsclass.network.ICert> {
const certificate = await this.cloudlyApiClient.getCertificateForDomain({
identity: this.identity,

6
ts/coreflow.connector.coretrafficconnector.ts
View File

@ -16,15 +16,15 @@ export class CoretrafficConnector {
}
public async setReverseConfigs(
reverseConfigsArg: plugins.servezoneInterfaces.data.IReverseProxyConfig[]
reverseConfigsArg: plugins.servezoneInterfaces.data.IReverseProxyConfig[],
) {
await this.start();
const reactionRequest =
this.coreflowRef.internalServer.typedsocketServer.createTypedRequest<plugins.servezoneInterfaces.requests.routing.IRequest_Coreflow_Coretraffic_RoutingUpdate>(
'updateRouting',
await this.coreflowRef.internalServer.typedsocketServer.findTargetConnection(
async (targetConnection) => targetConnection.alias === 'coretraffic'
)
async (targetConnection) => targetConnection.alias === 'coretraffic',
),
);
const response = await reactionRequest.fire({
reverseConfigs: reverseConfigsArg,

2
ts/coreflow.paths.ts
View File

@ -2,5 +2,5 @@ import * as plugins from './coreflow.plugins.js';
export const packageDir = plugins.path.join(
plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url),
'../'
'../',
);

2
ts/index.ts
View File

@ -18,7 +18,7 @@ export const runCli = async () => {
'info',
`trying to start coreflow@v${projectInfoNpm.version} on ${
(await smartnetworkInstance.getPublicIps()).v4
}`
}`,
);
coreflowInstance = new Coreflow();
await coreflowInstance.start();