import { execFile } from 'node:child_process'; import { mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { promisify } from 'node:util'; import { tapNodeTools } from '@git.zone/tstest/tapbundle_node'; import { SmartNetwork } from '@push.rocks/smartnetwork'; import * as cloudlyApiClient from '../../../api/ts/index.js'; import { Cloudly, type ICloudlyConfig } from '../../../cloudly/ts/index.js'; import { Coreflow } from '../../../coreflow/ts/coreflow.classes.coreflow.js'; const execFileAsync = promisify(execFile); const scenarioDir = dirname(fileURLToPath(import.meta.url)); const testingDir = resolve(scenarioDir, '../..'); const repoRoot = resolve(testingDir, '..'); const smokeId = `szn-e2e-smoke-${Date.now().toString(36)}`; const buildDir = join(testingDir, '.nogit', 'registry-deploy-on-push', smokeId); const coretrafficServiceName = `${smokeId}-coretraffic`; const coreflowProxyServiceName = 'coreflow'; const stopFunctions: Array<() => Promise> = []; const delayFor = async (millisecondsArg: number) => { await new Promise((resolveArg) => setTimeout(resolveArg, millisecondsArg)); }; const run = async (commandArg: string, argsArg: string[]) => { const { stdout, stderr } = await execFileAsync(commandArg, argsArg, { maxBuffer: 1024 * 1024 * 20, }); if (stdout.trim()) { console.log(stdout.trim()); } if (stderr.trim()) { console.log(stderr.trim()); } }; const waitFor = async (checkFunctionArg: () => boolean | Promise, messageArg: string) => { const startTime = Date.now(); while (Date.now() - startTime < 90000) { if (await checkFunctionArg()) { return; } await delayFor(500); } throw new Error(`Timed out waiting for ${messageArg}`); }; const getDockerSafeName = (valueArg: string, maxLengthArg = 64) => { const safeName = valueArg .replace(/[^a-zA-Z0-9_.-]+/g, '-') .replace(/^[^a-zA-Z0-9]+|[^a-zA-Z0-9]+$/g, '') .slice(0, maxLengthArg) .replace(/[^a-zA-Z0-9]+$/g, ''); return safeName || 'resource'; }; const getWorkloadSecretName = (serviceArg: { id: string; data: { name: string } }) => { const serviceName = getDockerSafeName(serviceArg.data.name, 36); const serviceId = getDockerSafeName(serviceArg.id, 20); return getDockerSafeName(`${serviceName}-${serviceId}-secret`); }; const dockerServiceExists = async (serviceNameArg: string) => { try { await execFileAsync('docker', ['service', 'inspect', serviceNameArg]); return true; } catch { return false; } }; const removeDockerService = async (serviceNameArg: string) => { if (await dockerServiceExists(serviceNameArg)) { await execFileAsync('docker', ['service', 'rm', serviceNameArg]).catch(() => null); await delayFor(2000); } }; const printDockerServiceLogs = async (serviceNameArg: string) => { if (!(await dockerServiceExists(serviceNameArg))) { return; } console.log(`[registry-deploy-on-push] Logs for Docker service ${serviceNameArg}:`); await execFileAsync('docker', ['service', 'logs', '--raw', '--tail', '120', serviceNameArg], { maxBuffer: 1024 * 1024 * 5, }) .then(({ stdout, stderr }) => { if (stdout.trim()) { console.log(stdout.trim()); } if (stderr.trim()) { console.log(stderr.trim()); } }) .catch((error) => { console.log(`[registry-deploy-on-push] Could not read ${serviceNameArg} logs: ${(error as Error).message}`); }); }; const printDockerServicePs = async (serviceNameArg: string) => { if (!(await dockerServiceExists(serviceNameArg))) { return; } console.log(`[registry-deploy-on-push] Tasks for Docker service ${serviceNameArg}:`); await execFileAsync('docker', ['service', 'ps', '--no-trunc', serviceNameArg], { maxBuffer: 1024 * 1024 * 5, }) .then(({ stdout, stderr }) => { if (stdout.trim()) { console.log(stdout.trim()); } if (stderr.trim()) { console.log(stderr.trim()); } }) .catch((error) => { console.log(`[registry-deploy-on-push] Could not read ${serviceNameArg} tasks: ${(error as Error).message}`); }); }; const printDockerNetworkContainers = async (networkNameArg: string) => { console.log(`[registry-deploy-on-push] Containers on Docker network ${networkNameArg}:`); await execFileAsync('docker', ['network', 'inspect', networkNameArg, '--format', '{{json .Containers}}'], { maxBuffer: 1024 * 1024 * 5, }) .then(({ stdout, stderr }) => { if (stdout.trim()) { console.log(stdout.trim()); } if (stderr.trim()) { console.log(stderr.trim()); } }) .catch((error) => { console.log(`[registry-deploy-on-push] Could not inspect ${networkNameArg}: ${(error as Error).message}`); }); }; const getDockerGwbridgeGatewayIp = async () => { const { stdout } = await execFileAsync('docker', [ 'network', 'inspect', 'docker_gwbridge', '--format', '{{(index .IPAM.Config 0).Gateway}}', ]); const gatewayIp = stdout.trim(); if (!gatewayIp) { throw new Error('Could not determine docker_gwbridge gateway IP'); } return gatewayIp; }; const createSelfSignedCertificate = async (domainNameArg: string) => { mkdirSync(buildDir, { recursive: true }); const keyPath = join(buildDir, 'route.key'); const certPath = join(buildDir, 'route.crt'); await run('openssl', [ 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-keyout', keyPath, '-out', certPath, '-subj', `/CN=${domainNameArg}`, '-days', '1', ]); return { privateKey: readFileSync(keyPath, 'utf8'), publicKey: readFileSync(certPath, 'utf8'), }; }; const waitForWorkloadContainer = async ( networkArg: Awaited>, serviceArg: Awaited>, ) => { if (!networkArg) { throw new Error('Missing Docker network while waiting for workload container'); } await waitFor(async () => { const containers = await networkArg.getContainersOnNetworkForService(serviceArg); return containers.length > 0; }, 'workload container on web gateway network'); }; const createCoreflowProxyService = async (corechatNetworkNameArg: string) => { if (await dockerServiceExists(coreflowProxyServiceName)) { throw new Error(`Docker service ${coreflowProxyServiceName} already exists; refusing to overwrite it`); } mkdirSync(buildDir, { recursive: true }); const gatewayIp = await getDockerGwbridgeGatewayIp(); const caddyfilePath = join(buildDir, 'coreflow-proxy.Caddyfile'); writeFileSync(caddyfilePath, `:3000 {\n reverse_proxy ${gatewayIp}:3000\n}\n`); await run('docker', [ 'service', 'create', '--name', coreflowProxyServiceName, '--label', `serve.zone.testing.id=${smokeId}`, '--network', corechatNetworkNameArg, '--mount', `type=bind,source=${caddyfilePath},target=/etc/caddy/Caddyfile,readonly`, 'caddy:2-alpine', ]); }; const createCoretrafficService = async ( corechatNetworkNameArg: string, webGatewayNetworkNameArg: string, httpsPortArg: number, ) => { const coretrafficDir = join(repoRoot, 'coretraffic'); await run('pnpm', ['--dir', coretrafficDir, 'build']); await run('docker', [ 'service', 'create', '--name', coretrafficServiceName, '--label', `serve.zone.testing.id=${smokeId}`, '--network', corechatNetworkNameArg, '--network', webGatewayNetworkNameArg, '--publish', `published=${httpsPortArg},target=8000,protocol=tcp`, '--mount', `type=bind,source=${coretrafficDir},target=/app`, 'node:22-trixie-slim', 'sh', '-lc', 'cd /app && node cli.js', ]); }; const requestCoretrafficRoute = async (domainNameArg: string, httpsPortArg: number) => { const curlArgs = [ '-k', '-sS', '--noproxy', '*', '--max-time', '10', '--resolve', `${domainNameArg}:${httpsPortArg}:127.0.0.1`, '-o', '-', '-w', '\n%{http_code}', `https://${domainNameArg}:${httpsPortArg}/`, ]; const { stdout, stderr } = await execFileAsync('curl', curlArgs); const lines = stdout.trim().split('\n'); const statusCode = lines[lines.length - 1]; const body = lines.slice(0, -1).join('\n'); return { statusCode, body, stderr, }; }; const waitForCoretrafficRoute = async ( domainNameArg: string, httpsPortArg: number, messageArg: string, backendServiceNameArg?: string, ) => { let lastResponse: Awaited> | undefined; let lastError: Error | undefined; try { await waitFor(async () => { try { lastResponse = await requestCoretrafficRoute(domainNameArg, httpsPortArg); lastError = undefined; return lastResponse.statusCode === '200' && /Caddy|serve/i.test(lastResponse.body); } catch (error) { lastError = error as Error; return false; } }, messageArg); } catch (error) { console.log(`[registry-deploy-on-push] Last route response: ${JSON.stringify(lastResponse)}`); if (lastError) { console.log(`[registry-deploy-on-push] Last route error: ${lastError.message}`); } if (backendServiceNameArg) { await printDockerServicePs(backendServiceNameArg); await printDockerServiceLogs(backendServiceNameArg); } await printDockerServicePs(coretrafficServiceName); await printDockerServiceLogs(coretrafficServiceName); await printDockerServicePs(coreflowProxyServiceName); await printDockerServiceLogs(coreflowProxyServiceName); await printDockerNetworkContainers('sznwebgateway'); throw error; } }; const ensureDockerReady = async () => { await run('docker', ['version']); const { stdout } = await execFileAsync('docker', ['info', '--format', '{{.Swarm.LocalNodeState}}']); if (stdout.trim() !== 'active') { throw new Error('Docker Swarm must be active. In Vagrant this is handled by scripts/provision-vm.sh.'); } }; const buildSmokeImage = async (revisionArg: string) => { mkdirSync(buildDir, { recursive: true }); const imageTag = `${smokeId}:${revisionArg}`; writeFileSync( join(buildDir, 'Dockerfile'), `FROM caddy:2-alpine\nLABEL serve.zone.smoke.id="${smokeId}"\nLABEL serve.zone.smoke.revision="${revisionArg}"\n`, ); await run('docker', ['build', '-t', imageTag, buildDir]); return imageTag; }; const dockerImageRemove = async (imageArg: string) => { await execFileAsync('docker', ['image', 'rm', imageArg]).catch(() => null); }; const dockerLogin = async (registryHostArg: string, usernameArg: string, passwordArg: string) => { await new Promise((resolveArg, rejectArg) => { const childProcess = execFile('docker', [ 'login', registryHostArg, '-u', usernameArg, '--password-stdin', ]); childProcess.stdin?.write(passwordArg); childProcess.stdin?.end(); let output = ''; childProcess.stdout?.on('data', (dataArg) => { output += dataArg.toString(); }); childProcess.stderr?.on('data', (dataArg) => { output += dataArg.toString(); }); childProcess.on('error', rejectArg); childProcess.on('exit', (codeArg) => { console.log(output.trim()); if (codeArg === 0) { resolveArg(); } else { rejectArg(new Error(`docker login exited with ${codeArg}`)); } }); }); }; const createCloudlyConfig = async (): Promise => { console.log('[registry-deploy-on-push] Starting isolated MongoDB and S3 helpers'); const smartmongo = await tapNodeTools.createSmartmongo(); stopFunctions.push(async () => { await smartmongo.stopAndDumpToDir(join(testingDir, '.nogit', 'mongodump', smokeId)); }); const smarts3 = await tapNodeTools.createSmarts3(); stopFunctions.push(async () => { await smarts3.stop(); }); const bucketName = `${smokeId}-bucket`; await smarts3.createBucket(bucketName); const smartnetwork = new SmartNetwork(); const publicPort = await smartnetwork.findFreePort(30000, 40000, { randomize: true }); if (!publicPort) { throw new Error('Could not find a free Cloudly scenario port'); } return { environment: 'integration', letsEncryptEmail: 'test@serve.zone', publicUrl: '127.0.0.1', publicPort: String(publicPort), mongoDescriptor: await smartmongo.getMongoDescriptor(), s3Descriptor: await smarts3.getS3Descriptor({ bucketName, }), sslMode: 'none', servezoneAdminaccount: 'smokeadmin:smokepassword', }; }; const main = async () => { let testCloudly: Cloudly | undefined; let testClient: cloudlyApiClient.CloudlyApiClient | undefined; let coreflow: Coreflow | undefined; let subscription: { unsubscribe: () => void } | undefined; let createdWebGatewayNetwork = false; let createdCorechatNetwork = false; let createdCoreflowProxyService = false; let createdCoretrafficService = false; let startedCoreflowInternalServer = false; let serviceName = ''; let registryImageUrl = ''; let localImageRevision1 = ''; let localImageRevision2 = ''; let serviceForCleanup: { id: string; data: { name: string } } | undefined; const routeDomain = `${smokeId}.test`; try { await ensureDockerReady(); const cloudlyConfig = await createCloudlyConfig(); testCloudly = new Cloudly(cloudlyConfig); console.log('[registry-deploy-on-push] Starting Cloudly'); await testCloudly.start(); const machineUser = new testCloudly.authManager.CUser(); machineUser.id = await testCloudly.authManager.CUser.getNewId(); machineUser.data = { type: 'machine', username: 'smoke-admin', password: 'smoke-admin-token', tokens: [ { token: 'smoke-admin-token', expiresAt: Date.now() + 3600 * 1000, assignedRoles: ['admin'], }, ], role: 'admin', }; await machineUser.save(); const cloudlyUrl = `http://${cloudlyConfig.publicUrl}:${cloudlyConfig.publicPort}`; testClient = new cloudlyApiClient.CloudlyApiClient({ registerAs: 'api', cloudlyUrl, }); await testClient.start(); await testClient.getIdentityByToken('smoke-admin-token'); console.log(`[registry-deploy-on-push] Cloudly started at ${cloudlyUrl}`); const cluster = await testClient.cluster.createCluster(`${smokeId} cluster`); const persistedCluster = await testCloudly.clusterManager.getConfigBy_ConfigID(cluster.id); const clusterUser = await testCloudly.authManager.CUser.getInstance({ id: persistedCluster.data.userId, }); const clusterToken = clusterUser.data.tokens?.[0]?.token; if (!clusterToken) { throw new Error('Cluster token was not created'); } const image = await testClient.image.createImage({ name: `${smokeId} image`, description: 'End-to-end registry/Coreflow smoke image', }); const secretBundle = await testClient.secretbundle.createSecretBundle({ name: `${smokeId} secrets`, description: 'End-to-end registry/Coreflow smoke secrets', type: 'service', includedSecretGroupIds: [], includedTags: [], imageClaims: [], authorizations: [ { environment: 'production', secretAccessKey: `${smokeId}-secret-access`, }, ], }); serviceName = smokeId; const service = await testClient.services.createService({ name: serviceName, description: 'End-to-end registry/Coreflow smoke service', imageId: image.id, imageVersion: 'latest', environment: {}, secretBundleId: secretBundle.id, serviceCategory: 'workload', deploymentStrategy: 'custom', scaleFactor: 1, balancingStrategy: 'round-robin', ports: { web: 80, }, domains: [ { name: routeDomain, port: 80, protocol: 'https', }, ], deploymentIds: [], }); serviceForCleanup = service; const registryTarget = await testClient.services.getRegistryTarget(service.id, 'latest'); registryImageUrl = registryTarget.imageUrl; console.log(`[registry-deploy-on-push] Registry target: ${registryImageUrl}`); process.env.CLOUDLY_URL = cloudlyUrl; process.env.JUMPCODE = clusterToken; coreflow = new Coreflow(); await coreflow.dockerHost.start(); await coreflow.internalServer.start(); startedCoreflowInternalServer = true; coreflow.cloudlyConnector.getCertificateForDomainFromCloudly = (async () => { return await createSelfSignedCertificate(routeDomain); }) as any; let webGatewayNetwork = await coreflow.dockerHost.getNetworkByName( coreflow.clusterManager.commonDockerData.networkNames.sznWebgateway, ); if (!webGatewayNetwork) { webGatewayNetwork = await coreflow.dockerHost.createNetwork({ Name: coreflow.clusterManager.commonDockerData.networkNames.sznWebgateway, }); createdWebGatewayNetwork = true; } let corechatNetwork = await coreflow.dockerHost.getNetworkByName( coreflow.clusterManager.commonDockerData.networkNames.sznCorechat, ); if (!corechatNetwork) { corechatNetwork = await coreflow.dockerHost.createNetwork({ Name: coreflow.clusterManager.commonDockerData.networkNames.sznCorechat, }); createdCorechatNetwork = true; } const smartnetwork = new SmartNetwork(); const coretrafficHttpsPort = await smartnetwork.findFreePort(41000, 43000, { randomize: true }); if (!coretrafficHttpsPort) { throw new Error('Could not find a free Coretraffic HTTPS test port'); } await createCoreflowProxyService(coreflow.clusterManager.commonDockerData.networkNames.sznCorechat); createdCoreflowProxyService = true; await createCoretrafficService( coreflow.clusterManager.commonDockerData.networkNames.sznCorechat, coreflow.clusterManager.commonDockerData.networkNames.sznWebgateway, coretrafficHttpsPort, ); createdCoretrafficService = true; await coreflow.cloudlyConnector.start(); console.log('[registry-deploy-on-push] Coreflow connector authenticated and tagged'); try { await waitFor(async () => { try { await coreflow!.corechatConnector.setReverseConfigs([]); return true; } catch { return false; } }, 'coretraffic connection to coreflow'); } catch (error) { await printDockerServiceLogs(coretrafficServiceName); await printDockerServiceLogs(coreflowProxyServiceName); throw error; } const configUpdates: Array> = []; subscription = coreflow.cloudlyConnector.cloudlyApiClient.configUpdateSubject.subscribe((updateArg) => { if (updateArg.services?.some((serviceArg: any) => serviceArg.id === service.id)) { configUpdates.push(updateArg); } }); await dockerLogin(registryTarget.registryHost, 'smoke-admin', 'smoke-admin-token'); localImageRevision1 = await buildSmokeImage('revision1'); await run('docker', ['tag', localImageRevision1, registryImageUrl]); await run('docker', ['push', registryImageUrl]); await waitFor(() => configUpdates.length >= 1, 'first registry config update'); console.log('[registry-deploy-on-push] First docker push produced a Cloudly config update'); let refreshedService = await testClient.services.getServiceById(service.id); await coreflow.clusterManager.provisionWorkloadService(refreshedService as any); let dockerService = await coreflow.dockerHost.getServiceByName(serviceName); await waitForWorkloadContainer(webGatewayNetwork, dockerService); await coreflow.clusterManager.updateTrafficRouting(persistedCluster as any); await waitForCoretrafficRoute( routeDomain, coretrafficHttpsPort, 'coretraffic HTTPS route to first deployment', serviceName, ); console.log(`[registry-deploy-on-push] Coretraffic routed ${routeDomain} to first deployment`); const firstDockerServiceId = dockerService.ID; const firstDigest = dockerService.Spec.Labels['serve.zone.registryDigest']; if (!firstDigest) { throw new Error('First deployment did not record a registry digest label'); } console.log(`[registry-deploy-on-push] First deployment: ${firstDockerServiceId} digest=${firstDigest}`); localImageRevision2 = await buildSmokeImage('revision2'); await run('docker', ['tag', localImageRevision2, registryImageUrl]); await run('docker', ['push', registryImageUrl]); await waitFor(() => configUpdates.length >= 2, 'second registry config update'); console.log('[registry-deploy-on-push] Second docker push produced a Cloudly config update'); refreshedService = await testClient.services.getServiceById(service.id); await coreflow.clusterManager.provisionWorkloadService(refreshedService as any); dockerService = await coreflow.dockerHost.getServiceByName(serviceName); await waitForWorkloadContainer(webGatewayNetwork, dockerService); await coreflow.clusterManager.updateTrafficRouting(persistedCluster as any); await waitForCoretrafficRoute( routeDomain, coretrafficHttpsPort, 'coretraffic HTTPS route to redeployment', serviceName, ); const secondDockerServiceId = dockerService.ID; const secondDigest = dockerService.Spec.Labels['serve.zone.registryDigest']; if (firstDockerServiceId === secondDockerServiceId) { throw new Error('Docker service ID did not change after same-tag digest update'); } if (firstDigest === secondDigest) { throw new Error('Registry digest label did not change after second push'); } console.log(`[registry-deploy-on-push] Redeployment: ${secondDockerServiceId} digest=${secondDigest}`); console.log(`[registry-deploy-on-push] Coretraffic routed ${routeDomain} to redeployment`); console.log('[registry-deploy-on-push] PASS'); } finally { subscription?.unsubscribe(); if (coreflow) { if (createdCoretrafficService) { await removeDockerService(coretrafficServiceName); } if (createdCoreflowProxyService) { await removeDockerService(coreflowProxyServiceName); } if (serviceName) { const dockerService = await coreflow.dockerHost.getServiceByName(serviceName).catch(() => null); if (dockerService) { await dockerService.remove(); await delayFor(5000); } } if (serviceForCleanup) { const dockerSecret = await coreflow.dockerHost .getSecretByName(getWorkloadSecretName(serviceForCleanup)) .catch(() => null); if (dockerSecret) { await dockerSecret.remove(); } } if (createdWebGatewayNetwork) { const webGatewayNetwork = await coreflow.dockerHost .getNetworkByName(coreflow.clusterManager.commonDockerData.networkNames.sznWebgateway) .catch(() => null); if (webGatewayNetwork) { await webGatewayNetwork.remove(); } } if (createdCorechatNetwork) { const corechatNetwork = await coreflow.dockerHost .getNetworkByName(coreflow.clusterManager.commonDockerData.networkNames.sznCorechat) .catch(() => null); if (corechatNetwork) { await corechatNetwork.remove(); } } await coreflow.cloudlyConnector.stop().catch(() => null); if (startedCoreflowInternalServer) { await coreflow.internalServer.stop().catch(() => null); } await coreflow.dockerHost.stop().catch(() => null); } if (testClient) { await testClient.stop().catch(() => null); } if (testCloudly) { await testCloudly.stop().catch(() => null); } await Promise.all(stopFunctions.map((stopFunction) => stopFunction().catch(() => null))); if (registryImageUrl) { await dockerImageRemove(registryImageUrl); } if (localImageRevision1) { await dockerImageRemove(localImageRevision1); } if (localImageRevision2) { await dockerImageRemove(localImageRevision2); } rmSync(buildDir, { force: true, recursive: true }); } }; await main();