testing/scenarios/registry-deploy-on-push/scenario.ts

import { execFile } from 'node:child_process';
import { mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { dirname, join, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { promisify } from 'node:util';

import { tapNodeTools } from '@git.zone/tstest/tapbundle_node';
import { SmartNetwork } from '@push.rocks/smartnetwork';

import * as cloudlyApiClient from '../../../api/ts/index.js';
import { Cloudly, type ICloudlyConfig } from '../../../cloudly/ts/index.js';
import { Coreflow } from '../../../coreflow/ts/coreflow.classes.coreflow.js';

const execFileAsync = promisify(execFile);
const scenarioDir = dirname(fileURLToPath(import.meta.url));
const testingDir = resolve(scenarioDir, '../..');
const repoRoot = resolve(testingDir, '..');
const smokeId = `szn-e2e-smoke-${Date.now().toString(36)}`;
const buildDir = join(testingDir, '.nogit', 'registry-deploy-on-push', smokeId);
const coretrafficServiceName = `${smokeId}-coretraffic`;
const coreflowProxyServiceName = 'coreflow';
const stopFunctions: Array<() => Promise<void>> = [];

const delayFor = async (millisecondsArg: number) => {
  await new Promise((resolveArg) => setTimeout(resolveArg, millisecondsArg));
};

const run = async (commandArg: string, argsArg: string[]) => {
  const { stdout, stderr } = await execFileAsync(commandArg, argsArg, {
    maxBuffer: 1024 * 1024 * 20,
  });
  if (stdout.trim()) {
    console.log(stdout.trim());
  }
  if (stderr.trim()) {
    console.log(stderr.trim());
  }
};

const waitFor = async (checkFunctionArg: () => boolean | Promise<boolean>, messageArg: string) => {
  const startTime = Date.now();
  while (Date.now() - startTime < 90000) {
    if (await checkFunctionArg()) {
      return;
    }
    await delayFor(500);
  }
  throw new Error(`Timed out waiting for ${messageArg}`);
};

const getDockerSafeName = (valueArg: string, maxLengthArg = 64) => {
  const safeName = valueArg
    .replace(/[^a-zA-Z0-9_.-]+/g, '-')
    .replace(/^[^a-zA-Z0-9]+|[^a-zA-Z0-9]+$/g, '')
    .slice(0, maxLengthArg)
    .replace(/[^a-zA-Z0-9]+$/g, '');
  return safeName || 'resource';
};

const getWorkloadSecretName = (serviceArg: { id: string; data: { name: string } }) => {
  const serviceName = getDockerSafeName(serviceArg.data.name, 36);
  const serviceId = getDockerSafeName(serviceArg.id, 20);
  return getDockerSafeName(`${serviceName}-${serviceId}-secret`);
};

const dockerServiceExists = async (serviceNameArg: string) => {
  try {
    await execFileAsync('docker', ['service', 'inspect', serviceNameArg]);
    return true;
  } catch {
    return false;
  }
};

const removeDockerService = async (serviceNameArg: string) => {
  if (await dockerServiceExists(serviceNameArg)) {
    await execFileAsync('docker', ['service', 'rm', serviceNameArg]).catch(() => null);
    await delayFor(2000);
  }
};

const printDockerServiceLogs = async (serviceNameArg: string) => {
  if (!(await dockerServiceExists(serviceNameArg))) {
    return;
  }
  console.log(`[registry-deploy-on-push] Logs for Docker service ${serviceNameArg}:`);
  await execFileAsync('docker', ['service', 'logs', '--raw', '--tail', '120', serviceNameArg], {
    maxBuffer: 1024 * 1024 * 5,
  })
    .then(({ stdout, stderr }) => {
      if (stdout.trim()) {
        console.log(stdout.trim());
      }
      if (stderr.trim()) {
        console.log(stderr.trim());
      }
    })
    .catch((error) => {
      console.log(`[registry-deploy-on-push] Could not read ${serviceNameArg} logs: ${(error as Error).message}`);
    });
};

const printDockerServicePs = async (serviceNameArg: string) => {
  if (!(await dockerServiceExists(serviceNameArg))) {
    return;
  }
  console.log(`[registry-deploy-on-push] Tasks for Docker service ${serviceNameArg}:`);
  await execFileAsync('docker', ['service', 'ps', '--no-trunc', serviceNameArg], {
    maxBuffer: 1024 * 1024 * 5,
  })
    .then(({ stdout, stderr }) => {
      if (stdout.trim()) {
        console.log(stdout.trim());
      }
      if (stderr.trim()) {
        console.log(stderr.trim());
      }
    })
    .catch((error) => {
      console.log(`[registry-deploy-on-push] Could not read ${serviceNameArg} tasks: ${(error as Error).message}`);
    });
};

const printDockerNetworkContainers = async (networkNameArg: string) => {
  console.log(`[registry-deploy-on-push] Containers on Docker network ${networkNameArg}:`);
  await execFileAsync('docker', ['network', 'inspect', networkNameArg, '--format', '{{json .Containers}}'], {
    maxBuffer: 1024 * 1024 * 5,
  })
    .then(({ stdout, stderr }) => {
      if (stdout.trim()) {
        console.log(stdout.trim());
      }
      if (stderr.trim()) {
        console.log(stderr.trim());
      }
    })
    .catch((error) => {
      console.log(`[registry-deploy-on-push] Could not inspect ${networkNameArg}: ${(error as Error).message}`);
    });
};

const getDockerGwbridgeGatewayIp = async () => {
  const { stdout } = await execFileAsync('docker', [
    'network',
    'inspect',
    'docker_gwbridge',
    '--format',
    '{{(index .IPAM.Config 0).Gateway}}',
  ]);
  const gatewayIp = stdout.trim();
  if (!gatewayIp) {
    throw new Error('Could not determine docker_gwbridge gateway IP');
  }
  return gatewayIp;
};

const createSelfSignedCertificate = async (domainNameArg: string) => {
  mkdirSync(buildDir, { recursive: true });
  const keyPath = join(buildDir, 'route.key');
  const certPath = join(buildDir, 'route.crt');
  await run('openssl', [
    'req',
    '-x509',
    '-newkey',
    'rsa:2048',
    '-nodes',
    '-keyout',
    keyPath,
    '-out',
    certPath,
    '-subj',
    `/CN=${domainNameArg}`,
    '-days',
    '1',
  ]);
  return {
    privateKey: readFileSync(keyPath, 'utf8'),
    publicKey: readFileSync(certPath, 'utf8'),
  };
};

const waitForWorkloadContainer = async (
  networkArg: Awaited<ReturnType<Coreflow['dockerHost']['getNetworkByName']>>,
  serviceArg: Awaited<ReturnType<Coreflow['dockerHost']['getServiceByName']>>,
) => {
  if (!networkArg) {
    throw new Error('Missing Docker network while waiting for workload container');
  }
  await waitFor(async () => {
    const containers = await networkArg.getContainersOnNetworkForService(serviceArg);
    return containers.length > 0;
  }, 'workload container on web gateway network');
};

const createCoreflowProxyService = async (corechatNetworkNameArg: string) => {
  if (await dockerServiceExists(coreflowProxyServiceName)) {
    throw new Error(`Docker service ${coreflowProxyServiceName} already exists; refusing to overwrite it`);
  }
  mkdirSync(buildDir, { recursive: true });
  const gatewayIp = await getDockerGwbridgeGatewayIp();
  const caddyfilePath = join(buildDir, 'coreflow-proxy.Caddyfile');
  writeFileSync(caddyfilePath, `:3000 {\n  reverse_proxy ${gatewayIp}:3000\n}\n`);
  await run('docker', [
    'service',
    'create',
    '--name',
    coreflowProxyServiceName,
    '--label',
    `serve.zone.testing.id=${smokeId}`,
    '--network',
    corechatNetworkNameArg,
    '--mount',
    `type=bind,source=${caddyfilePath},target=/etc/caddy/Caddyfile,readonly`,
    'caddy:2-alpine',
  ]);
};

const createCoretrafficService = async (
  corechatNetworkNameArg: string,
  webGatewayNetworkNameArg: string,
  httpsPortArg: number,
) => {
  const coretrafficDir = join(repoRoot, 'coretraffic');
  await run('pnpm', ['--dir', coretrafficDir, 'build']);
  await run('docker', [
    'service',
    'create',
    '--name',
    coretrafficServiceName,
    '--label',
    `serve.zone.testing.id=${smokeId}`,
    '--network',
    corechatNetworkNameArg,
    '--network',
    webGatewayNetworkNameArg,
    '--publish',
    `published=${httpsPortArg},target=8000,protocol=tcp`,
    '--mount',
    `type=bind,source=${coretrafficDir},target=/app`,
    'node:22-trixie-slim',
    'sh',
    '-lc',
    'cd /app && node cli.js',
  ]);
};

const requestCoretrafficRoute = async (domainNameArg: string, httpsPortArg: number) => {
  const curlArgs = [
    '-k',
    '-sS',
    '--noproxy',
    '*',
    '--max-time',
    '10',
    '--resolve',
    `${domainNameArg}:${httpsPortArg}:127.0.0.1`,
    '-o',
    '-',
    '-w',
    '\n%{http_code}',
    `https://${domainNameArg}:${httpsPortArg}/`,
  ];
  const { stdout, stderr } = await execFileAsync('curl', curlArgs);
  const lines = stdout.trim().split('\n');
  const statusCode = lines[lines.length - 1];
  const body = lines.slice(0, -1).join('\n');
  return {
    statusCode,
    body,
    stderr,
  };
};

const waitForCoretrafficRoute = async (
  domainNameArg: string,
  httpsPortArg: number,
  messageArg: string,
  backendServiceNameArg?: string,
) => {
  let lastResponse: Awaited<ReturnType<typeof requestCoretrafficRoute>> | undefined;
  let lastError: Error | undefined;
  try {
    await waitFor(async () => {
      try {
        lastResponse = await requestCoretrafficRoute(domainNameArg, httpsPortArg);
        lastError = undefined;
        return lastResponse.statusCode === '200' && /Caddy|serve/i.test(lastResponse.body);
      } catch (error) {
        lastError = error as Error;
        return false;
      }
    }, messageArg);
  } catch (error) {
    console.log(`[registry-deploy-on-push] Last route response: ${JSON.stringify(lastResponse)}`);
    if (lastError) {
      console.log(`[registry-deploy-on-push] Last route error: ${lastError.message}`);
    }
    if (backendServiceNameArg) {
      await printDockerServicePs(backendServiceNameArg);
      await printDockerServiceLogs(backendServiceNameArg);
    }
    await printDockerServicePs(coretrafficServiceName);
    await printDockerServiceLogs(coretrafficServiceName);
    await printDockerServicePs(coreflowProxyServiceName);
    await printDockerServiceLogs(coreflowProxyServiceName);
    await printDockerNetworkContainers('sznwebgateway');
    throw error;
  }
};

const ensureDockerReady = async () => {
  await run('docker', ['version']);
  const { stdout } = await execFileAsync('docker', ['info', '--format', '{{.Swarm.LocalNodeState}}']);
  if (stdout.trim() !== 'active') {
    throw new Error('Docker Swarm must be active. In Vagrant this is handled by scripts/provision-vm.sh.');
  }
};

const buildSmokeImage = async (revisionArg: string) => {
  mkdirSync(buildDir, { recursive: true });
  const imageTag = `${smokeId}:${revisionArg}`;
  writeFileSync(
    join(buildDir, 'Dockerfile'),
    `FROM caddy:2-alpine\nLABEL serve.zone.smoke.id="${smokeId}"\nLABEL serve.zone.smoke.revision="${revisionArg}"\n`,
  );
  await run('docker', ['build', '-t', imageTag, buildDir]);
  return imageTag;
};

const dockerImageRemove = async (imageArg: string) => {
  await execFileAsync('docker', ['image', 'rm', imageArg]).catch(() => null);
};

const dockerLogin = async (registryHostArg: string, usernameArg: string, passwordArg: string) => {
  await new Promise<void>((resolveArg, rejectArg) => {
    const childProcess = execFile('docker', [
      'login',
      registryHostArg,
      '-u',
      usernameArg,
      '--password-stdin',
    ]);
    childProcess.stdin?.write(passwordArg);
    childProcess.stdin?.end();
    let output = '';
    childProcess.stdout?.on('data', (dataArg) => {
      output += dataArg.toString();
    });
    childProcess.stderr?.on('data', (dataArg) => {
      output += dataArg.toString();
    });
    childProcess.on('error', rejectArg);
    childProcess.on('exit', (codeArg) => {
      console.log(output.trim());
      if (codeArg === 0) {
        resolveArg();
      } else {
        rejectArg(new Error(`docker login exited with ${codeArg}`));
      }
    });
  });
};

const createCloudlyConfig = async (): Promise<ICloudlyConfig> => {
  console.log('[registry-deploy-on-push] Starting isolated MongoDB and S3 helpers');
  const smartmongo = await tapNodeTools.createSmartmongo();
  stopFunctions.push(async () => {
    await smartmongo.stopAndDumpToDir(join(testingDir, '.nogit', 'mongodump', smokeId));
  });
  const smarts3 = await tapNodeTools.createSmarts3();
  stopFunctions.push(async () => {
    await smarts3.stop();
  });

  const bucketName = `${smokeId}-bucket`;
  await smarts3.createBucket(bucketName);
  const smartnetwork = new SmartNetwork();
  const publicPort = await smartnetwork.findFreePort(30000, 40000, { randomize: true });
  if (!publicPort) {
    throw new Error('Could not find a free Cloudly scenario port');
  }

  return {
    environment: 'integration',
    letsEncryptEmail: 'test@serve.zone',
    publicUrl: '127.0.0.1',
    publicPort: String(publicPort),
    mongoDescriptor: await smartmongo.getMongoDescriptor(),
    s3Descriptor: await smarts3.getS3Descriptor({
      bucketName,
    }),
    sslMode: 'none',
    servezoneAdminaccount: 'smokeadmin:smokepassword',
  };
};

const main = async () => {
  let testCloudly: Cloudly | undefined;
  let testClient: cloudlyApiClient.CloudlyApiClient | undefined;
  let coreflow: Coreflow | undefined;
  let subscription: { unsubscribe: () => void } | undefined;
  let createdWebGatewayNetwork = false;
  let createdCorechatNetwork = false;
  let createdCoreflowProxyService = false;
  let createdCoretrafficService = false;
  let startedCoreflowInternalServer = false;
  let serviceName = '';
  let registryImageUrl = '';
  let localImageRevision1 = '';
  let localImageRevision2 = '';
  let serviceForCleanup: { id: string; data: { name: string } } | undefined;
  const routeDomain = `${smokeId}.test`;

  try {
    await ensureDockerReady();

    const cloudlyConfig = await createCloudlyConfig();
    testCloudly = new Cloudly(cloudlyConfig);
    console.log('[registry-deploy-on-push] Starting Cloudly');
    await testCloudly.start();

    const machineUser = new testCloudly.authManager.CUser();
    machineUser.id = await testCloudly.authManager.CUser.getNewId();
    machineUser.data = {
      type: 'machine',
      username: 'smoke-admin',
      password: 'smoke-admin-token',
      tokens: [
        {
          token: 'smoke-admin-token',
          expiresAt: Date.now() + 3600 * 1000,
          assignedRoles: ['admin'],
        },
      ],
      role: 'admin',
    };
    await machineUser.save();

    const cloudlyUrl = `http://${cloudlyConfig.publicUrl}:${cloudlyConfig.publicPort}`;
    testClient = new cloudlyApiClient.CloudlyApiClient({
      registerAs: 'api',
      cloudlyUrl,
    });
    await testClient.start();
    await testClient.getIdentityByToken('smoke-admin-token');
    console.log(`[registry-deploy-on-push] Cloudly started at ${cloudlyUrl}`);

    const cluster = await testClient.cluster.createCluster(`${smokeId} cluster`);
    const persistedCluster = await testCloudly.clusterManager.getConfigBy_ConfigID(cluster.id);
    const clusterUser = await testCloudly.authManager.CUser.getInstance({
      id: persistedCluster.data.userId,
    });
    const clusterToken = clusterUser.data.tokens?.[0]?.token;
    if (!clusterToken) {
      throw new Error('Cluster token was not created');
    }

    const image = await testClient.image.createImage({
      name: `${smokeId} image`,
      description: 'End-to-end registry/Coreflow smoke image',
    });
    const secretBundle = await testClient.secretbundle.createSecretBundle({
      name: `${smokeId} secrets`,
      description: 'End-to-end registry/Coreflow smoke secrets',
      type: 'service',
      includedSecretGroupIds: [],
      includedTags: [],
      imageClaims: [],
      authorizations: [
        {
          environment: 'production',
          secretAccessKey: `${smokeId}-secret-access`,
        },
      ],
    });

    serviceName = smokeId;
    const service = await testClient.services.createService({
      name: serviceName,
      description: 'End-to-end registry/Coreflow smoke service',
      imageId: image.id,
      imageVersion: 'latest',
      environment: {},
      secretBundleId: secretBundle.id,
      serviceCategory: 'workload',
      deploymentStrategy: 'custom',
      scaleFactor: 1,
      balancingStrategy: 'round-robin',
      ports: {
        web: 80,
      },
      domains: [
        {
          name: routeDomain,
          port: 80,
          protocol: 'https',
        },
      ],
      deploymentIds: [],
    });
    serviceForCleanup = service;

    const registryTarget = await testClient.services.getRegistryTarget(service.id, 'latest');
    registryImageUrl = registryTarget.imageUrl;
    console.log(`[registry-deploy-on-push] Registry target: ${registryImageUrl}`);

    process.env.CLOUDLY_URL = cloudlyUrl;
    process.env.JUMPCODE = clusterToken;
    coreflow = new Coreflow();
    await coreflow.dockerHost.start();
    await coreflow.internalServer.start();
    startedCoreflowInternalServer = true;
    coreflow.cloudlyConnector.getCertificateForDomainFromCloudly = (async () => {
      return await createSelfSignedCertificate(routeDomain);
    }) as any;

    let webGatewayNetwork = await coreflow.dockerHost.getNetworkByName(
      coreflow.clusterManager.commonDockerData.networkNames.sznWebgateway,
    );
    if (!webGatewayNetwork) {
      webGatewayNetwork = await coreflow.dockerHost.createNetwork({
        Name: coreflow.clusterManager.commonDockerData.networkNames.sznWebgateway,
      });
      createdWebGatewayNetwork = true;
    }
    let corechatNetwork = await coreflow.dockerHost.getNetworkByName(
      coreflow.clusterManager.commonDockerData.networkNames.sznCorechat,
    );
    if (!corechatNetwork) {
      corechatNetwork = await coreflow.dockerHost.createNetwork({
        Name: coreflow.clusterManager.commonDockerData.networkNames.sznCorechat,
      });
      createdCorechatNetwork = true;
    }

    const smartnetwork = new SmartNetwork();
    const coretrafficHttpsPort = await smartnetwork.findFreePort(41000, 43000, { randomize: true });
    if (!coretrafficHttpsPort) {
      throw new Error('Could not find a free Coretraffic HTTPS test port');
    }

    await createCoreflowProxyService(coreflow.clusterManager.commonDockerData.networkNames.sznCorechat);
    createdCoreflowProxyService = true;
    await createCoretrafficService(
      coreflow.clusterManager.commonDockerData.networkNames.sznCorechat,
      coreflow.clusterManager.commonDockerData.networkNames.sznWebgateway,
      coretrafficHttpsPort,
    );
    createdCoretrafficService = true;
    await coreflow.cloudlyConnector.start();
    console.log('[registry-deploy-on-push] Coreflow connector authenticated and tagged');
    try {
      await waitFor(async () => {
        try {
          await coreflow!.corechatConnector.setReverseConfigs([]);
          return true;
        } catch {
          return false;
        }
      }, 'coretraffic connection to coreflow');
    } catch (error) {
      await printDockerServiceLogs(coretrafficServiceName);
      await printDockerServiceLogs(coreflowProxyServiceName);
      throw error;
    }

    const configUpdates: Array<Record<string, any>> = [];
    subscription = coreflow.cloudlyConnector.cloudlyApiClient.configUpdateSubject.subscribe((updateArg) => {
      if (updateArg.services?.some((serviceArg: any) => serviceArg.id === service.id)) {
        configUpdates.push(updateArg);
      }
    });

    await dockerLogin(registryTarget.registryHost, 'smoke-admin', 'smoke-admin-token');
    localImageRevision1 = await buildSmokeImage('revision1');
    await run('docker', ['tag', localImageRevision1, registryImageUrl]);
    await run('docker', ['push', registryImageUrl]);
    await waitFor(() => configUpdates.length >= 1, 'first registry config update');
    console.log('[registry-deploy-on-push] First docker push produced a Cloudly config update');

    let refreshedService = await testClient.services.getServiceById(service.id);
    await coreflow.clusterManager.provisionWorkloadService(refreshedService as any);
    let dockerService = await coreflow.dockerHost.getServiceByName(serviceName);
    await waitForWorkloadContainer(webGatewayNetwork, dockerService);
    await coreflow.clusterManager.updateTrafficRouting(persistedCluster as any);
    await waitForCoretrafficRoute(
      routeDomain,
      coretrafficHttpsPort,
      'coretraffic HTTPS route to first deployment',
      serviceName,
    );
    console.log(`[registry-deploy-on-push] Coretraffic routed ${routeDomain} to first deployment`);
    const firstDockerServiceId = dockerService.ID;
    const firstDigest = dockerService.Spec.Labels['serve.zone.registryDigest'];
    if (!firstDigest) {
      throw new Error('First deployment did not record a registry digest label');
    }
    console.log(`[registry-deploy-on-push] First deployment: ${firstDockerServiceId} digest=${firstDigest}`);

    localImageRevision2 = await buildSmokeImage('revision2');
    await run('docker', ['tag', localImageRevision2, registryImageUrl]);
    await run('docker', ['push', registryImageUrl]);
    await waitFor(() => configUpdates.length >= 2, 'second registry config update');
    console.log('[registry-deploy-on-push] Second docker push produced a Cloudly config update');

    refreshedService = await testClient.services.getServiceById(service.id);
    await coreflow.clusterManager.provisionWorkloadService(refreshedService as any);
    dockerService = await coreflow.dockerHost.getServiceByName(serviceName);
    await waitForWorkloadContainer(webGatewayNetwork, dockerService);
    await coreflow.clusterManager.updateTrafficRouting(persistedCluster as any);
    await waitForCoretrafficRoute(
      routeDomain,
      coretrafficHttpsPort,
      'coretraffic HTTPS route to redeployment',
      serviceName,
    );
    const secondDockerServiceId = dockerService.ID;
    const secondDigest = dockerService.Spec.Labels['serve.zone.registryDigest'];
    if (firstDockerServiceId === secondDockerServiceId) {
      throw new Error('Docker service ID did not change after same-tag digest update');
    }
    if (firstDigest === secondDigest) {
      throw new Error('Registry digest label did not change after second push');
    }
    console.log(`[registry-deploy-on-push] Redeployment: ${secondDockerServiceId} digest=${secondDigest}`);
    console.log(`[registry-deploy-on-push] Coretraffic routed ${routeDomain} to redeployment`);
    console.log('[registry-deploy-on-push] PASS');
  } finally {
    subscription?.unsubscribe();
    if (coreflow) {
      if (createdCoretrafficService) {
        await removeDockerService(coretrafficServiceName);
      }
      if (createdCoreflowProxyService) {
        await removeDockerService(coreflowProxyServiceName);
      }
      if (serviceName) {
        const dockerService = await coreflow.dockerHost.getServiceByName(serviceName).catch(() => null);
        if (dockerService) {
          await dockerService.remove();
          await delayFor(5000);
        }
      }
      if (serviceForCleanup) {
        const dockerSecret = await coreflow.dockerHost
          .getSecretByName(getWorkloadSecretName(serviceForCleanup))
          .catch(() => null);
        if (dockerSecret) {
          await dockerSecret.remove();
        }
      }
      if (createdWebGatewayNetwork) {
        const webGatewayNetwork = await coreflow.dockerHost
          .getNetworkByName(coreflow.clusterManager.commonDockerData.networkNames.sznWebgateway)
          .catch(() => null);
        if (webGatewayNetwork) {
          await webGatewayNetwork.remove();
        }
      }
      if (createdCorechatNetwork) {
        const corechatNetwork = await coreflow.dockerHost
          .getNetworkByName(coreflow.clusterManager.commonDockerData.networkNames.sznCorechat)
          .catch(() => null);
        if (corechatNetwork) {
          await corechatNetwork.remove();
        }
      }
      await coreflow.cloudlyConnector.stop().catch(() => null);
      if (startedCoreflowInternalServer) {
        await coreflow.internalServer.stop().catch(() => null);
      }
      await coreflow.dockerHost.stop().catch(() => null);
    }
    if (testClient) {
      await testClient.stop().catch(() => null);
    }
    if (testCloudly) {
      await testCloudly.stop().catch(() => null);
    }
    await Promise.all(stopFunctions.map((stopFunction) => stopFunction().catch(() => null)));
    if (registryImageUrl) {
      await dockerImageRemove(registryImageUrl);
    }
    if (localImageRevision1) {
      await dockerImageRemove(localImageRevision1);
    }
    if (localImageRevision2) {
      await dockerImageRemove(localImageRevision2);
    }
    rmSync(buildDir, { force: true, recursive: true });
  }
};

await main();