testing/scenarios/onebox-cloudly-appstore-worker/scenario.ts

import { dirname, fromFileUrl, join, resolve } from '@std/path';

import { Onebox } from '../../../onebox/ts/classes/onebox.ts';
import {
  disableManagedDcRouterForScenario,
  useLocalAppStoreForScenario,
} from '../onebox-test-helpers.ts';

const scenarioName = 'onebox-cloudly-appstore-worker';
const smokeId = `cloudly-appstore-${Date.now().toString(36)}`;
const testingDir = resolve(dirname(fromFileUrl(import.meta.url)), '../..');
const repoRoot = resolve(testingDir, '..');
const appCatalogDir = join(repoRoot, 'appstore-apptemplates');
const buildDir = join(testingDir, '.nogit', scenarioName, smokeId);
const serviceName = `cloudly-${Date.now().toString(36)}`;
const dockerServiceName = `onebox-${serviceName}`;
const routeDomain = `${serviceName}.test`;

const delayFor = async (millisecondsArg: number) => {
  await new Promise((resolveArg) => setTimeout(resolveArg, millisecondsArg));
};

const run = async (commandArg: string, argsArg: string[]) => {
  const command = new Deno.Command(commandArg, {
    args: argsArg,
    stdout: 'piped',
    stderr: 'piped',
  });
  const output = await command.output();
  const stdout = new TextDecoder().decode(output.stdout).trim();
  const stderr = new TextDecoder().decode(output.stderr).trim();
  if (stdout) {
    console.log(stdout);
  }
  if (stderr) {
    console.log(stderr);
  }
  if (!output.success) {
    throw new Error(`${commandArg} ${argsArg.join(' ')} exited with ${output.code}`);
  }
  return { stdout, stderr };
};

const waitFor = async (
  checkFunctionArg: () => boolean | Promise<boolean>,
  messageArg: string,
  timeoutMsArg = 180000,
) => {
  const startTime = Date.now();
  while (Date.now() - startTime < timeoutMsArg) {
    if (await checkFunctionArg()) {
      return;
    }
    await delayFor(1000);
  }
  throw new Error(`Timed out waiting for ${messageArg}`);
};

const dockerServiceExists = async (serviceNameArg: string) => {
  const command = new Deno.Command('docker', {
    args: ['service', 'inspect', serviceNameArg],
    stdout: 'null',
    stderr: 'null',
  });
  return (await command.output()).success;
};

const removeDockerService = async (serviceNameArg: string) => {
  if (await dockerServiceExists(serviceNameArg)) {
    await run('docker', ['service', 'rm', serviceNameArg]).catch(() => null);
    await delayFor(2000);
  }
};

const dockerContainerRunning = async (containerNameArg: string) => {
  const command = new Deno.Command('docker', {
    args: ['container', 'inspect', containerNameArg, '--format', '{{.State.Running}}'],
    stdout: 'piped',
    stderr: 'null',
  });
  const output = await command.output();
  if (!output.success) {
    return false;
  }
  return new TextDecoder().decode(output.stdout).trim() === 'true';
};

const dockerContainerExists = async (containerNameArg: string) => {
  const command = new Deno.Command('docker', {
    args: ['container', 'inspect', containerNameArg],
    stdout: 'null',
    stderr: 'null',
  });
  return (await command.output()).success;
};

const removeDockerContainer = async (containerNameArg: string) => {
  if (await dockerContainerExists(containerNameArg)) {
    await run('docker', ['container', 'rm', '-f', containerNameArg]).catch(() => null);
    await delayFor(1000);
  }
};

const assertNoPreexistingScenarioServices = async () => {
  for (const serviceNameArg of [dockerServiceName, 'onebox-smartproxy']) {
    if (await dockerServiceExists(serviceNameArg)) {
      throw new Error(`${serviceNameArg} already exists; refusing to overwrite a running service`);
    }
  }
  for (const containerNameArg of ['onebox-mongodb', 'onebox-minio']) {
    if (await dockerContainerExists(containerNameArg)) {
      const state = await dockerContainerRunning(containerNameArg) ? 'running' : 'present';
      throw new Error(
        `${containerNameArg} is already ${state}; refusing to overwrite a platform container`,
      );
    }
  }
};

const waitForDockerServiceRunning = async (serviceNameArg: string) => {
  try {
    await waitFor(async () => {
      const command = new Deno.Command('docker', {
        args: ['service', 'ps', serviceNameArg, '--format', '{{.CurrentState}}'],
        stdout: 'piped',
        stderr: 'null',
      });
      const output = await command.output();
      if (!output.success) {
        return false;
      }
      return new TextDecoder().decode(output.stdout).includes('Running');
    }, `${serviceNameArg} running task`);
  } catch (error) {
    await run('docker', ['service', 'ps', '--no-trunc', serviceNameArg]).catch(() => null);
    await run('docker', ['service', 'logs', '--raw', '--tail', '120', serviceNameArg]).catch(() =>
      null
    );
    throw error;
  }
};

const waitForDockerContainerRunning = async (containerNameArg: string) => {
  await waitFor(
    async () => dockerContainerRunning(containerNameArg),
    `${containerNameArg} running container`,
  );
};

const ensureDockerReady = async () => {
  await run('docker', ['version']);
  const { stdout } = await run('docker', ['info', '--format', '{{.Swarm.LocalNodeState}}']);
  if (stdout.trim() !== 'active') {
    throw new Error(
      'Docker Swarm must be active. In Vagrant this is handled by scripts/provision-vm.sh.',
    );
  }
};

const requestCloudlyBootstrapScript = async () => {
  const curlArgs = [
    '-sS',
    '--noproxy',
    '*',
    '--max-time',
    '10',
    '--resolve',
    `${routeDomain}:8080:127.0.0.1`,
    '-o',
    '-',
    '-w',
    '\n%{http_code}',
    `http://${routeDomain}:8080/curlfresh/setup.sh`,
  ];

  const command = new Deno.Command('curl', {
    args: curlArgs,
    stdout: 'piped',
    stderr: 'piped',
  });
  const output = await command.output();
  const stdout = new TextDecoder().decode(output.stdout).trim();
  const stderr = new TextDecoder().decode(output.stderr).trim();
  if (!output.success) {
    throw new Error(`curl failed: ${stderr || stdout}`);
  }
  const lines = stdout.split('\n');
  const statusCode = lines[lines.length - 1];
  const body = lines.slice(0, -1).join('\n');
  return { statusCode, body, stderr };
};

const waitForCloudlyWorkerBootstrapRoute = async () => {
  let lastResponse: Awaited<ReturnType<typeof requestCloudlyBootstrapScript>> | undefined;
  let lastError: Error | undefined;
  try {
    await waitFor(async () => {
      try {
        lastResponse = await requestCloudlyBootstrapScript();
        lastError = undefined;
        return (
          lastResponse.statusCode === '200' &&
          lastResponse.body.includes('pnpm install -g @serve.zone/spark') &&
          lastResponse.body.includes('spark installdaemon')
        );
      } catch (error) {
        lastError = error as Error;
        return false;
      }
    }, 'Cloudly worker bootstrap route through Onebox ingress');
  } catch (error) {
    console.log(`[${scenarioName}] Last bootstrap response: ${JSON.stringify(lastResponse)}`);
    if (lastError) {
      console.log(`[${scenarioName}] Last bootstrap error: ${lastError.message}`);
    }
    await run('docker', ['service', 'ps', 'onebox-smartproxy']).catch(() => null);
    await run('docker', ['service', 'logs', '--raw', '--tail', '120', 'onebox-smartproxy']).catch(
      () => null,
    );
    await run('docker', ['service', 'ps', dockerServiceName]).catch(() => null);
    await run('docker', ['service', 'logs', '--raw', '--tail', '120', dockerServiceName]).catch(
      () => null,
    );
    throw error;
  }
};

const main = async () => {
  let onebox: Onebox | undefined;
  let deployedService = false;

  try {
    await ensureDockerReady();
    await assertNoPreexistingScenarioServices();
    await Deno.mkdir(buildDir, { recursive: true });
    Deno.chdir(buildDir);
    Deno.env.set('ONEBOX_DEV', 'true');

    console.log(`[${scenarioName}] Starting Onebox from ${buildDir}`);
    onebox = new Onebox();
    disableManagedDcRouterForScenario(onebox);
    useLocalAppStoreForScenario(onebox, appCatalogDir);
    await onebox.init();
    await waitForDockerServiceRunning('onebox-smartproxy');

    const appMeta = await onebox.appStore.getAppMeta('cloudly');
    const version = appMeta.latestVersion;
    const config = await onebox.appStore.getAppVersionConfig('cloudly', version);
    if (!config.platformRequirements?.mongodb || !config.platformRequirements?.s3) {
      throw new Error(
        `Cloudly template must require MongoDB and S3: ${
          JSON.stringify(config.platformRequirements)
        }`,
      );
    }

    console.log(`[${scenarioName}] Installing Cloudly template ${version} as ${serviceName}`);
    const service = await onebox.appStore.installApp({
      appId: 'cloudly',
      version,
      serviceName,
      domain: routeDomain,
      autoDNS: false,
      envVars: {
        SERVEZONE_ADMINACCOUNT: 'testadmin:testpassword',
      },
    });
    deployedService = true;

    if (service.appTemplateId !== 'cloudly' || service.appTemplateVersion !== version) {
      throw new Error(`Cloudly template tracking missing on service: ${JSON.stringify(service)}`);
    }

    await waitForDockerContainerRunning('onebox-mongodb');
    await waitForDockerContainerRunning('onebox-minio');
    await waitForDockerServiceRunning(dockerServiceName);
    await waitForCloudlyWorkerBootstrapRoute();

    const deployedServiceRecord = onebox.services.getService(serviceName);
    if (!deployedServiceRecord?.id) {
      throw new Error(
        `Cloudly service missing after deploy: ${JSON.stringify(deployedServiceRecord)}`,
      );
    }
    if (deployedServiceRecord.envVars.SERVEZONE_URL !== routeDomain) {
      throw new Error(
        `Cloudly SERVICE_DOMAIN template was not resolved: ${
          JSON.stringify(deployedServiceRecord.envVars)
        }`,
      );
    }
    if (
      !deployedServiceRecord.envVars.MONGODB_URL ||
      deployedServiceRecord.envVars.MONGODB_URL.includes('${')
    ) {
      throw new Error(
        `Cloudly MongoDB template was not resolved: ${
          JSON.stringify(deployedServiceRecord.envVars)
        }`,
      );
    }

    const platformResources = await onebox.platformServices.getResourcesForService(
      deployedServiceRecord.id,
    );
    const platformResourceTypes = platformResources.map((resourceArg) =>
      resourceArg.platformService.type
    ).sort();
    if (platformResourceTypes.join(',') !== 'minio,mongodb') {
      throw new Error(
        `Unexpected Cloudly platform resources: ${JSON.stringify(platformResourceTypes)}`,
      );
    }

    console.log(`[${scenarioName}] Creating Cloudly platform-resource backup for ${serviceName}`);
    onebox.database.updateService(deployedServiceRecord.id, { includeImageInBackup: false });
    const backupResult = await onebox.backupManager.createBackup(serviceName);
    const backupId = backupResult.backup.id;
    if (!backupId || !backupResult.snapshotId || !backupResult.backup.snapshotId) {
      throw new Error(
        `Cloudly backup did not produce a ContainerArchive snapshot: ${
          JSON.stringify(backupResult)
        }`,
      );
    }
    if (backupResult.backup.includesImage) {
      throw new Error(
        `Cloudly backup unexpectedly included the large app image: ${
          JSON.stringify(backupResult.backup)
        }`,
      );
    }
    if (backupResult.backup.platformResources.sort().join(',') !== 'minio,mongodb') {
      throw new Error(
        `Cloudly backup missing platform resources: ${JSON.stringify(backupResult.backup)}`,
      );
    }

    console.log(`[${scenarioName}] Restoring Cloudly platform-resource backup for ${serviceName}`);
    const restoreResult = await onebox.backupManager.restoreBackup(backupId, {
      mode: 'restore',
      overwriteExisting: true,
    });
    if (restoreResult.service.name !== serviceName) {
      throw new Error(
        `Cloudly restore returned unexpected service: ${JSON.stringify(restoreResult.service)}`,
      );
    }
    if (restoreResult.platformResourcesRestored !== 2 || restoreResult.warnings.length > 0) {
      throw new Error(
        `Cloudly restore failed platform resource validation: ${JSON.stringify(restoreResult)}`,
      );
    }

    await waitForCloudlyWorkerBootstrapRoute();

    console.log(`[${scenarioName}] Scenario passed`);
  } finally {
    if (onebox) {
      if (deployedService) {
        await onebox.services.removeService(serviceName).catch((error) => {
          console.log(`[${scenarioName}] Service cleanup failed: ${(error as Error).message}`);
        });
      }
      await onebox.platformServices.stopPlatformService('mongodb').catch(() => null);
      await onebox.platformServices.stopPlatformService('minio').catch(() => null);
      await onebox.shutdown();
    }

    await removeDockerService(dockerServiceName);
    await removeDockerContainer('onebox-mongodb');
    await removeDockerContainer('onebox-minio');
    await removeDockerService('onebox-smartproxy');
  }
};

try {
  await main();
  Deno.exit(0);
} catch (error) {
  console.error(error);
  Deno.exit(1);
}