testing/scenarios/onebox-backup-restore/scenario.ts

import { dirname, fromFileUrl, join, resolve } from '@std/path';

import { Onebox } from '../../../onebox/ts/classes/onebox.ts';
import { disableManagedDcRouterForScenario } from '../onebox-test-helpers.ts';

const scenarioName = 'onebox-backup-restore';
const smokeId = `onebox-backup-${Date.now().toString(36)}`;
const testingDir = resolve(dirname(fromFileUrl(import.meta.url)), '../..');
const buildDir = join(testingDir, '.nogit', scenarioName, smokeId);
const serviceName = `app-${Date.now().toString(36)}`;
const cloneServiceName = `${serviceName}-clone`;
const dockerServiceName = `onebox-${serviceName}`;
const cloneDockerServiceName = `onebox-${cloneServiceName}`;
const volumeName = `onebox-${serviceName}-data`;
const cloneVolumeName = `onebox-${cloneServiceName}-data`;

const delayFor = async (millisecondsArg: number) => {
  await new Promise((resolveArg) => setTimeout(resolveArg, millisecondsArg));
};

const outputCommand = async (
  commandArg: string,
  argsArg: string[],
  optionsArg: Pick<Deno.CommandOptions, 'stdout' | 'stderr'>,
  timeoutMsArg = 30000,
) => {
  const controller = new AbortController();
  const timeoutId = setTimeout(() => controller.abort(), timeoutMsArg);
  try {
    const command = new Deno.Command(commandArg, {
      args: argsArg,
      signal: controller.signal,
      ...optionsArg,
    });
    return await command.output();
  } catch (error) {
    if ((error as Error).name === 'AbortError') {
      throw new Error(`${commandArg} ${argsArg.join(' ')} timed out after ${timeoutMsArg}ms`);
    }
    throw error;
  } finally {
    clearTimeout(timeoutId);
  }
};

const run = async (commandArg: string, argsArg: string[]) => {
  const output = await outputCommand(commandArg, argsArg, {
    stdout: 'piped',
    stderr: 'piped',
  });
  const stdout = new TextDecoder().decode(output.stdout).trim();
  const stderr = new TextDecoder().decode(output.stderr).trim();
  if (stdout) {
    console.log(stdout);
  }
  if (stderr) {
    console.log(stderr);
  }
  if (!output.success) {
    throw new Error(`${commandArg} ${argsArg.join(' ')} exited with ${output.code}`);
  }
  return { stdout, stderr };
};

const waitFor = async (checkFunctionArg: () => boolean | Promise<boolean>, messageArg: string) => {
  const startTime = Date.now();
  while (Date.now() - startTime < 120000) {
    if (await checkFunctionArg()) {
      return;
    }
    await delayFor(500);
  }
  throw new Error(`Timed out waiting for ${messageArg}`);
};

const dockerServiceExists = async (serviceNameArg: string) => {
  const output = await outputCommand('docker', ['service', 'inspect', serviceNameArg], {
    stdout: 'null',
    stderr: 'null',
  }, 15000);
  return output.success;
};

const removeDockerService = async (serviceNameArg: string) => {
  if (await dockerServiceExists(serviceNameArg)) {
    await run('docker', ['service', 'rm', serviceNameArg]).catch(() => null);
    await delayFor(2000);
  }
};

const removeDockerVolume = async (volumeNameArg: string) => {
  let lastErrorText = '';
  const startTime = Date.now();
  while (Date.now() - startTime < 30000) {
    const output = await outputCommand(
      'docker',
      ['volume', 'rm', '-f', volumeNameArg],
      {
        stdout: 'piped',
        stderr: 'piped',
      },
      15000,
    );
    if (output.success) {
      return;
    }
    lastErrorText = new TextDecoder().decode(output.stderr).trim();
    await delayFor(1000);
  }
  console.log(
    `[${scenarioName}] Failed to remove Docker volume ${volumeNameArg}: ${lastErrorText}`,
  );
};

const assertNoPreexistingOneboxIngress = async () => {
  if (await dockerServiceExists('onebox-smartproxy')) {
    throw new Error(
      'onebox-smartproxy already exists; refusing to overwrite a running Onebox ingress service',
    );
  }
};

const waitForDockerServiceRunning = async (serviceNameArg: string) => {
  await waitFor(async () => {
    const output = await outputCommand('docker', [
      'service',
      'ps',
      serviceNameArg,
      '--format',
      '{{.CurrentState}}',
    ], {
      stdout: 'piped',
      stderr: 'null',
    }, 15000);
    if (!output.success) {
      return false;
    }
    return new TextDecoder().decode(output.stdout).includes('Running');
  }, `${serviceNameArg} running task`);
};

const waitForDockerServiceRemoved = async (serviceNameArg: string) => {
  await waitFor(
    async () => !(await dockerServiceExists(serviceNameArg)),
    `${serviceNameArg} removal`,
  );
};

const inspectDockerServiceJson = async <T>(serviceNameArg: string, formatArg: string) => {
  const output = await outputCommand('docker', [
    'service',
    'inspect',
    serviceNameArg,
    '--format',
    formatArg,
  ], {
    stdout: 'piped',
    stderr: 'piped',
  }, 15000);
  if (!output.success) {
    throw new Error(`docker service inspect ${serviceNameArg} exited with ${output.code}`);
  }
  return JSON.parse(new TextDecoder().decode(output.stdout).trim()) as T;
};

const assertCloneVolumeMount = async () => {
  const mounts = await inspectDockerServiceJson<Array<Record<string, unknown>>>(
    cloneDockerServiceName,
    '{{json .Spec.TaskTemplate.ContainerSpec.Mounts}}',
  );
  if (
    !mounts.some((mountArg) => mountArg.Source === cloneVolumeName && mountArg.Target === '/data')
  ) {
    throw new Error(
      `Cloned service volume mount missing from Docker service: ${JSON.stringify(mounts)}`,
    );
  }
};

const ensureDockerReady = async () => {
  await run('docker', ['version']);
  const { stdout } = await run('docker', ['info', '--format', '{{.Swarm.LocalNodeState}}']);
  if (stdout.trim() !== 'active') {
    throw new Error(
      'Docker Swarm must be active. In Vagrant this is handled by scripts/provision-vm.sh.',
    );
  }
};

const main = async () => {
  let onebox: Onebox | undefined;
  let deployedService = false;
  let clonedService = false;

  try {
    await ensureDockerReady();
    await assertNoPreexistingOneboxIngress();
    await Deno.mkdir(buildDir, { recursive: true });
    Deno.chdir(buildDir);
    Deno.env.set('ONEBOX_DEV', 'true');

    console.log(`[${scenarioName}] Starting Onebox from ${buildDir}`);
    onebox = new Onebox();
    disableManagedDcRouterForScenario(onebox);
    await onebox.init();

    await waitForDockerServiceRunning('onebox-smartproxy');

    console.log(`[${scenarioName}] Deploying workload ${serviceName}`);
    const service = await onebox.services.deployService({
      name: serviceName,
      image: 'caddy:2-alpine',
      port: 80,
      autoDNS: false,
      envVars: {
        ONEBOX_BACKUP_SCENARIO: smokeId,
      },
      volumes: [{ mountPath: '/data', backup: true }],
    });
    deployedService = true;

    if (service.status !== 'running') {
      throw new Error(`Unexpected deployed service state: ${JSON.stringify(service)}`);
    }

    await waitForDockerServiceRunning(dockerServiceName);

    console.log(`[${scenarioName}] Creating backup for ${serviceName}`);
    const backupResult = await onebox.backupManager.createBackup(serviceName);
    const backupId = backupResult.backup.id;
    if (!backupId || !backupResult.snapshotId || !backupResult.backup.snapshotId) {
      throw new Error(
        `Backup did not produce a ContainerArchive snapshot: ${JSON.stringify(backupResult)}`,
      );
    }
    if (!backupResult.backup.includesImage) {
      throw new Error(
        `Backup did not include Docker image: ${JSON.stringify(backupResult.backup)}`,
      );
    }
    if ((backupResult.backup.storedSizeBytes ?? 0) <= 0 || backupResult.backup.sizeBytes <= 0) {
      throw new Error(`Backup size metadata is invalid: ${JSON.stringify(backupResult.backup)}`);
    }

    const backups = onebox.backupManager.listBackups(serviceName);
    if (
      !backups.some((backupArg) =>
        backupArg.id === backupId && backupArg.snapshotId === backupResult.snapshotId
      )
    ) {
      throw new Error(`Backup not listed for service: ${JSON.stringify(backups)}`);
    }

    console.log(`[${scenarioName}] Restoring backup as ${cloneServiceName}`);
    const restoreResult = await onebox.backupManager.restoreBackup(backupId, {
      mode: 'clone',
      newServiceName: cloneServiceName,
    });
    clonedService = true;

    if (
      restoreResult.service.name !== cloneServiceName || restoreResult.service.status !== 'running'
    ) {
      throw new Error(
        `Unexpected restored service state: ${JSON.stringify(restoreResult.service)}`,
      );
    }
    if (restoreResult.service.domain !== undefined) {
      throw new Error(
        `Clone unexpectedly retained a domain: ${JSON.stringify(restoreResult.service)}`,
      );
    }
    if (restoreResult.service.envVars.ONEBOX_BACKUP_SCENARIO !== smokeId) {
      throw new Error(
        `Clone did not preserve env vars: ${JSON.stringify(restoreResult.service.envVars)}`,
      );
    }
    if (!restoreResult.service.volumes?.some((volumeArg) => volumeArg.mountPath === '/data')) {
      throw new Error(
        `Clone did not preserve volumes: ${JSON.stringify(restoreResult.service.volumes)}`,
      );
    }
    if (restoreResult.warnings.length > 0) {
      throw new Error(`Restore completed with warnings: ${restoreResult.warnings.join('; ')}`);
    }

    await waitForDockerServiceRunning(cloneDockerServiceName);
    await assertCloneVolumeMount();

    console.log(`[${scenarioName}] Removing restored workload ${cloneServiceName}`);
    await onebox.services.removeService(cloneServiceName);
    clonedService = false;
    await waitForDockerServiceRemoved(cloneDockerServiceName);

    console.log(`[${scenarioName}] Removing workload ${serviceName}`);
    await onebox.services.removeService(serviceName);
    deployedService = false;
    await waitForDockerServiceRemoved(dockerServiceName);

    console.log(`[${scenarioName}] Scenario passed`);
  } finally {
    if (onebox && clonedService) {
      await onebox.services.removeService(cloneServiceName).catch((error) => {
        console.log(
          `[${scenarioName}] Failed to remove restored Onebox service: ${(error as Error).message}`,
        );
      });
    }
    if (onebox && deployedService) {
      await onebox.services.removeService(serviceName).catch((error) => {
        console.log(
          `[${scenarioName}] Failed to remove Onebox service: ${(error as Error).message}`,
        );
      });
    }
    if (onebox) {
      await onebox.shutdown().catch((error) => {
        console.log(`[${scenarioName}] Failed to shut down Onebox: ${(error as Error).message}`);
      });
    }
    await removeDockerService(cloneDockerServiceName);
    await removeDockerService(dockerServiceName);
    await removeDockerService('onebox-smartproxy');
    await removeDockerVolume(cloneVolumeName);
    await removeDockerVolume(volumeName);
  }
};

try {
  await main();
  Deno.exit(0);
} catch (error) {
  console.error(error);
  Deno.exit(1);
}