feat: add onebox backup restore scenarios

package.json

@@ -6,10 +6,11 @@
   "description": "Whole-system integration scenarios for serve.zone components.",
   "scripts": {
     "bootstrap:components": "pnpm --dir ../interfaces install && pnpm --dir ../api install && pnpm --dir ../cloudly install && pnpm --dir ../coreflow install && pnpm --dir ../coretraffic install && pnpm --dir ../onebox exec deno install --config deno.json && pnpm install",
-    "test": "pnpm scenario:registry-deploy-on-push && pnpm scenario:onebox-basic-lifecycle",
+    "test": "pnpm scenario:registry-deploy-on-push && pnpm scenario:onebox-basic-lifecycle && pnpm scenario:onebox-backup-restore",
     "test:full": "pnpm test && pnpm scenario:onebox-cloudly-appstore-worker",
     "scenario:registry-deploy-on-push": "tsx --tsconfig ../cloudly/tsconfig.json scenarios/registry-deploy-on-push/scenario.ts",
     "scenario:onebox-basic-lifecycle": "deno run --allow-all --config ../onebox/deno.json scenarios/onebox-basic-lifecycle/scenario.ts",
+    "scenario:onebox-backup-restore": "deno run --allow-all --config ../onebox/deno.json scenarios/onebox-backup-restore/scenario.ts",
     "scenario:onebox-cloudly-appstore-worker": "deno run --allow-all --config ../onebox/deno.json scenarios/onebox-cloudly-appstore-worker/scenario.ts",
     "vagrant:up": "vagrant up",
     "vagrant:test": "vagrant ssh -c 'cd /serve.zone/testing && pnpm bootstrap:components && pnpm test'",

scenarios/onebox-backup-restore/scenario.ts

@@ -0,0 +1,208 @@
+import { dirname, fromFileUrl, join, resolve } from '@std/path';
+
+import { Onebox } from '../../../onebox/ts/classes/onebox.ts';
+
+const scenarioName = 'onebox-backup-restore';
+const smokeId = `onebox-backup-${Date.now().toString(36)}`;
+const testingDir = resolve(dirname(fromFileUrl(import.meta.url)), '../..');
+const buildDir = join(testingDir, '.nogit', scenarioName, smokeId);
+const serviceName = `app-${Date.now().toString(36)}`;
+const cloneServiceName = `${serviceName}-clone`;
+const dockerServiceName = `onebox-${serviceName}`;
+const cloneDockerServiceName = `onebox-${cloneServiceName}`;
+
+const delayFor = async (millisecondsArg: number) => {
+  await new Promise((resolveArg) => setTimeout(resolveArg, millisecondsArg));
+};
+
+const run = async (commandArg: string, argsArg: string[]) => {
+  const command = new Deno.Command(commandArg, {
+    args: argsArg,
+    stdout: 'piped',
+    stderr: 'piped',
+  });
+  const output = await command.output();
+  const stdout = new TextDecoder().decode(output.stdout).trim();
+  const stderr = new TextDecoder().decode(output.stderr).trim();
+  if (stdout) {
+    console.log(stdout);
+  }
+  if (stderr) {
+    console.log(stderr);
+  }
+  if (!output.success) {
+    throw new Error(`${commandArg} ${argsArg.join(' ')} exited with ${output.code}`);
+  }
+  return { stdout, stderr };
+};
+
+const waitFor = async (checkFunctionArg: () => boolean | Promise<boolean>, messageArg: string) => {
+  const startTime = Date.now();
+  while (Date.now() - startTime < 120000) {
+    if (await checkFunctionArg()) {
+      return;
+    }
+    await delayFor(500);
+  }
+  throw new Error(`Timed out waiting for ${messageArg}`);
+};
+
+const dockerServiceExists = async (serviceNameArg: string) => {
+  const command = new Deno.Command('docker', {
+    args: ['service', 'inspect', serviceNameArg],
+    stdout: 'null',
+    stderr: 'null',
+  });
+  return (await command.output()).success;
+};
+
+const removeDockerService = async (serviceNameArg: string) => {
+  if (await dockerServiceExists(serviceNameArg)) {
+    await run('docker', ['service', 'rm', serviceNameArg]).catch(() => null);
+    await delayFor(2000);
+  }
+};
+
+const assertNoPreexistingOneboxIngress = async () => {
+  if (await dockerServiceExists('onebox-smartproxy')) {
+    throw new Error('onebox-smartproxy already exists; refusing to overwrite a running Onebox ingress service');
+  }
+};
+
+const waitForDockerServiceRunning = async (serviceNameArg: string) => {
+  await waitFor(async () => {
+    const command = new Deno.Command('docker', {
+      args: ['service', 'ps', serviceNameArg, '--format', '{{.CurrentState}}'],
+      stdout: 'piped',
+      stderr: 'null',
+    });
+    const output = await command.output();
+    if (!output.success) {
+      return false;
+    }
+    return new TextDecoder().decode(output.stdout).includes('Running');
+  }, `${serviceNameArg} running task`);
+};
+
+const waitForDockerServiceRemoved = async (serviceNameArg: string) => {
+  await waitFor(async () => !(await dockerServiceExists(serviceNameArg)), `${serviceNameArg} removal`);
+};
+
+const ensureDockerReady = async () => {
+  await run('docker', ['version']);
+  const { stdout } = await run('docker', ['info', '--format', '{{.Swarm.LocalNodeState}}']);
+  if (stdout.trim() !== 'active') {
+    throw new Error('Docker Swarm must be active. In Vagrant this is handled by scripts/provision-vm.sh.');
+  }
+};
+
+const main = async () => {
+  let onebox: Onebox | undefined;
+  let deployedService = false;
+  let clonedService = false;
+
+  try {
+    await ensureDockerReady();
+    await assertNoPreexistingOneboxIngress();
+    await Deno.mkdir(buildDir, { recursive: true });
+    Deno.chdir(buildDir);
+    Deno.env.set('ONEBOX_DEV', 'true');
+
+    console.log(`[${scenarioName}] Starting Onebox from ${buildDir}`);
+    onebox = new Onebox();
+    await onebox.init();
+
+    await waitForDockerServiceRunning('onebox-smartproxy');
+
+    console.log(`[${scenarioName}] Deploying workload ${serviceName}`);
+    const service = await onebox.services.deployService({
+      name: serviceName,
+      image: 'caddy:2-alpine',
+      port: 80,
+      autoDNS: false,
+      envVars: {
+        ONEBOX_BACKUP_SCENARIO: smokeId,
+      },
+    });
+    deployedService = true;
+
+    if (service.status !== 'running') {
+      throw new Error(`Unexpected deployed service state: ${JSON.stringify(service)}`);
+    }
+
+    await waitForDockerServiceRunning(dockerServiceName);
+
+    console.log(`[${scenarioName}] Creating backup for ${serviceName}`);
+    const backupResult = await onebox.backupManager.createBackup(serviceName);
+    const backupId = backupResult.backup.id;
+    if (!backupId || !backupResult.snapshotId || !backupResult.backup.snapshotId) {
+      throw new Error(`Backup did not produce a ContainerArchive snapshot: ${JSON.stringify(backupResult)}`);
+    }
+    if (!backupResult.backup.includesImage) {
+      throw new Error(`Backup did not include Docker image: ${JSON.stringify(backupResult.backup)}`);
+    }
+    if ((backupResult.backup.storedSizeBytes ?? 0) <= 0 || backupResult.backup.sizeBytes <= 0) {
+      throw new Error(`Backup size metadata is invalid: ${JSON.stringify(backupResult.backup)}`);
+    }
+
+    const backups = onebox.backupManager.listBackups(serviceName);
+    if (!backups.some((backupArg) => backupArg.id === backupId && backupArg.snapshotId === backupResult.snapshotId)) {
+      throw new Error(`Backup not listed for service: ${JSON.stringify(backups)}`);
+    }
+
+    console.log(`[${scenarioName}] Restoring backup as ${cloneServiceName}`);
+    const restoreResult = await onebox.backupManager.restoreBackup(backupId, {
+      mode: 'clone',
+      newServiceName: cloneServiceName,
+    });
+    clonedService = true;
+
+    if (restoreResult.service.name !== cloneServiceName || restoreResult.service.status !== 'running') {
+      throw new Error(`Unexpected restored service state: ${JSON.stringify(restoreResult.service)}`);
+    }
+    if (restoreResult.service.domain !== undefined) {
+      throw new Error(`Clone unexpectedly retained a domain: ${JSON.stringify(restoreResult.service)}`);
+    }
+    if (restoreResult.service.envVars.ONEBOX_BACKUP_SCENARIO !== smokeId) {
+      throw new Error(`Clone did not preserve env vars: ${JSON.stringify(restoreResult.service.envVars)}`);
+    }
+    if (restoreResult.warnings.length > 0) {
+      throw new Error(`Restore completed with warnings: ${restoreResult.warnings.join('; ')}`);
+    }
+
+    await waitForDockerServiceRunning(cloneDockerServiceName);
+
+    console.log(`[${scenarioName}] Removing restored workload ${cloneServiceName}`);
+    await onebox.services.removeService(cloneServiceName);
+    clonedService = false;
+    await waitForDockerServiceRemoved(cloneDockerServiceName);
+
+    console.log(`[${scenarioName}] Removing workload ${serviceName}`);
+    await onebox.services.removeService(serviceName);
+    deployedService = false;
+    await waitForDockerServiceRemoved(dockerServiceName);
+
+    console.log(`[${scenarioName}] Scenario passed`);
+  } finally {
+    if (onebox && clonedService) {
+      await onebox.services.removeService(cloneServiceName).catch((error) => {
+        console.log(`[${scenarioName}] Failed to remove restored Onebox service: ${(error as Error).message}`);
+      });
+    }
+    if (onebox && deployedService) {
+      await onebox.services.removeService(serviceName).catch((error) => {
+        console.log(`[${scenarioName}] Failed to remove Onebox service: ${(error as Error).message}`);
+      });
+    }
+    if (onebox) {
+      await onebox.shutdown().catch((error) => {
+        console.log(`[${scenarioName}] Failed to shut down Onebox: ${(error as Error).message}`);
+      });
+    }
+    await removeDockerService(cloneDockerServiceName);
+    await removeDockerService(dockerServiceName);
+    await removeDockerService('onebox-smartproxy');
+  }
+};
+
+await main();

scenarios/onebox-cloudly-appstore-worker/scenario.ts

@@ -270,6 +270,45 @@ const main = async () => {
     await waitForDockerServiceRunning(dockerServiceName);
     await waitForCloudlyWorkerBootstrapRoute();
 
+    const deployedServiceRecord = onebox.services.getService(serviceName);
+    if (!deployedServiceRecord?.id) {
+      throw new Error(`Cloudly service missing after deploy: ${JSON.stringify(deployedServiceRecord)}`);
+    }
+
+    const platformResources = await onebox.platformServices.getResourcesForService(deployedServiceRecord.id);
+    const platformResourceTypes = platformResources.map((resourceArg) => resourceArg.platformService.type).sort();
+    if (platformResourceTypes.join(',') !== 'minio,mongodb') {
+      throw new Error(`Unexpected Cloudly platform resources: ${JSON.stringify(platformResourceTypes)}`);
+    }
+
+    console.log(`[${scenarioName}] Creating Cloudly platform-resource backup for ${serviceName}`);
+    onebox.database.updateService(deployedServiceRecord.id, { includeImageInBackup: false });
+    const backupResult = await onebox.backupManager.createBackup(serviceName);
+    const backupId = backupResult.backup.id;
+    if (!backupId || !backupResult.snapshotId || !backupResult.backup.snapshotId) {
+      throw new Error(`Cloudly backup did not produce a ContainerArchive snapshot: ${JSON.stringify(backupResult)}`);
+    }
+    if (backupResult.backup.includesImage) {
+      throw new Error(`Cloudly backup unexpectedly included the large app image: ${JSON.stringify(backupResult.backup)}`);
+    }
+    if (backupResult.backup.platformResources.sort().join(',') !== 'minio,mongodb') {
+      throw new Error(`Cloudly backup missing platform resources: ${JSON.stringify(backupResult.backup)}`);
+    }
+
+    console.log(`[${scenarioName}] Restoring Cloudly platform-resource backup for ${serviceName}`);
+    const restoreResult = await onebox.backupManager.restoreBackup(backupId, {
+      mode: 'restore',
+      overwriteExisting: true,
+    });
+    if (restoreResult.service.name !== serviceName) {
+      throw new Error(`Cloudly restore returned unexpected service: ${JSON.stringify(restoreResult.service)}`);
+    }
+    if (restoreResult.platformResourcesRestored !== 2 || restoreResult.warnings.length > 0) {
+      throw new Error(`Cloudly restore failed platform resource validation: ${JSON.stringify(restoreResult)}`);
+    }
+
+    await waitForCloudlyWorkerBootstrapRoute();
+
     console.log(`[${scenarioName}] Scenario passed`);
   } finally {
     if (onebox) {