diff --git a/package.json b/package.json index 6dfe9ea..bfb78e5 100644 --- a/package.json +++ b/package.json @@ -6,10 +6,11 @@ "description": "Whole-system integration scenarios for serve.zone components.", "scripts": { "bootstrap:components": "pnpm --dir ../interfaces install && pnpm --dir ../api install && pnpm --dir ../cloudly install && pnpm --dir ../coreflow install && pnpm --dir ../coretraffic install && pnpm --dir ../onebox exec deno install --config deno.json && pnpm install", - "test": "pnpm scenario:registry-deploy-on-push && pnpm scenario:onebox-basic-lifecycle", + "test": "pnpm scenario:registry-deploy-on-push && pnpm scenario:onebox-basic-lifecycle && pnpm scenario:onebox-backup-restore", "test:full": "pnpm test && pnpm scenario:onebox-cloudly-appstore-worker", "scenario:registry-deploy-on-push": "tsx --tsconfig ../cloudly/tsconfig.json scenarios/registry-deploy-on-push/scenario.ts", "scenario:onebox-basic-lifecycle": "deno run --allow-all --config ../onebox/deno.json scenarios/onebox-basic-lifecycle/scenario.ts", + "scenario:onebox-backup-restore": "deno run --allow-all --config ../onebox/deno.json scenarios/onebox-backup-restore/scenario.ts", "scenario:onebox-cloudly-appstore-worker": "deno run --allow-all --config ../onebox/deno.json scenarios/onebox-cloudly-appstore-worker/scenario.ts", "vagrant:up": "vagrant up", "vagrant:test": "vagrant ssh -c 'cd /serve.zone/testing && pnpm bootstrap:components && pnpm test'", diff --git a/scenarios/onebox-backup-restore/scenario.ts b/scenarios/onebox-backup-restore/scenario.ts new file mode 100644 index 0000000..e8fd1b8 --- /dev/null +++ b/scenarios/onebox-backup-restore/scenario.ts @@ -0,0 +1,208 @@ +import { dirname, fromFileUrl, join, resolve } from '@std/path'; + +import { Onebox } from '../../../onebox/ts/classes/onebox.ts'; + +const scenarioName = 'onebox-backup-restore'; +const smokeId = `onebox-backup-${Date.now().toString(36)}`; +const testingDir = resolve(dirname(fromFileUrl(import.meta.url)), '../..'); +const buildDir = join(testingDir, '.nogit', scenarioName, smokeId); +const serviceName = `app-${Date.now().toString(36)}`; +const cloneServiceName = `${serviceName}-clone`; +const dockerServiceName = `onebox-${serviceName}`; +const cloneDockerServiceName = `onebox-${cloneServiceName}`; + +const delayFor = async (millisecondsArg: number) => { + await new Promise((resolveArg) => setTimeout(resolveArg, millisecondsArg)); +}; + +const run = async (commandArg: string, argsArg: string[]) => { + const command = new Deno.Command(commandArg, { + args: argsArg, + stdout: 'piped', + stderr: 'piped', + }); + const output = await command.output(); + const stdout = new TextDecoder().decode(output.stdout).trim(); + const stderr = new TextDecoder().decode(output.stderr).trim(); + if (stdout) { + console.log(stdout); + } + if (stderr) { + console.log(stderr); + } + if (!output.success) { + throw new Error(`${commandArg} ${argsArg.join(' ')} exited with ${output.code}`); + } + return { stdout, stderr }; +}; + +const waitFor = async (checkFunctionArg: () => boolean | Promise, messageArg: string) => { + const startTime = Date.now(); + while (Date.now() - startTime < 120000) { + if (await checkFunctionArg()) { + return; + } + await delayFor(500); + } + throw new Error(`Timed out waiting for ${messageArg}`); +}; + +const dockerServiceExists = async (serviceNameArg: string) => { + const command = new Deno.Command('docker', { + args: ['service', 'inspect', serviceNameArg], + stdout: 'null', + stderr: 'null', + }); + return (await command.output()).success; +}; + +const removeDockerService = async (serviceNameArg: string) => { + if (await dockerServiceExists(serviceNameArg)) { + await run('docker', ['service', 'rm', serviceNameArg]).catch(() => null); + await delayFor(2000); + } +}; + +const assertNoPreexistingOneboxIngress = async () => { + if (await dockerServiceExists('onebox-smartproxy')) { + throw new Error('onebox-smartproxy already exists; refusing to overwrite a running Onebox ingress service'); + } +}; + +const waitForDockerServiceRunning = async (serviceNameArg: string) => { + await waitFor(async () => { + const command = new Deno.Command('docker', { + args: ['service', 'ps', serviceNameArg, '--format', '{{.CurrentState}}'], + stdout: 'piped', + stderr: 'null', + }); + const output = await command.output(); + if (!output.success) { + return false; + } + return new TextDecoder().decode(output.stdout).includes('Running'); + }, `${serviceNameArg} running task`); +}; + +const waitForDockerServiceRemoved = async (serviceNameArg: string) => { + await waitFor(async () => !(await dockerServiceExists(serviceNameArg)), `${serviceNameArg} removal`); +}; + +const ensureDockerReady = async () => { + await run('docker', ['version']); + const { stdout } = await run('docker', ['info', '--format', '{{.Swarm.LocalNodeState}}']); + if (stdout.trim() !== 'active') { + throw new Error('Docker Swarm must be active. In Vagrant this is handled by scripts/provision-vm.sh.'); + } +}; + +const main = async () => { + let onebox: Onebox | undefined; + let deployedService = false; + let clonedService = false; + + try { + await ensureDockerReady(); + await assertNoPreexistingOneboxIngress(); + await Deno.mkdir(buildDir, { recursive: true }); + Deno.chdir(buildDir); + Deno.env.set('ONEBOX_DEV', 'true'); + + console.log(`[${scenarioName}] Starting Onebox from ${buildDir}`); + onebox = new Onebox(); + await onebox.init(); + + await waitForDockerServiceRunning('onebox-smartproxy'); + + console.log(`[${scenarioName}] Deploying workload ${serviceName}`); + const service = await onebox.services.deployService({ + name: serviceName, + image: 'caddy:2-alpine', + port: 80, + autoDNS: false, + envVars: { + ONEBOX_BACKUP_SCENARIO: smokeId, + }, + }); + deployedService = true; + + if (service.status !== 'running') { + throw new Error(`Unexpected deployed service state: ${JSON.stringify(service)}`); + } + + await waitForDockerServiceRunning(dockerServiceName); + + console.log(`[${scenarioName}] Creating backup for ${serviceName}`); + const backupResult = await onebox.backupManager.createBackup(serviceName); + const backupId = backupResult.backup.id; + if (!backupId || !backupResult.snapshotId || !backupResult.backup.snapshotId) { + throw new Error(`Backup did not produce a ContainerArchive snapshot: ${JSON.stringify(backupResult)}`); + } + if (!backupResult.backup.includesImage) { + throw new Error(`Backup did not include Docker image: ${JSON.stringify(backupResult.backup)}`); + } + if ((backupResult.backup.storedSizeBytes ?? 0) <= 0 || backupResult.backup.sizeBytes <= 0) { + throw new Error(`Backup size metadata is invalid: ${JSON.stringify(backupResult.backup)}`); + } + + const backups = onebox.backupManager.listBackups(serviceName); + if (!backups.some((backupArg) => backupArg.id === backupId && backupArg.snapshotId === backupResult.snapshotId)) { + throw new Error(`Backup not listed for service: ${JSON.stringify(backups)}`); + } + + console.log(`[${scenarioName}] Restoring backup as ${cloneServiceName}`); + const restoreResult = await onebox.backupManager.restoreBackup(backupId, { + mode: 'clone', + newServiceName: cloneServiceName, + }); + clonedService = true; + + if (restoreResult.service.name !== cloneServiceName || restoreResult.service.status !== 'running') { + throw new Error(`Unexpected restored service state: ${JSON.stringify(restoreResult.service)}`); + } + if (restoreResult.service.domain !== undefined) { + throw new Error(`Clone unexpectedly retained a domain: ${JSON.stringify(restoreResult.service)}`); + } + if (restoreResult.service.envVars.ONEBOX_BACKUP_SCENARIO !== smokeId) { + throw new Error(`Clone did not preserve env vars: ${JSON.stringify(restoreResult.service.envVars)}`); + } + if (restoreResult.warnings.length > 0) { + throw new Error(`Restore completed with warnings: ${restoreResult.warnings.join('; ')}`); + } + + await waitForDockerServiceRunning(cloneDockerServiceName); + + console.log(`[${scenarioName}] Removing restored workload ${cloneServiceName}`); + await onebox.services.removeService(cloneServiceName); + clonedService = false; + await waitForDockerServiceRemoved(cloneDockerServiceName); + + console.log(`[${scenarioName}] Removing workload ${serviceName}`); + await onebox.services.removeService(serviceName); + deployedService = false; + await waitForDockerServiceRemoved(dockerServiceName); + + console.log(`[${scenarioName}] Scenario passed`); + } finally { + if (onebox && clonedService) { + await onebox.services.removeService(cloneServiceName).catch((error) => { + console.log(`[${scenarioName}] Failed to remove restored Onebox service: ${(error as Error).message}`); + }); + } + if (onebox && deployedService) { + await onebox.services.removeService(serviceName).catch((error) => { + console.log(`[${scenarioName}] Failed to remove Onebox service: ${(error as Error).message}`); + }); + } + if (onebox) { + await onebox.shutdown().catch((error) => { + console.log(`[${scenarioName}] Failed to shut down Onebox: ${(error as Error).message}`); + }); + } + await removeDockerService(cloneDockerServiceName); + await removeDockerService(dockerServiceName); + await removeDockerService('onebox-smartproxy'); + } +}; + +await main(); diff --git a/scenarios/onebox-cloudly-appstore-worker/scenario.ts b/scenarios/onebox-cloudly-appstore-worker/scenario.ts index f1d5ebf..0f925c2 100644 --- a/scenarios/onebox-cloudly-appstore-worker/scenario.ts +++ b/scenarios/onebox-cloudly-appstore-worker/scenario.ts @@ -270,6 +270,45 @@ const main = async () => { await waitForDockerServiceRunning(dockerServiceName); await waitForCloudlyWorkerBootstrapRoute(); + const deployedServiceRecord = onebox.services.getService(serviceName); + if (!deployedServiceRecord?.id) { + throw new Error(`Cloudly service missing after deploy: ${JSON.stringify(deployedServiceRecord)}`); + } + + const platformResources = await onebox.platformServices.getResourcesForService(deployedServiceRecord.id); + const platformResourceTypes = platformResources.map((resourceArg) => resourceArg.platformService.type).sort(); + if (platformResourceTypes.join(',') !== 'minio,mongodb') { + throw new Error(`Unexpected Cloudly platform resources: ${JSON.stringify(platformResourceTypes)}`); + } + + console.log(`[${scenarioName}] Creating Cloudly platform-resource backup for ${serviceName}`); + onebox.database.updateService(deployedServiceRecord.id, { includeImageInBackup: false }); + const backupResult = await onebox.backupManager.createBackup(serviceName); + const backupId = backupResult.backup.id; + if (!backupId || !backupResult.snapshotId || !backupResult.backup.snapshotId) { + throw new Error(`Cloudly backup did not produce a ContainerArchive snapshot: ${JSON.stringify(backupResult)}`); + } + if (backupResult.backup.includesImage) { + throw new Error(`Cloudly backup unexpectedly included the large app image: ${JSON.stringify(backupResult.backup)}`); + } + if (backupResult.backup.platformResources.sort().join(',') !== 'minio,mongodb') { + throw new Error(`Cloudly backup missing platform resources: ${JSON.stringify(backupResult.backup)}`); + } + + console.log(`[${scenarioName}] Restoring Cloudly platform-resource backup for ${serviceName}`); + const restoreResult = await onebox.backupManager.restoreBackup(backupId, { + mode: 'restore', + overwriteExisting: true, + }); + if (restoreResult.service.name !== serviceName) { + throw new Error(`Cloudly restore returned unexpected service: ${JSON.stringify(restoreResult.service)}`); + } + if (restoreResult.platformResourcesRestored !== 2 || restoreResult.warnings.length > 0) { + throw new Error(`Cloudly restore failed platform resource validation: ${JSON.stringify(restoreResult)}`); + } + + await waitForCloudlyWorkerBootstrapRoute(); + console.log(`[${scenarioName}] Scenario passed`); } finally { if (onebox) {