import { dirname, fromFileUrl, join, resolve } from '@std/path'; import { Onebox } from '../../../onebox/ts/classes/onebox.ts'; import { disableManagedDcRouterForScenario } from '../onebox-test-helpers.ts'; const scenarioName = 'onebox-backup-restore'; const smokeId = `onebox-backup-${Date.now().toString(36)}`; const testingDir = resolve(dirname(fromFileUrl(import.meta.url)), '../..'); const buildDir = join(testingDir, '.nogit', scenarioName, smokeId); const serviceName = `app-${Date.now().toString(36)}`; const cloneServiceName = `${serviceName}-clone`; const dockerServiceName = `onebox-${serviceName}`; const cloneDockerServiceName = `onebox-${cloneServiceName}`; const volumeName = `onebox-${serviceName}-data`; const cloneVolumeName = `onebox-${cloneServiceName}-data`; const delayFor = async (millisecondsArg: number) => { await new Promise((resolveArg) => setTimeout(resolveArg, millisecondsArg)); }; const outputCommand = async ( commandArg: string, argsArg: string[], optionsArg: Pick, timeoutMsArg = 30000, ) => { const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), timeoutMsArg); try { const command = new Deno.Command(commandArg, { args: argsArg, signal: controller.signal, ...optionsArg, }); return await command.output(); } catch (error) { if ((error as Error).name === 'AbortError') { throw new Error(`${commandArg} ${argsArg.join(' ')} timed out after ${timeoutMsArg}ms`); } throw error; } finally { clearTimeout(timeoutId); } }; const run = async (commandArg: string, argsArg: string[]) => { const output = await outputCommand(commandArg, argsArg, { stdout: 'piped', stderr: 'piped', }); const stdout = new TextDecoder().decode(output.stdout).trim(); const stderr = new TextDecoder().decode(output.stderr).trim(); if (stdout) { console.log(stdout); } if (stderr) { console.log(stderr); } if (!output.success) { throw new Error(`${commandArg} ${argsArg.join(' ')} exited with ${output.code}`); } return { stdout, stderr }; }; const waitFor = async (checkFunctionArg: () => boolean | Promise, messageArg: string) => { const startTime = Date.now(); while (Date.now() - startTime < 120000) { if (await checkFunctionArg()) { return; } await delayFor(500); } throw new Error(`Timed out waiting for ${messageArg}`); }; const dockerServiceExists = async (serviceNameArg: string) => { const output = await outputCommand('docker', ['service', 'inspect', serviceNameArg], { stdout: 'null', stderr: 'null', }, 15000); return output.success; }; const removeDockerService = async (serviceNameArg: string) => { if (await dockerServiceExists(serviceNameArg)) { await run('docker', ['service', 'rm', serviceNameArg]).catch(() => null); await delayFor(2000); } }; const removeDockerVolume = async (volumeNameArg: string) => { let lastErrorText = ''; const startTime = Date.now(); while (Date.now() - startTime < 30000) { const output = await outputCommand( 'docker', ['volume', 'rm', '-f', volumeNameArg], { stdout: 'piped', stderr: 'piped', }, 15000, ); if (output.success) { return; } lastErrorText = new TextDecoder().decode(output.stderr).trim(); await delayFor(1000); } console.log( `[${scenarioName}] Failed to remove Docker volume ${volumeNameArg}: ${lastErrorText}`, ); }; const assertNoPreexistingOneboxIngress = async () => { if (await dockerServiceExists('onebox-smartproxy')) { throw new Error( 'onebox-smartproxy already exists; refusing to overwrite a running Onebox ingress service', ); } }; const waitForDockerServiceRunning = async (serviceNameArg: string) => { await waitFor(async () => { const output = await outputCommand('docker', [ 'service', 'ps', serviceNameArg, '--format', '{{.CurrentState}}', ], { stdout: 'piped', stderr: 'null', }, 15000); if (!output.success) { return false; } return new TextDecoder().decode(output.stdout).includes('Running'); }, `${serviceNameArg} running task`); }; const waitForDockerServiceRemoved = async (serviceNameArg: string) => { await waitFor( async () => !(await dockerServiceExists(serviceNameArg)), `${serviceNameArg} removal`, ); }; const inspectDockerServiceJson = async (serviceNameArg: string, formatArg: string) => { const output = await outputCommand('docker', [ 'service', 'inspect', serviceNameArg, '--format', formatArg, ], { stdout: 'piped', stderr: 'piped', }, 15000); if (!output.success) { throw new Error(`docker service inspect ${serviceNameArg} exited with ${output.code}`); } return JSON.parse(new TextDecoder().decode(output.stdout).trim()) as T; }; const assertCloneVolumeMount = async () => { const mounts = await inspectDockerServiceJson>>( cloneDockerServiceName, '{{json .Spec.TaskTemplate.ContainerSpec.Mounts}}', ); if ( !mounts.some((mountArg) => mountArg.Source === cloneVolumeName && mountArg.Target === '/data') ) { throw new Error( `Cloned service volume mount missing from Docker service: ${JSON.stringify(mounts)}`, ); } }; const ensureDockerReady = async () => { await run('docker', ['version']); const { stdout } = await run('docker', ['info', '--format', '{{.Swarm.LocalNodeState}}']); if (stdout.trim() !== 'active') { throw new Error( 'Docker Swarm must be active. In Vagrant this is handled by scripts/provision-vm.sh.', ); } }; const main = async () => { let onebox: Onebox | undefined; let deployedService = false; let clonedService = false; try { await ensureDockerReady(); await assertNoPreexistingOneboxIngress(); await Deno.mkdir(buildDir, { recursive: true }); Deno.chdir(buildDir); Deno.env.set('ONEBOX_DEV', 'true'); console.log(`[${scenarioName}] Starting Onebox from ${buildDir}`); onebox = new Onebox(); disableManagedDcRouterForScenario(onebox); await onebox.init(); await waitForDockerServiceRunning('onebox-smartproxy'); console.log(`[${scenarioName}] Deploying workload ${serviceName}`); const service = await onebox.services.deployService({ name: serviceName, image: 'caddy:2-alpine', port: 80, autoDNS: false, envVars: { ONEBOX_BACKUP_SCENARIO: smokeId, }, volumes: [{ mountPath: '/data', backup: true }], }); deployedService = true; if (service.status !== 'running') { throw new Error(`Unexpected deployed service state: ${JSON.stringify(service)}`); } await waitForDockerServiceRunning(dockerServiceName); console.log(`[${scenarioName}] Creating backup for ${serviceName}`); const backupResult = await onebox.backupManager.createBackup(serviceName); const backupId = backupResult.backup.id; if (!backupId || !backupResult.snapshotId || !backupResult.backup.snapshotId) { throw new Error( `Backup did not produce a ContainerArchive snapshot: ${JSON.stringify(backupResult)}`, ); } if (!backupResult.backup.includesImage) { throw new Error( `Backup did not include Docker image: ${JSON.stringify(backupResult.backup)}`, ); } if ((backupResult.backup.storedSizeBytes ?? 0) <= 0 || backupResult.backup.sizeBytes <= 0) { throw new Error(`Backup size metadata is invalid: ${JSON.stringify(backupResult.backup)}`); } const backups = onebox.backupManager.listBackups(serviceName); if ( !backups.some((backupArg) => backupArg.id === backupId && backupArg.snapshotId === backupResult.snapshotId ) ) { throw new Error(`Backup not listed for service: ${JSON.stringify(backups)}`); } console.log(`[${scenarioName}] Restoring backup as ${cloneServiceName}`); const restoreResult = await onebox.backupManager.restoreBackup(backupId, { mode: 'clone', newServiceName: cloneServiceName, }); clonedService = true; if ( restoreResult.service.name !== cloneServiceName || restoreResult.service.status !== 'running' ) { throw new Error( `Unexpected restored service state: ${JSON.stringify(restoreResult.service)}`, ); } if (restoreResult.service.domain !== undefined) { throw new Error( `Clone unexpectedly retained a domain: ${JSON.stringify(restoreResult.service)}`, ); } if (restoreResult.service.envVars.ONEBOX_BACKUP_SCENARIO !== smokeId) { throw new Error( `Clone did not preserve env vars: ${JSON.stringify(restoreResult.service.envVars)}`, ); } if (!restoreResult.service.volumes?.some((volumeArg) => volumeArg.mountPath === '/data')) { throw new Error( `Clone did not preserve volumes: ${JSON.stringify(restoreResult.service.volumes)}`, ); } if (restoreResult.warnings.length > 0) { throw new Error(`Restore completed with warnings: ${restoreResult.warnings.join('; ')}`); } await waitForDockerServiceRunning(cloneDockerServiceName); await assertCloneVolumeMount(); console.log(`[${scenarioName}] Removing restored workload ${cloneServiceName}`); await onebox.services.removeService(cloneServiceName); clonedService = false; await waitForDockerServiceRemoved(cloneDockerServiceName); console.log(`[${scenarioName}] Removing workload ${serviceName}`); await onebox.services.removeService(serviceName); deployedService = false; await waitForDockerServiceRemoved(dockerServiceName); console.log(`[${scenarioName}] Scenario passed`); } finally { if (onebox && clonedService) { await onebox.services.removeService(cloneServiceName).catch((error) => { console.log( `[${scenarioName}] Failed to remove restored Onebox service: ${(error as Error).message}`, ); }); } if (onebox && deployedService) { await onebox.services.removeService(serviceName).catch((error) => { console.log( `[${scenarioName}] Failed to remove Onebox service: ${(error as Error).message}`, ); }); } if (onebox) { await onebox.shutdown().catch((error) => { console.log(`[${scenarioName}] Failed to shut down Onebox: ${(error as Error).message}`); }); } await removeDockerService(cloneDockerServiceName); await removeDockerService(dockerServiceName); await removeDockerService('onebox-smartproxy'); await removeDockerVolume(cloneVolumeName); await removeDockerVolume(volumeName); } }; try { await main(); Deno.exit(0); } catch (error) { console.error(error); Deno.exit(1); }