336 lines
11 KiB
TypeScript
336 lines
11 KiB
TypeScript
import { dirname, fromFileUrl, join, resolve } from '@std/path';
|
|
|
|
import { Onebox } from '../../../onebox/ts/classes/onebox.ts';
|
|
import { disableManagedDcRouterForScenario } from '../onebox-test-helpers.ts';
|
|
|
|
const scenarioName = 'onebox-backup-restore';
|
|
const smokeId = `onebox-backup-${Date.now().toString(36)}`;
|
|
const testingDir = resolve(dirname(fromFileUrl(import.meta.url)), '../..');
|
|
const buildDir = join(testingDir, '.nogit', scenarioName, smokeId);
|
|
const serviceName = `app-${Date.now().toString(36)}`;
|
|
const cloneServiceName = `${serviceName}-clone`;
|
|
const dockerServiceName = `onebox-${serviceName}`;
|
|
const cloneDockerServiceName = `onebox-${cloneServiceName}`;
|
|
const volumeName = `onebox-${serviceName}-data`;
|
|
const cloneVolumeName = `onebox-${cloneServiceName}-data`;
|
|
|
|
const delayFor = async (millisecondsArg: number) => {
|
|
await new Promise((resolveArg) => setTimeout(resolveArg, millisecondsArg));
|
|
};
|
|
|
|
const outputCommand = async (
|
|
commandArg: string,
|
|
argsArg: string[],
|
|
optionsArg: Pick<Deno.CommandOptions, 'stdout' | 'stderr'>,
|
|
timeoutMsArg = 30000,
|
|
) => {
|
|
const controller = new AbortController();
|
|
const timeoutId = setTimeout(() => controller.abort(), timeoutMsArg);
|
|
try {
|
|
const command = new Deno.Command(commandArg, {
|
|
args: argsArg,
|
|
signal: controller.signal,
|
|
...optionsArg,
|
|
});
|
|
return await command.output();
|
|
} catch (error) {
|
|
if ((error as Error).name === 'AbortError') {
|
|
throw new Error(`${commandArg} ${argsArg.join(' ')} timed out after ${timeoutMsArg}ms`);
|
|
}
|
|
throw error;
|
|
} finally {
|
|
clearTimeout(timeoutId);
|
|
}
|
|
};
|
|
|
|
const run = async (commandArg: string, argsArg: string[]) => {
|
|
const output = await outputCommand(commandArg, argsArg, {
|
|
stdout: 'piped',
|
|
stderr: 'piped',
|
|
});
|
|
const stdout = new TextDecoder().decode(output.stdout).trim();
|
|
const stderr = new TextDecoder().decode(output.stderr).trim();
|
|
if (stdout) {
|
|
console.log(stdout);
|
|
}
|
|
if (stderr) {
|
|
console.log(stderr);
|
|
}
|
|
if (!output.success) {
|
|
throw new Error(`${commandArg} ${argsArg.join(' ')} exited with ${output.code}`);
|
|
}
|
|
return { stdout, stderr };
|
|
};
|
|
|
|
const waitFor = async (checkFunctionArg: () => boolean | Promise<boolean>, messageArg: string) => {
|
|
const startTime = Date.now();
|
|
while (Date.now() - startTime < 120000) {
|
|
if (await checkFunctionArg()) {
|
|
return;
|
|
}
|
|
await delayFor(500);
|
|
}
|
|
throw new Error(`Timed out waiting for ${messageArg}`);
|
|
};
|
|
|
|
const dockerServiceExists = async (serviceNameArg: string) => {
|
|
const output = await outputCommand('docker', ['service', 'inspect', serviceNameArg], {
|
|
stdout: 'null',
|
|
stderr: 'null',
|
|
}, 15000);
|
|
return output.success;
|
|
};
|
|
|
|
const removeDockerService = async (serviceNameArg: string) => {
|
|
if (await dockerServiceExists(serviceNameArg)) {
|
|
await run('docker', ['service', 'rm', serviceNameArg]).catch(() => null);
|
|
await delayFor(2000);
|
|
}
|
|
};
|
|
|
|
const removeDockerVolume = async (volumeNameArg: string) => {
|
|
let lastErrorText = '';
|
|
const startTime = Date.now();
|
|
while (Date.now() - startTime < 30000) {
|
|
const output = await outputCommand(
|
|
'docker',
|
|
['volume', 'rm', '-f', volumeNameArg],
|
|
{
|
|
stdout: 'piped',
|
|
stderr: 'piped',
|
|
},
|
|
15000,
|
|
);
|
|
if (output.success) {
|
|
return;
|
|
}
|
|
lastErrorText = new TextDecoder().decode(output.stderr).trim();
|
|
await delayFor(1000);
|
|
}
|
|
console.log(
|
|
`[${scenarioName}] Failed to remove Docker volume ${volumeNameArg}: ${lastErrorText}`,
|
|
);
|
|
};
|
|
|
|
const assertNoPreexistingOneboxIngress = async () => {
|
|
if (await dockerServiceExists('onebox-smartproxy')) {
|
|
throw new Error(
|
|
'onebox-smartproxy already exists; refusing to overwrite a running Onebox ingress service',
|
|
);
|
|
}
|
|
};
|
|
|
|
const waitForDockerServiceRunning = async (serviceNameArg: string) => {
|
|
await waitFor(async () => {
|
|
const output = await outputCommand('docker', [
|
|
'service',
|
|
'ps',
|
|
serviceNameArg,
|
|
'--format',
|
|
'{{.CurrentState}}',
|
|
], {
|
|
stdout: 'piped',
|
|
stderr: 'null',
|
|
}, 15000);
|
|
if (!output.success) {
|
|
return false;
|
|
}
|
|
return new TextDecoder().decode(output.stdout).includes('Running');
|
|
}, `${serviceNameArg} running task`);
|
|
};
|
|
|
|
const waitForDockerServiceRemoved = async (serviceNameArg: string) => {
|
|
await waitFor(
|
|
async () => !(await dockerServiceExists(serviceNameArg)),
|
|
`${serviceNameArg} removal`,
|
|
);
|
|
};
|
|
|
|
const inspectDockerServiceJson = async <T>(serviceNameArg: string, formatArg: string) => {
|
|
const output = await outputCommand('docker', [
|
|
'service',
|
|
'inspect',
|
|
serviceNameArg,
|
|
'--format',
|
|
formatArg,
|
|
], {
|
|
stdout: 'piped',
|
|
stderr: 'piped',
|
|
}, 15000);
|
|
if (!output.success) {
|
|
throw new Error(`docker service inspect ${serviceNameArg} exited with ${output.code}`);
|
|
}
|
|
return JSON.parse(new TextDecoder().decode(output.stdout).trim()) as T;
|
|
};
|
|
|
|
const assertCloneVolumeMount = async () => {
|
|
const mounts = await inspectDockerServiceJson<Array<Record<string, unknown>>>(
|
|
cloneDockerServiceName,
|
|
'{{json .Spec.TaskTemplate.ContainerSpec.Mounts}}',
|
|
);
|
|
if (
|
|
!mounts.some((mountArg) => mountArg.Source === cloneVolumeName && mountArg.Target === '/data')
|
|
) {
|
|
throw new Error(
|
|
`Cloned service volume mount missing from Docker service: ${JSON.stringify(mounts)}`,
|
|
);
|
|
}
|
|
};
|
|
|
|
const ensureDockerReady = async () => {
|
|
await run('docker', ['version']);
|
|
const { stdout } = await run('docker', ['info', '--format', '{{.Swarm.LocalNodeState}}']);
|
|
if (stdout.trim() !== 'active') {
|
|
throw new Error(
|
|
'Docker Swarm must be active. In Vagrant this is handled by scripts/provision-vm.sh.',
|
|
);
|
|
}
|
|
};
|
|
|
|
const main = async () => {
|
|
let onebox: Onebox | undefined;
|
|
let deployedService = false;
|
|
let clonedService = false;
|
|
|
|
try {
|
|
await ensureDockerReady();
|
|
await assertNoPreexistingOneboxIngress();
|
|
await Deno.mkdir(buildDir, { recursive: true });
|
|
Deno.chdir(buildDir);
|
|
Deno.env.set('ONEBOX_DEV', 'true');
|
|
|
|
console.log(`[${scenarioName}] Starting Onebox from ${buildDir}`);
|
|
onebox = new Onebox();
|
|
disableManagedDcRouterForScenario(onebox);
|
|
await onebox.init();
|
|
|
|
await waitForDockerServiceRunning('onebox-smartproxy');
|
|
|
|
console.log(`[${scenarioName}] Deploying workload ${serviceName}`);
|
|
const service = await onebox.services.deployService({
|
|
name: serviceName,
|
|
image: 'caddy:2-alpine',
|
|
port: 80,
|
|
autoDNS: false,
|
|
envVars: {
|
|
ONEBOX_BACKUP_SCENARIO: smokeId,
|
|
},
|
|
volumes: [{ mountPath: '/data', backup: true }],
|
|
});
|
|
deployedService = true;
|
|
|
|
if (service.status !== 'running') {
|
|
throw new Error(`Unexpected deployed service state: ${JSON.stringify(service)}`);
|
|
}
|
|
|
|
await waitForDockerServiceRunning(dockerServiceName);
|
|
|
|
console.log(`[${scenarioName}] Creating backup for ${serviceName}`);
|
|
const backupResult = await onebox.backupManager.createBackup(serviceName);
|
|
const backupId = backupResult.backup.id;
|
|
if (!backupId || !backupResult.snapshotId || !backupResult.backup.snapshotId) {
|
|
throw new Error(
|
|
`Backup did not produce a ContainerArchive snapshot: ${JSON.stringify(backupResult)}`,
|
|
);
|
|
}
|
|
if (!backupResult.backup.includesImage) {
|
|
throw new Error(
|
|
`Backup did not include Docker image: ${JSON.stringify(backupResult.backup)}`,
|
|
);
|
|
}
|
|
if ((backupResult.backup.storedSizeBytes ?? 0) <= 0 || backupResult.backup.sizeBytes <= 0) {
|
|
throw new Error(`Backup size metadata is invalid: ${JSON.stringify(backupResult.backup)}`);
|
|
}
|
|
|
|
const backups = onebox.backupManager.listBackups(serviceName);
|
|
if (
|
|
!backups.some((backupArg) =>
|
|
backupArg.id === backupId && backupArg.snapshotId === backupResult.snapshotId
|
|
)
|
|
) {
|
|
throw new Error(`Backup not listed for service: ${JSON.stringify(backups)}`);
|
|
}
|
|
|
|
console.log(`[${scenarioName}] Restoring backup as ${cloneServiceName}`);
|
|
const restoreResult = await onebox.backupManager.restoreBackup(backupId, {
|
|
mode: 'clone',
|
|
newServiceName: cloneServiceName,
|
|
});
|
|
clonedService = true;
|
|
|
|
if (
|
|
restoreResult.service.name !== cloneServiceName || restoreResult.service.status !== 'running'
|
|
) {
|
|
throw new Error(
|
|
`Unexpected restored service state: ${JSON.stringify(restoreResult.service)}`,
|
|
);
|
|
}
|
|
if (restoreResult.service.domain !== undefined) {
|
|
throw new Error(
|
|
`Clone unexpectedly retained a domain: ${JSON.stringify(restoreResult.service)}`,
|
|
);
|
|
}
|
|
if (restoreResult.service.envVars.ONEBOX_BACKUP_SCENARIO !== smokeId) {
|
|
throw new Error(
|
|
`Clone did not preserve env vars: ${JSON.stringify(restoreResult.service.envVars)}`,
|
|
);
|
|
}
|
|
if (!restoreResult.service.volumes?.some((volumeArg) => volumeArg.mountPath === '/data')) {
|
|
throw new Error(
|
|
`Clone did not preserve volumes: ${JSON.stringify(restoreResult.service.volumes)}`,
|
|
);
|
|
}
|
|
if (restoreResult.warnings.length > 0) {
|
|
throw new Error(`Restore completed with warnings: ${restoreResult.warnings.join('; ')}`);
|
|
}
|
|
|
|
await waitForDockerServiceRunning(cloneDockerServiceName);
|
|
await assertCloneVolumeMount();
|
|
|
|
console.log(`[${scenarioName}] Removing restored workload ${cloneServiceName}`);
|
|
await onebox.services.removeService(cloneServiceName);
|
|
clonedService = false;
|
|
await waitForDockerServiceRemoved(cloneDockerServiceName);
|
|
|
|
console.log(`[${scenarioName}] Removing workload ${serviceName}`);
|
|
await onebox.services.removeService(serviceName);
|
|
deployedService = false;
|
|
await waitForDockerServiceRemoved(dockerServiceName);
|
|
|
|
console.log(`[${scenarioName}] Scenario passed`);
|
|
} finally {
|
|
if (onebox && clonedService) {
|
|
await onebox.services.removeService(cloneServiceName).catch((error) => {
|
|
console.log(
|
|
`[${scenarioName}] Failed to remove restored Onebox service: ${(error as Error).message}`,
|
|
);
|
|
});
|
|
}
|
|
if (onebox && deployedService) {
|
|
await onebox.services.removeService(serviceName).catch((error) => {
|
|
console.log(
|
|
`[${scenarioName}] Failed to remove Onebox service: ${(error as Error).message}`,
|
|
);
|
|
});
|
|
}
|
|
if (onebox) {
|
|
await onebox.shutdown().catch((error) => {
|
|
console.log(`[${scenarioName}] Failed to shut down Onebox: ${(error as Error).message}`);
|
|
});
|
|
}
|
|
await removeDockerService(cloneDockerServiceName);
|
|
await removeDockerService(dockerServiceName);
|
|
await removeDockerService('onebox-smartproxy');
|
|
await removeDockerVolume(cloneVolumeName);
|
|
await removeDockerVolume(volumeName);
|
|
}
|
|
};
|
|
|
|
try {
|
|
await main();
|
|
Deno.exit(0);
|
|
} catch (error) {
|
|
console.error(error);
|
|
Deno.exit(1);
|
|
}
|