import { dirname, fromFileUrl, join, resolve } from '@std/path'; import { Onebox } from '../../../onebox/ts/classes/onebox.ts'; import { disableManagedDcRouterForScenario, useLocalAppStoreForScenario, } from '../onebox-test-helpers.ts'; const scenarioName = 'onebox-cloudly-appstore-worker'; const smokeId = `cloudly-appstore-${Date.now().toString(36)}`; const testingDir = resolve(dirname(fromFileUrl(import.meta.url)), '../..'); const repoRoot = resolve(testingDir, '..'); const appCatalogDir = join(repoRoot, 'appstore-apptemplates'); const buildDir = join(testingDir, '.nogit', scenarioName, smokeId); const serviceName = `cloudly-${Date.now().toString(36)}`; const dockerServiceName = `onebox-${serviceName}`; const routeDomain = `${serviceName}.test`; const delayFor = async (millisecondsArg: number) => { await new Promise((resolveArg) => setTimeout(resolveArg, millisecondsArg)); }; const run = async (commandArg: string, argsArg: string[]) => { const command = new Deno.Command(commandArg, { args: argsArg, stdout: 'piped', stderr: 'piped', }); const output = await command.output(); const stdout = new TextDecoder().decode(output.stdout).trim(); const stderr = new TextDecoder().decode(output.stderr).trim(); if (stdout) { console.log(stdout); } if (stderr) { console.log(stderr); } if (!output.success) { throw new Error(`${commandArg} ${argsArg.join(' ')} exited with ${output.code}`); } return { stdout, stderr }; }; const waitFor = async ( checkFunctionArg: () => boolean | Promise, messageArg: string, timeoutMsArg = 180000, ) => { const startTime = Date.now(); while (Date.now() - startTime < timeoutMsArg) { if (await checkFunctionArg()) { return; } await delayFor(1000); } throw new Error(`Timed out waiting for ${messageArg}`); }; const dockerServiceExists = async (serviceNameArg: string) => { const command = new Deno.Command('docker', { args: ['service', 'inspect', serviceNameArg], stdout: 'null', stderr: 'null', }); return (await command.output()).success; }; const removeDockerService = async (serviceNameArg: string) => { if (await dockerServiceExists(serviceNameArg)) { await run('docker', ['service', 'rm', serviceNameArg]).catch(() => null); await delayFor(2000); } }; const dockerContainerRunning = async (containerNameArg: string) => { const command = new Deno.Command('docker', { args: ['container', 'inspect', containerNameArg, '--format', '{{.State.Running}}'], stdout: 'piped', stderr: 'null', }); const output = await command.output(); if (!output.success) { return false; } return new TextDecoder().decode(output.stdout).trim() === 'true'; }; const dockerContainerExists = async (containerNameArg: string) => { const command = new Deno.Command('docker', { args: ['container', 'inspect', containerNameArg], stdout: 'null', stderr: 'null', }); return (await command.output()).success; }; const removeDockerContainer = async (containerNameArg: string) => { if (await dockerContainerExists(containerNameArg)) { await run('docker', ['container', 'rm', '-f', containerNameArg]).catch(() => null); await delayFor(1000); } }; const assertNoPreexistingScenarioServices = async () => { for (const serviceNameArg of [dockerServiceName, 'onebox-smartproxy']) { if (await dockerServiceExists(serviceNameArg)) { throw new Error(`${serviceNameArg} already exists; refusing to overwrite a running service`); } } for (const containerNameArg of ['onebox-mongodb', 'onebox-minio']) { if (await dockerContainerExists(containerNameArg)) { const state = await dockerContainerRunning(containerNameArg) ? 'running' : 'present'; throw new Error( `${containerNameArg} is already ${state}; refusing to overwrite a platform container`, ); } } }; const waitForDockerServiceRunning = async (serviceNameArg: string) => { try { await waitFor(async () => { const command = new Deno.Command('docker', { args: ['service', 'ps', serviceNameArg, '--format', '{{.CurrentState}}'], stdout: 'piped', stderr: 'null', }); const output = await command.output(); if (!output.success) { return false; } return new TextDecoder().decode(output.stdout).includes('Running'); }, `${serviceNameArg} running task`); } catch (error) { await run('docker', ['service', 'ps', '--no-trunc', serviceNameArg]).catch(() => null); await run('docker', ['service', 'logs', '--raw', '--tail', '120', serviceNameArg]).catch(() => null ); throw error; } }; const waitForDockerContainerRunning = async (containerNameArg: string) => { await waitFor( async () => dockerContainerRunning(containerNameArg), `${containerNameArg} running container`, ); }; const ensureDockerReady = async () => { await run('docker', ['version']); const { stdout } = await run('docker', ['info', '--format', '{{.Swarm.LocalNodeState}}']); if (stdout.trim() !== 'active') { throw new Error( 'Docker Swarm must be active. In Vagrant this is handled by scripts/provision-vm.sh.', ); } }; const requestCloudlyBootstrapScript = async () => { const curlArgs = [ '-sS', '--noproxy', '*', '--max-time', '10', '--resolve', `${routeDomain}:8080:127.0.0.1`, '-o', '-', '-w', '\n%{http_code}', `http://${routeDomain}:8080/curlfresh/setup.sh`, ]; const command = new Deno.Command('curl', { args: curlArgs, stdout: 'piped', stderr: 'piped', }); const output = await command.output(); const stdout = new TextDecoder().decode(output.stdout).trim(); const stderr = new TextDecoder().decode(output.stderr).trim(); if (!output.success) { throw new Error(`curl failed: ${stderr || stdout}`); } const lines = stdout.split('\n'); const statusCode = lines[lines.length - 1]; const body = lines.slice(0, -1).join('\n'); return { statusCode, body, stderr }; }; const waitForCloudlyWorkerBootstrapRoute = async () => { let lastResponse: Awaited> | undefined; let lastError: Error | undefined; try { await waitFor(async () => { try { lastResponse = await requestCloudlyBootstrapScript(); lastError = undefined; return ( lastResponse.statusCode === '200' && lastResponse.body.includes('pnpm install -g @serve.zone/spark') && lastResponse.body.includes('spark installdaemon') ); } catch (error) { lastError = error as Error; return false; } }, 'Cloudly worker bootstrap route through Onebox ingress'); } catch (error) { console.log(`[${scenarioName}] Last bootstrap response: ${JSON.stringify(lastResponse)}`); if (lastError) { console.log(`[${scenarioName}] Last bootstrap error: ${lastError.message}`); } await run('docker', ['service', 'ps', 'onebox-smartproxy']).catch(() => null); await run('docker', ['service', 'logs', '--raw', '--tail', '120', 'onebox-smartproxy']).catch( () => null, ); await run('docker', ['service', 'ps', dockerServiceName]).catch(() => null); await run('docker', ['service', 'logs', '--raw', '--tail', '120', dockerServiceName]).catch( () => null, ); throw error; } }; const main = async () => { let onebox: Onebox | undefined; let deployedService = false; try { await ensureDockerReady(); await assertNoPreexistingScenarioServices(); await Deno.mkdir(buildDir, { recursive: true }); Deno.chdir(buildDir); Deno.env.set('ONEBOX_DEV', 'true'); console.log(`[${scenarioName}] Starting Onebox from ${buildDir}`); onebox = new Onebox(); disableManagedDcRouterForScenario(onebox); useLocalAppStoreForScenario(onebox, appCatalogDir); await onebox.init(); await waitForDockerServiceRunning('onebox-smartproxy'); const appMeta = await onebox.appStore.getAppMeta('cloudly'); const version = appMeta.latestVersion; const config = await onebox.appStore.getAppVersionConfig('cloudly', version); if (!config.platformRequirements?.mongodb || !config.platformRequirements?.s3) { throw new Error( `Cloudly template must require MongoDB and S3: ${ JSON.stringify(config.platformRequirements) }`, ); } console.log(`[${scenarioName}] Installing Cloudly template ${version} as ${serviceName}`); const service = await onebox.appStore.installApp({ appId: 'cloudly', version, serviceName, domain: routeDomain, autoDNS: false, envVars: { SERVEZONE_ADMINACCOUNT: 'testadmin:testpassword', }, }); deployedService = true; if (service.appTemplateId !== 'cloudly' || service.appTemplateVersion !== version) { throw new Error(`Cloudly template tracking missing on service: ${JSON.stringify(service)}`); } await waitForDockerContainerRunning('onebox-mongodb'); await waitForDockerContainerRunning('onebox-minio'); await waitForDockerServiceRunning(dockerServiceName); await waitForCloudlyWorkerBootstrapRoute(); const deployedServiceRecord = onebox.services.getService(serviceName); if (!deployedServiceRecord?.id) { throw new Error( `Cloudly service missing after deploy: ${JSON.stringify(deployedServiceRecord)}`, ); } if (deployedServiceRecord.envVars.SERVEZONE_URL !== routeDomain) { throw new Error( `Cloudly SERVICE_DOMAIN template was not resolved: ${ JSON.stringify(deployedServiceRecord.envVars) }`, ); } if ( !deployedServiceRecord.envVars.MONGODB_URL || deployedServiceRecord.envVars.MONGODB_URL.includes('${') ) { throw new Error( `Cloudly MongoDB template was not resolved: ${ JSON.stringify(deployedServiceRecord.envVars) }`, ); } const platformResources = await onebox.platformServices.getResourcesForService( deployedServiceRecord.id, ); const platformResourceTypes = platformResources.map((resourceArg) => resourceArg.platformService.type ).sort(); if (platformResourceTypes.join(',') !== 'minio,mongodb') { throw new Error( `Unexpected Cloudly platform resources: ${JSON.stringify(platformResourceTypes)}`, ); } console.log(`[${scenarioName}] Creating Cloudly platform-resource backup for ${serviceName}`); onebox.database.updateService(deployedServiceRecord.id, { includeImageInBackup: false }); const backupResult = await onebox.backupManager.createBackup(serviceName); const backupId = backupResult.backup.id; if (!backupId || !backupResult.snapshotId || !backupResult.backup.snapshotId) { throw new Error( `Cloudly backup did not produce a ContainerArchive snapshot: ${ JSON.stringify(backupResult) }`, ); } if (backupResult.backup.includesImage) { throw new Error( `Cloudly backup unexpectedly included the large app image: ${ JSON.stringify(backupResult.backup) }`, ); } if (backupResult.backup.platformResources.sort().join(',') !== 'minio,mongodb') { throw new Error( `Cloudly backup missing platform resources: ${JSON.stringify(backupResult.backup)}`, ); } console.log(`[${scenarioName}] Restoring Cloudly platform-resource backup for ${serviceName}`); const restoreResult = await onebox.backupManager.restoreBackup(backupId, { mode: 'restore', overwriteExisting: true, }); if (restoreResult.service.name !== serviceName) { throw new Error( `Cloudly restore returned unexpected service: ${JSON.stringify(restoreResult.service)}`, ); } if (restoreResult.platformResourcesRestored !== 2 || restoreResult.warnings.length > 0) { throw new Error( `Cloudly restore failed platform resource validation: ${JSON.stringify(restoreResult)}`, ); } await waitForCloudlyWorkerBootstrapRoute(); console.log(`[${scenarioName}] Scenario passed`); } finally { if (onebox) { if (deployedService) { await onebox.services.removeService(serviceName).catch((error) => { console.log(`[${scenarioName}] Service cleanup failed: ${(error as Error).message}`); }); } await onebox.platformServices.stopPlatformService('mongodb').catch(() => null); await onebox.platformServices.stopPlatformService('minio').catch(() => null); await onebox.shutdown(); } await removeDockerService(dockerServiceName); await removeDockerContainer('onebox-mongodb'); await removeDockerContainer('onebox-minio'); await removeDockerService('onebox-smartproxy'); } }; try { await main(); Deno.exit(0); } catch (error) { console.error(error); Deno.exit(1); }