diff --git a/package.json b/package.json index 9959f9a..6dfe9ea 100644 --- a/package.json +++ b/package.json @@ -7,10 +7,13 @@ "scripts": { "bootstrap:components": "pnpm --dir ../interfaces install && pnpm --dir ../api install && pnpm --dir ../cloudly install && pnpm --dir ../coreflow install && pnpm --dir ../coretraffic install && pnpm --dir ../onebox exec deno install --config deno.json && pnpm install", "test": "pnpm scenario:registry-deploy-on-push && pnpm scenario:onebox-basic-lifecycle", + "test:full": "pnpm test && pnpm scenario:onebox-cloudly-appstore-worker", "scenario:registry-deploy-on-push": "tsx --tsconfig ../cloudly/tsconfig.json scenarios/registry-deploy-on-push/scenario.ts", "scenario:onebox-basic-lifecycle": "deno run --allow-all --config ../onebox/deno.json scenarios/onebox-basic-lifecycle/scenario.ts", + "scenario:onebox-cloudly-appstore-worker": "deno run --allow-all --config ../onebox/deno.json scenarios/onebox-cloudly-appstore-worker/scenario.ts", "vagrant:up": "vagrant up", "vagrant:test": "vagrant ssh -c 'cd /serve.zone/testing && pnpm bootstrap:components && pnpm test'", + "vagrant:test:full": "vagrant ssh -c 'cd /serve.zone/testing && pnpm bootstrap:components && pnpm test:full'", "vagrant:destroy": "vagrant destroy -f" }, "devDependencies": { diff --git a/scenarios/onebox-cloudly-appstore-worker/readme.md b/scenarios/onebox-cloudly-appstore-worker/readme.md new file mode 100644 index 0000000..e8e821f --- /dev/null +++ b/scenarios/onebox-cloudly-appstore-worker/readme.md @@ -0,0 +1,20 @@ +# onebox-cloudly-appstore-worker + +This full scenario verifies the Onebox-to-Cloudly app-store path and the worker bootstrap endpoint exposed by the installed Cloudly service. + +Assertions: + +- Onebox starts from an isolated bare working directory in dev mode. +- The remote app-store catalog exposes the Cloudly template. +- Installing Cloudly from the template provisions MongoDB and MinIO platform resources. +- Cloudly starts as a Onebox-managed Docker service with the template version tracked on the service record. +- Onebox ingress routes HTTP traffic to the installed Cloudly service. +- Cloudly exposes `/curlfresh/setup.sh`, which is the current worker-node bootstrap script used by freshly provisioned nodes. + +Run with: + +```bash +pnpm scenario:onebox-cloudly-appstore-worker +``` + +This scenario is intentionally excluded from the default `pnpm test` because it pulls and starts the Cloudly image plus platform services. diff --git a/scenarios/onebox-cloudly-appstore-worker/scenario.ts b/scenarios/onebox-cloudly-appstore-worker/scenario.ts new file mode 100644 index 0000000..f1d5ebf --- /dev/null +++ b/scenarios/onebox-cloudly-appstore-worker/scenario.ts @@ -0,0 +1,293 @@ +import { dirname, fromFileUrl, join, resolve } from '@std/path'; + +import { Onebox } from '../../../onebox/ts/classes/onebox.ts'; +import type { IAppVersionConfig } from '../../../onebox/ts/classes/appstore-types.ts'; + +const scenarioName = 'onebox-cloudly-appstore-worker'; +const smokeId = `cloudly-appstore-${Date.now().toString(36)}`; +const testingDir = resolve(dirname(fromFileUrl(import.meta.url)), '../..'); +const buildDir = join(testingDir, '.nogit', scenarioName, smokeId); +const serviceName = `cloudly-${Date.now().toString(36)}`; +const dockerServiceName = `onebox-${serviceName}`; +const routeDomain = `${serviceName}.test`; + +const delayFor = async (millisecondsArg: number) => { + await new Promise((resolveArg) => setTimeout(resolveArg, millisecondsArg)); +}; + +const run = async (commandArg: string, argsArg: string[]) => { + const command = new Deno.Command(commandArg, { + args: argsArg, + stdout: 'piped', + stderr: 'piped', + }); + const output = await command.output(); + const stdout = new TextDecoder().decode(output.stdout).trim(); + const stderr = new TextDecoder().decode(output.stderr).trim(); + if (stdout) { + console.log(stdout); + } + if (stderr) { + console.log(stderr); + } + if (!output.success) { + throw new Error(`${commandArg} ${argsArg.join(' ')} exited with ${output.code}`); + } + return { stdout, stderr }; +}; + +const waitFor = async ( + checkFunctionArg: () => boolean | Promise, + messageArg: string, + timeoutMsArg = 180000, +) => { + const startTime = Date.now(); + while (Date.now() - startTime < timeoutMsArg) { + if (await checkFunctionArg()) { + return; + } + await delayFor(1000); + } + throw new Error(`Timed out waiting for ${messageArg}`); +}; + +const dockerServiceExists = async (serviceNameArg: string) => { + const command = new Deno.Command('docker', { + args: ['service', 'inspect', serviceNameArg], + stdout: 'null', + stderr: 'null', + }); + return (await command.output()).success; +}; + +const removeDockerService = async (serviceNameArg: string) => { + if (await dockerServiceExists(serviceNameArg)) { + await run('docker', ['service', 'rm', serviceNameArg]).catch(() => null); + await delayFor(2000); + } +}; + +const dockerContainerRunning = async (containerNameArg: string) => { + const command = new Deno.Command('docker', { + args: ['container', 'inspect', containerNameArg, '--format', '{{.State.Running}}'], + stdout: 'piped', + stderr: 'null', + }); + const output = await command.output(); + if (!output.success) { + return false; + } + return new TextDecoder().decode(output.stdout).trim() === 'true'; +}; + +const assertNoPreexistingScenarioServices = async () => { + for (const serviceNameArg of [dockerServiceName, 'onebox-smartproxy']) { + if (await dockerServiceExists(serviceNameArg)) { + throw new Error(`${serviceNameArg} already exists; refusing to overwrite a running service`); + } + } + for (const containerNameArg of ['onebox-mongodb', 'onebox-minio']) { + if (await dockerContainerRunning(containerNameArg)) { + throw new Error(`${containerNameArg} is already running; refusing to overwrite a platform container`); + } + } +}; + +const waitForDockerServiceRunning = async (serviceNameArg: string) => { + try { + await waitFor(async () => { + const command = new Deno.Command('docker', { + args: ['service', 'ps', serviceNameArg, '--format', '{{.CurrentState}}'], + stdout: 'piped', + stderr: 'null', + }); + const output = await command.output(); + if (!output.success) { + return false; + } + return new TextDecoder().decode(output.stdout).includes('Running'); + }, `${serviceNameArg} running task`); + } catch (error) { + await run('docker', ['service', 'ps', '--no-trunc', serviceNameArg]).catch(() => null); + await run('docker', ['service', 'logs', '--raw', '--tail', '120', serviceNameArg]).catch(() => null); + throw error; + } +}; + +const waitForDockerContainerRunning = async (containerNameArg: string) => { + await waitFor(async () => dockerContainerRunning(containerNameArg), `${containerNameArg} running container`); +}; + +const ensureDockerReady = async () => { + await run('docker', ['version']); + const { stdout } = await run('docker', ['info', '--format', '{{.Swarm.LocalNodeState}}']); + if (stdout.trim() !== 'active') { + throw new Error('Docker Swarm must be active. In Vagrant this is handled by scripts/provision-vm.sh.'); + } +}; + +const getTemplateEnvVars = ( + configArg: IAppVersionConfig, + overridesArg: Record, +) => { + const envVars: Record = {}; + const missingRequiredEnvVars: string[] = []; + + for (const envVar of configArg.envVars || []) { + const value = overridesArg[envVar.key] ?? envVar.value ?? ''; + if (envVar.required && !value) { + missingRequiredEnvVars.push(envVar.key); + } + envVars[envVar.key] = value; + } + + for (const [key, value] of Object.entries(overridesArg)) { + envVars[key] = value; + } + + if (missingRequiredEnvVars.length > 0) { + throw new Error(`Missing required template env vars: ${missingRequiredEnvVars.join(', ')}`); + } + + return envVars; +}; + +const requestCloudlyBootstrapScript = async () => { + const curlArgs = [ + '-sS', + '--noproxy', + '*', + '--max-time', + '10', + '--resolve', + `${routeDomain}:8080:127.0.0.1`, + '-o', + '-', + '-w', + '\n%{http_code}', + `http://${routeDomain}:8080/curlfresh/setup.sh`, + ]; + + const command = new Deno.Command('curl', { + args: curlArgs, + stdout: 'piped', + stderr: 'piped', + }); + const output = await command.output(); + const stdout = new TextDecoder().decode(output.stdout).trim(); + const stderr = new TextDecoder().decode(output.stderr).trim(); + if (!output.success) { + throw new Error(`curl failed: ${stderr || stdout}`); + } + const lines = stdout.split('\n'); + const statusCode = lines[lines.length - 1]; + const body = lines.slice(0, -1).join('\n'); + return { statusCode, body, stderr }; +}; + +const waitForCloudlyWorkerBootstrapRoute = async () => { + let lastResponse: Awaited> | undefined; + let lastError: Error | undefined; + try { + await waitFor(async () => { + try { + lastResponse = await requestCloudlyBootstrapScript(); + lastError = undefined; + return ( + lastResponse.statusCode === '200' && + lastResponse.body.includes('pnpm install -g @serve.zone/spark') && + lastResponse.body.includes('spark installdaemon') + ); + } catch (error) { + lastError = error as Error; + return false; + } + }, 'Cloudly worker bootstrap route through Onebox ingress'); + } catch (error) { + console.log(`[${scenarioName}] Last bootstrap response: ${JSON.stringify(lastResponse)}`); + if (lastError) { + console.log(`[${scenarioName}] Last bootstrap error: ${lastError.message}`); + } + await run('docker', ['service', 'ps', 'onebox-smartproxy']).catch(() => null); + await run('docker', ['service', 'logs', '--raw', '--tail', '120', 'onebox-smartproxy']).catch(() => null); + await run('docker', ['service', 'ps', dockerServiceName]).catch(() => null); + await run('docker', ['service', 'logs', '--raw', '--tail', '120', dockerServiceName]).catch(() => null); + throw error; + } +}; + +const main = async () => { + let onebox: Onebox | undefined; + let deployedService = false; + + try { + await ensureDockerReady(); + await assertNoPreexistingScenarioServices(); + await Deno.mkdir(buildDir, { recursive: true }); + Deno.chdir(buildDir); + Deno.env.set('ONEBOX_DEV', 'true'); + + console.log(`[${scenarioName}] Starting Onebox from ${buildDir}`); + onebox = new Onebox(); + await onebox.init(); + await waitForDockerServiceRunning('onebox-smartproxy'); + + const appMeta = await onebox.appStore.getAppMeta('cloudly'); + const version = appMeta.latestVersion; + const config = await onebox.appStore.getAppVersionConfig('cloudly', version); + if (!config.platformRequirements?.mongodb || !config.platformRequirements?.s3) { + throw new Error(`Cloudly template must require MongoDB and S3: ${JSON.stringify(config.platformRequirements)}`); + } + + console.log(`[${scenarioName}] Installing Cloudly template ${version} as ${serviceName}`); + const envVars = getTemplateEnvVars(config, { + SERVEZONE_ADMINACCOUNT: 'testadmin:testpassword', + }); + + const service = await onebox.services.deployService({ + name: serviceName, + image: config.image, + port: config.port, + domain: routeDomain, + autoDNS: false, + envVars, + enableMongoDB: Boolean(config.platformRequirements.mongodb), + enableS3: Boolean(config.platformRequirements.s3), + enableClickHouse: Boolean(config.platformRequirements.clickhouse), + enableRedis: Boolean(config.platformRequirements.redis), + enableMariaDB: Boolean(config.platformRequirements.mariadb), + appTemplateId: 'cloudly', + appTemplateVersion: version, + }); + deployedService = true; + + if (service.appTemplateId !== 'cloudly' || service.appTemplateVersion !== version) { + throw new Error(`Cloudly template tracking missing on service: ${JSON.stringify(service)}`); + } + + await waitForDockerContainerRunning('onebox-mongodb'); + await waitForDockerContainerRunning('onebox-minio'); + await waitForDockerServiceRunning(dockerServiceName); + await waitForCloudlyWorkerBootstrapRoute(); + + console.log(`[${scenarioName}] Scenario passed`); + } finally { + if (onebox) { + if (deployedService) { + await onebox.services.removeService(serviceName).catch((error) => { + console.log(`[${scenarioName}] Service cleanup failed: ${(error as Error).message}`); + }); + } + await onebox.platformServices.stopPlatformService('mongodb').catch(() => null); + await onebox.platformServices.stopPlatformService('minio').catch(() => null); + await onebox.shutdown(); + } + + await removeDockerService(dockerServiceName); + await removeDockerService('onebox-mongodb'); + await removeDockerService('onebox-minio'); + await removeDockerService('onebox-smartproxy'); + } +}; + +await main();