feat: add cloudly appstore worker scenario

package.json

@@ -7,10 +7,13 @@
   "scripts": {
     "bootstrap:components": "pnpm --dir ../interfaces install && pnpm --dir ../api install && pnpm --dir ../cloudly install && pnpm --dir ../coreflow install && pnpm --dir ../coretraffic install && pnpm --dir ../onebox exec deno install --config deno.json && pnpm install",
     "test": "pnpm scenario:registry-deploy-on-push && pnpm scenario:onebox-basic-lifecycle",
+    "test:full": "pnpm test && pnpm scenario:onebox-cloudly-appstore-worker",
     "scenario:registry-deploy-on-push": "tsx --tsconfig ../cloudly/tsconfig.json scenarios/registry-deploy-on-push/scenario.ts",
     "scenario:onebox-basic-lifecycle": "deno run --allow-all --config ../onebox/deno.json scenarios/onebox-basic-lifecycle/scenario.ts",
+    "scenario:onebox-cloudly-appstore-worker": "deno run --allow-all --config ../onebox/deno.json scenarios/onebox-cloudly-appstore-worker/scenario.ts",
     "vagrant:up": "vagrant up",
     "vagrant:test": "vagrant ssh -c 'cd /serve.zone/testing && pnpm bootstrap:components && pnpm test'",
+    "vagrant:test:full": "vagrant ssh -c 'cd /serve.zone/testing && pnpm bootstrap:components && pnpm test:full'",
     "vagrant:destroy": "vagrant destroy -f"
   },
   "devDependencies": {

scenarios/onebox-cloudly-appstore-worker/readme.md

@@ -0,0 +1,20 @@
+# onebox-cloudly-appstore-worker
+
+This full scenario verifies the Onebox-to-Cloudly app-store path and the worker bootstrap endpoint exposed by the installed Cloudly service.
+
+Assertions:
+
+- Onebox starts from an isolated bare working directory in dev mode.
+- The remote app-store catalog exposes the Cloudly template.
+- Installing Cloudly from the template provisions MongoDB and MinIO platform resources.
+- Cloudly starts as a Onebox-managed Docker service with the template version tracked on the service record.
+- Onebox ingress routes HTTP traffic to the installed Cloudly service.
+- Cloudly exposes `/curlfresh/setup.sh`, which is the current worker-node bootstrap script used by freshly provisioned nodes.
+
+Run with:
+
+```bash
+pnpm scenario:onebox-cloudly-appstore-worker
+```
+
+This scenario is intentionally excluded from the default `pnpm test` because it pulls and starts the Cloudly image plus platform services.

scenarios/onebox-cloudly-appstore-worker/scenario.ts

@@ -0,0 +1,293 @@
+import { dirname, fromFileUrl, join, resolve } from '@std/path';
+
+import { Onebox } from '../../../onebox/ts/classes/onebox.ts';
+import type { IAppVersionConfig } from '../../../onebox/ts/classes/appstore-types.ts';
+
+const scenarioName = 'onebox-cloudly-appstore-worker';
+const smokeId = `cloudly-appstore-${Date.now().toString(36)}`;
+const testingDir = resolve(dirname(fromFileUrl(import.meta.url)), '../..');
+const buildDir = join(testingDir, '.nogit', scenarioName, smokeId);
+const serviceName = `cloudly-${Date.now().toString(36)}`;
+const dockerServiceName = `onebox-${serviceName}`;
+const routeDomain = `${serviceName}.test`;
+
+const delayFor = async (millisecondsArg: number) => {
+  await new Promise((resolveArg) => setTimeout(resolveArg, millisecondsArg));
+};
+
+const run = async (commandArg: string, argsArg: string[]) => {
+  const command = new Deno.Command(commandArg, {
+    args: argsArg,
+    stdout: 'piped',
+    stderr: 'piped',
+  });
+  const output = await command.output();
+  const stdout = new TextDecoder().decode(output.stdout).trim();
+  const stderr = new TextDecoder().decode(output.stderr).trim();
+  if (stdout) {
+    console.log(stdout);
+  }
+  if (stderr) {
+    console.log(stderr);
+  }
+  if (!output.success) {
+    throw new Error(`${commandArg} ${argsArg.join(' ')} exited with ${output.code}`);
+  }
+  return { stdout, stderr };
+};
+
+const waitFor = async (
+  checkFunctionArg: () => boolean | Promise<boolean>,
+  messageArg: string,
+  timeoutMsArg = 180000,
+) => {
+  const startTime = Date.now();
+  while (Date.now() - startTime < timeoutMsArg) {
+    if (await checkFunctionArg()) {
+      return;
+    }
+    await delayFor(1000);
+  }
+  throw new Error(`Timed out waiting for ${messageArg}`);
+};
+
+const dockerServiceExists = async (serviceNameArg: string) => {
+  const command = new Deno.Command('docker', {
+    args: ['service', 'inspect', serviceNameArg],
+    stdout: 'null',
+    stderr: 'null',
+  });
+  return (await command.output()).success;
+};
+
+const removeDockerService = async (serviceNameArg: string) => {
+  if (await dockerServiceExists(serviceNameArg)) {
+    await run('docker', ['service', 'rm', serviceNameArg]).catch(() => null);
+    await delayFor(2000);
+  }
+};
+
+const dockerContainerRunning = async (containerNameArg: string) => {
+  const command = new Deno.Command('docker', {
+    args: ['container', 'inspect', containerNameArg, '--format', '{{.State.Running}}'],
+    stdout: 'piped',
+    stderr: 'null',
+  });
+  const output = await command.output();
+  if (!output.success) {
+    return false;
+  }
+  return new TextDecoder().decode(output.stdout).trim() === 'true';
+};
+
+const assertNoPreexistingScenarioServices = async () => {
+  for (const serviceNameArg of [dockerServiceName, 'onebox-smartproxy']) {
+    if (await dockerServiceExists(serviceNameArg)) {
+      throw new Error(`${serviceNameArg} already exists; refusing to overwrite a running service`);
+    }
+  }
+  for (const containerNameArg of ['onebox-mongodb', 'onebox-minio']) {
+    if (await dockerContainerRunning(containerNameArg)) {
+      throw new Error(`${containerNameArg} is already running; refusing to overwrite a platform container`);
+    }
+  }
+};
+
+const waitForDockerServiceRunning = async (serviceNameArg: string) => {
+  try {
+    await waitFor(async () => {
+      const command = new Deno.Command('docker', {
+        args: ['service', 'ps', serviceNameArg, '--format', '{{.CurrentState}}'],
+        stdout: 'piped',
+        stderr: 'null',
+      });
+      const output = await command.output();
+      if (!output.success) {
+        return false;
+      }
+      return new TextDecoder().decode(output.stdout).includes('Running');
+    }, `${serviceNameArg} running task`);
+  } catch (error) {
+    await run('docker', ['service', 'ps', '--no-trunc', serviceNameArg]).catch(() => null);
+    await run('docker', ['service', 'logs', '--raw', '--tail', '120', serviceNameArg]).catch(() => null);
+    throw error;
+  }
+};
+
+const waitForDockerContainerRunning = async (containerNameArg: string) => {
+  await waitFor(async () => dockerContainerRunning(containerNameArg), `${containerNameArg} running container`);
+};
+
+const ensureDockerReady = async () => {
+  await run('docker', ['version']);
+  const { stdout } = await run('docker', ['info', '--format', '{{.Swarm.LocalNodeState}}']);
+  if (stdout.trim() !== 'active') {
+    throw new Error('Docker Swarm must be active. In Vagrant this is handled by scripts/provision-vm.sh.');
+  }
+};
+
+const getTemplateEnvVars = (
+  configArg: IAppVersionConfig,
+  overridesArg: Record<string, string>,
+) => {
+  const envVars: Record<string, string> = {};
+  const missingRequiredEnvVars: string[] = [];
+
+  for (const envVar of configArg.envVars || []) {
+    const value = overridesArg[envVar.key] ?? envVar.value ?? '';
+    if (envVar.required && !value) {
+      missingRequiredEnvVars.push(envVar.key);
+    }
+    envVars[envVar.key] = value;
+  }
+
+  for (const [key, value] of Object.entries(overridesArg)) {
+    envVars[key] = value;
+  }
+
+  if (missingRequiredEnvVars.length > 0) {
+    throw new Error(`Missing required template env vars: ${missingRequiredEnvVars.join(', ')}`);
+  }
+
+  return envVars;
+};
+
+const requestCloudlyBootstrapScript = async () => {
+  const curlArgs = [
+    '-sS',
+    '--noproxy',
+    '*',
+    '--max-time',
+    '10',
+    '--resolve',
+    `${routeDomain}:8080:127.0.0.1`,
+    '-o',
+    '-',
+    '-w',
+    '\n%{http_code}',
+    `http://${routeDomain}:8080/curlfresh/setup.sh`,
+  ];
+
+  const command = new Deno.Command('curl', {
+    args: curlArgs,
+    stdout: 'piped',
+    stderr: 'piped',
+  });
+  const output = await command.output();
+  const stdout = new TextDecoder().decode(output.stdout).trim();
+  const stderr = new TextDecoder().decode(output.stderr).trim();
+  if (!output.success) {
+    throw new Error(`curl failed: ${stderr || stdout}`);
+  }
+  const lines = stdout.split('\n');
+  const statusCode = lines[lines.length - 1];
+  const body = lines.slice(0, -1).join('\n');
+  return { statusCode, body, stderr };
+};
+
+const waitForCloudlyWorkerBootstrapRoute = async () => {
+  let lastResponse: Awaited<ReturnType<typeof requestCloudlyBootstrapScript>> | undefined;
+  let lastError: Error | undefined;
+  try {
+    await waitFor(async () => {
+      try {
+        lastResponse = await requestCloudlyBootstrapScript();
+        lastError = undefined;
+        return (
+          lastResponse.statusCode === '200' &&
+          lastResponse.body.includes('pnpm install -g @serve.zone/spark') &&
+          lastResponse.body.includes('spark installdaemon')
+        );
+      } catch (error) {
+        lastError = error as Error;
+        return false;
+      }
+    }, 'Cloudly worker bootstrap route through Onebox ingress');
+  } catch (error) {
+    console.log(`[${scenarioName}] Last bootstrap response: ${JSON.stringify(lastResponse)}`);
+    if (lastError) {
+      console.log(`[${scenarioName}] Last bootstrap error: ${lastError.message}`);
+    }
+    await run('docker', ['service', 'ps', 'onebox-smartproxy']).catch(() => null);
+    await run('docker', ['service', 'logs', '--raw', '--tail', '120', 'onebox-smartproxy']).catch(() => null);
+    await run('docker', ['service', 'ps', dockerServiceName]).catch(() => null);
+    await run('docker', ['service', 'logs', '--raw', '--tail', '120', dockerServiceName]).catch(() => null);
+    throw error;
+  }
+};
+
+const main = async () => {
+  let onebox: Onebox | undefined;
+  let deployedService = false;
+
+  try {
+    await ensureDockerReady();
+    await assertNoPreexistingScenarioServices();
+    await Deno.mkdir(buildDir, { recursive: true });
+    Deno.chdir(buildDir);
+    Deno.env.set('ONEBOX_DEV', 'true');
+
+    console.log(`[${scenarioName}] Starting Onebox from ${buildDir}`);
+    onebox = new Onebox();
+    await onebox.init();
+    await waitForDockerServiceRunning('onebox-smartproxy');
+
+    const appMeta = await onebox.appStore.getAppMeta('cloudly');
+    const version = appMeta.latestVersion;
+    const config = await onebox.appStore.getAppVersionConfig('cloudly', version);
+    if (!config.platformRequirements?.mongodb || !config.platformRequirements?.s3) {
+      throw new Error(`Cloudly template must require MongoDB and S3: ${JSON.stringify(config.platformRequirements)}`);
+    }
+
+    console.log(`[${scenarioName}] Installing Cloudly template ${version} as ${serviceName}`);
+    const envVars = getTemplateEnvVars(config, {
+      SERVEZONE_ADMINACCOUNT: 'testadmin:testpassword',
+    });
+
+    const service = await onebox.services.deployService({
+      name: serviceName,
+      image: config.image,
+      port: config.port,
+      domain: routeDomain,
+      autoDNS: false,
+      envVars,
+      enableMongoDB: Boolean(config.platformRequirements.mongodb),
+      enableS3: Boolean(config.platformRequirements.s3),
+      enableClickHouse: Boolean(config.platformRequirements.clickhouse),
+      enableRedis: Boolean(config.platformRequirements.redis),
+      enableMariaDB: Boolean(config.platformRequirements.mariadb),
+      appTemplateId: 'cloudly',
+      appTemplateVersion: version,
+    });
+    deployedService = true;
+
+    if (service.appTemplateId !== 'cloudly' || service.appTemplateVersion !== version) {
+      throw new Error(`Cloudly template tracking missing on service: ${JSON.stringify(service)}`);
+    }
+
+    await waitForDockerContainerRunning('onebox-mongodb');
+    await waitForDockerContainerRunning('onebox-minio');
+    await waitForDockerServiceRunning(dockerServiceName);
+    await waitForCloudlyWorkerBootstrapRoute();
+
+    console.log(`[${scenarioName}] Scenario passed`);
+  } finally {
+    if (onebox) {
+      if (deployedService) {
+        await onebox.services.removeService(serviceName).catch((error) => {
+          console.log(`[${scenarioName}] Service cleanup failed: ${(error as Error).message}`);
+        });
+      }
+      await onebox.platformServices.stopPlatformService('mongodb').catch(() => null);
+      await onebox.platformServices.stopPlatformService('minio').catch(() => null);
+      await onebox.shutdown();
+    }
+
+    await removeDockerService(dockerServiceName);
+    await removeDockerService('onebox-mongodb');
+    await removeDockerService('onebox-minio');
+    await removeDockerService('onebox-smartproxy');
+  }
+};
+
+await main();