test: expand onebox integration coverage

readme.md

@@ -14,12 +14,12 @@ Component repositories keep their fast unit and package tests locally. This repo
 
 | Scenario | Command | What it validates |
 | --- | --- | --- |
-| `codebase-regressions` | `pnpm scenario:codebase-regressions` | Static cross-package regression checks for security defaults, registry/image stream fixes, package metadata, and ContainerArchive prune safety. |
+| `codebase-regressions` | `pnpm scenario:codebase-regressions` | Static cross-package regression checks for security defaults, registry/image stream fixes, app catalog schema/templates, deployment runtime hooks, package metadata, and ContainerArchive prune safety. |
 | `corestore-volume-driver` | `pnpm scenario:corestore-volume-driver` | Corestore control API, Docker VolumeDriver protocol, snapshot/restore through `containerarchive`, Coreflow volume mount generation, backup/restore orchestration, and remote replication against a fake Cloudly API. |
 | `registry-deploy-on-push` | `pnpm scenario:registry-deploy-on-push` | Cloudly startup with isolated Mongo/S3 helpers, cluster and registry setup, Docker image push, Cloudly metadata update, Coreflow workload provisioning, Coretraffic HTTPS routing, and same-tag redeploy after digest changes. |
-| `onebox-basic-lifecycle` | `pnpm scenario:onebox-basic-lifecycle` | Onebox startup in dev mode, SmartProxy ingress, local registry status, workload deploy/remove, HTTP route check, and HTTPS route check with a temporary certificate. |
-| `onebox-backup-restore` | `pnpm scenario:onebox-backup-restore` | Onebox service backup creation through ContainerArchive, image-in-backup metadata, clone restore, environment preservation, and cleanup. |
-| `onebox-cloudly-appstore-worker` | `pnpm scenario:onebox-cloudly-appstore-worker` | Cloudly App Store template installation through Onebox, MongoDB and MinIO platform resources, worker bootstrap route availability, backup without app image, and platform-resource restore. |
+| `onebox-basic-lifecycle` | `pnpm scenario:onebox-basic-lifecycle` | Onebox startup in dev mode, SmartProxy ingress, local registry status, workload deploy/remove, persisted service volumes, raw published ports, HTTP route check, and HTTPS route check with a temporary certificate. |
+| `onebox-backup-restore` | `pnpm scenario:onebox-backup-restore` | Onebox service backup creation through ContainerArchive, image-in-backup metadata, clone restore, environment and volume preservation, and cleanup. |
+| `onebox-cloudly-appstore-worker` | `pnpm scenario:onebox-cloudly-appstore-worker` | Cloudly App Store template installation through Onebox's AppStore manager using the local app catalog, MongoDB and MinIO platform resources, env template resolution, worker bootstrap route availability, backup without app image, and platform-resource restore. |
 | `baseos-image-pipeline` | `pnpm scenario:baseos-image-pipeline` | CoreBuild BaseOS raw-image job API, S3 artifact upload, Raspberry Pi image customization, WiFi config, SSH key injection, and baserunner environment generation. |
 
 `pnpm test` runs the core host scenarios. `pnpm test:full` adds the Cloudly App Store worker scenario. The BaseOS image pipeline is intentionally separate because it needs image tooling such as `qemu-img`, `guestfish`, and `xz`.

scenarios/codebase-regressions/scenario.ts

@@ -1,5 +1,5 @@
 import assert from 'assert/strict';
-import { readFileSync } from 'fs';
+import { existsSync, readFileSync, readdirSync } from 'fs';
 import { dirname, resolve } from 'path';
 import { fileURLToPath } from 'url';
 
@@ -9,6 +9,10 @@ const readWorkspaceFile = (pathFromRoot: string) => {
   return readFileSync(resolve(root, pathFromRoot), 'utf8');
 };
 
+const readWorkspaceJson = <T>(pathFromRoot: string) => {
+  return JSON.parse(readWorkspaceFile(pathFromRoot)) as T;
+};
+
 const assertIncludes = (filePath: string, expected: string) => {
   const content = readWorkspaceFile(filePath);
   assert.ok(content.includes(expected), `${filePath} should include ${expected}`);
@@ -19,6 +23,86 @@ const assertExcludes = (filePath: string, forbidden: string) => {
   assert.ok(!content.includes(forbidden), `${filePath} should not include ${forbidden}`);
 };
 
+const assertPort = (valueArg: unknown, labelArg: string) => {
+  assert.ok(Number.isInteger(valueArg), `${labelArg} should be an integer`);
+  assert.ok((valueArg as number) >= 1 && (valueArg as number) <= 65535, `${labelArg} should be a valid TCP/UDP port`);
+};
+
+const assertValidPublishedPort = (portArg: any, labelArg: string) => {
+  assertPort(portArg.targetPort, `${labelArg}.targetPort`);
+  const targetEnd = portArg.targetPortEnd ?? portArg.targetPort;
+  assertPort(targetEnd, `${labelArg}.targetPortEnd`);
+  assert.ok(targetEnd >= portArg.targetPort, `${labelArg} target range should be ascending`);
+
+  const publishedStart = portArg.publishedPort ?? portArg.targetPort;
+  const publishedEnd = portArg.publishedPortEnd ?? (publishedStart + (targetEnd - portArg.targetPort));
+  assertPort(publishedStart, `${labelArg}.publishedPort`);
+  assertPort(publishedEnd, `${labelArg}.publishedPortEnd`);
+  assert.ok(publishedEnd >= publishedStart, `${labelArg} published range should be ascending`);
+  assert.equal(targetEnd - portArg.targetPort, publishedEnd - publishedStart, `${labelArg} target and published ranges should have the same size`);
+  assert.ok(['tcp', 'udp'].includes(portArg.protocol ?? 'tcp'), `${labelArg}.protocol should be tcp or udp`);
+};
+
+const assertValidAppCatalog = () => {
+  const catalog = readWorkspaceJson<{
+    schemaVersion: number;
+    apps: Array<{ id: string; latestVersion: string }>;
+  }>('appstore-apptemplates/catalog.json');
+  assert.equal(catalog.schemaVersion, 1, 'App catalog schema version should be 1');
+
+  const appIds = catalog.apps.map((appArg) => appArg.id);
+  assert.equal(new Set(appIds).size, appIds.length, 'App catalog ids should be unique');
+
+  const appDirs = readdirSync(resolve(root, 'appstore-apptemplates/apps'), { withFileTypes: true })
+    .filter((direntArg) => direntArg.isDirectory())
+    .map((direntArg) => direntArg.name)
+    .sort();
+  assert.deepEqual([...appIds].sort(), appDirs, 'Every app directory should be represented in catalog.json');
+
+  for (const catalogApp of catalog.apps) {
+    const appPath = `appstore-apptemplates/apps/${catalogApp.id}`;
+    const metaPath = `${appPath}/app.json`;
+    assert.ok(existsSync(resolve(root, metaPath)), `${metaPath} should exist`);
+    const appMeta = readWorkspaceJson<{
+      id: string;
+      latestVersion: string;
+      versions: string[];
+    }>(metaPath);
+    assert.equal(appMeta.id, catalogApp.id, `${metaPath} id should match its directory and catalog entry`);
+    assert.equal(appMeta.latestVersion, catalogApp.latestVersion, `${metaPath} latestVersion should match catalog.json`);
+    assert.ok(appMeta.versions.includes(appMeta.latestVersion), `${metaPath} versions should include latestVersion`);
+
+    const configPath = `${appPath}/versions/${appMeta.latestVersion}/config.json`;
+    assert.ok(existsSync(resolve(root, configPath)), `${configPath} should exist`);
+    const config = readWorkspaceJson<{
+      image: string;
+      port: number;
+      envVars?: Array<{ key: string; value?: string; description: string; required?: boolean }>;
+      volumes?: Array<string | { mountPath: string }>;
+      publishedPorts?: any[];
+    }>(configPath);
+    assert.ok(config.image, `${configPath} should define an image`);
+    assertPort(config.port, `${configPath}.port`);
+
+    const envKeys = new Set<string>();
+    for (const envVar of config.envVars || []) {
+      assert.ok(envVar.key, `${configPath} env var should define a key`);
+      assert.ok(!envKeys.has(envVar.key), `${configPath} env var ${envVar.key} should be unique`);
+      envKeys.add(envVar.key);
+      assert.equal(typeof envVar.description, 'string', `${configPath} env var ${envVar.key} should describe its purpose`);
+    }
+
+    for (const [index, volume] of (config.volumes || []).entries()) {
+      const mountPath = typeof volume === 'string' ? volume : volume.mountPath;
+      assert.ok(mountPath?.startsWith('/'), `${configPath} volume ${index} should define an absolute mountPath`);
+    }
+
+    for (const [index, publishedPort] of (config.publishedPorts || []).entries()) {
+      assertValidPublishedPort(publishedPort, `${configPath}.publishedPorts[${index}]`);
+    }
+  }
+};
+
 assertIncludes('containerarchive/rust/src/prune.rs', 'rewritten_offsets');
 assertExcludes('containerarchive/rust/src/prune.rs', 'offset: entry.offset, // Note: offset in the new pack may differ');
 assertIncludes('containerarchive/test/test.ts', 'partial-pack GC rewrites chunks');
@@ -35,17 +119,56 @@ assertIncludes('api/ts/classes.image.ts', 'versionString: imageVersion');
 assertIncludes('cloudly/ts/manager.image/classes.imagemanager.ts', 'readFromWebstream(readable)');
 assertIncludes('cloudly/ts/manager.service/classes.servicemanager.ts', 'adminIdentityGuard');
 assertIncludes('cloudly/ts/manager.cluster/classes.clustermanager.ts', 'getClusterById');
+assertIncludes('cloudly/ts/manager.appcatalog/classes.appcatalogmanager.ts', 'installAppCatalogApp');
+assertIncludes('cloudly/ts/manager.appcatalog/classes.appcatalogmanager.ts', 'externalImageRef');
+assertIncludes('cloudly/ts/manager.appcatalog/classes.appcatalogmanager.ts', 'SERVICE_DOMAIN');
+assertIncludes('cloudly/ts/manager.deployment/classes.deploymentmanager.ts', 'deploymentWorkspaceExec');
+assertIncludes('coreflow/ts/coreflow.classes.coreflow.ts', 'deploymentRuntimeManager');
+assertIncludes('coreflow/ts/coreflow.classes.deploymentruntime.ts', 'coreflowDeploymentWorkspaceExec');
+assertIncludes('coreflow/ts/coreflow.classes.clustermanager.ts', 'externalImageRef');
+assertIncludes('coreflow/ts/coreflow.classes.clustermanager.ts', 'resolveEnvTemplates');
+assertIncludes('corestore/ts/corestore.classes.corestore.ts', "'/archive/manifest'");
+assertIncludes('corestore/ts/corestore.classes.corestore.ts', "'/archive/object/read'");
+assertIncludes('corestore/ts/corestore.classes.corestore.ts', "'/archive/object/write'");
 
 assertIncludes('platformclient/ts/email/classes.emailconnector.ts', 'return response.responseId');
 assertIncludes('platformclient/ts/email/classes.letterconnector.ts', 'return response.processId');
 assertIncludes('platformclient/ts/email/classes.pushnotificationconnector.ts', "typeof process !== 'undefined'");
 
 assertIncludes('siprouter/ts/sipproxy.ts', 'return { id: callId };');
+assertIncludes('siprouter/ts/storage.ts', 'MONGODB_URI');
+assertIncludes('siprouter/ts/storage.ts', 'S3_ENDPOINT');
+assertIncludes('siprouter/Dockerfile', 'libjpeg-dev');
+assertIncludes('siprouter/Dockerfile', 'libtiff-dev');
+assertIncludes('siprouter/Dockerfile', 'libjpeg-turbo');
+assertIncludes('siprouter/.smartconfig.json', 'linux/amd64');
+assertIncludes('siprouter/.smartconfig.json', 'linux/arm64');
 assertIncludes('onebox/ts/opsserver/classes.opsserver.ts', "'/v2/*'");
+assertIncludes('onebox/ts/classes/appstore.ts', 'installApp(optionsArg: IAppInstallOptions)');
+assertIncludes('onebox/ts/classes/appstore.ts', 'publishedPorts: optionsArg.publishedPorts || config.publishedPorts');
+assertIncludes('onebox/ts/classes/docker.ts', 'expandPublishedPorts');
+assertIncludes('onebox/ts/classes/docker.ts', 'Mounts: this.getSwarmVolumeMounts(service)');
+assertIncludes('onebox/ts/database/migrations/migration-016-service-volumes.ts', 'volumes TEXT');
+assertIncludes('onebox/ts/database/migrations/migration-017-service-published-ports.ts', 'published_ports TEXT');
+assertIncludes('onebox/ts_interfaces/requests/appstore.ts', "method: 'installAppTemplate'");
+assertIncludes('onebox/ts_interfaces/requests/appstore.ts', 'publishedPorts?: data.IServicePublishedPort[]');
+assertIncludes('onebox/ts/classes/managed-dcrouter.ts', 'DCROUTER_ADMIN_API_TOKEN');
+assertIncludes('api/ts/classes.cloudlyapiclient.ts', 'createNodeJumpCommand');
+assertIncludes('interfaces/ts/appcatalog/types.ts', 'publishedPorts?: IAppCatalogPublishedPort[]');
 assertIncludes('spark/ts/spark.cli.ts', 'CLOUDLY_URL: cloudlyUrl');
 
+assertValidAppCatalog();
+
 const rustdeskConfig = JSON.parse(readWorkspaceFile('appstore-apptemplates/apps/rustdesk-server/versions/1.0.0/config.json'));
 const relay = rustdeskConfig.envVars.find((entry: { key: string }) => entry.key === 'RELAY');
 assert.equal(relay?.value, '${SERVICE_DOMAIN}:21116');
 
+const siprouterConfig = readWorkspaceJson<any>('appstore-apptemplates/apps/siprouter/versions/1.0.0/config.json');
+assert.equal(siprouterConfig.platformRequirements?.mongodb, true);
+assert.equal(siprouterConfig.platformRequirements?.s3, true);
+assert.ok(siprouterConfig.publishedPorts.some((portArg: any) => portArg.targetPort === 20000 && portArg.targetPortEnd === 20200 && portArg.protocol === 'udp'));
+
+const gitopsConfig = readWorkspaceJson<any>('appstore-apptemplates/apps/gitops/versions/1.0.0/config.json');
+assert.ok(gitopsConfig.volumes.includes('/data/.serve.zone/gitops'));
+
 console.log('codebase regression checks passed');

scenarios/onebox-backup-restore/scenario.ts

@@ -1,6 +1,7 @@
 import { dirname, fromFileUrl, join, resolve } from '@std/path';
 
 import { Onebox } from '../../../onebox/ts/classes/onebox.ts';
+import { disableManagedDcRouterForScenario } from '../onebox-test-helpers.ts';
 
 const scenarioName = 'onebox-backup-restore';
 const smokeId = `onebox-backup-${Date.now().toString(36)}`;
@@ -10,6 +11,8 @@ const serviceName = `app-${Date.now().toString(36)}`;
 const cloneServiceName = `${serviceName}-clone`;
 const dockerServiceName = `onebox-${serviceName}`;
 const cloneDockerServiceName = `onebox-${cloneServiceName}`;
+const volumeName = `onebox-${serviceName}-data`;
+const cloneVolumeName = `onebox-${cloneServiceName}-data`;
 
 const delayFor = async (millisecondsArg: number) => {
   await new Promise((resolveArg) => setTimeout(resolveArg, millisecondsArg));
@@ -85,9 +88,35 @@ const removeDockerService = async (serviceNameArg: string) => {
   }
 };
 
+const removeDockerVolume = async (volumeNameArg: string) => {
+  let lastErrorText = '';
+  const startTime = Date.now();
+  while (Date.now() - startTime < 30000) {
+    const output = await outputCommand(
+      'docker',
+      ['volume', 'rm', '-f', volumeNameArg],
+      {
+        stdout: 'piped',
+        stderr: 'piped',
+      },
+      15000,
+    );
+    if (output.success) {
+      return;
+    }
+    lastErrorText = new TextDecoder().decode(output.stderr).trim();
+    await delayFor(1000);
+  }
+  console.log(
+    `[${scenarioName}] Failed to remove Docker volume ${volumeNameArg}: ${lastErrorText}`,
+  );
+};
+
 const assertNoPreexistingOneboxIngress = async () => {
   if (await dockerServiceExists('onebox-smartproxy')) {
-    throw new Error('onebox-smartproxy already exists; refusing to overwrite a running Onebox ingress service');
+    throw new Error(
+      'onebox-smartproxy already exists; refusing to overwrite a running Onebox ingress service',
+    );
   }
 };
 
@@ -111,14 +140,50 @@ const waitForDockerServiceRunning = async (serviceNameArg: string) => {
 };
 
 const waitForDockerServiceRemoved = async (serviceNameArg: string) => {
-  await waitFor(async () => !(await dockerServiceExists(serviceNameArg)), `${serviceNameArg} removal`);
+  await waitFor(
+    async () => !(await dockerServiceExists(serviceNameArg)),
+    `${serviceNameArg} removal`,
+  );
+};
+
+const inspectDockerServiceJson = async <T>(serviceNameArg: string, formatArg: string) => {
+  const output = await outputCommand('docker', [
+    'service',
+    'inspect',
+    serviceNameArg,
+    '--format',
+    formatArg,
+  ], {
+    stdout: 'piped',
+    stderr: 'piped',
+  }, 15000);
+  if (!output.success) {
+    throw new Error(`docker service inspect ${serviceNameArg} exited with ${output.code}`);
+  }
+  return JSON.parse(new TextDecoder().decode(output.stdout).trim()) as T;
+};
+
+const assertCloneVolumeMount = async () => {
+  const mounts = await inspectDockerServiceJson<Array<Record<string, unknown>>>(
+    cloneDockerServiceName,
+    '{{json .Spec.TaskTemplate.ContainerSpec.Mounts}}',
+  );
+  if (
+    !mounts.some((mountArg) => mountArg.Source === cloneVolumeName && mountArg.Target === '/data')
+  ) {
+    throw new Error(
+      `Cloned service volume mount missing from Docker service: ${JSON.stringify(mounts)}`,
+    );
+  }
 };
 
 const ensureDockerReady = async () => {
   await run('docker', ['version']);
   const { stdout } = await run('docker', ['info', '--format', '{{.Swarm.LocalNodeState}}']);
   if (stdout.trim() !== 'active') {
-    throw new Error('Docker Swarm must be active. In Vagrant this is handled by scripts/provision-vm.sh.');
+    throw new Error(
+      'Docker Swarm must be active. In Vagrant this is handled by scripts/provision-vm.sh.',
+    );
   }
 };
 
@@ -136,6 +201,7 @@ const main = async () => {
 
     console.log(`[${scenarioName}] Starting Onebox from ${buildDir}`);
     onebox = new Onebox();
+    disableManagedDcRouterForScenario(onebox);
     await onebox.init();
 
     await waitForDockerServiceRunning('onebox-smartproxy');
@@ -149,6 +215,7 @@ const main = async () => {
       envVars: {
         ONEBOX_BACKUP_SCENARIO: smokeId,
       },
+      volumes: [{ mountPath: '/data', backup: true }],
     });
     deployedService = true;
 
@@ -162,17 +229,25 @@ const main = async () => {
     const backupResult = await onebox.backupManager.createBackup(serviceName);
     const backupId = backupResult.backup.id;
     if (!backupId || !backupResult.snapshotId || !backupResult.backup.snapshotId) {
-      throw new Error(`Backup did not produce a ContainerArchive snapshot: ${JSON.stringify(backupResult)}`);
+      throw new Error(
+        `Backup did not produce a ContainerArchive snapshot: ${JSON.stringify(backupResult)}`,
+      );
     }
     if (!backupResult.backup.includesImage) {
-      throw new Error(`Backup did not include Docker image: ${JSON.stringify(backupResult.backup)}`);
+      throw new Error(
+        `Backup did not include Docker image: ${JSON.stringify(backupResult.backup)}`,
+      );
     }
     if ((backupResult.backup.storedSizeBytes ?? 0) <= 0 || backupResult.backup.sizeBytes <= 0) {
       throw new Error(`Backup size metadata is invalid: ${JSON.stringify(backupResult.backup)}`);
     }
 
     const backups = onebox.backupManager.listBackups(serviceName);
-    if (!backups.some((backupArg) => backupArg.id === backupId && backupArg.snapshotId === backupResult.snapshotId)) {
+    if (
+      !backups.some((backupArg) =>
+        backupArg.id === backupId && backupArg.snapshotId === backupResult.snapshotId
+      )
+    ) {
       throw new Error(`Backup not listed for service: ${JSON.stringify(backups)}`);
     }
 
@@ -183,20 +258,34 @@ const main = async () => {
     });
     clonedService = true;
 
-    if (restoreResult.service.name !== cloneServiceName || restoreResult.service.status !== 'running') {
-      throw new Error(`Unexpected restored service state: ${JSON.stringify(restoreResult.service)}`);
+    if (
+      restoreResult.service.name !== cloneServiceName || restoreResult.service.status !== 'running'
+    ) {
+      throw new Error(
+        `Unexpected restored service state: ${JSON.stringify(restoreResult.service)}`,
+      );
     }
     if (restoreResult.service.domain !== undefined) {
-      throw new Error(`Clone unexpectedly retained a domain: ${JSON.stringify(restoreResult.service)}`);
+      throw new Error(
+        `Clone unexpectedly retained a domain: ${JSON.stringify(restoreResult.service)}`,
+      );
     }
     if (restoreResult.service.envVars.ONEBOX_BACKUP_SCENARIO !== smokeId) {
-      throw new Error(`Clone did not preserve env vars: ${JSON.stringify(restoreResult.service.envVars)}`);
+      throw new Error(
+        `Clone did not preserve env vars: ${JSON.stringify(restoreResult.service.envVars)}`,
+      );
+    }
+    if (!restoreResult.service.volumes?.some((volumeArg) => volumeArg.mountPath === '/data')) {
+      throw new Error(
+        `Clone did not preserve volumes: ${JSON.stringify(restoreResult.service.volumes)}`,
+      );
     }
     if (restoreResult.warnings.length > 0) {
       throw new Error(`Restore completed with warnings: ${restoreResult.warnings.join('; ')}`);
     }
 
     await waitForDockerServiceRunning(cloneDockerServiceName);
+    await assertCloneVolumeMount();
 
     console.log(`[${scenarioName}] Removing restored workload ${cloneServiceName}`);
     await onebox.services.removeService(cloneServiceName);
@@ -212,12 +301,16 @@ const main = async () => {
   } finally {
     if (onebox && clonedService) {
       await onebox.services.removeService(cloneServiceName).catch((error) => {
-        console.log(`[${scenarioName}] Failed to remove restored Onebox service: ${(error as Error).message}`);
+        console.log(
+          `[${scenarioName}] Failed to remove restored Onebox service: ${(error as Error).message}`,
+        );
       });
     }
     if (onebox && deployedService) {
       await onebox.services.removeService(serviceName).catch((error) => {
-        console.log(`[${scenarioName}] Failed to remove Onebox service: ${(error as Error).message}`);
+        console.log(
+          `[${scenarioName}] Failed to remove Onebox service: ${(error as Error).message}`,
+        );
       });
     }
     if (onebox) {
@@ -228,6 +321,8 @@ const main = async () => {
     await removeDockerService(cloneDockerServiceName);
     await removeDockerService(dockerServiceName);
     await removeDockerService('onebox-smartproxy');
+    await removeDockerVolume(cloneVolumeName);
+    await removeDockerVolume(volumeName);
   }
 };
 

scenarios/onebox-basic-lifecycle/scenario.ts

@@ -1,6 +1,7 @@
 import { dirname, fromFileUrl, join, resolve } from '@std/path';
 
 import { Onebox } from '../../../onebox/ts/classes/onebox.ts';
+import { disableManagedDcRouterForScenario } from '../onebox-test-helpers.ts';
 
 const scenarioName = 'onebox-basic-lifecycle';
 const smokeId = `onebox-basic-${Date.now().toString(36)}`;
@@ -9,6 +10,7 @@ const buildDir = join(testingDir, '.nogit', scenarioName, smokeId);
 const serviceName = `app-${Date.now().toString(36)}`;
 const dockerServiceName = `onebox-${serviceName}`;
 const routeDomain = `${serviceName}.test`;
+const volumeName = `onebox-${serviceName}-data`;
 
 const delayFor = async (millisecondsArg: number) => {
   await new Promise((resolveArg) => setTimeout(resolveArg, millisecondsArg));
@@ -69,6 +71,13 @@ const waitFor = async (checkFunctionArg: () => boolean | Promise<boolean>, messa
   throw new Error(`Timed out waiting for ${messageArg}`);
 };
 
+const getFreeTcpPort = () => {
+  const listener = Deno.listen({ hostname: '127.0.0.1', port: 0 });
+  const port = (listener.addr as Deno.NetAddr).port;
+  listener.close();
+  return port;
+};
+
 const dockerServiceExists = async (serviceNameArg: string) => {
   const output = await outputCommand('docker', ['service', 'inspect', serviceNameArg], {
     stdout: 'null',
@@ -84,9 +93,35 @@ const removeDockerService = async (serviceNameArg: string) => {
   }
 };
 
+const removeDockerVolume = async (volumeNameArg: string) => {
+  let lastErrorText = '';
+  const startTime = Date.now();
+  while (Date.now() - startTime < 30000) {
+    const output = await outputCommand(
+      'docker',
+      ['volume', 'rm', '-f', volumeNameArg],
+      {
+        stdout: 'piped',
+        stderr: 'piped',
+      },
+      15000,
+    );
+    if (output.success) {
+      return;
+    }
+    lastErrorText = new TextDecoder().decode(output.stderr).trim();
+    await delayFor(1000);
+  }
+  console.log(
+    `[${scenarioName}] Failed to remove Docker volume ${volumeNameArg}: ${lastErrorText}`,
+  );
+};
+
 const assertNoPreexistingOneboxIngress = async () => {
   if (await dockerServiceExists('onebox-smartproxy')) {
-    throw new Error('onebox-smartproxy already exists; refusing to overwrite a running Onebox ingress service');
+    throw new Error(
+      'onebox-smartproxy already exists; refusing to overwrite a running Onebox ingress service',
+    );
   }
 };
 
@@ -110,7 +145,27 @@ const waitForDockerServiceRunning = async (serviceNameArg: string) => {
 };
 
 const waitForDockerServiceRemoved = async (serviceNameArg: string) => {
-  await waitFor(async () => !(await dockerServiceExists(serviceNameArg)), `${serviceNameArg} removal`);
+  await waitFor(
+    async () => !(await dockerServiceExists(serviceNameArg)),
+    `${serviceNameArg} removal`,
+  );
+};
+
+const inspectDockerServiceJson = async <T>(serviceNameArg: string, formatArg: string) => {
+  const output = await outputCommand('docker', [
+    'service',
+    'inspect',
+    serviceNameArg,
+    '--format',
+    formatArg,
+  ], {
+    stdout: 'piped',
+    stderr: 'piped',
+  }, 15000);
+  if (!output.success) {
+    throw new Error(`docker service inspect ${serviceNameArg} exited with ${output.code}`);
+  }
+  return JSON.parse(new TextDecoder().decode(output.stdout).trim()) as T;
 };
 
 const requestRoute = async (protocolArg: 'http' | 'https', portArg: number) => {
@@ -168,9 +223,13 @@ const waitForRoute = async (protocolArg: 'http' | 'https', portArg: number) => {
       console.log(`[${scenarioName}] Last route error: ${lastError.message}`);
     }
     await run('docker', ['service', 'ps', 'onebox-smartproxy']).catch(() => null);
-    await run('docker', ['service', 'logs', '--raw', '--tail', '120', 'onebox-smartproxy']).catch(() => null);
+    await run('docker', ['service', 'logs', '--raw', '--tail', '120', 'onebox-smartproxy']).catch(
+      () => null,
+    );
     await run('docker', ['service', 'ps', dockerServiceName]).catch(() => null);
-    await run('docker', ['service', 'logs', '--raw', '--tail', '120', dockerServiceName]).catch(() => null);
+    await run('docker', ['service', 'logs', '--raw', '--tail', '120', dockerServiceName]).catch(
+      () => null,
+    );
     throw error;
   }
 };
@@ -205,13 +264,19 @@ const ensureDockerReady = async () => {
   await run('docker', ['version']);
   const { stdout } = await run('docker', ['info', '--format', '{{.Swarm.LocalNodeState}}']);
   if (stdout.trim() !== 'active') {
-    throw new Error('Docker Swarm must be active. In Vagrant this is handled by scripts/provision-vm.sh.');
+    throw new Error(
+      'Docker Swarm must be active. In Vagrant this is handled by scripts/provision-vm.sh.',
+    );
   }
 };
 
 const assertOneboxCoreReady = async (oneboxArg: Onebox) => {
   const status = await oneboxArg.getSystemStatus();
-  const reverseProxy = status.reverseProxy as { backend: string; http: { running: boolean }; https: { running: boolean } };
+  const reverseProxy = status.reverseProxy as {
+    backend: string;
+    http: { running: boolean };
+    https: { running: boolean };
+  };
   if (reverseProxy.backend !== 'smartproxy-docker') {
     throw new Error(`Unexpected reverse proxy backend: ${JSON.stringify(reverseProxy)}`);
   }
@@ -225,11 +290,59 @@ const assertOneboxCoreReady = async (oneboxArg: Onebox) => {
   }
 
   const platformServices = status.platformServices as Array<{ type: string; status: string }>;
-  if (!platformServices.some((serviceArg) => serviceArg.type === 'smartproxy' && serviceArg.status === 'running')) {
-    throw new Error(`SmartProxy platform service not reported as running: ${JSON.stringify(platformServices)}`);
+  if (
+    !platformServices.some((serviceArg) =>
+      serviceArg.type === 'smartproxy' && serviceArg.status === 'running'
+    )
+  ) {
+    throw new Error(
+      `SmartProxy platform service not reported as running: ${JSON.stringify(platformServices)}`,
+    );
   }
   if (platformServices.some((serviceArg) => serviceArg.type === 'caddy')) {
-    throw new Error(`Unexpected legacy caddy platform service: ${JSON.stringify(platformServices)}`);
+    throw new Error(
+      `Unexpected legacy caddy platform service: ${JSON.stringify(platformServices)}`,
+    );
+  }
+};
+
+const assertDockerStorageAndPorts = async (publishedPortArg: number) => {
+  const mounts = await inspectDockerServiceJson<Array<Record<string, unknown>>>(
+    dockerServiceName,
+    '{{json .Spec.TaskTemplate.ContainerSpec.Mounts}}',
+  );
+  if (!mounts.some((mountArg) => mountArg.Source === volumeName && mountArg.Target === '/data')) {
+    throw new Error(`Onebox volume mount missing from Docker service: ${JSON.stringify(mounts)}`);
+  }
+
+  const ports = await inspectDockerServiceJson<Array<Record<string, unknown>>>(
+    dockerServiceName,
+    '{{json .Spec.EndpointSpec.Ports}}',
+  );
+  if (
+    !ports.some((portArg) =>
+      portArg.TargetPort === 80 && portArg.PublishedPort === publishedPortArg &&
+      portArg.Protocol === 'tcp'
+    )
+  ) {
+    throw new Error(`Onebox published port missing from Docker service: ${JSON.stringify(ports)}`);
+  }
+};
+
+const assertServicePersistence = (oneboxArg: Onebox, publishedPortArg: number) => {
+  const service = oneboxArg.services.getService(serviceName);
+  if (!service) {
+    throw new Error(`Service was not persisted: ${serviceName}`);
+  }
+  if (!service.volumes?.some((volumeArg) => volumeArg.mountPath === '/data')) {
+    throw new Error(`Service volume was not persisted: ${JSON.stringify(service)}`);
+  }
+  if (
+    !service.publishedPorts?.some((portArg) =>
+      portArg.targetPort === 80 && portArg.publishedPort === publishedPortArg
+    )
+  ) {
+    throw new Error(`Service published port was not persisted: ${JSON.stringify(service)}`);
   }
 };
 
@@ -246,12 +359,14 @@ const main = async () => {
 
     console.log(`[${scenarioName}] Starting Onebox from ${buildDir}`);
     onebox = new Onebox();
+    disableManagedDcRouterForScenario(onebox);
     await onebox.init();
 
     await waitForDockerServiceRunning('onebox-smartproxy');
     await assertOneboxCoreReady(onebox);
 
     console.log(`[${scenarioName}] Deploying workload ${serviceName}`);
+    const publishedPort = getFreeTcpPort();
     const service = await onebox.services.deployService({
       name: serviceName,
       image: 'caddy:2-alpine',
@@ -259,6 +374,8 @@ const main = async () => {
       domain: routeDomain,
       autoDNS: false,
       envVars: {},
+      volumes: [{ mountPath: '/data', backup: true }],
+      publishedPorts: [{ targetPort: 80, publishedPort, protocol: 'tcp' }],
     });
     deployedService = true;
 
@@ -270,10 +387,16 @@ const main = async () => {
     }
 
     await waitForDockerServiceRunning(dockerServiceName);
+    assertServicePersistence(onebox, publishedPort);
+    await assertDockerStorageAndPorts(publishedPort);
     await waitForRoute('http', 8080);
 
     const certificate = await createSelfSignedCertificate();
-    await onebox.reverseProxy.addCertificate(routeDomain, certificate.publicKey, certificate.privateKey);
+    await onebox.reverseProxy.addCertificate(
+      routeDomain,
+      certificate.publicKey,
+      certificate.privateKey,
+    );
     await waitForRoute('https', 8443);
 
     console.log(`[${scenarioName}] Removing workload ${serviceName}`);
@@ -288,7 +411,9 @@ const main = async () => {
   } finally {
     if (onebox && deployedService) {
       await onebox.services.removeService(serviceName).catch((error) => {
-        console.log(`[${scenarioName}] Failed to remove Onebox service: ${(error as Error).message}`);
+        console.log(
+          `[${scenarioName}] Failed to remove Onebox service: ${(error as Error).message}`,
+        );
       });
     }
     if (onebox) {
@@ -298,6 +423,7 @@ const main = async () => {
     }
     await removeDockerService(dockerServiceName);
     await removeDockerService('onebox-smartproxy');
+    await removeDockerVolume(volumeName);
   }
 };
 

scenarios/onebox-cloudly-appstore-worker/scenario.ts

@@ -1,11 +1,16 @@
 import { dirname, fromFileUrl, join, resolve } from '@std/path';
 
 import { Onebox } from '../../../onebox/ts/classes/onebox.ts';
-import type { IAppVersionConfig } from '../../../onebox/ts/classes/appstore-types.ts';
+import {
+  disableManagedDcRouterForScenario,
+  useLocalAppStoreForScenario,
+} from '../onebox-test-helpers.ts';
 
 const scenarioName = 'onebox-cloudly-appstore-worker';
 const smokeId = `cloudly-appstore-${Date.now().toString(36)}`;
 const testingDir = resolve(dirname(fromFileUrl(import.meta.url)), '../..');
+const repoRoot = resolve(testingDir, '..');
+const appCatalogDir = join(repoRoot, 'appstore-apptemplates');
 const buildDir = join(testingDir, '.nogit', scenarioName, smokeId);
 const serviceName = `cloudly-${Date.now().toString(36)}`;
 const dockerServiceName = `onebox-${serviceName}`;
@@ -80,6 +85,22 @@ const dockerContainerRunning = async (containerNameArg: string) => {
   return new TextDecoder().decode(output.stdout).trim() === 'true';
 };
 
+const dockerContainerExists = async (containerNameArg: string) => {
+  const command = new Deno.Command('docker', {
+    args: ['container', 'inspect', containerNameArg],
+    stdout: 'null',
+    stderr: 'null',
+  });
+  return (await command.output()).success;
+};
+
+const removeDockerContainer = async (containerNameArg: string) => {
+  if (await dockerContainerExists(containerNameArg)) {
+    await run('docker', ['container', 'rm', '-f', containerNameArg]).catch(() => null);
+    await delayFor(1000);
+  }
+};
+
 const assertNoPreexistingScenarioServices = async () => {
   for (const serviceNameArg of [dockerServiceName, 'onebox-smartproxy']) {
     if (await dockerServiceExists(serviceNameArg)) {
@@ -87,8 +108,11 @@ const assertNoPreexistingScenarioServices = async () => {
     }
   }
   for (const containerNameArg of ['onebox-mongodb', 'onebox-minio']) {
-    if (await dockerContainerRunning(containerNameArg)) {
-      throw new Error(`${containerNameArg} is already running; refusing to overwrite a platform container`);
+    if (await dockerContainerExists(containerNameArg)) {
+      const state = await dockerContainerRunning(containerNameArg) ? 'running' : 'present';
+      throw new Error(
+        `${containerNameArg} is already ${state}; refusing to overwrite a platform container`,
+      );
     }
   }
 };
@@ -109,49 +133,30 @@ const waitForDockerServiceRunning = async (serviceNameArg: string) => {
     }, `${serviceNameArg} running task`);
   } catch (error) {
     await run('docker', ['service', 'ps', '--no-trunc', serviceNameArg]).catch(() => null);
-    await run('docker', ['service', 'logs', '--raw', '--tail', '120', serviceNameArg]).catch(() => null);
+    await run('docker', ['service', 'logs', '--raw', '--tail', '120', serviceNameArg]).catch(() =>
+      null
+    );
     throw error;
   }
 };
 
 const waitForDockerContainerRunning = async (containerNameArg: string) => {
-  await waitFor(async () => dockerContainerRunning(containerNameArg), `${containerNameArg} running container`);
+  await waitFor(
+    async () => dockerContainerRunning(containerNameArg),
+    `${containerNameArg} running container`,
+  );
 };
 
 const ensureDockerReady = async () => {
   await run('docker', ['version']);
   const { stdout } = await run('docker', ['info', '--format', '{{.Swarm.LocalNodeState}}']);
   if (stdout.trim() !== 'active') {
-    throw new Error('Docker Swarm must be active. In Vagrant this is handled by scripts/provision-vm.sh.');
+    throw new Error(
+      'Docker Swarm must be active. In Vagrant this is handled by scripts/provision-vm.sh.',
+    );
   }
 };
 
-const getTemplateEnvVars = (
-  configArg: IAppVersionConfig,
-  overridesArg: Record<string, string>,
-) => {
-  const envVars: Record<string, string> = {};
-  const missingRequiredEnvVars: string[] = [];
-
-  for (const envVar of configArg.envVars || []) {
-    const value = overridesArg[envVar.key] ?? envVar.value ?? '';
-    if (envVar.required && !value) {
-      missingRequiredEnvVars.push(envVar.key);
-    }
-    envVars[envVar.key] = value;
-  }
-
-  for (const [key, value] of Object.entries(overridesArg)) {
-    envVars[key] = value;
-  }
-
-  if (missingRequiredEnvVars.length > 0) {
-    throw new Error(`Missing required template env vars: ${missingRequiredEnvVars.join(', ')}`);
-  }
-
-  return envVars;
-};
-
 const requestCloudlyBootstrapScript = async () => {
   const curlArgs = [
     '-sS',
@@ -209,9 +214,13 @@ const waitForCloudlyWorkerBootstrapRoute = async () => {
       console.log(`[${scenarioName}] Last bootstrap error: ${lastError.message}`);
     }
     await run('docker', ['service', 'ps', 'onebox-smartproxy']).catch(() => null);
-    await run('docker', ['service', 'logs', '--raw', '--tail', '120', 'onebox-smartproxy']).catch(() => null);
+    await run('docker', ['service', 'logs', '--raw', '--tail', '120', 'onebox-smartproxy']).catch(
+      () => null,
+    );
     await run('docker', ['service', 'ps', dockerServiceName]).catch(() => null);
-    await run('docker', ['service', 'logs', '--raw', '--tail', '120', dockerServiceName]).catch(() => null);
+    await run('docker', ['service', 'logs', '--raw', '--tail', '120', dockerServiceName]).catch(
+      () => null,
+    );
     throw error;
   }
 };
@@ -229,6 +238,8 @@ const main = async () => {
 
     console.log(`[${scenarioName}] Starting Onebox from ${buildDir}`);
     onebox = new Onebox();
+    disableManagedDcRouterForScenario(onebox);
+    useLocalAppStoreForScenario(onebox, appCatalogDir);
     await onebox.init();
     await waitForDockerServiceRunning('onebox-smartproxy');
 
@@ -236,28 +247,23 @@ const main = async () => {
     const version = appMeta.latestVersion;
     const config = await onebox.appStore.getAppVersionConfig('cloudly', version);
     if (!config.platformRequirements?.mongodb || !config.platformRequirements?.s3) {
-      throw new Error(`Cloudly template must require MongoDB and S3: ${JSON.stringify(config.platformRequirements)}`);
+      throw new Error(
+        `Cloudly template must require MongoDB and S3: ${
+          JSON.stringify(config.platformRequirements)
+        }`,
+      );
     }
 
     console.log(`[${scenarioName}] Installing Cloudly template ${version} as ${serviceName}`);
-    const envVars = getTemplateEnvVars(config, {
-      SERVEZONE_ADMINACCOUNT: 'testadmin:testpassword',
-    });
-
-    const service = await onebox.services.deployService({
-      name: serviceName,
-      image: config.image,
-      port: config.port,
+    const service = await onebox.appStore.installApp({
+      appId: 'cloudly',
+      version,
+      serviceName,
       domain: routeDomain,
       autoDNS: false,
-      envVars,
-      enableMongoDB: Boolean(config.platformRequirements.mongodb),
-      enableS3: Boolean(config.platformRequirements.s3),
-      enableClickHouse: Boolean(config.platformRequirements.clickhouse),
-      enableRedis: Boolean(config.platformRequirements.redis),
-      enableMariaDB: Boolean(config.platformRequirements.mariadb),
-      appTemplateId: 'cloudly',
-      appTemplateVersion: version,
+      envVars: {
+        SERVEZONE_ADMINACCOUNT: 'testadmin:testpassword',
+      },
     });
     deployedService = true;
 
@@ -272,13 +278,38 @@ const main = async () => {
 
     const deployedServiceRecord = onebox.services.getService(serviceName);
     if (!deployedServiceRecord?.id) {
-      throw new Error(`Cloudly service missing after deploy: ${JSON.stringify(deployedServiceRecord)}`);
+      throw new Error(
+        `Cloudly service missing after deploy: ${JSON.stringify(deployedServiceRecord)}`,
+      );
+    }
+    if (deployedServiceRecord.envVars.SERVEZONE_URL !== routeDomain) {
+      throw new Error(
+        `Cloudly SERVICE_DOMAIN template was not resolved: ${
+          JSON.stringify(deployedServiceRecord.envVars)
+        }`,
+      );
+    }
+    if (
+      !deployedServiceRecord.envVars.MONGODB_URL ||
+      deployedServiceRecord.envVars.MONGODB_URL.includes('${')
+    ) {
+      throw new Error(
+        `Cloudly MongoDB template was not resolved: ${
+          JSON.stringify(deployedServiceRecord.envVars)
+        }`,
+      );
     }
 
-    const platformResources = await onebox.platformServices.getResourcesForService(deployedServiceRecord.id);
-    const platformResourceTypes = platformResources.map((resourceArg) => resourceArg.platformService.type).sort();
+    const platformResources = await onebox.platformServices.getResourcesForService(
+      deployedServiceRecord.id,
+    );
+    const platformResourceTypes = platformResources.map((resourceArg) =>
+      resourceArg.platformService.type
+    ).sort();
     if (platformResourceTypes.join(',') !== 'minio,mongodb') {
-      throw new Error(`Unexpected Cloudly platform resources: ${JSON.stringify(platformResourceTypes)}`);
+      throw new Error(
+        `Unexpected Cloudly platform resources: ${JSON.stringify(platformResourceTypes)}`,
+      );
     }
 
     console.log(`[${scenarioName}] Creating Cloudly platform-resource backup for ${serviceName}`);
@@ -286,13 +317,23 @@ const main = async () => {
     const backupResult = await onebox.backupManager.createBackup(serviceName);
     const backupId = backupResult.backup.id;
     if (!backupId || !backupResult.snapshotId || !backupResult.backup.snapshotId) {
-      throw new Error(`Cloudly backup did not produce a ContainerArchive snapshot: ${JSON.stringify(backupResult)}`);
+      throw new Error(
+        `Cloudly backup did not produce a ContainerArchive snapshot: ${
+          JSON.stringify(backupResult)
+        }`,
+      );
     }
     if (backupResult.backup.includesImage) {
-      throw new Error(`Cloudly backup unexpectedly included the large app image: ${JSON.stringify(backupResult.backup)}`);
+      throw new Error(
+        `Cloudly backup unexpectedly included the large app image: ${
+          JSON.stringify(backupResult.backup)
+        }`,
+      );
     }
     if (backupResult.backup.platformResources.sort().join(',') !== 'minio,mongodb') {
-      throw new Error(`Cloudly backup missing platform resources: ${JSON.stringify(backupResult.backup)}`);
+      throw new Error(
+        `Cloudly backup missing platform resources: ${JSON.stringify(backupResult.backup)}`,
+      );
     }
 
     console.log(`[${scenarioName}] Restoring Cloudly platform-resource backup for ${serviceName}`);
@@ -301,10 +342,14 @@ const main = async () => {
       overwriteExisting: true,
     });
     if (restoreResult.service.name !== serviceName) {
-      throw new Error(`Cloudly restore returned unexpected service: ${JSON.stringify(restoreResult.service)}`);
+      throw new Error(
+        `Cloudly restore returned unexpected service: ${JSON.stringify(restoreResult.service)}`,
+      );
     }
     if (restoreResult.platformResourcesRestored !== 2 || restoreResult.warnings.length > 0) {
-      throw new Error(`Cloudly restore failed platform resource validation: ${JSON.stringify(restoreResult)}`);
+      throw new Error(
+        `Cloudly restore failed platform resource validation: ${JSON.stringify(restoreResult)}`,
+      );
     }
 
     await waitForCloudlyWorkerBootstrapRoute();
@@ -323,8 +368,8 @@ const main = async () => {
     }
 
     await removeDockerService(dockerServiceName);
-    await removeDockerService('onebox-mongodb');
-    await removeDockerService('onebox-minio');
+    await removeDockerContainer('onebox-mongodb');
+    await removeDockerContainer('onebox-minio');
     await removeDockerService('onebox-smartproxy');
   }
 };

scenarios/onebox-test-helpers.ts

@@ -0,0 +1,49 @@
+import { join } from '@std/path';
+
+import type { Onebox } from '../../onebox/ts/classes/onebox.ts';
+
+const disabledDcRouterStatus = {
+  mode: 'disabled' as const,
+  configured: false,
+  running: false,
+  healthy: false,
+  image: '',
+  gatewayUrl: '',
+  opsPort: 0,
+  httpPort: 0,
+  httpsPort: 0,
+};
+
+export const disableManagedDcRouterForScenario = (oneboxArg: Onebox) => {
+  const manager = oneboxArg.managedDcRouter as unknown as {
+    getMode: () => 'disabled';
+    prepareGatewaySettings: () => Promise<void>;
+    init: () => Promise<typeof disabledDcRouterStatus>;
+    stop: () => Promise<typeof disabledDcRouterStatus>;
+    getStatus: () => Promise<typeof disabledDcRouterStatus>;
+  };
+
+  manager.getMode = () => 'disabled';
+  manager.prepareGatewaySettings = async () => {};
+  manager.init = async () => disabledDcRouterStatus;
+  manager.stop = async () => disabledDcRouterStatus;
+  manager.getStatus = async () => disabledDcRouterStatus;
+};
+
+export const useLocalAppStoreForScenario = (oneboxArg: Onebox, appCatalogDirArg: string) => {
+  const appStore = oneboxArg.appStore as unknown as {
+    catalogCache: unknown;
+    lastFetchTime: number;
+    fetchJson: (pathArg: string) => Promise<unknown>;
+    fetchText: (pathArg: string) => Promise<string>;
+  };
+
+  appStore.catalogCache = null;
+  appStore.lastFetchTime = 0;
+  appStore.fetchText = async (pathArg: string) => {
+    return await Deno.readTextFile(join(appCatalogDirArg, pathArg));
+  };
+  appStore.fetchJson = async (pathArg: string) => {
+    return JSON.parse(await appStore.fetchText(pathArg)) as unknown;
+  };
+};