test: expand onebox integration coverage

scenarios/codebase-regressions/scenario.ts

@@ -1,5 +1,5 @@
 import assert from 'assert/strict';
-import { readFileSync } from 'fs';
+import { existsSync, readFileSync, readdirSync } from 'fs';
 import { dirname, resolve } from 'path';
 import { fileURLToPath } from 'url';
 
@@ -9,6 +9,10 @@ const readWorkspaceFile = (pathFromRoot: string) => {
   return readFileSync(resolve(root, pathFromRoot), 'utf8');
 };
 
+const readWorkspaceJson = <T>(pathFromRoot: string) => {
+  return JSON.parse(readWorkspaceFile(pathFromRoot)) as T;
+};
+
 const assertIncludes = (filePath: string, expected: string) => {
   const content = readWorkspaceFile(filePath);
   assert.ok(content.includes(expected), `${filePath} should include ${expected}`);
@@ -19,6 +23,86 @@ const assertExcludes = (filePath: string, forbidden: string) => {
   assert.ok(!content.includes(forbidden), `${filePath} should not include ${forbidden}`);
 };
 
+const assertPort = (valueArg: unknown, labelArg: string) => {
+  assert.ok(Number.isInteger(valueArg), `${labelArg} should be an integer`);
+  assert.ok((valueArg as number) >= 1 && (valueArg as number) <= 65535, `${labelArg} should be a valid TCP/UDP port`);
+};
+
+const assertValidPublishedPort = (portArg: any, labelArg: string) => {
+  assertPort(portArg.targetPort, `${labelArg}.targetPort`);
+  const targetEnd = portArg.targetPortEnd ?? portArg.targetPort;
+  assertPort(targetEnd, `${labelArg}.targetPortEnd`);
+  assert.ok(targetEnd >= portArg.targetPort, `${labelArg} target range should be ascending`);
+
+  const publishedStart = portArg.publishedPort ?? portArg.targetPort;
+  const publishedEnd = portArg.publishedPortEnd ?? (publishedStart + (targetEnd - portArg.targetPort));
+  assertPort(publishedStart, `${labelArg}.publishedPort`);
+  assertPort(publishedEnd, `${labelArg}.publishedPortEnd`);
+  assert.ok(publishedEnd >= publishedStart, `${labelArg} published range should be ascending`);
+  assert.equal(targetEnd - portArg.targetPort, publishedEnd - publishedStart, `${labelArg} target and published ranges should have the same size`);
+  assert.ok(['tcp', 'udp'].includes(portArg.protocol ?? 'tcp'), `${labelArg}.protocol should be tcp or udp`);
+};
+
+const assertValidAppCatalog = () => {
+  const catalog = readWorkspaceJson<{
+    schemaVersion: number;
+    apps: Array<{ id: string; latestVersion: string }>;
+  }>('appstore-apptemplates/catalog.json');
+  assert.equal(catalog.schemaVersion, 1, 'App catalog schema version should be 1');
+
+  const appIds = catalog.apps.map((appArg) => appArg.id);
+  assert.equal(new Set(appIds).size, appIds.length, 'App catalog ids should be unique');
+
+  const appDirs = readdirSync(resolve(root, 'appstore-apptemplates/apps'), { withFileTypes: true })
+    .filter((direntArg) => direntArg.isDirectory())
+    .map((direntArg) => direntArg.name)
+    .sort();
+  assert.deepEqual([...appIds].sort(), appDirs, 'Every app directory should be represented in catalog.json');
+
+  for (const catalogApp of catalog.apps) {
+    const appPath = `appstore-apptemplates/apps/${catalogApp.id}`;
+    const metaPath = `${appPath}/app.json`;
+    assert.ok(existsSync(resolve(root, metaPath)), `${metaPath} should exist`);
+    const appMeta = readWorkspaceJson<{
+      id: string;
+      latestVersion: string;
+      versions: string[];
+    }>(metaPath);
+    assert.equal(appMeta.id, catalogApp.id, `${metaPath} id should match its directory and catalog entry`);
+    assert.equal(appMeta.latestVersion, catalogApp.latestVersion, `${metaPath} latestVersion should match catalog.json`);
+    assert.ok(appMeta.versions.includes(appMeta.latestVersion), `${metaPath} versions should include latestVersion`);
+
+    const configPath = `${appPath}/versions/${appMeta.latestVersion}/config.json`;
+    assert.ok(existsSync(resolve(root, configPath)), `${configPath} should exist`);
+    const config = readWorkspaceJson<{
+      image: string;
+      port: number;
+      envVars?: Array<{ key: string; value?: string; description: string; required?: boolean }>;
+      volumes?: Array<string | { mountPath: string }>;
+      publishedPorts?: any[];
+    }>(configPath);
+    assert.ok(config.image, `${configPath} should define an image`);
+    assertPort(config.port, `${configPath}.port`);
+
+    const envKeys = new Set<string>();
+    for (const envVar of config.envVars || []) {
+      assert.ok(envVar.key, `${configPath} env var should define a key`);
+      assert.ok(!envKeys.has(envVar.key), `${configPath} env var ${envVar.key} should be unique`);
+      envKeys.add(envVar.key);
+      assert.equal(typeof envVar.description, 'string', `${configPath} env var ${envVar.key} should describe its purpose`);
+    }
+
+    for (const [index, volume] of (config.volumes || []).entries()) {
+      const mountPath = typeof volume === 'string' ? volume : volume.mountPath;
+      assert.ok(mountPath?.startsWith('/'), `${configPath} volume ${index} should define an absolute mountPath`);
+    }
+
+    for (const [index, publishedPort] of (config.publishedPorts || []).entries()) {
+      assertValidPublishedPort(publishedPort, `${configPath}.publishedPorts[${index}]`);
+    }
+  }
+};
+
 assertIncludes('containerarchive/rust/src/prune.rs', 'rewritten_offsets');
 assertExcludes('containerarchive/rust/src/prune.rs', 'offset: entry.offset, // Note: offset in the new pack may differ');
 assertIncludes('containerarchive/test/test.ts', 'partial-pack GC rewrites chunks');
@@ -35,17 +119,56 @@ assertIncludes('api/ts/classes.image.ts', 'versionString: imageVersion');
 assertIncludes('cloudly/ts/manager.image/classes.imagemanager.ts', 'readFromWebstream(readable)');
 assertIncludes('cloudly/ts/manager.service/classes.servicemanager.ts', 'adminIdentityGuard');
 assertIncludes('cloudly/ts/manager.cluster/classes.clustermanager.ts', 'getClusterById');
+assertIncludes('cloudly/ts/manager.appcatalog/classes.appcatalogmanager.ts', 'installAppCatalogApp');
+assertIncludes('cloudly/ts/manager.appcatalog/classes.appcatalogmanager.ts', 'externalImageRef');
+assertIncludes('cloudly/ts/manager.appcatalog/classes.appcatalogmanager.ts', 'SERVICE_DOMAIN');
+assertIncludes('cloudly/ts/manager.deployment/classes.deploymentmanager.ts', 'deploymentWorkspaceExec');
+assertIncludes('coreflow/ts/coreflow.classes.coreflow.ts', 'deploymentRuntimeManager');
+assertIncludes('coreflow/ts/coreflow.classes.deploymentruntime.ts', 'coreflowDeploymentWorkspaceExec');
+assertIncludes('coreflow/ts/coreflow.classes.clustermanager.ts', 'externalImageRef');
+assertIncludes('coreflow/ts/coreflow.classes.clustermanager.ts', 'resolveEnvTemplates');
+assertIncludes('corestore/ts/corestore.classes.corestore.ts', "'/archive/manifest'");
+assertIncludes('corestore/ts/corestore.classes.corestore.ts', "'/archive/object/read'");
+assertIncludes('corestore/ts/corestore.classes.corestore.ts', "'/archive/object/write'");
 
 assertIncludes('platformclient/ts/email/classes.emailconnector.ts', 'return response.responseId');
 assertIncludes('platformclient/ts/email/classes.letterconnector.ts', 'return response.processId');
 assertIncludes('platformclient/ts/email/classes.pushnotificationconnector.ts', "typeof process !== 'undefined'");
 
 assertIncludes('siprouter/ts/sipproxy.ts', 'return { id: callId };');
+assertIncludes('siprouter/ts/storage.ts', 'MONGODB_URI');
+assertIncludes('siprouter/ts/storage.ts', 'S3_ENDPOINT');
+assertIncludes('siprouter/Dockerfile', 'libjpeg-dev');
+assertIncludes('siprouter/Dockerfile', 'libtiff-dev');
+assertIncludes('siprouter/Dockerfile', 'libjpeg-turbo');
+assertIncludes('siprouter/.smartconfig.json', 'linux/amd64');
+assertIncludes('siprouter/.smartconfig.json', 'linux/arm64');
 assertIncludes('onebox/ts/opsserver/classes.opsserver.ts', "'/v2/*'");
+assertIncludes('onebox/ts/classes/appstore.ts', 'installApp(optionsArg: IAppInstallOptions)');
+assertIncludes('onebox/ts/classes/appstore.ts', 'publishedPorts: optionsArg.publishedPorts || config.publishedPorts');
+assertIncludes('onebox/ts/classes/docker.ts', 'expandPublishedPorts');
+assertIncludes('onebox/ts/classes/docker.ts', 'Mounts: this.getSwarmVolumeMounts(service)');
+assertIncludes('onebox/ts/database/migrations/migration-016-service-volumes.ts', 'volumes TEXT');
+assertIncludes('onebox/ts/database/migrations/migration-017-service-published-ports.ts', 'published_ports TEXT');
+assertIncludes('onebox/ts_interfaces/requests/appstore.ts', "method: 'installAppTemplate'");
+assertIncludes('onebox/ts_interfaces/requests/appstore.ts', 'publishedPorts?: data.IServicePublishedPort[]');
+assertIncludes('onebox/ts/classes/managed-dcrouter.ts', 'DCROUTER_ADMIN_API_TOKEN');
+assertIncludes('api/ts/classes.cloudlyapiclient.ts', 'createNodeJumpCommand');
+assertIncludes('interfaces/ts/appcatalog/types.ts', 'publishedPorts?: IAppCatalogPublishedPort[]');
 assertIncludes('spark/ts/spark.cli.ts', 'CLOUDLY_URL: cloudlyUrl');
 
+assertValidAppCatalog();
+
 const rustdeskConfig = JSON.parse(readWorkspaceFile('appstore-apptemplates/apps/rustdesk-server/versions/1.0.0/config.json'));
 const relay = rustdeskConfig.envVars.find((entry: { key: string }) => entry.key === 'RELAY');
 assert.equal(relay?.value, '${SERVICE_DOMAIN}:21116');
 
+const siprouterConfig = readWorkspaceJson<any>('appstore-apptemplates/apps/siprouter/versions/1.0.0/config.json');
+assert.equal(siprouterConfig.platformRequirements?.mongodb, true);
+assert.equal(siprouterConfig.platformRequirements?.s3, true);
+assert.ok(siprouterConfig.publishedPorts.some((portArg: any) => portArg.targetPort === 20000 && portArg.targetPortEnd === 20200 && portArg.protocol === 'udp'));
+
+const gitopsConfig = readWorkspaceJson<any>('appstore-apptemplates/apps/gitops/versions/1.0.0/config.json');
+assert.ok(gitopsConfig.volumes.includes('/data/.serve.zone/gitops'));
+
 console.log('codebase regression checks passed');

scenarios/onebox-backup-restore/scenario.ts

@@ -1,6 +1,7 @@
 import { dirname, fromFileUrl, join, resolve } from '@std/path';
 
 import { Onebox } from '../../../onebox/ts/classes/onebox.ts';
+import { disableManagedDcRouterForScenario } from '../onebox-test-helpers.ts';
 
 const scenarioName = 'onebox-backup-restore';
 const smokeId = `onebox-backup-${Date.now().toString(36)}`;
@@ -10,6 +11,8 @@ const serviceName = `app-${Date.now().toString(36)}`;
 const cloneServiceName = `${serviceName}-clone`;
 const dockerServiceName = `onebox-${serviceName}`;
 const cloneDockerServiceName = `onebox-${cloneServiceName}`;
+const volumeName = `onebox-${serviceName}-data`;
+const cloneVolumeName = `onebox-${cloneServiceName}-data`;
 
 const delayFor = async (millisecondsArg: number) => {
   await new Promise((resolveArg) => setTimeout(resolveArg, millisecondsArg));
@@ -85,9 +88,35 @@ const removeDockerService = async (serviceNameArg: string) => {
   }
 };
 
+const removeDockerVolume = async (volumeNameArg: string) => {
+  let lastErrorText = '';
+  const startTime = Date.now();
+  while (Date.now() - startTime < 30000) {
+    const output = await outputCommand(
+      'docker',
+      ['volume', 'rm', '-f', volumeNameArg],
+      {
+        stdout: 'piped',
+        stderr: 'piped',
+      },
+      15000,
+    );
+    if (output.success) {
+      return;
+    }
+    lastErrorText = new TextDecoder().decode(output.stderr).trim();
+    await delayFor(1000);
+  }
+  console.log(
+    `[${scenarioName}] Failed to remove Docker volume ${volumeNameArg}: ${lastErrorText}`,
+  );
+};
+
 const assertNoPreexistingOneboxIngress = async () => {
   if (await dockerServiceExists('onebox-smartproxy')) {
-    throw new Error('onebox-smartproxy already exists; refusing to overwrite a running Onebox ingress service');
+    throw new Error(
+      'onebox-smartproxy already exists; refusing to overwrite a running Onebox ingress service',
+    );
   }
 };
 
@@ -111,14 +140,50 @@ const waitForDockerServiceRunning = async (serviceNameArg: string) => {
 };
 
 const waitForDockerServiceRemoved = async (serviceNameArg: string) => {
-  await waitFor(async () => !(await dockerServiceExists(serviceNameArg)), `${serviceNameArg} removal`);
+  await waitFor(
+    async () => !(await dockerServiceExists(serviceNameArg)),
+    `${serviceNameArg} removal`,
+  );
+};
+
+const inspectDockerServiceJson = async <T>(serviceNameArg: string, formatArg: string) => {
+  const output = await outputCommand('docker', [
+    'service',
+    'inspect',
+    serviceNameArg,
+    '--format',
+    formatArg,
+  ], {
+    stdout: 'piped',
+    stderr: 'piped',
+  }, 15000);
+  if (!output.success) {
+    throw new Error(`docker service inspect ${serviceNameArg} exited with ${output.code}`);
+  }
+  return JSON.parse(new TextDecoder().decode(output.stdout).trim()) as T;
+};
+
+const assertCloneVolumeMount = async () => {
+  const mounts = await inspectDockerServiceJson<Array<Record<string, unknown>>>(
+    cloneDockerServiceName,
+    '{{json .Spec.TaskTemplate.ContainerSpec.Mounts}}',
+  );
+  if (
+    !mounts.some((mountArg) => mountArg.Source === cloneVolumeName && mountArg.Target === '/data')
+  ) {
+    throw new Error(
+      `Cloned service volume mount missing from Docker service: ${JSON.stringify(mounts)}`,
+    );
+  }
 };
 
 const ensureDockerReady = async () => {
   await run('docker', ['version']);
   const { stdout } = await run('docker', ['info', '--format', '{{.Swarm.LocalNodeState}}']);
   if (stdout.trim() !== 'active') {
-    throw new Error('Docker Swarm must be active. In Vagrant this is handled by scripts/provision-vm.sh.');
+    throw new Error(
+      'Docker Swarm must be active. In Vagrant this is handled by scripts/provision-vm.sh.',
+    );
   }
 };
 
@@ -136,6 +201,7 @@ const main = async () => {
 
     console.log(`[${scenarioName}] Starting Onebox from ${buildDir}`);
     onebox = new Onebox();
+    disableManagedDcRouterForScenario(onebox);
     await onebox.init();
 
     await waitForDockerServiceRunning('onebox-smartproxy');
@@ -149,6 +215,7 @@ const main = async () => {
       envVars: {
         ONEBOX_BACKUP_SCENARIO: smokeId,
       },
+      volumes: [{ mountPath: '/data', backup: true }],
     });
     deployedService = true;
 
@@ -162,17 +229,25 @@ const main = async () => {
     const backupResult = await onebox.backupManager.createBackup(serviceName);
     const backupId = backupResult.backup.id;
     if (!backupId || !backupResult.snapshotId || !backupResult.backup.snapshotId) {
-      throw new Error(`Backup did not produce a ContainerArchive snapshot: ${JSON.stringify(backupResult)}`);
+      throw new Error(
+        `Backup did not produce a ContainerArchive snapshot: ${JSON.stringify(backupResult)}`,
+      );
     }
     if (!backupResult.backup.includesImage) {
-      throw new Error(`Backup did not include Docker image: ${JSON.stringify(backupResult.backup)}`);
+      throw new Error(
+        `Backup did not include Docker image: ${JSON.stringify(backupResult.backup)}`,
+      );
     }
     if ((backupResult.backup.storedSizeBytes ?? 0) <= 0 || backupResult.backup.sizeBytes <= 0) {
       throw new Error(`Backup size metadata is invalid: ${JSON.stringify(backupResult.backup)}`);
     }
 
     const backups = onebox.backupManager.listBackups(serviceName);
-    if (!backups.some((backupArg) => backupArg.id === backupId && backupArg.snapshotId === backupResult.snapshotId)) {
+    if (
+      !backups.some((backupArg) =>
+        backupArg.id === backupId && backupArg.snapshotId === backupResult.snapshotId
+      )
+    ) {
       throw new Error(`Backup not listed for service: ${JSON.stringify(backups)}`);
     }
 
@@ -183,20 +258,34 @@ const main = async () => {
     });
     clonedService = true;
 
-    if (restoreResult.service.name !== cloneServiceName || restoreResult.service.status !== 'running') {
-      throw new Error(`Unexpected restored service state: ${JSON.stringify(restoreResult.service)}`);
+    if (
+      restoreResult.service.name !== cloneServiceName || restoreResult.service.status !== 'running'
+    ) {
+      throw new Error(
+        `Unexpected restored service state: ${JSON.stringify(restoreResult.service)}`,
+      );
     }
     if (restoreResult.service.domain !== undefined) {
-      throw new Error(`Clone unexpectedly retained a domain: ${JSON.stringify(restoreResult.service)}`);
+      throw new Error(
+        `Clone unexpectedly retained a domain: ${JSON.stringify(restoreResult.service)}`,
+      );
     }
     if (restoreResult.service.envVars.ONEBOX_BACKUP_SCENARIO !== smokeId) {
-      throw new Error(`Clone did not preserve env vars: ${JSON.stringify(restoreResult.service.envVars)}`);
+      throw new Error(
+        `Clone did not preserve env vars: ${JSON.stringify(restoreResult.service.envVars)}`,
+      );
+    }
+    if (!restoreResult.service.volumes?.some((volumeArg) => volumeArg.mountPath === '/data')) {
+      throw new Error(
+        `Clone did not preserve volumes: ${JSON.stringify(restoreResult.service.volumes)}`,
+      );
     }
     if (restoreResult.warnings.length > 0) {
       throw new Error(`Restore completed with warnings: ${restoreResult.warnings.join('; ')}`);
     }
 
     await waitForDockerServiceRunning(cloneDockerServiceName);
+    await assertCloneVolumeMount();
 
     console.log(`[${scenarioName}] Removing restored workload ${cloneServiceName}`);
     await onebox.services.removeService(cloneServiceName);
@@ -212,12 +301,16 @@ const main = async () => {
   } finally {
     if (onebox && clonedService) {
       await onebox.services.removeService(cloneServiceName).catch((error) => {
-        console.log(`[${scenarioName}] Failed to remove restored Onebox service: ${(error as Error).message}`);
+        console.log(
+          `[${scenarioName}] Failed to remove restored Onebox service: ${(error as Error).message}`,
+        );
       });
     }
     if (onebox && deployedService) {
       await onebox.services.removeService(serviceName).catch((error) => {
-        console.log(`[${scenarioName}] Failed to remove Onebox service: ${(error as Error).message}`);
+        console.log(
+          `[${scenarioName}] Failed to remove Onebox service: ${(error as Error).message}`,
+        );
       });
     }
     if (onebox) {
@@ -228,6 +321,8 @@ const main = async () => {
     await removeDockerService(cloneDockerServiceName);
     await removeDockerService(dockerServiceName);
     await removeDockerService('onebox-smartproxy');
+    await removeDockerVolume(cloneVolumeName);
+    await removeDockerVolume(volumeName);
   }
 };
 

scenarios/onebox-basic-lifecycle/scenario.ts

@@ -1,6 +1,7 @@
 import { dirname, fromFileUrl, join, resolve } from '@std/path';
 
 import { Onebox } from '../../../onebox/ts/classes/onebox.ts';
+import { disableManagedDcRouterForScenario } from '../onebox-test-helpers.ts';
 
 const scenarioName = 'onebox-basic-lifecycle';
 const smokeId = `onebox-basic-${Date.now().toString(36)}`;
@@ -9,6 +10,7 @@ const buildDir = join(testingDir, '.nogit', scenarioName, smokeId);
 const serviceName = `app-${Date.now().toString(36)}`;
 const dockerServiceName = `onebox-${serviceName}`;
 const routeDomain = `${serviceName}.test`;
+const volumeName = `onebox-${serviceName}-data`;
 
 const delayFor = async (millisecondsArg: number) => {
   await new Promise((resolveArg) => setTimeout(resolveArg, millisecondsArg));
@@ -69,6 +71,13 @@ const waitFor = async (checkFunctionArg: () => boolean | Promise<boolean>, messa
   throw new Error(`Timed out waiting for ${messageArg}`);
 };
 
+const getFreeTcpPort = () => {
+  const listener = Deno.listen({ hostname: '127.0.0.1', port: 0 });
+  const port = (listener.addr as Deno.NetAddr).port;
+  listener.close();
+  return port;
+};
+
 const dockerServiceExists = async (serviceNameArg: string) => {
   const output = await outputCommand('docker', ['service', 'inspect', serviceNameArg], {
     stdout: 'null',
@@ -84,9 +93,35 @@ const removeDockerService = async (serviceNameArg: string) => {
   }
 };
 
+const removeDockerVolume = async (volumeNameArg: string) => {
+  let lastErrorText = '';
+  const startTime = Date.now();
+  while (Date.now() - startTime < 30000) {
+    const output = await outputCommand(
+      'docker',
+      ['volume', 'rm', '-f', volumeNameArg],
+      {
+        stdout: 'piped',
+        stderr: 'piped',
+      },
+      15000,
+    );
+    if (output.success) {
+      return;
+    }
+    lastErrorText = new TextDecoder().decode(output.stderr).trim();
+    await delayFor(1000);
+  }
+  console.log(
+    `[${scenarioName}] Failed to remove Docker volume ${volumeNameArg}: ${lastErrorText}`,
+  );
+};
+
 const assertNoPreexistingOneboxIngress = async () => {
   if (await dockerServiceExists('onebox-smartproxy')) {
-    throw new Error('onebox-smartproxy already exists; refusing to overwrite a running Onebox ingress service');
+    throw new Error(
+      'onebox-smartproxy already exists; refusing to overwrite a running Onebox ingress service',
+    );
   }
 };
 
@@ -110,7 +145,27 @@ const waitForDockerServiceRunning = async (serviceNameArg: string) => {
 };
 
 const waitForDockerServiceRemoved = async (serviceNameArg: string) => {
-  await waitFor(async () => !(await dockerServiceExists(serviceNameArg)), `${serviceNameArg} removal`);
+  await waitFor(
+    async () => !(await dockerServiceExists(serviceNameArg)),
+    `${serviceNameArg} removal`,
+  );
+};
+
+const inspectDockerServiceJson = async <T>(serviceNameArg: string, formatArg: string) => {
+  const output = await outputCommand('docker', [
+    'service',
+    'inspect',
+    serviceNameArg,
+    '--format',
+    formatArg,
+  ], {
+    stdout: 'piped',
+    stderr: 'piped',
+  }, 15000);
+  if (!output.success) {
+    throw new Error(`docker service inspect ${serviceNameArg} exited with ${output.code}`);
+  }
+  return JSON.parse(new TextDecoder().decode(output.stdout).trim()) as T;
 };
 
 const requestRoute = async (protocolArg: 'http' | 'https', portArg: number) => {
@@ -168,9 +223,13 @@ const waitForRoute = async (protocolArg: 'http' | 'https', portArg: number) => {
       console.log(`[${scenarioName}] Last route error: ${lastError.message}`);
     }
     await run('docker', ['service', 'ps', 'onebox-smartproxy']).catch(() => null);
-    await run('docker', ['service', 'logs', '--raw', '--tail', '120', 'onebox-smartproxy']).catch(() => null);
+    await run('docker', ['service', 'logs', '--raw', '--tail', '120', 'onebox-smartproxy']).catch(
+      () => null,
+    );
     await run('docker', ['service', 'ps', dockerServiceName]).catch(() => null);
-    await run('docker', ['service', 'logs', '--raw', '--tail', '120', dockerServiceName]).catch(() => null);
+    await run('docker', ['service', 'logs', '--raw', '--tail', '120', dockerServiceName]).catch(
+      () => null,
+    );
     throw error;
   }
 };
@@ -205,13 +264,19 @@ const ensureDockerReady = async () => {
   await run('docker', ['version']);
   const { stdout } = await run('docker', ['info', '--format', '{{.Swarm.LocalNodeState}}']);
   if (stdout.trim() !== 'active') {
-    throw new Error('Docker Swarm must be active. In Vagrant this is handled by scripts/provision-vm.sh.');
+    throw new Error(
+      'Docker Swarm must be active. In Vagrant this is handled by scripts/provision-vm.sh.',
+    );
   }
 };
 
 const assertOneboxCoreReady = async (oneboxArg: Onebox) => {
   const status = await oneboxArg.getSystemStatus();
-  const reverseProxy = status.reverseProxy as { backend: string; http: { running: boolean }; https: { running: boolean } };
+  const reverseProxy = status.reverseProxy as {
+    backend: string;
+    http: { running: boolean };
+    https: { running: boolean };
+  };
   if (reverseProxy.backend !== 'smartproxy-docker') {
     throw new Error(`Unexpected reverse proxy backend: ${JSON.stringify(reverseProxy)}`);
   }
@@ -225,11 +290,59 @@ const assertOneboxCoreReady = async (oneboxArg: Onebox) => {
   }
 
   const platformServices = status.platformServices as Array<{ type: string; status: string }>;
-  if (!platformServices.some((serviceArg) => serviceArg.type === 'smartproxy' && serviceArg.status === 'running')) {
-    throw new Error(`SmartProxy platform service not reported as running: ${JSON.stringify(platformServices)}`);
+  if (
+    !platformServices.some((serviceArg) =>
+      serviceArg.type === 'smartproxy' && serviceArg.status === 'running'
+    )
+  ) {
+    throw new Error(
+      `SmartProxy platform service not reported as running: ${JSON.stringify(platformServices)}`,
+    );
   }
   if (platformServices.some((serviceArg) => serviceArg.type === 'caddy')) {
-    throw new Error(`Unexpected legacy caddy platform service: ${JSON.stringify(platformServices)}`);
+    throw new Error(
+      `Unexpected legacy caddy platform service: ${JSON.stringify(platformServices)}`,
+    );
+  }
+};
+
+const assertDockerStorageAndPorts = async (publishedPortArg: number) => {
+  const mounts = await inspectDockerServiceJson<Array<Record<string, unknown>>>(
+    dockerServiceName,
+    '{{json .Spec.TaskTemplate.ContainerSpec.Mounts}}',
+  );
+  if (!mounts.some((mountArg) => mountArg.Source === volumeName && mountArg.Target === '/data')) {
+    throw new Error(`Onebox volume mount missing from Docker service: ${JSON.stringify(mounts)}`);
+  }
+
+  const ports = await inspectDockerServiceJson<Array<Record<string, unknown>>>(
+    dockerServiceName,
+    '{{json .Spec.EndpointSpec.Ports}}',
+  );
+  if (
+    !ports.some((portArg) =>
+      portArg.TargetPort === 80 && portArg.PublishedPort === publishedPortArg &&
+      portArg.Protocol === 'tcp'
+    )
+  ) {
+    throw new Error(`Onebox published port missing from Docker service: ${JSON.stringify(ports)}`);
+  }
+};
+
+const assertServicePersistence = (oneboxArg: Onebox, publishedPortArg: number) => {
+  const service = oneboxArg.services.getService(serviceName);
+  if (!service) {
+    throw new Error(`Service was not persisted: ${serviceName}`);
+  }
+  if (!service.volumes?.some((volumeArg) => volumeArg.mountPath === '/data')) {
+    throw new Error(`Service volume was not persisted: ${JSON.stringify(service)}`);
+  }
+  if (
+    !service.publishedPorts?.some((portArg) =>
+      portArg.targetPort === 80 && portArg.publishedPort === publishedPortArg
+    )
+  ) {
+    throw new Error(`Service published port was not persisted: ${JSON.stringify(service)}`);
   }
 };
 
@@ -246,12 +359,14 @@ const main = async () => {
 
     console.log(`[${scenarioName}] Starting Onebox from ${buildDir}`);
     onebox = new Onebox();
+    disableManagedDcRouterForScenario(onebox);
     await onebox.init();
 
     await waitForDockerServiceRunning('onebox-smartproxy');
     await assertOneboxCoreReady(onebox);
 
     console.log(`[${scenarioName}] Deploying workload ${serviceName}`);
+    const publishedPort = getFreeTcpPort();
     const service = await onebox.services.deployService({
       name: serviceName,
       image: 'caddy:2-alpine',
@@ -259,6 +374,8 @@ const main = async () => {
       domain: routeDomain,
       autoDNS: false,
       envVars: {},
+      volumes: [{ mountPath: '/data', backup: true }],
+      publishedPorts: [{ targetPort: 80, publishedPort, protocol: 'tcp' }],
     });
     deployedService = true;
 
@@ -270,10 +387,16 @@ const main = async () => {
     }
 
     await waitForDockerServiceRunning(dockerServiceName);
+    assertServicePersistence(onebox, publishedPort);
+    await assertDockerStorageAndPorts(publishedPort);
     await waitForRoute('http', 8080);
 
     const certificate = await createSelfSignedCertificate();
-    await onebox.reverseProxy.addCertificate(routeDomain, certificate.publicKey, certificate.privateKey);
+    await onebox.reverseProxy.addCertificate(
+      routeDomain,
+      certificate.publicKey,
+      certificate.privateKey,
+    );
     await waitForRoute('https', 8443);
 
     console.log(`[${scenarioName}] Removing workload ${serviceName}`);
@@ -288,7 +411,9 @@ const main = async () => {
   } finally {
     if (onebox && deployedService) {
       await onebox.services.removeService(serviceName).catch((error) => {
-        console.log(`[${scenarioName}] Failed to remove Onebox service: ${(error as Error).message}`);
+        console.log(
+          `[${scenarioName}] Failed to remove Onebox service: ${(error as Error).message}`,
+        );
       });
     }
     if (onebox) {
@@ -298,6 +423,7 @@ const main = async () => {
     }
     await removeDockerService(dockerServiceName);
     await removeDockerService('onebox-smartproxy');
+    await removeDockerVolume(volumeName);
   }
 };
 

scenarios/onebox-cloudly-appstore-worker/scenario.ts

@@ -1,11 +1,16 @@
 import { dirname, fromFileUrl, join, resolve } from '@std/path';
 
 import { Onebox } from '../../../onebox/ts/classes/onebox.ts';
-import type { IAppVersionConfig } from '../../../onebox/ts/classes/appstore-types.ts';
+import {
+  disableManagedDcRouterForScenario,
+  useLocalAppStoreForScenario,
+} from '../onebox-test-helpers.ts';
 
 const scenarioName = 'onebox-cloudly-appstore-worker';
 const smokeId = `cloudly-appstore-${Date.now().toString(36)}`;
 const testingDir = resolve(dirname(fromFileUrl(import.meta.url)), '../..');
+const repoRoot = resolve(testingDir, '..');
+const appCatalogDir = join(repoRoot, 'appstore-apptemplates');
 const buildDir = join(testingDir, '.nogit', scenarioName, smokeId);
 const serviceName = `cloudly-${Date.now().toString(36)}`;
 const dockerServiceName = `onebox-${serviceName}`;
@@ -80,6 +85,22 @@ const dockerContainerRunning = async (containerNameArg: string) => {
   return new TextDecoder().decode(output.stdout).trim() === 'true';
 };
 
+const dockerContainerExists = async (containerNameArg: string) => {
+  const command = new Deno.Command('docker', {
+    args: ['container', 'inspect', containerNameArg],
+    stdout: 'null',
+    stderr: 'null',
+  });
+  return (await command.output()).success;
+};
+
+const removeDockerContainer = async (containerNameArg: string) => {
+  if (await dockerContainerExists(containerNameArg)) {
+    await run('docker', ['container', 'rm', '-f', containerNameArg]).catch(() => null);
+    await delayFor(1000);
+  }
+};
+
 const assertNoPreexistingScenarioServices = async () => {
   for (const serviceNameArg of [dockerServiceName, 'onebox-smartproxy']) {
     if (await dockerServiceExists(serviceNameArg)) {
@@ -87,8 +108,11 @@ const assertNoPreexistingScenarioServices = async () => {
     }
   }
   for (const containerNameArg of ['onebox-mongodb', 'onebox-minio']) {
-    if (await dockerContainerRunning(containerNameArg)) {
-      throw new Error(`${containerNameArg} is already running; refusing to overwrite a platform container`);
+    if (await dockerContainerExists(containerNameArg)) {
+      const state = await dockerContainerRunning(containerNameArg) ? 'running' : 'present';
+      throw new Error(
+        `${containerNameArg} is already ${state}; refusing to overwrite a platform container`,
+      );
     }
   }
 };
@@ -109,49 +133,30 @@ const waitForDockerServiceRunning = async (serviceNameArg: string) => {
     }, `${serviceNameArg} running task`);
   } catch (error) {
     await run('docker', ['service', 'ps', '--no-trunc', serviceNameArg]).catch(() => null);
-    await run('docker', ['service', 'logs', '--raw', '--tail', '120', serviceNameArg]).catch(() => null);
+    await run('docker', ['service', 'logs', '--raw', '--tail', '120', serviceNameArg]).catch(() =>
+      null
+    );
     throw error;
   }
 };
 
 const waitForDockerContainerRunning = async (containerNameArg: string) => {
-  await waitFor(async () => dockerContainerRunning(containerNameArg), `${containerNameArg} running container`);
+  await waitFor(
+    async () => dockerContainerRunning(containerNameArg),
+    `${containerNameArg} running container`,
+  );
 };
 
 const ensureDockerReady = async () => {
   await run('docker', ['version']);
   const { stdout } = await run('docker', ['info', '--format', '{{.Swarm.LocalNodeState}}']);
   if (stdout.trim() !== 'active') {
-    throw new Error('Docker Swarm must be active. In Vagrant this is handled by scripts/provision-vm.sh.');
+    throw new Error(
+      'Docker Swarm must be active. In Vagrant this is handled by scripts/provision-vm.sh.',
+    );
   }
 };
 
-const getTemplateEnvVars = (
-  configArg: IAppVersionConfig,
-  overridesArg: Record<string, string>,
-) => {
-  const envVars: Record<string, string> = {};
-  const missingRequiredEnvVars: string[] = [];
-
-  for (const envVar of configArg.envVars || []) {
-    const value = overridesArg[envVar.key] ?? envVar.value ?? '';
-    if (envVar.required && !value) {
-      missingRequiredEnvVars.push(envVar.key);
-    }
-    envVars[envVar.key] = value;
-  }
-
-  for (const [key, value] of Object.entries(overridesArg)) {
-    envVars[key] = value;
-  }
-
-  if (missingRequiredEnvVars.length > 0) {
-    throw new Error(`Missing required template env vars: ${missingRequiredEnvVars.join(', ')}`);
-  }
-
-  return envVars;
-};
-
 const requestCloudlyBootstrapScript = async () => {
   const curlArgs = [
     '-sS',
@@ -209,9 +214,13 @@ const waitForCloudlyWorkerBootstrapRoute = async () => {
       console.log(`[${scenarioName}] Last bootstrap error: ${lastError.message}`);
     }
     await run('docker', ['service', 'ps', 'onebox-smartproxy']).catch(() => null);
-    await run('docker', ['service', 'logs', '--raw', '--tail', '120', 'onebox-smartproxy']).catch(() => null);
+    await run('docker', ['service', 'logs', '--raw', '--tail', '120', 'onebox-smartproxy']).catch(
+      () => null,
+    );
     await run('docker', ['service', 'ps', dockerServiceName]).catch(() => null);
-    await run('docker', ['service', 'logs', '--raw', '--tail', '120', dockerServiceName]).catch(() => null);
+    await run('docker', ['service', 'logs', '--raw', '--tail', '120', dockerServiceName]).catch(
+      () => null,
+    );
     throw error;
   }
 };
@@ -229,6 +238,8 @@ const main = async () => {
 
     console.log(`[${scenarioName}] Starting Onebox from ${buildDir}`);
     onebox = new Onebox();
+    disableManagedDcRouterForScenario(onebox);
+    useLocalAppStoreForScenario(onebox, appCatalogDir);
     await onebox.init();
     await waitForDockerServiceRunning('onebox-smartproxy');
 
@@ -236,28 +247,23 @@ const main = async () => {
     const version = appMeta.latestVersion;
     const config = await onebox.appStore.getAppVersionConfig('cloudly', version);
     if (!config.platformRequirements?.mongodb || !config.platformRequirements?.s3) {
-      throw new Error(`Cloudly template must require MongoDB and S3: ${JSON.stringify(config.platformRequirements)}`);
+      throw new Error(
+        `Cloudly template must require MongoDB and S3: ${
+          JSON.stringify(config.platformRequirements)
+        }`,
+      );
     }
 
     console.log(`[${scenarioName}] Installing Cloudly template ${version} as ${serviceName}`);
-    const envVars = getTemplateEnvVars(config, {
-      SERVEZONE_ADMINACCOUNT: 'testadmin:testpassword',
-    });
-
-    const service = await onebox.services.deployService({
-      name: serviceName,
-      image: config.image,
-      port: config.port,
+    const service = await onebox.appStore.installApp({
+      appId: 'cloudly',
+      version,
+      serviceName,
       domain: routeDomain,
       autoDNS: false,
-      envVars,
-      enableMongoDB: Boolean(config.platformRequirements.mongodb),
-      enableS3: Boolean(config.platformRequirements.s3),
-      enableClickHouse: Boolean(config.platformRequirements.clickhouse),
-      enableRedis: Boolean(config.platformRequirements.redis),
-      enableMariaDB: Boolean(config.platformRequirements.mariadb),
-      appTemplateId: 'cloudly',
-      appTemplateVersion: version,
+      envVars: {
+        SERVEZONE_ADMINACCOUNT: 'testadmin:testpassword',
+      },
     });
     deployedService = true;
 
@@ -272,13 +278,38 @@ const main = async () => {
 
     const deployedServiceRecord = onebox.services.getService(serviceName);
     if (!deployedServiceRecord?.id) {
-      throw new Error(`Cloudly service missing after deploy: ${JSON.stringify(deployedServiceRecord)}`);
+      throw new Error(
+        `Cloudly service missing after deploy: ${JSON.stringify(deployedServiceRecord)}`,
+      );
+    }
+    if (deployedServiceRecord.envVars.SERVEZONE_URL !== routeDomain) {
+      throw new Error(
+        `Cloudly SERVICE_DOMAIN template was not resolved: ${
+          JSON.stringify(deployedServiceRecord.envVars)
+        }`,
+      );
+    }
+    if (
+      !deployedServiceRecord.envVars.MONGODB_URL ||
+      deployedServiceRecord.envVars.MONGODB_URL.includes('${')
+    ) {
+      throw new Error(
+        `Cloudly MongoDB template was not resolved: ${
+          JSON.stringify(deployedServiceRecord.envVars)
+        }`,
+      );
     }
 
-    const platformResources = await onebox.platformServices.getResourcesForService(deployedServiceRecord.id);
-    const platformResourceTypes = platformResources.map((resourceArg) => resourceArg.platformService.type).sort();
+    const platformResources = await onebox.platformServices.getResourcesForService(
+      deployedServiceRecord.id,
+    );
+    const platformResourceTypes = platformResources.map((resourceArg) =>
+      resourceArg.platformService.type
+    ).sort();
     if (platformResourceTypes.join(',') !== 'minio,mongodb') {
-      throw new Error(`Unexpected Cloudly platform resources: ${JSON.stringify(platformResourceTypes)}`);
+      throw new Error(
+        `Unexpected Cloudly platform resources: ${JSON.stringify(platformResourceTypes)}`,
+      );
     }
 
     console.log(`[${scenarioName}] Creating Cloudly platform-resource backup for ${serviceName}`);
@@ -286,13 +317,23 @@ const main = async () => {
     const backupResult = await onebox.backupManager.createBackup(serviceName);
     const backupId = backupResult.backup.id;
     if (!backupId || !backupResult.snapshotId || !backupResult.backup.snapshotId) {
-      throw new Error(`Cloudly backup did not produce a ContainerArchive snapshot: ${JSON.stringify(backupResult)}`);
+      throw new Error(
+        `Cloudly backup did not produce a ContainerArchive snapshot: ${
+          JSON.stringify(backupResult)
+        }`,
+      );
     }
     if (backupResult.backup.includesImage) {
-      throw new Error(`Cloudly backup unexpectedly included the large app image: ${JSON.stringify(backupResult.backup)}`);
+      throw new Error(
+        `Cloudly backup unexpectedly included the large app image: ${
+          JSON.stringify(backupResult.backup)
+        }`,
+      );
     }
     if (backupResult.backup.platformResources.sort().join(',') !== 'minio,mongodb') {
-      throw new Error(`Cloudly backup missing platform resources: ${JSON.stringify(backupResult.backup)}`);
+      throw new Error(
+        `Cloudly backup missing platform resources: ${JSON.stringify(backupResult.backup)}`,
+      );
     }
 
     console.log(`[${scenarioName}] Restoring Cloudly platform-resource backup for ${serviceName}`);
@@ -301,10 +342,14 @@ const main = async () => {
       overwriteExisting: true,
     });
     if (restoreResult.service.name !== serviceName) {
-      throw new Error(`Cloudly restore returned unexpected service: ${JSON.stringify(restoreResult.service)}`);
+      throw new Error(
+        `Cloudly restore returned unexpected service: ${JSON.stringify(restoreResult.service)}`,
+      );
     }
     if (restoreResult.platformResourcesRestored !== 2 || restoreResult.warnings.length > 0) {
-      throw new Error(`Cloudly restore failed platform resource validation: ${JSON.stringify(restoreResult)}`);
+      throw new Error(
+        `Cloudly restore failed platform resource validation: ${JSON.stringify(restoreResult)}`,
+      );
     }
 
     await waitForCloudlyWorkerBootstrapRoute();
@@ -323,8 +368,8 @@ const main = async () => {
     }
 
     await removeDockerService(dockerServiceName);
-    await removeDockerService('onebox-mongodb');
-    await removeDockerService('onebox-minio');
+    await removeDockerContainer('onebox-mongodb');
+    await removeDockerContainer('onebox-minio');
     await removeDockerService('onebox-smartproxy');
   }
 };

scenarios/onebox-test-helpers.ts

@@ -0,0 +1,49 @@
+import { join } from '@std/path';
+
+import type { Onebox } from '../../onebox/ts/classes/onebox.ts';
+
+const disabledDcRouterStatus = {
+  mode: 'disabled' as const,
+  configured: false,
+  running: false,
+  healthy: false,
+  image: '',
+  gatewayUrl: '',
+  opsPort: 0,
+  httpPort: 0,
+  httpsPort: 0,
+};
+
+export const disableManagedDcRouterForScenario = (oneboxArg: Onebox) => {
+  const manager = oneboxArg.managedDcRouter as unknown as {
+    getMode: () => 'disabled';
+    prepareGatewaySettings: () => Promise<void>;
+    init: () => Promise<typeof disabledDcRouterStatus>;
+    stop: () => Promise<typeof disabledDcRouterStatus>;
+    getStatus: () => Promise<typeof disabledDcRouterStatus>;
+  };
+
+  manager.getMode = () => 'disabled';
+  manager.prepareGatewaySettings = async () => {};
+  manager.init = async () => disabledDcRouterStatus;
+  manager.stop = async () => disabledDcRouterStatus;
+  manager.getStatus = async () => disabledDcRouterStatus;
+};
+
+export const useLocalAppStoreForScenario = (oneboxArg: Onebox, appCatalogDirArg: string) => {
+  const appStore = oneboxArg.appStore as unknown as {
+    catalogCache: unknown;
+    lastFetchTime: number;
+    fetchJson: (pathArg: string) => Promise<unknown>;
+    fetchText: (pathArg: string) => Promise<string>;
+  };
+
+  appStore.catalogCache = null;
+  appStore.lastFetchTime = 0;
+  appStore.fetchText = async (pathArg: string) => {
+    return await Deno.readTextFile(join(appCatalogDirArg, pathArg));
+  };
+  appStore.fetchJson = async (pathArg: string) => {
+    return JSON.parse(await appStore.fetchText(pathArg)) as unknown;
+  };
+};