test: add regression scenario coverage

2026-05-08 16:24:45 +00:00
parent 0116c4972d
commit 08ab7fea8e
9 changed files with 1069 additions and 2700 deletions
@@ -1,5 +1,6 @@
 import * as http from 'node:http';
 import * as net from 'node:net';
+import * as crypto from 'node:crypto';
 import { access, mkdir, readFile, rm, writeFile } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
 import { dirname, join } from 'node:path';
@@ -120,6 +121,10 @@ const pathExists = async (pathArg: string) => {
  }
 };

+const sha256Base64 = (base64Arg: string) => {
+  return crypto.createHash('sha256').update(Buffer.from(base64Arg, 'base64')).digest('hex');
+};
+
 const assertCoreflowVolumeMounts = () => {
  const coreflow = new Coreflow();
  const service: IService = {
@@ -439,12 +444,207 @@ const assertCoreflowBackupOrchestration = async () => {
  }
 };

+const assertCoreflowRemoteReplication = async () => {
+  const socketPath = join(buildDir, 'replication-plugins', 'corestore.sock');
+  const dataDir = join(buildDir, 'replication-data');
+  await mkdir(dirname(socketPath), { recursive: true });
+
+  const controlPort = await getFreePort();
+  const s3Port = await getFreePort();
+  const dbPort = await getFreePort();
+  let corestore = new CoreStore({
+    dataDir,
+    bindAddress: '127.0.0.1',
+    publicHost: '127.0.0.1',
+    controlPort,
+    s3Port,
+    dbPort,
+    apiToken,
+    volumePluginSocketPath: socketPath,
+  });
+  const previousControlUrl = process.env.CORESTORE_CONTROL_URL;
+  const previousApiToken = process.env.CORESTORE_API_TOKEN;
+
+  try {
+    await corestore.start();
+    process.env.CORESTORE_CONTROL_URL = `http://127.0.0.1:${controlPort}`;
+    process.env.CORESTORE_API_TOKEN = apiToken;
+
+    const volumeName = 'remote-replication-data';
+    const service: IService = {
+      id: 'svc-remote-replication',
+      data: {
+        name: 'remote-replication',
+        description: 'Coreflow remote replication service',
+        imageId: 'image-remote-replication',
+        imageVersion: 'latest',
+        environment: {},
+        secretBundleId: 'secret-remote-replication',
+        serviceCategory: 'workload',
+        deploymentStrategy: 'custom',
+        scaleFactor: 1,
+        balancingStrategy: 'round-robin',
+        ports: {
+          web: 80,
+        },
+        volumes: [
+          {
+            source: volumeName,
+            mountPath: '/data',
+            backup: true,
+          },
+        ],
+        domains: [],
+        deploymentIds: [],
+      },
+    };
+
+    const createVolumeResponse = await controlPost<any>(controlPort, '/volumes/create', {
+      name: volumeName,
+      serviceId: service.id,
+      serviceName: service.data.name,
+      mountPath: '/data',
+      backup: true,
+    });
+    const mountpoint = createVolumeResponse.volume?.Mountpoint;
+    assert(typeof mountpoint === 'string', `Could not create replication test volume: ${JSON.stringify(createVolumeResponse)}`);
+    await writeFile(join(mountpoint, 'state.txt'), 'before remote backup\n');
+
+    const remoteObjects = new Map<string, { object: any; contentsBase64: string }>();
+    let remoteManifest: any;
+    const coreflow = new Coreflow();
+    const fakeCloudlyApiClient = {
+      platform: {
+        getPlatformDesiredState: async () => ({ capabilities: [], providerConfigs: [], bindings: [] }),
+      },
+      services: {
+        getServices: async () => [service],
+      },
+      typedsocketClient: {
+        createTypedRequest: (methodArg: string) => ({
+          fire: async (payloadArg: any) => {
+            if (methodArg === 'prepareBackupReplication') {
+              remoteManifest = payloadArg.manifest;
+              return {
+                missingObjects: payloadArg.manifest.objects.filter((objectArg: any) => {
+                  const existing = remoteObjects.get(objectArg.path);
+                  return !existing || existing.object.sha256 !== objectArg.sha256;
+                }),
+              };
+            }
+            if (methodArg === 'uploadBackupArchiveObject') {
+              assert(payloadArg.object.sha256 === sha256Base64(payloadArg.contentsBase64), `Remote upload checksum mismatch for ${payloadArg.object.path}`);
+              remoteObjects.set(payloadArg.object.path, {
+                object: payloadArg.object,
+                contentsBase64: payloadArg.contentsBase64,
+              });
+              return { accepted: true };
+            }
+            if (methodArg === 'completeBackupReplication') {
+              remoteManifest = payloadArg.manifest;
+              for (const object of remoteManifest.objects) {
+                assert(remoteObjects.has(object.path), `Missing replicated object ${object.path}`);
+              }
+              const manifestBody = JSON.stringify(remoteManifest);
+              return {
+                replication: {
+                  targetType: 's3',
+                  targetPath: 'fake-target',
+                  manifestPath: 'fake-target/manifest.json',
+                  manifestSha256: crypto.createHash('sha256').update(manifestBody).digest('hex'),
+                  objectCount: remoteManifest.objects.length,
+                  totalSize: remoteManifest.totalSize,
+                  completedAt: Date.now(),
+                },
+              };
+            }
+            if (methodArg === 'getBackupArchiveManifest') {
+              return { manifest: remoteManifest };
+            }
+            if (methodArg === 'downloadBackupArchiveObject') {
+              const remoteObject = remoteObjects.get(payloadArg.object.path);
+              assert(remoteObject, `Remote object ${payloadArg.object.path} not found`);
+              return remoteObject;
+            }
+            throw new Error(`Unexpected fake Cloudly request ${methodArg}`);
+          },
+        }),
+      },
+    };
+    (coreflow.cloudlyConnector as any).cloudlyApiClient = fakeCloudlyApiClient;
+    (coreflow.cloudlyConnector as any).identity = {
+      name: 'replication-test-coreflow',
+      role: 'cluster',
+      type: 'machine',
+      userId: 'cluster-remote-replication',
+      expiresAt: Date.now() + 3600 * 1000,
+      jwt: '',
+      clusterId: 'cluster-remote-replication',
+    };
+
+    const backupResult = await coreflow.backupManager.executeServiceBackup({
+      backupId: 'remote-replication-smoke',
+      service,
+      replication: {
+        enabled: true,
+      },
+      tags: {
+        scenario: scenarioName,
+      },
+    });
+    assert(backupResult.replication?.objectCount === remoteObjects.size, `Unexpected replication result: ${JSON.stringify(backupResult.replication)}`);
+    assert(remoteObjects.size > 0, 'Remote replication did not upload archive objects');
+
+    await corestore.stop();
+    await rm(join(dataDir, 'volume-archive'), { recursive: true, force: true });
+    await writeFile(join(mountpoint, 'state.txt'), 'after local archive loss\n');
+
+    corestore = new CoreStore({
+      dataDir,
+      bindAddress: '127.0.0.1',
+      publicHost: '127.0.0.1',
+      controlPort,
+      s3Port,
+      dbPort,
+      apiToken,
+      volumePluginSocketPath: socketPath,
+    });
+    await corestore.start();
+
+    await coreflow.backupManager.executeServiceRestore({
+      backupId: 'remote-replication-smoke',
+      service,
+      snapshots: backupResult.snapshots,
+      clear: true,
+      replication: {
+        enabled: true,
+      },
+    });
+    assert(await readFile(join(mountpoint, 'state.txt'), 'utf8') === 'before remote backup\n', 'Remote archive restore did not restore volume data');
+  } finally {
+    if (previousControlUrl === undefined) {
+      delete process.env.CORESTORE_CONTROL_URL;
+    } else {
+      process.env.CORESTORE_CONTROL_URL = previousControlUrl;
+    }
+    if (previousApiToken === undefined) {
+      delete process.env.CORESTORE_API_TOKEN;
+    } else {
+      process.env.CORESTORE_API_TOKEN = previousApiToken;
+    }
+    await corestore.stop().catch((errorArg) => {
+      console.log(`[${scenarioName}] Failed to stop replication Corestore: ${(errorArg as Error).message}`);
+    });
+  }
+};
+
 const main = async () => {
  try {
    await mkdir(buildDir, { recursive: true });
    assertCoreflowVolumeMounts();
    await assertCorestoreVolumeDriver();
    await assertCoreflowBackupOrchestration();
+    await assertCoreflowRemoteReplication();
    console.log(`[${scenarioName}] PASS`);
  } finally {
    await rm(buildDir, { recursive: true, force: true });