fix: honor backup and cluster config updates

ts/coreflow.classes.backupmanager.ts

@@ -19,6 +19,30 @@ type TBackupSnapshot = {
   bucketName?: string;
 };
 
+type TBackupArchiveObject = {
+  path: string;
+  size: number;
+  sha256: string;
+};
+
+type TBackupArchiveManifest = {
+  version: 1;
+  backupId: string;
+  createdAt: number;
+  objects: TBackupArchiveObject[];
+  totalSize: number;
+};
+
+type TBackupReplicationResult = {
+  targetType: 's3' | 'smb';
+  targetPath: string;
+  manifestPath: string;
+  manifestSha256: string;
+  objectCount: number;
+  totalSize: number;
+  completedAt: number;
+};
+
 type TServiceVolumeConfig = {
   name?: string;
   source?: string;
@@ -56,14 +80,20 @@ export class CoreflowBackupManager {
     backupId: string;
     service: plugins.servezoneInterfaces.data.IService;
     tags?: Record<string, string>;
+    replication?: {
+      enabled: boolean;
+    };
   }) {
     await this.provisionCorestoreBindingsIfAvailable(requestArg.service);
 
     const snapshots: TBackupSnapshot[] = [];
     snapshots.push(...await this.snapshotServiceVolumes(requestArg));
     snapshots.push(...await this.snapshotServiceResources(requestArg));
+    const replication = requestArg.replication?.enabled
+      ? await this.replicateBackupArchive(requestArg.backupId)
+      : undefined;
 
-    return { snapshots };
+    return { snapshots, replication };
   }
 
   public async executeServiceRestore(requestArg: {
@@ -72,8 +102,14 @@ export class CoreflowBackupManager {
     snapshots: TBackupSnapshot[];
     clear?: boolean;
     resourceTypes?: TBackupSnapshotType[];
+    replication?: {
+      enabled: boolean;
+    };
   }) {
     await this.provisionCorestoreBindingsIfAvailable(requestArg.service);
+    if (requestArg.replication?.enabled) {
+      await this.ensureBackupArchiveAvailable(requestArg.backupId);
+    }
 
     const selectedSnapshots = (requestArg.snapshots || []).filter((snapshotArg) => {
       return !requestArg.resourceTypes?.length || requestArg.resourceTypes.includes(snapshotArg.type);
@@ -153,6 +189,88 @@ export class CoreflowBackupManager {
     return responseText ? JSON.parse(responseText) as T : ({} as T);
   }
 
+  private async fireCloudlyBackupRequest<T = any>(methodArg: string, payloadArg: Record<string, unknown>) {
+    const connector = this.coreflowRef.cloudlyConnector as any;
+    if (!connector.cloudlyApiClient || !connector.identity) {
+      throw new Error('Cloudly connection is required for backup replication');
+    }
+    const request = connector.cloudlyApiClient.typedsocketClient.createTypedRequest(methodArg);
+    return await request.fire({
+      identity: connector.identity,
+      ...payloadArg,
+    }) as T;
+  }
+
+  private async getCorestoreArchiveManifest(backupIdArg: string) {
+    const response = await this.postCorestore<{ manifest: TBackupArchiveManifest }>('/archive/manifest', {
+      backupId: backupIdArg,
+    });
+    return response.manifest;
+  }
+
+  private async replicateBackupArchive(backupIdArg: string): Promise<TBackupReplicationResult> {
+    const manifest = await this.getCorestoreArchiveManifest(backupIdArg);
+    const prepareResult = await this.fireCloudlyBackupRequest<{ missingObjects: TBackupArchiveObject[] }>(
+      'prepareBackupReplication',
+      {
+        backupId: backupIdArg,
+        manifest,
+      },
+    );
+
+    for (const object of prepareResult.missingObjects || []) {
+      const readResult = await this.postCorestore<{
+        object: TBackupArchiveObject;
+        contentsBase64: string;
+      }>('/archive/object/read', { path: object.path });
+      if (readResult.object.sha256 !== object.sha256 || readResult.object.size !== object.size) {
+        throw new Error(`Corestore archive object changed during replication: ${object.path}`);
+      }
+      await this.fireCloudlyBackupRequest('uploadBackupArchiveObject', {
+        backupId: backupIdArg,
+        object,
+        contentsBase64: readResult.contentsBase64,
+      });
+    }
+
+    const completeResult = await this.fireCloudlyBackupRequest<{ replication: TBackupReplicationResult }>(
+      'completeBackupReplication',
+      {
+        backupId: backupIdArg,
+        manifest,
+      },
+    );
+    return completeResult.replication;
+  }
+
+  private async ensureBackupArchiveAvailable(backupIdArg: string) {
+    const manifestResult = await this.fireCloudlyBackupRequest<{ manifest: TBackupArchiveManifest }>(
+      'getBackupArchiveManifest',
+      { backupId: backupIdArg },
+    );
+    const remoteManifest = manifestResult.manifest;
+    const localManifest = await this.getCorestoreArchiveManifest(backupIdArg);
+    const localObjectMap = new Map(localManifest.objects.map((objectArg) => [objectArg.path, objectArg]));
+
+    for (const remoteObject of remoteManifest.objects || []) {
+      const localObject = localObjectMap.get(remoteObject.path);
+      if (localObject?.sha256 === remoteObject.sha256 && localObject.size === remoteObject.size) {
+        continue;
+      }
+      const downloadResult = await this.fireCloudlyBackupRequest<{
+        object: TBackupArchiveObject;
+        contentsBase64: string;
+      }>('downloadBackupArchiveObject', {
+        backupId: backupIdArg,
+        object: remoteObject,
+      });
+      await this.postCorestore('/archive/object/write', {
+        ...downloadResult.object,
+        contentsBase64: downloadResult.contentsBase64,
+      });
+    }
+  }
+
   private getDockerSafeName(valueArg: string, maxLengthArg = 64) {
     const safeName = valueArg
       .replace(/[^a-zA-Z0-9_.-]+/g, '-')
@@ -163,7 +281,7 @@ export class CoreflowBackupManager {
   }
 
   private getServiceVolumeConfigs(serviceArg: plugins.servezoneInterfaces.data.IService) {
-    const serviceData = serviceArg.data as plugins.servezoneInterfaces.data.IService['data'] & {
+    const serviceData = serviceArg.data as Omit<plugins.servezoneInterfaces.data.IService['data'], 'volumes'> & {
       volumes?: TServiceVolumeConfig[];
     };
     return (serviceData.volumes || []).filter((volumeArg) => {