import type { Cloudly } from '../classes.cloudly.js'; import * as plugins from '../plugins.js'; import { BackupRecord } from './classes.backuprecord.js'; import { createBackupTargetWriterFromEnv, type IBackupTargetWriter } from './classes.replicationtarget.js'; export type TBackupStatus = | 'pending' | 'running' | 'replicating' | 'replicated' | 'ready' | 'failed' | 'restoring' | 'restored'; export type TBackupResourceType = 'volume' | 'database' | 'objectstorage'; export type TBackupReplicationTargetType = 's3' | 'smb'; export interface IBackupArchiveObject { path: string; size: number; sha256: string; } export interface IBackupArchiveManifest { version: 1; backupId: string; createdAt: number; objects: IBackupArchiveObject[]; totalSize: number; } export interface IBackupReplicationResult { targetType: TBackupReplicationTargetType; targetPath: string; manifestPath: string; manifestSha256: string; objectCount: number; totalSize: number; completedAt: number; } export interface IBackupSnapshotData { type: TBackupResourceType; snapshotId: string; snapshotName?: string; originalSize: number; storedSize: number; createdAt: number; tags?: Record; volumeName?: string; mountPath?: string; resourceName?: string; databaseName?: string; bucketName?: string; } export interface IBackupRecordData { id: string; serviceId: string; serviceName?: string; clusterId?: string; status: TBackupStatus; trigger: 'manual' | 'scheduled'; snapshots: IBackupSnapshotData[]; replication?: IBackupReplicationResult; createdAt: number; updatedAt: number; completedAt?: number; requestedBy?: string; errorText?: string; restoreHistory?: Array<{ restoredAt: number; status: 'restored' | 'failed'; errorText?: string; }>; tags?: Record; } export class CloudlyBackupManager { public typedrouter = new plugins.typedrequest.TypedRouter(); public cloudlyRef: Cloudly; private backupTargetWriter?: IBackupTargetWriter; get db() { return this.cloudlyRef.mongodbConnector.smartdataDb; } public CBackupRecord = plugins.smartdata.setDefaultManagerForDoc(this, BackupRecord); constructor(cloudlyRefArg: Cloudly) { this.cloudlyRef = cloudlyRefArg; this.cloudlyRef.typedrouter.addTypedRouter(this.typedrouter); this.typedrouter.addTypedHandler( new plugins.typedrequest.TypedHandler('createServiceBackup', async (requestArg) => { await this.passAdminIdentity(requestArg); return { backup: await this.createServiceBackup(requestArg), }; }), ); this.typedrouter.addTypedHandler( new plugins.typedrequest.TypedHandler('getServiceBackups', async (requestArg) => { await this.passValidIdentity(requestArg); return { backups: await this.getBackups({ ...(requestArg.serviceId ? { serviceId: requestArg.serviceId } : {}), ...(requestArg.status ? { status: requestArg.status } : {}), }), }; }), ); this.typedrouter.addTypedHandler( new plugins.typedrequest.TypedHandler('getBackupById', async (requestArg) => { await this.passValidIdentity(requestArg); return { backup: await this.getBackupById(requestArg.backupId), }; }), ); this.typedrouter.addTypedHandler( new plugins.typedrequest.TypedHandler('restoreServiceBackup', async (requestArg) => { await this.passAdminIdentity(requestArg); return { backup: await this.restoreServiceBackup(requestArg), }; }), ); this.typedrouter.addTypedHandler( new plugins.typedrequest.TypedHandler('prepareBackupReplication', async (requestArg) => { await this.passClusterIdentity(requestArg); return await this.prepareBackupReplication(requestArg); }), ); this.typedrouter.addTypedHandler( new plugins.typedrequest.TypedHandler('uploadBackupArchiveObject', async (requestArg) => { await this.passClusterIdentity(requestArg); return await this.uploadBackupArchiveObject(requestArg); }), ); this.typedrouter.addTypedHandler( new plugins.typedrequest.TypedHandler('completeBackupReplication', async (requestArg) => { await this.passClusterIdentity(requestArg); return await this.completeBackupReplication(requestArg); }), ); this.typedrouter.addTypedHandler( new plugins.typedrequest.TypedHandler('getBackupArchiveManifest', async (requestArg) => { await this.passClusterIdentity(requestArg); return await this.getBackupArchiveManifest(requestArg); }), ); this.typedrouter.addTypedHandler( new plugins.typedrequest.TypedHandler('downloadBackupArchiveObject', async (requestArg) => { await this.passClusterIdentity(requestArg); return await this.downloadBackupArchiveObject(requestArg); }), ); } public async start() { const schedule = process.env.CLOUDLY_BACKUP_CRON; this.cloudlyRef.taskManager.registerTask( 'backup-all-services', new plugins.taskbuffer.Task({ name: 'backup-all-services', taskFunction: async () => await this.backupAllServices(), }), { description: 'Create backups for every workload service with backup-enabled resources.', category: 'backup', schedule, enabled: Boolean(schedule), }, ); } public async stop() {} public async getBackups(queryArg: Record = {}) { const backups = await this.CBackupRecord.getInstances(queryArg); return await Promise.all(backups.map((backupArg) => backupArg.createSavableObject())); } public async getBackupById(backupIdArg: string) { const backup = await BackupRecord.getInstance({ id: backupIdArg }); if (!backup) { throw new plugins.typedrequest.TypedResponseError(`Backup ${backupIdArg} not found`); } return await backup.createSavableObject(); } public async backupAllServices() { const services = await this.cloudlyRef.serviceManager.CService.getInstances({}); const results: Array<{ serviceId: string; backupId?: string; errorText?: string }> = []; for (const service of services) { if (service.data.serviceCategory && service.data.serviceCategory !== 'workload') { continue; } try { const backup = await this.createServiceBackup({ identity: { name: 'cloudly-backup-scheduler', role: 'admin', type: 'machine', userId: 'system', expiresAt: Date.now() + 3600 * 1000, jwt: '', }, serviceId: service.id, tags: { trigger: 'scheduled', }, }); results.push({ serviceId: service.id, backupId: backup.id }); } catch (error) { results.push({ serviceId: service.id, errorText: (error as Error).message }); } } return { results }; } public async createServiceBackup(requestArg: { identity: plugins.servezoneInterfaces.data.IIdentity; serviceId: string; clusterId?: string; tags?: Record; }) { const service = await this.cloudlyRef.serviceManager.CService.getInstance({ id: requestArg.serviceId, }); if (!service) { throw new plugins.typedrequest.TypedResponseError(`Service ${requestArg.serviceId} not found`); } const now = Date.now(); const backup = new BackupRecord(); backup.id = await BackupRecord.getNewId(); backup.serviceId = service.id; backup.serviceName = service.data.name; backup.clusterId = requestArg.clusterId || (requestArg.identity as any).clusterId; backup.status = 'running'; backup.trigger = 'manual'; backup.snapshots = []; backup.createdAt = now; backup.updatedAt = now; backup.requestedBy = requestArg.identity.userId; backup.tags = requestArg.tags; await backup.save(); const replicationEnabled = (requestArg as any).replicate !== false && !!process.env.CLOUDLY_BACKUP_TARGET_TYPE; try { const result = await this.fireCoreflowRequest('executeServiceBackup', { backupId: backup.id, service: await service.createSavableObject(), tags: requestArg.tags, replication: { enabled: replicationEnabled, }, }, backup.clusterId); backup.snapshots = result.snapshots || []; if (replicationEnabled && !result.replication) { throw new Error('Coreflow did not complete remote backup replication'); } backup.replication = result.replication; backup.status = 'replicated'; backup.completedAt = Date.now(); backup.updatedAt = Date.now(); await backup.save(); await this.applyRetention(backup.serviceId); } catch (error) { backup.status = 'failed'; backup.errorText = (error as Error).message; backup.completedAt = Date.now(); backup.updatedAt = Date.now(); await backup.save(); throw error; } return await backup.createSavableObject(); } private async applyRetention(serviceIdArg: string) { const keepLast = Number(process.env.CLOUDLY_BACKUP_KEEP_LAST || '24'); if (!Number.isInteger(keepLast) || keepLast <= 0) { return; } const backups = await this.CBackupRecord.getInstances({ serviceId: serviceIdArg, }); const completedBackups = backups .filter((backupArg) => backupArg.status === 'replicated' || backupArg.status === 'restored' || backupArg.status === 'failed') .sort((a, b) => (b.createdAt || 0) - (a.createdAt || 0)); for (const backup of completedBackups.slice(keepLast)) { await backup.delete(); } } public async restoreServiceBackup(requestArg: { identity: plugins.servezoneInterfaces.data.IIdentity; backupId: string; clear?: boolean; resourceTypes?: TBackupResourceType[]; }) { const backup = await BackupRecord.getInstance({ id: requestArg.backupId }); if (!backup) { throw new plugins.typedrequest.TypedResponseError(`Backup ${requestArg.backupId} not found`); } if (backup.status !== 'replicated' && backup.status !== 'restored') { throw new plugins.typedrequest.TypedResponseError(`Backup ${backup.id} is not restorable in status ${backup.status}`); } const service = await this.cloudlyRef.serviceManager.CService.getInstance({ id: backup.serviceId, }); if (!service) { throw new plugins.typedrequest.TypedResponseError(`Service ${backup.serviceId} not found`); } const previousStatus = backup.status; backup.status = 'restoring'; backup.updatedAt = Date.now(); await backup.save(); try { await this.fireCoreflowRequest('executeServiceRestore', { backupId: backup.id, service: await service.createSavableObject(), snapshots: backup.snapshots || [], clear: requestArg.clear, resourceTypes: requestArg.resourceTypes, replication: { enabled: true, }, }, backup.clusterId); backup.status = 'restored'; backup.restoreHistory = [ ...(backup.restoreHistory || []), { restoredAt: Date.now(), status: 'restored', }, ]; backup.updatedAt = Date.now(); await backup.save(); } catch (error) { backup.status = previousStatus; backup.restoreHistory = [ ...(backup.restoreHistory || []), { restoredAt: Date.now(), status: 'failed', errorText: (error as Error).message, }, ]; backup.updatedAt = Date.now(); await backup.save(); throw error; } return await backup.createSavableObject(); } private getBackupTargetWriter() { if (!this.backupTargetWriter) { this.backupTargetWriter = createBackupTargetWriterFromEnv(); } return this.backupTargetWriter; } private normalizeTargetPath(pathArg: string) { const normalized = plugins.path.posix .normalize(String(pathArg || '').replace(/\\/g, '/').trim()) .replace(/^\/+/, ''); if (!normalized || normalized === '.' || normalized.startsWith('../') || normalized.includes('/../')) { throw new Error(`Invalid backup target path ${pathArg}`); } return normalized; } private getBackupTargetPath(backupArg: BackupRecord) { return this.normalizeTargetPath([ process.env.CLOUDLY_BACKUP_TARGET_PREFIX || 'serve.zone-backups', 'clusters', backupArg.clusterId || 'default', 'services', backupArg.serviceId, 'backups', backupArg.id, ].filter(Boolean).join('/')); } private getArchiveObjectTargetPath(backupArg: BackupRecord, objectPathArg: string) { return this.normalizeTargetPath(`${this.getBackupTargetPath(backupArg)}/archive/${objectPathArg}`); } private getManifestTargetPath(backupArg: BackupRecord) { return this.normalizeTargetPath(`${this.getBackupTargetPath(backupArg)}/manifest.json`); } private getSha256(contentsArg: Buffer) { return plugins.crypto.createHash('sha256').update(contentsArg).digest('hex'); } private assertObjectMatches(objectArg: IBackupArchiveObject, contentsArg: Buffer) { if (contentsArg.length !== objectArg.size || this.getSha256(contentsArg) !== objectArg.sha256) { throw new Error(`Backup archive object checksum mismatch for ${objectArg.path}`); } } private createManifestBuffer(backupArg: BackupRecord, manifestArg: IBackupArchiveManifest) { return Buffer.from(`${JSON.stringify({ version: 1, backupId: backupArg.id, serviceId: backupArg.serviceId, serviceName: backupArg.serviceName, clusterId: backupArg.clusterId, archive: manifestArg, }, null, 2)}\n`); } private async getBackupForClusterRequest(backupIdArg: string, identityArg: plugins.servezoneInterfaces.data.IIdentity) { const backup = await BackupRecord.getInstance({ id: backupIdArg }); if (!backup) { throw new plugins.typedrequest.TypedResponseError(`Backup ${backupIdArg} not found`); } const identityClusterId = (identityArg as any).clusterId; if (backup.clusterId && identityClusterId && backup.clusterId !== identityClusterId) { throw new plugins.typedrequest.TypedResponseError(`Backup ${backupIdArg} does not belong to this cluster`); } return backup; } public async prepareBackupReplication(requestArg: { identity: plugins.servezoneInterfaces.data.IIdentity; backupId: string; manifest: IBackupArchiveManifest; }) { const backup = await this.getBackupForClusterRequest(requestArg.backupId, requestArg.identity); const targetWriter = this.getBackupTargetWriter(); const missingObjects: IBackupArchiveObject[] = []; for (const object of requestArg.manifest.objects || []) { const targetPath = this.getArchiveObjectTargetPath(backup, object.path); if (!await targetWriter.hasObject(targetPath, object)) { missingObjects.push(object); } } backup.status = 'replicating'; backup.updatedAt = Date.now(); await backup.save(); return { missingObjects }; } public async uploadBackupArchiveObject(requestArg: { identity: plugins.servezoneInterfaces.data.IIdentity; backupId: string; object: IBackupArchiveObject; contentsBase64: string; }) { const backup = await this.getBackupForClusterRequest(requestArg.backupId, requestArg.identity); const contents = Buffer.from(requestArg.contentsBase64 || '', 'base64'); this.assertObjectMatches(requestArg.object, contents); await this.getBackupTargetWriter().putObject( this.getArchiveObjectTargetPath(backup, requestArg.object.path), requestArg.object, contents, ); return { accepted: true }; } public async completeBackupReplication(requestArg: { identity: plugins.servezoneInterfaces.data.IIdentity; backupId: string; manifest: IBackupArchiveManifest; }) { const backup = await this.getBackupForClusterRequest(requestArg.backupId, requestArg.identity); const targetWriter = this.getBackupTargetWriter(); for (const object of requestArg.manifest.objects || []) { const targetPath = this.getArchiveObjectTargetPath(backup, object.path); if (!await targetWriter.hasObject(targetPath, object)) { throw new Error(`Remote backup target is missing archive object ${object.path}`); } } const manifestPath = this.getManifestTargetPath(backup); const manifestBuffer = this.createManifestBuffer(backup, requestArg.manifest); const manifestObject = { path: 'manifest.json', size: manifestBuffer.length, sha256: this.getSha256(manifestBuffer), }; await targetWriter.putObject(manifestPath, manifestObject, manifestBuffer); const replication: IBackupReplicationResult = { targetType: targetWriter.targetType, targetPath: this.getBackupTargetPath(backup), manifestPath, manifestSha256: manifestObject.sha256, objectCount: requestArg.manifest.objects.length, totalSize: requestArg.manifest.totalSize, completedAt: Date.now(), }; backup.replication = replication; backup.status = 'replicated'; backup.completedAt = replication.completedAt; backup.updatedAt = replication.completedAt; await backup.save(); return { replication }; } public async getBackupArchiveManifest(requestArg: { identity: plugins.servezoneInterfaces.data.IIdentity; backupId: string; }) { const backup = await this.getBackupForClusterRequest(requestArg.backupId, requestArg.identity); if (!backup.replication) { throw new plugins.typedrequest.TypedResponseError(`Backup ${backup.id} has not been replicated`); } const manifestBuffer = await this.getBackupTargetWriter().readObject(backup.replication.manifestPath); if (this.getSha256(manifestBuffer) !== backup.replication.manifestSha256) { throw new Error(`Remote manifest checksum mismatch for backup ${backup.id}`); } const parsedManifest = JSON.parse(manifestBuffer.toString('utf8')); return { manifest: parsedManifest.archive as IBackupArchiveManifest }; } public async downloadBackupArchiveObject(requestArg: { identity: plugins.servezoneInterfaces.data.IIdentity; backupId: string; object: IBackupArchiveObject; }) { const backup = await this.getBackupForClusterRequest(requestArg.backupId, requestArg.identity); if (!backup.replication) { throw new plugins.typedrequest.TypedResponseError(`Backup ${backup.id} has not been replicated`); } const contents = await this.getBackupTargetWriter().readObject( this.getArchiveObjectTargetPath(backup, requestArg.object.path), ); this.assertObjectMatches(requestArg.object, contents); return { object: requestArg.object, contentsBase64: contents.toString('base64'), }; } private async fireCoreflowRequest(methodArg: string, payloadArg: Record, clusterIdArg?: string) { const typedsocket = this.cloudlyRef.server.typedServer?.typedsocket; if (!typedsocket) { throw new Error('Cloudly TypedSocket server is not running'); } const connections = await typedsocket.findAllTargetConnections(async (connectionArg) => { const identityTag = await connectionArg.getTagById('identity'); const identity = identityTag?.payload as plugins.servezoneInterfaces.data.IIdentity | undefined; return identity?.role === 'cluster' && (!clusterIdArg || (identity as any).clusterId === clusterIdArg); }); if (connections.length === 0) { throw new Error(clusterIdArg ? `No connected coreflow for cluster ${clusterIdArg}` : 'No connected coreflow'); } const request = typedsocket.createTypedRequest(methodArg, connections[0]); return await request.fire(payloadArg as any); } private async passValidIdentity(requestData: { identity: plugins.servezoneInterfaces.data.IIdentity }) { await plugins.smartguard.passGuardsOrReject(requestData, [ this.cloudlyRef.authManager.validIdentityGuard, ]); } private async passAdminIdentity(requestData: { identity: plugins.servezoneInterfaces.data.IIdentity }) { await plugins.smartguard.passGuardsOrReject(requestData, [ this.cloudlyRef.authManager.adminIdentityGuard, ]); } private async passClusterIdentity(requestData: { identity: plugins.servezoneInterfaces.data.IIdentity }) { await this.passValidIdentity(requestData); if (requestData.identity.role !== 'cluster') { throw new plugins.typedrequest.TypedResponseError('Cluster identity required'); } } }