diff --git a/ts/classes.cloudly.ts b/ts/classes.cloudly.ts index af7ca27..a42b4b6 100644 --- a/ts/classes.cloudly.ts +++ b/ts/classes.cloudly.ts @@ -32,6 +32,7 @@ import { logger } from './logger.js'; import { CloudlyAuthManager } from './manager.auth/classes.authmanager.js'; import { CloudlySettingsManager } from './manager.settings/classes.settingsmanager.js'; import { CloudlyPlatformManager } from './manager.platform/classes.platformmanager.js'; +import { CloudlyBackupManager } from './manager.backup/classes.backupmanager.js'; /** * Cloudly class can be used to instantiate a cloudly server. @@ -73,6 +74,7 @@ export class Cloudly { public dnsManager: DnsManager; public domainManager: DomainManager; public taskManager: CloudlyTaskManager; + public backupManager: CloudlyBackupManager; public nodeManager: CloudlyNodeManager; public baremetalManager: CloudlyBaremetalManager; @@ -108,6 +110,7 @@ export class Cloudly { this.dnsManager = new DnsManager(this); this.domainManager = new DomainManager(this); this.taskManager = new CloudlyTaskManager(this); + this.backupManager = new CloudlyBackupManager(this); this.secretManager = new CloudlySecretManager(this); this.nodeManager = new CloudlyNodeManager(this); this.baremetalManager = new CloudlyBaremetalManager(this); @@ -136,6 +139,7 @@ export class Cloudly { await this.platformManager.start(); await this.deploymentManager.start(); await this.taskManager.init(); + await this.backupManager.start(); await this.registryManager.start(); await this.domainManager.init(); @@ -162,6 +166,7 @@ export class Cloudly { await this.platformManager.stop(); await this.deploymentManager.stop(); await this.taskManager.stop(); + await this.backupManager.stop(); await this.registryManager.stop(); await this.externalRegistryManager.stop(); } diff --git a/ts/manager.backup/classes.backupmanager.ts b/ts/manager.backup/classes.backupmanager.ts new file mode 100644 index 0000000..373352f --- /dev/null +++ b/ts/manager.backup/classes.backupmanager.ts @@ -0,0 +1,320 @@ +import type { Cloudly } from '../classes.cloudly.js'; +import * as plugins from '../plugins.js'; +import { BackupRecord } from './classes.backuprecord.js'; + +export type TBackupStatus = 'pending' | 'running' | 'ready' | 'failed' | 'restoring' | 'restored'; +export type TBackupResourceType = 'volume' | 'database' | 'objectstorage'; + +export interface IBackupSnapshotData { + type: TBackupResourceType; + snapshotId: string; + snapshotName?: string; + originalSize: number; + storedSize: number; + createdAt: number; + tags?: Record; + volumeName?: string; + mountPath?: string; + resourceName?: string; + databaseName?: string; + bucketName?: string; +} + +export interface IBackupRecordData { + id: string; + serviceId: string; + serviceName?: string; + clusterId?: string; + status: TBackupStatus; + trigger: 'manual' | 'scheduled'; + snapshots: IBackupSnapshotData[]; + createdAt: number; + updatedAt: number; + completedAt?: number; + requestedBy?: string; + errorText?: string; + restoreHistory?: Array<{ + restoredAt: number; + status: 'restored' | 'failed'; + errorText?: string; + }>; + tags?: Record; +} + +export class CloudlyBackupManager { + public typedrouter = new plugins.typedrequest.TypedRouter(); + public cloudlyRef: Cloudly; + + get db() { + return this.cloudlyRef.mongodbConnector.smartdataDb; + } + + public CBackupRecord = plugins.smartdata.setDefaultManagerForDoc(this, BackupRecord); + + constructor(cloudlyRefArg: Cloudly) { + this.cloudlyRef = cloudlyRefArg; + this.cloudlyRef.typedrouter.addTypedRouter(this.typedrouter); + + this.typedrouter.addTypedHandler( + new plugins.typedrequest.TypedHandler('createServiceBackup', async (requestArg) => { + await this.passAdminIdentity(requestArg); + return { + backup: await this.createServiceBackup(requestArg), + }; + }), + ); + + this.typedrouter.addTypedHandler( + new plugins.typedrequest.TypedHandler('getServiceBackups', async (requestArg) => { + await this.passValidIdentity(requestArg); + return { + backups: await this.getBackups({ + ...(requestArg.serviceId ? { serviceId: requestArg.serviceId } : {}), + ...(requestArg.status ? { status: requestArg.status } : {}), + }), + }; + }), + ); + + this.typedrouter.addTypedHandler( + new plugins.typedrequest.TypedHandler('getBackupById', async (requestArg) => { + await this.passValidIdentity(requestArg); + return { + backup: await this.getBackupById(requestArg.backupId), + }; + }), + ); + + this.typedrouter.addTypedHandler( + new plugins.typedrequest.TypedHandler('restoreServiceBackup', async (requestArg) => { + await this.passAdminIdentity(requestArg); + return { + backup: await this.restoreServiceBackup(requestArg), + }; + }), + ); + } + + public async start() { + const schedule = process.env.CLOUDLY_BACKUP_CRON; + this.cloudlyRef.taskManager.registerTask( + 'backup-all-services', + new plugins.taskbuffer.Task({ + name: 'backup-all-services', + taskFunction: async () => await this.backupAllServices(), + }), + { + description: 'Create backups for every workload service with backup-enabled resources.', + category: 'backup', + schedule, + enabled: Boolean(schedule), + }, + ); + } + + public async stop() {} + + public async getBackups(queryArg: Record = {}) { + const backups = await this.CBackupRecord.getInstances(queryArg); + return await Promise.all(backups.map((backupArg) => backupArg.createSavableObject())); + } + + public async getBackupById(backupIdArg: string) { + const backup = await BackupRecord.getInstance({ id: backupIdArg }); + if (!backup) { + throw new plugins.typedrequest.TypedResponseError(`Backup ${backupIdArg} not found`); + } + return await backup.createSavableObject(); + } + + public async backupAllServices() { + const services = await this.cloudlyRef.serviceManager.CService.getInstances({}); + const results: Array<{ serviceId: string; backupId?: string; errorText?: string }> = []; + for (const service of services) { + if (service.data.serviceCategory && service.data.serviceCategory !== 'workload') { + continue; + } + try { + const backup = await this.createServiceBackup({ + identity: { + name: 'cloudly-backup-scheduler', + role: 'admin', + type: 'machine', + userId: 'system', + expiresAt: Date.now() + 3600 * 1000, + jwt: '', + }, + serviceId: service.id, + tags: { + trigger: 'scheduled', + }, + }); + results.push({ serviceId: service.id, backupId: backup.id }); + } catch (error) { + results.push({ serviceId: service.id, errorText: (error as Error).message }); + } + } + return { results }; + } + + public async createServiceBackup(requestArg: { + identity: plugins.servezoneInterfaces.data.IIdentity; + serviceId: string; + clusterId?: string; + tags?: Record; + }) { + const service = await this.cloudlyRef.serviceManager.CService.getInstance({ + id: requestArg.serviceId, + }); + if (!service) { + throw new plugins.typedrequest.TypedResponseError(`Service ${requestArg.serviceId} not found`); + } + + const now = Date.now(); + const backup = new BackupRecord(); + backup.id = await BackupRecord.getNewId(); + backup.serviceId = service.id; + backup.serviceName = service.data.name; + backup.clusterId = requestArg.clusterId || (requestArg.identity as any).clusterId; + backup.status = 'running'; + backup.trigger = 'manual'; + backup.snapshots = []; + backup.createdAt = now; + backup.updatedAt = now; + backup.requestedBy = requestArg.identity.userId; + backup.tags = requestArg.tags; + await backup.save(); + + try { + const result = await this.fireCoreflowRequest('executeServiceBackup', { + backupId: backup.id, + service: await service.createSavableObject(), + tags: requestArg.tags, + }, backup.clusterId); + backup.snapshots = result.snapshots || []; + backup.status = 'ready'; + backup.completedAt = Date.now(); + backup.updatedAt = Date.now(); + await backup.save(); + await this.applyRetention(backup.serviceId); + } catch (error) { + backup.status = 'failed'; + backup.errorText = (error as Error).message; + backup.completedAt = Date.now(); + backup.updatedAt = Date.now(); + await backup.save(); + throw error; + } + + return await backup.createSavableObject(); + } + + private async applyRetention(serviceIdArg: string) { + const keepLast = Number(process.env.CLOUDLY_BACKUP_KEEP_LAST || '24'); + if (!Number.isInteger(keepLast) || keepLast <= 0) { + return; + } + const backups = await this.CBackupRecord.getInstances({ + serviceId: serviceIdArg, + }); + const completedBackups = backups + .filter((backupArg) => backupArg.status === 'ready' || backupArg.status === 'restored' || backupArg.status === 'failed') + .sort((a, b) => (b.createdAt || 0) - (a.createdAt || 0)); + for (const backup of completedBackups.slice(keepLast)) { + await backup.delete(); + } + } + + public async restoreServiceBackup(requestArg: { + identity: plugins.servezoneInterfaces.data.IIdentity; + backupId: string; + clear?: boolean; + resourceTypes?: TBackupResourceType[]; + }) { + const backup = await BackupRecord.getInstance({ id: requestArg.backupId }); + if (!backup) { + throw new plugins.typedrequest.TypedResponseError(`Backup ${requestArg.backupId} not found`); + } + if (backup.status !== 'ready' && backup.status !== 'restored') { + throw new plugins.typedrequest.TypedResponseError(`Backup ${backup.id} is not restorable in status ${backup.status}`); + } + const service = await this.cloudlyRef.serviceManager.CService.getInstance({ + id: backup.serviceId, + }); + if (!service) { + throw new plugins.typedrequest.TypedResponseError(`Service ${backup.serviceId} not found`); + } + + backup.status = 'restoring'; + backup.updatedAt = Date.now(); + await backup.save(); + + try { + await this.fireCoreflowRequest('executeServiceRestore', { + backupId: backup.id, + service: await service.createSavableObject(), + snapshots: backup.snapshots || [], + clear: requestArg.clear, + resourceTypes: requestArg.resourceTypes, + }, backup.clusterId); + backup.status = 'restored'; + backup.restoreHistory = [ + ...(backup.restoreHistory || []), + { + restoredAt: Date.now(), + status: 'restored', + }, + ]; + backup.updatedAt = Date.now(); + await backup.save(); + } catch (error) { + backup.status = 'ready'; + backup.restoreHistory = [ + ...(backup.restoreHistory || []), + { + restoredAt: Date.now(), + status: 'failed', + errorText: (error as Error).message, + }, + ]; + backup.updatedAt = Date.now(); + await backup.save(); + throw error; + } + + return await backup.createSavableObject(); + } + + private async fireCoreflowRequest(methodArg: string, payloadArg: Record, clusterIdArg?: string) { + const typedsocket = this.cloudlyRef.server.typedServer?.typedsocket; + if (!typedsocket) { + throw new Error('Cloudly TypedSocket server is not running'); + } + + const connections = await typedsocket.findAllTargetConnections(async (connectionArg) => { + const identityTag = await connectionArg.getTagById('identity'); + const identity = identityTag?.payload as plugins.servezoneInterfaces.data.IIdentity | undefined; + return identity?.role === 'cluster' && (!clusterIdArg || (identity as any).clusterId === clusterIdArg); + }); + if (connections.length === 0) { + throw new Error(clusterIdArg + ? `No connected coreflow for cluster ${clusterIdArg}` + : 'No connected coreflow'); + } + + const request = typedsocket.createTypedRequest(methodArg, connections[0]); + return await request.fire(payloadArg as any); + } + + private async passValidIdentity(requestData: { identity: plugins.servezoneInterfaces.data.IIdentity }) { + await plugins.smartguard.passGuardsOrReject(requestData, [ + this.cloudlyRef.authManager.validIdentityGuard, + ]); + } + + private async passAdminIdentity(requestData: { identity: plugins.servezoneInterfaces.data.IIdentity }) { + await plugins.smartguard.passGuardsOrReject(requestData, [ + this.cloudlyRef.authManager.adminIdentityGuard, + ]); + } +} diff --git a/ts/manager.backup/classes.backuprecord.ts b/ts/manager.backup/classes.backuprecord.ts new file mode 100644 index 0000000..074e6d0 --- /dev/null +++ b/ts/manager.backup/classes.backuprecord.ts @@ -0,0 +1,51 @@ +import * as plugins from '../plugins.js'; +import type { CloudlyBackupManager, IBackupRecordData } from './classes.backupmanager.js'; + +@plugins.smartdata.managed() +export class BackupRecord extends plugins.smartdata.SmartDataDbDoc< + BackupRecord, + IBackupRecordData, + CloudlyBackupManager +> { + @plugins.smartdata.unI() + public id!: string; + + @plugins.smartdata.svDb() + public serviceId!: string; + + @plugins.smartdata.svDb() + public serviceName?: string; + + @plugins.smartdata.svDb() + public clusterId?: string; + + @plugins.smartdata.svDb() + public status!: IBackupRecordData['status']; + + @plugins.smartdata.svDb() + public trigger!: IBackupRecordData['trigger']; + + @plugins.smartdata.svDb() + public snapshots!: IBackupRecordData['snapshots']; + + @plugins.smartdata.svDb() + public createdAt!: number; + + @plugins.smartdata.svDb() + public updatedAt!: number; + + @plugins.smartdata.svDb() + public completedAt?: number; + + @plugins.smartdata.svDb() + public requestedBy?: string; + + @plugins.smartdata.svDb() + public errorText?: string; + + @plugins.smartdata.svDb() + public restoreHistory?: IBackupRecordData['restoreHistory']; + + @plugins.smartdata.svDb() + public tags?: Record; +}