/** * Backup Manager for Onebox * * Handles service backup and restore operations including: * - Service configuration export/import * - MongoDB database dumps * - MinIO bucket contents * - ClickHouse database dumps * - Docker image export/import * - Encrypted archive creation */ import * as plugins from '../plugins.ts'; import type { IService, IBackup, IBackupManifest, IBackupServiceConfig, IBackupPlatformResource, IBackupResult, IRestoreOptions, IRestoreResult, TPlatformServiceType, IPlatformResource, } from '../types.ts'; import { logger } from '../logging.ts'; import { getErrorMessage } from '../utils/error.ts'; import { credentialEncryption } from './encryption.ts'; import type { Onebox } from './onebox.ts'; import { projectInfo } from '../info.ts'; // Backup archive encryption parameters const ENCRYPTION_ALGORITHM = 'AES-GCM'; const KEY_LENGTH = 256; const IV_LENGTH = 12; const SALT_LENGTH = 32; const PBKDF2_ITERATIONS = 100000; export class BackupManager { private oneboxRef: Onebox; constructor(oneboxRef: Onebox) { this.oneboxRef = oneboxRef; } /** * Create a backup for a service */ async createBackup(serviceName: string): Promise { const service = this.oneboxRef.database.getServiceByName(serviceName); if (!service) { throw new Error(`Service not found: ${serviceName}`); } // Verify backup password is configured const backupPassword = this.getBackupPassword(); if (!backupPassword) { throw new Error('Backup password not configured. Set a backup password in settings first.'); } logger.info(`Creating backup for service: ${serviceName}`); // Create temp directory for backup contents const timestamp = Date.now(); const tempDir = `/tmp/onebox-backup-${serviceName}-${timestamp}`; await Deno.mkdir(tempDir, { recursive: true }); try { // 1. Export service configuration const serviceConfig = await this.exportServiceConfig(service); await Deno.writeTextFile( `${tempDir}/service.json`, JSON.stringify(serviceConfig, null, 2) ); // 2. Export platform resources metadata and data const platformResources: IBackupPlatformResource[] = []; const resourceTypes: TPlatformServiceType[] = []; if (service.platformRequirements) { const resources = await this.oneboxRef.platformServices.getResourcesForService(service.id!); for (const { resource, platformService, credentials } of resources) { // Store resource metadata platformResources.push({ resourceType: resource.resourceType, resourceName: resource.resourceName, platformServiceType: platformService.type, credentials, }); // Track resource types if (!resourceTypes.includes(platformService.type)) { resourceTypes.push(platformService.type); } // Create data directory const dataDir = `${tempDir}/data/${platformService.type}`; await Deno.mkdir(dataDir, { recursive: true }); // Export data based on type switch (platformService.type) { case 'mongodb': await this.exportMongoDatabase(dataDir, resource, credentials); break; case 'minio': await this.exportMinioBucket(dataDir, resource, credentials); break; case 'clickhouse': await this.exportClickHouseDatabase(dataDir, resource, credentials); break; } } } await Deno.writeTextFile( `${tempDir}/platform-resources.json`, JSON.stringify(platformResources, null, 2) ); // 3. Export Docker image if configured const includeImage = service.includeImageInBackup !== false; // Default true if (includeImage && service.image) { await Deno.mkdir(`${tempDir}/data/image`, { recursive: true }); await this.exportDockerImage(service.image, `${tempDir}/data/image/image.tar`); } // 4. Create manifest const manifest: IBackupManifest = { version: '1.0', createdAt: timestamp, oneboxVersion: projectInfo.version, serviceName: service.name, includesImage: includeImage, platformResources: resourceTypes, checksum: '', // Will be computed after archive creation }; await Deno.writeTextFile( `${tempDir}/manifest.json`, JSON.stringify(manifest, null, 2) ); // 5. Create tar archive const tarPath = `/tmp/onebox-backup-${serviceName}-${timestamp}.tar`; await this.createTarArchive(tempDir, tarPath); // 6. Compute checksum of tar const tarData = await Deno.readFile(tarPath); const checksum = await this.computeChecksum(tarData); manifest.checksum = checksum; // Update manifest with checksum await Deno.writeTextFile( `${tempDir}/manifest.json`, JSON.stringify(manifest, null, 2) ); // Recreate tar with updated manifest await this.createTarArchive(tempDir, tarPath); // 7. Encrypt the archive const backupsDir = this.getBackupsDirectory(); await Deno.mkdir(backupsDir, { recursive: true }); const encryptedFilename = `${serviceName}-${timestamp}.tar.enc`; const encryptedPath = `${backupsDir}/${encryptedFilename}`; await this.encryptFile(tarPath, encryptedPath, backupPassword); // Get encrypted file size const stat = await Deno.stat(encryptedPath); const sizeBytes = stat.size; // 8. Store backup record in database const backup: IBackup = { serviceId: service.id!, serviceName: service.name, filename: encryptedFilename, sizeBytes, createdAt: timestamp, includesImage: includeImage, platformResources: resourceTypes, checksum, }; const createdBackup = this.oneboxRef.database.createBackup(backup); // Cleanup temp files await Deno.remove(tempDir, { recursive: true }); await Deno.remove(tarPath); logger.success(`Backup created for service ${serviceName}: ${encryptedFilename}`); return { backup: createdBackup, filePath: encryptedPath, }; } catch (error) { // Cleanup on error try { await Deno.remove(tempDir, { recursive: true }); } catch { // Ignore cleanup errors } logger.error(`Failed to create backup for ${serviceName}: ${getErrorMessage(error)}`); throw error; } } /** * Restore a backup */ async restoreBackup(backupPath: string, options: IRestoreOptions): Promise { // Verify backup password const backupPassword = this.getBackupPassword(); if (!backupPassword) { throw new Error('Backup password not configured.'); } logger.info(`Restoring backup from: ${backupPath}`); // Create temp directory for extraction const timestamp = Date.now(); const tempDir = `/tmp/onebox-restore-${timestamp}`; await Deno.mkdir(tempDir, { recursive: true }); const warnings: string[] = []; try { // 1. Decrypt the archive const tarPath = `${tempDir}/backup.tar`; await this.decryptFile(backupPath, tarPath, backupPassword); // 2. Extract tar archive await this.extractTarArchive(tarPath, tempDir); // 3. Read and validate manifest const manifestPath = `${tempDir}/manifest.json`; const manifestData = await Deno.readTextFile(manifestPath); const manifest: IBackupManifest = JSON.parse(manifestData); // Verify checksum (excluding manifest itself) // Note: For simplicity, we trust the manifest here // In production, you'd want to verify the checksum of specific files // 4. Read service config const serviceConfigPath = `${tempDir}/service.json`; const serviceConfigData = await Deno.readTextFile(serviceConfigPath); const serviceConfig: IBackupServiceConfig = JSON.parse(serviceConfigData); // 5. Read platform resources const platformResourcesPath = `${tempDir}/platform-resources.json`; let platformResources: IBackupPlatformResource[] = []; try { const resourcesData = await Deno.readTextFile(platformResourcesPath); platformResources = JSON.parse(resourcesData); } catch { // No platform resources in backup } // 6. Determine service name based on mode let serviceName: string; let existingService: IService | null = null; switch (options.mode) { case 'restore': serviceName = manifest.serviceName; existingService = this.oneboxRef.database.getServiceByName(serviceName); if (!existingService) { throw new Error(`Service '${serviceName}' not found. Use 'import' mode to create a new service.`); } if (!options.overwriteExisting) { throw new Error(`Service '${serviceName}' exists. Set overwriteExisting=true to proceed.`); } break; case 'import': case 'clone': if (!options.newServiceName) { throw new Error(`New service name required for '${options.mode}' mode.`); } serviceName = options.newServiceName; existingService = this.oneboxRef.database.getServiceByName(serviceName); if (existingService) { throw new Error(`Service '${serviceName}' already exists. Choose a different name.`); } break; default: throw new Error(`Invalid restore mode: ${options.mode}`); } // 7. Import Docker image if present if (manifest.includesImage) { const imagePath = `${tempDir}/data/image/image.tar`; try { await Deno.stat(imagePath); const newImageTag = await this.importDockerImage(imagePath); // Update service config with the imported image tag serviceConfig.image = newImageTag; logger.info(`Docker image imported: ${newImageTag}`); } catch (error) { warnings.push(`Docker image import failed: ${getErrorMessage(error)}`); } } // 8. Create or update service let service: IService; let platformResourcesRestored = 0; if (options.mode === 'restore' && existingService) { // Update existing service this.oneboxRef.database.updateService(existingService.id!, { image: serviceConfig.image, registry: serviceConfig.registry, port: serviceConfig.port, domain: serviceConfig.domain, useOneboxRegistry: serviceConfig.useOneboxRegistry, registryRepository: serviceConfig.registryRepository, registryImageTag: serviceConfig.registryImageTag, autoUpdateOnPush: serviceConfig.autoUpdateOnPush, platformRequirements: serviceConfig.platformRequirements, updatedAt: Date.now(), }); // Restore env vars (merge with platform provisioned vars later) const updatedEnvVars = { ...serviceConfig.envVars }; // Handle platform data restore if (!options.skipPlatformData && platformResources.length > 0) { platformResourcesRestored = await this.restorePlatformResources( existingService.id!, platformResources, tempDir, warnings ); } this.oneboxRef.database.updateService(existingService.id!, { envVars: updatedEnvVars }); service = this.oneboxRef.database.getServiceByName(serviceName)!; } else { // Create new service const deployOptions = { name: serviceName, image: serviceConfig.image, registry: serviceConfig.registry, port: serviceConfig.port, domain: options.mode === 'clone' ? undefined : serviceConfig.domain, // Don't duplicate domain for clones envVars: serviceConfig.envVars, useOneboxRegistry: serviceConfig.useOneboxRegistry, registryImageTag: serviceConfig.registryImageTag, autoUpdateOnPush: serviceConfig.autoUpdateOnPush, enableMongoDB: serviceConfig.platformRequirements?.mongodb, enableS3: serviceConfig.platformRequirements?.s3, enableClickHouse: serviceConfig.platformRequirements?.clickhouse, }; service = await this.oneboxRef.services.deployService(deployOptions); // Import platform data if not skipping if (!options.skipPlatformData && platformResources.length > 0) { // Wait a moment for platform resources to be provisioned await new Promise((resolve) => setTimeout(resolve, 2000)); platformResourcesRestored = await this.restorePlatformData( service.id!, platformResources, tempDir, warnings ); } } // Cleanup await Deno.remove(tempDir, { recursive: true }); logger.success(`Backup restored successfully as service '${serviceName}'`); return { service, platformResourcesRestored, warnings, }; } catch (error) { // Cleanup on error try { await Deno.remove(tempDir, { recursive: true }); } catch { // Ignore cleanup errors } logger.error(`Failed to restore backup: ${getErrorMessage(error)}`); throw error; } } /** * List all backups */ listBackups(serviceName?: string): IBackup[] { if (serviceName) { const service = this.oneboxRef.database.getServiceByName(serviceName); if (!service) { return []; } return this.oneboxRef.database.getBackupsByService(service.id!); } return this.oneboxRef.database.getAllBackups(); } /** * Delete a backup */ async deleteBackup(backupId: number): Promise { const backup = this.oneboxRef.database.getBackupById(backupId); if (!backup) { throw new Error(`Backup not found: ${backupId}`); } // Delete file const backupsDir = this.getBackupsDirectory(); const filePath = `${backupsDir}/${backup.filename}`; try { await Deno.remove(filePath); } catch { logger.warn(`Could not delete backup file: ${filePath}`); } // Delete database record this.oneboxRef.database.deleteBackup(backupId); logger.info(`Backup deleted: ${backup.filename}`); } /** * Get backup file path for download */ getBackupFilePath(backupId: number): string | null { const backup = this.oneboxRef.database.getBackupById(backupId); if (!backup) { return null; } const backupsDir = this.getBackupsDirectory(); return `${backupsDir}/${backup.filename}`; } // ========== Private Methods ========== /** * Get backup password from settings */ private getBackupPassword(): string | null { return this.oneboxRef.database.getSetting('backup_encryption_password'); } /** * Get backups directory */ private getBackupsDirectory(): string { const dataDir = this.oneboxRef.database.getSetting('dataDir') || './.nogit'; return `${dataDir}/backups`; } /** * Export service configuration */ private async exportServiceConfig(service: IService): Promise { return { name: service.name, image: service.image, registry: service.registry, envVars: service.envVars, port: service.port, domain: service.domain, useOneboxRegistry: service.useOneboxRegistry, registryRepository: service.registryRepository, registryImageTag: service.registryImageTag, autoUpdateOnPush: service.autoUpdateOnPush, platformRequirements: service.platformRequirements, includeImageInBackup: service.includeImageInBackup, }; } /** * Export MongoDB database */ private async exportMongoDatabase( dataDir: string, resource: IPlatformResource, credentials: Record ): Promise { logger.info(`Exporting MongoDB database: ${resource.resourceName}`); const mongoService = this.oneboxRef.database.getPlatformServiceById(resource.platformServiceId); if (!mongoService || !mongoService.containerId) { throw new Error('MongoDB service not running'); } // Build connection URI const connectionUri = credentials.connectionUri || credentials.MONGODB_URI; if (!connectionUri) { throw new Error('MongoDB connection URI not found in credentials'); } // Use mongodump via docker exec const archivePath = `/tmp/${resource.resourceName}.archive`; const result = await this.oneboxRef.docker.execInContainer(mongoService.containerId, [ 'mongodump', `--uri=${connectionUri}`, `--archive=${archivePath}`, '--gzip', ]); if (result.exitCode !== 0) { throw new Error(`mongodump failed: ${result.stderr}`); } // Copy archive out of container const container = await this.oneboxRef.docker.getContainerById(mongoService.containerId); if (!container) { throw new Error('MongoDB container not found'); } // Read the archive from container and write to local file const copyResult = await this.oneboxRef.docker.execInContainer(mongoService.containerId, [ 'cat', archivePath, ]); // Write base64-decoded content (stdout is binary data encoded) const localPath = `${dataDir}/${resource.resourceName}.archive`; const encoder = new TextEncoder(); await Deno.writeFile(localPath, encoder.encode(copyResult.stdout)); // Cleanup inside container await this.oneboxRef.docker.execInContainer(mongoService.containerId, ['rm', archivePath]); logger.success(`MongoDB database exported: ${resource.resourceName}`); } /** * Export MinIO bucket */ private async exportMinioBucket( dataDir: string, resource: IPlatformResource, credentials: Record ): Promise { logger.info(`Exporting MinIO bucket: ${resource.resourceName}`); const bucketDir = `${dataDir}/${resource.resourceName}`; await Deno.mkdir(bucketDir, { recursive: true }); // Use S3 client to download all objects const endpoint = credentials.endpoint || credentials.S3_ENDPOINT; const accessKey = credentials.accessKey || credentials.S3_ACCESS_KEY; const secretKey = credentials.secretKey || credentials.S3_SECRET_KEY; const bucket = credentials.bucket || credentials.S3_BUCKET; if (!endpoint || !accessKey || !secretKey || !bucket) { throw new Error('MinIO credentials incomplete'); } // Initialize S3 client const s3Client = new plugins.smarts3.SmartS3({ endpoint, accessKey, secretKey, bucket, }); await s3Client.start(); // List and download all objects const objects = await s3Client.listObjects(); for (const obj of objects) { const objectKey = obj.Key; if (!objectKey) continue; const objectData = await s3Client.getObject(objectKey); if (objectData) { const objectPath = `${bucketDir}/${objectKey}`; // Create parent directories if needed const parentDir = plugins.path.dirname(objectPath); await Deno.mkdir(parentDir, { recursive: true }); await Deno.writeFile(objectPath, objectData); } } await s3Client.stop(); // Also save bucket metadata await Deno.writeTextFile( `${bucketDir}/_metadata.json`, JSON.stringify({ bucket, objectCount: objects.length }, null, 2) ); logger.success(`MinIO bucket exported: ${resource.resourceName} (${objects.length} objects)`); } /** * Export ClickHouse database */ private async exportClickHouseDatabase( dataDir: string, resource: IPlatformResource, credentials: Record ): Promise { logger.info(`Exporting ClickHouse database: ${resource.resourceName}`); const clickhouseService = this.oneboxRef.database.getPlatformServiceByType('clickhouse'); if (!clickhouseService || !clickhouseService.containerId) { throw new Error('ClickHouse service not running'); } const dbName = credentials.database || credentials.CLICKHOUSE_DB; const user = credentials.username || credentials.CLICKHOUSE_USER || 'default'; const password = credentials.password || credentials.CLICKHOUSE_PASSWORD || ''; if (!dbName) { throw new Error('ClickHouse database name not found in credentials'); } // Get list of tables const tablesResult = await this.oneboxRef.docker.execInContainer(clickhouseService.containerId, [ 'clickhouse-client', `--user=${user}`, `--password=${password}`, '--query', `SELECT name FROM system.tables WHERE database = '${dbName}'`, ]); if (tablesResult.exitCode !== 0) { throw new Error(`Failed to list ClickHouse tables: ${tablesResult.stderr}`); } const tables = tablesResult.stdout.trim().split('\n').filter(Boolean); const dumpPath = `${dataDir}/${resource.resourceName}.sql`; let dumpContent = `-- ClickHouse backup for database: ${dbName}\n`; dumpContent += `-- Created: ${new Date().toISOString()}\n\n`; dumpContent += `CREATE DATABASE IF NOT EXISTS ${dbName};\n\n`; for (const table of tables) { // Get CREATE TABLE statement const createResult = await this.oneboxRef.docker.execInContainer(clickhouseService.containerId, [ 'clickhouse-client', `--user=${user}`, `--password=${password}`, '--query', `SHOW CREATE TABLE ${dbName}.${table}`, ]); if (createResult.exitCode === 0) { dumpContent += `-- Table: ${table}\n`; dumpContent += createResult.stdout + ';\n\n'; } // Get table data in TSV format const dataResult = await this.oneboxRef.docker.execInContainer(clickhouseService.containerId, [ 'clickhouse-client', `--user=${user}`, `--password=${password}`, '--query', `SELECT * FROM ${dbName}.${table} FORMAT TabSeparatedWithNames`, ]); if (dataResult.exitCode === 0 && dataResult.stdout.trim()) { // Save data to separate file for large datasets const tableDataPath = `${dataDir}/${resource.resourceName}_${table}.tsv`; await Deno.writeTextFile(tableDataPath, dataResult.stdout); } } await Deno.writeTextFile(dumpPath, dumpContent); logger.success(`ClickHouse database exported: ${resource.resourceName} (${tables.length} tables)`); } /** * Export Docker image */ private async exportDockerImage(imageName: string, outputPath: string): Promise { logger.info(`Exporting Docker image: ${imageName}`); // Use docker save command via shell const command = new Deno.Command('docker', { args: ['save', '-o', outputPath, imageName], }); const result = await command.output(); if (!result.success) { const stderr = new TextDecoder().decode(result.stderr); throw new Error(`docker save failed: ${stderr}`); } logger.success(`Docker image exported: ${imageName}`); } /** * Import Docker image */ private async importDockerImage(imagePath: string): Promise { logger.info(`Importing Docker image from: ${imagePath}`); // Use docker load command const command = new Deno.Command('docker', { args: ['load', '-i', imagePath], }); const result = await command.output(); if (!result.success) { const stderr = new TextDecoder().decode(result.stderr); throw new Error(`docker load failed: ${stderr}`); } const stdout = new TextDecoder().decode(result.stdout); // Parse image name from output like "Loaded image: nginx:latest" const match = stdout.match(/Loaded image: (.+)/); const imageName = match ? match[1].trim() : 'unknown'; logger.success(`Docker image imported: ${imageName}`); return imageName; } /** * Restore platform resources for existing service (restore mode) */ private async restorePlatformResources( serviceId: number, backupResources: IBackupPlatformResource[], tempDir: string, warnings: string[] ): Promise { let restoredCount = 0; // Get existing resources for this service const existingResources = await this.oneboxRef.platformServices.getResourcesForService(serviceId); for (const backupResource of backupResources) { try { // Find matching existing resource const existing = existingResources.find( (e) => e.platformService.type === backupResource.platformServiceType && e.resource.resourceType === backupResource.resourceType ); if (!existing) { warnings.push( `Platform resource ${backupResource.platformServiceType}/${backupResource.resourceName} not provisioned. Skipping data import.` ); continue; } // Import data based on type const dataDir = `${tempDir}/data/${backupResource.platformServiceType}`; switch (backupResource.platformServiceType) { case 'mongodb': await this.importMongoDatabase( dataDir, existing.resource, existing.credentials, backupResource.resourceName ); restoredCount++; break; case 'minio': await this.importMinioBucket( dataDir, existing.resource, existing.credentials, backupResource.resourceName ); restoredCount++; break; case 'clickhouse': await this.importClickHouseDatabase( dataDir, existing.resource, existing.credentials, backupResource.resourceName ); restoredCount++; break; } } catch (error) { warnings.push( `Failed to restore ${backupResource.platformServiceType} resource: ${getErrorMessage(error)}` ); } } return restoredCount; } /** * Restore platform data for new service (import/clone mode) */ private async restorePlatformData( serviceId: number, backupResources: IBackupPlatformResource[], tempDir: string, warnings: string[] ): Promise { // For new services, platform resources should have been provisioned during deployment return this.restorePlatformResources(serviceId, backupResources, tempDir, warnings); } /** * Import MongoDB database */ private async importMongoDatabase( dataDir: string, resource: IPlatformResource, credentials: Record, backupResourceName: string ): Promise { logger.info(`Importing MongoDB database: ${resource.resourceName}`); const mongoService = this.oneboxRef.database.getPlatformServiceById(resource.platformServiceId); if (!mongoService || !mongoService.containerId) { throw new Error('MongoDB service not running'); } const archivePath = `${dataDir}/${backupResourceName}.archive`; const connectionUri = credentials.connectionUri || credentials.MONGODB_URI; if (!connectionUri) { throw new Error('MongoDB connection URI not found'); } // Read local archive and copy to container const archiveData = await Deno.readFile(archivePath); const containerArchivePath = `/tmp/${resource.resourceName}.archive`; // Write archive to container via exec + stdin (simplified - use cat) // For production, use Docker API copy endpoint const base64Data = btoa(String.fromCharCode(...archiveData)); await this.oneboxRef.docker.execInContainer(mongoService.containerId, [ 'bash', '-c', `echo '${base64Data}' | base64 -d > ${containerArchivePath}`, ]); // Run mongorestore const result = await this.oneboxRef.docker.execInContainer(mongoService.containerId, [ 'mongorestore', `--uri=${connectionUri}`, `--archive=${containerArchivePath}`, '--gzip', '--drop', ]); if (result.exitCode !== 0) { throw new Error(`mongorestore failed: ${result.stderr}`); } // Cleanup await this.oneboxRef.docker.execInContainer(mongoService.containerId, ['rm', containerArchivePath]); logger.success(`MongoDB database imported: ${resource.resourceName}`); } /** * Import MinIO bucket */ private async importMinioBucket( dataDir: string, resource: IPlatformResource, credentials: Record, backupResourceName: string ): Promise { logger.info(`Importing MinIO bucket: ${resource.resourceName}`); const bucketDir = `${dataDir}/${backupResourceName}`; const endpoint = credentials.endpoint || credentials.S3_ENDPOINT; const accessKey = credentials.accessKey || credentials.S3_ACCESS_KEY; const secretKey = credentials.secretKey || credentials.S3_SECRET_KEY; const bucket = credentials.bucket || credentials.S3_BUCKET; if (!endpoint || !accessKey || !secretKey || !bucket) { throw new Error('MinIO credentials incomplete'); } const s3Client = new plugins.smarts3.SmartS3({ endpoint, accessKey, secretKey, bucket, }); await s3Client.start(); // Walk directory and upload all files let uploadedCount = 0; for await (const entry of Deno.readDir(bucketDir)) { if (entry.name === '_metadata.json') continue; const filePath = `${bucketDir}/${entry.name}`; if (entry.isFile) { const fileData = await Deno.readFile(filePath); await s3Client.putObject(entry.name, fileData); uploadedCount++; } // Note: For nested directories, would need recursive handling } await s3Client.stop(); logger.success(`MinIO bucket imported: ${resource.resourceName} (${uploadedCount} objects)`); } /** * Import ClickHouse database */ private async importClickHouseDatabase( dataDir: string, resource: IPlatformResource, credentials: Record, backupResourceName: string ): Promise { logger.info(`Importing ClickHouse database: ${resource.resourceName}`); const clickhouseService = this.oneboxRef.database.getPlatformServiceByType('clickhouse'); if (!clickhouseService || !clickhouseService.containerId) { throw new Error('ClickHouse service not running'); } const dbName = credentials.database || credentials.CLICKHOUSE_DB; const user = credentials.username || credentials.CLICKHOUSE_USER || 'default'; const password = credentials.password || credentials.CLICKHOUSE_PASSWORD || ''; if (!dbName) { throw new Error('ClickHouse database name not found'); } // Read SQL dump const sqlPath = `${dataDir}/${backupResourceName}.sql`; const sqlContent = await Deno.readTextFile(sqlPath); // Execute SQL statements const statements = sqlContent.split(';').filter((s) => s.trim()); for (const statement of statements) { if (statement.trim().startsWith('--')) continue; const result = await this.oneboxRef.docker.execInContainer(clickhouseService.containerId, [ 'clickhouse-client', `--user=${user}`, `--password=${password}`, '--query', statement.trim(), ]); if (result.exitCode !== 0) { logger.warn(`ClickHouse statement failed: ${result.stderr}`); } } // Import TSV data files for each table try { for await (const entry of Deno.readDir(dataDir)) { if (entry.name.endsWith('.tsv') && entry.name.startsWith(`${backupResourceName}_`)) { const tableName = entry.name.replace(`${backupResourceName}_`, '').replace('.tsv', ''); const tsvPath = `${dataDir}/${entry.name}`; const tsvContent = await Deno.readTextFile(tsvPath); // Skip header line and insert data const lines = tsvContent.split('\n'); if (lines.length > 1) { const dataLines = lines.slice(1).join('\n'); const result = await this.oneboxRef.docker.execInContainer(clickhouseService.containerId, [ 'clickhouse-client', `--user=${user}`, `--password=${password}`, '--query', `INSERT INTO ${dbName}.${tableName} FORMAT TabSeparated`, ]); // Note: Would need to pipe data via stdin for proper import if (result.exitCode !== 0) { logger.warn(`ClickHouse data import failed for ${tableName}: ${result.stderr}`); } } } } } catch { // No TSV files found } logger.success(`ClickHouse database imported: ${resource.resourceName}`); } /** * Create tar archive from directory */ private async createTarArchive(sourceDir: string, outputPath: string): Promise { const command = new Deno.Command('tar', { args: ['-cf', outputPath, '-C', sourceDir, '.'], }); const result = await command.output(); if (!result.success) { const stderr = new TextDecoder().decode(result.stderr); throw new Error(`tar create failed: ${stderr}`); } } /** * Extract tar archive to directory */ private async extractTarArchive(archivePath: string, outputDir: string): Promise { const command = new Deno.Command('tar', { args: ['-xf', archivePath, '-C', outputDir], }); const result = await command.output(); if (!result.success) { const stderr = new TextDecoder().decode(result.stderr); throw new Error(`tar extract failed: ${stderr}`); } } /** * Encrypt a file using AES-256-GCM */ private async encryptFile(inputPath: string, outputPath: string, password: string): Promise { const data = await Deno.readFile(inputPath); // Generate salt and derive key const salt = crypto.getRandomValues(new Uint8Array(SALT_LENGTH)); const key = await this.deriveKey(password, salt); // Generate IV const iv = crypto.getRandomValues(new Uint8Array(IV_LENGTH)); // Encrypt const ciphertext = await crypto.subtle.encrypt({ name: ENCRYPTION_ALGORITHM, iv }, key, data); // Combine: salt (32) + iv (12) + ciphertext const combined = new Uint8Array(salt.length + iv.length + ciphertext.byteLength); combined.set(salt, 0); combined.set(iv, salt.length); combined.set(new Uint8Array(ciphertext), salt.length + iv.length); await Deno.writeFile(outputPath, combined); } /** * Decrypt a file using AES-256-GCM */ private async decryptFile(inputPath: string, outputPath: string, password: string): Promise { const combined = await Deno.readFile(inputPath); // Extract salt, iv, and ciphertext const salt = combined.slice(0, SALT_LENGTH); const iv = combined.slice(SALT_LENGTH, SALT_LENGTH + IV_LENGTH); const ciphertext = combined.slice(SALT_LENGTH + IV_LENGTH); // Derive key const key = await this.deriveKey(password, salt); // Decrypt try { const decrypted = await crypto.subtle.decrypt({ name: ENCRYPTION_ALGORITHM, iv }, key, ciphertext); await Deno.writeFile(outputPath, new Uint8Array(decrypted)); } catch { throw new Error('Decryption failed. Invalid backup password or corrupted file.'); } } /** * Derive encryption key from password using PBKDF2 */ private async deriveKey(password: string, salt: Uint8Array): Promise { const encoder = new TextEncoder(); const passwordBytes = encoder.encode(password); const baseKey = await crypto.subtle.importKey('raw', passwordBytes, 'PBKDF2', false, ['deriveKey']); return await crypto.subtle.deriveKey( { name: 'PBKDF2', salt, iterations: PBKDF2_ITERATIONS, hash: 'SHA-256', }, baseKey, { name: ENCRYPTION_ALGORITHM, length: KEY_LENGTH }, false, ['encrypt', 'decrypt'] ); } /** * Compute SHA-256 checksum */ private async computeChecksum(data: Uint8Array): Promise { const hashBuffer = await crypto.subtle.digest('SHA-256', data); const hashArray = new Uint8Array(hashBuffer); return 'sha256:' + Array.from(hashArray).map((b) => b.toString(16).padStart(2, '0')).join(''); } }