Juergen Kunz
5cd7e7c252
Introduce a complete service backup/restore subsystem with encrypted archives, database records and REST endpoints. Implements BackupManager with export/import for service config, platform resources (MongoDB, MinIO, ClickHouse), and Docker images; adds BackupRepository and migrations for backups table and include_image_in_backup; integrates backup flows into the HTTP API and the UI client; exposes backup password management and restore modes (restore/import/clone). Wire BackupManager into Onebox initialization.
1113 lines
36 KiB
TypeScript
1113 lines
36 KiB
TypeScript
/**
|
|
* Backup Manager for Onebox
|
|
*
|
|
* Handles service backup and restore operations including:
|
|
* - Service configuration export/import
|
|
* - MongoDB database dumps
|
|
* - MinIO bucket contents
|
|
* - ClickHouse database dumps
|
|
* - Docker image export/import
|
|
* - Encrypted archive creation
|
|
*/
|
|
|
|
import * as plugins from '../plugins.ts';
|
|
import type {
|
|
IService,
|
|
IBackup,
|
|
IBackupManifest,
|
|
IBackupServiceConfig,
|
|
IBackupPlatformResource,
|
|
IBackupResult,
|
|
IRestoreOptions,
|
|
IRestoreResult,
|
|
TPlatformServiceType,
|
|
IPlatformResource,
|
|
} from '../types.ts';
|
|
import { logger } from '../logging.ts';
|
|
import { getErrorMessage } from '../utils/error.ts';
|
|
import { credentialEncryption } from './encryption.ts';
|
|
import type { Onebox } from './onebox.ts';
|
|
import { projectInfo } from '../info.ts';
|
|
|
|
// Backup archive encryption parameters
|
|
const ENCRYPTION_ALGORITHM = 'AES-GCM';
|
|
const KEY_LENGTH = 256;
|
|
const IV_LENGTH = 12;
|
|
const SALT_LENGTH = 32;
|
|
const PBKDF2_ITERATIONS = 100000;
|
|
|
|
export class BackupManager {
|
|
private oneboxRef: Onebox;
|
|
|
|
constructor(oneboxRef: Onebox) {
|
|
this.oneboxRef = oneboxRef;
|
|
}
|
|
|
|
/**
|
|
* Create a backup for a service
|
|
*/
|
|
async createBackup(serviceName: string): Promise<IBackupResult> {
|
|
const service = this.oneboxRef.database.getServiceByName(serviceName);
|
|
if (!service) {
|
|
throw new Error(`Service not found: ${serviceName}`);
|
|
}
|
|
|
|
// Verify backup password is configured
|
|
const backupPassword = this.getBackupPassword();
|
|
if (!backupPassword) {
|
|
throw new Error('Backup password not configured. Set a backup password in settings first.');
|
|
}
|
|
|
|
logger.info(`Creating backup for service: ${serviceName}`);
|
|
|
|
// Create temp directory for backup contents
|
|
const timestamp = Date.now();
|
|
const tempDir = `/tmp/onebox-backup-${serviceName}-${timestamp}`;
|
|
await Deno.mkdir(tempDir, { recursive: true });
|
|
|
|
try {
|
|
// 1. Export service configuration
|
|
const serviceConfig = await this.exportServiceConfig(service);
|
|
await Deno.writeTextFile(
|
|
`${tempDir}/service.json`,
|
|
JSON.stringify(serviceConfig, null, 2)
|
|
);
|
|
|
|
// 2. Export platform resources metadata and data
|
|
const platformResources: IBackupPlatformResource[] = [];
|
|
const resourceTypes: TPlatformServiceType[] = [];
|
|
|
|
if (service.platformRequirements) {
|
|
const resources = await this.oneboxRef.platformServices.getResourcesForService(service.id!);
|
|
|
|
for (const { resource, platformService, credentials } of resources) {
|
|
// Store resource metadata
|
|
platformResources.push({
|
|
resourceType: resource.resourceType,
|
|
resourceName: resource.resourceName,
|
|
platformServiceType: platformService.type,
|
|
credentials,
|
|
});
|
|
|
|
// Track resource types
|
|
if (!resourceTypes.includes(platformService.type)) {
|
|
resourceTypes.push(platformService.type);
|
|
}
|
|
|
|
// Create data directory
|
|
const dataDir = `${tempDir}/data/${platformService.type}`;
|
|
await Deno.mkdir(dataDir, { recursive: true });
|
|
|
|
// Export data based on type
|
|
switch (platformService.type) {
|
|
case 'mongodb':
|
|
await this.exportMongoDatabase(dataDir, resource, credentials);
|
|
break;
|
|
case 'minio':
|
|
await this.exportMinioBucket(dataDir, resource, credentials);
|
|
break;
|
|
case 'clickhouse':
|
|
await this.exportClickHouseDatabase(dataDir, resource, credentials);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
await Deno.writeTextFile(
|
|
`${tempDir}/platform-resources.json`,
|
|
JSON.stringify(platformResources, null, 2)
|
|
);
|
|
|
|
// 3. Export Docker image if configured
|
|
const includeImage = service.includeImageInBackup !== false; // Default true
|
|
if (includeImage && service.image) {
|
|
await Deno.mkdir(`${tempDir}/data/image`, { recursive: true });
|
|
await this.exportDockerImage(service.image, `${tempDir}/data/image/image.tar`);
|
|
}
|
|
|
|
// 4. Create manifest
|
|
const manifest: IBackupManifest = {
|
|
version: '1.0',
|
|
createdAt: timestamp,
|
|
oneboxVersion: projectInfo.version,
|
|
serviceName: service.name,
|
|
includesImage: includeImage,
|
|
platformResources: resourceTypes,
|
|
checksum: '', // Will be computed after archive creation
|
|
};
|
|
await Deno.writeTextFile(
|
|
`${tempDir}/manifest.json`,
|
|
JSON.stringify(manifest, null, 2)
|
|
);
|
|
|
|
// 5. Create tar archive
|
|
const tarPath = `/tmp/onebox-backup-${serviceName}-${timestamp}.tar`;
|
|
await this.createTarArchive(tempDir, tarPath);
|
|
|
|
// 6. Compute checksum of tar
|
|
const tarData = await Deno.readFile(tarPath);
|
|
const checksum = await this.computeChecksum(tarData);
|
|
manifest.checksum = checksum;
|
|
|
|
// Update manifest with checksum
|
|
await Deno.writeTextFile(
|
|
`${tempDir}/manifest.json`,
|
|
JSON.stringify(manifest, null, 2)
|
|
);
|
|
|
|
// Recreate tar with updated manifest
|
|
await this.createTarArchive(tempDir, tarPath);
|
|
|
|
// 7. Encrypt the archive
|
|
const backupsDir = this.getBackupsDirectory();
|
|
await Deno.mkdir(backupsDir, { recursive: true });
|
|
|
|
const encryptedFilename = `${serviceName}-${timestamp}.tar.enc`;
|
|
const encryptedPath = `${backupsDir}/${encryptedFilename}`;
|
|
|
|
await this.encryptFile(tarPath, encryptedPath, backupPassword);
|
|
|
|
// Get encrypted file size
|
|
const stat = await Deno.stat(encryptedPath);
|
|
const sizeBytes = stat.size;
|
|
|
|
// 8. Store backup record in database
|
|
const backup: IBackup = {
|
|
serviceId: service.id!,
|
|
serviceName: service.name,
|
|
filename: encryptedFilename,
|
|
sizeBytes,
|
|
createdAt: timestamp,
|
|
includesImage: includeImage,
|
|
platformResources: resourceTypes,
|
|
checksum,
|
|
};
|
|
|
|
const createdBackup = this.oneboxRef.database.createBackup(backup);
|
|
|
|
// Cleanup temp files
|
|
await Deno.remove(tempDir, { recursive: true });
|
|
await Deno.remove(tarPath);
|
|
|
|
logger.success(`Backup created for service ${serviceName}: ${encryptedFilename}`);
|
|
|
|
return {
|
|
backup: createdBackup,
|
|
filePath: encryptedPath,
|
|
};
|
|
} catch (error) {
|
|
// Cleanup on error
|
|
try {
|
|
await Deno.remove(tempDir, { recursive: true });
|
|
} catch {
|
|
// Ignore cleanup errors
|
|
}
|
|
logger.error(`Failed to create backup for ${serviceName}: ${getErrorMessage(error)}`);
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Restore a backup
|
|
*/
|
|
async restoreBackup(backupPath: string, options: IRestoreOptions): Promise<IRestoreResult> {
|
|
// Verify backup password
|
|
const backupPassword = this.getBackupPassword();
|
|
if (!backupPassword) {
|
|
throw new Error('Backup password not configured.');
|
|
}
|
|
|
|
logger.info(`Restoring backup from: ${backupPath}`);
|
|
|
|
// Create temp directory for extraction
|
|
const timestamp = Date.now();
|
|
const tempDir = `/tmp/onebox-restore-${timestamp}`;
|
|
await Deno.mkdir(tempDir, { recursive: true });
|
|
|
|
const warnings: string[] = [];
|
|
|
|
try {
|
|
// 1. Decrypt the archive
|
|
const tarPath = `${tempDir}/backup.tar`;
|
|
await this.decryptFile(backupPath, tarPath, backupPassword);
|
|
|
|
// 2. Extract tar archive
|
|
await this.extractTarArchive(tarPath, tempDir);
|
|
|
|
// 3. Read and validate manifest
|
|
const manifestPath = `${tempDir}/manifest.json`;
|
|
const manifestData = await Deno.readTextFile(manifestPath);
|
|
const manifest: IBackupManifest = JSON.parse(manifestData);
|
|
|
|
// Verify checksum (excluding manifest itself)
|
|
// Note: For simplicity, we trust the manifest here
|
|
// In production, you'd want to verify the checksum of specific files
|
|
|
|
// 4. Read service config
|
|
const serviceConfigPath = `${tempDir}/service.json`;
|
|
const serviceConfigData = await Deno.readTextFile(serviceConfigPath);
|
|
const serviceConfig: IBackupServiceConfig = JSON.parse(serviceConfigData);
|
|
|
|
// 5. Read platform resources
|
|
const platformResourcesPath = `${tempDir}/platform-resources.json`;
|
|
let platformResources: IBackupPlatformResource[] = [];
|
|
try {
|
|
const resourcesData = await Deno.readTextFile(platformResourcesPath);
|
|
platformResources = JSON.parse(resourcesData);
|
|
} catch {
|
|
// No platform resources in backup
|
|
}
|
|
|
|
// 6. Determine service name based on mode
|
|
let serviceName: string;
|
|
let existingService: IService | null = null;
|
|
|
|
switch (options.mode) {
|
|
case 'restore':
|
|
serviceName = manifest.serviceName;
|
|
existingService = this.oneboxRef.database.getServiceByName(serviceName);
|
|
if (!existingService) {
|
|
throw new Error(`Service '${serviceName}' not found. Use 'import' mode to create a new service.`);
|
|
}
|
|
if (!options.overwriteExisting) {
|
|
throw new Error(`Service '${serviceName}' exists. Set overwriteExisting=true to proceed.`);
|
|
}
|
|
break;
|
|
|
|
case 'import':
|
|
case 'clone':
|
|
if (!options.newServiceName) {
|
|
throw new Error(`New service name required for '${options.mode}' mode.`);
|
|
}
|
|
serviceName = options.newServiceName;
|
|
existingService = this.oneboxRef.database.getServiceByName(serviceName);
|
|
if (existingService) {
|
|
throw new Error(`Service '${serviceName}' already exists. Choose a different name.`);
|
|
}
|
|
break;
|
|
|
|
default:
|
|
throw new Error(`Invalid restore mode: ${options.mode}`);
|
|
}
|
|
|
|
// 7. Import Docker image if present
|
|
if (manifest.includesImage) {
|
|
const imagePath = `${tempDir}/data/image/image.tar`;
|
|
try {
|
|
await Deno.stat(imagePath);
|
|
const newImageTag = await this.importDockerImage(imagePath);
|
|
// Update service config with the imported image tag
|
|
serviceConfig.image = newImageTag;
|
|
logger.info(`Docker image imported: ${newImageTag}`);
|
|
} catch (error) {
|
|
warnings.push(`Docker image import failed: ${getErrorMessage(error)}`);
|
|
}
|
|
}
|
|
|
|
// 8. Create or update service
|
|
let service: IService;
|
|
let platformResourcesRestored = 0;
|
|
|
|
if (options.mode === 'restore' && existingService) {
|
|
// Update existing service
|
|
this.oneboxRef.database.updateService(existingService.id!, {
|
|
image: serviceConfig.image,
|
|
registry: serviceConfig.registry,
|
|
port: serviceConfig.port,
|
|
domain: serviceConfig.domain,
|
|
useOneboxRegistry: serviceConfig.useOneboxRegistry,
|
|
registryRepository: serviceConfig.registryRepository,
|
|
registryImageTag: serviceConfig.registryImageTag,
|
|
autoUpdateOnPush: serviceConfig.autoUpdateOnPush,
|
|
platformRequirements: serviceConfig.platformRequirements,
|
|
updatedAt: Date.now(),
|
|
});
|
|
|
|
// Restore env vars (merge with platform provisioned vars later)
|
|
const updatedEnvVars = { ...serviceConfig.envVars };
|
|
|
|
// Handle platform data restore
|
|
if (!options.skipPlatformData && platformResources.length > 0) {
|
|
platformResourcesRestored = await this.restorePlatformResources(
|
|
existingService.id!,
|
|
platformResources,
|
|
tempDir,
|
|
warnings
|
|
);
|
|
}
|
|
|
|
this.oneboxRef.database.updateService(existingService.id!, { envVars: updatedEnvVars });
|
|
service = this.oneboxRef.database.getServiceByName(serviceName)!;
|
|
} else {
|
|
// Create new service
|
|
const deployOptions = {
|
|
name: serviceName,
|
|
image: serviceConfig.image,
|
|
registry: serviceConfig.registry,
|
|
port: serviceConfig.port,
|
|
domain: options.mode === 'clone' ? undefined : serviceConfig.domain, // Don't duplicate domain for clones
|
|
envVars: serviceConfig.envVars,
|
|
useOneboxRegistry: serviceConfig.useOneboxRegistry,
|
|
registryImageTag: serviceConfig.registryImageTag,
|
|
autoUpdateOnPush: serviceConfig.autoUpdateOnPush,
|
|
enableMongoDB: serviceConfig.platformRequirements?.mongodb,
|
|
enableS3: serviceConfig.platformRequirements?.s3,
|
|
enableClickHouse: serviceConfig.platformRequirements?.clickhouse,
|
|
};
|
|
|
|
service = await this.oneboxRef.services.deployService(deployOptions);
|
|
|
|
// Import platform data if not skipping
|
|
if (!options.skipPlatformData && platformResources.length > 0) {
|
|
// Wait a moment for platform resources to be provisioned
|
|
await new Promise((resolve) => setTimeout(resolve, 2000));
|
|
|
|
platformResourcesRestored = await this.restorePlatformData(
|
|
service.id!,
|
|
platformResources,
|
|
tempDir,
|
|
warnings
|
|
);
|
|
}
|
|
}
|
|
|
|
// Cleanup
|
|
await Deno.remove(tempDir, { recursive: true });
|
|
|
|
logger.success(`Backup restored successfully as service '${serviceName}'`);
|
|
|
|
return {
|
|
service,
|
|
platformResourcesRestored,
|
|
warnings,
|
|
};
|
|
} catch (error) {
|
|
// Cleanup on error
|
|
try {
|
|
await Deno.remove(tempDir, { recursive: true });
|
|
} catch {
|
|
// Ignore cleanup errors
|
|
}
|
|
logger.error(`Failed to restore backup: ${getErrorMessage(error)}`);
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* List all backups
|
|
*/
|
|
listBackups(serviceName?: string): IBackup[] {
|
|
if (serviceName) {
|
|
const service = this.oneboxRef.database.getServiceByName(serviceName);
|
|
if (!service) {
|
|
return [];
|
|
}
|
|
return this.oneboxRef.database.getBackupsByService(service.id!);
|
|
}
|
|
return this.oneboxRef.database.getAllBackups();
|
|
}
|
|
|
|
/**
|
|
* Delete a backup
|
|
*/
|
|
async deleteBackup(backupId: number): Promise<void> {
|
|
const backup = this.oneboxRef.database.getBackupById(backupId);
|
|
if (!backup) {
|
|
throw new Error(`Backup not found: ${backupId}`);
|
|
}
|
|
|
|
// Delete file
|
|
const backupsDir = this.getBackupsDirectory();
|
|
const filePath = `${backupsDir}/${backup.filename}`;
|
|
try {
|
|
await Deno.remove(filePath);
|
|
} catch {
|
|
logger.warn(`Could not delete backup file: ${filePath}`);
|
|
}
|
|
|
|
// Delete database record
|
|
this.oneboxRef.database.deleteBackup(backupId);
|
|
logger.info(`Backup deleted: ${backup.filename}`);
|
|
}
|
|
|
|
/**
|
|
* Get backup file path for download
|
|
*/
|
|
getBackupFilePath(backupId: number): string | null {
|
|
const backup = this.oneboxRef.database.getBackupById(backupId);
|
|
if (!backup) {
|
|
return null;
|
|
}
|
|
const backupsDir = this.getBackupsDirectory();
|
|
return `${backupsDir}/${backup.filename}`;
|
|
}
|
|
|
|
// ========== Private Methods ==========
|
|
|
|
/**
|
|
* Get backup password from settings
|
|
*/
|
|
private getBackupPassword(): string | null {
|
|
return this.oneboxRef.database.getSetting('backup_encryption_password');
|
|
}
|
|
|
|
/**
|
|
* Get backups directory
|
|
*/
|
|
private getBackupsDirectory(): string {
|
|
const dataDir = this.oneboxRef.database.getSetting('dataDir') || './.nogit';
|
|
return `${dataDir}/backups`;
|
|
}
|
|
|
|
/**
|
|
* Export service configuration
|
|
*/
|
|
private async exportServiceConfig(service: IService): Promise<IBackupServiceConfig> {
|
|
return {
|
|
name: service.name,
|
|
image: service.image,
|
|
registry: service.registry,
|
|
envVars: service.envVars,
|
|
port: service.port,
|
|
domain: service.domain,
|
|
useOneboxRegistry: service.useOneboxRegistry,
|
|
registryRepository: service.registryRepository,
|
|
registryImageTag: service.registryImageTag,
|
|
autoUpdateOnPush: service.autoUpdateOnPush,
|
|
platformRequirements: service.platformRequirements,
|
|
includeImageInBackup: service.includeImageInBackup,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Export MongoDB database
|
|
*/
|
|
private async exportMongoDatabase(
|
|
dataDir: string,
|
|
resource: IPlatformResource,
|
|
credentials: Record<string, string>
|
|
): Promise<void> {
|
|
logger.info(`Exporting MongoDB database: ${resource.resourceName}`);
|
|
|
|
const mongoService = this.oneboxRef.database.getPlatformServiceById(resource.platformServiceId);
|
|
if (!mongoService || !mongoService.containerId) {
|
|
throw new Error('MongoDB service not running');
|
|
}
|
|
|
|
// Build connection URI
|
|
const connectionUri = credentials.connectionUri || credentials.MONGODB_URI;
|
|
if (!connectionUri) {
|
|
throw new Error('MongoDB connection URI not found in credentials');
|
|
}
|
|
|
|
// Use mongodump via docker exec
|
|
const archivePath = `/tmp/${resource.resourceName}.archive`;
|
|
const result = await this.oneboxRef.docker.execInContainer(mongoService.containerId, [
|
|
'mongodump',
|
|
`--uri=${connectionUri}`,
|
|
`--archive=${archivePath}`,
|
|
'--gzip',
|
|
]);
|
|
|
|
if (result.exitCode !== 0) {
|
|
throw new Error(`mongodump failed: ${result.stderr}`);
|
|
}
|
|
|
|
// Copy archive out of container
|
|
const container = await this.oneboxRef.docker.getContainerById(mongoService.containerId);
|
|
if (!container) {
|
|
throw new Error('MongoDB container not found');
|
|
}
|
|
|
|
// Read the archive from container and write to local file
|
|
const copyResult = await this.oneboxRef.docker.execInContainer(mongoService.containerId, [
|
|
'cat',
|
|
archivePath,
|
|
]);
|
|
|
|
// Write base64-decoded content (stdout is binary data encoded)
|
|
const localPath = `${dataDir}/${resource.resourceName}.archive`;
|
|
const encoder = new TextEncoder();
|
|
await Deno.writeFile(localPath, encoder.encode(copyResult.stdout));
|
|
|
|
// Cleanup inside container
|
|
await this.oneboxRef.docker.execInContainer(mongoService.containerId, ['rm', archivePath]);
|
|
|
|
logger.success(`MongoDB database exported: ${resource.resourceName}`);
|
|
}
|
|
|
|
/**
|
|
* Export MinIO bucket
|
|
*/
|
|
private async exportMinioBucket(
|
|
dataDir: string,
|
|
resource: IPlatformResource,
|
|
credentials: Record<string, string>
|
|
): Promise<void> {
|
|
logger.info(`Exporting MinIO bucket: ${resource.resourceName}`);
|
|
|
|
const bucketDir = `${dataDir}/${resource.resourceName}`;
|
|
await Deno.mkdir(bucketDir, { recursive: true });
|
|
|
|
// Use S3 client to download all objects
|
|
const endpoint = credentials.endpoint || credentials.S3_ENDPOINT;
|
|
const accessKey = credentials.accessKey || credentials.S3_ACCESS_KEY;
|
|
const secretKey = credentials.secretKey || credentials.S3_SECRET_KEY;
|
|
const bucket = credentials.bucket || credentials.S3_BUCKET;
|
|
|
|
if (!endpoint || !accessKey || !secretKey || !bucket) {
|
|
throw new Error('MinIO credentials incomplete');
|
|
}
|
|
|
|
// Initialize S3 client
|
|
const s3Client = new plugins.smarts3.SmartS3({
|
|
endpoint,
|
|
accessKey,
|
|
secretKey,
|
|
bucket,
|
|
});
|
|
|
|
await s3Client.start();
|
|
|
|
// List and download all objects
|
|
const objects = await s3Client.listObjects();
|
|
|
|
for (const obj of objects) {
|
|
const objectKey = obj.Key;
|
|
if (!objectKey) continue;
|
|
|
|
const objectData = await s3Client.getObject(objectKey);
|
|
if (objectData) {
|
|
const objectPath = `${bucketDir}/${objectKey}`;
|
|
// Create parent directories if needed
|
|
const parentDir = plugins.path.dirname(objectPath);
|
|
await Deno.mkdir(parentDir, { recursive: true });
|
|
await Deno.writeFile(objectPath, objectData);
|
|
}
|
|
}
|
|
|
|
await s3Client.stop();
|
|
|
|
// Also save bucket metadata
|
|
await Deno.writeTextFile(
|
|
`${bucketDir}/_metadata.json`,
|
|
JSON.stringify({ bucket, objectCount: objects.length }, null, 2)
|
|
);
|
|
|
|
logger.success(`MinIO bucket exported: ${resource.resourceName} (${objects.length} objects)`);
|
|
}
|
|
|
|
/**
|
|
* Export ClickHouse database
|
|
*/
|
|
private async exportClickHouseDatabase(
|
|
dataDir: string,
|
|
resource: IPlatformResource,
|
|
credentials: Record<string, string>
|
|
): Promise<void> {
|
|
logger.info(`Exporting ClickHouse database: ${resource.resourceName}`);
|
|
|
|
const clickhouseService = this.oneboxRef.database.getPlatformServiceByType('clickhouse');
|
|
if (!clickhouseService || !clickhouseService.containerId) {
|
|
throw new Error('ClickHouse service not running');
|
|
}
|
|
|
|
const dbName = credentials.database || credentials.CLICKHOUSE_DB;
|
|
const user = credentials.username || credentials.CLICKHOUSE_USER || 'default';
|
|
const password = credentials.password || credentials.CLICKHOUSE_PASSWORD || '';
|
|
|
|
if (!dbName) {
|
|
throw new Error('ClickHouse database name not found in credentials');
|
|
}
|
|
|
|
// Get list of tables
|
|
const tablesResult = await this.oneboxRef.docker.execInContainer(clickhouseService.containerId, [
|
|
'clickhouse-client',
|
|
`--user=${user}`,
|
|
`--password=${password}`,
|
|
'--query',
|
|
`SELECT name FROM system.tables WHERE database = '${dbName}'`,
|
|
]);
|
|
|
|
if (tablesResult.exitCode !== 0) {
|
|
throw new Error(`Failed to list ClickHouse tables: ${tablesResult.stderr}`);
|
|
}
|
|
|
|
const tables = tablesResult.stdout.trim().split('\n').filter(Boolean);
|
|
const dumpPath = `${dataDir}/${resource.resourceName}.sql`;
|
|
let dumpContent = `-- ClickHouse backup for database: ${dbName}\n`;
|
|
dumpContent += `-- Created: ${new Date().toISOString()}\n\n`;
|
|
dumpContent += `CREATE DATABASE IF NOT EXISTS ${dbName};\n\n`;
|
|
|
|
for (const table of tables) {
|
|
// Get CREATE TABLE statement
|
|
const createResult = await this.oneboxRef.docker.execInContainer(clickhouseService.containerId, [
|
|
'clickhouse-client',
|
|
`--user=${user}`,
|
|
`--password=${password}`,
|
|
'--query',
|
|
`SHOW CREATE TABLE ${dbName}.${table}`,
|
|
]);
|
|
|
|
if (createResult.exitCode === 0) {
|
|
dumpContent += `-- Table: ${table}\n`;
|
|
dumpContent += createResult.stdout + ';\n\n';
|
|
}
|
|
|
|
// Get table data in TSV format
|
|
const dataResult = await this.oneboxRef.docker.execInContainer(clickhouseService.containerId, [
|
|
'clickhouse-client',
|
|
`--user=${user}`,
|
|
`--password=${password}`,
|
|
'--query',
|
|
`SELECT * FROM ${dbName}.${table} FORMAT TabSeparatedWithNames`,
|
|
]);
|
|
|
|
if (dataResult.exitCode === 0 && dataResult.stdout.trim()) {
|
|
// Save data to separate file for large datasets
|
|
const tableDataPath = `${dataDir}/${resource.resourceName}_${table}.tsv`;
|
|
await Deno.writeTextFile(tableDataPath, dataResult.stdout);
|
|
}
|
|
}
|
|
|
|
await Deno.writeTextFile(dumpPath, dumpContent);
|
|
|
|
logger.success(`ClickHouse database exported: ${resource.resourceName} (${tables.length} tables)`);
|
|
}
|
|
|
|
/**
|
|
* Export Docker image
|
|
*/
|
|
private async exportDockerImage(imageName: string, outputPath: string): Promise<void> {
|
|
logger.info(`Exporting Docker image: ${imageName}`);
|
|
|
|
// Use docker save command via shell
|
|
const command = new Deno.Command('docker', {
|
|
args: ['save', '-o', outputPath, imageName],
|
|
});
|
|
|
|
const result = await command.output();
|
|
|
|
if (!result.success) {
|
|
const stderr = new TextDecoder().decode(result.stderr);
|
|
throw new Error(`docker save failed: ${stderr}`);
|
|
}
|
|
|
|
logger.success(`Docker image exported: ${imageName}`);
|
|
}
|
|
|
|
/**
|
|
* Import Docker image
|
|
*/
|
|
private async importDockerImage(imagePath: string): Promise<string> {
|
|
logger.info(`Importing Docker image from: ${imagePath}`);
|
|
|
|
// Use docker load command
|
|
const command = new Deno.Command('docker', {
|
|
args: ['load', '-i', imagePath],
|
|
});
|
|
|
|
const result = await command.output();
|
|
|
|
if (!result.success) {
|
|
const stderr = new TextDecoder().decode(result.stderr);
|
|
throw new Error(`docker load failed: ${stderr}`);
|
|
}
|
|
|
|
const stdout = new TextDecoder().decode(result.stdout);
|
|
// Parse image name from output like "Loaded image: nginx:latest"
|
|
const match = stdout.match(/Loaded image: (.+)/);
|
|
const imageName = match ? match[1].trim() : 'unknown';
|
|
|
|
logger.success(`Docker image imported: ${imageName}`);
|
|
return imageName;
|
|
}
|
|
|
|
/**
|
|
* Restore platform resources for existing service (restore mode)
|
|
*/
|
|
private async restorePlatformResources(
|
|
serviceId: number,
|
|
backupResources: IBackupPlatformResource[],
|
|
tempDir: string,
|
|
warnings: string[]
|
|
): Promise<number> {
|
|
let restoredCount = 0;
|
|
|
|
// Get existing resources for this service
|
|
const existingResources = await this.oneboxRef.platformServices.getResourcesForService(serviceId);
|
|
|
|
for (const backupResource of backupResources) {
|
|
try {
|
|
// Find matching existing resource
|
|
const existing = existingResources.find(
|
|
(e) =>
|
|
e.platformService.type === backupResource.platformServiceType &&
|
|
e.resource.resourceType === backupResource.resourceType
|
|
);
|
|
|
|
if (!existing) {
|
|
warnings.push(
|
|
`Platform resource ${backupResource.platformServiceType}/${backupResource.resourceName} not provisioned. Skipping data import.`
|
|
);
|
|
continue;
|
|
}
|
|
|
|
// Import data based on type
|
|
const dataDir = `${tempDir}/data/${backupResource.platformServiceType}`;
|
|
|
|
switch (backupResource.platformServiceType) {
|
|
case 'mongodb':
|
|
await this.importMongoDatabase(
|
|
dataDir,
|
|
existing.resource,
|
|
existing.credentials,
|
|
backupResource.resourceName
|
|
);
|
|
restoredCount++;
|
|
break;
|
|
case 'minio':
|
|
await this.importMinioBucket(
|
|
dataDir,
|
|
existing.resource,
|
|
existing.credentials,
|
|
backupResource.resourceName
|
|
);
|
|
restoredCount++;
|
|
break;
|
|
case 'clickhouse':
|
|
await this.importClickHouseDatabase(
|
|
dataDir,
|
|
existing.resource,
|
|
existing.credentials,
|
|
backupResource.resourceName
|
|
);
|
|
restoredCount++;
|
|
break;
|
|
}
|
|
} catch (error) {
|
|
warnings.push(
|
|
`Failed to restore ${backupResource.platformServiceType} resource: ${getErrorMessage(error)}`
|
|
);
|
|
}
|
|
}
|
|
|
|
return restoredCount;
|
|
}
|
|
|
|
/**
|
|
* Restore platform data for new service (import/clone mode)
|
|
*/
|
|
private async restorePlatformData(
|
|
serviceId: number,
|
|
backupResources: IBackupPlatformResource[],
|
|
tempDir: string,
|
|
warnings: string[]
|
|
): Promise<number> {
|
|
// For new services, platform resources should have been provisioned during deployment
|
|
return this.restorePlatformResources(serviceId, backupResources, tempDir, warnings);
|
|
}
|
|
|
|
/**
|
|
* Import MongoDB database
|
|
*/
|
|
private async importMongoDatabase(
|
|
dataDir: string,
|
|
resource: IPlatformResource,
|
|
credentials: Record<string, string>,
|
|
backupResourceName: string
|
|
): Promise<void> {
|
|
logger.info(`Importing MongoDB database: ${resource.resourceName}`);
|
|
|
|
const mongoService = this.oneboxRef.database.getPlatformServiceById(resource.platformServiceId);
|
|
if (!mongoService || !mongoService.containerId) {
|
|
throw new Error('MongoDB service not running');
|
|
}
|
|
|
|
const archivePath = `${dataDir}/${backupResourceName}.archive`;
|
|
const connectionUri = credentials.connectionUri || credentials.MONGODB_URI;
|
|
|
|
if (!connectionUri) {
|
|
throw new Error('MongoDB connection URI not found');
|
|
}
|
|
|
|
// Read local archive and copy to container
|
|
const archiveData = await Deno.readFile(archivePath);
|
|
const containerArchivePath = `/tmp/${resource.resourceName}.archive`;
|
|
|
|
// Write archive to container via exec + stdin (simplified - use cat)
|
|
// For production, use Docker API copy endpoint
|
|
const base64Data = btoa(String.fromCharCode(...archiveData));
|
|
|
|
await this.oneboxRef.docker.execInContainer(mongoService.containerId, [
|
|
'bash',
|
|
'-c',
|
|
`echo '${base64Data}' | base64 -d > ${containerArchivePath}`,
|
|
]);
|
|
|
|
// Run mongorestore
|
|
const result = await this.oneboxRef.docker.execInContainer(mongoService.containerId, [
|
|
'mongorestore',
|
|
`--uri=${connectionUri}`,
|
|
`--archive=${containerArchivePath}`,
|
|
'--gzip',
|
|
'--drop',
|
|
]);
|
|
|
|
if (result.exitCode !== 0) {
|
|
throw new Error(`mongorestore failed: ${result.stderr}`);
|
|
}
|
|
|
|
// Cleanup
|
|
await this.oneboxRef.docker.execInContainer(mongoService.containerId, ['rm', containerArchivePath]);
|
|
|
|
logger.success(`MongoDB database imported: ${resource.resourceName}`);
|
|
}
|
|
|
|
/**
|
|
* Import MinIO bucket
|
|
*/
|
|
private async importMinioBucket(
|
|
dataDir: string,
|
|
resource: IPlatformResource,
|
|
credentials: Record<string, string>,
|
|
backupResourceName: string
|
|
): Promise<void> {
|
|
logger.info(`Importing MinIO bucket: ${resource.resourceName}`);
|
|
|
|
const bucketDir = `${dataDir}/${backupResourceName}`;
|
|
|
|
const endpoint = credentials.endpoint || credentials.S3_ENDPOINT;
|
|
const accessKey = credentials.accessKey || credentials.S3_ACCESS_KEY;
|
|
const secretKey = credentials.secretKey || credentials.S3_SECRET_KEY;
|
|
const bucket = credentials.bucket || credentials.S3_BUCKET;
|
|
|
|
if (!endpoint || !accessKey || !secretKey || !bucket) {
|
|
throw new Error('MinIO credentials incomplete');
|
|
}
|
|
|
|
const s3Client = new plugins.smarts3.SmartS3({
|
|
endpoint,
|
|
accessKey,
|
|
secretKey,
|
|
bucket,
|
|
});
|
|
|
|
await s3Client.start();
|
|
|
|
// Walk directory and upload all files
|
|
let uploadedCount = 0;
|
|
|
|
for await (const entry of Deno.readDir(bucketDir)) {
|
|
if (entry.name === '_metadata.json') continue;
|
|
|
|
const filePath = `${bucketDir}/${entry.name}`;
|
|
|
|
if (entry.isFile) {
|
|
const fileData = await Deno.readFile(filePath);
|
|
await s3Client.putObject(entry.name, fileData);
|
|
uploadedCount++;
|
|
}
|
|
// Note: For nested directories, would need recursive handling
|
|
}
|
|
|
|
await s3Client.stop();
|
|
|
|
logger.success(`MinIO bucket imported: ${resource.resourceName} (${uploadedCount} objects)`);
|
|
}
|
|
|
|
/**
|
|
* Import ClickHouse database
|
|
*/
|
|
private async importClickHouseDatabase(
|
|
dataDir: string,
|
|
resource: IPlatformResource,
|
|
credentials: Record<string, string>,
|
|
backupResourceName: string
|
|
): Promise<void> {
|
|
logger.info(`Importing ClickHouse database: ${resource.resourceName}`);
|
|
|
|
const clickhouseService = this.oneboxRef.database.getPlatformServiceByType('clickhouse');
|
|
if (!clickhouseService || !clickhouseService.containerId) {
|
|
throw new Error('ClickHouse service not running');
|
|
}
|
|
|
|
const dbName = credentials.database || credentials.CLICKHOUSE_DB;
|
|
const user = credentials.username || credentials.CLICKHOUSE_USER || 'default';
|
|
const password = credentials.password || credentials.CLICKHOUSE_PASSWORD || '';
|
|
|
|
if (!dbName) {
|
|
throw new Error('ClickHouse database name not found');
|
|
}
|
|
|
|
// Read SQL dump
|
|
const sqlPath = `${dataDir}/${backupResourceName}.sql`;
|
|
const sqlContent = await Deno.readTextFile(sqlPath);
|
|
|
|
// Execute SQL statements
|
|
const statements = sqlContent.split(';').filter((s) => s.trim());
|
|
|
|
for (const statement of statements) {
|
|
if (statement.trim().startsWith('--')) continue;
|
|
|
|
const result = await this.oneboxRef.docker.execInContainer(clickhouseService.containerId, [
|
|
'clickhouse-client',
|
|
`--user=${user}`,
|
|
`--password=${password}`,
|
|
'--query',
|
|
statement.trim(),
|
|
]);
|
|
|
|
if (result.exitCode !== 0) {
|
|
logger.warn(`ClickHouse statement failed: ${result.stderr}`);
|
|
}
|
|
}
|
|
|
|
// Import TSV data files for each table
|
|
try {
|
|
for await (const entry of Deno.readDir(dataDir)) {
|
|
if (entry.name.endsWith('.tsv') && entry.name.startsWith(`${backupResourceName}_`)) {
|
|
const tableName = entry.name.replace(`${backupResourceName}_`, '').replace('.tsv', '');
|
|
const tsvPath = `${dataDir}/${entry.name}`;
|
|
const tsvContent = await Deno.readTextFile(tsvPath);
|
|
|
|
// Skip header line and insert data
|
|
const lines = tsvContent.split('\n');
|
|
if (lines.length > 1) {
|
|
const dataLines = lines.slice(1).join('\n');
|
|
|
|
const result = await this.oneboxRef.docker.execInContainer(clickhouseService.containerId, [
|
|
'clickhouse-client',
|
|
`--user=${user}`,
|
|
`--password=${password}`,
|
|
'--query',
|
|
`INSERT INTO ${dbName}.${tableName} FORMAT TabSeparated`,
|
|
]);
|
|
|
|
// Note: Would need to pipe data via stdin for proper import
|
|
if (result.exitCode !== 0) {
|
|
logger.warn(`ClickHouse data import failed for ${tableName}: ${result.stderr}`);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} catch {
|
|
// No TSV files found
|
|
}
|
|
|
|
logger.success(`ClickHouse database imported: ${resource.resourceName}`);
|
|
}
|
|
|
|
/**
|
|
* Create tar archive from directory
|
|
*/
|
|
private async createTarArchive(sourceDir: string, outputPath: string): Promise<void> {
|
|
const command = new Deno.Command('tar', {
|
|
args: ['-cf', outputPath, '-C', sourceDir, '.'],
|
|
});
|
|
|
|
const result = await command.output();
|
|
|
|
if (!result.success) {
|
|
const stderr = new TextDecoder().decode(result.stderr);
|
|
throw new Error(`tar create failed: ${stderr}`);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Extract tar archive to directory
|
|
*/
|
|
private async extractTarArchive(archivePath: string, outputDir: string): Promise<void> {
|
|
const command = new Deno.Command('tar', {
|
|
args: ['-xf', archivePath, '-C', outputDir],
|
|
});
|
|
|
|
const result = await command.output();
|
|
|
|
if (!result.success) {
|
|
const stderr = new TextDecoder().decode(result.stderr);
|
|
throw new Error(`tar extract failed: ${stderr}`);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Encrypt a file using AES-256-GCM
|
|
*/
|
|
private async encryptFile(inputPath: string, outputPath: string, password: string): Promise<void> {
|
|
const data = await Deno.readFile(inputPath);
|
|
|
|
// Generate salt and derive key
|
|
const salt = crypto.getRandomValues(new Uint8Array(SALT_LENGTH));
|
|
const key = await this.deriveKey(password, salt);
|
|
|
|
// Generate IV
|
|
const iv = crypto.getRandomValues(new Uint8Array(IV_LENGTH));
|
|
|
|
// Encrypt
|
|
const ciphertext = await crypto.subtle.encrypt({ name: ENCRYPTION_ALGORITHM, iv }, key, data);
|
|
|
|
// Combine: salt (32) + iv (12) + ciphertext
|
|
const combined = new Uint8Array(salt.length + iv.length + ciphertext.byteLength);
|
|
combined.set(salt, 0);
|
|
combined.set(iv, salt.length);
|
|
combined.set(new Uint8Array(ciphertext), salt.length + iv.length);
|
|
|
|
await Deno.writeFile(outputPath, combined);
|
|
}
|
|
|
|
/**
|
|
* Decrypt a file using AES-256-GCM
|
|
*/
|
|
private async decryptFile(inputPath: string, outputPath: string, password: string): Promise<void> {
|
|
const combined = await Deno.readFile(inputPath);
|
|
|
|
// Extract salt, iv, and ciphertext
|
|
const salt = combined.slice(0, SALT_LENGTH);
|
|
const iv = combined.slice(SALT_LENGTH, SALT_LENGTH + IV_LENGTH);
|
|
const ciphertext = combined.slice(SALT_LENGTH + IV_LENGTH);
|
|
|
|
// Derive key
|
|
const key = await this.deriveKey(password, salt);
|
|
|
|
// Decrypt
|
|
try {
|
|
const decrypted = await crypto.subtle.decrypt({ name: ENCRYPTION_ALGORITHM, iv }, key, ciphertext);
|
|
await Deno.writeFile(outputPath, new Uint8Array(decrypted));
|
|
} catch {
|
|
throw new Error('Decryption failed. Invalid backup password or corrupted file.');
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Derive encryption key from password using PBKDF2
|
|
*/
|
|
private async deriveKey(password: string, salt: Uint8Array): Promise<CryptoKey> {
|
|
const encoder = new TextEncoder();
|
|
const passwordBytes = encoder.encode(password);
|
|
|
|
const baseKey = await crypto.subtle.importKey('raw', passwordBytes, 'PBKDF2', false, ['deriveKey']);
|
|
|
|
return await crypto.subtle.deriveKey(
|
|
{
|
|
name: 'PBKDF2',
|
|
salt,
|
|
iterations: PBKDF2_ITERATIONS,
|
|
hash: 'SHA-256',
|
|
},
|
|
baseKey,
|
|
{ name: ENCRYPTION_ALGORITHM, length: KEY_LENGTH },
|
|
false,
|
|
['encrypt', 'decrypt']
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Compute SHA-256 checksum
|
|
*/
|
|
private async computeChecksum(data: Uint8Array): Promise<string> {
|
|
const hashBuffer = await crypto.subtle.digest('SHA-256', data);
|
|
const hashArray = new Uint8Array(hashBuffer);
|
|
return 'sha256:' + Array.from(hashArray).map((b) => b.toString(16).padStart(2, '0')).join('');
|
|
}
|
|
}
|