diff --git a/changelog.md b/changelog.md index 79a5544..ec5ee9d 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,16 @@ # Changelog +## 2025-11-26 - 1.3.0 - feat(platform-services) +Add ClickHouse platform service support (provider, types, provisioning, UI and port mappings) + +- Introduce ClickHouse as a first-class platform service: added ClickHouseProvider and registered it in PlatformServicesManager +- Support provisioning ClickHouse resources for user services and storing encrypted credentials in platform_resources +- Add ClickHouse to core types (TPlatformServiceType, IPlatformRequirements, IServiceDeployOptions) and service DB handling so services can request ClickHouse +- Inject ClickHouse-related environment variables into deployed services (CLICKHOUSE_* mappings) when provisioning resources +- Expose ClickHouse default port (8123) in platform port mappings / network targets +- UI: add checkbox and description for enabling ClickHouse during service creation; form now submits enableClickHouse +- Add VS Code recommendations and launch/tasks for the UI development workflow + ## 2025-11-26 - 1.2.1 - fix(platform-services/minio) Improve MinIO provider: reuse existing data and credentials, use host-bound port for provisioning, and safer provisioning/deprovisioning diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index beae010..90451e5 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@serve.zone/onebox', - version: '1.2.1', + version: '1.3.0', description: 'Self-hosted container platform with automatic SSL and DNS - a mini Heroku for single servers' } diff --git a/ts/classes/httpserver.ts b/ts/classes/httpserver.ts index 3ea7fc7..7da0e94 100644 --- a/ts/classes/httpserver.ts +++ b/ts/classes/httpserver.ts @@ -1323,6 +1323,7 @@ export class OneboxHttpServer { postgresql: 5432, rabbitmq: 5672, caddy: 80, + clickhouse: 8123, }; return ports[type] || 0; } diff --git a/ts/classes/platform-services/manager.ts b/ts/classes/platform-services/manager.ts index 439c352..7251673 100644 --- a/ts/classes/platform-services/manager.ts +++ b/ts/classes/platform-services/manager.ts @@ -15,6 +15,7 @@ import type { IPlatformServiceProvider } from './providers/base.ts'; import { MongoDBProvider } from './providers/mongodb.ts'; import { MinioProvider } from './providers/minio.ts'; import { CaddyProvider } from './providers/caddy.ts'; +import { ClickHouseProvider } from './providers/clickhouse.ts'; import { logger } from '../../logging.ts'; import { getErrorMessage } from '../../utils/error.ts'; import { credentialEncryption } from '../encryption.ts'; @@ -39,6 +40,7 @@ export class PlatformServicesManager { this.registerProvider(new MongoDBProvider(this.oneboxRef)); this.registerProvider(new MinioProvider(this.oneboxRef)); this.registerProvider(new CaddyProvider(this.oneboxRef)); + this.registerProvider(new ClickHouseProvider(this.oneboxRef)); logger.info(`Platform services manager initialized with ${this.providers.size} providers`); } @@ -275,6 +277,33 @@ export class PlatformServicesManager { logger.success(`S3 storage provisioned for service '${service.name}'`); } + // Provision ClickHouse if requested + if (requirements.clickhouse) { + logger.info(`Provisioning ClickHouse for service '${service.name}'...`); + + // Ensure ClickHouse is running + const clickhouseService = await this.ensureRunning('clickhouse'); + const provider = this.providers.get('clickhouse')!; + + // Provision database + const result = await provider.provisionResource(service); + + // Store resource record + const encryptedCreds = await credentialEncryption.encrypt(result.credentials); + this.oneboxRef.database.createPlatformResource({ + platformServiceId: clickhouseService.id!, + serviceId: service.id!, + resourceType: result.type, + resourceName: result.name, + credentialsEncrypted: encryptedCreds, + createdAt: Date.now(), + }); + + // Merge env vars + Object.assign(allEnvVars, result.envVars); + logger.success(`ClickHouse provisioned for service '${service.name}'`); + } + return allEnvVars; } diff --git a/ts/classes/platform-services/providers/clickhouse.ts b/ts/classes/platform-services/providers/clickhouse.ts new file mode 100644 index 0000000..ebfc56d --- /dev/null +++ b/ts/classes/platform-services/providers/clickhouse.ts @@ -0,0 +1,338 @@ +/** + * ClickHouse Platform Service Provider + */ + +import { BasePlatformServiceProvider } from './base.ts'; +import type { + IService, + IPlatformResource, + IPlatformServiceConfig, + IProvisionedResource, + IEnvVarMapping, + TPlatformServiceType, + TPlatformResourceType, +} from '../../../types.ts'; +import { logger } from '../../../logging.ts'; +import { getErrorMessage } from '../../../utils/error.ts'; +import { credentialEncryption } from '../../encryption.ts'; +import type { Onebox } from '../../onebox.ts'; + +export class ClickHouseProvider extends BasePlatformServiceProvider { + readonly type: TPlatformServiceType = 'clickhouse'; + readonly displayName = 'ClickHouse'; + readonly resourceTypes: TPlatformResourceType[] = ['database']; + + constructor(oneboxRef: Onebox) { + super(oneboxRef); + } + + getDefaultConfig(): IPlatformServiceConfig { + return { + image: 'clickhouse/clickhouse-server:latest', + port: 8123, // HTTP interface + volumes: ['/var/lib/onebox/clickhouse:/var/lib/clickhouse'], + environment: { + CLICKHOUSE_DB: 'default', + // Password will be generated and stored encrypted + }, + }; + } + + getEnvVarMappings(): IEnvVarMapping[] { + return [ + { envVar: 'CLICKHOUSE_HOST', credentialPath: 'host' }, + { envVar: 'CLICKHOUSE_PORT', credentialPath: 'port' }, + { envVar: 'CLICKHOUSE_HTTP_PORT', credentialPath: 'httpPort' }, + { envVar: 'CLICKHOUSE_DATABASE', credentialPath: 'database' }, + { envVar: 'CLICKHOUSE_USER', credentialPath: 'username' }, + { envVar: 'CLICKHOUSE_PASSWORD', credentialPath: 'password' }, + { envVar: 'CLICKHOUSE_URL', credentialPath: 'connectionUrl' }, + ]; + } + + async deployContainer(): Promise { + const config = this.getDefaultConfig(); + const containerName = this.getContainerName(); + const dataDir = '/var/lib/onebox/clickhouse'; + + logger.info(`Deploying ClickHouse platform service as ${containerName}...`); + + // Check if we have existing data and stored credentials + const platformService = this.oneboxRef.database.getPlatformServiceByType(this.type); + let adminCredentials: { username: string; password: string }; + let dataExists = false; + + // Check if data directory has existing ClickHouse data + // ClickHouse creates 'metadata' directory on first startup + try { + const stat = await Deno.stat(`${dataDir}/metadata`); + dataExists = stat.isDirectory; + logger.info(`ClickHouse data directory exists with metadata folder`); + } catch { + // metadata directory doesn't exist, this is a fresh install + dataExists = false; + } + + if (dataExists && platformService?.adminCredentialsEncrypted) { + // Reuse existing credentials from database + logger.info('Reusing existing ClickHouse credentials (data directory already initialized)'); + adminCredentials = await credentialEncryption.decrypt(platformService.adminCredentialsEncrypted); + } else { + // Generate new credentials for fresh deployment + logger.info('Generating new ClickHouse admin credentials'); + adminCredentials = { + username: 'default', + password: credentialEncryption.generatePassword(32), + }; + + // If data exists but we don't have credentials, we need to wipe the data + if (dataExists) { + logger.warn('ClickHouse data exists but no credentials in database - wiping data directory'); + try { + await Deno.remove(dataDir, { recursive: true }); + } catch (e) { + logger.error(`Failed to wipe ClickHouse data directory: ${getErrorMessage(e)}`); + throw new Error('Cannot deploy ClickHouse: data directory exists without credentials'); + } + } + } + + // Ensure data directory exists + try { + await Deno.mkdir(dataDir, { recursive: true }); + } catch (e) { + // Directory might already exist + if (!(e instanceof Deno.errors.AlreadyExists)) { + logger.warn(`Could not create ClickHouse data directory: ${getErrorMessage(e)}`); + } + } + + // Create container using Docker API + // ClickHouse uses environment variables for initial setup + const envVars = [ + `CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1`, + `CLICKHOUSE_USER=${adminCredentials.username}`, + `CLICKHOUSE_PASSWORD=${adminCredentials.password}`, + ]; + + const containerId = await this.oneboxRef.docker.createPlatformContainer({ + name: containerName, + image: config.image, + port: config.port, + env: envVars, + volumes: config.volumes, + network: this.getNetworkName(), + exposePorts: [8123, 9000], // HTTP and native TCP ports + }); + + // Store encrypted admin credentials (only update if new or changed) + const encryptedCreds = await credentialEncryption.encrypt(adminCredentials); + if (platformService) { + this.oneboxRef.database.updatePlatformService(platformService.id!, { + containerId, + adminCredentialsEncrypted: encryptedCreds, + status: 'starting', + }); + } + + logger.success(`ClickHouse container created: ${containerId}`); + return containerId; + } + + async stopContainer(containerId: string): Promise { + logger.info(`Stopping ClickHouse container ${containerId}...`); + await this.oneboxRef.docker.stopContainer(containerId); + logger.success('ClickHouse container stopped'); + } + + async healthCheck(): Promise { + try { + logger.info('ClickHouse health check: starting...'); + const platformService = this.oneboxRef.database.getPlatformServiceByType(this.type); + if (!platformService) { + logger.info('ClickHouse health check: platform service not found in database'); + return false; + } + if (!platformService.adminCredentialsEncrypted) { + logger.info('ClickHouse health check: no admin credentials stored'); + return false; + } + if (!platformService.containerId) { + logger.info('ClickHouse health check: no container ID in database record'); + return false; + } + + logger.info(`ClickHouse health check: using container ID ${platformService.containerId.substring(0, 12)}...`); + + // Use docker exec to run health check inside the container + // This avoids network issues with overlay networks + // Note: ClickHouse image has wget but not curl + const result = await this.oneboxRef.docker.execInContainer( + platformService.containerId, + ['wget', '-q', '-O', '-', 'http://localhost:8123/ping'] + ); + + if (result.exitCode === 0) { + logger.info('ClickHouse health check: success'); + return true; + } else { + logger.info(`ClickHouse health check failed: exit code ${result.exitCode}, stderr: ${result.stderr.substring(0, 200)}`); + return false; + } + } catch (error) { + logger.info(`ClickHouse health check exception: ${getErrorMessage(error)}`); + return false; + } + } + + async provisionResource(userService: IService): Promise { + const platformService = this.oneboxRef.database.getPlatformServiceByType(this.type); + if (!platformService || !platformService.adminCredentialsEncrypted || !platformService.containerId) { + throw new Error('ClickHouse platform service not found or not configured'); + } + + const adminCreds = await credentialEncryption.decrypt(platformService.adminCredentialsEncrypted); + const containerName = this.getContainerName(); + + // Get container host port for connection from host (overlay network IPs not accessible from host) + const hostPort = await this.oneboxRef.docker.getContainerHostPort(platformService.containerId, 8123); + if (!hostPort) { + throw new Error('Could not get ClickHouse container host port'); + } + + // Generate resource names and credentials + const dbName = this.generateResourceName(userService.name); + const username = this.generateResourceName(userService.name); + const password = credentialEncryption.generatePassword(32); + + logger.info(`Provisioning ClickHouse database '${dbName}' for service '${userService.name}'...`); + + // Connect to ClickHouse via localhost and the mapped host port + const baseUrl = `http://127.0.0.1:${hostPort}`; + + // Create database + await this.executeQuery( + baseUrl, + adminCreds.username, + adminCreds.password, + `CREATE DATABASE IF NOT EXISTS ${dbName}` + ); + logger.info(`Created ClickHouse database '${dbName}'`); + + // Create user with access to this database + await this.executeQuery( + baseUrl, + adminCreds.username, + adminCreds.password, + `CREATE USER IF NOT EXISTS ${username} IDENTIFIED BY '${password}'` + ); + logger.info(`Created ClickHouse user '${username}'`); + + // Grant permissions on the database + await this.executeQuery( + baseUrl, + adminCreds.username, + adminCreds.password, + `GRANT ALL ON ${dbName}.* TO ${username}` + ); + logger.info(`Granted permissions to user '${username}' on database '${dbName}'`); + + logger.success(`ClickHouse database '${dbName}' provisioned with user '${username}'`); + + // Build the credentials and env vars + const credentials: Record = { + host: containerName, + port: '9000', // Native TCP port + httpPort: '8123', + database: dbName, + username, + password, + connectionUrl: `http://${username}:${password}@${containerName}:8123/?database=${dbName}`, + }; + + // Map credentials to env vars + const envVars: Record = {}; + for (const mapping of this.getEnvVarMappings()) { + if (credentials[mapping.credentialPath]) { + envVars[mapping.envVar] = credentials[mapping.credentialPath]; + } + } + + return { + type: 'database', + name: dbName, + credentials, + envVars, + }; + } + + async deprovisionResource(resource: IPlatformResource, credentials: Record): Promise { + const platformService = this.oneboxRef.database.getPlatformServiceByType(this.type); + if (!platformService || !platformService.adminCredentialsEncrypted || !platformService.containerId) { + throw new Error('ClickHouse platform service not found or not configured'); + } + + const adminCreds = await credentialEncryption.decrypt(platformService.adminCredentialsEncrypted); + + // Get container host port for connection from host (overlay network IPs not accessible from host) + const hostPort = await this.oneboxRef.docker.getContainerHostPort(platformService.containerId, 8123); + if (!hostPort) { + throw new Error('Could not get ClickHouse container host port'); + } + + logger.info(`Deprovisioning ClickHouse database '${resource.resourceName}'...`); + + const baseUrl = `http://127.0.0.1:${hostPort}`; + + try { + // Drop the user + try { + await this.executeQuery( + baseUrl, + adminCreds.username, + adminCreds.password, + `DROP USER IF EXISTS ${credentials.username}` + ); + logger.info(`Dropped ClickHouse user '${credentials.username}'`); + } catch (e) { + logger.warn(`Could not drop ClickHouse user: ${getErrorMessage(e)}`); + } + + // Drop the database + await this.executeQuery( + baseUrl, + adminCreds.username, + adminCreds.password, + `DROP DATABASE IF EXISTS ${resource.resourceName}` + ); + logger.success(`ClickHouse database '${resource.resourceName}' dropped`); + } catch (e) { + logger.error(`Failed to deprovision ClickHouse database: ${getErrorMessage(e)}`); + throw e; + } + } + + /** + * Execute a ClickHouse SQL query via HTTP interface + */ + private async executeQuery( + baseUrl: string, + username: string, + password: string, + query: string + ): Promise { + const url = `${baseUrl}/?user=${encodeURIComponent(username)}&password=${encodeURIComponent(password)}`; + + const response = await fetch(url, { + method: 'POST', + body: query, + }); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`ClickHouse query failed: ${errorText}`); + } + + return await response.text(); + } +} diff --git a/ts/classes/services.ts b/ts/classes/services.ts index 378e109..ea30d9e 100644 --- a/ts/classes/services.ts +++ b/ts/classes/services.ts @@ -49,10 +49,11 @@ export class OneboxServicesManager { // Build platform requirements const platformRequirements: IPlatformRequirements | undefined = - (options.enableMongoDB || options.enableS3) + (options.enableMongoDB || options.enableS3 || options.enableClickHouse) ? { mongodb: options.enableMongoDB, s3: options.enableS3, + clickhouse: options.enableClickHouse, } : undefined; diff --git a/ts/types.ts b/ts/types.ts index af0c953..412793b 100644 --- a/ts/types.ts +++ b/ts/types.ts @@ -73,7 +73,7 @@ export interface ITokenCreatedResponse { } // Platform service types -export type TPlatformServiceType = 'mongodb' | 'minio' | 'redis' | 'postgresql' | 'rabbitmq' | 'caddy'; +export type TPlatformServiceType = 'mongodb' | 'minio' | 'redis' | 'postgresql' | 'rabbitmq' | 'caddy' | 'clickhouse'; export type TPlatformResourceType = 'database' | 'bucket' | 'cache' | 'queue'; export type TPlatformServiceStatus = 'stopped' | 'starting' | 'running' | 'stopping' | 'failed'; @@ -110,6 +110,7 @@ export interface IPlatformResource { export interface IPlatformRequirements { mongodb?: boolean; s3?: boolean; + clickhouse?: boolean; } export interface IProvisionedResource { @@ -287,6 +288,7 @@ export interface IServiceDeployOptions { // Platform service requirements enableMongoDB?: boolean; enableS3?: boolean; + enableClickHouse?: boolean; } // HTTP API request/response types diff --git a/ui/src/app/core/types/api.types.ts b/ui/src/app/core/types/api.types.ts index 89f0efe..fdac3f3 100644 --- a/ui/src/app/core/types/api.types.ts +++ b/ui/src/app/core/types/api.types.ts @@ -16,13 +16,14 @@ export interface ILoginResponse { } // Platform Service Types (defined early for use in ISystemStatus) -export type TPlatformServiceType = 'mongodb' | 'minio' | 'redis' | 'postgresql' | 'rabbitmq' | 'caddy'; +export type TPlatformServiceType = 'mongodb' | 'minio' | 'redis' | 'postgresql' | 'rabbitmq' | 'caddy' | 'clickhouse'; export type TPlatformServiceStatus = 'not-deployed' | 'stopped' | 'starting' | 'running' | 'stopping' | 'failed'; export type TPlatformResourceType = 'database' | 'bucket' | 'cache' | 'queue'; export interface IPlatformRequirements { mongodb?: boolean; s3?: boolean; + clickhouse?: boolean; } export interface IService { @@ -56,6 +57,7 @@ export interface IServiceCreate { autoUpdateOnPush?: boolean; enableMongoDB?: boolean; enableS3?: boolean; + enableClickHouse?: boolean; // ClickHouse analytics database } export interface IServiceUpdate { diff --git a/ui/src/app/features/services/platform-service-detail.component.ts b/ui/src/app/features/services/platform-service-detail.component.ts index ac09a08..8f57345 100644 --- a/ui/src/app/features/services/platform-service-detail.component.ts +++ b/ui/src/app/features/services/platform-service-detail.component.ts @@ -357,6 +357,7 @@ export class PlatformServiceDetailComponent implements OnInit, OnDestroy { postgresql: 'PostgreSQL is a powerful, open-source object-relational database system with over 35 years of active development.', rabbitmq: 'RabbitMQ is a message broker that enables applications to communicate with each other using messages through queues.', caddy: 'Caddy is a powerful, enterprise-ready, open-source web server with automatic HTTPS. It serves as the reverse proxy for Onebox.', + clickhouse: 'ClickHouse is a fast, open-source columnar database management system optimized for real-time analytics and data warehousing.', }; return descriptions[type] || 'A platform service managed by Onebox.'; } diff --git a/ui/src/app/features/services/service-create.component.ts b/ui/src/app/features/services/service-create.component.ts index 0262ade..1616a43 100644 --- a/ui/src/app/features/services/service-create.component.ts +++ b/ui/src/app/features/services/service-create.component.ts @@ -215,9 +215,20 @@ interface EnvVar {

A dedicated bucket will be created and credentials injected as S3_* and AWS_* env vars

+ +
+ +
+ +

A dedicated database will be created and credentials injected as CLICKHOUSE_* env vars

+
+
- @if (form.enableMongoDB || form.enableS3) { + @if (form.enableMongoDB || form.enableS3 || form.enableClickHouse) { Platform services will be auto-deployed if not already running. Credentials are automatically injected as environment variables. @@ -301,6 +312,7 @@ export class ServiceCreateComponent implements OnInit { autoUpdateOnPush: false, enableMongoDB: false, enableS3: false, + enableClickHouse: false, }; envVars = signal([]);