serve.zone/onebox
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(platform-services): Add ClickHouse platform service support (provider, types, provisioning, UI and port mappings)

Browse Source
This commit is contained in:
Jürgen Kunz
2025-11-26 18:54:20 +00:00
parent 38b5462b09
commit 0d932239d2
10 changed files with 402 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
changelog.md
View File

@@ -1,5 +1,16 @@
# Changelog
## 2025-11-26 - 1.3.0 - feat(platform-services)
Add ClickHouse platform service support (provider, types, provisioning, UI and port mappings)
- Introduce ClickHouse as a first-class platform service: added ClickHouseProvider and registered it in PlatformServicesManager
- Support provisioning ClickHouse resources for user services and storing encrypted credentials in platform_resources
- Add ClickHouse to core types (TPlatformServiceType, IPlatformRequirements, IServiceDeployOptions) and service DB handling so services can request ClickHouse
- Inject ClickHouse-related environment variables into deployed services (CLICKHOUSE_* mappings) when provisioning resources
- Expose ClickHouse default port (8123) in platform port mappings / network targets
- UI: add checkbox and description for enabling ClickHouse during service creation; form now submits enableClickHouse
- Add VS Code recommendations and launch/tasks for the UI development workflow
## 2025-11-26 - 1.2.1 - fix(platform-services/minio)
Improve MinIO provider: reuse existing data and credentials, use host-bound port for provisioning, and safer provisioning/deprovisioning

2
ts/00_commitinfo_data.ts
View File

@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@serve.zone/onebox',
version: '1.2.1',
version: '1.3.0',
description: 'Self-hosted container platform with automatic SSL and DNS - a mini Heroku for single servers'
}

1
ts/classes/httpserver.ts
View File

@@ -1323,6 +1323,7 @@ export class OneboxHttpServer {
postgresql: 5432,
rabbitmq: 5672,
caddy: 80,
clickhouse: 8123,
};
return ports[type] || 0;
}

29
ts/classes/platform-services/manager.ts
View File

@@ -15,6 +15,7 @@ import type { IPlatformServiceProvider } from './providers/base.ts';
import { MongoDBProvider } from './providers/mongodb.ts';
import { MinioProvider } from './providers/minio.ts';
import { CaddyProvider } from './providers/caddy.ts';
import { ClickHouseProvider } from './providers/clickhouse.ts';
import { logger } from '../../logging.ts';
import { getErrorMessage } from '../../utils/error.ts';
import { credentialEncryption } from '../encryption.ts';
@@ -39,6 +40,7 @@ export class PlatformServicesManager {
this.registerProvider(new MongoDBProvider(this.oneboxRef));
this.registerProvider(new MinioProvider(this.oneboxRef));
this.registerProvider(new CaddyProvider(this.oneboxRef));
this.registerProvider(new ClickHouseProvider(this.oneboxRef));
logger.info(`Platform services manager initialized with ${this.providers.size} providers`);
}
@@ -275,6 +277,33 @@ export class PlatformServicesManager {
logger.success(`S3 storage provisioned for service '${service.name}'`);
}
// Provision ClickHouse if requested
if (requirements.clickhouse) {
logger.info(`Provisioning ClickHouse for service '${service.name}'...`);
// Ensure ClickHouse is running
const clickhouseService = await this.ensureRunning('clickhouse');
const provider = this.providers.get('clickhouse')!;
// Provision database
const result = await provider.provisionResource(service);
// Store resource record
const encryptedCreds = await credentialEncryption.encrypt(result.credentials);
this.oneboxRef.database.createPlatformResource({
platformServiceId: clickhouseService.id!,
serviceId: service.id!,
resourceType: result.type,
resourceName: result.name,
credentialsEncrypted: encryptedCreds,
createdAt: Date.now(),
});
// Merge env vars
Object.assign(allEnvVars, result.envVars);
logger.success(`ClickHouse provisioned for service '${service.name}'`);
}
return allEnvVars;
}

338
ts/classes/platform-services/providers/clickhouse.ts Normal file
View File

@@ -0,0 +1,338 @@
/**
* ClickHouse Platform Service Provider
*/
import { BasePlatformServiceProvider } from './base.ts';
import type {
IService,
IPlatformResource,
IPlatformServiceConfig,
IProvisionedResource,
IEnvVarMapping,
TPlatformServiceType,
TPlatformResourceType,
} from '../../../types.ts';
import { logger } from '../../../logging.ts';
import { getErrorMessage } from '../../../utils/error.ts';
import { credentialEncryption } from '../../encryption.ts';
import type { Onebox } from '../../onebox.ts';
export class ClickHouseProvider extends BasePlatformServiceProvider {
readonly type: TPlatformServiceType = 'clickhouse';
readonly displayName = 'ClickHouse';
readonly resourceTypes: TPlatformResourceType[] = ['database'];
constructor(oneboxRef: Onebox) {
super(oneboxRef);
}
getDefaultConfig(): IPlatformServiceConfig {
return {
image: 'clickhouse/clickhouse-server:latest',
port: 8123, // HTTP interface
volumes: ['/var/lib/onebox/clickhouse:/var/lib/clickhouse'],
environment: {
CLICKHOUSE_DB: 'default',
// Password will be generated and stored encrypted
},
};
}
getEnvVarMappings(): IEnvVarMapping[] {
return [
{ envVar: 'CLICKHOUSE_HOST', credentialPath: 'host' },
{ envVar: 'CLICKHOUSE_PORT', credentialPath: 'port' },
{ envVar: 'CLICKHOUSE_HTTP_PORT', credentialPath: 'httpPort' },
{ envVar: 'CLICKHOUSE_DATABASE', credentialPath: 'database' },
{ envVar: 'CLICKHOUSE_USER', credentialPath: 'username' },
{ envVar: 'CLICKHOUSE_PASSWORD', credentialPath: 'password' },
{ envVar: 'CLICKHOUSE_URL', credentialPath: 'connectionUrl' },
];
}
async deployContainer(): Promise<string> {
const config = this.getDefaultConfig();
const containerName = this.getContainerName();
const dataDir = '/var/lib/onebox/clickhouse';
logger.info(`Deploying ClickHouse platform service as ${containerName}...`);
// Check if we have existing data and stored credentials
const platformService = this.oneboxRef.database.getPlatformServiceByType(this.type);
let adminCredentials: { username: string; password: string };
let dataExists = false;
// Check if data directory has existing ClickHouse data
// ClickHouse creates 'metadata' directory on first startup
try {
const stat = await Deno.stat(`${dataDir}/metadata`);
dataExists = stat.isDirectory;
logger.info(`ClickHouse data directory exists with metadata folder`);
} catch {
// metadata directory doesn't exist, this is a fresh install
dataExists = false;
}
if (dataExists && platformService?.adminCredentialsEncrypted) {
// Reuse existing credentials from database
logger.info('Reusing existing ClickHouse credentials (data directory already initialized)');
adminCredentials = await credentialEncryption.decrypt(platformService.adminCredentialsEncrypted);
} else {
// Generate new credentials for fresh deployment
logger.info('Generating new ClickHouse admin credentials');
adminCredentials = {
username: 'default',
password: credentialEncryption.generatePassword(32),
};
// If data exists but we don't have credentials, we need to wipe the data
if (dataExists) {
logger.warn('ClickHouse data exists but no credentials in database - wiping data directory');
try {
await Deno.remove(dataDir, { recursive: true });
} catch (e) {
logger.error(`Failed to wipe ClickHouse data directory: ${getErrorMessage(e)}`);
throw new Error('Cannot deploy ClickHouse: data directory exists without credentials');
}
}
}
// Ensure data directory exists
try {
await Deno.mkdir(dataDir, { recursive: true });
} catch (e) {
// Directory might already exist
if (!(e instanceof Deno.errors.AlreadyExists)) {
logger.warn(`Could not create ClickHouse data directory: ${getErrorMessage(e)}`);
}
}
// Create container using Docker API
// ClickHouse uses environment variables for initial setup
const envVars = [
`CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1`,
`CLICKHOUSE_USER=${adminCredentials.username}`,
`CLICKHOUSE_PASSWORD=${adminCredentials.password}`,
];
const containerId = await this.oneboxRef.docker.createPlatformContainer({
name: containerName,
image: config.image,
port: config.port,
env: envVars,
volumes: config.volumes,
network: this.getNetworkName(),
exposePorts: [8123, 9000], // HTTP and native TCP ports
});
// Store encrypted admin credentials (only update if new or changed)
const encryptedCreds = await credentialEncryption.encrypt(adminCredentials);
if (platformService) {
this.oneboxRef.database.updatePlatformService(platformService.id!, {
containerId,
adminCredentialsEncrypted: encryptedCreds,
status: 'starting',
});
}
logger.success(`ClickHouse container created: ${containerId}`);
return containerId;
}
async stopContainer(containerId: string): Promise<void> {
logger.info(`Stopping ClickHouse container ${containerId}...`);
await this.oneboxRef.docker.stopContainer(containerId);
logger.success('ClickHouse container stopped');
}
async healthCheck(): Promise<boolean> {
try {
logger.info('ClickHouse health check: starting...');
const platformService = this.oneboxRef.database.getPlatformServiceByType(this.type);
if (!platformService) {
logger.info('ClickHouse health check: platform service not found in database');
return false;
}
if (!platformService.adminCredentialsEncrypted) {
logger.info('ClickHouse health check: no admin credentials stored');
return false;
}
if (!platformService.containerId) {
logger.info('ClickHouse health check: no container ID in database record');
return false;
}
logger.info(`ClickHouse health check: using container ID ${platformService.containerId.substring(0, 12)}...`);
// Use docker exec to run health check inside the container
// This avoids network issues with overlay networks
// Note: ClickHouse image has wget but not curl
const result = await this.oneboxRef.docker.execInContainer(
platformService.containerId,
['wget', '-q', '-O', '-', 'http://localhost:8123/ping']
);
if (result.exitCode === 0) {
logger.info('ClickHouse health check: success');
return true;
} else {
logger.info(`ClickHouse health check failed: exit code ${result.exitCode}, stderr: ${result.stderr.substring(0, 200)}`);
return false;
}
} catch (error) {
logger.info(`ClickHouse health check exception: ${getErrorMessage(error)}`);
return false;
}
}
async provisionResource(userService: IService): Promise<IProvisionedResource> {
const platformService = this.oneboxRef.database.getPlatformServiceByType(this.type);
if (!platformService || !platformService.adminCredentialsEncrypted || !platformService.containerId) {
throw new Error('ClickHouse platform service not found or not configured');
}
const adminCreds = await credentialEncryption.decrypt(platformService.adminCredentialsEncrypted);
const containerName = this.getContainerName();
// Get container host port for connection from host (overlay network IPs not accessible from host)
const hostPort = await this.oneboxRef.docker.getContainerHostPort(platformService.containerId, 8123);
if (!hostPort) {
throw new Error('Could not get ClickHouse container host port');
}
// Generate resource names and credentials
const dbName = this.generateResourceName(userService.name);
const username = this.generateResourceName(userService.name);
const password = credentialEncryption.generatePassword(32);
logger.info(`Provisioning ClickHouse database '${dbName}' for service '${userService.name}'...`);
// Connect to ClickHouse via localhost and the mapped host port
const baseUrl = `http://127.0.0.1:${hostPort}`;
// Create database
await this.executeQuery(
baseUrl,
adminCreds.username,
adminCreds.password,
`CREATE DATABASE IF NOT EXISTS ${dbName}`
);
logger.info(`Created ClickHouse database '${dbName}'`);
// Create user with access to this database
await this.executeQuery(
baseUrl,
adminCreds.username,
adminCreds.password,
`CREATE USER IF NOT EXISTS ${username} IDENTIFIED BY '${password}'`
);
logger.info(`Created ClickHouse user '${username}'`);
// Grant permissions on the database
await this.executeQuery(
baseUrl,
adminCreds.username,
adminCreds.password,
`GRANT ALL ON ${dbName}.* TO ${username}`
);
logger.info(`Granted permissions to user '${username}' on database '${dbName}'`);
logger.success(`ClickHouse database '${dbName}' provisioned with user '${username}'`);
// Build the credentials and env vars
const credentials: Record<string, string> = {
host: containerName,
port: '9000', // Native TCP port
httpPort: '8123',
database: dbName,
username,
password,
connectionUrl: `http://${username}:${password}@${containerName}:8123/?database=${dbName}`,
};
// Map credentials to env vars
const envVars: Record<string, string> = {};
for (const mapping of this.getEnvVarMappings()) {
if (credentials[mapping.credentialPath]) {
envVars[mapping.envVar] = credentials[mapping.credentialPath];
}
}
return {
type: 'database',
name: dbName,
credentials,
envVars,
};
}
async deprovisionResource(resource: IPlatformResource, credentials: Record<string, string>): Promise<void> {
const platformService = this.oneboxRef.database.getPlatformServiceByType(this.type);
if (!platformService || !platformService.adminCredentialsEncrypted || !platformService.containerId) {
throw new Error('ClickHouse platform service not found or not configured');
}
const adminCreds = await credentialEncryption.decrypt(platformService.adminCredentialsEncrypted);
// Get container host port for connection from host (overlay network IPs not accessible from host)
const hostPort = await this.oneboxRef.docker.getContainerHostPort(platformService.containerId, 8123);
if (!hostPort) {
throw new Error('Could not get ClickHouse container host port');
}
logger.info(`Deprovisioning ClickHouse database '${resource.resourceName}'...`);
const baseUrl = `http://127.0.0.1:${hostPort}`;
try {
// Drop the user
try {
await this.executeQuery(
baseUrl,
adminCreds.username,
adminCreds.password,
`DROP USER IF EXISTS ${credentials.username}`
);
logger.info(`Dropped ClickHouse user '${credentials.username}'`);
} catch (e) {
logger.warn(`Could not drop ClickHouse user: ${getErrorMessage(e)}`);
}
// Drop the database
await this.executeQuery(
baseUrl,
adminCreds.username,
adminCreds.password,
`DROP DATABASE IF EXISTS ${resource.resourceName}`
);
logger.success(`ClickHouse database '${resource.resourceName}' dropped`);
} catch (e) {
logger.error(`Failed to deprovision ClickHouse database: ${getErrorMessage(e)}`);
throw e;
}
}
/**
* Execute a ClickHouse SQL query via HTTP interface
*/
private async executeQuery(
baseUrl: string,
username: string,
password: string,
query: string
): Promise<string> {
const url = `${baseUrl}/?user=${encodeURIComponent(username)}&password=${encodeURIComponent(password)}`;
const response = await fetch(url, {
method: 'POST',
body: query,
});
if (!response.ok) {
const errorText = await response.text();
throw new Error(`ClickHouse query failed: ${errorText}`);
}
return await response.text();
}
}

3
ts/classes/services.ts
View File

@@ -49,10 +49,11 @@ export class OneboxServicesManager {
// Build platform requirements
const platformRequirements: IPlatformRequirements | undefined =
(options.enableMongoDB || options.enableS3)
(options.enableMongoDB || options.enableS3 || options.enableClickHouse)
? {
mongodb: options.enableMongoDB,
s3: options.enableS3,
clickhouse: options.enableClickHouse,
}
: undefined;

4
ts/types.ts
View File

@@ -73,7 +73,7 @@ export interface ITokenCreatedResponse {
}
// Platform service types
export type TPlatformServiceType = 'mongodb' | 'minio' | 'redis' | 'postgresql' | 'rabbitmq' | 'caddy';
export type TPlatformServiceType = 'mongodb' | 'minio' | 'redis' | 'postgresql' | 'rabbitmq' | 'caddy' | 'clickhouse';
export type TPlatformResourceType = 'database' | 'bucket' | 'cache' | 'queue';
export type TPlatformServiceStatus = 'stopped' | 'starting' | 'running' | 'stopping' | 'failed';
@@ -110,6 +110,7 @@ export interface IPlatformResource {
export interface IPlatformRequirements {
mongodb?: boolean;
s3?: boolean;
clickhouse?: boolean;
}
export interface IProvisionedResource {
@@ -287,6 +288,7 @@ export interface IServiceDeployOptions {
// Platform service requirements
enableMongoDB?: boolean;
enableS3?: boolean;
enableClickHouse?: boolean;
}
// HTTP API request/response types

4
ui/src/app/core/types/api.types.ts
View File

@@ -16,13 +16,14 @@ export interface ILoginResponse {
}
// Platform Service Types (defined early for use in ISystemStatus)
export type TPlatformServiceType = 'mongodb' | 'minio' | 'redis' | 'postgresql' | 'rabbitmq' | 'caddy';
export type TPlatformServiceType = 'mongodb' | 'minio' | 'redis' | 'postgresql' | 'rabbitmq' | 'caddy' | 'clickhouse';
export type TPlatformServiceStatus = 'not-deployed' | 'stopped' | 'starting' | 'running' | 'stopping' | 'failed';
export type TPlatformResourceType = 'database' | 'bucket' | 'cache' | 'queue';
export interface IPlatformRequirements {
mongodb?: boolean;
s3?: boolean;
clickhouse?: boolean;
}
export interface IService {
@@ -56,6 +57,7 @@ export interface IServiceCreate {
autoUpdateOnPush?: boolean;
enableMongoDB?: boolean;
enableS3?: boolean;
enableClickHouse?: boolean; // ClickHouse analytics database
}
export interface IServiceUpdate {

1
ui/src/app/features/services/platform-service-detail.component.ts
View File

@@ -357,6 +357,7 @@ export class PlatformServiceDetailComponent implements OnInit, OnDestroy {
postgresql: 'PostgreSQL is a powerful, open-source object-relational database system with over 35 years of active development.',
rabbitmq: 'RabbitMQ is a message broker that enables applications to communicate with each other using messages through queues.',
caddy: 'Caddy is a powerful, enterprise-ready, open-source web server with automatic HTTPS. It serves as the reverse proxy for Onebox.',
clickhouse: 'ClickHouse is a fast, open-source columnar database management system optimized for real-time analytics and data warehousing.',
};
return descriptions[type] || 'A platform service managed by Onebox.';
}

14
ui/src/app/features/services/service-create.component.ts
View File

@@ -215,9 +215,20 @@ interface EnvVar {
<p class="text-xs text-muted-foreground">A dedicated bucket will be created and credentials injected as S3_* and AWS_* env vars</p>
</div>
</div>
<div class="flex items-center gap-3">
<ui-checkbox
[checked]="form.enableClickHouse ?? false"
(checkedChange)="form.enableClickHouse = $event"
/>
<div>
<label uiLabel class="cursor-pointer">ClickHouse Database</label>
<p class="text-xs text-muted-foreground">A dedicated database will be created and credentials injected as CLICKHOUSE_* env vars</p>
</div>
</div>
</div>
@if (form.enableMongoDB || form.enableS3) {
@if (form.enableMongoDB || form.enableS3 || form.enableClickHouse) {
<ui-alert variant="default">
<ui-alert-description>
Platform services will be auto-deployed if not already running. Credentials are automatically injected as environment variables.
@@ -301,6 +312,7 @@ export class ServiceCreateComponent implements OnInit {
autoUpdateOnPush: false,
enableMongoDB: false,
enableS3: false,
enableClickHouse: false,
};
envVars = signal<EnvVar[]>([]);