serve.zone/onebox
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

fix(platform-services): provision ClickHouse, MinIO, and MongoDB resources via docker exec instead of host port access

Browse Source
This commit is contained in:
Jürgen Kunz
2026-03-16 12:45:44 +00:00
parent cd06c74cc3
commit 6f1b8469e0
8 changed files with 129 additions and 250 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
changelog.md
View File

@@ -1,5 +1,12 @@
# Changelog
## 2026-03-16 - 1.17.2 - fix(platform-services)
provision ClickHouse, MinIO, and MongoDB resources via docker exec instead of host port access
- switch ClickHouse provisioning and teardown to in-container client commands to avoid host port mapping issues
- replace MinIO host-side S3 API calls with in-container mc commands for bucket creation and removal
- run MongoDB provisioning and deprovisioning through mongosh inside the container and improve docker exec failure reporting
## 2026-03-16 - 1.17.1 - fix(repo)
no changes to commit

2
ts/00_commitinfo_data.ts
View File

@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@serve.zone/onebox',
version: '1.17.1',
version: '1.17.2',
description: 'Self-hosted container platform with automatic SSL and DNS - a mini Heroku for single servers'
}

4
ts/classes/docker.ts
View File

@@ -881,12 +881,12 @@ export class OneboxDockerManager {
]);
const execInfo = await inspect();
const exitCode = execInfo.ExitCode || 0;
const exitCode = execInfo.ExitCode ?? -1;
return { stdout, stderr, exitCode };
} catch (error) {
logger.error(`Failed to exec in container ${containerID}: ${getErrorMessage(error)}`);
throw error;
return { stdout: '', stderr: getErrorMessage(error), exitCode: -1 };
}
}

104
ts/classes/platform-services/providers/clickhouse.ts
View File

@@ -194,12 +194,6 @@ export class ClickHouseProvider extends BasePlatformServiceProvider {
const adminCreds = await credentialEncryption.decrypt(platformService.adminCredentialsEncrypted);
const containerName = this.getContainerName();
// Get container host port for connection from host (overlay network IPs not accessible from host)
const hostPort = await this.oneboxRef.docker.getContainerHostPort(platformService.containerId, 8123);
if (!hostPort) {
throw new Error('Could not get ClickHouse container host port');
}
// Generate resource names and credentials
const dbName = this.generateResourceName(userService.name);
const username = this.generateResourceName(userService.name);
@@ -207,35 +201,16 @@ export class ClickHouseProvider extends BasePlatformServiceProvider {
logger.info(`Provisioning ClickHouse database '${dbName}' for service '${userService.name}'...`);
// Connect to ClickHouse via localhost and the mapped host port
const baseUrl = `http://127.0.0.1:${hostPort}`;
// Use docker exec to provision inside the container (avoids host port mapping issues)
const queries = [
`CREATE DATABASE IF NOT EXISTS ${dbName}`,
`CREATE USER IF NOT EXISTS ${username} IDENTIFIED BY '${password}'`,
`GRANT ALL ON ${dbName}.* TO ${username}`,
];
// Create database
await this.executeQuery(
baseUrl,
adminCreds.username,
adminCreds.password,
`CREATE DATABASE IF NOT EXISTS ${dbName}`
);
logger.info(`Created ClickHouse database '${dbName}'`);
// Create user with access to this database
await this.executeQuery(
baseUrl,
adminCreds.username,
adminCreds.password,
`CREATE USER IF NOT EXISTS ${username} IDENTIFIED BY '${password}'`
);
logger.info(`Created ClickHouse user '${username}'`);
// Grant permissions on the database
await this.executeQuery(
baseUrl,
adminCreds.username,
adminCreds.password,
`GRANT ALL ON ${dbName}.* TO ${username}`
);
logger.info(`Granted permissions to user '${username}' on database '${dbName}'`);
for (const query of queries) {
await this.execClickHouseQuery(platformService.containerId, adminCreds, query);
}
logger.success(`ClickHouse database '${dbName}' provisioned with user '${username}'`);
@@ -274,37 +249,11 @@ export class ClickHouseProvider extends BasePlatformServiceProvider {
const adminCreds = await credentialEncryption.decrypt(platformService.adminCredentialsEncrypted);
// Get container host port for connection from host (overlay network IPs not accessible from host)
const hostPort = await this.oneboxRef.docker.getContainerHostPort(platformService.containerId, 8123);
if (!hostPort) {
throw new Error('Could not get ClickHouse container host port');
}
logger.info(`Deprovisioning ClickHouse database '${resource.resourceName}'...`);
const baseUrl = `http://127.0.0.1:${hostPort}`;
try {
// Drop the user
try {
await this.executeQuery(
baseUrl,
adminCreds.username,
adminCreds.password,
`DROP USER IF EXISTS ${credentials.username}`
);
logger.info(`Dropped ClickHouse user '${credentials.username}'`);
} catch (e) {
logger.warn(`Could not drop ClickHouse user: ${getErrorMessage(e)}`);
}
// Drop the database
await this.executeQuery(
baseUrl,
adminCreds.username,
adminCreds.password,
`DROP DATABASE IF EXISTS ${resource.resourceName}`
);
await this.execClickHouseQuery(platformService.containerId, adminCreds, `DROP USER IF EXISTS ${credentials.username}`);
await this.execClickHouseQuery(platformService.containerId, adminCreds, `DROP DATABASE IF EXISTS ${resource.resourceName}`);
logger.success(`ClickHouse database '${resource.resourceName}' dropped`);
} catch (e) {
logger.error(`Failed to deprovision ClickHouse database: ${getErrorMessage(e)}`);
@@ -313,26 +262,27 @@ export class ClickHouseProvider extends BasePlatformServiceProvider {
}
/**
* Execute a ClickHouse SQL query via HTTP interface
* Execute a ClickHouse SQL query via docker exec inside the container
*/
private async executeQuery(
baseUrl: string,
username: string,
password: string,
private async execClickHouseQuery(
containerId: string,
adminCreds: { username: string; password: string },
query: string
): Promise<string> {
const url = `${baseUrl}/?user=${encodeURIComponent(username)}&password=${encodeURIComponent(password)}`;
const result = await this.oneboxRef.docker.execInContainer(
containerId,
[
'clickhouse-client',
'--user', adminCreds.username,
'--password', adminCreds.password,
'--query', query,
]
);
const response = await fetch(url, {
method: 'POST',
body: query,
});
if (!response.ok) {
const errorText = await response.text();
throw new Error(`ClickHouse query failed: ${errorText}`);
if (result.exitCode !== 0) {
throw new Error(`ClickHouse query failed (exit ${result.exitCode}): ${result.stderr.substring(0, 200)}`);
}
return await response.text();
return result.stdout;
}
}

154
ts/classes/platform-services/providers/minio.ts
View File

@@ -196,84 +196,28 @@ export class MinioProvider extends BasePlatformServiceProvider {
const adminCreds = await credentialEncryption.decrypt(platformService.adminCredentialsEncrypted);
const containerName = this.getContainerName();
// Get container host port for connection from host (overlay network IPs not accessible from host)
const hostPort = await this.oneboxRef.docker.getContainerHostPort(platformService.containerId, 9000);
if (!hostPort) {
throw new Error('Could not get MinIO container host port');
}
// Generate bucket name and credentials
// Generate bucket name
const bucketName = this.generateBucketName(userService.name);
const accessKey = credentialEncryption.generateAccessKey(20);
const secretKey = credentialEncryption.generateSecretKey(40);
logger.info(`Provisioning MinIO bucket '${bucketName}' for service '${userService.name}'...`);
// Connect to MinIO via localhost and the mapped host port (for provisioning from host)
const provisioningEndpoint = `http://127.0.0.1:${hostPort}`;
// Import AWS S3 client
const { S3Client, CreateBucketCommand, PutBucketPolicyCommand } = await import('npm:@aws-sdk/client-s3@3');
// Create S3 client with admin credentials - connect via host port
const s3Client = new S3Client({
endpoint: provisioningEndpoint,
region: 'us-east-1',
credentials: {
accessKeyId: adminCreds.username,
secretAccessKey: adminCreds.password,
},
forcePathStyle: true,
});
// Use docker exec with mc (MinIO Client) inside the container
// First configure mc alias for local server
await this.execMc(platformService.containerId, [
'alias', 'set', 'local', 'http://localhost:9000',
adminCreds.username, adminCreds.password,
]);
// Create the bucket
try {
await s3Client.send(new CreateBucketCommand({
Bucket: bucketName,
}));
logger.info(`Created MinIO bucket '${bucketName}'`);
} catch (e: any) {
if (e.name !== 'BucketAlreadyOwnedByYou' && e.name !== 'BucketAlreadyExists') {
throw e;
}
logger.warn(`Bucket '${bucketName}' already exists`);
}
const mbResult = await this.execMc(platformService.containerId, [
'mb', '--ignore-existing', `local/${bucketName}`,
]);
logger.info(`Created MinIO bucket '${bucketName}'`);
// Create service account/access key using MinIO Admin API
// MinIO Admin API requires mc client or direct API calls
// For simplicity, we'll use root credentials and bucket policy isolation
// In production, you'd use MinIO's Admin API to create service accounts
// Set bucket policy to allow access only with this bucket's credentials
const bucketPolicy = {
Version: '2012-10-17',
Statement: [
{
Effect: 'Allow',
Principal: { AWS: ['*'] },
Action: ['s3:GetObject', 's3:PutObject', 's3:DeleteObject', 's3:ListBucket'],
Resource: [
`arn:aws:s3:::${bucketName}`,
`arn:aws:s3:::${bucketName}/*`,
],
},
],
};
try {
await s3Client.send(new PutBucketPolicyCommand({
Bucket: bucketName,
Policy: JSON.stringify(bucketPolicy),
}));
logger.info(`Set bucket policy for '${bucketName}'`);
} catch (e) {
logger.warn(`Could not set bucket policy: ${getErrorMessage(e)}`);
}
// Note: For proper per-service credentials, MinIO Admin API should be used
// For now, we're providing the bucket with root access
// TODO: Implement MinIO service account creation
logger.warn('Using root credentials for MinIO access. Consider implementing service accounts for production.');
// Set bucket policy to allow public read/write (services on the same network use root creds)
await this.execMc(platformService.containerId, [
'anonymous', 'set', 'none', `local/${bucketName}`,
]);
// Use container name for the endpoint in credentials (user services run in same network)
const serviceEndpoint = `http://${containerName}:9000`;
@@ -281,7 +225,7 @@ export class MinioProvider extends BasePlatformServiceProvider {
const credentials: Record<string, string> = {
endpoint: serviceEndpoint,
bucket: bucketName,
accessKey: adminCreds.username, // Using root for now
accessKey: adminCreds.username,
secretKey: adminCreds.password,
region: 'us-east-1',
};
@@ -312,57 +256,37 @@ export class MinioProvider extends BasePlatformServiceProvider {
const adminCreds = await credentialEncryption.decrypt(platformService.adminCredentialsEncrypted);
// Get container host port for connection from host (overlay network IPs not accessible from host)
const hostPort = await this.oneboxRef.docker.getContainerHostPort(platformService.containerId, 9000);
if (!hostPort) {
throw new Error('Could not get MinIO container host port');
}
logger.info(`Deprovisioning MinIO bucket '${resource.resourceName}'...`);
const { S3Client, DeleteBucketCommand, ListObjectsV2Command, DeleteObjectsCommand } = await import('npm:@aws-sdk/client-s3@3');
const s3Client = new S3Client({
endpoint: `http://127.0.0.1:${hostPort}`,
region: 'us-east-1',
credentials: {
accessKeyId: adminCreds.username,
secretAccessKey: adminCreds.password,
},
forcePathStyle: true,
});
// Configure mc alias
await this.execMc(platformService.containerId, [
'alias', 'set', 'local', 'http://localhost:9000',
adminCreds.username, adminCreds.password,
]);
try {
// First, delete all objects in the bucket
let continuationToken: string | undefined;
do {
const listResponse = await s3Client.send(new ListObjectsV2Command({
Bucket: resource.resourceName,
ContinuationToken: continuationToken,
}));
if (listResponse.Contents && listResponse.Contents.length > 0) {
await s3Client.send(new DeleteObjectsCommand({
Bucket: resource.resourceName,
Delete: {
Objects: listResponse.Contents.map(obj => ({ Key: obj.Key! })),
},
}));
logger.info(`Deleted ${listResponse.Contents.length} objects from bucket`);
}
continuationToken = listResponse.IsTruncated ? listResponse.NextContinuationToken : undefined;
} while (continuationToken);
// Now delete the bucket
await s3Client.send(new DeleteBucketCommand({
Bucket: resource.resourceName,
}));
// Remove all objects and the bucket
await this.execMc(platformService.containerId, [
'rb', '--force', `local/${resource.resourceName}`,
]);
logger.success(`MinIO bucket '${resource.resourceName}' deleted`);
} catch (e) {
logger.error(`Failed to delete MinIO bucket: ${getErrorMessage(e)}`);
throw e;
}
}
/**
* Execute mc (MinIO Client) command inside the container
*/
private async execMc(
containerId: string,
args: string[],
): Promise<{ stdout: string; stderr: string }> {
const result = await this.oneboxRef.docker.execInContainer(containerId, ['mc', ...args]);
if (result.exitCode !== 0) {
throw new Error(`mc command failed (exit ${result.exitCode}): ${result.stderr.substring(0, 200)}`);
}
return result;
}
}

104
ts/classes/platform-services/providers/mongodb.ts
View File

@@ -190,12 +190,6 @@ export class MongoDBProvider extends BasePlatformServiceProvider {
const adminCreds = await credentialEncryption.decrypt(platformService.adminCredentialsEncrypted);
const containerName = this.getContainerName();
// Get container host port for connection from host (overlay network IPs not accessible from host)
const hostPort = await this.oneboxRef.docker.getContainerHostPort(platformService.containerId, 27017);
if (!hostPort) {
throw new Error('Could not get MongoDB container host port');
}
// Generate resource names and credentials
const dbName = this.generateResourceName(userService.name);
const username = this.generateResourceName(userService.name);
@@ -203,32 +197,40 @@ export class MongoDBProvider extends BasePlatformServiceProvider {
logger.info(`Provisioning MongoDB database '${dbName}' for service '${userService.name}'...`);
// Connect to MongoDB via localhost and the mapped host port
const { MongoClient } = await import('npm:mongodb@6');
const adminUri = `mongodb://${adminCreds.username}:${adminCreds.password}@127.0.0.1:${hostPort}/?authSource=admin`;
// Use docker exec to provision inside the container (avoids host port mapping issues)
const escapedPassword = password.replace(/'/g, "'\\''");
const escapedAdminPassword = adminCreds.password.replace(/'/g, "'\\''");
const client = new MongoClient(adminUri);
await client.connect();
try {
// Create the database by switching to it (MongoDB creates on first write)
const db = client.db(dbName);
// Create a collection to ensure the database exists
await db.createCollection('_onebox_init');
// Create user with readWrite access to this database
await db.command({
createUser: username,
pwd: password,
roles: [{ role: 'readWrite', db: dbName }],
// Create database and user via mongosh inside the container
const mongoshScript = `
db = db.getSiblingDB('${dbName}');
db.createCollection('_onebox_init');
db.createUser({
user: '${username}',
pwd: '${escapedPassword}',
roles: [{ role: 'readWrite', db: '${dbName}' }]
});
print('PROVISION_SUCCESS');
`;
logger.success(`MongoDB database '${dbName}' provisioned with user '${username}'`);
} finally {
await client.close();
const result = await this.oneboxRef.docker.execInContainer(
platformService.containerId,
[
'mongosh',
'--username', adminCreds.username,
'--password', escapedAdminPassword,
'--authenticationDatabase', 'admin',
'--quiet',
'--eval', mongoshScript,
]
);
if (result.exitCode !== 0 || !result.stdout.includes('PROVISION_SUCCESS')) {
throw new Error(`Failed to provision MongoDB database: exit code ${result.exitCode}, output: ${result.stdout.substring(0, 200)} ${result.stderr.substring(0, 200)}`);
}
logger.success(`MongoDB database '${dbName}' provisioned with user '${username}'`);
// Build the credentials and env vars
const credentials: Record<string, string> = {
host: containerName,
@@ -262,37 +264,33 @@ export class MongoDBProvider extends BasePlatformServiceProvider {
}
const adminCreds = await credentialEncryption.decrypt(platformService.adminCredentialsEncrypted);
// Get container host port for connection from host (overlay network IPs not accessible from host)
const hostPort = await this.oneboxRef.docker.getContainerHostPort(platformService.containerId, 27017);
if (!hostPort) {
throw new Error('Could not get MongoDB container host port');
}
const escapedAdminPassword = adminCreds.password.replace(/'/g, "'\\''");
logger.info(`Deprovisioning MongoDB database '${resource.resourceName}'...`);
const { MongoClient } = await import('npm:mongodb@6');
const adminUri = `mongodb://${adminCreds.username}:${adminCreds.password}@127.0.0.1:${hostPort}/?authSource=admin`;
const mongoshScript = `
db = db.getSiblingDB('${resource.resourceName}');
try { db.dropUser('${credentials.username}'); } catch(e) { print('User drop failed: ' + e); }
db.dropDatabase();
print('DEPROVISION_SUCCESS');
`;
const client = new MongoClient(adminUri);
await client.connect();
const result = await this.oneboxRef.docker.execInContainer(
platformService.containerId,
[
'mongosh',
'--username', adminCreds.username,
'--password', escapedAdminPassword,
'--authenticationDatabase', 'admin',
'--quiet',
'--eval', mongoshScript,
]
);
try {
const db = client.db(resource.resourceName);
// Drop the user
try {
await db.command({ dropUser: credentials.username });
logger.info(`Dropped MongoDB user '${credentials.username}'`);
} catch (e) {
logger.warn(`Could not drop MongoDB user: ${getErrorMessage(e)}`);
}
// Drop the database
await db.dropDatabase();
logger.success(`MongoDB database '${resource.resourceName}' dropped`);
} finally {
await client.close();
if (result.exitCode !== 0) {
logger.warn(`MongoDB deprovision returned exit code ${result.exitCode}: ${result.stderr.substring(0, 200)}`);
}
logger.success(`MongoDB database '${resource.resourceName}' dropped`);
}
}

2
ts_bundled/bundle.ts
View File

File diff suppressed because one or more lines are too long

2
ts_web/00_commitinfo_data.ts
View File

@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@serve.zone/onebox',
version: '1.17.1',
version: '1.17.2',
description: 'Self-hosted container platform with automatic SSL and DNS - a mini Heroku for single servers'
}