serve.zone/onebox
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(platform-services): Add platform service log streaming, improve health checks and provisioning robustness

Browse Source
This commit is contained in:
Jürgen Kunz
2025-11-26 18:20:02 +00:00
parent 9de32cd00d
commit 3fbcaee56e
9 changed files with 515 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
ts/classes/platform-services/manager.ts
View File

@@ -93,6 +93,8 @@ export class PlatformServicesManager {
}
// Check if already running
let needsDeploy = platformService.status !== 'running';
if (platformService.status === 'running') {
// Verify it's actually healthy
const isHealthy = await provider.healthCheck();
@@ -100,11 +102,14 @@ export class PlatformServicesManager {
logger.debug(`${provider.displayName} is already running and healthy`);
return platformService;
}
logger.warn(`${provider.displayName} reports running but health check failed, restarting...`);
logger.warn(`${provider.displayName} reports running but health check failed, will redeploy...`);
// Mark status as needing redeploy - container may have been recreated with different credentials
this.oneboxRef.database.updatePlatformService(platformService.id!, { status: 'stopped' });
needsDeploy = true;
}
// Deploy if not running
if (platformService.status !== 'running') {
// Deploy if needed
if (needsDeploy) {
logger.info(`Starting ${provider.displayName} platform service...`);
try {
@@ -143,19 +148,28 @@ export class PlatformServicesManager {
*/
private async waitForHealthy(type: TPlatformServiceType, timeoutMs: number): Promise<boolean> {
const provider = this.providers.get(type);
if (!provider) return false;
if (!provider) {
logger.warn(`waitForHealthy: no provider for type ${type}`);
return false;
}
logger.info(`waitForHealthy: starting health check loop for ${type} (timeout: ${timeoutMs}ms)`);
const startTime = Date.now();
const checkInterval = 2000; // Check every 2 seconds
let checkCount = 0;
while (Date.now() - startTime < timeoutMs) {
checkCount++;
logger.info(`waitForHealthy: health check attempt #${checkCount} for ${type}`);
const isHealthy = await provider.healthCheck();
if (isHealthy) {
logger.info(`waitForHealthy: ${type} became healthy after ${checkCount} attempts`);
return true;
}
await new Promise((resolve) => setTimeout(resolve, checkInterval));
}
logger.warn(`waitForHealthy: ${type} did not become healthy after ${checkCount} attempts (${timeoutMs}ms)`);
return false;
}

15
ts/classes/platform-services/providers/minio.ts
View File

@@ -118,8 +118,19 @@ export class MinioProvider extends BasePlatformServiceProvider {
async healthCheck(): Promise<boolean> {
try {
const containerName = this.getContainerName();
const endpoint = `http://${containerName}:9000/minio/health/live`;
const platformService = this.oneboxRef.database.getPlatformServiceByType(this.type);
if (!platformService || !platformService.containerId) {
return false;
}
// Get container IP for health check (hostname won't resolve from host)
const containerIP = await this.oneboxRef.docker.getContainerIP(platformService.containerId);
if (!containerIP) {
logger.debug('MinIO health check: could not get container IP');
return false;
}
const endpoint = `http://${containerIP}:9000/minio/health/live`;
const response = await fetch(endpoint, {
method: 'GET',

119
ts/classes/platform-services/providers/mongodb.ts
View File

@@ -52,21 +52,52 @@ export class MongoDBProvider extends BasePlatformServiceProvider {
async deployContainer(): Promise<string> {
const config = this.getDefaultConfig();
const containerName = this.getContainerName();
// Generate admin password
const adminPassword = credentialEncryption.generatePassword(32);
// Store admin credentials encrypted in the platform service record
const adminCredentials = {
username: 'admin',
password: adminPassword,
};
const dataDir = '/var/lib/onebox/mongodb';
logger.info(`Deploying MongoDB platform service as ${containerName}...`);
// Check if we have existing data and stored credentials
const platformService = this.oneboxRef.database.getPlatformServiceByType(this.type);
let adminCredentials: { username: string; password: string };
let dataExists = false;
// Check if data directory has existing MongoDB data
try {
const stat = await Deno.stat(`${dataDir}/WiredTiger`);
dataExists = stat.isFile;
logger.info(`MongoDB data directory exists with WiredTiger file`);
} catch {
// WiredTiger file doesn't exist, this is a fresh install
dataExists = false;
}
if (dataExists && platformService?.adminCredentialsEncrypted) {
// Reuse existing credentials from database
logger.info('Reusing existing MongoDB credentials (data directory already initialized)');
adminCredentials = await credentialEncryption.decrypt(platformService.adminCredentialsEncrypted);
} else {
// Generate new credentials for fresh deployment
logger.info('Generating new MongoDB admin credentials');
adminCredentials = {
username: 'admin',
password: credentialEncryption.generatePassword(32),
};
// If data exists but we don't have credentials, we need to wipe the data
if (dataExists) {
logger.warn('MongoDB data exists but no credentials in database - wiping data directory');
try {
await Deno.remove(dataDir, { recursive: true });
} catch (e) {
logger.error(`Failed to wipe MongoDB data directory: ${getErrorMessage(e)}`);
throw new Error('Cannot deploy MongoDB: data directory exists without credentials');
}
}
}
// Ensure data directory exists
try {
await Deno.mkdir('/var/lib/onebox/mongodb', { recursive: true });
await Deno.mkdir(dataDir, { recursive: true });
} catch (e) {
// Directory might already exist
if (!(e instanceof Deno.errors.AlreadyExists)) {
@@ -90,9 +121,8 @@ export class MongoDBProvider extends BasePlatformServiceProvider {
network: this.getNetworkName(),
});
// Store encrypted admin credentials
// Store encrypted admin credentials (only update if new or changed)
const encryptedCreds = await credentialEncryption.encrypt(adminCredentials);
const platformService = this.oneboxRef.database.getPlatformServiceByType(this.type);
if (platformService) {
this.oneboxRef.database.updatePlatformService(platformService.id!, {
containerId,
@@ -113,43 +143,59 @@ export class MongoDBProvider extends BasePlatformServiceProvider {
async healthCheck(): Promise<boolean> {
try {
logger.info('MongoDB health check: starting...');
const platformService = this.oneboxRef.database.getPlatformServiceByType(this.type);
if (!platformService || !platformService.adminCredentialsEncrypted) {
if (!platformService) {
logger.info('MongoDB health check: platform service not found in database');
return false;
}
if (!platformService.adminCredentialsEncrypted) {
logger.info('MongoDB health check: no admin credentials stored');
return false;
}
if (!platformService.containerId) {
logger.info('MongoDB health check: no container ID in database record');
return false;
}
logger.info(`MongoDB health check: using container ID ${platformService.containerId.substring(0, 12)}...`);
const adminCreds = await credentialEncryption.decrypt(platformService.adminCredentialsEncrypted);
const containerName = this.getContainerName();
// Try to connect to MongoDB using mongosh ping
const { MongoClient } = await import('npm:mongodb@6');
const uri = `mongodb://${adminCreds.username}:${adminCreds.password}@${containerName}:27017/?authSource=admin`;
// Use docker exec to run health check inside the container
// This avoids network issues with overlay networks
const result = await this.oneboxRef.docker.execInContainer(
platformService.containerId,
['mongosh', '--eval', 'db.adminCommand("ping")', '--username', adminCreds.username, '--password', adminCreds.password, '--authenticationDatabase', 'admin', '--quiet']
);
const client = new MongoClient(uri, {
serverSelectionTimeoutMS: 5000,
connectTimeoutMS: 5000,
});
await client.connect();
await client.db('admin').command({ ping: 1 });
await client.close();
return true;
if (result.exitCode === 0) {
logger.info('MongoDB health check: success');
return true;
} else {
logger.info(`MongoDB health check failed: exit code ${result.exitCode}, stderr: ${result.stderr.substring(0, 200)}`);
return false;
}
} catch (error) {
logger.debug(`MongoDB health check failed: ${getErrorMessage(error)}`);
logger.info(`MongoDB health check exception: ${getErrorMessage(error)}`);
return false;
}
}
async provisionResource(userService: IService): Promise<IProvisionedResource> {
const platformService = this.oneboxRef.database.getPlatformServiceByType(this.type);
if (!platformService || !platformService.adminCredentialsEncrypted) {
if (!platformService || !platformService.adminCredentialsEncrypted || !platformService.containerId) {
throw new Error('MongoDB platform service not found or not configured');
}
const adminCreds = await credentialEncryption.decrypt(platformService.adminCredentialsEncrypted);
const containerName = this.getContainerName();
// Get container host port for connection from host (overlay network IPs not accessible from host)
const hostPort = await this.oneboxRef.docker.getContainerHostPort(platformService.containerId, 27017);
if (!hostPort) {
throw new Error('Could not get MongoDB container host port');
}
// Generate resource names and credentials
const dbName = this.generateResourceName(userService.name);
const username = this.generateResourceName(userService.name);
@@ -157,9 +203,9 @@ export class MongoDBProvider extends BasePlatformServiceProvider {
logger.info(`Provisioning MongoDB database '${dbName}' for service '${userService.name}'...`);
// Connect to MongoDB and create database/user
// Connect to MongoDB via localhost and the mapped host port
const { MongoClient } = await import('npm:mongodb@6');
const adminUri = `mongodb://${adminCreds.username}:${adminCreds.password}@${containerName}:27017/?authSource=admin`;
const adminUri = `mongodb://${adminCreds.username}:${adminCreds.password}@127.0.0.1:${hostPort}/?authSource=admin`;
const client = new MongoClient(adminUri);
await client.connect();
@@ -211,17 +257,22 @@ export class MongoDBProvider extends BasePlatformServiceProvider {
async deprovisionResource(resource: IPlatformResource, credentials: Record<string, string>): Promise<void> {
const platformService = this.oneboxRef.database.getPlatformServiceByType(this.type);
if (!platformService || !platformService.adminCredentialsEncrypted) {
if (!platformService || !platformService.adminCredentialsEncrypted || !platformService.containerId) {
throw new Error('MongoDB platform service not found or not configured');
}
const adminCreds = await credentialEncryption.decrypt(platformService.adminCredentialsEncrypted);
const containerName = this.getContainerName();
// Get container host port for connection from host (overlay network IPs not accessible from host)
const hostPort = await this.oneboxRef.docker.getContainerHostPort(platformService.containerId, 27017);
if (!hostPort) {
throw new Error('Could not get MongoDB container host port');
}
logger.info(`Deprovisioning MongoDB database '${resource.resourceName}'...`);
const { MongoClient } = await import('npm:mongodb@6');
const adminUri = `mongodb://${adminCreds.username}:${adminCreds.password}@${containerName}:27017/?authSource=admin`;
const adminUri = `mongodb://${adminCreds.username}:${adminCreds.password}@127.0.0.1:${hostPort}/?authSource=admin`;
const client = new MongoClient(adminUri);
await client.connect();