serve.zone/dcrouter
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(dcrouter): add service manager lifecycle orchestration and health-based ops status reporting

Browse Source
This commit is contained in:
Jürgen Kunz
2026-03-20 15:35:10 +00:00
parent b62bad3616
commit 2cf362020f
8 changed files with 354 additions and 196 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ts/00_commitinfo_data.ts
View File

@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@serve.zone/dcrouter',
version: '11.8.11',
version: '11.9.0',
description: 'A multifaceted routing service handling mail and SMS delivery functions.'
}

442
ts/classes.dcrouter.ts
View File

@@ -252,6 +252,10 @@ export class DcRouter {
// Certificate provisioning scheduler with per-domain backoff
public certProvisionScheduler?: CertProvisionScheduler;
// Service lifecycle management
public serviceManager: plugins.taskbuffer.ServiceManager;
public smartAcmeReady = false;
// TypedRouter for API endpoints
public typedrouter = new plugins.typedrequest.TypedRouter();
@@ -279,67 +283,253 @@ export class DcRouter {
// Initialize storage manager
this.storageManager = new StorageManager(this.options.storage);
// Initialize service manager and register all services
this.serviceManager = new plugins.taskbuffer.ServiceManager({
name: 'dcrouter',
startupTimeoutMs: 120_000,
shutdownTimeoutMs: 30_000,
});
this.registerServices();
}
/**
* Register all dcrouter services with the ServiceManager.
* Services are started in dependency order, with failure isolation for optional services.
*/
private registerServices(): void {
// OpsServer: critical, no dependencies — provides visibility
this.serviceManager.addService(
new plugins.taskbuffer.Service('OpsServer')
.critical()
.withStart(async () => {
this.opsServer = new OpsServer(this);
await this.opsServer.start();
})
.withStop(async () => {
await this.opsServer?.stop();
})
.withRetry({ maxRetries: 0 }),
);
// CacheDb: optional, no dependencies
if (this.options.cacheConfig?.enabled !== false) {
this.serviceManager.addService(
new plugins.taskbuffer.Service('CacheDb')
.optional()
.withStart(async () => {
await this.setupCacheDb();
})
.withStop(async () => {
if (this.cacheCleaner) {
this.cacheCleaner.stop();
this.cacheCleaner = undefined;
}
if (this.cacheDb) {
await this.cacheDb.stop();
CacheDb.resetInstance();
this.cacheDb = undefined;
}
})
.withRetry({ maxRetries: 2, baseDelayMs: 1000, maxDelayMs: 5000 }),
);
}
// MetricsManager: optional, depends on OpsServer
this.serviceManager.addService(
new plugins.taskbuffer.Service('MetricsManager')
.optional()
.dependsOn('OpsServer')
.withStart(async () => {
this.metricsManager = new MetricsManager(this);
await this.metricsManager.start();
})
.withStop(async () => {
if (this.metricsManager) {
await this.metricsManager.stop();
this.metricsManager = undefined;
}
})
.withRetry({ maxRetries: 1, baseDelayMs: 1000 }),
);
// SmartProxy: critical, depends on CacheDb (if enabled)
const smartProxyDeps: string[] = [];
if (this.options.cacheConfig?.enabled !== false) {
smartProxyDeps.push('CacheDb');
}
this.serviceManager.addService(
new plugins.taskbuffer.Service('SmartProxy')
.critical()
.dependsOn(...smartProxyDeps)
.withStart(async () => {
await this.setupSmartProxy();
})
.withStop(async () => {
if (this.smartProxy) {
this.smartProxy.removeAllListeners();
await this.smartProxy.stop();
this.smartProxy = undefined;
}
})
.withRetry({ maxRetries: 0 }),
);
// SmartAcme: optional, depends on SmartProxy — aggressive retry for rate limits
// Only registered if DNS challenge is configured
if (this.options.dnsChallenge?.cloudflareApiKey) {
this.serviceManager.addService(
new plugins.taskbuffer.Service('SmartAcme')
.optional()
.dependsOn('SmartProxy')
.withStart(async () => {
if (this.smartAcme) {
await this.smartAcme.start();
this.smartAcmeReady = true;
logger.log('info', 'SmartAcme DNS-01 provider is now ready');
}
})
.withStop(async () => {
this.smartAcmeReady = false;
if (this.smartAcme) {
await this.smartAcme.stop();
this.smartAcme = undefined;
}
})
.withRetry({ maxRetries: 20, baseDelayMs: 5000, maxDelayMs: 3_600_000, backoffFactor: 2 }),
);
}
// ConfigManagers: optional, depends on SmartProxy
this.serviceManager.addService(
new plugins.taskbuffer.Service('ConfigManagers')
.optional()
.dependsOn('SmartProxy')
.withStart(async () => {
this.routeConfigManager = new RouteConfigManager(
this.storageManager,
() => this.getConstructorRoutes(),
() => this.smartProxy,
() => this.options.http3,
);
this.apiTokenManager = new ApiTokenManager(this.storageManager);
await this.apiTokenManager.initialize();
await this.routeConfigManager.initialize();
})
.withStop(async () => {
this.routeConfigManager = undefined;
this.apiTokenManager = undefined;
})
.withRetry({ maxRetries: 2, baseDelayMs: 1000 }),
);
// Email Server: optional, depends on SmartProxy
if (this.options.emailConfig) {
this.serviceManager.addService(
new plugins.taskbuffer.Service('EmailServer')
.optional()
.dependsOn('SmartProxy')
.withStart(async () => {
await this.setupUnifiedEmailHandling();
})
.withStop(async () => {
if (this.emailServer) {
if ((this.emailServer as any).deliverySystem) {
(this.emailServer as any).deliverySystem.removeAllListeners();
}
this.emailServer.removeAllListeners();
await this.emailServer.stop();
this.emailServer = undefined;
}
})
.withRetry({ maxRetries: 3, baseDelayMs: 2000, maxDelayMs: 30_000 }),
);
}
// DNS Server: optional, depends on SmartProxy
if (this.options.dnsNsDomains?.length > 0 && this.options.dnsScopes?.length > 0) {
this.serviceManager.addService(
new plugins.taskbuffer.Service('DnsServer')
.optional()
.dependsOn('SmartProxy')
.withStart(async () => {
await this.setupDnsWithSocketHandler();
})
.withStop(async () => {
// Flush pending DNS batch log
if (this.dnsBatchTimer) {
clearTimeout(this.dnsBatchTimer);
if (this.dnsBatchCount > 0) {
logger.log('info', `DNS: ${this.dnsBatchCount} queries processed (final flush)`, { zone: 'dns' });
}
this.dnsBatchTimer = null;
this.dnsBatchCount = 0;
this.dnsLogWindowSecond = 0;
this.dnsLogWindowCount = 0;
}
if (this.dnsServer) {
this.dnsServer.removeAllListeners();
await this.dnsServer.stop();
this.dnsServer = undefined;
}
})
.withRetry({ maxRetries: 3, baseDelayMs: 2000, maxDelayMs: 30_000 }),
);
}
// RADIUS Server: optional, no dependency on SmartProxy
if (this.options.radiusConfig) {
this.serviceManager.addService(
new plugins.taskbuffer.Service('RadiusServer')
.optional()
.withStart(async () => {
await this.setupRadiusServer();
})
.withStop(async () => {
if (this.radiusServer) {
await this.radiusServer.stop();
this.radiusServer = undefined;
}
})
.withRetry({ maxRetries: 3, baseDelayMs: 2000, maxDelayMs: 30_000 }),
);
}
// Remote Ingress: optional, depends on SmartProxy
if (this.options.remoteIngressConfig?.enabled) {
this.serviceManager.addService(
new plugins.taskbuffer.Service('RemoteIngress')
.optional()
.dependsOn('SmartProxy')
.withStart(async () => {
await this.setupRemoteIngress();
})
.withStop(async () => {
if (this.tunnelManager) {
await this.tunnelManager.stop();
this.tunnelManager = undefined;
}
this.remoteIngressManager = undefined;
})
.withRetry({ maxRetries: 3, baseDelayMs: 2000, maxDelayMs: 30_000 }),
);
}
// Wire up aggregated events for logging
this.serviceManager.serviceSubject.subscribe((event) => {
const level = event.type === 'failed' ? 'error' : event.type === 'retrying' ? 'warn' : 'info';
logger.log(level as any, `Service '${event.serviceName}': ${event.type}`, {
state: event.state,
...(event.error ? { error: event.error } : {}),
...(event.attempt ? { attempt: event.attempt } : {}),
});
});
}
public async start() {
logger.log('info', 'Starting DcRouter Services');
this.opsServer = new OpsServer(this);
await this.opsServer.start();
try {
// Initialize cache database if enabled (default: enabled)
if (this.options.cacheConfig?.enabled !== false) {
await this.setupCacheDb();
}
// Initialize MetricsManager
this.metricsManager = new MetricsManager(this);
await this.metricsManager.start();
// Set up SmartProxy for HTTP/HTTPS and all traffic including email routes
await this.setupSmartProxy();
// Initialize programmatic config API managers
this.routeConfigManager = new RouteConfigManager(
this.storageManager,
() => this.getConstructorRoutes(),
() => this.smartProxy,
() => this.options.http3,
);
this.apiTokenManager = new ApiTokenManager(this.storageManager);
await this.apiTokenManager.initialize();
await this.routeConfigManager.initialize();
// Set up unified email handling if configured
if (this.options.emailConfig) {
await this.setupUnifiedEmailHandling();
}
// Set up DNS server if configured with nameservers and scopes
if (this.options.dnsNsDomains && this.options.dnsNsDomains.length > 0 &&
this.options.dnsScopes && this.options.dnsScopes.length > 0) {
await this.setupDnsWithSocketHandler();
}
// Set up RADIUS server if configured
if (this.options.radiusConfig) {
await this.setupRadiusServer();
}
// Set up Remote Ingress hub if configured
if (this.options.remoteIngressConfig?.enabled) {
await this.setupRemoteIngress();
}
this.logStartupSummary();
} catch (error) {
logger.log('error', 'Error starting DcRouter', { error: String(error) });
// Try to clean up any services that may have started
await this.stop();
throw error;
}
await this.serviceManager.start();
this.logStartupSummary();
}
/**
@@ -399,7 +589,21 @@ export class DcRouter {
logger.log('info', `Cache Database: storage=${this.cacheDb.getStoragePath()}, db=${this.cacheDb.getDbName()}, cleaner=${this.cacheCleaner?.isActive() ? 'active' : 'inactive'} (${(this.options.cacheConfig?.cleanupIntervalHours || 1)}h interval)`);
}
logger.log('info', 'All services are running');
// Service status summary from ServiceManager
const health = this.serviceManager.getHealth();
const statuses = health.services;
const running = statuses.filter(s => s.state === 'running').length;
const failed = statuses.filter(s => s.state === 'failed').length;
const retrying = statuses.filter(s => s.state === 'starting' || s.state === 'degraded').length;
if (failed > 0) {
const failedNames = statuses.filter(s => s.state === 'failed').map(s => `${s.name}: ${s.lastError || 'unknown'}`);
logger.log('warn', `DcRouter started in degraded mode — ${running} running, ${failed} failed: ${failedNames.join('; ')}`);
} else if (retrying > 0) {
logger.log('info', `DcRouter started — ${running} running, ${retrying} still initializing`);
} else {
logger.log('info', `All ${running} services are running`);
}
}
/**
@@ -535,10 +739,13 @@ export class DcRouter {
// Initialize cert provision scheduler
this.certProvisionScheduler = new CertProvisionScheduler(this.storageManager);
// If we have DNS challenge handlers, create SmartAcme and wire to certProvisionFunction
// If we have DNS challenge handlers, create SmartAcme instance and wire certProvisionFunction
// Note: SmartAcme.start() is NOT called here — it runs as a separate optional service
// via the ServiceManager, with aggressive retry for rate-limit resilience.
if (challengeHandlers.length > 0) {
// Stop old SmartAcme if it exists (e.g., during updateSmartProxyConfig)
if (this.smartAcme) {
this.smartAcmeReady = false;
await this.smartAcme.stop().catch(err =>
logger.log('error', 'Error stopping old SmartAcme', { error: String(err) })
);
@@ -550,10 +757,15 @@ export class DcRouter {
challengeHandlers: challengeHandlers,
challengePriority: ['dns-01'],
});
await this.smartAcme.start();
const scheduler = this.certProvisionScheduler;
smartProxyConfig.certProvisionFunction = async (domain, eventComms) => {
// If SmartAcme is not yet ready (still starting or retrying), fall back to HTTP-01
if (!this.smartAcmeReady) {
eventComms.warn(`SmartAcme not yet initialized, falling back to http-01 for ${domain}`);
return 'http01';
}
// Check backoff before attempting provision
if (await scheduler.isInBackoff(domain)) {
const info = await scheduler.getBackoffInfo(domain);
@@ -914,105 +1126,23 @@ export class DcRouter {
public async stop() {
logger.log('info', 'Stopping DcRouter services...');
// Flush pending DNS batch log
if (this.dnsBatchTimer) {
clearTimeout(this.dnsBatchTimer);
if (this.dnsBatchCount > 0) {
logger.log('info', `DNS: ${this.dnsBatchCount} queries processed (rate limited, final flush)`, { zone: 'dns' });
}
this.dnsBatchTimer = null;
this.dnsBatchCount = 0;
this.dnsLogWindowSecond = 0;
this.dnsLogWindowCount = 0;
}
// ServiceManager handles reverse-dependency-ordered shutdown
await this.serviceManager.stop();
await this.opsServer.stop();
try {
// Remove event listeners before stopping services to prevent leaks
if (this.smartProxy) {
this.smartProxy.removeAllListeners();
}
if (this.emailServer) {
if ((this.emailServer as any).deliverySystem) {
(this.emailServer as any).deliverySystem.removeAllListeners();
}
this.emailServer.removeAllListeners();
}
if (this.dnsServer) {
this.dnsServer.removeAllListeners();
}
// Stop all services in parallel for faster shutdown
await Promise.all([
// Stop cache cleaner if running
this.cacheCleaner ? Promise.resolve(this.cacheCleaner.stop()) : Promise.resolve(),
// Stop metrics manager if running
this.metricsManager ? this.metricsManager.stop().catch(err => logger.log('error', 'Error stopping MetricsManager', { error: String(err) })) : Promise.resolve(),
// Stop unified email server if running
this.emailServer ? this.emailServer.stop().catch(err => logger.log('error', 'Error stopping email server', { error: String(err) })) : Promise.resolve(),
// Stop SmartAcme if running
this.smartAcme ? this.smartAcme.stop().catch(err => logger.log('error', 'Error stopping SmartAcme', { error: String(err) })) : Promise.resolve(),
// Stop HTTP SmartProxy if running
this.smartProxy ? this.smartProxy.stop().catch(err => logger.log('error', 'Error stopping SmartProxy', { error: String(err) })) : Promise.resolve(),
// Stop DNS server if running
this.dnsServer ?
this.dnsServer.stop().catch(err => logger.log('error', 'Error stopping DNS server', { error: String(err) })) :
Promise.resolve(),
// Stop RADIUS server if running
this.radiusServer ?
this.radiusServer.stop().catch(err => logger.log('error', 'Error stopping RADIUS server', { error: String(err) })) :
Promise.resolve(),
// Stop Remote Ingress tunnel manager if running
this.tunnelManager ?
this.tunnelManager.stop().catch(err => logger.log('error', 'Error stopping TunnelManager', { error: String(err) })) :
Promise.resolve()
]);
// Stop cache database after other services (they may need it during shutdown)
if (this.cacheDb) {
await this.cacheDb.stop().catch(err => logger.log('error', 'Error stopping CacheDb', { error: String(err) }));
CacheDb.resetInstance();
}
// Clear backoff cache in cert scheduler
if (this.certProvisionScheduler) {
this.certProvisionScheduler.clear();
}
// Allow GC of stopped services by nulling references
this.smartProxy = undefined;
this.emailServer = undefined;
this.dnsServer = undefined;
this.metricsManager = undefined;
this.cacheCleaner = undefined;
this.cacheDb = undefined;
this.tunnelManager = undefined;
this.radiusServer = undefined;
this.smartAcme = undefined;
// Clear backoff cache in cert scheduler
if (this.certProvisionScheduler) {
this.certProvisionScheduler.clear();
this.certProvisionScheduler = undefined;
this.remoteIngressManager = undefined;
this.routeConfigManager = undefined;
this.apiTokenManager = undefined;
this.certificateStatusMap.clear();
// Reset security singletons to allow GC
SecurityLogger.resetInstance();
ContentScanner.resetInstance();
IPReputationChecker.resetInstance();
logger.log('info', 'All DcRouter services stopped');
} catch (error) {
logger.log('error', 'Error during DcRouter shutdown', { error: String(error) });
throw error;
}
this.certificateStatusMap.clear();
// Reset security singletons to allow GC
SecurityLogger.resetInstance();
ContentScanner.resetInstance();
IPReputationChecker.resetInstance();
logger.log('info', 'All DcRouter services stopped');
}
/**

71
ts/opsserver/handlers/stats.handler.ts
View File

@@ -489,44 +489,41 @@ export class StatsHandler {
message?: string;
}>;
}> {
const services: Array<{
name: string;
status: 'healthy' | 'degraded' | 'unhealthy';
message?: string;
}> = [];
// Check HTTP Proxy
if (this.opsServerRef.dcRouterRef.smartProxy) {
services.push({
name: 'HTTP/HTTPS Proxy',
status: 'healthy',
});
}
// Check Email Server
if (this.opsServerRef.dcRouterRef.emailServer) {
services.push({
name: 'Email Server',
status: 'healthy',
});
}
// Check DNS Server
if (this.opsServerRef.dcRouterRef.dnsServer) {
services.push({
name: 'DNS Server',
status: 'healthy',
});
}
// Check OpsServer
services.push({
name: 'OpsServer',
status: 'healthy',
const dcRouter = this.opsServerRef.dcRouterRef;
const health = dcRouter.serviceManager.getHealth();
const services = health.services.map((svc) => {
let status: 'healthy' | 'degraded' | 'unhealthy';
switch (svc.state) {
case 'running':
status = 'healthy';
break;
case 'starting':
case 'degraded':
status = 'degraded';
break;
case 'failed':
status = svc.criticality === 'critical' ? 'unhealthy' : 'degraded';
break;
case 'stopped':
case 'stopping':
default:
status = 'degraded';
break;
}
let message: string | undefined;
if (svc.state === 'failed' && svc.lastError) {
message = svc.lastError;
} else if (svc.retryCount > 0 && svc.state !== 'running') {
message = `Retry attempt ${svc.retryCount}`;
}
return { name: svc.name, status, message };
});
const healthy = services.every(s => s.status === 'healthy');
const healthy = health.overall === 'healthy';
return {
healthy,
services,

3
ts/plugins.ts
View File

@@ -62,8 +62,9 @@ import * as smartradius from '@push.rocks/smartradius';
import * as smartrequest from '@push.rocks/smartrequest';
import * as smartrx from '@push.rocks/smartrx';
import * as smartunique from '@push.rocks/smartunique';
import * as taskbuffer from '@push.rocks/taskbuffer';
export { projectinfo, qenv, smartacme, smartdata, smartdns, smartfile, smartguard, smartjwt, smartlog, smartmetrics, smartmongo, smartmta, smartnetwork, smartpath, smartproxy, smartpromise, smartradius, smartrequest, smartrx, smartunique };
export { projectinfo, qenv, smartacme, smartdata, smartdns, smartfile, smartguard, smartjwt, smartlog, smartmetrics, smartmongo, smartmta, smartnetwork, smartpath, smartproxy, smartpromise, smartradius, smartrequest, smartrx, smartunique, taskbuffer };
// Define SmartLog types for use in error handling
export type TLogLevel = 'error' | 'warn' | 'info' | 'success' | 'debug';