v11.9.1

changelog.md

@@ -1,5 +1,76 @@
 # Changelog
 
+## 2026-03-21 - 11.9.1 - fix(lifecycle)
+clean up service subscriptions, proxy retries, and stale runtime state on shutdown
+
+- unsubscribe from ServiceManager event streams and use one-time signal handlers to avoid duplicate shutdown execution
+- reset existing SmartProxy instances before retry setup and prune expired certificate backoff cache entries
+- add periodic sweeping and shutdown cleanup for stale RADIUS accounting sessions
+
+## 2026-03-20 - 11.9.0 - feat(dcrouter)
+add service manager lifecycle orchestration and health-based ops status reporting
+
+- register dcrouter components with a taskbuffer ServiceManager using dependencies, retries, and critical/optional service roles
+- update ops stats health output to reflect aggregated service manager state and per-service error or retry details
+- add @push.rocks/taskbuffer to shared plugins and project dependencies for service lifecycle management
+
+## 2026-03-20 - 11.8.11 - fix(deps)
+bump @push.rocks/smartproxy to ^25.17.10
+
+- Updates the @push.rocks/smartproxy dependency from ^25.17.9 to ^25.17.10 in package.json
+
+## 2026-03-20 - 11.8.10 - fix(deps)
+bump @push.rocks/smartproxy to ^25.17.9
+
+- Updates @push.rocks/smartproxy from ^25.17.8 to ^25.17.9 in package.json
+
+## 2026-03-20 - 11.8.9 - fix(deps)
+bump @push.rocks/smartproxy to ^25.17.8
+
+- Updates the @push.rocks/smartproxy dependency from ^25.17.7 to ^25.17.8.
+
+## 2026-03-20 - 11.8.8 - fix(deps)
+bump @push.rocks/smartproxy to ^25.17.7
+
+- Updates the @push.rocks/smartproxy dependency from ^25.17.4 to ^25.17.7 in package.json.
+
+## 2026-03-20 - 11.8.7 - fix(deps)
+bump @push.rocks/smartproxy to ^25.17.4
+
+- updates @push.rocks/smartproxy from ^25.17.3 to ^25.17.4 in package.json
+
+## 2026-03-20 - 11.8.6 - fix(deps)
+bump @push.rocks/smartproxy to ^25.17.3
+
+- updates @push.rocks/smartproxy from ^25.17.1 to ^25.17.3 in package.json
+
+## 2026-03-20 - 11.8.5 - fix(deps)
+bump @push.rocks/smartproxy to ^25.17.1
+
+- Updates the @push.rocks/smartproxy dependency from ^25.17.0 to ^25.17.1.
+
+## 2026-03-20 - 11.8.4 - fix(deps)
+bump @serve.zone/remoteingress to ^4.14.0
+
+- Updates the @serve.zone/remoteingress dependency from ^4.13.2 to ^4.14.0 in package.json.
+
+## 2026-03-20 - 11.8.3 - fix(deps)
+bump @serve.zone/remoteingress to ^4.13.2
+
+- Updates the @serve.zone/remoteingress dependency from ^4.13.1 to ^4.13.2.
+
+## 2026-03-19 - 11.8.2 - fix(deps)
+bump smartproxy and remoteingress dependencies
+
+- updates @push.rocks/smartproxy from ^25.16.3 to ^25.17.0
+- updates @serve.zone/remoteingress from ^4.13.0 to ^4.13.1
+
+## 2026-03-19 - 11.8.1 - fix(dcrouter)
+use constructor routes for remote ingress setup and bump smartproxy dependency
+
+- Switch remote ingress initialization to use constructorRoutes instead of smartProxyConfig routes so derived edge ports are based on the active route set.
+- Update @push.rocks/smartproxy from ^25.16.2 to ^25.16.3.
+
 ## 2026-03-19 - 11.8.0 - feat(remoteingress)
 add UDP listen port derivation and edge configuration support
 

package.json

@@ -1,7 +1,7 @@
 {
   "name": "@serve.zone/dcrouter",
   "private": false,
-  "version": "11.8.0",
+  "version": "11.9.1",
   "description": "A multifaceted routing service handling mail and SMS delivery functions.",
   "type": "module",
   "exports": {
@@ -53,15 +53,16 @@
     "@push.rocks/smartnetwork": "^4.4.0",
     "@push.rocks/smartpath": "^6.0.0",
     "@push.rocks/smartpromise": "^4.2.3",
-    "@push.rocks/smartproxy": "^25.16.2",
+    "@push.rocks/smartproxy": "^26.0.0",
     "@push.rocks/smartradius": "^1.1.1",
     "@push.rocks/smartrequest": "^5.0.1",
     "@push.rocks/smartrx": "^3.0.10",
     "@push.rocks/smartstate": "^2.2.0",
     "@push.rocks/smartunique": "^3.0.9",
+    "@push.rocks/taskbuffer": "^8.0.0",
     "@serve.zone/catalog": "^2.9.0",
     "@serve.zone/interfaces": "^5.3.0",
-    "@serve.zone/remoteingress": "^4.13.0",
+    "@serve.zone/remoteingress": "^4.14.1",
     "@tsclass/tsclass": "^9.4.0",
     "lru-cache": "^11.2.7",
     "uuid": "^13.0.0"

pnpm-lock.yaml

@@ -78,8 +78,8 @@ importers:
         specifier: ^4.2.3
         version: 4.2.3
       '@push.rocks/smartproxy':
-        specifier: ^25.16.2
+        specifier: ^26.0.0
-        version: 25.16.2
+        version: 26.0.0
       '@push.rocks/smartradius':
         specifier: ^1.1.1
         version: 1.1.1
@@ -95,6 +95,9 @@ importers:
       '@push.rocks/smartunique':
         specifier: ^3.0.9
         version: 3.0.9
+      '@push.rocks/taskbuffer':
+        specifier: ^8.0.0
+        version: 8.0.0
       '@serve.zone/catalog':
         specifier: ^2.9.0
         version: 2.9.0(@tiptap/pm@2.27.2)
@@ -102,8 +105,8 @@ importers:
         specifier: ^5.3.0
         version: 5.3.0
       '@serve.zone/remoteingress':
-        specifier: ^4.13.0
+        specifier: ^4.14.1
-        version: 4.13.0
+        version: 4.14.1
       '@tsclass/tsclass':
         specifier: ^9.4.0
         version: 9.5.0
@@ -1256,8 +1259,8 @@ packages:
   '@push.rocks/smartpromise@4.2.3':
     resolution: {integrity: sha512-Ycg/TJR+tMt+S3wSFurOpEoW6nXv12QBtKXgBcjMZ4RsdO28geN46U09osPn9N9WuwQy1PkmTV5J/V4F9U8qEw==}
 
-  '@push.rocks/smartproxy@25.16.2':
+  '@push.rocks/smartproxy@26.0.0':
-    resolution: {integrity: sha512-o42ipMKmgfl5dOVWF3lCLYrCxLTl+OGvwmtK8smqFG6BCfUvJrd38zDH2j3fA6bxKdIgr5rJFDSjJypznL+6Gg==}
+    resolution: {integrity: sha512-fGLSVGCMEnmRFzt1iwiOjaOv6fB94fJgmtU13c9IHrpcuoPL2BhJqY+vj0bEgh2ee1F1fos3oARHKf4dwoeS6w==}
 
   '@push.rocks/smartpuppeteer@2.0.5':
     resolution: {integrity: sha512-yK/qSeWVHIGWRp3c8S5tfdGP6WCKllZC4DR8d8CQlEjszOSBmHtlTdyyqOMBZ/BA4kd+eU5f3A1r4K2tGYty1g==}
@@ -1341,6 +1344,9 @@ packages:
   '@push.rocks/taskbuffer@6.1.2':
     resolution: {integrity: sha512-sdqKd8N/GidztQ1k3r8A86rLvD8Afyir5FjYCNJXDD9837JLoqzHaOKGltUSBsCGh2gjsZn6GydsY6HhXQgvZQ==}
 
+  '@push.rocks/taskbuffer@8.0.0':
+    resolution: {integrity: sha512-ay4iXz0JmvsCQCmh5vvuu6KAl8FEZm5EpDXMQbeU+563d89xn+vMhh4+PtwxrVCogMEULWgGnavDYPTuzWtJOA==}
+
   '@push.rocks/webrequest@4.0.5':
     resolution: {integrity: sha512-wVSCaXqJ9Vh+rbwVz0wDl46dYz4rnwwSrm5vbVXKbuH6oKTPF0YRoujeJPqRltIn64RVGdLeY9/6ix+ZCrzhsg==}
 
@@ -1550,8 +1556,8 @@ packages:
   '@serve.zone/interfaces@5.3.0':
     resolution: {integrity: sha512-venO7wtDR9ixzD9NhdERBGjNKbFA5LL0yHw4eqGh0UpmvtXVc3SFG0uuHDilOKMZqZ8bttV88qVsFy1aSTJrtA==}
 
-  '@serve.zone/remoteingress@4.13.0':
+  '@serve.zone/remoteingress@4.14.1':
-    resolution: {integrity: sha512-Gw/yIgCukh3kImIco3u9B+b2cQP4l88RgCdP7NhYpwTDrI9jrsKmrzq0cRXo/Lnja35RZ1D7fBmNvaaAqEToVQ==}
+    resolution: {integrity: sha512-rYM4msFwo9SPxgNp/qXkJCQ8uXvQiMcH3cQEyciLKJ+7HwqKwQLCK4kbl45r/rzRAVOjyxXi0ae3hjgOBzTbyw==}
 
   '@sindresorhus/is@5.6.0':
     resolution: {integrity: sha512-TV7t8GKYaJWsn00tFDqBw8+Uqmr8A0fRU1tvTQhyZzGv0sJCGRQL3JGMI3ucuKo3XIZdUP+Lx7/gh2t3lewy7g==}
@@ -6539,7 +6545,7 @@ snapshots:
 
   '@push.rocks/smartpromise@4.2.3': {}
 
-  '@push.rocks/smartproxy@25.16.2':
+  '@push.rocks/smartproxy@26.0.0':
     dependencies:
       '@push.rocks/smartcrypto': 2.0.4
       '@push.rocks/smartlog': 3.2.1
@@ -6758,6 +6764,22 @@ snapshots:
       - supports-color
       - vue
 
+  '@push.rocks/taskbuffer@8.0.0':
+    dependencies:
+      '@design.estate/dees-element': 2.2.3
+      '@push.rocks/lik': 6.3.1
+      '@push.rocks/smartdelay': 3.0.5
+      '@push.rocks/smartlog': 3.2.1
+      '@push.rocks/smartpromise': 4.2.3
+      '@push.rocks/smartrx': 3.0.10
+      '@push.rocks/smarttime': 4.2.3
+      '@push.rocks/smartunique': 3.0.9
+    transitivePeerDependencies:
+      - '@nuxt/kit'
+      - react
+      - supports-color
+      - vue
+
   '@push.rocks/webrequest@4.0.5':
     dependencies:
       '@push.rocks/smartdelay': 3.0.5
@@ -6956,7 +6978,7 @@ snapshots:
       '@push.rocks/smartlog-interfaces': 3.0.2
       '@tsclass/tsclass': 9.5.0
 
-  '@serve.zone/remoteingress@4.13.0':
+  '@serve.zone/remoteingress@4.14.1':
     dependencies:
       '@push.rocks/qenv': 6.1.3
       '@push.rocks/smartrust': 1.3.2

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@serve.zone/dcrouter',
-  version: '11.8.0',
+  version: '11.9.1',
   description: 'A multifaceted routing service handling mail and SMS delivery functions.'
 }

ts/classes.cert-provision-scheduler.ts

@@ -61,14 +61,21 @@ export class CertProvisionScheduler {
   }
 
   /**
-   * Check if a domain is currently in backoff
+   * Check if a domain is currently in backoff.
+   * Expired entries are pruned from the cache to prevent unbounded growth.
    */
   async isInBackoff(domain: string): Promise<boolean> {
     const entry = await this.loadBackoff(domain);
     if (!entry) return false;
 
     const retryAfter = new Date(entry.retryAfter);
-    return retryAfter.getTime() > Date.now();
+    if (retryAfter.getTime() > Date.now()) {
+      return true;
+    }
+
+    // Backoff has expired — prune the stale entry
+    this.backoffCache.delete(domain);
+    return false;
   }
 
   /**
@@ -124,9 +131,12 @@ export class CertProvisionScheduler {
     const entry = await this.loadBackoff(domain);
     if (!entry) return null;
 
-    // Only return if still in backoff
+    // Only return if still in backoff — prune expired entries
     const retryAfter = new Date(entry.retryAfter);
-    if (retryAfter.getTime() <= Date.now()) return null;
+    if (retryAfter.getTime() <= Date.now()) {
+      this.backoffCache.delete(domain);
+      return null;
+    }
 
     return {
       failures: entry.failures,

ts/classes.dcrouter.ts

@@ -252,6 +252,11 @@ export class DcRouter {
   // Certificate provisioning scheduler with per-domain backoff
   public certProvisionScheduler?: CertProvisionScheduler;
 
+  // Service lifecycle management
+  public serviceManager: plugins.taskbuffer.ServiceManager;
+  private serviceSubjectSubscription?: plugins.smartrx.rxjs.Subscription;
+  public smartAcmeReady = false;
+
   // TypedRouter for API endpoints
   public typedrouter = new plugins.typedrequest.TypedRouter();
 
@@ -279,67 +284,253 @@ export class DcRouter {
 
     // Initialize storage manager
     this.storageManager = new StorageManager(this.options.storage);
+
+    // Initialize service manager and register all services
+    this.serviceManager = new plugins.taskbuffer.ServiceManager({
+      name: 'dcrouter',
+      startupTimeoutMs: 120_000,
+      shutdownTimeoutMs: 30_000,
+    });
+    this.registerServices();
+  }
+
+  /**
+   * Register all dcrouter services with the ServiceManager.
+   * Services are started in dependency order, with failure isolation for optional services.
+   */
+  private registerServices(): void {
+    // OpsServer: critical, no dependencies — provides visibility
+    this.serviceManager.addService(
+      new plugins.taskbuffer.Service('OpsServer')
+        .critical()
+        .withStart(async () => {
+          this.opsServer = new OpsServer(this);
+          await this.opsServer.start();
+        })
+        .withStop(async () => {
+          await this.opsServer?.stop();
+        })
+        .withRetry({ maxRetries: 0 }),
+    );
+
+    // CacheDb: optional, no dependencies
+    if (this.options.cacheConfig?.enabled !== false) {
+      this.serviceManager.addService(
+        new plugins.taskbuffer.Service('CacheDb')
+          .optional()
+          .withStart(async () => {
+            await this.setupCacheDb();
+          })
+          .withStop(async () => {
+            if (this.cacheCleaner) {
+              this.cacheCleaner.stop();
+              this.cacheCleaner = undefined;
+            }
+            if (this.cacheDb) {
+              await this.cacheDb.stop();
+              CacheDb.resetInstance();
+              this.cacheDb = undefined;
+            }
+          })
+          .withRetry({ maxRetries: 2, baseDelayMs: 1000, maxDelayMs: 5000 }),
+      );
+    }
+
+    // MetricsManager: optional, depends on OpsServer
+    this.serviceManager.addService(
+      new plugins.taskbuffer.Service('MetricsManager')
+        .optional()
+        .dependsOn('OpsServer')
+        .withStart(async () => {
+          this.metricsManager = new MetricsManager(this);
+          await this.metricsManager.start();
+        })
+        .withStop(async () => {
+          if (this.metricsManager) {
+            await this.metricsManager.stop();
+            this.metricsManager = undefined;
+          }
+        })
+        .withRetry({ maxRetries: 1, baseDelayMs: 1000 }),
+    );
+
+    // SmartProxy: critical, depends on CacheDb (if enabled)
+    const smartProxyDeps: string[] = [];
+    if (this.options.cacheConfig?.enabled !== false) {
+      smartProxyDeps.push('CacheDb');
+    }
+    this.serviceManager.addService(
+      new plugins.taskbuffer.Service('SmartProxy')
+        .critical()
+        .dependsOn(...smartProxyDeps)
+        .withStart(async () => {
+          await this.setupSmartProxy();
+        })
+        .withStop(async () => {
+          if (this.smartProxy) {
+            this.smartProxy.removeAllListeners();
+            await this.smartProxy.stop();
+            this.smartProxy = undefined;
+          }
+        })
+        .withRetry({ maxRetries: 0 }),
+    );
+
+    // SmartAcme: optional, depends on SmartProxy — aggressive retry for rate limits
+    // Only registered if DNS challenge is configured
+    if (this.options.dnsChallenge?.cloudflareApiKey) {
+      this.serviceManager.addService(
+        new plugins.taskbuffer.Service('SmartAcme')
+          .optional()
+          .dependsOn('SmartProxy')
+          .withStart(async () => {
+            if (this.smartAcme) {
+              await this.smartAcme.start();
+              this.smartAcmeReady = true;
+              logger.log('info', 'SmartAcme DNS-01 provider is now ready');
+            }
+          })
+          .withStop(async () => {
+            this.smartAcmeReady = false;
+            if (this.smartAcme) {
+              await this.smartAcme.stop();
+              this.smartAcme = undefined;
+            }
+          })
+          .withRetry({ maxRetries: 20, baseDelayMs: 5000, maxDelayMs: 3_600_000, backoffFactor: 2 }),
+      );
+    }
+
+    // ConfigManagers: optional, depends on SmartProxy
+    this.serviceManager.addService(
+      new plugins.taskbuffer.Service('ConfigManagers')
+        .optional()
+        .dependsOn('SmartProxy')
+        .withStart(async () => {
+          this.routeConfigManager = new RouteConfigManager(
+            this.storageManager,
+            () => this.getConstructorRoutes(),
+            () => this.smartProxy,
+            () => this.options.http3,
+          );
+          this.apiTokenManager = new ApiTokenManager(this.storageManager);
+          await this.apiTokenManager.initialize();
+          await this.routeConfigManager.initialize();
+        })
+        .withStop(async () => {
+          this.routeConfigManager = undefined;
+          this.apiTokenManager = undefined;
+        })
+        .withRetry({ maxRetries: 2, baseDelayMs: 1000 }),
+    );
+
+    // Email Server: optional, depends on SmartProxy
+    if (this.options.emailConfig) {
+      this.serviceManager.addService(
+        new plugins.taskbuffer.Service('EmailServer')
+          .optional()
+          .dependsOn('SmartProxy')
+          .withStart(async () => {
+            await this.setupUnifiedEmailHandling();
+          })
+          .withStop(async () => {
+            if (this.emailServer) {
+              if ((this.emailServer as any).deliverySystem) {
+                (this.emailServer as any).deliverySystem.removeAllListeners();
+              }
+              this.emailServer.removeAllListeners();
+              await this.emailServer.stop();
+              this.emailServer = undefined;
+            }
+          })
+          .withRetry({ maxRetries: 3, baseDelayMs: 2000, maxDelayMs: 30_000 }),
+      );
+    }
+
+    // DNS Server: optional, depends on SmartProxy
+    if (this.options.dnsNsDomains?.length > 0 && this.options.dnsScopes?.length > 0) {
+      this.serviceManager.addService(
+        new plugins.taskbuffer.Service('DnsServer')
+          .optional()
+          .dependsOn('SmartProxy')
+          .withStart(async () => {
+            await this.setupDnsWithSocketHandler();
+          })
+          .withStop(async () => {
+            // Flush pending DNS batch log
+            if (this.dnsBatchTimer) {
+              clearTimeout(this.dnsBatchTimer);
+              if (this.dnsBatchCount > 0) {
+                logger.log('info', `DNS: ${this.dnsBatchCount} queries processed (final flush)`, { zone: 'dns' });
+              }
+              this.dnsBatchTimer = null;
+              this.dnsBatchCount = 0;
+              this.dnsLogWindowSecond = 0;
+              this.dnsLogWindowCount = 0;
+            }
+            if (this.dnsServer) {
+              this.dnsServer.removeAllListeners();
+              await this.dnsServer.stop();
+              this.dnsServer = undefined;
+            }
+          })
+          .withRetry({ maxRetries: 3, baseDelayMs: 2000, maxDelayMs: 30_000 }),
+      );
+    }
+
+    // RADIUS Server: optional, no dependency on SmartProxy
+    if (this.options.radiusConfig) {
+      this.serviceManager.addService(
+        new plugins.taskbuffer.Service('RadiusServer')
+          .optional()
+          .withStart(async () => {
+            await this.setupRadiusServer();
+          })
+          .withStop(async () => {
+            if (this.radiusServer) {
+              await this.radiusServer.stop();
+              this.radiusServer = undefined;
+            }
+          })
+          .withRetry({ maxRetries: 3, baseDelayMs: 2000, maxDelayMs: 30_000 }),
+      );
+    }
+
+    // Remote Ingress: optional, depends on SmartProxy
+    if (this.options.remoteIngressConfig?.enabled) {
+      this.serviceManager.addService(
+        new plugins.taskbuffer.Service('RemoteIngress')
+          .optional()
+          .dependsOn('SmartProxy')
+          .withStart(async () => {
+            await this.setupRemoteIngress();
+          })
+          .withStop(async () => {
+            if (this.tunnelManager) {
+              await this.tunnelManager.stop();
+              this.tunnelManager = undefined;
+            }
+            this.remoteIngressManager = undefined;
+          })
+          .withRetry({ maxRetries: 3, baseDelayMs: 2000, maxDelayMs: 30_000 }),
+      );
+    }
+
+    // Wire up aggregated events for logging
+    this.serviceSubjectSubscription = this.serviceManager.serviceSubject.subscribe((event) => {
+      const level = event.type === 'failed' ? 'error' : event.type === 'retrying' ? 'warn' : 'info';
+      logger.log(level as any, `Service '${event.serviceName}': ${event.type}`, {
+        state: event.state,
+        ...(event.error ? { error: event.error } : {}),
+        ...(event.attempt ? { attempt: event.attempt } : {}),
+      });
+    });
   }
 
   public async start() {
     logger.log('info', 'Starting DcRouter Services');
-    
+    await this.serviceManager.start();
-
+    this.logStartupSummary();
-    this.opsServer = new OpsServer(this);
-    await this.opsServer.start();
-
-    try {
-      // Initialize cache database if enabled (default: enabled)
-      if (this.options.cacheConfig?.enabled !== false) {
-        await this.setupCacheDb();
-      }
-
-      // Initialize MetricsManager
-      this.metricsManager = new MetricsManager(this);
-      await this.metricsManager.start();
-      
-      // Set up SmartProxy for HTTP/HTTPS and all traffic including email routes
-      await this.setupSmartProxy();
-
-      // Initialize programmatic config API managers
-      this.routeConfigManager = new RouteConfigManager(
-        this.storageManager,
-        () => this.getConstructorRoutes(),
-        () => this.smartProxy,
-        () => this.options.http3,
-      );
-      this.apiTokenManager = new ApiTokenManager(this.storageManager);
-      await this.apiTokenManager.initialize();
-      await this.routeConfigManager.initialize();
-
-      // Set up unified email handling if configured
-      if (this.options.emailConfig) {
-        await this.setupUnifiedEmailHandling();
-      }
-      
-      // Set up DNS server if configured with nameservers and scopes
-      if (this.options.dnsNsDomains && this.options.dnsNsDomains.length > 0 &&
-          this.options.dnsScopes && this.options.dnsScopes.length > 0) {
-        await this.setupDnsWithSocketHandler();
-      }
-
-      // Set up RADIUS server if configured
-      if (this.options.radiusConfig) {
-        await this.setupRadiusServer();
-      }
-
-      // Set up Remote Ingress hub if configured
-      if (this.options.remoteIngressConfig?.enabled) {
-        await this.setupRemoteIngress();
-      }
-
-      this.logStartupSummary();
-    } catch (error) {
-      logger.log('error', 'Error starting DcRouter', { error: String(error) });
-      // Try to clean up any services that may have started
-      await this.stop();
-      throw error;
-    }
   }
   
   /**
@@ -399,7 +590,21 @@ export class DcRouter {
       logger.log('info', `Cache Database: storage=${this.cacheDb.getStoragePath()}, db=${this.cacheDb.getDbName()}, cleaner=${this.cacheCleaner?.isActive() ? 'active' : 'inactive'} (${(this.options.cacheConfig?.cleanupIntervalHours || 1)}h interval)`);
     }
 
-    logger.log('info', 'All services are running');
+    // Service status summary from ServiceManager
+    const health = this.serviceManager.getHealth();
+    const statuses = health.services;
+    const running = statuses.filter(s => s.state === 'running').length;
+    const failed = statuses.filter(s => s.state === 'failed').length;
+    const retrying = statuses.filter(s => s.state === 'starting' || s.state === 'degraded').length;
+
+    if (failed > 0) {
+      const failedNames = statuses.filter(s => s.state === 'failed').map(s => `${s.name}: ${s.lastError || 'unknown'}`);
+      logger.log('warn', `DcRouter started in degraded mode — ${running} running, ${failed} failed: ${failedNames.join('; ')}`);
+    } else if (retrying > 0) {
+      logger.log('info', `DcRouter started — ${running} running, ${retrying} still initializing`);
+    } else {
+      logger.log('info', `All ${running} services are running`);
+    }
   }
   
   /**
@@ -435,6 +640,13 @@ export class DcRouter {
    */
   private async setupSmartProxy(): Promise<void> {
     logger.log('info', 'Setting up SmartProxy...');
+
+    // Clean up any existing SmartProxy instance (e.g. from a retry)
+    if (this.smartProxy) {
+      this.smartProxy.removeAllListeners();
+      this.smartProxy = undefined;
+    }
+
     let routes: plugins.smartproxy.IRouteConfig[] = [];
     let acmeConfig: plugins.smartproxy.IAcmeOptions | undefined;
     
@@ -535,10 +747,13 @@ export class DcRouter {
       // Initialize cert provision scheduler
       this.certProvisionScheduler = new CertProvisionScheduler(this.storageManager);
 
-      // If we have DNS challenge handlers, create SmartAcme and wire to certProvisionFunction
+      // If we have DNS challenge handlers, create SmartAcme instance and wire certProvisionFunction
+      // Note: SmartAcme.start() is NOT called here — it runs as a separate optional service
+      // via the ServiceManager, with aggressive retry for rate-limit resilience.
       if (challengeHandlers.length > 0) {
         // Stop old SmartAcme if it exists (e.g., during updateSmartProxyConfig)
         if (this.smartAcme) {
+          this.smartAcmeReady = false;
           await this.smartAcme.stop().catch(err =>
             logger.log('error', 'Error stopping old SmartAcme', { error: String(err) })
           );
@@ -550,10 +765,15 @@ export class DcRouter {
           challengeHandlers: challengeHandlers,
           challengePriority: ['dns-01'],
         });
-        await this.smartAcme.start();
 
         const scheduler = this.certProvisionScheduler;
         smartProxyConfig.certProvisionFunction = async (domain, eventComms) => {
+          // If SmartAcme is not yet ready (still starting or retrying), fall back to HTTP-01
+          if (!this.smartAcmeReady) {
+            eventComms.warn(`SmartAcme not yet initialized, falling back to http-01 for ${domain}`);
+            return 'http01';
+          }
+
           // Check backoff before attempting provision
           if (await scheduler.isInBackoff(domain)) {
             const info = await scheduler.getBackoffInfo(domain);
@@ -914,105 +1134,29 @@ export class DcRouter {
   public async stop() {
     logger.log('info', 'Stopping DcRouter services...');
 
-    // Flush pending DNS batch log
+    // Unsubscribe from service events before stopping services
-    if (this.dnsBatchTimer) {
+    if (this.serviceSubjectSubscription) {
-      clearTimeout(this.dnsBatchTimer);
+      this.serviceSubjectSubscription.unsubscribe();
-      if (this.dnsBatchCount > 0) {
+      this.serviceSubjectSubscription = undefined;
-        logger.log('info', `DNS: ${this.dnsBatchCount} queries processed (rate limited, final flush)`, { zone: 'dns' });
-      }
-      this.dnsBatchTimer = null;
-      this.dnsBatchCount = 0;
-      this.dnsLogWindowSecond = 0;
-      this.dnsLogWindowCount = 0;
     }
 
-    await this.opsServer.stop();
+    // ServiceManager handles reverse-dependency-ordered shutdown
+    await this.serviceManager.stop();
 
-    try {
+    // Clear backoff cache in cert scheduler
-      // Remove event listeners before stopping services to prevent leaks
+    if (this.certProvisionScheduler) {
-      if (this.smartProxy) {
+      this.certProvisionScheduler.clear();
-        this.smartProxy.removeAllListeners();
-      }
-      if (this.emailServer) {
-        if ((this.emailServer as any).deliverySystem) {
-          (this.emailServer as any).deliverySystem.removeAllListeners();
-        }
-        this.emailServer.removeAllListeners();
-      }
-      if (this.dnsServer) {
-        this.dnsServer.removeAllListeners();
-      }
-
-      // Stop all services in parallel for faster shutdown
-      await Promise.all([
-        // Stop cache cleaner if running
-        this.cacheCleaner ? Promise.resolve(this.cacheCleaner.stop()) : Promise.resolve(),
-
-        // Stop metrics manager if running
-        this.metricsManager ? this.metricsManager.stop().catch(err => logger.log('error', 'Error stopping MetricsManager', { error: String(err) })) : Promise.resolve(),
-
-        // Stop unified email server if running
-        this.emailServer ? this.emailServer.stop().catch(err => logger.log('error', 'Error stopping email server', { error: String(err) })) : Promise.resolve(),
-
-        // Stop SmartAcme if running
-        this.smartAcme ? this.smartAcme.stop().catch(err => logger.log('error', 'Error stopping SmartAcme', { error: String(err) })) : Promise.resolve(),
-
-        // Stop HTTP SmartProxy if running
-        this.smartProxy ? this.smartProxy.stop().catch(err => logger.log('error', 'Error stopping SmartProxy', { error: String(err) })) : Promise.resolve(),
-
-        // Stop DNS server if running
-        this.dnsServer ?
-          this.dnsServer.stop().catch(err => logger.log('error', 'Error stopping DNS server', { error: String(err) })) :
-          Promise.resolve(),
-
-        // Stop RADIUS server if running
-        this.radiusServer ?
-          this.radiusServer.stop().catch(err => logger.log('error', 'Error stopping RADIUS server', { error: String(err) })) :
-          Promise.resolve(),
-
-        // Stop Remote Ingress tunnel manager if running
-        this.tunnelManager ?
-          this.tunnelManager.stop().catch(err => logger.log('error', 'Error stopping TunnelManager', { error: String(err) })) :
-          Promise.resolve()
-      ]);
-
-      // Stop cache database after other services (they may need it during shutdown)
-      if (this.cacheDb) {
-        await this.cacheDb.stop().catch(err => logger.log('error', 'Error stopping CacheDb', { error: String(err) }));
-        CacheDb.resetInstance();
-      }
-
-      // Clear backoff cache in cert scheduler
-      if (this.certProvisionScheduler) {
-        this.certProvisionScheduler.clear();
-      }
-
-      // Allow GC of stopped services by nulling references
-      this.smartProxy = undefined;
-      this.emailServer = undefined;
-      this.dnsServer = undefined;
-      this.metricsManager = undefined;
-      this.cacheCleaner = undefined;
-      this.cacheDb = undefined;
-      this.tunnelManager = undefined;
-      this.radiusServer = undefined;
-      this.smartAcme = undefined;
       this.certProvisionScheduler = undefined;
-      this.remoteIngressManager = undefined;
-      this.routeConfigManager = undefined;
-      this.apiTokenManager = undefined;
-      this.certificateStatusMap.clear();
-
-      // Reset security singletons to allow GC
-      SecurityLogger.resetInstance();
-      ContentScanner.resetInstance();
-      IPReputationChecker.resetInstance();
-
-      logger.log('info', 'All DcRouter services stopped');
-    } catch (error) {
-      logger.log('error', 'Error during DcRouter shutdown', { error: String(error) });
-      throw error;
     }
+
+    this.certificateStatusMap.clear();
+
+    // Reset security singletons to allow GC
+    SecurityLogger.resetInstance();
+    ContentScanner.resetInstance();
+    IPReputationChecker.resetInstance();
+
+    logger.log('info', 'All DcRouter services stopped');
   }
 
   /**
@@ -1753,7 +1897,7 @@ export class DcRouter {
     await this.remoteIngressManager.initialize();
 
     // Pass current routes so the manager can derive edge ports from remoteIngress-tagged routes
-    const currentRoutes = this.options.smartProxyConfig?.routes || [];
+    const currentRoutes = this.constructorRoutes;
     this.remoteIngressManager.setRoutes(currentRoutes as any[]);
 
     // Resolve TLS certs for tunnel: explicit paths > ACME for hubDomain > self-signed (Rust default)

ts/index.ts

@@ -35,6 +35,6 @@ export const runCli = async () => {
     await dcRouter.stop();
     process.exit(0);
   };
-  process.on('SIGINT', shutdown);
+  process.once('SIGINT', shutdown);
-  process.on('SIGTERM', shutdown);
+  process.once('SIGTERM', shutdown);
 };

ts/opsserver/handlers/stats.handler.ts

@@ -489,44 +489,41 @@ export class StatsHandler {
       message?: string;
     }>;
   }> {
-    const services: Array<{
+    const dcRouter = this.opsServerRef.dcRouterRef;
-      name: string;
+    const health = dcRouter.serviceManager.getHealth();
-      status: 'healthy' | 'degraded' | 'unhealthy';
+
-      message?: string;
+    const services = health.services.map((svc) => {
-    }> = [];
+      let status: 'healthy' | 'degraded' | 'unhealthy';
-    
+      switch (svc.state) {
-    // Check HTTP Proxy
+        case 'running':
-    if (this.opsServerRef.dcRouterRef.smartProxy) {
+          status = 'healthy';
-      services.push({
+          break;
-        name: 'HTTP/HTTPS Proxy',
+        case 'starting':
-        status: 'healthy',
+        case 'degraded':
-      });
+          status = 'degraded';
-    }
+          break;
-    
+        case 'failed':
-    // Check Email Server
+          status = svc.criticality === 'critical' ? 'unhealthy' : 'degraded';
-    if (this.opsServerRef.dcRouterRef.emailServer) {
+          break;
-      services.push({
+        case 'stopped':
-        name: 'Email Server',
+        case 'stopping':
-        status: 'healthy',
+        default:
-      });
+          status = 'degraded';
-    }
+          break;
-    
+      }
-    // Check DNS Server
+
-    if (this.opsServerRef.dcRouterRef.dnsServer) {
+      let message: string | undefined;
-      services.push({
+      if (svc.state === 'failed' && svc.lastError) {
-        name: 'DNS Server',
+        message = svc.lastError;
-        status: 'healthy',
+      } else if (svc.retryCount > 0 && svc.state !== 'running') {
-      });
+        message = `Retry attempt ${svc.retryCount}`;
-    }
+      }
-    
+
-    // Check OpsServer
+      return { name: svc.name, status, message };
-    services.push({
-      name: 'OpsServer',
-      status: 'healthy',
     });
-    
+
-    const healthy = services.every(s => s.status === 'healthy');
+    const healthy = health.overall === 'healthy';
-    
+
     return {
       healthy,
       services,

ts/plugins.ts

@@ -62,8 +62,9 @@ import * as smartradius from '@push.rocks/smartradius';
 import * as smartrequest from '@push.rocks/smartrequest';
 import * as smartrx from '@push.rocks/smartrx';
 import * as smartunique from '@push.rocks/smartunique';
+import * as taskbuffer from '@push.rocks/taskbuffer';
 
-export { projectinfo, qenv, smartacme, smartdata, smartdns, smartfile, smartguard, smartjwt, smartlog, smartmetrics, smartmongo, smartmta, smartnetwork, smartpath, smartproxy, smartpromise, smartradius, smartrequest, smartrx, smartunique };
+export { projectinfo, qenv, smartacme, smartdata, smartdns, smartfile, smartguard, smartjwt, smartlog, smartmetrics, smartmongo, smartmta, smartnetwork, smartpath, smartproxy, smartpromise, smartradius, smartrequest, smartrx, smartunique, taskbuffer };
 
 // Define SmartLog types for use in error handling
 export type TLogLevel = 'error' | 'warn' | 'info' | 'success' | 'debug';

ts/radius/classes.accounting.manager.ts

@@ -92,6 +92,8 @@ export interface IAccountingManagerConfig {
   detailedLogging?: boolean;
   /** Maximum active sessions to track in memory */
   maxActiveSessions?: number;
+  /** Stale session timeout in hours — sessions with no update for this long are evicted (default: 24) */
+  staleSessionTimeoutHours?: number;
 }
 
 /**
@@ -105,6 +107,7 @@ export class AccountingManager {
   private activeSessions: Map<string, IAccountingSession> = new Map();
   private config: Required<IAccountingManagerConfig>;
   private storageManager?: StorageManager;
+  private staleSessionSweepTimer?: ReturnType<typeof setInterval>;
 
   // Counters for statistics
   private stats = {
@@ -121,6 +124,7 @@ export class AccountingManager {
       retentionDays: config?.retentionDays ?? 30,
       detailedLogging: config?.detailedLogging ?? false,
       maxActiveSessions: config?.maxActiveSessions ?? 10000,
+      staleSessionTimeoutHours: config?.staleSessionTimeoutHours ?? 24,
     };
     this.storageManager = storageManager;
   }
@@ -132,9 +136,60 @@ export class AccountingManager {
     if (this.storageManager) {
       await this.loadActiveSessions();
     }
+
+    // Start periodic sweep to evict stale sessions (every 15 minutes)
+    this.staleSessionSweepTimer = setInterval(() => {
+      this.sweepStaleSessions();
+    }, 15 * 60 * 1000);
+    // Allow the process to exit even if the timer is pending
+    if (this.staleSessionSweepTimer.unref) {
+      this.staleSessionSweepTimer.unref();
+    }
+
     logger.log('info', `AccountingManager initialized with ${this.activeSessions.size} active sessions`);
   }
 
+  /**
+   * Stop the accounting manager and clean up timers
+   */
+  stop(): void {
+    if (this.staleSessionSweepTimer) {
+      clearInterval(this.staleSessionSweepTimer);
+      this.staleSessionSweepTimer = undefined;
+    }
+  }
+
+  /**
+   * Sweep stale active sessions that have not received any update
+   * within the configured timeout. These are orphaned sessions where
+   * the Stop packet was never received.
+   */
+  private sweepStaleSessions(): void {
+    const timeoutMs = this.config.staleSessionTimeoutHours * 60 * 60 * 1000;
+    const cutoff = Date.now() - timeoutMs;
+    let swept = 0;
+
+    for (const [sessionId, session] of this.activeSessions) {
+      if (session.lastUpdateTime < cutoff) {
+        session.status = 'terminated';
+        session.terminateCause = 'StaleSessionTimeout';
+        session.endTime = Date.now();
+        session.sessionTime = Math.floor((session.endTime - session.startTime) / 1000);
+
+        if (this.storageManager) {
+          this.archiveSession(session).catch(() => {});
+        }
+
+        this.activeSessions.delete(sessionId);
+        swept++;
+      }
+    }
+
+    if (swept > 0) {
+      logger.log('info', `Swept ${swept} stale RADIUS sessions (no update for ${this.config.staleSessionTimeoutHours}h)`);
+    }
+  }
+
   /**
    * Handle accounting start request
    */

ts/radius/classes.radius.server.ts

@@ -183,6 +183,8 @@ export class RadiusServer {
       this.radiusServer = undefined;
     }
 
+    this.accountingManager.stop();
+
     this.running = false;
     logger.log('info', 'RADIUS server stopped');
   }

ts_web/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@serve.zone/dcrouter',
-  version: '11.8.0',
+  version: '11.9.1',
   description: 'A multifaceted routing service handling mail and SMS delivery functions.'
 }