fix(core): improve shutdown cleanup, socket/stream robustness, and memory/cache handling

2026-03-01 00:44:01 +00:00
parent 59a3f7978e
commit 74ad5cec90
13 changed files with 150 additions and 59 deletions
--- a/changelog.md
+++ b/changelog.md
@@ -1,5 +1,17 @@
 # Changelog

+## 2026-03-01 - 10.1.2 - fix(core)
+improve shutdown cleanup, socket/stream robustness, and memory/cache handling
+
+- Reset security singletons and CacheDb on shutdown to allow GC (SecurityLogger, ContentScanner, IPReputationChecker, CacheDb).
+- Add DNS socket 'error' handler and only destroy socket when not already destroyed to avoid uncaught exceptions.
+- Move pruning of dnsMetrics.queryTimestamps to a periodic interval to avoid O(n) work on every query.
+- Debounce IPReputationChecker cache saves (save timer + reset on instance reset) to reduce IO and prevent duplicate saves.
+- Fix virtualStream send timeout handling by keeping/clearing a timeout handle to avoid leaks and hung promises.
+- Add memory store eviction in StorageManager to cap entries (MAX_MEMORY_ENTRIES) and evict oldest entries when exceeded.
+- Add terminal-ready timeout in ops-view-logs to avoid blocking UI initialization if xterm CDN fails to initialize.
+- Bump dev dependency @types/node and push.rocks/smartstate versions.
+
 ## 2026-02-27 - 10.1.1 - fix(ops-view-apitokens)
 replace lucide:refresh-cw with lucide:rotate-cw for Roll action icon

--- a/package.json
+++ b/package.json
@@ -24,7 +24,7 @@
    "@git.zone/tsrun": "^2.0.1",
    "@git.zone/tstest": "^3.1.8",
    "@git.zone/tswatch": "^3.2.0",
-    "@types/node": "^25.3.0"
+    "@types/node": "^25.3.3"
  },
  "dependencies": {
    "@api.global/typedrequest": "^3.2.6",
@@ -53,7 +53,7 @@
    "@push.rocks/smartradius": "^1.1.1",
    "@push.rocks/smartrequest": "^5.0.1",
    "@push.rocks/smartrx": "^3.0.10",
-    "@push.rocks/smartstate": "^2.0.30",
+    "@push.rocks/smartstate": "^2.1.1",
    "@push.rocks/smartunique": "^3.0.9",
    "@serve.zone/catalog": "^2.5.0",
    "@serve.zone/interfaces": "^5.3.0",
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -87,8 +87,8 @@ importers:
        specifier: ^3.0.10
        version: 3.0.10
      '@push.rocks/smartstate':
-        specifier: ^2.0.30
-        version: 2.0.30
+        specifier: ^2.1.1
+        version: 2.1.1
      '@push.rocks/smartunique':
        specifier: ^3.0.9
        version: 3.0.9
@@ -127,8 +127,8 @@ importers:
        specifier: ^3.2.0
        version: 3.2.0(@tiptap/pm@2.27.2)
      '@types/node':
-        specifier: ^25.3.0
-        version: 25.3.0
+        specifier: ^25.3.3
+        version: 25.3.3

 packages:

@@ -1083,8 +1083,8 @@ packages:
  '@push.rocks/smartspawn@3.0.3':
    resolution: {integrity: sha512-DyrGPV69wwOiJgKkyruk5hS3UEGZ99xFAqBE9O2nM8VXCRLbbty3xt1Ug5Z092ZZmJYaaGMSnMw3ijyZJFCT0Q==}

-  '@push.rocks/smartstate@2.0.30':
-    resolution: {integrity: sha512-IuNW8XtSumXIr7g7MIFyWg5PBwLF2mwsymTJbSEycK2Pa9ZLk4yjRHnR907xCilxgiMU9ixQZyNdpa5MMF999A==}
+  '@push.rocks/smartstate@2.1.1':
+    resolution: {integrity: sha512-4OM9TXfiiSYIgVz2pQdM2UCTurXwd8o9LCtyZ/o+rnntnXp/X8UTWZ+WyTxgnfuzXhpIYXt83t34bVBJ2EPUOw==}

  '@push.rocks/smartstream@2.0.8':
    resolution: {integrity: sha512-GlF/9cCkvBHwKa3DK4DO5wjfSgqkj6gAS4TrY9uD5NMHu9RQv4WiNrElTYj7iCEpnZgUnLO3tzw1JA3NRIMnnA==}
@@ -1835,11 +1835,11 @@ packages:
  '@types/node@18.19.130':
    resolution: {integrity: sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==}

-  '@types/node@22.19.11':
-    resolution: {integrity: sha512-BH7YwL6rA93ReqeQS1c4bsPpcfOmJasG+Fkr6Y59q83f9M1WcBRHR2vM+P9eOisYRcN3ujQoiZY8uk5W+1WL8w==}
+  '@types/node@22.19.13':
+    resolution: {integrity: sha512-akNQMv0wW5uyRpD2v2IEyRSZiR+BeGuoB6L310EgGObO44HSMNT8z1xzio28V8qOrgYaopIDNA18YgdXd+qTiw==}

-  '@types/node@25.3.0':
-    resolution: {integrity: sha512-4K3bqJpXpqfg2XKGK9bpDTc6xO/xoUP/RBWS7AtRMug6zZFaRekiLzjVtAoZMquxoAbzBvy5nxQ7veS5eYzf8A==}
+  '@types/node@25.3.3':
+    resolution: {integrity: sha512-DpzbrH7wIcBaJibpKo9nnSQL0MTRdnWttGyE5haGwK86xgMOkFLp7vEyfQPGLOJh5wNYiJ3V9PmUMDhV9u8kkQ==}

  '@types/ping@0.4.4':
    resolution: {integrity: sha512-ifvo6w2f5eJYlXm+HiVx67iJe8WZp87sfa683nlqED5Vnt9Z93onkokNoWqOG21EaE8fMxyKPobE+mkPEyxsdw==}
@@ -5010,7 +5010,7 @@ snapshots:
      '@push.rocks/smartpromise': 4.2.3
      '@push.rocks/smartrouter': 1.3.3
      '@push.rocks/smartrx': 3.0.10
-      '@push.rocks/smartstate': 2.0.30
+      '@push.rocks/smartstate': 2.1.1
      '@push.rocks/smartstring': 4.1.0
      '@push.rocks/smarturl': 3.1.0
      '@push.rocks/webrequest': 3.0.37
@@ -5334,7 +5334,7 @@ snapshots:
      '@inquirer/figures': 1.0.15
      '@inquirer/type': 2.0.0
      '@types/mute-stream': 0.0.4
-      '@types/node': 22.19.11
+      '@types/node': 22.19.13
      '@types/wrap-ansi': 3.0.0
      ansi-escapes: 4.3.2
      cli-width: 4.1.0
@@ -6487,9 +6487,8 @@ snapshots:
    transitivePeerDependencies:
      - supports-color

-  '@push.rocks/smartstate@2.0.30':
+  '@push.rocks/smartstate@2.1.1':
    dependencies:
-      '@push.rocks/lik': 6.2.2
      '@push.rocks/smarthash': 3.2.6
      '@push.rocks/smartjson': 6.0.0
      '@push.rocks/smartpromise': 4.2.3
@@ -7359,22 +7358,22 @@ snapshots:
  '@types/body-parser@1.19.6':
    dependencies:
      '@types/connect': 3.4.38
-      '@types/node': 25.3.0
+      '@types/node': 25.3.3

  '@types/buffer-json@2.0.3': {}

  '@types/clean-css@4.2.11':
    dependencies:
-      '@types/node': 25.3.0
+      '@types/node': 25.3.3
      source-map: 0.6.1

  '@types/connect@3.4.38':
    dependencies:
-      '@types/node': 25.3.0
+      '@types/node': 25.3.3

  '@types/cors@2.8.19':
    dependencies:
-      '@types/node': 25.3.0
+      '@types/node': 25.3.3

  '@types/debug@4.1.12':
    dependencies:
@@ -7382,7 +7381,7 @@ snapshots:

  '@types/express-serve-static-core@5.1.1':
    dependencies:
-      '@types/node': 25.3.0
+      '@types/node': 25.3.3
      '@types/qs': 6.14.0
      '@types/range-parser': 1.2.7
      '@types/send': 1.2.1
@@ -7395,17 +7394,17 @@ snapshots:

  '@types/from2@2.3.6':
    dependencies:
-      '@types/node': 25.3.0
+      '@types/node': 25.3.3

  '@types/fs-extra@11.0.4':
    dependencies:
      '@types/jsonfile': 6.1.4
-      '@types/node': 25.3.0
+      '@types/node': 25.3.3

  '@types/glob@8.1.0':
    dependencies:
      '@types/minimatch': 5.1.2
-      '@types/node': 25.3.0
+      '@types/node': 25.3.3

  '@types/hast@3.0.4':
    dependencies:
@@ -7427,12 +7426,12 @@ snapshots:

  '@types/jsonfile@6.1.4':
    dependencies:
-      '@types/node': 25.3.0
+      '@types/node': 25.3.3

  '@types/jsonwebtoken@9.0.10':
    dependencies:
      '@types/ms': 2.1.0
-      '@types/node': 25.3.0
+      '@types/node': 25.3.3

  '@types/linkify-it@5.0.0': {}

@@ -7455,26 +7454,26 @@ snapshots:

  '@types/mute-stream@0.0.4':
    dependencies:
-      '@types/node': 25.3.0
+      '@types/node': 25.3.3

  '@types/node-fetch@2.6.13':
    dependencies:
-      '@types/node': 25.3.0
+      '@types/node': 25.3.3
      form-data: 4.0.5

  '@types/node-forge@1.3.14':
    dependencies:
-      '@types/node': 25.3.0
+      '@types/node': 25.3.3

  '@types/node@18.19.130':
    dependencies:
      undici-types: 5.26.5

-  '@types/node@22.19.11':
+  '@types/node@22.19.13':
    dependencies:
      undici-types: 6.21.0

-  '@types/node@25.3.0':
+  '@types/node@25.3.3':
    dependencies:
      undici-types: 7.18.2

@@ -7492,22 +7491,22 @@ snapshots:

  '@types/send@1.2.1':
    dependencies:
-      '@types/node': 25.3.0
+      '@types/node': 25.3.3

  '@types/serve-static@2.2.0':
    dependencies:
      '@types/http-errors': 2.0.5
-      '@types/node': 25.3.0
+      '@types/node': 25.3.3

  '@types/symbol-tree@3.2.5': {}

  '@types/tar-stream@3.1.4':
    dependencies:
-      '@types/node': 25.3.0
+      '@types/node': 25.3.3

  '@types/through2@2.0.41':
    dependencies:
-      '@types/node': 25.3.0
+      '@types/node': 25.3.3

  '@types/trusted-types@2.0.7': {}

@@ -7537,11 +7536,11 @@ snapshots:

  '@types/ws@8.18.1':
    dependencies:
-      '@types/node': 25.3.0
+      '@types/node': 25.3.3

  '@types/yauzl@2.10.3':
    dependencies:
-      '@types/node': 25.3.0
+      '@types/node': 25.3.3
    optional: true

  '@ungap/structured-clone@1.3.0': {}
@@ -8018,7 +8017,7 @@ snapshots:
  engine.io@6.6.4:
    dependencies:
      '@types/cors': 2.8.19
-      '@types/node': 25.3.0
+      '@types/node': 25.3.3
      accepts: 1.3.8
      base64id: 2.0.0
      cookie: 0.7.2
--- a/ts/00_commitinfo_data.ts
+++ b/ts/00_commitinfo_data.ts
@@ -3,6 +3,6 @@
 */
 export const commitinfo = {
  name: '@serve.zone/dcrouter',
-  version: '10.1.1',
+  version: '10.1.2',
  description: 'A multifaceted routing service handling mail and SMS delivery functions.'
 }
--- a/ts/classes.dcrouter.ts
+++ b/ts/classes.dcrouter.ts
@@ -23,6 +23,7 @@ import { MetricsManager } from './monitoring/index.js';
 import { RadiusServer, type IRadiusServerConfig } from './radius/index.js';
 import { RemoteIngressManager, TunnelManager } from './remoteingress/index.js';
 import { RouteConfigManager, ApiTokenManager } from './config/index.js';
+import { SecurityLogger, ContentScanner, IPReputationChecker } from './security/index.js';

 export interface IDcRouterOptions {
  /** Base directory for all dcrouter data. Defaults to ~/.serve.zone/dcrouter */
@@ -956,6 +957,7 @@ export class DcRouter {
      // Stop cache database after other services (they may need it during shutdown)
      if (this.cacheDb) {
        await this.cacheDb.stop().catch(err => logger.log('error', 'Error stopping CacheDb', { error: String(err) }));
+        CacheDb.resetInstance();
      }

      // Clear backoff cache in cert scheduler
@@ -979,6 +981,11 @@ export class DcRouter {
      this.apiTokenManager = undefined;
      this.certificateStatusMap.clear();

+      // Reset security singletons to allow GC
+      SecurityLogger.resetInstance();
+      ContentScanner.resetInstance();
+      IPReputationChecker.resetInstance();
+
      logger.log('info', 'All DcRouter services stopped');
    } catch (error) {
      logger.log('error', 'Error during DcRouter shutdown', { error: String(error) });
@@ -1363,15 +1370,25 @@ export class DcRouter {
        return;
      }
      
+      // Prevent uncaught exception from socket 'error' events
+      socket.on('error', (err) => {
+        logger.log('error', `DNS socket error: ${err.message}`);
+        if (!socket.destroyed) {
+          socket.destroy();
+        }
+      });
+
      logger.log('debug', 'DNS socket handler: passing socket to DnsServer');
-      
+
      try {
        // Use the built-in socket handler from smartdns
        // This handles HTTP/2, DoH protocol, etc.
        await (this.dnsServer as any).handleHttpsSocket(socket);
      } catch (error) {
        logger.log('error', `DNS socket handler error: ${error.message}`);
-        socket.destroy();
+        if (!socket.destroyed) {
+          socket.destroy();
+        }
      }
    };
  }
--- a/ts/monitoring/classes.metricsmanager.ts
+++ b/ts/monitoring/classes.metricsmanager.ts
@@ -111,6 +111,15 @@ export class MetricsManager {
        this.securityMetrics.lastResetDate = currentDate;
      }

+      // Prune old query timestamps (keep last 5 minutes)
+      const fiveMinutesAgo = Date.now() - 300000;
+      const idx = this.dnsMetrics.queryTimestamps.findIndex(ts => ts >= fiveMinutesAgo);
+      if (idx > 0) {
+        this.dnsMetrics.queryTimestamps = this.dnsMetrics.queryTimestamps.slice(idx);
+      } else if (idx === -1) {
+        this.dnsMetrics.queryTimestamps = [];
+      }
+
      // Prune old time-series buckets every minute (don't wait for lazy query)
      this.pruneOldBuckets();
    }, 60000); // Check every minute
@@ -427,13 +436,9 @@ export class MetricsManager {
      this.dnsMetrics.cacheMisses++;
    }
    
-    // Track query timestamp
+    // Track query timestamp (pruning moved to resetInterval to avoid O(n) per query)
    this.dnsMetrics.queryTimestamps.push(Date.now());
    
-    // Keep only timestamps from last 5 minutes
-    const fiveMinutesAgo = Date.now() - 300000;
-    this.dnsMetrics.queryTimestamps = this.dnsMetrics.queryTimestamps.filter(ts => ts >= fiveMinutesAgo);
-    
    // Track response time if provided
    if (responseTimeMs) {
      this.dnsMetrics.responseTimes.push(responseTimeMs);
--- a/ts/opsserver/handlers/logs.handler.ts
+++ b/ts/opsserver/handlers/logs.handler.ts
@@ -318,11 +318,15 @@ export class LogsHandler {
        try {
          // Use a timeout to detect hung streams (sendData can hang if the
          // VirtualStream's keepAlive loop has ended)
+          let timeoutHandle: ReturnType<typeof setTimeout>;
          await Promise.race([
-            virtualStream.sendData(encoder.encode(logData)),
-            new Promise<never>((_, reject) =>
-              setTimeout(() => reject(new Error('stream send timeout')), 10_000)
-            ),
+            virtualStream.sendData(encoder.encode(logData)).then((result) => {
+              clearTimeout(timeoutHandle);
+              return result;
+            }),
+            new Promise<never>((_, reject) => {
+              timeoutHandle = setTimeout(() => reject(new Error('stream send timeout')), 10_000);
+            }),
          ]);
        } catch {
          // Stream closed, errored, or timed out — clean up
--- a/ts/security/classes.contentscanner.ts
+++ b/ts/security/classes.contentscanner.ts
@@ -182,7 +182,14 @@ export class ContentScanner {
    }
    return ContentScanner.instance;
  }
-  
+
+  /**
+   * Reset the singleton instance (for shutdown/testing)
+   */
+  public static resetInstance(): void {
+    ContentScanner.instance = undefined;
+  }
+
  /**
   * Scan an email for malicious content
   * @param email The email to scan
--- a/ts/security/classes.ipreputationchecker.ts
+++ b/ts/security/classes.ipreputationchecker.ts
@@ -65,6 +65,8 @@ export class IPReputationChecker {
  private reputationCache: LRUCache<string, IReputationResult>;
  private options: Required<IIPReputationOptions>;
  private storageManager?: any; // StorageManager instance
+  private saveCacheTimer: ReturnType<typeof setTimeout> | null = null;
+  private static readonly SAVE_CACHE_DEBOUNCE_MS = 30_000;
  
  // Default DNSBL servers
  private static readonly DEFAULT_DNSBL_SERVERS = [
@@ -143,7 +145,20 @@ export class IPReputationChecker {
    }
    return IPReputationChecker.instance;
  }
-  
+
+  /**
+   * Reset the singleton instance (for shutdown/testing)
+   */
+  public static resetInstance(): void {
+    if (IPReputationChecker.instance) {
+      if (IPReputationChecker.instance.saveCacheTimer) {
+        clearTimeout(IPReputationChecker.instance.saveCacheTimer);
+        IPReputationChecker.instance.saveCacheTimer = null;
+      }
+    }
+    IPReputationChecker.instance = undefined;
+  }
+
  /**
   * Check an IP address's reputation
   * @param ip IP address to check
@@ -213,12 +228,9 @@ export class IPReputationChecker {
      // Update cache with result
      this.reputationCache.set(ip, result);
      
-      // Save cache if enabled
+      // Schedule debounced cache save if enabled
      if (this.options.enableLocalCache) {
-        // Fire and forget the save operation
-        this.saveCache().catch(error => {
-          logger.log('error', `Failed to save IP reputation cache: ${error.message}`);
-        });
+        this.debouncedSaveCache();
      }
      
      // Log the reputation check
@@ -447,6 +459,21 @@ export class IPReputationChecker {
    });
  }
  
+  /**
+   * Schedule a debounced cache save (at most once per SAVE_CACHE_DEBOUNCE_MS)
+   */
+  private debouncedSaveCache(): void {
+    if (this.saveCacheTimer) {
+      return; // already scheduled
+    }
+    this.saveCacheTimer = setTimeout(() => {
+      this.saveCacheTimer = null;
+      this.saveCache().catch(error => {
+        logger.log('error', `Failed to save IP reputation cache: ${error.message}`);
+      });
+    }, IPReputationChecker.SAVE_CACHE_DEBOUNCE_MS);
+  }
+
  /**
   * Save cache to disk or storage manager
   */
--- a/ts/security/classes.securitylogger.ts
+++ b/ts/security/classes.securitylogger.ts
@@ -83,7 +83,14 @@ export class SecurityLogger {
    }
    return SecurityLogger.instance;
  }
-  
+
+  /**
+   * Reset the singleton instance (for shutdown/testing)
+   */
+  public static resetInstance(): void {
+    SecurityLogger.instance = undefined;
+  }
+
  /**
   * Log a security event
   * @param event The security event to log
--- a/ts/storage/classes.storagemanager.ts
+++ b/ts/storage/classes.storagemanager.ts
@@ -30,6 +30,7 @@ export type StorageBackend = 'filesystem' | 'custom' | 'memory';
 * Provides unified key-value storage with multiple backend support
 */
 export class StorageManager {
+  private static readonly MAX_MEMORY_ENTRIES = 10_000;
  private backend: StorageBackend;
  private memoryStore: Map<string, string> = new Map();
  private config: IStorageConfig;
@@ -227,6 +228,11 @@ export class StorageManager {
        
        case 'memory': {
          this.memoryStore.set(key, value);
+          // Evict oldest entries if memory store exceeds limit
+          while (this.memoryStore.size > StorageManager.MAX_MEMORY_ENTRIES) {
+            const firstKey = this.memoryStore.keys().next().value;
+            this.memoryStore.delete(firstKey);
+          }
          break;
        }
        
--- a/ts_web/00_commitinfo_data.ts
+++ b/ts_web/00_commitinfo_data.ts
@@ -3,6 +3,6 @@
 */
 export const commitinfo = {
  name: '@serve.zone/dcrouter',
-  version: '10.1.1',
+  version: '10.1.2',
  description: 'A multifaceted routing service handling mail and SMS delivery functions.'
 }
--- a/ts_web/elements/ops-view-logs.ts
+++ b/ts_web/elements/ops-view-logs.ts
@@ -76,8 +76,15 @@ export class OpsViewLogs extends DeesElement {
    // Wait for xterm terminal to finish initializing (CDN load)
    if (!chartLog.terminalReady) {
      await new Promise<void>((resolve) => {
+        let attempts = 0;
+        const maxAttempts = 200; // 200 * 50ms = 10 seconds
        const check = () => {
          if (chartLog.terminalReady) { resolve(); return; }
+          if (++attempts >= maxAttempts) {
+            console.warn('ops-view-logs: terminal ready timeout after 10s');
+            resolve(); // resolve gracefully to avoid blocking
+            return;
+          }
          setTimeout(check, 50);
        };
        check();