feat(monitoring): improve network activity metrics with live domain request rates and backend identifiers

2026-04-25 20:37:28 +00:00
parent 97017ede98
commit b3751abd17
18 changed files with 318 additions and 93 deletions
@@ -3,6 +3,6 @@
 */
 export const commitinfo = {
  name: '@serve.zone/dcrouter',
-  version: '13.20.2',
+  version: '13.21.0',
  description: 'A multifaceted routing service handling mail and SMS delivery functions.'
 }
@@ -560,7 +560,9 @@ export class MetricsManager {
          requestsPerSecond: 0,
          requestsTotal: 0,
          backends: [] as Array<any>,
-          domainActivity: [] as Array<{ domain: string; bytesInPerSecond: number; bytesOutPerSecond: number; activeConnections: number; routeCount: number; requestCount: number }>,
+          domainActivity: [] as Array<{ domain: string; bytesInPerSecond: number; bytesOutPerSecond: number; activeConnections: number; routeCount: number; requestCount: number; requestsPerSecond?: number; requestsLastMinute?: number }>,
+          frontendProtocols: null,
+          backendProtocols: null,
        };
      }

@@ -592,6 +594,7 @@ export class MetricsManager {
      // Get HTTP request rates
      const requestsPerSecond = proxyMetrics.requests.perSecond();
      const requestsTotal = proxyMetrics.requests.total();
+      const domainRequestRates = proxyMetrics.requests.byDomain();

      // Get frontend/backend protocol distribution
      const frontendProtocols = proxyMetrics.connections.frontendProtocols() ?? null;
@@ -619,47 +622,48 @@ export class MetricsManager {
      const seenCacheKeys = new Set<string>();

      for (const [key, bm] of backendMetrics) {
+        backends.push({
+          id: `backend:${key}`,
+          backend: key,
+          domain: null,
+          protocol: bm.protocol,
+          activeConnections: bm.activeConnections,
+          totalConnections: bm.totalConnections,
+          connectErrors: bm.connectErrors,
+          handshakeErrors: bm.handshakeErrors,
+          requestErrors: bm.requestErrors,
+          avgConnectTimeMs: Math.round(bm.avgConnectTimeMs * 10) / 10,
+          poolHitRate: Math.round(bm.poolHitRate * 1000) / 1000,
+          h2Failures: bm.h2Failures,
+          h2Suppressed: false,
+          h3Suppressed: false,
+          h2CooldownRemainingSecs: null,
+          h3CooldownRemainingSecs: null,
+          h2ConsecutiveFailures: null,
+          h3ConsecutiveFailures: null,
+          h3Port: null,
+          cacheAgeSecs: null,
+        });
+
        const cacheEntries = cacheByBackend.get(key);
-        if (!cacheEntries || cacheEntries.length === 0) {
-          // No protocol cache entry — emit one row with backend metrics only
-          backends.push({
-            backend: key,
-            domain: null,
-            protocol: bm.protocol,
-            activeConnections: bm.activeConnections,
-            totalConnections: bm.totalConnections,
-            connectErrors: bm.connectErrors,
-            handshakeErrors: bm.handshakeErrors,
-            requestErrors: bm.requestErrors,
-            avgConnectTimeMs: Math.round(bm.avgConnectTimeMs * 10) / 10,
-            poolHitRate: Math.round(bm.poolHitRate * 1000) / 1000,
-            h2Failures: bm.h2Failures,
-            h2Suppressed: false,
-            h3Suppressed: false,
-            h2CooldownRemainingSecs: null,
-            h3CooldownRemainingSecs: null,
-            h2ConsecutiveFailures: null,
-            h3ConsecutiveFailures: null,
-            h3Port: null,
-            cacheAgeSecs: null,
-          });
-        } else {
-          // One row per domain, each enriched with the shared backend metrics
+        if (cacheEntries && cacheEntries.length > 0) {
+          // Protocol cache rows are domain-scoped metadata, not live backend connections.
          for (const cache of cacheEntries) {
            const compositeKey = `${cache.host}:${cache.port}:${cache.domain ?? ''}`;
            seenCacheKeys.add(compositeKey);
            backends.push({
+              id: `cache:${compositeKey}`,
              backend: key,
              domain: cache.domain ?? null,
              protocol: cache.protocol ?? bm.protocol,
-              activeConnections: bm.activeConnections,
-              totalConnections: bm.totalConnections,
-              connectErrors: bm.connectErrors,
-              handshakeErrors: bm.handshakeErrors,
-              requestErrors: bm.requestErrors,
-              avgConnectTimeMs: Math.round(bm.avgConnectTimeMs * 10) / 10,
-              poolHitRate: Math.round(bm.poolHitRate * 1000) / 1000,
-              h2Failures: bm.h2Failures,
+              activeConnections: 0,
+              totalConnections: 0,
+              connectErrors: 0,
+              handshakeErrors: 0,
+              requestErrors: 0,
+              avgConnectTimeMs: 0,
+              poolHitRate: 0,
+              h2Failures: 0,
              h2Suppressed: cache.h2Suppressed,
              h3Suppressed: cache.h3Suppressed,
              h2CooldownRemainingSecs: cache.h2CooldownRemainingSecs,
@@ -678,6 +682,7 @@ export class MetricsManager {
        const compositeKey = `${entry.host}:${entry.port}:${entry.domain ?? ''}`;
        if (!seenCacheKeys.has(compositeKey)) {
          backends.push({
+            id: `cache:${compositeKey}`,
            backend: `${entry.host}:${entry.port}`,
            domain: entry.domain,
            protocol: entry.protocol,
@@ -750,6 +755,9 @@ export class MetricsManager {

      // Resolve wildcards using domains seen in request metrics
      const allKnownDomains = new Set<string>(domainRequestTotals.keys());
+      for (const domain of domainRequestRates.keys()) {
+        allKnownDomains.add(domain);
+      }
      for (const entry of protocolCache) {
        if (entry.domain) allKnownDomains.add(entry.domain);
      }
@@ -775,11 +783,20 @@ export class MetricsManager {
        }
      }

-      // For each route, compute the total request count across all its resolved domains
-      // so we can distribute throughput/connections proportionally
+      const hasLiveDomainRates = domainRequestRates.size > 0;
+      const getDomainWeight = (domain: string): number => {
+        const liveRate = domainRequestRates.get(domain);
+        return hasLiveDomainRates
+          ? (liveRate?.lastMinute ?? 0)
+          : (domainRequestTotals.get(domain) || 0);
+      };
+
+      // For each route, compute the total activity weight across all resolved domains
+      // so we can distribute route-level throughput/connections. Prefer live domain
+      // request rates from SmartProxy 27.8+, falling back to lifetime counters.
      const routeTotalRequests = new Map<string, number>();
      for (const [domain, routeKeys] of domainToRoutes) {
-        const reqs = domainRequestTotals.get(domain) || 0;
+        const reqs = getDomainWeight(domain);
        for (const routeKey of routeKeys) {
          routeTotalRequests.set(routeKey, (routeTotalRequests.get(routeKey) || 0) + reqs);
        }
@@ -792,10 +809,13 @@ export class MetricsManager {
        bytesOutPerSec: number;
        routeCount: number;
        requestCount: number;
+        requestsPerSecond: number;
+        requestsLastMinute: number;
      }>();

      for (const [domain, routeKeys] of domainToRoutes) {
-        const domainReqs = domainRequestTotals.get(domain) || 0;
+        const domainReqs = getDomainWeight(domain);
+        const requestRate = domainRequestRates.get(domain);
        let totalConns = 0;
        let totalIn = 0;
        let totalOut = 0;
@@ -816,7 +836,9 @@ export class MetricsManager {
          bytesInPerSec: totalIn,
          bytesOutPerSec: totalOut,
          routeCount: routeKeys.length,
-          requestCount: domainReqs,
+          requestCount: domainRequestTotals.get(domain) || 0,
+          requestsPerSecond: requestRate?.perSecond ?? 0,
+          requestsLastMinute: requestRate?.lastMinute ?? 0,
        });
      }

@@ -828,8 +850,17 @@ export class MetricsManager {
          activeConnections: data.activeConnections,
          routeCount: data.routeCount,
          requestCount: data.requestCount,
+          requestsPerSecond: data.requestsPerSecond,
+          requestsLastMinute: data.requestsLastMinute,
        }))
-        .sort((a, b) => (b.bytesInPerSecond + b.bytesOutPerSecond) - (a.bytesInPerSecond + a.bytesOutPerSecond));
+        .sort((a, b) => {
+          if (hasLiveDomainRates) {
+            return (b.requestsPerSecond - a.requestsPerSecond) ||
+              (b.requestsLastMinute - a.requestsLastMinute) ||
+              ((b.bytesInPerSecond + b.bytesOutPerSecond) - (a.bytesInPerSecond + a.bytesOutPerSecond));
+          }
+          return (b.bytesInPerSecond + b.bytesOutPerSecond) - (a.bytesInPerSecond + a.bytesOutPerSecond);
+        });

      return {
        connectionsByIP,
@@ -50,19 +50,21 @@ export class SecurityHandler {
            localAddress: conn.destination.ip,
            startTime: conn.startTime,
            protocol: conn.type === 'http' ? 'https' : conn.type as any,
-            state: conn.status as any,
+            state: conn.status === 'active' ? 'connected' : conn.status as any,
            bytesReceived: (conn as any)._throughputIn || 0,
            bytesSent: (conn as any)._throughputOut || 0,
+            connectionCount: conn.bytesTransferred || 1,
          }));
+          const totalConnections = connectionInfos.reduce((sum, conn) => sum + (conn.connectionCount || 1), 0);
          
          const summary = {
-            total: connectionInfos.length,
+            total: totalConnections,
            byProtocol: connectionInfos.reduce((acc, conn) => {
-              acc[conn.protocol] = (acc[conn.protocol] || 0) + 1;
+              acc[conn.protocol] = (acc[conn.protocol] || 0) + (conn.connectionCount || 1);
              return acc;
            }, {} as { [protocol: string]: number }),
            byState: connectionInfos.reduce((acc, conn) => {
-              acc[conn.state] = (acc[conn.state] || 0) + 1;
+              acc[conn.state] = (acc[conn.state] || 0) + (conn.connectionCount || 1);
              return acc;
            }, {} as { [state: string]: number }),
          };
@@ -104,6 +106,8 @@ export class SecurityHandler {
              requestsPerSecond: networkStats.requestsPerSecond || 0,
              requestsTotal: networkStats.requestsTotal || 0,
              backends: networkStats.backends || [],
+              frontendProtocols: networkStats.frontendProtocols || null,
+              backendProtocols: networkStats.backendProtocols || null,
            };
          }

@@ -120,6 +124,8 @@ export class SecurityHandler {
            requestsPerSecond: 0,
            requestsTotal: 0,
            backends: [],
+            frontendProtocols: null,
+            backendProtocols: null,
          };
        }
      )
@@ -335,4 +341,4 @@ export class SecurityHandler {
      limits: [],
    };
  }
-}
+}
@@ -302,6 +302,7 @@ export class StatsHandler {
                    startTime: 0,
                    bytesIn: tp?.in || 0,
                    bytesOut: tp?.out || 0,
+                    connectionCount: count,
                  });
                }