19.6.16

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@push.rocks/smartproxy",
-  "version": "19.6.14",
+  "version": "19.6.16",
   "private": false,
   "description": "A powerful proxy package with unified route-based configuration for high traffic management. Features include SSL/TLS support, flexible routing patterns, WebSocket handling, advanced security options, and automatic ACME certificate management.",
   "main": "dist_ts/index.js",

readme.hints.md

@@ -263,3 +263,36 @@ To reduce log spam during high-traffic scenarios or attacks, SmartProxy implemen
 - Improves log readability and analysis
 - Prevents log storage overflow
 - Maintains detailed information in aggregated form
+
+### Log Output Examples
+
+Instead of hundreds of individual logs:
+```
+Connection rejected
+Connection rejected
+Connection rejected
+... (repeated 500 times)
+```
+
+You'll see:
+```
+[SUMMARY] Rejected 500 connections from 10 IPs in 5s (rate-limit: 350, per-ip-limit: 150) (top offenders: 192.168.1.100 (200x, rate-limit), 10.0.0.1 (150x, per-ip-limit))
+```
+
+Instead of:
+```
+Connection terminated: ::ffff:127.0.0.1 (client_closed). Active: 266
+Connection terminated: ::ffff:127.0.0.1 (client_closed). Active: 265
+... (repeated 266 times)
+```
+
+You'll see:
+```
+[SUMMARY] 266 HttpProxy connections terminated in 5s (reasons: client_closed: 266, activeConnections: 0)
+```
+
+### Rapid Event Handling
+- During attacks or high-volume scenarios, logs are flushed more frequently
+- If 50+ events occur within 1 second, immediate flush is triggered
+- Prevents memory buildup during flooding attacks
+- Maintains real-time visibility during incidents

readme.plan.md

@@ -37,9 +37,17 @@ Command to re-read CLAUDE.md: `cat /home/philkunz/.claude/CLAUDE.md`
 - [x] Test cleanup queue edge cases
 - [x] Test memory usage with many unique IPs
 
+### 6. Log Deduplication for High-Volume Scenarios ✓
+- [x] Implement LogDeduplicator utility for batching similar events
+- [x] Add deduplication for connection rejections, terminations, and cleanups
+- [x] Include rejection reasons in IP rejection summaries
+- [x] Provide aggregated summaries with meaningful context
+
 ## Notes
 
 - All connection limiting is now consistent across SmartProxy and HttpProxy
 - Route-level limits provide additional granular control
 - Memory usage is optimized for high-traffic scenarios
 - Comprehensive test coverage ensures reliability
+- Log deduplication reduces spam during attacks or high-traffic periods
+- IP rejection summaries now include rejection reasons in main message

ts/core/utils/log-deduplicator.ts

@@ -23,6 +23,8 @@ export class LogDeduplicator {
   private aggregatedEvents: Map<string, IAggregatedEvent> = new Map();
   private flushInterval: number = 5000; // 5 seconds
   private maxBatchSize: number = 100;
+  private rapidEventThreshold: number = 50; // Flush early if this many events in 1 second
+  private lastRapidCheck: number = Date.now();
   
   constructor(flushInterval?: number) {
     if (flushInterval) {
@@ -85,8 +87,15 @@ export class LogDeduplicator {
       });
     }
     
-    // Check if we should flush due to size
+    // Check for rapid events (many events in short time)
-    if (aggregated.events.size >= this.maxBatchSize) {
+    const totalEvents = Array.from(aggregated.events.values()).reduce((sum, e) => sum + e.count, 0);
+    
+    // If we're getting flooded with events, flush more frequently
+    if (now - this.lastRapidCheck < 1000 && totalEvents >= this.rapidEventThreshold) {
+      this.flush(key);
+      this.lastRapidCheck = now;
+    } else if (aggregated.events.size >= this.maxBatchSize) {
+      // Check if we should flush due to size
       this.flush(key);
     } else if (!aggregated.flushTimer) {
       // Schedule flush
@@ -98,6 +107,11 @@ export class LogDeduplicator {
         aggregated.flushTimer.unref();
       }
     }
+    
+    // Update rapid check time
+    if (now - this.lastRapidCheck >= 1000) {
+      this.lastRapidCheck = now;
+    }
   }
   
   /**
@@ -122,6 +136,9 @@ export class LogDeduplicator {
       case 'connection-cleanup':
         this.flushConnectionCleanups(aggregated);
         break;
+      case 'connection-terminated':
+        this.flushConnectionTerminations(aggregated);
+        break;
       case 'ip-rejected':
         this.flushIPRejections(aggregated);
         break;
@@ -156,10 +173,10 @@ export class LogDeduplicator {
       .map(([reason, count]) => `${reason}: ${count}`)
       .join(', ');
     
-    logger.log('warn', `Rejected ${totalCount} connections`, {
+    const duration = Date.now() - Math.min(...Array.from(aggregated.events.values()).map(e => e.firstSeen));
+    logger.log('warn', `[SUMMARY] Rejected ${totalCount} connections in ${Math.round(duration/1000)}s`, {
       reasons: reasonSummary,
       uniqueIPs: aggregated.events.size,
-      duration: Date.now() - Math.min(...Array.from(aggregated.events.values()).map(e => e.firstSeen)),
       component: 'connection-dedup'
     });
   }
@@ -186,8 +203,73 @@ export class LogDeduplicator {
     });
   }
   
+  private flushConnectionTerminations(aggregated: IAggregatedEvent): void {
+    const totalCount = Array.from(aggregated.events.values()).reduce((sum, e) => sum + e.count, 0);
+    const byReason = new Map<string, number>();
+    const byIP = new Map<string, number>();
+    let lastActiveCount = 0;
+    
+    for (const [, event] of aggregated.events) {
+      const reason = event.data?.reason || 'unknown';
+      const ip = event.data?.remoteIP || 'unknown';
+      
+      byReason.set(reason, (byReason.get(reason) || 0) + event.count);
+      
+      // Track by IP
+      if (ip !== 'unknown') {
+        byIP.set(ip, (byIP.get(ip) || 0) + event.count);
+      }
+      
+      // Track the last active connection count
+      if (event.data?.activeConnections !== undefined) {
+        lastActiveCount = event.data.activeConnections;
+      }
+    }
+    
+    const reasonSummary = Array.from(byReason.entries())
+      .sort((a, b) => b[1] - a[1])
+      .slice(0, 5) // Top 5 reasons
+      .map(([reason, count]) => `${reason}: ${count}`)
+      .join(', ');
+    
+    // Show top IPs if there are many different ones
+    let ipInfo = '';
+    if (byIP.size > 3) {
+      const topIPs = Array.from(byIP.entries())
+        .sort((a, b) => b[1] - a[1])
+        .slice(0, 3)
+        .map(([ip, count]) => `${ip} (${count})`)
+        .join(', ');
+      ipInfo = `, from ${byIP.size} IPs (top: ${topIPs})`;
+    } else if (byIP.size > 0) {
+      ipInfo = `, IPs: ${Array.from(byIP.keys()).join(', ')}`;
+    }
+    
+    const duration = Date.now() - Math.min(...Array.from(aggregated.events.values()).map(e => e.firstSeen));
+    
+    // Special handling for localhost connections (HttpProxy)
+    const localhostCount = byIP.get('::ffff:127.0.0.1') || 0;
+    if (localhostCount > 0 && byIP.size === 1) {
+      // All connections are from localhost (HttpProxy)
+      logger.log('info', `[SUMMARY] ${totalCount} HttpProxy connections terminated in ${Math.round(duration/1000)}s`, {
+        reasons: reasonSummary,
+        activeConnections: lastActiveCount,
+        component: 'connection-dedup'
+      });
+    } else {
+      logger.log('info', `[SUMMARY] ${totalCount} connections terminated in ${Math.round(duration/1000)}s`, {
+        reasons: reasonSummary,
+        activeConnections: lastActiveCount,
+        uniqueReasons: byReason.size,
+        ...(ipInfo ? { ips: ipInfo } : {}),
+        component: 'connection-dedup'
+      });
+    }
+  }
+  
   private flushIPRejections(aggregated: IAggregatedEvent): void {
     const byIP = new Map<string, { count: number; reasons: Set<string> }>();
+    const allReasons = new Map<string, number>();
     
     for (const [ip, event] of aggregated.events) {
       if (!byIP.has(ip)) {
@@ -197,9 +279,17 @@ export class LogDeduplicator {
       ipData.count += event.count;
       if (event.data?.reason) {
         ipData.reasons.add(event.data.reason);
+        // Track overall reason counts
+        allReasons.set(event.data.reason, (allReasons.get(event.data.reason) || 0) + event.count);
       }
     }
     
+    // Create reason summary
+    const reasonSummary = Array.from(allReasons.entries())
+      .sort((a, b) => b[1] - a[1])
+      .map(([reason, count]) => `${reason}: ${count}`)
+      .join(', ');
+    
     // Log top offenders
     const topOffenders = Array.from(byIP.entries())
       .sort((a, b) => b[1].count - a[1].count)
@@ -209,9 +299,9 @@ export class LogDeduplicator {
     
     const totalRejections = Array.from(byIP.values()).reduce((sum, data) => sum + data.count, 0);
     
-    logger.log('warn', `Rejected ${totalRejections} connections from ${byIP.size} IPs`, {
+    const duration = Date.now() - Math.min(...Array.from(aggregated.events.values()).map(e => e.firstSeen));
+    logger.log('warn', `[SUMMARY] Rejected ${totalRejections} connections from ${byIP.size} IPs in ${Math.round(duration/1000)}s (${reasonSummary})`, {
       topOffenders,
-      duration: Date.now() - Math.min(...Array.from(aggregated.events.values()).map(e => e.firstSeen)),
       component: 'ip-dedup'
     });
   }

ts/proxies/smart-proxy/connection-manager.ts

@@ -401,23 +401,34 @@ export class ConnectionManager extends LifecycleComponent {
       // Remove the record from the tracking map
       this.connectionRecords.delete(record.id);
 
-      // Log connection details
+      // Use deduplicated logging for connection termination
       if (this.smartProxy.settings.enableDetailedLogging) {
-        logger.log('info', 
+        // For detailed logging, include more info but still deduplicate by IP+reason
-          `Connection terminated: ${record.remoteIP}:${record.localPort} (${reason}) - ` +
+        connectionLogDeduplicator.log(
-          `${plugins.prettyMs(duration)}, IN: ${record.bytesReceived}B, OUT: ${record.bytesSent}B`, 
+          'connection-terminated',
-          logData
+          'info',
+          `Connection terminated: ${record.remoteIP}:${record.localPort}`,
+          {
+            ...logData,
+            duration_ms: duration,
+            bytesIn: record.bytesReceived,
+            bytesOut: record.bytesSent
+          },
+          `${record.remoteIP}-${reason}`
         );
       } else {
-        logger.log('info', 
+        // For normal logging, deduplicate by termination reason
-          `Connection terminated: ${record.remoteIP} (${reason}). Active: ${this.connectionRecords.size}`, 
+        connectionLogDeduplicator.log(
+          'connection-terminated',
+          'info',
+          `Connection terminated`,
           {
-            connectionId: record.id,
             remoteIP: record.remoteIP,
             reason,
             activeConnections: this.connectionRecords.size,
             component: 'connection-manager'
-          }
+          },
+          reason // Group by termination reason
         );
       }
     }

ts/proxies/smart-proxy/route-connection-handler.ts

@@ -90,7 +90,13 @@ export class RouteConnectionHandler {
     // Note: For wrapped sockets, this will use the underlying socket IP until PROXY protocol is parsed
     const ipValidation = this.smartProxy.securityManager.validateIP(wrappedSocket.remoteAddress || '');
     if (!ipValidation.allowed) {
-      logger.log('warn', `Connection rejected`, { remoteIP: wrappedSocket.remoteAddress, reason: ipValidation.reason, component: 'route-handler' });
+      connectionLogDeduplicator.log(
+        'ip-rejected',
+        'warn',
+        `Connection rejected from ${wrappedSocket.remoteAddress}`,
+        { remoteIP: wrappedSocket.remoteAddress, reason: ipValidation.reason, component: 'route-handler' },
+        wrappedSocket.remoteAddress
+      );
       cleanupSocket(wrappedSocket.socket, `rejected-${ipValidation.reason}`, { immediate: true });
       return;
     }