update logs

readme.hints.md

@@ -276,7 +276,7 @@ Connection rejected
 
 You'll see:
 ```
-[SUMMARY] Rejected 500 connections from 10 IPs in 5s (top offenders: 192.168.1.100 (200x, rate-limit), 10.0.0.1 (150x, per-ip-limit))
+[SUMMARY] Rejected 500 connections from 10 IPs in 5s (rate-limit: 350, per-ip-limit: 150) (top offenders: 192.168.1.100 (200x, rate-limit), 10.0.0.1 (150x, per-ip-limit))
 ```
 
 Instead of:

readme.plan.md

@@ -37,9 +37,17 @@ Command to re-read CLAUDE.md: `cat /home/philkunz/.claude/CLAUDE.md`
 - [x] Test cleanup queue edge cases
 - [x] Test memory usage with many unique IPs
 
+### 6. Log Deduplication for High-Volume Scenarios ✓
+- [x] Implement LogDeduplicator utility for batching similar events
+- [x] Add deduplication for connection rejections, terminations, and cleanups
+- [x] Include rejection reasons in IP rejection summaries
+- [x] Provide aggregated summaries with meaningful context
+
 ## Notes
 
 - All connection limiting is now consistent across SmartProxy and HttpProxy
 - Route-level limits provide additional granular control
 - Memory usage is optimized for high-traffic scenarios
-- Comprehensive test coverage ensures reliability
+- Comprehensive test coverage ensures reliability
+- Log deduplication reduces spam during attacks or high-traffic periods
+- IP rejection summaries now include rejection reasons in main message

ts/core/utils/log-deduplicator.ts

@@ -269,6 +269,7 @@ export class LogDeduplicator {
   
   private flushIPRejections(aggregated: IAggregatedEvent): void {
     const byIP = new Map<string, { count: number; reasons: Set<string> }>();
+    const allReasons = new Map<string, number>();
     
     for (const [ip, event] of aggregated.events) {
       if (!byIP.has(ip)) {
@@ -278,9 +279,17 @@ export class LogDeduplicator {
       ipData.count += event.count;
       if (event.data?.reason) {
         ipData.reasons.add(event.data.reason);
+        // Track overall reason counts
+        allReasons.set(event.data.reason, (allReasons.get(event.data.reason) || 0) + event.count);
       }
     }
     
+    // Create reason summary
+    const reasonSummary = Array.from(allReasons.entries())
+      .sort((a, b) => b[1] - a[1])
+      .map(([reason, count]) => `${reason}: ${count}`)
+      .join(', ');
+    
     // Log top offenders
     const topOffenders = Array.from(byIP.entries())
       .sort((a, b) => b[1].count - a[1].count)
@@ -291,7 +300,7 @@ export class LogDeduplicator {
     const totalRejections = Array.from(byIP.values()).reduce((sum, data) => sum + data.count, 0);
     
     const duration = Date.now() - Math.min(...Array.from(aggregated.events.values()).map(e => e.firstSeen));
-    logger.log('warn', `[SUMMARY] Rejected ${totalRejections} connections from ${byIP.size} IPs in ${Math.round(duration/1000)}s`, {
+    logger.log('warn', `[SUMMARY] Rejected ${totalRejections} connections from ${byIP.size} IPs in ${Math.round(duration/1000)}s (${reasonSummary})`, {
       topOffenders,
       component: 'ip-dedup'
     });