diff --git a/readme.hints.md b/readme.hints.md index 6fa7b73..6f55fc9 100644 --- a/readme.hints.md +++ b/readme.hints.md @@ -276,7 +276,7 @@ Connection rejected You'll see: ``` -[SUMMARY] Rejected 500 connections from 10 IPs in 5s (top offenders: 192.168.1.100 (200x, rate-limit), 10.0.0.1 (150x, per-ip-limit)) +[SUMMARY] Rejected 500 connections from 10 IPs in 5s (rate-limit: 350, per-ip-limit: 150) (top offenders: 192.168.1.100 (200x, rate-limit), 10.0.0.1 (150x, per-ip-limit)) ``` Instead of: diff --git a/readme.plan.md b/readme.plan.md index b4b86cc..304950a 100644 --- a/readme.plan.md +++ b/readme.plan.md @@ -37,9 +37,17 @@ Command to re-read CLAUDE.md: `cat /home/philkunz/.claude/CLAUDE.md` - [x] Test cleanup queue edge cases - [x] Test memory usage with many unique IPs +### 6. Log Deduplication for High-Volume Scenarios ✓ +- [x] Implement LogDeduplicator utility for batching similar events +- [x] Add deduplication for connection rejections, terminations, and cleanups +- [x] Include rejection reasons in IP rejection summaries +- [x] Provide aggregated summaries with meaningful context + ## Notes - All connection limiting is now consistent across SmartProxy and HttpProxy - Route-level limits provide additional granular control - Memory usage is optimized for high-traffic scenarios -- Comprehensive test coverage ensures reliability \ No newline at end of file +- Comprehensive test coverage ensures reliability +- Log deduplication reduces spam during attacks or high-traffic periods +- IP rejection summaries now include rejection reasons in main message \ No newline at end of file diff --git a/ts/core/utils/log-deduplicator.ts b/ts/core/utils/log-deduplicator.ts index c090a5e..3a33995 100644 --- a/ts/core/utils/log-deduplicator.ts +++ b/ts/core/utils/log-deduplicator.ts @@ -269,6 +269,7 @@ export class LogDeduplicator { private flushIPRejections(aggregated: IAggregatedEvent): void { const byIP = new Map }>(); + const allReasons = new Map(); for (const [ip, event] of aggregated.events) { if (!byIP.has(ip)) { @@ -278,9 +279,17 @@ export class LogDeduplicator { ipData.count += event.count; if (event.data?.reason) { ipData.reasons.add(event.data.reason); + // Track overall reason counts + allReasons.set(event.data.reason, (allReasons.get(event.data.reason) || 0) + event.count); } } + // Create reason summary + const reasonSummary = Array.from(allReasons.entries()) + .sort((a, b) => b[1] - a[1]) + .map(([reason, count]) => `${reason}: ${count}`) + .join(', '); + // Log top offenders const topOffenders = Array.from(byIP.entries()) .sort((a, b) => b[1].count - a[1].count) @@ -291,7 +300,7 @@ export class LogDeduplicator { const totalRejections = Array.from(byIP.values()).reduce((sum, data) => sum + data.count, 0); const duration = Date.now() - Math.min(...Array.from(aggregated.events.values()).map(e => e.firstSeen)); - logger.log('warn', `[SUMMARY] Rejected ${totalRejections} connections from ${byIP.size} IPs in ${Math.round(duration/1000)}s`, { + logger.log('warn', `[SUMMARY] Rejected ${totalRejections} connections from ${byIP.size} IPs in ${Math.round(duration/1000)}s (${reasonSummary})`, { topOffenders, component: 'ip-dedup' });