diff --git a/readme.hints.md b/readme.hints.md index 220589d..6fa7b73 100644 --- a/readme.hints.md +++ b/readme.hints.md @@ -262,4 +262,37 @@ To reduce log spam during high-traffic scenarios or attacks, SmartProxy implemen - Provides better overview of patterns (e.g., which IPs are attacking) - Improves log readability and analysis - Prevents log storage overflow -- Maintains detailed information in aggregated form \ No newline at end of file +- Maintains detailed information in aggregated form + +### Log Output Examples + +Instead of hundreds of individual logs: +``` +Connection rejected +Connection rejected +Connection rejected +... (repeated 500 times) +``` + +You'll see: +``` +[SUMMARY] Rejected 500 connections from 10 IPs in 5s (top offenders: 192.168.1.100 (200x, rate-limit), 10.0.0.1 (150x, per-ip-limit)) +``` + +Instead of: +``` +Connection terminated: ::ffff:127.0.0.1 (client_closed). Active: 266 +Connection terminated: ::ffff:127.0.0.1 (client_closed). Active: 265 +... (repeated 266 times) +``` + +You'll see: +``` +[SUMMARY] 266 HttpProxy connections terminated in 5s (reasons: client_closed: 266, activeConnections: 0) +``` + +### Rapid Event Handling +- During attacks or high-volume scenarios, logs are flushed more frequently +- If 50+ events occur within 1 second, immediate flush is triggered +- Prevents memory buildup during flooding attacks +- Maintains real-time visibility during incidents \ No newline at end of file diff --git a/ts/core/utils/log-deduplicator.ts b/ts/core/utils/log-deduplicator.ts index 101f5b0..c090a5e 100644 --- a/ts/core/utils/log-deduplicator.ts +++ b/ts/core/utils/log-deduplicator.ts @@ -23,6 +23,8 @@ export class LogDeduplicator { private aggregatedEvents: Map = new Map(); private flushInterval: number = 5000; // 5 seconds private maxBatchSize: number = 100; + private rapidEventThreshold: number = 50; // Flush early if this many events in 1 second + private lastRapidCheck: number = Date.now(); constructor(flushInterval?: number) { if (flushInterval) { @@ -85,8 +87,15 @@ export class LogDeduplicator { }); } - // Check if we should flush due to size - if (aggregated.events.size >= this.maxBatchSize) { + // Check for rapid events (many events in short time) + const totalEvents = Array.from(aggregated.events.values()).reduce((sum, e) => sum + e.count, 0); + + // If we're getting flooded with events, flush more frequently + if (now - this.lastRapidCheck < 1000 && totalEvents >= this.rapidEventThreshold) { + this.flush(key); + this.lastRapidCheck = now; + } else if (aggregated.events.size >= this.maxBatchSize) { + // Check if we should flush due to size this.flush(key); } else if (!aggregated.flushTimer) { // Schedule flush @@ -98,6 +107,11 @@ export class LogDeduplicator { aggregated.flushTimer.unref(); } } + + // Update rapid check time + if (now - this.lastRapidCheck >= 1000) { + this.lastRapidCheck = now; + } } /** @@ -122,6 +136,9 @@ export class LogDeduplicator { case 'connection-cleanup': this.flushConnectionCleanups(aggregated); break; + case 'connection-terminated': + this.flushConnectionTerminations(aggregated); + break; case 'ip-rejected': this.flushIPRejections(aggregated); break; @@ -156,10 +173,10 @@ export class LogDeduplicator { .map(([reason, count]) => `${reason}: ${count}`) .join(', '); - logger.log('warn', `Rejected ${totalCount} connections`, { + const duration = Date.now() - Math.min(...Array.from(aggregated.events.values()).map(e => e.firstSeen)); + logger.log('warn', `[SUMMARY] Rejected ${totalCount} connections in ${Math.round(duration/1000)}s`, { reasons: reasonSummary, uniqueIPs: aggregated.events.size, - duration: Date.now() - Math.min(...Array.from(aggregated.events.values()).map(e => e.firstSeen)), component: 'connection-dedup' }); } @@ -186,6 +203,70 @@ export class LogDeduplicator { }); } + private flushConnectionTerminations(aggregated: IAggregatedEvent): void { + const totalCount = Array.from(aggregated.events.values()).reduce((sum, e) => sum + e.count, 0); + const byReason = new Map(); + const byIP = new Map(); + let lastActiveCount = 0; + + for (const [, event] of aggregated.events) { + const reason = event.data?.reason || 'unknown'; + const ip = event.data?.remoteIP || 'unknown'; + + byReason.set(reason, (byReason.get(reason) || 0) + event.count); + + // Track by IP + if (ip !== 'unknown') { + byIP.set(ip, (byIP.get(ip) || 0) + event.count); + } + + // Track the last active connection count + if (event.data?.activeConnections !== undefined) { + lastActiveCount = event.data.activeConnections; + } + } + + const reasonSummary = Array.from(byReason.entries()) + .sort((a, b) => b[1] - a[1]) + .slice(0, 5) // Top 5 reasons + .map(([reason, count]) => `${reason}: ${count}`) + .join(', '); + + // Show top IPs if there are many different ones + let ipInfo = ''; + if (byIP.size > 3) { + const topIPs = Array.from(byIP.entries()) + .sort((a, b) => b[1] - a[1]) + .slice(0, 3) + .map(([ip, count]) => `${ip} (${count})`) + .join(', '); + ipInfo = `, from ${byIP.size} IPs (top: ${topIPs})`; + } else if (byIP.size > 0) { + ipInfo = `, IPs: ${Array.from(byIP.keys()).join(', ')}`; + } + + const duration = Date.now() - Math.min(...Array.from(aggregated.events.values()).map(e => e.firstSeen)); + + // Special handling for localhost connections (HttpProxy) + const localhostCount = byIP.get('::ffff:127.0.0.1') || 0; + if (localhostCount > 0 && byIP.size === 1) { + // All connections are from localhost (HttpProxy) + logger.log('info', `[SUMMARY] ${totalCount} HttpProxy connections terminated in ${Math.round(duration/1000)}s`, { + reasons: reasonSummary, + activeConnections: lastActiveCount, + component: 'connection-dedup' + }); + } else { + logger.log('info', `[SUMMARY] ${totalCount} connections terminated in ${Math.round(duration/1000)}s`, { + reasons: reasonSummary, + activeConnections: lastActiveCount, + uniqueReasons: byReason.size, + ...(ipInfo ? { ips: ipInfo } : {}), + component: 'connection-dedup' + }); + } + } + private flushIPRejections(aggregated: IAggregatedEvent): void { const byIP = new Map }>(); @@ -209,9 +290,9 @@ export class LogDeduplicator { const totalRejections = Array.from(byIP.values()).reduce((sum, data) => sum + data.count, 0); - logger.log('warn', `Rejected ${totalRejections} connections from ${byIP.size} IPs`, { + const duration = Date.now() - Math.min(...Array.from(aggregated.events.values()).map(e => e.firstSeen)); + logger.log('warn', `[SUMMARY] Rejected ${totalRejections} connections from ${byIP.size} IPs in ${Math.round(duration/1000)}s`, { topOffenders, - duration: Date.now() - Math.min(...Array.from(aggregated.events.values()).map(e => e.firstSeen)), component: 'ip-dedup' }); } diff --git a/ts/proxies/smart-proxy/connection-manager.ts b/ts/proxies/smart-proxy/connection-manager.ts index 60b773f..1f05086 100644 --- a/ts/proxies/smart-proxy/connection-manager.ts +++ b/ts/proxies/smart-proxy/connection-manager.ts @@ -401,23 +401,34 @@ export class ConnectionManager extends LifecycleComponent { // Remove the record from the tracking map this.connectionRecords.delete(record.id); - // Log connection details + // Use deduplicated logging for connection termination if (this.smartProxy.settings.enableDetailedLogging) { - logger.log('info', - `Connection terminated: ${record.remoteIP}:${record.localPort} (${reason}) - ` + - `${plugins.prettyMs(duration)}, IN: ${record.bytesReceived}B, OUT: ${record.bytesSent}B`, - logData + // For detailed logging, include more info but still deduplicate by IP+reason + connectionLogDeduplicator.log( + 'connection-terminated', + 'info', + `Connection terminated: ${record.remoteIP}:${record.localPort}`, + { + ...logData, + duration_ms: duration, + bytesIn: record.bytesReceived, + bytesOut: record.bytesSent + }, + `${record.remoteIP}-${reason}` ); } else { - logger.log('info', - `Connection terminated: ${record.remoteIP} (${reason}). Active: ${this.connectionRecords.size}`, + // For normal logging, deduplicate by termination reason + connectionLogDeduplicator.log( + 'connection-terminated', + 'info', + `Connection terminated`, { - connectionId: record.id, remoteIP: record.remoteIP, reason, activeConnections: this.connectionRecords.size, component: 'connection-manager' - } + }, + reason // Group by termination reason ); } } diff --git a/ts/proxies/smart-proxy/route-connection-handler.ts b/ts/proxies/smart-proxy/route-connection-handler.ts index be96ea3..b6a8445 100644 --- a/ts/proxies/smart-proxy/route-connection-handler.ts +++ b/ts/proxies/smart-proxy/route-connection-handler.ts @@ -90,7 +90,13 @@ export class RouteConnectionHandler { // Note: For wrapped sockets, this will use the underlying socket IP until PROXY protocol is parsed const ipValidation = this.smartProxy.securityManager.validateIP(wrappedSocket.remoteAddress || ''); if (!ipValidation.allowed) { - logger.log('warn', `Connection rejected`, { remoteIP: wrappedSocket.remoteAddress, reason: ipValidation.reason, component: 'route-handler' }); + connectionLogDeduplicator.log( + 'ip-rejected', + 'warn', + `Connection rejected from ${wrappedSocket.remoteAddress}`, + { remoteIP: wrappedSocket.remoteAddress, reason: ipValidation.reason, component: 'route-handler' }, + wrappedSocket.remoteAddress + ); cleanupSocket(wrappedSocket.socket, `rejected-${ipValidation.reason}`, { immediate: true }); return; }