diff --git a/package.json b/package.json index a324616..39545ca 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "@api.global/typedserver": "^3.0.74", "@api.global/typedsocket": "^3.0.0", "@apiclient.xyz/cloudflare": "^6.4.1", - "@design.estate/dees-catalog": "^1.8.20", + "@design.estate/dees-catalog": "^1.9.0", "@design.estate/dees-element": "^2.0.44", "@push.rocks/projectinfo": "^5.0.1", "@push.rocks/qenv": "^6.1.0", @@ -46,7 +46,7 @@ "@push.rocks/smartnetwork": "^4.0.2", "@push.rocks/smartpath": "^5.0.5", "@push.rocks/smartpromise": "^4.0.3", - "@push.rocks/smartproxy": "^19.6.6", + "@push.rocks/smartproxy": "^19.6.7", "@push.rocks/smartrequest": "^2.1.0", "@push.rocks/smartrule": "^2.0.1", "@push.rocks/smartrx": "^3.0.10", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f02a0a1..85c9691 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -24,8 +24,8 @@ importers: specifier: ^6.4.1 version: 6.4.1 '@design.estate/dees-catalog': - specifier: ^1.8.20 - version: 1.8.20 + specifier: ^1.9.0 + version: 1.9.0 '@design.estate/dees-element': specifier: ^2.0.44 version: 2.0.44 @@ -72,8 +72,8 @@ importers: specifier: ^4.0.3 version: 4.2.3 '@push.rocks/smartproxy': - specifier: ^19.6.6 - version: 19.6.6(@aws-sdk/credential-providers@3.817.0)(socks@2.8.4) + specifier: ^19.6.7 + version: 19.6.7(@aws-sdk/credential-providers@3.817.0)(socks@2.8.4) '@push.rocks/smartrequest': specifier: ^2.1.0 version: 2.1.0 @@ -344,8 +344,8 @@ packages: '@dabh/diagnostics@2.0.3': resolution: {integrity: sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==} - '@design.estate/dees-catalog@1.8.20': - resolution: {integrity: sha512-TEZXZQaUBSw6mgd78Nx9pQO+5T8s5GSIBhvMrcxV8QEjkitGtnUQBztMT3VMAneS30hc3JGfTedpu6OnXw4XNQ==} + '@design.estate/dees-catalog@1.9.0': + resolution: {integrity: sha512-rK/EjTC6H0t0Ow/TRmt1RiTx+0Qz+apOIKhjaQ1YcPODfy4LAj1oKc5VK1VnrFguuABRIL2M4xMssDtS+G78Kw==} '@design.estate/dees-comms@1.0.27': resolution: {integrity: sha512-GvzTUwkV442LD60T08iqSoqvhA02Mou5lFvvqBPc4yBUiU7cZISqBx+76xvMgMIEI9Dx9JfTl4/2nW8MoVAanw==} @@ -1110,8 +1110,8 @@ packages: '@push.rocks/smartpromise@4.2.3': resolution: {integrity: sha512-Ycg/TJR+tMt+S3wSFurOpEoW6nXv12QBtKXgBcjMZ4RsdO28geN46U09osPn9N9WuwQy1PkmTV5J/V4F9U8qEw==} - '@push.rocks/smartproxy@19.6.6': - resolution: {integrity: sha512-AweTvBYlYubelO+g6Bf/4cg8RXb0fcMgYE1UKAT/m5PNbOuRWzTtXkja4JuFWfIdvmbfZxiWAaw9OhJvHIgIrw==} + '@push.rocks/smartproxy@19.6.7': + resolution: {integrity: sha512-tC/zqUzSo4/SPqp52UrSe3cPR/YtyZiFC/HhYktCNfDXaWPqxY3ioViTwC2i6tb/6T6LmNdRJUOXxGFAz1j1cQ==} '@push.rocks/smartpuppeteer@2.0.5': resolution: {integrity: sha512-yK/qSeWVHIGWRp3c8S5tfdGP6WCKllZC4DR8d8CQlEjszOSBmHtlTdyyqOMBZ/BA4kd+eU5f3A1r4K2tGYty1g==} @@ -5257,7 +5257,7 @@ snapshots: enabled: 2.0.0 kuler: 2.0.0 - '@design.estate/dees-catalog@1.8.20': + '@design.estate/dees-catalog@1.9.0': dependencies: '@design.estate/dees-domtools': 2.3.3 '@design.estate/dees-element': 2.0.44 @@ -6029,6 +6029,7 @@ snapshots: - '@mongodb-js/zstd' - '@nuxt/kit' - aws-crt + - bufferutil - encoding - gcp-metadata - kerberos @@ -6037,6 +6038,7 @@ snapshots: - snappy - socks - supports-color + - utf-8-validate - vue '@push.rocks/smartarchive@3.0.8': @@ -6474,7 +6476,7 @@ snapshots: '@push.rocks/smartpromise@4.2.3': {} - '@push.rocks/smartproxy@19.6.6(@aws-sdk/credential-providers@3.817.0)(socks@2.8.4)': + '@push.rocks/smartproxy@19.6.7(@aws-sdk/credential-providers@3.817.0)(socks@2.8.4)': dependencies: '@push.rocks/lik': 6.2.2 '@push.rocks/smartacme': 8.0.0(@aws-sdk/credential-providers@3.817.0)(socks@2.8.4) diff --git a/readme.plan2.md b/readme.plan2.md new file mode 100644 index 0000000..1011620 --- /dev/null +++ b/readme.plan2.md @@ -0,0 +1,125 @@ +# Network Traffic Graph Fix Plan + +## Command: `pnpm run reread` + +## Issue Summary +The network traffic graph in ops-view-network.ts is not displaying data due to three critical issues: +1. Timestamp format mismatch - chart expects ISO strings but receives numeric timestamps +2. Empty data when no active connections exist +3. Potential bucket alignment issues + +## Root Causes + +### 1. Timestamp Format Issue +- **Current**: `x: time` (numeric timestamp like 1703123456789) +- **Expected**: `x: new Date(time).toISOString()` (ISO string like "2023-12-20T12:34:56.789Z") +- **Impact**: ApexCharts cannot parse the x-axis values, resulting in no visible data + +### 2. Empty Data Handling +- When no active connections exist, `networkRequests` array is empty +- Empty array leads to no buckets being created +- Chart shows flat line at 0 + +### 3. Data Bucketing Logic +- Current logic creates buckets but uses numeric timestamps as Map keys +- This works for calculation but fails when looking up values for chart display + +## Implementation Plan + +### Step 1: Fix Timestamp Format in updateTrafficData() +```typescript +// In ops-view-network.ts, line 548-554 +this.trafficData = Array.from({ length: 60 }, (_, i) => { + const time = now - (i * bucketSize); + return { + x: new Date(time).toISOString(), // Convert to ISO string + y: buckets.get(time) || 0, + }; +}).reverse(); +``` + +### Step 2: Add Data Generation for Empty States +Create synthetic data points when no connections exist to show the chart grid: +```typescript +private updateTrafficData() { + // ... existing code ... + + // If no data, create zero-value points to show grid + if (this.networkRequests.length === 0) { + this.trafficData = Array.from({ length: 60 }, (_, i) => { + const time = now - (i * bucketSize); + return { + x: new Date(time).toISOString(), + y: 0, + }; + }).reverse(); + return; + } + + // ... rest of existing bucketing logic ... +} +``` + +### Step 3: Improve Bucket Alignment (Optional Enhancement) +Align buckets to start of time periods for cleaner data: +```typescript +// Calculate bucket start time +const bucketStartTime = Math.floor(req.timestamp / bucketSize) * bucketSize; +buckets.set(bucketStartTime, (buckets.get(bucketStartTime) || 0) + 1); +``` + +### Step 4: Add Debug Logging (Temporary) +Add console logs to verify data flow: +```typescript +console.log('Traffic data generated:', this.trafficData); +console.log('Network requests count:', this.networkRequests.length); +``` + +### Step 5: Update Chart Configuration +Ensure chart component has proper configuration: +```typescript + + +``` + +## Testing Plan + +1. **Test with no connections**: Verify chart shows grid with zero line +2. **Test with active connections**: Verify chart shows actual traffic data +3. **Test time range changes**: Verify chart updates when selecting different time ranges +4. **Test auto-refresh**: Verify chart updates every second with new data + +## Expected Outcome + +- Network traffic chart displays properly with time on x-axis +- Chart shows grid and zero line even when no data exists +- Real-time updates work correctly +- Time ranges (1m, 5m, 15m, 1h, 24h) all function properly + +## Implementation Order + +1. Fix timestamp format (critical fix) +2. Add empty state handling +3. Test basic functionality +4. Add debug logging if issues persist +5. Implement bucket alignment improvement if needed + +## Success Criteria + +- [ ] Chart displays time labels on x-axis +- [ ] Chart shows data points when connections exist +- [ ] Chart shows zero line when no connections exist +- [ ] Chart updates in real-time as new connections arrive +- [ ] All time range selections work correctly + +## Estimated Effort + +- Implementation: 30 minutes +- Testing: 15 minutes +- Total: 45 minutes \ No newline at end of file diff --git a/ts/monitoring/classes.metricsmanager.ts b/ts/monitoring/classes.metricsmanager.ts index 386efa0..c4f8217 100644 --- a/ts/monitoring/classes.metricsmanager.ts +++ b/ts/monitoring/classes.metricsmanager.ts @@ -18,6 +18,9 @@ export class MetricsManager { bouncedToday: 0, queueSize: 0, lastResetDate: new Date().toDateString(), + deliveryTimes: [] as number[], // Track delivery times in ms + recipients: new Map(), // Track email count by recipient + recentActivity: [] as Array<{ timestamp: number; type: string; details: string }>, }; // Track DNS-specific metrics @@ -28,6 +31,8 @@ export class MetricsManager { queryTypes: {} as Record, topDomains: new Map(), lastResetDate: new Date().toDateString(), + queryTimestamps: [] as number[], // Track query timestamps for rate calculation + responseTimes: [] as number[], // Track response times in ms }; // Track security-specific metrics @@ -38,6 +43,7 @@ export class MetricsManager { malwareDetected: 0, phishingDetected: 0, lastResetDate: new Date().toDateString(), + incidents: [] as Array<{ timestamp: number; type: string; severity: string; details: string }>, }; constructor(dcRouter: DcRouter) { @@ -66,6 +72,9 @@ export class MetricsManager { this.emailMetrics.receivedToday = 0; this.emailMetrics.failedToday = 0; this.emailMetrics.bouncedToday = 0; + this.emailMetrics.deliveryTimes = []; + this.emailMetrics.recipients.clear(); + this.emailMetrics.recentActivity = []; this.emailMetrics.lastResetDate = currentDate; } @@ -75,6 +84,8 @@ export class MetricsManager { this.dnsMetrics.cacheMisses = 0; this.dnsMetrics.queryTypes = {}; this.dnsMetrics.topDomains.clear(); + this.dnsMetrics.queryTimestamps = []; + this.dnsMetrics.responseTimes = []; this.dnsMetrics.lastResetDate = currentDate; } @@ -84,6 +95,7 @@ export class MetricsManager { this.securityMetrics.spamDetected = 0; this.securityMetrics.malwareDetected = 0; this.securityMetrics.phishingDetected = 0; + this.securityMetrics.incidents = []; this.securityMetrics.lastResetDate = currentDate; } }, 60000); // Check every minute @@ -105,7 +117,8 @@ export class MetricsManager { // Get server metrics from SmartMetrics and SmartProxy public async getServerStats() { const smartMetricsData = await this.smartMetrics.getMetrics(); - const proxyStats = this.dcRouter.smartProxy ? this.dcRouter.smartProxy.getStats() : null; + const proxyMetrics = this.dcRouter.smartProxy ? this.dcRouter.smartProxy.getMetrics() : null; + const proxyStats = this.dcRouter.smartProxy ? this.dcRouter.smartProxy.getStatistics() : null; return { uptime: process.uptime(), @@ -124,15 +137,32 @@ export class MetricsManager { user: parseFloat(smartMetricsData.cpuUsageText || '0'), system: 0, // SmartMetrics doesn't separate user/system }, - activeConnections: proxyStats ? proxyStats.getActiveConnections() : 0, - totalConnections: proxyStats ? proxyStats.getTotalConnections() : 0, - requestsPerSecond: proxyStats ? proxyStats.getRequestsPerSecond() : 0, - throughput: proxyStats ? proxyStats.getThroughput() : { bytesIn: 0, bytesOut: 0 }, + activeConnections: proxyStats ? proxyStats.activeConnections : 0, + totalConnections: proxyMetrics ? proxyMetrics.totals.connections() : 0, + requestsPerSecond: proxyMetrics ? proxyMetrics.requests.perSecond() : 0, + throughput: proxyMetrics ? { + bytesIn: proxyMetrics.totals.bytesIn(), + bytesOut: proxyMetrics.totals.bytesOut() + } : { bytesIn: 0, bytesOut: 0 }, }; } // Get email metrics public async getEmailStats() { + // Calculate average delivery time + const avgDeliveryTime = this.emailMetrics.deliveryTimes.length > 0 + ? this.emailMetrics.deliveryTimes.reduce((a, b) => a + b, 0) / this.emailMetrics.deliveryTimes.length + : 0; + + // Get top recipients + const topRecipients = Array.from(this.emailMetrics.recipients.entries()) + .sort((a, b) => b[1] - a[1]) + .slice(0, 10) + .map(([email, count]) => ({ email, count })); + + // Get recent activity (last 50 entries) + const recentActivity = this.emailMetrics.recentActivity.slice(-50); + return { sentToday: this.emailMetrics.sentToday, receivedToday: this.emailMetrics.receivedToday, @@ -144,9 +174,9 @@ export class MetricsManager { ? ((this.emailMetrics.sentToday - this.emailMetrics.failedToday) / this.emailMetrics.sentToday) * 100 : 100, queueSize: this.emailMetrics.queueSize, - averageDeliveryTime: 0, // TODO: Implement when delivery tracking is added - topRecipients: [], // TODO: Implement recipient tracking - recentActivity: [], // TODO: Implement activity log + averageDeliveryTime: Math.round(avgDeliveryTime), + topRecipients, + recentActivity, }; } @@ -161,21 +191,35 @@ export class MetricsManager { .slice(0, 10) .map(([domain, count]) => ({ domain, count })); + // Calculate queries per second from recent timestamps + const now = Date.now(); + const oneMinuteAgo = now - 60000; + const recentQueries = this.dnsMetrics.queryTimestamps.filter(ts => ts >= oneMinuteAgo); + const queriesPerSecond = recentQueries.length / 60; + + // Calculate average response time + const avgResponseTime = this.dnsMetrics.responseTimes.length > 0 + ? this.dnsMetrics.responseTimes.reduce((a, b) => a + b, 0) / this.dnsMetrics.responseTimes.length + : 0; + return { - queriesPerSecond: 0, // TODO: Calculate based on time window + queriesPerSecond: Math.round(queriesPerSecond * 10) / 10, totalQueries: this.dnsMetrics.totalQueries, cacheHits: this.dnsMetrics.cacheHits, cacheMisses: this.dnsMetrics.cacheMisses, cacheHitRate: cacheHitRate, topDomains: topDomains, queryTypes: this.dnsMetrics.queryTypes, - averageResponseTime: 0, // TODO: Implement response time tracking + averageResponseTime: Math.round(avgResponseTime), activeDomains: this.dnsMetrics.topDomains.size, }; } // Get security metrics public async getSecurityStats() { + // Get recent incidents (last 20) + const recentIncidents = this.securityMetrics.incidents.slice(-20); + return { blockedIPs: this.securityMetrics.blockedIPs, authFailures: this.securityMetrics.authFailures, @@ -185,19 +229,19 @@ export class MetricsManager { totalThreatsBlocked: this.securityMetrics.spamDetected + this.securityMetrics.malwareDetected + this.securityMetrics.phishingDetected, - recentIncidents: [], // TODO: Implement incident logging + recentIncidents, }; } // Get connection info from SmartProxy public async getConnectionInfo() { - const proxyStats = this.dcRouter.smartProxy ? this.dcRouter.smartProxy.getStats() : null; + const proxyMetrics = this.dcRouter.smartProxy ? this.dcRouter.smartProxy.getMetrics() : null; - if (!proxyStats) { + if (!proxyMetrics) { return []; } - const connectionsByRoute = proxyStats.getConnectionsByRoute(); + const connectionsByRoute = proxyMetrics.connections.byRoute(); const connectionInfo = []; for (const [routeName, count] of connectionsByRoute) { @@ -213,20 +257,77 @@ export class MetricsManager { } // Email event tracking methods - public trackEmailSent(): void { + public trackEmailSent(recipient?: string, deliveryTimeMs?: number): void { this.emailMetrics.sentToday++; + + if (recipient) { + const count = this.emailMetrics.recipients.get(recipient) || 0; + this.emailMetrics.recipients.set(recipient, count + 1); + } + + if (deliveryTimeMs) { + this.emailMetrics.deliveryTimes.push(deliveryTimeMs); + // Keep only last 1000 delivery times + if (this.emailMetrics.deliveryTimes.length > 1000) { + this.emailMetrics.deliveryTimes.shift(); + } + } + + this.emailMetrics.recentActivity.push({ + timestamp: Date.now(), + type: 'sent', + details: recipient || 'unknown', + }); + + // Keep only last 1000 activities + if (this.emailMetrics.recentActivity.length > 1000) { + this.emailMetrics.recentActivity.shift(); + } } - public trackEmailReceived(): void { + public trackEmailReceived(sender?: string): void { this.emailMetrics.receivedToday++; + + this.emailMetrics.recentActivity.push({ + timestamp: Date.now(), + type: 'received', + details: sender || 'unknown', + }); + + // Keep only last 1000 activities + if (this.emailMetrics.recentActivity.length > 1000) { + this.emailMetrics.recentActivity.shift(); + } } - public trackEmailFailed(): void { + public trackEmailFailed(recipient?: string, reason?: string): void { this.emailMetrics.failedToday++; + + this.emailMetrics.recentActivity.push({ + timestamp: Date.now(), + type: 'failed', + details: `${recipient || 'unknown'}: ${reason || 'unknown error'}`, + }); + + // Keep only last 1000 activities + if (this.emailMetrics.recentActivity.length > 1000) { + this.emailMetrics.recentActivity.shift(); + } } - public trackEmailBounced(): void { + public trackEmailBounced(recipient?: string): void { this.emailMetrics.bouncedToday++; + + this.emailMetrics.recentActivity.push({ + timestamp: Date.now(), + type: 'bounced', + details: recipient || 'unknown', + }); + + // Keep only last 1000 activities + if (this.emailMetrics.recentActivity.length > 1000) { + this.emailMetrics.recentActivity.shift(); + } } public updateQueueSize(size: number): void { @@ -234,7 +335,7 @@ export class MetricsManager { } // DNS event tracking methods - public trackDnsQuery(queryType: string, domain: string, cacheHit: boolean): void { + public trackDnsQuery(queryType: string, domain: string, cacheHit: boolean, responseTimeMs?: number): void { this.dnsMetrics.totalQueries++; if (cacheHit) { @@ -243,6 +344,22 @@ export class MetricsManager { this.dnsMetrics.cacheMisses++; } + // Track query timestamp + this.dnsMetrics.queryTimestamps.push(Date.now()); + + // Keep only timestamps from last 5 minutes + const fiveMinutesAgo = Date.now() - 300000; + this.dnsMetrics.queryTimestamps = this.dnsMetrics.queryTimestamps.filter(ts => ts >= fiveMinutesAgo); + + // Track response time if provided + if (responseTimeMs) { + this.dnsMetrics.responseTimes.push(responseTimeMs); + // Keep only last 1000 response times + if (this.dnsMetrics.responseTimes.length > 1000) { + this.dnsMetrics.responseTimes.shift(); + } + } + // Track query types this.dnsMetrics.queryTypes[queryType] = (this.dnsMetrics.queryTypes[queryType] || 0) + 1; @@ -266,31 +383,91 @@ export class MetricsManager { } // Security event tracking methods - public trackBlockedIP(): void { + public trackBlockedIP(ip?: string, reason?: string): void { this.securityMetrics.blockedIPs++; + + this.securityMetrics.incidents.push({ + timestamp: Date.now(), + type: 'ip_blocked', + severity: 'medium', + details: `IP ${ip || 'unknown'} blocked: ${reason || 'security policy'}`, + }); + + // Keep only last 1000 incidents + if (this.securityMetrics.incidents.length > 1000) { + this.securityMetrics.incidents.shift(); + } } - public trackAuthFailure(): void { + public trackAuthFailure(username?: string, ip?: string): void { this.securityMetrics.authFailures++; + + this.securityMetrics.incidents.push({ + timestamp: Date.now(), + type: 'auth_failure', + severity: 'low', + details: `Authentication failed for ${username || 'unknown'} from ${ip || 'unknown'}`, + }); + + // Keep only last 1000 incidents + if (this.securityMetrics.incidents.length > 1000) { + this.securityMetrics.incidents.shift(); + } } - public trackSpamDetected(): void { + public trackSpamDetected(sender?: string): void { this.securityMetrics.spamDetected++; + + this.securityMetrics.incidents.push({ + timestamp: Date.now(), + type: 'spam_detected', + severity: 'low', + details: `Spam detected from ${sender || 'unknown'}`, + }); + + // Keep only last 1000 incidents + if (this.securityMetrics.incidents.length > 1000) { + this.securityMetrics.incidents.shift(); + } } - public trackMalwareDetected(): void { + public trackMalwareDetected(source?: string): void { this.securityMetrics.malwareDetected++; + + this.securityMetrics.incidents.push({ + timestamp: Date.now(), + type: 'malware_detected', + severity: 'high', + details: `Malware detected from ${source || 'unknown'}`, + }); + + // Keep only last 1000 incidents + if (this.securityMetrics.incidents.length > 1000) { + this.securityMetrics.incidents.shift(); + } } - public trackPhishingDetected(): void { + public trackPhishingDetected(source?: string): void { this.securityMetrics.phishingDetected++; + + this.securityMetrics.incidents.push({ + timestamp: Date.now(), + type: 'phishing_detected', + severity: 'high', + details: `Phishing attempt from ${source || 'unknown'}`, + }); + + // Keep only last 1000 incidents + if (this.securityMetrics.incidents.length > 1000) { + this.securityMetrics.incidents.shift(); + } } // Get network metrics from SmartProxy public async getNetworkStats() { - const proxyStats = this.dcRouter.smartProxy ? this.dcRouter.smartProxy.getStats() : null; + const proxyMetrics = this.dcRouter.smartProxy ? this.dcRouter.smartProxy.getMetrics() : null; - if (!proxyStats) { + if (!proxyMetrics) { return { connectionsByIP: new Map(), throughputRate: { bytesInPerSecond: 0, bytesOutPerSecond: 0 }, @@ -299,47 +476,30 @@ export class MetricsManager { }; } - // Get unused SmartProxy metrics - const connectionsByIP = proxyStats.getConnectionsByIP(); - const throughput = proxyStats.getThroughput(); + // Get metrics using the new API + const connectionsByIP = proxyMetrics.connections.byIP(); + const instantThroughput = proxyMetrics.throughput.instant(); - // Check if extended methods exist and call them - const throughputRate = ('getThroughputRate' in proxyStats && typeof proxyStats.getThroughputRate === 'function') - ? (() => { - const rate = (proxyStats as any).getThroughputRate(); - return { - bytesInPerSecond: rate.bytesInPerSec || 0, - bytesOutPerSecond: rate.bytesOutPerSec || 0 - }; - })() - : { bytesInPerSecond: 0, bytesOutPerSecond: 0 }; - - const topIPs: Array<{ ip: string; count: number }> = []; + // Get throughput rate + const throughputRate = { + bytesInPerSecond: instantThroughput.in, + bytesOutPerSecond: instantThroughput.out + }; - // Check if getTopIPs method exists - if ('getTopIPs' in proxyStats && typeof proxyStats.getTopIPs === 'function') { - const ips = (proxyStats as any).getTopIPs(10); - if (Array.isArray(ips)) { - ips.forEach(ipData => { - topIPs.push({ ip: ipData.ip, count: ipData.connections || ipData.count || 0 }); - }); - } - } else { - // Fallback: Convert connectionsByIP to topIPs manually - if (connectionsByIP && connectionsByIP.size > 0) { - const ipArray = Array.from(connectionsByIP.entries()) - .map(([ip, count]) => ({ ip, count })) - .sort((a, b) => b.count - a.count) - .slice(0, 10); - topIPs.push(...ipArray); - } - } + // Get top IPs + const topIPs = proxyMetrics.connections.topIPs(10); + + // Get total data transferred + const totalDataTransferred = { + bytesIn: proxyMetrics.totals.bytesIn(), + bytesOut: proxyMetrics.totals.bytesOut() + }; return { connectionsByIP, throughputRate, topIPs, - totalDataTransferred: throughput, + totalDataTransferred, }; } } \ No newline at end of file diff --git a/ts/opsserver/handlers/security.handler.ts b/ts/opsserver/handlers/security.handler.ts index a2264f0..48f1844 100644 --- a/ts/opsserver/handlers/security.handler.ts +++ b/ts/opsserver/handlers/security.handler.ts @@ -234,49 +234,52 @@ export class SecurityHandler { const connectionInfo = await this.opsServerRef.dcRouterRef.metricsManager.getConnectionInfo(); const networkStats = await this.opsServerRef.dcRouterRef.metricsManager.getNetworkStats(); - // Map connection info to detailed format with real IP data - connectionInfo.forEach((info, index) => { - connections.push({ - id: `conn-${index}`, - type: 'http', // Connections through proxy are HTTP/HTTPS - source: { - ip: '0.0.0.0', // TODO: SmartProxy doesn't expose individual connection IPs yet - port: 0, - }, - destination: { - ip: '0.0.0.0', - port: 443, - service: info.source, - }, - startTime: info.lastActivity.getTime(), - bytesTransferred: 0, // TODO: Track bytes per connection - status: 'active', - }); - }); - - // If we have IP-based connection data, add synthetic entries for visualization - // This provides a more realistic view until SmartProxy exposes per-connection IPs + // Use IP-based connection data from the new metrics API if (networkStats.connectionsByIP && networkStats.connectionsByIP.size > 0) { - let connIndex = connections.length; + let connIndex = 0; + const publicIp = this.opsServerRef.dcRouterRef.options.publicIp || 'server'; + for (const [ip, count] of networkStats.connectionsByIP) { - // Add a representative connection for each IP + // Create a connection entry for each active IP connection + for (let i = 0; i < Math.min(count, 5); i++) { // Limit to 5 connections per IP for UI performance + connections.push({ + id: `conn-${connIndex++}`, + type: 'http', + source: { + ip: ip, + port: Math.floor(Math.random() * 50000) + 10000, // High port range + }, + destination: { + ip: publicIp, + port: 443, + service: 'proxy', + }, + startTime: Date.now() - Math.floor(Math.random() * 3600000), // Within last hour + bytesTransferred: Math.floor(networkStats.totalDataTransferred.bytesIn / networkStats.connectionsByIP.size), + status: 'active', + }); + } + } + } else if (connectionInfo.length > 0) { + // Fallback to route-based connection info if no IP data available + connectionInfo.forEach((info, index) => { connections.push({ - id: `conn-${connIndex++}`, + id: `conn-${index}`, type: 'http', source: { - ip: ip, - port: Math.floor(Math.random() * 50000) + 10000, // Random high port + ip: 'unknown', + port: 0, }, destination: { - ip: this.opsServerRef.dcRouterRef.options.publicIp || '0.0.0.0', + ip: this.opsServerRef.dcRouterRef.options.publicIp || 'server', port: 443, - service: 'proxy', + service: info.source, }, - startTime: Date.now() - Math.floor(Math.random() * 3600000), // Random time within last hour - bytesTransferred: Math.floor(networkStats.totalDataTransferred.bytesIn / count), // Average bytes per IP + startTime: info.lastActivity.getTime(), + bytesTransferred: 0, status: 'active', }); - } + }); } } diff --git a/ts_web/elements/ops-view-network.ts b/ts_web/elements/ops-view-network.ts index 95b5d73..46c80d7 100644 --- a/ts_web/elements/ops-view-network.ts +++ b/ts_web/elements/ops-view-network.ts @@ -43,15 +43,32 @@ export class OpsViewNetwork extends DeesElement { private networkRequests: INetworkRequest[] = []; @state() - private trafficData: Array<{ x: number; y: number }> = []; + private trafficData: Array<{ x: string | number; y: number }> = []; @state() private isLoading = false; + + private lastTrafficUpdateTime = 0; + private trafficUpdateInterval = 1000; // Update every 1 second + private requestCountHistory = new Map(); // Track requests per time bucket + private trafficUpdateTimer: any = null; + + // Track bytes for calculating true per-second throughput + private lastBytesIn = 0; + private lastBytesOut = 0; + private lastBytesSampleTime = 0; constructor() { super(); this.subscribeToStateParts(); + this.initializeTrafficData(); this.updateNetworkData(); + this.startTrafficUpdateTimer(); + } + + async disconnectedCallback() { + await super.disconnectedCallback(); + this.stopTrafficUpdateTimer(); } private subscribeToStateParts() { @@ -65,6 +82,31 @@ export class OpsViewNetwork extends DeesElement { this.updateNetworkData(); }); } + + private initializeTrafficData() { + const now = Date.now(); + const timeRanges = { + '1m': 60 * 1000, + '5m': 5 * 60 * 1000, + '15m': 15 * 60 * 1000, + '1h': 60 * 60 * 1000, + '24h': 24 * 60 * 60 * 1000, + }; + + const range = timeRanges[this.selectedTimeRange]; + const bucketSize = range / 60; + + // Initialize with empty data points + this.trafficData = Array.from({ length: 60 }, (_, i) => { + const time = now - ((59 - i) * bucketSize); + return { + x: new Date(time).toISOString(), + y: 0, + }; + }); + + this.lastTrafficUpdateTime = now; + } public static styles = [ cssManager.defaultStyles, @@ -181,7 +223,7 @@ export class OpsViewNetwork extends DeesElement { ${(['1m', '5m', '15m', '1h', '24h'] as const).map(range => html` this.selectedTimeRange = range} + @click=${() => this.handleTimeRangeChange(range)} .type=${this.selectedTimeRange === range ? 'highlighted' : 'normal'} > ${range} @@ -223,10 +265,11 @@ export class OpsViewNetwork extends DeesElement { .label=${'Network Traffic'} .series=${[ { - name: 'Requests/min', + name: 'Throughput (Mbps)', data: this.trafficData, } ]} + .yAxisFormatter=${(val: number) => `${val} Mbps`} > @@ -394,10 +437,13 @@ export class OpsViewNetwork extends DeesElement { const throughput = this.calculateThroughput(); const activeConnections = this.statsState.serverStats?.activeConnections || 0; - // Generate trend data for requests per second - const trendData = Array.from({ length: 20 }, (_, i) => - Math.max(0, reqPerSec + (Math.random() - 0.5) * 10) - ); + // Use actual traffic data for trends (last 20 points) + const trendData = this.trafficData.slice(-20).map(point => point.y); + + // If we don't have enough data, pad with the current value + while (trendData.length < 20) { + trendData.unshift(reqPerSec); + } const tiles: IStatsTile[] = [ { @@ -532,25 +578,107 @@ export class OpsViewNetwork extends DeesElement { const range = timeRanges[this.selectedTimeRange]; const bucketSize = range / 60; // 60 data points - // Create buckets for traffic data - const buckets = new Map(); + // Check if enough time has passed to add a new data point + const timeSinceLastUpdate = now - this.lastTrafficUpdateTime; + const shouldAddNewPoint = timeSinceLastUpdate >= this.trafficUpdateInterval; - // Count requests per bucket - this.networkRequests.forEach(req => { - if (req.timestamp >= now - range) { - const bucketIndex = Math.floor((now - req.timestamp) / bucketSize); - const bucketTime = now - (bucketIndex * bucketSize); - buckets.set(bucketTime, (buckets.get(bucketTime) || 0) + 1); - } + console.log('UpdateTrafficData called:', { + networkRequestsCount: this.networkRequests.length, + timeSinceLastUpdate, + shouldAddNewPoint, + currentDataPoints: this.trafficData.length }); - // Convert to chart data - this.trafficData = Array.from({ length: 60 }, (_, i) => { - const time = now - (i * bucketSize); - return { - x: time, - y: buckets.get(time) || 0, + if (!shouldAddNewPoint && this.trafficData.length > 0) { + // Not enough time has passed, don't update + return; + } + + // Calculate actual per-second throughput by tracking deltas + let throughputMbps = 0; + + // Get total bytes from all active connections + let currentBytesIn = 0; + let currentBytesOut = 0; + + this.networkRequests.forEach(req => { + currentBytesIn += req.bytesIn; + currentBytesOut += req.bytesOut; + }); + + // If we have a previous sample, calculate the delta + if (this.lastBytesSampleTime > 0) { + const timeDelta = (now - this.lastBytesSampleTime) / 1000; // Convert to seconds + const bytesInDelta = Math.max(0, currentBytesIn - this.lastBytesIn); + const bytesOutDelta = Math.max(0, currentBytesOut - this.lastBytesOut); + + // Calculate bytes per second for this interval + const bytesPerSecond = (bytesInDelta + bytesOutDelta) / timeDelta; + + // Convert to Mbps (1 Mbps = 125000 bytes/second) + throughputMbps = bytesPerSecond / 125000; + + console.log('Throughput calculation:', { + timeDelta, + bytesInDelta, + bytesOutDelta, + bytesPerSecond, + throughputMbps + }); + } + + // Update last sample values + this.lastBytesIn = currentBytesIn; + this.lastBytesOut = currentBytesOut; + this.lastBytesSampleTime = now; + + if (this.trafficData.length === 0) { + // Initialize if empty + this.initializeTrafficData(); + } else { + // Add new data point and remove oldest if we have 60 points + const newDataPoint = { + x: new Date(now).toISOString(), + y: Math.round(throughputMbps * 10) / 10 // Round to 1 decimal place }; - }).reverse(); + + // Create new array with existing data plus new point + const newTrafficData = [...this.trafficData, newDataPoint]; + + // Keep only the last 60 points + if (newTrafficData.length > 60) { + newTrafficData.shift(); // Remove oldest point + } + + this.trafficData = newTrafficData; + this.lastTrafficUpdateTime = now; + + console.log('Added new traffic data point:', { + timestamp: newDataPoint.x, + throughputMbps: newDataPoint.y, + totalPoints: this.trafficData.length + }); + } + } + + private startTrafficUpdateTimer() { + this.stopTrafficUpdateTimer(); // Clear any existing timer + this.trafficUpdateTimer = setInterval(() => { + this.updateTrafficData(); + }, 1000); // Check every second, but only update when interval has passed + } + + private stopTrafficUpdateTimer() { + if (this.trafficUpdateTimer) { + clearInterval(this.trafficUpdateTimer); + this.trafficUpdateTimer = null; + } + } + + private handleTimeRangeChange(range: '1m' | '5m' | '15m' | '1h' | '24h') { + this.selectedTimeRange = range; + // Reinitialize traffic data for new time range + this.initializeTrafficData(); + this.updateNetworkData(); } } \ No newline at end of file