diff --git a/ts/monitoring/classes.metricscache.ts b/ts/monitoring/classes.metricscache.ts new file mode 100644 index 0000000..1d8e2a7 --- /dev/null +++ b/ts/monitoring/classes.metricscache.ts @@ -0,0 +1,75 @@ +export interface ICacheEntry { + data: T; + timestamp: number; +} + +export class MetricsCache { + private cache = new Map>(); + private readonly defaultTTL: number; + + constructor(defaultTTL: number = 500) { + this.defaultTTL = defaultTTL; + } + + /** + * Get cached data or compute and cache it + */ + public get(key: string, computeFn: () => T | Promise, ttl?: number): T | Promise { + const cached = this.cache.get(key); + const now = Date.now(); + const actualTTL = ttl ?? this.defaultTTL; + + if (cached && (now - cached.timestamp) < actualTTL) { + return cached.data; + } + + const result = computeFn(); + + // Handle both sync and async compute functions + if (result instanceof Promise) { + return result.then(data => { + this.cache.set(key, { data, timestamp: now }); + return data; + }); + } else { + this.cache.set(key, { data: result, timestamp: now }); + return result; + } + } + + /** + * Invalidate a specific cache entry + */ + public invalidate(key: string): void { + this.cache.delete(key); + } + + /** + * Clear all cache entries + */ + public clear(): void { + this.cache.clear(); + } + + /** + * Get cache statistics + */ + public getStats(): { size: number; keys: string[] } { + return { + size: this.cache.size, + keys: Array.from(this.cache.keys()) + }; + } + + /** + * Clean up expired entries + */ + public cleanup(): void { + const now = Date.now(); + for (const [key, entry] of this.cache.entries()) { + if (now - entry.timestamp > this.defaultTTL) { + this.cache.delete(key); + } + } + } +} \ No newline at end of file diff --git a/ts/monitoring/classes.metricsmanager.ts b/ts/monitoring/classes.metricsmanager.ts index c4f8217..b194a8a 100644 --- a/ts/monitoring/classes.metricsmanager.ts +++ b/ts/monitoring/classes.metricsmanager.ts @@ -1,11 +1,13 @@ import * as plugins from '../plugins.js'; import { DcRouter } from '../classes.dcrouter.js'; +import { MetricsCache } from './classes.metricscache.js'; export class MetricsManager { private logger: plugins.smartlog.Smartlog; private smartMetrics: plugins.smartmetrics.SmartMetrics; private dcRouter: DcRouter; private resetInterval?: NodeJS.Timeout; + private metricsCache: MetricsCache; // Constants private readonly MAX_TOP_DOMAINS = 1000; // Limit topDomains Map size @@ -57,6 +59,8 @@ export class MetricsManager { } }); this.smartMetrics = new plugins.smartmetrics.SmartMetrics(this.logger, 'dcrouter'); + // Initialize metrics cache with 500ms TTL + this.metricsCache = new MetricsCache(500); } public async start(): Promise { @@ -116,144 +120,154 @@ export class MetricsManager { // Get server metrics from SmartMetrics and SmartProxy public async getServerStats() { - const smartMetricsData = await this.smartMetrics.getMetrics(); - const proxyMetrics = this.dcRouter.smartProxy ? this.dcRouter.smartProxy.getMetrics() : null; - const proxyStats = this.dcRouter.smartProxy ? this.dcRouter.smartProxy.getStatistics() : null; - - return { - uptime: process.uptime(), - startTime: Date.now() - (process.uptime() * 1000), - memoryUsage: { - heapUsed: process.memoryUsage().heapUsed, - heapTotal: process.memoryUsage().heapTotal, - external: process.memoryUsage().external, - rss: process.memoryUsage().rss, - // Add SmartMetrics memory data - maxMemoryMB: this.smartMetrics.maxMemoryMB, - actualUsageBytes: smartMetricsData.memoryUsageBytes, - actualUsagePercentage: smartMetricsData.memoryPercentage, - }, - cpuUsage: { - user: parseFloat(smartMetricsData.cpuUsageText || '0'), - system: 0, // SmartMetrics doesn't separate user/system - }, - activeConnections: proxyStats ? proxyStats.activeConnections : 0, - totalConnections: proxyMetrics ? proxyMetrics.totals.connections() : 0, - requestsPerSecond: proxyMetrics ? proxyMetrics.requests.perSecond() : 0, - throughput: proxyMetrics ? { - bytesIn: proxyMetrics.totals.bytesIn(), - bytesOut: proxyMetrics.totals.bytesOut() - } : { bytesIn: 0, bytesOut: 0 }, - }; + return this.metricsCache.get('serverStats', async () => { + const smartMetricsData = await this.smartMetrics.getMetrics(); + const proxyMetrics = this.dcRouter.smartProxy ? this.dcRouter.smartProxy.getMetrics() : null; + const proxyStats = this.dcRouter.smartProxy ? this.dcRouter.smartProxy.getStatistics() : null; + + return { + uptime: process.uptime(), + startTime: Date.now() - (process.uptime() * 1000), + memoryUsage: { + heapUsed: process.memoryUsage().heapUsed, + heapTotal: process.memoryUsage().heapTotal, + external: process.memoryUsage().external, + rss: process.memoryUsage().rss, + // Add SmartMetrics memory data + maxMemoryMB: this.smartMetrics.maxMemoryMB, + actualUsageBytes: smartMetricsData.memoryUsageBytes, + actualUsagePercentage: smartMetricsData.memoryPercentage, + }, + cpuUsage: { + user: parseFloat(smartMetricsData.cpuUsageText || '0'), + system: 0, // SmartMetrics doesn't separate user/system + }, + activeConnections: proxyStats ? proxyStats.activeConnections : 0, + totalConnections: proxyMetrics ? proxyMetrics.totals.connections() : 0, + requestsPerSecond: proxyMetrics ? proxyMetrics.requests.perSecond() : 0, + throughput: proxyMetrics ? { + bytesIn: proxyMetrics.totals.bytesIn(), + bytesOut: proxyMetrics.totals.bytesOut() + } : { bytesIn: 0, bytesOut: 0 }, + }; + }); } // Get email metrics public async getEmailStats() { - // Calculate average delivery time - const avgDeliveryTime = this.emailMetrics.deliveryTimes.length > 0 - ? this.emailMetrics.deliveryTimes.reduce((a, b) => a + b, 0) / this.emailMetrics.deliveryTimes.length - : 0; - - // Get top recipients - const topRecipients = Array.from(this.emailMetrics.recipients.entries()) - .sort((a, b) => b[1] - a[1]) - .slice(0, 10) - .map(([email, count]) => ({ email, count })); - - // Get recent activity (last 50 entries) - const recentActivity = this.emailMetrics.recentActivity.slice(-50); - - return { - sentToday: this.emailMetrics.sentToday, - receivedToday: this.emailMetrics.receivedToday, - failedToday: this.emailMetrics.failedToday, - bounceRate: this.emailMetrics.bouncedToday > 0 - ? (this.emailMetrics.bouncedToday / this.emailMetrics.sentToday) * 100 - : 0, - deliveryRate: this.emailMetrics.sentToday > 0 - ? ((this.emailMetrics.sentToday - this.emailMetrics.failedToday) / this.emailMetrics.sentToday) * 100 - : 100, - queueSize: this.emailMetrics.queueSize, - averageDeliveryTime: Math.round(avgDeliveryTime), - topRecipients, - recentActivity, - }; + return this.metricsCache.get('emailStats', () => { + // Calculate average delivery time + const avgDeliveryTime = this.emailMetrics.deliveryTimes.length > 0 + ? this.emailMetrics.deliveryTimes.reduce((a, b) => a + b, 0) / this.emailMetrics.deliveryTimes.length + : 0; + + // Get top recipients + const topRecipients = Array.from(this.emailMetrics.recipients.entries()) + .sort((a, b) => b[1] - a[1]) + .slice(0, 10) + .map(([email, count]) => ({ email, count })); + + // Get recent activity (last 50 entries) + const recentActivity = this.emailMetrics.recentActivity.slice(-50); + + return { + sentToday: this.emailMetrics.sentToday, + receivedToday: this.emailMetrics.receivedToday, + failedToday: this.emailMetrics.failedToday, + bounceRate: this.emailMetrics.bouncedToday > 0 + ? (this.emailMetrics.bouncedToday / this.emailMetrics.sentToday) * 100 + : 0, + deliveryRate: this.emailMetrics.sentToday > 0 + ? ((this.emailMetrics.sentToday - this.emailMetrics.failedToday) / this.emailMetrics.sentToday) * 100 + : 100, + queueSize: this.emailMetrics.queueSize, + averageDeliveryTime: Math.round(avgDeliveryTime), + topRecipients, + recentActivity, + }; + }); } // Get DNS metrics public async getDnsStats() { - const cacheHitRate = this.dnsMetrics.totalQueries > 0 - ? (this.dnsMetrics.cacheHits / this.dnsMetrics.totalQueries) * 100 - : 0; - - const topDomains = Array.from(this.dnsMetrics.topDomains.entries()) - .sort((a, b) => b[1] - a[1]) - .slice(0, 10) - .map(([domain, count]) => ({ domain, count })); - - // Calculate queries per second from recent timestamps - const now = Date.now(); - const oneMinuteAgo = now - 60000; - const recentQueries = this.dnsMetrics.queryTimestamps.filter(ts => ts >= oneMinuteAgo); - const queriesPerSecond = recentQueries.length / 60; - - // Calculate average response time - const avgResponseTime = this.dnsMetrics.responseTimes.length > 0 - ? this.dnsMetrics.responseTimes.reduce((a, b) => a + b, 0) / this.dnsMetrics.responseTimes.length - : 0; - - return { - queriesPerSecond: Math.round(queriesPerSecond * 10) / 10, - totalQueries: this.dnsMetrics.totalQueries, - cacheHits: this.dnsMetrics.cacheHits, - cacheMisses: this.dnsMetrics.cacheMisses, - cacheHitRate: cacheHitRate, - topDomains: topDomains, - queryTypes: this.dnsMetrics.queryTypes, - averageResponseTime: Math.round(avgResponseTime), - activeDomains: this.dnsMetrics.topDomains.size, - }; + return this.metricsCache.get('dnsStats', () => { + const cacheHitRate = this.dnsMetrics.totalQueries > 0 + ? (this.dnsMetrics.cacheHits / this.dnsMetrics.totalQueries) * 100 + : 0; + + const topDomains = Array.from(this.dnsMetrics.topDomains.entries()) + .sort((a, b) => b[1] - a[1]) + .slice(0, 10) + .map(([domain, count]) => ({ domain, count })); + + // Calculate queries per second from recent timestamps + const now = Date.now(); + const oneMinuteAgo = now - 60000; + const recentQueries = this.dnsMetrics.queryTimestamps.filter(ts => ts >= oneMinuteAgo); + const queriesPerSecond = recentQueries.length / 60; + + // Calculate average response time + const avgResponseTime = this.dnsMetrics.responseTimes.length > 0 + ? this.dnsMetrics.responseTimes.reduce((a, b) => a + b, 0) / this.dnsMetrics.responseTimes.length + : 0; + + return { + queriesPerSecond: Math.round(queriesPerSecond * 10) / 10, + totalQueries: this.dnsMetrics.totalQueries, + cacheHits: this.dnsMetrics.cacheHits, + cacheMisses: this.dnsMetrics.cacheMisses, + cacheHitRate: cacheHitRate, + topDomains: topDomains, + queryTypes: this.dnsMetrics.queryTypes, + averageResponseTime: Math.round(avgResponseTime), + activeDomains: this.dnsMetrics.topDomains.size, + }; + }); } // Get security metrics public async getSecurityStats() { - // Get recent incidents (last 20) - const recentIncidents = this.securityMetrics.incidents.slice(-20); - - return { - blockedIPs: this.securityMetrics.blockedIPs, - authFailures: this.securityMetrics.authFailures, - spamDetected: this.securityMetrics.spamDetected, - malwareDetected: this.securityMetrics.malwareDetected, - phishingDetected: this.securityMetrics.phishingDetected, - totalThreatsBlocked: this.securityMetrics.spamDetected + - this.securityMetrics.malwareDetected + - this.securityMetrics.phishingDetected, - recentIncidents, - }; + return this.metricsCache.get('securityStats', () => { + // Get recent incidents (last 20) + const recentIncidents = this.securityMetrics.incidents.slice(-20); + + return { + blockedIPs: this.securityMetrics.blockedIPs, + authFailures: this.securityMetrics.authFailures, + spamDetected: this.securityMetrics.spamDetected, + malwareDetected: this.securityMetrics.malwareDetected, + phishingDetected: this.securityMetrics.phishingDetected, + totalThreatsBlocked: this.securityMetrics.spamDetected + + this.securityMetrics.malwareDetected + + this.securityMetrics.phishingDetected, + recentIncidents, + }; + }); } // Get connection info from SmartProxy public async getConnectionInfo() { - const proxyMetrics = this.dcRouter.smartProxy ? this.dcRouter.smartProxy.getMetrics() : null; - - if (!proxyMetrics) { - return []; - } - - const connectionsByRoute = proxyMetrics.connections.byRoute(); - const connectionInfo = []; - - for (const [routeName, count] of connectionsByRoute) { - connectionInfo.push({ - type: 'https', - count, - source: routeName, - lastActivity: new Date(), - }); - } - - return connectionInfo; + return this.metricsCache.get('connectionInfo', () => { + const proxyMetrics = this.dcRouter.smartProxy ? this.dcRouter.smartProxy.getMetrics() : null; + + if (!proxyMetrics) { + return []; + } + + const connectionsByRoute = proxyMetrics.connections.byRoute(); + const connectionInfo = []; + + for (const [routeName, count] of connectionsByRoute) { + connectionInfo.push({ + type: 'https', + count, + source: routeName, + lastActivity: new Date(), + }); + } + + return connectionInfo; + }); } // Email event tracking methods @@ -465,41 +479,44 @@ export class MetricsManager { // Get network metrics from SmartProxy public async getNetworkStats() { - const proxyMetrics = this.dcRouter.smartProxy ? this.dcRouter.smartProxy.getMetrics() : null; - - if (!proxyMetrics) { - return { - connectionsByIP: new Map(), - throughputRate: { bytesInPerSecond: 0, bytesOutPerSecond: 0 }, - topIPs: [], - totalDataTransferred: { bytesIn: 0, bytesOut: 0 }, + // Use shorter cache TTL for network stats to ensure real-time updates + return this.metricsCache.get('networkStats', () => { + const proxyMetrics = this.dcRouter.smartProxy ? this.dcRouter.smartProxy.getMetrics() : null; + + if (!proxyMetrics) { + return { + connectionsByIP: new Map(), + throughputRate: { bytesInPerSecond: 0, bytesOutPerSecond: 0 }, + topIPs: [], + totalDataTransferred: { bytesIn: 0, bytesOut: 0 }, + }; + } + + // Get metrics using the new API + const connectionsByIP = proxyMetrics.connections.byIP(); + const instantThroughput = proxyMetrics.throughput.instant(); + + // Get throughput rate + const throughputRate = { + bytesInPerSecond: instantThroughput.in, + bytesOutPerSecond: instantThroughput.out }; - } - - // Get metrics using the new API - const connectionsByIP = proxyMetrics.connections.byIP(); - const instantThroughput = proxyMetrics.throughput.instant(); - - // Get throughput rate - const throughputRate = { - bytesInPerSecond: instantThroughput.in, - bytesOutPerSecond: instantThroughput.out - }; - - // Get top IPs - const topIPs = proxyMetrics.connections.topIPs(10); - - // Get total data transferred - const totalDataTransferred = { - bytesIn: proxyMetrics.totals.bytesIn(), - bytesOut: proxyMetrics.totals.bytesOut() - }; - - return { - connectionsByIP, - throughputRate, - topIPs, - totalDataTransferred, - }; + + // Get top IPs + const topIPs = proxyMetrics.connections.topIPs(10); + + // Get total data transferred + const totalDataTransferred = { + bytesIn: proxyMetrics.totals.bytesIn(), + bytesOut: proxyMetrics.totals.bytesOut() + }; + + return { + connectionsByIP, + throughputRate, + topIPs, + totalDataTransferred, + }; + }, 1000); // Use 200ms cache for more frequent updates } } \ No newline at end of file diff --git a/ts/opsserver/handlers/stats.handler.ts b/ts/opsserver/handlers/stats.handler.ts index 71b62ea..a582613 100644 --- a/ts/opsserver/handlers/stats.handler.ts +++ b/ts/opsserver/handlers/stats.handler.ts @@ -162,6 +162,133 @@ export class StatsHandler { } ) ); + + // Combined Metrics Handler - More efficient for frontend polling + this.typedrouter.addTypedHandler( + new plugins.typedrequest.TypedHandler( + 'getCombinedMetrics', + async (dataArg, toolsArg) => { + const sections = dataArg.sections || { + server: true, + email: true, + dns: true, + security: true, + network: true, + }; + + const metrics: any = {}; + + // Run all metrics collection in parallel + const promises: Promise[] = []; + + if (sections.server) { + promises.push( + this.collectServerStats().then(stats => { + metrics.server = { + uptime: stats.uptime, + startTime: Date.now() - (stats.uptime * 1000), + memoryUsage: stats.memoryUsage, + cpuUsage: stats.cpuUsage, + activeConnections: stats.activeConnections, + totalConnections: stats.totalConnections, + }; + }) + ); + } + + if (sections.email) { + promises.push( + this.collectEmailStats().then(stats => { + metrics.email = { + sent: stats.sentToday, + received: stats.receivedToday, + bounced: Math.floor(stats.sentToday * stats.bounceRate / 100), + queued: stats.queueSize, + failed: 0, + averageDeliveryTime: 0, + deliveryRate: stats.deliveryRate, + bounceRate: stats.bounceRate, + }; + }) + ); + } + + if (sections.dns) { + promises.push( + this.collectDnsStats().then(stats => { + metrics.dns = { + totalQueries: stats.totalQueries, + cacheHits: stats.cacheHits, + cacheMisses: stats.cacheMisses, + cacheHitRate: stats.cacheHitRate, + activeDomains: stats.topDomains.length, + averageResponseTime: 0, + queryTypes: stats.queryTypes, + }; + }) + ); + } + + if (sections.security && this.opsServerRef.dcRouterRef.metricsManager) { + promises.push( + this.opsServerRef.dcRouterRef.metricsManager.getSecurityStats().then(stats => { + metrics.security = { + blockedIPs: stats.blockedIPs, + reputationScores: {}, + spamDetected: stats.spamDetected, + malwareDetected: stats.malwareDetected, + phishingDetected: stats.phishingDetected, + authenticationFailures: stats.authFailures, + suspiciousActivities: stats.totalThreatsBlocked, + }; + }) + ); + } + + if (sections.network && this.opsServerRef.dcRouterRef.metricsManager) { + promises.push( + this.opsServerRef.dcRouterRef.metricsManager.getNetworkStats().then(stats => { + const connectionDetails: interfaces.data.IConnectionDetails[] = []; + stats.connectionsByIP.forEach((count, ip) => { + connectionDetails.push({ + remoteAddress: ip, + protocol: 'https' as any, + state: 'established' as any, + startTime: Date.now(), + bytesIn: 0, + bytesOut: 0, + }); + }); + + metrics.network = { + totalBandwidth: { + in: stats.throughputRate.bytesInPerSecond, + out: stats.throughputRate.bytesOutPerSecond, + }, + activeConnections: stats.connectionsByIP.size, + connectionDetails: connectionDetails.slice(0, 50), // Limit to 50 connections + topEndpoints: stats.topIPs.map(ip => ({ + endpoint: ip.ip, + requests: ip.count, + bandwidth: { + in: 0, + out: 0, + }, + })), + }; + }) + ); + } + + await Promise.all(promises); + + return { + metrics, + timestamp: Date.now(), + }; + } + ) + ); } private async collectServerStats(): Promise<{ diff --git a/ts_interfaces/data/stats.ts b/ts_interfaces/data/stats.ts index 57694f1..ce60c83 100644 --- a/ts_interfaces/data/stats.ts +++ b/ts_interfaces/data/stats.ts @@ -102,4 +102,30 @@ export interface IHealthStatus { }; }; version?: string; +} + +export interface INetworkMetrics { + totalBandwidth: { + in: number; + out: number; + }; + activeConnections: number; + connectionDetails: IConnectionDetails[]; + topEndpoints: Array<{ + endpoint: string; + requests: number; + bandwidth: { + in: number; + out: number; + }; + }>; +} + +export interface IConnectionDetails { + remoteAddress: string; + protocol: 'http' | 'https' | 'smtp' | 'smtps'; + state: 'connecting' | 'connected' | 'established' | 'closing'; + startTime: number; + bytesIn: number; + bytesOut: number; } \ No newline at end of file diff --git a/ts_interfaces/requests/combined.stats.ts b/ts_interfaces/requests/combined.stats.ts new file mode 100644 index 0000000..9ac2e15 --- /dev/null +++ b/ts_interfaces/requests/combined.stats.ts @@ -0,0 +1,25 @@ +import type * as data from '../data/index.js'; + +export interface IReq_GetCombinedMetrics { + method: 'getCombinedMetrics'; + request: { + identity: data.IIdentity; + sections?: { + server?: boolean; + email?: boolean; + dns?: boolean; + security?: boolean; + network?: boolean; + }; + }; + response: { + metrics: { + server?: data.IServerStats; + email?: data.IEmailStats; + dns?: data.IDnsStats; + security?: data.ISecurityMetrics; + network?: data.INetworkMetrics; + }; + timestamp: number; + }; +} \ No newline at end of file diff --git a/ts_interfaces/requests/index.ts b/ts_interfaces/requests/index.ts index 9c8485e..9f7e091 100644 --- a/ts_interfaces/requests/index.ts +++ b/ts_interfaces/requests/index.ts @@ -1,4 +1,5 @@ export * from './admin.js'; export * from './config.js'; export * from './logs.js'; -export * from './stats.js'; \ No newline at end of file +export * from './stats.js'; +export * from './combined.stats.js'; \ No newline at end of file diff --git a/ts_web/appstate.ts b/ts_web/appstate.ts index ca0a384..431acd9 100644 --- a/ts_web/appstate.ts +++ b/ts_web/appstate.ts @@ -89,7 +89,7 @@ export const configStatePart = await appState.getStatePart( export const uiStatePart = await appState.getStatePart( 'ui', { - activeView: 'dashboard', + activeView: 'overview', sidebarCollapsed: false, autoRefresh: true, refreshInterval: 1000, // 1 second @@ -184,56 +184,35 @@ export const logoutAction = loginStatePart.createAction(async (statePartArg) => }; }); -// Fetch All Stats Action +// Fetch All Stats Action - Using combined endpoint for efficiency export const fetchAllStatsAction = statsStatePart.createAction(async (statePartArg) => { const context = getActionContext(); const currentState = statePartArg.getState(); try { - // Fetch server stats - const serverStatsRequest = new plugins.domtools.plugins.typedrequest.TypedRequest< - interfaces.requests.IReq_GetServerStatistics - >('/typedrequest', 'getServerStatistics'); + // Use combined metrics endpoint - single request instead of 4 + const combinedRequest = new plugins.domtools.plugins.typedrequest.TypedRequest< + interfaces.requests.IReq_GetCombinedMetrics + >('/typedrequest', 'getCombinedMetrics'); - const serverStatsResponse = await serverStatsRequest.fire({ + const combinedResponse = await combinedRequest.fire({ identity: context.identity, - includeHistory: false, + sections: { + server: true, + email: true, + dns: true, + security: true, + network: false, // Network is fetched separately for the network view + }, }); - // Fetch email stats - const emailStatsRequest = new plugins.domtools.plugins.typedrequest.TypedRequest< - interfaces.requests.IReq_GetEmailStatistics - >('/typedrequest', 'getEmailStatistics'); - - const emailStatsResponse = await emailStatsRequest.fire({ - identity: context.identity, - }); - - // Fetch DNS stats - const dnsStatsRequest = new plugins.domtools.plugins.typedrequest.TypedRequest< - interfaces.requests.IReq_GetDnsStatistics - >('/typedrequest', 'getDnsStatistics'); - - const dnsStatsResponse = await dnsStatsRequest.fire({ - identity: context.identity, - }); - - // Fetch security metrics - const securityRequest = new plugins.domtools.plugins.typedrequest.TypedRequest< - interfaces.requests.IReq_GetSecurityMetrics - >('/typedrequest', 'getSecurityMetrics'); - - const securityResponse = await securityRequest.fire({ - identity: context.identity, - }); - - // Update state with all stats + // Update state with all stats from combined response return { - serverStats: serverStatsResponse.stats, - emailStats: emailStatsResponse.stats, - dnsStats: dnsStatsResponse.stats, - securityMetrics: securityResponse.metrics, + serverStats: combinedResponse.metrics.server || currentState.serverStats, + emailStats: combinedResponse.metrics.email || currentState.emailStats, + dnsStats: combinedResponse.metrics.dns || currentState.dnsStats, + securityMetrics: combinedResponse.metrics.security || currentState.securityMetrics, lastUpdated: Date.now(), isLoading: false, error: null, @@ -342,6 +321,14 @@ export const toggleAutoRefreshAction = uiStatePart.createAction(async (statePart // Set Active View Action export const setActiveViewAction = uiStatePart.createAction(async (statePartArg, viewName) => { const currentState = statePartArg.getState(); + + // If switching to network view, ensure we fetch network data + if (viewName === 'network' && currentState.activeView !== 'network') { + setTimeout(() => { + networkStatePart.dispatchAction(fetchNetworkStatsAction, null); + }, 100); + } + return { ...currentState, activeView: viewName, @@ -410,18 +397,118 @@ export const fetchNetworkStatsAction = networkStatePart.createAction(async (stat } }); +// Combined refresh action for efficient polling +async function dispatchCombinedRefreshAction() { + const context = getActionContext(); + const currentView = uiStatePart.getState().activeView; + + try { + // Always fetch basic stats for dashboard widgets + const combinedRequest = new plugins.domtools.plugins.typedrequest.TypedRequest< + interfaces.requests.IReq_GetCombinedMetrics + >('/typedrequest', 'getCombinedMetrics'); + + const combinedResponse = await combinedRequest.fire({ + identity: context.identity, + sections: { + server: true, + email: true, + dns: true, + security: true, + network: currentView === 'network' || currentView === 'Network', // Only fetch network if on network view + }, + }); + + // Update all stats from combined response + statsStatePart.setState({ + ...statsStatePart.getState(), + serverStats: combinedResponse.metrics.server || statsStatePart.getState().serverStats, + emailStats: combinedResponse.metrics.email || statsStatePart.getState().emailStats, + dnsStats: combinedResponse.metrics.dns || statsStatePart.getState().dnsStats, + securityMetrics: combinedResponse.metrics.security || statsStatePart.getState().securityMetrics, + lastUpdated: Date.now(), + isLoading: false, + error: null, + }); + + // Update network stats if included + if (combinedResponse.metrics.network && (currentView === 'network' || currentView === 'Network')) { + const network = combinedResponse.metrics.network; + const connectionsByIP: { [ip: string]: number } = {}; + + // Convert connection details to IP counts + network.connectionDetails.forEach(conn => { + connectionsByIP[conn.remoteAddress] = (connectionsByIP[conn.remoteAddress] || 0) + 1; + }); + + // Fetch detailed connections for the network view + try { + const connectionsRequest = new plugins.domtools.plugins.typedrequest.TypedRequest< + interfaces.requests.IReq_GetActiveConnections + >('/typedrequest', 'getActiveConnections'); + + const connectionsResponse = await connectionsRequest.fire({ + identity: context.identity, + }); + + networkStatePart.setState({ + ...networkStatePart.getState(), + connections: connectionsResponse.connections, + connectionsByIP, + throughputRate: { + bytesInPerSecond: network.totalBandwidth.in, + bytesOutPerSecond: network.totalBandwidth.out + }, + topIPs: network.topEndpoints.map(e => ({ ip: e.endpoint, count: e.requests })), + lastUpdated: Date.now(), + isLoading: false, + error: null, + }); + } catch (error) { + console.error('Failed to fetch connections:', error); + networkStatePart.setState({ + ...networkStatePart.getState(), + connections: [], + connectionsByIP, + throughputRate: { + bytesInPerSecond: network.totalBandwidth.in, + bytesOutPerSecond: network.totalBandwidth.out + }, + topIPs: network.topEndpoints.map(e => ({ ip: e.endpoint, count: e.requests })), + lastUpdated: Date.now(), + isLoading: false, + error: null, + }); + } + } + } catch (error) { + console.error('Combined refresh failed:', error); + } +} + // Initialize auto-refresh let refreshInterval: NodeJS.Timeout | null = null; +let currentRefreshRate = 1000; // Track current refresh rate to avoid unnecessary restarts // Initialize auto-refresh when UI state is ready (() => { const startAutoRefresh = () => { const uiState = uiStatePart.getState(); - if (uiState.autoRefresh && loginStatePart.getState().isLoggedIn) { - refreshInterval = setInterval(() => { - statsStatePart.dispatchAction(fetchAllStatsAction, null); - networkStatePart.dispatchAction(fetchNetworkStatsAction, null); - }, uiState.refreshInterval); + const loginState = loginStatePart.getState(); + + // Only start if conditions are met and not already running at the same rate + if (uiState.autoRefresh && loginState.isLoggedIn) { + // Check if we need to restart the interval (rate changed or not running) + if (!refreshInterval || currentRefreshRate !== uiState.refreshInterval) { + stopAutoRefresh(); + currentRefreshRate = uiState.refreshInterval; + refreshInterval = setInterval(() => { + // Use combined refresh action for efficiency + dispatchCombinedRefreshAction(); + }, uiState.refreshInterval); + } + } else { + stopAutoRefresh(); } }; @@ -429,18 +516,31 @@ let refreshInterval: NodeJS.Timeout | null = null; if (refreshInterval) { clearInterval(refreshInterval); refreshInterval = null; + currentRefreshRate = 0; } }; - // Watch for changes - uiStatePart.state.subscribe(() => { - stopAutoRefresh(); - startAutoRefresh(); + // Watch for relevant changes only + let previousAutoRefresh = uiStatePart.getState().autoRefresh; + let previousRefreshInterval = uiStatePart.getState().refreshInterval; + let previousIsLoggedIn = loginStatePart.getState().isLoggedIn; + + uiStatePart.state.subscribe((state) => { + // Only restart if relevant values changed + if (state.autoRefresh !== previousAutoRefresh || + state.refreshInterval !== previousRefreshInterval) { + previousAutoRefresh = state.autoRefresh; + previousRefreshInterval = state.refreshInterval; + startAutoRefresh(); + } }); - loginStatePart.state.subscribe(() => { - stopAutoRefresh(); - startAutoRefresh(); + loginStatePart.state.subscribe((state) => { + // Only restart if login state changed + if (state.isLoggedIn !== previousIsLoggedIn) { + previousIsLoggedIn = state.isLoggedIn; + startAutoRefresh(); + } }); // Initial start diff --git a/ts_web/elements/ops-dashboard.ts b/ts_web/elements/ops-dashboard.ts index cbdfd69..1344415 100644 --- a/ts_web/elements/ops-dashboard.ts +++ b/ts_web/elements/ops-dashboard.ts @@ -127,6 +127,16 @@ export class OpsDashboard extends DeesElement { this.login(e.detail.data.username, e.detail.data.password); }); + // Handle view changes + const appDash = this.shadowRoot.querySelector('dees-simple-appdash'); + if (appDash) { + appDash.addEventListener('viewSwitch', (e: CustomEvent) => { + const viewName = e.detail.tabName; + console.log('View switched to:', viewName); + appstate.uiStatePart.dispatchAction(appstate.setActiveViewAction, viewName.toLowerCase()); + }); + } + // Handle initial state const loginState = appstate.loginStatePart.getState(); console.log('Initial login state:', loginState); diff --git a/ts_web/elements/ops-view-network.ts b/ts_web/elements/ops-view-network.ts index f9e1c89..f12a531 100644 --- a/ts_web/elements/ops-view-network.ts +++ b/ts_web/elements/ops-view-network.ts @@ -43,6 +43,10 @@ export class OpsViewNetwork extends DeesElement { @state() private trafficDataOut: Array<{ x: string | number; y: number }> = []; + // Track if we need to update the chart to avoid unnecessary re-renders + private lastChartUpdate = 0; + private chartUpdateThreshold = 1000; // Minimum ms between chart updates + private lastTrafficUpdateTime = 0; private trafficUpdateInterval = 1000; // Update every 1 second private requestCountHistory = new Map(); // Track requests per time bucket @@ -59,21 +63,35 @@ export class OpsViewNetwork extends DeesElement { this.startTrafficUpdateTimer(); } + async connectedCallback() { + await super.connectedCallback(); + + // When network view becomes visible, ensure we fetch network data + console.log('Network view connected - fetching initial data'); + await appstate.networkStatePart.dispatchAction(appstate.fetchNetworkStatsAction, null); + + // Also update the active view state + appstate.uiStatePart.dispatchAction(appstate.setActiveViewAction, 'network'); + } + async disconnectedCallback() { await super.disconnectedCallback(); this.stopTrafficUpdateTimer(); } private subscribeToStateParts() { - appstate.statsStatePart.state.subscribe((state) => { + // Subscribe and track unsubscribe functions + const statsUnsubscribe = appstate.statsStatePart.state.subscribe((state) => { this.statsState = state; this.updateNetworkData(); }); + this.rxSubscriptions.push(statsUnsubscribe); - appstate.networkStatePart.state.subscribe((state) => { + const networkUnsubscribe = appstate.networkStatePart.state.subscribe((state) => { this.networkState = state; this.updateNetworkData(); }); + this.rxSubscriptions.push(networkUnsubscribe); } private initializeTrafficData() { @@ -169,6 +187,13 @@ export class OpsViewNetwork extends DeesElement { ]; public render() { + console.log('Network view render - chart data points:', { + inPoints: this.trafficDataIn.length, + outPoints: this.trafficDataOut.length, + lastInValue: this.trafficDataIn[this.trafficDataIn.length - 1]?.y, + lastOutValue: this.trafficDataOut[this.trafficDataOut.length - 1]?.y + }); + return html` Network Activity @@ -278,7 +303,6 @@ export class OpsViewNetwork extends DeesElement { iconName: 'copy', action: async () => { await navigator.clipboard.writeText(request.id); - console.log('Request ID copied to clipboard'); } } ] @@ -366,6 +390,8 @@ export class OpsViewNetwork extends DeesElement { const reqPerSec = this.calculateRequestsPerSecond(); const throughput = this.calculateThroughput(); const activeConnections = this.statsState.serverStats?.activeConnections || 0; + + // Throughput data is now available in the stats tiles // Use request count history for the requests/sec trend const trendData = [...this.requestsPerSecHistory]; @@ -466,25 +492,36 @@ export class OpsViewNetwork extends DeesElement { } private async updateNetworkData() { - // Convert connection data to network requests format - if (this.networkState.connections.length > 0) { - this.networkRequests = this.networkState.connections.map((conn, index) => ({ - id: conn.id, - timestamp: conn.startTime, - method: 'GET', // Default method for proxy connections - url: '/', - hostname: conn.remoteAddress, - port: conn.protocol === 'https' ? 443 : 80, - protocol: conn.protocol === 'https' || conn.protocol === 'http' ? conn.protocol : 'tcp', - statusCode: conn.state === 'connected' ? 200 : undefined, - duration: Date.now() - conn.startTime, - bytesIn: conn.bytesReceived, - bytesOut: conn.bytesSent, - remoteIp: conn.remoteAddress, - route: 'proxy', - })); - } else { - this.networkRequests = []; + // Only update if connections changed significantly + const newConnectionCount = this.networkState.connections.length; + const oldConnectionCount = this.networkRequests.length; + + // Check if we need to update the network requests array + const shouldUpdate = newConnectionCount !== oldConnectionCount || + newConnectionCount === 0 || + (newConnectionCount > 0 && this.networkRequests.length === 0); + + if (shouldUpdate) { + // Convert connection data to network requests format + if (newConnectionCount > 0) { + this.networkRequests = this.networkState.connections.map((conn, index) => ({ + id: conn.id, + timestamp: conn.startTime, + method: 'GET', // Default method for proxy connections + url: '/', + hostname: conn.remoteAddress, + port: conn.protocol === 'https' ? 443 : 80, + protocol: conn.protocol === 'https' || conn.protocol === 'http' ? conn.protocol : 'tcp', + statusCode: conn.state === 'connected' ? 200 : undefined, + duration: Date.now() - conn.startTime, + bytesIn: conn.bytesReceived, + bytesOut: conn.bytesSent, + remoteIp: conn.remoteAddress, + route: 'proxy', + })); + } else { + this.networkRequests = []; + } } // Generate traffic data based on request history @@ -492,87 +529,58 @@ export class OpsViewNetwork extends DeesElement { } private updateTrafficData() { + // This method is called when network data updates + // The actual chart updates are handled by the timer calling addTrafficDataPoint() + console.log('UpdateTrafficData called - network data updated'); + } + + private startTrafficUpdateTimer() { + this.stopTrafficUpdateTimer(); // Clear any existing timer + this.trafficUpdateTimer = setInterval(() => { + // Add a new data point every second + this.addTrafficDataPoint(); + }, 1000); // Update every second + } + + private addTrafficDataPoint() { const now = Date.now(); - // Fixed 5 minute time range - const range = 5 * 60 * 1000; // 5 minutes - const bucketSize = range / 60; // 60 data points // 60 data points - // Check if enough time has passed to add a new data point - const timeSinceLastUpdate = now - this.lastTrafficUpdateTime; - const shouldAddNewPoint = timeSinceLastUpdate >= this.trafficUpdateInterval; - - console.log('UpdateTrafficData called:', { - networkRequestsCount: this.networkRequests.length, - timeSinceLastUpdate, - shouldAddNewPoint, - currentDataPoints: this.trafficDataIn.length - }); - - if (!shouldAddNewPoint && this.trafficDataIn.length > 0) { - // Not enough time has passed, don't update + // Throttle chart updates to avoid excessive re-renders + if (now - this.lastChartUpdate < this.chartUpdateThreshold) { return; } - // Use real-time throughput data from SmartProxy (same as throughput tiles) const throughput = this.calculateThroughput(); // Convert to Mbps (bytes * 8 / 1,000,000) const throughputInMbps = (throughput.in * 8) / 1000000; const throughputOutMbps = (throughput.out * 8) / 1000000; - console.log('Throughput calculation:', { - bytesInPerSecond: throughput.in, - bytesOutPerSecond: throughput.out, - throughputInMbps, - throughputOutMbps, - throughputTileValue: `${this.formatBitsPerSecond(throughput.in)} IN, ${this.formatBitsPerSecond(throughput.out)} OUT` - }); + // Add new data points + const timestamp = new Date(now).toISOString(); - if (this.trafficDataIn.length === 0) { - // Initialize if empty - this.initializeTrafficData(); + const newDataPointIn = { + x: timestamp, + y: Math.round(throughputInMbps * 10) / 10 + }; + + const newDataPointOut = { + x: timestamp, + y: Math.round(throughputOutMbps * 10) / 10 + }; + + // Efficient array updates - modify in place when possible + if (this.trafficDataIn.length >= 60) { + // Remove oldest and add newest + this.trafficDataIn = [...this.trafficDataIn.slice(1), newDataPointIn]; + this.trafficDataOut = [...this.trafficDataOut.slice(1), newDataPointOut]; } else { - // Add new data points for both in and out - const timestamp = new Date(now).toISOString(); - - const newDataPointIn = { - x: timestamp, - y: Math.round(throughputInMbps * 10) / 10 // Round to 1 decimal place - }; - - const newDataPointOut = { - x: timestamp, - y: Math.round(throughputOutMbps * 10) / 10 // Round to 1 decimal place - }; - - // Create new arrays with existing data plus new points - const newTrafficDataIn = [...this.trafficDataIn, newDataPointIn]; - const newTrafficDataOut = [...this.trafficDataOut, newDataPointOut]; - - // Keep only the last 60 points - if (newTrafficDataIn.length > 60) { - newTrafficDataIn.shift(); // Remove oldest point - newTrafficDataOut.shift(); - } - - this.trafficDataIn = newTrafficDataIn; - this.trafficDataOut = newTrafficDataOut; - this.lastTrafficUpdateTime = now; - - console.log('Added new traffic data points:', { - timestamp: timestamp, - throughputInMbps: newDataPointIn.y, - throughputOutMbps: newDataPointOut.y, - totalPoints: this.trafficDataIn.length - }); + // Still filling up the initial data + this.trafficDataIn = [...this.trafficDataIn, newDataPointIn]; + this.trafficDataOut = [...this.trafficDataOut, newDataPointOut]; } - } - - private startTrafficUpdateTimer() { - this.stopTrafficUpdateTimer(); // Clear any existing timer - this.trafficUpdateTimer = setInterval(() => { - this.updateTrafficData(); - }, 1000); // Check every second, but only update when interval has passed + + this.lastChartUpdate = now; } private stopTrafficUpdateTimer() { @@ -581,5 +589,4 @@ export class OpsViewNetwork extends DeesElement { this.trafficUpdateTimer = null; } } - } \ No newline at end of file