1009 lines
36 KiB
TypeScript
1009 lines
36 KiB
TypeScript
import * as plugins from '../plugins.js';
|
|
import { DcRouter } from '../classes.dcrouter.js';
|
|
import { MetricsCache } from './classes.metricscache.js';
|
|
import { SecurityLogger, SecurityEventType } from '../security/classes.securitylogger.js';
|
|
import { logger } from '../logger.js';
|
|
|
|
export class MetricsManager {
|
|
private metricsLogger: plugins.smartlog.Smartlog;
|
|
private smartMetrics: plugins.smartmetrics.SmartMetrics;
|
|
private dcRouter: DcRouter;
|
|
private resetInterval?: NodeJS.Timeout;
|
|
private metricsCache: MetricsCache;
|
|
|
|
// Constants
|
|
private readonly MAX_TOP_DOMAINS = 1000; // Limit topDomains Map size
|
|
|
|
// Track email-specific metrics
|
|
private emailMetrics = {
|
|
sentToday: 0,
|
|
receivedToday: 0,
|
|
failedToday: 0,
|
|
bouncedToday: 0,
|
|
queueSize: 0,
|
|
lastResetDate: new Date().toDateString(),
|
|
deliveryTimes: [] as number[], // Track delivery times in ms
|
|
recipients: new Map<string, number>(), // Track email count by recipient
|
|
recentActivity: [] as Array<{ timestamp: number; type: string; details: string }>,
|
|
};
|
|
|
|
// Track DNS-specific metrics
|
|
private dnsMetrics = {
|
|
totalQueries: 0,
|
|
cacheHits: 0,
|
|
cacheMisses: 0,
|
|
queryTypes: {} as Record<string, number>,
|
|
topDomains: new Map<string, number>(),
|
|
lastResetDate: new Date().toDateString(),
|
|
// Per-second query count ring buffer (300 entries = 5 minutes)
|
|
queryRing: new Int32Array(300),
|
|
queryRingLastSecond: 0, // last epoch second that was written
|
|
responseTimes: [] as number[], // Track response times in ms
|
|
recentQueries: [] as Array<{ timestamp: number; domain: string; type: string; answered: boolean; responseTimeMs: number }>,
|
|
};
|
|
|
|
// Per-minute time-series buckets for charts
|
|
private emailMinuteBuckets = new Map<number, { sent: number; received: number; failed: number }>();
|
|
private dnsMinuteBuckets = new Map<number, { queries: number }>();
|
|
|
|
// Track security-specific metrics
|
|
private securityMetrics = {
|
|
blockedIPs: 0,
|
|
authFailures: 0,
|
|
spamDetected: 0,
|
|
malwareDetected: 0,
|
|
phishingDetected: 0,
|
|
lastResetDate: new Date().toDateString(),
|
|
incidents: [] as Array<{ timestamp: number; type: string; severity: string; details: string }>,
|
|
};
|
|
|
|
constructor(dcRouter: DcRouter) {
|
|
this.dcRouter = dcRouter;
|
|
// Create a Smartlog instance for SmartMetrics (requires its own instance)
|
|
this.metricsLogger = new plugins.smartlog.Smartlog({
|
|
logContext: {
|
|
environment: 'production',
|
|
runtime: 'node',
|
|
zone: 'dcrouter-metrics',
|
|
}
|
|
});
|
|
this.smartMetrics = new plugins.smartmetrics.SmartMetrics(this.metricsLogger, 'dcrouter');
|
|
// Initialize metrics cache with 500ms TTL
|
|
this.metricsCache = new MetricsCache(500);
|
|
}
|
|
|
|
public async start(): Promise<void> {
|
|
// Start SmartMetrics collection
|
|
this.smartMetrics.start();
|
|
|
|
// Reset daily counters at midnight
|
|
this.resetInterval = setInterval(() => {
|
|
const currentDate = new Date().toDateString();
|
|
|
|
if (currentDate !== this.emailMetrics.lastResetDate) {
|
|
this.emailMetrics.sentToday = 0;
|
|
this.emailMetrics.receivedToday = 0;
|
|
this.emailMetrics.failedToday = 0;
|
|
this.emailMetrics.bouncedToday = 0;
|
|
this.emailMetrics.deliveryTimes = [];
|
|
this.emailMetrics.recipients.clear();
|
|
this.emailMetrics.recentActivity = [];
|
|
this.emailMetrics.lastResetDate = currentDate;
|
|
}
|
|
|
|
if (currentDate !== this.dnsMetrics.lastResetDate) {
|
|
this.dnsMetrics.totalQueries = 0;
|
|
this.dnsMetrics.cacheHits = 0;
|
|
this.dnsMetrics.cacheMisses = 0;
|
|
this.dnsMetrics.queryTypes = {};
|
|
this.dnsMetrics.topDomains.clear();
|
|
this.dnsMetrics.queryRing.fill(0);
|
|
this.dnsMetrics.queryRingLastSecond = 0;
|
|
this.dnsMetrics.responseTimes = [];
|
|
this.dnsMetrics.recentQueries = [];
|
|
this.dnsMetrics.lastResetDate = currentDate;
|
|
}
|
|
|
|
if (currentDate !== this.securityMetrics.lastResetDate) {
|
|
this.securityMetrics.blockedIPs = 0;
|
|
this.securityMetrics.authFailures = 0;
|
|
this.securityMetrics.spamDetected = 0;
|
|
this.securityMetrics.malwareDetected = 0;
|
|
this.securityMetrics.phishingDetected = 0;
|
|
this.securityMetrics.incidents = [];
|
|
this.securityMetrics.lastResetDate = currentDate;
|
|
}
|
|
|
|
// Prune old time-series buckets every minute (don't wait for lazy query)
|
|
this.pruneOldBuckets();
|
|
}, 60000); // Check every minute
|
|
|
|
logger.log('info', 'MetricsManager started');
|
|
}
|
|
|
|
public async stop(): Promise<void> {
|
|
// Clear the reset interval
|
|
if (this.resetInterval) {
|
|
clearInterval(this.resetInterval);
|
|
this.resetInterval = undefined;
|
|
}
|
|
|
|
this.smartMetrics.stop();
|
|
|
|
// Clear caches and time-series buckets on shutdown
|
|
this.metricsCache.clear();
|
|
this.emailMinuteBuckets.clear();
|
|
this.dnsMinuteBuckets.clear();
|
|
|
|
logger.log('info', 'MetricsManager stopped');
|
|
}
|
|
|
|
// Get server metrics from SmartMetrics and SmartProxy
|
|
public async getServerStats() {
|
|
return this.metricsCache.get('serverStats', async () => {
|
|
const smartMetricsData = await this.smartMetrics.getMetrics();
|
|
const proxyMetrics = this.dcRouter.smartProxy ? this.dcRouter.smartProxy.getMetrics() : null;
|
|
const proxyStats = this.dcRouter.smartProxy ? await this.dcRouter.smartProxy.getStatistics() : null;
|
|
const { heapUsed, heapTotal, external, rss } = process.memoryUsage();
|
|
|
|
return {
|
|
uptime: process.uptime(),
|
|
startTime: Date.now() - (process.uptime() * 1000),
|
|
memoryUsage: {
|
|
heapUsed,
|
|
heapTotal,
|
|
external,
|
|
rss,
|
|
maxMemoryMB: this.smartMetrics.maxMemoryMB,
|
|
actualUsageBytes: smartMetricsData.memoryUsageBytes,
|
|
actualUsagePercentage: smartMetricsData.memoryPercentage,
|
|
},
|
|
cpuUsage: {
|
|
user: smartMetricsData.cpuPercentage,
|
|
system: 0,
|
|
},
|
|
activeConnections: proxyStats ? proxyStats.activeConnections : 0,
|
|
totalConnections: proxyMetrics ? proxyMetrics.totals.connections() : 0,
|
|
requestsPerSecond: proxyMetrics ? proxyMetrics.requests.perSecond() : 0,
|
|
throughput: proxyMetrics ? {
|
|
bytesIn: proxyMetrics.totals.bytesIn(),
|
|
bytesOut: proxyMetrics.totals.bytesOut(),
|
|
bytesInPerSecond: proxyMetrics.throughput.instant().in,
|
|
bytesOutPerSecond: proxyMetrics.throughput.instant().out,
|
|
} : { bytesIn: 0, bytesOut: 0, bytesInPerSecond: 0, bytesOutPerSecond: 0 },
|
|
};
|
|
});
|
|
}
|
|
|
|
// Get email metrics
|
|
public async getEmailStats() {
|
|
return this.metricsCache.get('emailStats', () => {
|
|
// Calculate average delivery time
|
|
const avgDeliveryTime = this.emailMetrics.deliveryTimes.length > 0
|
|
? this.emailMetrics.deliveryTimes.reduce((a, b) => a + b, 0) / this.emailMetrics.deliveryTimes.length
|
|
: 0;
|
|
|
|
// Get top recipients
|
|
const topRecipients = Array.from(this.emailMetrics.recipients.entries())
|
|
.sort((a, b) => b[1] - a[1])
|
|
.slice(0, 10)
|
|
.map(([email, count]) => ({ email, count }));
|
|
|
|
// Get recent activity (last 50 entries)
|
|
const recentActivity = this.emailMetrics.recentActivity.slice(-50);
|
|
|
|
return {
|
|
sentToday: this.emailMetrics.sentToday,
|
|
receivedToday: this.emailMetrics.receivedToday,
|
|
failedToday: this.emailMetrics.failedToday,
|
|
bounceRate: this.emailMetrics.bouncedToday > 0
|
|
? (this.emailMetrics.bouncedToday / this.emailMetrics.sentToday) * 100
|
|
: 0,
|
|
deliveryRate: this.emailMetrics.sentToday > 0
|
|
? ((this.emailMetrics.sentToday - this.emailMetrics.failedToday) / this.emailMetrics.sentToday) * 100
|
|
: 100,
|
|
queueSize: this.emailMetrics.queueSize,
|
|
averageDeliveryTime: Math.round(avgDeliveryTime),
|
|
topRecipients,
|
|
recentActivity,
|
|
};
|
|
});
|
|
}
|
|
|
|
// Get DNS metrics
|
|
public async getDnsStats() {
|
|
return this.metricsCache.get('dnsStats', () => {
|
|
const cacheHitRate = this.dnsMetrics.totalQueries > 0
|
|
? (this.dnsMetrics.cacheHits / this.dnsMetrics.totalQueries) * 100
|
|
: 0;
|
|
|
|
const topDomains = Array.from(this.dnsMetrics.topDomains.entries())
|
|
.sort((a, b) => b[1] - a[1])
|
|
.slice(0, 10)
|
|
.map(([domain, count]) => ({ domain, count }));
|
|
|
|
// Calculate queries per second from ring buffer (sum last 60 seconds)
|
|
const queriesPerSecond = this.getQueryRingSum(60) / 60;
|
|
|
|
// Calculate average response time
|
|
const avgResponseTime = this.dnsMetrics.responseTimes.length > 0
|
|
? this.dnsMetrics.responseTimes.reduce((a, b) => a + b, 0) / this.dnsMetrics.responseTimes.length
|
|
: 0;
|
|
|
|
return {
|
|
queriesPerSecond: Math.round(queriesPerSecond * 10) / 10,
|
|
totalQueries: this.dnsMetrics.totalQueries,
|
|
cacheHits: this.dnsMetrics.cacheHits,
|
|
cacheMisses: this.dnsMetrics.cacheMisses,
|
|
cacheHitRate: cacheHitRate,
|
|
topDomains: topDomains,
|
|
queryTypes: this.dnsMetrics.queryTypes,
|
|
averageResponseTime: Math.round(avgResponseTime),
|
|
activeDomains: this.dnsMetrics.topDomains.size,
|
|
recentQueries: this.dnsMetrics.recentQueries.slice(),
|
|
};
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Sync security metrics from the SecurityLogger singleton (last 24h).
|
|
* Called before returning security stats so counters reflect real events.
|
|
*/
|
|
private syncFromSecurityLogger(): void {
|
|
try {
|
|
const securityLogger = SecurityLogger.getInstance();
|
|
const summary = securityLogger.getEventsSummary(86400000); // last 24h
|
|
|
|
this.securityMetrics.spamDetected = summary.byType[SecurityEventType.SPAM] || 0;
|
|
this.securityMetrics.malwareDetected = summary.byType[SecurityEventType.MALWARE] || 0;
|
|
this.securityMetrics.phishingDetected = summary.byType[SecurityEventType.DMARC] || 0; // phishing via DMARC
|
|
this.securityMetrics.authFailures =
|
|
summary.byType[SecurityEventType.AUTHENTICATION] || 0;
|
|
this.securityMetrics.blockedIPs =
|
|
(summary.byType[SecurityEventType.IP_REPUTATION] || 0) +
|
|
(summary.byType[SecurityEventType.REJECTED_CONNECTION] || 0);
|
|
} catch {
|
|
// SecurityLogger may not be initialized yet — ignore
|
|
}
|
|
}
|
|
|
|
// Get security metrics
|
|
public async getSecurityStats() {
|
|
return this.metricsCache.get('securityStats', () => {
|
|
// Sync counters from the real SecurityLogger events
|
|
this.syncFromSecurityLogger();
|
|
|
|
// Get recent incidents (last 20)
|
|
const recentIncidents = this.securityMetrics.incidents.slice(-20);
|
|
|
|
return {
|
|
blockedIPs: this.securityMetrics.blockedIPs,
|
|
authFailures: this.securityMetrics.authFailures,
|
|
spamDetected: this.securityMetrics.spamDetected,
|
|
malwareDetected: this.securityMetrics.malwareDetected,
|
|
phishingDetected: this.securityMetrics.phishingDetected,
|
|
totalThreatsBlocked: this.securityMetrics.spamDetected +
|
|
this.securityMetrics.malwareDetected +
|
|
this.securityMetrics.phishingDetected,
|
|
recentIncidents,
|
|
};
|
|
});
|
|
}
|
|
|
|
// Get connection info from SmartProxy
|
|
public async getConnectionInfo() {
|
|
return this.metricsCache.get('connectionInfo', () => {
|
|
const proxyMetrics = this.dcRouter.smartProxy ? this.dcRouter.smartProxy.getMetrics() : null;
|
|
|
|
if (!proxyMetrics) {
|
|
return [] as Array<{ type: string; count: number; source: string; lastActivity: Date }>;
|
|
}
|
|
|
|
const connectionsByRoute = proxyMetrics.connections.byRoute();
|
|
const connectionInfo: Array<{ type: string; count: number; source: string; lastActivity: Date }> = [];
|
|
|
|
for (const [routeName, count] of connectionsByRoute) {
|
|
connectionInfo.push({
|
|
type: 'https',
|
|
count,
|
|
source: routeName,
|
|
lastActivity: new Date(),
|
|
});
|
|
}
|
|
|
|
return connectionInfo;
|
|
});
|
|
}
|
|
|
|
// Email event tracking methods
|
|
public trackEmailSent(recipient?: string, deliveryTimeMs?: number): void {
|
|
this.emailMetrics.sentToday++;
|
|
this.incrementEmailBucket('sent');
|
|
|
|
if (recipient) {
|
|
const count = this.emailMetrics.recipients.get(recipient) || 0;
|
|
this.emailMetrics.recipients.set(recipient, count + 1);
|
|
|
|
// Cap recipients map to prevent unbounded growth within a day
|
|
if (this.emailMetrics.recipients.size > this.MAX_TOP_DOMAINS) {
|
|
const sorted = Array.from(this.emailMetrics.recipients.entries())
|
|
.sort((a, b) => b[1] - a[1])
|
|
.slice(0, Math.floor(this.MAX_TOP_DOMAINS * 0.8));
|
|
this.emailMetrics.recipients = new Map(sorted);
|
|
}
|
|
}
|
|
|
|
if (deliveryTimeMs) {
|
|
this.emailMetrics.deliveryTimes.push(deliveryTimeMs);
|
|
// Keep only last 1000 delivery times
|
|
if (this.emailMetrics.deliveryTimes.length > 1000) {
|
|
this.emailMetrics.deliveryTimes.shift();
|
|
}
|
|
}
|
|
|
|
this.emailMetrics.recentActivity.push({
|
|
timestamp: Date.now(),
|
|
type: 'sent',
|
|
details: recipient || 'unknown',
|
|
});
|
|
|
|
// Keep only last 1000 activities
|
|
if (this.emailMetrics.recentActivity.length > 1000) {
|
|
this.emailMetrics.recentActivity.shift();
|
|
}
|
|
}
|
|
|
|
public trackEmailReceived(sender?: string): void {
|
|
this.emailMetrics.receivedToday++;
|
|
this.incrementEmailBucket('received');
|
|
|
|
this.emailMetrics.recentActivity.push({
|
|
timestamp: Date.now(),
|
|
type: 'received',
|
|
details: sender || 'unknown',
|
|
});
|
|
|
|
// Keep only last 1000 activities
|
|
if (this.emailMetrics.recentActivity.length > 1000) {
|
|
this.emailMetrics.recentActivity.shift();
|
|
}
|
|
}
|
|
|
|
public trackEmailFailed(recipient?: string, reason?: string): void {
|
|
this.emailMetrics.failedToday++;
|
|
this.incrementEmailBucket('failed');
|
|
|
|
this.emailMetrics.recentActivity.push({
|
|
timestamp: Date.now(),
|
|
type: 'failed',
|
|
details: `${recipient || 'unknown'}: ${reason || 'unknown error'}`,
|
|
});
|
|
|
|
// Keep only last 1000 activities
|
|
if (this.emailMetrics.recentActivity.length > 1000) {
|
|
this.emailMetrics.recentActivity.shift();
|
|
}
|
|
}
|
|
|
|
public trackEmailBounced(recipient?: string): void {
|
|
this.emailMetrics.bouncedToday++;
|
|
|
|
this.emailMetrics.recentActivity.push({
|
|
timestamp: Date.now(),
|
|
type: 'bounced',
|
|
details: recipient || 'unknown',
|
|
});
|
|
|
|
// Keep only last 1000 activities
|
|
if (this.emailMetrics.recentActivity.length > 1000) {
|
|
this.emailMetrics.recentActivity.shift();
|
|
}
|
|
}
|
|
|
|
public updateQueueSize(size: number): void {
|
|
this.emailMetrics.queueSize = size;
|
|
}
|
|
|
|
// DNS event tracking methods
|
|
public trackDnsQuery(queryType: string, domain: string, cacheHit: boolean, responseTimeMs?: number, answered?: boolean): void {
|
|
this.dnsMetrics.totalQueries++;
|
|
this.incrementDnsBucket();
|
|
|
|
// Store recent query entry
|
|
this.dnsMetrics.recentQueries.push({
|
|
timestamp: Date.now(),
|
|
domain,
|
|
type: queryType,
|
|
answered: answered ?? true,
|
|
responseTimeMs: responseTimeMs ?? 0,
|
|
});
|
|
if (this.dnsMetrics.recentQueries.length > 100) {
|
|
this.dnsMetrics.recentQueries.shift();
|
|
}
|
|
|
|
if (cacheHit) {
|
|
this.dnsMetrics.cacheHits++;
|
|
} else {
|
|
this.dnsMetrics.cacheMisses++;
|
|
}
|
|
|
|
// Increment per-second query counter in ring buffer
|
|
this.incrementQueryRing();
|
|
|
|
// Track response time if provided
|
|
if (responseTimeMs) {
|
|
this.dnsMetrics.responseTimes.push(responseTimeMs);
|
|
// Keep only last 1000 response times
|
|
if (this.dnsMetrics.responseTimes.length > 1000) {
|
|
this.dnsMetrics.responseTimes.shift();
|
|
}
|
|
}
|
|
|
|
// Track query types
|
|
this.dnsMetrics.queryTypes[queryType] = (this.dnsMetrics.queryTypes[queryType] || 0) + 1;
|
|
|
|
// Track top domains with size limit
|
|
const currentCount = this.dnsMetrics.topDomains.get(domain) || 0;
|
|
this.dnsMetrics.topDomains.set(domain, currentCount + 1);
|
|
|
|
// If we've exceeded the limit, remove the least accessed domains
|
|
if (this.dnsMetrics.topDomains.size > this.MAX_TOP_DOMAINS) {
|
|
// Convert to array, sort by count, and keep only top domains
|
|
const sortedDomains = Array.from(this.dnsMetrics.topDomains.entries())
|
|
.sort((a, b) => b[1] - a[1])
|
|
.slice(0, Math.floor(this.MAX_TOP_DOMAINS * 0.8)); // Keep 80% to avoid frequent cleanup
|
|
|
|
// Clear and repopulate with top domains
|
|
this.dnsMetrics.topDomains.clear();
|
|
sortedDomains.forEach(([domain, count]) => {
|
|
this.dnsMetrics.topDomains.set(domain, count);
|
|
});
|
|
}
|
|
}
|
|
|
|
// Security event tracking methods
|
|
public trackBlockedIP(ip?: string, reason?: string): void {
|
|
this.securityMetrics.blockedIPs++;
|
|
|
|
this.securityMetrics.incidents.push({
|
|
timestamp: Date.now(),
|
|
type: 'ip_blocked',
|
|
severity: 'medium',
|
|
details: `IP ${ip || 'unknown'} blocked: ${reason || 'security policy'}`,
|
|
});
|
|
|
|
// Keep only last 1000 incidents
|
|
if (this.securityMetrics.incidents.length > 1000) {
|
|
this.securityMetrics.incidents.shift();
|
|
}
|
|
}
|
|
|
|
public trackAuthFailure(username?: string, ip?: string): void {
|
|
this.securityMetrics.authFailures++;
|
|
|
|
this.securityMetrics.incidents.push({
|
|
timestamp: Date.now(),
|
|
type: 'auth_failure',
|
|
severity: 'low',
|
|
details: `Authentication failed for ${username || 'unknown'} from ${ip || 'unknown'}`,
|
|
});
|
|
|
|
// Keep only last 1000 incidents
|
|
if (this.securityMetrics.incidents.length > 1000) {
|
|
this.securityMetrics.incidents.shift();
|
|
}
|
|
}
|
|
|
|
public trackSpamDetected(sender?: string): void {
|
|
this.securityMetrics.spamDetected++;
|
|
|
|
this.securityMetrics.incidents.push({
|
|
timestamp: Date.now(),
|
|
type: 'spam_detected',
|
|
severity: 'low',
|
|
details: `Spam detected from ${sender || 'unknown'}`,
|
|
});
|
|
|
|
// Keep only last 1000 incidents
|
|
if (this.securityMetrics.incidents.length > 1000) {
|
|
this.securityMetrics.incidents.shift();
|
|
}
|
|
}
|
|
|
|
public trackMalwareDetected(source?: string): void {
|
|
this.securityMetrics.malwareDetected++;
|
|
|
|
this.securityMetrics.incidents.push({
|
|
timestamp: Date.now(),
|
|
type: 'malware_detected',
|
|
severity: 'high',
|
|
details: `Malware detected from ${source || 'unknown'}`,
|
|
});
|
|
|
|
// Keep only last 1000 incidents
|
|
if (this.securityMetrics.incidents.length > 1000) {
|
|
this.securityMetrics.incidents.shift();
|
|
}
|
|
}
|
|
|
|
public trackPhishingDetected(source?: string): void {
|
|
this.securityMetrics.phishingDetected++;
|
|
|
|
this.securityMetrics.incidents.push({
|
|
timestamp: Date.now(),
|
|
type: 'phishing_detected',
|
|
severity: 'high',
|
|
details: `Phishing attempt from ${source || 'unknown'}`,
|
|
});
|
|
|
|
// Keep only last 1000 incidents
|
|
if (this.securityMetrics.incidents.length > 1000) {
|
|
this.securityMetrics.incidents.shift();
|
|
}
|
|
}
|
|
|
|
// Get network metrics from SmartProxy
|
|
public async getNetworkStats() {
|
|
// Use shorter cache TTL for network stats to ensure real-time updates
|
|
return this.metricsCache.get('networkStats', () => {
|
|
const proxyMetrics = this.dcRouter.smartProxy ? this.dcRouter.smartProxy.getMetrics() : null;
|
|
|
|
if (!proxyMetrics) {
|
|
return {
|
|
connectionsByIP: new Map<string, number>(),
|
|
throughputRate: { bytesInPerSecond: 0, bytesOutPerSecond: 0 },
|
|
topIPs: [] as Array<{ ip: string; count: number }>,
|
|
topIPsByBandwidth: [] as Array<{ ip: string; count: number; bwIn: number; bwOut: number }>,
|
|
totalDataTransferred: { bytesIn: 0, bytesOut: 0 },
|
|
throughputHistory: [] as Array<{ timestamp: number; in: number; out: number }>,
|
|
throughputByIP: new Map<string, { in: number; out: number }>(),
|
|
requestsPerSecond: 0,
|
|
requestsTotal: 0,
|
|
backends: [] as Array<any>,
|
|
domainActivity: [] as Array<{ domain: string; bytesInPerSecond: number; bytesOutPerSecond: number; activeConnections: number; routeCount: number }>,
|
|
};
|
|
}
|
|
|
|
// Get metrics using the new API
|
|
const connectionsByIP = proxyMetrics.connections.byIP();
|
|
const instantThroughput = proxyMetrics.throughput.instant();
|
|
|
|
// Get throughput rate
|
|
const throughputRate = {
|
|
bytesInPerSecond: instantThroughput.in,
|
|
bytesOutPerSecond: instantThroughput.out
|
|
};
|
|
|
|
// Get top IPs by connection count
|
|
const topIPs = proxyMetrics.connections.topIPs(10);
|
|
|
|
// Get total data transferred
|
|
const totalDataTransferred = {
|
|
bytesIn: proxyMetrics.totals.bytesIn(),
|
|
bytesOut: proxyMetrics.totals.bytesOut()
|
|
};
|
|
|
|
// Get throughput history from Rust engine (up to 300 seconds)
|
|
const throughputHistory = proxyMetrics.throughput.history(300);
|
|
|
|
// Get per-IP throughput
|
|
const throughputByIP = proxyMetrics.throughput.byIP();
|
|
|
|
// Get HTTP request rates
|
|
const requestsPerSecond = proxyMetrics.requests.perSecond();
|
|
const requestsTotal = proxyMetrics.requests.total();
|
|
|
|
// Get frontend/backend protocol distribution
|
|
const frontendProtocols = proxyMetrics.connections.frontendProtocols() ?? null;
|
|
const backendProtocols = proxyMetrics.connections.backendProtocols() ?? null;
|
|
|
|
// Collect backend protocol data
|
|
const backendMetrics = proxyMetrics.backends.byBackend();
|
|
const protocolCache = proxyMetrics.backends.detectedProtocols();
|
|
|
|
// Group protocol cache entries by host:port so we can match them to backend metrics.
|
|
// The protocol cache is keyed by (host, port, domain) in Rust, so the same host:port
|
|
// can have multiple entries for different domains.
|
|
const cacheByBackend = new Map<string, (typeof protocolCache)[number][]>();
|
|
for (const entry of protocolCache) {
|
|
const backendKey = `${entry.host}:${entry.port}`;
|
|
let entries = cacheByBackend.get(backendKey);
|
|
if (!entries) {
|
|
entries = [];
|
|
cacheByBackend.set(backendKey, entries);
|
|
}
|
|
entries.push(entry);
|
|
}
|
|
|
|
const backends: Array<any> = [];
|
|
const seenCacheKeys = new Set<string>();
|
|
|
|
for (const [key, bm] of backendMetrics) {
|
|
const cacheEntries = cacheByBackend.get(key);
|
|
if (!cacheEntries || cacheEntries.length === 0) {
|
|
// No protocol cache entry — emit one row with backend metrics only
|
|
backends.push({
|
|
backend: key,
|
|
domain: null,
|
|
protocol: bm.protocol,
|
|
activeConnections: bm.activeConnections,
|
|
totalConnections: bm.totalConnections,
|
|
connectErrors: bm.connectErrors,
|
|
handshakeErrors: bm.handshakeErrors,
|
|
requestErrors: bm.requestErrors,
|
|
avgConnectTimeMs: Math.round(bm.avgConnectTimeMs * 10) / 10,
|
|
poolHitRate: Math.round(bm.poolHitRate * 1000) / 1000,
|
|
h2Failures: bm.h2Failures,
|
|
h2Suppressed: false,
|
|
h3Suppressed: false,
|
|
h2CooldownRemainingSecs: null,
|
|
h3CooldownRemainingSecs: null,
|
|
h2ConsecutiveFailures: null,
|
|
h3ConsecutiveFailures: null,
|
|
h3Port: null,
|
|
cacheAgeSecs: null,
|
|
});
|
|
} else {
|
|
// One row per domain, each enriched with the shared backend metrics
|
|
for (const cache of cacheEntries) {
|
|
const compositeKey = `${cache.host}:${cache.port}:${cache.domain ?? ''}`;
|
|
seenCacheKeys.add(compositeKey);
|
|
backends.push({
|
|
backend: key,
|
|
domain: cache.domain ?? null,
|
|
protocol: cache.protocol ?? bm.protocol,
|
|
activeConnections: bm.activeConnections,
|
|
totalConnections: bm.totalConnections,
|
|
connectErrors: bm.connectErrors,
|
|
handshakeErrors: bm.handshakeErrors,
|
|
requestErrors: bm.requestErrors,
|
|
avgConnectTimeMs: Math.round(bm.avgConnectTimeMs * 10) / 10,
|
|
poolHitRate: Math.round(bm.poolHitRate * 1000) / 1000,
|
|
h2Failures: bm.h2Failures,
|
|
h2Suppressed: cache.h2Suppressed,
|
|
h3Suppressed: cache.h3Suppressed,
|
|
h2CooldownRemainingSecs: cache.h2CooldownRemainingSecs,
|
|
h3CooldownRemainingSecs: cache.h3CooldownRemainingSecs,
|
|
h2ConsecutiveFailures: cache.h2ConsecutiveFailures,
|
|
h3ConsecutiveFailures: cache.h3ConsecutiveFailures,
|
|
h3Port: cache.h3Port,
|
|
cacheAgeSecs: cache.ageSecs,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
// Include protocol cache entries with no matching backend metric
|
|
for (const entry of protocolCache) {
|
|
const compositeKey = `${entry.host}:${entry.port}:${entry.domain ?? ''}`;
|
|
if (!seenCacheKeys.has(compositeKey)) {
|
|
backends.push({
|
|
backend: `${entry.host}:${entry.port}`,
|
|
domain: entry.domain,
|
|
protocol: entry.protocol,
|
|
activeConnections: 0,
|
|
totalConnections: 0,
|
|
connectErrors: 0,
|
|
handshakeErrors: 0,
|
|
requestErrors: 0,
|
|
avgConnectTimeMs: 0,
|
|
poolHitRate: 0,
|
|
h2Failures: 0,
|
|
h2Suppressed: entry.h2Suppressed,
|
|
h3Suppressed: entry.h3Suppressed,
|
|
h2CooldownRemainingSecs: entry.h2CooldownRemainingSecs,
|
|
h3CooldownRemainingSecs: entry.h3CooldownRemainingSecs,
|
|
h2ConsecutiveFailures: entry.h2ConsecutiveFailures,
|
|
h3ConsecutiveFailures: entry.h3ConsecutiveFailures,
|
|
h3Port: entry.h3Port,
|
|
cacheAgeSecs: entry.ageSecs,
|
|
});
|
|
}
|
|
}
|
|
|
|
// Build top 10 IPs by bandwidth (sorted by total throughput desc)
|
|
const allIPData = new Map<string, { count: number; bwIn: number; bwOut: number }>();
|
|
for (const [ip, count] of connectionsByIP) {
|
|
allIPData.set(ip, { count, bwIn: 0, bwOut: 0 });
|
|
}
|
|
for (const [ip, tp] of throughputByIP) {
|
|
const existing = allIPData.get(ip);
|
|
if (existing) {
|
|
existing.bwIn = tp.in;
|
|
existing.bwOut = tp.out;
|
|
} else {
|
|
allIPData.set(ip, { count: 0, bwIn: tp.in, bwOut: tp.out });
|
|
}
|
|
}
|
|
const topIPsByBandwidth = Array.from(allIPData.entries())
|
|
.sort((a, b) => (b[1].bwIn + b[1].bwOut) - (a[1].bwIn + a[1].bwOut))
|
|
.slice(0, 10)
|
|
.map(([ip, data]) => ({ ip, count: data.count, bwIn: data.bwIn, bwOut: data.bwOut }));
|
|
|
|
// Build domain activity from per-route metrics
|
|
const connectionsByRoute = proxyMetrics.connections.byRoute();
|
|
const throughputByRoute = proxyMetrics.throughput.byRoute();
|
|
|
|
// Map route name → ALL its domains (not just the first one)
|
|
const routeDomains = new Map<string, string[]>();
|
|
if (this.dcRouter.smartProxy) {
|
|
for (const route of this.dcRouter.smartProxy.routeManager.getRoutes()) {
|
|
if (!route.name || !route.match.domains) continue;
|
|
const domains = Array.isArray(route.match.domains)
|
|
? route.match.domains
|
|
: [route.match.domains];
|
|
if (domains.length > 0) {
|
|
routeDomains.set(route.name, domains);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Use protocol cache to discover actual active domains (resolves wildcards)
|
|
const activeDomains = new Set<string>();
|
|
const domainToBackend = new Map<string, string>(); // domain → host:port
|
|
for (const entry of protocolCache) {
|
|
if (entry.domain) {
|
|
activeDomains.add(entry.domain);
|
|
domainToBackend.set(entry.domain, `${entry.host}:${entry.port}`);
|
|
}
|
|
}
|
|
|
|
// Build reverse map: domain → route name(s) that handle it
|
|
// For concrete domains: direct lookup from route config
|
|
// For wildcard patterns: match active domains from protocol cache
|
|
const domainToRoutes = new Map<string, string[]>();
|
|
for (const [routeName, domains] of routeDomains) {
|
|
for (const pattern of domains) {
|
|
if (pattern.includes('*')) {
|
|
// Wildcard pattern — match against active domains from protocol cache
|
|
const regex = new RegExp('^' + pattern.replace(/\./g, '\\.').replace(/\*/g, '[^.]+') + '$');
|
|
for (const activeDomain of activeDomains) {
|
|
if (regex.test(activeDomain)) {
|
|
const existing = domainToRoutes.get(activeDomain);
|
|
if (existing) { existing.push(routeName); }
|
|
else { domainToRoutes.set(activeDomain, [routeName]); }
|
|
}
|
|
}
|
|
} else {
|
|
// Concrete domain
|
|
const existing = domainToRoutes.get(pattern);
|
|
if (existing) { existing.push(routeName); }
|
|
else { domainToRoutes.set(pattern, [routeName]); }
|
|
}
|
|
}
|
|
}
|
|
|
|
// Aggregate metrics per domain
|
|
// For each domain, sum metrics from all routes that serve it,
|
|
// divided by the number of domains each route serves
|
|
const domainAgg = new Map<string, {
|
|
activeConnections: number;
|
|
bytesInPerSec: number;
|
|
bytesOutPerSec: number;
|
|
routeCount: number;
|
|
}>();
|
|
|
|
// Track which routes are accounted for
|
|
const accountedRoutes = new Set<string>();
|
|
|
|
for (const [domain, routeNames] of domainToRoutes) {
|
|
let totalConns = 0;
|
|
let totalIn = 0;
|
|
let totalOut = 0;
|
|
|
|
for (const routeName of routeNames) {
|
|
accountedRoutes.add(routeName);
|
|
const conns = connectionsByRoute.get(routeName) || 0;
|
|
const tp = throughputByRoute.get(routeName) || { in: 0, out: 0 };
|
|
// Count how many resolved domains share this route
|
|
let domainsInRoute = 0;
|
|
for (const [, routes] of domainToRoutes) {
|
|
if (routes.includes(routeName)) domainsInRoute++;
|
|
}
|
|
const share = Math.max(domainsInRoute, 1);
|
|
totalConns += conns / share;
|
|
totalIn += tp.in / share;
|
|
totalOut += tp.out / share;
|
|
}
|
|
|
|
domainAgg.set(domain, {
|
|
activeConnections: Math.round(totalConns),
|
|
bytesInPerSec: totalIn,
|
|
bytesOutPerSec: totalOut,
|
|
routeCount: routeNames.length,
|
|
});
|
|
}
|
|
|
|
// Include routes with no domain config (fallback: use route name)
|
|
for (const [routeName, activeConns] of connectionsByRoute) {
|
|
if (accountedRoutes.has(routeName)) continue;
|
|
if (routeDomains.has(routeName)) continue; // has domains but no traffic matched
|
|
const tp = throughputByRoute.get(routeName) || { in: 0, out: 0 };
|
|
if (activeConns === 0 && tp.in === 0 && tp.out === 0) continue;
|
|
const existing = domainAgg.get(routeName);
|
|
if (existing) {
|
|
existing.activeConnections += activeConns;
|
|
existing.bytesInPerSec += tp.in;
|
|
existing.bytesOutPerSec += tp.out;
|
|
existing.routeCount++;
|
|
} else {
|
|
domainAgg.set(routeName, {
|
|
activeConnections: activeConns,
|
|
bytesInPerSec: tp.in,
|
|
bytesOutPerSec: tp.out,
|
|
routeCount: 1,
|
|
});
|
|
}
|
|
}
|
|
|
|
const domainActivity = Array.from(domainAgg.entries())
|
|
.map(([domain, data]) => ({
|
|
domain,
|
|
bytesInPerSecond: data.bytesInPerSec,
|
|
bytesOutPerSecond: data.bytesOutPerSec,
|
|
activeConnections: data.activeConnections,
|
|
routeCount: data.routeCount,
|
|
}))
|
|
.sort((a, b) => (b.bytesInPerSecond + b.bytesOutPerSecond) - (a.bytesInPerSecond + a.bytesOutPerSecond));
|
|
|
|
return {
|
|
connectionsByIP,
|
|
throughputRate,
|
|
topIPs,
|
|
topIPsByBandwidth,
|
|
totalDataTransferred,
|
|
throughputHistory,
|
|
throughputByIP,
|
|
requestsPerSecond,
|
|
requestsTotal,
|
|
backends,
|
|
frontendProtocols,
|
|
backendProtocols,
|
|
domainActivity,
|
|
};
|
|
}, 1000); // 1s cache — matches typical dashboard poll interval
|
|
}
|
|
|
|
// --- Time-series helpers ---
|
|
|
|
private static minuteKey(ts: number = Date.now()): number {
|
|
return Math.floor(ts / 60000) * 60000;
|
|
}
|
|
|
|
private incrementEmailBucket(field: 'sent' | 'received' | 'failed'): void {
|
|
const key = MetricsManager.minuteKey();
|
|
let bucket = this.emailMinuteBuckets.get(key);
|
|
if (!bucket) {
|
|
bucket = { sent: 0, received: 0, failed: 0 };
|
|
this.emailMinuteBuckets.set(key, bucket);
|
|
}
|
|
bucket[field]++;
|
|
}
|
|
|
|
private incrementDnsBucket(): void {
|
|
const key = MetricsManager.minuteKey();
|
|
let bucket = this.dnsMinuteBuckets.get(key);
|
|
if (!bucket) {
|
|
bucket = { queries: 0 };
|
|
this.dnsMinuteBuckets.set(key, bucket);
|
|
}
|
|
bucket.queries++;
|
|
}
|
|
|
|
/**
|
|
* Increment the per-second query counter in the ring buffer.
|
|
* Zeros any stale slots between the last write and the current second.
|
|
*/
|
|
private incrementQueryRing(): void {
|
|
const currentSecond = Math.floor(Date.now() / 1000);
|
|
const ring = this.dnsMetrics.queryRing;
|
|
const last = this.dnsMetrics.queryRingLastSecond;
|
|
|
|
if (last === 0) {
|
|
// First call — zero and anchor
|
|
ring.fill(0);
|
|
this.dnsMetrics.queryRingLastSecond = currentSecond;
|
|
ring[currentSecond % ring.length] = 1;
|
|
return;
|
|
}
|
|
|
|
const gap = currentSecond - last;
|
|
if (gap >= ring.length) {
|
|
// Entire ring is stale — clear all
|
|
ring.fill(0);
|
|
} else if (gap > 0) {
|
|
// Zero slots from (last+1) to currentSecond (inclusive)
|
|
for (let s = last + 1; s <= currentSecond; s++) {
|
|
ring[s % ring.length] = 0;
|
|
}
|
|
}
|
|
|
|
this.dnsMetrics.queryRingLastSecond = currentSecond;
|
|
ring[currentSecond % ring.length]++;
|
|
}
|
|
|
|
/**
|
|
* Sum query counts from the ring buffer for the last N seconds.
|
|
*/
|
|
private getQueryRingSum(seconds: number): number {
|
|
const currentSecond = Math.floor(Date.now() / 1000);
|
|
const ring = this.dnsMetrics.queryRing;
|
|
const last = this.dnsMetrics.queryRingLastSecond;
|
|
|
|
if (last === 0) return 0;
|
|
|
|
// First, zero stale slots so reads are accurate even without writes
|
|
const gap = currentSecond - last;
|
|
if (gap >= ring.length) return 0; // all data is stale
|
|
|
|
let sum = 0;
|
|
const limit = Math.min(seconds, ring.length);
|
|
for (let i = 0; i < limit; i++) {
|
|
const sec = currentSecond - i;
|
|
if (sec < last - (ring.length - 1)) break; // slot is from older cycle
|
|
if (sec > last) continue; // no writes yet for this second
|
|
sum += ring[sec % ring.length];
|
|
}
|
|
return sum;
|
|
}
|
|
|
|
private pruneOldBuckets(): void {
|
|
const cutoff = Date.now() - 86400000; // 24h
|
|
for (const key of this.emailMinuteBuckets.keys()) {
|
|
if (key < cutoff) this.emailMinuteBuckets.delete(key);
|
|
}
|
|
for (const key of this.dnsMinuteBuckets.keys()) {
|
|
if (key < cutoff) this.dnsMinuteBuckets.delete(key);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get email time-series data for the last N hours, aggregated per minute.
|
|
*/
|
|
public getEmailTimeSeries(hours: number = 24): {
|
|
sent: Array<{ timestamp: number; value: number }>;
|
|
received: Array<{ timestamp: number; value: number }>;
|
|
failed: Array<{ timestamp: number; value: number }>;
|
|
} {
|
|
this.pruneOldBuckets();
|
|
const cutoff = Date.now() - hours * 3600000;
|
|
const sent: Array<{ timestamp: number; value: number }> = [];
|
|
const received: Array<{ timestamp: number; value: number }> = [];
|
|
const failed: Array<{ timestamp: number; value: number }> = [];
|
|
|
|
const sortedKeys = Array.from(this.emailMinuteBuckets.keys())
|
|
.filter((k) => k >= cutoff)
|
|
.sort((a, b) => a - b);
|
|
|
|
for (const key of sortedKeys) {
|
|
const bucket = this.emailMinuteBuckets.get(key)!;
|
|
sent.push({ timestamp: key, value: bucket.sent });
|
|
received.push({ timestamp: key, value: bucket.received });
|
|
failed.push({ timestamp: key, value: bucket.failed });
|
|
}
|
|
|
|
return { sent, received, failed };
|
|
}
|
|
|
|
/**
|
|
* Get DNS time-series data for the last N hours, aggregated per minute.
|
|
*/
|
|
public getDnsTimeSeries(hours: number = 24): {
|
|
queries: Array<{ timestamp: number; value: number }>;
|
|
} {
|
|
this.pruneOldBuckets();
|
|
const cutoff = Date.now() - hours * 3600000;
|
|
const queries: Array<{ timestamp: number; value: number }> = [];
|
|
|
|
const sortedKeys = Array.from(this.dnsMinuteBuckets.keys())
|
|
.filter((k) => k >= cutoff)
|
|
.sort((a, b) => a - b);
|
|
|
|
for (const key of sortedKeys) {
|
|
const bucket = this.dnsMinuteBuckets.get(key)!;
|
|
queries.push({ timestamp: key, value: bucket.queries });
|
|
}
|
|
|
|
return { queries };
|
|
}
|
|
} |