Implement Metrics Manager and Integrate Metrics Collection
- Removed the existing readme.opsserver.md file as it is no longer needed. - Added a new MetricsManager class to handle metrics collection using @push.rocks/smartmetrics. - Integrated MetricsManager into the DcRouter and OpsServer classes. - Updated StatsHandler and SecurityHandler to retrieve metrics from MetricsManager. - Implemented methods for tracking email, DNS, and security metrics. - Added connection tracking capabilities to the MetricsManager. - Created a new readme.metrics.md file outlining the metrics implementation plan. - Adjusted plugins.ts to include smartmetrics. - Added a new monitoring directory with classes for metrics management. - Created readme.module-adjustments.md to document necessary adjustments for SmartProxy and SmartDNS.
This commit is contained in:
@ -13,6 +13,7 @@ import { configureEmailStorage, configureEmailServer } from './mail/delivery/ind
|
||||
import { StorageManager, type IStorageConfig } from './storage/index.js';
|
||||
|
||||
import { OpsServer } from './opsserver/index.js';
|
||||
import { MetricsManager } from './monitoring/index.js';
|
||||
|
||||
export interface IDcRouterOptions {
|
||||
/**
|
||||
@ -133,6 +134,7 @@ export class DcRouter {
|
||||
public emailServer?: UnifiedEmailServer;
|
||||
public storageManager: StorageManager;
|
||||
public opsServer: OpsServer;
|
||||
public metricsManager?: MetricsManager;
|
||||
|
||||
// TypedRouter for API endpoints
|
||||
public typedrouter = new plugins.typedrequest.TypedRouter();
|
||||
@ -160,6 +162,10 @@ export class DcRouter {
|
||||
await this.opsServer.start();
|
||||
|
||||
try {
|
||||
// Initialize MetricsManager
|
||||
this.metricsManager = new MetricsManager(this);
|
||||
await this.metricsManager.start();
|
||||
|
||||
// Set up SmartProxy for HTTP/HTTPS and all traffic including email routes
|
||||
await this.setupSmartProxy();
|
||||
|
||||
@ -197,6 +203,14 @@ export class DcRouter {
|
||||
console.log('║ DcRouter Started Successfully ║');
|
||||
console.log('╚═══════════════════════════════════════════════════════════════════╝\n');
|
||||
|
||||
// Metrics summary
|
||||
if (this.metricsManager) {
|
||||
console.log('📊 Metrics Service:');
|
||||
console.log(' ├─ SmartMetrics: Active');
|
||||
console.log(' ├─ SmartProxy Stats: Active');
|
||||
console.log(' └─ Real-time tracking: Enabled');
|
||||
}
|
||||
|
||||
// SmartProxy summary
|
||||
if (this.smartProxy) {
|
||||
console.log('🌐 SmartProxy Service:');
|
||||
@ -566,6 +580,9 @@ export class DcRouter {
|
||||
try {
|
||||
// Stop all services in parallel for faster shutdown
|
||||
await Promise.all([
|
||||
// Stop metrics manager if running
|
||||
this.metricsManager ? this.metricsManager.stop().catch(err => console.error('Error stopping MetricsManager:', err)) : Promise.resolve(),
|
||||
|
||||
// Stop unified email server if running
|
||||
this.emailServer ? this.emailServer.stop().catch(err => console.error('Error stopping email server:', err)) : Promise.resolve(),
|
||||
|
||||
|
||||
260
ts/monitoring/classes.metricsmanager.ts
Normal file
260
ts/monitoring/classes.metricsmanager.ts
Normal file
@ -0,0 +1,260 @@
|
||||
import * as plugins from '../plugins.js';
|
||||
import { DcRouter } from '../classes.dcrouter.js';
|
||||
|
||||
export class MetricsManager {
|
||||
private logger: plugins.smartlog.Smartlog;
|
||||
private smartMetrics: plugins.smartmetrics.SmartMetrics;
|
||||
private dcRouter: DcRouter;
|
||||
|
||||
// Track email-specific metrics
|
||||
private emailMetrics = {
|
||||
sentToday: 0,
|
||||
receivedToday: 0,
|
||||
failedToday: 0,
|
||||
bouncedToday: 0,
|
||||
queueSize: 0,
|
||||
lastResetDate: new Date().toDateString(),
|
||||
};
|
||||
|
||||
// Track DNS-specific metrics
|
||||
private dnsMetrics = {
|
||||
totalQueries: 0,
|
||||
cacheHits: 0,
|
||||
cacheMisses: 0,
|
||||
queryTypes: {} as Record<string, number>,
|
||||
topDomains: new Map<string, number>(),
|
||||
lastResetDate: new Date().toDateString(),
|
||||
};
|
||||
|
||||
// Track security-specific metrics
|
||||
private securityMetrics = {
|
||||
blockedIPs: 0,
|
||||
authFailures: 0,
|
||||
spamDetected: 0,
|
||||
malwareDetected: 0,
|
||||
phishingDetected: 0,
|
||||
lastResetDate: new Date().toDateString(),
|
||||
};
|
||||
|
||||
constructor(dcRouter: DcRouter) {
|
||||
this.dcRouter = dcRouter;
|
||||
// Create a new Smartlog instance for metrics
|
||||
this.logger = new plugins.smartlog.Smartlog({
|
||||
logContext: {
|
||||
environment: 'production',
|
||||
runtime: 'node',
|
||||
zone: 'dcrouter-metrics',
|
||||
}
|
||||
});
|
||||
this.smartMetrics = new plugins.smartmetrics.SmartMetrics(this.logger, 'dcrouter');
|
||||
}
|
||||
|
||||
public async start(): Promise<void> {
|
||||
// Start SmartMetrics collection
|
||||
this.smartMetrics.start();
|
||||
|
||||
// Reset daily counters at midnight
|
||||
setInterval(() => {
|
||||
const currentDate = new Date().toDateString();
|
||||
|
||||
if (currentDate !== this.emailMetrics.lastResetDate) {
|
||||
this.emailMetrics.sentToday = 0;
|
||||
this.emailMetrics.receivedToday = 0;
|
||||
this.emailMetrics.failedToday = 0;
|
||||
this.emailMetrics.bouncedToday = 0;
|
||||
this.emailMetrics.lastResetDate = currentDate;
|
||||
}
|
||||
|
||||
if (currentDate !== this.dnsMetrics.lastResetDate) {
|
||||
this.dnsMetrics.totalQueries = 0;
|
||||
this.dnsMetrics.cacheHits = 0;
|
||||
this.dnsMetrics.cacheMisses = 0;
|
||||
this.dnsMetrics.queryTypes = {};
|
||||
this.dnsMetrics.topDomains.clear();
|
||||
this.dnsMetrics.lastResetDate = currentDate;
|
||||
}
|
||||
|
||||
if (currentDate !== this.securityMetrics.lastResetDate) {
|
||||
this.securityMetrics.blockedIPs = 0;
|
||||
this.securityMetrics.authFailures = 0;
|
||||
this.securityMetrics.spamDetected = 0;
|
||||
this.securityMetrics.malwareDetected = 0;
|
||||
this.securityMetrics.phishingDetected = 0;
|
||||
this.securityMetrics.lastResetDate = currentDate;
|
||||
}
|
||||
}, 60000); // Check every minute
|
||||
|
||||
this.logger.log('info', 'MetricsManager started');
|
||||
}
|
||||
|
||||
public async stop(): Promise<void> {
|
||||
this.smartMetrics.stop();
|
||||
this.logger.log('info', 'MetricsManager stopped');
|
||||
}
|
||||
|
||||
// Get server metrics from SmartMetrics and SmartProxy
|
||||
public async getServerStats() {
|
||||
const smartMetricsData = await this.smartMetrics.getMetrics();
|
||||
const proxyStats = this.dcRouter.smartProxy ? this.dcRouter.smartProxy.getStats() : null;
|
||||
|
||||
return {
|
||||
uptime: process.uptime(),
|
||||
startTime: Date.now() - (process.uptime() * 1000),
|
||||
memoryUsage: {
|
||||
heapUsed: process.memoryUsage().heapUsed,
|
||||
heapTotal: process.memoryUsage().heapTotal,
|
||||
external: process.memoryUsage().external,
|
||||
rss: process.memoryUsage().rss,
|
||||
},
|
||||
cpuUsage: {
|
||||
user: parseFloat(smartMetricsData.cpuUsageText || '0'),
|
||||
system: 0, // SmartMetrics doesn't separate user/system
|
||||
},
|
||||
activeConnections: proxyStats ? proxyStats.getActiveConnections() : 0,
|
||||
totalConnections: proxyStats ? proxyStats.getTotalConnections() : 0,
|
||||
requestsPerSecond: proxyStats ? proxyStats.getRequestsPerSecond() : 0,
|
||||
throughput: proxyStats ? proxyStats.getThroughput() : { bytesIn: 0, bytesOut: 0 },
|
||||
};
|
||||
}
|
||||
|
||||
// Get email metrics
|
||||
public async getEmailStats() {
|
||||
return {
|
||||
sentToday: this.emailMetrics.sentToday,
|
||||
receivedToday: this.emailMetrics.receivedToday,
|
||||
failedToday: this.emailMetrics.failedToday,
|
||||
bounceRate: this.emailMetrics.bouncedToday > 0
|
||||
? (this.emailMetrics.bouncedToday / this.emailMetrics.sentToday) * 100
|
||||
: 0,
|
||||
deliveryRate: this.emailMetrics.sentToday > 0
|
||||
? ((this.emailMetrics.sentToday - this.emailMetrics.failedToday) / this.emailMetrics.sentToday) * 100
|
||||
: 100,
|
||||
queueSize: this.emailMetrics.queueSize,
|
||||
averageDeliveryTime: 0, // TODO: Implement when delivery tracking is added
|
||||
topRecipients: [], // TODO: Implement recipient tracking
|
||||
recentActivity: [], // TODO: Implement activity log
|
||||
};
|
||||
}
|
||||
|
||||
// Get DNS metrics
|
||||
public async getDnsStats() {
|
||||
const cacheHitRate = this.dnsMetrics.totalQueries > 0
|
||||
? (this.dnsMetrics.cacheHits / this.dnsMetrics.totalQueries) * 100
|
||||
: 0;
|
||||
|
||||
const topDomains = Array.from(this.dnsMetrics.topDomains.entries())
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.slice(0, 10)
|
||||
.map(([domain, count]) => ({ domain, count }));
|
||||
|
||||
return {
|
||||
queriesPerSecond: 0, // TODO: Calculate based on time window
|
||||
totalQueries: this.dnsMetrics.totalQueries,
|
||||
cacheHits: this.dnsMetrics.cacheHits,
|
||||
cacheMisses: this.dnsMetrics.cacheMisses,
|
||||
cacheHitRate: cacheHitRate,
|
||||
topDomains: topDomains,
|
||||
queryTypes: this.dnsMetrics.queryTypes,
|
||||
averageResponseTime: 0, // TODO: Implement response time tracking
|
||||
activeDomains: this.dnsMetrics.topDomains.size,
|
||||
};
|
||||
}
|
||||
|
||||
// Get security metrics
|
||||
public async getSecurityStats() {
|
||||
return {
|
||||
blockedIPs: this.securityMetrics.blockedIPs,
|
||||
authFailures: this.securityMetrics.authFailures,
|
||||
spamDetected: this.securityMetrics.spamDetected,
|
||||
malwareDetected: this.securityMetrics.malwareDetected,
|
||||
phishingDetected: this.securityMetrics.phishingDetected,
|
||||
totalThreatsBlocked: this.securityMetrics.spamDetected +
|
||||
this.securityMetrics.malwareDetected +
|
||||
this.securityMetrics.phishingDetected,
|
||||
recentIncidents: [], // TODO: Implement incident logging
|
||||
};
|
||||
}
|
||||
|
||||
// Get connection info from SmartProxy
|
||||
public async getConnectionInfo() {
|
||||
const proxyStats = this.dcRouter.smartProxy ? this.dcRouter.smartProxy.getStats() : null;
|
||||
|
||||
if (!proxyStats) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const connectionsByRoute = proxyStats.getConnectionsByRoute();
|
||||
const connectionInfo = [];
|
||||
|
||||
for (const [routeName, count] of connectionsByRoute) {
|
||||
connectionInfo.push({
|
||||
type: 'https',
|
||||
count,
|
||||
source: routeName,
|
||||
lastActivity: new Date(),
|
||||
});
|
||||
}
|
||||
|
||||
return connectionInfo;
|
||||
}
|
||||
|
||||
// Email event tracking methods
|
||||
public trackEmailSent(): void {
|
||||
this.emailMetrics.sentToday++;
|
||||
}
|
||||
|
||||
public trackEmailReceived(): void {
|
||||
this.emailMetrics.receivedToday++;
|
||||
}
|
||||
|
||||
public trackEmailFailed(): void {
|
||||
this.emailMetrics.failedToday++;
|
||||
}
|
||||
|
||||
public trackEmailBounced(): void {
|
||||
this.emailMetrics.bouncedToday++;
|
||||
}
|
||||
|
||||
public updateQueueSize(size: number): void {
|
||||
this.emailMetrics.queueSize = size;
|
||||
}
|
||||
|
||||
// DNS event tracking methods
|
||||
public trackDnsQuery(queryType: string, domain: string, cacheHit: boolean): void {
|
||||
this.dnsMetrics.totalQueries++;
|
||||
|
||||
if (cacheHit) {
|
||||
this.dnsMetrics.cacheHits++;
|
||||
} else {
|
||||
this.dnsMetrics.cacheMisses++;
|
||||
}
|
||||
|
||||
// Track query types
|
||||
this.dnsMetrics.queryTypes[queryType] = (this.dnsMetrics.queryTypes[queryType] || 0) + 1;
|
||||
|
||||
// Track top domains
|
||||
const currentCount = this.dnsMetrics.topDomains.get(domain) || 0;
|
||||
this.dnsMetrics.topDomains.set(domain, currentCount + 1);
|
||||
}
|
||||
|
||||
// Security event tracking methods
|
||||
public trackBlockedIP(): void {
|
||||
this.securityMetrics.blockedIPs++;
|
||||
}
|
||||
|
||||
public trackAuthFailure(): void {
|
||||
this.securityMetrics.authFailures++;
|
||||
}
|
||||
|
||||
public trackSpamDetected(): void {
|
||||
this.securityMetrics.spamDetected++;
|
||||
}
|
||||
|
||||
public trackMalwareDetected(): void {
|
||||
this.securityMetrics.malwareDetected++;
|
||||
}
|
||||
|
||||
public trackPhishingDetected(): void {
|
||||
this.securityMetrics.phishingDetected++;
|
||||
}
|
||||
}
|
||||
1
ts/monitoring/index.ts
Normal file
1
ts/monitoring/index.ts
Normal file
@ -0,0 +1 @@
|
||||
export * from './classes.metricsmanager.js';
|
||||
@ -1,6 +1,7 @@
|
||||
import * as plugins from '../../plugins.js';
|
||||
import type { OpsServer } from '../classes.opsserver.js';
|
||||
import * as interfaces from '../../../ts_interfaces/index.js';
|
||||
import { MetricsManager } from '../../monitoring/index.js';
|
||||
|
||||
export class SecurityHandler {
|
||||
public typedrouter = new plugins.typedrequest.TypedRouter();
|
||||
@ -120,7 +121,29 @@ export class SecurityHandler {
|
||||
phishing: Array<{ timestamp: number; value: number }>;
|
||||
};
|
||||
}> {
|
||||
// TODO: Implement actual security metrics collection
|
||||
// Get metrics from MetricsManager if available
|
||||
if (this.opsServerRef.dcRouterRef.metricsManager) {
|
||||
const securityStats = await this.opsServerRef.dcRouterRef.metricsManager.getSecurityStats();
|
||||
return {
|
||||
blockedIPs: [], // TODO: Track actual blocked IPs
|
||||
reputationScores: {},
|
||||
spamDetection: {
|
||||
detected: securityStats.spamDetected,
|
||||
falsePositives: 0,
|
||||
},
|
||||
malwareDetected: securityStats.malwareDetected,
|
||||
phishingDetected: securityStats.phishingDetected,
|
||||
authFailures: securityStats.authFailures,
|
||||
suspiciousActivities: 0,
|
||||
trends: {
|
||||
spam: [],
|
||||
malware: [],
|
||||
phishing: [],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// Fallback if MetricsManager not available
|
||||
return {
|
||||
blockedIPs: [],
|
||||
reputationScores: {},
|
||||
@ -178,11 +201,31 @@ export class SecurityHandler {
|
||||
status: 'active' | 'idle' | 'closing';
|
||||
}> = [];
|
||||
|
||||
// TODO: Implement actual connection tracking
|
||||
// This would collect from:
|
||||
// - SmartProxy connections
|
||||
// - Email server connections
|
||||
// - DNS server connections
|
||||
// Get connection info from MetricsManager if available
|
||||
if (this.opsServerRef.dcRouterRef.metricsManager) {
|
||||
const connectionInfo = await this.opsServerRef.dcRouterRef.metricsManager.getConnectionInfo();
|
||||
|
||||
// Map connection info to detailed format
|
||||
// Note: Some fields will be placeholder values until more detailed tracking is implemented
|
||||
connectionInfo.forEach((info, index) => {
|
||||
connections.push({
|
||||
id: `conn-${index}`,
|
||||
type: 'http', // TODO: Determine from source/protocol
|
||||
source: {
|
||||
ip: '0.0.0.0', // TODO: Track actual source IPs
|
||||
port: 0,
|
||||
},
|
||||
destination: {
|
||||
ip: '0.0.0.0',
|
||||
port: 443,
|
||||
service: info.source,
|
||||
},
|
||||
startTime: info.lastActivity.getTime(),
|
||||
bytesTransferred: 0, // TODO: Track bytes per connection
|
||||
status: 'active',
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
return connections;
|
||||
}
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
import * as plugins from '../../plugins.js';
|
||||
import type { OpsServer } from '../classes.opsserver.js';
|
||||
import * as interfaces from '../../../ts_interfaces/index.js';
|
||||
import { MetricsManager } from '../../monitoring/index.js';
|
||||
|
||||
export class StatsHandler {
|
||||
public typedrouter = new plugins.typedrequest.TypedRouter();
|
||||
@ -178,25 +179,30 @@ export class StatsHandler {
|
||||
value: number;
|
||||
}>;
|
||||
}> {
|
||||
// Get metrics from MetricsManager if available
|
||||
if (this.opsServerRef.dcRouterRef.metricsManager) {
|
||||
const serverStats = await this.opsServerRef.dcRouterRef.metricsManager.getServerStats();
|
||||
return {
|
||||
uptime: serverStats.uptime,
|
||||
cpuUsage: serverStats.cpuUsage,
|
||||
memoryUsage: serverStats.memoryUsage,
|
||||
requestsPerSecond: serverStats.requestsPerSecond,
|
||||
activeConnections: serverStats.activeConnections,
|
||||
totalConnections: serverStats.totalConnections,
|
||||
history: [], // TODO: Implement history tracking
|
||||
};
|
||||
}
|
||||
|
||||
// Fallback to basic stats if MetricsManager not available
|
||||
const uptime = process.uptime();
|
||||
const memUsage = process.memoryUsage();
|
||||
const totalMem = plugins.os.totalmem();
|
||||
const freeMem = plugins.os.freemem();
|
||||
const usedMem = totalMem - freeMem;
|
||||
|
||||
// Get CPU usage (simplified - in production would use proper monitoring)
|
||||
const cpuUsage = plugins.os.loadavg()[0] * 100 / plugins.os.cpus().length;
|
||||
|
||||
// TODO: Implement proper request tracking
|
||||
const requestsPerSecond = 0;
|
||||
const activeConnections = 0;
|
||||
const totalConnections = 0;
|
||||
|
||||
return {
|
||||
uptime,
|
||||
cpuUsage: {
|
||||
user: cpuUsage * 0.7, // Approximate user CPU
|
||||
system: cpuUsage * 0.3, // Approximate system CPU
|
||||
user: cpuUsage * 0.7,
|
||||
system: cpuUsage * 0.3,
|
||||
},
|
||||
memoryUsage: {
|
||||
heapUsed: memUsage.heapUsed,
|
||||
@ -204,10 +210,10 @@ export class StatsHandler {
|
||||
external: memUsage.external,
|
||||
rss: memUsage.rss,
|
||||
},
|
||||
requestsPerSecond,
|
||||
activeConnections,
|
||||
totalConnections,
|
||||
history: [], // TODO: Implement history tracking
|
||||
requestsPerSecond: 0,
|
||||
activeConnections: 0,
|
||||
totalConnections: 0,
|
||||
history: [],
|
||||
};
|
||||
}
|
||||
|
||||
@ -219,7 +225,19 @@ export class StatsHandler {
|
||||
queueSize: number;
|
||||
domainBreakdown?: { [domain: string]: interfaces.data.IEmailStats };
|
||||
}> {
|
||||
// TODO: Implement actual email statistics collection
|
||||
// Get metrics from MetricsManager if available
|
||||
if (this.opsServerRef.dcRouterRef.metricsManager) {
|
||||
const emailStats = await this.opsServerRef.dcRouterRef.metricsManager.getEmailStats();
|
||||
return {
|
||||
sentToday: emailStats.sentToday,
|
||||
receivedToday: emailStats.receivedToday,
|
||||
bounceRate: emailStats.bounceRate,
|
||||
deliveryRate: emailStats.deliveryRate,
|
||||
queueSize: emailStats.queueSize,
|
||||
};
|
||||
}
|
||||
|
||||
// Fallback if MetricsManager not available
|
||||
return {
|
||||
sentToday: 0,
|
||||
receivedToday: 0,
|
||||
@ -242,7 +260,21 @@ export class StatsHandler {
|
||||
queryTypes: { [key: string]: number };
|
||||
domainBreakdown?: { [domain: string]: interfaces.data.IDnsStats };
|
||||
}> {
|
||||
// TODO: Implement actual DNS statistics collection
|
||||
// Get metrics from MetricsManager if available
|
||||
if (this.opsServerRef.dcRouterRef.metricsManager) {
|
||||
const dnsStats = await this.opsServerRef.dcRouterRef.metricsManager.getDnsStats();
|
||||
return {
|
||||
queriesPerSecond: dnsStats.queriesPerSecond,
|
||||
totalQueries: dnsStats.totalQueries,
|
||||
cacheHits: dnsStats.cacheHits,
|
||||
cacheMisses: dnsStats.cacheMisses,
|
||||
cacheHitRate: dnsStats.cacheHitRate,
|
||||
topDomains: dnsStats.topDomains,
|
||||
queryTypes: dnsStats.queryTypes,
|
||||
};
|
||||
}
|
||||
|
||||
// Fallback if MetricsManager not available
|
||||
return {
|
||||
queriesPerSecond: 0,
|
||||
totalQueries: 0,
|
||||
|
||||
@ -50,6 +50,7 @@ import * as smartguard from '@push.rocks/smartguard';
|
||||
import * as smartjwt from '@push.rocks/smartjwt';
|
||||
import * as smartlog from '@push.rocks/smartlog';
|
||||
import * as smartmail from '@push.rocks/smartmail';
|
||||
import * as smartmetrics from '@push.rocks/smartmetrics';
|
||||
import * as smartnetwork from '@push.rocks/smartnetwork';
|
||||
import * as smartpath from '@push.rocks/smartpath';
|
||||
import * as smartproxy from '@push.rocks/smartproxy';
|
||||
@ -59,7 +60,7 @@ import * as smartrule from '@push.rocks/smartrule';
|
||||
import * as smartrx from '@push.rocks/smartrx';
|
||||
import * as smartunique from '@push.rocks/smartunique';
|
||||
|
||||
export { projectinfo, qenv, smartacme, smartdata, smartdns, smartfile, smartguard, smartjwt, smartlog, smartmail, smartnetwork, smartpath, smartproxy, smartpromise, smartrequest, smartrule, smartrx, smartunique };
|
||||
export { projectinfo, qenv, smartacme, smartdata, smartdns, smartfile, smartguard, smartjwt, smartlog, smartmail, smartmetrics, smartnetwork, smartpath, smartproxy, smartpromise, smartrequest, smartrule, smartrx, smartunique };
|
||||
|
||||
// Define SmartLog types for use in error handling
|
||||
export type TLogLevel = 'error' | 'warn' | 'info' | 'success' | 'debug';
|
||||
|
||||
Reference in New Issue
Block a user