From 93995d503110ff06942ae68c78379b87fc398e41 Mon Sep 17 00:00:00 2001 From: Philipp Kunz Date: Mon, 9 Jun 2025 16:03:27 +0000 Subject: [PATCH] Implement Metrics Manager and Integrate Metrics Collection - Removed the existing readme.opsserver.md file as it is no longer needed. - Added a new MetricsManager class to handle metrics collection using @push.rocks/smartmetrics. - Integrated MetricsManager into the DcRouter and OpsServer classes. - Updated StatsHandler and SecurityHandler to retrieve metrics from MetricsManager. - Implemented methods for tracking email, DNS, and security metrics. - Added connection tracking capabilities to the MetricsManager. - Created a new readme.metrics.md file outlining the metrics implementation plan. - Adjusted plugins.ts to include smartmetrics. - Added a new monitoring directory with classes for metrics management. - Created readme.module-adjustments.md to document necessary adjustments for SmartProxy and SmartDNS. --- package.json | 3 +- pnpm-lock.yaml | 77 ++++- readme.metrics.md | 202 +++++++++++++ readme.module-adjustments.md | 173 +++++++++++ readme.opsserver.md | 351 ---------------------- ts/classes.dcrouter.ts | 17 ++ ts/monitoring/classes.metricsmanager.ts | 260 ++++++++++++++++ ts/monitoring/index.ts | 1 + ts/opsserver/handlers/security.handler.ts | 55 +++- ts/opsserver/handlers/stats.handler.ts | 68 +++-- ts/plugins.ts | 3 +- 11 files changed, 826 insertions(+), 384 deletions(-) create mode 100644 readme.metrics.md create mode 100644 readme.module-adjustments.md delete mode 100644 readme.opsserver.md create mode 100644 ts/monitoring/classes.metricsmanager.ts create mode 100644 ts/monitoring/index.ts diff --git a/package.json b/package.json index f588d69..695b899 100644 --- a/package.json +++ b/package.json @@ -41,10 +41,11 @@ "@push.rocks/smartjwt": "^2.2.1", "@push.rocks/smartlog": "^3.1.8", "@push.rocks/smartmail": "^2.1.0", + "@push.rocks/smartmetrics": "^2.0.9", "@push.rocks/smartnetwork": "^4.0.2", "@push.rocks/smartpath": "^5.0.5", "@push.rocks/smartpromise": "^4.0.3", - "@push.rocks/smartproxy": "^19.5.26", + "@push.rocks/smartproxy": "^19.6.0", "@push.rocks/smartrequest": "^2.1.0", "@push.rocks/smartrule": "^2.0.1", "@push.rocks/smartrx": "^3.0.10", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 032321a..8b9b357 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -59,6 +59,9 @@ importers: '@push.rocks/smartmail': specifier: ^2.1.0 version: 2.1.0 + '@push.rocks/smartmetrics': + specifier: ^2.0.9 + version: 2.0.9 '@push.rocks/smartnetwork': specifier: ^4.0.2 version: 4.0.2 @@ -69,8 +72,8 @@ importers: specifier: ^4.0.3 version: 4.2.3 '@push.rocks/smartproxy': - specifier: ^19.5.26 - version: 19.5.26(@aws-sdk/credential-providers@3.817.0)(socks@2.8.4) + specifier: ^19.6.0 + version: 19.6.0(@aws-sdk/credential-providers@3.817.0)(socks@2.8.4) '@push.rocks/smartrequest': specifier: ^2.1.0 version: 2.1.0 @@ -814,6 +817,10 @@ packages: resolution: {integrity: sha512-T8TbSnGsxo6TDBJx/Sgv/BlVJL3tshxZP7Aq5R1mSnM5OcHY2dQaxLMu2+E8u3gN0MLOzdjurqN4ZRVuzQycOQ==} engines: {node: '>=8.0'} + '@opentelemetry/api@1.9.0': + resolution: {integrity: sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==} + engines: {node: '>=8.0.0'} + '@pdf-lib/standard-fonts@1.0.0': resolution: {integrity: sha512-hU30BK9IUN/su0Mn9VdlVKsWBS6GyhVfqjwl1FjZN4TxP6cCw0jP2w7V3Hf5uX7M0AZJ16vey9yE0ny7Sa59ZA==} @@ -1010,6 +1017,9 @@ packages: '@push.rocks/smartmatch@2.0.0': resolution: {integrity: sha512-MBzP++1yNIBeox71X6VxpIgZ8m4bXnJpZJ4nWVH6IWpmO38MXTu4X0QF8tQnyT4LFcwvc9iiWaD15cstHa7Mmw==} + '@push.rocks/smartmetrics@2.0.9': + resolution: {integrity: sha512-3rKHItgIL4Pt/kFH12qCmTxsnB2EcBgO6H0MYe9dPjDI7yl7IDiLzZg2Abp4q9LEPVw009ATJsxklPQSNqI5gQ==} + '@push.rocks/smartmime@1.0.6': resolution: {integrity: sha512-PHd+I4UcsnOATNg8wjDsSAmmJ4CwQFrQCNzd0HSJMs4ZpiK3Ya91almd6GLpDPU370U4HFh4FaPF4eEAI6vkJQ==} @@ -1052,8 +1062,8 @@ packages: '@push.rocks/smartpromise@4.2.3': resolution: {integrity: sha512-Ycg/TJR+tMt+S3wSFurOpEoW6nXv12QBtKXgBcjMZ4RsdO28geN46U09osPn9N9WuwQy1PkmTV5J/V4F9U8qEw==} - '@push.rocks/smartproxy@19.5.26': - resolution: {integrity: sha512-hCQcIZWX6wjKuom7HYYeGhuRijgU+R958WACDauKfQEDTTzF00STH4GB5KcL7kbaF+20Tx5IBR7pvzHTRjJmQg==} + '@push.rocks/smartproxy@19.6.0': + resolution: {integrity: sha512-qMayuTkKpgQM14Z2cksjG8NmF6KLogcmXQIKUSFx/W7kKoGSkZ+/kYbnngDpc87n86nathr0p2H2euUt0lHfRQ==} '@push.rocks/smartpuppeteer@2.0.5': resolution: {integrity: sha512-yK/qSeWVHIGWRp3c8S5tfdGP6WCKllZC4DR8d8CQlEjszOSBmHtlTdyyqOMBZ/BA4kd+eU5f3A1r4K2tGYty1g==} @@ -1654,6 +1664,9 @@ packages: '@types/node@22.15.30': resolution: {integrity: sha512-6Q7lr06bEHdlfplU6YRbgG1SFBdlsfNC4/lX+SkhiTs0cpJkOElmWls8PxDFv4yY/xKb8Y6SO0OmSX4wgqTZbA==} + '@types/pidusage@2.0.5': + resolution: {integrity: sha512-MIiyZI4/MK9UGUXWt0jJcCZhVw7YdhBuTOuqP/BjuLDLZ2PmmViMIQgZiWxtaMicQfAz/kMrZ5T7PKxFSkTeUA==} + '@types/ping@0.4.4': resolution: {integrity: sha512-ifvo6w2f5eJYlXm+HiVx67iJe8WZp87sfa683nlqED5Vnt9Z93onkokNoWqOG21EaE8fMxyKPobE+mkPEyxsdw==} @@ -1865,6 +1878,9 @@ packages: resolution: {integrity: sha512-4Bcg1P8xhUuqcii/S0Z9wiHIrQVPMermM1any+MX5GeGD7faD3/msQUDGLol9wOcz4/jbg/WJnGqoJF6LiBdtg==} engines: {node: '>=10.0.0'} + bintrees@1.0.2: + resolution: {integrity: sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==} + bn.js@4.12.2: resolution: {integrity: sha512-n4DSx829VRTRByMRGdjQ9iqsN0Bh4OolPsFnaZBLcbi8iXcB+kJ9s7EnRt4wILZNV3kPLHkRVfOc/HvhC3ovDw==} @@ -3615,6 +3631,15 @@ packages: picocolors@1.1.1: resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} + pidtree@0.6.0: + resolution: {integrity: sha512-eG2dWTVw5bzqGRztnHExczNxt5VGsE6OwTeCG3fdUf9KBsZzO3R5OIIIzWR+iZA0NtZ+RDVdaoE2dK1cn6jH4g==} + engines: {node: '>=0.10'} + hasBin: true + + pidusage@4.0.1: + resolution: {integrity: sha512-yCH2dtLHfEBnzlHUJymR/Z1nN2ePG3m392Mv8TFlTP1B0xkpMQNHAnfkY0n2tAi6ceKO6YWhxYfZ96V4vVkh/g==} + engines: {node: '>=18'} + ping@0.4.4: resolution: {integrity: sha512-56ZMC0j7SCsMMLdOoUg12VZCfj/+ZO+yfOSjaNCRrmZZr6GLbN2X/Ui56T15dI8NhiHckaw5X2pvyfAomanwqQ==} engines: {node: '>=4.0.0'} @@ -3642,6 +3667,10 @@ packages: resolution: {integrity: sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==} engines: {node: '>=0.4.0'} + prom-client@15.1.3: + resolution: {integrity: sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==} + engines: {node: ^16 || ^18 || >=20} + property-information@7.1.0: resolution: {integrity: sha512-TwEZ+X+yCJmYfL7TPUOcvBZ4QfoT5YenQiJuX//0th53DE6w0xxLEtfK3iyryQFddXuvkIk51EEgrJQ0WJkOmQ==} @@ -4031,6 +4060,9 @@ packages: resolution: {integrity: sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==} engines: {node: '>=10'} + tdigest@0.1.2: + resolution: {integrity: sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==} + text-decoder@1.2.3: resolution: {integrity: sha512-3/o9z3X0X0fTupwsYvR03pJ/DjWuqqrfwBgTQzdWDiQSm9KitAyz/9WqsT2JQW7KV2m+bC2ol/zqpW37NHxLaA==} @@ -5408,8 +5440,10 @@ snapshots: '@push.rocks/taskbuffer': 3.1.7 transitivePeerDependencies: - '@nuxt/kit' + - bufferutil - react - supports-color + - utf-8-validate - vue '@hapi/hoek@9.3.0': {} @@ -5516,6 +5550,8 @@ snapshots: '@oozcitak/util@8.3.8': {} + '@opentelemetry/api@1.9.0': {} + '@pdf-lib/standard-fonts@1.0.0': dependencies: pako: 1.0.11 @@ -5750,7 +5786,6 @@ snapshots: - '@mongodb-js/zstd' - '@nuxt/kit' - aws-crt - - bufferutil - encoding - gcp-metadata - kerberos @@ -5759,7 +5794,6 @@ snapshots: - snappy - socks - supports-color - - utf-8-validate - vue '@push.rocks/smartarchive@3.0.8': @@ -6060,6 +6094,15 @@ snapshots: dependencies: matcher: 5.0.0 + '@push.rocks/smartmetrics@2.0.9': + dependencies: + '@push.rocks/smartdelay': 3.0.5 + '@push.rocks/smartlog': 3.1.8 + '@types/pidusage': 2.0.5 + pidtree: 0.6.0 + pidusage: 4.0.1 + prom-client: 15.1.3 + '@push.rocks/smartmime@1.0.6': dependencies: '@types/mime-types': 2.1.4 @@ -6180,7 +6223,7 @@ snapshots: '@push.rocks/smartpromise@4.2.3': {} - '@push.rocks/smartproxy@19.5.26(@aws-sdk/credential-providers@3.817.0)(socks@2.8.4)': + '@push.rocks/smartproxy@19.6.0(@aws-sdk/credential-providers@3.817.0)(socks@2.8.4)': dependencies: '@push.rocks/lik': 6.2.2 '@push.rocks/smartacme': 8.0.0(@aws-sdk/credential-providers@3.817.0)(socks@2.8.4) @@ -6191,6 +6234,7 @@ snapshots: '@push.rocks/smartnetwork': 4.0.2 '@push.rocks/smartpromise': 4.2.3 '@push.rocks/smartrequest': 2.1.0 + '@push.rocks/smartrx': 3.0.10 '@push.rocks/smartstring': 4.0.15 '@push.rocks/taskbuffer': 3.1.7 '@tsclass/tsclass': 9.2.0 @@ -7147,6 +7191,8 @@ snapshots: dependencies: undici-types: 6.21.0 + '@types/pidusage@2.0.5': {} + '@types/ping@0.4.4': {} '@types/qs@6.14.0': {} @@ -7352,6 +7398,8 @@ snapshots: basic-ftp@5.0.5: {} + bintrees@1.0.2: {} + bn.js@4.12.2: {} body-parser@1.20.3: @@ -9441,6 +9489,12 @@ snapshots: picocolors@1.1.1: {} + pidtree@0.6.0: {} + + pidusage@4.0.1: + dependencies: + safe-buffer: 5.2.1 + ping@0.4.4: {} pkg-dir@4.2.0: @@ -9463,6 +9517,11 @@ snapshots: progress@2.0.3: {} + prom-client@15.1.3: + dependencies: + '@opentelemetry/api': 1.9.0 + tdigest: 0.1.2 + property-information@7.1.0: {} proto-list@1.2.4: {} @@ -9992,6 +10051,10 @@ snapshots: mkdirp: 1.0.4 yallist: 4.0.0 + tdigest@0.1.2: + dependencies: + bintrees: 1.0.2 + text-decoder@1.2.3: dependencies: b4a: 1.6.7 diff --git a/readme.metrics.md b/readme.metrics.md new file mode 100644 index 0000000..789d32c --- /dev/null +++ b/readme.metrics.md @@ -0,0 +1,202 @@ +# Metrics Implementation Plan with @push.rocks/smartmetrics + +Command to reread CLAUDE.md: `cat /home/centraluser/eu.central.ingress-2/CLAUDE.md` + +## Overview +This plan outlines the migration from placeholder/demo metrics to real metrics using @push.rocks/smartmetrics for the dcrouter project. + +## Current State Analysis + +### Currently Implemented (Real Data) +- CPU usage (basic calculation from os.loadavg) +- Memory usage (from process.memoryUsage) +- System uptime + +### Currently Stubbed (Returns 0 or Demo Data) +- Active connections (HTTP/HTTPS/WebSocket) +- Total connections +- Requests per second +- Email statistics (sent/received/failed/queued/bounce rate) +- DNS statistics (queries/cache hits/response times) +- Security metrics (blocked IPs/auth failures/spam detection) + +## Implementation Plan + +### Phase 1: Core Infrastructure Setup + +1. **Install Dependencies** + ```bash + pnpm install --save @push.rocks/smartmetrics + ``` + +2. **Update plugins.ts** + - Add smartmetrics to ts/plugins.ts + - Import as: `import * as smartmetrics from '@push.rocks/smartmetrics';` + +3. **Create Metrics Manager Class** + - Location: `ts/monitoring/classes.metricsmanager.ts` + - Initialize SmartMetrics with existing logger + - Configure for dcrouter service identification + - Set up automatic metric collection intervals + +### Phase 2: Connection Tracking Implementation + +1. **HTTP/HTTPS Connection Tracking** + - Instrument the SmartProxy connection handlers + - Track active connections in real-time + - Monitor connection lifecycle (open/close events) + - Location: Update connection managers in routing system + +2. **Email Connection Tracking** + - Instrument SMTP server connection handlers + - Track both incoming and outgoing connections + - Location: `ts/mail/delivery/smtpserver/connection-manager.ts` + +3. **DNS Query Tracking** + - Instrument DNS server handlers + - Track query counts and response times + - Location: `ts/mail/routing/classes.dns.manager.ts` + +### Phase 3: Email Metrics Collection + +1. **Email Processing Metrics** + - Track sent/received/failed emails + - Monitor queue sizes + - Calculate delivery and bounce rates + - Location: Instrument `classes.delivery.queue.ts` and `classes.emailsendjob.ts` + +2. **Email Performance Metrics** + - Track processing times + - Monitor queue throughput + - Location: Update delivery system classes + +### Phase 4: Security Metrics Integration + +1. **Security Event Tracking** + - Track blocked IPs from IPReputationChecker + - Monitor authentication failures + - Count spam/malware/phishing detections + - Location: Instrument security classes in `ts/security/` + +### Phase 5: Stats Handler Refactoring + +1. **Update Stats Handler** + - Location: `ts/opsserver/handlers/stats.handler.ts` + - Replace all stub implementations with MetricsManager calls + - Maintain existing API interface structure + +2. **Metrics Aggregation** + - Implement proper time-window aggregations + - Add historical data storage (last hour/day) + - Calculate rates and percentages accurately + +### Phase 6: Prometheus Integration (Optional Enhancement) + +1. **Enable Prometheus Endpoint** + - Add Prometheus metrics endpoint + - Configure port (default: 9090) + - Document metrics for monitoring systems + +## Implementation Details + +### MetricsManager Core Structure +```typescript +export class MetricsManager { + private smartMetrics: smartmetrics.SmartMetrics; + private connectionTrackers: Map; + private emailMetrics: EmailMetricsCollector; + private dnsMetrics: DnsMetricsCollector; + private securityMetrics: SecurityMetricsCollector; + + // Real-time counters + private activeConnections = { + http: 0, + https: 0, + websocket: 0, + smtp: 0 + }; + + // Initialize and start collection + public async start(): Promise; + + // Get aggregated metrics for stats handler + public async getServerStats(): Promise; + public async getEmailStats(): Promise; + public async getDnsStats(): Promise; + public async getSecurityStats(): Promise; +} +``` + +### Connection Tracking Pattern +```typescript +// Example for HTTP connections +onConnectionOpen(type: string) { + this.activeConnections[type]++; + this.totalConnections[type]++; +} + +onConnectionClose(type: string) { + this.activeConnections[type]--; +} +``` + +### Email Metrics Pattern +```typescript +// Track email events +onEmailSent() { this.emailsSentToday++; } +onEmailReceived() { this.emailsReceivedToday++; } +onEmailFailed() { this.emailsFailedToday++; } +onEmailQueued() { this.queueSize++; } +onEmailDequeued() { this.queueSize--; } +``` + +## Testing Strategy + +1. **Unit Tests** + - Test MetricsManager initialization + - Test metric collection accuracy + - Test aggregation calculations + +2. **Integration Tests** + - Test metrics flow from source to API + - Verify real-time updates + - Test under load conditions + +3. **Debug Utilities** + - Create `.nogit/debug/test-metrics.ts` for quick testing + - Add metrics dump endpoint for debugging + +## Migration Steps + +1. Implement MetricsManager without breaking existing code +2. Wire up one metric type at a time +3. Verify each metric shows real data +4. Remove TODO comments from stats handler +5. Update tests to expect real values + +## Success Criteria + +- [ ] All metrics show real, accurate data +- [ ] No performance degradation +- [ ] Metrics update in real-time +- [ ] Historical data is collected +- [ ] All TODO comments removed from stats handler +- [ ] Tests pass with real metric values + +## Notes + +- SmartMetrics provides CPU and memory metrics out of the box +- We'll need custom collectors for application-specific metrics +- Consider adding metric persistence for historical data +- Prometheus integration provides industry-standard monitoring + +## Questions to Address + +1. Should we persist metrics to disk for historical analysis? +2. What time windows should we support (5min, 1hour, 1day)? +3. Should we add alerting thresholds? +4. Do we need custom metric types beyond the current interface? + +--- + +This plan ensures a systematic migration from demo metrics to real, actionable data using @push.rocks/smartmetrics while maintaining the existing API structure and adding powerful monitoring capabilities. \ No newline at end of file diff --git a/readme.module-adjustments.md b/readme.module-adjustments.md new file mode 100644 index 0000000..6421ed9 --- /dev/null +++ b/readme.module-adjustments.md @@ -0,0 +1,173 @@ +# Module Adjustments for Metrics Collection + +Command to reread CLAUDE.md: `cat /home/centraluser/eu.central.ingress-2/CLAUDE.md` + +## SmartProxy Adjustments + +### Current State +SmartProxy (@push.rocks/smartproxy) provides: +- Route-level `maxConnections` limiting +- Event emission system (currently only for certificates) +- NFTables integration with packet statistics +- Connection monitoring during active sessions + +### Missing Capabilities for Metrics +1. **No Connection Lifecycle Events** + - No `connection-open` or `connection-close` events + - No way to track active connections in real-time + - No exposure of internal connection tracking + +2. **No Statistics API** + - No methods like `getActiveConnections()` or `getConnectionStats()` + - No access to connection counts per route + - No throughput or performance metrics exposed + +3. **Limited Event System** + - Currently only emits certificate-related events + - No connection, request, or performance events + +### Required Adjustments +1. **Add Connection Tracking Events** + ```typescript + // Emit on new connection + smartProxy.emit('connection-open', { + type: 'http' | 'https' | 'websocket', + routeName: string, + clientIp: string, + timestamp: Date + }); + + // Emit on connection close + smartProxy.emit('connection-close', { + connectionId: string, + duration: number, + bytesTransferred: number + }); + ``` + +2. **Add Statistics API** + ```typescript + interface IProxyStats { + getActiveConnections(): number; + getConnectionsByRoute(): Map; + getTotalConnections(): number; + getRequestsPerSecond(): number; + getThroughput(): { bytesIn: number, bytesOut: number }; + } + ``` + +3. **Expose Internal Metrics** + - Make connection pools accessible + - Expose route-level statistics + - Provide request/response metrics + +### Alternative Approach +Since SmartProxy is already used with socket handlers for email routing, we could: +1. Wrap all SmartProxy socket handlers with a metrics-aware wrapper +2. Use the existing socket-handler pattern to intercept all connections +3. Track connections at the dcrouter level rather than modifying SmartProxy + +## SmartDNS Adjustments + +### Current State +SmartDNS (@push.rocks/smartdns) provides: +- DNS query handling via registered handlers +- Support for UDP (port 53) and DNS-over-HTTPS +- Domain pattern matching and routing +- DNSSEC support + +### Missing Capabilities for Metrics +1. **No Query Tracking** + - No counters for total queries + - No breakdown by query type (A, AAAA, MX, etc.) + - No domain popularity tracking + +2. **No Performance Metrics** + - No response time tracking + - No cache hit/miss statistics + - No error rate tracking + +3. **No Event Emission** + - No query lifecycle events + - No cache events + - No error events + +### Required Adjustments +1. **Add Query Interceptor/Middleware** + ```typescript + // Wrap handler registration to add metrics + smartDns.use((query, next) => { + metricsCollector.trackQuery(query); + const startTime = Date.now(); + + next((response) => { + metricsCollector.trackResponse(response, Date.now() - startTime); + }); + }); + ``` + +2. **Add Event Emissions** + ```typescript + // Query events + smartDns.emit('query-received', { + type: query.type, + domain: query.domain, + source: 'udp' | 'https', + clientIp: string + }); + + smartDns.emit('query-answered', { + cached: boolean, + responseTime: number, + responseCode: string + }); + ``` + +3. **Add Statistics API** + ```typescript + interface IDnsStats { + getTotalQueries(): number; + getQueriesPerSecond(): number; + getCacheStats(): { hits: number, misses: number, hitRate: number }; + getTopDomains(limit: number): Array<{ domain: string, count: number }>; + getQueryTypeBreakdown(): Record; + } + ``` + +### Alternative Approach +Since we control the handler registration in dcrouter: +1. Create a metrics-aware handler wrapper at the dcrouter level +2. Wrap all DNS handlers before registration +3. Track metrics without modifying SmartDNS itself + +## Implementation Strategy + +### Option 1: Fork and Modify Dependencies +- Fork @push.rocks/smartproxy and @push.rocks/smartdns +- Add metrics capabilities directly +- Maintain custom versions +- **Pros**: Clean integration, full control +- **Cons**: Maintenance burden, divergence from upstream + +### Option 2: Wrapper Approach at DcRouter Level +- Create wrapper classes that intercept all operations +- Track metrics at the application level +- No modifications to dependencies +- **Pros**: No dependency modifications, easier to maintain +- **Cons**: May miss some internal events, slightly higher overhead + +### Option 3: Contribute Back to Upstream +- Submit PRs to add metrics capabilities to original packages +- Work with maintainers to add event emissions and stats APIs +- **Pros**: Benefits everyone, no fork maintenance +- **Cons**: Slower process, may not align with maintainer vision + +## Recommendation + +**Use Option 2 (Wrapper Approach)** for immediate implementation: +1. Create `MetricsAwareProxy` and `MetricsAwareDns` wrapper classes +2. Intercept all operations and track metrics +3. Minimal changes to existing codebase +4. Can migrate to Option 3 later if upstream accepts contributions + +This approach allows us to implement comprehensive metrics collection without modifying external dependencies, maintaining compatibility and reducing maintenance burden. \ No newline at end of file diff --git a/readme.opsserver.md b/readme.opsserver.md deleted file mode 100644 index c4822f2..0000000 --- a/readme.opsserver.md +++ /dev/null @@ -1,351 +0,0 @@ -# DCRouter OpsServer Implementation Plan - -**Command to reread CLAUDE.md: `cat /home/philkunz/.claude/CLAUDE.md`** - -## Overview - -This document outlines the implementation plan for adding a TypedRequest-based API to the DCRouter OpsServer, following the patterns established in the cloudly project. The goal is to create a type-safe, reactive management dashboard with real-time statistics and monitoring capabilities. - -## Architecture Overview - -The implementation follows a clear separation of concerns: -- **Backend**: TypedRequest handlers in OpsServer -- **Frontend**: Reactive web components with Smartstate -- **Communication**: Type-safe requests via TypedRequest pattern -- **State Management**: Centralized state with reactive updates - -## Implementation Phases - -### Phase 1: Interface Definition ✓ - -Create TypeScript interfaces for all API operations: - -#### Directory Structure ✓ -``` -ts_interfaces/ - plugins.ts # TypedRequest interfaces import - data/ # Data type definitions - auth.ts # IIdentity interface - stats.ts # Server, Email, DNS, Security types - index.ts # Exports - requests/ # Request interfaces - admin.ts # Authentication requests - config.ts # Configuration management - logs.ts # Log retrieval with IVirtualStream - stats.ts # Statistics endpoints - index.ts # Exports -``` - -#### Key Interfaces Defined ✓ -- **Server Statistics** - - [x] `IReq_GetServerStatistics` - Server metrics with history - -- **Email Operations** - - [x] `IReq_GetEmailStatistics` - Email delivery stats - - [x] `IReq_GetQueueStatus` - Queue monitoring - -- **DNS Management** - - [x] `IReq_GetDnsStatistics` - DNS query metrics - -- **Rate Limiting** - - [x] `IReq_GetRateLimitStatus` - Rate limit info - -- **Security Metrics** - - [x] `IReq_GetSecurityMetrics` - Security stats and trends - - [x] `IReq_GetActiveConnections` - Connection monitoring - -- **Logging** - - [x] `IReq_GetRecentLogs` - Paginated log retrieval - - [x] `IReq_GetLogStream` - Real-time log streaming with IVirtualStream - -- **Configuration** - - [x] `IReq_GetConfiguration` - Read config - - [x] `IReq_UpdateConfiguration` - Update config - -- **Authentication** - - [x] `IReq_AdminLoginWithUsernameAndPassword` - Admin login - - [x] `IReq_AdminLogout` - Logout - - [x] `IReq_VerifyIdentity` - Token verification - -- **Health Check** - - [x] `IReq_GetHealthStatus` - Service health monitoring - -### Phase 2: Backend Implementation ✓ - -#### 2.1 Enhance OpsServer (`ts/opsserver/classes.opsserver.ts`) ✓ - -- [x] Add TypedRouter initialization -- [x] Use TypedServer's built-in typedrouter -- [x] CORS is already handled by TypedServer -- [x] Add handler registration method - -```typescript -// Example structure following cloudly pattern -public typedrouter = new plugins.typedrequest.TypedRouter(); - -constructor(private dcRouterRef: DcRouter) { - // Add our typedrouter to the dcRouter's main typedrouter - this.dcRouterRef.typedrouter.addTypedRouter(this.typedrouter); -} - -public async start() { - // TypedServer already has a built-in typedrouter at /typedrequest - this.server = new plugins.typedserver.utilityservers.UtilityWebsiteServer({ - domain: 'localhost', - feedMetadata: null, - serveDir: paths.distServe, - }); - - // The server's typedrouter is automatically available - // Add the main dcRouter typedrouter to the server's typedrouter - this.server.typedrouter.addTypedRouter(this.dcRouterRef.typedrouter); - - this.setupHandlers(); - await this.server.start(3000); -} -``` - -**Note**: TypedServer automatically provides the `/typedrequest` endpoint with its built-in typedrouter. We just need to add our routers to it using the `addTypedRouter()` method. - -#### Hierarchical TypedRouter Structure - -Following cloudly's pattern, we'll use a hierarchical router structure: - -``` -TypedServer (built-in typedrouter at /typedrequest) - └── DcRouter.typedrouter (main router) - └── OpsServer.typedrouter (ops-specific handlers) - ├── StatsHandler.typedrouter - ├── ConfigHandler.typedrouter - └── SecurityHandler.typedrouter -``` - -This allows clean separation of concerns while keeping all handlers accessible through the single `/typedrequest` endpoint. - -#### 2.2 Create Handler Classes ✓ - -Create modular handlers in `ts/opsserver/handlers/`: - -- [x] `stats.handler.ts` - Server and performance statistics -- [x] `security.handler.ts` - Security and reputation metrics -- [x] `config.handler.ts` - Configuration management -- [x] `logs.handler.ts` - Log retrieval and streaming -- [x] `admin.handler.ts` - Authentication and session management - -Each handler should: -- Have its own typedrouter that gets added to OpsServer's router -- Access the main DCRouter instance -- Register handlers using TypedHandler instances -- Format responses according to interfaces -- Handle errors gracefully - -Example handler structure: -```typescript -export class StatsHandler { - public typedrouter = new plugins.typedrequest.TypedRouter(); - - constructor(private opsServerRef: OpsServer) { - // Add this handler's router to the parent - this.opsServerRef.typedrouter.addTypedRouter(this.typedrouter); - this.registerHandlers(); - } - - private registerHandlers() { - this.typedrouter.addTypedHandler( - new plugins.typedrequest.TypedHandler( - 'getServerStatistics', - async (dataArg, toolsArg) => { - const stats = await this.collectServerStats(); - return stats; - } - ) - ); - } -} -``` - -### Phase 3: Frontend State Management ✓ - -#### 3.1 Set up Smartstate (`ts_web/appstate.ts`) ✓ - -- [x] Initialize Smartstate instance -- [x] Create state parts with appropriate persistence -- [x] Define initial state structures - -```typescript -// State structure example -interface IStatsState { - serverStats: IRes_ServerStatistics | null; - emailStats: IRes_EmailStatistics | null; - dnsStats: IRes_DnsStatistics | null; - lastUpdated: number; - isLoading: boolean; - error: string | null; -} -``` - -#### 3.2 State Parts to Create ✓ - -- [x] `statsState` - Runtime statistics (soft persistence) -- [x] `configState` - Configuration data (soft persistence) -- [x] `uiState` - UI preferences (persistent) -- [x] `loginState` - Authentication state (persistent) - -### Phase 4: Frontend Integration ✓ - -#### 4.1 API Client Setup ✓ - -- [x] TypedRequest instances created inline within actions -- [x] Base URL handled through relative paths -- [x] Error handling integrated in actions -- [x] Following cloudly pattern of creating requests within actions - -#### 4.2 Create Actions (`ts_web/appstate.ts`) ✓ - -- [x] `loginAction` - Authentication with JWT -- [x] `logoutAction` - Clear authentication state -- [x] `fetchAllStatsAction` - Batch fetch all statistics -- [x] `fetchConfigurationAction` - Get configuration -- [x] `updateConfigurationAction` - Update configuration -- [x] `fetchRecentLogsAction` - Get recent logs -- [x] `toggleAutoRefreshAction` - Toggle auto-refresh -- [x] `setActiveViewAction` - Change active view -- [x] Error handling in all actions - -#### 4.3 Update Dashboard Component (`ts_web/elements/ops-dashboard.ts`) ✓ - -- [x] Subscribe to state changes (login and UI state) -- [x] Implement reactive UI updates -- [x] Use dees-simple-login and dees-simple-appdash components -- [x] Create view components for different sections -- [x] Implement auto-refresh timer functionality - -### Phase 5: Component Structure ✓ - -Created modular view components in `ts_web/elements/`: - -- [x] `ops-view-overview.ts` - Overview with server, email, and DNS statistics -- [x] `ops-view-stats.ts` - Detailed statistics with tables and metrics -- [x] `ops-view-logs.ts` - Log viewer with filtering and search -- [x] `ops-view-config.ts` - Configuration editor with JSON editing -- [x] `ops-view-security.ts` - Security metrics and threat monitoring -- [x] `shared/ops-sectionheading.ts` - Reusable section heading component -- [x] `shared/css.ts` - Shared CSS styles - -### Phase 6: Optional Enhancements - -#### 6.1 Authentication ✓ (Implemented) -- [x] JWT-based authentication using `@push.rocks/smartjwt` -- [x] Guards for identity validation and admin access -- [x] Login/logout endpoints following cloudly pattern -- [ ] Login component (frontend) -- [ ] Protected route handling (frontend) -- [ ] Session persistence (frontend) - -#### 6.2 Real-time Updates (future) -- [ ] WebSocket integration for live stats -- [ ] Push notifications for critical events -- [ ] Event streaming for logs - -## Technical Stack - -### Dependencies to Use -- `@api.global/typedserver` - Server with built-in typedrouter at `/typedrequest` -- `@api.global/typedrequest` - TypedRouter and TypedHandler classes -- `@design.estate/dees-domtools` - Frontend TypedRequest client -- `@push.rocks/smartstate` - State management -- `@design.estate/dees-element` - Web components -- `@design.estate/dees-catalog` - UI components - -### Existing Dependencies to Leverage -- Current DCRouter instance and statistics -- Existing error handling patterns -- Logger infrastructure -- Security modules - -## Development Workflow - -1. **Start with interfaces** - Define all types first -2. **Implement one handler** - Start with server stats -3. **Create minimal frontend** - Test with one endpoint -4. **Iterate** - Add more handlers and UI components -5. **Polish** - Add error handling, loading states, etc. - -## Testing Strategy - -- [ ] Unit tests for handlers -- [ ] Integration tests for API endpoints -- [ ] Frontend component tests -- [ ] End-to-end testing with real DCRouter instance - -## Success Criteria - -- Type-safe communication between frontend and backend -- Real-time statistics display -- Responsive and reactive UI -- Clean, maintainable code structure -- Consistent with cloudly patterns -- Easy to extend with new features - -## Notes - -- Follow existing code conventions in the project -- Use pnpm for all package management -- Ensure all tests pass before marking complete -- Document any deviations from the plan - ---- - -## Progress Status - -### Completed ✓ -- **Phase 1: Interface Definition** - All TypedRequest interfaces created following cloudly pattern - - Created proper TypedRequest interfaces with `method`, `request`, and `response` properties - - Used `IVirtualStream` for log streaming - - Added `@api.global/typedrequest-interfaces` dependency - - All interfaces compile successfully - -- **Phase 2: Backend Implementation** - TypedRouter integration and handlers - - Enhanced OpsServer with hierarchical TypedRouter structure - - Created all handler classes with proper TypedHandler registration - - Implemented mock data responses for all endpoints - - Fixed all TypeScript compilation errors - - VirtualStream used for log streaming with Uint8Array encoding - - **JWT Authentication** - Following cloudly pattern: - - Added `@push.rocks/smartjwt` and `@push.rocks/smartguard` dependencies - - Updated IIdentity interface to match cloudly structure - - Implemented JWT-based authentication with RSA keypairs - - Created validIdentityGuard and adminIdentityGuard - - Added guard helpers for protecting endpoints - - Full test coverage for JWT authentication flows - -- **Phase 3: Frontend State Management** - Smartstate implementation - - Initialized Smartstate with proper state parts - - Created state interfaces for all data types - - Implemented persistent vs soft state persistence - - Set up reactive subscriptions - -- **Phase 4: Frontend Integration** - Complete dashboard implementation - - Created all state management actions with TypedRequest - - Implemented JWT authentication flow in frontend - - Built reactive dashboard with dees-simple-login and dees-simple-appdash - - Added auto-refresh functionality - - Fixed all interface import issues (using dist_ts_interfaces) - -- **Phase 5: Component Structure** - View components - - Created all view components following cloudly patterns - - Implemented reactive data binding with state subscriptions - - Added interactive features (filtering, editing, refresh controls) - - Used @design.estate/dees-catalog components throughout - - Created shared components and styles - -### Next Steps -- Write comprehensive tests for handlers and frontend components -- Implement real data sources (replace mock data) -- Add WebSocket support for real-time updates -- Enhance error handling and user feedback -- Add more detailed charts and visualizations - ---- - -*This plan is a living document. Update it as implementation progresses.* \ No newline at end of file diff --git a/ts/classes.dcrouter.ts b/ts/classes.dcrouter.ts index 40885e5..7e54f79 100644 --- a/ts/classes.dcrouter.ts +++ b/ts/classes.dcrouter.ts @@ -13,6 +13,7 @@ import { configureEmailStorage, configureEmailServer } from './mail/delivery/ind import { StorageManager, type IStorageConfig } from './storage/index.js'; import { OpsServer } from './opsserver/index.js'; +import { MetricsManager } from './monitoring/index.js'; export interface IDcRouterOptions { /** @@ -133,6 +134,7 @@ export class DcRouter { public emailServer?: UnifiedEmailServer; public storageManager: StorageManager; public opsServer: OpsServer; + public metricsManager?: MetricsManager; // TypedRouter for API endpoints public typedrouter = new plugins.typedrequest.TypedRouter(); @@ -160,6 +162,10 @@ export class DcRouter { await this.opsServer.start(); try { + // Initialize MetricsManager + this.metricsManager = new MetricsManager(this); + await this.metricsManager.start(); + // Set up SmartProxy for HTTP/HTTPS and all traffic including email routes await this.setupSmartProxy(); @@ -197,6 +203,14 @@ export class DcRouter { console.log('║ DcRouter Started Successfully ║'); console.log('╚═══════════════════════════════════════════════════════════════════╝\n'); + // Metrics summary + if (this.metricsManager) { + console.log('📊 Metrics Service:'); + console.log(' ├─ SmartMetrics: Active'); + console.log(' ├─ SmartProxy Stats: Active'); + console.log(' └─ Real-time tracking: Enabled'); + } + // SmartProxy summary if (this.smartProxy) { console.log('🌐 SmartProxy Service:'); @@ -566,6 +580,9 @@ export class DcRouter { try { // Stop all services in parallel for faster shutdown await Promise.all([ + // Stop metrics manager if running + this.metricsManager ? this.metricsManager.stop().catch(err => console.error('Error stopping MetricsManager:', err)) : Promise.resolve(), + // Stop unified email server if running this.emailServer ? this.emailServer.stop().catch(err => console.error('Error stopping email server:', err)) : Promise.resolve(), diff --git a/ts/monitoring/classes.metricsmanager.ts b/ts/monitoring/classes.metricsmanager.ts new file mode 100644 index 0000000..3ec36d1 --- /dev/null +++ b/ts/monitoring/classes.metricsmanager.ts @@ -0,0 +1,260 @@ +import * as plugins from '../plugins.js'; +import { DcRouter } from '../classes.dcrouter.js'; + +export class MetricsManager { + private logger: plugins.smartlog.Smartlog; + private smartMetrics: plugins.smartmetrics.SmartMetrics; + private dcRouter: DcRouter; + + // Track email-specific metrics + private emailMetrics = { + sentToday: 0, + receivedToday: 0, + failedToday: 0, + bouncedToday: 0, + queueSize: 0, + lastResetDate: new Date().toDateString(), + }; + + // Track DNS-specific metrics + private dnsMetrics = { + totalQueries: 0, + cacheHits: 0, + cacheMisses: 0, + queryTypes: {} as Record, + topDomains: new Map(), + lastResetDate: new Date().toDateString(), + }; + + // Track security-specific metrics + private securityMetrics = { + blockedIPs: 0, + authFailures: 0, + spamDetected: 0, + malwareDetected: 0, + phishingDetected: 0, + lastResetDate: new Date().toDateString(), + }; + + constructor(dcRouter: DcRouter) { + this.dcRouter = dcRouter; + // Create a new Smartlog instance for metrics + this.logger = new plugins.smartlog.Smartlog({ + logContext: { + environment: 'production', + runtime: 'node', + zone: 'dcrouter-metrics', + } + }); + this.smartMetrics = new plugins.smartmetrics.SmartMetrics(this.logger, 'dcrouter'); + } + + public async start(): Promise { + // Start SmartMetrics collection + this.smartMetrics.start(); + + // Reset daily counters at midnight + setInterval(() => { + const currentDate = new Date().toDateString(); + + if (currentDate !== this.emailMetrics.lastResetDate) { + this.emailMetrics.sentToday = 0; + this.emailMetrics.receivedToday = 0; + this.emailMetrics.failedToday = 0; + this.emailMetrics.bouncedToday = 0; + this.emailMetrics.lastResetDate = currentDate; + } + + if (currentDate !== this.dnsMetrics.lastResetDate) { + this.dnsMetrics.totalQueries = 0; + this.dnsMetrics.cacheHits = 0; + this.dnsMetrics.cacheMisses = 0; + this.dnsMetrics.queryTypes = {}; + this.dnsMetrics.topDomains.clear(); + this.dnsMetrics.lastResetDate = currentDate; + } + + if (currentDate !== this.securityMetrics.lastResetDate) { + this.securityMetrics.blockedIPs = 0; + this.securityMetrics.authFailures = 0; + this.securityMetrics.spamDetected = 0; + this.securityMetrics.malwareDetected = 0; + this.securityMetrics.phishingDetected = 0; + this.securityMetrics.lastResetDate = currentDate; + } + }, 60000); // Check every minute + + this.logger.log('info', 'MetricsManager started'); + } + + public async stop(): Promise { + this.smartMetrics.stop(); + this.logger.log('info', 'MetricsManager stopped'); + } + + // Get server metrics from SmartMetrics and SmartProxy + public async getServerStats() { + const smartMetricsData = await this.smartMetrics.getMetrics(); + const proxyStats = this.dcRouter.smartProxy ? this.dcRouter.smartProxy.getStats() : null; + + return { + uptime: process.uptime(), + startTime: Date.now() - (process.uptime() * 1000), + memoryUsage: { + heapUsed: process.memoryUsage().heapUsed, + heapTotal: process.memoryUsage().heapTotal, + external: process.memoryUsage().external, + rss: process.memoryUsage().rss, + }, + cpuUsage: { + user: parseFloat(smartMetricsData.cpuUsageText || '0'), + system: 0, // SmartMetrics doesn't separate user/system + }, + activeConnections: proxyStats ? proxyStats.getActiveConnections() : 0, + totalConnections: proxyStats ? proxyStats.getTotalConnections() : 0, + requestsPerSecond: proxyStats ? proxyStats.getRequestsPerSecond() : 0, + throughput: proxyStats ? proxyStats.getThroughput() : { bytesIn: 0, bytesOut: 0 }, + }; + } + + // Get email metrics + public async getEmailStats() { + return { + sentToday: this.emailMetrics.sentToday, + receivedToday: this.emailMetrics.receivedToday, + failedToday: this.emailMetrics.failedToday, + bounceRate: this.emailMetrics.bouncedToday > 0 + ? (this.emailMetrics.bouncedToday / this.emailMetrics.sentToday) * 100 + : 0, + deliveryRate: this.emailMetrics.sentToday > 0 + ? ((this.emailMetrics.sentToday - this.emailMetrics.failedToday) / this.emailMetrics.sentToday) * 100 + : 100, + queueSize: this.emailMetrics.queueSize, + averageDeliveryTime: 0, // TODO: Implement when delivery tracking is added + topRecipients: [], // TODO: Implement recipient tracking + recentActivity: [], // TODO: Implement activity log + }; + } + + // Get DNS metrics + public async getDnsStats() { + const cacheHitRate = this.dnsMetrics.totalQueries > 0 + ? (this.dnsMetrics.cacheHits / this.dnsMetrics.totalQueries) * 100 + : 0; + + const topDomains = Array.from(this.dnsMetrics.topDomains.entries()) + .sort((a, b) => b[1] - a[1]) + .slice(0, 10) + .map(([domain, count]) => ({ domain, count })); + + return { + queriesPerSecond: 0, // TODO: Calculate based on time window + totalQueries: this.dnsMetrics.totalQueries, + cacheHits: this.dnsMetrics.cacheHits, + cacheMisses: this.dnsMetrics.cacheMisses, + cacheHitRate: cacheHitRate, + topDomains: topDomains, + queryTypes: this.dnsMetrics.queryTypes, + averageResponseTime: 0, // TODO: Implement response time tracking + activeDomains: this.dnsMetrics.topDomains.size, + }; + } + + // Get security metrics + public async getSecurityStats() { + return { + blockedIPs: this.securityMetrics.blockedIPs, + authFailures: this.securityMetrics.authFailures, + spamDetected: this.securityMetrics.spamDetected, + malwareDetected: this.securityMetrics.malwareDetected, + phishingDetected: this.securityMetrics.phishingDetected, + totalThreatsBlocked: this.securityMetrics.spamDetected + + this.securityMetrics.malwareDetected + + this.securityMetrics.phishingDetected, + recentIncidents: [], // TODO: Implement incident logging + }; + } + + // Get connection info from SmartProxy + public async getConnectionInfo() { + const proxyStats = this.dcRouter.smartProxy ? this.dcRouter.smartProxy.getStats() : null; + + if (!proxyStats) { + return []; + } + + const connectionsByRoute = proxyStats.getConnectionsByRoute(); + const connectionInfo = []; + + for (const [routeName, count] of connectionsByRoute) { + connectionInfo.push({ + type: 'https', + count, + source: routeName, + lastActivity: new Date(), + }); + } + + return connectionInfo; + } + + // Email event tracking methods + public trackEmailSent(): void { + this.emailMetrics.sentToday++; + } + + public trackEmailReceived(): void { + this.emailMetrics.receivedToday++; + } + + public trackEmailFailed(): void { + this.emailMetrics.failedToday++; + } + + public trackEmailBounced(): void { + this.emailMetrics.bouncedToday++; + } + + public updateQueueSize(size: number): void { + this.emailMetrics.queueSize = size; + } + + // DNS event tracking methods + public trackDnsQuery(queryType: string, domain: string, cacheHit: boolean): void { + this.dnsMetrics.totalQueries++; + + if (cacheHit) { + this.dnsMetrics.cacheHits++; + } else { + this.dnsMetrics.cacheMisses++; + } + + // Track query types + this.dnsMetrics.queryTypes[queryType] = (this.dnsMetrics.queryTypes[queryType] || 0) + 1; + + // Track top domains + const currentCount = this.dnsMetrics.topDomains.get(domain) || 0; + this.dnsMetrics.topDomains.set(domain, currentCount + 1); + } + + // Security event tracking methods + public trackBlockedIP(): void { + this.securityMetrics.blockedIPs++; + } + + public trackAuthFailure(): void { + this.securityMetrics.authFailures++; + } + + public trackSpamDetected(): void { + this.securityMetrics.spamDetected++; + } + + public trackMalwareDetected(): void { + this.securityMetrics.malwareDetected++; + } + + public trackPhishingDetected(): void { + this.securityMetrics.phishingDetected++; + } +} \ No newline at end of file diff --git a/ts/monitoring/index.ts b/ts/monitoring/index.ts new file mode 100644 index 0000000..4df123c --- /dev/null +++ b/ts/monitoring/index.ts @@ -0,0 +1 @@ +export * from './classes.metricsmanager.js'; \ No newline at end of file diff --git a/ts/opsserver/handlers/security.handler.ts b/ts/opsserver/handlers/security.handler.ts index 24f70d2..68dfca6 100644 --- a/ts/opsserver/handlers/security.handler.ts +++ b/ts/opsserver/handlers/security.handler.ts @@ -1,6 +1,7 @@ import * as plugins from '../../plugins.js'; import type { OpsServer } from '../classes.opsserver.js'; import * as interfaces from '../../../ts_interfaces/index.js'; +import { MetricsManager } from '../../monitoring/index.js'; export class SecurityHandler { public typedrouter = new plugins.typedrequest.TypedRouter(); @@ -120,7 +121,29 @@ export class SecurityHandler { phishing: Array<{ timestamp: number; value: number }>; }; }> { - // TODO: Implement actual security metrics collection + // Get metrics from MetricsManager if available + if (this.opsServerRef.dcRouterRef.metricsManager) { + const securityStats = await this.opsServerRef.dcRouterRef.metricsManager.getSecurityStats(); + return { + blockedIPs: [], // TODO: Track actual blocked IPs + reputationScores: {}, + spamDetection: { + detected: securityStats.spamDetected, + falsePositives: 0, + }, + malwareDetected: securityStats.malwareDetected, + phishingDetected: securityStats.phishingDetected, + authFailures: securityStats.authFailures, + suspiciousActivities: 0, + trends: { + spam: [], + malware: [], + phishing: [], + }, + }; + } + + // Fallback if MetricsManager not available return { blockedIPs: [], reputationScores: {}, @@ -178,11 +201,31 @@ export class SecurityHandler { status: 'active' | 'idle' | 'closing'; }> = []; - // TODO: Implement actual connection tracking - // This would collect from: - // - SmartProxy connections - // - Email server connections - // - DNS server connections + // Get connection info from MetricsManager if available + if (this.opsServerRef.dcRouterRef.metricsManager) { + const connectionInfo = await this.opsServerRef.dcRouterRef.metricsManager.getConnectionInfo(); + + // Map connection info to detailed format + // Note: Some fields will be placeholder values until more detailed tracking is implemented + connectionInfo.forEach((info, index) => { + connections.push({ + id: `conn-${index}`, + type: 'http', // TODO: Determine from source/protocol + source: { + ip: '0.0.0.0', // TODO: Track actual source IPs + port: 0, + }, + destination: { + ip: '0.0.0.0', + port: 443, + service: info.source, + }, + startTime: info.lastActivity.getTime(), + bytesTransferred: 0, // TODO: Track bytes per connection + status: 'active', + }); + }); + } return connections; } diff --git a/ts/opsserver/handlers/stats.handler.ts b/ts/opsserver/handlers/stats.handler.ts index 431e840..71b62ea 100644 --- a/ts/opsserver/handlers/stats.handler.ts +++ b/ts/opsserver/handlers/stats.handler.ts @@ -1,6 +1,7 @@ import * as plugins from '../../plugins.js'; import type { OpsServer } from '../classes.opsserver.js'; import * as interfaces from '../../../ts_interfaces/index.js'; +import { MetricsManager } from '../../monitoring/index.js'; export class StatsHandler { public typedrouter = new plugins.typedrequest.TypedRouter(); @@ -178,25 +179,30 @@ export class StatsHandler { value: number; }>; }> { + // Get metrics from MetricsManager if available + if (this.opsServerRef.dcRouterRef.metricsManager) { + const serverStats = await this.opsServerRef.dcRouterRef.metricsManager.getServerStats(); + return { + uptime: serverStats.uptime, + cpuUsage: serverStats.cpuUsage, + memoryUsage: serverStats.memoryUsage, + requestsPerSecond: serverStats.requestsPerSecond, + activeConnections: serverStats.activeConnections, + totalConnections: serverStats.totalConnections, + history: [], // TODO: Implement history tracking + }; + } + + // Fallback to basic stats if MetricsManager not available const uptime = process.uptime(); const memUsage = process.memoryUsage(); - const totalMem = plugins.os.totalmem(); - const freeMem = plugins.os.freemem(); - const usedMem = totalMem - freeMem; - - // Get CPU usage (simplified - in production would use proper monitoring) const cpuUsage = plugins.os.loadavg()[0] * 100 / plugins.os.cpus().length; - // TODO: Implement proper request tracking - const requestsPerSecond = 0; - const activeConnections = 0; - const totalConnections = 0; - return { uptime, cpuUsage: { - user: cpuUsage * 0.7, // Approximate user CPU - system: cpuUsage * 0.3, // Approximate system CPU + user: cpuUsage * 0.7, + system: cpuUsage * 0.3, }, memoryUsage: { heapUsed: memUsage.heapUsed, @@ -204,10 +210,10 @@ export class StatsHandler { external: memUsage.external, rss: memUsage.rss, }, - requestsPerSecond, - activeConnections, - totalConnections, - history: [], // TODO: Implement history tracking + requestsPerSecond: 0, + activeConnections: 0, + totalConnections: 0, + history: [], }; } @@ -219,7 +225,19 @@ export class StatsHandler { queueSize: number; domainBreakdown?: { [domain: string]: interfaces.data.IEmailStats }; }> { - // TODO: Implement actual email statistics collection + // Get metrics from MetricsManager if available + if (this.opsServerRef.dcRouterRef.metricsManager) { + const emailStats = await this.opsServerRef.dcRouterRef.metricsManager.getEmailStats(); + return { + sentToday: emailStats.sentToday, + receivedToday: emailStats.receivedToday, + bounceRate: emailStats.bounceRate, + deliveryRate: emailStats.deliveryRate, + queueSize: emailStats.queueSize, + }; + } + + // Fallback if MetricsManager not available return { sentToday: 0, receivedToday: 0, @@ -242,7 +260,21 @@ export class StatsHandler { queryTypes: { [key: string]: number }; domainBreakdown?: { [domain: string]: interfaces.data.IDnsStats }; }> { - // TODO: Implement actual DNS statistics collection + // Get metrics from MetricsManager if available + if (this.opsServerRef.dcRouterRef.metricsManager) { + const dnsStats = await this.opsServerRef.dcRouterRef.metricsManager.getDnsStats(); + return { + queriesPerSecond: dnsStats.queriesPerSecond, + totalQueries: dnsStats.totalQueries, + cacheHits: dnsStats.cacheHits, + cacheMisses: dnsStats.cacheMisses, + cacheHitRate: dnsStats.cacheHitRate, + topDomains: dnsStats.topDomains, + queryTypes: dnsStats.queryTypes, + }; + } + + // Fallback if MetricsManager not available return { queriesPerSecond: 0, totalQueries: 0, diff --git a/ts/plugins.ts b/ts/plugins.ts index efd7ecc..3aa43bc 100644 --- a/ts/plugins.ts +++ b/ts/plugins.ts @@ -50,6 +50,7 @@ import * as smartguard from '@push.rocks/smartguard'; import * as smartjwt from '@push.rocks/smartjwt'; import * as smartlog from '@push.rocks/smartlog'; import * as smartmail from '@push.rocks/smartmail'; +import * as smartmetrics from '@push.rocks/smartmetrics'; import * as smartnetwork from '@push.rocks/smartnetwork'; import * as smartpath from '@push.rocks/smartpath'; import * as smartproxy from '@push.rocks/smartproxy'; @@ -59,7 +60,7 @@ import * as smartrule from '@push.rocks/smartrule'; import * as smartrx from '@push.rocks/smartrx'; import * as smartunique from '@push.rocks/smartunique'; -export { projectinfo, qenv, smartacme, smartdata, smartdns, smartfile, smartguard, smartjwt, smartlog, smartmail, smartnetwork, smartpath, smartproxy, smartpromise, smartrequest, smartrule, smartrx, smartunique }; +export { projectinfo, qenv, smartacme, smartdata, smartdns, smartfile, smartguard, smartjwt, smartlog, smartmail, smartmetrics, smartnetwork, smartpath, smartproxy, smartpromise, smartrequest, smartrule, smartrx, smartunique }; // Define SmartLog types for use in error handling export type TLogLevel = 'error' | 'warn' | 'info' | 'success' | 'debug';