From fb424d814c6b94dc1129c59372a4e58ae4be4e70 Mon Sep 17 00:00:00 2001 From: Philipp Kunz Date: Fri, 16 May 2025 15:50:46 +0000 Subject: [PATCH] feat(smartproxy): Update documentation and configuration guides to adopt new route-based SmartProxy architecture --- changelog.md | 8 + readme.hints.md | 168 +++- readme.plan.md | 1662 ++++++++------------------------- readme.smartlog.md | 107 --- ts/00_commitinfo_data.ts | 2 +- ts/classes.dcrouter.ts | 252 +++-- ts/classes.smtp.portconfig.ts | 100 +- ts_web/00_commitinfo_data.ts | 2 +- 8 files changed, 785 insertions(+), 1516 deletions(-) delete mode 100644 readme.smartlog.md diff --git a/changelog.md b/changelog.md index 4d7d962..27be75e 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,13 @@ # Changelog +## 2025-05-16 - 2.12.0 - feat(smartproxy) +Update documentation and configuration guides to adopt new route-based SmartProxy architecture + +- Revise SmartProxy implementation hints in readme.hints.md to describe route-based configuration with glob pattern matching +- Add migration examples showing transition from old direct configuration to new route-based style +- Update DcRouter and SMTP port configuration to generate SmartProxy routes for email handling (ports 25, 587, 465 mapped to internal services) +- Enhance integration documentation with examples for HTTP and email services using the new SmartProxy routes + ## 2025-05-16 - 2.11.2 - fix(dependencies) Update dependency versions and adjust test imports to use new packages diff --git a/readme.hints.md b/readme.hints.md index 3be345d..0c3b473 100644 --- a/readme.hints.md +++ b/readme.hints.md @@ -2,31 +2,76 @@ ## SmartProxy Usage +### New Route-Based Architecture (v18+) +- SmartProxy now uses a route-based configuration system +- Routes define match criteria and actions instead of simple port-to-port forwarding +- All traffic types (HTTP, HTTPS, TCP, WebSocket) are configured through routes + +```typescript +// NEW: Route-based SmartProxy configuration +const smartProxy = new plugins.smartproxy.SmartProxy({ + routes: [ + { + name: 'https-traffic', + match: { + ports: 443, + domains: ['example.com', '*.example.com'] + }, + action: { + type: 'forward', + target: { + host: 'backend.server.com', + port: 8080 + } + }, + tls: { + mode: 'terminate', + certificate: 'auto' + } + } + ], + defaults: { + target: { + host: 'fallback.server.com', + port: 8080 + } + }, + acme: { + accountEmail: 'admin@example.com', + enabled: true, + useProduction: true + } +}); +``` + +### Migration from Old to New +```typescript +// OLD configuration style (deprecated) +{ + fromPort: 443, + toPort: 8080, + targetIP: 'backend.server.com', + domainConfigs: [...] +} + +// NEW route-based style +{ + routes: [{ + name: 'main-route', + match: { ports: 443 }, + action: { + type: 'forward', + target: { host: 'backend.server.com', port: 8080 } + } + }] +} +``` + ### Direct Component Usage - Use SmartProxy components directly instead of creating your own wrappers - SmartProxy already includes Port80Handler and NetworkProxy functionality - When using SmartProxy, configure it directly rather than instantiating Port80Handler or NetworkProxy separately -```typescript -// PREFERRED: Use SmartProxy with built-in ACME support -const smartProxy = new plugins.smartproxy.SmartProxy({ - fromPort: 443, - toPort: targetPort, - targetIP: targetServer, - sniEnabled: true, - acme: { - port: 80, - enabled: true, - autoRenew: true, - useProduction: true, - renewThresholdDays: 30, - accountEmail: contactEmail - }, - globalPortRanges: [{ from: 443, to: 443 }], - domainConfigs: [/* domain configurations */] -}); -``` - ### Certificate Management - SmartProxy has built-in ACME certificate management - Configure it in the `acme` property of SmartProxy options @@ -48,15 +93,48 @@ const value = await qenv.getEnvVarOnDemand('ENV_VAR_NAME'); ### SmartProxy Interfaces - Always check the interfaces from the node_modules to ensure correct property names -- Important interfaces: - - `ISmartProxyOptions`: Main configuration for SmartProxy +- Important interfaces for the new architecture: + - `ISmartProxyOptions`: Main configuration with `routes` array + - `IRouteConfig`: Individual route configuration + - `IRouteMatch`: Match criteria for routes + - `IRouteTarget`: Target configuration for forwarding - `IAcmeOptions`: ACME certificate configuration - - `IDomainConfig`: Domain-specific configuration + - `TTlsMode`: TLS handling modes ('passthrough' | 'terminate' | 'terminate-and-reencrypt') + +### New Route Configuration +```typescript +interface IRouteConfig { + name: string; + match: { + ports: number | number[]; + domains?: string | string[]; + path?: string; + headers?: Record; + }; + action: { + type: 'forward' | 'redirect' | 'block' | 'static'; + target?: { + host: string | string[] | ((context) => string); + port: number | 'preserve' | ((context) => number); + }; + }; + tls?: { + mode: TTlsMode; + certificate?: 'auto' | { key: string; cert: string; }; + }; + security?: { + authentication?: IRouteAuthentication; + rateLimit?: IRouteRateLimit; + ipAllowList?: string[]; + ipBlockList?: string[]; + }; +} +``` ### Required Properties -- Remember to include all required properties in your interface implementations -- For `ISmartProxyOptions`, `globalPortRanges` is required +- For `ISmartProxyOptions`, `routes` array is the main configuration - For `IAcmeOptions`, use `accountEmail` for the contact email +- Routes must have `name`, `match`, and `action` properties ## Testing @@ -93,4 +171,42 @@ tap.test('stop', async () => { ### Component Integration - Leverage built-in integrations between components (like SmartProxy's ACME handling) - Use parallel operations for performance (like in the `stop()` method) -- Separate concerns clearly (HTTP handling vs. SMTP handling) \ No newline at end of file +- Separate concerns clearly (HTTP handling vs. SMTP handling) + +## Email Integration with SmartProxy + +### Architecture +- Email traffic is routed through SmartProxy using automatic route generation +- Email server runs on internal ports and receives forwarded traffic from SmartProxy +- SmartProxy handles external ports (25, 587, 465) and forwards to internal ports + +### Email Route Generation +```typescript +// Email configuration automatically generates SmartProxy routes +emailConfig: { + ports: [25, 587, 465], + hostname: 'mail.example.com', + domainRules: [...] +} + +// Generates routes like: +{ + name: 'smtp-route', + match: { ports: [25] }, + action: { + type: 'forward', + target: { host: 'localhost', port: 10025 } + }, + tls: { mode: 'passthrough' } // STARTTLS handled by email server +} +``` + +### Port Mapping +- External port 25 → Internal port 10025 (SMTP) +- External port 587 → Internal port 10587 (Submission) +- External port 465 → Internal port 10465 (SMTPS) + +### TLS Handling +- Ports 25 and 587: Use 'passthrough' mode (STARTTLS handled by email server) +- Port 465: Use 'terminate' mode (SmartProxy handles TLS termination) +- Domain-specific TLS can be configured per email rule \ No newline at end of file diff --git a/readme.plan.md b/readme.plan.md index 3251f97..cec0047 100644 --- a/readme.plan.md +++ b/readme.plan.md @@ -1,1308 +1,428 @@ -# PlatformService Roadmap +# Platform Service - SmartProxy Architecture Update Plan -## Latest Changes - -### API Type Safety Improvements -- [x] Create comprehensive TypeScript interfaces for all API methods -- [x] Replace any types with specific interfaces in EmailService and MtaConnector -- [x] Align interface types with their implementations -- [x] Document all interface properties with detailed JSDoc comments -- [x] Use type imports to ensure consistency between services -- [x] Fix TypeScript build errors from interface placements -- [x] Add proper method signatures for API endpoint handlers -- [x] Implement type-safe API responses - -### Test Stability Improvements -- [x] Fix race conditions in SenderReputationMonitor tests -- [x] Disable filesystem operations during tests to prevent race conditions -- [x] Add proper cleanup of singleton instances and timeouts -- [x] Ensure all tests properly clean up shared resources -- [x] Set consistent test environment with NODE_ENV=test - -### Error Handling Improvements -- [x] Add structured error types for better error reporting -- [x] Implement consistent error handling patterns across services -- [x] Add detailed error context for debugging -- [x] Create retry mechanisms for transient failures -- [x] Improve error logging with structured data - -### Configuration Interface Standardization -- [x] Define consistent configuration interfaces across services -- [x] Implement validation for all configuration objects -- [x] Add default values for optional configuration -- [x] Create documentation for all configuration options -- [x] Add migration helpers for configuration format changes - -### Logging Enhancements -- [ ] Implement structured logging throughout the codebase -- [ ] Add context information to all log messages -- [ ] Create consistent log levels and usage patterns -- [ ] Add correlation IDs for request tracking -- [ ] Implement log filtering and sampling options - -### Mailgun Removal -- [x] Remove Mailgun integration from keywords in package.json and npmextra.json -- [x] Update EmailService comments to remove mentions of Mailgun -- [x] Update error messages to reference MTA instead of generic email providers -- [x] Update the readme email example to use PlatformService reference instead of Mailgun API key - -# DcRouter Consolidated Email Configuration Plan +**Command to reread CLAUDE.md: `cat /home/philkunz/.claude/CLAUDE.md`** ## Overview -This plan outlines the consolidation of all email-related functionality in DcRouter under a unified `emailConfig` interface. This new approach combines MTA, SMTP forwarding, and store-and-forward processing into a single, pattern-based routing system that: -1. Uses glob patterns for domain matching (e.g., `*@task.vc` or `*@*.example.net`) -2. Shares ports across all email handling methods (25, 587, 465) -3. Allows different handling modes based on email domain patterns -4. Provides a flexible configuration interface for all email-related functionality +SmartProxy has undergone a major architectural update, moving from a direct port-to-port proxy configuration to a flexible route-based system. This plan outlines the necessary updates to integrate these changes into the platformservice while retaining all SmartProxy functionality. -This consolidated approach simplifies configuration while enhancing flexibility, allowing domain-specific handling where, for example, `*@task.vc` emails are forwarded to another SMTP server while `*@lossless.digital` emails are processed by the MTA for programmatic use. - -## 0. Configuration Approaches - -### 0.1 Core SmartProxy Direct Configuration -DcRouter should leverage SmartProxy's configuration directly, exposing SmartProxy's full domain configuration options to give users maximum flexibility for all HTTP/HTTPS and TCP/SNI traffic: +## New SmartProxy Architecture +### Previous Architecture (Legacy) ```typescript -interface IDcRouterOptions { - // Direct SmartProxy configuration for general HTTP/HTTPS and TCP/SNI traffic - smartProxyConfig?: plugins.smartproxy.ISmartProxyOptions; - - // Consolidated email configuration - emailConfig?: IEmailConfig; - - // Other DcRouter options... +// Old configuration style +{ + fromPort: 443, + toPort: 8080, + targetIP: 'backend.server.com', + sniEnabled: true, + domainConfigs: [ + { + domain: 'example.com', + target: 'backend1.server.com', + port: 8081 + } + ] } ``` -This approach allows direct configuration of SmartProxy's powerful domain-based routing, giving full control over HTTP/HTTPS and SNI-based traffic: - +### New Architecture (Route-Based) ```typescript -const dcRouter = new DcRouter({ - // Direct SmartProxy configuration for HTTP/HTTPS traffic - smartProxyConfig: { - fromPort: 443, - toPort: 8080, - targetIP: '10.0.0.10', - sniEnabled: true, - acme: { - port: 80, - enabled: true, - autoRenew: true, - useProduction: true, - renewThresholdDays: 30, - accountEmail: 'admin@example.com' - }, - globalPortRanges: [ - { from: 80, to: 80 }, - { from: 443, to: 443 } - ], - - // SmartProxy's full domain configuration flexibility - domainConfigs: [ - { - domains: ['example.com', 'www.example.com'], - allowedIPs: ['0.0.0.0/0'], - blockedIPs: ['1.2.3.4/32'], - targetIPs: ['10.0.0.10', '10.0.0.11'], - portRanges: [ - { from: 80, to: 80 }, - { from: 443, to: 443 } - ], - connectionTimeout: 60000, - useNetworkProxy: true +// New configuration style - FULL SmartProxy functionality +{ + routes: [ + { + name: 'https-traffic', + match: { + ports: 443, + domains: ['example.com', '*.example.com'] }, - // Additional domain configurations... - ] - }, - - // Consolidated email configuration - emailConfig: { - // Email handling configuration... - } -}); -``` - -### 0.2 Consolidated Email Configuration -We'll implement a unified email configuration approach that combines MTA, SMTP forwarding, and store-and-forward processing into a single pattern-based routing system: - -```typescript -interface IDcRouterOptions { - // HTTP/HTTPS configuration - smartProxyConfig?: plugins.smartproxy.ISmartProxyOptions; - - // Consolidated email handling - emailConfig?: { - // Global email server settings - ports: number[]; - hostname: string; - maxMessageSize?: number; - - // TLS configuration for all email services - tls?: { - certPath?: string; - keyPath?: string; - caPath?: string; - minVersion?: string; - }; - - // Authentication for inbound connections - auth?: { - required?: boolean; - methods?: ('PLAIN' | 'LOGIN' | 'OAUTH2')[]; - users?: Array<{username: string, password: string}>; - }; - - // Default routing for unmatched domains - defaultMode: 'forward' | 'mta' | 'process'; - defaultServer?: string; - defaultPort?: number; - defaultTls?: boolean; - - // Domain-specific rules with glob pattern support - domainRules: Array<{ - // Domain pattern (e.g., "*@example.com", "*@*.example.net") - pattern: string; - - // Handling mode for this pattern - mode: 'forward' | 'mta' | 'process'; - - // Forward mode configuration - target?: { - server: string; - port?: number; - useTls?: boolean; - authentication?: { - user?: string; - pass?: string; - }; - }; - - // MTA mode configuration - mtaOptions?: { - domain?: string; - allowLocalDelivery?: boolean; - localDeliveryPath?: string; - dkimSign?: boolean; - dkimOptions?: { - domainName: string; - keySelector: string; - privateKey: string; - }; - }; - - // Process mode configuration - contentScanning?: boolean; - scanners?: Array<{ - type: 'spam' | 'virus' | 'attachment'; - threshold?: number; - action: 'tag' | 'reject'; - blockedExtensions?: string[]; - }>; - - transformations?: Array<{ - type: string; - [key: string]: any; - }>; - - // Rate limits for this domain - rateLimits?: { - maxMessagesPerMinute?: number; - maxRecipientsPerMessage?: number; - }; - }>; - - // Queue configuration for all email processing - queue?: { - storageType?: 'memory' | 'disk'; - persistentPath?: string; - maxRetries?: number; - baseRetryDelay?: number; - maxRetryDelay?: number; - }; - - // Advanced MTA settings - mtaGlobalOptions?: { - smtpBanner?: string; - maxConnections?: number; - connTimeout?: number; - spoolDir?: string; - dkimKeyPath?: string; - }; - }; -} - -## 1. Core Architecture for Consolidated Email Processing - -### 1.1 Unified Email Server -- [x] Create a unified email server component - - Build on existing SmtpServer class but with enhanced routing capabilities - - Configure to listen on standard ports (25, 587, 465) for all email handling - - Implement TLS support (STARTTLS and implicit TLS) - - Add support for authentication methods (PLAIN, LOGIN, OAUTH2) - - Set up size limits and connection timeouts - -### 1.2 Pattern-Based Domain Router -- [x] Create pattern matching system for email domains - - Implement glob pattern matching for email addresses - - Support patterns like `*@domain.com`, `*@*.domain.com` - - Create priority-based matching system (most specific match wins) - - Build rule evaluation engine to determine processing mode - - Implement a fast lookup mechanism for incoming emails - -### 1.3 Multi-Modal Processing System -- [x] Create a unified processing system with multiple modes - - Forward mode: SMTP proxy functionality with enhanced routing - - MTA mode: Programmatic email handling with local delivery options - - Process mode: Full store-and-forward pipeline with content scanning - - Implement mode-specific configuration and handlers - - Create fallback handling for unmatched domains - -### 1.4 Shared Infrastructure -- [x] Develop shared components across all email handling modes - - [x] Create unified delivery queue for all outbound email - - [x] Implement shared authentication system - - [x] Build common TLS and certificate management - - [x] Create uniform logging and metrics collection - - [x] Develop shared rate limiting and throttling - -## 2. Consolidated Email Processing Features - -### 2.1 Pattern-Based Routing -- [x] Implement glob pattern-based email routing - - Create glob pattern matching for both domains and full email addresses - - Support wildcards for domains, subdomains, and local parts (e.g., `*@domain.com`, `user@*.domain.com`) - - Add support for pattern matching priorities (most specific wins) - - Implement cacheable pattern matching for performance - - Create comprehensive test suite for pattern matching - -### 2.2 Multi-Modal Processing -- [x] Develop multiple email handling modes - - Forward mode: Simple SMTP forwarding to another server with enhanced routing - - MTA mode: Process with the MTA for programmatic handling and local delivery - - Process mode: Full store-and-forward processing with content scanning - - Add mode-specific configuration validation - - Implement seamless mode transitions based on patterns - -### 2.3 Content Inspection and Transformation -- [x] Enhance content inspection for processing mode - - [x] Improve MIME parsing and content extraction capabilities - - [x] Enhance attachment scanning and filtering - - [x] Add text analysis for spam and phishing detection - - [x] Create more robust transformation framework - - [x] Support content-based routing decisions - -### 2.4 Unified Rate Limiting and Traffic Control -- [x] Build unified rate limiting across all modes - - [x] Implement pattern-based rate limits - - [x] Create hierarchical rate limiting (global, pattern, IP) - - [x] Add real-time rate limit monitoring - - [x] Develop traffic shaping capabilities - - [x] Implement backpressure mechanisms for overload protection - -## 3. DcRouter Integration - -### 3.1 Unified Configuration Interface -- [x] Implement the consolidated emailConfig interface - - Create the IEmailConfig interface with all required components - - Replace existing SMTP, forwarding, and MTA configs with unified approach - - Add backward compatibility layer for existing configurations - - Provide comprehensive validation for the new configuration format - - Add clear documentation and examples in code comments - -### 3.2 Enhanced Management API -- [x] Develop enhanced management API for consolidated email handling - - Create unified status reporting across all modes - - Implement pattern-based rule management (add, update, remove) - - Add comprehensive queue management across all modes - - Create mode-specific monitoring endpoints - - Implement enhanced configuration update methods - -### 3.3 Unified Metrics and Logging -- [x] Create a unified metrics system for all email handling - - Develop pattern-based metrics collection - - Implement mode-specific performance metrics - - Create pattern rule effectiveness measurements - - Add real-time monitoring capabilities - - Design comprehensive logging with correlation IDs - -## 4. Detailed Component Specifications - -### 4.0 DcRouter Configuration Extension - -```typescript -export interface IDcRouterOptions { - // Core configuration options - - // Direct SmartProxy configuration - gives full control over all TCP/SNI traffic - // including HTTP, HTTPS, and any other TCP-based protocol - smartProxyConfig?: plugins.smartproxy.ISmartProxyOptions; - - // Unified email configuration for all email handling modes - emailConfig?: IEmailConfig; - - // Shared configurations - tls?: { - contactEmail: string; - domain?: string; - certPath?: string; - keyPath?: string; - }; - - // Other DcRouter options - dnsServerConfig?: plugins.smartdns.IDnsServerOptions; -} - -/** - * Consolidated email configuration interface - */ -export interface IEmailConfig { - // Email server settings - ports: number[]; - hostname: string; - maxMessageSize?: number; - - // TLS configuration for email server - tls?: { - certPath?: string; - keyPath?: string; - caPath?: string; - minVersion?: string; - }; - - // Authentication for inbound connections - auth?: { - required?: boolean; - methods?: ('PLAIN' | 'LOGIN' | 'OAUTH2')[]; - users?: Array<{username: string, password: string}>; - }; - - // Default routing for unmatched domains - defaultMode: EmailProcessingMode; - defaultServer?: string; - defaultPort?: number; - defaultTls?: boolean; - - // Domain rules with glob pattern support - domainRules: IDomainRule[]; - - // Queue configuration for all email processing - queue?: { - storageType?: 'memory' | 'disk'; - persistentPath?: string; - maxRetries?: number; - baseRetryDelay?: number; - maxRetryDelay?: number; - }; - - // Advanced MTA settings - mtaGlobalOptions?: IMtaOptions; -} - -/** - * Email processing modes - */ -export type EmailProcessingMode = 'forward' | 'mta' | 'process'; - -/** - * Domain rule interface for pattern-based routing - */ -export interface IDomainRule { - // Domain pattern (e.g., "*@example.com", "*@*.example.net") - pattern: string; - - // Handling mode for this pattern - mode: EmailProcessingMode; - - // Forward mode configuration - target?: { - server: string; - port?: number; - useTls?: boolean; - authentication?: { - user?: string; - pass?: string; - }; - }; - - // MTA mode configuration - mtaOptions?: IMtaOptions; - - // Process mode configuration - contentScanning?: boolean; - scanners?: IContentScanner[]; - transformations?: ITransformation[]; - - // Rate limits for this domain - rateLimits?: { - maxMessagesPerMinute?: number; - maxRecipientsPerMessage?: number; - }; -} - -/** - * MTA options interface - */ -export interface IMtaOptions { - domain?: string; - allowLocalDelivery?: boolean; - localDeliveryPath?: string; - dkimSign?: boolean; - dkimOptions?: { - domainName: string; - keySelector: string; - privateKey: string; - }; - smtpBanner?: string; - maxConnections?: number; - connTimeout?: number; - spoolDir?: string; -} - -/** - * Content scanner interface - */ -export interface IContentScanner { - type: 'spam' | 'virus' | 'attachment'; - threshold?: number; - action: 'tag' | 'reject'; - blockedExtensions?: string[]; -} - -/** - * Transformation interface - */ -export interface ITransformation { - type: string; - [key: string]: any; -} -``` - -### 4.1 UnifiedEmailServer Class - -```typescript -/** - * Options for the unified email server - */ -export interface IUnifiedEmailServerOptions { - // Base server options - ports: number[]; - hostname: string; - banner?: string; - - // Authentication options - auth?: { - required?: boolean; - methods?: ('PLAIN' | 'LOGIN' | 'OAUTH2')[]; - users?: Array<{username: string, password: string}>; - }; - - // TLS options - tls?: { - certPath?: string; - keyPath?: string; - caPath?: string; - minVersion?: string; - ciphers?: string; - }; - - // Limits - maxMessageSize?: number; - maxClients?: number; - maxConnections?: number; - - // Connection options - connectionTimeout?: number; - socketTimeout?: number; - - // Domain rules - domainRules: IDomainRule[]; - - // Default handling for unmatched domains - defaultMode: EmailProcessingMode; - defaultServer?: string; - defaultPort?: number; - defaultTls?: boolean; -} - -/** - * Unified email server that handles all email traffic with pattern-based routing - */ -export class UnifiedEmailServer extends EventEmitter { - constructor(options: IUnifiedEmailServerOptions); - - // Start and stop the server - public start(): Promise; - public stop(): Promise; - - // Core event handlers - private onConnect(session: ISmtpSession, callback: (err?: Error) => void): void; - private onAuth(auth: IAuthData, session: ISmtpSession, callback: (err?: Error, user?: any) => void): void; - private onMailFrom(address: {address: string}, session: ISmtpSession, callback: (err?: Error) => void): void; - private onRcptTo(address: {address: string}, session: ISmtpSession, callback: (err?: Error) => void): void; - private onData(stream: Readable, session: ISmtpSession, callback: (err?: Error) => void): void; - - // Pattern matching and routing - private matchDomainRule(address: string): IDomainRule | null; - private determineProcessingMode(session: ISmtpSession): EmailProcessingMode; - - // Mode-specific processing - private handleForwardMode(message: any, session: ISmtpSession, rule: IDomainRule): Promise; - private handleMtaMode(message: any, session: ISmtpSession, rule: IDomainRule): Promise; - private handleProcessMode(message: any, session: ISmtpSession, rule: IDomainRule): Promise; - - // Helper methods - private parseEmail(rawData: string): Promise; - private isIpAllowed(ip: string, rule: IDomainRule): boolean; - private createDeliveryJob(message: any, rule: IDomainRule): IDeliveryJob; - - // Configuration updates - public updateOptions(options: Partial): void; - public updateDomainRules(rules: IDomainRule[]): void; - - // Server stats - public getStats(): IServerStats; -} -``` - -### 4.2 DomainRouter Class - -```typescript -/** - * Options for the domain-based router - */ -export interface IDomainRouterOptions { - // Domain rules with glob pattern matching - domainRules: IDomainRule[]; - - // Default handling for unmatched domains - defaultMode: EmailProcessingMode; - defaultServer?: string; - defaultPort?: number; - defaultTls?: boolean; - - // Pattern matching options - caseSensitive?: boolean; - priorityOrder?: 'most-specific' | 'first-match'; - - // Cache settings for pattern matching - enableCache?: boolean; - cacheSize?: number; -} - -/** - * A pattern matching and routing class for email domains - */ -export class DomainRouter { - constructor(options: IDomainRouterOptions); - - /** - * Match an email address against defined rules - * @param email Email address to match - * @returns The matching rule or null if no match - */ - public matchRule(email: string): IDomainRule | null; - - /** - * Check if email matches a specific pattern - * @param email Email address to check - * @param pattern Pattern to check against - * @returns True if matching, false otherwise - */ - public matchesPattern(email: string, pattern: string): boolean; - - /** - * Get all rules that match an email address - * @param email Email address to match - * @returns Array of matching rules - */ - public getAllMatchingRules(email: string): IDomainRule[]; - - /** - * Add a new routing rule - * @param rule Domain rule to add - */ - public addRule(rule: IDomainRule): void; - - /** - * Update an existing rule - * @param pattern Pattern to update - * @param updates Updates to apply - */ - public updateRule(pattern: string, updates: Partial): boolean; - - /** - * Remove a rule - * @param pattern Pattern to remove - */ - public removeRule(pattern: string): boolean; - - /** - * Get rule by pattern - * @param pattern Pattern to find - */ - public getRule(pattern: string): IDomainRule | null; - - /** - * Get all rules - */ - public getRules(): IDomainRule[]; - - /** - * Update options - * @param options New options - */ - public updateOptions(options: Partial): void; - - /** - * Clear pattern matching cache - */ - public clearCache(): void; -} -``` - -### 4.3 MultiModeProcessor Class - -```typescript -/** - * Processing modes - */ -export type EmailProcessingMode = 'forward' | 'mta' | 'process'; - -/** - * Processor options - */ -export interface IMultiModeProcessorOptions { - // Processing options - maxParallelProcessing?: number; - processingTimeout?: number; - - // Mode handlers - forwardHandler?: IForwardHandler; - mtaHandler?: IMtaHandler; - processHandler?: IProcessHandler; - - // Queue configuration - queue?: { - storageType?: 'memory' | 'disk'; - persistentPath?: string; - maxRetries?: number; - baseRetryDelay?: number; - maxRetryDelay?: number; - }; - - // Shared services - sharedServices?: { - dkimSigner?: IDkimSigner; - contentScanner?: IContentScanner; - rateLimiter?: IRateLimiter; - }; -} - -/** - * Processes emails using different modes based on domain rules - */ -export class MultiModeProcessor extends EventEmitter { - constructor(options: IMultiModeProcessorOptions); - - /** - * Process an email using the appropriate mode - * @param message Parsed email message - * @param session SMTP session - * @param rule Matching domain rule - * @param mode Processing mode - */ - public async processEmail( - message: any, - session: ISmtpSession, - rule: IDomainRule, - mode: EmailProcessingMode - ): Promise; - - /** - * Handle email in forward mode - * @param message Parsed email message - * @param session SMTP session - * @param rule Matching domain rule - */ - private async handleForwardMode( - message: any, - session: ISmtpSession, - rule: IDomainRule - ): Promise; - - /** - * Handle email in MTA mode - * @param message Parsed email message - * @param session SMTP session - * @param rule Matching domain rule - */ - private async handleMtaMode( - message: any, - session: ISmtpSession, - rule: IDomainRule - ): Promise; - - /** - * Handle email in process mode - * @param message Parsed email message - * @param session SMTP session - * @param rule Matching domain rule - */ - private async handleProcessMode( - message: any, - session: ISmtpSession, - rule: IDomainRule - ): Promise; - - /** - * Update processor options - * @param options New options - */ - public updateOptions(options: Partial): void; - - /** - * Get processor statistics - */ - public getStats(): IProcessorStats; -} -``` - -### 4.4 UnifiedDeliveryQueue Class - -```typescript -/** - * Queue item status - */ -export type QueueItemStatus = 'pending' | 'processing' | 'delivered' | 'failed' | 'deferred'; - -/** - * Queue item interface - */ -export interface IQueueItem { - id: string; - processingMode: EmailProcessingMode; - processingResult: any; - rule: IDomainRule; - status: QueueItemStatus; - attempts: number; - nextAttempt: Date; - lastError?: string; - createdAt: Date; - updatedAt: Date; - deliveredAt?: Date; -} - -/** - * Queue options interface - */ -export interface IQueueOptions { - // Storage options - storageType?: 'memory' | 'disk'; - persistentPath?: string; - - // Queue behavior - checkInterval?: number; - maxQueueSize?: number; - maxPerDestination?: number; - - // Delivery attempts - maxRetries?: number; - baseRetryDelay?: number; - maxRetryDelay?: number; -} - -/** - * A unified queue for all email modes - */ -export class UnifiedDeliveryQueue extends EventEmitter { - constructor(options: IQueueOptions); - - /** - * Initialize the queue - */ - public async initialize(): Promise; - - /** - * Add an item to the queue - * @param processingResult Processing result to queue - * @param mode Processing mode - * @param rule Domain rule - */ - public async enqueue(processingResult: any, mode: EmailProcessingMode, rule: IDomainRule): Promise; - - /** - * Get an item from the queue - * @param id Item ID - */ - public getItem(id: string): IQueueItem | undefined; - - /** - * Mark an item as delivered - * @param id Item ID - */ - public async markDelivered(id: string): Promise; - - /** - * Mark an item as failed - * @param id Item ID - * @param error Error message - */ - public async markFailed(id: string, error: string): Promise; - - /** - * Get queue statistics - */ - public getStats(): IQueueStats; - - /** - * Pause queue processing - */ - public pause(): void; - - /** - * Resume queue processing - */ - public resume(): void; - - /** - * Shutdown the queue - */ - public async shutdown(): Promise; -} -``` - -### 4.5 MultiModeDeliverySystem Class - -```typescript -/** - * Delivery options - */ -export interface IMultiModeDeliveryOptions { - // Connection options - connectionPoolSize?: number; - socketTimeout?: number; - - // Delivery behavior - concurrentDeliveries?: number; - sendTimeout?: number; - - // TLS options - verifyCertificates?: boolean; - tlsMinVersion?: string; - - // Mode-specific handlers - forwardHandler?: IForwardDeliveryHandler; - mtaHandler?: IMtaDeliveryHandler; - processHandler?: IProcessDeliveryHandler; - - // Rate limiting - globalRateLimit?: number; - perPatternRateLimit?: Record; - - // Event hooks - onDeliveryStart?: (item: IQueueItem) => Promise; - onDeliverySuccess?: (item: IQueueItem, result: any) => Promise; - onDeliveryFailed?: (item: IQueueItem, error: string) => Promise; -} - -/** - * Handles delivery for all email processing modes - */ -export class MultiModeDeliverySystem extends EventEmitter { - constructor(queue: UnifiedDeliveryQueue, options: IMultiModeDeliveryOptions); - - /** - * Start the delivery system - */ - public async start(): Promise; - - /** - * Stop the delivery system - */ - public async stop(): Promise; - - /** - * Deliver an item from the queue - * @param item Queue item to deliver - */ - private async deliverItem(item: IQueueItem): Promise; - - /** - * Handle delivery in forward mode - * @param item Queue item - */ - private async handleForwardDelivery(item: IQueueItem): Promise; - - /** - * Handle delivery in MTA mode - * @param item Queue item - */ - private async handleMtaDelivery(item: IQueueItem): Promise; - - /** - * Handle delivery in process mode - * @param item Queue item - */ - private async handleProcessDelivery(item: IQueueItem): Promise; - - /** - * Update delivery options - * @param options New options - */ - public updateOptions(options: Partial): void; - - /** - * Get delivery statistics - */ - public getStats(): IDeliveryStats; -} -``` - -### 4.6 DcRouter Integration with EmailConfig - -```typescript -/** - * DcRouter options with emailConfig - */ -export interface IDcRouterOptions { - // Direct SmartProxy configuration for general HTTP/HTTPS and TCP/SNI traffic - smartProxyConfig?: plugins.smartproxy.ISmartProxyOptions; - - // Consolidated email configuration - emailConfig?: IEmailConfig; - - // Shared TLS configuration - tls?: { - contactEmail: string; - domain?: string; - certPath?: string; - keyPath?: string; - }; - - // DNS server configuration - dnsServerConfig?: plugins.smartdns.IDnsServerOptions; -} - -/** - * DcRouter with consolidated email handling - */ -export class DcRouter { - // Core services - public smartProxy?: plugins.smartproxy.SmartProxy; - public dnsServer?: plugins.smartdns.DnsServer; - - // Unified email components - public emailServer?: UnifiedEmailServer; - public domainRouter?: DomainRouter; - public multiModeProcessor?: MultiModeProcessor; - public deliveryQueue?: UnifiedDeliveryQueue; - public deliverySystem?: MultiModeDeliverySystem; - - constructor(options: IDcRouterOptions); - - /** - * Start DcRouter services - */ - public async start(): Promise; - - /** - * Stop DcRouter services - */ - public async stop(): Promise; - - /** - * Set up email handling - */ - private async setupEmailHandling(): Promise; - - /** - * Update email configuration - * @param config New email configuration - */ - public async updateEmailConfig(config: IEmailConfig): Promise; - - /** - * Update domain rules - * @param rules New domain rules - */ - public async updateDomainRules(rules: IDomainRule[]): Promise; - - /** - * Get DcRouter statistics - */ - public getStats(): IDcRouterStats; -} -``` - -## 5. Implementation Phases - -### Phase 1: Core Architecture and Pattern Matching -- [x] Create the UnifiedEmailServer class foundation -- [x] Implement the DomainRouter with glob pattern matching -- [x] Build pattern priority system (most specific match first) -- [x] Create pattern caching mechanism for performance -- [x] Implement validation for email patterns -- [x] Build test suite for pattern matching system - -### Phase 2: Multi-Modal Processing Framework -- [x] Build the MultiModeProcessor class -- [x] Implement mode-specific handlers (forward, MTA, process) -- [x] Create processing pipeline for each mode -- [x] Implement content scanning for process mode -- [x] Build shared services infrastructure -- [x] Add validation for mode-specific configurations - -### Phase 3: Unified Queue and Delivery System -- [x] Implement the UnifiedDeliveryQueue -- [x] Create persistent storage for all processing modes -- [x] Build the MultiModeDeliverySystem -- [x] Implement mode-specific delivery handlers -- [x] Create shared retry logic with exponential backoff -- [x] Add delivery tracking and notification - -### Phase 4: DcRouter Integration -- [x] Implement the consolidated emailConfig interface -- [x] Integrate all components into DcRouter -- [x] Add configuration validation -- [x] Create management APIs for updating rules -- [x] Implement migration support for existing configurations -- [x] Build mode-specific metrics and logging - -### Phase 5: Testing and Documentation -- [x] Create comprehensive unit tests for all components -- [x] Implement integration tests for all processing modes -- [x] Test pattern matching with complex scenarios -- [x] Create performance tests for high-volume scenarios -- [x] Build detailed documentation and examples -- [x] Identify and document legacy components to be deprecated (EmailDomainRouter) -- [x] Remove deprecated components (EmailDomainRouter) - -## 6. Technical Requirements - -### 6.1 Dependencies -- SMTP server library (smtp-server or similar) -- Email parsing library (mailparser or similar) -- MIME handling library -- DKIM signing library -- Queue management system (optional Redis support) -- Cryptographic libraries for TLS and authentication - -### 6.2 Performance Targets -- Handle 1000+ concurrent SMTP connections -- Process 100+ messages per second on standard hardware -- Support message sizes up to 50MB -- Maintain delivery queue of 100,000+ messages -- Sub-second processing time for standard emails - -### 6.3 Security Requirements -- Full TLS support with modern cipher configurations -- Authentication verification and rate limiting -- Input validation for all SMTP commands -- Secure storage of queued emails -- Proper error handling to prevent information leakage -- Access controls based on IP addresses and authentication - -## 7. API Examples - -### 7.1 Basic DcRouter SMTP Configuration - -```typescript -const dcRouter = new DcRouter({ - // HTTP configuration... - - smtpConfig: { - ports: [25, 587, 465], - hostname: 'mail.example.com', - maxMessageSize: 50 * 1024 * 1024, // 50MB - - // TLS configuration - tls: { - certPath: '/path/to/cert.pem', - keyPath: '/path/to/key.pem' - }, - - // Domain routing - domainConfigs: [ - { - domains: ['example.com', '*.example.com'], - targetIPs: ['mail1.example.com', 'mail2.example.com'], - port: 25, - useTls: true + action: { + type: 'forward', + target: { + host: 'backend.server.com', + port: 8080 + } + }, + tls: { + mode: 'terminate', + certificate: 'auto' } - ], - - // Default routing - defaultServer: 'fallback-mail.example.com', - defaultPort: 25, - useTls: true, - - // Queue settings - queueStorage: 'disk', - persistentPath: '/var/mail/queue', - maxRetries: 5 + } + ], + defaults: { + target: { + host: 'fallback.server.com', + port: 8080 + }, + security: { + ipAllowList: ['192.168.1.0/24'], + maxConnections: 1000 + } } -}); +} ``` -### 7.2 Advanced Configuration with Processing +## Integration Approach + +### 1. Direct SmartProxy Configuration + +The DcRouter will expose the full SmartProxy configuration directly, with additional email integration that automatically adds email routes to SmartProxy: ```typescript -const dcRouter = new DcRouter({ - // HTTP configuration... +export interface IDcRouterOptions { + /** + * Full SmartProxy configuration - ALL SmartProxy features available + * This handles HTTP/HTTPS and general TCP/SNI traffic + */ + smartProxyConfig?: plugins.smartproxy.ISmartProxyOptions; - smtpConfig: { - // Basic settings - ports: [25, 587, 465], - hostname: 'mail.example.com', - - // Domain routing with advanced features - domainConfigs: [ + /** + * Email configuration - automatically creates SmartProxy routes for email ports + */ + emailConfig?: IEmailConfig; + + /** + * Additional configuration options + */ + tls?: { + contactEmail: string; + domain?: string; + certPath?: string; + keyPath?: string; + }; + + /** + * DNS server configuration + */ + dnsServerConfig?: plugins.smartdns.IDnsServerOptions; +} +``` + +### 2. Email Route Auto-Generation + +When email configuration is provided, DcRouter will automatically generate SmartProxy routes for email traffic: + +```typescript +private async setupSmartProxy() { + let routes: plugins.smartproxy.IRouteConfig[] = []; + let acmeConfig: plugins.smartproxy.IAcmeOptions | undefined; + + // If user provides full SmartProxy config, use it directly + if (this.options.smartProxyConfig) { + routes = this.options.smartProxyConfig.routes || []; + acmeConfig = this.options.smartProxyConfig.acme; + } + + // If email config exists, automatically add email routes + if (this.options.emailConfig) { + const emailRoutes = this.generateEmailRoutes(this.options.emailConfig); + routes = [...routes, ...emailRoutes]; + } + + // Merge TLS/ACME configuration if provided at root level + if (this.options.tls && !acmeConfig) { + acmeConfig = { + accountEmail: this.options.tls.contactEmail, + enabled: true, + useProduction: true, + autoRenew: true, + renewThresholdDays: 30 + }; + } + + // Initialize SmartProxy with combined configuration + this.smartProxy = new plugins.smartproxy.SmartProxy({ + ...this.options.smartProxyConfig, + routes, + acme: acmeConfig + }); + + await this.smartProxy.start(); + logger.info('SmartProxy started with route-based configuration'); +} + +private generateEmailRoutes(emailConfig: IEmailConfig): plugins.smartproxy.IRouteConfig[] { + const emailRoutes: plugins.smartproxy.IRouteConfig[] = []; + + // Create routes for each email port + for (const port of emailConfig.ports) { + // Handle different email ports differently + switch (port) { + case 25: // SMTP + emailRoutes.push({ + name: 'smtp-route', + match: { + ports: [25] + }, + action: { + type: 'forward', + target: { + host: 'localhost', // Forward to internal email server + port: 10025 // Internal email server port + } + }, + // No TLS termination for port 25 (STARTTLS handled by email server) + tls: { + mode: 'passthrough' + } + }); + break; + + case 587: // Submission + emailRoutes.push({ + name: 'submission-route', + match: { + ports: [587] + }, + action: { + type: 'forward', + target: { + host: 'localhost', + port: 10587 + } + }, + tls: { + mode: 'passthrough' // STARTTLS handled by email server + } + }); + break; + + case 465: // SMTPS + emailRoutes.push({ + name: 'smtps-route', + match: { + ports: [465] + }, + action: { + type: 'forward', + target: { + host: 'localhost', + port: 10465 + } + }, + tls: { + mode: 'terminate', // Terminate TLS and re-encrypt to email server + certificate: 'auto' + } + }); + break; + } + } + + // Add domain-specific email routes if configured + if (emailConfig.domainRules) { + for (const rule of emailConfig.domainRules) { + // Extract domain from pattern (e.g., "*@example.com" -> "example.com") + const domain = rule.pattern.split('@')[1]; + + if (domain && rule.mode === 'forward' && rule.target) { + emailRoutes.push({ + name: `email-forward-${domain}`, + match: { + ports: emailConfig.ports, + domains: [domain] + }, + action: { + type: 'forward', + target: { + host: rule.target.server, + port: rule.target.port || 25 + } + }, + tls: { + mode: rule.target.useTls ? 'terminate-and-reencrypt' : 'passthrough' + } + }); + } + } + } + + return emailRoutes; +} +``` + +### 3. Email Server Integration + +The email server will run on internal ports and receive traffic from SmartProxy: + +```typescript +private async setupUnifiedEmailHandling() { + if (!this.options.emailConfig) { + logger.info('No email configuration provided'); + return; + } + + const emailConfig = this.options.emailConfig; + + // Map external ports to internal ports + const portMapping = { + 25: 10025, // SMTP + 587: 10587, // Submission + 465: 10465 // SMTPS + }; + + // Create internal email server configuration + const internalEmailConfig: IEmailConfig = { + ...emailConfig, + ports: emailConfig.ports.map(port => portMapping[port] || port + 10000), + hostname: 'localhost' // Listen on localhost for SmartProxy forwarding + }; + + // Initialize email components with internal configuration + this.domainRouter = new DomainRouter({ + domainRules: emailConfig.domainRules, + defaultMode: emailConfig.defaultMode, + defaultServer: emailConfig.defaultServer, + defaultPort: emailConfig.defaultPort, + defaultTls: emailConfig.defaultTls + }); + + this.unifiedEmailServer = new UnifiedEmailServer({ + ...internalEmailConfig, + domainRouter: this.domainRouter + }); + + await this.unifiedEmailServer.start(); + logger.info('Unified email server started on internal ports'); +} +``` + +## Usage Examples + +### Example 1: Combined HTTP and Email Configuration +```typescript +const dcRouter = new DcRouter({ + // Full SmartProxy configuration for HTTP/HTTPS + smartProxyConfig: { + routes: [ { - domains: ['example.com', '*.example.com'], - targetIPs: ['mail1.example.com', 'mail2.example.com'], - port: 25, - useTls: true, - // Add custom headers - addHeaders: true, - headerInfo: [ - { name: 'X-Processed-By', value: 'gateway' }, - { name: 'X-Scanned', value: 'true' } - ], - // Sign with DKIM - signDkim: true, - dkimOptions: { - domainName: 'example.com', - keySelector: 'mail', - privateKey: '...' + name: 'web-traffic', + match: { + ports: [80, 443], + domains: ['www.example.com'] }, - // Rate limiting - rateLimits: { - maxMessagesPerMinute: 100, - maxRecipientsPerMessage: 50 + action: { + type: 'forward', + target: { + host: 'web-backend.example.com', + port: 8080 + } + }, + tls: { + mode: 'terminate', + certificate: 'auto' } } ], - - // Content scanning - contentScanning: true, - scanners: [ + acme: { + accountEmail: 'admin@example.com', + enabled: true, + useProduction: true + } + }, + + // Email configuration - automatically creates SmartProxy routes + emailConfig: { + ports: [25, 587, 465], + hostname: 'mail.example.com', + domainRules: [ { - type: 'spam', - threshold: 5.0, - action: 'tag' - }, - { - type: 'virus', - action: 'reject' - }, - { - type: 'attachment', - blockedExtensions: ['.exe', '.bat', '.vbs'], - action: 'reject' + pattern: '*@example.com', + mode: 'mta', + mtaOptions: { + dkimSign: true, + dkimOptions: { + domainName: 'example.com', + keySelector: 'mail', + privateKey: '...' + } + } } ], - - // Transformations - transformations: [ - { - type: 'addHeader', - header: 'X-Gateway', - value: 'DcRouter 1.0' - }, - { - type: 'dkimSign', - domains: ['example.com'] - } - ] + defaultMode: 'forward', + defaultServer: 'backup-mail.example.com' } }); ``` -## 8. Extensibility Points +### Example 2: Advanced SmartProxy Features with Email +```typescript +const dcRouter = new DcRouter({ + smartProxyConfig: { + routes: [ + // Custom API route with authentication + { + name: 'api-route', + match: { + ports: 443, + domains: 'api.example.com', + path: '/v1/*' + }, + action: { + type: 'forward', + target: { + host: (context) => { + // Dynamic host selection based on context + return context.headers['x-api-version'] === 'v2' + ? 'api-v2.backend.com' + : 'api-v1.backend.com'; + }, + port: 9090 + } + }, + security: { + authentication: { + type: 'jwt', + jwtSecret: process.env.JWT_SECRET + }, + rateLimit: { + maxRequestsPerMinute: 100, + maxRequestsPerSecond: 10 + } + } + } + ], + // Advanced SmartProxy options + preserveSourceIP: true, + enableDetailedLogging: true, + maxConnectionsPerIP: 50, + connectionRateLimitPerMinute: 1000 + }, + + // Email automatically integrates with SmartProxy + emailConfig: { + ports: [25, 587, 465], + hostname: 'mail.example.com', + domainRules: [] + } +}); +``` -### 8.1 Plugin Architecture -- Custom content scanners -- Custom transformation handlers -- Routing rule extensions -- Authentication providers -- Queue storage backends +## Key Benefits -### 8.2 Event System -- Connection events (connect, disconnect, error) -- Message events (received, processed, queued, delivered) -- Error events (delivery failure, processing error) -- Performance events (queue size, processing time) -- Security events (authentication failure, policy violation) +1. **Full SmartProxy Power**: All SmartProxy features remain available +2. **Automatic Email Integration**: Email ports are automatically routed through SmartProxy +3. **Unified TLS Management**: SmartProxy handles all TLS termination and certificates +4. **Flexible Configuration**: Mix custom SmartProxy routes with automatic email routes +5. **Performance**: SmartProxy's efficient routing benefits email traffic too -## 9. Migration Plan +## Implementation Tasks -### 9.1 From Simple Proxy to Store-and-Forward -- [x] Create compatibility layer for existing configurations -- [x] Implement graceful transition from connection proxy to full processing -- [x] Add configuration validation to ensure smooth migration -- [x] Allow components to coexist for flexible deployment options -- [x] Provide documentation with comments for migrating existing deployments -- [x] Remove deprecated files after migration to consolidated approach - - [x] Removed EmailDomainRouter class - - [x] Updated imports and references to use the new DomainRouter +- [ ] Update DcRouter to preserve full SmartProxy configuration +- [ ] Implement automatic email route generation +- [ ] Configure email server to run on internal ports +- [ ] Test integration between SmartProxy and email server +- [ ] Update documentation with full SmartProxy examples +- [ ] Create migration guide showing how to use new features +- [ ] Test advanced SmartProxy features with email routing +- [ ] Verify TLS handling for different email ports +- [ ] Test domain-specific email routing through SmartProxy -### 9.2 Backward Compatibility -- [x] Maintain support for basic proxy functionality -- [x] Allow MTA, SMTP forwarding, and store-and-forward to work together -- [x] Support multiple concurrent email handling approaches -- [x] Enable hybrid deployments with different services running simultaneously +## Notes -### 9.3 Enhanced Coexistence Support -- [x] Modified the startup sequence to enable concurrent operation of multiple services: - - MTA service can now run alongside SMTP forwarding and store-and-forward processing - - Store-and-forward SMTP processing can run alongside MTA and SMTP forwarding - - SMTP forwarding now uses its own dedicated SmartProxy instance (smtpProxy) -- [x] Updated component lifecycle management to properly start/stop all services -- [x] Added clear separation between service instances to avoid conflicts -- [x] Ensured configuration updates for one component don't affect others - -## 10. SmartProxy Integration - -### 10.1 SmartProxy Configuration Handling -- [x] Implement comprehensive support for SmartProxy configuration - - Passed through all SmartProxy options directly in DcRouter's configuration - - Added support for all SmartProxy domain configuration features - - Implemented proper handling of SmartProxy events and callbacks -- [x] Added documentation on SmartProxy configuration: - - Documented how all SmartProxy features are available through DcRouter - - Added examples of different configuration approaches - - Provided guidance in code comments - -### 10.2 SMTP Integration with SmartProxy -- [x] Ensured store-and-forward SMTP works alongside SmartProxy - - Handled SMTP ports separately from HTTP/HTTPS ports - - Prevented port conflicts between SmartProxy and SMTP server - - Created code structure showing SmartProxy and SMTP working together -- [x] Implemented combined usage model: - - HTTP/HTTPS traffic using SmartProxy configuration - - SMTP traffic using store-and-forward for advanced processing - - Added support for multi-service environments - -## 11. Documentation Requirements - -### 11.1 Code Documentation -- [x] Comprehensive JSDoc comments for all classes and methods -- [x] Interface definitions with detailed parameter descriptions -- [x] Example code snippets for common operations -- [x] Architecture documentation with component diagrams -- [x] Decision logs for key design choices - -### 11.2 User Documentation -- [x] Getting started guide with configuration approach selection guidance -- [x] Complete configuration reference for both approaches -- [x] Deployment scenarios and examples -- [x] Troubleshooting guide -- [x] Performance tuning recommendations -- [x] Security best practices - -### 11.3 Direct SmartProxy Configuration Guide -- [x] Detailed guide on using SmartProxy's domain configuration capabilities -- [x] Examples of complex routing scenarios with SmartProxy -- [x] Performance optimization tips for SmartProxy configurations -- [x] Security settings for SmartProxy deployments \ No newline at end of file +- SmartProxy acts as the single entry point for all traffic (HTTP, HTTPS, and email) +- Email server runs on internal ports and receives forwarded traffic from SmartProxy +- TLS termination is handled by SmartProxy where appropriate +- STARTTLS for email is handled by the email server itself +- All SmartProxy features (rate limiting, authentication, dynamic routing) are available \ No newline at end of file diff --git a/readme.smartlog.md b/readme.smartlog.md deleted file mode 100644 index ffb2eb0..0000000 --- a/readme.smartlog.md +++ /dev/null @@ -1,107 +0,0 @@ -# Smartlog Improvement Plan - -## Overview - -This document outlines a plan for enhancing the `@push.rocks/smartlog` module to incorporate the advanced features currently implemented in the custom `EnhancedLogger` wrapper. By moving these features directly into `smartlog`, we can eliminate the need for wrapper classes while providing a more comprehensive logging solution. - -## Current Limitations in Smartlog - -- Limited context management (no hierarchical contexts) -- No correlation ID tracking for distributed tracing -- No built-in filtering or log level management -- No log sampling capabilities -- No middleware for HTTP request/response logging -- No timing utilities for performance tracking -- No child logger functionality with context inheritance - -## Proposed Enhancements - -### 1. Context Management - -- Add hierarchical context support -- Implement methods for manipulating context: - - `setContext(context, overwrite = false)` - - `addToContext(key, value)` - - `removeFromContext(key)` - -### 2. Correlation ID Tracking - -- Add correlation ID support for distributed tracing -- Implement methods for correlation management: - - `setCorrelationId(id = null)` - - `getCorrelationId()` - - `clearCorrelationId()` - -### 3. Log Filtering - -- Implement configurable log filtering based on: - - Minimum log level - - Pattern-based exclusion rules - - Custom filtering functions - -### 4. Log Sampling - -- Add probabilistic log sampling for high-volume environments -- Support for enforcing critical logs (e.g., errors) regardless of sampling - -### 5. Child Loggers - -- Support creating child loggers with inherited context -- Allow context overrides in child loggers - -### 6. Timing Utilities - -- Add methods for timing operations: - - `logTimed(level, message, fn, context)` - - Support for both async and sync operations - -### 7. HTTP Request Logging - -- Add middleware for Express/Fastify/other HTTP frameworks -- Auto-capture request/response data -- Auto-propagate correlation IDs - -### 8. Log Standardization - -- Ensure consistent output format -- Add standard fields like timestamp, correlation ID -- Support for custom formatters - -## Implementation Plan - -1. **Core Enhancements** - - Implement context management - - Add correlation ID tracking - - Develop filtering and sampling capabilities - -2. **Extended Features** - - Build child logger functionality - - Create timing utility methods - - Implement HTTP middleware - -3. **Compatibility** - - Ensure backward compatibility - - Provide migration guide - - Add TypeScript declarations - -4. **Documentation** - - Update README with new features - - Add examples for each feature - - Document best practices - -## Migration Path - -After implementing these enhancements to `smartlog`, the migration would involve: - -1. Update to latest `smartlog` version -2. Replace `EnhancedLogger` instances with `smartlog.Smartlog` -3. Update configuration to use new capabilities -4. Replace middleware with `smartlog`'s built-in solutions - -## Benefits - -- Simplified dependency tree -- Better maintainability with single logging solution -- Improved performance with native implementation -- Enhanced type safety through TypeScript -- Standardized logging across projects \ No newline at end of file diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index 2800857..49ce47d 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@serve.zone/platformservice', - version: '2.11.2', + version: '2.12.0', description: 'A multifaceted platform service handling mail, SMS, letter delivery, and AI services.' } diff --git a/ts/classes.dcrouter.ts b/ts/classes.dcrouter.ts index 7f0cec1..e6972d3 100644 --- a/ts/classes.dcrouter.ts +++ b/ts/classes.dcrouter.ts @@ -1,6 +1,5 @@ import * as plugins from './plugins.js'; import * as paths from './paths.js'; -import { SmtpPortConfig, type ISmtpPortSettings } from './classes.smtp.portconfig.js'; // Certificate types are available via plugins.tsclass @@ -53,7 +52,7 @@ export interface IDcRouterOptions { */ export interface PortProxyRuleContext { proxy: plugins.smartproxy.SmartProxy; - configs: plugins.smartproxy.IPortProxySettings['domainConfigs']; + routes: plugins.smartproxy.IRouteConfig[]; } export class DcRouter { @@ -84,17 +83,15 @@ export class DcRouter { console.log('Starting DcRouter services...'); try { - // Set up SmartProxy for HTTP/HTTPS and general TCP/SNI traffic - if (this.options.smartProxyConfig) { - await this.setupSmartProxy(); - } + // Set up SmartProxy for HTTP/HTTPS and all traffic including email routes + await this.setupSmartProxy(); // Set up unified email handling if configured if (this.options.emailConfig) { await this.setupUnifiedEmailHandling(); } - // 3. Set up DNS server if configured + // Set up DNS server if configured if (this.options.dnsServerConfig) { this.dnsServer = new plugins.smartdns.DnsServer(this.options.dnsServerConfig); await this.dnsServer.start(); @@ -111,41 +108,175 @@ export class DcRouter { } /** - * Set up SmartProxy with direct configuration + * Set up SmartProxy with direct configuration and automatic email routes */ private async setupSmartProxy(): Promise { - if (!this.options.smartProxyConfig) { - return; + let routes: plugins.smartproxy.IRouteConfig[] = []; + let acmeConfig: plugins.smartproxy.IAcmeOptions | undefined; + + // If user provides full SmartProxy config, use it directly + if (this.options.smartProxyConfig) { + routes = this.options.smartProxyConfig.routes || []; + acmeConfig = this.options.smartProxyConfig.acme; } - console.log('Setting up SmartProxy with direct configuration'); + // If email config exists, automatically add email routes + if (this.options.emailConfig) { + const emailRoutes = this.generateEmailRoutes(this.options.emailConfig); + routes = [...routes, ...emailRoutes]; + } - // Create SmartProxy instance with full configuration - this.smartProxy = new plugins.smartproxy.SmartProxy(this.options.smartProxyConfig); + // Merge TLS/ACME configuration if provided at root level + if (this.options.tls && !acmeConfig) { + acmeConfig = { + accountEmail: this.options.tls.contactEmail, + enabled: true, + useProduction: true, + autoRenew: true, + renewThresholdDays: 30 + }; + } - // Set up event listeners - this.smartProxy.on('error', (err) => { - console.error('SmartProxy error:', err); - }); - - if (this.options.smartProxyConfig.acme) { - this.smartProxy.on('certificate-issued', (event) => { - console.log(`Certificate issued for ${event.domain}, expires ${event.expiryDate}`); + // If we have routes or need a basic SmartProxy instance, create it + if (routes.length > 0 || this.options.smartProxyConfig) { + console.log('Setting up SmartProxy with combined configuration'); + + // Create SmartProxy configuration + const smartProxyConfig: plugins.smartproxy.ISmartProxyOptions = { + ...this.options.smartProxyConfig, + routes, + acme: acmeConfig + }; + + // Create SmartProxy instance + this.smartProxy = new plugins.smartproxy.SmartProxy(smartProxyConfig); + + // Set up event listeners + this.smartProxy.on('error', (err) => { + console.error('SmartProxy error:', err); }); - this.smartProxy.on('certificate-renewed', (event) => { - console.log(`Certificate renewed for ${event.domain}, expires ${event.expiryDate}`); - }); + if (acmeConfig) { + this.smartProxy.on('certificate-issued', (event) => { + console.log(`Certificate issued for ${event.domain}, expires ${event.expiryDate}`); + }); + + this.smartProxy.on('certificate-renewed', (event) => { + console.log(`Certificate renewed for ${event.domain}, expires ${event.expiryDate}`); + }); + } + + // Start SmartProxy + await this.smartProxy.start(); + + console.log(`SmartProxy started with ${routes.length} routes`); } - - // Start SmartProxy - await this.smartProxy.start(); - - console.log('SmartProxy started successfully'); } + /** + * Generate SmartProxy routes for email configuration + */ + private generateEmailRoutes(emailConfig: IEmailConfig): plugins.smartproxy.IRouteConfig[] { + const emailRoutes: plugins.smartproxy.IRouteConfig[] = []; + + // Create routes for each email port + for (const port of emailConfig.ports) { + // Handle different email ports differently + switch (port) { + case 25: // SMTP + emailRoutes.push({ + name: 'smtp-route', + match: { + ports: [25] + }, + action: { + type: 'forward', + target: { + host: 'localhost', // Forward to internal email server + port: 10025 // Internal email server port + }, + // No TLS termination for port 25 (STARTTLS handled by email server) + tls: { + mode: 'passthrough' + } + } + }); + break; + + case 587: // Submission + emailRoutes.push({ + name: 'submission-route', + match: { + ports: [587] + }, + action: { + type: 'forward', + target: { + host: 'localhost', + port: 10587 + }, + tls: { + mode: 'passthrough' // STARTTLS handled by email server + } + } + }); + break; + + case 465: // SMTPS + emailRoutes.push({ + name: 'smtps-route', + match: { + ports: [465] + }, + action: { + type: 'forward', + target: { + host: 'localhost', + port: 10465 + }, + tls: { + mode: 'terminate', // Terminate TLS and re-encrypt to email server + certificate: 'auto' + } + } + }); + break; + } + } + + // Add domain-specific email routes if configured + if (emailConfig.domainRules) { + for (const rule of emailConfig.domainRules) { + // Extract domain from pattern (e.g., "*@example.com" -> "example.com") + const domain = rule.pattern.split('@')[1]; + + if (domain && rule.mode === 'forward' && rule.target) { + emailRoutes.push({ + name: `email-forward-${domain}`, + match: { + ports: emailConfig.ports, + domains: [domain] + }, + action: { + type: 'forward', + target: { + host: rule.target.server, + port: rule.target.port || 25 + }, + tls: { + mode: rule.target.useTls ? 'terminate-and-reencrypt' : 'passthrough' + } + } + }); + } + } + } + + return emailRoutes; + } + /** * Check if a domain matches a pattern (including wildcard support) * @param domain The domain to check @@ -213,7 +344,7 @@ export class DcRouter { // Update configuration this.options.smartProxyConfig = config; - // Start new SmartProxy with updated configuration + // Start new SmartProxy with updated configuration (will include email routes if configured) await this.setupSmartProxy(); console.log('SmartProxy configuration updated'); @@ -231,15 +362,31 @@ export class DcRouter { if (!this.options.emailConfig) { throw new Error('Email configuration is required for unified email handling'); } + + const emailConfig = this.options.emailConfig; + + // Map external ports to internal ports + const portMapping = { + 25: 10025, // SMTP + 587: 10587, // Submission + 465: 10465 // SMTPS + }; + + // Create internal email server configuration + const internalEmailConfig: IEmailConfig = { + ...emailConfig, + ports: emailConfig.ports.map(port => portMapping[port] || port + 10000), + hostname: 'localhost' // Listen on localhost for SmartProxy forwarding + }; try { // Create domain router for pattern matching this.domainRouter = new DomainRouter({ - domainRules: this.options.emailConfig.domainRules, - defaultMode: this.options.emailConfig.defaultMode, - defaultServer: this.options.emailConfig.defaultServer, - defaultPort: this.options.emailConfig.defaultPort, - defaultTls: this.options.emailConfig.defaultTls + domainRules: emailConfig.domainRules, + defaultMode: emailConfig.defaultMode, + defaultServer: emailConfig.defaultServer, + defaultPort: emailConfig.defaultPort, + defaultTls: emailConfig.defaultTls }); // Initialize the rate limiter @@ -255,11 +402,11 @@ export class DcRouter { // Initialize the unified delivery queue const queueOptions: IQueueOptions = { - storageType: this.options.emailConfig.queue?.storageType || 'memory', - persistentPath: this.options.emailConfig.queue?.persistentPath, - maxRetries: this.options.emailConfig.queue?.maxRetries, - baseRetryDelay: this.options.emailConfig.queue?.baseRetryDelay, - maxRetryDelay: this.options.emailConfig.queue?.maxRetryDelay + storageType: emailConfig.queue?.storageType || 'memory', + persistentPath: emailConfig.queue?.persistentPath, + maxRetries: emailConfig.queue?.maxRetries, + baseRetryDelay: emailConfig.queue?.baseRetryDelay, + maxRetryDelay: emailConfig.queue?.maxRetryDelay }; this.deliveryQueue = new UnifiedDeliveryQueue(queueOptions); @@ -274,18 +421,18 @@ export class DcRouter { this.deliverySystem = new MultiModeDeliverySystem(this.deliveryQueue, deliveryOptions); await this.deliverySystem.start(); - // Initialize the unified email server + // Initialize the unified email server with internal configuration this.unifiedEmailServer = new UnifiedEmailServer({ - ports: this.options.emailConfig.ports, - hostname: this.options.emailConfig.hostname, - maxMessageSize: this.options.emailConfig.maxMessageSize, - auth: this.options.emailConfig.auth, - tls: this.options.emailConfig.tls, - domainRules: this.options.emailConfig.domainRules, - defaultMode: this.options.emailConfig.defaultMode, - defaultServer: this.options.emailConfig.defaultServer, - defaultPort: this.options.emailConfig.defaultPort, - defaultTls: this.options.emailConfig.defaultTls + ports: internalEmailConfig.ports, + hostname: internalEmailConfig.hostname, + maxMessageSize: emailConfig.maxMessageSize, + auth: emailConfig.auth, + tls: emailConfig.tls, + domainRules: emailConfig.domainRules, + defaultMode: emailConfig.defaultMode, + defaultServer: emailConfig.defaultServer, + defaultPort: emailConfig.defaultPort, + defaultTls: emailConfig.defaultTls }); // Set up event listeners @@ -303,7 +450,8 @@ export class DcRouter { // Start the unified email server await this.unifiedEmailServer.start(); - logger.log('info', `Unified email handling configured with ${this.options.emailConfig.domainRules.length} domain rules`); + logger.log('info', `Unified email handling configured with ${emailConfig.domainRules.length} domain rules on internal ports`); + logger.log('info', `Email server listening on ports: ${internalEmailConfig.ports.join(', ')}`); } catch (error) { logger.log('error', `Error setting up unified email handling: ${error.message}`); throw error; diff --git a/ts/classes.smtp.portconfig.ts b/ts/classes.smtp.portconfig.ts index 3ce7a75..ce1c7a1 100644 --- a/ts/classes.smtp.portconfig.ts +++ b/ts/classes.smtp.portconfig.ts @@ -239,73 +239,57 @@ export class SmtpPortConfig { } /** - * Apply port configurations to SmartProxy settings - * @param smartProxy SmartProxy instance + * Convert port configurations to SmartProxy routes + * @returns Array of SmartProxy routes */ - public applyToSmartProxy(smartProxy: plugins.smartproxy.SmartProxy): void { - if (!smartProxy) return; - + public toSmartProxyRoutes(): plugins.smartproxy.IRouteConfig[] { const enabledPorts = this.getEnabledPortConfigs(); - const settings = smartProxy.settings; + const routes: plugins.smartproxy.IRouteConfig[] = []; - // Initialize globalPortRanges if needed - if (!settings.globalPortRanges) { - settings.globalPortRanges = []; - } - - // Add configured ports to globalPortRanges + // Add configured ports as routes for (const portConfig of enabledPorts) { - // Add port to global port ranges if not already present - if (!settings.globalPortRanges.some((r) => r.from <= portConfig.port && portConfig.port <= r.to)) { - settings.globalPortRanges.push({ from: portConfig.port, to: portConfig.port }); - } - - // Apply TLS settings at SmartProxy level + // Create a route for each SMTP port + const route: plugins.smartproxy.IRouteConfig = { + name: `smtp-port-${portConfig.port}`, + match: { + ports: [portConfig.port] + }, + action: { + type: 'forward', + target: { + host: 'localhost', + port: portConfig.port + 10000 // Map to internal port (e.g., 25 -> 10025) + } + } + }; + + // Apply TLS settings if (portConfig.port === 465 && portConfig.tls?.enabled) { // For implicit TLS on port 465 - settings.sniEnabled = true; + route.action.tls = { + mode: 'terminate', + certificate: 'auto' + }; + } else if (portConfig.tls?.useStartTls) { + // For STARTTLS on ports 25 and 587 + route.action.tls = { + mode: 'passthrough' + }; } + + routes.push(route); } - // Group ports by TLS configuration to log them - const starttlsPorts = enabledPorts - .filter(p => p.tls?.enabled && p.tls?.useStartTls) - .map(p => p.port); + return routes; + } - const implicitTlsPorts = enabledPorts - .filter(p => p.tls?.enabled && !p.tls?.useStartTls) - .map(p => p.port); - - const nonTlsPorts = enabledPorts - .filter(p => !p.tls?.enabled) - .map(p => p.port); - - if (starttlsPorts.length > 0) { - console.log(`Configured STARTTLS SMTP ports: ${starttlsPorts.join(', ')}`); - } - - if (implicitTlsPorts.length > 0) { - console.log(`Configured Implicit TLS SMTP ports: ${implicitTlsPorts.join(', ')}`); - } - - if (nonTlsPorts.length > 0) { - console.log(`Configured Plain SMTP ports: ${nonTlsPorts.join(', ')}`); - } - - // Setup connection listeners for different port types - smartProxy.on('connection', (connection) => { - const port = connection.localPort; - - // Check which type of port this is - if (implicitTlsPorts.includes(port)) { - console.log(`Implicit TLS SMTP connection on port ${port} from ${connection.remoteIP}`); - } else if (starttlsPorts.includes(port)) { - console.log(`STARTTLS SMTP connection on port ${port} from ${connection.remoteIP}`); - } else if (nonTlsPorts.includes(port)) { - console.log(`Plain SMTP connection on port ${port} from ${connection.remoteIP}`); - } - }); - - console.log(`Applied SMTP port configurations to SmartProxy: ${enabledPorts.map(p => p.port).join(', ')}`); + /** + * Apply port configurations to SmartProxy settings + * @param smartProxy SmartProxy instance + * @deprecated Use toSmartProxyRoutes() instead to generate routes + */ + public applyToSmartProxy(smartProxy: plugins.smartproxy.SmartProxy): void { + console.warn('SmtpPortConfig.applyToSmartProxy() is deprecated. Use toSmartProxyRoutes() instead.'); + // This method is deprecated and no longer functional } } \ No newline at end of file diff --git a/ts_web/00_commitinfo_data.ts b/ts_web/00_commitinfo_data.ts index 2800857..49ce47d 100644 --- a/ts_web/00_commitinfo_data.ts +++ b/ts_web/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@serve.zone/platformservice', - version: '2.11.2', + version: '2.12.0', description: 'A multifaceted platform service handling mail, SMS, letter delivery, and AI services.' }