19.6.17

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@push.rocks/smartproxy",
-  "version": "19.6.14",
+  "version": "19.6.17",
   "private": false,
   "description": "A powerful proxy package with unified route-based configuration for high traffic management. Features include SSL/TLS support, flexible routing patterns, WebSocket handling, advanced security options, and automatic ACME certificate management.",
   "main": "dist_ts/index.js",

readme.hints.md

@@ -262,4 +262,87 @@ To reduce log spam during high-traffic scenarios or attacks, SmartProxy implemen
 - Provides better overview of patterns (e.g., which IPs are attacking)
 - Improves log readability and analysis
 - Prevents log storage overflow
-- Maintains detailed information in aggregated form
+- Maintains detailed information in aggregated form
+
+### Log Output Examples
+
+Instead of hundreds of individual logs:
+```
+Connection rejected
+Connection rejected
+Connection rejected
+... (repeated 500 times)
+```
+
+You'll see:
+```
+[SUMMARY] Rejected 500 connections from 10 IPs in 5s (rate-limit: 350, per-ip-limit: 150) (top offenders: 192.168.1.100 (200x, rate-limit), 10.0.0.1 (150x, per-ip-limit))
+```
+
+Instead of:
+```
+Connection terminated: ::ffff:127.0.0.1 (client_closed). Active: 266
+Connection terminated: ::ffff:127.0.0.1 (client_closed). Active: 265
+... (repeated 266 times)
+```
+
+You'll see:
+```
+[SUMMARY] 266 HttpProxy connections terminated in 5s (reasons: client_closed: 266, activeConnections: 0)
+```
+
+### Rapid Event Handling
+- During attacks or high-volume scenarios, logs are flushed more frequently
+- If 50+ events occur within 1 second, immediate flush is triggered
+- Prevents memory buildup during flooding attacks
+- Maintains real-time visibility during incidents
+
+## Custom Certificate Provision Function
+
+The `certProvisionFunction` feature has been implemented to allow users to provide their own certificate generation logic.
+
+### Implementation Details
+
+1. **Type Definition**: The function must return `Promise<TSmartProxyCertProvisionObject>` where:
+   - `TSmartProxyCertProvisionObject = plugins.tsclass.network.ICert | 'http01'`
+   - Return `'http01'` to fallback to Let's Encrypt
+   - Return a certificate object for custom certificates
+
+2. **Certificate Manager Changes**:
+   - Added `certProvisionFunction` property to CertificateManager
+   - Modified `provisionAcmeCertificate()` to check custom function first
+   - Custom certificates are stored with source type 'custom'
+   - Expiry date extraction currently defaults to 90 days
+
+3. **Configuration Options**:
+   - `certProvisionFunction`: The custom provision function
+   - `certProvisionFallbackToAcme`: Whether to fallback to ACME on error (default: true)
+
+4. **Usage Example**:
+```typescript
+new SmartProxy({
+  certProvisionFunction: async (domain: string) => {
+    if (domain === 'internal.example.com') {
+      return {
+        cert: customCert,
+        key: customKey,
+        ca: customCA
+      } as unknown as TSmartProxyCertProvisionObject;
+    }
+    return 'http01'; // Use Let's Encrypt
+  },
+  certProvisionFallbackToAcme: true
+})
+```
+
+5. **Testing Notes**:
+   - Type assertions through `unknown` are needed in tests due to strict interface typing
+   - Mock certificate objects work for testing but need proper type casting
+   - The actual certificate parsing for expiry dates would need a proper X.509 parser
+
+### Future Improvements
+
+1. Implement proper certificate expiry date extraction using X.509 parsing
+2. Add support for returning expiry date with custom certificates
+3. Consider adding validation for custom certificate format
+4. Add events/hooks for certificate provisioning lifecycle

readme.md

@@ -2336,14 +2336,117 @@ sequenceDiagram
   • Efficient SNI extraction
   • Minimal overhead routing
 
-## Certificate Hooks & Events
+## Certificate Management
+
+### Custom Certificate Provision Function
+
+SmartProxy supports a custom certificate provision function that allows you to provide your own certificate generation logic while maintaining compatibility with Let's Encrypt:
+
+```typescript
+const proxy = new SmartProxy({
+  certProvisionFunction: async (domain: string): Promise<TSmartProxyCertProvisionObject> => {
+    // Option 1: Return a custom certificate
+    if (domain === 'internal.example.com') {
+      return {
+        cert: customCertPEM,
+        key: customKeyPEM,
+        ca: customCAPEM       // Optional CA chain
+      };
+    }
+    
+    // Option 2: Fallback to Let's Encrypt
+    return 'http01';
+  },
+  
+  // Control fallback behavior when custom provision fails
+  certProvisionFallbackToAcme: true,  // Default: true
+  
+  routes: [...]
+});
+```
+
+**Key Features:**
+- Called for any route with `certificate: 'auto'`
+- Return custom certificate object or `'http01'` to use Let's Encrypt
+- Participates in automatic renewal cycle (checked every 12 hours)
+- Custom certificates stored with source type 'custom' for tracking
+
+**Configuration Options:**
+- `certProvisionFunction`: Async function that receives domain and returns certificate or 'http01'
+- `certProvisionFallbackToAcme`: Whether to fallback to Let's Encrypt if custom provision fails (default: true)
+
+**Advanced Example with Certificate Manager:**
+
+```typescript
+const certManager = new MyCertificateManager();
+
+const proxy = new SmartProxy({
+  certProvisionFunction: async (domain: string) => {
+    try {
+      // Check if we have a custom certificate for this domain
+      if (await certManager.hasCustomCert(domain)) {
+        const cert = await certManager.getCertificate(domain);
+        return {
+          cert: cert.certificate,
+          key: cert.privateKey,
+          ca: cert.chain
+        };
+      }
+      
+      // Use Let's Encrypt for public domains
+      if (domain.endsWith('.example.com')) {
+        return 'http01';
+      }
+      
+      // Generate self-signed for internal domains
+      if (domain.endsWith('.internal')) {
+        const selfSigned = await certManager.generateSelfSigned(domain);
+        return {
+          cert: selfSigned.cert,
+          key: selfSigned.key,
+          ca: ''
+        };
+      }
+      
+      // Default to Let's Encrypt
+      return 'http01';
+    } catch (error) {
+      console.error(`Certificate provision failed for ${domain}:`, error);
+      // Will fallback to Let's Encrypt if certProvisionFallbackToAcme is true
+      throw error;
+    }
+  },
+  
+  certProvisionFallbackToAcme: true,
+  
+  routes: [
+    // Routes that use automatic certificates
+    {
+      match: { ports: 443, domains: ['app.example.com', '*.internal'] },
+      action: {
+        type: 'forward',
+        target: { host: 'localhost', port: 8080 },
+        tls: { mode: 'terminate', certificate: 'auto' }
+      }
+    }
+  ]
+});
+```
+
+### Certificate Events
 
 Listen for certificate events via EventEmitter:
 - **SmartProxy**:
   - `certificate` (domain, publicKey, privateKey, expiryDate, source, isRenewal)
   - Events from CertManager are propagated
 
-Provide a `certProvisionFunction(domain)` in SmartProxy settings to supply static certs or return `'http01'`.
+```typescript
+proxy.on('certificate', (domain, cert, key, expiryDate, source, isRenewal) => {
+  console.log(`Certificate ${isRenewal ? 'renewed' : 'provisioned'} for ${domain}`);
+  console.log(`Source: ${source}`); // 'acme', 'static', or 'custom'
+  console.log(`Expires: ${expiryDate}`);
+});
+```
 
 ## SmartProxy: Common Use Cases
 

readme.plan.md

@@ -1,45 +1,281 @@
-# SmartProxy Connection Limiting Improvements Plan
+# SmartProxy Implementation Plan
 
-Command to re-read CLAUDE.md: `cat /home/philkunz/.claude/CLAUDE.md`
+## Feature: Custom Certificate Provision Function
 
-## Issues Identified
+### Summary
+This plan implements the `certProvisionFunction` feature that allows users to provide their own certificate generation logic. The function can either return a custom certificate or delegate back to Let's Encrypt by returning 'http01'.
 
-1. **HttpProxy Bypass**: Connections forwarded to HttpProxy for TLS termination only check global limits, not per-IP limits
-2. **Missing Route-Level Connection Enforcement**: Routes can define `security.maxConnections` but it's never enforced
-3. **Cleanup Queue Race Condition**: New connections can be added to cleanup queue while processing
-4. **IP Tracking Memory Optimization**: IP entries remain in map even without active connections
+### Key Changes
+1. Add `certProvisionFunction` support to CertificateManager
+2. Modify `provisionAcmeCertificate()` to check custom function first
+3. Add certificate expiry parsing for custom certificates
+4. Support both initial provisioning and renewal
+5. Add fallback configuration option
 
-## Implementation Steps
+### Overview
+Implement the `certProvisionFunction` callback that's defined in the interface but currently not implemented. This will allow users to provide custom certificate generation logic while maintaining backward compatibility with the existing Let's Encrypt integration.
 
-### 1. Fix HttpProxy Per-IP Validation ✓
-- [x] Pass IP information to HttpProxy when forwarding connections
-- [x] Add per-IP validation in HttpProxy connection handler
-- [x] Ensure connection tracking is consistent between SmartProxy and HttpProxy
+### Requirements
+1. The function should be called for any new certificate provisioning or renewal
+2. Must support returning custom certificates or falling back to Let's Encrypt
+3. Should integrate seamlessly with the existing certificate lifecycle
+4. Must maintain backward compatibility
 
-### 2. Implement Route-Level Connection Limits ✓
-- [x] Add connection count tracking per route in ConnectionManager
-- [x] Update SharedSecurityManager.isAllowed() to check route-specific maxConnections
-- [x] Add route connection limit validation in route-connection-handler.ts
+### Implementation Steps
 
-### 3. Fix Cleanup Queue Race Condition ✓
-- [x] Implement proper queue snapshotting before processing
-- [x] Ensure new connections added during processing aren't missed
-- [x] Add proper synchronization for cleanup operations
+#### 1. Update Certificate Manager to Support Custom Provision Function
+**File**: `ts/proxies/smart-proxy/certificate-manager.ts`
 
-### 4. Optimize IP Tracking Memory Usage ✓
-- [x] Add periodic cleanup for IPs with no active connections
-- [x] Implement expiry for rate limit timestamps
-- [x] Add memory-efficient data structures for IP tracking
+- [ ] Add `certProvisionFunction` property to CertificateManager class
+- [ ] Pass the function from SmartProxy options during initialization
+- [ ] Modify `provisionCertificate()` method to check for custom function first
 
-### 5. Add Comprehensive Tests ✓
-- [x] Test per-IP limits with HttpProxy forwarding
-- [x] Test route-level connection limits
-- [x] Test cleanup queue edge cases
-- [x] Test memory usage with many unique IPs
+#### 2. Implement Custom Certificate Provisioning Logic
+**Location**: Modify `provisionAcmeCertificate()` method
 
-## Notes
+```typescript
+private async provisionAcmeCertificate(
+  route: IRouteConfig, 
+  domains: string[]
+): Promise<void> {
+  const primaryDomain = domains[0];
+  const routeName = route.name || primaryDomain;
+  
+  // Check for custom provision function first
+  if (this.certProvisionFunction) {
+    try {
+      logger.log('info', `Attempting custom certificate provision for ${primaryDomain}`, { domain: primaryDomain });
+      const result = await this.certProvisionFunction(primaryDomain);
+      
+      if (result === 'http01') {
+        logger.log('info', `Custom function returned 'http01', falling back to Let's Encrypt for ${primaryDomain}`);
+        // Continue with existing ACME logic below
+      } else {
+        // Use custom certificate
+        const customCert = result as plugins.tsclass.network.ICert;
+        
+        // Convert to internal certificate format
+        const certData: ICertificateData = {
+          cert: customCert.cert,
+          key: customCert.key,
+          ca: customCert.ca || '',
+          issueDate: new Date(),
+          expiryDate: this.extractExpiryDate(customCert.cert)
+        };
+        
+        // Store and apply certificate
+        await this.certStore.saveCertificate(routeName, certData);
+        await this.applyCertificate(primaryDomain, certData);
+        this.updateCertStatus(routeName, 'valid', 'custom', certData);
+        
+        logger.log('info', `Custom certificate applied for ${primaryDomain}`, { 
+          domain: primaryDomain,
+          expiryDate: certData.expiryDate
+        });
+        return;
+      }
+    } catch (error) {
+      logger.log('error', `Custom cert provision failed for ${primaryDomain}: ${error.message}`, {
+        domain: primaryDomain,
+        error: error.message
+      });
+      // Configuration option to control fallback behavior
+      if (this.smartProxy.settings.certProvisionFallbackToAcme !== false) {
+        logger.log('info', `Falling back to Let's Encrypt for ${primaryDomain}`);
+      } else {
+        throw error;
+      }
+    }
+  }
+  
+  // Existing Let's Encrypt logic continues here...
+  if (!this.smartAcme) {
+    throw new Error('SmartAcme not initialized...');
+  }
+  // ... rest of existing code
+}
+```
 
-- All connection limiting is now consistent across SmartProxy and HttpProxy
-- Route-level limits provide additional granular control
-- Memory usage is optimized for high-traffic scenarios
-- Comprehensive test coverage ensures reliability
+#### 3. Add Helper Method for Certificate Expiry Extraction
+**New method**: `extractExpiryDate()`
+
+- [ ] Parse PEM certificate to extract expiry date
+- [ ] Use existing certificate parsing utilities
+- [ ] Handle parse errors gracefully
+
+```typescript
+private extractExpiryDate(certPem: string): Date {
+  try {
+    // Use forge or similar library to parse certificate
+    const cert = forge.pki.certificateFromPem(certPem);
+    return cert.validity.notAfter;
+  } catch (error) {
+    // Default to 90 days if parsing fails
+    return new Date(Date.now() + 90 * 24 * 60 * 60 * 1000);
+  }
+}
+```
+
+#### 4. Update SmartProxy Initialization
+**File**: `ts/proxies/smart-proxy/index.ts`
+
+- [ ] Pass `certProvisionFunction` from options to CertificateManager
+- [ ] Validate function if provided
+
+#### 5. Add Type Safety and Validation
+**Tasks**:
+- [ ] Validate returned certificate has required fields (cert, key, ca)
+- [ ] Check certificate validity dates
+- [ ] Ensure certificate matches requested domain
+
+#### 6. Update Certificate Renewal Logic
+**Location**: `checkAndRenewCertificates()`
+
+- [ ] Ensure renewal checks work for both ACME and custom certificates
+- [ ] Custom certificates should go through the same `provisionAcmeCertificate()` path
+- [ ] The existing renewal logic already calls `provisionCertificate()` which will use our modified flow
+
+```typescript
+// No changes needed here - the existing renewal logic will automatically
+// use the custom provision function when calling provisionCertificate()
+private async checkAndRenewCertificates(): Promise<void> {
+  // Existing code already handles this correctly
+  for (const route of routes) {
+    if (this.shouldRenewCertificate(cert, renewThreshold)) {
+      // This will call provisionCertificate -> provisionAcmeCertificate
+      // which now includes our custom function check
+      await this.provisionCertificate(route);
+    }
+  }
+}
+```
+
+#### 7. Add Integration Tests
+**File**: `test/test.certificate-provision.ts`
+
+- [ ] Test custom certificate provision
+- [ ] Test fallback to Let's Encrypt ('http01' return)
+- [ ] Test error handling
+- [ ] Test renewal with custom function
+
+#### 8. Update Documentation
+**Files**: 
+- [ ] Update interface documentation
+- [ ] Add examples to README
+- [ ] Document ICert structure requirements
+
+### API Design
+
+```typescript
+// Example usage
+const proxy = new SmartProxy({
+  certProvisionFunction: async (domain: string) => {
+    // Option 1: Return custom certificate
+    const customCert = await myCustomCA.generateCert(domain);
+    return {
+      cert: customCert.certificate,
+      key: customCert.privateKey,
+      ca: customCert.chain
+    };
+    
+    // Option 2: Use Let's Encrypt for certain domains
+    if (domain.endsWith('.internal')) {
+      return customCert;
+    }
+    return 'http01'; // Fallback to Let's Encrypt
+  },
+  certProvisionFallbackToAcme: true, // Default: true
+  routes: [...]
+});
+```
+
+### Configuration Options to Add
+
+```typescript
+interface ISmartProxyOptions {
+  // Existing options...
+  
+  // Custom certificate provision function
+  certProvisionFunction?: (domain: string) => Promise<TSmartProxyCertProvisionObject>;
+  
+  // Whether to fallback to ACME if custom provision fails
+  certProvisionFallbackToAcme?: boolean; // Default: true
+}
+```
+
+### Error Handling Strategy
+
+1. **Custom Function Errors**:
+   - Log detailed error with domain context
+   - Option A: Fallback to Let's Encrypt (safer)
+   - Option B: Fail certificate provisioning (stricter)
+   - Make this configurable via option?
+
+2. **Invalid Certificate Returns**:
+   - Validate certificate structure
+   - Check expiry dates
+   - Verify domain match
+
+### Testing Plan
+
+1. **Unit Tests**:
+   - Mock certProvisionFunction returns
+   - Test validation logic
+   - Test error scenarios
+
+2. **Integration Tests**:
+   - Real certificate generation
+   - Renewal cycle testing
+   - Mixed custom/Let's Encrypt scenarios
+
+### Backward Compatibility
+
+- If no `certProvisionFunction` provided, behavior unchanged
+- Existing routes with 'auto' certificates continue using Let's Encrypt
+- No breaking changes to existing API
+
+### Future Enhancements
+
+1. **Per-Route Custom Functions**:
+   - Allow different provision functions per route
+   - Override global function at route level
+
+2. **Certificate Events**:
+   - Emit events for custom cert provisioning
+   - Allow monitoring/logging hooks
+
+3. **Async Certificate Updates**:
+   - Support updating certificates outside renewal cycle
+   - Hot-reload certificates without restart
+
+### Implementation Notes
+
+1. **Certificate Status Tracking**:
+   - The `updateCertStatus()` method needs to support a new type: 'custom'
+   - Current types are 'acme' and 'static'
+   - This helps distinguish custom certificates in monitoring/logs
+
+2. **Certificate Store Integration**:
+   - Custom certificates are stored the same way as ACME certificates
+   - They participate in the same renewal cycle
+   - The store handles persistence across restarts
+
+3. **Existing Methods to Reuse**:
+   - `applyCertificate()` - Already handles applying certs to routes
+   - `isCertificateValid()` - Can validate custom certificates
+   - `certStore.saveCertificate()` - Handles storage
+
+### Implementation Priority
+
+1. Core functionality (steps 1-3)
+2. Type safety and validation (step 5)
+3. Renewal support (step 6)
+4. Tests (step 7)
+5. Documentation (step 8)
+
+### Estimated Effort
+
+- Core implementation: 4-6 hours
+- Testing: 2-3 hours
+- Documentation: 1 hour
+- Total: ~8-10 hours

test/test.certificate-provision.ts

@@ -0,0 +1,360 @@
+import { expect, tap } from '@git.zone/tstest/tapbundle';
+import { SmartProxy } from '../ts/index.js';
+import type { TSmartProxyCertProvisionObject } from '../ts/index.js';
+import * as fs from 'fs';
+import * as path from 'path';
+import { fileURLToPath } from 'url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+let testProxy: SmartProxy;
+
+// Load test certificates from helpers
+const testCert = fs.readFileSync(path.join(__dirname, 'helpers/test-cert.pem'), 'utf8');
+const testKey = fs.readFileSync(path.join(__dirname, 'helpers/test-key.pem'), 'utf8');
+
+tap.test('SmartProxy should support custom certificate provision function', async () => {
+  // Create test certificate object matching ICert interface
+  const testCertObject = {
+    id: 'test-cert-1',
+    domainName: 'test.example.com',
+    created: Date.now(),
+    validUntil: Date.now() + 90 * 24 * 60 * 60 * 1000, // 90 days
+    privateKey: testKey,
+    publicKey: testCert,
+    csr: ''
+  };
+  
+  // Custom certificate store for testing
+  const customCerts = new Map<string, typeof testCertObject>();
+  customCerts.set('test.example.com', testCertObject);
+  
+  // Create proxy with custom certificate provision
+  testProxy = new SmartProxy({
+    certProvisionFunction: async (domain: string): Promise<TSmartProxyCertProvisionObject> => {
+      console.log(`Custom cert provision called for domain: ${domain}`);
+      
+      // Return custom cert for known domains
+      if (customCerts.has(domain)) {
+        console.log(`Returning custom certificate for ${domain}`);
+        return customCerts.get(domain)!;
+      }
+      
+      // Fallback to Let's Encrypt for other domains
+      console.log(`Falling back to Let's Encrypt for ${domain}`);
+      return 'http01';
+    },
+    certProvisionFallbackToAcme: true,
+    acme: {
+      email: 'test@example.com',
+      useProduction: false
+    },
+    routes: [
+      {
+        name: 'test-route',
+        match: {
+          ports: [443],
+          domains: ['test.example.com']
+        },
+        action: {
+          type: 'forward',
+          target: {
+            host: 'localhost',
+            port: 8080
+          },
+          tls: {
+            mode: 'terminate',
+            certificate: 'auto'
+          }
+        }
+      }
+    ]
+  });
+  
+  expect(testProxy).toBeInstanceOf(SmartProxy);
+});
+
+tap.test('Custom certificate provision function should be called', async () => {
+  let provisionCalled = false;
+  const provisionedDomains: string[] = [];
+  
+  const testProxy2 = new SmartProxy({
+    certProvisionFunction: async (domain: string): Promise<TSmartProxyCertProvisionObject> => {
+      provisionCalled = true;
+      provisionedDomains.push(domain);
+      
+      // Return a test certificate matching ICert interface
+      return {
+        id: `test-cert-${domain}`,
+        domainName: domain,
+        created: Date.now(),
+        validUntil: Date.now() + 90 * 24 * 60 * 60 * 1000,
+        privateKey: testKey,
+        publicKey: testCert,
+        csr: ''
+      };
+    },
+    acme: {
+      email: 'test@example.com',
+      useProduction: false,
+      port: 9080
+    },
+    routes: [
+      {
+        name: 'custom-cert-route',
+        match: {
+          ports: [9443],
+          domains: ['custom.example.com']
+        },
+        action: {
+          type: 'forward',
+          target: {
+            host: 'localhost',
+            port: 8080
+          },
+          tls: {
+            mode: 'terminate',
+            certificate: 'auto'
+          }
+        }
+      }
+    ]
+  });
+  
+  // Mock the certificate manager to test our custom provision function
+  let certManagerCalled = false;
+  const origCreateCertManager = (testProxy2 as any).createCertificateManager;
+  (testProxy2 as any).createCertificateManager = async function(...args: any[]) {
+    const certManager = await origCreateCertManager.apply(testProxy2, args);
+    
+    // Override provisionAllCertificates to track calls
+    const origProvisionAll = certManager.provisionAllCertificates;
+    certManager.provisionAllCertificates = async function() {
+      certManagerCalled = true;
+      await origProvisionAll.call(certManager);
+    };
+    
+    return certManager;
+  };
+  
+  // Start the proxy (this will trigger certificate provisioning)
+  await testProxy2.start();
+  
+  expect(certManagerCalled).toBeTrue();
+  expect(provisionCalled).toBeTrue();
+  expect(provisionedDomains).toContain('custom.example.com');
+  
+  await testProxy2.stop();
+});
+
+tap.test('Should fallback to ACME when custom provision fails', async () => {
+  const failedDomains: string[] = [];
+  let acmeAttempted = false;
+  
+  const testProxy3 = new SmartProxy({
+    certProvisionFunction: async (domain: string): Promise<TSmartProxyCertProvisionObject> => {
+      failedDomains.push(domain);
+      throw new Error('Custom provision failed for testing');
+    },
+    certProvisionFallbackToAcme: true,
+    acme: {
+      email: 'test@example.com',
+      useProduction: false,
+      port: 9080
+    },
+    routes: [
+      {
+        name: 'fallback-route',
+        match: {
+          ports: [9444],
+          domains: ['fallback.example.com']
+        },
+        action: {
+          type: 'forward',
+          target: {
+            host: 'localhost',
+            port: 8080
+          },
+          tls: {
+            mode: 'terminate',
+            certificate: 'auto'
+          }
+        }
+      }
+    ]
+  });
+  
+  // Mock to track ACME attempts
+  const origCreateCertManager = (testProxy3 as any).createCertificateManager;
+  (testProxy3 as any).createCertificateManager = async function(...args: any[]) {
+    const certManager = await origCreateCertManager.apply(testProxy3, args);
+    
+    // Mock SmartAcme to avoid real ACME calls
+    (certManager as any).smartAcme = {
+      getCertificateForDomain: async () => {
+        acmeAttempted = true;
+        throw new Error('Mocked ACME failure');
+      }
+    };
+    
+    return certManager;
+  };
+  
+  // Start the proxy
+  await testProxy3.start();
+  
+  // Custom provision should have failed
+  expect(failedDomains).toContain('fallback.example.com');
+  
+  // ACME should have been attempted as fallback
+  expect(acmeAttempted).toBeTrue();
+  
+  await testProxy3.stop();
+});
+
+tap.test('Should not fallback when certProvisionFallbackToAcme is false', async () => {
+  let errorThrown = false;
+  let errorMessage = '';
+  
+  const testProxy4 = new SmartProxy({
+    certProvisionFunction: async (_domain: string): Promise<TSmartProxyCertProvisionObject> => {
+      throw new Error('Custom provision failed for testing');
+    },
+    certProvisionFallbackToAcme: false,
+    routes: [
+      {
+        name: 'no-fallback-route',
+        match: {
+          ports: [9445],
+          domains: ['no-fallback.example.com']
+        },
+        action: {
+          type: 'forward',
+          target: {
+            host: 'localhost',
+            port: 8080
+          },
+          tls: {
+            mode: 'terminate',
+            certificate: 'auto'
+          }
+        }
+      }
+    ]
+  });
+  
+  // Mock certificate manager to capture errors
+  const origCreateCertManager = (testProxy4 as any).createCertificateManager;
+  (testProxy4 as any).createCertificateManager = async function(...args: any[]) {
+    const certManager = await origCreateCertManager.apply(testProxy4, args);
+    
+    // Override provisionAllCertificates to capture errors
+    const origProvisionAll = certManager.provisionAllCertificates;
+    certManager.provisionAllCertificates = async function() {
+      try {
+        await origProvisionAll.call(certManager);
+      } catch (e) {
+        errorThrown = true;
+        errorMessage = e.message;
+        throw e;
+      }
+    };
+    
+    return certManager;
+  };
+  
+  try {
+    await testProxy4.start();
+  } catch (e) {
+    // Expected to fail
+  }
+  
+  expect(errorThrown).toBeTrue();
+  expect(errorMessage).toInclude('Custom provision failed for testing');
+  
+  await testProxy4.stop();
+});
+
+tap.test('Should return http01 for unknown domains', async () => {
+  let returnedHttp01 = false;
+  let acmeAttempted = false;
+  
+  const testProxy5 = new SmartProxy({
+    certProvisionFunction: async (domain: string): Promise<TSmartProxyCertProvisionObject> => {
+      if (domain === 'known.example.com') {
+        return {
+          id: `test-cert-${domain}`,
+          domainName: domain,
+          created: Date.now(),
+          validUntil: Date.now() + 90 * 24 * 60 * 60 * 1000,
+          privateKey: testKey,
+          publicKey: testCert,
+          csr: ''
+        };
+      }
+      returnedHttp01 = true;
+      return 'http01';
+    },
+    acme: {
+      email: 'test@example.com',
+      useProduction: false,
+      port: 9081
+    },
+    routes: [
+      {
+        name: 'unknown-domain-route',
+        match: {
+          ports: [9446],
+          domains: ['unknown.example.com']
+        },
+        action: {
+          type: 'forward',
+          target: {
+            host: 'localhost',
+            port: 8080
+          },
+          tls: {
+            mode: 'terminate',
+            certificate: 'auto'
+          }
+        }
+      }
+    ]
+  });
+  
+  // Mock to track ACME attempts
+  const origCreateCertManager = (testProxy5 as any).createCertificateManager;
+  (testProxy5 as any).createCertificateManager = async function(...args: any[]) {
+    const certManager = await origCreateCertManager.apply(testProxy5, args);
+    
+    // Mock SmartAcme to track attempts
+    (certManager as any).smartAcme = {
+      getCertificateForDomain: async () => {
+        acmeAttempted = true;
+        throw new Error('Mocked ACME failure');
+      }
+    };
+    
+    return certManager;
+  };
+  
+  await testProxy5.start();
+  
+  // Should have returned http01 for unknown domain
+  expect(returnedHttp01).toBeTrue();
+  
+  // ACME should have been attempted
+  expect(acmeAttempted).toBeTrue();
+  
+  await testProxy5.stop();
+});
+
+tap.test('cleanup', async () => {
+  // Clean up any test proxies
+  if (testProxy) {
+    await testProxy.stop();
+  }
+});
+
+export default tap.start();

ts/core/utils/log-deduplicator.ts

@@ -23,6 +23,8 @@ export class LogDeduplicator {
   private aggregatedEvents: Map<string, IAggregatedEvent> = new Map();
   private flushInterval: number = 5000; // 5 seconds
   private maxBatchSize: number = 100;
+  private rapidEventThreshold: number = 50; // Flush early if this many events in 1 second
+  private lastRapidCheck: number = Date.now();
   
   constructor(flushInterval?: number) {
     if (flushInterval) {
@@ -85,8 +87,15 @@ export class LogDeduplicator {
       });
     }
     
-    // Check if we should flush due to size
-    if (aggregated.events.size >= this.maxBatchSize) {
+    // Check for rapid events (many events in short time)
+    const totalEvents = Array.from(aggregated.events.values()).reduce((sum, e) => sum + e.count, 0);
+    
+    // If we're getting flooded with events, flush more frequently
+    if (now - this.lastRapidCheck < 1000 && totalEvents >= this.rapidEventThreshold) {
+      this.flush(key);
+      this.lastRapidCheck = now;
+    } else if (aggregated.events.size >= this.maxBatchSize) {
+      // Check if we should flush due to size
       this.flush(key);
     } else if (!aggregated.flushTimer) {
       // Schedule flush
@@ -98,6 +107,11 @@ export class LogDeduplicator {
         aggregated.flushTimer.unref();
       }
     }
+    
+    // Update rapid check time
+    if (now - this.lastRapidCheck >= 1000) {
+      this.lastRapidCheck = now;
+    }
   }
   
   /**
@@ -122,6 +136,9 @@ export class LogDeduplicator {
       case 'connection-cleanup':
         this.flushConnectionCleanups(aggregated);
         break;
+      case 'connection-terminated':
+        this.flushConnectionTerminations(aggregated);
+        break;
       case 'ip-rejected':
         this.flushIPRejections(aggregated);
         break;
@@ -156,10 +173,10 @@ export class LogDeduplicator {
       .map(([reason, count]) => `${reason}: ${count}`)
       .join(', ');
     
-    logger.log('warn', `Rejected ${totalCount} connections`, {
+    const duration = Date.now() - Math.min(...Array.from(aggregated.events.values()).map(e => e.firstSeen));
+    logger.log('warn', `[SUMMARY] Rejected ${totalCount} connections in ${Math.round(duration/1000)}s`, {
       reasons: reasonSummary,
       uniqueIPs: aggregated.events.size,
-      duration: Date.now() - Math.min(...Array.from(aggregated.events.values()).map(e => e.firstSeen)),
       component: 'connection-dedup'
     });
   }
@@ -186,8 +203,73 @@ export class LogDeduplicator {
     });
   }
   
+  private flushConnectionTerminations(aggregated: IAggregatedEvent): void {
+    const totalCount = Array.from(aggregated.events.values()).reduce((sum, e) => sum + e.count, 0);
+    const byReason = new Map<string, number>();
+    const byIP = new Map<string, number>();
+    let lastActiveCount = 0;
+    
+    for (const [, event] of aggregated.events) {
+      const reason = event.data?.reason || 'unknown';
+      const ip = event.data?.remoteIP || 'unknown';
+      
+      byReason.set(reason, (byReason.get(reason) || 0) + event.count);
+      
+      // Track by IP
+      if (ip !== 'unknown') {
+        byIP.set(ip, (byIP.get(ip) || 0) + event.count);
+      }
+      
+      // Track the last active connection count
+      if (event.data?.activeConnections !== undefined) {
+        lastActiveCount = event.data.activeConnections;
+      }
+    }
+    
+    const reasonSummary = Array.from(byReason.entries())
+      .sort((a, b) => b[1] - a[1])
+      .slice(0, 5) // Top 5 reasons
+      .map(([reason, count]) => `${reason}: ${count}`)
+      .join(', ');
+    
+    // Show top IPs if there are many different ones
+    let ipInfo = '';
+    if (byIP.size > 3) {
+      const topIPs = Array.from(byIP.entries())
+        .sort((a, b) => b[1] - a[1])
+        .slice(0, 3)
+        .map(([ip, count]) => `${ip} (${count})`)
+        .join(', ');
+      ipInfo = `, from ${byIP.size} IPs (top: ${topIPs})`;
+    } else if (byIP.size > 0) {
+      ipInfo = `, IPs: ${Array.from(byIP.keys()).join(', ')}`;
+    }
+    
+    const duration = Date.now() - Math.min(...Array.from(aggregated.events.values()).map(e => e.firstSeen));
+    
+    // Special handling for localhost connections (HttpProxy)
+    const localhostCount = byIP.get('::ffff:127.0.0.1') || 0;
+    if (localhostCount > 0 && byIP.size === 1) {
+      // All connections are from localhost (HttpProxy)
+      logger.log('info', `[SUMMARY] ${totalCount} HttpProxy connections terminated in ${Math.round(duration/1000)}s`, {
+        reasons: reasonSummary,
+        activeConnections: lastActiveCount,
+        component: 'connection-dedup'
+      });
+    } else {
+      logger.log('info', `[SUMMARY] ${totalCount} connections terminated in ${Math.round(duration/1000)}s`, {
+        reasons: reasonSummary,
+        activeConnections: lastActiveCount,
+        uniqueReasons: byReason.size,
+        ...(ipInfo ? { ips: ipInfo } : {}),
+        component: 'connection-dedup'
+      });
+    }
+  }
+  
   private flushIPRejections(aggregated: IAggregatedEvent): void {
     const byIP = new Map<string, { count: number; reasons: Set<string> }>();
+    const allReasons = new Map<string, number>();
     
     for (const [ip, event] of aggregated.events) {
       if (!byIP.has(ip)) {
@@ -197,9 +279,17 @@ export class LogDeduplicator {
       ipData.count += event.count;
       if (event.data?.reason) {
         ipData.reasons.add(event.data.reason);
+        // Track overall reason counts
+        allReasons.set(event.data.reason, (allReasons.get(event.data.reason) || 0) + event.count);
       }
     }
     
+    // Create reason summary
+    const reasonSummary = Array.from(allReasons.entries())
+      .sort((a, b) => b[1] - a[1])
+      .map(([reason, count]) => `${reason}: ${count}`)
+      .join(', ');
+    
     // Log top offenders
     const topOffenders = Array.from(byIP.entries())
       .sort((a, b) => b[1].count - a[1].count)
@@ -209,9 +299,9 @@ export class LogDeduplicator {
     
     const totalRejections = Array.from(byIP.values()).reduce((sum, data) => sum + data.count, 0);
     
-    logger.log('warn', `Rejected ${totalRejections} connections from ${byIP.size} IPs`, {
+    const duration = Date.now() - Math.min(...Array.from(aggregated.events.values()).map(e => e.firstSeen));
+    logger.log('warn', `[SUMMARY] Rejected ${totalRejections} connections from ${byIP.size} IPs in ${Math.round(duration/1000)}s (${reasonSummary})`, {
       topOffenders,
-      duration: Date.now() - Math.min(...Array.from(aggregated.events.values()).map(e => e.firstSeen)),
       component: 'ip-dedup'
     });
   }

ts/proxies/smart-proxy/certificate-manager.ts

@@ -12,7 +12,7 @@ export interface ICertStatus {
   status: 'valid' | 'pending' | 'expired' | 'error';
   expiryDate?: Date;
   issueDate?: Date;
-  source: 'static' | 'acme';
+  source: 'static' | 'acme' | 'custom';
   error?: string;
 }
 
@@ -22,6 +22,7 @@ export interface ICertificateData {
   ca?: string;
   expiryDate: Date;
   issueDate: Date;
+  source?: 'static' | 'acme' | 'custom';
 }
 
 export class SmartCertManager {
@@ -50,6 +51,12 @@ export class SmartCertManager {
   // ACME state manager reference
   private acmeStateManager: AcmeStateManager | null = null;
   
+  // Custom certificate provision function
+  private certProvisionFunction?: (domain: string) => Promise<plugins.tsclass.network.ICert | 'http01'>;
+  
+  // Whether to fallback to ACME if custom provision fails
+  private certProvisionFallbackToAcme: boolean = true;
+  
   constructor(
     private routes: IRouteConfig[],
     private certDir: string = './certs',
@@ -89,6 +96,20 @@ export class SmartCertManager {
     this.globalAcmeDefaults = defaults;
   }
   
+  /**
+   * Set custom certificate provision function
+   */
+  public setCertProvisionFunction(fn: (domain: string) => Promise<plugins.tsclass.network.ICert | 'http01'>): void {
+    this.certProvisionFunction = fn;
+  }
+  
+  /**
+   * Set whether to fallback to ACME if custom provision fails
+   */
+  public setCertProvisionFallbackToAcme(fallback: boolean): void {
+    this.certProvisionFallbackToAcme = fallback;
+  }
+  
   /**
    * Set callback for updating routes (used for challenge routes)
    */
@@ -212,15 +233,6 @@ export class SmartCertManager {
     route: IRouteConfig, 
     domains: string[]
   ): Promise<void> {
-    if (!this.smartAcme) {
-      throw new Error(
-        'SmartAcme not initialized. This usually means no ACME email was provided. ' +
-        'Please ensure you have configured ACME with an email address either:\n' +
-        '1. In the top-level "acme" configuration\n' +
-        '2. In the route\'s "tls.acme" configuration'
-      );
-    }
-    
     const primaryDomain = domains[0];
     const routeName = route.name || primaryDomain;
     
@@ -229,10 +241,68 @@ export class SmartCertManager {
     if (existingCert && this.isCertificateValid(existingCert)) {
       logger.log('info', `Using existing valid certificate for ${primaryDomain}`, { domain: primaryDomain, component: 'certificate-manager' });
       await this.applyCertificate(primaryDomain, existingCert);
-      this.updateCertStatus(routeName, 'valid', 'acme', existingCert);
+      this.updateCertStatus(routeName, 'valid', existingCert.source || 'acme', existingCert);
       return;
     }
     
+    // Check for custom provision function first
+    if (this.certProvisionFunction) {
+      try {
+        logger.log('info', `Attempting custom certificate provision for ${primaryDomain}`, { domain: primaryDomain, component: 'certificate-manager' });
+        const result = await this.certProvisionFunction(primaryDomain);
+        
+        if (result === 'http01') {
+          logger.log('info', `Custom function returned 'http01', falling back to Let's Encrypt for ${primaryDomain}`, { domain: primaryDomain, component: 'certificate-manager' });
+          // Continue with existing ACME logic below
+        } else {
+          // Use custom certificate
+          const customCert = result as plugins.tsclass.network.ICert;
+          
+          // Convert to internal certificate format
+          const certData: ICertificateData = {
+            cert: customCert.publicKey,
+            key: customCert.privateKey,
+            ca: '',
+            issueDate: new Date(),
+            expiryDate: this.extractExpiryDate(customCert.publicKey),
+            source: 'custom'
+          };
+          
+          // Store and apply certificate
+          await this.certStore.saveCertificate(routeName, certData);
+          await this.applyCertificate(primaryDomain, certData);
+          this.updateCertStatus(routeName, 'valid', 'custom', certData);
+          
+          logger.log('info', `Custom certificate applied for ${primaryDomain}`, { 
+            domain: primaryDomain,
+            expiryDate: certData.expiryDate,
+            component: 'certificate-manager'
+          });
+          return;
+        }
+      } catch (error) {
+        logger.log('error', `Custom cert provision failed for ${primaryDomain}: ${error.message}`, {
+          domain: primaryDomain,
+          error: error.message,
+          component: 'certificate-manager'
+        });
+        // Check if we should fallback to ACME
+        if (!this.certProvisionFallbackToAcme) {
+          throw error;
+        }
+        logger.log('info', `Falling back to Let's Encrypt for ${primaryDomain}`, { domain: primaryDomain, component: 'certificate-manager' });
+      }
+    }
+    
+    if (!this.smartAcme) {
+      throw new Error(
+        'SmartAcme not initialized. This usually means no ACME email was provided. ' +
+        'Please ensure you have configured ACME with an email address either:\n' +
+        '1. In the top-level "acme" configuration\n' +
+        '2. In the route\'s "tls.acme" configuration'
+      );
+    }
+    
     // Apply renewal threshold from global defaults or route config
     const renewThreshold = route.action.tls?.acme?.renewBeforeDays || 
                          this.globalAcmeDefaults?.renewThresholdDays || 
@@ -280,7 +350,8 @@ export class SmartCertManager {
         key: cert.privateKey, 
         ca: cert.publicKey, // Use same as cert for now
         expiryDate: new Date(cert.validUntil),
-        issueDate: new Date(cert.created)
+        issueDate: new Date(cert.created),
+        source: 'acme'
       };
       
       await this.certStore.saveCertificate(routeName, certData);
@@ -328,7 +399,8 @@ export class SmartCertManager {
         cert,
         key,
         expiryDate: certInfo.validTo,
-        issueDate: certInfo.validFrom
+        issueDate: certInfo.validFrom,
+        source: 'static'
       };
       
       // Save to store for consistency
@@ -399,6 +471,19 @@ export class SmartCertManager {
     return cert.expiryDate > expiryThreshold;
   }
   
+  /**
+   * Extract expiry date from a PEM certificate
+   */
+  private extractExpiryDate(_certPem: string): Date {
+    // For now, we'll default to 90 days for custom certificates
+    // In production, you might want to use a proper X.509 parser
+    // or require the custom cert provider to include expiry info
+    logger.log('info', 'Using default 90-day expiry for custom certificate', {
+      component: 'certificate-manager'
+    });
+    return new Date(Date.now() + 90 * 24 * 60 * 60 * 1000);
+  }
+  
   
   /**
    * Add challenge route to SmartProxy

ts/proxies/smart-proxy/connection-manager.ts

@@ -401,23 +401,34 @@ export class ConnectionManager extends LifecycleComponent {
       // Remove the record from the tracking map
       this.connectionRecords.delete(record.id);
 
-      // Log connection details
+      // Use deduplicated logging for connection termination
       if (this.smartProxy.settings.enableDetailedLogging) {
-        logger.log('info', 
-          `Connection terminated: ${record.remoteIP}:${record.localPort} (${reason}) - ` +
-          `${plugins.prettyMs(duration)}, IN: ${record.bytesReceived}B, OUT: ${record.bytesSent}B`, 
-          logData
+        // For detailed logging, include more info but still deduplicate by IP+reason
+        connectionLogDeduplicator.log(
+          'connection-terminated',
+          'info',
+          `Connection terminated: ${record.remoteIP}:${record.localPort}`,
+          {
+            ...logData,
+            duration_ms: duration,
+            bytesIn: record.bytesReceived,
+            bytesOut: record.bytesSent
+          },
+          `${record.remoteIP}-${reason}`
         );
       } else {
-        logger.log('info', 
-          `Connection terminated: ${record.remoteIP} (${reason}). Active: ${this.connectionRecords.size}`, 
+        // For normal logging, deduplicate by termination reason
+        connectionLogDeduplicator.log(
+          'connection-terminated',
+          'info',
+          `Connection terminated`,
           {
-            connectionId: record.id,
             remoteIP: record.remoteIP,
             reason,
             activeConnections: this.connectionRecords.size,
             component: 'connection-manager'
-          }
+          },
+          reason // Group by termination reason
         );
       }
     }

ts/proxies/smart-proxy/models/interfaces.ts

@@ -135,6 +135,12 @@ export interface ISmartProxyOptions {
    * or a static certificate object for immediate provisioning.
    */
   certProvisionFunction?: (domain: string) => Promise<TSmartProxyCertProvisionObject>;
+  
+  /**
+   * Whether to fallback to ACME if custom certificate provision fails.
+   * Default: true
+   */
+  certProvisionFallbackToAcme?: boolean;
 }
 
 /**

ts/proxies/smart-proxy/route-connection-handler.ts

@@ -90,7 +90,13 @@ export class RouteConnectionHandler {
     // Note: For wrapped sockets, this will use the underlying socket IP until PROXY protocol is parsed
     const ipValidation = this.smartProxy.securityManager.validateIP(wrappedSocket.remoteAddress || '');
     if (!ipValidation.allowed) {
-      logger.log('warn', `Connection rejected`, { remoteIP: wrappedSocket.remoteAddress, reason: ipValidation.reason, component: 'route-handler' });
+      connectionLogDeduplicator.log(
+        'ip-rejected',
+        'warn',
+        `Connection rejected from ${wrappedSocket.remoteAddress}`,
+        { remoteIP: wrappedSocket.remoteAddress, reason: ipValidation.reason, component: 'route-handler' },
+        wrappedSocket.remoteAddress
+      );
       cleanupSocket(wrappedSocket.socket, `rejected-${ipValidation.reason}`, { immediate: true });
       return;
     }

ts/proxies/smart-proxy/smart-proxy.ts

@@ -243,6 +243,16 @@ export class SmartProxy extends plugins.EventEmitter {
       certManager.setGlobalAcmeDefaults(this.settings.acme);
     }
     
+    // Pass down the custom certificate provision function if available
+    if (this.settings.certProvisionFunction) {
+      certManager.setCertProvisionFunction(this.settings.certProvisionFunction);
+    }
+    
+    // Pass down the fallback to ACME setting
+    if (this.settings.certProvisionFallbackToAcme !== undefined) {
+      certManager.setCertProvisionFallbackToAcme(this.settings.certProvisionFallbackToAcme);
+    }
+    
     await certManager.initialize();
     return certManager;
   }