From 02e77655ad8c9a961c084704111d2e9a8990d415 Mon Sep 17 00:00:00 2001 From: Philipp Kunz Date: Mon, 19 May 2025 12:04:26 +0000 Subject: [PATCH] update --- Connection-Cleanup-Patterns.md | 341 ++++++++++++++++++ Connection-Termination-Issues.md | 248 +++++++++++++ ...kProxy-SmartProxy-Connection-Management.md | 153 ++++++++ summary-acme-simplification.md | 86 ----- summary-nftables-naming-consolidation.md | 34 -- test/core/utils/test.event-system.ts | 2 +- test/core/utils/test.ip-utils.ts | 2 +- test/core/utils/test.route-utils.ts | 2 +- .../utils/test.shared-security-manager.ts | 2 +- test/core/utils/test.validation-utils.ts | 2 +- test/test.acme-state-manager.node.ts | 2 +- test/test.certificate-provisioning.ts | 2 +- test/test.certificate-simple.ts | 2 +- test/test.forwarding.examples.ts | 2 +- test/test.forwarding.ts | 2 +- test/test.forwarding.unit.ts | 2 +- test/test.networkproxy.function-targets.ts | 2 +- test/test.networkproxy.ts | 2 +- test/test.nftables-integration.simple.ts | 2 +- test/test.nftables-integration.ts | 2 +- test/test.nftables-manager.ts | 2 +- test/test.nftables-status.ts | 2 +- test/test.port-mapping.ts | 2 +- test/test.port80-management.node.ts | 2 +- test/test.race-conditions.node.ts | 2 +- test/test.route-config.ts | 2 +- test/test.route-update-callback.node.ts | 2 +- test/test.route-utils.ts | 2 +- test/test.router.ts | 2 +- test/test.smartacme-integration.ts | 2 +- test/test.smartproxy.ts | 2 +- ts/proxies/network-proxy/network-proxy.ts | 9 - ts/proxies/smart-proxy/models/route-types.ts | 1 + .../smart-proxy/route-connection-handler.ts | 164 +++++++-- 34 files changed, 898 insertions(+), 190 deletions(-) create mode 100644 Connection-Cleanup-Patterns.md create mode 100644 Connection-Termination-Issues.md create mode 100644 NetworkProxy-SmartProxy-Connection-Management.md delete mode 100644 summary-acme-simplification.md delete mode 100644 summary-nftables-naming-consolidation.md diff --git a/Connection-Cleanup-Patterns.md b/Connection-Cleanup-Patterns.md new file mode 100644 index 0000000..a55ace9 --- /dev/null +++ b/Connection-Cleanup-Patterns.md @@ -0,0 +1,341 @@ +# Connection Cleanup Code Patterns + +## Pattern 1: Safe Connection Cleanup + +```typescript +public initiateCleanupOnce(record: IConnectionRecord, reason: string = 'normal'): void { + // Prevent duplicate cleanup + if (record.incomingTerminationReason === null || + record.incomingTerminationReason === undefined) { + record.incomingTerminationReason = reason; + this.incrementTerminationStat('incoming', reason); + } + + this.cleanupConnection(record, reason); +} + +public cleanupConnection(record: IConnectionRecord, reason: string = 'normal'): void { + if (!record.connectionClosed) { + record.connectionClosed = true; + + // Remove from tracking immediately + this.connectionRecords.delete(record.id); + this.securityManager.removeConnectionByIP(record.remoteIP, record.id); + + // Clear timers + if (record.cleanupTimer) { + clearTimeout(record.cleanupTimer); + record.cleanupTimer = undefined; + } + + // Clean up sockets + this.cleanupSocket(record, 'incoming', record.incoming); + if (record.outgoing) { + this.cleanupSocket(record, 'outgoing', record.outgoing); + } + + // Clear memory + record.pendingData = []; + record.pendingDataSize = 0; + } +} +``` + +## Pattern 2: Socket Cleanup with Retry + +```typescript +private cleanupSocket( + record: IConnectionRecord, + side: 'incoming' | 'outgoing', + socket: plugins.net.Socket +): void { + try { + if (!socket.destroyed) { + // Graceful shutdown first + socket.end(); + + // Force destroy after timeout + const socketTimeout = setTimeout(() => { + try { + if (!socket.destroyed) { + socket.destroy(); + } + } catch (err) { + console.log(`[${record.id}] Error destroying ${side} socket: ${err}`); + } + }, 1000); + + // Don't block process exit + if (socketTimeout.unref) { + socketTimeout.unref(); + } + } + } catch (err) { + console.log(`[${record.id}] Error closing ${side} socket: ${err}`); + // Fallback to destroy + try { + if (!socket.destroyed) { + socket.destroy(); + } + } catch (destroyErr) { + console.log(`[${record.id}] Error destroying ${side} socket: ${destroyErr}`); + } + } +} +``` + +## Pattern 3: NetworkProxy Bridge Cleanup + +```typescript +public async forwardToNetworkProxy( + connectionId: string, + socket: plugins.net.Socket, + record: IConnectionRecord, + initialChunk: Buffer, + networkProxyPort: number, + cleanupCallback: (reason: string) => void +): Promise { + const proxySocket = new plugins.net.Socket(); + + // Connect to NetworkProxy + await new Promise((resolve, reject) => { + proxySocket.connect(networkProxyPort, 'localhost', () => { + resolve(); + }); + proxySocket.on('error', reject); + }); + + // Send initial data + if (initialChunk) { + proxySocket.write(initialChunk); + } + + // Setup bidirectional piping + socket.pipe(proxySocket); + proxySocket.pipe(socket); + + // Comprehensive cleanup handler + const cleanup = (reason: string) => { + // Unpipe to prevent data loss + socket.unpipe(proxySocket); + proxySocket.unpipe(socket); + + // Destroy proxy socket + proxySocket.destroy(); + + // Notify SmartProxy + cleanupCallback(reason); + }; + + // Setup all cleanup triggers + socket.on('end', () => cleanup('socket_end')); + socket.on('error', () => cleanup('socket_error')); + proxySocket.on('end', () => cleanup('proxy_end')); + proxySocket.on('error', () => cleanup('proxy_error')); +} +``` + +## Pattern 4: Error Handler with Cleanup + +```typescript +public handleError(side: 'incoming' | 'outgoing', record: IConnectionRecord) { + return (err: Error) => { + const code = (err as any).code; + let reason = 'error'; + + // Map error codes to reasons + switch (code) { + case 'ECONNRESET': + reason = 'econnreset'; + break; + case 'ETIMEDOUT': + reason = 'etimedout'; + break; + case 'ECONNREFUSED': + reason = 'connection_refused'; + break; + case 'EHOSTUNREACH': + reason = 'host_unreachable'; + break; + } + + // Log with context + const duration = Date.now() - record.incomingStartTime; + console.log( + `[${record.id}] ${code} on ${side} side from ${record.remoteIP}. ` + + `Duration: ${plugins.prettyMs(duration)}` + ); + + // Track termination reason + if (side === 'incoming' && record.incomingTerminationReason === null) { + record.incomingTerminationReason = reason; + this.incrementTerminationStat('incoming', reason); + } + + // Initiate cleanup + this.initiateCleanupOnce(record, reason); + }; +} +``` + +## Pattern 5: Inactivity Check with Cleanup + +```typescript +public performInactivityCheck(): void { + const now = Date.now(); + const connectionIds = [...this.connectionRecords.keys()]; + + for (const id of connectionIds) { + const record = this.connectionRecords.get(id); + if (!record) continue; + + // Skip if disabled or immortal + if (this.settings.disableInactivityCheck || + (record.hasKeepAlive && this.settings.keepAliveTreatment === 'immortal')) { + continue; + } + + const inactivityTime = now - record.lastActivity; + let effectiveTimeout = this.settings.inactivityTimeout!; + + // Extended timeout for keep-alive + if (record.hasKeepAlive && this.settings.keepAliveTreatment === 'extended') { + effectiveTimeout *= (this.settings.keepAliveInactivityMultiplier || 6); + } + + if (inactivityTime > effectiveTimeout && !record.connectionClosed) { + // Warn before closing keep-alive connections + if (record.hasKeepAlive && !record.inactivityWarningIssued) { + console.log(`[${id}] Warning: Keep-alive connection inactive`); + record.inactivityWarningIssued = true; + // Grace period + record.lastActivity = now - (effectiveTimeout - 600000); + } else { + // Close the connection + console.log(`[${id}] Closing due to inactivity`); + this.cleanupConnection(record, 'inactivity'); + } + } + } +} +``` + +## Pattern 6: Complete Shutdown + +```typescript +public clearConnections(): void { + const connectionIds = [...this.connectionRecords.keys()]; + + // Phase 1: Graceful end + for (const id of connectionIds) { + const record = this.connectionRecords.get(id); + if (record) { + try { + // Clear timers + if (record.cleanupTimer) { + clearTimeout(record.cleanupTimer); + record.cleanupTimer = undefined; + } + + // Graceful socket end + if (record.incoming && !record.incoming.destroyed) { + record.incoming.end(); + } + if (record.outgoing && !record.outgoing.destroyed) { + record.outgoing.end(); + } + } catch (err) { + console.log(`Error during graceful end: ${err}`); + } + } + } + + // Phase 2: Force destroy after delay + setTimeout(() => { + for (const id of connectionIds) { + const record = this.connectionRecords.get(id); + if (record) { + try { + // Remove all listeners + if (record.incoming) { + record.incoming.removeAllListeners(); + if (!record.incoming.destroyed) { + record.incoming.destroy(); + } + } + if (record.outgoing) { + record.outgoing.removeAllListeners(); + if (!record.outgoing.destroyed) { + record.outgoing.destroy(); + } + } + } catch (err) { + console.log(`Error during forced destruction: ${err}`); + } + } + } + + // Clear all tracking + this.connectionRecords.clear(); + this.terminationStats = { incoming: {}, outgoing: {} }; + }, 100); +} +``` + +## Pattern 7: Safe Event Handler Removal + +```typescript +// Store handlers for later removal +record.renegotiationHandler = this.tlsManager.createRenegotiationHandler( + connectionId, + serverName, + connInfo, + (connectionId, reason) => this.connectionManager.initiateCleanupOnce(record, reason) +); + +// Add the handler +socket.on('data', record.renegotiationHandler); + +// Remove during cleanup +if (record.incoming) { + try { + record.incoming.removeAllListeners('data'); + record.renegotiationHandler = undefined; + } catch (err) { + console.log(`[${record.id}] Error removing data handlers: ${err}`); + } +} +``` + +## Pattern 8: Connection State Tracking + +```typescript +interface IConnectionRecord { + id: string; + connectionClosed: boolean; + incomingTerminationReason: string | null; + outgoingTerminationReason: string | null; + cleanupTimer?: NodeJS.Timeout; + renegotiationHandler?: Function; + // ... other fields +} + +// Check state before operations +if (!record.connectionClosed) { + // Safe to perform operations +} + +// Track cleanup state +record.connectionClosed = true; +``` + +## Key Principles + +1. **Idempotency**: Cleanup operations should be safe to call multiple times +2. **State Tracking**: Always track connection and cleanup state +3. **Error Resilience**: Handle errors during cleanup gracefully +4. **Resource Release**: Clear all references (timers, handlers, buffers) +5. **Graceful First**: Try graceful shutdown before forced destroy +6. **Comprehensive Coverage**: Handle all possible termination scenarios +7. **Logging**: Track termination reasons for debugging +8. **Memory Safety**: Clear data structures to prevent leaks \ No newline at end of file diff --git a/Connection-Termination-Issues.md b/Connection-Termination-Issues.md new file mode 100644 index 0000000..3ffe934 --- /dev/null +++ b/Connection-Termination-Issues.md @@ -0,0 +1,248 @@ +# Connection Termination Issues and Solutions in SmartProxy/NetworkProxy + +## Common Connection Termination Scenarios + +### 1. Normal Connection Closure + +**Flow**: +- Client or server initiates graceful close +- 'close' event triggers cleanup +- Connection removed from tracking +- Resources freed + +**Code Path**: +```typescript +// In ConnectionManager +handleClose(side: 'incoming' | 'outgoing', record: IConnectionRecord) { + record.incomingTerminationReason = 'normal'; + this.initiateCleanupOnce(record, 'closed_' + side); +} +``` + +### 2. Error-Based Termination + +**Common Errors**: +- ECONNRESET: Connection reset by peer +- ETIMEDOUT: Connection timed out +- ECONNREFUSED: Connection refused +- EHOSTUNREACH: Host unreachable + +**Handling**: +```typescript +handleError(side: 'incoming' | 'outgoing', record: IConnectionRecord) { + return (err: Error) => { + const code = (err as any).code; + let reason = 'error'; + + if (code === 'ECONNRESET') { + reason = 'econnreset'; + } else if (code === 'ETIMEDOUT') { + reason = 'etimedout'; + } + + this.initiateCleanupOnce(record, reason); + }; +} +``` + +### 3. Inactivity Timeout + +**Detection**: +```typescript +performInactivityCheck(): void { + const now = Date.now(); + for (const record of this.connectionRecords.values()) { + const inactivityTime = now - record.lastActivity; + if (inactivityTime > effectiveTimeout) { + this.cleanupConnection(record, 'inactivity'); + } + } +} +``` + +**Special Cases**: +- Keep-alive connections get extended timeouts +- "Immortal" connections bypass inactivity checks +- Warning issued before closure for keep-alive connections + +### 4. NFTables-Handled Connections + +**Special Handling**: +```typescript +if (route.action.forwardingEngine === 'nftables') { + socket.end(); + record.nftablesHandled = true; + this.connectionManager.initiateCleanupOnce(record, 'nftables_handled'); + return; +} +``` + +These connections are: +- Handled at kernel level +- Closed immediately at application level +- Tracked for metrics only + +### 5. NetworkProxy Bridge Termination + +**Bridge Cleanup**: +```typescript +const cleanup = (reason: string) => { + socket.unpipe(proxySocket); + proxySocket.unpipe(socket); + proxySocket.destroy(); + cleanupCallback(reason); +}; + +socket.on('end', () => cleanup('socket_end')); +socket.on('error', () => cleanup('socket_error')); +proxySocket.on('end', () => cleanup('proxy_end')); +proxySocket.on('error', () => cleanup('proxy_error')); +``` + +## Preventing Connection Leaks + +### 1. Always Remove Event Listeners + +```typescript +cleanupConnection(record: IConnectionRecord, reason: string): void { + if (record.incoming) { + record.incoming.removeAllListeners('data'); + record.renegotiationHandler = undefined; + } +} +``` + +### 2. Clear Timers + +```typescript +if (record.cleanupTimer) { + clearTimeout(record.cleanupTimer); + record.cleanupTimer = undefined; +} +``` + +### 3. Proper Socket Cleanup + +```typescript +private cleanupSocket(record: IConnectionRecord, side: string, socket: net.Socket): void { + try { + if (!socket.destroyed) { + socket.end(); // Graceful + setTimeout(() => { + if (!socket.destroyed) { + socket.destroy(); // Forced + } + }, 1000); + } + } catch (err) { + console.log(`Error closing ${side} socket: ${err}`); + } +} +``` + +### 4. Connection Record Cleanup + +```typescript +// Clear pending data to prevent memory leaks +record.pendingData = []; +record.pendingDataSize = 0; + +// Remove from tracking map +this.connectionRecords.delete(record.id); +``` + +## Monitoring and Debugging + +### 1. Termination Statistics + +```typescript +private terminationStats: { + incoming: Record; + outgoing: Record; +} = { incoming: {}, outgoing: {} }; + +incrementTerminationStat(side: 'incoming' | 'outgoing', reason: string): void { + this.terminationStats[side][reason] = (this.terminationStats[side][reason] || 0) + 1; +} +``` + +### 2. Connection Logging + +**Detailed Logging**: +```typescript +console.log( + `[${record.id}] Connection from ${record.remoteIP} terminated (${reason}).` + + ` Duration: ${prettyMs(duration)}, Bytes IN: ${bytesReceived}, OUT: ${bytesSent}` +); +``` + +### 3. Active Connection Tracking + +```typescript +getConnectionCount(): number { + return this.connectionRecords.size; +} + +// In NetworkProxy +metrics = { + activeConnections: this.connectedClients, + portProxyConnections: this.portProxyConnections, + tlsTerminatedConnections: this.tlsTerminatedConnections +}; +``` + +## Best Practices for Connection Termination + +1. **Always Use initiateCleanupOnce()**: + - Prevents duplicate cleanup operations + - Ensures proper termination reason tracking + +2. **Handle All Socket Events**: + - 'error', 'close', 'end' events + - Both incoming and outgoing sockets + +3. **Implement Proper Timeouts**: + - Initial data timeout + - Inactivity timeout + - Maximum connection lifetime + +4. **Track Resources**: + - Connection records + - Socket maps + - Timer references + +5. **Log Termination Reasons**: + - Helps debug connection issues + - Provides metrics for monitoring + +6. **Graceful Shutdown**: + - Try socket.end() before socket.destroy() + - Allow time for graceful closure + +7. **Memory Management**: + - Clear pending data buffers + - Remove event listeners + - Delete connection records + +## Common Issues and Solutions + +### Issue: Memory Leaks from Event Listeners +**Solution**: Always call removeAllListeners() during cleanup + +### Issue: Orphaned Connections +**Solution**: Implement multiple cleanup triggers (timeout, error, close) + +### Issue: Duplicate Cleanup Operations +**Solution**: Use connectionClosed flag and initiateCleanupOnce() + +### Issue: Hanging Connections +**Solution**: Implement inactivity checks and maximum lifetime limits + +### Issue: Resource Exhaustion +**Solution**: Track connection counts and implement limits + +### Issue: Lost Data During Cleanup +**Solution**: Use proper unpipe operations and graceful shutdown + +### Issue: Debugging Connection Issues +**Solution**: Track termination reasons and maintain detailed logs \ No newline at end of file diff --git a/NetworkProxy-SmartProxy-Connection-Management.md b/NetworkProxy-SmartProxy-Connection-Management.md new file mode 100644 index 0000000..0a57098 --- /dev/null +++ b/NetworkProxy-SmartProxy-Connection-Management.md @@ -0,0 +1,153 @@ +# NetworkProxy Connection Termination and SmartProxy Connection Handling + +## Overview + +The connection management between NetworkProxy and SmartProxy involves complex coordination to handle TLS termination, connection forwarding, and proper cleanup. This document outlines how these systems work together. + +## SmartProxy Connection Management + +### Connection Tracking (ConnectionManager) + +1. **Connection Lifecycle**: + - New connections are registered in `ConnectionManager.createConnection()` + - Each connection gets a unique ID and tracking record + - Connection records track both incoming (client) and outgoing (target) sockets + - Connections are removed from tracking upon cleanup + +2. **Connection Cleanup Flow**: + ``` + initiateCleanupOnce() -> cleanupConnection() -> cleanupSocket() + ``` + - `initiateCleanupOnce()`: Prevents duplicate cleanup operations + - `cleanupConnection()`: Main cleanup logic, removes connections from tracking + - `cleanupSocket()`: Handles socket termination (graceful end, then forced destroy) + +3. **Cleanup Triggers**: + - Socket errors (ECONNRESET, ETIMEDOUT, etc.) + - Socket close events + - Inactivity timeouts + - Connection lifetime limits + - Manual cleanup (e.g., NFTables-handled connections) + +## NetworkProxy Integration + +### NetworkProxyBridge + +The `NetworkProxyBridge` class manages the connection between SmartProxy and NetworkProxy: + +1. **Connection Forwarding**: + ```typescript + forwardToNetworkProxy( + connectionId: string, + socket: net.Socket, + record: IConnectionRecord, + initialChunk: Buffer, + networkProxyPort: number, + cleanupCallback: (reason: string) => void + ) + ``` + - Creates a new socket connection to NetworkProxy + - Pipes data between client and NetworkProxy sockets + - Sets up cleanup handlers for both sockets + +2. **Cleanup Coordination**: + - When either socket ends or errors, both are cleaned up + - Cleanup callback notifies SmartProxy's ConnectionManager + - Proper unpipe operations prevent memory leaks + +## NetworkProxy Connection Tracking + +### Connection Tracking in NetworkProxy + +1. **Raw TCP Connection Tracking**: + ```typescript + setupConnectionTracking(): void { + this.httpsServer.on('connection', (connection: net.Socket) => { + // Track connections in socketMap + this.socketMap.add(connection); + + // Setup cleanup handlers + connection.on('close', cleanupConnection); + connection.on('error', cleanupConnection); + connection.on('end', cleanupConnection); + }); + } + ``` + +2. **SmartProxy Connection Detection**: + - Connections from localhost (127.0.0.1) are identified as SmartProxy connections + - Special counter tracks `portProxyConnections` + - Connection counts are updated when connections close + +3. **Metrics and Monitoring**: + - Active connections tracked in `connectedClients` + - TLS handshake completions tracked in `tlsTerminatedConnections` + - Connection pool status monitored periodically + +## Connection Termination Flow + +### Typical TLS Termination Flow: + +1. Client connects to SmartProxy +2. SmartProxy creates connection record and tracks socket +3. SmartProxy determines route requires TLS termination +4. NetworkProxyBridge forwards connection to NetworkProxy +5. NetworkProxy performs TLS termination +6. Data flows through piped sockets +7. When connection ends: + - NetworkProxy cleans up its socket tracking + - NetworkProxyBridge handles cleanup coordination + - SmartProxy's ConnectionManager removes connection record + - All resources are properly released + +### Cleanup Coordination Points: + +1. **SmartProxy Cleanup**: + - ConnectionManager tracks all cleanup reasons + - Socket handlers removed to prevent memory leaks + - Timeout timers cleared + - Connection records removed from maps + - Security manager notified of connection removal + +2. **NetworkProxy Cleanup**: + - Sockets removed from tracking map + - Connection counters updated + - Metrics updated for monitoring + - Connection pool resources freed + +3. **Bridge Cleanup**: + - Unpipe operations prevent data loss + - Both sockets properly destroyed + - Cleanup callback ensures SmartProxy is notified + +## Important Considerations + +1. **Memory Management**: + - All event listeners must be removed during cleanup + - Proper unpipe operations prevent memory leaks + - Connection records cleared from all tracking maps + +2. **Error Handling**: + - Multiple cleanup mechanisms prevent orphaned connections + - Graceful shutdown attempted before forced destruction + - Timeout mechanisms ensure cleanup even in edge cases + +3. **State Consistency**: + - Connection closed flags prevent duplicate cleanup + - Termination reasons tracked for debugging + - Activity timestamps updated for accurate timeout handling + +4. **Performance**: + - Connection pools minimize TCP handshake overhead + - Efficient socket tracking using Maps + - Periodic cleanup prevents resource accumulation + +## Best Practices + +1. Always use `initiateCleanupOnce()` to prevent duplicate cleanup operations +2. Track termination reasons for debugging and monitoring +3. Ensure all event listeners are removed during cleanup +4. Use proper unpipe operations when breaking socket connections +5. Monitor connection counts and cleanup statistics +6. Implement proper timeout handling for all connection types +7. Keep socket tracking maps synchronized with actual socket state \ No newline at end of file diff --git a/summary-acme-simplification.md b/summary-acme-simplification.md deleted file mode 100644 index 549eaa7..0000000 --- a/summary-acme-simplification.md +++ /dev/null @@ -1,86 +0,0 @@ -# ACME/Certificate Simplification Summary - -## What Was Done - -We successfully implemented the ACME/Certificate simplification plan for SmartProxy: - -### 1. Created New Certificate Management System - -- **SmartCertManager** (`ts/proxies/smart-proxy/certificate-manager.ts`): A unified certificate manager that handles both ACME and static certificates -- **CertStore** (`ts/proxies/smart-proxy/cert-store.ts`): File-based certificate storage system - -### 2. Updated Route Types - -- Added `IRouteAcme` interface for ACME configuration -- Added `IStaticResponse` interface for static route responses -- Extended `IRouteTls` with comprehensive certificate options -- Added `handler` property to `IRouteAction` for static routes - -### 3. Implemented Static Route Handler - -- Added `handleStaticAction` method to route-connection-handler.ts -- Added support for 'static' route type in the action switch statement -- Implemented proper HTTP response formatting - -### 4. Updated SmartProxy Integration - -- Removed old CertProvisioner and Port80Handler dependencies -- Added `initializeCertificateManager` method -- Updated `start` and `stop` methods to use new certificate manager -- Added `provisionCertificate`, `renewCertificate`, and `getCertificateStatus` methods - -### 5. Simplified NetworkProxyBridge - -- Removed all certificate-related logic -- Simplified to only handle network proxy forwarding -- Updated to use port-based matching for network proxy routes - -### 6. Cleaned Up HTTP Module - -- Removed exports for port80 subdirectory -- Kept only router and redirect functionality - -### 7. Created Tests - -- Created simplified test for certificate functionality -- Test demonstrates static route handling and basic certificate configuration - -## Key Improvements - -1. **No Backward Compatibility**: Clean break from legacy implementations -2. **Direct SmartAcme Integration**: Uses @push.rocks/smartacme directly without custom wrappers -3. **Route-Based ACME Challenges**: No separate HTTP server needed -4. **Simplified Architecture**: Removed unnecessary abstraction layers -5. **Unified Configuration**: Certificate configuration is part of route definitions - -## Configuration Example - -```typescript -const proxy = new SmartProxy({ - routes: [{ - name: 'secure-site', - match: { ports: 443, domains: 'example.com' }, - action: { - type: 'forward', - target: { host: 'backend', port: 8080 }, - tls: { - mode: 'terminate', - certificate: 'auto', - acme: { - email: 'admin@example.com', - useProduction: true - } - } - } - }] -}); -``` - -## Next Steps - -1. Remove old certificate module and port80 directory -2. Update documentation with new configuration format -3. Test with real ACME certificates in staging environment -4. Add more comprehensive tests for renewal and edge cases - -The implementation is complete and builds successfully! \ No newline at end of file diff --git a/summary-nftables-naming-consolidation.md b/summary-nftables-naming-consolidation.md deleted file mode 100644 index 6d26c1d..0000000 --- a/summary-nftables-naming-consolidation.md +++ /dev/null @@ -1,34 +0,0 @@ -# NFTables Naming Consolidation Summary - -This document summarizes the changes made to consolidate the naming convention for IP allow/block lists in the NFTables integration. - -## Changes Made - -1. **Updated NFTablesProxy interface** (`ts/proxies/nftables-proxy/models/interfaces.ts`): - - Changed `allowedSourceIPs` to `ipAllowList` - - Changed `bannedSourceIPs` to `ipBlockList` - -2. **Updated NFTablesProxy implementation** (`ts/proxies/nftables-proxy/nftables-proxy.ts`): - - Updated all references from `allowedSourceIPs` to `ipAllowList` - - Updated all references from `bannedSourceIPs` to `ipBlockList` - -3. **Updated NFTablesManager** (`ts/proxies/smart-proxy/nftables-manager.ts`): - - Changed mapping from `allowedSourceIPs` to `ipAllowList` - - Changed mapping from `bannedSourceIPs` to `ipBlockList` - -## Files Already Using Consistent Naming - -The following files already used the consistent naming convention `ipAllowList` and `ipBlockList`: - -1. **Route helpers** (`ts/proxies/smart-proxy/utils/route-helpers.ts`) -2. **Integration test** (`test/test.nftables-integration.ts`) -3. **NFTables example** (`examples/nftables-integration.ts`) -4. **Route types** (`ts/proxies/smart-proxy/models/route-types.ts`) - -## Result - -The naming is now consistent throughout the codebase: -- `ipAllowList` is used for lists of allowed IP addresses -- `ipBlockList` is used for lists of blocked IP addresses - -This matches the naming convention already established in SmartProxy's core routing system. \ No newline at end of file diff --git a/test/core/utils/test.event-system.ts b/test/core/utils/test.event-system.ts index 813fb04..59f233d 100644 --- a/test/core/utils/test.event-system.ts +++ b/test/core/utils/test.event-system.ts @@ -1,4 +1,4 @@ -import { expect, tap } from '@push.rocks/tapbundle'; +import { expect, tap } from '@git.zone/tstest/tapbundle'; import { EventSystem, ProxyEvents, diff --git a/test/core/utils/test.ip-utils.ts b/test/core/utils/test.ip-utils.ts index 7d2823c..659887d 100644 --- a/test/core/utils/test.ip-utils.ts +++ b/test/core/utils/test.ip-utils.ts @@ -1,4 +1,4 @@ -import { expect, tap } from '@push.rocks/tapbundle'; +import { expect, tap } from '@git.zone/tstest/tapbundle'; import { IpUtils } from '../../../ts/core/utils/ip-utils.js'; tap.test('ip-utils - normalizeIP', async () => { diff --git a/test/core/utils/test.route-utils.ts b/test/core/utils/test.route-utils.ts index aab0411..03cbf94 100644 --- a/test/core/utils/test.route-utils.ts +++ b/test/core/utils/test.route-utils.ts @@ -1,4 +1,4 @@ -import { expect, tap } from '@push.rocks/tapbundle'; +import { expect, tap } from '@git.zone/tstest/tapbundle'; import * as routeUtils from '../../../ts/core/utils/route-utils.js'; // Test domain matching diff --git a/test/core/utils/test.shared-security-manager.ts b/test/core/utils/test.shared-security-manager.ts index 87c6c6c..38cfdb9 100644 --- a/test/core/utils/test.shared-security-manager.ts +++ b/test/core/utils/test.shared-security-manager.ts @@ -1,4 +1,4 @@ -import { expect, tap } from '@push.rocks/tapbundle'; +import { expect, tap } from '@git.zone/tstest/tapbundle'; import { SharedSecurityManager } from '../../../ts/core/utils/shared-security-manager.js'; import type { IRouteConfig, IRouteContext } from '../../../ts/proxies/smart-proxy/models/route-types.js'; diff --git a/test/core/utils/test.validation-utils.ts b/test/core/utils/test.validation-utils.ts index be96ccd..34c3a94 100644 --- a/test/core/utils/test.validation-utils.ts +++ b/test/core/utils/test.validation-utils.ts @@ -1,4 +1,4 @@ -import { expect, tap } from '@push.rocks/tapbundle'; +import { expect, tap } from '@git.zone/tstest/tapbundle'; import { ValidationUtils } from '../../../ts/core/utils/validation-utils.js'; import type { IDomainOptions, IAcmeOptions } from '../../../ts/core/models/common-types.js'; diff --git a/test/test.acme-state-manager.node.ts b/test/test.acme-state-manager.node.ts index 9182bd8..1ec31b2 100644 --- a/test/test.acme-state-manager.node.ts +++ b/test/test.acme-state-manager.node.ts @@ -1,4 +1,4 @@ -import { expect, tap } from '@push.rocks/tapbundle'; +import { expect, tap } from '@git.zone/tstest/tapbundle'; import { AcmeStateManager } from '../ts/proxies/smart-proxy/acme-state-manager.js'; import type { IRouteConfig } from '../ts/proxies/smart-proxy/models/route-types.js'; diff --git a/test/test.certificate-provisioning.ts b/test/test.certificate-provisioning.ts index 23a6246..9010c6d 100644 --- a/test/test.certificate-provisioning.ts +++ b/test/test.certificate-provisioning.ts @@ -1,5 +1,5 @@ import { SmartProxy } from '../ts/proxies/smart-proxy/index.js'; -import { expect, tap } from '@push.rocks/tapbundle'; +import { expect, tap } from '@git.zone/tstest/tapbundle'; const testProxy = new SmartProxy({ routes: [{ diff --git a/test/test.certificate-simple.ts b/test/test.certificate-simple.ts index 8aacd30..0b584a3 100644 --- a/test/test.certificate-simple.ts +++ b/test/test.certificate-simple.ts @@ -1,5 +1,5 @@ import { SmartProxy } from '../ts/proxies/smart-proxy/index.js'; -import { expect, tap } from '@push.rocks/tapbundle'; +import { expect, tap } from '@git.zone/tstest/tapbundle'; tap.test('should create SmartProxy with certificate routes', async () => { const proxy = new SmartProxy({ diff --git a/test/test.forwarding.examples.ts b/test/test.forwarding.examples.ts index c7fe55d..4c9a83a 100644 --- a/test/test.forwarding.examples.ts +++ b/test/test.forwarding.examples.ts @@ -1,5 +1,5 @@ import * as path from 'path'; -import { tap, expect } from '@push.rocks/tapbundle'; +import { tap, expect } from '@git.zone/tstest/tapbundle'; import { SmartProxy } from '../ts/proxies/smart-proxy/index.js'; import { diff --git a/test/test.forwarding.ts b/test/test.forwarding.ts index d1be3bd..135795c 100644 --- a/test/test.forwarding.ts +++ b/test/test.forwarding.ts @@ -1,4 +1,4 @@ -import { tap, expect } from '@push.rocks/tapbundle'; +import { tap, expect } from '@git.zone/tstest/tapbundle'; import * as plugins from '../ts/plugins.js'; import type { IForwardConfig, TForwardingType } from '../ts/forwarding/config/forwarding-types.js'; diff --git a/test/test.forwarding.unit.ts b/test/test.forwarding.unit.ts index ce96fb1..e114d6f 100644 --- a/test/test.forwarding.unit.ts +++ b/test/test.forwarding.unit.ts @@ -1,4 +1,4 @@ -import { tap, expect } from '@push.rocks/tapbundle'; +import { tap, expect } from '@git.zone/tstest/tapbundle'; import * as plugins from '../ts/plugins.js'; // First, import the components directly to avoid issues with compiled modules diff --git a/test/test.networkproxy.function-targets.ts b/test/test.networkproxy.function-targets.ts index 8773c29..6fc597d 100644 --- a/test/test.networkproxy.function-targets.ts +++ b/test/test.networkproxy.function-targets.ts @@ -1,4 +1,4 @@ -import { expect, tap } from '@push.rocks/tapbundle'; +import { expect, tap } from '@git.zone/tstest/tapbundle'; import * as plugins from '../ts/plugins.js'; import { NetworkProxy } from '../ts/proxies/network-proxy/index.js'; import type { IRouteConfig } from '../ts/proxies/smart-proxy/models/route-types.js'; diff --git a/test/test.networkproxy.ts b/test/test.networkproxy.ts index 9c66cd9..a3d499b 100644 --- a/test/test.networkproxy.ts +++ b/test/test.networkproxy.ts @@ -1,4 +1,4 @@ -import { expect, tap } from '@push.rocks/tapbundle'; +import { expect, tap } from '@git.zone/tstest/tapbundle'; import * as smartproxy from '../ts/index.js'; import { loadTestCertificates } from './helpers/certificates.js'; import * as https from 'https'; diff --git a/test/test.nftables-integration.simple.ts b/test/test.nftables-integration.simple.ts index 9456fc6..dca9062 100644 --- a/test/test.nftables-integration.simple.ts +++ b/test/test.nftables-integration.simple.ts @@ -1,6 +1,6 @@ import { SmartProxy } from '../ts/proxies/smart-proxy/index.js'; import { createNfTablesRoute, createNfTablesTerminateRoute } from '../ts/proxies/smart-proxy/utils/route-helpers.js'; -import { expect, tap } from '@push.rocks/tapbundle'; +import { expect, tap } from '@git.zone/tstest/tapbundle'; import * as child_process from 'child_process'; import { promisify } from 'util'; diff --git a/test/test.nftables-integration.ts b/test/test.nftables-integration.ts index fd05ffc..31ed206 100644 --- a/test/test.nftables-integration.ts +++ b/test/test.nftables-integration.ts @@ -1,6 +1,6 @@ import { SmartProxy } from '../ts/proxies/smart-proxy/index.js'; import { createNfTablesRoute, createNfTablesTerminateRoute } from '../ts/proxies/smart-proxy/utils/route-helpers.js'; -import { expect, tap } from '@push.rocks/tapbundle'; +import { expect, tap } from '@git.zone/tstest/tapbundle'; import * as net from 'net'; import * as http from 'http'; import * as https from 'https'; diff --git a/test/test.nftables-manager.ts b/test/test.nftables-manager.ts index 2d190e1..d838bbb 100644 --- a/test/test.nftables-manager.ts +++ b/test/test.nftables-manager.ts @@ -1,4 +1,4 @@ -import { expect, tap } from '@push.rocks/tapbundle'; +import { expect, tap } from '@git.zone/tstest/tapbundle'; import { NFTablesManager } from '../ts/proxies/smart-proxy/nftables-manager.js'; import type { IRouteConfig } from '../ts/proxies/smart-proxy/models/route-types.js'; import type { ISmartProxyOptions } from '../ts/proxies/smart-proxy/models/interfaces.js'; diff --git a/test/test.nftables-status.ts b/test/test.nftables-status.ts index 0bf5cfd..ccd2738 100644 --- a/test/test.nftables-status.ts +++ b/test/test.nftables-status.ts @@ -1,7 +1,7 @@ import { SmartProxy } from '../ts/proxies/smart-proxy/index.js'; import { NFTablesManager } from '../ts/proxies/smart-proxy/nftables-manager.js'; import { createNfTablesRoute } from '../ts/proxies/smart-proxy/utils/route-helpers.js'; -import { expect, tap } from '@push.rocks/tapbundle'; +import { expect, tap } from '@git.zone/tstest/tapbundle'; import * as child_process from 'child_process'; import { promisify } from 'util'; diff --git a/test/test.port-mapping.ts b/test/test.port-mapping.ts index 635b3b5..89860aa 100644 --- a/test/test.port-mapping.ts +++ b/test/test.port-mapping.ts @@ -1,4 +1,4 @@ -import { expect, tap } from '@push.rocks/tapbundle'; +import { expect, tap } from '@git.zone/tstest/tapbundle'; import * as net from 'net'; import { SmartProxy } from '../ts/proxies/smart-proxy/index.js'; import { diff --git a/test/test.port80-management.node.ts b/test/test.port80-management.node.ts index 859ce42..250b596 100644 --- a/test/test.port80-management.node.ts +++ b/test/test.port80-management.node.ts @@ -1,4 +1,4 @@ -import { expect, tap } from '@push.rocks/tapbundle'; +import { expect, tap } from '@git.zone/tstest/tapbundle'; import { SmartProxy } from '../ts/index.js'; /** diff --git a/test/test.race-conditions.node.ts b/test/test.race-conditions.node.ts index de4d913..6a006a2 100644 --- a/test/test.race-conditions.node.ts +++ b/test/test.race-conditions.node.ts @@ -1,4 +1,4 @@ -import { expect, tap } from '@push.rocks/tapbundle'; +import { expect, tap } from '@git.zone/tstest/tapbundle'; import { SmartProxy } from '../ts/index.js'; /** diff --git a/test/test.route-config.ts b/test/test.route-config.ts index 267f200..690c6cf 100644 --- a/test/test.route-config.ts +++ b/test/test.route-config.ts @@ -1,7 +1,7 @@ /** * Tests for the unified route-based configuration system */ -import { expect, tap } from '@push.rocks/tapbundle'; +import { expect, tap } from '@git.zone/tstest/tapbundle'; // Import from core modules import { SmartProxy } from '../ts/proxies/smart-proxy/index.js'; diff --git a/test/test.route-update-callback.node.ts b/test/test.route-update-callback.node.ts index 62d770b..5ed8ca2 100644 --- a/test/test.route-update-callback.node.ts +++ b/test/test.route-update-callback.node.ts @@ -1,6 +1,6 @@ import * as plugins from '../ts/plugins.js'; import { SmartProxy } from '../ts/index.js'; -import { tap, expect } from '@push.rocks/tapbundle'; +import { tap, expect } from '@git.zone/tstest/tapbundle'; let testProxy: SmartProxy; diff --git a/test/test.route-utils.ts b/test/test.route-utils.ts index 85c0e7f..21c7c68 100644 --- a/test/test.route-utils.ts +++ b/test/test.route-utils.ts @@ -1,4 +1,4 @@ -import { tap, expect } from '@push.rocks/tapbundle'; +import { tap, expect } from '@git.zone/tstest/tapbundle'; import * as plugins from '../ts/plugins.js'; // Import from individual modules to avoid naming conflicts diff --git a/test/test.router.ts b/test/test.router.ts index 4eef48c..2570c5e 100644 --- a/test/test.router.ts +++ b/test/test.router.ts @@ -1,4 +1,4 @@ -import { expect, tap } from '@push.rocks/tapbundle'; +import { expect, tap } from '@git.zone/tstest/tapbundle'; import * as tsclass from '@tsclass/tsclass'; import * as http from 'http'; import { ProxyRouter, type RouterResult } from '../ts/http/router/proxy-router.js'; diff --git a/test/test.smartacme-integration.ts b/test/test.smartacme-integration.ts index bc3e1c4..abab35f 100644 --- a/test/test.smartacme-integration.ts +++ b/test/test.smartacme-integration.ts @@ -1,5 +1,5 @@ import * as plugins from '../ts/plugins.js'; -import { tap, expect } from '@push.rocks/tapbundle'; +import { tap, expect } from '@git.zone/tstest/tapbundle'; import { SmartCertManager } from '../ts/proxies/smart-proxy/certificate-manager.js'; import type { IRouteConfig } from '../ts/proxies/smart-proxy/models/route-types.js'; diff --git a/test/test.smartproxy.ts b/test/test.smartproxy.ts index 328f4f4..4df917c 100644 --- a/test/test.smartproxy.ts +++ b/test/test.smartproxy.ts @@ -1,4 +1,4 @@ -import { expect, tap } from '@push.rocks/tapbundle'; +import { expect, tap } from '@git.zone/tstest/tapbundle'; import * as net from 'net'; import { SmartProxy } from '../ts/proxies/smart-proxy/index.js'; diff --git a/ts/proxies/network-proxy/network-proxy.ts b/ts/proxies/network-proxy/network-proxy.ts index cf7d6db..2279d9c 100644 --- a/ts/proxies/network-proxy/network-proxy.ts +++ b/ts/proxies/network-proxy/network-proxy.ts @@ -219,21 +219,12 @@ export class NetworkProxy implements IMetricsTracker { }; } - /** - * @deprecated Use SmartCertManager instead - */ - public setExternalPort80Handler(handler: any): void { - this.logger.warn('Port80Handler is deprecated - use SmartCertManager instead'); - } - /** * Starts the proxy server */ public async start(): Promise { this.startTime = Date.now(); - // Certificate management is now handled by SmartCertManager - // Create HTTP/2 server with HTTP/1 fallback this.httpsServer = plugins.http2.createSecureServer( { diff --git a/ts/proxies/smart-proxy/models/route-types.ts b/ts/proxies/smart-proxy/models/route-types.ts index ee6f81b..012e920 100644 --- a/ts/proxies/smart-proxy/models/route-types.ts +++ b/ts/proxies/smart-proxy/models/route-types.ts @@ -47,6 +47,7 @@ export interface IRouteContext { path?: string; // URL path (for HTTP connections) query?: string; // Query string (for HTTP connections) headers?: Record; // HTTP headers (for HTTP connections) + method?: string; // HTTP method (for HTTP connections) // TLS information isTls: boolean; // Whether the connection is TLS diff --git a/ts/proxies/smart-proxy/route-connection-handler.ts b/ts/proxies/smart-proxy/route-connection-handler.ts index 1e15178..711fcac 100644 --- a/ts/proxies/smart-proxy/route-connection-handler.ts +++ b/ts/proxies/smart-proxy/route-connection-handler.ts @@ -728,45 +728,139 @@ export class RouteConnectionHandler { return; } - try { - // Build route context - const context: IRouteContext = { - port: record.localPort, - domain: record.lockedDomain, - clientIp: record.remoteIP, - serverIp: socket.localAddress!, - path: undefined, // Will need to be extracted from HTTP request - isTls: record.isTLS, - tlsVersion: record.tlsVersion, - routeName: route.name, - routeId: route.name, - timestamp: Date.now(), - connectionId - }; + let buffer = Buffer.alloc(0); + + const handleHttpData = async (chunk: Buffer) => { + buffer = Buffer.concat([buffer, chunk]); - // Call the handler - const response = await route.action.handler(context); - - // Send HTTP response - const headers = response.headers || {}; - headers['Content-Length'] = Buffer.byteLength(response.body).toString(); - - let httpResponse = `HTTP/1.1 ${response.status} ${getStatusText(response.status)}\r\n`; - for (const [key, value] of Object.entries(headers)) { - httpResponse += `${key}: ${value}\r\n`; + // Look for end of HTTP headers + const headerEndIndex = buffer.indexOf('\r\n\r\n'); + if (headerEndIndex === -1) { + // Need more data + if (buffer.length > 8192) { // Prevent excessive buffering + console.error(`[${connectionId}] HTTP headers too large`); + socket.end(); + this.connectionManager.cleanupConnection(record, 'headers_too_large'); + } + return; } - httpResponse += '\r\n'; - socket.write(httpResponse); - socket.write(response.body); - socket.end(); + // Parse the HTTP request + const headerBuffer = buffer.slice(0, headerEndIndex); + const headers = headerBuffer.toString(); + const lines = headers.split('\r\n'); - this.connectionManager.cleanupConnection(record, 'completed'); - } catch (error) { - console.error(`[${connectionId}] Error in static handler: ${error}`); - socket.end(); - this.connectionManager.cleanupConnection(record, 'handler_error'); - } + if (lines.length === 0) { + console.error(`[${connectionId}] Invalid HTTP request`); + socket.end(); + this.connectionManager.cleanupConnection(record, 'invalid_request'); + return; + } + + // Parse request line + const requestLine = lines[0]; + const requestParts = requestLine.split(' '); + if (requestParts.length < 3) { + console.error(`[${connectionId}] Invalid HTTP request line`); + socket.end(); + this.connectionManager.cleanupConnection(record, 'invalid_request_line'); + return; + } + + const [method, path, httpVersion] = requestParts; + + // Parse headers + const headersMap: Record = {}; + for (let i = 1; i < lines.length; i++) { + const colonIndex = lines[i].indexOf(':'); + if (colonIndex > 0) { + const key = lines[i].slice(0, colonIndex).trim().toLowerCase(); + const value = lines[i].slice(colonIndex + 1).trim(); + headersMap[key] = value; + } + } + + // Extract query string if present + let pathname = path; + let query: string | undefined; + const queryIndex = path.indexOf('?'); + if (queryIndex !== -1) { + pathname = path.slice(0, queryIndex); + query = path.slice(queryIndex + 1); + } + + try { + // Build route context with parsed HTTP information + const context: IRouteContext = { + port: record.localPort, + domain: record.lockedDomain || headersMap['host']?.split(':')[0], + clientIp: record.remoteIP, + serverIp: socket.localAddress!, + path: pathname, + query: query, + headers: headersMap, + method: method, + isTls: record.isTLS, + tlsVersion: record.tlsVersion, + routeName: route.name, + routeId: route.name, + timestamp: Date.now(), + connectionId + }; + + // Remove the data listener since we're handling the request + socket.removeListener('data', handleHttpData); + + // Call the handler with the properly parsed context + const response = await route.action.handler(context); + + // Prepare the HTTP response + const responseHeaders = response.headers || {}; + const contentLength = Buffer.byteLength(response.body || ''); + responseHeaders['Content-Length'] = contentLength.toString(); + + if (!responseHeaders['Content-Type']) { + responseHeaders['Content-Type'] = 'text/plain'; + } + + // Build the response + let httpResponse = `HTTP/1.1 ${response.status} ${getStatusText(response.status)}\r\n`; + for (const [key, value] of Object.entries(responseHeaders)) { + httpResponse += `${key}: ${value}\r\n`; + } + httpResponse += '\r\n'; + + // Send response + socket.write(httpResponse); + if (response.body) { + socket.write(response.body); + } + socket.end(); + + this.connectionManager.cleanupConnection(record, 'completed'); + } catch (error) { + console.error(`[${connectionId}] Error in static handler: ${error}`); + + // Send error response + const errorResponse = 'HTTP/1.1 500 Internal Server Error\r\n' + + 'Content-Type: text/plain\r\n' + + 'Content-Length: 21\r\n' + + '\r\n' + + 'Internal Server Error'; + socket.write(errorResponse); + socket.end(); + + this.connectionManager.cleanupConnection(record, 'handler_error'); + } + }; + + // Listen for data + socket.on('data', handleHttpData); + + // Ensure cleanup on socket close + socket.once('close', () => { + socket.removeListener('data', handleHttpData); + }); } /**