Enhance connection cleanup and error handling in RouteConnectionHandler

- Implement immediate cleanup for connection failures to prevent leaks
- Add NFTables cleanup on socket close to manage memory usage
- Fix connection limit bypass by checking record after creation
- Introduce tests for rapid connection retries and routing failures

ts/proxies/smart-proxy/route-connection-handler.ts

@@ -90,6 +90,10 @@ export class RouteConnectionHandler {
 
     // Create a new connection record
     const record = this.connectionManager.createConnection(socket);
+    if (!record) {
+      // Connection was rejected due to limit - socket already destroyed by connection manager
+      return;
+    }
     const connectionId = record.id;
 
     // Apply socket optimizations
@@ -546,6 +550,12 @@ export class RouteConnectionHandler {
       
       // We don't close the socket - just let it remain open
       // The kernel-level NFTables rules will handle the actual forwarding
+      
+      // Set up cleanup when the socket eventually closes
+      socket.once('close', () => {
+        this.connectionManager.cleanupConnection(record, 'nftables_closed');
+      });
+      
       return;
     }
 
@@ -687,7 +697,7 @@ export class RouteConnectionHandler {
                 record,
                 initialChunk,
                 this.settings.httpProxyPort || 8443,
-                (reason) => this.connectionManager.initiateCleanupOnce(record, reason)
+                (reason) => this.connectionManager.cleanupConnection(record, reason)
               );
               return;
             }
@@ -742,7 +752,7 @@ export class RouteConnectionHandler {
           record,
           initialChunk,
           this.settings.httpProxyPort || 8443,
-          (reason) => this.connectionManager.initiateCleanupOnce(record, reason)
+          (reason) => this.connectionManager.cleanupConnection(record, reason)
         );
         return;
       } else {
@@ -919,6 +929,7 @@ export class RouteConnectionHandler {
 
   /**
    * Setup improved error handling for the outgoing connection
+   * @deprecated This method is no longer used - error handling is done in createSocketWithErrorHandler
    */
   private setupOutgoingErrorHandler(
     connectionId: string,
@@ -1074,8 +1085,6 @@ export class RouteConnectionHandler {
     }
 
     // Create the target socket with immediate error handling
-    let connectionEstablished = false;
-    
     const targetSocket = createSocketWithErrorHandler({
       port: finalTargetPort,
       host: finalTargetHost,
@@ -1119,8 +1128,6 @@ export class RouteConnectionHandler {
         this.connectionManager.cleanupConnection(record, `connection_failed_${(error as any).code || 'unknown'}`);
       },
       onConnect: () => {
-        connectionEstablished = true;
-        
         if (this.settings.enableDetailedLogging) {
           logger.log('info', `Connection ${connectionId} established to target ${finalTargetHost}:${finalTargetPort}`, {
             connectionId,
@@ -1154,7 +1161,7 @@ export class RouteConnectionHandler {
                 error: err.message,
                 component: 'route-handler'
               });
-              return this.connectionManager.initiateCleanupOnce(record, 'write_error');
+              return this.connectionManager.cleanupConnection(record, 'write_error');
             }
           });
 
@@ -1168,7 +1175,7 @@ export class RouteConnectionHandler {
           socket,
           targetSocket,
           (reason) => {
-            this.connectionManager.initiateCleanupOnce(record, reason);
+            this.connectionManager.cleanupConnection(record, reason);
           }
         );
 
@@ -1252,7 +1259,7 @@ export class RouteConnectionHandler {
             connectionId,
             serverName,
             connInfo,
-            (_connectionId, reason) => this.connectionManager.initiateCleanupOnce(record, reason)
+            (_connectionId, reason) => this.connectionManager.cleanupConnection(record, reason)
           );
 
           // Store the handler in the connection record so we can remove it during cleanup
@@ -1277,7 +1284,7 @@ export class RouteConnectionHandler {
             remoteIP: record.remoteIP,
             component: 'route-handler'
           });
-          this.connectionManager.initiateCleanupOnce(record, reason);
+          this.connectionManager.cleanupConnection(record, reason);
         });
 
         // Mark TLS handshake as complete for TLS connections
@@ -1348,7 +1355,7 @@ export class RouteConnectionHandler {
         record.incomingTerminationReason = 'timeout';
         this.connectionManager.incrementTerminationStat('incoming', 'timeout');
       }
-      this.connectionManager.initiateCleanupOnce(record, 'timeout_incoming');
+      this.connectionManager.cleanupConnection(record, 'timeout_incoming');
     });
 
     targetSocket.on('timeout', () => {
@@ -1375,7 +1382,7 @@ export class RouteConnectionHandler {
         record.outgoingTerminationReason = 'timeout';
         this.connectionManager.incrementTerminationStat('outgoing', 'timeout');
       }
-      this.connectionManager.initiateCleanupOnce(record, 'timeout_outgoing');
+      this.connectionManager.cleanupConnection(record, 'timeout_outgoing');
     });
 
     // Apply socket timeouts