smartproxy/test/test.zombie-connection-cleanup.node.ts

import { tap, expect } from '@git.zone/tstest/tapbundle';
import * as net from 'net';
import * as plugins from '../ts/plugins.js';

// Import SmartProxy
import { SmartProxy } from '../ts/index.js';

// Import types through type-only imports
import type { ConnectionManager } from '../ts/proxies/smart-proxy/connection-manager.js';
import type { IConnectionRecord } from '../ts/proxies/smart-proxy/models/interfaces.js';

tap.test('zombie connection cleanup - verify inactivity check detects and cleans destroyed sockets', async () => {
  console.log('\n=== Zombie Connection Cleanup Test ===');
  console.log('Purpose: Verify that connections with destroyed sockets are detected and cleaned up');
  console.log('Setup: Client → OuterProxy (8590) → InnerProxy (8591) → Backend (9998)');
  
  // Create backend server that can be controlled
  let acceptConnections = true;
  let destroyImmediately = false;
  const backendConnections: net.Socket[] = [];
  
  const backend = net.createServer((socket) => {
    console.log('Backend: Connection received');
    backendConnections.push(socket);
    
    if (destroyImmediately) {
      console.log('Backend: Destroying connection immediately');
      socket.destroy();
    } else {
      socket.on('data', (data) => {
        console.log('Backend: Received data, echoing back');
        socket.write(data);
      });
    }
  });
  
  await new Promise<void>((resolve) => {
    backend.listen(9998, () => {
      console.log('✓ Backend server started on port 9998');
      resolve();
    });
  });
  
  // Create InnerProxy with faster inactivity check for testing
  const innerProxy = new SmartProxy({
    ports: [8591],
    enableDetailedLogging: true,
    inactivityTimeout: 5000, // 5 seconds for faster testing
    inactivityCheckInterval: 1000, // Check every second
    routes: [{
      name: 'to-backend',
      match: { ports: 8591 },
      action: {
        type: 'forward',
        target: {
          host: 'localhost',
          port: 9998
        }
      }
    }]
  });
  
  // Create OuterProxy with faster inactivity check
  const outerProxy = new SmartProxy({
    ports: [8590],
    enableDetailedLogging: true,
    inactivityTimeout: 5000, // 5 seconds for faster testing
    inactivityCheckInterval: 1000, // Check every second
    routes: [{
      name: 'to-inner',
      match: { ports: 8590 },
      action: {
        type: 'forward',
        target: {
          host: 'localhost',
          port: 8591
        }
      }
    }]
  });
  
  await innerProxy.start();
  console.log('✓ InnerProxy started on port 8591');
  
  await outerProxy.start();
  console.log('✓ OuterProxy started on port 8590');
  
  // Helper to get connection details
  const getConnectionDetails = () => {
    const outerConnMgr = (outerProxy as any).connectionManager as ConnectionManager;
    const innerConnMgr = (innerProxy as any).connectionManager as ConnectionManager;
    
    const outerRecords = Array.from((outerConnMgr as any).connectionRecords.values()) as IConnectionRecord[];
    const innerRecords = Array.from((innerConnMgr as any).connectionRecords.values()) as IConnectionRecord[];
    
    return {
      outer: {
        count: outerConnMgr.getConnectionCount(),
        records: outerRecords,
        zombies: outerRecords.filter(r => 
          !r.connectionClosed && 
          r.incoming?.destroyed && 
          (r.outgoing?.destroyed ?? true)
        ),
        halfZombies: outerRecords.filter(r => 
          !r.connectionClosed && 
          (r.incoming?.destroyed || r.outgoing?.destroyed) &&
          !(r.incoming?.destroyed && (r.outgoing?.destroyed ?? true))
        )
      },
      inner: {
        count: innerConnMgr.getConnectionCount(),
        records: innerRecords,
        zombies: innerRecords.filter(r => 
          !r.connectionClosed && 
          r.incoming?.destroyed && 
          (r.outgoing?.destroyed ?? true)
        ),
        halfZombies: innerRecords.filter(r => 
          !r.connectionClosed && 
          (r.incoming?.destroyed || r.outgoing?.destroyed) &&
          !(r.incoming?.destroyed && (r.outgoing?.destroyed ?? true))
        )
      }
    };
  };
  
  console.log('\n--- Test 1: Create zombie by destroying sockets without events ---');
  
  // Create a connection and forcefully destroy sockets to create zombies
  const client1 = new net.Socket();
  await new Promise<void>((resolve) => {
    client1.connect(8590, 'localhost', () => {
      console.log('Client1 connected to OuterProxy');
      client1.write('GET / HTTP/1.1\r\nHost: test.com\r\n\r\n');
      
      // Wait for connection to be established through the chain
      setTimeout(() => {
        console.log('Forcefully destroying backend connections to create zombies');
        
        // Get connection details before destruction
        const beforeDetails = getConnectionDetails();
        console.log(`Before destruction: Outer=${beforeDetails.outer.count}, Inner=${beforeDetails.inner.count}`);
        
        // Destroy all backend connections without proper close events
        backendConnections.forEach(conn => {
          if (!conn.destroyed) {
            // Remove all listeners to prevent proper cleanup
            conn.removeAllListeners();
            conn.destroy();
          }
        });
        
        // Also destroy the client socket abruptly
        client1.removeAllListeners();
        client1.destroy();
        
        resolve();
      }, 500);
    });
  });
  
  // Check immediately after destruction
  await new Promise(resolve => setTimeout(resolve, 100));
  let details = getConnectionDetails();
  console.log(`\nAfter destruction:`);
  console.log(`  Outer: ${details.outer.count} connections, ${details.outer.zombies.length} zombies, ${details.outer.halfZombies.length} half-zombies`);
  console.log(`  Inner: ${details.inner.count} connections, ${details.inner.zombies.length} zombies, ${details.inner.halfZombies.length} half-zombies`);
  
  // Wait for inactivity check to run (should detect zombies)
  console.log('\nWaiting for inactivity check to detect zombies...');
  await new Promise(resolve => setTimeout(resolve, 2000));
  
  details = getConnectionDetails();
  console.log(`\nAfter first inactivity check:`);
  console.log(`  Outer: ${details.outer.count} connections, ${details.outer.zombies.length} zombies, ${details.outer.halfZombies.length} half-zombies`);
  console.log(`  Inner: ${details.inner.count} connections, ${details.inner.zombies.length} zombies, ${details.inner.halfZombies.length} half-zombies`);
  
  console.log('\n--- Test 2: Create half-zombie by destroying only one socket ---');
  
  // Clear backend connections array
  backendConnections.length = 0;
  
  const client2 = new net.Socket();
  await new Promise<void>((resolve) => {
    client2.connect(8590, 'localhost', () => {
      console.log('Client2 connected to OuterProxy');
      client2.write('GET / HTTP/1.1\r\nHost: test.com\r\n\r\n');
      
      setTimeout(() => {
        console.log('Creating half-zombie by destroying only outgoing socket on outer proxy');
        
        // Access the connection records directly
        const outerConnMgr = (outerProxy as any).connectionManager as ConnectionManager;
        const outerRecords = Array.from((outerConnMgr as any).connectionRecords.values()) as IConnectionRecord[];
        
        // Find the active connection and destroy only its outgoing socket
        const activeRecord = outerRecords.find(r => !r.connectionClosed && r.outgoing && !r.outgoing.destroyed);
        if (activeRecord && activeRecord.outgoing) {
          console.log('Found active connection, destroying outgoing socket');
          activeRecord.outgoing.removeAllListeners();
          activeRecord.outgoing.destroy();
        }
        
        resolve();
      }, 500);
    });
  });
  
  // Check half-zombie state
  await new Promise(resolve => setTimeout(resolve, 100));
  details = getConnectionDetails();
  console.log(`\nAfter creating half-zombie:`);
  console.log(`  Outer: ${details.outer.count} connections, ${details.outer.zombies.length} zombies, ${details.outer.halfZombies.length} half-zombies`);
  console.log(`  Inner: ${details.inner.count} connections, ${details.inner.zombies.length} zombies, ${details.inner.halfZombies.length} half-zombies`);
  
  // Wait for 30-second grace period (simulated by multiple checks)
  console.log('\nWaiting for half-zombie grace period (30 seconds simulated)...');
  
  // Manually age the connection to trigger half-zombie cleanup
  const outerConnMgr = (outerProxy as any).connectionManager as ConnectionManager;
  const records = Array.from((outerConnMgr as any).connectionRecords.values()) as IConnectionRecord[];
  records.forEach(record => {
    if (!record.connectionClosed) {
      // Age the connection by 35 seconds
      record.incomingStartTime -= 35000;
    }
  });
  
  // Trigger inactivity check
  await new Promise(resolve => setTimeout(resolve, 2000));
  
  details = getConnectionDetails();
  console.log(`\nAfter half-zombie cleanup:`);
  console.log(`  Outer: ${details.outer.count} connections, ${details.outer.zombies.length} zombies, ${details.outer.halfZombies.length} half-zombies`);
  console.log(`  Inner: ${details.inner.count} connections, ${details.inner.zombies.length} zombies, ${details.inner.halfZombies.length} half-zombies`);
  
  // Clean up client2 properly
  if (!client2.destroyed) {
    client2.destroy();
  }
  
  console.log('\n--- Test 3: Rapid zombie creation under load ---');
  
  // Create multiple connections rapidly and destroy them
  const rapidClients: net.Socket[] = [];
  
  for (let i = 0; i < 5; i++) {
    const client = new net.Socket();
    rapidClients.push(client);
    
    client.connect(8590, 'localhost', () => {
      console.log(`Rapid client ${i} connected`);
      client.write('GET / HTTP/1.1\r\nHost: test.com\r\n\r\n');
      
      // Destroy after random delay
      setTimeout(() => {
        client.removeAllListeners();
        client.destroy();
      }, Math.random() * 500);
    });
    
    // Small delay between connections
    await new Promise(resolve => setTimeout(resolve, 50));
  }
  
  // Wait a bit
  await new Promise(resolve => setTimeout(resolve, 1000));
  
  details = getConnectionDetails();
  console.log(`\nAfter rapid connections:`);
  console.log(`  Outer: ${details.outer.count} connections, ${details.outer.zombies.length} zombies, ${details.outer.halfZombies.length} half-zombies`);
  console.log(`  Inner: ${details.inner.count} connections, ${details.inner.zombies.length} zombies, ${details.inner.halfZombies.length} half-zombies`);
  
  // Wait for cleanup
  console.log('\nWaiting for final cleanup...');
  await new Promise(resolve => setTimeout(resolve, 3000));
  
  details = getConnectionDetails();
  console.log(`\nFinal state:`);
  console.log(`  Outer: ${details.outer.count} connections, ${details.outer.zombies.length} zombies, ${details.outer.halfZombies.length} half-zombies`);
  console.log(`  Inner: ${details.inner.count} connections, ${details.inner.zombies.length} zombies, ${details.inner.halfZombies.length} half-zombies`);
  
  // Cleanup
  await outerProxy.stop();
  await innerProxy.stop();
  backend.close();
  
  // Verify all connections are cleaned up
  console.log('\n--- Verification ---');
  
  if (details.outer.count === 0 && details.inner.count === 0) {
    console.log('✅ PASS: All zombie connections were cleaned up');
  } else {
    console.log('❌ FAIL: Some connections remain');
  }
  
  expect(details.outer.count).toEqual(0);
  expect(details.inner.count).toEqual(0);
  expect(details.outer.zombies.length).toEqual(0);
  expect(details.inner.zombies.length).toEqual(0);
  expect(details.outer.halfZombies.length).toEqual(0);
  expect(details.inner.halfZombies.length).toEqual(0);
});

tap.start();