dcrouter/test/suite/reliability/test.restart-recovery.ts

import * as plugins from '@push.rocks/tapbundle';
import { expect, tap } from '@push.rocks/tapbundle';
import * as net from 'net';
import { startTestServer, stopTestServer } from '../server.loader.js';

const TEST_PORT = 2525;

tap.test('prepare server', async () => {
  await startTestServer();
  await new Promise(resolve => setTimeout(resolve, 100));
});

tap.test('REL-02: Restart recovery - Server state after restart', async (tools) => {
  const done = tools.defer();

  try {
    console.log('Testing server state and recovery capabilities...');

    // First, establish that server is working normally
    const socket1 = net.createConnection({
      host: 'localhost',
      port: TEST_PORT,
      timeout: 30000
    });

    await new Promise<void>((resolve, reject) => {
      socket1.once('connect', resolve);
      socket1.once('error', reject);
    });

    // Read greeting
    const greeting1 = await new Promise<string>((resolve) => {
      socket1.once('data', (chunk) => {
        resolve(chunk.toString());
      });
    });

    expect(greeting1).toInclude('220');
    console.log('Initial connection successful');

    // Send EHLO
    socket1.write('EHLO testhost\r\n');
    
    await new Promise<void>((resolve) => {
      let data = '';
      const handleData = (chunk: Buffer) => {
        data += chunk.toString();
        if (data.includes('250 ') && !data.includes('250-')) {
          socket1.removeListener('data', handleData);
          resolve();
        }
      };
      socket1.on('data', handleData);
    });

    // Complete a transaction
    socket1.write('MAIL FROM:<sender@example.com>\r\n');
    
    await new Promise<void>((resolve) => {
      socket1.once('data', (chunk) => {
        const response = chunk.toString();
        expect(response).toInclude('250');
        resolve();
      });
    });

    socket1.write('RCPT TO:<recipient@example.com>\r\n');
    
    await new Promise<void>((resolve) => {
      socket1.once('data', (chunk) => {
        const response = chunk.toString();
        expect(response).toInclude('250');
        resolve();
      });
    });

    socket1.write('DATA\r\n');
    
    await new Promise<void>((resolve) => {
      socket1.once('data', (chunk) => {
        const response = chunk.toString();
        expect(response).toInclude('354');
        resolve();
      });
    });

    const emailContent = [
      'From: sender@example.com',
      'To: recipient@example.com',
      'Subject: Pre-restart test',
      '',
      'Testing server state before restart.',
      '.',
      ''
    ].join('\r\n');

    socket1.write(emailContent);

    await new Promise<void>((resolve) => {
      socket1.once('data', (chunk) => {
        const response = chunk.toString();
        expect(response).toInclude('250');
        resolve();
      });
    });

    socket1.write('QUIT\r\n');
    socket1.end();

    console.log('Pre-restart transaction completed successfully');

    // Simulate server restart by closing and reopening connections
    console.log('\nSimulating server restart scenario...');
    
    // Wait a moment to simulate restart time
    await new Promise(resolve => setTimeout(resolve, 2000));

    // Test recovery after simulated restart
    const socket2 = net.createConnection({
      host: 'localhost',
      port: TEST_PORT,
      timeout: 30000
    });

    await new Promise<void>((resolve, reject) => {
      socket2.once('connect', resolve);
      socket2.once('error', reject);
    });

    // Read greeting after "restart"
    const greeting2 = await new Promise<string>((resolve) => {
      socket2.once('data', (chunk) => {
        resolve(chunk.toString());
      });
    });

    expect(greeting2).toInclude('220');
    console.log('Post-restart connection successful');

    // Verify server is fully functional after restart
    socket2.write('EHLO testhost-postrestart\r\n');
    
    await new Promise<void>((resolve) => {
      let data = '';
      const handleData = (chunk: Buffer) => {
        data += chunk.toString();
        if (data.includes('250 ') && !data.includes('250-')) {
          socket2.removeListener('data', handleData);
          resolve();
        }
      };
      socket2.on('data', handleData);
    });

    // Complete another transaction to verify full recovery
    socket2.write('MAIL FROM:<sender2@example.com>\r\n');
    
    await new Promise<void>((resolve) => {
      socket2.once('data', (chunk) => {
        const response = chunk.toString();
        expect(response).toInclude('250');
        resolve();
      });
    });

    socket2.write('RCPT TO:<recipient2@example.com>\r\n');
    
    await new Promise<void>((resolve) => {
      socket2.once('data', (chunk) => {
        const response = chunk.toString();
        expect(response).toInclude('250');
        resolve();
      });
    });

    socket2.write('DATA\r\n');
    
    await new Promise<void>((resolve) => {
      socket2.once('data', (chunk) => {
        const response = chunk.toString();
        expect(response).toInclude('354');
        resolve();
      });
    });

    const postRestartEmail = [
      'From: sender2@example.com',
      'To: recipient2@example.com',
      'Subject: Post-restart recovery test',
      '',
      'Testing server recovery after restart.',
      '.',
      ''
    ].join('\r\n');

    socket2.write(postRestartEmail);

    await new Promise<void>((resolve) => {
      socket2.once('data', (chunk) => {
        const response = chunk.toString();
        expect(response).toInclude('250');
        resolve();
      });
    });

    socket2.write('QUIT\r\n');
    socket2.end();

    console.log('Post-restart transaction completed successfully');
    console.log('Server recovered successfully from restart');

    done.resolve();
  } catch (error) {
    done.reject(error);
  }
});

tap.test('REL-02: Restart recovery - Multiple rapid reconnections', async (tools) => {
  const done = tools.defer();
  const rapidConnections = 10;
  let successfulReconnects = 0;

  try {
    console.log(`\nTesting rapid reconnection after disruption (${rapidConnections} attempts)...`);

    for (let i = 0; i < rapidConnections; i++) {
      try {
        const socket = net.createConnection({
          host: 'localhost',
          port: TEST_PORT,
          timeout: 5000
        });

        await new Promise<void>((resolve, reject) => {
          const timeout = setTimeout(() => {
            socket.destroy();
            reject(new Error('Connection timeout'));
          }, 5000);

          socket.once('connect', () => {
            clearTimeout(timeout);
            resolve();
          });
          socket.once('error', (err) => {
            clearTimeout(timeout);
            reject(err);
          });
        });

        // Read greeting
        const greeting = await new Promise<string>((resolve, reject) => {
          const timeout = setTimeout(() => {
            reject(new Error('Greeting timeout'));
          }, 3000);

          socket.once('data', (chunk) => {
            clearTimeout(timeout);
            resolve(chunk.toString());
          });
        });

        if (greeting.includes('220')) {
          successfulReconnects++;
          socket.write('QUIT\r\n');
          socket.end();
        } else {
          socket.destroy();
        }

        // Very short delay between attempts
        await new Promise(resolve => setTimeout(resolve, 100));

      } catch (error) {
        console.log(`Reconnection ${i + 1} failed:`, error.message);
      }
    }

    const reconnectRate = successfulReconnects / rapidConnections;
    console.log(`Successful reconnections: ${successfulReconnects}/${rapidConnections} (${(reconnectRate * 100).toFixed(1)}%)`);

    // Expect high success rate for good recovery
    expect(reconnectRate).toBeGreaterThanOrEqual(0.8);
    done.resolve();
  } catch (error) {
    done.reject(error);
  }
});

tap.test('REL-02: Restart recovery - State persistence check', async (tools) => {
  const done = tools.defer();

  try {
    console.log('\nTesting server state persistence across connections...');

    // Create initial connection and start transaction
    const socket1 = net.createConnection({
      host: 'localhost',
      port: TEST_PORT,
      timeout: 30000
    });

    await new Promise<void>((resolve, reject) => {
      socket1.once('connect', resolve);
      socket1.once('error', reject);
    });

    // Read greeting
    await new Promise<void>((resolve) => {
      socket1.once('data', () => resolve());
    });

    // Send EHLO
    socket1.write('EHLO persistence-test\r\n');
    
    await new Promise<void>((resolve) => {
      let data = '';
      const handleData = (chunk: Buffer) => {
        data += chunk.toString();
        if (data.includes('250 ') && !data.includes('250-')) {
          socket1.removeListener('data', handleData);
          resolve();
        }
      };
      socket1.on('data', handleData);
    });

    // Start transaction but don't complete it
    socket1.write('MAIL FROM:<incomplete@example.com>\r\n');
    
    await new Promise<void>((resolve) => {
      socket1.once('data', (chunk) => {
        const response = chunk.toString();
        expect(response).toInclude('250');
        resolve();
      });
    });

    // Abruptly close connection
    socket1.destroy();
    console.log('Abruptly closed connection with incomplete transaction');

    // Wait briefly
    await new Promise(resolve => setTimeout(resolve, 1000));

    // Create new connection and verify server recovered
    const socket2 = net.createConnection({
      host: 'localhost',
      port: TEST_PORT,
      timeout: 30000
    });

    await new Promise<void>((resolve, reject) => {
      socket2.once('connect', resolve);
      socket2.once('error', reject);
    });

    // Read greeting
    await new Promise<void>((resolve) => {
      socket2.once('data', () => resolve());
    });

    // Send EHLO
    socket2.write('EHLO recovery-test\r\n');
    
    await new Promise<void>((resolve) => {
      let data = '';
      const handleData = (chunk: Buffer) => {
        data += chunk.toString();
        if (data.includes('250 ') && !data.includes('250-')) {
          socket2.removeListener('data', handleData);
          resolve();
        }
      };
      socket2.on('data', handleData);
    });

    // Try new transaction - should work without issues from previous incomplete one
    socket2.write('MAIL FROM:<recovery@example.com>\r\n');
    
    const mailResponse = await new Promise<string>((resolve) => {
      socket2.once('data', (chunk) => {
        resolve(chunk.toString());
      });
    });

    expect(mailResponse).toInclude('250');
    console.log('Server recovered successfully - new transaction started without issues');

    socket2.write('QUIT\r\n');
    socket2.end();

    done.resolve();
  } catch (error) {
    done.reject(error);
  }
});

tap.test('cleanup server', async () => {
  await stopTestServer();
});

tap.start();