dcrouter/test/suite/smtpserver_reliability/test.rel-02.restart-recovery.ts

import * as plugins from '@git.zone/tstest/tapbundle';
import { expect, tap } from '@git.zone/tstest/tapbundle';
import * as net from 'net';
import { startTestServer, stopTestServer } from '../../helpers/server.loader.js';

const TEST_PORT = 2525;

let testServer;

// Helper function to wait for SMTP response
const waitForResponse = (socket: net.Socket, expectedCode?: string, timeout = 5000): Promise<string> => {
  return new Promise((resolve, reject) => {
    let buffer = '';
    const timer = setTimeout(() => {
      socket.removeListener('data', handler);
      reject(new Error(`Timeout waiting for ${expectedCode || 'any'} response`));
    }, timeout);
    
    const handler = (data: Buffer) => {
      buffer += data.toString();
      const lines = buffer.split('\r\n');
      
      // Check if we have a complete response
      for (const line of lines) {
        if (expectedCode) {
          if (line.startsWith(expectedCode + ' ')) {
            clearTimeout(timer);
            socket.removeListener('data', handler);
            resolve(buffer);
            return;
          }
        } else {
          // Any complete response line
          if (line.match(/^\d{3} /)) {
            clearTimeout(timer);
            socket.removeListener('data', handler);
            resolve(buffer);
            return;
          }
        }
      }
    };
    
    socket.on('data', handler);
  });
};

tap.test('prepare server', async () => {
  testServer = await startTestServer({ port: TEST_PORT });
  await new Promise(resolve => setTimeout(resolve, 100));
});

tap.test('REL-02: Restart recovery - Server state after restart', async (tools) => {
  const done = tools.defer();

  try {
    console.log('Testing server state and recovery capabilities...');

    // First, establish that server is working normally
    const socket1 = net.createConnection({
      host: 'localhost',
      port: TEST_PORT,
      timeout: 30000
    });

    await new Promise<void>((resolve, reject) => {
      socket1.once('connect', resolve);
      socket1.once('error', reject);
    });

    // Read greeting
    const greeting1 = await waitForResponse(socket1, '220');
    expect(greeting1).toInclude('220');
    console.log('Initial connection successful');

    // Send EHLO
    socket1.write('EHLO testhost\r\n');
    await waitForResponse(socket1, '250');

    // Complete a transaction
    socket1.write('MAIL FROM:<sender@example.com>\r\n');
    const mailResp1 = await waitForResponse(socket1, '250');
    expect(mailResp1).toInclude('250');

    socket1.write('RCPT TO:<recipient@example.com>\r\n');
    const rcptResp1 = await waitForResponse(socket1, '250');
    expect(rcptResp1).toInclude('250');

    socket1.write('DATA\r\n');
    const dataResp1 = await waitForResponse(socket1, '354');
    expect(dataResp1).toInclude('354');

    const emailContent = [
      'From: sender@example.com',
      'To: recipient@example.com',
      'Subject: Pre-restart test',
      '',
      'Testing server state before restart.',
      '.',
      ''
    ].join('\r\n');

    socket1.write(emailContent);
    const sendResp1 = await waitForResponse(socket1, '250');
    expect(sendResp1).toInclude('250');

    socket1.write('QUIT\r\n');
    await waitForResponse(socket1, '221');
    socket1.end();

    console.log('Pre-restart transaction completed successfully');

    // Simulate server restart by closing and reopening connections
    console.log('\nSimulating server restart scenario...');
    
    // Wait a moment to simulate restart time
    await new Promise(resolve => setTimeout(resolve, 2000));

    // Test recovery after simulated restart
    const socket2 = net.createConnection({
      host: 'localhost',
      port: TEST_PORT,
      timeout: 30000
    });

    await new Promise<void>((resolve, reject) => {
      socket2.once('connect', resolve);
      socket2.once('error', reject);
    });

    // Read greeting after "restart"
    const greeting2 = await new Promise<string>((resolve) => {
      socket2.once('data', (chunk) => {
        resolve(chunk.toString());
      });
    });

    expect(greeting2).toInclude('220');
    console.log('Post-restart connection successful');

    // Verify server is fully functional after restart
    socket2.write('EHLO testhost-postrestart\r\n');
    await waitForResponse(socket2, '250');

    // Complete another transaction to verify full recovery
    socket2.write('MAIL FROM:<sender2@example.com>\r\n');
    const mailResp2 = await waitForResponse(socket2, '250');
    expect(mailResp2).toInclude('250');

    socket2.write('RCPT TO:<recipient2@example.com>\r\n');
    const rcptResp2 = await waitForResponse(socket2, '250');
    expect(rcptResp2).toInclude('250');

    socket2.write('DATA\r\n');
    const dataResp2 = await waitForResponse(socket2, '354');
    expect(dataResp2).toInclude('354');

    const postRestartEmail = [
      'From: sender2@example.com',
      'To: recipient2@example.com',
      'Subject: Post-restart recovery test',
      '',
      'Testing server recovery after restart.',
      '.',
      ''
    ].join('\r\n');

    socket2.write(postRestartEmail);
    const sendResp2 = await waitForResponse(socket2, '250');
    expect(sendResp2).toInclude('250');

    socket2.write('QUIT\r\n');
    await waitForResponse(socket2, '221');
    socket2.end();

    console.log('Post-restart transaction completed successfully');
    console.log('Server recovered successfully from restart');

    done.resolve();
  } catch (error) {
    done.reject(error);
  }
});

tap.test('REL-02: Restart recovery - Multiple rapid reconnections', async (tools) => {
  const done = tools.defer();
  const rapidConnections = 10;
  let successfulReconnects = 0;

  try {
    console.log(`\nTesting rapid reconnection after disruption (${rapidConnections} attempts)...`);

    for (let i = 0; i < rapidConnections; i++) {
      try {
        const socket = net.createConnection({
          host: 'localhost',
          port: TEST_PORT,
          timeout: 5000
        });

        await new Promise<void>((resolve, reject) => {
          const timeout = setTimeout(() => {
            socket.destroy();
            reject(new Error('Connection timeout'));
          }, 5000);

          socket.once('connect', () => {
            clearTimeout(timeout);
            resolve();
          });
          socket.once('error', (err) => {
            clearTimeout(timeout);
            reject(err);
          });
        });

        // Read greeting
        try {
          const greeting = await waitForResponse(socket, '220', 3000);
          if (greeting.includes('220')) {
            successfulReconnects++;
            socket.write('QUIT\r\n');
            await waitForResponse(socket, '221', 1000).catch(() => {});
            socket.end();
          } else {
            socket.destroy();
          }
        } catch (error) {
          socket.destroy();
          throw error;
        }

        // Very short delay between attempts
        await new Promise(resolve => setTimeout(resolve, 100));

      } catch (error) {
        console.log(`Reconnection ${i + 1} failed:`, error.message);
      }
    }

    const reconnectRate = successfulReconnects / rapidConnections;
    console.log(`Successful reconnections: ${successfulReconnects}/${rapidConnections} (${(reconnectRate * 100).toFixed(1)}%)`);

    // Expect high success rate for good recovery
    expect(reconnectRate).toBeGreaterThanOrEqual(0.8);
    done.resolve();
  } catch (error) {
    done.reject(error);
  }
});

tap.test('REL-02: Restart recovery - State persistence check', async (tools) => {
  const done = tools.defer();

  try {
    console.log('\nTesting server state persistence across connections...');

    // Create initial connection and start transaction
    const socket1 = net.createConnection({
      host: 'localhost',
      port: TEST_PORT,
      timeout: 30000
    });

    await new Promise<void>((resolve, reject) => {
      socket1.once('connect', resolve);
      socket1.once('error', reject);
    });

    // Read greeting
    await waitForResponse(socket1, '220');

    // Send EHLO
    socket1.write('EHLO persistence-test\r\n');
    await waitForResponse(socket1, '250');

    // Start transaction but don't complete it
    socket1.write('MAIL FROM:<incomplete@example.com>\r\n');
    const mailResp = await waitForResponse(socket1, '250');
    expect(mailResp).toInclude('250');

    // Abruptly close connection
    socket1.destroy();
    console.log('Abruptly closed connection with incomplete transaction');

    // Wait briefly
    await new Promise(resolve => setTimeout(resolve, 1000));

    // Create new connection and verify server recovered
    const socket2 = net.createConnection({
      host: 'localhost',
      port: TEST_PORT,
      timeout: 30000
    });

    await new Promise<void>((resolve, reject) => {
      socket2.once('connect', resolve);
      socket2.once('error', reject);
    });

    // Read greeting
    await waitForResponse(socket2, '220');

    // Send EHLO
    socket2.write('EHLO recovery-test\r\n');
    await waitForResponse(socket2, '250');

    // Try new transaction - should work without issues from previous incomplete one
    socket2.write('MAIL FROM:<recovery@example.com>\r\n');
    const mailResponse = await waitForResponse(socket2, '250');
    expect(mailResponse).toInclude('250');
    console.log('Server recovered successfully - new transaction started without issues');

    socket2.write('QUIT\r\n');
    await waitForResponse(socket2, '221');
    socket2.end();

    done.resolve();
  } catch (error) {
    done.reject(error);
  }
});

tap.test('cleanup server', async () => {
  await stopTestServer(testServer);
});

export default tap.start();