dcrouter/test/suite/reliability/test.error-recovery.ts

import * as plugins from '@git.zone/tstest/tapbundle';
import { expect, tap } from '@git.zone/tstest/tapbundle';
import * as net from 'net';
import { startTestServer, stopTestServer } from '../server.loader.js';

const TEST_PORT = 2525;

const createConnection = async (): Promise<net.Socket> => {
  const socket = net.createConnection({
    host: 'localhost',
    port: TEST_PORT,
    timeout: 5000
  });

  await new Promise<void>((resolve, reject) => {
    socket.once('connect', resolve);
    socket.once('error', reject);
  });

  return socket;
};

const getResponse = (socket: net.Socket, commandName: string): Promise<string> => {
  return new Promise((resolve, reject) => {
    const timeout = setTimeout(() => {
      reject(new Error(`${commandName} response timeout`));
    }, 3000);

    socket.once('data', (chunk: Buffer) => {
      clearTimeout(timeout);
      resolve(chunk.toString());
    });
  });
};

const testBasicSmtpFlow = async (socket: net.Socket): Promise<boolean> => {
  try {
    // Read greeting
    await getResponse(socket, 'GREETING');

    // Send EHLO
    socket.write('EHLO recovery-test\r\n');
    const ehloResp = await getResponse(socket, 'EHLO');
    if (!ehloResp.includes('250')) return false;

    // Wait for complete EHLO response
    if (ehloResp.includes('250-')) {
      await new Promise(resolve => setTimeout(resolve, 100));
    }

    socket.write('MAIL FROM:<sender@example.com>\r\n');
    const mailResp = await getResponse(socket, 'MAIL FROM');
    if (!mailResp.includes('250')) return false;

    socket.write('RCPT TO:<recipient@example.com>\r\n');
    const rcptResp = await getResponse(socket, 'RCPT TO');
    if (!rcptResp.includes('250')) return false;

    socket.write('DATA\r\n');
    const dataResp = await getResponse(socket, 'DATA');
    if (!dataResp.includes('354')) return false;

    const testEmail = [
      'From: sender@example.com',
      'To: recipient@example.com',
      'Subject: Recovery Test Email',
      '',
      'This email tests server recovery.',
      '.',
      ''
    ].join('\r\n');

    socket.write(testEmail);
    const finalResp = await getResponse(socket, 'EMAIL DATA');

    socket.write('QUIT\r\n');
    socket.end();

    return finalResp.includes('250');
  } catch (error) {
    console.log('Basic SMTP flow error:', error);
    return false;
  }
};

tap.test('prepare server', async () => {
  await startTestServer();
  await new Promise(resolve => setTimeout(resolve, 100));
});

tap.test('REL-04: Error recovery - Invalid command recovery', async (tools) => {
  const done = tools.defer();

  try {
    console.log('Testing recovery from invalid commands...');

    // Phase 1: Send invalid commands
    const socket1 = await createConnection();
    await getResponse(socket1, 'GREETING');

    // Send multiple invalid commands
    socket1.write('INVALID_COMMAND\r\n');
    const response1 = await getResponse(socket1, 'INVALID');
    expect(response1).toMatch(/50[0-3]/); // Should get error response

    socket1.write('ANOTHER_INVALID\r\n');
    const response2 = await getResponse(socket1, 'INVALID');
    expect(response2).toMatch(/50[0-3]/);

    socket1.write('YET_ANOTHER_BAD_CMD\r\n');
    const response3 = await getResponse(socket1, 'INVALID');
    expect(response3).toMatch(/50[0-3]/);

    socket1.end();

    // Phase 2: Test recovery - server should still work normally
    await new Promise(resolve => setTimeout(resolve, 500));
    
    const socket2 = await createConnection();
    const recoverySuccess = await testBasicSmtpFlow(socket2);

    expect(recoverySuccess).toBeTrue();
    console.log('✓ Server recovered from invalid commands');
    done.resolve();
  } catch (error) {
    done.reject(error);
  }
});

tap.test('REL-04: Error recovery - Malformed data recovery', async (tools) => {
  const done = tools.defer();

  try {
    console.log('\nTesting recovery from malformed data...');

    // Phase 1: Send malformed data
    const socket1 = await createConnection();
    await getResponse(socket1, 'GREETING');

    socket1.write('EHLO testhost\r\n');
    let data = '';
    await new Promise<void>((resolve) => {
      const handleData = (chunk: Buffer) => {
        data += chunk.toString();
        if (data.includes('250 ') && !data.includes('250-')) {
          socket1.removeListener('data', handleData);
          resolve();
        }
      };
      socket1.on('data', handleData);
    });

    // Send malformed MAIL FROM
    socket1.write('MAIL FROM: invalid-format\r\n');
    const response1 = await getResponse(socket1, 'MALFORMED');
    expect(response1).toMatch(/50[0-3]/);

    // Send malformed RCPT TO
    socket1.write('RCPT TO: also-invalid\r\n');
    const response2 = await getResponse(socket1, 'MALFORMED');
    expect(response2).toMatch(/50[0-3]/);

    // Send malformed DATA with binary
    socket1.write('DATA\x00\x01\x02CORRUPTED\r\n');
    const response3 = await getResponse(socket1, 'CORRUPTED');
    expect(response3).toMatch(/50[0-3]/);

    socket1.end();

    // Phase 2: Test recovery
    await new Promise(resolve => setTimeout(resolve, 500));
    
    const socket2 = await createConnection();
    const recoverySuccess = await testBasicSmtpFlow(socket2);

    expect(recoverySuccess).toBeTrue();
    console.log('✓ Server recovered from malformed data');
    done.resolve();
  } catch (error) {
    done.reject(error);
  }
});

tap.test('REL-04: Error recovery - Premature disconnection recovery', async (tools) => {
  const done = tools.defer();

  try {
    console.log('\nTesting recovery from premature disconnection...');

    // Phase 1: Create incomplete transactions
    for (let i = 0; i < 3; i++) {
      const socket = await createConnection();
      await getResponse(socket, 'GREETING');

      socket.write('EHLO abrupt-test\r\n');
      let data = '';
      await new Promise<void>((resolve) => {
        const handleData = (chunk: Buffer) => {
          data += chunk.toString();
          if (data.includes('250 ') && !data.includes('250-')) {
            socket.removeListener('data', handleData);
            resolve();
          }
        };
        socket.on('data', handleData);
      });

      socket.write('MAIL FROM:<test@example.com>\r\n');
      await getResponse(socket, 'MAIL FROM');

      // Abruptly close connection during transaction
      socket.destroy();
      console.log(`  Abruptly closed connection ${i + 1}`);

      await new Promise(resolve => setTimeout(resolve, 200));
    }

    // Phase 2: Test recovery
    await new Promise(resolve => setTimeout(resolve, 1000));
    
    const socket2 = await createConnection();
    const recoverySuccess = await testBasicSmtpFlow(socket2);

    expect(recoverySuccess).toBeTrue();
    console.log('✓ Server recovered from premature disconnections');
    done.resolve();
  } catch (error) {
    done.reject(error);
  }
});

tap.test('REL-04: Error recovery - Data corruption recovery', async (tools) => {
  const done = tools.defer();

  try {
    console.log('\nTesting recovery from data corruption...');

    const socket1 = await createConnection();
    await getResponse(socket1, 'GREETING');

    socket1.write('EHLO corruption-test\r\n');
    let data = '';
    await new Promise<void>((resolve) => {
      const handleData = (chunk: Buffer) => {
        data += chunk.toString();
        if (data.includes('250 ') && !data.includes('250-')) {
          socket1.removeListener('data', handleData);
          resolve();
        }
      };
      socket1.on('data', handleData);
    });

    socket1.write('MAIL FROM:<sender@example.com>\r\n');
    await getResponse(socket1, 'MAIL FROM');

    socket1.write('RCPT TO:<recipient@example.com>\r\n');
    await getResponse(socket1, 'RCPT TO');

    socket1.write('DATA\r\n');
    const dataResp = await getResponse(socket1, 'DATA');
    expect(dataResp).toInclude('354');

    // Send corrupted email data with null bytes and invalid characters
    socket1.write('From: test\r\n\0\0\0CORRUPTED DATA\xff\xfe\r\n');
    socket1.write('Subject: \x01\x02\x03Invalid\r\n');
    socket1.write('\r\n');
    socket1.write('Body with \0null bytes\r\n');
    socket1.write('.\r\n');

    try {
      const response = await getResponse(socket1, 'CORRUPTED DATA');
      console.log('  Server response to corrupted data:', response.substring(0, 50));
    } catch (error) {
      console.log('  Server rejected corrupted data (expected)');
    }

    socket1.end();

    // Phase 2: Test recovery
    await new Promise(resolve => setTimeout(resolve, 1000));
    
    const socket2 = await createConnection();
    const recoverySuccess = await testBasicSmtpFlow(socket2);

    expect(recoverySuccess).toBeTrue();
    console.log('✓ Server recovered from data corruption');
    done.resolve();
  } catch (error) {
    done.reject(error);
  }
});

tap.test('REL-04: Error recovery - Connection flooding recovery', async (tools) => {
  const done = tools.defer();
  const connections: net.Socket[] = [];

  try {
    console.log('\nTesting recovery from connection flooding...');

    // Phase 1: Create multiple rapid connections
    console.log('  Creating 15 rapid connections...');
    for (let i = 0; i < 15; i++) {
      try {
        const socket = net.createConnection({
          host: 'localhost',
          port: TEST_PORT,
          timeout: 2000
        });
        connections.push(socket);
        
        // Don't wait for connection to complete
        await new Promise(resolve => setTimeout(resolve, 50));
      } catch (error) {
        // Some connections might fail - that's expected
        console.log(`  Connection ${i + 1} failed (expected during flooding)`);
      }
    }

    console.log(`  Created ${connections.length} connections`);

    // Close all connections
    connections.forEach(conn => {
      try {
        conn.destroy();
      } catch (e) {
        // Ignore errors
      }
    });

    // Phase 2: Test recovery
    console.log('  Waiting for server to recover...');
    await new Promise(resolve => setTimeout(resolve, 3000));
    
    const socket2 = await createConnection();
    const recoverySuccess = await testBasicSmtpFlow(socket2);

    expect(recoverySuccess).toBeTrue();
    console.log('✓ Server recovered from connection flooding');
    done.resolve();
  } catch (error) {
    connections.forEach(conn => conn.destroy());
    done.reject(error);
  }
});

tap.test('REL-04: Error recovery - Mixed error scenario', async (tools) => {
  const done = tools.defer();

  try {
    console.log('\nTesting recovery from mixed error scenarios...');

    // Create multiple error conditions simultaneously
    const errorPromises = [];

    // Invalid command connection
    errorPromises.push((async () => {
      const socket = await createConnection();
      await getResponse(socket, 'GREETING');
      socket.write('TOTALLY_WRONG\r\n');
      await getResponse(socket, 'WRONG');
      socket.destroy();
    })());

    // Malformed data connection
    errorPromises.push((async () => {
      const socket = await createConnection();
      await getResponse(socket, 'GREETING');
      socket.write('MAIL FROM:<<<invalid>>>\r\n');
      try {
        await getResponse(socket, 'INVALID');
      } catch (e) {
        // Expected
      }
      socket.destroy();
    })());

    // Abrupt disconnection
    errorPromises.push((async () => {
      const socket = await createConnection();
      socket.destroy();
    })());

    // Wait for all errors to execute
    await Promise.allSettled(errorPromises);

    console.log('  All error scenarios executed');

    // Test recovery
    await new Promise(resolve => setTimeout(resolve, 2000));
    
    const socket = await createConnection();
    const recoverySuccess = await testBasicSmtpFlow(socket);

    expect(recoverySuccess).toBeTrue();
    console.log('✓ Server recovered from mixed error scenarios');
    done.resolve();
  } catch (error) {
    done.reject(error);
  }
});

tap.test('cleanup server', async () => {
  await stopTestServer();
});

tap.start();