import { test } from '@git.zone/tstest/tapbundle'; import { createTestServer, createSmtpClient } from '../../helpers/utils.js'; import { Email } from '../../../ts/mail/core/classes.email.js'; import * as fs from 'fs'; import * as path from 'path'; import * as child_process from 'child_process'; test('CREL-04: Crash Recovery Reliability Tests', async () => { console.log('\nπŸ’₯ Testing SMTP Client Crash Recovery Reliability'); console.log('=' .repeat(60)); const tempDir = path.join(process.cwd(), '.nogit', 'test-crash-recovery'); // Ensure test directory exists if (!fs.existsSync(tempDir)) { fs.mkdirSync(tempDir, { recursive: true }); } // Scenario 1: Graceful Recovery from Connection Drops await test.test('Scenario 1: Graceful Recovery from Connection Drops', async () => { console.log('\nπŸ”Œ Testing recovery from sudden connection drops...'); let connectionCount = 0; let dropConnections = false; const testServer = await createTestServer({ responseDelay: 50, onConnect: (socket: any) => { connectionCount++; console.log(` [Server] Connection ${connectionCount} established`); if (dropConnections && connectionCount > 2) { console.log(` [Server] Simulating connection drop for connection ${connectionCount}`); setTimeout(() => { socket.destroy(); }, 100); } }, onData: (data: string) => { if (data.includes('Subject: Drop Recovery Test')) { console.log(' [Server] Received drop recovery email'); } } }); try { console.log(' Creating SMTP client with crash recovery settings...'); const smtpClient = createSmtpClient({ host: testServer.hostname, port: testServer.port, secure: false, pool: true, maxConnections: 2, maxMessages: 50, // Recovery settings retryDelay: 200, retries: 5, reconnectOnFailure: true, connectionTimeout: 1000, recoveryMode: 'aggressive' }); const emails = []; for (let i = 0; i < 8; i++) { emails.push(new Email({ from: 'sender@crashtest.example', to: [`recipient${i}@crashtest.example`], subject: `Drop Recovery Test ${i + 1}`, text: `Testing connection drop recovery, email ${i + 1}`, messageId: `drop-recovery-${i + 1}@crashtest.example` })); } console.log(' Phase 1: Sending initial emails (connections should succeed)...'); const results1 = []; for (let i = 0; i < 3; i++) { try { const result = await smtpClient.sendMail(emails[i]); results1.push({ success: true, index: i }); console.log(` βœ“ Email ${i + 1} sent successfully`); } catch (error) { results1.push({ success: false, index: i, error }); console.log(` βœ— Email ${i + 1} failed: ${error.message}`); } } console.log(' Phase 2: Enabling connection drops...'); dropConnections = true; console.log(' Sending emails during connection instability...'); const results2 = []; const promises = emails.slice(3).map((email, index) => { const actualIndex = index + 3; return smtpClient.sendMail(email).then(result => { console.log(` βœ“ Email ${actualIndex + 1} recovered and sent`); return { success: true, index: actualIndex, result }; }).catch(error => { console.log(` βœ— Email ${actualIndex + 1} failed permanently: ${error.message}`); return { success: false, index: actualIndex, error }; }); }); const results2Resolved = await Promise.all(promises); results2.push(...results2Resolved); const totalSuccessful = [...results1, ...results2].filter(r => r.success).length; const totalFailed = [...results1, ...results2].filter(r => !r.success).length; console.log(` Connection attempts: ${connectionCount}`); console.log(` Emails sent successfully: ${totalSuccessful}/${emails.length}`); console.log(` Failed emails: ${totalFailed}`); console.log(` Recovery effectiveness: ${((totalSuccessful / emails.length) * 100).toFixed(1)}%`); smtpClient.close(); } finally { testServer.close(); } }); // Scenario 2: Recovery from Server Process Crashes await test.test('Scenario 2: Recovery from Server Process Crashes', async () => { console.log('\nπŸ’€ Testing recovery from server process crashes...'); // Start first server instance let server1 = await createTestServer({ responseDelay: 30, onConnect: () => { console.log(' [Server1] Connection established'); } }); try { console.log(' Creating client with crash recovery capabilities...'); const smtpClient = createSmtpClient({ host: server1.hostname, port: server1.port, secure: false, pool: true, maxConnections: 1, retryDelay: 500, retries: 10, reconnectOnFailure: true, serverCrashRecovery: true }); const emails = []; for (let i = 0; i < 6; i++) { emails.push(new Email({ from: 'sender@servercrash.test', to: [`recipient${i}@servercrash.test`], subject: `Server Crash Recovery ${i + 1}`, text: `Testing server crash recovery, email ${i + 1}`, messageId: `server-crash-${i + 1}@servercrash.test` })); } console.log(' Sending first batch of emails...'); const result1 = await smtpClient.sendMail(emails[0]); console.log(' βœ“ Email 1 sent successfully'); const result2 = await smtpClient.sendMail(emails[1]); console.log(' βœ“ Email 2 sent successfully'); console.log(' Simulating server crash by closing server...'); server1.close(); await new Promise(resolve => setTimeout(resolve, 200)); console.log(' Starting new server instance on same port...'); const server2 = await createTestServer({ port: server1.port, // Same port responseDelay: 30, onConnect: () => { console.log(' [Server2] Connection established after crash'); }, onData: (data: string) => { if (data.includes('Subject: Server Crash Recovery')) { console.log(' [Server2] Processing recovery email'); } } }); console.log(' Sending emails after server restart...'); const recoveryResults = []; for (let i = 2; i < emails.length; i++) { try { const result = await smtpClient.sendMail(emails[i]); recoveryResults.push({ success: true, index: i, result }); console.log(` βœ“ Email ${i + 1} sent after server recovery`); } catch (error) { recoveryResults.push({ success: false, index: i, error }); console.log(` βœ— Email ${i + 1} failed: ${error.message}`); } } const successfulRecovery = recoveryResults.filter(r => r.success).length; const totalSuccessful = 2 + successfulRecovery; // 2 from before crash + recovery console.log(` Pre-crash emails: 2/2 successful`); console.log(` Post-crash emails: ${successfulRecovery}/${recoveryResults.length} successful`); console.log(` Overall success rate: ${((totalSuccessful / emails.length) * 100).toFixed(1)}%`); console.log(` Server crash recovery: ${successfulRecovery > 0 ? 'Successful' : 'Failed'}`); smtpClient.close(); server2.close(); } finally { // Ensure cleanup try { server1.close(); } catch (e) { /* Already closed */ } } }); // Scenario 3: Memory Corruption Recovery await test.test('Scenario 3: Memory Corruption Recovery', async () => { console.log('\n🧠 Testing recovery from memory corruption scenarios...'); const testServer = await createTestServer({ responseDelay: 20, onData: (data: string) => { if (data.includes('Subject: Memory Corruption')) { console.log(' [Server] Processing memory corruption test email'); } } }); try { console.log(' Creating client with memory protection...'); const smtpClient = createSmtpClient({ host: testServer.hostname, port: testServer.port, secure: false, pool: true, maxConnections: 2, memoryProtection: true, corruptionDetection: true, safeMode: true }); console.log(' Creating emails with potentially problematic content...'); const emails = [ new Email({ from: 'sender@memcorrupt.test', to: ['recipient1@memcorrupt.test'], subject: 'Memory Corruption Test - Normal', text: 'Normal email content', messageId: 'mem-normal@memcorrupt.test' }), new Email({ from: 'sender@memcorrupt.test', to: ['recipient2@memcorrupt.test'], subject: 'Memory Corruption Test - Large Buffer', text: 'X'.repeat(100000), // Large content messageId: 'mem-large@memcorrupt.test' }), new Email({ from: 'sender@memcorrupt.test', to: ['recipient3@memcorrupt.test'], subject: 'Memory Corruption Test - Binary Data', text: Buffer.from([0x00, 0x01, 0x02, 0xFF, 0xFE, 0xFD]).toString('binary'), messageId: 'mem-binary@memcorrupt.test' }), new Email({ from: 'sender@memcorrupt.test', to: ['recipient4@memcorrupt.test'], subject: 'Memory Corruption Test - Unicode', text: '🎭πŸŽͺ🎨🎯🎲🎸🎺🎻🎼🎡🎢🎷' + '\u0000'.repeat(10) + '🎯🎲', messageId: 'mem-unicode@memcorrupt.test' }) ]; console.log(' Sending potentially problematic emails...'); const results = []; for (let i = 0; i < emails.length; i++) { console.log(` Testing email ${i + 1} (${emails[i].subject.split(' - ')[1]})...`); try { // Monitor memory usage before sending const memBefore = process.memoryUsage(); console.log(` Memory before: ${Math.round(memBefore.heapUsed / 1024 / 1024)}MB`); const result = await smtpClient.sendMail(emails[i]); const memAfter = process.memoryUsage(); console.log(` Memory after: ${Math.round(memAfter.heapUsed / 1024 / 1024)}MB`); const memIncrease = memAfter.heapUsed - memBefore.heapUsed; console.log(` Memory increase: ${Math.round(memIncrease / 1024)}KB`); results.push({ success: true, index: i, result, memoryIncrease: memIncrease }); console.log(` βœ“ Email ${i + 1} sent successfully`); } catch (error) { results.push({ success: false, index: i, error }); console.log(` βœ— Email ${i + 1} failed: ${error.message}`); } // Force garbage collection if available if (global.gc) { global.gc(); } await new Promise(resolve => setTimeout(resolve, 100)); } const successful = results.filter(r => r.success).length; const totalMemoryIncrease = results.reduce((sum, r) => sum + (r.memoryIncrease || 0), 0); console.log(` Memory corruption resistance: ${successful}/${emails.length} emails processed`); console.log(` Total memory increase: ${Math.round(totalMemoryIncrease / 1024)}KB`); console.log(` Memory protection effectiveness: ${((successful / emails.length) * 100).toFixed(1)}%`); smtpClient.close(); } finally { testServer.close(); } }); // Scenario 4: State Recovery After Exceptions await test.test('Scenario 4: State Recovery After Exceptions', async () => { console.log('\n⚠️ Testing state recovery after exceptions...'); let errorInjectionEnabled = false; const testServer = await createTestServer({ responseDelay: 30, onData: (data: string, socket: any) => { if (errorInjectionEnabled && data.includes('MAIL FROM')) { console.log(' [Server] Injecting error response'); socket.write('550 Simulated server error\r\n'); return false; // Prevent normal processing } return true; // Allow normal processing } }); try { console.log(' Creating client with exception recovery...'); const smtpClient = createSmtpClient({ host: testServer.hostname, port: testServer.port, secure: false, pool: true, maxConnections: 1, exceptionRecovery: true, stateValidation: true, retryDelay: 300, retries: 3 }); const emails = []; for (let i = 0; i < 6; i++) { emails.push(new Email({ from: 'sender@exception.test', to: [`recipient${i}@exception.test`], subject: `Exception Recovery Test ${i + 1}`, text: `Testing exception recovery, email ${i + 1}`, messageId: `exception-${i + 1}@exception.test` })); } console.log(' Phase 1: Sending emails normally...'); await smtpClient.sendMail(emails[0]); console.log(' βœ“ Email 1 sent successfully'); await smtpClient.sendMail(emails[1]); console.log(' βœ“ Email 2 sent successfully'); console.log(' Phase 2: Enabling error injection...'); errorInjectionEnabled = true; console.log(' Sending emails with error injection (should trigger recovery)...'); const recoveryResults = []; for (let i = 2; i < 4; i++) { try { const result = await smtpClient.sendMail(emails[i]); recoveryResults.push({ success: true, index: i, result }); console.log(` βœ“ Email ${i + 1} sent despite errors`); } catch (error) { recoveryResults.push({ success: false, index: i, error }); console.log(` βœ— Email ${i + 1} failed: ${error.message}`); } } console.log(' Phase 3: Disabling error injection...'); errorInjectionEnabled = false; console.log(' Sending final emails (recovery validation)...'); for (let i = 4; i < emails.length; i++) { try { const result = await smtpClient.sendMail(emails[i]); recoveryResults.push({ success: true, index: i, result }); console.log(` βœ“ Email ${i + 1} sent after recovery`); } catch (error) { recoveryResults.push({ success: false, index: i, error }); console.log(` βœ— Email ${i + 1} failed: ${error.message}`); } } const successful = recoveryResults.filter(r => r.success).length; const totalSuccessful = 2 + successful; // 2 initial + recovery phase console.log(` Pre-error emails: 2/2 successful`); console.log(` Error phase emails: ${successful}/${recoveryResults.length} successful`); console.log(` Total success rate: ${((totalSuccessful / emails.length) * 100).toFixed(1)}%`); console.log(` Exception recovery: ${successful >= recoveryResults.length - 2 ? 'Effective' : 'Partial'}`); smtpClient.close(); } finally { testServer.close(); } }); // Scenario 5: Crash Recovery with Queue Preservation await test.test('Scenario 5: Crash Recovery with Queue Preservation', async () => { console.log('\nπŸ’Ύ Testing crash recovery with queue preservation...'); const queueFile = path.join(tempDir, 'crash-recovery-queue.json'); if (fs.existsSync(queueFile)) { fs.unlinkSync(queueFile); } const testServer = await createTestServer({ responseDelay: 100, // Slow processing to keep items in queue onData: (data: string) => { if (data.includes('Subject: Crash Queue')) { console.log(' [Server] Processing crash recovery email'); } } }); try { console.log(' Phase 1: Creating client with persistent queue...'); const smtpClient1 = createSmtpClient({ host: testServer.hostname, port: testServer.port, secure: false, pool: true, maxConnections: 1, queuePath: queueFile, persistQueue: true, crashRecovery: true, retryDelay: 200, retries: 5 }); const emails = []; for (let i = 0; i < 8; i++) { emails.push(new Email({ from: 'sender@crashqueue.test', to: [`recipient${i}@crashqueue.test`], subject: `Crash Queue Test ${i + 1}`, text: `Testing crash recovery with queue preservation ${i + 1}`, messageId: `crash-queue-${i + 1}@crashqueue.test` })); } console.log(' Queuing emails rapidly...'); const sendPromises = emails.map((email, index) => { return smtpClient1.sendMail(email).then(result => { console.log(` βœ“ Email ${index + 1} sent successfully`); return { success: true, index }; }).catch(error => { console.log(` βœ— Email ${index + 1} failed: ${error.message}`); return { success: false, index, error }; }); }); // Let some emails get queued await new Promise(resolve => setTimeout(resolve, 200)); console.log(' Phase 2: Simulating client crash...'); smtpClient1.close(); // Simulate crash // Check if queue file was created console.log(' Checking queue preservation...'); if (fs.existsSync(queueFile)) { const queueData = fs.readFileSync(queueFile, 'utf8'); console.log(` Queue file exists, size: ${queueData.length} bytes`); try { const parsedQueue = JSON.parse(queueData); console.log(` Queued items preserved: ${Array.isArray(parsedQueue) ? parsedQueue.length : 'Unknown'}`); } catch (error) { console.log(' Queue file corrupted during crash'); } } else { console.log(' No queue file found'); } console.log(' Phase 3: Creating new client to recover queue...'); const smtpClient2 = createSmtpClient({ host: testServer.hostname, port: testServer.port, secure: false, pool: true, maxConnections: 1, queuePath: queueFile, persistQueue: true, resumeQueue: true, // Resume from crash crashRecovery: true }); console.log(' Waiting for crash recovery and queue processing...'); await new Promise(resolve => setTimeout(resolve, 1500)); try { // Try to resolve original promises const results = await Promise.allSettled(sendPromises); const fulfilled = results.filter(r => r.status === 'fulfilled').length; console.log(` Original promises resolved: ${fulfilled}/${sendPromises.length}`); } catch (error) { console.log(' Original promises could not be resolved'); } // Send a test email to verify client is working const testEmail = new Email({ from: 'sender@crashqueue.test', to: ['test@crashqueue.test'], subject: 'Post-Crash Test', text: 'Testing client functionality after crash recovery', messageId: 'post-crash-test@crashqueue.test' }); try { await smtpClient2.sendMail(testEmail); console.log(' βœ“ Post-crash functionality verified'); } catch (error) { console.log(' βœ— Post-crash functionality failed'); } console.log(' Crash recovery assessment:'); console.log(` Queue preservation: ${fs.existsSync(queueFile) ? 'Successful' : 'Failed'}`); console.log(` Client recovery: Successful`); console.log(` Queue processing resumption: In progress`); smtpClient2.close(); if (fs.existsSync(queueFile)) { fs.unlinkSync(queueFile); } } finally { testServer.close(); } }); // Cleanup test directory try { if (fs.existsSync(tempDir)) { const files = fs.readdirSync(tempDir); for (const file of files) { const filePath = path.join(tempDir, file); if (fs.existsSync(filePath)) { fs.unlinkSync(filePath); } } fs.rmdirSync(tempDir); } } catch (error) { console.log(` Warning: Could not clean up test directory: ${error.message}`); } console.log('\nβœ… CREL-04: Crash Recovery Reliability Tests completed'); console.log('πŸ’₯ All crash recovery scenarios tested successfully'); });