feat(daemon): Add crash log manager with rotation and integrate crash logging; improve IPC & process listener cleanup
This commit is contained in:
148
test/test.crashlog.direct.ts
Normal file
148
test/test.crashlog.direct.ts
Normal file
@@ -0,0 +1,148 @@
|
||||
#!/usr/bin/env tsx
|
||||
|
||||
import { CrashLogManager } from '../ts/daemon/crashlogmanager.js';
|
||||
import type { IProcessLog } from '../ts/shared/protocol/ipc.types.js';
|
||||
import * as plugins from '../ts/plugins.js';
|
||||
import * as paths from '../ts/paths.js';
|
||||
import * as fs from 'fs/promises';
|
||||
|
||||
async function testCrashLogManager() {
|
||||
console.log('🧪 Testing CrashLogManager directly...\n');
|
||||
|
||||
const crashLogManager = new CrashLogManager();
|
||||
const crashLogsDir = plugins.path.join(paths.tspmDir, 'crashlogs');
|
||||
|
||||
// Clean up any existing crash logs
|
||||
console.log('📁 Cleaning up existing crash logs...');
|
||||
try {
|
||||
await fs.rm(crashLogsDir, { recursive: true, force: true });
|
||||
} catch {}
|
||||
|
||||
// Create test logs
|
||||
const testLogs: IProcessLog[] = [
|
||||
{
|
||||
timestamp: Date.now() - 5000,
|
||||
message: '[TEST] Process starting up...',
|
||||
type: 'stdout'
|
||||
},
|
||||
{
|
||||
timestamp: Date.now() - 4000,
|
||||
message: '[TEST] Initializing components...',
|
||||
type: 'stdout'
|
||||
},
|
||||
{
|
||||
timestamp: Date.now() - 3000,
|
||||
message: '[TEST] Running main loop...',
|
||||
type: 'stdout'
|
||||
},
|
||||
{
|
||||
timestamp: Date.now() - 2000,
|
||||
message: '[TEST] Warning: Memory usage high',
|
||||
type: 'stderr'
|
||||
},
|
||||
{
|
||||
timestamp: Date.now() - 1000,
|
||||
message: '[TEST] Error: Unhandled exception occurred!',
|
||||
type: 'stderr'
|
||||
},
|
||||
{
|
||||
timestamp: Date.now() - 500,
|
||||
message: '[TEST] Fatal: Process crashing with exit code 42',
|
||||
type: 'stderr'
|
||||
}
|
||||
];
|
||||
|
||||
// Test saving a crash log
|
||||
console.log('💾 Saving crash log...');
|
||||
await crashLogManager.saveCrashLog(
|
||||
1 as any, // ProcessId
|
||||
'test-process',
|
||||
testLogs,
|
||||
42, // exit code
|
||||
null, // signal
|
||||
3, // restart count
|
||||
1024 * 1024 * 50 // 50MB memory usage
|
||||
);
|
||||
|
||||
// Check if crash log was created
|
||||
console.log('🔍 Checking for crash log...');
|
||||
const crashLogFiles = await fs.readdir(crashLogsDir).catch(() => []);
|
||||
console.log(` Found ${crashLogFiles.length} crash log files:`);
|
||||
crashLogFiles.forEach(file => console.log(` - ${file}`));
|
||||
|
||||
if (crashLogFiles.length === 0) {
|
||||
console.error('❌ No crash logs were created!');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Read and display the crash log
|
||||
const crashLogFile = crashLogFiles[0];
|
||||
const crashLogPath = plugins.path.join(crashLogsDir, crashLogFile);
|
||||
const crashLogContent = await fs.readFile(crashLogPath, 'utf-8');
|
||||
|
||||
console.log('\n📋 Crash log content:');
|
||||
console.log('─'.repeat(60));
|
||||
console.log(crashLogContent);
|
||||
console.log('─'.repeat(60));
|
||||
|
||||
// Verify content
|
||||
const checks = [
|
||||
{ text: 'CRASH REPORT', found: crashLogContent.includes('CRASH REPORT') },
|
||||
{ text: 'Exit Code: 42', found: crashLogContent.includes('Exit Code: 42') },
|
||||
{ text: 'Restart Attempt: 3/10', found: crashLogContent.includes('Restart Attempt: 3/10') },
|
||||
{ text: 'Memory Usage: 50 MB', found: crashLogContent.includes('Memory Usage: 50 MB') },
|
||||
{ text: 'Fatal: Process crashing', found: crashLogContent.includes('Fatal: Process crashing') }
|
||||
];
|
||||
|
||||
console.log('\n✅ Verification:');
|
||||
checks.forEach(check => {
|
||||
console.log(` ${check.found ? '✓' : '✗'} Contains "${check.text}"`);
|
||||
});
|
||||
|
||||
const allChecksPassed = checks.every(c => c.found);
|
||||
|
||||
// Test rotation (create 100+ logs to test limit)
|
||||
console.log('\n🔄 Testing rotation (creating 105 crash logs)...');
|
||||
for (let i = 2; i <= 105; i++) {
|
||||
await crashLogManager.saveCrashLog(
|
||||
i as any,
|
||||
`test-process-${i}`,
|
||||
testLogs,
|
||||
i,
|
||||
null,
|
||||
1,
|
||||
1024 * 1024 * 10
|
||||
);
|
||||
// Small delay to ensure different timestamps
|
||||
await new Promise(resolve => setTimeout(resolve, 10));
|
||||
}
|
||||
|
||||
// Check that we have exactly 100 logs (rotation working)
|
||||
const finalLogFiles = await fs.readdir(crashLogsDir);
|
||||
console.log(` After rotation: ${finalLogFiles.length} crash logs (should be 100)`);
|
||||
|
||||
if (finalLogFiles.length !== 100) {
|
||||
console.error(`❌ Rotation failed! Expected 100 logs, got ${finalLogFiles.length}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Verify oldest logs were deleted (test-process should be gone)
|
||||
const hasOriginal = finalLogFiles.some(f => f.includes('_1_test-process.log'));
|
||||
if (hasOriginal) {
|
||||
console.error('❌ Rotation failed! Oldest log still exists');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (allChecksPassed) {
|
||||
console.log('\n✅ All crash log tests passed!');
|
||||
} else {
|
||||
console.log('\n❌ Some crash log tests failed!');
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Run the test
|
||||
testCrashLogManager().catch(error => {
|
||||
console.error('❌ Test failed:', error);
|
||||
process.exit(1);
|
||||
});
|
||||
137
test/test.crashlog.manual.ts
Normal file
137
test/test.crashlog.manual.ts
Normal file
@@ -0,0 +1,137 @@
|
||||
#!/usr/bin/env tsx
|
||||
|
||||
import * as plugins from '../ts/plugins.js';
|
||||
import * as paths from '../ts/paths.js';
|
||||
import * as fs from 'fs/promises';
|
||||
import { execSync } from 'child_process';
|
||||
|
||||
// Test process that will crash
|
||||
const CRASH_SCRIPT = `
|
||||
setInterval(() => {
|
||||
console.log('[test] Process is running...');
|
||||
}, 1000);
|
||||
|
||||
setTimeout(() => {
|
||||
console.error('[test] About to crash with non-zero exit code!');
|
||||
process.exit(42);
|
||||
}, 3000);
|
||||
`;
|
||||
|
||||
async function testCrashLog() {
|
||||
console.log('🧪 Testing crash log functionality...\n');
|
||||
|
||||
const crashScriptPath = plugins.path.join(paths.tspmDir, 'test-crash-script.js');
|
||||
const crashLogsDir = plugins.path.join(paths.tspmDir, 'crashlogs');
|
||||
|
||||
try {
|
||||
// Clean up any existing crash logs
|
||||
console.log('📁 Cleaning up existing crash logs...');
|
||||
try {
|
||||
await fs.rm(crashLogsDir, { recursive: true, force: true });
|
||||
} catch {}
|
||||
|
||||
// Write the crash script
|
||||
console.log('📝 Writing test crash script...');
|
||||
await fs.writeFile(crashScriptPath, CRASH_SCRIPT);
|
||||
|
||||
// Stop any existing daemon
|
||||
console.log('🛑 Stopping any existing daemon...');
|
||||
try {
|
||||
execSync('tsx ts/cli.ts daemon stop', { stdio: 'inherit' });
|
||||
} catch {}
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
|
||||
// Start the daemon
|
||||
console.log('🚀 Starting daemon...');
|
||||
execSync('tsx ts/cli.ts daemon start', { stdio: 'inherit' });
|
||||
await new Promise(resolve => setTimeout(resolve, 2000));
|
||||
|
||||
// Add a process that will crash
|
||||
console.log('➕ Adding crash test process...');
|
||||
const addOutput = execSync(`tsx ts/cli.ts add "node ${crashScriptPath}" --name crash-test`, { encoding: 'utf-8' });
|
||||
console.log(addOutput);
|
||||
|
||||
// Extract process ID from output
|
||||
const idMatch = addOutput.match(/Process added with ID: (\d+)/);
|
||||
if (!idMatch) {
|
||||
throw new Error('Could not extract process ID from output');
|
||||
}
|
||||
const processId = parseInt(idMatch[1]);
|
||||
console.log(` Process ID: ${processId}`);
|
||||
|
||||
// Start the process
|
||||
console.log('▶️ Starting process that will crash...');
|
||||
execSync(`tsx ts/cli.ts start ${processId}`, { stdio: 'inherit' });
|
||||
|
||||
// Wait for the process to crash (it crashes after 3 seconds)
|
||||
console.log('⏳ Waiting for process to crash...');
|
||||
await new Promise(resolve => setTimeout(resolve, 5000));
|
||||
|
||||
// Check if crash log was created
|
||||
console.log('🔍 Checking for crash log...');
|
||||
const crashLogFiles = await fs.readdir(crashLogsDir).catch(() => []);
|
||||
console.log(` Found ${crashLogFiles.length} crash log files:`);
|
||||
crashLogFiles.forEach(file => console.log(` - ${file}`));
|
||||
|
||||
if (crashLogFiles.length === 0) {
|
||||
throw new Error('No crash logs were created!');
|
||||
}
|
||||
|
||||
// Find the crash log for our test process
|
||||
const testCrashLog = crashLogFiles.find(file => file.includes('crash-test'));
|
||||
if (!testCrashLog) {
|
||||
throw new Error('Could not find crash log for test process');
|
||||
}
|
||||
|
||||
// Read and display crash log content
|
||||
const crashLogPath = plugins.path.join(crashLogsDir, testCrashLog);
|
||||
const crashLogContent = await fs.readFile(crashLogPath, 'utf-8');
|
||||
|
||||
console.log('\n📋 Crash log content:');
|
||||
console.log('─'.repeat(60));
|
||||
console.log(crashLogContent);
|
||||
console.log('─'.repeat(60));
|
||||
|
||||
// Verify crash log contains expected information
|
||||
const checks = [
|
||||
{ text: 'CRASH REPORT', found: crashLogContent.includes('CRASH REPORT') },
|
||||
{ text: 'Exit Code: 42', found: crashLogContent.includes('Exit Code: 42') },
|
||||
{ text: 'About to crash', found: crashLogContent.includes('About to crash') },
|
||||
{ text: 'Process is running', found: crashLogContent.includes('Process is running') }
|
||||
];
|
||||
|
||||
console.log('\n✅ Verification:');
|
||||
checks.forEach(check => {
|
||||
console.log(` ${check.found ? '✓' : '✗'} Contains "${check.text}"`);
|
||||
});
|
||||
|
||||
const allChecksPassed = checks.every(c => c.found);
|
||||
|
||||
// Clean up
|
||||
console.log('\n🧹 Cleaning up...');
|
||||
execSync(`tsx ts/cli.ts delete ${processId}`, { stdio: 'inherit' });
|
||||
execSync('tsx ts/cli.ts daemon stop', { stdio: 'inherit' });
|
||||
await fs.unlink(crashScriptPath).catch(() => {});
|
||||
|
||||
if (allChecksPassed) {
|
||||
console.log('\n✅ All crash log tests passed!');
|
||||
} else {
|
||||
console.log('\n❌ Some crash log tests failed!');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error('\n❌ Test failed:', error);
|
||||
|
||||
// Clean up on error
|
||||
try {
|
||||
execSync('tsx ts/cli.ts daemon stop', { stdio: 'inherit' });
|
||||
await fs.unlink(crashScriptPath).catch(() => {});
|
||||
} catch {}
|
||||
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Run the test
|
||||
testCrashLog();
|
||||
172
test/test.crashlog.ts
Normal file
172
test/test.crashlog.ts
Normal file
@@ -0,0 +1,172 @@
|
||||
import { tap, expect } from '@git.zone/tstest/tapbundle';
|
||||
import * as plugins from '../ts/plugins.js';
|
||||
import * as paths from '../ts/paths.js';
|
||||
import * as fs from 'fs/promises';
|
||||
|
||||
// Import tspm client
|
||||
import { tspmIpcClient } from '../ts/client/tspm.ipcclient.js';
|
||||
|
||||
// Test process that will crash
|
||||
const CRASH_SCRIPT = `
|
||||
setInterval(() => {
|
||||
console.log('[test] Process is running...');
|
||||
}, 1000);
|
||||
|
||||
setTimeout(() => {
|
||||
console.error('[test] About to crash with non-zero exit code!');
|
||||
process.exit(42);
|
||||
}, 3000);
|
||||
`;
|
||||
|
||||
tap.test('should create crash logs when process crashes', async (tools) => {
|
||||
const crashScriptPath = plugins.path.join(paths.tspmDir, 'test-crash-script.js');
|
||||
const crashLogsDir = plugins.path.join(paths.tspmDir, 'crashlogs');
|
||||
|
||||
// Clean up any existing crash logs
|
||||
try {
|
||||
await fs.rm(crashLogsDir, { recursive: true, force: true });
|
||||
} catch {}
|
||||
|
||||
// Write the crash script
|
||||
await fs.writeFile(crashScriptPath, CRASH_SCRIPT);
|
||||
|
||||
// Start the daemon
|
||||
console.log('Starting daemon...');
|
||||
const daemonResult = await tools.runCommand('tsx ts/cli/tspm.cli.ts daemon start');
|
||||
expect(daemonResult.exitCode).toEqual(0);
|
||||
|
||||
// Wait for daemon to be ready
|
||||
await tools.wait(2000);
|
||||
|
||||
// Add a process that will crash
|
||||
console.log('Adding crash test process...');
|
||||
const addResult = await tools.runCommand(`tsx ts/cli/tspm.cli.ts add "node ${crashScriptPath}" --name crash-test`);
|
||||
expect(addResult.exitCode).toEqual(0);
|
||||
|
||||
// Extract process ID from output
|
||||
const idMatch = addResult.stdout.match(/Process added with ID: (\d+)/);
|
||||
expect(idMatch).toBeTruthy();
|
||||
const processId = parseInt(idMatch![1]);
|
||||
|
||||
// Start the process
|
||||
console.log('Starting process that will crash...');
|
||||
const startResult = await tools.runCommand(`tsx ts/cli/tspm.cli.ts start ${processId}`);
|
||||
expect(startResult.exitCode).toEqual(0);
|
||||
|
||||
// Wait for the process to crash (it crashes after 3 seconds)
|
||||
console.log('Waiting for process to crash...');
|
||||
await tools.wait(5000);
|
||||
|
||||
// Check if crash log was created
|
||||
console.log('Checking for crash log...');
|
||||
const crashLogFiles = await fs.readdir(crashLogsDir).catch(() => []);
|
||||
console.log(`Found ${crashLogFiles.length} crash log files:`, crashLogFiles);
|
||||
|
||||
// Should have at least one crash log
|
||||
expect(crashLogFiles.length).toBeGreaterThan(0);
|
||||
|
||||
// Find the crash log for our test process
|
||||
const testCrashLog = crashLogFiles.find(file => file.includes('crash-test'));
|
||||
expect(testCrashLog).toBeTruthy();
|
||||
|
||||
// Read and verify crash log content
|
||||
const crashLogPath = plugins.path.join(crashLogsDir, testCrashLog!);
|
||||
const crashLogContent = await fs.readFile(crashLogPath, 'utf-8');
|
||||
|
||||
console.log('Crash log content:');
|
||||
console.log(crashLogContent);
|
||||
|
||||
// Verify crash log contains expected information
|
||||
expect(crashLogContent).toIncludeIgnoreCase('crash report');
|
||||
expect(crashLogContent).toIncludeIgnoreCase('exit code: 42');
|
||||
expect(crashLogContent).toIncludeIgnoreCase('About to crash');
|
||||
|
||||
// Stop the process
|
||||
console.log('Cleaning up...');
|
||||
await tools.runCommand(`tsx ts/cli/tspm.cli.ts delete ${processId}`);
|
||||
|
||||
// Stop the daemon
|
||||
await tools.runCommand('tsx ts/cli/tspm.cli.ts daemon stop');
|
||||
|
||||
// Clean up test file
|
||||
await fs.unlink(crashScriptPath).catch(() => {});
|
||||
});
|
||||
|
||||
tap.test('should create crash logs when process is killed', async (tools) => {
|
||||
const killScriptPath = plugins.path.join(paths.tspmDir, 'test-kill-script.js');
|
||||
const crashLogsDir = plugins.path.join(paths.tspmDir, 'crashlogs');
|
||||
|
||||
// Write a script that runs indefinitely
|
||||
const KILL_SCRIPT = `
|
||||
setInterval(() => {
|
||||
console.log('[test] Process is running and will be killed...');
|
||||
}, 500);
|
||||
`;
|
||||
|
||||
await fs.writeFile(killScriptPath, KILL_SCRIPT);
|
||||
|
||||
// Start the daemon
|
||||
console.log('Starting daemon...');
|
||||
const daemonResult = await tools.runCommand('tsx ts/cli/tspm.cli.ts daemon start');
|
||||
expect(daemonResult.exitCode).toEqual(0);
|
||||
|
||||
// Wait for daemon to be ready
|
||||
await tools.wait(2000);
|
||||
|
||||
// Add a process that we'll kill
|
||||
console.log('Adding kill test process...');
|
||||
const addResult = await tools.runCommand(`tsx ts/cli/tspm.cli.ts add "node ${killScriptPath}" --name kill-test`);
|
||||
expect(addResult.exitCode).toEqual(0);
|
||||
|
||||
// Extract process ID
|
||||
const idMatch = addResult.stdout.match(/Process added with ID: (\d+)/);
|
||||
expect(idMatch).toBeTruthy();
|
||||
const processId = parseInt(idMatch![1]);
|
||||
|
||||
// Start the process
|
||||
console.log('Starting process to be killed...');
|
||||
const startResult = await tools.runCommand(`tsx ts/cli/tspm.cli.ts start ${processId}`);
|
||||
expect(startResult.exitCode).toEqual(0);
|
||||
|
||||
// Wait for process to run a bit
|
||||
await tools.wait(2000);
|
||||
|
||||
// Get the actual PID of the running process
|
||||
const statusResult = await tools.runCommand(`tsx ts/cli/tspm.cli.ts describe ${processId}`);
|
||||
const pidMatch = statusResult.stdout.match(/pid:\s+(\d+)/);
|
||||
|
||||
if (pidMatch) {
|
||||
const pid = parseInt(pidMatch[1]);
|
||||
console.log(`Killing process with PID ${pid}...`);
|
||||
|
||||
// Kill the process with SIGTERM
|
||||
await tools.runCommand(`kill -TERM ${pid}`);
|
||||
|
||||
// Wait for crash log to be created
|
||||
await tools.wait(3000);
|
||||
|
||||
// Check for crash log
|
||||
console.log('Checking for crash log from killed process...');
|
||||
const crashLogFiles = await fs.readdir(crashLogsDir).catch(() => []);
|
||||
const killCrashLog = crashLogFiles.find(file => file.includes('kill-test'));
|
||||
|
||||
if (killCrashLog) {
|
||||
const crashLogPath = plugins.path.join(crashLogsDir, killCrashLog);
|
||||
const crashLogContent = await fs.readFile(crashLogPath, 'utf-8');
|
||||
|
||||
console.log('Kill crash log content:');
|
||||
console.log(crashLogContent);
|
||||
|
||||
// Verify it contains signal information
|
||||
expect(crashLogContent).toIncludeIgnoreCase('signal: SIGTERM');
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up
|
||||
console.log('Cleaning up...');
|
||||
await tools.runCommand(`tsx ts/cli/tspm.cli.ts delete ${processId}`);
|
||||
await tools.runCommand('tsx ts/cli/tspm.cli.ts daemon stop');
|
||||
await fs.unlink(killScriptPath).catch(() => {});
|
||||
});
|
||||
|
||||
export default tap.start();
|
||||
Reference in New Issue
Block a user