feat(daemon): Add crash log manager with rotation and integrate crash logging; improve IPC & process listener cleanup

This commit is contained in:
2025-09-01 10:32:51 +00:00
parent 9473924fcc
commit c9d924811d
9 changed files with 932 additions and 31 deletions

View File

@@ -2,6 +2,7 @@ import * as plugins from '../plugins.js';
import { EventEmitter } from 'events';
import { ProcessWrapper } from './processwrapper.js';
import { LogPersistence } from './logpersistence.js';
import { CrashLogManager } from './crashlogmanager.js';
import { Logger, ProcessError, handleError } from '../shared/common/utils.errorhandler.js';
import type { IMonitorConfig, IProcessLog } from '../shared/protocol/ipc.types.js';
import type { ProcessId } from '../shared/protocol/id.js';
@@ -15,6 +16,7 @@ export class ProcessMonitor extends EventEmitter {
private logger: Logger;
private logs: IProcessLog[] = [];
private logPersistence: LogPersistence;
private crashLogManager: CrashLogManager;
private processId?: ProcessId;
private currentLogMemorySize: number = 0;
private readonly MAX_LOG_MEMORY_SIZE = 10 * 1024 * 1024; // 10MB
@@ -26,6 +28,11 @@ export class ProcessMonitor extends EventEmitter {
private readonly RESET_WINDOW_MS = 60 * 60 * 1000; // 1 hour
private lastMemoryUsage: number = 0;
private lastCpuUsage: number = 0;
// Store event listeners for cleanup
private logHandler?: (log: IProcessLog) => void;
private startHandler?: (pid: number) => void;
private exitHandler?: (code: number | null, signal: string | null) => Promise<void>;
private errorHandler?: (error: Error | ProcessError) => Promise<void>;
constructor(config: IMonitorConfig & { id?: ProcessId }) {
super();
@@ -33,6 +40,7 @@ export class ProcessMonitor extends EventEmitter {
this.logger = new Logger(`ProcessMonitor:${config.name || 'unnamed'}`);
this.logs = [];
this.logPersistence = new LogPersistence();
this.crashLogManager = new CrashLogManager();
this.processId = config.id;
this.currentLogMemorySize = 0;
}
@@ -83,6 +91,14 @@ export class ProcessMonitor extends EventEmitter {
this.logger.info(`Spawning process: ${this.config.command}`);
// Clear any orphaned pidusage cache entries before spawning
try {
(plugins.pidusage as any)?.clearAll?.();
} catch {}
// Clean up previous listeners if any
this.cleanupListeners();
// Create a new process wrapper
this.processWrapper = new ProcessWrapper({
name: this.config.name || 'unnamed-process',
@@ -94,7 +110,7 @@ export class ProcessMonitor extends EventEmitter {
});
// Set up event handlers
this.processWrapper.on('log', (log: IProcessLog): void => {
this.logHandler = (log: IProcessLog): void => {
// Store the log in our buffer
this.logs.push(log);
if (process.env.TSPM_DEBUG) {
@@ -117,6 +133,7 @@ export class ProcessMonitor extends EventEmitter {
// Remove oldest logs until we're under the memory limit
const removed = this.logs.shift()!;
const removedSize = this.logSizeMap.get(removed) ?? this.estimateLogSize(removed);
this.logSizeMap.delete(removed); // Clean up map entry to prevent memory leak
this.currentLogMemorySize -= removedSize;
}
@@ -127,16 +144,16 @@ export class ProcessMonitor extends EventEmitter {
if (log.type === 'system') {
this.log(log.message);
}
});
};
this.processWrapper.on('log', this.logHandler);
// Re-emit start event with PID for upstream handlers
this.processWrapper.on('start', (pid: number): void => {
this.startHandler = (pid: number): void => {
this.emit('start', pid);
});
};
this.processWrapper.on('start', this.startHandler);
this.processWrapper.on(
'exit',
async (code: number | null, signal: string | null): Promise<void> => {
this.exitHandler = async (code: number | null, signal: string | null): Promise<void> => {
const exitMsg = `Process exited with code ${code}, signal ${signal}.`;
this.logger.info(exitMsg);
this.log(exitMsg);
@@ -149,6 +166,27 @@ export class ProcessMonitor extends EventEmitter {
}
} catch {}
// Detect if this was a crash (non-zero exit code or killed by signal)
const isCrash = (code !== null && code !== 0) || signal !== null;
// Save crash log if this was a crash
if (isCrash && this.processId && this.config.name) {
try {
await this.crashLogManager.saveCrashLog(
this.processId,
this.config.name,
this.logs,
code,
signal,
this.restartCount,
this.lastMemoryUsage
);
this.logger.info(`Saved crash log for process ${this.config.name}`);
} catch (error) {
this.logger.error(`Failed to save crash log: ${error}`);
}
}
// Flush logs to disk on exit
if (this.processId && this.logs.length > 0) {
try {
@@ -169,10 +207,10 @@ export class ProcessMonitor extends EventEmitter {
'Not restarting process because monitor is stopped',
);
}
},
);
};
this.processWrapper.on('exit', this.exitHandler);
this.processWrapper.on('error', async (error: Error | ProcessError): Promise<void> => {
this.errorHandler = async (error: Error | ProcessError): Promise<void> => {
const errorMsg =
error instanceof ProcessError
? `Process error: ${error.toString()}`
@@ -181,6 +219,24 @@ export class ProcessMonitor extends EventEmitter {
this.logger.error(error);
this.log(errorMsg);
// Save crash log for errors
if (this.processId && this.config.name) {
try {
await this.crashLogManager.saveCrashLog(
this.processId,
this.config.name,
this.logs,
null, // no exit code for errors
null, // no signal for errors
this.restartCount,
this.lastMemoryUsage
);
this.logger.info(`Saved crash log for process ${this.config.name} due to error`);
} catch (crashLogError) {
this.logger.error(`Failed to save crash log: ${crashLogError}`);
}
}
// Flush logs to disk on error
if (this.processId && this.logs.length > 0) {
try {
@@ -196,7 +252,8 @@ export class ProcessMonitor extends EventEmitter {
} else {
this.logger.debug('Not restarting process because monitor is stopped');
}
});
};
this.processWrapper.on('error', this.errorHandler);
// Start the process
try {
@@ -210,6 +267,31 @@ export class ProcessMonitor extends EventEmitter {
}
}
/**
* Clean up event listeners from process wrapper
*/
private cleanupListeners(): void {
if (this.processWrapper) {
if (this.logHandler) {
this.processWrapper.removeListener('log', this.logHandler);
}
if (this.startHandler) {
this.processWrapper.removeListener('start', this.startHandler);
}
if (this.exitHandler) {
this.processWrapper.removeListener('exit', this.exitHandler);
}
if (this.errorHandler) {
this.processWrapper.removeListener('error', this.errorHandler);
}
}
// Clear references
this.logHandler = undefined;
this.startHandler = undefined;
this.exitHandler = undefined;
this.errorHandler = undefined;
}
/**
* Schedule a restart with incremental debounce and failure cutoff.
*/
@@ -360,6 +442,14 @@ export class ProcessMonitor extends EventEmitter {
this.logger.debug(
`Total memory for process group: ${this.humanReadableBytes(totalMemory)}`,
);
// Clear pidusage cache for all PIDs to prevent memory leaks
for (const pid of pids) {
try {
(plugins.pidusage as any)?.clear?.(pid);
} catch {}
}
resolve({ memory: totalMemory, cpu: totalCpu });
},
);
@@ -387,6 +477,9 @@ export class ProcessMonitor extends EventEmitter {
this.log('Stopping process monitor.');
this.stopped = true;
// Clean up event listeners
this.cleanupListeners();
// Flush logs to disk before stopping
if (this.processId && this.logs.length > 0) {
try {