feat(daemon): Add crash log manager with rotation and integrate crash logging; improve IPC & process listener cleanup
This commit is contained in:
@@ -2,6 +2,7 @@ import * as plugins from '../plugins.js';
|
||||
import { EventEmitter } from 'events';
|
||||
import { ProcessWrapper } from './processwrapper.js';
|
||||
import { LogPersistence } from './logpersistence.js';
|
||||
import { CrashLogManager } from './crashlogmanager.js';
|
||||
import { Logger, ProcessError, handleError } from '../shared/common/utils.errorhandler.js';
|
||||
import type { IMonitorConfig, IProcessLog } from '../shared/protocol/ipc.types.js';
|
||||
import type { ProcessId } from '../shared/protocol/id.js';
|
||||
@@ -15,6 +16,7 @@ export class ProcessMonitor extends EventEmitter {
|
||||
private logger: Logger;
|
||||
private logs: IProcessLog[] = [];
|
||||
private logPersistence: LogPersistence;
|
||||
private crashLogManager: CrashLogManager;
|
||||
private processId?: ProcessId;
|
||||
private currentLogMemorySize: number = 0;
|
||||
private readonly MAX_LOG_MEMORY_SIZE = 10 * 1024 * 1024; // 10MB
|
||||
@@ -26,6 +28,11 @@ export class ProcessMonitor extends EventEmitter {
|
||||
private readonly RESET_WINDOW_MS = 60 * 60 * 1000; // 1 hour
|
||||
private lastMemoryUsage: number = 0;
|
||||
private lastCpuUsage: number = 0;
|
||||
// Store event listeners for cleanup
|
||||
private logHandler?: (log: IProcessLog) => void;
|
||||
private startHandler?: (pid: number) => void;
|
||||
private exitHandler?: (code: number | null, signal: string | null) => Promise<void>;
|
||||
private errorHandler?: (error: Error | ProcessError) => Promise<void>;
|
||||
|
||||
constructor(config: IMonitorConfig & { id?: ProcessId }) {
|
||||
super();
|
||||
@@ -33,6 +40,7 @@ export class ProcessMonitor extends EventEmitter {
|
||||
this.logger = new Logger(`ProcessMonitor:${config.name || 'unnamed'}`);
|
||||
this.logs = [];
|
||||
this.logPersistence = new LogPersistence();
|
||||
this.crashLogManager = new CrashLogManager();
|
||||
this.processId = config.id;
|
||||
this.currentLogMemorySize = 0;
|
||||
}
|
||||
@@ -83,6 +91,14 @@ export class ProcessMonitor extends EventEmitter {
|
||||
|
||||
this.logger.info(`Spawning process: ${this.config.command}`);
|
||||
|
||||
// Clear any orphaned pidusage cache entries before spawning
|
||||
try {
|
||||
(plugins.pidusage as any)?.clearAll?.();
|
||||
} catch {}
|
||||
|
||||
// Clean up previous listeners if any
|
||||
this.cleanupListeners();
|
||||
|
||||
// Create a new process wrapper
|
||||
this.processWrapper = new ProcessWrapper({
|
||||
name: this.config.name || 'unnamed-process',
|
||||
@@ -94,7 +110,7 @@ export class ProcessMonitor extends EventEmitter {
|
||||
});
|
||||
|
||||
// Set up event handlers
|
||||
this.processWrapper.on('log', (log: IProcessLog): void => {
|
||||
this.logHandler = (log: IProcessLog): void => {
|
||||
// Store the log in our buffer
|
||||
this.logs.push(log);
|
||||
if (process.env.TSPM_DEBUG) {
|
||||
@@ -117,6 +133,7 @@ export class ProcessMonitor extends EventEmitter {
|
||||
// Remove oldest logs until we're under the memory limit
|
||||
const removed = this.logs.shift()!;
|
||||
const removedSize = this.logSizeMap.get(removed) ?? this.estimateLogSize(removed);
|
||||
this.logSizeMap.delete(removed); // Clean up map entry to prevent memory leak
|
||||
this.currentLogMemorySize -= removedSize;
|
||||
}
|
||||
|
||||
@@ -127,16 +144,16 @@ export class ProcessMonitor extends EventEmitter {
|
||||
if (log.type === 'system') {
|
||||
this.log(log.message);
|
||||
}
|
||||
});
|
||||
};
|
||||
this.processWrapper.on('log', this.logHandler);
|
||||
|
||||
// Re-emit start event with PID for upstream handlers
|
||||
this.processWrapper.on('start', (pid: number): void => {
|
||||
this.startHandler = (pid: number): void => {
|
||||
this.emit('start', pid);
|
||||
});
|
||||
};
|
||||
this.processWrapper.on('start', this.startHandler);
|
||||
|
||||
this.processWrapper.on(
|
||||
'exit',
|
||||
async (code: number | null, signal: string | null): Promise<void> => {
|
||||
this.exitHandler = async (code: number | null, signal: string | null): Promise<void> => {
|
||||
const exitMsg = `Process exited with code ${code}, signal ${signal}.`;
|
||||
this.logger.info(exitMsg);
|
||||
this.log(exitMsg);
|
||||
@@ -149,6 +166,27 @@ export class ProcessMonitor extends EventEmitter {
|
||||
}
|
||||
} catch {}
|
||||
|
||||
// Detect if this was a crash (non-zero exit code or killed by signal)
|
||||
const isCrash = (code !== null && code !== 0) || signal !== null;
|
||||
|
||||
// Save crash log if this was a crash
|
||||
if (isCrash && this.processId && this.config.name) {
|
||||
try {
|
||||
await this.crashLogManager.saveCrashLog(
|
||||
this.processId,
|
||||
this.config.name,
|
||||
this.logs,
|
||||
code,
|
||||
signal,
|
||||
this.restartCount,
|
||||
this.lastMemoryUsage
|
||||
);
|
||||
this.logger.info(`Saved crash log for process ${this.config.name}`);
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to save crash log: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Flush logs to disk on exit
|
||||
if (this.processId && this.logs.length > 0) {
|
||||
try {
|
||||
@@ -169,10 +207,10 @@ export class ProcessMonitor extends EventEmitter {
|
||||
'Not restarting process because monitor is stopped',
|
||||
);
|
||||
}
|
||||
},
|
||||
);
|
||||
};
|
||||
this.processWrapper.on('exit', this.exitHandler);
|
||||
|
||||
this.processWrapper.on('error', async (error: Error | ProcessError): Promise<void> => {
|
||||
this.errorHandler = async (error: Error | ProcessError): Promise<void> => {
|
||||
const errorMsg =
|
||||
error instanceof ProcessError
|
||||
? `Process error: ${error.toString()}`
|
||||
@@ -181,6 +219,24 @@ export class ProcessMonitor extends EventEmitter {
|
||||
this.logger.error(error);
|
||||
this.log(errorMsg);
|
||||
|
||||
// Save crash log for errors
|
||||
if (this.processId && this.config.name) {
|
||||
try {
|
||||
await this.crashLogManager.saveCrashLog(
|
||||
this.processId,
|
||||
this.config.name,
|
||||
this.logs,
|
||||
null, // no exit code for errors
|
||||
null, // no signal for errors
|
||||
this.restartCount,
|
||||
this.lastMemoryUsage
|
||||
);
|
||||
this.logger.info(`Saved crash log for process ${this.config.name} due to error`);
|
||||
} catch (crashLogError) {
|
||||
this.logger.error(`Failed to save crash log: ${crashLogError}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Flush logs to disk on error
|
||||
if (this.processId && this.logs.length > 0) {
|
||||
try {
|
||||
@@ -196,7 +252,8 @@ export class ProcessMonitor extends EventEmitter {
|
||||
} else {
|
||||
this.logger.debug('Not restarting process because monitor is stopped');
|
||||
}
|
||||
});
|
||||
};
|
||||
this.processWrapper.on('error', this.errorHandler);
|
||||
|
||||
// Start the process
|
||||
try {
|
||||
@@ -210,6 +267,31 @@ export class ProcessMonitor extends EventEmitter {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up event listeners from process wrapper
|
||||
*/
|
||||
private cleanupListeners(): void {
|
||||
if (this.processWrapper) {
|
||||
if (this.logHandler) {
|
||||
this.processWrapper.removeListener('log', this.logHandler);
|
||||
}
|
||||
if (this.startHandler) {
|
||||
this.processWrapper.removeListener('start', this.startHandler);
|
||||
}
|
||||
if (this.exitHandler) {
|
||||
this.processWrapper.removeListener('exit', this.exitHandler);
|
||||
}
|
||||
if (this.errorHandler) {
|
||||
this.processWrapper.removeListener('error', this.errorHandler);
|
||||
}
|
||||
}
|
||||
// Clear references
|
||||
this.logHandler = undefined;
|
||||
this.startHandler = undefined;
|
||||
this.exitHandler = undefined;
|
||||
this.errorHandler = undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Schedule a restart with incremental debounce and failure cutoff.
|
||||
*/
|
||||
@@ -360,6 +442,14 @@ export class ProcessMonitor extends EventEmitter {
|
||||
this.logger.debug(
|
||||
`Total memory for process group: ${this.humanReadableBytes(totalMemory)}`,
|
||||
);
|
||||
|
||||
// Clear pidusage cache for all PIDs to prevent memory leaks
|
||||
for (const pid of pids) {
|
||||
try {
|
||||
(plugins.pidusage as any)?.clear?.(pid);
|
||||
} catch {}
|
||||
}
|
||||
|
||||
resolve({ memory: totalMemory, cpu: totalCpu });
|
||||
},
|
||||
);
|
||||
@@ -387,6 +477,9 @@ export class ProcessMonitor extends EventEmitter {
|
||||
this.log('Stopping process monitor.');
|
||||
this.stopped = true;
|
||||
|
||||
// Clean up event listeners
|
||||
this.cleanupListeners();
|
||||
|
||||
// Flush logs to disk before stopping
|
||||
if (this.processId && this.logs.length > 0) {
|
||||
try {
|
||||
|
Reference in New Issue
Block a user