feat(daemon): Reorganize and refactor core into client/daemon/shared modules; add IPC protocol and tests
This commit is contained in:
308
ts/daemon/processmonitor.ts
Normal file
308
ts/daemon/processmonitor.ts
Normal file
@@ -0,0 +1,308 @@
|
||||
import * as plugins from '../plugins.js';
|
||||
import { EventEmitter } from 'events';
|
||||
import { ProcessWrapper, type IProcessLog } from './processwrapper.js';
|
||||
import { Logger, ProcessError, handleError } from '../shared/common/utils.errorhandler.js';
|
||||
|
||||
export interface IMonitorConfig {
|
||||
name?: string; // Optional name to identify the instance
|
||||
projectDir: string; // Directory where the command will run
|
||||
command: string; // Full command to run (e.g., "npm run xyz")
|
||||
args?: string[]; // Optional: arguments for the command
|
||||
memoryLimitBytes: number; // Maximum allowed memory (in bytes) for the process group
|
||||
monitorIntervalMs?: number; // Interval (in ms) at which memory is checked (default: 5000)
|
||||
env?: NodeJS.ProcessEnv; // Optional: custom environment variables
|
||||
logBufferSize?: number; // Optional: number of log lines to keep (default: 100)
|
||||
}
|
||||
|
||||
export class ProcessMonitor extends EventEmitter {
|
||||
private processWrapper: ProcessWrapper | null = null;
|
||||
private config: IMonitorConfig;
|
||||
private intervalId: NodeJS.Timeout | null = null;
|
||||
private stopped: boolean = true; // Initially stopped until start() is called
|
||||
private restartCount: number = 0;
|
||||
private logger: Logger;
|
||||
|
||||
constructor(config: IMonitorConfig) {
|
||||
super();
|
||||
this.config = config;
|
||||
this.logger = new Logger(`ProcessMonitor:${config.name || 'unnamed'}`);
|
||||
}
|
||||
|
||||
public start(): void {
|
||||
// Reset the stopped flag so that new processes can spawn.
|
||||
this.stopped = false;
|
||||
this.log(`Starting process monitor.`);
|
||||
this.spawnProcess();
|
||||
|
||||
// Set the monitoring interval.
|
||||
const interval = this.config.monitorIntervalMs || 5000;
|
||||
this.intervalId = setInterval((): void => {
|
||||
if (this.processWrapper && this.processWrapper.getPid()) {
|
||||
this.monitorProcessGroup(
|
||||
this.processWrapper.getPid()!,
|
||||
this.config.memoryLimitBytes,
|
||||
);
|
||||
}
|
||||
}, interval);
|
||||
}
|
||||
|
||||
private spawnProcess(): void {
|
||||
// Don't spawn if the monitor has been stopped.
|
||||
if (this.stopped) {
|
||||
this.logger.debug('Not spawning process because monitor is stopped');
|
||||
return;
|
||||
}
|
||||
|
||||
this.logger.info(`Spawning process: ${this.config.command}`);
|
||||
|
||||
// Create a new process wrapper
|
||||
this.processWrapper = new ProcessWrapper({
|
||||
name: this.config.name || 'unnamed-process',
|
||||
command: this.config.command,
|
||||
args: this.config.args,
|
||||
cwd: this.config.projectDir,
|
||||
env: this.config.env,
|
||||
logBuffer: this.config.logBufferSize,
|
||||
});
|
||||
|
||||
// Set up event handlers
|
||||
this.processWrapper.on('log', (log: IProcessLog): void => {
|
||||
// Re-emit the log event for upstream handlers
|
||||
this.emit('log', log);
|
||||
|
||||
// Log system messages to the console
|
||||
if (log.type === 'system') {
|
||||
this.log(log.message);
|
||||
}
|
||||
});
|
||||
|
||||
this.processWrapper.on(
|
||||
'exit',
|
||||
(code: number | null, signal: string | null): void => {
|
||||
const exitMsg = `Process exited with code ${code}, signal ${signal}.`;
|
||||
this.logger.info(exitMsg);
|
||||
this.log(exitMsg);
|
||||
|
||||
if (!this.stopped) {
|
||||
this.logger.info('Restarting process...');
|
||||
this.log('Restarting process...');
|
||||
this.restartCount++;
|
||||
this.spawnProcess();
|
||||
} else {
|
||||
this.logger.debug(
|
||||
'Not restarting process because monitor is stopped',
|
||||
);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
this.processWrapper.on('error', (error: Error | ProcessError): void => {
|
||||
const errorMsg =
|
||||
error instanceof ProcessError
|
||||
? `Process error: ${error.toString()}`
|
||||
: `Process error: ${error.message}`;
|
||||
|
||||
this.logger.error(error);
|
||||
this.log(errorMsg);
|
||||
|
||||
if (!this.stopped) {
|
||||
this.logger.info('Restarting process due to error...');
|
||||
this.log('Restarting process due to error...');
|
||||
this.restartCount++;
|
||||
this.spawnProcess();
|
||||
} else {
|
||||
this.logger.debug('Not restarting process because monitor is stopped');
|
||||
}
|
||||
});
|
||||
|
||||
// Start the process
|
||||
try {
|
||||
this.processWrapper.start();
|
||||
} catch (error: Error | unknown) {
|
||||
// The process wrapper will handle logging the error
|
||||
// Just prevent it from bubbling up further
|
||||
this.logger.error(
|
||||
`Failed to start process: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Monitor the process group's memory usage. If the total memory exceeds the limit,
|
||||
* kill the process group so that the 'exit' handler can restart it.
|
||||
*/
|
||||
private async monitorProcessGroup(
|
||||
pid: number,
|
||||
memoryLimit: number,
|
||||
): Promise<void> {
|
||||
try {
|
||||
const memoryUsage = await this.getProcessGroupMemory(pid);
|
||||
|
||||
this.logger.debug(
|
||||
`Memory usage for PID ${pid}: ${this.humanReadableBytes(memoryUsage)} (${memoryUsage} bytes)`,
|
||||
);
|
||||
|
||||
// Only log to the process log at longer intervals to avoid spamming
|
||||
this.log(
|
||||
`Current memory usage for process group (PID ${pid}): ${this.humanReadableBytes(
|
||||
memoryUsage,
|
||||
)} (${memoryUsage} bytes)`,
|
||||
);
|
||||
|
||||
if (memoryUsage > memoryLimit) {
|
||||
const memoryLimitMsg = `Memory usage ${this.humanReadableBytes(
|
||||
memoryUsage,
|
||||
)} exceeds limit of ${this.humanReadableBytes(memoryLimit)}. Restarting process.`;
|
||||
|
||||
this.logger.warn(memoryLimitMsg);
|
||||
this.log(memoryLimitMsg);
|
||||
|
||||
// Stop the process wrapper, which will trigger the exit handler and restart
|
||||
if (this.processWrapper) {
|
||||
this.processWrapper.stop();
|
||||
}
|
||||
}
|
||||
} catch (error: Error | unknown) {
|
||||
const processError = new ProcessError(
|
||||
error instanceof Error ? error.message : String(error),
|
||||
'ERR_MEMORY_MONITORING_FAILED',
|
||||
{ pid },
|
||||
);
|
||||
|
||||
this.logger.error(processError);
|
||||
this.log(`Error monitoring process group: ${processError.toString()}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the total memory usage (in bytes) for the process group (the main process and its children).
|
||||
*/
|
||||
private getProcessGroupMemory(pid: number): Promise<number> {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.logger.debug(
|
||||
`Getting memory usage for process group with PID ${pid}`,
|
||||
);
|
||||
|
||||
plugins.psTree(
|
||||
pid,
|
||||
(err: Error | null, children: Array<{ PID: string }>) => {
|
||||
if (err) {
|
||||
const processError = new ProcessError(
|
||||
`Failed to get process tree: ${err.message}`,
|
||||
'ERR_PSTREE_FAILED',
|
||||
{ pid },
|
||||
);
|
||||
this.logger.debug(`psTree error: ${err.message}`);
|
||||
return reject(processError);
|
||||
}
|
||||
|
||||
// Include the main process and its children.
|
||||
const pids: number[] = [
|
||||
pid,
|
||||
...children.map((child) => Number(child.PID)),
|
||||
];
|
||||
this.logger.debug(
|
||||
`Found ${pids.length} processes in group with parent PID ${pid}`,
|
||||
);
|
||||
|
||||
plugins.pidusage(
|
||||
pids,
|
||||
(err: Error | null, stats: Record<string, { memory: number }>) => {
|
||||
if (err) {
|
||||
const processError = new ProcessError(
|
||||
`Failed to get process usage stats: ${err.message}`,
|
||||
'ERR_PIDUSAGE_FAILED',
|
||||
{ pids },
|
||||
);
|
||||
this.logger.debug(`pidusage error: ${err.message}`);
|
||||
return reject(processError);
|
||||
}
|
||||
|
||||
let totalMemory = 0;
|
||||
for (const key in stats) {
|
||||
totalMemory += stats[key].memory;
|
||||
}
|
||||
|
||||
this.logger.debug(
|
||||
`Total memory for process group: ${this.humanReadableBytes(totalMemory)}`,
|
||||
);
|
||||
resolve(totalMemory);
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a number of bytes into a human-readable string (e.g. "1.23 MB").
|
||||
*/
|
||||
private humanReadableBytes(bytes: number, decimals: number = 2): string {
|
||||
if (bytes === 0) return '0 Bytes';
|
||||
const k = 1024;
|
||||
const dm = decimals < 0 ? 0 : decimals;
|
||||
const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
|
||||
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
||||
return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i];
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the monitor and prevent any further respawns.
|
||||
*/
|
||||
public stop(): void {
|
||||
this.log('Stopping process monitor.');
|
||||
this.stopped = true;
|
||||
if (this.intervalId) {
|
||||
clearInterval(this.intervalId);
|
||||
}
|
||||
if (this.processWrapper) {
|
||||
this.processWrapper.stop();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current logs from the process
|
||||
*/
|
||||
public getLogs(limit?: number): IProcessLog[] {
|
||||
if (!this.processWrapper) {
|
||||
return [];
|
||||
}
|
||||
return this.processWrapper.getLogs(limit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the number of times the process has been restarted
|
||||
*/
|
||||
public getRestartCount(): number {
|
||||
return this.restartCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the process ID if running
|
||||
*/
|
||||
public getPid(): number | null {
|
||||
return this.processWrapper?.getPid() || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get process uptime in milliseconds
|
||||
*/
|
||||
public getUptime(): number {
|
||||
return this.processWrapper?.getUptime() || 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the process is currently running
|
||||
*/
|
||||
public isRunning(): boolean {
|
||||
return this.processWrapper?.isRunning() || false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper method for logging messages with the instance name.
|
||||
*/
|
||||
private log(message: string): void {
|
||||
const prefix = this.config.name ? `[${this.config.name}] ` : '';
|
||||
console.log(prefix + message);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user