feat(cli/daemon/processmonitor): Add flexible target resolution and search command; improve restart/backoff and error handling
This commit is contained in:
@@ -18,6 +18,10 @@ export class ProcessMonitor extends EventEmitter {
|
||||
private processId?: ProcessId;
|
||||
private currentLogMemorySize: number = 0;
|
||||
private readonly MAX_LOG_MEMORY_SIZE = 10 * 1024 * 1024; // 10MB
|
||||
private restartTimer: NodeJS.Timeout | null = null;
|
||||
private lastRetryAt: number | null = null;
|
||||
private readonly MAX_RETRIES = 10;
|
||||
private readonly RESET_WINDOW_MS = 60 * 60 * 1000; // 1 hour
|
||||
|
||||
constructor(config: IMonitorConfig & { id?: ProcessId }) {
|
||||
super();
|
||||
@@ -132,10 +136,7 @@ export class ProcessMonitor extends EventEmitter {
|
||||
this.emit('exit', code, signal);
|
||||
|
||||
if (!this.stopped) {
|
||||
this.logger.info('Restarting process...');
|
||||
this.log('Restarting process...');
|
||||
this.restartCount++;
|
||||
this.spawnProcess();
|
||||
this.scheduleRestart('exit');
|
||||
} else {
|
||||
this.logger.debug(
|
||||
'Not restarting process because monitor is stopped',
|
||||
@@ -164,10 +165,7 @@ export class ProcessMonitor extends EventEmitter {
|
||||
}
|
||||
|
||||
if (!this.stopped) {
|
||||
this.logger.info('Restarting process due to error...');
|
||||
this.log('Restarting process due to error...');
|
||||
this.restartCount++;
|
||||
this.spawnProcess();
|
||||
this.scheduleRestart('error');
|
||||
} else {
|
||||
this.logger.debug('Not restarting process because monitor is stopped');
|
||||
}
|
||||
@@ -185,6 +183,49 @@ export class ProcessMonitor extends EventEmitter {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Schedule a restart with incremental debounce and failure cutoff.
|
||||
*/
|
||||
private scheduleRestart(reason: 'exit' | 'error'): void {
|
||||
const now = Date.now();
|
||||
// Reset window: if last retry was more than 1 hour ago, reset counter
|
||||
if (this.lastRetryAt && now - this.lastRetryAt >= this.RESET_WINDOW_MS) {
|
||||
this.logger.info('Resetting retry counter after 1 hour window');
|
||||
this.restartCount = 0;
|
||||
}
|
||||
|
||||
// Already at or above max retries?
|
||||
if (this.restartCount >= this.MAX_RETRIES) {
|
||||
const msg = 'Maximum restart attempts reached. Marking process as failed.';
|
||||
this.logger.warn(msg);
|
||||
this.log(msg);
|
||||
this.stopped = true;
|
||||
// Emit a specific event so manager can set status to errored
|
||||
this.emit('failed');
|
||||
return;
|
||||
}
|
||||
|
||||
// Increment and compute delay (1..10 seconds)
|
||||
this.restartCount++;
|
||||
const delaySec = Math.min(this.restartCount, 10);
|
||||
const msg = `Restarting process in ${delaySec}s (attempt ${this.restartCount}/${this.MAX_RETRIES}) due to ${reason}...`;
|
||||
this.logger.info(msg);
|
||||
this.log(msg);
|
||||
|
||||
// Clear existing timer if any, then schedule
|
||||
if (this.restartTimer) {
|
||||
clearTimeout(this.restartTimer);
|
||||
}
|
||||
this.lastRetryAt = now;
|
||||
this.restartTimer = setTimeout(() => {
|
||||
// If stopped in the meantime, do not spawn
|
||||
if (this.stopped) {
|
||||
return;
|
||||
}
|
||||
this.spawnProcess();
|
||||
}, delaySec * 1000);
|
||||
}
|
||||
|
||||
/**
|
||||
* Monitor the process group's memory usage. If the total memory exceeds the limit,
|
||||
* kill the process group so that the 'exit' handler can restart it.
|
||||
|
Reference in New Issue
Block a user