feat(cli/daemon/processmonitor): Add flexible target resolution and search command; improve restart/backoff and error handling

This commit is contained in:
2025-08-30 16:55:10 +00:00
parent 22a43204d4
commit ebc20a9232
17 changed files with 327 additions and 109 deletions

View File

@@ -18,6 +18,10 @@ export class ProcessMonitor extends EventEmitter {
private processId?: ProcessId;
private currentLogMemorySize: number = 0;
private readonly MAX_LOG_MEMORY_SIZE = 10 * 1024 * 1024; // 10MB
private restartTimer: NodeJS.Timeout | null = null;
private lastRetryAt: number | null = null;
private readonly MAX_RETRIES = 10;
private readonly RESET_WINDOW_MS = 60 * 60 * 1000; // 1 hour
constructor(config: IMonitorConfig & { id?: ProcessId }) {
super();
@@ -132,10 +136,7 @@ export class ProcessMonitor extends EventEmitter {
this.emit('exit', code, signal);
if (!this.stopped) {
this.logger.info('Restarting process...');
this.log('Restarting process...');
this.restartCount++;
this.spawnProcess();
this.scheduleRestart('exit');
} else {
this.logger.debug(
'Not restarting process because monitor is stopped',
@@ -164,10 +165,7 @@ export class ProcessMonitor extends EventEmitter {
}
if (!this.stopped) {
this.logger.info('Restarting process due to error...');
this.log('Restarting process due to error...');
this.restartCount++;
this.spawnProcess();
this.scheduleRestart('error');
} else {
this.logger.debug('Not restarting process because monitor is stopped');
}
@@ -185,6 +183,49 @@ export class ProcessMonitor extends EventEmitter {
}
}
/**
* Schedule a restart with incremental debounce and failure cutoff.
*/
private scheduleRestart(reason: 'exit' | 'error'): void {
const now = Date.now();
// Reset window: if last retry was more than 1 hour ago, reset counter
if (this.lastRetryAt && now - this.lastRetryAt >= this.RESET_WINDOW_MS) {
this.logger.info('Resetting retry counter after 1 hour window');
this.restartCount = 0;
}
// Already at or above max retries?
if (this.restartCount >= this.MAX_RETRIES) {
const msg = 'Maximum restart attempts reached. Marking process as failed.';
this.logger.warn(msg);
this.log(msg);
this.stopped = true;
// Emit a specific event so manager can set status to errored
this.emit('failed');
return;
}
// Increment and compute delay (1..10 seconds)
this.restartCount++;
const delaySec = Math.min(this.restartCount, 10);
const msg = `Restarting process in ${delaySec}s (attempt ${this.restartCount}/${this.MAX_RETRIES}) due to ${reason}...`;
this.logger.info(msg);
this.log(msg);
// Clear existing timer if any, then schedule
if (this.restartTimer) {
clearTimeout(this.restartTimer);
}
this.lastRetryAt = now;
this.restartTimer = setTimeout(() => {
// If stopped in the meantime, do not spawn
if (this.stopped) {
return;
}
this.spawnProcess();
}, delaySec * 1000);
}
/**
* Monitor the process group's memory usage. If the total memory exceeds the limit,
* kill the process group so that the 'exit' handler can restart it.