fix(processmanager): Improve process lifecycle handling and cleanup in daemon, monitors and wrappers
This commit is contained in:
@@ -1,5 +1,13 @@
|
|||||||
# Changelog
|
# Changelog
|
||||||
|
|
||||||
|
## 2025-08-31 - 5.6.2 - fix(processmanager)
|
||||||
|
Improve process lifecycle handling and cleanup in daemon, monitors and wrappers
|
||||||
|
|
||||||
|
- StartAll: when a monitor exists but is not running, restart it instead of skipping — ensures saved processes are reliably brought online.
|
||||||
|
- ProcessMonitor.stop: cancel any pending restart timers to prevent stray restarts after explicit stop.
|
||||||
|
- ProcessWrapper: add killProcessTree helper and use it for graceful (SIGTERM) and force (SIGKILL) shutdowns to reliably signal child processes.
|
||||||
|
- Daemon stopAll: yield briefly after stopping processes and inspect monitors (not only processInfo) to accurately report stopped vs failed processes.
|
||||||
|
|
||||||
## 2025-08-31 - 5.6.1 - fix(daemon)
|
## 2025-08-31 - 5.6.1 - fix(daemon)
|
||||||
Ensure robust process shutdown and improve logs/subscriber diagnostics
|
Ensure robust process shutdown and improve logs/subscriber diagnostics
|
||||||
|
|
||||||
|
@@ -3,6 +3,6 @@
|
|||||||
*/
|
*/
|
||||||
export const commitinfo = {
|
export const commitinfo = {
|
||||||
name: '@git.zone/tspm',
|
name: '@git.zone/tspm',
|
||||||
version: '5.6.1',
|
version: '5.6.2',
|
||||||
description: 'a no fuzz process manager'
|
description: 'a no fuzz process manager'
|
||||||
}
|
}
|
||||||
|
@@ -499,8 +499,12 @@ export class ProcessManager extends EventEmitter {
|
|||||||
*/
|
*/
|
||||||
public async startAll(): Promise<void> {
|
public async startAll(): Promise<void> {
|
||||||
for (const [id, config] of this.processConfigs.entries()) {
|
for (const [id, config] of this.processConfigs.entries()) {
|
||||||
if (!this.processes.has(id)) {
|
const monitor = this.processes.get(id);
|
||||||
|
if (!monitor) {
|
||||||
await this.start(config);
|
await this.start(config);
|
||||||
|
} else if (!monitor.isRunning()) {
|
||||||
|
// If a monitor exists but is not running, restart the process to ensure a clean start
|
||||||
|
await this.restart(id);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@@ -400,6 +400,11 @@ export class ProcessMonitor extends EventEmitter {
|
|||||||
if (this.intervalId) {
|
if (this.intervalId) {
|
||||||
clearInterval(this.intervalId);
|
clearInterval(this.intervalId);
|
||||||
}
|
}
|
||||||
|
// Cancel any pending restart timer
|
||||||
|
if (this.restartTimer) {
|
||||||
|
clearTimeout(this.restartTimer);
|
||||||
|
this.restartTimer = null;
|
||||||
|
}
|
||||||
if (this.processWrapper) {
|
if (this.processWrapper) {
|
||||||
// Clear pidusage state for current PID before stopping to avoid leaks
|
// Clear pidusage state for current PID before stopping to avoid leaks
|
||||||
try {
|
try {
|
||||||
|
@@ -23,6 +23,26 @@ export class ProcessWrapper extends EventEmitter {
|
|||||||
private runId: string = '';
|
private runId: string = '';
|
||||||
private stdoutRemainder: string = '';
|
private stdoutRemainder: string = '';
|
||||||
private stderrRemainder: string = '';
|
private stderrRemainder: string = '';
|
||||||
|
|
||||||
|
// Helper: send a signal to the process and all its children (best-effort)
|
||||||
|
private async killProcessTree(signal: NodeJS.Signals): Promise<void> {
|
||||||
|
if (!this.process || !this.process.pid) return;
|
||||||
|
const rootPid = this.process.pid;
|
||||||
|
await new Promise<void>((resolve) => {
|
||||||
|
plugins.psTree(rootPid, (err: any, children: ReadonlyArray<{ PID: string }>) => {
|
||||||
|
const pids: number[] = [rootPid, ...children.map((c) => Number(c.PID)).filter((n) => Number.isFinite(n))];
|
||||||
|
for (const pid of pids) {
|
||||||
|
try {
|
||||||
|
// Always signal individual PIDs to avoid accidentally targeting unrelated groups
|
||||||
|
process.kill(pid, signal);
|
||||||
|
} catch {
|
||||||
|
// ignore ESRCH/EPERM
|
||||||
|
}
|
||||||
|
}
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
constructor(options: IProcessWrapperOptions) {
|
constructor(options: IProcessWrapperOptions) {
|
||||||
super();
|
super();
|
||||||
@@ -193,17 +213,13 @@ export class ProcessWrapper extends EventEmitter {
|
|||||||
// First try SIGTERM for graceful shutdown
|
// First try SIGTERM for graceful shutdown
|
||||||
if (this.process.pid) {
|
if (this.process.pid) {
|
||||||
try {
|
try {
|
||||||
this.logger.debug(`Sending SIGTERM to process ${this.process.pid}`);
|
this.logger.debug(`Sending SIGTERM to process tree rooted at ${this.process.pid}`);
|
||||||
try {
|
await this.killProcessTree('SIGTERM');
|
||||||
// Try to signal the whole process group on POSIX to ensure children get the signal too
|
|
||||||
if (process.platform !== 'win32') {
|
// If the process already exited, return immediately
|
||||||
process.kill(-Math.abs(this.process.pid), 'SIGTERM');
|
if (typeof this.process.exitCode === 'number') {
|
||||||
} else {
|
this.logger.debug('Process already exited, no need to wait');
|
||||||
process.kill(this.process.pid, 'SIGTERM');
|
return;
|
||||||
}
|
|
||||||
} catch {
|
|
||||||
// Fallback to direct process kill if group kill fails
|
|
||||||
process.kill(this.process.pid, 'SIGTERM');
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Wait for exit or escalate
|
// Wait for exit or escalate
|
||||||
@@ -218,26 +234,15 @@ export class ProcessWrapper extends EventEmitter {
|
|||||||
const onExit = () => cleanup();
|
const onExit = () => cleanup();
|
||||||
this.process!.once('exit', onExit);
|
this.process!.once('exit', onExit);
|
||||||
|
|
||||||
const killTimer = setTimeout(() => {
|
const killTimer = setTimeout(async () => {
|
||||||
if (!this.process || !this.process.pid) return cleanup();
|
if (!this.process || !this.process.pid) return cleanup();
|
||||||
this.logger.warn(
|
this.logger.warn(
|
||||||
`Process ${this.process.pid} did not exit gracefully, force killing...`,
|
`Process ${this.process.pid} did not exit gracefully, force killing tree...`,
|
||||||
);
|
|
||||||
this.addSystemLog(
|
|
||||||
'Process did not exit gracefully, force killing...',
|
|
||||||
);
|
);
|
||||||
|
this.addSystemLog('Process did not exit gracefully, force killing...');
|
||||||
try {
|
try {
|
||||||
if (process.platform !== 'win32') {
|
await this.killProcessTree('SIGKILL');
|
||||||
process.kill(-Math.abs(this.process.pid), 'SIGKILL');
|
} catch {}
|
||||||
} else {
|
|
||||||
process.kill(this.process.pid, 'SIGKILL');
|
|
||||||
}
|
|
||||||
} catch (error: any) {
|
|
||||||
this.logger.debug(
|
|
||||||
`Failed to send SIGKILL, process probably already exited: ${error?.message || String(error)}`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Give a short grace period after SIGKILL
|
// Give a short grace period after SIGKILL
|
||||||
setTimeout(() => cleanup(), 500);
|
setTimeout(() => cleanup(), 500);
|
||||||
}, 5000);
|
}, 5000);
|
||||||
|
@@ -450,10 +450,12 @@ export class TspmDaemon {
|
|||||||
|
|
||||||
await this.tspmInstance.setDesiredStateForAll('stopped');
|
await this.tspmInstance.setDesiredStateForAll('stopped');
|
||||||
await this.tspmInstance.stopAll();
|
await this.tspmInstance.stopAll();
|
||||||
|
// Yield briefly to allow any pending exit events to settle
|
||||||
|
await new Promise((r) => setTimeout(r, 50));
|
||||||
|
|
||||||
// Get status of all processes
|
// Determine which monitors are no longer running
|
||||||
for (const [id, processInfo] of this.tspmInstance.processInfo) {
|
for (const [id, monitor] of this.tspmInstance.processes) {
|
||||||
if (processInfo.status === 'stopped') {
|
if (!monitor.isRunning()) {
|
||||||
stopped.push(id);
|
stopped.push(id);
|
||||||
} else {
|
} else {
|
||||||
failed.push({ id, error: 'Failed to stop' });
|
failed.push({ id, error: 'Failed to stop' });
|
||||||
|
Reference in New Issue
Block a user