fix(remoteingress edge/hub crash recovery): prevent duplicate crash recovery listeners and reset saved runtime state on shutdown

This commit is contained in:
2026-03-21 19:49:05 +00:00
parent bf3418d0ed
commit 331b5c8d3f
4 changed files with 38 additions and 3 deletions

View File

@@ -1,5 +1,12 @@
# Changelog # Changelog
## 2026-03-21 - 4.14.1 - fix(remoteingress edge/hub crash recovery)
prevent duplicate crash recovery listeners and reset saved runtime state on shutdown
- Removes existing exit listeners before re-registering crash recovery handlers for edge and hub processes.
- Clears saved edge and hub configuration on stop to avoid stale restart state.
- Resets orphaned edge status intervals and restarts periodic status logging after successful crash recovery.
## 2026-03-20 - 4.14.0 - feat(quic) ## 2026-03-20 - 4.14.0 - feat(quic)
add QUIC stability test coverage and bridge logging for hub and edge add QUIC stability test coverage and bridge logging for hub and edge

View File

@@ -3,6 +3,6 @@
*/ */
export const commitinfo = { export const commitinfo = {
name: '@serve.zone/remoteingress', name: '@serve.zone/remoteingress',
version: '4.14.0', version: '4.14.1',
description: 'Edge ingress tunnel for DcRouter - tunnels TCP and UDP traffic from the network edge to SmartProxy over TLS or QUIC, preserving client IP via PROXY protocol.' description: 'Edge ingress tunnel for DcRouter - tunnels TCP and UDP traffic from the network edge to SmartProxy over TLS or QUIC, preserving client IP via PROXY protocol.'
} }

View File

@@ -139,7 +139,8 @@ export class RemoteIngressEdge extends EventEmitter {
throw new Error('Failed to spawn remoteingress-bin'); throw new Error('Failed to spawn remoteingress-bin');
} }
// Register crash recovery handler // Register crash recovery handler (remove first to avoid duplicates)
this.bridge.removeListener('exit', this.handleCrashRecovery);
this.bridge.on('exit', this.handleCrashRecovery); this.bridge.on('exit', this.handleCrashRecovery);
await this.bridge.sendCommand('startEdge', { await this.bridge.sendCommand('startEdge', {
@@ -189,6 +190,7 @@ export class RemoteIngressEdge extends EventEmitter {
this.bridge.kill(); this.bridge.kill();
this.started = false; this.started = false;
} }
this.savedConfig = null;
} }
/** /**
@@ -220,6 +222,12 @@ export class RemoteIngressEdge extends EventEmitter {
this.started = false; this.started = false;
// Clear orphaned status interval from previous run
if (this.statusInterval) {
clearInterval(this.statusInterval);
this.statusInterval = undefined;
}
if (this.restartAttempts >= MAX_RESTART_ATTEMPTS) { if (this.restartAttempts >= MAX_RESTART_ATTEMPTS) {
console.error('[RemoteIngressEdge] Max restart attempts reached, giving up'); console.error('[RemoteIngressEdge] Max restart attempts reached, giving up');
this.emit('crashRecoveryFailed'); this.emit('crashRecoveryFailed');
@@ -237,6 +245,7 @@ export class RemoteIngressEdge extends EventEmitter {
return; return;
} }
this.bridge.removeListener('exit', this.handleCrashRecovery);
this.bridge.on('exit', this.handleCrashRecovery); this.bridge.on('exit', this.handleCrashRecovery);
await this.bridge.sendCommand('startEdge', { await this.bridge.sendCommand('startEdge', {
@@ -251,6 +260,21 @@ export class RemoteIngressEdge extends EventEmitter {
this.started = true; this.started = true;
this.restartAttempts = 0; this.restartAttempts = 0;
this.restartBackoffMs = 1000; this.restartBackoffMs = 1000;
// Restart periodic status logging
this.statusInterval = setInterval(async () => {
try {
const status = await this.getStatus();
console.log(
`[RemoteIngressEdge] Status: connected=${status.connected}, ` +
`streams=${status.activeStreams}, ports=[${status.listenPorts.join(',')}], ` +
`publicIp=${status.publicIp ?? 'unknown'}`
);
} catch {
// Bridge may be shutting down
}
}, 60_000);
console.log('[RemoteIngressEdge] Successfully recovered from crash'); console.log('[RemoteIngressEdge] Successfully recovered from crash');
this.emit('crashRecovered'); this.emit('crashRecovered');
} catch (err) { } catch (err) {

View File

@@ -127,7 +127,8 @@ export class RemoteIngressHub extends EventEmitter {
throw new Error('Failed to spawn remoteingress-bin'); throw new Error('Failed to spawn remoteingress-bin');
} }
// Register crash recovery handler // Register crash recovery handler (remove first to avoid duplicates)
this.bridge.removeListener('exit', this.handleCrashRecovery);
this.bridge.on('exit', this.handleCrashRecovery); this.bridge.on('exit', this.handleCrashRecovery);
await this.bridge.sendCommand('startHub', { await this.bridge.sendCommand('startHub', {
@@ -158,6 +159,8 @@ export class RemoteIngressHub extends EventEmitter {
this.bridge.kill(); this.bridge.kill();
this.started = false; this.started = false;
} }
this.savedConfig = null;
this.savedEdges = [];
} }
/** /**
@@ -214,6 +217,7 @@ export class RemoteIngressHub extends EventEmitter {
return; return;
} }
this.bridge.removeListener('exit', this.handleCrashRecovery);
this.bridge.on('exit', this.handleCrashRecovery); this.bridge.on('exit', this.handleCrashRecovery);
const config = this.savedConfig; const config = this.savedConfig;