fix(lifecycle): clean up service subscriptions, proxy retries, and stale runtime state on shutdown

This commit is contained in:
2026-03-21 22:30:30 +00:00
parent c7fe7aeb50
commit 39ff159bf7
10 changed files with 115 additions and 27 deletions

View File

@@ -1,5 +1,12 @@
# Changelog
## 2026-03-21 - 11.9.1 - fix(lifecycle)
clean up service subscriptions, proxy retries, and stale runtime state on shutdown
- unsubscribe from ServiceManager event streams and use one-time signal handlers to avoid duplicate shutdown execution
- reset existing SmartProxy instances before retry setup and prune expired certificate backoff cache entries
- add periodic sweeping and shutdown cleanup for stale RADIUS accounting sessions
## 2026-03-20 - 11.9.0 - feat(dcrouter)
add service manager lifecycle orchestration and health-based ops status reporting

View File

@@ -53,16 +53,16 @@
"@push.rocks/smartnetwork": "^4.4.0",
"@push.rocks/smartpath": "^6.0.0",
"@push.rocks/smartpromise": "^4.2.3",
"@push.rocks/smartproxy": "^25.17.10",
"@push.rocks/smartproxy": "^26.0.0",
"@push.rocks/smartradius": "^1.1.1",
"@push.rocks/smartrequest": "^5.0.1",
"@push.rocks/smartrx": "^3.0.10",
"@push.rocks/smartstate": "^2.2.0",
"@push.rocks/smartunique": "^3.0.9",
"@push.rocks/taskbuffer": "7.0.0",
"@push.rocks/taskbuffer": "^8.0.0",
"@serve.zone/catalog": "^2.9.0",
"@serve.zone/interfaces": "^5.3.0",
"@serve.zone/remoteingress": "^4.14.0",
"@serve.zone/remoteingress": "^4.14.1",
"@tsclass/tsclass": "^9.4.0",
"lru-cache": "^11.2.7",
"uuid": "^13.0.0"

30
pnpm-lock.yaml generated
View File

@@ -78,8 +78,8 @@ importers:
specifier: ^4.2.3
version: 4.2.3
'@push.rocks/smartproxy':
specifier: ^25.17.10
version: 25.17.10
specifier: ^26.0.0
version: 26.0.0
'@push.rocks/smartradius':
specifier: ^1.1.1
version: 1.1.1
@@ -96,8 +96,8 @@ importers:
specifier: ^3.0.9
version: 3.0.9
'@push.rocks/taskbuffer':
specifier: 7.0.0
version: 7.0.0
specifier: ^8.0.0
version: 8.0.0
'@serve.zone/catalog':
specifier: ^2.9.0
version: 2.9.0(@tiptap/pm@2.27.2)
@@ -105,8 +105,8 @@ importers:
specifier: ^5.3.0
version: 5.3.0
'@serve.zone/remoteingress':
specifier: ^4.14.0
version: 4.14.0
specifier: ^4.14.1
version: 4.14.1
'@tsclass/tsclass':
specifier: ^9.4.0
version: 9.5.0
@@ -1259,8 +1259,8 @@ packages:
'@push.rocks/smartpromise@4.2.3':
resolution: {integrity: sha512-Ycg/TJR+tMt+S3wSFurOpEoW6nXv12QBtKXgBcjMZ4RsdO28geN46U09osPn9N9WuwQy1PkmTV5J/V4F9U8qEw==}
'@push.rocks/smartproxy@25.17.10':
resolution: {integrity: sha512-7tONZ77+Jlp4j2bGzp7ZpnS7nZC2Z8qbL23TYVQIR5KXt9GJuJV4w3CBsULtxL9PzmQajcY7jbmjFnjq2hcVqQ==}
'@push.rocks/smartproxy@26.0.0':
resolution: {integrity: sha512-fGLSVGCMEnmRFzt1iwiOjaOv6fB94fJgmtU13c9IHrpcuoPL2BhJqY+vj0bEgh2ee1F1fos3oARHKf4dwoeS6w==}
'@push.rocks/smartpuppeteer@2.0.5':
resolution: {integrity: sha512-yK/qSeWVHIGWRp3c8S5tfdGP6WCKllZC4DR8d8CQlEjszOSBmHtlTdyyqOMBZ/BA4kd+eU5f3A1r4K2tGYty1g==}
@@ -1344,8 +1344,8 @@ packages:
'@push.rocks/taskbuffer@6.1.2':
resolution: {integrity: sha512-sdqKd8N/GidztQ1k3r8A86rLvD8Afyir5FjYCNJXDD9837JLoqzHaOKGltUSBsCGh2gjsZn6GydsY6HhXQgvZQ==}
'@push.rocks/taskbuffer@7.0.0':
resolution: {integrity: sha512-cmjGwC/K7SzAcJrQChWSLTbIYl6YORbUkA/gyUTPVj/7Z7/BL7GzLyhYRk3ZHBS0AiCeTiP2WWNl+QJrf2WP9g==}
'@push.rocks/taskbuffer@8.0.0':
resolution: {integrity: sha512-ay4iXz0JmvsCQCmh5vvuu6KAl8FEZm5EpDXMQbeU+563d89xn+vMhh4+PtwxrVCogMEULWgGnavDYPTuzWtJOA==}
'@push.rocks/webrequest@4.0.5':
resolution: {integrity: sha512-wVSCaXqJ9Vh+rbwVz0wDl46dYz4rnwwSrm5vbVXKbuH6oKTPF0YRoujeJPqRltIn64RVGdLeY9/6ix+ZCrzhsg==}
@@ -1556,8 +1556,8 @@ packages:
'@serve.zone/interfaces@5.3.0':
resolution: {integrity: sha512-venO7wtDR9ixzD9NhdERBGjNKbFA5LL0yHw4eqGh0UpmvtXVc3SFG0uuHDilOKMZqZ8bttV88qVsFy1aSTJrtA==}
'@serve.zone/remoteingress@4.14.0':
resolution: {integrity: sha512-oDbKHhhvN2LxCcvmSgYhRLF+0FknEcPN+zg5kO4I0pfNpW/zgUYiaZns4TcYStZMS5/4i9j1uVR7QEO0a571/w==}
'@serve.zone/remoteingress@4.14.1':
resolution: {integrity: sha512-rYM4msFwo9SPxgNp/qXkJCQ8uXvQiMcH3cQEyciLKJ+7HwqKwQLCK4kbl45r/rzRAVOjyxXi0ae3hjgOBzTbyw==}
'@sindresorhus/is@5.6.0':
resolution: {integrity: sha512-TV7t8GKYaJWsn00tFDqBw8+Uqmr8A0fRU1tvTQhyZzGv0sJCGRQL3JGMI3ucuKo3XIZdUP+Lx7/gh2t3lewy7g==}
@@ -6545,7 +6545,7 @@ snapshots:
'@push.rocks/smartpromise@4.2.3': {}
'@push.rocks/smartproxy@25.17.10':
'@push.rocks/smartproxy@26.0.0':
dependencies:
'@push.rocks/smartcrypto': 2.0.4
'@push.rocks/smartlog': 3.2.1
@@ -6764,7 +6764,7 @@ snapshots:
- supports-color
- vue
'@push.rocks/taskbuffer@7.0.0':
'@push.rocks/taskbuffer@8.0.0':
dependencies:
'@design.estate/dees-element': 2.2.3
'@push.rocks/lik': 6.3.1
@@ -6978,7 +6978,7 @@ snapshots:
'@push.rocks/smartlog-interfaces': 3.0.2
'@tsclass/tsclass': 9.5.0
'@serve.zone/remoteingress@4.14.0':
'@serve.zone/remoteingress@4.14.1':
dependencies:
'@push.rocks/qenv': 6.1.3
'@push.rocks/smartrust': 1.3.2

View File

@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@serve.zone/dcrouter',
version: '11.9.0',
version: '11.9.1',
description: 'A multifaceted routing service handling mail and SMS delivery functions.'
}

View File

@@ -61,14 +61,21 @@ export class CertProvisionScheduler {
}
/**
* Check if a domain is currently in backoff
* Check if a domain is currently in backoff.
* Expired entries are pruned from the cache to prevent unbounded growth.
*/
async isInBackoff(domain: string): Promise<boolean> {
const entry = await this.loadBackoff(domain);
if (!entry) return false;
const retryAfter = new Date(entry.retryAfter);
return retryAfter.getTime() > Date.now();
if (retryAfter.getTime() > Date.now()) {
return true;
}
// Backoff has expired — prune the stale entry
this.backoffCache.delete(domain);
return false;
}
/**
@@ -124,9 +131,12 @@ export class CertProvisionScheduler {
const entry = await this.loadBackoff(domain);
if (!entry) return null;
// Only return if still in backoff
// Only return if still in backoff — prune expired entries
const retryAfter = new Date(entry.retryAfter);
if (retryAfter.getTime() <= Date.now()) return null;
if (retryAfter.getTime() <= Date.now()) {
this.backoffCache.delete(domain);
return null;
}
return {
failures: entry.failures,

View File

@@ -254,6 +254,7 @@ export class DcRouter {
// Service lifecycle management
public serviceManager: plugins.taskbuffer.ServiceManager;
private serviceSubjectSubscription?: plugins.smartrx.rxjs.Subscription;
public smartAcmeReady = false;
// TypedRouter for API endpoints
@@ -516,7 +517,7 @@ export class DcRouter {
}
// Wire up aggregated events for logging
this.serviceManager.serviceSubject.subscribe((event) => {
this.serviceSubjectSubscription = this.serviceManager.serviceSubject.subscribe((event) => {
const level = event.type === 'failed' ? 'error' : event.type === 'retrying' ? 'warn' : 'info';
logger.log(level as any, `Service '${event.serviceName}': ${event.type}`, {
state: event.state,
@@ -639,6 +640,13 @@ export class DcRouter {
*/
private async setupSmartProxy(): Promise<void> {
logger.log('info', 'Setting up SmartProxy...');
// Clean up any existing SmartProxy instance (e.g. from a retry)
if (this.smartProxy) {
this.smartProxy.removeAllListeners();
this.smartProxy = undefined;
}
let routes: plugins.smartproxy.IRouteConfig[] = [];
let acmeConfig: plugins.smartproxy.IAcmeOptions | undefined;
@@ -1126,6 +1134,12 @@ export class DcRouter {
public async stop() {
logger.log('info', 'Stopping DcRouter services...');
// Unsubscribe from service events before stopping services
if (this.serviceSubjectSubscription) {
this.serviceSubjectSubscription.unsubscribe();
this.serviceSubjectSubscription = undefined;
}
// ServiceManager handles reverse-dependency-ordered shutdown
await this.serviceManager.stop();

View File

@@ -35,6 +35,6 @@ export const runCli = async () => {
await dcRouter.stop();
process.exit(0);
};
process.on('SIGINT', shutdown);
process.on('SIGTERM', shutdown);
process.once('SIGINT', shutdown);
process.once('SIGTERM', shutdown);
};

View File

@@ -92,6 +92,8 @@ export interface IAccountingManagerConfig {
detailedLogging?: boolean;
/** Maximum active sessions to track in memory */
maxActiveSessions?: number;
/** Stale session timeout in hours — sessions with no update for this long are evicted (default: 24) */
staleSessionTimeoutHours?: number;
}
/**
@@ -105,6 +107,7 @@ export class AccountingManager {
private activeSessions: Map<string, IAccountingSession> = new Map();
private config: Required<IAccountingManagerConfig>;
private storageManager?: StorageManager;
private staleSessionSweepTimer?: ReturnType<typeof setInterval>;
// Counters for statistics
private stats = {
@@ -121,6 +124,7 @@ export class AccountingManager {
retentionDays: config?.retentionDays ?? 30,
detailedLogging: config?.detailedLogging ?? false,
maxActiveSessions: config?.maxActiveSessions ?? 10000,
staleSessionTimeoutHours: config?.staleSessionTimeoutHours ?? 24,
};
this.storageManager = storageManager;
}
@@ -132,9 +136,60 @@ export class AccountingManager {
if (this.storageManager) {
await this.loadActiveSessions();
}
// Start periodic sweep to evict stale sessions (every 15 minutes)
this.staleSessionSweepTimer = setInterval(() => {
this.sweepStaleSessions();
}, 15 * 60 * 1000);
// Allow the process to exit even if the timer is pending
if (this.staleSessionSweepTimer.unref) {
this.staleSessionSweepTimer.unref();
}
logger.log('info', `AccountingManager initialized with ${this.activeSessions.size} active sessions`);
}
/**
* Stop the accounting manager and clean up timers
*/
stop(): void {
if (this.staleSessionSweepTimer) {
clearInterval(this.staleSessionSweepTimer);
this.staleSessionSweepTimer = undefined;
}
}
/**
* Sweep stale active sessions that have not received any update
* within the configured timeout. These are orphaned sessions where
* the Stop packet was never received.
*/
private sweepStaleSessions(): void {
const timeoutMs = this.config.staleSessionTimeoutHours * 60 * 60 * 1000;
const cutoff = Date.now() - timeoutMs;
let swept = 0;
for (const [sessionId, session] of this.activeSessions) {
if (session.lastUpdateTime < cutoff) {
session.status = 'terminated';
session.terminateCause = 'StaleSessionTimeout';
session.endTime = Date.now();
session.sessionTime = Math.floor((session.endTime - session.startTime) / 1000);
if (this.storageManager) {
this.archiveSession(session).catch(() => {});
}
this.activeSessions.delete(sessionId);
swept++;
}
}
if (swept > 0) {
logger.log('info', `Swept ${swept} stale RADIUS sessions (no update for ${this.config.staleSessionTimeoutHours}h)`);
}
}
/**
* Handle accounting start request
*/

View File

@@ -183,6 +183,8 @@ export class RadiusServer {
this.radiusServer = undefined;
}
this.accountingManager.stop();
this.running = false;
logger.log('info', 'RADIUS server stopped');
}

View File

@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@serve.zone/dcrouter',
version: '11.9.0',
version: '11.9.1',
description: 'A multifaceted routing service handling mail and SMS delivery functions.'
}