From 7b3545d1b579b85476e08eb1f54412f184545f2e Mon Sep 17 00:00:00 2001 From: Juergen Kunz Date: Sat, 14 Feb 2026 14:02:25 +0000 Subject: [PATCH] feat(smart-proxy): add background concurrent certificate provisioning with per-domain timeouts and concurrency control --- changelog.md | 10 + ts/00_commitinfo_data.ts | 2 +- ts/proxies/smart-proxy/models/interfaces.ts | 15 ++ ts/proxies/smart-proxy/smart-proxy.ts | 228 +++++++++++------- .../utils/concurrency-semaphore.ts | 28 +++ ts/proxies/smart-proxy/utils/index.ts | 3 + 6 files changed, 202 insertions(+), 84 deletions(-) create mode 100644 ts/proxies/smart-proxy/utils/concurrency-semaphore.ts diff --git a/changelog.md b/changelog.md index 6b8c684..b11ad48 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,15 @@ # Changelog +## 2026-02-14 - 25.3.0 - feat(smart-proxy) +add background concurrent certificate provisioning with per-domain timeouts and concurrency control + +- Add ISmartProxyOptions settings: certProvisionTimeout (ms) and certProvisionConcurrency (default 4) +- Run certProvisionFunction as fire-and-forget background tasks (stores promise on start/route-update and awaited on stop) +- Provision certificates in parallel with a concurrency limit using a new ConcurrencySemaphore utility +- Introduce per-domain timeout handling (default 300000ms) via withTimeout and surface timeout errors as certificate-failed events +- Refactor provisioning into provisionSingleDomain to isolate domain handling, ACME fallback preserved +- Run provisioning outside route update mutex so route updates are not blocked by slow provisioning + ## 2026-02-14 - 25.2.2 - fix(smart-proxy) start metrics polling before certificate provisioning to avoid blocking metrics collection diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index 53ca93d..15906cb 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@push.rocks/smartproxy', - version: '25.2.2', + version: '25.3.0', description: 'A powerful proxy package with unified route-based configuration for high traffic management. Features include SSL/TLS support, flexible routing patterns, WebSocket handling, advanced security options, and automatic ACME certificate management.' } diff --git a/ts/proxies/smart-proxy/models/interfaces.ts b/ts/proxies/smart-proxy/models/interfaces.ts index eb5d0bb..b3ff900 100644 --- a/ts/proxies/smart-proxy/models/interfaces.ts +++ b/ts/proxies/smart-proxy/models/interfaces.ts @@ -180,6 +180,21 @@ export interface ISmartProxyOptions { */ certProvisionFallbackToAcme?: boolean; + /** + * Per-domain timeout in ms for certProvisionFunction calls. + * If a single domain's provisioning takes longer than this, it's aborted + * and a certificate-failed event is emitted. + * Default: 300000 (5 minutes) + */ + certProvisionTimeout?: number; + + /** + * Maximum number of domains to provision certificates for concurrently. + * Prevents overwhelming ACME providers when many domains provision at once. + * Default: 4 + */ + certProvisionConcurrency?: number; + /** * Disable the default self-signed fallback certificate. * When false (default), a self-signed cert is generated at startup and loaded diff --git a/ts/proxies/smart-proxy/smart-proxy.ts b/ts/proxies/smart-proxy/smart-proxy.ts index 35bfc58..8db22c4 100644 --- a/ts/proxies/smart-proxy/smart-proxy.ts +++ b/ts/proxies/smart-proxy/smart-proxy.ts @@ -12,6 +12,7 @@ import { SharedRouteManager as RouteManager } from '../../core/routing/route-man import { RouteValidator } from './utils/route-validator.js'; import { generateDefaultCertificate } from './utils/default-cert-generator.js'; import { Mutex } from './utils/mutex.js'; +import { ConcurrencySemaphore } from './utils/concurrency-semaphore.js'; // Types import type { ISmartProxyOptions, TSmartProxyCertProvisionObject, IAcmeOptions, ICertProvisionEventComms, ICertificateIssuedEvent, ICertificateFailedEvent } from './models/interfaces.js'; @@ -38,6 +39,7 @@ export class SmartProxy extends plugins.EventEmitter { private metricsAdapter: RustMetricsAdapter; private routeUpdateLock: Mutex; private stopping = false; + private certProvisionPromise: Promise | null = null; constructor(settingsArg: ISmartProxyOptions) { super(); @@ -199,8 +201,10 @@ export class SmartProxy extends plugins.EventEmitter { logger.log('info', 'SmartProxy started (Rust engine)', { component: 'smart-proxy' }); - // Handle certProvisionFunction (may be slow — runs after startup is complete) - await this.provisionCertificatesViaCallback(preloadedDomains); + // Fire-and-forget cert provisioning — Rust engine is already running and serving traffic. + // Events (certificate-issued / certificate-failed) fire independently per domain. + this.certProvisionPromise = this.provisionCertificatesViaCallback(preloadedDomains) + .catch((err) => logger.log('error', `Unexpected error in cert provisioning: ${err.message}`, { component: 'smart-proxy' })); } /** @@ -210,6 +214,12 @@ export class SmartProxy extends plugins.EventEmitter { logger.log('info', 'SmartProxy shutting down...', { component: 'smart-proxy' }); this.stopping = true; + // Wait for in-flight cert provisioning to bail out (it checks this.stopping) + if (this.certProvisionPromise) { + await this.certProvisionPromise; + this.certProvisionPromise = null; + } + // Stop metrics polling this.metricsAdapter.stopPolling(); @@ -237,7 +247,7 @@ export class SmartProxy extends plugins.EventEmitter { * Update routes atomically. */ public async updateRoutes(newRoutes: IRouteConfig[]): Promise { - return this.routeUpdateLock.runExclusive(async () => { + await this.routeUpdateLock.runExclusive(async () => { // Validate const validation = RouteValidator.validateRoutes(newRoutes); if (!validation.valid) { @@ -273,11 +283,13 @@ export class SmartProxy extends plugins.EventEmitter { // Update stored routes this.settings.routes = newRoutes; - // Handle cert provisioning for new routes - await this.provisionCertificatesViaCallback(); - logger.log('info', `Routes updated (${newRoutes.length} routes)`, { component: 'smart-proxy' }); }); + + // Fire-and-forget cert provisioning outside the mutex — routes are already updated, + // cert provisioning doesn't need the route update lock and may be slow. + this.certProvisionPromise = this.provisionCertificatesViaCallback() + .catch((err) => logger.log('error', `Unexpected error in cert provisioning after route update: ${err.message}`, { component: 'smart-proxy' })); } /** @@ -412,7 +424,9 @@ export class SmartProxy extends plugins.EventEmitter { const provisionFn = this.settings.certProvisionFunction; if (!provisionFn) return; - const provisionedDomains = new Set(skipDomains); + // Phase 1: Collect all unique (domain, route) pairs that need provisioning + const seen = new Set(skipDomains); + const tasks: Array<{ domain: string; route: IRouteConfig }> = []; for (const route of this.settings.routes) { if (route.action.tls?.certificate !== 'auto') continue; @@ -422,91 +436,139 @@ export class SmartProxy extends plugins.EventEmitter { const certDomains = this.normalizeDomainsForCertProvisioning(rawDomains); for (const domain of certDomains) { - if (provisionedDomains.has(domain)) continue; - provisionedDomains.add(domain); + if (seen.has(domain)) continue; + seen.add(domain); + tasks.push({ domain, route }); + } + } - // Build eventComms channel for this domain - let expiryDate: string | undefined; - let source = 'certProvisionFunction'; + if (tasks.length === 0) return; - const eventComms: ICertProvisionEventComms = { - log: (msg) => logger.log('info', `[certProvision ${domain}] ${msg}`, { component: 'smart-proxy' }), - warn: (msg) => logger.log('warn', `[certProvision ${domain}] ${msg}`, { component: 'smart-proxy' }), - error: (msg) => logger.log('error', `[certProvision ${domain}] ${msg}`, { component: 'smart-proxy' }), - setExpiryDate: (date) => { expiryDate = date.toISOString(); }, - setSource: (s) => { source = s; }, - }; + // Phase 2: Process all domains in parallel with concurrency limit + const concurrency = this.settings.certProvisionConcurrency ?? 4; + const semaphore = new ConcurrencySemaphore(concurrency); + const promises = tasks.map(async ({ domain, route }) => { + await semaphore.acquire(); + try { + await this.provisionSingleDomain(domain, route, provisionFn); + } finally { + semaphore.release(); + } + }); + + await Promise.allSettled(promises); + } + + /** + * Provision a single domain's certificate via the callback. + * Includes per-domain timeout and shutdown checks. + */ + private async provisionSingleDomain( + domain: string, + route: IRouteConfig, + provisionFn: (domain: string, eventComms: ICertProvisionEventComms) => Promise, + ): Promise { + if (this.stopping) return; + + let expiryDate: string | undefined; + let source = 'certProvisionFunction'; + + const eventComms: ICertProvisionEventComms = { + log: (msg) => logger.log('info', `[certProvision ${domain}] ${msg}`, { component: 'smart-proxy' }), + warn: (msg) => logger.log('warn', `[certProvision ${domain}] ${msg}`, { component: 'smart-proxy' }), + error: (msg) => logger.log('error', `[certProvision ${domain}] ${msg}`, { component: 'smart-proxy' }), + setExpiryDate: (date) => { expiryDate = date.toISOString(); }, + setSource: (s) => { source = s; }, + }; + + const timeoutMs = this.settings.certProvisionTimeout ?? 300_000; // 5 min default + + try { + const result: TSmartProxyCertProvisionObject = await this.withTimeout( + provisionFn(domain, eventComms), + timeoutMs, + `Certificate provisioning timed out for ${domain} after ${timeoutMs}ms`, + ); + + if (this.stopping) return; + + if (result === 'http01') { + if (route.name) { + try { + await this.bridge.provisionCertificate(route.name); + logger.log('info', `Triggered Rust ACME for ${domain} (route: ${route.name})`, { component: 'smart-proxy' }); + } catch (provisionErr: any) { + logger.log('warn', `Cannot provision cert for ${domain} — callback returned 'http01' but Rust ACME failed: ${provisionErr.message}. ` + + 'Note: Rust ACME is disabled when certProvisionFunction is set.', { component: 'smart-proxy' }); + } + } + return; + } + + if (result && typeof result === 'object') { + if (this.stopping) return; + + const certObj = result as plugins.tsclass.network.ICert; + await this.bridge.loadCertificate( + domain, + certObj.publicKey, + certObj.privateKey, + ); + logger.log('info', `Certificate loaded via provision function for ${domain}`, { component: 'smart-proxy' }); + + // Persist to consumer store + if (this.settings.certStore?.save) { + try { + await this.settings.certStore.save(domain, certObj.publicKey, certObj.privateKey); + } catch (storeErr: any) { + logger.log('warn', `certStore.save() failed for ${domain}: ${storeErr.message}`, { component: 'smart-proxy' }); + } + } + + this.emit('certificate-issued', { + domain, + expiryDate: expiryDate || (certObj.validUntil ? new Date(certObj.validUntil).toISOString() : undefined), + source, + } satisfies ICertificateIssuedEvent); + } + } catch (err: any) { + logger.log('warn', `certProvisionFunction failed for ${domain}: ${err.message}`, { component: 'smart-proxy' }); + + this.emit('certificate-failed', { + domain, + error: err.message, + source, + } satisfies ICertificateFailedEvent); + + // Fallback to ACME if enabled and route has a name + if (this.settings.certProvisionFallbackToAcme !== false && route.name) { try { - const result: TSmartProxyCertProvisionObject = await provisionFn(domain, eventComms); - - if (result === 'http01') { - // Callback wants HTTP-01 for this domain — trigger Rust ACME explicitly - if (route.name) { - try { - await this.bridge.provisionCertificate(route.name); - logger.log('info', `Triggered Rust ACME for ${domain} (route: ${route.name})`, { component: 'smart-proxy' }); - } catch (provisionErr: any) { - logger.log('warn', `Cannot provision cert for ${domain} — callback returned 'http01' but Rust ACME failed: ${provisionErr.message}. ` + - 'Note: Rust ACME is disabled when certProvisionFunction is set.', { component: 'smart-proxy' }); - } - } - continue; - } - - // Got a static cert object - load it into Rust - if (result && typeof result === 'object') { - const certObj = result as plugins.tsclass.network.ICert; - await this.bridge.loadCertificate( - domain, - certObj.publicKey, - certObj.privateKey, - ); - logger.log('info', `Certificate loaded via provision function for ${domain}`, { component: 'smart-proxy' }); - - // Persist to consumer store - if (this.settings.certStore?.save) { - try { - await this.settings.certStore.save(domain, certObj.publicKey, certObj.privateKey); - } catch (storeErr: any) { - logger.log('warn', `certStore.save() failed for ${domain}: ${storeErr.message}`, { component: 'smart-proxy' }); - } - } - - // Emit certificate-issued event - this.emit('certificate-issued', { - domain, - expiryDate: expiryDate || (certObj.validUntil ? new Date(certObj.validUntil).toISOString() : undefined), - source, - } satisfies ICertificateIssuedEvent); - } - } catch (err: any) { - logger.log('warn', `certProvisionFunction failed for ${domain}: ${err.message}`, { component: 'smart-proxy' }); - - // Emit certificate-failed event - this.emit('certificate-failed', { - domain, - error: err.message, - source, - } satisfies ICertificateFailedEvent); - - // Fallback to ACME if enabled and route has a name - if (this.settings.certProvisionFallbackToAcme !== false && route.name) { - try { - await this.bridge.provisionCertificate(route.name); - logger.log('info', `Falling back to Rust ACME for ${domain} (route: ${route.name})`, { component: 'smart-proxy' }); - } catch (acmeErr: any) { - logger.log('warn', `ACME fallback also failed for ${domain}: ${acmeErr.message}` + - (this.settings.disableDefaultCert - ? ' — TLS will fail for this domain (disableDefaultCert is true)' - : ' — default self-signed fallback cert will be used'), { component: 'smart-proxy' }); - } - } + await this.bridge.provisionCertificate(route.name); + logger.log('info', `Falling back to Rust ACME for ${domain} (route: ${route.name})`, { component: 'smart-proxy' }); + } catch (acmeErr: any) { + logger.log('warn', `ACME fallback also failed for ${domain}: ${acmeErr.message}` + + (this.settings.disableDefaultCert + ? ' — TLS will fail for this domain (disableDefaultCert is true)' + : ' — default self-signed fallback cert will be used'), { component: 'smart-proxy' }); } } } } + /** + * Race a promise against a timeout. Rejects with the given message if the timeout fires first. + */ + private withTimeout(promise: Promise, ms: number, message: string): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error(message)), ms); + promise.then( + (val) => { clearTimeout(timer); resolve(val); }, + (err) => { clearTimeout(timer); reject(err); }, + ); + }); + } + /** * Normalize routing glob patterns into valid domain identifiers for cert provisioning. * - `*nevermind.cloud` → `['nevermind.cloud', '*.nevermind.cloud']` diff --git a/ts/proxies/smart-proxy/utils/concurrency-semaphore.ts b/ts/proxies/smart-proxy/utils/concurrency-semaphore.ts new file mode 100644 index 0000000..a7a2aff --- /dev/null +++ b/ts/proxies/smart-proxy/utils/concurrency-semaphore.ts @@ -0,0 +1,28 @@ +/** + * Async concurrency semaphore — limits the number of concurrent async operations. + */ +export class ConcurrencySemaphore { + private running = 0; + private waitQueue: Array<() => void> = []; + + constructor(private readonly maxConcurrency: number) {} + + async acquire(): Promise { + if (this.running < this.maxConcurrency) { + this.running++; + return; + } + return new Promise((resolve) => { + this.waitQueue.push(() => { + this.running++; + resolve(); + }); + }); + } + + release(): void { + this.running--; + const next = this.waitQueue.shift(); + if (next) next(); + } +} diff --git a/ts/proxies/smart-proxy/utils/index.ts b/ts/proxies/smart-proxy/utils/index.ts index d5fb631..6884ff1 100644 --- a/ts/proxies/smart-proxy/utils/index.ts +++ b/ts/proxies/smart-proxy/utils/index.ts @@ -17,6 +17,9 @@ export * from './route-utils.js'; // Export default certificate generator export { generateDefaultCertificate } from './default-cert-generator.js'; +// Export concurrency semaphore +export { ConcurrencySemaphore } from './concurrency-semaphore.js'; + // Export additional functions from route-helpers that weren't already exported export { createApiGatewayRoute,