feat(smart-proxy): add background concurrent certificate provisioning with per-domain timeouts and concurrency control

changelog.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## 2026-02-14 - 25.3.0 - feat(smart-proxy)
+add background concurrent certificate provisioning with per-domain timeouts and concurrency control
+
+- Add ISmartProxyOptions settings: certProvisionTimeout (ms) and certProvisionConcurrency (default 4)
+- Run certProvisionFunction as fire-and-forget background tasks (stores promise on start/route-update and awaited on stop)
+- Provision certificates in parallel with a concurrency limit using a new ConcurrencySemaphore utility
+- Introduce per-domain timeout handling (default 300000ms) via withTimeout and surface timeout errors as certificate-failed events
+- Refactor provisioning into provisionSingleDomain to isolate domain handling, ACME fallback preserved
+- Run provisioning outside route update mutex so route updates are not blocked by slow provisioning
+
 ## 2026-02-14 - 25.2.2 - fix(smart-proxy)
 start metrics polling before certificate provisioning to avoid blocking metrics collection
 

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@push.rocks/smartproxy',
-  version: '25.2.2',
+  version: '25.3.0',
   description: 'A powerful proxy package with unified route-based configuration for high traffic management. Features include SSL/TLS support, flexible routing patterns, WebSocket handling, advanced security options, and automatic ACME certificate management.'
 }

ts/proxies/smart-proxy/models/interfaces.ts

@@ -180,6 +180,21 @@ export interface ISmartProxyOptions {
    */
   certProvisionFallbackToAcme?: boolean;
 
+  /**
+   * Per-domain timeout in ms for certProvisionFunction calls.
+   * If a single domain's provisioning takes longer than this, it's aborted
+   * and a certificate-failed event is emitted.
+   * Default: 300000 (5 minutes)
+   */
+  certProvisionTimeout?: number;
+
+  /**
+   * Maximum number of domains to provision certificates for concurrently.
+   * Prevents overwhelming ACME providers when many domains provision at once.
+   * Default: 4
+   */
+  certProvisionConcurrency?: number;
+
   /**
    * Disable the default self-signed fallback certificate.
    * When false (default), a self-signed cert is generated at startup and loaded

ts/proxies/smart-proxy/smart-proxy.ts

@@ -12,6 +12,7 @@ import { SharedRouteManager as RouteManager } from '../../core/routing/route-man
 import { RouteValidator } from './utils/route-validator.js';
 import { generateDefaultCertificate } from './utils/default-cert-generator.js';
 import { Mutex } from './utils/mutex.js';
+import { ConcurrencySemaphore } from './utils/concurrency-semaphore.js';
 
 // Types
 import type { ISmartProxyOptions, TSmartProxyCertProvisionObject, IAcmeOptions, ICertProvisionEventComms, ICertificateIssuedEvent, ICertificateFailedEvent } from './models/interfaces.js';
@@ -38,6 +39,7 @@ export class SmartProxy extends plugins.EventEmitter {
   private metricsAdapter: RustMetricsAdapter;
   private routeUpdateLock: Mutex;
   private stopping = false;
+  private certProvisionPromise: Promise<void> | null = null;
 
   constructor(settingsArg: ISmartProxyOptions) {
     super();
@@ -199,8 +201,10 @@ export class SmartProxy extends plugins.EventEmitter {
 
     logger.log('info', 'SmartProxy started (Rust engine)', { component: 'smart-proxy' });
 
-    // Handle certProvisionFunction (may be slow — runs after startup is complete)
+    // Fire-and-forget cert provisioning — Rust engine is already running and serving traffic.
-    await this.provisionCertificatesViaCallback(preloadedDomains);
+    // Events (certificate-issued / certificate-failed) fire independently per domain.
+    this.certProvisionPromise = this.provisionCertificatesViaCallback(preloadedDomains)
+      .catch((err) => logger.log('error', `Unexpected error in cert provisioning: ${err.message}`, { component: 'smart-proxy' }));
   }
 
   /**
@@ -210,6 +214,12 @@ export class SmartProxy extends plugins.EventEmitter {
     logger.log('info', 'SmartProxy shutting down...', { component: 'smart-proxy' });
     this.stopping = true;
 
+    // Wait for in-flight cert provisioning to bail out (it checks this.stopping)
+    if (this.certProvisionPromise) {
+      await this.certProvisionPromise;
+      this.certProvisionPromise = null;
+    }
+
     // Stop metrics polling
     this.metricsAdapter.stopPolling();
 
@@ -237,7 +247,7 @@ export class SmartProxy extends plugins.EventEmitter {
    * Update routes atomically.
    */
   public async updateRoutes(newRoutes: IRouteConfig[]): Promise<void> {
-    return this.routeUpdateLock.runExclusive(async () => {
+    await this.routeUpdateLock.runExclusive(async () => {
       // Validate
       const validation = RouteValidator.validateRoutes(newRoutes);
       if (!validation.valid) {
@@ -273,11 +283,13 @@ export class SmartProxy extends plugins.EventEmitter {
       // Update stored routes
       this.settings.routes = newRoutes;
 
-      // Handle cert provisioning for new routes
-      await this.provisionCertificatesViaCallback();
-
       logger.log('info', `Routes updated (${newRoutes.length} routes)`, { component: 'smart-proxy' });
     });
+
+    // Fire-and-forget cert provisioning outside the mutex — routes are already updated,
+    // cert provisioning doesn't need the route update lock and may be slow.
+    this.certProvisionPromise = this.provisionCertificatesViaCallback()
+      .catch((err) => logger.log('error', `Unexpected error in cert provisioning after route update: ${err.message}`, { component: 'smart-proxy' }));
   }
 
   /**
@@ -412,7 +424,9 @@ export class SmartProxy extends plugins.EventEmitter {
     const provisionFn = this.settings.certProvisionFunction;
     if (!provisionFn) return;
 
-    const provisionedDomains = new Set<string>(skipDomains);
+    // Phase 1: Collect all unique (domain, route) pairs that need provisioning
+    const seen = new Set<string>(skipDomains);
+    const tasks: Array<{ domain: string; route: IRouteConfig }> = [];
 
     for (const route of this.settings.routes) {
       if (route.action.tls?.certificate !== 'auto') continue;
@@ -422,91 +436,139 @@ export class SmartProxy extends plugins.EventEmitter {
       const certDomains = this.normalizeDomainsForCertProvisioning(rawDomains);
 
       for (const domain of certDomains) {
-        if (provisionedDomains.has(domain)) continue;
+        if (seen.has(domain)) continue;
-        provisionedDomains.add(domain);
+        seen.add(domain);
+        tasks.push({ domain, route });
+      }
+    }
 
-        // Build eventComms channel for this domain
+    if (tasks.length === 0) return;
-        let expiryDate: string | undefined;
-        let source = 'certProvisionFunction';
 
-        const eventComms: ICertProvisionEventComms = {
+    // Phase 2: Process all domains in parallel with concurrency limit
-          log: (msg) => logger.log('info', `[certProvision ${domain}] ${msg}`, { component: 'smart-proxy' }),
+    const concurrency = this.settings.certProvisionConcurrency ?? 4;
-          warn: (msg) => logger.log('warn', `[certProvision ${domain}] ${msg}`, { component: 'smart-proxy' }),
+    const semaphore = new ConcurrencySemaphore(concurrency);
-          error: (msg) => logger.log('error', `[certProvision ${domain}] ${msg}`, { component: 'smart-proxy' }),
-          setExpiryDate: (date) => { expiryDate = date.toISOString(); },
-          setSource: (s) => { source = s; },
-        };
 
+    const promises = tasks.map(async ({ domain, route }) => {
+      await semaphore.acquire();
+      try {
+        await this.provisionSingleDomain(domain, route, provisionFn);
+      } finally {
+        semaphore.release();
+      }
+    });
+
+    await Promise.allSettled(promises);
+  }
+
+  /**
+   * Provision a single domain's certificate via the callback.
+   * Includes per-domain timeout and shutdown checks.
+   */
+  private async provisionSingleDomain(
+    domain: string,
+    route: IRouteConfig,
+    provisionFn: (domain: string, eventComms: ICertProvisionEventComms) => Promise<TSmartProxyCertProvisionObject>,
+  ): Promise<void> {
+    if (this.stopping) return;
+
+    let expiryDate: string | undefined;
+    let source = 'certProvisionFunction';
+
+    const eventComms: ICertProvisionEventComms = {
+      log: (msg) => logger.log('info', `[certProvision ${domain}] ${msg}`, { component: 'smart-proxy' }),
+      warn: (msg) => logger.log('warn', `[certProvision ${domain}] ${msg}`, { component: 'smart-proxy' }),
+      error: (msg) => logger.log('error', `[certProvision ${domain}] ${msg}`, { component: 'smart-proxy' }),
+      setExpiryDate: (date) => { expiryDate = date.toISOString(); },
+      setSource: (s) => { source = s; },
+    };
+
+    const timeoutMs = this.settings.certProvisionTimeout ?? 300_000; // 5 min default
+
+    try {
+      const result: TSmartProxyCertProvisionObject = await this.withTimeout(
+        provisionFn(domain, eventComms),
+        timeoutMs,
+        `Certificate provisioning timed out for ${domain} after ${timeoutMs}ms`,
+      );
+
+      if (this.stopping) return;
+
+      if (result === 'http01') {
+        if (route.name) {
+          try {
+            await this.bridge.provisionCertificate(route.name);
+            logger.log('info', `Triggered Rust ACME for ${domain} (route: ${route.name})`, { component: 'smart-proxy' });
+          } catch (provisionErr: any) {
+            logger.log('warn', `Cannot provision cert for ${domain} — callback returned 'http01' but Rust ACME failed: ${provisionErr.message}. ` +
+              'Note: Rust ACME is disabled when certProvisionFunction is set.', { component: 'smart-proxy' });
+          }
+        }
+        return;
+      }
+
+      if (result && typeof result === 'object') {
+        if (this.stopping) return;
+
+        const certObj = result as plugins.tsclass.network.ICert;
+        await this.bridge.loadCertificate(
+          domain,
+          certObj.publicKey,
+          certObj.privateKey,
+        );
+        logger.log('info', `Certificate loaded via provision function for ${domain}`, { component: 'smart-proxy' });
+
+        // Persist to consumer store
+        if (this.settings.certStore?.save) {
+          try {
+            await this.settings.certStore.save(domain, certObj.publicKey, certObj.privateKey);
+          } catch (storeErr: any) {
+            logger.log('warn', `certStore.save() failed for ${domain}: ${storeErr.message}`, { component: 'smart-proxy' });
+          }
+        }
+
+        this.emit('certificate-issued', {
+          domain,
+          expiryDate: expiryDate || (certObj.validUntil ? new Date(certObj.validUntil).toISOString() : undefined),
+          source,
+        } satisfies ICertificateIssuedEvent);
+      }
+    } catch (err: any) {
+      logger.log('warn', `certProvisionFunction failed for ${domain}: ${err.message}`, { component: 'smart-proxy' });
+
+      this.emit('certificate-failed', {
+        domain,
+        error: err.message,
+        source,
+      } satisfies ICertificateFailedEvent);
+
+      // Fallback to ACME if enabled and route has a name
+      if (this.settings.certProvisionFallbackToAcme !== false && route.name) {
         try {
-          const result: TSmartProxyCertProvisionObject = await provisionFn(domain, eventComms);
+          await this.bridge.provisionCertificate(route.name);
-
+          logger.log('info', `Falling back to Rust ACME for ${domain} (route: ${route.name})`, { component: 'smart-proxy' });
-          if (result === 'http01') {
+        } catch (acmeErr: any) {
-            // Callback wants HTTP-01 for this domain — trigger Rust ACME explicitly
+          logger.log('warn', `ACME fallback also failed for ${domain}: ${acmeErr.message}` +
-            if (route.name) {
+            (this.settings.disableDefaultCert
-              try {
+              ? ' — TLS will fail for this domain (disableDefaultCert is true)'
-                await this.bridge.provisionCertificate(route.name);
+              : ' — default self-signed fallback cert will be used'), { component: 'smart-proxy' });
-                logger.log('info', `Triggered Rust ACME for ${domain} (route: ${route.name})`, { component: 'smart-proxy' });
-              } catch (provisionErr: any) {
-                logger.log('warn', `Cannot provision cert for ${domain} — callback returned 'http01' but Rust ACME failed: ${provisionErr.message}. ` +
-                  'Note: Rust ACME is disabled when certProvisionFunction is set.', { component: 'smart-proxy' });
-              }
-            }
-            continue;
-          }
-
-          // Got a static cert object - load it into Rust
-          if (result && typeof result === 'object') {
-            const certObj = result as plugins.tsclass.network.ICert;
-            await this.bridge.loadCertificate(
-              domain,
-              certObj.publicKey,
-              certObj.privateKey,
-            );
-            logger.log('info', `Certificate loaded via provision function for ${domain}`, { component: 'smart-proxy' });
-
-            // Persist to consumer store
-            if (this.settings.certStore?.save) {
-              try {
-                await this.settings.certStore.save(domain, certObj.publicKey, certObj.privateKey);
-              } catch (storeErr: any) {
-                logger.log('warn', `certStore.save() failed for ${domain}: ${storeErr.message}`, { component: 'smart-proxy' });
-              }
-            }
-
-            // Emit certificate-issued event
-            this.emit('certificate-issued', {
-              domain,
-              expiryDate: expiryDate || (certObj.validUntil ? new Date(certObj.validUntil).toISOString() : undefined),
-              source,
-            } satisfies ICertificateIssuedEvent);
-          }
-        } catch (err: any) {
-          logger.log('warn', `certProvisionFunction failed for ${domain}: ${err.message}`, { component: 'smart-proxy' });
-
-          // Emit certificate-failed event
-          this.emit('certificate-failed', {
-            domain,
-            error: err.message,
-            source,
-          } satisfies ICertificateFailedEvent);
-
-          // Fallback to ACME if enabled and route has a name
-          if (this.settings.certProvisionFallbackToAcme !== false && route.name) {
-            try {
-              await this.bridge.provisionCertificate(route.name);
-              logger.log('info', `Falling back to Rust ACME for ${domain} (route: ${route.name})`, { component: 'smart-proxy' });
-            } catch (acmeErr: any) {
-              logger.log('warn', `ACME fallback also failed for ${domain}: ${acmeErr.message}` +
-                (this.settings.disableDefaultCert
-                  ? ' — TLS will fail for this domain (disableDefaultCert is true)'
-                  : ' — default self-signed fallback cert will be used'), { component: 'smart-proxy' });
-            }
-          }
         }
       }
     }
   }
 
+  /**
+   * Race a promise against a timeout. Rejects with the given message if the timeout fires first.
+   */
+  private withTimeout<T>(promise: Promise<T>, ms: number, message: string): Promise<T> {
+    return new Promise<T>((resolve, reject) => {
+      const timer = setTimeout(() => reject(new Error(message)), ms);
+      promise.then(
+        (val) => { clearTimeout(timer); resolve(val); },
+        (err) => { clearTimeout(timer); reject(err); },
+      );
+    });
+  }
+
   /**
    * Normalize routing glob patterns into valid domain identifiers for cert provisioning.
    * - `*nevermind.cloud` → `['nevermind.cloud', '*.nevermind.cloud']`

ts/proxies/smart-proxy/utils/concurrency-semaphore.ts

@@ -0,0 +1,28 @@
+/**
+ * Async concurrency semaphore — limits the number of concurrent async operations.
+ */
+export class ConcurrencySemaphore {
+  private running = 0;
+  private waitQueue: Array<() => void> = [];
+
+  constructor(private readonly maxConcurrency: number) {}
+
+  async acquire(): Promise<void> {
+    if (this.running < this.maxConcurrency) {
+      this.running++;
+      return;
+    }
+    return new Promise<void>((resolve) => {
+      this.waitQueue.push(() => {
+        this.running++;
+        resolve();
+      });
+    });
+  }
+
+  release(): void {
+    this.running--;
+    const next = this.waitQueue.shift();
+    if (next) next();
+  }
+}

ts/proxies/smart-proxy/utils/index.ts

@@ -17,6 +17,9 @@ export * from './route-utils.js';
 // Export default certificate generator
 export { generateDefaultCertificate } from './default-cert-generator.js';
 
+// Export concurrency semaphore
+export { ConcurrencySemaphore } from './concurrency-semaphore.js';
+
 // Export additional functions from route-helpers that weren't already exported
 export {
   createApiGatewayRoute,