BREAKING CHANGE(certs): Introduce domain-centric certificate provisioning with per-domain exponential backoff and a staggered serial scheduler; add domain-based reprovision API and UI backoff display; change certificate overview API to be domain-first and include backoff info; bump related deps.

ts/classes.dcrouter.ts

@@ -14,6 +14,7 @@ import { logger } from './logger.js';
 // Import storage manager
 import { StorageManager, type IStorageConfig } from './storage/index.js';
 import { StorageBackedCertManager } from './classes.storage-cert-manager.js';
+import { CertProvisionScheduler } from './classes.cert-provision-scheduler.js';
 // Import cache system
 import { CacheDb, CacheCleaner, type ICacheDbOptions } from './cache/index.js';
 
@@ -184,16 +185,19 @@ export class DcRouter {
   public cacheDb?: CacheDb;
   public cacheCleaner?: CacheCleaner;
 
-  // Certificate status tracking from SmartProxy events
+  // Certificate status tracking from SmartProxy events (keyed by domain)
   public certificateStatusMap = new Map<string, {
     status: 'valid' | 'failed';
-    domain: string;
+    routeNames: string[];
     expiryDate?: string;
     issuedAt?: string;
     source?: string;
     error?: string;
   }>();
 
+  // Certificate provisioning scheduler with backoff + stagger
+  public certProvisionScheduler?: CertProvisionScheduler;
+
   // TypedRouter for API endpoints
   public typedrouter = new plugins.typedrequest.TypedRouter();
 
@@ -467,6 +471,9 @@ export class DcRouter {
         },
       };
       
+      // Initialize cert provision scheduler
+      this.certProvisionScheduler = new CertProvisionScheduler(this.storageManager);
+
       // If we have DNS challenge handlers, create SmartAcme and wire to certProvisionFunction
       if (challengeHandlers.length > 0) {
         this.smartAcme = new plugins.smartacme.SmartAcme({
@@ -478,24 +485,41 @@ export class DcRouter {
         });
         await this.smartAcme.start();
 
+        const scheduler = this.certProvisionScheduler;
         smartProxyConfig.certProvisionFunction = async (domain, eventComms) => {
+          // Check backoff before attempting provision
+          if (await scheduler.isInBackoff(domain)) {
+            const info = await scheduler.getBackoffInfo(domain);
+            const msg = `Domain ${domain} is in backoff (${info?.failures} failures), retry after ${info?.retryAfter}`;
+            eventComms.warn(msg);
+            throw new Error(msg);
+          }
+
           try {
-            eventComms.log(`Attempting DNS-01 via SmartAcme for ${domain}`);
-            eventComms.setSource('smartacme-dns-01');
-            const cert = await this.smartAcme.getCertificateForDomain(domain);
-            if (cert.validUntil) {
-              eventComms.setExpiryDate(new Date(cert.validUntil));
-            }
-            return {
-              id: cert.id,
-              domainName: cert.domainName,
-              created: cert.created,
-              validUntil: cert.validUntil,
-              privateKey: cert.privateKey,
-              publicKey: cert.publicKey,
-              csr: cert.csr,
-            };
+            const result = await scheduler.enqueueProvision(domain, async () => {
+              eventComms.log(`Attempting DNS-01 via SmartAcme for ${domain}`);
+              eventComms.setSource('smartacme-dns-01');
+              const cert = await this.smartAcme.getCertificateForDomain(domain);
+              if (cert.validUntil) {
+                eventComms.setExpiryDate(new Date(cert.validUntil));
+              }
+              return {
+                id: cert.id,
+                domainName: cert.domainName,
+                created: cert.created,
+                validUntil: cert.validUntil,
+                privateKey: cert.privateKey,
+                publicKey: cert.publicKey,
+                csr: cert.csr,
+              };
+            });
+
+            // Success — clear any backoff
+            await scheduler.clearBackoff(domain);
+            return result;
           } catch (err) {
+            // Record failure for backoff tracking
+            await scheduler.recordFailure(domain, err.message);
             eventComms.warn(`SmartAcme DNS-01 failed for ${domain}: ${err.message}, falling back to http-01`);
             return 'http01';
           }
@@ -519,39 +543,34 @@ export class DcRouter {
       });
       
       // Always listen for certificate events — emitted by both ACME and certProvisionFunction paths
+      // Events are keyed by domain for domain-centric certificate tracking
       this.smartProxy.on('certificate-issued', (event: plugins.smartproxy.ICertificateIssuedEvent) => {
         console.log(`[DcRouter] Certificate issued for ${event.domain} via ${event.source}, expires ${event.expiryDate}`);
-        const routeName = this.findRouteNameForDomain(event.domain);
-        if (routeName) {
-          this.certificateStatusMap.set(routeName, {
-            status: 'valid', domain: event.domain,
-            expiryDate: event.expiryDate, issuedAt: new Date().toISOString(),
-            source: event.source,
-          });
-        }
+        const routeNames = this.findRouteNamesForDomain(event.domain);
+        this.certificateStatusMap.set(event.domain, {
+          status: 'valid', routeNames,
+          expiryDate: event.expiryDate, issuedAt: new Date().toISOString(),
+          source: event.source,
+        });
       });
 
       this.smartProxy.on('certificate-renewed', (event: plugins.smartproxy.ICertificateIssuedEvent) => {
         console.log(`[DcRouter] Certificate renewed for ${event.domain} via ${event.source}, expires ${event.expiryDate}`);
-        const routeName = this.findRouteNameForDomain(event.domain);
-        if (routeName) {
-          this.certificateStatusMap.set(routeName, {
-            status: 'valid', domain: event.domain,
-            expiryDate: event.expiryDate, issuedAt: new Date().toISOString(),
-            source: event.source,
-          });
-        }
+        const routeNames = this.findRouteNamesForDomain(event.domain);
+        this.certificateStatusMap.set(event.domain, {
+          status: 'valid', routeNames,
+          expiryDate: event.expiryDate, issuedAt: new Date().toISOString(),
+          source: event.source,
+        });
       });
 
       this.smartProxy.on('certificate-failed', (event: plugins.smartproxy.ICertificateFailedEvent) => {
         console.error(`[DcRouter] Certificate failed for ${event.domain} (${event.source}):`, event.error);
-        const routeName = this.findRouteNameForDomain(event.domain);
-        if (routeName) {
-          this.certificateStatusMap.set(routeName, {
-            status: 'failed', domain: event.domain, error: event.error,
-            source: event.source,
-          });
-        }
+        const routeNames = this.findRouteNamesForDomain(event.domain);
+        this.certificateStatusMap.set(event.domain, {
+          status: 'failed', routeNames, error: event.error,
+          source: event.source,
+        });
       });
       
       // Start SmartProxy
@@ -724,7 +743,7 @@ export class DcRouter {
   }
 
   /**
-   * Find the route name that matches a given domain
+   * Find the first route name that matches a given domain
    */
   private findRouteNameForDomain(domain: string): string | undefined {
     if (!this.smartProxy) return undefined;
@@ -740,6 +759,27 @@ export class DcRouter {
     return undefined;
   }
 
+  /**
+   * Find ALL route names that match a given domain
+   */
+  public findRouteNamesForDomain(domain: string): string[] {
+    if (!this.smartProxy) return [];
+    const names: string[] = [];
+    for (const route of this.smartProxy.routeManager.getRoutes()) {
+      if (!route.match.domains || !route.name) continue;
+      const routeDomains = Array.isArray(route.match.domains)
+        ? route.match.domains
+        : [route.match.domains];
+      for (const pattern of routeDomains) {
+        if (this.isDomainMatch(domain, pattern)) {
+          names.push(route.name);
+          break; // This route already matched, no need to check other patterns
+        }
+      }
+    }
+    return names;
+  }
+
   public async stop() {
     console.log('Stopping DcRouter services...');