BREAKING CHANGE(certs): Introduce domain-centric certificate provisioning with per-domain exponential backoff and a staggered serial scheduler; add domain-based reprovision API and UI backoff display; change certificate overview API to be domain-first and include backoff info; bump related deps.

ts/classes.cert-provision-scheduler.ts

@@ -0,0 +1,176 @@
+import { logger } from './logger.js';
+import type { StorageManager } from './storage/index.js';
+
+interface IBackoffEntry {
+  failures: number;
+  lastFailure: string; // ISO string
+  retryAfter: string;  // ISO string
+  lastError?: string;
+}
+
+/**
+ * Manages certificate provisioning scheduling with:
+ * - Per-domain exponential backoff persisted in StorageManager
+ * - Serial stagger queue with configurable delay between provisions
+ */
+export class CertProvisionScheduler {
+  private storageManager: StorageManager;
+  private staggerDelayMs: number;
+  private maxBackoffHours: number;
+
+  // In-memory serial queue
+  private queue: Array<{
+    domain: string;
+    fn: () => Promise<any>;
+    resolve: (value: any) => void;
+    reject: (err: any) => void;
+  }> = [];
+  private processing = false;
+
+  // In-memory backoff cache (mirrors storage for fast lookups)
+  private backoffCache = new Map<string, IBackoffEntry>();
+
+  constructor(
+    storageManager: StorageManager,
+    options?: { staggerDelayMs?: number; maxBackoffHours?: number }
+  ) {
+    this.storageManager = storageManager;
+    this.staggerDelayMs = options?.staggerDelayMs ?? 3000;
+    this.maxBackoffHours = options?.maxBackoffHours ?? 24;
+  }
+
+  /**
+   * Storage key for a domain's backoff entry
+   */
+  private backoffKey(domain: string): string {
+    const clean = domain.replace(/\*/g, '_wildcard_').replace(/[^a-zA-Z0-9._-]/g, '_');
+    return `/cert-backoff/${clean}`;
+  }
+
+  /**
+   * Load backoff entry from storage (with in-memory cache)
+   */
+  private async loadBackoff(domain: string): Promise<IBackoffEntry | null> {
+    const cached = this.backoffCache.get(domain);
+    if (cached) return cached;
+
+    const entry = await this.storageManager.getJSON<IBackoffEntry>(this.backoffKey(domain));
+    if (entry) {
+      this.backoffCache.set(domain, entry);
+    }
+    return entry;
+  }
+
+  /**
+   * Save backoff entry to both cache and storage
+   */
+  private async saveBackoff(domain: string, entry: IBackoffEntry): Promise<void> {
+    this.backoffCache.set(domain, entry);
+    await this.storageManager.setJSON(this.backoffKey(domain), entry);
+  }
+
+  /**
+   * Check if a domain is currently in backoff
+   */
+  async isInBackoff(domain: string): Promise<boolean> {
+    const entry = await this.loadBackoff(domain);
+    if (!entry) return false;
+
+    const retryAfter = new Date(entry.retryAfter);
+    return retryAfter.getTime() > Date.now();
+  }
+
+  /**
+   * Record a provisioning failure for a domain.
+   * Sets exponential backoff: min(failures^2 * 1h, maxBackoffHours)
+   */
+  async recordFailure(domain: string, error?: string): Promise<void> {
+    const existing = await this.loadBackoff(domain);
+    const failures = (existing?.failures ?? 0) + 1;
+
+    // Exponential backoff: failures^2 hours, capped
+    const backoffHours = Math.min(failures * failures, this.maxBackoffHours);
+    const retryAfter = new Date(Date.now() + backoffHours * 60 * 60 * 1000);
+
+    const entry: IBackoffEntry = {
+      failures,
+      lastFailure: new Date().toISOString(),
+      retryAfter: retryAfter.toISOString(),
+      lastError: error,
+    };
+
+    await this.saveBackoff(domain, entry);
+    logger.log('warn', `Cert backoff for ${domain}: ${failures} failures, retry after ${retryAfter.toISOString()}`);
+  }
+
+  /**
+   * Clear backoff for a domain (on success or manual override)
+   */
+  async clearBackoff(domain: string): Promise<void> {
+    this.backoffCache.delete(domain);
+    try {
+      await this.storageManager.delete(this.backoffKey(domain));
+    } catch {
+      // Ignore delete errors (key may not exist)
+    }
+  }
+
+  /**
+   * Get backoff info for UI display
+   */
+  async getBackoffInfo(domain: string): Promise<{
+    failures: number;
+    retryAfter?: string;
+    lastError?: string;
+  } | null> {
+    const entry = await this.loadBackoff(domain);
+    if (!entry) return null;
+
+    // Only return if still in backoff
+    const retryAfter = new Date(entry.retryAfter);
+    if (retryAfter.getTime() <= Date.now()) return null;
+
+    return {
+      failures: entry.failures,
+      retryAfter: entry.retryAfter,
+      lastError: entry.lastError,
+    };
+  }
+
+  /**
+   * Enqueue a provision operation for serial execution with stagger delay.
+   * Returns the result of the provision function.
+   */
+  enqueueProvision<T>(domain: string, fn: () => Promise<T>): Promise<T> {
+    return new Promise<T>((resolve, reject) => {
+      this.queue.push({ domain, fn, resolve, reject });
+      this.processQueue();
+    });
+  }
+
+  /**
+   * Process the stagger queue serially
+   */
+  private async processQueue(): Promise<void> {
+    if (this.processing) return;
+    this.processing = true;
+
+    while (this.queue.length > 0) {
+      const item = this.queue.shift()!;
+      try {
+        logger.log('info', `Processing cert provision for ${item.domain}`);
+        const result = await item.fn();
+        item.resolve(result);
+      } catch (err) {
+        item.reject(err);
+      }
+
+      // Stagger delay between provisions
+      if (this.queue.length > 0) {
+        await new Promise<void>((r) => setTimeout(r, this.staggerDelayMs));
+      }
+    }
+
+    this.processing = false;
+  }
+}