dcrouter/ts/classes.cert-provision-scheduler.ts

import { logger } from './logger.js';
import type { StorageManager } from './storage/index.js';

interface IBackoffEntry {
  failures: number;
  lastFailure: string; // ISO string
  retryAfter: string;  // ISO string
  lastError?: string;
}

/**
 * Manages certificate provisioning scheduling with:
 * - Per-domain exponential backoff persisted in StorageManager
 * - Serial stagger queue with configurable delay between provisions
 */
export class CertProvisionScheduler {
  private storageManager: StorageManager;
  private staggerDelayMs: number;
  private maxBackoffHours: number;

  // In-memory serial queue
  private queue: Array<{
    domain: string;
    fn: () => Promise<any>;
    resolve: (value: any) => void;
    reject: (err: any) => void;
  }> = [];
  private processing = false;

  // In-memory backoff cache (mirrors storage for fast lookups)
  private backoffCache = new Map<string, IBackoffEntry>();

  constructor(
    storageManager: StorageManager,
    options?: { staggerDelayMs?: number; maxBackoffHours?: number }
  ) {
    this.storageManager = storageManager;
    this.staggerDelayMs = options?.staggerDelayMs ?? 3000;
    this.maxBackoffHours = options?.maxBackoffHours ?? 24;
  }

  /**
   * Storage key for a domain's backoff entry
   */
  private backoffKey(domain: string): string {
    const clean = domain.replace(/\*/g, '_wildcard_').replace(/[^a-zA-Z0-9._-]/g, '_');
    return `/cert-backoff/${clean}`;
  }

  /**
   * Load backoff entry from storage (with in-memory cache)
   */
  private async loadBackoff(domain: string): Promise<IBackoffEntry | null> {
    const cached = this.backoffCache.get(domain);
    if (cached) return cached;

    const entry = await this.storageManager.getJSON<IBackoffEntry>(this.backoffKey(domain));
    if (entry) {
      this.backoffCache.set(domain, entry);
    }
    return entry;
  }

  /**
   * Save backoff entry to both cache and storage
   */
  private async saveBackoff(domain: string, entry: IBackoffEntry): Promise<void> {
    this.backoffCache.set(domain, entry);
    await this.storageManager.setJSON(this.backoffKey(domain), entry);
  }

  /**
   * Check if a domain is currently in backoff
   */
  async isInBackoff(domain: string): Promise<boolean> {
    const entry = await this.loadBackoff(domain);
    if (!entry) return false;

    const retryAfter = new Date(entry.retryAfter);
    return retryAfter.getTime() > Date.now();
  }

  /**
   * Record a provisioning failure for a domain.
   * Sets exponential backoff: min(failures^2 * 1h, maxBackoffHours)
   */
  async recordFailure(domain: string, error?: string): Promise<void> {
    const existing = await this.loadBackoff(domain);
    const failures = (existing?.failures ?? 0) + 1;

    // Exponential backoff: failures^2 hours, capped
    const backoffHours = Math.min(failures * failures, this.maxBackoffHours);
    const retryAfter = new Date(Date.now() + backoffHours * 60 * 60 * 1000);

    const entry: IBackoffEntry = {
      failures,
      lastFailure: new Date().toISOString(),
      retryAfter: retryAfter.toISOString(),
      lastError: error,
    };

    await this.saveBackoff(domain, entry);
    logger.log('warn', `Cert backoff for ${domain}: ${failures} failures, retry after ${retryAfter.toISOString()}`);
  }

  /**
   * Clear backoff for a domain (on success or manual override)
   */
  async clearBackoff(domain: string): Promise<void> {
    this.backoffCache.delete(domain);
    try {
      await this.storageManager.delete(this.backoffKey(domain));
    } catch {
      // Ignore delete errors (key may not exist)
    }
  }

  /**
   * Get backoff info for UI display
   */
  async getBackoffInfo(domain: string): Promise<{
    failures: number;
    retryAfter?: string;
    lastError?: string;
  } | null> {
    const entry = await this.loadBackoff(domain);
    if (!entry) return null;

    // Only return if still in backoff
    const retryAfter = new Date(entry.retryAfter);
    if (retryAfter.getTime() <= Date.now()) return null;

    return {
      failures: entry.failures,
      retryAfter: entry.retryAfter,
      lastError: entry.lastError,
    };
  }

  /**
   * Enqueue a provision operation for serial execution with stagger delay.
   * Returns the result of the provision function.
   */
  enqueueProvision<T>(domain: string, fn: () => Promise<T>): Promise<T> {
    return new Promise<T>((resolve, reject) => {
      this.queue.push({ domain, fn, resolve, reject });
      this.processQueue();
    });
  }

  /**
   * Process the stagger queue serially
   */
  private async processQueue(): Promise<void> {
    if (this.processing) return;
    this.processing = true;

    while (this.queue.length > 0) {
      const item = this.queue.shift()!;
      try {
        logger.log('info', `Processing cert provision for ${item.domain}`);
        const result = await item.fn();
        item.resolve(result);
      } catch (err) {
        item.reject(err);
      }

      // Stagger delay between provisions
      if (this.queue.length > 0) {
        await new Promise<void>((r) => setTimeout(r, this.staggerDelayMs));
      }
    }

    this.processing = false;
  }
}