2026-02-15 16:03:13 +00:00
|
|
|
import { logger } from './logger.js';
|
|
|
|
|
import type { StorageManager } from './storage/index.js';
|
|
|
|
|
|
|
|
|
|
interface IBackoffEntry {
|
|
|
|
|
failures: number;
|
|
|
|
|
lastFailure: string; // ISO string
|
|
|
|
|
retryAfter: string; // ISO string
|
|
|
|
|
lastError?: string;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Manages certificate provisioning scheduling with:
|
|
|
|
|
* - Per-domain exponential backoff persisted in StorageManager
|
2026-02-16 00:22:23 +00:00
|
|
|
*
|
|
|
|
|
* Note: Serial stagger queue was removed — smartacme v9 handles
|
|
|
|
|
* concurrency, per-domain dedup, and rate limiting internally.
|
2026-02-15 16:03:13 +00:00
|
|
|
*/
|
|
|
|
|
export class CertProvisionScheduler {
|
|
|
|
|
private storageManager: StorageManager;
|
|
|
|
|
private maxBackoffHours: number;
|
|
|
|
|
|
|
|
|
|
// In-memory backoff cache (mirrors storage for fast lookups)
|
|
|
|
|
private backoffCache = new Map<string, IBackoffEntry>();
|
|
|
|
|
|
|
|
|
|
constructor(
|
|
|
|
|
storageManager: StorageManager,
|
2026-02-16 00:22:23 +00:00
|
|
|
options?: { maxBackoffHours?: number }
|
2026-02-15 16:03:13 +00:00
|
|
|
) {
|
|
|
|
|
this.storageManager = storageManager;
|
|
|
|
|
this.maxBackoffHours = options?.maxBackoffHours ?? 24;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Storage key for a domain's backoff entry
|
|
|
|
|
*/
|
|
|
|
|
private backoffKey(domain: string): string {
|
|
|
|
|
const clean = domain.replace(/\*/g, '_wildcard_').replace(/[^a-zA-Z0-9._-]/g, '_');
|
|
|
|
|
return `/cert-backoff/${clean}`;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Load backoff entry from storage (with in-memory cache)
|
|
|
|
|
*/
|
|
|
|
|
private async loadBackoff(domain: string): Promise<IBackoffEntry | null> {
|
|
|
|
|
const cached = this.backoffCache.get(domain);
|
|
|
|
|
if (cached) return cached;
|
|
|
|
|
|
|
|
|
|
const entry = await this.storageManager.getJSON<IBackoffEntry>(this.backoffKey(domain));
|
|
|
|
|
if (entry) {
|
|
|
|
|
this.backoffCache.set(domain, entry);
|
|
|
|
|
}
|
|
|
|
|
return entry;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Save backoff entry to both cache and storage
|
|
|
|
|
*/
|
|
|
|
|
private async saveBackoff(domain: string, entry: IBackoffEntry): Promise<void> {
|
|
|
|
|
this.backoffCache.set(domain, entry);
|
|
|
|
|
await this.storageManager.setJSON(this.backoffKey(domain), entry);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Check if a domain is currently in backoff
|
|
|
|
|
*/
|
|
|
|
|
async isInBackoff(domain: string): Promise<boolean> {
|
|
|
|
|
const entry = await this.loadBackoff(domain);
|
|
|
|
|
if (!entry) return false;
|
|
|
|
|
|
|
|
|
|
const retryAfter = new Date(entry.retryAfter);
|
|
|
|
|
return retryAfter.getTime() > Date.now();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Record a provisioning failure for a domain.
|
|
|
|
|
* Sets exponential backoff: min(failures^2 * 1h, maxBackoffHours)
|
|
|
|
|
*/
|
|
|
|
|
async recordFailure(domain: string, error?: string): Promise<void> {
|
|
|
|
|
const existing = await this.loadBackoff(domain);
|
|
|
|
|
const failures = (existing?.failures ?? 0) + 1;
|
|
|
|
|
|
|
|
|
|
// Exponential backoff: failures^2 hours, capped
|
|
|
|
|
const backoffHours = Math.min(failures * failures, this.maxBackoffHours);
|
|
|
|
|
const retryAfter = new Date(Date.now() + backoffHours * 60 * 60 * 1000);
|
|
|
|
|
|
|
|
|
|
const entry: IBackoffEntry = {
|
|
|
|
|
failures,
|
|
|
|
|
lastFailure: new Date().toISOString(),
|
|
|
|
|
retryAfter: retryAfter.toISOString(),
|
|
|
|
|
lastError: error,
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
await this.saveBackoff(domain, entry);
|
|
|
|
|
logger.log('warn', `Cert backoff for ${domain}: ${failures} failures, retry after ${retryAfter.toISOString()}`);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Clear backoff for a domain (on success or manual override)
|
|
|
|
|
*/
|
|
|
|
|
async clearBackoff(domain: string): Promise<void> {
|
|
|
|
|
this.backoffCache.delete(domain);
|
|
|
|
|
try {
|
|
|
|
|
await this.storageManager.delete(this.backoffKey(domain));
|
|
|
|
|
} catch {
|
|
|
|
|
// Ignore delete errors (key may not exist)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Get backoff info for UI display
|
|
|
|
|
*/
|
|
|
|
|
async getBackoffInfo(domain: string): Promise<{
|
|
|
|
|
failures: number;
|
|
|
|
|
retryAfter?: string;
|
|
|
|
|
lastError?: string;
|
|
|
|
|
} | null> {
|
|
|
|
|
const entry = await this.loadBackoff(domain);
|
|
|
|
|
if (!entry) return null;
|
|
|
|
|
|
|
|
|
|
// Only return if still in backoff
|
|
|
|
|
const retryAfter = new Date(entry.retryAfter);
|
|
|
|
|
if (retryAfter.getTime() <= Date.now()) return null;
|
|
|
|
|
|
|
|
|
|
return {
|
|
|
|
|
failures: entry.failures,
|
|
|
|
|
retryAfter: entry.retryAfter,
|
|
|
|
|
lastError: entry.lastError,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
}
|