serve.zone/dcrouter
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

BREAKING CHANGE(certs): Introduce domain-centric certificate provisioning with per-domain exponential backoff and a staggered serial scheduler; add domain-based reprovision API and UI backoff display; change certificate overview API to be domain-first and include backoff info; bump related deps.

Browse Source
This commit is contained in:
Jürgen Kunz
2026-02-15 16:03:13 +00:00
parent 2d44528345
commit 8e9de46cd2
11 changed files with 529 additions and 182 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ts/00_commitinfo_data.ts
View File

@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@serve.zone/dcrouter',
version: '5.5.0',
version: '6.0.0',
description: 'A multifaceted routing service handling mail and SMS delivery functions.'
}

176
ts/classes.cert-provision-scheduler.ts Normal file
View File

@@ -0,0 +1,176 @@
import { logger } from './logger.js';
import type { StorageManager } from './storage/index.js';
interface IBackoffEntry {
failures: number;
lastFailure: string; // ISO string
retryAfter: string; // ISO string
lastError?: string;
}
/**
* Manages certificate provisioning scheduling with:
* - Per-domain exponential backoff persisted in StorageManager
* - Serial stagger queue with configurable delay between provisions
*/
export class CertProvisionScheduler {
private storageManager: StorageManager;
private staggerDelayMs: number;
private maxBackoffHours: number;
// In-memory serial queue
private queue: Array<{
domain: string;
fn: () => Promise<any>;
resolve: (value: any) => void;
reject: (err: any) => void;
}> = [];
private processing = false;
// In-memory backoff cache (mirrors storage for fast lookups)
private backoffCache = new Map<string, IBackoffEntry>();
constructor(
storageManager: StorageManager,
options?: { staggerDelayMs?: number; maxBackoffHours?: number }
) {
this.storageManager = storageManager;
this.staggerDelayMs = options?.staggerDelayMs ?? 3000;
this.maxBackoffHours = options?.maxBackoffHours ?? 24;
}
/**
* Storage key for a domain's backoff entry
*/
private backoffKey(domain: string): string {
const clean = domain.replace(/\*/g, '_wildcard_').replace(/[^a-zA-Z0-9._-]/g, '_');
return `/cert-backoff/${clean}`;
}
/**
* Load backoff entry from storage (with in-memory cache)
*/
private async loadBackoff(domain: string): Promise<IBackoffEntry | null> {
const cached = this.backoffCache.get(domain);
if (cached) return cached;
const entry = await this.storageManager.getJSON<IBackoffEntry>(this.backoffKey(domain));
if (entry) {
this.backoffCache.set(domain, entry);
}
return entry;
}
/**
* Save backoff entry to both cache and storage
*/
private async saveBackoff(domain: string, entry: IBackoffEntry): Promise<void> {
this.backoffCache.set(domain, entry);
await this.storageManager.setJSON(this.backoffKey(domain), entry);
}
/**
* Check if a domain is currently in backoff
*/
async isInBackoff(domain: string): Promise<boolean> {
const entry = await this.loadBackoff(domain);
if (!entry) return false;
const retryAfter = new Date(entry.retryAfter);
return retryAfter.getTime() > Date.now();
}
/**
* Record a provisioning failure for a domain.
* Sets exponential backoff: min(failures^2 * 1h, maxBackoffHours)
*/
async recordFailure(domain: string, error?: string): Promise<void> {
const existing = await this.loadBackoff(domain);
const failures = (existing?.failures ?? 0) + 1;
// Exponential backoff: failures^2 hours, capped
const backoffHours = Math.min(failures * failures, this.maxBackoffHours);
const retryAfter = new Date(Date.now() + backoffHours * 60 * 60 * 1000);
const entry: IBackoffEntry = {
failures,
lastFailure: new Date().toISOString(),
retryAfter: retryAfter.toISOString(),
lastError: error,
};
await this.saveBackoff(domain, entry);
logger.log('warn', `Cert backoff for ${domain}: ${failures} failures, retry after ${retryAfter.toISOString()}`);
}
/**
* Clear backoff for a domain (on success or manual override)
*/
async clearBackoff(domain: string): Promise<void> {
this.backoffCache.delete(domain);
try {
await this.storageManager.delete(this.backoffKey(domain));
} catch {
// Ignore delete errors (key may not exist)
}
}
/**
* Get backoff info for UI display
*/
async getBackoffInfo(domain: string): Promise<{
failures: number;
retryAfter?: string;
lastError?: string;
} | null> {
const entry = await this.loadBackoff(domain);
if (!entry) return null;
// Only return if still in backoff
const retryAfter = new Date(entry.retryAfter);
if (retryAfter.getTime() <= Date.now()) return null;
return {
failures: entry.failures,
retryAfter: entry.retryAfter,
lastError: entry.lastError,
};
}
/**
* Enqueue a provision operation for serial execution with stagger delay.
* Returns the result of the provision function.
*/
enqueueProvision<T>(domain: string, fn: () => Promise<T>): Promise<T> {
return new Promise<T>((resolve, reject) => {
this.queue.push({ domain, fn, resolve, reject });
this.processQueue();
});
}
/**
* Process the stagger queue serially
*/
private async processQueue(): Promise<void> {
if (this.processing) return;
this.processing = true;
while (this.queue.length > 0) {
const item = this.queue.shift()!;
try {
logger.log('info', `Processing cert provision for ${item.domain}`);
const result = await item.fn();
item.resolve(result);
} catch (err) {
item.reject(err);
}
// Stagger delay between provisions
if (this.queue.length > 0) {
await new Promise<void>((r) => setTimeout(r, this.staggerDelayMs));
}
}
this.processing = false;
}
}

122
ts/classes.dcrouter.ts
View File

@@ -14,6 +14,7 @@ import { logger } from './logger.js';
// Import storage manager
import { StorageManager, type IStorageConfig } from './storage/index.js';
import { StorageBackedCertManager } from './classes.storage-cert-manager.js';
import { CertProvisionScheduler } from './classes.cert-provision-scheduler.js';
// Import cache system
import { CacheDb, CacheCleaner, type ICacheDbOptions } from './cache/index.js';
@@ -184,16 +185,19 @@ export class DcRouter {
public cacheDb?: CacheDb;
public cacheCleaner?: CacheCleaner;
// Certificate status tracking from SmartProxy events
// Certificate status tracking from SmartProxy events (keyed by domain)
public certificateStatusMap = new Map<string, {
status: 'valid' | 'failed';
domain: string;
routeNames: string[];
expiryDate?: string;
issuedAt?: string;
source?: string;
error?: string;
}>();
// Certificate provisioning scheduler with backoff + stagger
public certProvisionScheduler?: CertProvisionScheduler;
// TypedRouter for API endpoints
public typedrouter = new plugins.typedrequest.TypedRouter();
@@ -467,6 +471,9 @@ export class DcRouter {
},
};
// Initialize cert provision scheduler
this.certProvisionScheduler = new CertProvisionScheduler(this.storageManager);
// If we have DNS challenge handlers, create SmartAcme and wire to certProvisionFunction
if (challengeHandlers.length > 0) {
this.smartAcme = new plugins.smartacme.SmartAcme({
@@ -478,24 +485,41 @@ export class DcRouter {
});
await this.smartAcme.start();
const scheduler = this.certProvisionScheduler;
smartProxyConfig.certProvisionFunction = async (domain, eventComms) => {
// Check backoff before attempting provision
if (await scheduler.isInBackoff(domain)) {
const info = await scheduler.getBackoffInfo(domain);
const msg = `Domain ${domain} is in backoff (${info?.failures} failures), retry after ${info?.retryAfter}`;
eventComms.warn(msg);
throw new Error(msg);
}
try {
eventComms.log(`Attempting DNS-01 via SmartAcme for ${domain}`);
eventComms.setSource('smartacme-dns-01');
const cert = await this.smartAcme.getCertificateForDomain(domain);
if (cert.validUntil) {
eventComms.setExpiryDate(new Date(cert.validUntil));
}
return {
id: cert.id,
domainName: cert.domainName,
created: cert.created,
validUntil: cert.validUntil,
privateKey: cert.privateKey,
publicKey: cert.publicKey,
csr: cert.csr,
};
const result = await scheduler.enqueueProvision(domain, async () => {
eventComms.log(`Attempting DNS-01 via SmartAcme for ${domain}`);
eventComms.setSource('smartacme-dns-01');
const cert = await this.smartAcme.getCertificateForDomain(domain);
if (cert.validUntil) {
eventComms.setExpiryDate(new Date(cert.validUntil));
}
return {
id: cert.id,
domainName: cert.domainName,
created: cert.created,
validUntil: cert.validUntil,
privateKey: cert.privateKey,
publicKey: cert.publicKey,
csr: cert.csr,
};
});
// Success — clear any backoff
await scheduler.clearBackoff(domain);
return result;
} catch (err) {
// Record failure for backoff tracking
await scheduler.recordFailure(domain, err.message);
eventComms.warn(`SmartAcme DNS-01 failed for ${domain}: ${err.message}, falling back to http-01`);
return 'http01';
}
@@ -519,39 +543,34 @@ export class DcRouter {
});
// Always listen for certificate events — emitted by both ACME and certProvisionFunction paths
// Events are keyed by domain for domain-centric certificate tracking
this.smartProxy.on('certificate-issued', (event: plugins.smartproxy.ICertificateIssuedEvent) => {
console.log(`[DcRouter] Certificate issued for ${event.domain} via ${event.source}, expires ${event.expiryDate}`);
const routeName = this.findRouteNameForDomain(event.domain);
if (routeName) {
this.certificateStatusMap.set(routeName, {
status: 'valid', domain: event.domain,
expiryDate: event.expiryDate, issuedAt: new Date().toISOString(),
source: event.source,
});
}
const routeNames = this.findRouteNamesForDomain(event.domain);
this.certificateStatusMap.set(event.domain, {
status: 'valid', routeNames,
expiryDate: event.expiryDate, issuedAt: new Date().toISOString(),
source: event.source,
});
});
this.smartProxy.on('certificate-renewed', (event: plugins.smartproxy.ICertificateIssuedEvent) => {
console.log(`[DcRouter] Certificate renewed for ${event.domain} via ${event.source}, expires ${event.expiryDate}`);
const routeName = this.findRouteNameForDomain(event.domain);
if (routeName) {
this.certificateStatusMap.set(routeName, {
status: 'valid', domain: event.domain,
expiryDate: event.expiryDate, issuedAt: new Date().toISOString(),
source: event.source,
});
}
const routeNames = this.findRouteNamesForDomain(event.domain);
this.certificateStatusMap.set(event.domain, {
status: 'valid', routeNames,
expiryDate: event.expiryDate, issuedAt: new Date().toISOString(),
source: event.source,
});
});
this.smartProxy.on('certificate-failed', (event: plugins.smartproxy.ICertificateFailedEvent) => {
console.error(`[DcRouter] Certificate failed for ${event.domain} (${event.source}):`, event.error);
const routeName = this.findRouteNameForDomain(event.domain);
if (routeName) {
this.certificateStatusMap.set(routeName, {
status: 'failed', domain: event.domain, error: event.error,
source: event.source,
});
}
const routeNames = this.findRouteNamesForDomain(event.domain);
this.certificateStatusMap.set(event.domain, {
status: 'failed', routeNames, error: event.error,
source: event.source,
});
});
// Start SmartProxy
@@ -724,7 +743,7 @@ export class DcRouter {
}
/**
* Find the route name that matches a given domain
* Find the first route name that matches a given domain
*/
private findRouteNameForDomain(domain: string): string | undefined {
if (!this.smartProxy) return undefined;
@@ -740,6 +759,27 @@ export class DcRouter {
return undefined;
}
/**
* Find ALL route names that match a given domain
*/
public findRouteNamesForDomain(domain: string): string[] {
if (!this.smartProxy) return [];
const names: string[] = [];
for (const route of this.smartProxy.routeManager.getRoutes()) {
if (!route.match.domains || !route.name) continue;
const routeDomains = Array.isArray(route.match.domains)
? route.match.domains
: [route.match.domains];
for (const pattern of routeDomains) {
if (this.isDomainMatch(domain, pattern)) {
names.push(route.name);
break; // This route already matched, no need to check other patterns
}
}
}
return names;
}
public async stop() {
console.log('Stopping DcRouter services...');

170
ts/opsserver/handlers/certificate.handler.ts
View File

@@ -23,24 +23,45 @@ export class CertificateHandler {
)
);
// Reprovision Certificate
// Legacy route-based reprovision (backward compat)
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_ReprovisionCertificate>(
'reprovisionCertificate',
async (dataArg) => {
return this.reprovisionCertificate(dataArg.routeName);
return this.reprovisionCertificateByRoute(dataArg.routeName);
}
)
);
// Domain-based reprovision (preferred)
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_ReprovisionCertificateDomain>(
'reprovisionCertificateDomain',
async (dataArg) => {
return this.reprovisionCertificateDomain(dataArg.domain);
}
)
);
}
/**
* Build domain-centric certificate overview.
* Instead of one row per route, we produce one row per unique domain.
*/
private async buildCertificateOverview(): Promise<interfaces.requests.ICertificateInfo[]> {
const dcRouter = this.opsServerRef.dcRouterRef;
const smartProxy = dcRouter.smartProxy;
if (!smartProxy) return [];
const routes = smartProxy.routeManager.getRoutes();
const certificates: interfaces.requests.ICertificateInfo[] = [];
// Phase 1: Collect unique domains with their associated route info
const domainMap = new Map<string, {
routeNames: string[];
source: interfaces.requests.TCertificateSource;
tlsMode: 'terminate' | 'terminate-and-reencrypt' | 'passthrough';
canReprovision: boolean;
}>();
for (const route of routes) {
if (!route.name) continue;
@@ -58,7 +79,6 @@ export class CertificateHandler {
// Determine source
let source: interfaces.requests.TCertificateSource = 'none';
if (tls.certificate === 'auto') {
// Check if a certProvisionFunction is configured
if ((smartProxy.settings as any).certProvisionFunction) {
source = 'provision-function';
} else {
@@ -68,15 +88,44 @@ export class CertificateHandler {
source = 'static';
}
// Start with unknown status
const canReprovision = source === 'acme' || source === 'provision-function';
const tlsMode = tls.mode as 'terminate' | 'terminate-and-reencrypt' | 'passthrough';
for (const domain of routeDomains) {
const existing = domainMap.get(domain);
if (existing) {
// Add this route name to the existing domain entry
if (!existing.routeNames.includes(route.name)) {
existing.routeNames.push(route.name);
}
// Upgrade source if more specific
if (existing.source === 'none' && source !== 'none') {
existing.source = source;
existing.canReprovision = canReprovision;
}
} else {
domainMap.set(domain, {
routeNames: [route.name],
source,
tlsMode,
canReprovision,
});
}
}
}
// Phase 2: Resolve status for each unique domain
const certificates: interfaces.requests.ICertificateInfo[] = [];
for (const [domain, info] of domainMap) {
let status: interfaces.requests.TCertificateStatus = 'unknown';
let expiryDate: string | undefined;
let issuedAt: string | undefined;
let issuer: string | undefined;
let error: string | undefined;
// Check event-based status from DcRouter's certificateStatusMap
const eventStatus = dcRouter.certificateStatusMap.get(route.name);
// Check event-based status from certificateStatusMap (now keyed by domain)
const eventStatus = dcRouter.certificateStatusMap.get(domain);
if (eventStatus) {
status = eventStatus.status;
expiryDate = eventStatus.expiryDate;
@@ -87,10 +136,10 @@ export class CertificateHandler {
}
}
// Try Rust-side certificate status if no event data
if (status === 'unknown') {
// Try SmartProxy certificate status if no event data
if (status === 'unknown' && info.routeNames.length > 0) {
try {
const rustStatus = await smartProxy.getCertificateStatus(route.name);
const rustStatus = await smartProxy.getCertificateStatus(info.routeNames[0]);
if (rustStatus) {
if (rustStatus.expiryDate) expiryDate = rustStatus.expiryDate;
if (rustStatus.issuer) issuer = rustStatus.issuer;
@@ -105,22 +154,19 @@ export class CertificateHandler {
}
// Check persisted cert data from StorageManager
if (status === 'unknown' && routeDomains.length > 0) {
for (const domain of routeDomains) {
if (expiryDate) break;
const cleanDomain = domain.replace(/^\*\.?/, '');
const certData = await dcRouter.storageManager.getJSON(`/certs/${cleanDomain}`);
if (certData?.validUntil) {
expiryDate = new Date(certData.validUntil).toISOString();
if (certData.created) {
issuedAt = new Date(certData.created).toISOString();
}
issuer = 'smartacme-dns-01';
if (status === 'unknown') {
const cleanDomain = domain.replace(/^\*\.?/, '');
const certData = await dcRouter.storageManager.getJSON(`/certs/${cleanDomain}`);
if (certData?.validUntil) {
expiryDate = new Date(certData.validUntil).toISOString();
if (certData.created) {
issuedAt = new Date(certData.created).toISOString();
}
issuer = 'smartacme-dns-01';
}
}
// Compute status from expiry date if we have one and status is still valid/unknown
// Compute status from expiry date
if (expiryDate && (status === 'valid' || status === 'unknown')) {
const expiry = new Date(expiryDate);
const now = new Date();
@@ -136,28 +182,36 @@ export class CertificateHandler {
}
// Static certs with no other info default to 'valid'
if (source === 'static' && status === 'unknown') {
if (info.source === 'static' && status === 'unknown') {
status = 'valid';
}
// ACME/provision-function routes with no cert data are still provisioning
if (status === 'unknown' && (source === 'acme' || source === 'provision-function')) {
if (status === 'unknown' && (info.source === 'acme' || info.source === 'provision-function')) {
status = 'provisioning';
}
const canReprovision = source === 'acme' || source === 'provision-function';
// Phase 3: Attach backoff info
let backoffInfo: interfaces.requests.ICertificateInfo['backoffInfo'];
if (dcRouter.certProvisionScheduler) {
const bi = await dcRouter.certProvisionScheduler.getBackoffInfo(domain);
if (bi) {
backoffInfo = bi;
}
}
certificates.push({
routeName: route.name,
domains: routeDomains,
domain,
routeNames: info.routeNames,
status,
source,
tlsMode: tls.mode as 'terminate' | 'terminate-and-reencrypt' | 'passthrough',
source: info.source,
tlsMode: info.tlsMode,
expiryDate,
issuer,
issuedAt,
error,
canReprovision,
canReprovision: info.canReprovision,
backoffInfo,
});
}
@@ -187,7 +241,10 @@ export class CertificateHandler {
return summary;
}
private async reprovisionCertificate(routeName: string): Promise<{ success: boolean; message?: string }> {
/**
* Legacy route-based reprovisioning
*/
private async reprovisionCertificateByRoute(routeName: string): Promise<{ success: boolean; message?: string }> {
const dcRouter = this.opsServerRef.dcRouterRef;
const smartProxy = dcRouter.smartProxy;
@@ -197,11 +254,58 @@ export class CertificateHandler {
try {
await smartProxy.provisionCertificate(routeName);
// Clear event-based status so it gets refreshed
dcRouter.certificateStatusMap.delete(routeName);
// Clear event-based status for domains in this route
for (const [domain, entry] of dcRouter.certificateStatusMap) {
if (entry.routeNames.includes(routeName)) {
dcRouter.certificateStatusMap.delete(domain);
}
}
return { success: true, message: `Certificate reprovisioning triggered for route '${routeName}'` };
} catch (err) {
return { success: false, message: err.message || 'Failed to reprovision certificate' };
}
}
/**
* Domain-based reprovisioning — clears backoff first, then triggers provision
*/
private async reprovisionCertificateDomain(domain: string): Promise<{ success: boolean; message?: string }> {
const dcRouter = this.opsServerRef.dcRouterRef;
const smartProxy = dcRouter.smartProxy;
if (!smartProxy) {
return { success: false, message: 'SmartProxy is not running' };
}
// Clear backoff for this domain (user override)
if (dcRouter.certProvisionScheduler) {
await dcRouter.certProvisionScheduler.clearBackoff(domain);
}
// Clear status map entry so it gets refreshed
dcRouter.certificateStatusMap.delete(domain);
// Try to provision via SmartAcme directly
if (dcRouter.smartAcme) {
try {
await dcRouter.smartAcme.getCertificateForDomain(domain);
return { success: true, message: `Certificate reprovisioning triggered for domain '${domain}'` };
} catch (err) {
return { success: false, message: err.message || `Failed to reprovision certificate for ${domain}` };
}
}
// Fallback: try provisioning via the first matching route
const routeNames = dcRouter.findRouteNamesForDomain(domain);
if (routeNames.length > 0) {
try {
await smartProxy.provisionCertificate(routeNames[0]);
return { success: true, message: `Certificate reprovisioning triggered for domain '${domain}' via route '${routeNames[0]}'` };
} catch (err) {
return { success: false, message: err.message || `Failed to reprovision certificate for ${domain}` };
}
}
return { success: false, message: `No routes found for domain '${domain}'` };
}
}