feat(smart-proxy): add background concurrent certificate provisioning with per-domain timeouts and concurrency control
This commit is contained in:
@@ -12,6 +12,7 @@ import { SharedRouteManager as RouteManager } from '../../core/routing/route-man
|
||||
import { RouteValidator } from './utils/route-validator.js';
|
||||
import { generateDefaultCertificate } from './utils/default-cert-generator.js';
|
||||
import { Mutex } from './utils/mutex.js';
|
||||
import { ConcurrencySemaphore } from './utils/concurrency-semaphore.js';
|
||||
|
||||
// Types
|
||||
import type { ISmartProxyOptions, TSmartProxyCertProvisionObject, IAcmeOptions, ICertProvisionEventComms, ICertificateIssuedEvent, ICertificateFailedEvent } from './models/interfaces.js';
|
||||
@@ -38,6 +39,7 @@ export class SmartProxy extends plugins.EventEmitter {
|
||||
private metricsAdapter: RustMetricsAdapter;
|
||||
private routeUpdateLock: Mutex;
|
||||
private stopping = false;
|
||||
private certProvisionPromise: Promise<void> | null = null;
|
||||
|
||||
constructor(settingsArg: ISmartProxyOptions) {
|
||||
super();
|
||||
@@ -199,8 +201,10 @@ export class SmartProxy extends plugins.EventEmitter {
|
||||
|
||||
logger.log('info', 'SmartProxy started (Rust engine)', { component: 'smart-proxy' });
|
||||
|
||||
// Handle certProvisionFunction (may be slow — runs after startup is complete)
|
||||
await this.provisionCertificatesViaCallback(preloadedDomains);
|
||||
// Fire-and-forget cert provisioning — Rust engine is already running and serving traffic.
|
||||
// Events (certificate-issued / certificate-failed) fire independently per domain.
|
||||
this.certProvisionPromise = this.provisionCertificatesViaCallback(preloadedDomains)
|
||||
.catch((err) => logger.log('error', `Unexpected error in cert provisioning: ${err.message}`, { component: 'smart-proxy' }));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -210,6 +214,12 @@ export class SmartProxy extends plugins.EventEmitter {
|
||||
logger.log('info', 'SmartProxy shutting down...', { component: 'smart-proxy' });
|
||||
this.stopping = true;
|
||||
|
||||
// Wait for in-flight cert provisioning to bail out (it checks this.stopping)
|
||||
if (this.certProvisionPromise) {
|
||||
await this.certProvisionPromise;
|
||||
this.certProvisionPromise = null;
|
||||
}
|
||||
|
||||
// Stop metrics polling
|
||||
this.metricsAdapter.stopPolling();
|
||||
|
||||
@@ -237,7 +247,7 @@ export class SmartProxy extends plugins.EventEmitter {
|
||||
* Update routes atomically.
|
||||
*/
|
||||
public async updateRoutes(newRoutes: IRouteConfig[]): Promise<void> {
|
||||
return this.routeUpdateLock.runExclusive(async () => {
|
||||
await this.routeUpdateLock.runExclusive(async () => {
|
||||
// Validate
|
||||
const validation = RouteValidator.validateRoutes(newRoutes);
|
||||
if (!validation.valid) {
|
||||
@@ -273,11 +283,13 @@ export class SmartProxy extends plugins.EventEmitter {
|
||||
// Update stored routes
|
||||
this.settings.routes = newRoutes;
|
||||
|
||||
// Handle cert provisioning for new routes
|
||||
await this.provisionCertificatesViaCallback();
|
||||
|
||||
logger.log('info', `Routes updated (${newRoutes.length} routes)`, { component: 'smart-proxy' });
|
||||
});
|
||||
|
||||
// Fire-and-forget cert provisioning outside the mutex — routes are already updated,
|
||||
// cert provisioning doesn't need the route update lock and may be slow.
|
||||
this.certProvisionPromise = this.provisionCertificatesViaCallback()
|
||||
.catch((err) => logger.log('error', `Unexpected error in cert provisioning after route update: ${err.message}`, { component: 'smart-proxy' }));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -412,7 +424,9 @@ export class SmartProxy extends plugins.EventEmitter {
|
||||
const provisionFn = this.settings.certProvisionFunction;
|
||||
if (!provisionFn) return;
|
||||
|
||||
const provisionedDomains = new Set<string>(skipDomains);
|
||||
// Phase 1: Collect all unique (domain, route) pairs that need provisioning
|
||||
const seen = new Set<string>(skipDomains);
|
||||
const tasks: Array<{ domain: string; route: IRouteConfig }> = [];
|
||||
|
||||
for (const route of this.settings.routes) {
|
||||
if (route.action.tls?.certificate !== 'auto') continue;
|
||||
@@ -422,91 +436,139 @@ export class SmartProxy extends plugins.EventEmitter {
|
||||
const certDomains = this.normalizeDomainsForCertProvisioning(rawDomains);
|
||||
|
||||
for (const domain of certDomains) {
|
||||
if (provisionedDomains.has(domain)) continue;
|
||||
provisionedDomains.add(domain);
|
||||
if (seen.has(domain)) continue;
|
||||
seen.add(domain);
|
||||
tasks.push({ domain, route });
|
||||
}
|
||||
}
|
||||
|
||||
// Build eventComms channel for this domain
|
||||
let expiryDate: string | undefined;
|
||||
let source = 'certProvisionFunction';
|
||||
if (tasks.length === 0) return;
|
||||
|
||||
const eventComms: ICertProvisionEventComms = {
|
||||
log: (msg) => logger.log('info', `[certProvision ${domain}] ${msg}`, { component: 'smart-proxy' }),
|
||||
warn: (msg) => logger.log('warn', `[certProvision ${domain}] ${msg}`, { component: 'smart-proxy' }),
|
||||
error: (msg) => logger.log('error', `[certProvision ${domain}] ${msg}`, { component: 'smart-proxy' }),
|
||||
setExpiryDate: (date) => { expiryDate = date.toISOString(); },
|
||||
setSource: (s) => { source = s; },
|
||||
};
|
||||
// Phase 2: Process all domains in parallel with concurrency limit
|
||||
const concurrency = this.settings.certProvisionConcurrency ?? 4;
|
||||
const semaphore = new ConcurrencySemaphore(concurrency);
|
||||
|
||||
const promises = tasks.map(async ({ domain, route }) => {
|
||||
await semaphore.acquire();
|
||||
try {
|
||||
await this.provisionSingleDomain(domain, route, provisionFn);
|
||||
} finally {
|
||||
semaphore.release();
|
||||
}
|
||||
});
|
||||
|
||||
await Promise.allSettled(promises);
|
||||
}
|
||||
|
||||
/**
|
||||
* Provision a single domain's certificate via the callback.
|
||||
* Includes per-domain timeout and shutdown checks.
|
||||
*/
|
||||
private async provisionSingleDomain(
|
||||
domain: string,
|
||||
route: IRouteConfig,
|
||||
provisionFn: (domain: string, eventComms: ICertProvisionEventComms) => Promise<TSmartProxyCertProvisionObject>,
|
||||
): Promise<void> {
|
||||
if (this.stopping) return;
|
||||
|
||||
let expiryDate: string | undefined;
|
||||
let source = 'certProvisionFunction';
|
||||
|
||||
const eventComms: ICertProvisionEventComms = {
|
||||
log: (msg) => logger.log('info', `[certProvision ${domain}] ${msg}`, { component: 'smart-proxy' }),
|
||||
warn: (msg) => logger.log('warn', `[certProvision ${domain}] ${msg}`, { component: 'smart-proxy' }),
|
||||
error: (msg) => logger.log('error', `[certProvision ${domain}] ${msg}`, { component: 'smart-proxy' }),
|
||||
setExpiryDate: (date) => { expiryDate = date.toISOString(); },
|
||||
setSource: (s) => { source = s; },
|
||||
};
|
||||
|
||||
const timeoutMs = this.settings.certProvisionTimeout ?? 300_000; // 5 min default
|
||||
|
||||
try {
|
||||
const result: TSmartProxyCertProvisionObject = await this.withTimeout(
|
||||
provisionFn(domain, eventComms),
|
||||
timeoutMs,
|
||||
`Certificate provisioning timed out for ${domain} after ${timeoutMs}ms`,
|
||||
);
|
||||
|
||||
if (this.stopping) return;
|
||||
|
||||
if (result === 'http01') {
|
||||
if (route.name) {
|
||||
try {
|
||||
await this.bridge.provisionCertificate(route.name);
|
||||
logger.log('info', `Triggered Rust ACME for ${domain} (route: ${route.name})`, { component: 'smart-proxy' });
|
||||
} catch (provisionErr: any) {
|
||||
logger.log('warn', `Cannot provision cert for ${domain} — callback returned 'http01' but Rust ACME failed: ${provisionErr.message}. ` +
|
||||
'Note: Rust ACME is disabled when certProvisionFunction is set.', { component: 'smart-proxy' });
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (result && typeof result === 'object') {
|
||||
if (this.stopping) return;
|
||||
|
||||
const certObj = result as plugins.tsclass.network.ICert;
|
||||
await this.bridge.loadCertificate(
|
||||
domain,
|
||||
certObj.publicKey,
|
||||
certObj.privateKey,
|
||||
);
|
||||
logger.log('info', `Certificate loaded via provision function for ${domain}`, { component: 'smart-proxy' });
|
||||
|
||||
// Persist to consumer store
|
||||
if (this.settings.certStore?.save) {
|
||||
try {
|
||||
await this.settings.certStore.save(domain, certObj.publicKey, certObj.privateKey);
|
||||
} catch (storeErr: any) {
|
||||
logger.log('warn', `certStore.save() failed for ${domain}: ${storeErr.message}`, { component: 'smart-proxy' });
|
||||
}
|
||||
}
|
||||
|
||||
this.emit('certificate-issued', {
|
||||
domain,
|
||||
expiryDate: expiryDate || (certObj.validUntil ? new Date(certObj.validUntil).toISOString() : undefined),
|
||||
source,
|
||||
} satisfies ICertificateIssuedEvent);
|
||||
}
|
||||
} catch (err: any) {
|
||||
logger.log('warn', `certProvisionFunction failed for ${domain}: ${err.message}`, { component: 'smart-proxy' });
|
||||
|
||||
this.emit('certificate-failed', {
|
||||
domain,
|
||||
error: err.message,
|
||||
source,
|
||||
} satisfies ICertificateFailedEvent);
|
||||
|
||||
// Fallback to ACME if enabled and route has a name
|
||||
if (this.settings.certProvisionFallbackToAcme !== false && route.name) {
|
||||
try {
|
||||
const result: TSmartProxyCertProvisionObject = await provisionFn(domain, eventComms);
|
||||
|
||||
if (result === 'http01') {
|
||||
// Callback wants HTTP-01 for this domain — trigger Rust ACME explicitly
|
||||
if (route.name) {
|
||||
try {
|
||||
await this.bridge.provisionCertificate(route.name);
|
||||
logger.log('info', `Triggered Rust ACME for ${domain} (route: ${route.name})`, { component: 'smart-proxy' });
|
||||
} catch (provisionErr: any) {
|
||||
logger.log('warn', `Cannot provision cert for ${domain} — callback returned 'http01' but Rust ACME failed: ${provisionErr.message}. ` +
|
||||
'Note: Rust ACME is disabled when certProvisionFunction is set.', { component: 'smart-proxy' });
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Got a static cert object - load it into Rust
|
||||
if (result && typeof result === 'object') {
|
||||
const certObj = result as plugins.tsclass.network.ICert;
|
||||
await this.bridge.loadCertificate(
|
||||
domain,
|
||||
certObj.publicKey,
|
||||
certObj.privateKey,
|
||||
);
|
||||
logger.log('info', `Certificate loaded via provision function for ${domain}`, { component: 'smart-proxy' });
|
||||
|
||||
// Persist to consumer store
|
||||
if (this.settings.certStore?.save) {
|
||||
try {
|
||||
await this.settings.certStore.save(domain, certObj.publicKey, certObj.privateKey);
|
||||
} catch (storeErr: any) {
|
||||
logger.log('warn', `certStore.save() failed for ${domain}: ${storeErr.message}`, { component: 'smart-proxy' });
|
||||
}
|
||||
}
|
||||
|
||||
// Emit certificate-issued event
|
||||
this.emit('certificate-issued', {
|
||||
domain,
|
||||
expiryDate: expiryDate || (certObj.validUntil ? new Date(certObj.validUntil).toISOString() : undefined),
|
||||
source,
|
||||
} satisfies ICertificateIssuedEvent);
|
||||
}
|
||||
} catch (err: any) {
|
||||
logger.log('warn', `certProvisionFunction failed for ${domain}: ${err.message}`, { component: 'smart-proxy' });
|
||||
|
||||
// Emit certificate-failed event
|
||||
this.emit('certificate-failed', {
|
||||
domain,
|
||||
error: err.message,
|
||||
source,
|
||||
} satisfies ICertificateFailedEvent);
|
||||
|
||||
// Fallback to ACME if enabled and route has a name
|
||||
if (this.settings.certProvisionFallbackToAcme !== false && route.name) {
|
||||
try {
|
||||
await this.bridge.provisionCertificate(route.name);
|
||||
logger.log('info', `Falling back to Rust ACME for ${domain} (route: ${route.name})`, { component: 'smart-proxy' });
|
||||
} catch (acmeErr: any) {
|
||||
logger.log('warn', `ACME fallback also failed for ${domain}: ${acmeErr.message}` +
|
||||
(this.settings.disableDefaultCert
|
||||
? ' — TLS will fail for this domain (disableDefaultCert is true)'
|
||||
: ' — default self-signed fallback cert will be used'), { component: 'smart-proxy' });
|
||||
}
|
||||
}
|
||||
await this.bridge.provisionCertificate(route.name);
|
||||
logger.log('info', `Falling back to Rust ACME for ${domain} (route: ${route.name})`, { component: 'smart-proxy' });
|
||||
} catch (acmeErr: any) {
|
||||
logger.log('warn', `ACME fallback also failed for ${domain}: ${acmeErr.message}` +
|
||||
(this.settings.disableDefaultCert
|
||||
? ' — TLS will fail for this domain (disableDefaultCert is true)'
|
||||
: ' — default self-signed fallback cert will be used'), { component: 'smart-proxy' });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Race a promise against a timeout. Rejects with the given message if the timeout fires first.
|
||||
*/
|
||||
private withTimeout<T>(promise: Promise<T>, ms: number, message: string): Promise<T> {
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
const timer = setTimeout(() => reject(new Error(message)), ms);
|
||||
promise.then(
|
||||
(val) => { clearTimeout(timer); resolve(val); },
|
||||
(err) => { clearTimeout(timer); reject(err); },
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize routing glob patterns into valid domain identifiers for cert provisioning.
|
||||
* - `*nevermind.cloud` → `['nevermind.cloud', '*.nevermind.cloud']`
|
||||
|
||||
Reference in New Issue
Block a user