BREAKING CHANGE(db): replace StorageManager and CacheDb with a unified smartdata-backed database layer
This commit is contained in:
@@ -11,12 +11,10 @@ import {
|
||||
type IEmailDomainConfig,
|
||||
} from '@push.rocks/smartmta';
|
||||
import { logger } from './logger.js';
|
||||
// Import storage manager
|
||||
import { StorageManager, type IStorageConfig } from './storage/index.js';
|
||||
import { StorageBackedCertManager } from './classes.storage-cert-manager.js';
|
||||
import { CertProvisionScheduler } from './classes.cert-provision-scheduler.js';
|
||||
// Import cache system
|
||||
import { CacheDb, CacheCleaner, type ICacheDbOptions } from './cache/index.js';
|
||||
// Import unified database
|
||||
import { DcRouterDb, type IDcRouterDbConfig, CacheCleaner, ProxyCertDoc, AcmeCertDoc } from './db/index.js';
|
||||
|
||||
import { OpsServer } from './opsserver/index.js';
|
||||
import { MetricsManager } from './monitoring/index.js';
|
||||
@@ -122,37 +120,23 @@ export interface IDcRouterOptions {
|
||||
/** Other DNS providers can be added here */
|
||||
};
|
||||
|
||||
/** Storage configuration */
|
||||
storage?: IStorageConfig;
|
||||
|
||||
/**
|
||||
* Cache database configuration using smartdata and LocalTsmDb
|
||||
* Provides persistent caching for emails, IP reputation, bounces, etc.
|
||||
* Unified database configuration.
|
||||
* All persistent data (config, certs, VPN, cache, etc.) is stored via smartdata.
|
||||
* If mongoDbUrl is provided, connects to external MongoDB.
|
||||
* Otherwise, starts an embedded LocalSmartDb automatically.
|
||||
*/
|
||||
cacheConfig?: {
|
||||
/** Enable cache database (default: true) */
|
||||
dbConfig?: {
|
||||
/** Enable database (default: true). Set to false in tests to skip DB startup. */
|
||||
enabled?: boolean;
|
||||
/** Storage path for TsmDB data (default: ~/.serve.zone/dcrouter/tsmdb) */
|
||||
/** External MongoDB connection URL. If absent, uses embedded LocalSmartDb. */
|
||||
mongoDbUrl?: string;
|
||||
/** Storage path for embedded database data (default: ~/.serve.zone/dcrouter/tsmdb) */
|
||||
storagePath?: string;
|
||||
/** Database name (default: dcrouter) */
|
||||
dbName?: string;
|
||||
/** Default TTL in days for cached items (default: 30) */
|
||||
defaultTTLDays?: number;
|
||||
/** Cleanup interval in hours (default: 1) */
|
||||
/** Cache cleanup interval in hours (default: 1) */
|
||||
cleanupIntervalHours?: number;
|
||||
/** TTL configuration per data type (in days) */
|
||||
ttlConfig?: {
|
||||
/** Email cache TTL (default: 30 days) */
|
||||
emails?: number;
|
||||
/** IP reputation cache TTL (default: 1 day) */
|
||||
ipReputation?: number;
|
||||
/** Bounce records TTL (default: 30 days) */
|
||||
bounces?: number;
|
||||
/** DKIM keys TTL (default: 90 days) */
|
||||
dkimKeys?: number;
|
||||
/** Suppression list TTL (default: 30 days, can be permanent) */
|
||||
suppression?: number;
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -248,12 +232,20 @@ export class DcRouter {
|
||||
public dnsServer?: plugins.smartdns.dnsServerMod.DnsServer;
|
||||
public emailServer?: UnifiedEmailServer;
|
||||
public radiusServer?: RadiusServer;
|
||||
public storageManager: StorageManager;
|
||||
public opsServer!: OpsServer;
|
||||
public metricsManager?: MetricsManager;
|
||||
|
||||
// Cache system (smartdata + LocalTsmDb)
|
||||
public cacheDb?: CacheDb;
|
||||
// Compatibility shim for smartmta's DkimManager which calls dcRouter.storageManager.set()
|
||||
public storageManager: any = {
|
||||
get: async (_key: string) => null,
|
||||
set: async (_key: string, _value: string) => {
|
||||
// DKIM keys from smartmta — logged but not yet migrated to smartdata
|
||||
logger.log('debug', `storageManager.set() called (compat shim) for key: ${_key}`);
|
||||
},
|
||||
};
|
||||
|
||||
// Unified database (smartdata + LocalSmartDb or external MongoDB)
|
||||
public dcRouterDb?: DcRouterDb;
|
||||
public cacheCleaner?: CacheCleaner;
|
||||
|
||||
// Remote Ingress
|
||||
@@ -312,16 +304,6 @@ export class DcRouter {
|
||||
// Resolve all data paths from baseDir
|
||||
this.resolvedPaths = paths.resolvePaths(this.options.baseDir);
|
||||
|
||||
// Default storage to filesystem if not configured
|
||||
if (!this.options.storage) {
|
||||
this.options.storage = {
|
||||
fsPath: this.resolvedPaths.defaultStoragePath,
|
||||
};
|
||||
}
|
||||
|
||||
// Initialize storage manager
|
||||
this.storageManager = new StorageManager(this.options.storage);
|
||||
|
||||
// Initialize service manager and register all services
|
||||
this.serviceManager = new plugins.taskbuffer.ServiceManager({
|
||||
name: 'dcrouter',
|
||||
@@ -350,23 +332,23 @@ export class DcRouter {
|
||||
.withRetry({ maxRetries: 0 }),
|
||||
);
|
||||
|
||||
// CacheDb: optional, no dependencies
|
||||
if (this.options.cacheConfig?.enabled !== false) {
|
||||
// DcRouterDb: optional, no dependencies — unified database for all persistence
|
||||
if (this.options.dbConfig?.enabled !== false) {
|
||||
this.serviceManager.addService(
|
||||
new plugins.taskbuffer.Service('CacheDb')
|
||||
new plugins.taskbuffer.Service('DcRouterDb')
|
||||
.optional()
|
||||
.withStart(async () => {
|
||||
await this.setupCacheDb();
|
||||
await this.setupDcRouterDb();
|
||||
})
|
||||
.withStop(async () => {
|
||||
if (this.cacheCleaner) {
|
||||
this.cacheCleaner.stop();
|
||||
this.cacheCleaner = undefined;
|
||||
}
|
||||
if (this.cacheDb) {
|
||||
await this.cacheDb.stop();
|
||||
CacheDb.resetInstance();
|
||||
this.cacheDb = undefined;
|
||||
if (this.dcRouterDb) {
|
||||
await this.dcRouterDb.stop();
|
||||
DcRouterDb.resetInstance();
|
||||
this.dcRouterDb = undefined;
|
||||
}
|
||||
})
|
||||
.withRetry({ maxRetries: 2, baseDelayMs: 1000, maxDelayMs: 5000 }),
|
||||
@@ -391,10 +373,10 @@ export class DcRouter {
|
||||
.withRetry({ maxRetries: 1, baseDelayMs: 1000 }),
|
||||
);
|
||||
|
||||
// SmartProxy: critical, depends on CacheDb (if enabled)
|
||||
// SmartProxy: critical, depends on DcRouterDb (if enabled)
|
||||
const smartProxyDeps: string[] = [];
|
||||
if (this.options.cacheConfig?.enabled !== false) {
|
||||
smartProxyDeps.push('CacheDb');
|
||||
if (this.options.dbConfig?.enabled !== false) {
|
||||
smartProxyDeps.push('DcRouterDb');
|
||||
}
|
||||
this.serviceManager.addService(
|
||||
new plugins.taskbuffer.Service('SmartProxy')
|
||||
@@ -455,36 +437,38 @@ export class DcRouter {
|
||||
);
|
||||
}
|
||||
|
||||
// ConfigManagers: optional, depends on SmartProxy
|
||||
this.serviceManager.addService(
|
||||
new plugins.taskbuffer.Service('ConfigManagers')
|
||||
.optional()
|
||||
.dependsOn('SmartProxy')
|
||||
.withStart(async () => {
|
||||
this.routeConfigManager = new RouteConfigManager(
|
||||
this.storageManager,
|
||||
() => this.getConstructorRoutes(),
|
||||
() => this.smartProxy,
|
||||
() => this.options.http3,
|
||||
this.options.vpnConfig?.enabled
|
||||
? (tags?: string[]) => {
|
||||
if (tags?.length && this.vpnManager) {
|
||||
return this.vpnManager.getClientIpsForServerDefinedTags(tags);
|
||||
// ConfigManagers: optional, depends on SmartProxy + DcRouterDb
|
||||
// Requires DcRouterDb to be enabled (document classes need the database)
|
||||
if (this.options.dbConfig?.enabled !== false) {
|
||||
this.serviceManager.addService(
|
||||
new plugins.taskbuffer.Service('ConfigManagers')
|
||||
.optional()
|
||||
.dependsOn('SmartProxy', 'DcRouterDb')
|
||||
.withStart(async () => {
|
||||
this.routeConfigManager = new RouteConfigManager(
|
||||
() => this.getConstructorRoutes(),
|
||||
() => this.smartProxy,
|
||||
() => this.options.http3,
|
||||
this.options.vpnConfig?.enabled
|
||||
? (tags?: string[]) => {
|
||||
if (tags?.length && this.vpnManager) {
|
||||
return this.vpnManager.getClientIpsForServerDefinedTags(tags);
|
||||
}
|
||||
return [this.options.vpnConfig?.subnet || '10.8.0.0/24'];
|
||||
}
|
||||
return [this.options.vpnConfig?.subnet || '10.8.0.0/24'];
|
||||
}
|
||||
: undefined,
|
||||
);
|
||||
this.apiTokenManager = new ApiTokenManager(this.storageManager);
|
||||
await this.apiTokenManager.initialize();
|
||||
await this.routeConfigManager.initialize();
|
||||
})
|
||||
.withStop(async () => {
|
||||
this.routeConfigManager = undefined;
|
||||
this.apiTokenManager = undefined;
|
||||
})
|
||||
.withRetry({ maxRetries: 2, baseDelayMs: 1000 }),
|
||||
);
|
||||
: undefined,
|
||||
);
|
||||
this.apiTokenManager = new ApiTokenManager();
|
||||
await this.apiTokenManager.initialize();
|
||||
await this.routeConfigManager.initialize();
|
||||
})
|
||||
.withStop(async () => {
|
||||
this.routeConfigManager = undefined;
|
||||
this.apiTokenManager = undefined;
|
||||
})
|
||||
.withRetry({ maxRetries: 2, baseDelayMs: 1000 }),
|
||||
);
|
||||
}
|
||||
|
||||
// Email Server: optional, depends on SmartProxy
|
||||
if (this.options.emailConfig) {
|
||||
@@ -695,14 +679,9 @@ export class DcRouter {
|
||||
logger.log('info', `Remote Ingress: tunnel port=${this.options.remoteIngressConfig.tunnelPort || 8443}, edges=${edgeCount} registered/${connectedCount} connected`);
|
||||
}
|
||||
|
||||
// Storage summary
|
||||
if (this.storageManager && this.options.storage) {
|
||||
logger.log('info', `Storage: path=${this.options.storage.fsPath || 'default'}`);
|
||||
}
|
||||
|
||||
// Cache database summary
|
||||
if (this.cacheDb) {
|
||||
logger.log('info', `Cache Database: storage=${this.cacheDb.getStoragePath()}, db=${this.cacheDb.getDbName()}, cleaner=${this.cacheCleaner?.isActive() ? 'active' : 'inactive'} (${(this.options.cacheConfig?.cleanupIntervalHours || 1)}h interval)`);
|
||||
// Database summary
|
||||
if (this.dcRouterDb) {
|
||||
logger.log('info', `Database: ${this.dcRouterDb.isEmbedded() ? 'embedded' : 'external'}, db=${this.dcRouterDb.getDbName()}, cleaner=${this.cacheCleaner?.isActive() ? 'active' : 'inactive'} (${(this.options.dbConfig?.cleanupIntervalHours || 1)}h interval)`);
|
||||
}
|
||||
|
||||
// Service status summary from ServiceManager
|
||||
@@ -723,31 +702,32 @@ export class DcRouter {
|
||||
}
|
||||
|
||||
/**
|
||||
* Set up the cache database (smartdata + LocalTsmDb)
|
||||
* Set up the unified database (smartdata + LocalSmartDb or external MongoDB)
|
||||
*/
|
||||
private async setupCacheDb(): Promise<void> {
|
||||
logger.log('info', 'Setting up CacheDb...');
|
||||
private async setupDcRouterDb(): Promise<void> {
|
||||
logger.log('info', 'Setting up DcRouterDb...');
|
||||
|
||||
const cacheConfig = this.options.cacheConfig || {};
|
||||
const dbConfig = this.options.dbConfig || {};
|
||||
|
||||
// Initialize CacheDb singleton
|
||||
this.cacheDb = CacheDb.getInstance({
|
||||
storagePath: cacheConfig.storagePath || this.resolvedPaths.defaultTsmDbPath,
|
||||
dbName: cacheConfig.dbName || 'dcrouter',
|
||||
// Initialize DcRouterDb singleton
|
||||
this.dcRouterDb = DcRouterDb.getInstance({
|
||||
mongoDbUrl: dbConfig.mongoDbUrl,
|
||||
storagePath: dbConfig.storagePath || this.resolvedPaths.defaultTsmDbPath,
|
||||
dbName: dbConfig.dbName || 'dcrouter',
|
||||
debug: false,
|
||||
});
|
||||
|
||||
await this.cacheDb.start();
|
||||
await this.dcRouterDb.start();
|
||||
|
||||
// Start the cache cleaner
|
||||
const cleanupIntervalMs = (cacheConfig.cleanupIntervalHours || 1) * 60 * 60 * 1000;
|
||||
this.cacheCleaner = new CacheCleaner(this.cacheDb, {
|
||||
// Start the cache cleaner for TTL-based document cleanup
|
||||
const cleanupIntervalMs = (dbConfig.cleanupIntervalHours || 1) * 60 * 60 * 1000;
|
||||
this.cacheCleaner = new CacheCleaner(this.dcRouterDb, {
|
||||
intervalMs: cleanupIntervalMs,
|
||||
verbose: false,
|
||||
});
|
||||
this.cacheCleaner.start();
|
||||
|
||||
logger.log('info', `CacheDb initialized at ${this.cacheDb.getStoragePath()}`);
|
||||
logger.log('info', `DcRouterDb ready (${this.dcRouterDb.isEmbedded() ? 'embedded' : 'external'})`);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -850,14 +830,11 @@ export class DcRouter {
|
||||
acme: acmeConfig,
|
||||
certStore: {
|
||||
loadAll: async () => {
|
||||
const keys = await this.storageManager.list('/proxy-certs/');
|
||||
const docs = await ProxyCertDoc.findAll();
|
||||
const certs: Array<{ domain: string; publicKey: string; privateKey: string; ca?: string }> = [];
|
||||
for (const key of keys) {
|
||||
const data = await this.storageManager.getJSON(key);
|
||||
if (data) {
|
||||
certs.push(data);
|
||||
loadedCertEntries.push({ domain: data.domain, publicKey: data.publicKey, validUntil: data.validUntil, validFrom: data.validFrom });
|
||||
}
|
||||
for (const doc of docs) {
|
||||
certs.push({ domain: doc.domain, publicKey: doc.publicKey, privateKey: doc.privateKey, ca: doc.ca });
|
||||
loadedCertEntries.push({ domain: doc.domain, publicKey: doc.publicKey, validUntil: doc.validUntil, validFrom: doc.validFrom });
|
||||
}
|
||||
return certs;
|
||||
},
|
||||
@@ -869,18 +846,29 @@ export class DcRouter {
|
||||
validUntil = new Date(x509.validTo).getTime();
|
||||
validFrom = new Date(x509.validFrom).getTime();
|
||||
} catch { /* PEM parsing failed */ }
|
||||
await this.storageManager.setJSON(`/proxy-certs/${domain}`, {
|
||||
domain, publicKey, privateKey, ca, validUntil, validFrom,
|
||||
});
|
||||
let doc = await ProxyCertDoc.findByDomain(domain);
|
||||
if (!doc) {
|
||||
doc = new ProxyCertDoc();
|
||||
doc.domain = domain;
|
||||
}
|
||||
doc.publicKey = publicKey;
|
||||
doc.privateKey = privateKey;
|
||||
doc.ca = ca || '';
|
||||
doc.validUntil = validUntil || 0;
|
||||
doc.validFrom = validFrom || 0;
|
||||
await doc.save();
|
||||
},
|
||||
remove: async (domain: string) => {
|
||||
await this.storageManager.delete(`/proxy-certs/${domain}`);
|
||||
const doc = await ProxyCertDoc.findByDomain(domain);
|
||||
if (doc) {
|
||||
await doc.delete();
|
||||
}
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
// Initialize cert provision scheduler
|
||||
this.certProvisionScheduler = new CertProvisionScheduler(this.storageManager);
|
||||
this.certProvisionScheduler = new CertProvisionScheduler();
|
||||
|
||||
// If we have DNS challenge handlers, create SmartAcme instance and wire certProvisionFunction
|
||||
// Note: SmartAcme.start() is NOT called here — it runs as a separate optional service
|
||||
@@ -895,7 +883,7 @@ export class DcRouter {
|
||||
}
|
||||
this.smartAcme = new plugins.smartacme.SmartAcme({
|
||||
accountEmail: acmeConfig?.accountEmail || this.options.tls?.contactEmail || 'admin@example.com',
|
||||
certManager: new StorageBackedCertManager(this.storageManager),
|
||||
certManager: new StorageBackedCertManager(),
|
||||
environment: 'production',
|
||||
challengeHandlers: challengeHandlers,
|
||||
challengePriority: ['dns-01'],
|
||||
@@ -1037,16 +1025,16 @@ export class DcRouter {
|
||||
issuedAt = new Date(entry.validFrom).toISOString();
|
||||
}
|
||||
|
||||
// Try SmartAcme /certs/ metadata as secondary source
|
||||
// Try SmartAcme AcmeCertDoc metadata as secondary source
|
||||
if (!expiryDate) {
|
||||
try {
|
||||
const cleanDomain = entry.domain.replace(/^\*\.?/, '');
|
||||
const certMeta = await this.storageManager.getJSON(`/certs/${cleanDomain}`);
|
||||
if (certMeta?.validUntil) {
|
||||
expiryDate = new Date(certMeta.validUntil).toISOString();
|
||||
const certDoc = await AcmeCertDoc.findByDomain(cleanDomain);
|
||||
if (certDoc?.validUntil) {
|
||||
expiryDate = new Date(certDoc.validUntil).toISOString();
|
||||
}
|
||||
if (certMeta?.created && !issuedAt) {
|
||||
issuedAt = new Date(certMeta.created).toISOString();
|
||||
if (certDoc?.created && !issuedAt) {
|
||||
issuedAt = new Date(certDoc.created).toISOString();
|
||||
}
|
||||
} catch { /* no metadata available */ }
|
||||
}
|
||||
@@ -2030,7 +2018,7 @@ export class DcRouter {
|
||||
logger.log('info', 'Setting up Remote Ingress hub...');
|
||||
|
||||
// Initialize the edge registration manager
|
||||
this.remoteIngressManager = new RemoteIngressManager(this.storageManager);
|
||||
this.remoteIngressManager = new RemoteIngressManager();
|
||||
await this.remoteIngressManager.initialize();
|
||||
|
||||
// Pass current routes so the manager can derive edge ports from remoteIngress-tagged routes
|
||||
@@ -2056,7 +2044,7 @@ export class DcRouter {
|
||||
// Priority 2: Existing cert from SmartProxy cert store for hubDomain
|
||||
if (!tlsConfig && riCfg.hubDomain) {
|
||||
try {
|
||||
const stored = await this.storageManager.getJSON(`/proxy-certs/${riCfg.hubDomain}`);
|
||||
const stored = await ProxyCertDoc.findByDomain(riCfg.hubDomain);
|
||||
if (stored?.publicKey && stored?.privateKey) {
|
||||
tlsConfig = { certPem: stored.publicKey, keyPem: stored.privateKey };
|
||||
logger.log('info', `Using stored ACME cert for RemoteIngress tunnel TLS: ${riCfg.hubDomain}`);
|
||||
@@ -2090,7 +2078,7 @@ export class DcRouter {
|
||||
|
||||
logger.log('info', 'Setting up VPN server...');
|
||||
|
||||
this.vpnManager = new VpnManager(this.storageManager, {
|
||||
this.vpnManager = new VpnManager({
|
||||
subnet: this.options.vpnConfig.subnet,
|
||||
wgListenPort: this.options.vpnConfig.wgListenPort,
|
||||
dns: this.options.vpnConfig.dns,
|
||||
@@ -2180,7 +2168,7 @@ export class DcRouter {
|
||||
|
||||
logger.log('info', 'Setting up RADIUS server...');
|
||||
|
||||
this.radiusServer = new RadiusServer(this.options.radiusConfig, this.storageManager);
|
||||
this.radiusServer = new RadiusServer(this.options.radiusConfig);
|
||||
await this.radiusServer.start();
|
||||
|
||||
logger.log('info', `RADIUS server started on ports ${this.options.radiusConfig.authPort || 1812} (auth) and ${this.options.radiusConfig.acctPort || 1813} (acct)`);
|
||||
|
||||
Reference in New Issue
Block a user