serve.zone/dcrouter
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(email): add persistent smartmta storage and runtime-managed email domain syncing

Browse Source
This commit is contained in:
Jürgen Kunz
2026-04-14 13:11:48 +00:00
parent 9a378ae87f
commit 1d7e5495fa
14 changed files with 690 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files

226
ts/classes.dcrouter.ts
View File

@@ -9,6 +9,7 @@ import {
type IUnifiedEmailServerOptions,
type IEmailRoute,
type IEmailDomainConfig,
type IStorageManagerLike,
} from '@push.rocks/smartmta';
import { logger } from './logger.js';
import { StorageBackedCertManager } from './classes.storage-cert-manager.js';
@@ -29,7 +30,7 @@ import { SecurityLogger, ContentScanner, IPReputationChecker } from './security/
import { type IHttp3Config, augmentRoutesWithHttp3 } from './http3/index.js';
import { DnsManager } from './dns/manager.dns.js';
import { AcmeConfigManager } from './acme/manager.acme-config.js';
import { EmailDomainManager } from './email/classes.email-domain.manager.js';
import { EmailDomainManager, SmartMtaStorageManager } from './email/index.js';
export interface IDcRouterOptions {
/** Base directory for all dcrouter data. Defaults to ~/.serve.zone/dcrouter */
@@ -248,15 +249,13 @@ export class DcRouter {
public radiusServer?: RadiusServer;
public opsServer!: OpsServer;
public metricsManager?: MetricsManager;
private emailEventSubscriptions: Array<{
emitter: { off(eventName: string, listener: (...args: any[]) => void): void };
eventName: string;
listener: (...args: any[]) => void;
}> = [];
// Compatibility shim for smartmta's DkimManager which calls dcRouter.storageManager.set()
public storageManager: any = {
get: async (_key: string) => null,
set: async (_key: string, _value: string) => {
// DKIM keys from smartmta — logged but not yet migrated to smartdata
logger.log('debug', `storageManager.set() called (compat shim) for key: ${_key}`);
},
};
public storageManager: IStorageManagerLike;
// Unified database (smartdata + LocalSmartDb or external MongoDB)
public dcRouterDb?: DcRouterDb;
@@ -329,6 +328,10 @@ export class DcRouter {
// Resolve all data paths from baseDir
this.resolvedPaths = paths.resolvePaths(this.options.baseDir);
paths.ensureDataDirectories(this.resolvedPaths);
this.storageManager = new SmartMtaStorageManager(
plugins.path.join(this.resolvedPaths.dataDir, 'smartmta-storage')
);
// Initialize service manager and register all services
this.serviceManager = new plugins.taskbuffer.ServiceManager({
@@ -452,9 +455,13 @@ export class DcRouter {
.dependsOn('DcRouterDb')
.withStart(async () => {
this.emailDomainManager = new EmailDomainManager(this);
await this.emailDomainManager.start();
})
.withStop(async () => {
this.emailDomainManager = undefined;
if (this.emailDomainManager) {
await this.emailDomainManager.stop();
this.emailDomainManager = undefined;
}
}),
);
}
@@ -610,19 +617,20 @@ export class DcRouter {
// Email Server: optional, depends on SmartProxy
if (this.options.emailConfig) {
const emailServiceDeps = ['SmartProxy', 'MetricsManager'];
if (this.options.dbConfig?.enabled !== false) {
emailServiceDeps.push('EmailDomainManager');
}
this.serviceManager.addService(
new plugins.taskbuffer.Service('EmailServer')
.optional()
.dependsOn('SmartProxy')
.dependsOn(...emailServiceDeps)
.withStart(async () => {
await this.setupUnifiedEmailHandling();
})
.withStop(async () => {
if (this.emailServer) {
if ((this.emailServer as any).deliverySystem) {
(this.emailServer as any).deliverySystem.removeAllListeners();
}
this.emailServer.removeAllListeners();
this.clearEmailEventSubscriptions();
await this.emailServer.stop();
this.emailServer = undefined;
}
@@ -636,7 +644,7 @@ export class DcRouter {
this.serviceManager.addService(
new plugins.taskbuffer.Service('DnsServer')
.optional()
.dependsOn('SmartProxy')
.dependsOn('SmartProxy', ...(this.options.emailConfig ? ['EmailServer'] : []))
.withStart(async () => {
await this.setupDnsWithSocketHandler();
})
@@ -1511,40 +1519,74 @@ export class DcRouter {
...this.options.emailConfig,
domains: transformedDomains,
ports: this.options.emailConfig.ports.map(port => portMapping[port] || port + 10000),
hostname: 'localhost' // Listen on localhost for SmartProxy forwarding
persistRoutes: this.options.emailConfig.persistRoutes ?? false,
queue: {
storageType: 'disk',
persistentPath: plugins.path.join(this.resolvedPaths.dataDir, 'smartmta-queue'),
...this.options.emailConfig.queue,
},
};
// Create unified email server
this.emailServer = new UnifiedEmailServer(this, emailConfig);
this.clearEmailEventSubscriptions();
// Set up error handling
this.emailServer.on('error', (err: Error) => {
this.addEmailEventSubscription(this.emailServer, 'error', (err: Error) => {
logger.log('error', `UnifiedEmailServer error: ${err.message}`);
});
// Start the server
await this.emailServer.start();
// Wire delivery events to MetricsManager and logger
if (this.metricsManager && this.emailServer.deliverySystem) {
this.emailServer.deliverySystem.on('deliveryStart', (item: any) => {
this.metricsManager!.trackEmailReceived(item?.from);
logger.log('info', `Email delivery started: ${item?.from} → ${item?.to}`, { zone: 'email' });
});
this.emailServer.deliverySystem.on('deliverySuccess', (item: any) => {
this.metricsManager!.trackEmailSent(item?.to);
logger.log('info', `Email delivered to ${item?.to}`, { zone: 'email' });
});
this.emailServer.deliverySystem.on('deliveryFailed', (item: any, error: any) => {
this.metricsManager!.trackEmailFailed(item?.to, error?.message);
logger.log('warn', `Email delivery failed to ${item?.to}: ${error?.message}`, { zone: 'email' });
});
}
// Wire delivery events to MetricsManager and logger using smartmta's public queue APIs.
if (this.metricsManager && this.emailServer) {
this.emailServer.on('bounceProcessed', () => {
const getEnvelope = (item: { processingResult?: any; lastError?: string }) => {
const emailLike = item?.processingResult;
const from = emailLike?.from || emailLike?.email?.from || '';
const recipients = Array.isArray(emailLike?.to)
? emailLike.to
: Array.isArray(emailLike?.email?.to)
? emailLike.email.to
: [];
return {
from,
recipients: recipients.filter(Boolean),
};
};
const updateQueueSize = () => {
this.metricsManager!.updateQueueSize(this.emailServer!.getQueueStats().queueSize);
};
this.addEmailEventSubscription(this.emailServer.deliveryQueue, 'itemEnqueued', (item: any) => {
const envelope = getEnvelope(item);
this.metricsManager!.trackEmailReceived(envelope.from);
updateQueueSize();
logger.log('info', `Email queued: ${envelope.from} → ${envelope.recipients.join(', ') || 'unknown'}`, { zone: 'email' });
});
this.addEmailEventSubscription(this.emailServer.deliveryQueue, 'itemDelivered', (item: any) => {
const envelope = getEnvelope(item);
this.metricsManager!.trackEmailSent(envelope.recipients[0]);
updateQueueSize();
logger.log('info', `Email delivered to ${envelope.recipients.join(', ') || 'unknown'}`, { zone: 'email' });
});
this.addEmailEventSubscription(this.emailServer.deliveryQueue, 'itemFailed', (item: any) => {
const envelope = getEnvelope(item);
this.metricsManager!.trackEmailFailed(envelope.recipients[0], item?.lastError);
updateQueueSize();
logger.log('warn', `Email delivery failed to ${envelope.recipients.join(', ') || 'unknown'}: ${item?.lastError || 'unknown error'}`, { zone: 'email' });
});
this.addEmailEventSubscription(this.emailServer.deliveryQueue, 'itemDeferred', () => {
updateQueueSize();
});
this.addEmailEventSubscription(this.emailServer.deliveryQueue, 'itemRemoved', () => {
updateQueueSize();
});
this.addEmailEventSubscription(this.emailServer, 'bounceProcessed', () => {
this.metricsManager!.trackEmailBounced();
logger.log('warn', 'Email bounce processed', { zone: 'email' });
});
updateQueueSize();
}
logger.log('info', `Email server started on ports: ${emailConfig.ports.join(', ')}`);
@@ -1574,11 +1616,7 @@ export class DcRouter {
try {
// Stop the unified email server which contains all components
if (this.emailServer) {
// Remove listeners before stopping to prevent leaks on config update cycles
if ((this.emailServer as any).deliverySystem) {
(this.emailServer as any).deliverySystem.removeAllListeners();
}
this.emailServer.removeAllListeners();
this.clearEmailEventSubscriptions();
await this.emailServer.stop();
logger.log('info', 'Unified email server stopped');
this.emailServer = undefined;
@@ -1783,14 +1821,14 @@ export class DcRouter {
// Generate and register authoritative records
const authoritativeRecords = await this.generateAuthoritativeRecords();
// Generate email DNS records
const emailDnsRecords = await this.generateEmailDnsRecords();
// Initialize DKIM for all email domains
await this.initializeDkimForEmailDomains();
// Load DKIM records from JSON files (they should now exist)
const dkimRecords = await this.loadDkimRecords();
// Generate email DNS records
const emailDnsRecords = await this.generateEmailDnsRecords();
// Ensure DKIM keys exist for internal-dns domains before generating records.
await this.initializeDkimForEmailDomains();
// Generate DKIM records directly from smartmta instead of scanning legacy JSON files.
const dkimRecords = await this.loadDkimRecords();
// Combine all records: authoritative, email, DKIM, and user-defined
const allRecords = [...authoritativeRecords, ...emailDnsRecords, ...dkimRecords];
@@ -1939,54 +1977,30 @@ export class DcRouter {
}
/**
* Load DKIM records from JSON files
* Reads all *.dkimrecord.json files from the DNS records directory
* Generate DKIM DNS records for internal-dns domains from smartmta's selector-aware DKIM state.
*/
private async loadDkimRecords(): Promise<Array<{name: string; type: string; value: string; ttl?: number}>> {
const records: Array<{name: string; type: string; value: string; ttl?: number}> = [];
try {
// Ensure paths are imported
const dnsDir = this.resolvedPaths.dnsRecordsDir;
// Check if directory exists
if (!plugins.fs.existsSync(dnsDir)) {
logger.log('debug', 'No DNS records directory found, skipping DKIM record loading');
return records;
if (!this.options.emailConfig?.domains || !this.emailServer?.dkimCreator) {
return records;
}
for (const domainConfig of this.options.emailConfig.domains) {
if (domainConfig.dnsMode !== 'internal-dns') {
continue;
}
// Read all files in the directory
const files = plugins.fs.readdirSync(dnsDir);
const dkimFiles = files.filter(f => f.endsWith('.dkimrecord.json'));
logger.log('info', `Found ${dkimFiles.length} DKIM record files`);
// Load each DKIM record
for (const file of dkimFiles) {
try {
const filePath = plugins.path.join(dnsDir, file);
const fileContent = plugins.fs.readFileSync(filePath, 'utf8');
const dkimRecord = JSON.parse(fileContent);
// Validate record structure
if (dkimRecord.name && dkimRecord.type === 'TXT' && dkimRecord.value) {
records.push({
name: dkimRecord.name,
type: 'TXT',
value: dkimRecord.value,
ttl: 3600 // Standard DKIM TTL
});
logger.log('info', `Loaded DKIM record for ${dkimRecord.name}`);
} else {
logger.log('warn', `Invalid DKIM record structure in ${file}`);
}
} catch (error: unknown) {
logger.log('error', `Failed to load DKIM record from ${file}: ${(error as Error).message}`);
}
const selector = domainConfig.dkim?.selector || 'default';
try {
const dkimRecord = await this.emailServer.dkimCreator.getDNSRecordForDomain(domainConfig.domain, selector);
records.push({
name: dkimRecord.name,
type: 'TXT',
value: dkimRecord.value,
ttl: domainConfig.dns?.internal?.ttl || 3600,
});
} catch (error: unknown) {
logger.log('error', `Failed to generate DKIM record for ${domainConfig.domain}: ${(error as Error).message}`);
}
} catch (error: unknown) {
logger.log('error', `Failed to load DKIM records: ${(error as Error).message}`);
}
return records;
@@ -2013,12 +2027,17 @@ export class DcRouter {
// Ensure necessary directories exist
paths.ensureDataDirectories(this.resolvedPaths);
// Generate DKIM keys for each email domain
// Generate DKIM keys for each internal-dns email domain using the configured selector.
for (const domainConfig of this.options.emailConfig.domains) {
if (domainConfig.dnsMode !== 'internal-dns') {
continue;
}
try {
// Generate DKIM keys for all domains, regardless of DNS mode
// This ensures keys are ready even if DNS mode changes later
await dkimCreator.handleDKIMKeysForDomain(domainConfig.domain);
await dkimCreator.handleDKIMKeysForSelector(
domainConfig.domain,
domainConfig.dkim?.selector || 'default',
domainConfig.dkim?.keySize || 2048,
);
logger.log('info', `DKIM keys initialized for ${domainConfig.domain}`);
} catch (error: unknown) {
logger.log('error', `Failed to initialize DKIM for ${domainConfig.domain}: ${(error as Error).message}`);
@@ -2148,6 +2167,25 @@ export class DcRouter {
}
}
}
private addEmailEventSubscription(
emitter: {
on(eventName: string, listener: (...args: any[]) => void): void;
off(eventName: string, listener: (...args: any[]) => void): void;
},
eventName: string,
listener: (...args: any[]) => void,
): void {
emitter.on(eventName, listener);
this.emailEventSubscriptions.push({ emitter, eventName, listener });
}
private clearEmailEventSubscriptions(): void {
for (const subscription of this.emailEventSubscriptions) {
subscription.emitter.off(subscription.eventName, subscription.listener);
}
this.emailEventSubscriptions = [];
}
/**
* Detect the server's public IP address