From 65ecd9454047a42e17bb9644afc3b3b23adfbedc Mon Sep 17 00:00:00 2001 From: Juergen Kunz Date: Tue, 14 Apr 2026 12:17:50 +0000 Subject: [PATCH] fix(mail): align queue, outbound hostname, and DKIM selector behavior across the mail server APIs --- changelog.md | 8 + readme.md | 17 ++ test/test.email.contracts.node.ts | 122 ++++++++++ test/test.mta.delivery.node.ts | 2 +- ts/00_commitinfo_data.ts | 2 +- ts/mail/core/classes.bouncemanager.ts | 13 +- ts/mail/delivery/classes.delivery.queue.ts | 10 +- ts/mail/index.ts | 3 +- ts/mail/interfaces.storage.ts | 13 ++ ts/mail/routing/classes.dkim.manager.ts | 35 ++- ts/mail/routing/classes.dns.manager.ts | 11 +- ts/mail/routing/classes.email.router.ts | 13 +- .../routing/classes.unified.email.server.ts | 64 +++++- ts/mail/security/classes.dkimcreator.ts | 208 ++++++++++-------- ts/security/classes.ipreputationchecker.ts | 13 +- 15 files changed, 387 insertions(+), 147 deletions(-) create mode 100644 test/test.email.contracts.node.ts create mode 100644 ts/mail/interfaces.storage.ts diff --git a/changelog.md b/changelog.md index be81a44..8d206a9 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,13 @@ # Changelog +## 2026-04-14 - 5.3.2 - fix(mail) +align queue, outbound hostname, and DKIM selector behavior across the mail server APIs + +- return the actual delivery queue item id from sendEmail() and add queue inspection/stat APIs on UnifiedEmailServer +- use outbound.hostname for outbound SMTP identity while keeping hostname as the advertised public server hostname +- fix DKIM selector handling so DNS record names, key storage, signing, and rotation stay selector-aware +- harden storage manager integration with a shared typed interface and capability checks across mail and security components + ## 2026-03-02 - 5.3.1 - fix(mail) add periodic cleanup timers and proper shutdown handling for bounce manager and delivery queue; avoid mutating maps during iteration and prune stale rate-limiter stats to prevent memory growth diff --git a/readme.md b/readme.md index 7a5f5a2..ea69341 100644 --- a/readme.md +++ b/readme.md @@ -95,6 +95,8 @@ import { UnifiedEmailServer } from '@push.rocks/smartmta'; const emailServer = new UnifiedEmailServer(dcRouterRef, { // Ports to listen on (465 = implicit TLS, 25/587 = STARTTLS) ports: [25, 587, 465], + + // Public SMTP hostname used for greeting/banner and as the default outbound identity hostname: 'mail.example.com', // Multi-domain configuration @@ -160,6 +162,16 @@ const emailServer = new UnifiedEmailServer(dcRouterRef, { keyPath: '/etc/ssl/mail.key', }, + outbound: { + // Optional override for outbound EHLO/HELO identity + hostname: 'smtp-out.example.com', + }, + + queue: { + storageType: 'disk', + persistentPath: '/var/lib/smartmta/email-queue', + }, + maxMessageSize: 25 * 1024 * 1024, // 25 MB maxClients: 500, }); @@ -169,6 +181,8 @@ await emailServer.start(); ``` > 🔒 **Note:** `start()` will throw if the Rust binary is not compiled. Run `pnpm build` first. +> +> `hostname` is the public SMTP identity for greetings and outbound delivery by default. It is not a bind address. ### 📧 Sending Emails (Automatic MX Discovery) @@ -201,6 +215,8 @@ const emailId = await emailServer.sendEmail(email); const emailId2 = await emailServer.sendEmail(email, 'mta'); ``` +`sendEmail()` returns the delivery queue item ID, which you can later use with queue/status APIs. + In MTA mode, smartmta: - 🔍 Resolves MX records for each recipient domain (e.g. `gmail.com`, `company.org`) - 📊 Sorts MX hosts by priority (lowest = highest priority per RFC 5321) @@ -254,6 +270,7 @@ The `sendOutboundEmail` method: - 🔑 Automatically resolves DKIM keys from the `DKIMCreator` for the specified domain - 🔗 Uses connection pooling in Rust — reuses TCP/TLS connections across sends - ⏱️ Configurable connection and socket timeouts via `outbound` options on the server +- 🪪 Uses `outbound.hostname` as the SMTP identity when configured, otherwise falls back to `hostname` ### 🔑 DKIM Signing & Key Management diff --git a/test/test.email.contracts.node.ts b/test/test.email.contracts.node.ts new file mode 100644 index 0000000..0f25f9e --- /dev/null +++ b/test/test.email.contracts.node.ts @@ -0,0 +1,122 @@ +import { tap, expect } from '@git.zone/tstest/tapbundle'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; + +import { Email } from '../ts/mail/core/classes.email.js'; +import { DKIMCreator } from '../ts/mail/security/classes.dkimcreator.js'; +import { UnifiedEmailServer } from '../ts/mail/routing/classes.unified.email.server.js'; + +const storageMap = new Map(); +const serversToCleanup: UnifiedEmailServer[] = []; +const mockDcRouter = { + storageManager: { + get: async (key: string) => storageMap.get(key) || null, + set: async (key: string, value: string) => { + storageMap.set(key, value); + }, + list: async (prefix: string) => Array.from(storageMap.keys()).filter((key) => key.startsWith(prefix)), + delete: async (key: string) => { + storageMap.delete(key); + }, + }, +}; + +tap.test('UnifiedEmailServer.sendEmail returns the actual queue item id', async () => { + const server = new UnifiedEmailServer(mockDcRouter, { + ports: [10025], + hostname: 'mail.example.com', + domains: [{ domain: 'example.com', dnsMode: 'forward' }], + routes: [], + }); + serversToCleanup.push(server); + + const route = { + name: 'test-deliver-route', + match: { recipients: '*@*' }, + action: { type: 'deliver' as const }, + }; + + const email = new Email({ + from: 'sender@example.com', + to: ['recipient@example.net'], + subject: 'Queue ID contract', + text: 'hello', + }); + + const queueId = await server.sendEmail(email, 'mta', route); + const queuedItem = server.getQueueItem(queueId); + + expect(queuedItem).toBeTruthy(); + expect(queuedItem?.id).toEqual(queueId); + expect(server.getQueueStats().queueSize).toEqual(1); + expect(server.getQueueItems().map((item) => item.id)).toContain(queueId); +}); + +tap.test('UnifiedEmailServer.sendOutboundEmail uses outbound.hostname when configured', async () => { + const server = new UnifiedEmailServer(mockDcRouter, { + ports: [10026], + hostname: 'mail.example.com', + outbound: { + hostname: 'outbound.example.com', + }, + domains: [{ domain: 'example.com', dnsMode: 'forward' }], + routes: [], + }); + serversToCleanup.push(server); + + const email = new Email({ + from: 'sender@example.com', + to: ['recipient@example.net'], + subject: 'Outbound hostname contract', + text: 'hello', + }); + + let capturedOptions: any; + (server as any).rustBridge.sendOutboundEmail = async (options: any) => { + capturedOptions = options; + return { + accepted: ['recipient@example.net'], + rejected: [], + messageId: 'test-message-id', + response: '250 2.0.0 queued', + envelope: { + from: 'sender@example.com', + to: ['recipient@example.net'], + }, + }; + }; + + await server.sendOutboundEmail('smtp.target.example', 25, email); + + expect(capturedOptions).toBeTruthy(); + expect(capturedOptions.domain).toEqual('outbound.example.com'); +}); + +tap.test('DKIMCreator returns selector-aligned DNS record names', async () => { + const tempDir = path.join(os.tmpdir(), `smartmta-dkim-${Date.now()}`); + fs.mkdirSync(tempDir, { recursive: true }); + + try { + const creator = new DKIMCreator(tempDir); + + await creator.createAndStoreDKIMKeys('example.com'); + const defaultRecord = await creator.getDNSRecordForDomain('example.com'); + expect(defaultRecord.name).toEqual('default._domainkey.example.com'); + + await creator.createAndStoreDKIMKeysForSelector('example.org', 'selector1'); + const selectorRecord = await creator.getDNSRecordForDomain('example.org', 'selector1'); + expect(selectorRecord.name).toEqual('selector1._domainkey.example.org'); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +}); + +tap.test('cleanup', async () => { + for (const server of serversToCleanup) { + await server.stop(); + } + await tap.stopForcefully(); +}); + +export default tap.start(); diff --git a/test/test.mta.delivery.node.ts b/test/test.mta.delivery.node.ts index 0f6d426..fd6af67 100644 --- a/test/test.mta.delivery.node.ts +++ b/test/test.mta.delivery.node.ts @@ -99,7 +99,7 @@ tap.test('setup - start server and mock SMTP', async () => { type: 'process', options: { contentScanning: true, - scanners: [{ type: 'spam' }], + scanners: [{ type: 'spam', action: 'tag' }], }, }, }, diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index fa0f489..237fda7 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@push.rocks/smartmta', - version: '5.3.1', + version: '5.3.2', description: 'A high-performance, enterprise-grade Mail Transfer Agent (MTA) built from scratch in TypeScript with Rust acceleration.' } diff --git a/ts/mail/core/classes.bouncemanager.ts b/ts/mail/core/classes.bouncemanager.ts index b2860b3..e437910 100644 --- a/ts/mail/core/classes.bouncemanager.ts +++ b/ts/mail/core/classes.bouncemanager.ts @@ -1,6 +1,7 @@ import * as plugins from '../../plugins.js'; import * as paths from '../../paths.js'; import { logger } from '../../logger.js'; +import { hasStorageManagerMethods, type IStorageManagerLike } from '../interfaces.storage.js'; import { SecurityLogger, SecurityLogLevel, SecurityEventType } from '../../security/index.js'; import { RustSecurityBridge } from '../../security/classes.rustsecuritybridge.js'; import { LRUCache } from 'lru-cache'; @@ -107,13 +108,13 @@ export class BounceManager { expiresAt?: number; // undefined means permanent }> = new Map(); - private storageManager?: any; // StorageManager instance + private storageManager?: IStorageManagerLike; constructor(options?: { retryStrategy?: Partial; maxCacheSize?: number; cacheTTL?: number; - storageManager?: any; + storageManager?: IStorageManagerLike; }) { // Set retry strategy with defaults if (options?.retryStrategy) { @@ -552,7 +553,7 @@ export class BounceManager { try { const suppressionData = JSON.stringify(Array.from(this.suppressionList.entries())); - if (this.storageManager) { + if (hasStorageManagerMethods(this.storageManager, ['set'])) { // Use storage manager await this.storageManager.set('/email/bounces/suppression-list.json', suppressionData); } else { @@ -574,7 +575,7 @@ export class BounceManager { let entries = null; let needsMigration = false; - if (this.storageManager) { + if (hasStorageManagerMethods(this.storageManager, ['get'])) { // Try to load from storage manager first const suppressionData = await this.storageManager.get('/email/bounces/suppression-list.json'); @@ -636,7 +637,7 @@ export class BounceManager { try { const bounceData = JSON.stringify(bounce, null, 2); - if (this.storageManager) { + if (hasStorageManagerMethods(this.storageManager, ['set'])) { // Use storage manager await this.storageManager.set(`/email/bounces/records/${bounce.id}.json`, bounceData); } else { @@ -750,4 +751,4 @@ export class BounceManager { this.cleanupInterval = undefined; } } -} \ No newline at end of file +} diff --git a/ts/mail/delivery/classes.delivery.queue.ts b/ts/mail/delivery/classes.delivery.queue.ts index 6b86c89..91fb750 100644 --- a/ts/mail/delivery/classes.delivery.queue.ts +++ b/ts/mail/delivery/classes.delivery.queue.ts @@ -18,7 +18,7 @@ export interface IQueueItem { id: string; processingMode: EmailProcessingMode; processingResult: any; - route: IEmailRoute; + route?: IEmailRoute; status: QueueItemStatus; attempts: number; nextAttempt: Date; @@ -237,7 +237,7 @@ export class UnifiedDeliveryQueue extends EventEmitter { * @param mode Processing mode * @param route Email route */ - public async enqueue(processingResult: any, mode: EmailProcessingMode, route: IEmailRoute): Promise { + public async enqueue(processingResult: any, mode: EmailProcessingMode, route?: IEmailRoute): Promise { // Check if queue is full if (this.queue.size >= this.options.maxQueueSize) { throw new Error('Queue is full'); @@ -284,6 +284,10 @@ export class UnifiedDeliveryQueue extends EventEmitter { public getItem(id: string): IQueueItem | undefined { return this.queue.get(id); } + + public listItems(): IQueueItem[] { + return Array.from(this.queue.values()).map((item) => ({ ...item })); + } /** * Mark an item as being processed @@ -657,4 +661,4 @@ export class UnifiedDeliveryQueue extends EventEmitter { this.emit('shutdown'); logger.log('info', 'UnifiedDeliveryQueue shut down successfully'); } -} \ No newline at end of file +} diff --git a/ts/mail/index.ts b/ts/mail/index.ts index 81e19de..edd1fb1 100644 --- a/ts/mail/index.ts +++ b/ts/mail/index.ts @@ -1,4 +1,5 @@ // Export all mail modules for simplified imports +export * from './interfaces.storage.js'; export * from './routing/index.js'; export * from './security/index.js'; @@ -14,4 +15,4 @@ import { Email } from './core/classes.email.js'; // Re-export commonly used classes export { Email, -}; \ No newline at end of file +}; diff --git a/ts/mail/interfaces.storage.ts b/ts/mail/interfaces.storage.ts new file mode 100644 index 0000000..53f7198 --- /dev/null +++ b/ts/mail/interfaces.storage.ts @@ -0,0 +1,13 @@ +export interface IStorageManagerLike { + get?(key: string): Promise; + set?(key: string, value: string): Promise; + list?(prefix: string): Promise; + delete?(key: string): Promise; +} + +export function hasStorageManagerMethods( + storageManager: IStorageManagerLike | undefined, + methods: T[], +): storageManager is IStorageManagerLike & Required> { + return !!storageManager && methods.every((method) => typeof storageManager[method] === 'function'); +} diff --git a/ts/mail/routing/classes.dkim.manager.ts b/ts/mail/routing/classes.dkim.manager.ts index 7c86d39..5500655 100644 --- a/ts/mail/routing/classes.dkim.manager.ts +++ b/ts/mail/routing/classes.dkim.manager.ts @@ -1,12 +1,13 @@ import { logger } from '../../logger.js'; import { DKIMCreator } from '../security/classes.dkimcreator.js'; +import { hasStorageManagerMethods, type IStorageManagerLike } from '../interfaces.storage.js'; import { DomainRegistry } from './classes.domain.registry.js'; import { RustSecurityBridge } from '../../security/classes.rustsecuritybridge.js'; import { Email } from '../core/classes.email.js'; /** External DcRouter interface shape used by DkimManager */ interface DcRouter { - storageManager: any; + storageManager?: IStorageManagerLike; dnsServer?: any; } @@ -39,11 +40,19 @@ export class DkimManager { let keyPair: { privateKey: string; publicKey: string }; try { - keyPair = await this.dkimCreator.readDKIMKeys(domain); + keyPair = selector === 'default' + ? await this.dkimCreator.readDKIMKeys(domain) + : await this.dkimCreator.readDKIMKeysForSelector(domain, selector); logger.log('info', `Using existing DKIM keys for domain: ${domain}`); - } catch (error) { - keyPair = await this.dkimCreator.createDKIMKeys(); - await this.dkimCreator.createAndStoreDKIMKeys(domain); + } catch { + await this.dkimCreator.handleDKIMKeysForSelector( + domain, + selector, + domainConfig.dkim?.keySize || 2048, + ); + keyPair = selector === 'default' + ? await this.dkimCreator.readDKIMKeys(domain) + : await this.dkimCreator.readDKIMKeysForSelector(domain, selector); logger.log('info', `Generated new DKIM keys for domain: ${domain}`); } @@ -106,10 +115,12 @@ export class DkimManager { logger.log('info', `DKIM DNS handler registered for new selector: ${newSelector}._domainkey.${domain}`); - await this.dcRouter.storageManager.set( - `/email/dkim/${domain}/public.key`, - keyPair.publicKey - ); + if (hasStorageManagerMethods(this.dcRouter.storageManager, ['set'])) { + await this.dcRouter.storageManager.set( + `/email/dkim/${domain}/public.key`, + keyPair.publicKey + ); + } } this.dkimCreator.cleanupOldKeys(domain, 30).catch(error => { @@ -127,8 +138,10 @@ export class DkimManager { async handleDkimSigning(email: Email, domain: string, selector: string): Promise { try { - await this.dkimCreator.handleDKIMKeysForDomain(domain); - const { privateKey } = await this.dkimCreator.readDKIMKeys(domain); + await this.dkimCreator.handleDKIMKeysForSelector(domain, selector); + const { privateKey } = selector === 'default' + ? await this.dkimCreator.readDKIMKeys(domain) + : await this.dkimCreator.readDKIMKeysForSelector(domain, selector); const rawEmail = email.toRFC822String(); // Detect key type from PEM header diff --git a/ts/mail/routing/classes.dns.manager.ts b/ts/mail/routing/classes.dns.manager.ts index aaa2440..71c2f48 100644 --- a/ts/mail/routing/classes.dns.manager.ts +++ b/ts/mail/routing/classes.dns.manager.ts @@ -1,5 +1,6 @@ import * as plugins from '../../plugins.js'; import type { IEmailDomainConfig } from './interfaces.js'; +import type { IStorageManagerLike } from '../interfaces.storage.js'; import { logger } from '../../logger.js'; /** External DcRouter interface shape used by DnsManager */ interface IDcRouterLike { @@ -8,12 +9,6 @@ interface IDcRouterLike { options?: { dnsNsDomains?: string[]; dnsScopes?: string[] }; } -/** External StorageManager interface shape used by DnsManager */ -interface IStorageManagerLike { - get(key: string): Promise; - set(key: string, value: string): Promise; -} - /** * DNS validation result */ @@ -528,7 +523,7 @@ export class DnsManager { try { // Get DKIM DNS record from DKIMCreator - const dnsRecord = await dkimCreator.getDNSRecordForDomain(domain); + const dnsRecord = await dkimCreator.getDNSRecordForDomain(domain, selector); // For internal-dns domains, register the DNS handler if (domainConfig.dnsMode === 'internal-dns' && this.dcRouter.dnsServer) { @@ -570,4 +565,4 @@ export class DnsManager { } } } -} \ No newline at end of file +} diff --git a/ts/mail/routing/classes.email.router.ts b/ts/mail/routing/classes.email.router.ts index 7883c9b..bd64b93 100644 --- a/ts/mail/routing/classes.email.router.ts +++ b/ts/mail/routing/classes.email.router.ts @@ -1,5 +1,6 @@ import * as plugins from '../../plugins.js'; import { EventEmitter } from 'node:events'; +import { hasStorageManagerMethods, type IStorageManagerLike } from '../interfaces.storage.js'; import type { IEmailRoute, IEmailMatch, IEmailAction, IEmailContext } from './interfaces.js'; import type { Email } from '../core/classes.email.js'; @@ -9,7 +10,7 @@ import type { Email } from '../core/classes.email.js'; export class EmailRouter extends EventEmitter { private routes: IEmailRoute[]; private patternCache: Map = new Map(); - private storageManager?: any; // StorageManager instance + private storageManager?: IStorageManagerLike; private persistChanges: boolean; /** @@ -18,7 +19,7 @@ export class EmailRouter extends EventEmitter { * @param options Router options */ constructor(routes: IEmailRoute[], options?: { - storageManager?: any; + storageManager?: IStorageManagerLike; persistChanges?: boolean; }) { super(); @@ -27,7 +28,7 @@ export class EmailRouter extends EventEmitter { this.persistChanges = options?.persistChanges ?? !!this.storageManager; // If storage manager is provided, try to load persisted routes - if (this.storageManager) { + if (hasStorageManagerMethods(this.storageManager, ['get'])) { this.loadRoutes({ merge: true }).catch(error => { console.error(`Failed to load persisted routes: ${error.message}`); }); @@ -394,7 +395,7 @@ export class EmailRouter extends EventEmitter { * Save current routes to storage */ public async saveRoutes(): Promise { - if (!this.storageManager) { + if (!hasStorageManagerMethods(this.storageManager, ['set'])) { this.emit('persistenceWarning', 'Cannot save routes: StorageManager not configured'); return; } @@ -425,7 +426,7 @@ export class EmailRouter extends EventEmitter { merge?: boolean; // Merge with existing routes replace?: boolean; // Replace existing routes }): Promise { - if (!this.storageManager) { + if (!hasStorageManagerMethods(this.storageManager, ['get'])) { this.emit('persistenceWarning', 'Cannot load routes: StorageManager not configured'); return []; } @@ -572,4 +573,4 @@ export class EmailRouter extends EventEmitter { public getRoute(name: string): IEmailRoute | undefined { return this.routes.find(r => r.name === name); } -} \ No newline at end of file +} diff --git a/ts/mail/routing/classes.unified.email.server.ts b/ts/mail/routing/classes.unified.email.server.ts index a8b367f..da5ebdd 100644 --- a/ts/mail/routing/classes.unified.email.server.ts +++ b/ts/mail/routing/classes.unified.email.server.ts @@ -8,17 +8,18 @@ import { SecurityEventType } from '../../security/index.js'; import { DKIMCreator } from '../security/classes.dkimcreator.js'; +import { hasStorageManagerMethods, type IStorageManagerLike } from '../interfaces.storage.js'; import { RustSecurityBridge } from '../../security/classes.rustsecuritybridge.js'; -import type { IEmailReceivedEvent, IAuthRequestEvent, IEmailData } from '../../security/classes.rustsecuritybridge.js'; +import type { IEmailReceivedEvent, IAuthRequestEvent } from '../../security/classes.rustsecuritybridge.js'; import { EmailRouter } from './classes.email.router.js'; -import type { IEmailRoute, IEmailAction, IEmailContext, IEmailDomainConfig } from './interfaces.js'; +import type { IEmailRoute, IEmailContext, IEmailDomainConfig } from './interfaces.js'; import { Email } from '../core/classes.email.js'; import { DomainRegistry } from './classes.domain.registry.js'; import { DnsManager } from './classes.dns.manager.js'; import { BounceManager, BounceType, BounceCategory } from '../core/classes.bouncemanager.js'; import type { ISmtpSendResult, IOutboundEmail } from '../../security/classes.rustsecuritybridge.js'; -import { MultiModeDeliverySystem, type IMultiModeDeliveryOptions } from '../delivery/classes.delivery.system.js'; -import { UnifiedDeliveryQueue, type IQueueOptions } from '../delivery/classes.delivery.queue.js'; +import { MultiModeDeliverySystem, type IDeliveryStats, type IMultiModeDeliveryOptions } from '../delivery/classes.delivery.system.js'; +import { UnifiedDeliveryQueue, type IQueueItem, type IQueueOptions, type IQueueStats } from '../delivery/classes.delivery.queue.js'; import { UnifiedRateLimiter, type IHierarchicalRateLimits } from '../delivery/classes.unified.rate.limiter.js'; import { SmtpState } from '../delivery/interfaces.js'; import type { EmailProcessingMode, ISmtpSession as IBaseSmtpSession } from '../delivery/interfaces.js'; @@ -28,7 +29,7 @@ import { DkimManager } from './classes.dkim.manager.js'; /** External DcRouter interface shape used by UnifiedEmailServer */ interface DcRouter { - storageManager: any; + storageManager: IStorageManagerLike; dnsServer?: any; options?: any; } @@ -49,11 +50,14 @@ export interface IExtendedSmtpSession extends ISmtpSession { export interface IUnifiedEmailServerOptions { // Base server options ports: number[]; + /** Public SMTP hostname used for greeting/banner and as the default outbound identity. */ hostname: string; domains: IEmailDomainConfig[]; // Domain configurations banner?: string; debug?: boolean; useSocketHandler?: boolean; // Use socket-handler mode instead of port listening + /** Persist router changes back into storage when a storage manager is available. */ + persistRoutes?: boolean; // Authentication options auth?: { @@ -92,6 +96,8 @@ export interface IUnifiedEmailServerOptions { // Outbound settings outbound?: { + /** Override the SMTP identity used for outbound delivery. Defaults to `hostname`. */ + hostname?: string; maxConnections?: number; connectionTimeout?: number; socketTimeout?: number; @@ -99,6 +105,9 @@ export interface IUnifiedEmailServerOptions { defaultFrom?: string; }; + // Delivery queue + queue?: IQueueOptions; + // Rate limiting (global limits, can be overridden per domain) rateLimits?: IHierarchicalRateLimits; } @@ -206,7 +215,7 @@ export class UnifiedEmailServer extends EventEmitter { // Initialize email router with routes and storage manager this.emailRouter = new EmailRouter(options.routes || [], { storageManager: dcRouter.storageManager, - persistChanges: true + persistChanges: options.persistRoutes ?? hasStorageManagerMethods(dcRouter.storageManager, ['get', 'set']) }); // Initialize rate limiter @@ -226,7 +235,8 @@ export class UnifiedEmailServer extends EventEmitter { storageType: 'memory', // Default to memory storage maxRetries: 3, baseRetryDelay: 300000, // 5 minutes - maxRetryDelay: 3600000 // 1 hour + maxRetryDelay: 3600000, // 1 hour + ...options.queue, }; this.deliveryQueue = new UnifiedDeliveryQueue(queueOptions); @@ -277,6 +287,14 @@ export class UnifiedEmailServer extends EventEmitter { // We'll create the SMTP servers during the start() method } + private getAdvertisedHostname(): string { + return this.options.hostname; + } + + private getOutboundHostname(): string { + return this.options.outbound?.hostname || this.options.hostname; + } + /** * Send an outbound email via the Rust SMTP client. * Uses connection pooling in the Rust binary for efficiency. @@ -314,7 +332,7 @@ export class UnifiedEmailServer extends EventEmitter { host, port, secure: port === 465, - domain: this.options.hostname, + domain: this.getOutboundHostname(), auth: options?.auth, email: outboundEmail, dkim, @@ -455,7 +473,7 @@ export class UnifiedEmailServer extends EventEmitter { const securePort = (this.options.ports as number[]).find(p => p === 465); const started = await this.rustBridge.startSmtpServer({ - hostname: this.options.hostname, + hostname: this.getAdvertisedHostname(), ports: smtpPorts, securePort: securePort, tlsCertPem, @@ -518,6 +536,9 @@ export class UnifiedEmailServer extends EventEmitter { logger.log('info', 'Email delivery queue shut down'); } + this.bounceManager.stop(); + logger.log('info', 'Bounce manager stopped'); + // Close all Rust SMTP client connection pools try { await this.rustBridge.closeSmtpPool(); @@ -973,6 +994,10 @@ export class UnifiedEmailServer extends EventEmitter { this.emailRouter.updateRoutes(routes); } + public getEmailRoutes(): IEmailRoute[] { + return this.emailRouter.getRoutes(); + } + /** * Get server statistics */ @@ -980,6 +1005,22 @@ export class UnifiedEmailServer extends EventEmitter { return { ...this.stats }; } + public getQueueStats(): IQueueStats { + return this.deliveryQueue.getStats(); + } + + public getQueueItems(): IQueueItem[] { + return this.deliveryQueue.listItems(); + } + + public getQueueItem(id: string): IQueueItem | undefined { + return this.deliveryQueue.getItem(id); + } + + public getDeliveryStats(): IDeliveryStats { + return this.deliverySystem.getStats(); + } + /** * Get domain registry */ @@ -1039,11 +1080,10 @@ export class UnifiedEmailServer extends EventEmitter { // Sign with DKIM if configured if (mode === 'mta' && route?.action.options?.mtaOptions?.dkimSign) { const domain = email.from.split('@')[1]; - await this.dkimManager.handleDkimSigning(email, domain, route.action.options.mtaOptions.dkimOptions?.keySelector || 'mta'); + await this.dkimManager.handleDkimSigning(email, domain, route.action.options.mtaOptions.dkimOptions?.keySelector || 'default'); } - const id = plugins.uuid.v4(); - await this.deliveryQueue.enqueue(email, mode, route); + const id = await this.deliveryQueue.enqueue(email, mode, route); logger.log('info', `Email queued with ID: ${id}`); return id; diff --git a/ts/mail/security/classes.dkimcreator.ts b/ts/mail/security/classes.dkimcreator.ts index 2cccec4..9b93b58 100644 --- a/ts/mail/security/classes.dkimcreator.ts +++ b/ts/mail/security/classes.dkimcreator.ts @@ -1,5 +1,6 @@ import * as plugins from '../../plugins.js'; import * as paths from '../../paths.js'; +import { type IStorageManagerLike, hasStorageManagerMethods } from '../interfaces.storage.js'; import { Email } from '../core/classes.email.js'; // MtaService reference removed @@ -24,13 +25,47 @@ export interface IDkimKeyMetadata { export class DKIMCreator { private keysDir: string; - private storageManager?: any; // StorageManager instance + private storageManager?: IStorageManagerLike; - constructor(keysDir = paths.keysDir, storageManager?: any) { + constructor(keysDir = paths.keysDir, storageManager?: IStorageManagerLike) { this.keysDir = keysDir; this.storageManager = storageManager; } + private async writeKeyPairToFilesystem( + privateKeyPath: string, + publicKeyPath: string, + privateKey: string, + publicKey: string, + ): Promise { + await Promise.all([writeFile(privateKeyPath, privateKey), writeFile(publicKeyPath, publicKey)]); + } + + private async storeLegacyKeysToStorage(domain: string, privateKey: string, publicKey: string): Promise { + if (!hasStorageManagerMethods(this.storageManager, ['set'])) { + return; + } + await Promise.all([ + this.storageManager.set(`/email/dkim/${domain}/private.key`, privateKey), + this.storageManager.set(`/email/dkim/${domain}/public.key`, publicKey), + ]); + } + + private async storeSelectorKeysToStorage( + domain: string, + selector: string, + privateKey: string, + publicKey: string, + ): Promise { + if (!hasStorageManagerMethods(this.storageManager, ['set'])) { + return; + } + await Promise.all([ + this.storageManager.set(`/email/dkim/${domain}/${selector}/private.key`, privateKey), + this.storageManager.set(`/email/dkim/${domain}/${selector}/public.key`, publicKey), + ]); + } + public async getKeyPathsForDomain(domainArg: string): Promise { return { privateKeyPath: plugins.path.join(this.keysDir, `${domainArg}-private.pem`), @@ -51,6 +86,20 @@ export class DKIMCreator { } } + public async handleDKIMKeysForSelector(domainArg: string, selector: string = 'default', keySize: number = 2048): Promise { + if (selector === 'default') { + await this.handleDKIMKeysForDomain(domainArg); + return; + } + + try { + await this.readDKIMKeysForSelector(domainArg, selector); + } catch { + console.log(`No DKIM keys found for ${domainArg}/${selector}. Generating...`); + await this.createAndStoreDKIMKeysForSelector(domainArg, selector, keySize); + } + } + public async handleDKIMKeysForEmail(email: Email): Promise { const domain = email.from.split('@')[1]; await this.handleDKIMKeysForDomain(domain); @@ -59,7 +108,7 @@ export class DKIMCreator { // Read DKIM keys - always use storage manager, migrate from filesystem if needed public async readDKIMKeys(domainArg: string): Promise<{ privateKey: string; publicKey: string }> { // Try to read from storage manager first - if (this.storageManager) { + if (hasStorageManagerMethods(this.storageManager, ['get', 'set'])) { try { const [privateKey, publicKey] = await Promise.all([ this.storageManager.get(`/email/dkim/${domainArg}/private.key`), @@ -87,10 +136,7 @@ export class DKIMCreator { // Migrate to storage manager console.log(`Migrating DKIM keys for ${domainArg} from filesystem to StorageManager`); - await Promise.all([ - this.storageManager.set(`/email/dkim/${domainArg}/private.key`, privateKey), - this.storageManager.set(`/email/dkim/${domainArg}/public.key`, publicKey) - ]); + await this.storeLegacyKeysToStorage(domainArg, privateKey, publicKey); return { privateKey, publicKey }; } catch (error) { @@ -116,9 +162,9 @@ export class DKIMCreator { } // Create an RSA DKIM key pair - changed to public for API access - public async createDKIMKeys(): Promise<{ privateKey: string; publicKey: string }> { + public async createDKIMKeys(keySize: number = 2048): Promise<{ privateKey: string; publicKey: string }> { const { privateKey, publicKey } = await generateKeyPair('rsa', { - modulusLength: 2048, + modulusLength: keySize, publicKeyEncoding: { type: 'spki', format: 'pem' }, privateKeyEncoding: { type: 'pkcs1', format: 'pem' }, }); @@ -136,75 +182,58 @@ export class DKIMCreator { return { privateKey, publicKey }; } - // Store a DKIM key pair - uses storage manager if available, else disk - public async storeDKIMKeys( - privateKey: string, - publicKey: string, - privateKeyPath: string, - publicKeyPath: string - ): Promise { - // Store in storage manager if available - if (this.storageManager) { - // Extract domain from path (e.g., /path/to/keys/example.com-private.pem -> example.com) - const match = privateKeyPath.match(/\/([^\/]+)-private\.pem$/); - if (match) { - const domain = match[1]; - await Promise.all([ - this.storageManager.set(`/email/dkim/${domain}/private.key`, privateKey), - this.storageManager.set(`/email/dkim/${domain}/public.key`, publicKey) - ]); - } - } - - // Also store to filesystem for backward compatibility - await Promise.all([writeFile(privateKeyPath, privateKey), writeFile(publicKeyPath, publicKey)]); - } - // Create a DKIM key pair and store it to disk - changed to public for API access - public async createAndStoreDKIMKeys(domain: string): Promise { - const { privateKey, publicKey } = await this.createDKIMKeys(); + public async createAndStoreDKIMKeys(domain: string, keySize: number = 2048): Promise { + const { privateKey, publicKey } = await this.createDKIMKeys(keySize); const keyPaths = await this.getKeyPathsForDomain(domain); - await this.storeDKIMKeys( - privateKey, - publicKey, - keyPaths.privateKeyPath, - keyPaths.publicKeyPath - ); + await this.storeLegacyKeysToStorage(domain, privateKey, publicKey); + await this.writeKeyPairToFilesystem(keyPaths.privateKeyPath, keyPaths.publicKeyPath, privateKey, publicKey); + await this.saveKeyMetadata({ + domain, + selector: 'default', + createdAt: Date.now(), + keySize, + }); console.log(`DKIM keys for ${domain} created and stored.`); } + public async createAndStoreDKIMKeysForSelector( + domain: string, + selector: string, + keySize: number = 2048, + ): Promise { + if (selector === 'default') { + await this.createAndStoreDKIMKeys(domain, keySize); + return; + } + + const { privateKey, publicKey } = await this.createDKIMKeys(keySize); + const keyPaths = await this.getKeyPathsForSelector(domain, selector); + await this.storeSelectorKeysToStorage(domain, selector, privateKey, publicKey); + await this.writeKeyPairToFilesystem(keyPaths.privateKeyPath, keyPaths.publicKeyPath, privateKey, publicKey); + await this.saveKeyMetadata({ + domain, + selector, + createdAt: Date.now(), + keySize, + }); + console.log(`DKIM keys for ${domain}/${selector} created and stored.`); + } + // Changed to public for API access - public async getDNSRecordForDomain(domainArg: string): Promise { - await this.handleDKIMKeysForDomain(domainArg); - const keys = await this.readDKIMKeys(domainArg); - - // Remove the PEM header and footer and newlines - const pemHeader = '-----BEGIN PUBLIC KEY-----'; - const pemFooter = '-----END PUBLIC KEY-----'; - const keyContents = keys.publicKey - .replace(pemHeader, '') - .replace(pemFooter, '') - .replace(/\n/g, ''); - - // Detect key type from PEM header - const keyAlgo = keys.privateKey.includes('ED25519') || keys.publicKey.length < 200 ? 'ed25519' : 'rsa'; - - // Now generate the DKIM DNS TXT record - const dnsRecordValue = `v=DKIM1; h=sha256; k=${keyAlgo}; p=${keyContents}`; - - return { - name: `mta._domainkey.${domainArg}`, - type: 'TXT', - dnsSecEnabled: null, - value: dnsRecordValue, - }; + public async getDNSRecordForDomain( + domainArg: string, + selector: string = 'default', + ): Promise { + await this.handleDKIMKeysForSelector(domainArg, selector); + return this.getDNSRecordForSelector(domainArg, selector); } /** * Get DKIM key metadata for a domain */ private async getKeyMetadata(domain: string, selector: string = 'default'): Promise { - if (!this.storageManager) { + if (!hasStorageManagerMethods(this.storageManager, ['get'])) { return null; } @@ -222,7 +251,7 @@ export class DKIMCreator { * Save DKIM key metadata */ private async saveKeyMetadata(metadata: IDkimKeyMetadata): Promise { - if (!this.storageManager) { + if (!hasStorageManagerMethods(this.storageManager, ['set'])) { return; } @@ -259,30 +288,16 @@ export class DKIMCreator { const newSelector = `key${now.getFullYear()}${String(now.getMonth() + 1).padStart(2, '0')}`; // Create new keys with custom key size - const { privateKey, publicKey } = await generateKeyPair('rsa', { - modulusLength: keySize, - publicKeyEncoding: { type: 'spki', format: 'pem' }, - privateKeyEncoding: { type: 'pkcs1', format: 'pem' }, - }); + const { privateKey, publicKey } = await this.createDKIMKeys(keySize); // Store new keys with new selector const newKeyPaths = await this.getKeyPathsForSelector(domain, newSelector); // Store in storage manager if available - if (this.storageManager) { - await Promise.all([ - this.storageManager.set(`/email/dkim/${domain}/${newSelector}/private.key`, privateKey), - this.storageManager.set(`/email/dkim/${domain}/${newSelector}/public.key`, publicKey) - ]); - } + await this.storeSelectorKeysToStorage(domain, newSelector, privateKey, publicKey); // Also store to filesystem - await this.storeDKIMKeys( - privateKey, - publicKey, - newKeyPaths.privateKeyPath, - newKeyPaths.publicKeyPath - ); + await this.writeKeyPairToFilesystem(newKeyPaths.privateKeyPath, newKeyPaths.publicKeyPath, privateKey, publicKey); // Save metadata for new keys const metadata: IDkimKeyMetadata = { @@ -320,7 +335,7 @@ export class DKIMCreator { */ public async readDKIMKeysForSelector(domain: string, selector: string): Promise<{ privateKey: string; publicKey: string }> { // Try to read from storage manager first - if (this.storageManager) { + if (hasStorageManagerMethods(this.storageManager, ['get', 'set'])) { try { const [privateKey, publicKey] = await Promise.all([ this.storageManager.get(`/email/dkim/${domain}/${selector}/private.key`), @@ -330,6 +345,10 @@ export class DKIMCreator { if (privateKey && publicKey) { return { privateKey, publicKey }; } + + if (selector === 'default') { + return await this.readDKIMKeys(domain); + } } catch (error) { // Fall through to migration check } @@ -347,10 +366,7 @@ export class DKIMCreator { // Migrate to storage manager console.log(`Migrating DKIM keys for ${domain}/${selector} from filesystem to StorageManager`); - await Promise.all([ - this.storageManager.set(`/email/dkim/${domain}/${selector}/private.key`, privateKey), - this.storageManager.set(`/email/dkim/${domain}/${selector}/public.key`, publicKey) - ]); + await this.storeSelectorKeysToStorage(domain, selector, privateKey, publicKey); return { privateKey, publicKey }; } catch (error) { @@ -361,6 +377,9 @@ export class DKIMCreator { } } else { // No storage manager, use filesystem directly + if (selector === 'default') { + return this.readDKIMKeys(domain); + } const keyPaths = await this.getKeyPathsForSelector(domain, selector); const [privateKeyBuffer, publicKeyBuffer] = await Promise.all([ readFile(keyPaths.privateKeyPath), @@ -406,7 +425,8 @@ export class DKIMCreator { * Clean up old DKIM keys after grace period */ public async cleanupOldKeys(domain: string, gracePeriodDays: number = 30): Promise { - if (!this.storageManager) { + if (!hasStorageManagerMethods(this.storageManager, ['get', 'list', 'delete'])) { + console.log(`StorageManager for ${domain} does not support list/delete. Skipping DKIM cleanup.`); return; } @@ -436,7 +456,11 @@ export class DKIMCreator { console.warn(`Failed to delete old key files: ${error.message}`); } - // Delete metadata + // Delete selector-specific storage keys and metadata + await Promise.all([ + this.storageManager.delete(`/email/dkim/${domain}/${metadata.selector}/private.key`), + this.storageManager.delete(`/email/dkim/${domain}/${metadata.selector}/public.key`), + ]); await this.storageManager.delete(key); } } @@ -444,4 +468,4 @@ export class DKIMCreator { } } } -} \ No newline at end of file +} diff --git a/ts/security/classes.ipreputationchecker.ts b/ts/security/classes.ipreputationchecker.ts index aaf5cca..60a2e62 100644 --- a/ts/security/classes.ipreputationchecker.ts +++ b/ts/security/classes.ipreputationchecker.ts @@ -1,6 +1,7 @@ import * as plugins from '../plugins.js'; import * as paths from '../paths.js'; import { logger } from '../logger.js'; +import { hasStorageManagerMethods, type IStorageManagerLike } from '../mail/interfaces.storage.js'; import { SecurityLogger, SecurityLogLevel, SecurityEventType } from './classes.securitylogger.js'; import { RustSecurityBridge } from './classes.rustsecuritybridge.js'; import { LRUCache } from 'lru-cache'; @@ -66,7 +67,7 @@ export class IPReputationChecker { private static instance: IPReputationChecker; private reputationCache: LRUCache; private options: Required; - private storageManager?: any; + private storageManager?: IStorageManagerLike; private static readonly DEFAULT_OPTIONS: Required = { maxCacheSize: 10000, @@ -80,7 +81,7 @@ export class IPReputationChecker { enableIPInfo: true }; - constructor(options: IIPReputationOptions = {}, storageManager?: any) { + constructor(options: IIPReputationOptions = {}, storageManager?: IStorageManagerLike) { this.options = { ...IPReputationChecker.DEFAULT_OPTIONS, ...options @@ -100,7 +101,7 @@ export class IPReputationChecker { } } - public static getInstance(options: IIPReputationOptions = {}, storageManager?: any): IPReputationChecker { + public static getInstance(options: IIPReputationOptions = {}, storageManager?: IStorageManagerLike): IPReputationChecker { if (!IPReputationChecker.instance) { IPReputationChecker.instance = new IPReputationChecker(options, storageManager); } @@ -219,7 +220,7 @@ export class IPReputationChecker { const cacheData = JSON.stringify(entries); - if (this.storageManager) { + if (hasStorageManagerMethods(this.storageManager, ['set'])) { await this.storageManager.set('/security/ip-reputation-cache.json', cacheData); logger.log('info', `Saved ${entries.length} IP reputation cache entries to StorageManager`); } else { @@ -239,7 +240,7 @@ export class IPReputationChecker { let cacheData: string | null = null; let fromFilesystem = false; - if (this.storageManager) { + if (hasStorageManagerMethods(this.storageManager, ['get', 'set'])) { try { cacheData = await this.storageManager.get('/security/ip-reputation-cache.json'); @@ -302,7 +303,7 @@ export class IPReputationChecker { } } - public updateStorageManager(storageManager: any): void { + public updateStorageManager(storageManager: IStorageManagerLike): void { this.storageManager = storageManager; logger.log('info', 'IPReputationChecker storage manager updated');