diff --git a/changelog.md b/changelog.md index 025619a..8f51d82 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,17 @@ # Changelog +## 2026-02-10 - 4.1.0 - feat(cache) +add persistent smartdata-backed cache with LocalTsmDb, cache cleaner, and DcRouter integration + +- Introduce CacheDb and CacheCleaner using @push.rocks/smartdata and @push.rocks/smartmongo (LocalTsmDb) for persistent caching +- Integrate cache initialization, console summary, and graceful shutdown into DcRouter (options.cacheConfig and setupCacheDb()) +- Require svDb() decorators on concrete cache document classes; remove decorators from the abstract CachedDocument base class +- Switch CacheCleaner to smartdata getInstances() + per-document delete() instead of deleteMany +- Adapt to LocalTsmDb API changes (folderPath option and start() returning connectionUri) and initialize SmartdataDb with mongoDbUrl/mongoDbName +- Remove experimentalDecorators and emitDecoratorMetadata from tsconfig to use TC39 Stage 3 decorators (smartdata v7+ compatibility) +- Add package.json exports mapping (remove main/typings entries) to expose dist entry points +- Add README documentation for the Smartdata Cache System and configuration/usage examples + ## 2026-02-03 - 4.0.0 - BREAKING CHANGE(config) convert configuration management to read-only; remove updateConfiguration endpoint and client-side editing diff --git a/package.json b/package.json index 7caf284..6c1ffde 100644 --- a/package.json +++ b/package.json @@ -3,9 +3,11 @@ "private": false, "version": "4.0.0", "description": "A multifaceted routing service handling mail and SMS delivery functions.", - "main": "dist_ts/index.js", - "typings": "dist_ts/index.d.ts", "type": "module", + "exports": { + ".": "./dist_ts/index.js", + "./interfaces": "./dist_ts_interfaces/index.js" + }, "author": "Task Venture Capital GmbH", "license": "MIT", "scripts": { diff --git a/readme.hints.md b/readme.hints.md index 52a2e3b..252a338 100644 --- a/readme.hints.md +++ b/readme.hints.md @@ -1316,4 +1316,87 @@ The configuration UI has been converted from an editable interface to a read-onl - Numbers formatted with locale separators - Byte sizes auto-formatted (B, KB, MB, GB) - Time values shown with "seconds" suffix -- Nested objects with visual indentation \ No newline at end of file +- Nested objects with visual indentation + +## Smartdata Cache System (2026-02-03) + +### Overview +DcRouter now uses smartdata + LocalTsmDb for persistent caching. Data is stored at `/etc/dcrouter/tsmdb`. + +### Technology Stack +| Layer | Package | Purpose | +|-------|---------|---------| +| ORM | `@push.rocks/smartdata` | Document classes, decorators, queries | +| Database | `@push.rocks/smartmongo` (LocalTsmDb) | Embedded TsmDB via Unix socket | + +### TC39 Decorators +The project uses TC39 Stage 3 decorators (not experimental decorators). The tsconfig was updated: +- Removed `experimentalDecorators: true` +- Removed `emitDecoratorMetadata: true` + +This is required for smartdata v7+ compatibility. + +### Cache Document Classes +Located in `ts/cache/documents/`: + +| Class | Purpose | Default TTL | +|-------|---------|-------------| +| `CachedEmail` | Email queue items | 30 days | +| `CachedIPReputation` | IP reputation lookups | 24 hours | +| `CachedBounce` | Bounce records | 30 days | +| `CachedSuppression` | Suppression list | 30 days / permanent | +| `CachedDKIMKey` | DKIM key pairs | 90 days | + +### Usage Pattern +```typescript +// Document classes use smartdata decorators +@plugins.smartdata.Collection(() => getDb()) +export class CachedEmail extends CachedDocument { + @plugins.smartdata.svDb() + public createdAt: Date = new Date(); + + @plugins.smartdata.svDb() + public expiresAt: Date = new Date(Date.now() + TTL.DAYS_30); + + @plugins.smartdata.unI() + @plugins.smartdata.svDb() + public id: string; + // ... +} + +// Query examples +const email = await CachedEmail.getInstance({ id: 'abc123' }); +const pending = await CachedEmail.getInstances({ status: 'pending' }); +await email.save(); +await email.delete(); +``` + +### Configuration +```typescript +const dcRouter = new DcRouter({ + cacheConfig: { + enabled: true, + storagePath: '/etc/dcrouter/tsmdb', + dbName: 'dcrouter', + cleanupIntervalHours: 1, + ttlConfig: { + emails: 30, // days + ipReputation: 1, // days + bounces: 30, // days + dkimKeys: 90, // days + suppression: 30 // days + } + } +}); +``` + +### Cache Cleaner +- Runs hourly by default (configurable via `cleanupIntervalHours`) +- Finds and deletes documents where `expiresAt < now()` +- Uses smartdata's `getInstances()` + `delete()` pattern + +### Key Files +- `ts/cache/classes.cachedb.ts` - CacheDb singleton wrapper +- `ts/cache/classes.cached.document.ts` - Base class with TTL support +- `ts/cache/classes.cache.cleaner.ts` - Periodic cleanup service +- `ts/cache/documents/*.ts` - Document class definitions \ No newline at end of file diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index 4640a75..e08957f 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@serve.zone/dcrouter', - version: '4.0.0', + version: '4.1.0', description: 'A multifaceted routing service handling mail and SMS delivery functions.' } diff --git a/ts/cache/classes.cache.cleaner.ts b/ts/cache/classes.cache.cleaner.ts index 4e17b52..61407a8 100644 --- a/ts/cache/classes.cache.cleaner.ts +++ b/ts/cache/classes.cache.cleaner.ts @@ -98,24 +98,20 @@ export class CacheCleaner { const results: { collection: string; deleted: number }[] = []; try { - // Clean CachedEmail documents - const emailsDeleted = await this.cleanCollection(CachedEmail, now); + // Clean each collection using smartdata's getInstances + delete pattern + const emailsDeleted = await this.cleanExpiredDocuments(CachedEmail, now); results.push({ collection: 'CachedEmail', deleted: emailsDeleted }); - // Clean CachedIPReputation documents - const ipReputationDeleted = await this.cleanCollection(CachedIPReputation, now); + const ipReputationDeleted = await this.cleanExpiredDocuments(CachedIPReputation, now); results.push({ collection: 'CachedIPReputation', deleted: ipReputationDeleted }); - // Clean CachedBounce documents - const bouncesDeleted = await this.cleanCollection(CachedBounce, now); + const bouncesDeleted = await this.cleanExpiredDocuments(CachedBounce, now); results.push({ collection: 'CachedBounce', deleted: bouncesDeleted }); - // Clean CachedSuppression documents (but not permanent ones) - const suppressionDeleted = await this.cleanCollection(CachedSuppression, now); + const suppressionDeleted = await this.cleanExpiredDocuments(CachedSuppression, now); results.push({ collection: 'CachedSuppression', deleted: suppressionDeleted }); - // Clean CachedDKIMKey documents - const dkimDeleted = await this.cleanCollection(CachedDKIMKey, now); + const dkimDeleted = await this.cleanExpiredDocuments(CachedDKIMKey, now); results.push({ collection: 'CachedDKIMKey', deleted: dkimDeleted }); // Log results @@ -137,17 +133,30 @@ export class CacheCleaner { } /** - * Clean expired documents from a specific collection + * Clean expired documents from a specific collection using smartdata API */ - private async cleanCollection( - documentClass: { deleteMany: (filter: any) => Promise }, + private async cleanExpiredDocuments Promise }>( + documentClass: { getInstances: (filter: any) => Promise }, now: Date ): Promise { try { - const result = await documentClass.deleteMany({ + // Find all expired documents + const expiredDocs = await documentClass.getInstances({ expiresAt: { $lt: now }, }); - return result?.deletedCount || 0; + + // Delete each expired document + let deletedCount = 0; + for (const doc of expiredDocs) { + try { + await doc.delete(); + deletedCount++; + } catch (deleteError) { + logger.log('warn', `Failed to delete expired document: ${deleteError.message}`); + } + } + + return deletedCount; } catch (error) { logger.log('error', `Error cleaning collection: ${error.message}`); return 0; diff --git a/ts/cache/classes.cached.document.ts b/ts/cache/classes.cached.document.ts index b7a434c..8e14775 100644 --- a/ts/cache/classes.cached.document.ts +++ b/ts/cache/classes.cached.document.ts @@ -7,24 +7,27 @@ import * as plugins from '../plugins.js'; * - Automatic timestamps (createdAt, lastAccessedAt) * - TTL/expiration support (expiresAt) * - Helper methods for TTL management + * + * NOTE: Subclasses MUST add @svDb() decorators to createdAt, expiresAt, and lastAccessedAt + * since decorators on abstract classes don't propagate correctly. */ export abstract class CachedDocument> extends plugins.smartdata.SmartDataDbDoc { /** * Timestamp when the document was created + * NOTE: Subclasses must add @svDb() decorator */ - @plugins.smartdata.svDb() public createdAt: Date = new Date(); /** * Timestamp when the document expires and should be cleaned up + * NOTE: Subclasses must add @svDb() decorator */ - @plugins.smartdata.svDb() public expiresAt: Date; /** * Timestamp of last access (for LRU-style eviction if needed) + * NOTE: Subclasses must add @svDb() decorator */ - @plugins.smartdata.svDb() public lastAccessedAt: Date = new Date(); /** diff --git a/ts/cache/classes.cachedb.ts b/ts/cache/classes.cachedb.ts index 6f3e672..3296c96 100644 --- a/ts/cache/classes.cachedb.ts +++ b/ts/cache/classes.cachedb.ts @@ -69,19 +69,21 @@ export class CacheDb { // Create LocalTsmDb instance this.localTsmDb = new plugins.smartmongo.LocalTsmDb({ - dbDir: this.options.storagePath, + folderPath: this.options.storagePath, }); - // Start LocalTsmDb and get connection URI - await this.localTsmDb.start(); - const mongoDescriptor = this.localTsmDb.mongoDescriptor; + // Start LocalTsmDb and get connection info + const connectionInfo = await this.localTsmDb.start(); if (this.options.debug) { - logger.log('debug', `LocalTsmDb started with descriptor: ${JSON.stringify(mongoDescriptor)}`); + logger.log('debug', `LocalTsmDb started with URI: ${connectionInfo.connectionUri}`); } - // Initialize smartdata with the connection - this.smartdataDb = new plugins.smartdata.SmartdataDb(mongoDescriptor); + // Initialize smartdata with the connection URI + this.smartdataDb = new plugins.smartdata.SmartdataDb({ + mongoDbUrl: connectionInfo.connectionUri, + mongoDbName: this.options.dbName, + }); await this.smartdataDb.init(); this.isStarted = true; diff --git a/ts/cache/documents/classes.cached.bounce.ts b/ts/cache/documents/classes.cached.bounce.ts index bdcf785..15bcb88 100644 --- a/ts/cache/documents/classes.cached.bounce.ts +++ b/ts/cache/documents/classes.cached.bounce.ts @@ -33,6 +33,16 @@ export type TBounceCategory = */ @plugins.smartdata.Collection(() => getDb()) export class CachedBounce extends CachedDocument { + // TTL fields from base class (decorators required on concrete class) + @plugins.smartdata.svDb() + public createdAt: Date = new Date(); + + @plugins.smartdata.svDb() + public expiresAt: Date = new Date(Date.now() + TTL.DAYS_30); + + @plugins.smartdata.svDb() + public lastAccessedAt: Date = new Date(); + /** * Unique identifier for this bounce record */ diff --git a/ts/cache/documents/classes.cached.dkim.ts b/ts/cache/documents/classes.cached.dkim.ts index 7120b4d..27915f9 100644 --- a/ts/cache/documents/classes.cached.dkim.ts +++ b/ts/cache/documents/classes.cached.dkim.ts @@ -15,6 +15,16 @@ const getDb = () => CacheDb.getInstance().getDb(); */ @plugins.smartdata.Collection(() => getDb()) export class CachedDKIMKey extends CachedDocument { + // TTL fields from base class (decorators required on concrete class) + @plugins.smartdata.svDb() + public createdAt: Date = new Date(); + + @plugins.smartdata.svDb() + public expiresAt: Date = new Date(Date.now() + TTL.DAYS_90); + + @plugins.smartdata.svDb() + public lastAccessedAt: Date = new Date(); + /** * Composite key: domain:selector */ diff --git a/ts/cache/documents/classes.cached.email.ts b/ts/cache/documents/classes.cached.email.ts index c33e33d..fb31e67 100644 --- a/ts/cache/documents/classes.cached.email.ts +++ b/ts/cache/documents/classes.cached.email.ts @@ -20,6 +20,16 @@ const getDb = () => CacheDb.getInstance().getDb(); */ @plugins.smartdata.Collection(() => getDb()) export class CachedEmail extends CachedDocument { + // TTL fields from base class (decorators required on concrete class) + @plugins.smartdata.svDb() + public createdAt: Date = new Date(); + + @plugins.smartdata.svDb() + public expiresAt: Date = new Date(Date.now() + TTL.DAYS_30); + + @plugins.smartdata.svDb() + public lastAccessedAt: Date = new Date(); + /** * Unique identifier for this email */ diff --git a/ts/cache/documents/classes.cached.ip.reputation.ts b/ts/cache/documents/classes.cached.ip.reputation.ts index bcd28c3..b4df275 100644 --- a/ts/cache/documents/classes.cached.ip.reputation.ts +++ b/ts/cache/documents/classes.cached.ip.reputation.ts @@ -30,6 +30,16 @@ export interface IIPReputationData { */ @plugins.smartdata.Collection(() => getDb()) export class CachedIPReputation extends CachedDocument { + // TTL fields from base class (decorators required on concrete class) + @plugins.smartdata.svDb() + public createdAt: Date = new Date(); + + @plugins.smartdata.svDb() + public expiresAt: Date = new Date(Date.now() + TTL.HOURS_24); + + @plugins.smartdata.svDb() + public lastAccessedAt: Date = new Date(); + /** * IP address (unique identifier) */ diff --git a/ts/cache/documents/classes.cached.suppression.ts b/ts/cache/documents/classes.cached.suppression.ts index 3ccdd3e..5a239f1 100644 --- a/ts/cache/documents/classes.cached.suppression.ts +++ b/ts/cache/documents/classes.cached.suppression.ts @@ -27,6 +27,16 @@ export type TSuppressionReason = */ @plugins.smartdata.Collection(() => getDb()) export class CachedSuppression extends CachedDocument { + // TTL fields from base class (decorators required on concrete class) + @plugins.smartdata.svDb() + public createdAt: Date = new Date(); + + @plugins.smartdata.svDb() + public expiresAt: Date = new Date(Date.now() + TTL.DAYS_30); + + @plugins.smartdata.svDb() + public lastAccessedAt: Date = new Date(); + /** * Email address to suppress (unique identifier) */ diff --git a/ts/classes.dcrouter.ts b/ts/classes.dcrouter.ts index 8651a5c..6f8e1cf 100644 --- a/ts/classes.dcrouter.ts +++ b/ts/classes.dcrouter.ts @@ -11,6 +11,8 @@ import { logger } from './logger.js'; import { configureEmailStorage, configureEmailServer } from './mail/delivery/index.js'; // Import storage manager import { StorageManager, type IStorageConfig } from './storage/index.js'; +// Import cache system +import { CacheDb, CacheCleaner, type ICacheDbOptions } from './cache/index.js'; import { OpsServer } from './opsserver/index.js'; import { MetricsManager } from './monitoring/index.js'; @@ -111,6 +113,36 @@ export interface IDcRouterOptions { /** Storage configuration */ storage?: IStorageConfig; + /** + * Cache database configuration using smartdata and LocalTsmDb + * Provides persistent caching for emails, IP reputation, bounces, etc. + */ + cacheConfig?: { + /** Enable cache database (default: true) */ + enabled?: boolean; + /** Storage path for TsmDB data (default: /etc/dcrouter/tsmdb) */ + storagePath?: string; + /** Database name (default: dcrouter) */ + dbName?: string; + /** Default TTL in days for cached items (default: 30) */ + defaultTTLDays?: number; + /** Cleanup interval in hours (default: 1) */ + cleanupIntervalHours?: number; + /** TTL configuration per data type (in days) */ + ttlConfig?: { + /** Email cache TTL (default: 30 days) */ + emails?: number; + /** IP reputation cache TTL (default: 1 day) */ + ipReputation?: number; + /** Bounce records TTL (default: 30 days) */ + bounces?: number; + /** DKIM keys TTL (default: 90 days) */ + dkimKeys?: number; + /** Suppression list TTL (default: 30 days, can be permanent) */ + suppression?: number; + }; + }; + /** * RADIUS server configuration for network authentication * Enables MAC Authentication Bypass (MAB) and VLAN assignment @@ -134,7 +166,7 @@ export interface PortProxyRuleContext { export class DcRouter { public options: IDcRouterOptions; - + // Core services public smartProxy?: plugins.smartproxy.SmartProxy; public dnsServer?: plugins.smartdns.dnsServerMod.DnsServer; @@ -143,11 +175,15 @@ export class DcRouter { public storageManager: StorageManager; public opsServer: OpsServer; public metricsManager?: MetricsManager; - + + // Cache system (smartdata + LocalTsmDb) + public cacheDb?: CacheDb; + public cacheCleaner?: CacheCleaner; + // TypedRouter for API endpoints public typedrouter = new plugins.typedrequest.TypedRouter(); - - // Environment access + + // Environment access private qenv = new plugins.qenv.Qenv('./', '.nogit/'); constructor(optionsArg: IDcRouterOptions) { @@ -170,6 +206,11 @@ export class DcRouter { await this.opsServer.start(); try { + // Initialize cache database if enabled (default: enabled) + if (this.options.cacheConfig?.enabled !== false) { + await this.setupCacheDb(); + } + // Initialize MetricsManager this.metricsManager = new MetricsManager(this); await this.metricsManager.start(); @@ -291,9 +332,45 @@ export class DcRouter { console.log(` ā””ā”€ Path: ${this.options.storage.fsPath || 'default'}`); } + // Cache database summary + if (this.cacheDb) { + console.log('\nšŸ—„ļø Cache Database (smartdata + LocalTsmDb):'); + console.log(` ā”œā”€ Storage: ${this.cacheDb.getStoragePath()}`); + console.log(` ā”œā”€ Database: ${this.cacheDb.getDbName()}`); + console.log(` ā””ā”€ Cleaner: ${this.cacheCleaner?.isActive() ? 'Active' : 'Inactive'} (${(this.options.cacheConfig?.cleanupIntervalHours || 1)}h interval)`); + } + console.log('\nāœ… All services are running\n'); } + /** + * Set up the cache database (smartdata + LocalTsmDb) + */ + private async setupCacheDb(): Promise { + logger.log('info', 'Setting up CacheDb...'); + + const cacheConfig = this.options.cacheConfig || {}; + + // Initialize CacheDb singleton + this.cacheDb = CacheDb.getInstance({ + storagePath: cacheConfig.storagePath || '/etc/dcrouter/tsmdb', + dbName: cacheConfig.dbName || 'dcrouter', + debug: false, + }); + + await this.cacheDb.start(); + + // Start the cache cleaner + const cleanupIntervalMs = (cacheConfig.cleanupIntervalHours || 1) * 60 * 60 * 1000; + this.cacheCleaner = new CacheCleaner(this.cacheDb, { + intervalMs: cleanupIntervalMs, + verbose: false, + }); + this.cacheCleaner.start(); + + logger.log('info', `CacheDb initialized at ${this.cacheDb.getStoragePath()}`); + } + /** * Set up SmartProxy with direct configuration and automatic email routes */ @@ -600,10 +677,13 @@ export class DcRouter { console.log('Stopping DcRouter services...'); await this.opsServer.stop(); - + try { // Stop all services in parallel for faster shutdown await Promise.all([ + // Stop cache cleaner if running + this.cacheCleaner ? Promise.resolve(this.cacheCleaner.stop()) : Promise.resolve(), + // Stop metrics manager if running this.metricsManager ? this.metricsManager.stop().catch(err => console.error('Error stopping MetricsManager:', err)) : Promise.resolve(), @@ -623,7 +703,12 @@ export class DcRouter { this.radiusServer.stop().catch(err => console.error('Error stopping RADIUS server:', err)) : Promise.resolve() ]); - + + // Stop cache database after other services (they may need it during shutdown) + if (this.cacheDb) { + await this.cacheDb.stop().catch(err => console.error('Error stopping CacheDb:', err)); + } + console.log('All DcRouter services stopped'); } catch (error) { console.error('Error during DcRouter shutdown:', error); diff --git a/ts_web/00_commitinfo_data.ts b/ts_web/00_commitinfo_data.ts index 4640a75..e08957f 100644 --- a/ts_web/00_commitinfo_data.ts +++ b/ts_web/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@serve.zone/dcrouter', - version: '4.0.0', + version: '4.1.0', description: 'A multifaceted routing service handling mail and SMS delivery functions.' } diff --git a/tsconfig.json b/tsconfig.json index d76da25..70d3f34 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,8 +1,5 @@ { "compilerOptions": { - "experimentalDecorators": true, - "emitDecoratorMetadata": true, - "useDefineForClassFields": false, "target": "ES2022", "module": "NodeNext", "moduleResolution": "NodeNext",