feat(cache): add persistent smartdata-backed cache with LocalTsmDb, cache cleaner, and DcRouter integration

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@serve.zone/dcrouter',
-  version: '4.0.0',
+  version: '4.1.0',
   description: 'A multifaceted routing service handling mail and SMS delivery functions.'
 }

ts/cache/classes.cache.cleaner.ts

@@ -98,24 +98,20 @@ export class CacheCleaner {
     const results: { collection: string; deleted: number }[] = [];
 
     try {
-      // Clean CachedEmail documents
-      const emailsDeleted = await this.cleanCollection(CachedEmail, now);
+      // Clean each collection using smartdata's getInstances + delete pattern
+      const emailsDeleted = await this.cleanExpiredDocuments(CachedEmail, now);
       results.push({ collection: 'CachedEmail', deleted: emailsDeleted });
 
-      // Clean CachedIPReputation documents
-      const ipReputationDeleted = await this.cleanCollection(CachedIPReputation, now);
+      const ipReputationDeleted = await this.cleanExpiredDocuments(CachedIPReputation, now);
       results.push({ collection: 'CachedIPReputation', deleted: ipReputationDeleted });
 
-      // Clean CachedBounce documents
-      const bouncesDeleted = await this.cleanCollection(CachedBounce, now);
+      const bouncesDeleted = await this.cleanExpiredDocuments(CachedBounce, now);
       results.push({ collection: 'CachedBounce', deleted: bouncesDeleted });
 
-      // Clean CachedSuppression documents (but not permanent ones)
-      const suppressionDeleted = await this.cleanCollection(CachedSuppression, now);
+      const suppressionDeleted = await this.cleanExpiredDocuments(CachedSuppression, now);
       results.push({ collection: 'CachedSuppression', deleted: suppressionDeleted });
 
-      // Clean CachedDKIMKey documents
-      const dkimDeleted = await this.cleanCollection(CachedDKIMKey, now);
+      const dkimDeleted = await this.cleanExpiredDocuments(CachedDKIMKey, now);
       results.push({ collection: 'CachedDKIMKey', deleted: dkimDeleted });
 
       // Log results
@@ -137,17 +133,30 @@ export class CacheCleaner {
   }
 
   /**
-   * Clean expired documents from a specific collection
+   * Clean expired documents from a specific collection using smartdata API
    */
-  private async cleanCollection<T>(
-    documentClass: { deleteMany: (filter: any) => Promise<any> },
+  private async cleanExpiredDocuments<T extends { delete: () => Promise<void> }>(
+    documentClass: { getInstances: (filter: any) => Promise<T[]> },
     now: Date
   ): Promise<number> {
     try {
-      const result = await documentClass.deleteMany({
+      // Find all expired documents
+      const expiredDocs = await documentClass.getInstances({
         expiresAt: { $lt: now },
       });
-      return result?.deletedCount || 0;
+
+      // Delete each expired document
+      let deletedCount = 0;
+      for (const doc of expiredDocs) {
+        try {
+          await doc.delete();
+          deletedCount++;
+        } catch (deleteError) {
+          logger.log('warn', `Failed to delete expired document: ${deleteError.message}`);
+        }
+      }
+
+      return deletedCount;
     } catch (error) {
       logger.log('error', `Error cleaning collection: ${error.message}`);
       return 0;

ts/cache/classes.cached.document.ts

@@ -7,24 +7,27 @@ import * as plugins from '../plugins.js';
  * - Automatic timestamps (createdAt, lastAccessedAt)
  * - TTL/expiration support (expiresAt)
  * - Helper methods for TTL management
+ *
+ * NOTE: Subclasses MUST add @svDb() decorators to createdAt, expiresAt, and lastAccessedAt
+ * since decorators on abstract classes don't propagate correctly.
  */
 export abstract class CachedDocument<T extends CachedDocument<T>> extends plugins.smartdata.SmartDataDbDoc<T, T> {
   /**
    * Timestamp when the document was created
+   * NOTE: Subclasses must add @svDb() decorator
    */
-  @plugins.smartdata.svDb()
   public createdAt: Date = new Date();
 
   /**
    * Timestamp when the document expires and should be cleaned up
+   * NOTE: Subclasses must add @svDb() decorator
    */
-  @plugins.smartdata.svDb()
   public expiresAt: Date;
 
   /**
    * Timestamp of last access (for LRU-style eviction if needed)
+   * NOTE: Subclasses must add @svDb() decorator
    */
-  @plugins.smartdata.svDb()
   public lastAccessedAt: Date = new Date();
 
   /**

ts/cache/classes.cachedb.ts

@@ -69,19 +69,21 @@ export class CacheDb {
 
       // Create LocalTsmDb instance
       this.localTsmDb = new plugins.smartmongo.LocalTsmDb({
-        dbDir: this.options.storagePath,
+        folderPath: this.options.storagePath,
       });
 
-      // Start LocalTsmDb and get connection URI
-      await this.localTsmDb.start();
-      const mongoDescriptor = this.localTsmDb.mongoDescriptor;
+      // Start LocalTsmDb and get connection info
+      const connectionInfo = await this.localTsmDb.start();
 
       if (this.options.debug) {
-        logger.log('debug', `LocalTsmDb started with descriptor: ${JSON.stringify(mongoDescriptor)}`);
+        logger.log('debug', `LocalTsmDb started with URI: ${connectionInfo.connectionUri}`);
       }
 
-      // Initialize smartdata with the connection
-      this.smartdataDb = new plugins.smartdata.SmartdataDb(mongoDescriptor);
+      // Initialize smartdata with the connection URI
+      this.smartdataDb = new plugins.smartdata.SmartdataDb({
+        mongoDbUrl: connectionInfo.connectionUri,
+        mongoDbName: this.options.dbName,
+      });
       await this.smartdataDb.init();
 
       this.isStarted = true;

ts/cache/documents/classes.cached.bounce.ts

@@ -33,6 +33,16 @@ export type TBounceCategory =
  */
 @plugins.smartdata.Collection(() => getDb())
 export class CachedBounce extends CachedDocument<CachedBounce> {
+  // TTL fields from base class (decorators required on concrete class)
+  @plugins.smartdata.svDb()
+  public createdAt: Date = new Date();
+
+  @plugins.smartdata.svDb()
+  public expiresAt: Date = new Date(Date.now() + TTL.DAYS_30);
+
+  @plugins.smartdata.svDb()
+  public lastAccessedAt: Date = new Date();
+
   /**
    * Unique identifier for this bounce record
    */

ts/cache/documents/classes.cached.dkim.ts

@@ -15,6 +15,16 @@ const getDb = () => CacheDb.getInstance().getDb();
  */
 @plugins.smartdata.Collection(() => getDb())
 export class CachedDKIMKey extends CachedDocument<CachedDKIMKey> {
+  // TTL fields from base class (decorators required on concrete class)
+  @plugins.smartdata.svDb()
+  public createdAt: Date = new Date();
+
+  @plugins.smartdata.svDb()
+  public expiresAt: Date = new Date(Date.now() + TTL.DAYS_90);
+
+  @plugins.smartdata.svDb()
+  public lastAccessedAt: Date = new Date();
+
   /**
    * Composite key: domain:selector
    */

ts/cache/documents/classes.cached.email.ts

@@ -20,6 +20,16 @@ const getDb = () => CacheDb.getInstance().getDb();
  */
 @plugins.smartdata.Collection(() => getDb())
 export class CachedEmail extends CachedDocument<CachedEmail> {
+  // TTL fields from base class (decorators required on concrete class)
+  @plugins.smartdata.svDb()
+  public createdAt: Date = new Date();
+
+  @plugins.smartdata.svDb()
+  public expiresAt: Date = new Date(Date.now() + TTL.DAYS_30);
+
+  @plugins.smartdata.svDb()
+  public lastAccessedAt: Date = new Date();
+
   /**
    * Unique identifier for this email
    */

ts/cache/documents/classes.cached.ip.reputation.ts

@@ -30,6 +30,16 @@ export interface IIPReputationData {
  */
 @plugins.smartdata.Collection(() => getDb())
 export class CachedIPReputation extends CachedDocument<CachedIPReputation> {
+  // TTL fields from base class (decorators required on concrete class)
+  @plugins.smartdata.svDb()
+  public createdAt: Date = new Date();
+
+  @plugins.smartdata.svDb()
+  public expiresAt: Date = new Date(Date.now() + TTL.HOURS_24);
+
+  @plugins.smartdata.svDb()
+  public lastAccessedAt: Date = new Date();
+
   /**
    * IP address (unique identifier)
    */

ts/cache/documents/classes.cached.suppression.ts

@@ -27,6 +27,16 @@ export type TSuppressionReason =
  */
 @plugins.smartdata.Collection(() => getDb())
 export class CachedSuppression extends CachedDocument<CachedSuppression> {
+  // TTL fields from base class (decorators required on concrete class)
+  @plugins.smartdata.svDb()
+  public createdAt: Date = new Date();
+
+  @plugins.smartdata.svDb()
+  public expiresAt: Date = new Date(Date.now() + TTL.DAYS_30);
+
+  @plugins.smartdata.svDb()
+  public lastAccessedAt: Date = new Date();
+
   /**
    * Email address to suppress (unique identifier)
    */

ts/classes.dcrouter.ts

@@ -11,6 +11,8 @@ import { logger } from './logger.js';
 import { configureEmailStorage, configureEmailServer } from './mail/delivery/index.js';
 // Import storage manager
 import { StorageManager, type IStorageConfig } from './storage/index.js';
+// Import cache system
+import { CacheDb, CacheCleaner, type ICacheDbOptions } from './cache/index.js';
 
 import { OpsServer } from './opsserver/index.js';
 import { MetricsManager } from './monitoring/index.js';
@@ -111,6 +113,36 @@ export interface IDcRouterOptions {
   /** Storage configuration */
   storage?: IStorageConfig;
 
+  /**
+   * Cache database configuration using smartdata and LocalTsmDb
+   * Provides persistent caching for emails, IP reputation, bounces, etc.
+   */
+  cacheConfig?: {
+    /** Enable cache database (default: true) */
+    enabled?: boolean;
+    /** Storage path for TsmDB data (default: /etc/dcrouter/tsmdb) */
+    storagePath?: string;
+    /** Database name (default: dcrouter) */
+    dbName?: string;
+    /** Default TTL in days for cached items (default: 30) */
+    defaultTTLDays?: number;
+    /** Cleanup interval in hours (default: 1) */
+    cleanupIntervalHours?: number;
+    /** TTL configuration per data type (in days) */
+    ttlConfig?: {
+      /** Email cache TTL (default: 30 days) */
+      emails?: number;
+      /** IP reputation cache TTL (default: 1 day) */
+      ipReputation?: number;
+      /** Bounce records TTL (default: 30 days) */
+      bounces?: number;
+      /** DKIM keys TTL (default: 90 days) */
+      dkimKeys?: number;
+      /** Suppression list TTL (default: 30 days, can be permanent) */
+      suppression?: number;
+    };
+  };
+
   /**
    * RADIUS server configuration for network authentication
    * Enables MAC Authentication Bypass (MAB) and VLAN assignment
@@ -134,7 +166,7 @@ export interface PortProxyRuleContext {
 
 export class DcRouter {
   public options: IDcRouterOptions;
-  
+
   // Core services
   public smartProxy?: plugins.smartproxy.SmartProxy;
   public dnsServer?: plugins.smartdns.dnsServerMod.DnsServer;
@@ -143,11 +175,15 @@ export class DcRouter {
   public storageManager: StorageManager;
   public opsServer: OpsServer;
   public metricsManager?: MetricsManager;
-  
+
+  // Cache system (smartdata + LocalTsmDb)
+  public cacheDb?: CacheDb;
+  public cacheCleaner?: CacheCleaner;
+
   // TypedRouter for API endpoints
   public typedrouter = new plugins.typedrequest.TypedRouter();
-  
-  // Environment access 
+
+  // Environment access
   private qenv = new plugins.qenv.Qenv('./', '.nogit/');
   
   constructor(optionsArg: IDcRouterOptions) {
@@ -170,6 +206,11 @@ export class DcRouter {
     await this.opsServer.start();
 
     try {
+      // Initialize cache database if enabled (default: enabled)
+      if (this.options.cacheConfig?.enabled !== false) {
+        await this.setupCacheDb();
+      }
+
       // Initialize MetricsManager
       this.metricsManager = new MetricsManager(this);
       await this.metricsManager.start();
@@ -291,9 +332,45 @@ export class DcRouter {
       console.log(`   └─ Path: ${this.options.storage.fsPath || 'default'}`);
     }
 
+    // Cache database summary
+    if (this.cacheDb) {
+      console.log('\n🗄️  Cache Database (smartdata + LocalTsmDb):');
+      console.log(`   ├─ Storage: ${this.cacheDb.getStoragePath()}`);
+      console.log(`   ├─ Database: ${this.cacheDb.getDbName()}`);
+      console.log(`   └─ Cleaner: ${this.cacheCleaner?.isActive() ? 'Active' : 'Inactive'} (${(this.options.cacheConfig?.cleanupIntervalHours || 1)}h interval)`);
+    }
+
     console.log('\n✅ All services are running\n');
   }
   
+  /**
+   * Set up the cache database (smartdata + LocalTsmDb)
+   */
+  private async setupCacheDb(): Promise<void> {
+    logger.log('info', 'Setting up CacheDb...');
+
+    const cacheConfig = this.options.cacheConfig || {};
+
+    // Initialize CacheDb singleton
+    this.cacheDb = CacheDb.getInstance({
+      storagePath: cacheConfig.storagePath || '/etc/dcrouter/tsmdb',
+      dbName: cacheConfig.dbName || 'dcrouter',
+      debug: false,
+    });
+
+    await this.cacheDb.start();
+
+    // Start the cache cleaner
+    const cleanupIntervalMs = (cacheConfig.cleanupIntervalHours || 1) * 60 * 60 * 1000;
+    this.cacheCleaner = new CacheCleaner(this.cacheDb, {
+      intervalMs: cleanupIntervalMs,
+      verbose: false,
+    });
+    this.cacheCleaner.start();
+
+    logger.log('info', `CacheDb initialized at ${this.cacheDb.getStoragePath()}`);
+  }
+
   /**
    * Set up SmartProxy with direct configuration and automatic email routes
    */
@@ -600,10 +677,13 @@ export class DcRouter {
     console.log('Stopping DcRouter services...');
 
     await this.opsServer.stop();
-    
+
     try {
       // Stop all services in parallel for faster shutdown
       await Promise.all([
+        // Stop cache cleaner if running
+        this.cacheCleaner ? Promise.resolve(this.cacheCleaner.stop()) : Promise.resolve(),
+
         // Stop metrics manager if running
         this.metricsManager ? this.metricsManager.stop().catch(err => console.error('Error stopping MetricsManager:', err)) : Promise.resolve(),
 
@@ -623,7 +703,12 @@ export class DcRouter {
           this.radiusServer.stop().catch(err => console.error('Error stopping RADIUS server:', err)) :
           Promise.resolve()
       ]);
-      
+
+      // Stop cache database after other services (they may need it during shutdown)
+      if (this.cacheDb) {
+        await this.cacheDb.stop().catch(err => console.error('Error stopping CacheDb:', err));
+      }
+
       console.log('All DcRouter services stopped');
     } catch (error) {
       console.error('Error during DcRouter shutdown:', error);