diff --git a/changelog.md b/changelog.md index fcf3ce6..4cafa79 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,13 @@ # Changelog +## 2026-03-31 - 12.0.0 - BREAKING CHANGE(db) +replace StorageManager and CacheDb with a unified smartdata-backed database layer + +- introduces DcRouterDb with embedded LocalSmartDb or external MongoDB support via dbConfig +- migrates persisted routes, API tokens, VPN data, certificates, remote ingress, VLAN mappings, RADIUS accounting, and cache records to smartdata document classes +- removes StorageManager and CacheDb modules and renames configuration from cacheConfig to dbConfig +- updates certificate, security, remote ingress, VPN, and RADIUS components to read and write through document models + ## 2026-03-31 - 11.23.5 - fix(config) correct VPN mandatory flag default handling in route config manager diff --git a/readme.storage.md b/readme.storage.md index ad0ef6e..5b0994b 100644 --- a/readme.storage.md +++ b/readme.storage.md @@ -1,120 +1,84 @@ # DCRouter Storage Overview -DCRouter uses two complementary storage systems: **StorageManager** for configuration and state, and **CacheDb** for time-limited cached data. +DCRouter uses a **unified database layer** backed by `@push.rocks/smartdata` for all persistent data. All data is stored as typed document classes in a single database. -## StorageManager (Key-Value Store) +## Database Modes -A lightweight, pluggable key-value store for configuration, credentials, and runtime state. Data is persisted as flat JSON files on disk by default. - -### Default Path - -``` -~/.serve.zone/dcrouter/storage/ -``` - -Configurable via `options.storage.fsPath` or `options.baseDir`. - -### Backends - -```typescript -// Filesystem (default) -storage: { fsPath: '/var/lib/dcrouter/data' } - -// Custom (Redis, S3, etc.) -storage: { - readFunction: async (key) => await redis.get(key), - writeFunction: async (key, value) => await redis.set(key, value), -} - -// In-memory (omit storage config — data lost on restart) -``` - -### What's Stored - -| Prefix | Contents | Managed By | -|--------|----------|------------| -| `/vpn/server-keys` | VPN server Noise + WireGuard keypairs | `VpnManager` | -| `/vpn/clients/{clientId}` | VPN client registrations (keys, tags, description, assigned IP) | `VpnManager` | -| `/config-api/routes/{uuid}.json` | Programmatic routes (created via OpsServer API) | `RouteConfigManager` | -| `/config-api/tokens/{uuid}.json` | API tokens (hashed secrets, scopes, expiry) | `ApiTokenManager` | -| `/config-api/overrides/{routeName}.json` | Hardcoded route overrides (enable/disable) | `RouteConfigManager` | -| `/email/bounces/suppression-list.json` | Email bounce suppression list | `smartmta` | -| `/certs/*` | TLS certificates and ACME state | `SmartAcme` (via `StorageBackedCertManager`) | - -### API - -```typescript -// Read/write JSON -await storageManager.getJSON(key); -await storageManager.setJSON(key, value); - -// Raw string read/write -await storageManager.get(key); -await storageManager.set(key, value); - -// List keys by prefix -await storageManager.list('/vpn/clients/'); - -// Delete -await storageManager.delete(key); -``` - -## CacheDb (Embedded MongoDB) - -An embedded MongoDB-compatible database (via `@push.rocks/smartdb` + `@push.rocks/smartdata`) for cached data with automatic TTL-based cleanup. - -### Default Path +### Embedded Mode (default) +When no external MongoDB URL is provided, DCRouter starts an embedded `LocalSmartDb` (Rust-based MongoDB-compatible engine) via `@push.rocks/smartdb`. ``` ~/.serve.zone/dcrouter/tsmdb/ ``` -Configurable via `options.cacheConfig.storagePath`. - -### What's Cached - -| Document Type | Default TTL | Purpose | -|--------------|-------------|---------| -| `CachedEmail` | 30 days | Email metadata cache for dashboard display | -| `CachedIPReputation` | 1 day | IP reputation lookup results (DNSBL checks) | - -### Configuration +### External Mode +Connect to any MongoDB-compatible database by providing a connection URL. ```typescript -cacheConfig: { - enabled: true, // default: true - storagePath: '~/.serve.zone/dcrouter/tsmdb', // default - dbName: 'dcrouter', // default - cleanupIntervalHours: 1, // how often to purge expired records - ttlConfig: { - emails: 30, // days - ipReputation: 1, // days - bounces: 30, // days (reserved) - dkimKeys: 90, // days (reserved) - suppression: 30, // days (reserved) - }, +dbConfig: { + mongoDbUrl: 'mongodb://host:27017', + dbName: 'dcrouter', } ``` -### How It Works - -1. `CacheDb` starts a `LocalSmartDb` instance (embedded MongoDB process) -2. `smartdata` connects to it via Unix socket -3. Document classes (`CachedEmail`, `CachedIPReputation`) are decorated with `@Collection` and use `smartdata` ORM -4. `CacheCleaner` runs on a timer, purging records older than their configured TTL - -### Disabling - -For development or lightweight deployments, disable the cache to avoid starting a MongoDB process: +## Configuration ```typescript -cacheConfig: { enabled: false } +dbConfig: { + enabled: true, // default: true + mongoDbUrl: undefined, // default: embedded LocalSmartDb + storagePath: '~/.serve.zone/dcrouter/tsmdb', // default (embedded mode only) + dbName: 'dcrouter', // default + cleanupIntervalHours: 1, // TTL cleanup interval +} ``` -## When to Use Which +## Document Classes -| Use Case | System | Why | -|----------|--------|-----| -| VPN keys, API tokens, routes, certs | **StorageManager** | Small JSON blobs, key-value access, no queries needed | -| Email metadata, IP reputation | **CacheDb** | Time-series data, TTL expiry, potential for queries/aggregation | -| Runtime state (connected clients, metrics) | **Neither** | In-memory only, rebuilt on startup | +All data is stored as smartdata document classes in `ts/db/documents/`. + +| Document Class | Collection | Unique Key | Purpose | +|---|---|---|---| +| `StoredRouteDoc` | storedRoutes | `id` | Programmatic routes (created via API) | +| `RouteOverrideDoc` | routeOverrides | `routeName` | Hardcoded route enable/disable overrides | +| `ApiTokenDoc` | apiTokens | `id` | API tokens (hashed secrets, scopes, expiry) | +| `VpnServerKeysDoc` | vpnServerKeys | `configId` (singleton) | VPN server Noise + WireGuard keypairs | +| `VpnClientDoc` | vpnClients | `clientId` | VPN client registrations | +| `AcmeCertDoc` | acmeCerts | `domainName` | ACME certificates and keys | +| `ProxyCertDoc` | proxyCerts | `domain` | SmartProxy TLS certificates | +| `CertBackoffDoc` | certBackoff | `domain` | Per-domain cert provision backoff state | +| `RemoteIngressEdgeDoc` | remoteIngressEdges | `id` | Edge node registrations | +| `VlanMappingsDoc` | vlanMappings | `configId` (singleton) | MAC-to-VLAN mapping table | +| `AccountingSessionDoc` | accountingSessions | `sessionId` | RADIUS accounting sessions | +| `CachedEmail` | cachedEmails | `id` | Email metadata (TTL: 30 days) | +| `CachedIPReputation` | cachedIPReputation | `ipAddress` | IP reputation results (TTL: 24 hours) | + +## Architecture + +``` +DcRouterDb (singleton) + ├── LocalSmartDb (embedded, Rust) ─── or ─── External MongoDB + └── SmartdataDb (ORM) + └── @Collection(() => getDb()) + ├── StoredRouteDoc + ├── RouteOverrideDoc + ├── ApiTokenDoc + ├── VpnServerKeysDoc / VpnClientDoc + ├── AcmeCertDoc / ProxyCertDoc / CertBackoffDoc + ├── RemoteIngressEdgeDoc + ├── VlanMappingsDoc / AccountingSessionDoc + ├── CachedEmail (TTL) + └── CachedIPReputation (TTL) +``` + +### TTL Cleanup + +`CacheCleaner` runs on a configurable interval (default: 1 hour) and removes expired documents where `expiresAt < now()`. + +## Disabling + +For tests or lightweight deployments without persistence: + +```typescript +dbConfig: { enabled: false } +``` diff --git a/test/test.dcrouter.email.ts b/test/test.dcrouter.email.ts index 6e84d7a..6f82bb0 100644 --- a/test/test.dcrouter.email.ts +++ b/test/test.dcrouter.email.ts @@ -130,7 +130,7 @@ tap.test('DcRouter class - Email config with domains and routes', async () => { contactEmail: 'test@example.com' }, opsServerPort: 3104, - cacheConfig: { + dbConfig: { enabled: false, } }; diff --git a/test/test.dns-socket-handler.ts b/test/test.dns-socket-handler.ts index 4ec01ac..0284ba3 100644 --- a/test/test.dns-socket-handler.ts +++ b/test/test.dns-socket-handler.ts @@ -10,7 +10,7 @@ tap.test('should NOT instantiate DNS server when dnsNsDomains is not set', async routes: [] }, opsServerPort: 3100, - cacheConfig: { enabled: false } + dbConfig: { enabled: false } }); await dcRouter.start(); diff --git a/test/test.jwt-auth.ts b/test/test.jwt-auth.ts index be336ef..3a31794 100644 --- a/test/test.jwt-auth.ts +++ b/test/test.jwt-auth.ts @@ -10,7 +10,7 @@ tap.test('should start DCRouter with OpsServer', async () => { testDcRouter = new DcRouter({ // Minimal config for testing opsServerPort: 3102, - cacheConfig: { enabled: false }, + dbConfig: { enabled: false }, }); await testDcRouter.start(); diff --git a/test/test.opsserver-api.ts b/test/test.opsserver-api.ts index 0f856ad..c6318c6 100644 --- a/test/test.opsserver-api.ts +++ b/test/test.opsserver-api.ts @@ -10,7 +10,7 @@ tap.test('should start DCRouter with OpsServer', async () => { testDcRouter = new DcRouter({ // Minimal config for testing opsServerPort: 3101, - cacheConfig: { enabled: false }, + dbConfig: { enabled: false }, }); await testDcRouter.start(); diff --git a/test/test.protected-endpoint.ts b/test/test.protected-endpoint.ts index 6b2756d..72fa655 100644 --- a/test/test.protected-endpoint.ts +++ b/test/test.protected-endpoint.ts @@ -10,7 +10,7 @@ tap.test('should start DCRouter with OpsServer', async () => { testDcRouter = new DcRouter({ // Minimal config for testing opsServerPort: 3103, - cacheConfig: { enabled: false }, + dbConfig: { enabled: false }, }); await testDcRouter.start(); diff --git a/test/test.storagemanager.ts b/test/test.storagemanager.ts deleted file mode 100644 index 12ca4fb..0000000 --- a/test/test.storagemanager.ts +++ /dev/null @@ -1,289 +0,0 @@ -import { tap, expect } from '@git.zone/tstest/tapbundle'; -import * as plugins from '../ts/plugins.js'; -import * as paths from '../ts/paths.js'; -import { StorageManager } from '../ts/storage/classes.storagemanager.js'; -import { promises as fs } from 'fs'; -import * as path from 'path'; - -// Test data -const testData = { - string: 'Hello, World!', - json: { name: 'test', value: 42, nested: { data: true } }, - largeString: 'x'.repeat(10000) -}; - -tap.test('Storage Manager - Memory Backend', async () => { - // Create StorageManager without config (defaults to memory) - const storage = new StorageManager(); - - // Test basic get/set - await storage.set('/test/key', testData.string); - const value = await storage.get('/test/key'); - expect(value).toEqual(testData.string); - - // Test JSON helpers - await storage.setJSON('/test/json', testData.json); - const jsonValue = await storage.getJSON('/test/json'); - expect(jsonValue).toEqual(testData.json); - - // Test exists - expect(await storage.exists('/test/key')).toEqual(true); - expect(await storage.exists('/nonexistent')).toEqual(false); - - // Test delete - await storage.delete('/test/key'); - expect(await storage.exists('/test/key')).toEqual(false); - - // Test list - await storage.set('/items/1', 'one'); - await storage.set('/items/2', 'two'); - await storage.set('/other/3', 'three'); - - const items = await storage.list('/items'); - expect(items.length).toEqual(2); - expect(items).toContain('/items/1'); - expect(items).toContain('/items/2'); - - // Verify memory backend - expect(storage.getBackend()).toEqual('memory'); -}); - -tap.test('Storage Manager - Filesystem Backend', async () => { - const testDir = path.join(paths.dataDir, '.test-storage'); - - // Clean up test directory if it exists - try { - await fs.rm(testDir, { recursive: true, force: true }); - } catch {} - - // Create StorageManager with filesystem path - const storage = new StorageManager({ fsPath: testDir }); - - // Test basic operations - await storage.set('/test/file', testData.string); - const value = await storage.get('/test/file'); - expect(value).toEqual(testData.string); - - // Verify file exists on disk - const filePath = path.join(testDir, 'test', 'file'); - const fileExists = await fs.access(filePath).then(() => true).catch(() => false); - expect(fileExists).toEqual(true); - - // Test atomic writes (temp file should not exist) - const tempPath = filePath + '.tmp'; - const tempExists = await fs.access(tempPath).then(() => true).catch(() => false); - expect(tempExists).toEqual(false); - - // Test nested paths - await storage.set('/deeply/nested/path/to/file', testData.largeString); - const nestedValue = await storage.get('/deeply/nested/path/to/file'); - expect(nestedValue).toEqual(testData.largeString); - - // Test list with filesystem - await storage.set('/fs/items/a', 'alpha'); - await storage.set('/fs/items/b', 'beta'); - await storage.set('/fs/other/c', 'gamma'); - - // Filesystem backend now properly supports list - const fsItems = await storage.list('/fs/items'); - expect(fsItems.length).toEqual(2); // Should find both items - - // Clean up - await fs.rm(testDir, { recursive: true, force: true }); -}); - -tap.test('Storage Manager - Custom Function Backend', async () => { - // Create in-memory storage for custom functions - const customStore = new Map(); - - const storage = new StorageManager({ - readFunction: async (key: string) => { - return customStore.get(key) || null; - }, - writeFunction: async (key: string, value: string) => { - customStore.set(key, value); - } - }); - - // Test basic operations - await storage.set('/custom/key', testData.string); - expect(customStore.has('/custom/key')).toEqual(true); - - const value = await storage.get('/custom/key'); - expect(value).toEqual(testData.string); - - // Test that delete sets empty value (as per implementation) - await storage.delete('/custom/key'); - expect(customStore.get('/custom/key')).toEqual(''); - - // Verify custom backend (filesystem is implemented as custom backend internally) - expect(storage.getBackend()).toEqual('custom'); -}); - -tap.test('Storage Manager - Key Validation', async () => { - const storage = new StorageManager(); - - // Test key normalization - await storage.set('test/key', 'value1'); // Missing leading slash - const value1 = await storage.get('/test/key'); - expect(value1).toEqual('value1'); - - // Test dangerous path elements are removed - await storage.set('/test/../danger/key', 'value2'); - const value2 = await storage.get('/test/danger/key'); // .. is removed, not the whole path segment - expect(value2).toEqual('value2'); - - // Test multiple slashes are normalized - await storage.set('/test///multiple////slashes', 'value3'); - const value3 = await storage.get('/test/multiple/slashes'); - expect(value3).toEqual('value3'); - - // Test invalid keys throw errors - let emptyKeyError: Error | null = null; - try { - await storage.set('', 'value'); - } catch (error) { - emptyKeyError = error as Error; - } - expect(emptyKeyError).toBeTruthy(); - expect(emptyKeyError?.message).toEqual('Storage key must be a non-empty string'); - - let nullKeyError: Error | null = null; - try { - await storage.set(null as any, 'value'); - } catch (error) { - nullKeyError = error as Error; - } - expect(nullKeyError).toBeTruthy(); - expect(nullKeyError?.message).toEqual('Storage key must be a non-empty string'); -}); - -tap.test('Storage Manager - Concurrent Access', async () => { - const storage = new StorageManager(); - const promises: Promise[] = []; - - // Simulate concurrent writes - for (let i = 0; i < 100; i++) { - promises.push(storage.set(`/concurrent/key${i}`, `value${i}`)); - } - - await Promise.all(promises); - - // Verify all writes succeeded - for (let i = 0; i < 100; i++) { - const value = await storage.get(`/concurrent/key${i}`); - expect(value).toEqual(`value${i}`); - } - - // Test concurrent reads - const readPromises: Promise[] = []; - for (let i = 0; i < 100; i++) { - readPromises.push(storage.get(`/concurrent/key${i}`)); - } - - const results = await Promise.all(readPromises); - for (let i = 0; i < 100; i++) { - expect(results[i]).toEqual(`value${i}`); - } -}); - -tap.test('Storage Manager - Backend Priority', async () => { - const testDir = path.join(paths.dataDir, '.test-storage-priority'); - - // Test that custom functions take priority over fsPath - let warningLogged = false; - const originalWarn = console.warn; - console.warn = (message: string) => { - if (message.includes('Using custom read/write functions')) { - warningLogged = true; - } - }; - - const storage = new StorageManager({ - fsPath: testDir, - readFunction: async () => 'custom-value', - writeFunction: async () => {} - }); - - console.warn = originalWarn; - - expect(warningLogged).toEqual(true); - expect(storage.getBackend()).toEqual('custom'); // Custom functions take priority - - // Clean up - try { - await fs.rm(testDir, { recursive: true, force: true }); - } catch {} -}); - -tap.test('Storage Manager - Error Handling', async () => { - // Test filesystem errors - const storage = new StorageManager({ - readFunction: async () => { - throw new Error('Read error'); - }, - writeFunction: async () => { - throw new Error('Write error'); - } - }); - - // Read errors should return null - const value = await storage.get('/error/key'); - expect(value).toEqual(null); - - // Write errors should propagate - let writeError: Error | null = null; - try { - await storage.set('/error/key', 'value'); - } catch (error) { - writeError = error as Error; - } - expect(writeError).toBeTruthy(); - expect(writeError?.message).toEqual('Write error'); - - // Test JSON parse errors - const jsonStorage = new StorageManager({ - readFunction: async () => 'invalid json', - writeFunction: async () => {} - }); - - // Test JSON parse errors - let jsonError: Error | null = null; - try { - await jsonStorage.getJSON('/invalid/json'); - } catch (error) { - jsonError = error as Error; - } - expect(jsonError).toBeTruthy(); - expect(jsonError?.message).toContain('JSON'); -}); - -tap.test('Storage Manager - List Operations', async () => { - const storage = new StorageManager(); - - // Populate storage with hierarchical data - await storage.set('/app/config/database', 'db-config'); - await storage.set('/app/config/cache', 'cache-config'); - await storage.set('/app/data/users/1', 'user1'); - await storage.set('/app/data/users/2', 'user2'); - await storage.set('/app/logs/error.log', 'errors'); - - // List root - const rootItems = await storage.list('/'); - expect(rootItems.length).toBeGreaterThanOrEqual(5); - - // List specific paths - const configItems = await storage.list('/app/config'); - expect(configItems.length).toEqual(2); - expect(configItems).toContain('/app/config/database'); - expect(configItems).toContain('/app/config/cache'); - - const userItems = await storage.list('/app/data/users'); - expect(userItems.length).toEqual(2); - - // List non-existent path - const emptyList = await storage.list('/nonexistent/path'); - expect(emptyList.length).toEqual(0); -}); - -export default tap.start(); \ No newline at end of file diff --git a/test_watch/devserver.ts b/test_watch/devserver.ts index c078e61..8535d73 100644 --- a/test_watch/devserver.ts +++ b/test_watch/devserver.ts @@ -49,8 +49,8 @@ const devRouter = new DcRouter({ { clientId: 'admin-desktop', serverDefinedClientTags: ['admin'], description: 'Admin workstation' }, ], }, - // Disable cache/mongo for dev - cacheConfig: { enabled: false }, + // Disable db/mongo for dev + dbConfig: { enabled: false }, }); console.log('Starting DcRouter in development mode...'); diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index cefdcf4..23d447d 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@serve.zone/dcrouter', - version: '11.23.5', + version: '12.0.0', description: 'A multifaceted routing service handling mail and SMS delivery functions.' } diff --git a/ts/cache/classes.cachedb.ts b/ts/cache/classes.cachedb.ts deleted file mode 100644 index 0fe29b6..0000000 --- a/ts/cache/classes.cachedb.ts +++ /dev/null @@ -1,155 +0,0 @@ -import * as plugins from '../plugins.js'; -import { logger } from '../logger.js'; -import { defaultTsmDbPath } from '../paths.js'; - -/** - * Configuration options for CacheDb - */ -export interface ICacheDbOptions { - /** Base storage path for TsmDB data (default: ~/.serve.zone/dcrouter/tsmdb) */ - storagePath?: string; - /** Database name (default: dcrouter) */ - dbName?: string; - /** Enable debug logging */ - debug?: boolean; -} - -/** - * CacheDb - Wrapper around LocalSmartDb and smartdata - * - * Provides persistent caching using smartdata as the ORM layer - * and LocalSmartDb as the embedded database engine. - */ -export class CacheDb { - private static instance: CacheDb | null = null; - - private localSmartDb!: plugins.smartdb.LocalSmartDb; - private smartdataDb!: plugins.smartdata.SmartdataDb; - private options: Required; - private isStarted: boolean = false; - - constructor(options: ICacheDbOptions = {}) { - this.options = { - storagePath: options.storagePath || defaultTsmDbPath, - dbName: options.dbName || 'dcrouter', - debug: options.debug || false, - }; - } - - /** - * Get or create the singleton instance - */ - public static getInstance(options?: ICacheDbOptions): CacheDb { - if (!CacheDb.instance) { - CacheDb.instance = new CacheDb(options); - } - return CacheDb.instance; - } - - /** - * Reset the singleton instance (useful for testing) - */ - public static resetInstance(): void { - CacheDb.instance = null; - } - - /** - * Start the cache database - * - Initializes LocalSmartDb with file persistence - * - Connects smartdata to the LocalSmartDb via Unix socket - */ - public async start(): Promise { - if (this.isStarted) { - logger.log('warn', 'CacheDb already started'); - return; - } - - try { - // Ensure storage directory exists - await plugins.fsUtils.ensureDir(this.options.storagePath); - - // Create LocalSmartDb instance - this.localSmartDb = new plugins.smartdb.LocalSmartDb({ - folderPath: this.options.storagePath, - }); - - // Start LocalSmartDb and get connection info - const connectionInfo = await this.localSmartDb.start(); - - if (this.options.debug) { - logger.log('debug', `LocalSmartDb started with URI: ${connectionInfo.connectionUri}`); - } - - // Initialize smartdata with the connection URI - this.smartdataDb = new plugins.smartdata.SmartdataDb({ - mongoDbUrl: connectionInfo.connectionUri, - mongoDbName: this.options.dbName, - }); - await this.smartdataDb.init(); - - this.isStarted = true; - logger.log('info', `CacheDb started at ${this.options.storagePath}`); - } catch (error: unknown) { - logger.log('error', `Failed to start CacheDb: ${(error as Error).message}`); - throw error; - } - } - - /** - * Stop the cache database - */ - public async stop(): Promise { - if (!this.isStarted) { - return; - } - - try { - // Close smartdata connection - if (this.smartdataDb) { - await this.smartdataDb.close(); - } - - // Stop LocalSmartDb - if (this.localSmartDb) { - await this.localSmartDb.stop(); - } - - this.isStarted = false; - logger.log('info', 'CacheDb stopped'); - } catch (error: unknown) { - logger.log('error', `Error stopping CacheDb: ${(error as Error).message}`); - throw error; - } - } - - /** - * Get the smartdata database instance - */ - public getDb(): plugins.smartdata.SmartdataDb { - if (!this.isStarted) { - throw new Error('CacheDb not started. Call start() first.'); - } - return this.smartdataDb; - } - - /** - * Check if the database is ready - */ - public isReady(): boolean { - return this.isStarted; - } - - /** - * Get the storage path - */ - public getStoragePath(): string { - return this.options.storagePath; - } - - /** - * Get the database name - */ - public getDbName(): string { - return this.options.dbName; - } -} diff --git a/ts/cache/documents/index.ts b/ts/cache/documents/index.ts deleted file mode 100644 index 10a0ad9..0000000 --- a/ts/cache/documents/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './classes.cached.email.js'; -export * from './classes.cached.ip.reputation.js'; diff --git a/ts/classes.cert-provision-scheduler.ts b/ts/classes.cert-provision-scheduler.ts index b3b7305..3dedfbc 100644 --- a/ts/classes.cert-provision-scheduler.ts +++ b/ts/classes.cert-provision-scheduler.ts @@ -1,5 +1,5 @@ import { logger } from './logger.js'; -import type { StorageManager } from './storage/index.js'; +import { CertBackoffDoc } from './db/index.js'; interface IBackoffEntry { failures: number; @@ -10,54 +10,68 @@ interface IBackoffEntry { /** * Manages certificate provisioning scheduling with: - * - Per-domain exponential backoff persisted in StorageManager + * - Per-domain exponential backoff persisted via CertBackoffDoc * * Note: Serial stagger queue was removed — smartacme v9 handles * concurrency, per-domain dedup, and rate limiting internally. */ export class CertProvisionScheduler { - private storageManager: StorageManager; private maxBackoffHours: number; // In-memory backoff cache (mirrors storage for fast lookups) private backoffCache = new Map(); constructor( - storageManager: StorageManager, options?: { maxBackoffHours?: number } ) { - this.storageManager = storageManager; this.maxBackoffHours = options?.maxBackoffHours ?? 24; } /** - * Storage key for a domain's backoff entry + * Sanitized domain key for storage lookups */ - private backoffKey(domain: string): string { - const clean = domain.replace(/\*/g, '_wildcard_').replace(/[^a-zA-Z0-9._-]/g, '_'); - return `/cert-backoff/${clean}`; + private sanitizeDomain(domain: string): string { + return domain.replace(/\*/g, '_wildcard_').replace(/[^a-zA-Z0-9._-]/g, '_'); } /** - * Load backoff entry from storage (with in-memory cache) + * Load backoff entry from database (with in-memory cache) */ private async loadBackoff(domain: string): Promise { const cached = this.backoffCache.get(domain); if (cached) return cached; - const entry = await this.storageManager.getJSON(this.backoffKey(domain)); - if (entry) { + const sanitized = this.sanitizeDomain(domain); + const doc = await CertBackoffDoc.findByDomain(sanitized); + if (doc) { + const entry: IBackoffEntry = { + failures: doc.failures, + lastFailure: doc.lastFailure, + retryAfter: doc.retryAfter, + lastError: doc.lastError, + }; this.backoffCache.set(domain, entry); + return entry; } - return entry; + return null; } /** - * Save backoff entry to both cache and storage + * Save backoff entry to both cache and database */ private async saveBackoff(domain: string, entry: IBackoffEntry): Promise { this.backoffCache.set(domain, entry); - await this.storageManager.setJSON(this.backoffKey(domain), entry); + const sanitized = this.sanitizeDomain(domain); + let doc = await CertBackoffDoc.findByDomain(sanitized); + if (!doc) { + doc = new CertBackoffDoc(); + doc.domain = sanitized; + } + doc.failures = entry.failures; + doc.lastFailure = entry.lastFailure; + doc.retryAfter = entry.retryAfter; + doc.lastError = entry.lastError || ''; + await doc.save(); } /** @@ -107,9 +121,13 @@ export class CertProvisionScheduler { async clearBackoff(domain: string): Promise { this.backoffCache.delete(domain); try { - await this.storageManager.delete(this.backoffKey(domain)); + const sanitized = this.sanitizeDomain(domain); + const doc = await CertBackoffDoc.findByDomain(sanitized); + if (doc) { + await doc.delete(); + } } catch { - // Ignore delete errors (key may not exist) + // Ignore delete errors (doc may not exist) } } diff --git a/ts/classes.dcrouter.ts b/ts/classes.dcrouter.ts index ce82941..932ce8d 100644 --- a/ts/classes.dcrouter.ts +++ b/ts/classes.dcrouter.ts @@ -11,12 +11,10 @@ import { type IEmailDomainConfig, } from '@push.rocks/smartmta'; import { logger } from './logger.js'; -// Import storage manager -import { StorageManager, type IStorageConfig } from './storage/index.js'; import { StorageBackedCertManager } from './classes.storage-cert-manager.js'; import { CertProvisionScheduler } from './classes.cert-provision-scheduler.js'; -// Import cache system -import { CacheDb, CacheCleaner, type ICacheDbOptions } from './cache/index.js'; +// Import unified database +import { DcRouterDb, type IDcRouterDbConfig, CacheCleaner, ProxyCertDoc, AcmeCertDoc } from './db/index.js'; import { OpsServer } from './opsserver/index.js'; import { MetricsManager } from './monitoring/index.js'; @@ -122,37 +120,23 @@ export interface IDcRouterOptions { /** Other DNS providers can be added here */ }; - /** Storage configuration */ - storage?: IStorageConfig; - /** - * Cache database configuration using smartdata and LocalTsmDb - * Provides persistent caching for emails, IP reputation, bounces, etc. + * Unified database configuration. + * All persistent data (config, certs, VPN, cache, etc.) is stored via smartdata. + * If mongoDbUrl is provided, connects to external MongoDB. + * Otherwise, starts an embedded LocalSmartDb automatically. */ - cacheConfig?: { - /** Enable cache database (default: true) */ + dbConfig?: { + /** Enable database (default: true). Set to false in tests to skip DB startup. */ enabled?: boolean; - /** Storage path for TsmDB data (default: ~/.serve.zone/dcrouter/tsmdb) */ + /** External MongoDB connection URL. If absent, uses embedded LocalSmartDb. */ + mongoDbUrl?: string; + /** Storage path for embedded database data (default: ~/.serve.zone/dcrouter/tsmdb) */ storagePath?: string; /** Database name (default: dcrouter) */ dbName?: string; - /** Default TTL in days for cached items (default: 30) */ - defaultTTLDays?: number; - /** Cleanup interval in hours (default: 1) */ + /** Cache cleanup interval in hours (default: 1) */ cleanupIntervalHours?: number; - /** TTL configuration per data type (in days) */ - ttlConfig?: { - /** Email cache TTL (default: 30 days) */ - emails?: number; - /** IP reputation cache TTL (default: 1 day) */ - ipReputation?: number; - /** Bounce records TTL (default: 30 days) */ - bounces?: number; - /** DKIM keys TTL (default: 90 days) */ - dkimKeys?: number; - /** Suppression list TTL (default: 30 days, can be permanent) */ - suppression?: number; - }; }; /** @@ -248,12 +232,20 @@ export class DcRouter { public dnsServer?: plugins.smartdns.dnsServerMod.DnsServer; public emailServer?: UnifiedEmailServer; public radiusServer?: RadiusServer; - public storageManager: StorageManager; public opsServer!: OpsServer; public metricsManager?: MetricsManager; - // Cache system (smartdata + LocalTsmDb) - public cacheDb?: CacheDb; + // Compatibility shim for smartmta's DkimManager which calls dcRouter.storageManager.set() + public storageManager: any = { + get: async (_key: string) => null, + set: async (_key: string, _value: string) => { + // DKIM keys from smartmta — logged but not yet migrated to smartdata + logger.log('debug', `storageManager.set() called (compat shim) for key: ${_key}`); + }, + }; + + // Unified database (smartdata + LocalSmartDb or external MongoDB) + public dcRouterDb?: DcRouterDb; public cacheCleaner?: CacheCleaner; // Remote Ingress @@ -312,16 +304,6 @@ export class DcRouter { // Resolve all data paths from baseDir this.resolvedPaths = paths.resolvePaths(this.options.baseDir); - // Default storage to filesystem if not configured - if (!this.options.storage) { - this.options.storage = { - fsPath: this.resolvedPaths.defaultStoragePath, - }; - } - - // Initialize storage manager - this.storageManager = new StorageManager(this.options.storage); - // Initialize service manager and register all services this.serviceManager = new plugins.taskbuffer.ServiceManager({ name: 'dcrouter', @@ -350,23 +332,23 @@ export class DcRouter { .withRetry({ maxRetries: 0 }), ); - // CacheDb: optional, no dependencies - if (this.options.cacheConfig?.enabled !== false) { + // DcRouterDb: optional, no dependencies — unified database for all persistence + if (this.options.dbConfig?.enabled !== false) { this.serviceManager.addService( - new plugins.taskbuffer.Service('CacheDb') + new plugins.taskbuffer.Service('DcRouterDb') .optional() .withStart(async () => { - await this.setupCacheDb(); + await this.setupDcRouterDb(); }) .withStop(async () => { if (this.cacheCleaner) { this.cacheCleaner.stop(); this.cacheCleaner = undefined; } - if (this.cacheDb) { - await this.cacheDb.stop(); - CacheDb.resetInstance(); - this.cacheDb = undefined; + if (this.dcRouterDb) { + await this.dcRouterDb.stop(); + DcRouterDb.resetInstance(); + this.dcRouterDb = undefined; } }) .withRetry({ maxRetries: 2, baseDelayMs: 1000, maxDelayMs: 5000 }), @@ -391,10 +373,10 @@ export class DcRouter { .withRetry({ maxRetries: 1, baseDelayMs: 1000 }), ); - // SmartProxy: critical, depends on CacheDb (if enabled) + // SmartProxy: critical, depends on DcRouterDb (if enabled) const smartProxyDeps: string[] = []; - if (this.options.cacheConfig?.enabled !== false) { - smartProxyDeps.push('CacheDb'); + if (this.options.dbConfig?.enabled !== false) { + smartProxyDeps.push('DcRouterDb'); } this.serviceManager.addService( new plugins.taskbuffer.Service('SmartProxy') @@ -455,36 +437,38 @@ export class DcRouter { ); } - // ConfigManagers: optional, depends on SmartProxy - this.serviceManager.addService( - new plugins.taskbuffer.Service('ConfigManagers') - .optional() - .dependsOn('SmartProxy') - .withStart(async () => { - this.routeConfigManager = new RouteConfigManager( - this.storageManager, - () => this.getConstructorRoutes(), - () => this.smartProxy, - () => this.options.http3, - this.options.vpnConfig?.enabled - ? (tags?: string[]) => { - if (tags?.length && this.vpnManager) { - return this.vpnManager.getClientIpsForServerDefinedTags(tags); + // ConfigManagers: optional, depends on SmartProxy + DcRouterDb + // Requires DcRouterDb to be enabled (document classes need the database) + if (this.options.dbConfig?.enabled !== false) { + this.serviceManager.addService( + new plugins.taskbuffer.Service('ConfigManagers') + .optional() + .dependsOn('SmartProxy', 'DcRouterDb') + .withStart(async () => { + this.routeConfigManager = new RouteConfigManager( + () => this.getConstructorRoutes(), + () => this.smartProxy, + () => this.options.http3, + this.options.vpnConfig?.enabled + ? (tags?: string[]) => { + if (tags?.length && this.vpnManager) { + return this.vpnManager.getClientIpsForServerDefinedTags(tags); + } + return [this.options.vpnConfig?.subnet || '10.8.0.0/24']; } - return [this.options.vpnConfig?.subnet || '10.8.0.0/24']; - } - : undefined, - ); - this.apiTokenManager = new ApiTokenManager(this.storageManager); - await this.apiTokenManager.initialize(); - await this.routeConfigManager.initialize(); - }) - .withStop(async () => { - this.routeConfigManager = undefined; - this.apiTokenManager = undefined; - }) - .withRetry({ maxRetries: 2, baseDelayMs: 1000 }), - ); + : undefined, + ); + this.apiTokenManager = new ApiTokenManager(); + await this.apiTokenManager.initialize(); + await this.routeConfigManager.initialize(); + }) + .withStop(async () => { + this.routeConfigManager = undefined; + this.apiTokenManager = undefined; + }) + .withRetry({ maxRetries: 2, baseDelayMs: 1000 }), + ); + } // Email Server: optional, depends on SmartProxy if (this.options.emailConfig) { @@ -695,14 +679,9 @@ export class DcRouter { logger.log('info', `Remote Ingress: tunnel port=${this.options.remoteIngressConfig.tunnelPort || 8443}, edges=${edgeCount} registered/${connectedCount} connected`); } - // Storage summary - if (this.storageManager && this.options.storage) { - logger.log('info', `Storage: path=${this.options.storage.fsPath || 'default'}`); - } - - // Cache database summary - if (this.cacheDb) { - logger.log('info', `Cache Database: storage=${this.cacheDb.getStoragePath()}, db=${this.cacheDb.getDbName()}, cleaner=${this.cacheCleaner?.isActive() ? 'active' : 'inactive'} (${(this.options.cacheConfig?.cleanupIntervalHours || 1)}h interval)`); + // Database summary + if (this.dcRouterDb) { + logger.log('info', `Database: ${this.dcRouterDb.isEmbedded() ? 'embedded' : 'external'}, db=${this.dcRouterDb.getDbName()}, cleaner=${this.cacheCleaner?.isActive() ? 'active' : 'inactive'} (${(this.options.dbConfig?.cleanupIntervalHours || 1)}h interval)`); } // Service status summary from ServiceManager @@ -723,31 +702,32 @@ export class DcRouter { } /** - * Set up the cache database (smartdata + LocalTsmDb) + * Set up the unified database (smartdata + LocalSmartDb or external MongoDB) */ - private async setupCacheDb(): Promise { - logger.log('info', 'Setting up CacheDb...'); + private async setupDcRouterDb(): Promise { + logger.log('info', 'Setting up DcRouterDb...'); - const cacheConfig = this.options.cacheConfig || {}; + const dbConfig = this.options.dbConfig || {}; - // Initialize CacheDb singleton - this.cacheDb = CacheDb.getInstance({ - storagePath: cacheConfig.storagePath || this.resolvedPaths.defaultTsmDbPath, - dbName: cacheConfig.dbName || 'dcrouter', + // Initialize DcRouterDb singleton + this.dcRouterDb = DcRouterDb.getInstance({ + mongoDbUrl: dbConfig.mongoDbUrl, + storagePath: dbConfig.storagePath || this.resolvedPaths.defaultTsmDbPath, + dbName: dbConfig.dbName || 'dcrouter', debug: false, }); - await this.cacheDb.start(); + await this.dcRouterDb.start(); - // Start the cache cleaner - const cleanupIntervalMs = (cacheConfig.cleanupIntervalHours || 1) * 60 * 60 * 1000; - this.cacheCleaner = new CacheCleaner(this.cacheDb, { + // Start the cache cleaner for TTL-based document cleanup + const cleanupIntervalMs = (dbConfig.cleanupIntervalHours || 1) * 60 * 60 * 1000; + this.cacheCleaner = new CacheCleaner(this.dcRouterDb, { intervalMs: cleanupIntervalMs, verbose: false, }); this.cacheCleaner.start(); - logger.log('info', `CacheDb initialized at ${this.cacheDb.getStoragePath()}`); + logger.log('info', `DcRouterDb ready (${this.dcRouterDb.isEmbedded() ? 'embedded' : 'external'})`); } /** @@ -850,14 +830,11 @@ export class DcRouter { acme: acmeConfig, certStore: { loadAll: async () => { - const keys = await this.storageManager.list('/proxy-certs/'); + const docs = await ProxyCertDoc.findAll(); const certs: Array<{ domain: string; publicKey: string; privateKey: string; ca?: string }> = []; - for (const key of keys) { - const data = await this.storageManager.getJSON(key); - if (data) { - certs.push(data); - loadedCertEntries.push({ domain: data.domain, publicKey: data.publicKey, validUntil: data.validUntil, validFrom: data.validFrom }); - } + for (const doc of docs) { + certs.push({ domain: doc.domain, publicKey: doc.publicKey, privateKey: doc.privateKey, ca: doc.ca }); + loadedCertEntries.push({ domain: doc.domain, publicKey: doc.publicKey, validUntil: doc.validUntil, validFrom: doc.validFrom }); } return certs; }, @@ -869,18 +846,29 @@ export class DcRouter { validUntil = new Date(x509.validTo).getTime(); validFrom = new Date(x509.validFrom).getTime(); } catch { /* PEM parsing failed */ } - await this.storageManager.setJSON(`/proxy-certs/${domain}`, { - domain, publicKey, privateKey, ca, validUntil, validFrom, - }); + let doc = await ProxyCertDoc.findByDomain(domain); + if (!doc) { + doc = new ProxyCertDoc(); + doc.domain = domain; + } + doc.publicKey = publicKey; + doc.privateKey = privateKey; + doc.ca = ca || ''; + doc.validUntil = validUntil || 0; + doc.validFrom = validFrom || 0; + await doc.save(); }, remove: async (domain: string) => { - await this.storageManager.delete(`/proxy-certs/${domain}`); + const doc = await ProxyCertDoc.findByDomain(domain); + if (doc) { + await doc.delete(); + } }, }, }; // Initialize cert provision scheduler - this.certProvisionScheduler = new CertProvisionScheduler(this.storageManager); + this.certProvisionScheduler = new CertProvisionScheduler(); // If we have DNS challenge handlers, create SmartAcme instance and wire certProvisionFunction // Note: SmartAcme.start() is NOT called here — it runs as a separate optional service @@ -895,7 +883,7 @@ export class DcRouter { } this.smartAcme = new plugins.smartacme.SmartAcme({ accountEmail: acmeConfig?.accountEmail || this.options.tls?.contactEmail || 'admin@example.com', - certManager: new StorageBackedCertManager(this.storageManager), + certManager: new StorageBackedCertManager(), environment: 'production', challengeHandlers: challengeHandlers, challengePriority: ['dns-01'], @@ -1037,16 +1025,16 @@ export class DcRouter { issuedAt = new Date(entry.validFrom).toISOString(); } - // Try SmartAcme /certs/ metadata as secondary source + // Try SmartAcme AcmeCertDoc metadata as secondary source if (!expiryDate) { try { const cleanDomain = entry.domain.replace(/^\*\.?/, ''); - const certMeta = await this.storageManager.getJSON(`/certs/${cleanDomain}`); - if (certMeta?.validUntil) { - expiryDate = new Date(certMeta.validUntil).toISOString(); + const certDoc = await AcmeCertDoc.findByDomain(cleanDomain); + if (certDoc?.validUntil) { + expiryDate = new Date(certDoc.validUntil).toISOString(); } - if (certMeta?.created && !issuedAt) { - issuedAt = new Date(certMeta.created).toISOString(); + if (certDoc?.created && !issuedAt) { + issuedAt = new Date(certDoc.created).toISOString(); } } catch { /* no metadata available */ } } @@ -2030,7 +2018,7 @@ export class DcRouter { logger.log('info', 'Setting up Remote Ingress hub...'); // Initialize the edge registration manager - this.remoteIngressManager = new RemoteIngressManager(this.storageManager); + this.remoteIngressManager = new RemoteIngressManager(); await this.remoteIngressManager.initialize(); // Pass current routes so the manager can derive edge ports from remoteIngress-tagged routes @@ -2056,7 +2044,7 @@ export class DcRouter { // Priority 2: Existing cert from SmartProxy cert store for hubDomain if (!tlsConfig && riCfg.hubDomain) { try { - const stored = await this.storageManager.getJSON(`/proxy-certs/${riCfg.hubDomain}`); + const stored = await ProxyCertDoc.findByDomain(riCfg.hubDomain); if (stored?.publicKey && stored?.privateKey) { tlsConfig = { certPem: stored.publicKey, keyPem: stored.privateKey }; logger.log('info', `Using stored ACME cert for RemoteIngress tunnel TLS: ${riCfg.hubDomain}`); @@ -2090,7 +2078,7 @@ export class DcRouter { logger.log('info', 'Setting up VPN server...'); - this.vpnManager = new VpnManager(this.storageManager, { + this.vpnManager = new VpnManager({ subnet: this.options.vpnConfig.subnet, wgListenPort: this.options.vpnConfig.wgListenPort, dns: this.options.vpnConfig.dns, @@ -2180,7 +2168,7 @@ export class DcRouter { logger.log('info', 'Setting up RADIUS server...'); - this.radiusServer = new RadiusServer(this.options.radiusConfig, this.storageManager); + this.radiusServer = new RadiusServer(this.options.radiusConfig); await this.radiusServer.start(); logger.log('info', `RADIUS server started on ports ${this.options.radiusConfig.authPort || 1812} (auth) and ${this.options.radiusConfig.acctPort || 1813} (acct)`); diff --git a/ts/classes.storage-cert-manager.ts b/ts/classes.storage-cert-manager.ts index e241fcd..8067207 100644 --- a/ts/classes.storage-cert-manager.ts +++ b/ts/classes.storage-cert-manager.ts @@ -1,46 +1,58 @@ import * as plugins from './plugins.js'; -import { StorageManager } from './storage/index.js'; +import { AcmeCertDoc } from './db/index.js'; /** - * ICertManager implementation backed by StorageManager. - * Persists SmartAcme certificates under a /certs/ key prefix so they + * ICertManager implementation backed by smartdata document classes. + * Persists SmartAcme certificates via AcmeCertDoc so they * survive process restarts without re-hitting ACME. */ export class StorageBackedCertManager implements plugins.smartacme.ICertManager { - private keyPrefix = '/certs/'; - - constructor(private storageManager: StorageManager) {} + constructor() {} async init(): Promise {} async retrieveCertificate(domainName: string): Promise { - const data = await this.storageManager.getJSON(this.keyPrefix + domainName); - if (!data) return null; - return new plugins.smartacme.Cert(data); - } - - async storeCertificate(cert: plugins.smartacme.Cert): Promise { - await this.storageManager.setJSON(this.keyPrefix + cert.domainName, { - id: cert.id, - domainName: cert.domainName, - created: cert.created, - privateKey: cert.privateKey, - publicKey: cert.publicKey, - csr: cert.csr, - validUntil: cert.validUntil, + const doc = await AcmeCertDoc.findByDomain(domainName); + if (!doc) return null; + return new plugins.smartacme.Cert({ + id: doc.id, + domainName: doc.domainName, + created: doc.created, + privateKey: doc.privateKey, + publicKey: doc.publicKey, + csr: doc.csr, + validUntil: doc.validUntil, }); } + async storeCertificate(cert: plugins.smartacme.Cert): Promise { + let doc = await AcmeCertDoc.findByDomain(cert.domainName); + if (!doc) { + doc = new AcmeCertDoc(); + doc.domainName = cert.domainName; + } + doc.id = cert.id; + doc.created = cert.created; + doc.privateKey = cert.privateKey; + doc.publicKey = cert.publicKey; + doc.csr = cert.csr; + doc.validUntil = cert.validUntil; + await doc.save(); + } + async deleteCertificate(domainName: string): Promise { - await this.storageManager.delete(this.keyPrefix + domainName); + const doc = await AcmeCertDoc.findByDomain(domainName); + if (doc) { + await doc.delete(); + } } async close(): Promise {} async wipe(): Promise { - const keys = await this.storageManager.list(this.keyPrefix); - for (const key of keys) { - await this.storageManager.delete(key); + const docs = await AcmeCertDoc.findAll(); + for (const doc of docs) { + await doc.delete(); } } } diff --git a/ts/config/classes.api-token-manager.ts b/ts/config/classes.api-token-manager.ts index 0d0e2c3..fd7068d 100644 --- a/ts/config/classes.api-token-manager.ts +++ b/ts/config/classes.api-token-manager.ts @@ -1,19 +1,18 @@ import * as plugins from '../plugins.js'; import { logger } from '../logger.js'; -import type { StorageManager } from '../storage/index.js'; +import { ApiTokenDoc } from '../db/index.js'; import type { IStoredApiToken, IApiTokenInfo, TApiTokenScope, } from '../../ts_interfaces/data/route-management.js'; -const TOKENS_PREFIX = '/config-api/tokens/'; const TOKEN_PREFIX_STR = 'dcr_'; export class ApiTokenManager { private tokens = new Map(); - constructor(private storageManager: StorageManager) {} + constructor() {} public async initialize(): Promise { await this.loadTokens(); @@ -117,7 +116,8 @@ export class ApiTokenManager { if (!this.tokens.has(id)) return false; const token = this.tokens.get(id)!; this.tokens.delete(id); - await this.storageManager.delete(`${TOKENS_PREFIX}${id}.json`); + const doc = await ApiTokenDoc.findById(id); + if (doc) await doc.delete(); logger.log('info', `API token '${token.name}' revoked (id: ${id})`); return true; } @@ -157,17 +157,48 @@ export class ApiTokenManager { // ========================================================================= private async loadTokens(): Promise { - const keys = await this.storageManager.list(TOKENS_PREFIX); - for (const key of keys) { - if (!key.endsWith('.json')) continue; - const stored = await this.storageManager.getJSON(key); - if (stored?.id) { - this.tokens.set(stored.id, stored); + const docs = await ApiTokenDoc.findAll(); + for (const doc of docs) { + if (doc.id) { + this.tokens.set(doc.id, { + id: doc.id, + name: doc.name, + tokenHash: doc.tokenHash, + scopes: doc.scopes, + createdAt: doc.createdAt, + expiresAt: doc.expiresAt, + lastUsedAt: doc.lastUsedAt, + createdBy: doc.createdBy, + enabled: doc.enabled, + }); } } } private async persistToken(stored: IStoredApiToken): Promise { - await this.storageManager.setJSON(`${TOKENS_PREFIX}${stored.id}.json`, stored); + const existing = await ApiTokenDoc.findById(stored.id); + if (existing) { + existing.name = stored.name; + existing.tokenHash = stored.tokenHash; + existing.scopes = stored.scopes; + existing.createdAt = stored.createdAt; + existing.expiresAt = stored.expiresAt; + existing.lastUsedAt = stored.lastUsedAt; + existing.createdBy = stored.createdBy; + existing.enabled = stored.enabled; + await existing.save(); + } else { + const doc = new ApiTokenDoc(); + doc.id = stored.id; + doc.name = stored.name; + doc.tokenHash = stored.tokenHash; + doc.scopes = stored.scopes; + doc.createdAt = stored.createdAt; + doc.expiresAt = stored.expiresAt; + doc.lastUsedAt = stored.lastUsedAt; + doc.createdBy = stored.createdBy; + doc.enabled = stored.enabled; + await doc.save(); + } } } diff --git a/ts/config/classes.route-config-manager.ts b/ts/config/classes.route-config-manager.ts index d913825..718f485 100644 --- a/ts/config/classes.route-config-manager.ts +++ b/ts/config/classes.route-config-manager.ts @@ -1,6 +1,6 @@ import * as plugins from '../plugins.js'; import { logger } from '../logger.js'; -import type { StorageManager } from '../storage/index.js'; +import { StoredRouteDoc, RouteOverrideDoc } from '../db/index.js'; import type { IStoredRoute, IRouteOverride, @@ -10,16 +10,12 @@ import type { import type { IDcRouterRouteConfig } from '../../ts_interfaces/data/remoteingress.js'; import { type IHttp3Config, augmentRouteWithHttp3 } from '../http3/index.js'; -const ROUTES_PREFIX = '/config-api/routes/'; -const OVERRIDES_PREFIX = '/config-api/overrides/'; - export class RouteConfigManager { private storedRoutes = new Map(); private overrides = new Map(); private warnings: IRouteWarning[] = []; constructor( - private storageManager: StorageManager, private getHardcodedRoutes: () => plugins.smartproxy.IRouteConfig[], private getSmartProxy: () => plugins.smartproxy.SmartProxy | undefined, private getHttp3Config?: () => IHttp3Config | undefined, @@ -127,7 +123,8 @@ export class RouteConfigManager { public async deleteRoute(id: string): Promise { if (!this.storedRoutes.has(id)) return false; this.storedRoutes.delete(id); - await this.storageManager.delete(`${ROUTES_PREFIX}${id}.json`); + const doc = await StoredRouteDoc.findById(id); + if (doc) await doc.delete(); await this.applyRoutes(); return true; } @@ -148,7 +145,20 @@ export class RouteConfigManager { updatedBy, }; this.overrides.set(routeName, override); - await this.storageManager.setJSON(`${OVERRIDES_PREFIX}${routeName}.json`, override); + const existingDoc = await RouteOverrideDoc.findByRouteName(routeName); + if (existingDoc) { + existingDoc.enabled = override.enabled; + existingDoc.updatedAt = override.updatedAt; + existingDoc.updatedBy = override.updatedBy; + await existingDoc.save(); + } else { + const doc = new RouteOverrideDoc(); + doc.routeName = override.routeName; + doc.enabled = override.enabled; + doc.updatedAt = override.updatedAt; + doc.updatedBy = override.updatedBy; + await doc.save(); + } this.computeWarnings(); await this.applyRoutes(); } @@ -156,7 +166,8 @@ export class RouteConfigManager { public async removeOverride(routeName: string): Promise { if (!this.overrides.has(routeName)) return false; this.overrides.delete(routeName); - await this.storageManager.delete(`${OVERRIDES_PREFIX}${routeName}.json`); + const doc = await RouteOverrideDoc.findByRouteName(routeName); + if (doc) await doc.delete(); this.computeWarnings(); await this.applyRoutes(); return true; @@ -167,12 +178,17 @@ export class RouteConfigManager { // ========================================================================= private async loadStoredRoutes(): Promise { - const keys = await this.storageManager.list(ROUTES_PREFIX); - for (const key of keys) { - if (!key.endsWith('.json')) continue; - const stored = await this.storageManager.getJSON(key); - if (stored?.id) { - this.storedRoutes.set(stored.id, stored); + const docs = await StoredRouteDoc.findAll(); + for (const doc of docs) { + if (doc.id) { + this.storedRoutes.set(doc.id, { + id: doc.id, + route: doc.route, + enabled: doc.enabled, + createdAt: doc.createdAt, + updatedAt: doc.updatedAt, + createdBy: doc.createdBy, + }); } } if (this.storedRoutes.size > 0) { @@ -181,12 +197,15 @@ export class RouteConfigManager { } private async loadOverrides(): Promise { - const keys = await this.storageManager.list(OVERRIDES_PREFIX); - for (const key of keys) { - if (!key.endsWith('.json')) continue; - const override = await this.storageManager.getJSON(key); - if (override?.routeName) { - this.overrides.set(override.routeName, override); + const docs = await RouteOverrideDoc.findAll(); + for (const doc of docs) { + if (doc.routeName) { + this.overrides.set(doc.routeName, { + routeName: doc.routeName, + enabled: doc.enabled, + updatedAt: doc.updatedAt, + updatedBy: doc.updatedBy, + }); } } if (this.overrides.size > 0) { @@ -195,7 +214,23 @@ export class RouteConfigManager { } private async persistRoute(stored: IStoredRoute): Promise { - await this.storageManager.setJSON(`${ROUTES_PREFIX}${stored.id}.json`, stored); + const existingDoc = await StoredRouteDoc.findById(stored.id); + if (existingDoc) { + existingDoc.route = stored.route; + existingDoc.enabled = stored.enabled; + existingDoc.updatedAt = stored.updatedAt; + existingDoc.createdBy = stored.createdBy; + await existingDoc.save(); + } else { + const doc = new StoredRouteDoc(); + doc.id = stored.id; + doc.route = stored.route; + doc.enabled = stored.enabled; + doc.createdAt = stored.createdAt; + doc.updatedAt = stored.updatedAt; + doc.createdBy = stored.createdBy; + await doc.save(); + } } // ========================================================================= diff --git a/ts/cache/classes.cache.cleaner.ts b/ts/db/classes.cache.cleaner.ts similarity index 93% rename from ts/cache/classes.cache.cleaner.ts rename to ts/db/classes.cache.cleaner.ts index d360c6c..b7c72bd 100644 --- a/ts/cache/classes.cache.cleaner.ts +++ b/ts/db/classes.cache.cleaner.ts @@ -1,6 +1,6 @@ import * as plugins from '../plugins.js'; import { logger } from '../logger.js'; -import { CacheDb } from './classes.cachedb.js'; +import { DcRouterDb } from './classes.dcrouter-db.js'; // Import document classes for cleanup import { CachedEmail } from './documents/classes.cached.email.js'; @@ -26,10 +26,10 @@ export class CacheCleaner { private cleanupInterval: ReturnType | null = null; private isRunning: boolean = false; private options: Required; - private cacheDb: CacheDb; + private dcRouterDb: DcRouterDb; - constructor(cacheDb: CacheDb, options: ICacheCleanerOptions = {}) { - this.cacheDb = cacheDb; + constructor(dcRouterDb: DcRouterDb, options: ICacheCleanerOptions = {}) { + this.dcRouterDb = dcRouterDb; this.options = { intervalMs: options.intervalMs || 60 * 60 * 1000, // 1 hour default verbose: options.verbose || false, @@ -86,8 +86,8 @@ export class CacheCleaner { * Run a single cleanup cycle */ public async runCleanup(): Promise { - if (!this.cacheDb.isReady()) { - logger.log('warn', 'CacheDb not ready, skipping cleanup'); + if (!this.dcRouterDb.isReady()) { + logger.log('warn', 'DcRouterDb not ready, skipping cleanup'); return; } diff --git a/ts/cache/classes.cached.document.ts b/ts/db/classes.cached.document.ts similarity index 100% rename from ts/cache/classes.cached.document.ts rename to ts/db/classes.cached.document.ts diff --git a/ts/db/classes.dcrouter-db.ts b/ts/db/classes.dcrouter-db.ts new file mode 100644 index 0000000..15ebc13 --- /dev/null +++ b/ts/db/classes.dcrouter-db.ts @@ -0,0 +1,179 @@ +import * as plugins from '../plugins.js'; +import { logger } from '../logger.js'; +import { defaultTsmDbPath } from '../paths.js'; + +/** + * Configuration options for the unified DCRouter database + */ +export interface IDcRouterDbConfig { + /** External MongoDB connection URL. If absent, uses embedded LocalSmartDb. */ + mongoDbUrl?: string; + /** Storage path for embedded LocalSmartDb data (default: ~/.serve.zone/dcrouter/tsmdb) */ + storagePath?: string; + /** Database name (default: dcrouter) */ + dbName?: string; + /** Enable debug logging */ + debug?: boolean; +} + +/** + * DcRouterDb - Unified database layer for DCRouter + * + * Replaces both StorageManager (flat-file key-value) and CacheDb (embedded MongoDB). + * All data is stored as smartdata document classes in a single database. + * + * Two modes: + * - **Embedded** (default): Spawns a LocalSmartDb (Rust-based MongoDB-compatible engine) + * - **External**: Connects to a provided MongoDB URL + */ +export class DcRouterDb { + private static instance: DcRouterDb | null = null; + + private localSmartDb: plugins.smartdb.LocalSmartDb | null = null; + private smartdataDb!: plugins.smartdata.SmartdataDb; + private options: Required; + private isStarted: boolean = false; + + constructor(options: IDcRouterDbConfig = {}) { + this.options = { + mongoDbUrl: options.mongoDbUrl || '', + storagePath: options.storagePath || defaultTsmDbPath, + dbName: options.dbName || 'dcrouter', + debug: options.debug || false, + }; + } + + /** + * Get or create the singleton instance + */ + public static getInstance(options?: IDcRouterDbConfig): DcRouterDb { + if (!DcRouterDb.instance) { + DcRouterDb.instance = new DcRouterDb(options); + } + return DcRouterDb.instance; + } + + /** + * Reset the singleton instance (useful for testing) + */ + public static resetInstance(): void { + DcRouterDb.instance = null; + } + + /** + * Start the database + * - If mongoDbUrl is provided, connects directly to external MongoDB + * - Otherwise, starts an embedded LocalSmartDb instance + */ + public async start(): Promise { + if (this.isStarted) { + logger.log('warn', 'DcRouterDb already started'); + return; + } + + try { + let connectionUri: string; + + if (this.options.mongoDbUrl) { + // External MongoDB mode + connectionUri = this.options.mongoDbUrl; + logger.log('info', `DcRouterDb connecting to external MongoDB`); + } else { + // Embedded LocalSmartDb mode + await plugins.fsUtils.ensureDir(this.options.storagePath); + + this.localSmartDb = new plugins.smartdb.LocalSmartDb({ + folderPath: this.options.storagePath, + }); + + const connectionInfo = await this.localSmartDb.start(); + connectionUri = connectionInfo.connectionUri; + + if (this.options.debug) { + logger.log('debug', `LocalSmartDb started with URI: ${connectionUri}`); + } + + logger.log('info', `DcRouterDb started embedded instance at ${this.options.storagePath}`); + } + + // Initialize smartdata ORM + this.smartdataDb = new plugins.smartdata.SmartdataDb({ + mongoDbUrl: connectionUri, + mongoDbName: this.options.dbName, + }); + await this.smartdataDb.init(); + + this.isStarted = true; + logger.log('info', `DcRouterDb ready (db: ${this.options.dbName})`); + } catch (error: unknown) { + logger.log('error', `Failed to start DcRouterDb: ${(error as Error).message}`); + throw error; + } + } + + /** + * Stop the database + */ + public async stop(): Promise { + if (!this.isStarted) { + return; + } + + try { + // Close smartdata connection + if (this.smartdataDb) { + await this.smartdataDb.close(); + } + + // Stop embedded LocalSmartDb if running + if (this.localSmartDb) { + await this.localSmartDb.stop(); + this.localSmartDb = null; + } + + this.isStarted = false; + logger.log('info', 'DcRouterDb stopped'); + } catch (error: unknown) { + logger.log('error', `Error stopping DcRouterDb: ${(error as Error).message}`); + throw error; + } + } + + /** + * Get the smartdata database instance for @Collection decorators + */ + public getDb(): plugins.smartdata.SmartdataDb { + if (!this.isStarted) { + throw new Error('DcRouterDb not started. Call start() first.'); + } + return this.smartdataDb; + } + + /** + * Check if the database is ready + */ + public isReady(): boolean { + return this.isStarted; + } + + /** + * Whether running in embedded mode (LocalSmartDb) vs external MongoDB + */ + public isEmbedded(): boolean { + return !this.options.mongoDbUrl; + } + + /** + * Get the storage path (only relevant for embedded mode) + */ + public getStoragePath(): string { + return this.options.storagePath; + } + + /** + * Get the database name + */ + public getDbName(): string { + return this.options.dbName; + } +} diff --git a/ts/db/documents/classes.accounting-session.doc.ts b/ts/db/documents/classes.accounting-session.doc.ts new file mode 100644 index 0000000..d55d2a5 --- /dev/null +++ b/ts/db/documents/classes.accounting-session.doc.ts @@ -0,0 +1,106 @@ +import * as plugins from '../../plugins.js'; +import { DcRouterDb } from '../classes.dcrouter-db.js'; + +const getDb = () => DcRouterDb.getInstance().getDb(); + +@plugins.smartdata.Collection(() => getDb()) +export class AccountingSessionDoc extends plugins.smartdata.SmartDataDbDoc { + @plugins.smartdata.unI() + @plugins.smartdata.svDb() + public sessionId!: string; + + @plugins.smartdata.svDb() + public username!: string; + + @plugins.smartdata.svDb() + public macAddress!: string; + + @plugins.smartdata.svDb() + public nasIpAddress!: string; + + @plugins.smartdata.svDb() + public nasPort!: number; + + @plugins.smartdata.svDb() + public nasPortType!: string; + + @plugins.smartdata.svDb() + public nasIdentifier!: string; + + @plugins.smartdata.svDb() + public vlanId!: number; + + @plugins.smartdata.svDb() + public framedIpAddress!: string; + + @plugins.smartdata.svDb() + public calledStationId!: string; + + @plugins.smartdata.svDb() + public callingStationId!: string; + + @plugins.smartdata.svDb() + public startTime!: number; + + @plugins.smartdata.svDb() + public endTime!: number; + + @plugins.smartdata.svDb() + public lastUpdateTime!: number; + + @plugins.smartdata.index() + @plugins.smartdata.svDb() + public status!: 'active' | 'stopped' | 'terminated'; + + @plugins.smartdata.svDb() + public terminateCause!: string; + + @plugins.smartdata.svDb() + public inputOctets!: number; + + @plugins.smartdata.svDb() + public outputOctets!: number; + + @plugins.smartdata.svDb() + public inputPackets!: number; + + @plugins.smartdata.svDb() + public outputPackets!: number; + + @plugins.smartdata.svDb() + public sessionTime!: number; + + @plugins.smartdata.svDb() + public serviceType!: string; + + constructor() { + super(); + } + + public static async findBySessionId(sessionId: string): Promise { + return await AccountingSessionDoc.getInstance({ sessionId }); + } + + public static async findActive(): Promise { + return await AccountingSessionDoc.getInstances({ status: 'active' }); + } + + public static async findByUsername(username: string): Promise { + return await AccountingSessionDoc.getInstances({ username }); + } + + public static async findByNas(nasIpAddress: string): Promise { + return await AccountingSessionDoc.getInstances({ nasIpAddress }); + } + + public static async findByVlan(vlanId: number): Promise { + return await AccountingSessionDoc.getInstances({ vlanId }); + } + + public static async findStoppedBefore(cutoffTime: number): Promise { + return await AccountingSessionDoc.getInstances({ + status: { $in: ['stopped', 'terminated'] } as any, + endTime: { $lt: cutoffTime, $gt: 0 } as any, + }); + } +} diff --git a/ts/db/documents/classes.acme-cert.doc.ts b/ts/db/documents/classes.acme-cert.doc.ts new file mode 100644 index 0000000..e4ac7fb --- /dev/null +++ b/ts/db/documents/classes.acme-cert.doc.ts @@ -0,0 +1,41 @@ +import * as plugins from '../../plugins.js'; +import { DcRouterDb } from '../classes.dcrouter-db.js'; + +const getDb = () => DcRouterDb.getInstance().getDb(); + +@plugins.smartdata.Collection(() => getDb()) +export class AcmeCertDoc extends plugins.smartdata.SmartDataDbDoc { + @plugins.smartdata.unI() + @plugins.smartdata.svDb() + public domainName!: string; + + @plugins.smartdata.svDb() + public id!: string; + + @plugins.smartdata.svDb() + public created!: number; + + @plugins.smartdata.svDb() + public privateKey!: string; + + @plugins.smartdata.svDb() + public publicKey!: string; + + @plugins.smartdata.svDb() + public csr!: string; + + @plugins.smartdata.svDb() + public validUntil!: number; + + constructor() { + super(); + } + + public static async findByDomain(domainName: string): Promise { + return await AcmeCertDoc.getInstance({ domainName }); + } + + public static async findAll(): Promise { + return await AcmeCertDoc.getInstances({}); + } +} diff --git a/ts/db/documents/classes.api-token.doc.ts b/ts/db/documents/classes.api-token.doc.ts new file mode 100644 index 0000000..4f1e676 --- /dev/null +++ b/ts/db/documents/classes.api-token.doc.ts @@ -0,0 +1,56 @@ +import * as plugins from '../../plugins.js'; +import { DcRouterDb } from '../classes.dcrouter-db.js'; +import type { TApiTokenScope } from '../../../ts_interfaces/data/route-management.js'; + +const getDb = () => DcRouterDb.getInstance().getDb(); + +@plugins.smartdata.Collection(() => getDb()) +export class ApiTokenDoc extends plugins.smartdata.SmartDataDbDoc { + @plugins.smartdata.unI() + @plugins.smartdata.svDb() + public id!: string; + + @plugins.smartdata.svDb() + public name: string = ''; + + @plugins.smartdata.svDb() + public tokenHash!: string; + + @plugins.smartdata.svDb() + public scopes!: TApiTokenScope[]; + + @plugins.smartdata.svDb() + public createdAt!: number; + + @plugins.smartdata.svDb() + public expiresAt!: number | null; + + @plugins.smartdata.svDb() + public lastUsedAt!: number | null; + + @plugins.smartdata.svDb() + public createdBy!: string; + + @plugins.smartdata.svDb() + public enabled!: boolean; + + constructor() { + super(); + } + + public static async findById(id: string): Promise { + return await ApiTokenDoc.getInstance({ id }); + } + + public static async findByTokenHash(tokenHash: string): Promise { + return await ApiTokenDoc.getInstance({ tokenHash }); + } + + public static async findAll(): Promise { + return await ApiTokenDoc.getInstances({}); + } + + public static async findEnabled(): Promise { + return await ApiTokenDoc.getInstances({ enabled: true }); + } +} diff --git a/ts/cache/documents/classes.cached.email.ts b/ts/db/documents/classes.cached.email.ts similarity index 97% rename from ts/cache/documents/classes.cached.email.ts rename to ts/db/documents/classes.cached.email.ts index 215d618..090c98b 100644 --- a/ts/cache/documents/classes.cached.email.ts +++ b/ts/db/documents/classes.cached.email.ts @@ -1,6 +1,6 @@ import * as plugins from '../../plugins.js'; import { CachedDocument, TTL } from '../classes.cached.document.js'; -import { CacheDb } from '../classes.cachedb.js'; +import { DcRouterDb } from '../classes.dcrouter-db.js'; /** * Email status in the cache @@ -10,7 +10,7 @@ export type TCachedEmailStatus = 'pending' | 'processing' | 'delivered' | 'faile /** * Helper to get the smartdata database instance */ -const getDb = () => CacheDb.getInstance().getDb(); +const getDb = () => DcRouterDb.getInstance().getDb(); /** * CachedEmail - Stores email queue items in the cache diff --git a/ts/cache/documents/classes.cached.ip.reputation.ts b/ts/db/documents/classes.cached.ip.reputation.ts similarity index 98% rename from ts/cache/documents/classes.cached.ip.reputation.ts rename to ts/db/documents/classes.cached.ip.reputation.ts index 7bfe60c..c3935aa 100644 --- a/ts/cache/documents/classes.cached.ip.reputation.ts +++ b/ts/db/documents/classes.cached.ip.reputation.ts @@ -1,11 +1,11 @@ import * as plugins from '../../plugins.js'; import { CachedDocument, TTL } from '../classes.cached.document.js'; -import { CacheDb } from '../classes.cachedb.js'; +import { DcRouterDb } from '../classes.dcrouter-db.js'; /** * Helper to get the smartdata database instance */ -const getDb = () => CacheDb.getInstance().getDb(); +const getDb = () => DcRouterDb.getInstance().getDb(); /** * IP reputation result data diff --git a/ts/db/documents/classes.cert-backoff.doc.ts b/ts/db/documents/classes.cert-backoff.doc.ts new file mode 100644 index 0000000..174f103 --- /dev/null +++ b/ts/db/documents/classes.cert-backoff.doc.ts @@ -0,0 +1,35 @@ +import * as plugins from '../../plugins.js'; +import { DcRouterDb } from '../classes.dcrouter-db.js'; + +const getDb = () => DcRouterDb.getInstance().getDb(); + +@plugins.smartdata.Collection(() => getDb()) +export class CertBackoffDoc extends plugins.smartdata.SmartDataDbDoc { + @plugins.smartdata.unI() + @plugins.smartdata.svDb() + public domain!: string; + + @plugins.smartdata.svDb() + public failures!: number; + + @plugins.smartdata.svDb() + public lastFailure!: string; + + @plugins.smartdata.svDb() + public retryAfter!: string; + + @plugins.smartdata.svDb() + public lastError!: string; + + constructor() { + super(); + } + + public static async findByDomain(domain: string): Promise { + return await CertBackoffDoc.getInstance({ domain }); + } + + public static async findAll(): Promise { + return await CertBackoffDoc.getInstances({}); + } +} diff --git a/ts/db/documents/classes.proxy-cert.doc.ts b/ts/db/documents/classes.proxy-cert.doc.ts new file mode 100644 index 0000000..64fe12f --- /dev/null +++ b/ts/db/documents/classes.proxy-cert.doc.ts @@ -0,0 +1,38 @@ +import * as plugins from '../../plugins.js'; +import { DcRouterDb } from '../classes.dcrouter-db.js'; + +const getDb = () => DcRouterDb.getInstance().getDb(); + +@plugins.smartdata.Collection(() => getDb()) +export class ProxyCertDoc extends plugins.smartdata.SmartDataDbDoc { + @plugins.smartdata.unI() + @plugins.smartdata.svDb() + public domain!: string; + + @plugins.smartdata.svDb() + public publicKey!: string; + + @plugins.smartdata.svDb() + public privateKey!: string; + + @plugins.smartdata.svDb() + public ca!: string; + + @plugins.smartdata.svDb() + public validUntil!: number; + + @plugins.smartdata.svDb() + public validFrom!: number; + + constructor() { + super(); + } + + public static async findByDomain(domain: string): Promise { + return await ProxyCertDoc.getInstance({ domain }); + } + + public static async findAll(): Promise { + return await ProxyCertDoc.getInstances({}); + } +} diff --git a/ts/db/documents/classes.remote-ingress-edge.doc.ts b/ts/db/documents/classes.remote-ingress-edge.doc.ts new file mode 100644 index 0000000..4cbb5b7 --- /dev/null +++ b/ts/db/documents/classes.remote-ingress-edge.doc.ts @@ -0,0 +1,54 @@ +import * as plugins from '../../plugins.js'; +import { DcRouterDb } from '../classes.dcrouter-db.js'; + +const getDb = () => DcRouterDb.getInstance().getDb(); + +@plugins.smartdata.Collection(() => getDb()) +export class RemoteIngressEdgeDoc extends plugins.smartdata.SmartDataDbDoc { + @plugins.smartdata.unI() + @plugins.smartdata.svDb() + public id!: string; + + @plugins.smartdata.svDb() + public name: string = ''; + + @plugins.smartdata.svDb() + public secret!: string; + + @plugins.smartdata.svDb() + public listenPorts!: number[]; + + @plugins.smartdata.svDb() + public listenPortsUdp!: number[]; + + @plugins.smartdata.svDb() + public enabled!: boolean; + + @plugins.smartdata.svDb() + public autoDerivePorts!: boolean; + + @plugins.smartdata.svDb() + public tags!: string[]; + + @plugins.smartdata.svDb() + public createdAt!: number; + + @plugins.smartdata.svDb() + public updatedAt!: number; + + constructor() { + super(); + } + + public static async findById(id: string): Promise { + return await RemoteIngressEdgeDoc.getInstance({ id }); + } + + public static async findAll(): Promise { + return await RemoteIngressEdgeDoc.getInstances({}); + } + + public static async findEnabled(): Promise { + return await RemoteIngressEdgeDoc.getInstances({ enabled: true }); + } +} diff --git a/ts/db/documents/classes.route-override.doc.ts b/ts/db/documents/classes.route-override.doc.ts new file mode 100644 index 0000000..30221b6 --- /dev/null +++ b/ts/db/documents/classes.route-override.doc.ts @@ -0,0 +1,32 @@ +import * as plugins from '../../plugins.js'; +import { DcRouterDb } from '../classes.dcrouter-db.js'; + +const getDb = () => DcRouterDb.getInstance().getDb(); + +@plugins.smartdata.Collection(() => getDb()) +export class RouteOverrideDoc extends plugins.smartdata.SmartDataDbDoc { + @plugins.smartdata.unI() + @plugins.smartdata.svDb() + public routeName!: string; + + @plugins.smartdata.svDb() + public enabled!: boolean; + + @plugins.smartdata.svDb() + public updatedAt!: number; + + @plugins.smartdata.svDb() + public updatedBy!: string; + + constructor() { + super(); + } + + public static async findByRouteName(routeName: string): Promise { + return await RouteOverrideDoc.getInstance({ routeName }); + } + + public static async findAll(): Promise { + return await RouteOverrideDoc.getInstances({}); + } +} diff --git a/ts/db/documents/classes.stored-route.doc.ts b/ts/db/documents/classes.stored-route.doc.ts new file mode 100644 index 0000000..1d0fbcc --- /dev/null +++ b/ts/db/documents/classes.stored-route.doc.ts @@ -0,0 +1,38 @@ +import * as plugins from '../../plugins.js'; +import { DcRouterDb } from '../classes.dcrouter-db.js'; + +const getDb = () => DcRouterDb.getInstance().getDb(); + +@plugins.smartdata.Collection(() => getDb()) +export class StoredRouteDoc extends plugins.smartdata.SmartDataDbDoc { + @plugins.smartdata.unI() + @plugins.smartdata.svDb() + public id!: string; + + @plugins.smartdata.svDb() + public route!: plugins.smartproxy.IRouteConfig; + + @plugins.smartdata.svDb() + public enabled!: boolean; + + @plugins.smartdata.svDb() + public createdAt!: number; + + @plugins.smartdata.svDb() + public updatedAt!: number; + + @plugins.smartdata.svDb() + public createdBy!: string; + + constructor() { + super(); + } + + public static async findById(id: string): Promise { + return await StoredRouteDoc.getInstance({ id }); + } + + public static async findAll(): Promise { + return await StoredRouteDoc.getInstances({}); + } +} diff --git a/ts/db/documents/classes.vlan-mappings.doc.ts b/ts/db/documents/classes.vlan-mappings.doc.ts new file mode 100644 index 0000000..e7f3d3d --- /dev/null +++ b/ts/db/documents/classes.vlan-mappings.doc.ts @@ -0,0 +1,32 @@ +import * as plugins from '../../plugins.js'; +import { DcRouterDb } from '../classes.dcrouter-db.js'; + +const getDb = () => DcRouterDb.getInstance().getDb(); + +export interface IMacVlanMapping { + mac: string; + vlan: number; + description?: string; + enabled: boolean; + createdAt: number; + updatedAt: number; +} + +@plugins.smartdata.Collection(() => getDb()) +export class VlanMappingsDoc extends plugins.smartdata.SmartDataDbDoc { + @plugins.smartdata.unI() + @plugins.smartdata.svDb() + public configId: string = 'vlan-mappings'; + + @plugins.smartdata.svDb() + public mappings!: IMacVlanMapping[]; + + constructor() { + super(); + this.mappings = []; + } + + public static async load(): Promise { + return await VlanMappingsDoc.getInstance({ configId: 'vlan-mappings' }); + } +} diff --git a/ts/db/documents/classes.vpn-client.doc.ts b/ts/db/documents/classes.vpn-client.doc.ts new file mode 100644 index 0000000..44c9b4c --- /dev/null +++ b/ts/db/documents/classes.vpn-client.doc.ts @@ -0,0 +1,57 @@ +import * as plugins from '../../plugins.js'; +import { DcRouterDb } from '../classes.dcrouter-db.js'; + +const getDb = () => DcRouterDb.getInstance().getDb(); + +@plugins.smartdata.Collection(() => getDb()) +export class VpnClientDoc extends plugins.smartdata.SmartDataDbDoc { + @plugins.smartdata.unI() + @plugins.smartdata.svDb() + public clientId!: string; + + @plugins.smartdata.svDb() + public enabled!: boolean; + + @plugins.smartdata.svDb() + public serverDefinedClientTags?: string[]; + + @plugins.smartdata.svDb() + public description?: string; + + @plugins.smartdata.svDb() + public assignedIp?: string; + + @plugins.smartdata.svDb() + public noisePublicKey!: string; + + @plugins.smartdata.svDb() + public wgPublicKey!: string; + + @plugins.smartdata.svDb() + public wgPrivateKey?: string; + + @plugins.smartdata.svDb() + public createdAt!: number; + + @plugins.smartdata.svDb() + public updatedAt!: number; + + @plugins.smartdata.svDb() + public expiresAt?: string; + + constructor() { + super(); + } + + public static async findByClientId(clientId: string): Promise { + return await VpnClientDoc.getInstance({ clientId }); + } + + public static async findAll(): Promise { + return await VpnClientDoc.getInstances({}); + } + + public static async findEnabled(): Promise { + return await VpnClientDoc.getInstances({ enabled: true }); + } +} diff --git a/ts/db/documents/classes.vpn-server-keys.doc.ts b/ts/db/documents/classes.vpn-server-keys.doc.ts new file mode 100644 index 0000000..5a41696 --- /dev/null +++ b/ts/db/documents/classes.vpn-server-keys.doc.ts @@ -0,0 +1,31 @@ +import * as plugins from '../../plugins.js'; +import { DcRouterDb } from '../classes.dcrouter-db.js'; + +const getDb = () => DcRouterDb.getInstance().getDb(); + +@plugins.smartdata.Collection(() => getDb()) +export class VpnServerKeysDoc extends plugins.smartdata.SmartDataDbDoc { + @plugins.smartdata.unI() + @plugins.smartdata.svDb() + public configId: string = 'vpn-server-keys'; + + @plugins.smartdata.svDb() + public noisePrivateKey!: string; + + @plugins.smartdata.svDb() + public noisePublicKey!: string; + + @plugins.smartdata.svDb() + public wgPrivateKey!: string; + + @plugins.smartdata.svDb() + public wgPublicKey!: string; + + constructor() { + super(); + } + + public static async load(): Promise { + return await VpnServerKeysDoc.getInstance({ configId: 'vpn-server-keys' }); + } +} diff --git a/ts/db/documents/index.ts b/ts/db/documents/index.ts new file mode 100644 index 0000000..bd0e2b1 --- /dev/null +++ b/ts/db/documents/index.ts @@ -0,0 +1,24 @@ +// Cached/TTL document classes +export * from './classes.cached.email.js'; +export * from './classes.cached.ip.reputation.js'; + +// Config document classes +export * from './classes.stored-route.doc.js'; +export * from './classes.route-override.doc.js'; +export * from './classes.api-token.doc.js'; + +// VPN document classes +export * from './classes.vpn-server-keys.doc.js'; +export * from './classes.vpn-client.doc.js'; + +// Certificate document classes +export * from './classes.acme-cert.doc.js'; +export * from './classes.proxy-cert.doc.js'; +export * from './classes.cert-backoff.doc.js'; + +// Remote ingress document classes +export * from './classes.remote-ingress-edge.doc.js'; + +// RADIUS document classes +export * from './classes.vlan-mappings.doc.js'; +export * from './classes.accounting-session.doc.js'; diff --git a/ts/cache/index.ts b/ts/db/index.ts similarity index 55% rename from ts/cache/index.ts rename to ts/db/index.ts index a398f7b..dd5bd7b 100644 --- a/ts/cache/index.ts +++ b/ts/db/index.ts @@ -1,6 +1,10 @@ -// Core cache infrastructure -export * from './classes.cachedb.js'; +// Unified database manager +export * from './classes.dcrouter-db.js'; + +// TTL base class and constants export * from './classes.cached.document.js'; + +// Cache cleaner export * from './classes.cache.cleaner.js'; // Document classes diff --git a/ts/opsserver/handlers/certificate.handler.ts b/ts/opsserver/handlers/certificate.handler.ts index 89ab017..e79c326 100644 --- a/ts/opsserver/handlers/certificate.handler.ts +++ b/ts/opsserver/handlers/certificate.handler.ts @@ -1,6 +1,7 @@ import * as plugins from '../../plugins.js'; import type { OpsServer } from '../classes.opsserver.js'; import * as interfaces from '../../../ts_interfaces/index.js'; +import { AcmeCertDoc, ProxyCertDoc } from '../../db/index.js'; export class CertificateHandler { constructor(private opsServerRef: OpsServer) { @@ -187,30 +188,28 @@ export class CertificateHandler { } } - // Check persisted cert data from StorageManager + // Check persisted cert data from smartdata document classes if (status === 'unknown') { const cleanDomain = domain.replace(/^\*\.?/, ''); - let certData = await dcRouter.storageManager.getJSON(`/certs/${cleanDomain}`); - if (!certData) { - // Also check certStore path (proxy-certs) - certData = await dcRouter.storageManager.getJSON(`/proxy-certs/${domain}`); - } - if (certData?.validUntil) { - expiryDate = new Date(certData.validUntil).toISOString(); - if (certData.created) { - issuedAt = new Date(certData.created).toISOString(); + const acmeDoc = await AcmeCertDoc.findByDomain(cleanDomain); + const proxyDoc = !acmeDoc ? await ProxyCertDoc.findByDomain(domain) : null; + + if (acmeDoc?.validUntil) { + expiryDate = new Date(acmeDoc.validUntil).toISOString(); + if (acmeDoc.created) { + issuedAt = new Date(acmeDoc.created).toISOString(); } issuer = 'smartacme-dns-01'; - } else if (certData?.publicKey) { + } else if (proxyDoc?.publicKey) { // certStore has the cert — parse PEM for expiry try { - const x509 = new plugins.crypto.X509Certificate(certData.publicKey); + const x509 = new plugins.crypto.X509Certificate(proxyDoc.publicKey); expiryDate = new Date(x509.validTo).toISOString(); issuedAt = new Date(x509.validFrom).toISOString(); } catch { /* PEM parsing failed */ } status = 'valid'; issuer = 'cert-store'; - } else if (certData) { + } else if (acmeDoc || proxyDoc) { status = 'valid'; issuer = 'cert-store'; } @@ -366,18 +365,17 @@ export class CertificateHandler { const dcRouter = this.opsServerRef.dcRouterRef; const cleanDomain = domain.replace(/^\*\.?/, ''); - // Delete from all known storage paths - const paths = [ - `/proxy-certs/${domain}`, - `/proxy-certs/${cleanDomain}`, - `/certs/${cleanDomain}`, - ]; + // Delete from smartdata document classes + const acmeDoc = await AcmeCertDoc.findByDomain(cleanDomain); + if (acmeDoc) { + await acmeDoc.delete(); + } - for (const path of paths) { - try { - await dcRouter.storageManager.delete(path); - } catch { - // Path may not exist — ignore + // Try both original domain and clean domain for proxy certs + for (const d of [domain, cleanDomain]) { + const proxyDoc = await ProxyCertDoc.findByDomain(d); + if (proxyDoc) { + await proxyDoc.delete(); } } @@ -408,43 +406,41 @@ export class CertificateHandler { }; message?: string; }> { - const dcRouter = this.opsServerRef.dcRouterRef; const cleanDomain = domain.replace(/^\*\.?/, ''); - // Try SmartAcme /certs/ path first (has full ICert fields) - let certData = await dcRouter.storageManager.getJSON(`/certs/${cleanDomain}`); - if (certData && certData.publicKey && certData.privateKey) { + // Try AcmeCertDoc first (has full ICert fields) + const acmeDoc = await AcmeCertDoc.findByDomain(cleanDomain); + if (acmeDoc && acmeDoc.publicKey && acmeDoc.privateKey) { return { success: true, cert: { - id: certData.id || plugins.crypto.randomUUID(), - domainName: certData.domainName || domain, - created: certData.created || Date.now(), - validUntil: certData.validUntil || 0, - privateKey: certData.privateKey, - publicKey: certData.publicKey, - csr: certData.csr || '', + id: acmeDoc.id || plugins.crypto.randomUUID(), + domainName: acmeDoc.domainName || domain, + created: acmeDoc.created || Date.now(), + validUntil: acmeDoc.validUntil || 0, + privateKey: acmeDoc.privateKey, + publicKey: acmeDoc.publicKey, + csr: acmeDoc.csr || '', }, }; } - // Fallback: try /proxy-certs/ with original domain - certData = await dcRouter.storageManager.getJSON(`/proxy-certs/${domain}`); - if (!certData || !certData.publicKey) { - // Try with clean domain - certData = await dcRouter.storageManager.getJSON(`/proxy-certs/${cleanDomain}`); + // Fallback: try ProxyCertDoc with original domain, then clean domain + let proxyDoc = await ProxyCertDoc.findByDomain(domain); + if (!proxyDoc || !proxyDoc.publicKey) { + proxyDoc = await ProxyCertDoc.findByDomain(cleanDomain); } - if (certData && certData.publicKey && certData.privateKey) { + if (proxyDoc && proxyDoc.publicKey && proxyDoc.privateKey) { return { success: true, cert: { id: plugins.crypto.randomUUID(), domainName: domain, - created: certData.validFrom || Date.now(), - validUntil: certData.validUntil || 0, - privateKey: certData.privateKey, - publicKey: certData.publicKey, + created: proxyDoc.validFrom || Date.now(), + validUntil: proxyDoc.validUntil || 0, + privateKey: proxyDoc.privateKey, + publicKey: proxyDoc.publicKey, csr: '', }, }; @@ -476,26 +472,32 @@ export class CertificateHandler { const dcRouter = this.opsServerRef.dcRouterRef; const cleanDomain = cert.domainName.replace(/^\*\.?/, ''); - // Save to /certs/ (SmartAcme-compatible path) - await dcRouter.storageManager.setJSON(`/certs/${cleanDomain}`, { - id: cert.id, - domainName: cert.domainName, - created: cert.created, - validUntil: cert.validUntil, - privateKey: cert.privateKey, - publicKey: cert.publicKey, - csr: cert.csr || '', - }); + // Save to AcmeCertDoc (SmartAcme-compatible) + let acmeDoc = await AcmeCertDoc.findByDomain(cleanDomain); + if (!acmeDoc) { + acmeDoc = new AcmeCertDoc(); + acmeDoc.domainName = cleanDomain; + } + acmeDoc.id = cert.id; + acmeDoc.created = cert.created; + acmeDoc.validUntil = cert.validUntil; + acmeDoc.privateKey = cert.privateKey; + acmeDoc.publicKey = cert.publicKey; + acmeDoc.csr = cert.csr || ''; + await acmeDoc.save(); - // Also save to /proxy-certs/ (proxy-cert format) - await dcRouter.storageManager.setJSON(`/proxy-certs/${cert.domainName}`, { - domain: cert.domainName, - publicKey: cert.publicKey, - privateKey: cert.privateKey, - ca: undefined, - validUntil: cert.validUntil, - validFrom: cert.created, - }); + // Also save to ProxyCertDoc (proxy-cert format) + let proxyDoc = await ProxyCertDoc.findByDomain(cert.domainName); + if (!proxyDoc) { + proxyDoc = new ProxyCertDoc(); + proxyDoc.domain = cert.domainName; + } + proxyDoc.publicKey = cert.publicKey; + proxyDoc.privateKey = cert.privateKey; + proxyDoc.ca = ''; + proxyDoc.validUntil = cert.validUntil; + proxyDoc.validFrom = cert.created; + await proxyDoc.save(); // Update in-memory status map dcRouter.certificateStatusMap.set(cert.domainName, { diff --git a/ts/opsserver/handlers/config.handler.ts b/ts/opsserver/handlers/config.handler.ts index 157df14..19e7fe6 100644 --- a/ts/opsserver/handlers/config.handler.ts +++ b/ts/opsserver/handlers/config.handler.ts @@ -33,11 +33,9 @@ export class ConfigHandler { const resolvedPaths = dcRouter.resolvedPaths; // --- System --- - const storageBackend: 'filesystem' | 'custom' | 'memory' = opts.storage?.readFunction + const storageBackend: 'filesystem' | 'custom' | 'memory' = opts.dbConfig?.mongoDbUrl ? 'custom' - : opts.storage?.fsPath - ? 'filesystem' - : 'memory'; + : 'filesystem'; // Resolve proxy IPs: fall back to SmartProxy's runtime proxyIPs if not in opts let proxyIps = opts.proxyIps || []; @@ -55,7 +53,7 @@ export class ConfigHandler { proxyIps, uptime: Math.floor(process.uptime()), storageBackend, - storagePath: opts.storage?.fsPath || null, + storagePath: opts.dbConfig?.storagePath || resolvedPaths.defaultTsmDbPath, }; // --- SmartProxy --- @@ -151,15 +149,15 @@ export class ConfigHandler { keyPath: opts.tls?.keyPath || null, }; - // --- Cache --- - const cacheConfig = opts.cacheConfig; + // --- Database --- + const dbConfig = opts.dbConfig; const cache: interfaces.requests.IConfigData['cache'] = { - enabled: cacheConfig?.enabled !== false, - storagePath: cacheConfig?.storagePath || resolvedPaths.defaultTsmDbPath, - dbName: cacheConfig?.dbName || 'dcrouter', - defaultTTLDays: cacheConfig?.defaultTTLDays || 30, - cleanupIntervalHours: cacheConfig?.cleanupIntervalHours || 1, - ttlConfig: cacheConfig?.ttlConfig ? { ...cacheConfig.ttlConfig } as Record : {}, + enabled: dbConfig?.enabled !== false, + storagePath: dbConfig?.storagePath || resolvedPaths.defaultTsmDbPath, + dbName: dbConfig?.dbName || 'dcrouter', + defaultTTLDays: 30, + cleanupIntervalHours: dbConfig?.cleanupIntervalHours || 1, + ttlConfig: {}, }; // --- RADIUS --- @@ -185,7 +183,8 @@ export class ConfigHandler { tlsMode = 'custom'; } else if (riCfg?.hubDomain) { try { - const stored = await dcRouter.storageManager.getJSON(`/proxy-certs/${riCfg.hubDomain}`); + const { ProxyCertDoc } = await import('../../db/index.js'); + const stored = await ProxyCertDoc.findByDomain(riCfg.hubDomain); if (stored?.publicKey && stored?.privateKey) { tlsMode = 'acme'; } diff --git a/ts/paths.ts b/ts/paths.ts index a739c19..88b7f0e 100644 --- a/ts/paths.ts +++ b/ts/paths.ts @@ -34,7 +34,6 @@ export function resolvePaths(baseDir?: string) { dcrouterHomeDir: root, dataDir: resolvedDataDir, defaultTsmDbPath: plugins.path.join(root, 'tsmdb'), - defaultStoragePath: plugins.path.join(root, 'storage'), dnsRecordsDir: plugins.path.join(resolvedDataDir, 'dns'), }; } diff --git a/ts/radius/classes.accounting.manager.ts b/ts/radius/classes.accounting.manager.ts index 2a4c0ee..d31374e 100644 --- a/ts/radius/classes.accounting.manager.ts +++ b/ts/radius/classes.accounting.manager.ts @@ -1,6 +1,6 @@ import * as plugins from '../plugins.js'; import { logger } from '../logger.js'; -import type { StorageManager } from '../storage/index.js'; +import { AccountingSessionDoc } from '../db/index.js'; /** * RADIUS accounting session @@ -84,8 +84,6 @@ export interface IAccountingSummary { * Accounting manager configuration */ export interface IAccountingManagerConfig { - /** Storage key prefix */ - storagePrefix?: string; /** Session retention period in days (default: 30) */ retentionDays?: number; /** Enable detailed session logging */ @@ -106,7 +104,6 @@ export interface IAccountingManagerConfig { export class AccountingManager { private activeSessions: Map = new Map(); private config: Required; - private storageManager?: StorageManager; private staleSessionSweepTimer?: ReturnType; // Counters for statistics @@ -118,24 +115,20 @@ export class AccountingManager { interimUpdatesReceived: 0, }; - constructor(config?: IAccountingManagerConfig, storageManager?: StorageManager) { + constructor(config?: IAccountingManagerConfig) { this.config = { - storagePrefix: config?.storagePrefix ?? '/radius/accounting', retentionDays: config?.retentionDays ?? 30, detailedLogging: config?.detailedLogging ?? false, maxActiveSessions: config?.maxActiveSessions ?? 10000, staleSessionTimeoutHours: config?.staleSessionTimeoutHours ?? 24, }; - this.storageManager = storageManager; } /** * Initialize the accounting manager */ async initialize(): Promise { - if (this.storageManager) { - await this.loadActiveSessions(); - } + await this.loadActiveSessions(); // Start periodic sweep to evict stale sessions (every 15 minutes) this.staleSessionSweepTimer = setInterval(() => { @@ -176,9 +169,7 @@ export class AccountingManager { session.endTime = Date.now(); session.sessionTime = Math.floor((session.endTime - session.startTime) / 1000); - if (this.storageManager) { - this.archiveSession(session).catch(() => {}); - } + this.persistSession(session).catch(() => {}); this.activeSessions.delete(sessionId); swept++; @@ -250,9 +241,7 @@ export class AccountingManager { } // Persist session - if (this.storageManager) { - await this.persistSession(session); - } + await this.persistSession(session); } /** @@ -298,9 +287,7 @@ export class AccountingManager { } // Update persisted session - if (this.storageManager) { - await this.persistSession(session); - } + await this.persistSession(session); } /** @@ -353,10 +340,8 @@ export class AccountingManager { logger.log('info', `Accounting Stop: session=${data.sessionId}, duration=${session.sessionTime}s, in=${session.inputOctets}, out=${session.outputOctets}`); } - // Archive the session - if (this.storageManager) { - await this.archiveSession(session); - } + // Update status in the database (single collection, no active->archive move needed) + await this.persistSession(session); // Remove from active sessions this.activeSessions.delete(data.sessionId); @@ -493,23 +478,16 @@ export class AccountingManager { * Clean up old archived sessions based on retention policy */ async cleanupOldSessions(): Promise { - if (!this.storageManager) { - return 0; - } - const cutoffTime = Date.now() - this.config.retentionDays * 24 * 60 * 60 * 1000; let deletedCount = 0; try { - const keys = await this.storageManager.list(`${this.config.storagePrefix}/archive/`); + const oldDocs = await AccountingSessionDoc.findStoppedBefore(cutoffTime); - for (const key of keys) { + for (const doc of oldDocs) { try { - const session = await this.storageManager.getJSON(key); - if (session && session.endTime > 0 && session.endTime < cutoffTime) { - await this.storageManager.delete(key); - deletedCount++; - } + await doc.delete(); + deletedCount++; } catch (error) { // Ignore individual errors } @@ -552,9 +530,7 @@ export class AccountingManager { session.terminateCause = 'SessionEvicted'; session.endTime = Date.now(); - if (this.storageManager) { - await this.archiveSession(session); - } + await this.persistSession(session); this.activeSessions.delete(sessionId); logger.log('warn', `Evicted session ${sessionId} due to capacity limit`); @@ -562,25 +538,38 @@ export class AccountingManager { } /** - * Load active sessions from storage + * Load active sessions from database */ private async loadActiveSessions(): Promise { - if (!this.storageManager) { - return; - } - try { - const keys = await this.storageManager.list(`${this.config.storagePrefix}/active/`); + const docs = await AccountingSessionDoc.findActive(); - for (const key of keys) { - try { - const session = await this.storageManager.getJSON(key); - if (session && session.status === 'active') { - this.activeSessions.set(session.sessionId, session); - } - } catch (error) { - // Ignore individual errors - } + for (const doc of docs) { + const session: IAccountingSession = { + sessionId: doc.sessionId, + username: doc.username, + macAddress: doc.macAddress, + nasIpAddress: doc.nasIpAddress, + nasPort: doc.nasPort, + nasPortType: doc.nasPortType, + nasIdentifier: doc.nasIdentifier, + vlanId: doc.vlanId, + framedIpAddress: doc.framedIpAddress, + calledStationId: doc.calledStationId, + callingStationId: doc.callingStationId, + startTime: doc.startTime, + endTime: doc.endTime, + lastUpdateTime: doc.lastUpdateTime, + status: doc.status, + terminateCause: doc.terminateCause, + inputOctets: doc.inputOctets, + outputOctets: doc.outputOctets, + inputPackets: doc.inputPackets, + outputPackets: doc.outputPackets, + sessionTime: doc.sessionTime, + serviceType: doc.serviceType, + }; + this.activeSessions.set(session.sessionId, session); } } catch (error: unknown) { logger.log('warn', `Failed to load active sessions: ${(error as Error).message}`); @@ -588,70 +577,59 @@ export class AccountingManager { } /** - * Persist a session to storage + * Persist a session to the database (create or update) */ private async persistSession(session: IAccountingSession): Promise { - if (!this.storageManager) { - return; - } - - const key = `${this.config.storagePrefix}/active/${session.sessionId}.json`; try { - await this.storageManager.setJSON(key, session); + let doc = await AccountingSessionDoc.findBySessionId(session.sessionId); + if (!doc) { + doc = new AccountingSessionDoc(); + } + Object.assign(doc, session); + await doc.save(); } catch (error: unknown) { logger.log('error', `Failed to persist session ${session.sessionId}: ${(error as Error).message}`); } } /** - * Archive a completed session - */ - private async archiveSession(session: IAccountingSession): Promise { - if (!this.storageManager) { - return; - } - - try { - // Remove from active - const activeKey = `${this.config.storagePrefix}/active/${session.sessionId}.json`; - await this.storageManager.delete(activeKey); - - // Add to archive with date-based path - const date = new Date(session.endTime); - const archiveKey = `${this.config.storagePrefix}/archive/${date.getFullYear()}/${String(date.getMonth() + 1).padStart(2, '0')}/${String(date.getDate()).padStart(2, '0')}/${session.sessionId}.json`; - await this.storageManager.setJSON(archiveKey, session); - } catch (error: unknown) { - logger.log('error', `Failed to archive session ${session.sessionId}: ${(error as Error).message}`); - } - } - - /** - * Get archived sessions for a time period + * Get archived (stopped/terminated) sessions for a time period */ private async getArchivedSessions(startTime: number, endTime: number): Promise { - if (!this.storageManager) { - return []; - } - const sessions: IAccountingSession[] = []; try { - const keys = await this.storageManager.list(`${this.config.storagePrefix}/archive/`); + const docs = await AccountingSessionDoc.getInstances({ + status: { $in: ['stopped', 'terminated'] } as any, + endTime: { $gt: 0, $gte: startTime } as any, + startTime: { $lte: endTime } as any, + }); - for (const key of keys) { - try { - const session = await this.storageManager.getJSON(key); - if ( - session && - session.endTime > 0 && - session.startTime <= endTime && - session.endTime >= startTime - ) { - sessions.push(session); - } - } catch (error) { - // Ignore individual errors - } + for (const doc of docs) { + sessions.push({ + sessionId: doc.sessionId, + username: doc.username, + macAddress: doc.macAddress, + nasIpAddress: doc.nasIpAddress, + nasPort: doc.nasPort, + nasPortType: doc.nasPortType, + nasIdentifier: doc.nasIdentifier, + vlanId: doc.vlanId, + framedIpAddress: doc.framedIpAddress, + calledStationId: doc.calledStationId, + callingStationId: doc.callingStationId, + startTime: doc.startTime, + endTime: doc.endTime, + lastUpdateTime: doc.lastUpdateTime, + status: doc.status, + terminateCause: doc.terminateCause, + inputOctets: doc.inputOctets, + outputOctets: doc.outputOctets, + inputPackets: doc.inputPackets, + outputPackets: doc.outputPackets, + sessionTime: doc.sessionTime, + serviceType: doc.serviceType, + }); } } catch (error: unknown) { logger.log('warn', `Failed to get archived sessions: ${(error as Error).message}`); diff --git a/ts/radius/classes.radius.server.ts b/ts/radius/classes.radius.server.ts index 96be83c..533ec71 100644 --- a/ts/radius/classes.radius.server.ts +++ b/ts/radius/classes.radius.server.ts @@ -1,6 +1,5 @@ import * as plugins from '../plugins.js'; import { logger } from '../logger.js'; -import type { StorageManager } from '../storage/index.js'; import { VlanManager, type IMacVlanMapping, type IVlanManagerConfig } from './classes.vlan.manager.js'; import { AccountingManager, type IAccountingSession, type IAccountingManagerConfig } from './classes.accounting.manager.js'; @@ -92,7 +91,6 @@ export class RadiusServer { private vlanManager: VlanManager; private accountingManager: AccountingManager; private config: IRadiusServerConfig; - private storageManager?: StorageManager; private clientSecrets: Map = new Map(); private running: boolean = false; @@ -105,20 +103,19 @@ export class RadiusServer { startTime: 0, }; - constructor(config: IRadiusServerConfig, storageManager?: StorageManager) { + constructor(config: IRadiusServerConfig) { this.config = { authPort: config.authPort ?? 1812, acctPort: config.acctPort ?? 1813, bindAddress: config.bindAddress ?? '0.0.0.0', ...config, }; - this.storageManager = storageManager; // Initialize VLAN manager - this.vlanManager = new VlanManager(config.vlanAssignment, storageManager); + this.vlanManager = new VlanManager(config.vlanAssignment); // Initialize accounting manager - this.accountingManager = new AccountingManager(config.accounting, storageManager); + this.accountingManager = new AccountingManager(config.accounting); } /** diff --git a/ts/radius/classes.vlan.manager.ts b/ts/radius/classes.vlan.manager.ts index 5f39e68..72457d4 100644 --- a/ts/radius/classes.vlan.manager.ts +++ b/ts/radius/classes.vlan.manager.ts @@ -1,6 +1,6 @@ import * as plugins from '../plugins.js'; import { logger } from '../logger.js'; -import type { StorageManager } from '../storage/index.js'; +import { VlanMappingsDoc } from '../db/index.js'; /** * MAC address to VLAN mapping @@ -42,8 +42,6 @@ export interface IVlanManagerConfig { defaultVlan?: number; /** Whether to allow unknown MACs (assign default VLAN) or reject */ allowUnknownMacs?: boolean; - /** Storage key prefix for persistence */ - storagePrefix?: string; } /** @@ -56,27 +54,22 @@ export interface IVlanManagerConfig { export class VlanManager { private mappings: Map = new Map(); private config: Required; - private storageManager?: StorageManager; // Cache for normalized MAC lookups private normalizedMacCache: Map = new Map(); - constructor(config?: IVlanManagerConfig, storageManager?: StorageManager) { + constructor(config?: IVlanManagerConfig) { this.config = { defaultVlan: config?.defaultVlan ?? 1, allowUnknownMacs: config?.allowUnknownMacs ?? true, - storagePrefix: config?.storagePrefix ?? '/radius/vlan-mappings', }; - this.storageManager = storageManager; } /** * Initialize the VLAN manager and load persisted mappings */ async initialize(): Promise { - if (this.storageManager) { - await this.loadMappings(); - } + await this.loadMappings(); logger.log('info', `VlanManager initialized with ${this.mappings.size} mappings, default VLAN: ${this.config.defaultVlan}`); } @@ -157,10 +150,8 @@ export class VlanManager { this.mappings.set(normalizedMac, fullMapping); - // Persist to storage - if (this.storageManager) { - await this.saveMappings(); - } + // Persist to database + await this.saveMappings(); logger.log('info', `VLAN mapping ${existingMapping ? 'updated' : 'added'}: ${normalizedMac} -> VLAN ${mapping.vlan}`); return fullMapping; @@ -173,7 +164,7 @@ export class VlanManager { const normalizedMac = this.normalizeMac(mac); const removed = this.mappings.delete(normalizedMac); - if (removed && this.storageManager) { + if (removed) { await this.saveMappings(); logger.log('info', `VLAN mapping removed: ${normalizedMac}`); } @@ -333,39 +324,36 @@ export class VlanManager { } /** - * Load mappings from storage + * Load mappings from database */ private async loadMappings(): Promise { - if (!this.storageManager) { - return; - } - try { - const data = await this.storageManager.getJSON(this.config.storagePrefix); - if (data && Array.isArray(data)) { - for (const mapping of data) { + const doc = await VlanMappingsDoc.load(); + if (doc && Array.isArray(doc.mappings)) { + for (const mapping of doc.mappings) { this.mappings.set(this.normalizeMac(mapping.mac), mapping); } - logger.log('info', `Loaded ${data.length} VLAN mappings from storage`); + logger.log('info', `Loaded ${doc.mappings.length} VLAN mappings from database`); } } catch (error: unknown) { - logger.log('warn', `Failed to load VLAN mappings from storage: ${(error as Error).message}`); + logger.log('warn', `Failed to load VLAN mappings from database: ${(error as Error).message}`); } } /** - * Save mappings to storage + * Save mappings to database */ private async saveMappings(): Promise { - if (!this.storageManager) { - return; - } - try { const mappings = Array.from(this.mappings.values()); - await this.storageManager.setJSON(this.config.storagePrefix, mappings); + let doc = await VlanMappingsDoc.load(); + if (!doc) { + doc = new VlanMappingsDoc(); + } + doc.mappings = mappings; + await doc.save(); } catch (error: unknown) { - logger.log('error', `Failed to save VLAN mappings to storage: ${(error as Error).message}`); + logger.log('error', `Failed to save VLAN mappings to database: ${(error as Error).message}`); } } } diff --git a/ts/radius/index.ts b/ts/radius/index.ts index fef2450..deede0a 100644 --- a/ts/radius/index.ts +++ b/ts/radius/index.ts @@ -6,7 +6,7 @@ * - VLAN assignment based on MAC addresses * - OUI (vendor prefix) pattern matching for device categorization * - RADIUS accounting for session tracking and billing - * - Integration with StorageManager for persistence + * - Integration with smartdata document classes for persistence */ export * from './classes.radius.server.js'; diff --git a/ts/remoteingress/classes.remoteingress-manager.ts b/ts/remoteingress/classes.remoteingress-manager.ts index 4e7f04f..a30537b 100644 --- a/ts/remoteingress/classes.remoteingress-manager.ts +++ b/ts/remoteingress/classes.remoteingress-manager.ts @@ -1,8 +1,6 @@ import * as plugins from '../plugins.js'; -import type { StorageManager } from '../storage/classes.storagemanager.js'; import type { IRemoteIngress, IDcRouterRouteConfig } from '../../ts_interfaces/data/remoteingress.js'; - -const STORAGE_PREFIX = '/remote-ingress/'; +import { RemoteIngressEdgeDoc } from '../db/index.js'; /** * Flatten a port range (number | number[] | Array<{from, to}>) to a sorted unique number array. @@ -27,33 +25,40 @@ function extractPorts(portRange: number | Array = new Map(); private routes: IDcRouterRouteConfig[] = []; - constructor(storageManager: StorageManager) { - this.storageManager = storageManager; + constructor() { } /** - * Load all edge registrations from storage into memory. + * Load all edge registrations from the database into memory. */ public async initialize(): Promise { - const keys = await this.storageManager.list(STORAGE_PREFIX); - for (const key of keys) { - const edge = await this.storageManager.getJSON(key); - if (edge) { - // Migration: old edges without autoDerivePorts default to true - if ((edge as any).autoDerivePorts === undefined) { - edge.autoDerivePorts = true; - await this.storageManager.setJSON(key, edge); - } - this.edges.set(edge.id, edge); + const docs = await RemoteIngressEdgeDoc.findAll(); + for (const doc of docs) { + // Migration: old edges without autoDerivePorts default to true + if ((doc as any).autoDerivePorts === undefined) { + doc.autoDerivePorts = true; + await doc.save(); } + const edge: IRemoteIngress = { + id: doc.id, + name: doc.name, + secret: doc.secret, + listenPorts: doc.listenPorts, + listenPortsUdp: doc.listenPortsUdp, + enabled: doc.enabled, + autoDerivePorts: doc.autoDerivePorts, + tags: doc.tags, + createdAt: doc.createdAt, + updatedAt: doc.updatedAt, + }; + this.edges.set(edge.id, edge); } } @@ -189,7 +194,9 @@ export class RemoteIngressManager { updatedAt: now, }; - await this.storageManager.setJSON(`${STORAGE_PREFIX}${id}`, edge); + const doc = new RemoteIngressEdgeDoc(); + Object.assign(doc, edge); + await doc.save(); this.edges.set(id, edge); return edge; } @@ -233,7 +240,11 @@ export class RemoteIngressManager { if (updates.tags !== undefined) edge.tags = updates.tags; edge.updatedAt = Date.now(); - await this.storageManager.setJSON(`${STORAGE_PREFIX}${id}`, edge); + const doc = await RemoteIngressEdgeDoc.findById(id); + if (doc) { + Object.assign(doc, edge); + await doc.save(); + } this.edges.set(id, edge); return edge; } @@ -245,7 +256,10 @@ export class RemoteIngressManager { if (!this.edges.has(id)) { return false; } - await this.storageManager.delete(`${STORAGE_PREFIX}${id}`); + const doc = await RemoteIngressEdgeDoc.findById(id); + if (doc) { + await doc.delete(); + } this.edges.delete(id); return true; } @@ -262,7 +276,11 @@ export class RemoteIngressManager { edge.secret = plugins.crypto.randomBytes(32).toString('hex'); edge.updatedAt = Date.now(); - await this.storageManager.setJSON(`${STORAGE_PREFIX}${id}`, edge); + const doc = await RemoteIngressEdgeDoc.findById(id); + if (doc) { + Object.assign(doc, edge); + await doc.save(); + } this.edges.set(id, edge); return edge.secret; } diff --git a/ts/security/classes.ipreputationchecker.ts b/ts/security/classes.ipreputationchecker.ts index 298724c..125323a 100644 --- a/ts/security/classes.ipreputationchecker.ts +++ b/ts/security/classes.ipreputationchecker.ts @@ -1,8 +1,8 @@ import * as plugins from '../plugins.js'; -import * as paths from '../paths.js'; import { logger } from '../logger.js'; import { SecurityLogger, SecurityLogLevel, SecurityEventType } from './classes.securitylogger.js'; import { LRUCache } from 'lru-cache'; +import { CachedIPReputation } from '../db/documents/classes.cached.ip.reputation.js'; /** * Reputation check result information @@ -52,7 +52,7 @@ export interface IIPReputationOptions { highRiskThreshold?: number; // Score below this is high risk mediumRiskThreshold?: number; // Score below this is medium risk lowRiskThreshold?: number; // Score below this is low risk - enableLocalCache?: boolean; // Whether to persist cache to disk (default: true) + enableLocalCache?: boolean; // Whether to persist cache to database (default: true) enableDNSBL?: boolean; // Whether to use DNSBL checks (default: true) enableIPInfo?: boolean; // Whether to use IP info service (default: true) } @@ -64,10 +64,7 @@ export class IPReputationChecker { private static instance: IPReputationChecker | undefined; private reputationCache: LRUCache; private options: Required; - private storageManager?: any; // StorageManager instance - private saveCacheTimer: ReturnType | null = null; - private static readonly SAVE_CACHE_DEBOUNCE_MS = 30_000; - + // Default DNSBL servers private static readonly DEFAULT_DNSBL_SERVERS = [ 'zen.spamhaus.org', // Spamhaus @@ -75,13 +72,13 @@ export class IPReputationChecker { 'b.barracudacentral.org', // Barracuda 'spam.dnsbl.sorbs.net', // SORBS 'dnsbl.sorbs.net', // SORBS (expanded) - 'cbl.abuseat.org', // Composite Blocking List + 'cbl.abuseat.org', // Composite Blocking List 'xbl.spamhaus.org', // Spamhaus XBL 'pbl.spamhaus.org', // Spamhaus PBL 'dnsbl-1.uceprotect.net', // UCEPROTECT 'psbl.surriel.com' // PSBL ]; - + // Default options private static readonly DEFAULT_OPTIONS: Required = { maxCacheSize: 10000, @@ -94,54 +91,40 @@ export class IPReputationChecker { enableDNSBL: true, enableIPInfo: true }; - + /** * Constructor for IPReputationChecker * @param options Configuration options - * @param storageManager Optional StorageManager instance for persistence */ - constructor(options: IIPReputationOptions = {}, storageManager?: any) { + constructor(options: IIPReputationOptions = {}) { // Merge with default options this.options = { ...IPReputationChecker.DEFAULT_OPTIONS, ...options }; - - this.storageManager = storageManager; - - // If no storage manager provided, log warning - if (!storageManager && this.options.enableLocalCache) { - logger.log('warn', - '⚠️ WARNING: IPReputationChecker initialized without StorageManager.\n' + - ' IP reputation cache will only be stored to filesystem.\n' + - ' Consider passing a StorageManager instance for better storage flexibility.' - ); - } - + // Initialize reputation cache this.reputationCache = new LRUCache({ max: this.options.maxCacheSize, ttl: this.options.cacheTTL, // Cache TTL }); - - // Load cache from disk if enabled + + // Load persisted reputations into in-memory cache if (this.options.enableLocalCache) { - // Fire and forget the load operation - this.loadCache().catch((error: unknown) => { + this.loadCacheFromDb().catch((error: unknown) => { logger.log('error', `Failed to load IP reputation cache during initialization: ${(error as Error).message}`); }); } } - + /** * Get the singleton instance of the checker * @param options Configuration options - * @param storageManager Optional StorageManager instance for persistence * @returns Singleton instance */ - public static getInstance(options: IIPReputationOptions = {}, storageManager?: any): IPReputationChecker { + public static getInstance(options: IIPReputationOptions = {}): IPReputationChecker { if (!IPReputationChecker.instance) { - IPReputationChecker.instance = new IPReputationChecker(options, storageManager); + IPReputationChecker.instance = new IPReputationChecker(options); } return IPReputationChecker.instance; } @@ -150,12 +133,6 @@ export class IPReputationChecker { * Reset the singleton instance (for shutdown/testing) */ public static resetInstance(): void { - if (IPReputationChecker.instance) { - if (IPReputationChecker.instance.saveCacheTimer) { - clearTimeout(IPReputationChecker.instance.saveCacheTimer); - IPReputationChecker.instance.saveCacheTimer = null; - } - } IPReputationChecker.instance = undefined; } @@ -171,8 +148,8 @@ export class IPReputationChecker { logger.log('warn', `Invalid IP address format: ${ip}`); return this.createErrorResult(ip, 'Invalid IP address format'); } - - // Check cache first + + // Check in-memory LRU cache first (fast path) const cachedResult = this.reputationCache.get(ip); if (cachedResult) { logger.log('info', `Using cached reputation data for IP ${ip}`, { @@ -181,7 +158,7 @@ export class IPReputationChecker { }); return cachedResult; } - + // Initialize empty result const result: IReputationResult = { score: 100, // Start with perfect score @@ -191,51 +168,53 @@ export class IPReputationChecker { isVPN: false, timestamp: Date.now() }; - + // Check IP against DNS blacklists if enabled if (this.options.enableDNSBL) { const dnsblResult = await this.checkDNSBL(ip); - + // Update result with DNSBL information result.score -= dnsblResult.listCount * 10; // Subtract 10 points per blacklist result.isSpam = dnsblResult.listCount > 0; result.blacklists = dnsblResult.lists; } - + // Get additional IP information if enabled if (this.options.enableIPInfo) { const ipInfo = await this.getIPInfo(ip); - + // Update result with IP info result.country = ipInfo.country; result.asn = ipInfo.asn; result.org = ipInfo.org; - + // Adjust score based on IP type if (ipInfo.type === IPType.PROXY || ipInfo.type === IPType.TOR || ipInfo.type === IPType.VPN) { result.score -= 30; // Subtract 30 points for proxies, Tor, VPNs - + // Set proxy flags result.isProxy = ipInfo.type === IPType.PROXY; result.isTor = ipInfo.type === IPType.TOR; result.isVPN = ipInfo.type === IPType.VPN; } } - + // Ensure score is between 0 and 100 result.score = Math.max(0, Math.min(100, result.score)); - - // Update cache with result + + // Update in-memory LRU cache this.reputationCache.set(ip, result); - - // Schedule debounced cache save if enabled + + // Persist to database if enabled (fire and forget) if (this.options.enableLocalCache) { - this.debouncedSaveCache(); + this.persistReputationToDb(ip, result).catch((error: unknown) => { + logger.log('error', `Failed to persist IP reputation for ${ip}: ${(error as Error).message}`); + }); } - + // Log the reputation check this.logReputationCheck(ip, result); - + return result; } catch (error: unknown) { logger.log('error', `Error checking IP reputation for ${ip}: ${(error as Error).message}`, { @@ -246,7 +225,7 @@ export class IPReputationChecker { return this.createErrorResult(ip, (error as Error).message); } } - + /** * Check an IP against DNS blacklists * @param ip IP address to check @@ -259,7 +238,7 @@ export class IPReputationChecker { try { // Reverse the IP for DNSBL queries const reversedIP = this.reverseIP(ip); - + const results = await Promise.allSettled( this.options.dnsblServers.map(async (server) => { try { @@ -274,14 +253,14 @@ export class IPReputationChecker { } }) ); - + // Extract successful lookups (listed in DNSBL) const lists = results - .filter((result): result is PromiseFulfilledResult => + .filter((result): result is PromiseFulfilledResult => result.status === 'fulfilled' && result.value !== null ) .map(result => result.value); - + return { listCount: lists.length, lists @@ -294,7 +273,7 @@ export class IPReputationChecker { }; } } - + /** * Get information about an IP address * @param ip IP address to check @@ -309,16 +288,16 @@ export class IPReputationChecker { try { // In a real implementation, this would use an IP data service API // For this implementation, we'll use a simplified approach - + // Check if it's a known Tor exit node (simplified) const isTor = ip.startsWith('171.25.') || ip.startsWith('185.220.') || ip.startsWith('95.216.'); - + // Check if it's a known VPN (simplified) const isVPN = ip.startsWith('185.156.') || ip.startsWith('37.120.'); - + // Check if it's a known proxy (simplified) const isProxy = ip.startsWith('34.92.') || ip.startsWith('34.206.'); - + // Determine IP type let type = IPType.UNKNOWN; if (isTor) { @@ -341,7 +320,7 @@ export class IPReputationChecker { type = IPType.RESIDENTIAL; } } - + // Return the information return { country: this.determineCountry(ip), // Simplified, would use geolocation service @@ -356,7 +335,7 @@ export class IPReputationChecker { }; } } - + /** * Simplified method to determine country from IP * In a real implementation, this would use a geolocation database or service @@ -371,7 +350,7 @@ export class IPReputationChecker { if (ip.startsWith('171.')) return 'DE'; return 'XX'; // Unknown } - + /** * Simplified method to determine organization from IP * In a real implementation, this would use an IP-to-org database or service @@ -387,7 +366,7 @@ export class IPReputationChecker { if (ip.startsWith('185.220.')) return 'Tor Exit Node'; return 'Unknown'; } - + /** * Reverse an IP address for DNSBL lookups (e.g., 1.2.3.4 -> 4.3.2.1) * @param ip IP address to reverse @@ -396,7 +375,7 @@ export class IPReputationChecker { private reverseIP(ip: string): string { return ip.split('.').reverse().join('.'); } - + /** * Create an error result for when reputation check fails * @param ip IP address @@ -414,7 +393,7 @@ export class IPReputationChecker { error: errorMessage }; } - + /** * Validate IP address format * @param ip IP address to validate @@ -425,7 +404,7 @@ export class IPReputationChecker { const ipv4Pattern = /^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/; return ipv4Pattern.test(ip); } - + /** * Log reputation check to security logger * @param ip IP address @@ -439,7 +418,7 @@ export class IPReputationChecker { } else if (result.score < this.options.mediumRiskThreshold) { logLevel = SecurityLogLevel.INFO; } - + // Log the check SecurityLogger.getInstance().logEvent({ level: logLevel, @@ -458,131 +437,76 @@ export class IPReputationChecker { success: !result.isSpam }); } - - /** - * Schedule a debounced cache save (at most once per SAVE_CACHE_DEBOUNCE_MS) - */ - private debouncedSaveCache(): void { - if (this.saveCacheTimer) { - return; // already scheduled - } - this.saveCacheTimer = setTimeout(() => { - this.saveCacheTimer = null; - this.saveCache().catch((error: unknown) => { - logger.log('error', `Failed to save IP reputation cache: ${(error as Error).message}`); - }); - }, IPReputationChecker.SAVE_CACHE_DEBOUNCE_MS); - } /** - * Save cache to disk or storage manager + * Persist a single IP reputation result to the database via CachedIPReputation */ - private async saveCache(): Promise { + private async persistReputationToDb(ip: string, result: IReputationResult): Promise { try { - // Convert cache entries to serializable array - const entries = Array.from(this.reputationCache.entries()).map(([ip, data]) => ({ - ip, - data - })); - - // Only save if we have entries - if (entries.length === 0) { - return; - } - - const cacheData = JSON.stringify(entries); - - // Save to storage manager if available - if (this.storageManager) { - await this.storageManager.set('/security/ip-reputation-cache.json', cacheData); - logger.log('info', `Saved ${entries.length} IP reputation cache entries to StorageManager`); + const data = { + score: result.score, + isSpam: result.isSpam, + isProxy: result.isProxy, + isTor: result.isTor, + isVPN: result.isVPN, + country: result.country, + asn: result.asn, + org: result.org, + blacklists: result.blacklists, + }; + + const existing = await CachedIPReputation.findByIP(ip); + if (existing) { + existing.updateReputation(data); + await existing.save(); } else { - // Fall back to filesystem - const cacheDir = plugins.path.join(paths.dataDir, 'security'); - plugins.fsUtils.ensureDirSync(cacheDir); - - const cacheFile = plugins.path.join(cacheDir, 'ip_reputation_cache.json'); - plugins.fsUtils.toFsSync(cacheData, cacheFile); - - logger.log('info', `Saved ${entries.length} IP reputation cache entries to disk`); + const doc = CachedIPReputation.fromReputationData(ip, data); + await doc.save(); } } catch (error: unknown) { - logger.log('error', `Failed to save IP reputation cache: ${(error as Error).message}`); + logger.log('error', `Failed to persist IP reputation for ${ip}: ${(error as Error).message}`); } } /** - * Load cache from disk or storage manager + * Load persisted reputations from CachedIPReputation documents into the in-memory LRU cache */ - private async loadCache(): Promise { + private async loadCacheFromDb(): Promise { try { - let cacheData: string | null = null; - let fromFilesystem = false; - - // Try to load from storage manager first - if (this.storageManager) { - try { - cacheData = await this.storageManager.get('/security/ip-reputation-cache.json'); - - if (!cacheData) { - // Check if data exists in filesystem and migrate it - const cacheFile = plugins.path.join(paths.dataDir, 'security', 'ip_reputation_cache.json'); - - if (plugins.fs.existsSync(cacheFile)) { - logger.log('info', 'Migrating IP reputation cache from filesystem to StorageManager'); - cacheData = plugins.fs.readFileSync(cacheFile, 'utf8'); - fromFilesystem = true; - - // Migrate to storage manager - await this.storageManager.set('/security/ip-reputation-cache.json', cacheData); - logger.log('info', 'IP reputation cache migrated to StorageManager successfully'); - - // Optionally delete the old file after successful migration - try { - plugins.fs.unlinkSync(cacheFile); - logger.log('info', 'Old cache file removed after migration'); - } catch (deleteError) { - logger.log('warn', `Could not delete old cache file: ${(deleteError as Error).message}`); - } - } - } - } catch (error: unknown) { - logger.log('error', `Error loading from StorageManager: ${(error as Error).message}`); - } - } else { - // No storage manager, load from filesystem - const cacheFile = plugins.path.join(paths.dataDir, 'security', 'ip_reputation_cache.json'); - - if (plugins.fs.existsSync(cacheFile)) { - cacheData = plugins.fs.readFileSync(cacheFile, 'utf8'); - fromFilesystem = true; + const docs = await CachedIPReputation.getInstances({}); + let loadedCount = 0; + + for (const doc of docs) { + // Skip expired documents + if (doc.isExpired()) { + continue; } + + const result: IReputationResult = { + score: doc.score, + isSpam: doc.isSpam, + isProxy: doc.isProxy, + isTor: doc.isTor, + isVPN: doc.isVPN, + country: doc.country || undefined, + asn: doc.asn || undefined, + org: doc.org || undefined, + blacklists: doc.blacklists || [], + timestamp: doc.lastAccessedAt?.getTime() ?? doc.createdAt?.getTime() ?? Date.now(), + }; + + this.reputationCache.set(doc.ipAddress, result); + loadedCount++; } - - // Parse and restore cache if data was found - if (cacheData) { - const entries = JSON.parse(cacheData); - - // Validate and filter entries - const now = Date.now(); - const validEntries = entries.filter(entry => { - const age = now - entry.data.timestamp; - return age < this.options.cacheTTL; // Only load entries that haven't expired - }); - - // Restore cache - for (const entry of validEntries) { - this.reputationCache.set(entry.ip, entry.data); - } - - const source = fromFilesystem ? 'disk' : 'StorageManager'; - logger.log('info', `Loaded ${validEntries.length} IP reputation cache entries from ${source}`); + + if (loadedCount > 0) { + logger.log('info', `Loaded ${loadedCount} IP reputation cache entries from database`); } } catch (error: unknown) { - logger.log('error', `Failed to load IP reputation cache: ${(error as Error).message}`); + logger.log('error', `Failed to load IP reputation cache from database: ${(error as Error).message}`); } } - + /** * Get the risk level for a reputation score * @param score Reputation score (0-100) @@ -599,21 +523,4 @@ export class IPReputationChecker { return 'trusted'; } } - - /** - * Update the storage manager after instantiation - * This is useful when the storage manager is not available at construction time - * @param storageManager The StorageManager instance to use - */ - public updateStorageManager(storageManager: any): void { - this.storageManager = storageManager; - logger.log('info', 'IPReputationChecker storage manager updated'); - - // If cache is enabled and we have entries, save them to the new storage manager - if (this.options.enableLocalCache && this.reputationCache.size > 0) { - this.saveCache().catch((error: unknown) => { - logger.log('error', `Failed to save cache to new storage manager: ${(error as Error).message}`); - }); - } - } -} \ No newline at end of file +} diff --git a/ts/storage/classes.storagemanager.ts b/ts/storage/classes.storagemanager.ts deleted file mode 100644 index c0f89d3..0000000 --- a/ts/storage/classes.storagemanager.ts +++ /dev/null @@ -1,404 +0,0 @@ -import * as plugins from '../plugins.js'; -import { logger } from '../logger.js'; - -// Promisify filesystem operations -const readFile = plugins.util.promisify(plugins.fs.readFile); -const writeFile = plugins.util.promisify(plugins.fs.writeFile); -const unlink = plugins.util.promisify(plugins.fs.unlink); -const rename = plugins.util.promisify(plugins.fs.rename); -const readdir = plugins.util.promisify(plugins.fs.readdir); - -/** - * Storage configuration interface - */ -export interface IStorageConfig { - /** Filesystem path for storage */ - fsPath?: string; - /** Custom read function */ - readFunction?: (key: string) => Promise; - /** Custom write function */ - writeFunction?: (key: string, value: string) => Promise; -} - -/** - * Storage backend type - */ -export type StorageBackend = 'filesystem' | 'custom' | 'memory'; - -/** - * Central storage manager for DcRouter - * Provides unified key-value storage with multiple backend support - */ -export class StorageManager { - private static readonly MAX_MEMORY_ENTRIES = 10_000; - private backend: StorageBackend; - private memoryStore: Map = new Map(); - private config: IStorageConfig; - private fsBasePath?: string; - - constructor(config?: IStorageConfig) { - this.config = config || {}; - - // Check if both fsPath and custom functions are provided - if (config?.fsPath && (config?.readFunction || config?.writeFunction)) { - console.warn( - '⚠️ WARNING: Both fsPath and custom read/write functions are configured.\n' + - ' Using custom read/write functions. fsPath will be ignored.' - ); - } - - // Determine backend based on configuration - if (config?.readFunction && config?.writeFunction) { - this.backend = 'custom'; - } else if (config?.fsPath) { - // Set up internal read/write functions for filesystem - this.backend = 'custom'; // Use custom backend with internal functions - this.fsBasePath = plugins.path.resolve(config.fsPath); - this.ensureDirectory(this.fsBasePath); - - // Set up internal filesystem read/write functions - this.config.readFunction = (key: string): Promise => this.fsRead(key); - this.config.writeFunction = async (key: string, value: string) => { - await this.fsWrite(key, value); - }; - } else { - this.backend = 'memory'; - this.showMemoryWarning(); - } - - logger.log('info', `StorageManager initialized with ${this.backend} backend`); - } - - /** - * Show warning when using memory backend - */ - private showMemoryWarning(): void { - console.warn( - '⚠️ WARNING: StorageManager is using in-memory storage.\n' + - ' Data will be lost when the process restarts.\n' + - ' Configure storage.fsPath or storage functions for persistence.' - ); - } - - /** - * Ensure directory exists for filesystem backend - */ - private async ensureDirectory(dirPath: string): Promise { - try { - await plugins.fsUtils.ensureDir(dirPath); - } catch (error: unknown) { - logger.log('error', `Failed to create storage directory: ${(error as Error).message}`); - throw error; - } - } - - /** - * Validate and sanitize storage key - */ - private validateKey(key: string): string { - if (!key || typeof key !== 'string') { - throw new Error('Storage key must be a non-empty string'); - } - - // Ensure key starts with / - if (!key.startsWith('/')) { - key = '/' + key; - } - - // Remove any dangerous path elements - key = key.replace(/\.\./g, '').replace(/\/+/g, '/'); - - return key; - } - - /** - * Convert key to filesystem path - */ - private keyToPath(key: string): string { - if (!this.fsBasePath) { - throw new Error('Filesystem base path not configured'); - } - - // Remove leading slash and convert to path - const relativePath = key.substring(1); - return plugins.path.join(this.fsBasePath, relativePath); - } - - /** - * Internal filesystem read function - */ - private async fsRead(key: string): Promise { - const filePath = this.keyToPath(key); - try { - const content = await readFile(filePath, 'utf8'); - return content; - } catch (error: unknown) { - if ((error as any).code === 'ENOENT') { - return null; - } - throw error; - } - } - - /** - * Internal filesystem write function - */ - private async fsWrite(key: string, value: string): Promise { - const filePath = this.keyToPath(key); - const dir = plugins.path.dirname(filePath); - - // Ensure directory exists - await plugins.fsUtils.ensureDir(dir); - - // Write atomically with temp file - const tempPath = `${filePath}.tmp`; - await writeFile(tempPath, value, 'utf8'); - await rename(tempPath, filePath); - } - - /** - * Get value by key - */ - async get(key: string): Promise { - key = this.validateKey(key); - - try { - switch (this.backend) { - - case 'custom': { - if (!this.config.readFunction) { - throw new Error('Read function not configured'); - } - try { - return await this.config.readFunction(key); - } catch (error) { - // Assume null if read fails (key doesn't exist) - return null; - } - } - - case 'memory': { - return this.memoryStore.get(key) || null; - } - - default: - throw new Error(`Unknown backend: ${this.backend}`); - } - } catch (error: unknown) { - logger.log('error', `Storage get error for key ${key}: ${(error as Error).message}`); - throw error; - } - } - - /** - * Set value by key - */ - async set(key: string, value: string): Promise { - key = this.validateKey(key); - - if (typeof value !== 'string') { - throw new Error('Storage value must be a string'); - } - - try { - switch (this.backend) { - case 'filesystem': { - const filePath = this.keyToPath(key); - const dirPath = plugins.path.dirname(filePath); - - // Ensure directory exists - await plugins.fsUtils.ensureDir(dirPath); - - // Write atomically - const tempPath = filePath + '.tmp'; - await writeFile(tempPath, value, 'utf8'); - await rename(tempPath, filePath); - break; - } - - case 'custom': { - if (!this.config.writeFunction) { - throw new Error('Write function not configured'); - } - await this.config.writeFunction(key, value); - break; - } - - case 'memory': { - this.memoryStore.set(key, value); - // Evict oldest entries if memory store exceeds limit - while (this.memoryStore.size > StorageManager.MAX_MEMORY_ENTRIES) { - const firstKey = this.memoryStore.keys().next().value!; - this.memoryStore.delete(firstKey); - } - break; - } - - default: - throw new Error(`Unknown backend: ${this.backend}`); - } - } catch (error: unknown) { - logger.log('error', `Storage set error for key ${key}: ${(error as Error).message}`); - throw error; - } - } - - /** - * Delete value by key - */ - async delete(key: string): Promise { - key = this.validateKey(key); - - try { - switch (this.backend) { - case 'filesystem': { - const filePath = this.keyToPath(key); - try { - await unlink(filePath); - } catch (error: unknown) { - if ((error as any).code !== 'ENOENT') { - throw error; - } - } - break; - } - - case 'custom': { - // Try to delete by setting empty value - if (this.config.writeFunction) { - await this.config.writeFunction(key, ''); - } - break; - } - - case 'memory': { - this.memoryStore.delete(key); - break; - } - - default: - throw new Error(`Unknown backend: ${this.backend}`); - } - } catch (error: unknown) { - logger.log('error', `Storage delete error for key ${key}: ${(error as Error).message}`); - throw error; - } - } - - /** - * List keys by prefix - */ - async list(prefix?: string): Promise { - prefix = prefix ? this.validateKey(prefix) : '/'; - - try { - switch (this.backend) { - case 'custom': { - // If we have fsBasePath, this is actually filesystem backend - if (this.fsBasePath) { - const basePath = this.keyToPath(prefix); - const keys: string[] = []; - - const walkDir = async (dir: string, baseDir: string): Promise => { - try { - const entries = await readdir(dir, { withFileTypes: true }); - - for (const entry of entries) { - const fullPath = plugins.path.join(dir, entry.name); - - if (entry.isDirectory()) { - await walkDir(fullPath, baseDir); - } else if (entry.isFile()) { - // Convert path back to key - const relativePath = plugins.path.relative(this.fsBasePath!, fullPath); - const key = '/' + relativePath.replace(/\\/g, '/'); - if (key.startsWith(prefix)) { - keys.push(key); - } - } - } - } catch (error: unknown) { - if ((error as any).code !== 'ENOENT') { - throw error; - } - } - }; - - await walkDir(basePath, basePath); - return keys.sort(); - } else { - // True custom backends need to implement their own listing - logger.log('warn', 'List operation not supported for custom backend'); - return []; - } - } - - case 'memory': { - const keys: string[] = []; - for (const key of this.memoryStore.keys()) { - if (key.startsWith(prefix)) { - keys.push(key); - } - } - return keys.sort(); - } - - default: - throw new Error(`Unknown backend: ${this.backend}`); - } - } catch (error: unknown) { - logger.log('error', `Storage list error for prefix ${prefix}: ${(error as Error).message}`); - throw error; - } - } - - /** - * Check if key exists - */ - async exists(key: string): Promise { - key = this.validateKey(key); - - try { - const value = await this.get(key); - return value !== null; - } catch (error) { - return false; - } - } - - /** - * Get storage backend type - */ - getBackend(): StorageBackend { - // If we're using custom backend with fsBasePath, report it as filesystem - if (this.backend === 'custom' && this.fsBasePath) { - return 'filesystem' as StorageBackend; - } - return this.backend; - } - - /** - * JSON helper: Get and parse JSON value - */ - async getJSON(key: string): Promise { - const value = await this.get(key); - if (value === null || value.trim() === '') { - return null; - } - - try { - return JSON.parse(value) as T; - } catch (error: unknown) { - logger.log('error', `Failed to parse JSON for key ${key}: ${(error as Error).message}`); - throw error; - } - } - - /** - * JSON helper: Set value as JSON - */ - async setJSON(key: string, value: any): Promise { - const jsonString = JSON.stringify(value, null, 2); - await this.set(key, jsonString); - } -} \ No newline at end of file diff --git a/ts/storage/index.ts b/ts/storage/index.ts deleted file mode 100644 index e8056ea..0000000 --- a/ts/storage/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -// Storage module exports -export * from './classes.storagemanager.js'; \ No newline at end of file diff --git a/ts/vpn/classes.vpn-manager.ts b/ts/vpn/classes.vpn-manager.ts index 7f0dc49..694ebc9 100644 --- a/ts/vpn/classes.vpn-manager.ts +++ b/ts/vpn/classes.vpn-manager.ts @@ -1,9 +1,6 @@ import * as plugins from '../plugins.js'; import { logger } from '../logger.js'; -import type { StorageManager } from '../storage/classes.storagemanager.js'; - -const STORAGE_PREFIX_KEYS = '/vpn/server-keys'; -const STORAGE_PREFIX_CLIENTS = '/vpn/clients/'; +import { VpnServerKeysDoc, VpnClientDoc } from '../db/index.js'; export interface IVpnManagerConfig { /** VPN subnet CIDR (default: '10.8.0.0/24') */ @@ -35,43 +32,17 @@ export interface IVpnManagerConfig { getClientAllowedIPs?: (clientTags: string[]) => Promise; } -interface IPersistedServerKeys { - noisePrivateKey: string; - noisePublicKey: string; - wgPrivateKey: string; - wgPublicKey: string; -} - -interface IPersistedClient { - clientId: string; - enabled: boolean; - serverDefinedClientTags?: string[]; - description?: string; - assignedIp?: string; - noisePublicKey: string; - wgPublicKey: string; - /** WireGuard private key — stored so exports and QR codes produce valid configs */ - wgPrivateKey?: string; - createdAt: number; - updatedAt: number; - expiresAt?: string; - /** @deprecated Legacy field — migrated to serverDefinedClientTags on load */ - tags?: string[]; -} - /** * Manages the SmartVPN server lifecycle and VPN client CRUD. - * Persists server keys and client registrations via StorageManager. + * Persists server keys and client registrations via smartdata document classes. */ export class VpnManager { - private storageManager: StorageManager; private config: IVpnManagerConfig; private vpnServer?: plugins.smartvpn.VpnServer; - private clients: Map = new Map(); - private serverKeys?: IPersistedServerKeys; + private clients: Map = new Map(); + private serverKeys?: VpnServerKeysDoc; - constructor(storageManager: StorageManager, config: IVpnManagerConfig) { - this.storageManager = storageManager; + constructor(config: IVpnManagerConfig) { this.config = config; } @@ -204,22 +175,21 @@ export class VpnManager { } // Persist client entry (including WG private key for export/QR) - const persisted: IPersistedClient = { - clientId: bundle.entry.clientId, - enabled: bundle.entry.enabled ?? true, - serverDefinedClientTags: bundle.entry.serverDefinedClientTags, - description: bundle.entry.description, - assignedIp: bundle.entry.assignedIp, - noisePublicKey: bundle.entry.publicKey, - wgPublicKey: bundle.entry.wgPublicKey || '', - wgPrivateKey: bundle.secrets?.wgPrivateKey - || bundle.wireguardConfig?.match(/PrivateKey\s*=\s*(.+)/)?.[1]?.trim(), - createdAt: Date.now(), - updatedAt: Date.now(), - expiresAt: bundle.entry.expiresAt, - }; - this.clients.set(persisted.clientId, persisted); - await this.persistClient(persisted); + const doc = new VpnClientDoc(); + doc.clientId = bundle.entry.clientId; + doc.enabled = bundle.entry.enabled ?? true; + doc.serverDefinedClientTags = bundle.entry.serverDefinedClientTags; + doc.description = bundle.entry.description; + doc.assignedIp = bundle.entry.assignedIp; + doc.noisePublicKey = bundle.entry.publicKey; + doc.wgPublicKey = bundle.entry.wgPublicKey || ''; + doc.wgPrivateKey = bundle.secrets?.wgPrivateKey + || bundle.wireguardConfig?.match(/PrivateKey\s*=\s*(.+)/)?.[1]?.trim(); + doc.createdAt = Date.now(); + doc.updatedAt = Date.now(); + doc.expiresAt = bundle.entry.expiresAt; + this.clients.set(doc.clientId, doc); + await this.persistClient(doc); this.config.onClientChanged?.(); return bundle; @@ -233,15 +203,18 @@ export class VpnManager { throw new Error('VPN server not running'); } await this.vpnServer.removeClient(clientId); + const doc = this.clients.get(clientId); this.clients.delete(clientId); - await this.storageManager.delete(`${STORAGE_PREFIX_CLIENTS}${clientId}`); + if (doc) { + await doc.delete(); + } this.config.onClientChanged?.(); } /** * List all registered clients (without secrets). */ - public listClients(): IPersistedClient[] { + public listClients(): VpnClientDoc[] { return [...this.clients.values()]; } @@ -407,8 +380,8 @@ export class VpnManager { // ── Private helpers ──────────────────────────────────────────────────── - private async loadOrGenerateServerKeys(): Promise { - const stored = await this.storageManager.getJSON(STORAGE_PREFIX_KEYS); + private async loadOrGenerateServerKeys(): Promise { + const stored = await VpnServerKeysDoc.load(); if (stored?.noisePrivateKey && stored?.wgPrivateKey) { logger.log('info', 'Loaded VPN server keys from storage'); return stored; @@ -424,38 +397,34 @@ export class VpnManager { const wgKeys = await tempServer.generateWgKeypair(); tempServer.stop(); - const keys: IPersistedServerKeys = { - noisePrivateKey: noiseKeys.privateKey, - noisePublicKey: noiseKeys.publicKey, - wgPrivateKey: wgKeys.privateKey, - wgPublicKey: wgKeys.publicKey, - }; + const doc = stored || new VpnServerKeysDoc(); + doc.noisePrivateKey = noiseKeys.privateKey; + doc.noisePublicKey = noiseKeys.publicKey; + doc.wgPrivateKey = wgKeys.privateKey; + doc.wgPublicKey = wgKeys.publicKey; + await doc.save(); - await this.storageManager.setJSON(STORAGE_PREFIX_KEYS, keys); logger.log('info', 'Generated and persisted new VPN server keys'); - return keys; + return doc; } private async loadPersistedClients(): Promise { - const keys = await this.storageManager.list(STORAGE_PREFIX_CLIENTS); - for (const key of keys) { - const client = await this.storageManager.getJSON(key); - if (client) { - // Migrate legacy `tags` → `serverDefinedClientTags` - if (!client.serverDefinedClientTags && client.tags) { - client.serverDefinedClientTags = client.tags; - delete client.tags; - await this.persistClient(client); - } - this.clients.set(client.clientId, client); + const docs = await VpnClientDoc.findAll(); + for (const doc of docs) { + // Migrate legacy `tags` → `serverDefinedClientTags` + if (!doc.serverDefinedClientTags && (doc as any).tags) { + doc.serverDefinedClientTags = (doc as any).tags; + (doc as any).tags = undefined; + await doc.save(); } + this.clients.set(doc.clientId, doc); } if (this.clients.size > 0) { logger.log('info', `Loaded ${this.clients.size} persisted VPN client(s)`); } } - private async persistClient(client: IPersistedClient): Promise { - await this.storageManager.setJSON(`${STORAGE_PREFIX_CLIENTS}${client.clientId}`, client); + private async persistClient(client: VpnClientDoc): Promise { + await client.save(); } } diff --git a/ts_oci_container/index.ts b/ts_oci_container/index.ts index b340de0..d094b8a 100644 --- a/ts_oci_container/index.ts +++ b/ts_oci_container/index.ts @@ -87,11 +87,11 @@ export function getOciContainerConfig(): IDcRouterOptions { } as IDcRouterOptions['emailConfig']; } - // Cache config + // DB config const cacheEnabled = process.env.DCROUTER_CACHE_ENABLED; if (cacheEnabled !== undefined) { - options.cacheConfig = { - ...options.cacheConfig, + options.dbConfig = { + ...options.dbConfig, enabled: cacheEnabled === 'true', }; } diff --git a/ts_web/00_commitinfo_data.ts b/ts_web/00_commitinfo_data.ts index cefdcf4..23d447d 100644 --- a/ts_web/00_commitinfo_data.ts +++ b/ts_web/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@serve.zone/dcrouter', - version: '11.23.5', + version: '12.0.0', description: 'A multifaceted routing service handling mail and SMS delivery functions.' }