Compare commits

..

6 Commits

Author SHA1 Message Date
81f8e543e1 v12.0.0
Some checks failed
Docker (tags) / security (push) Failing after 3s
Docker (tags) / test (push) Has been skipped
Docker (tags) / release (push) Has been skipped
Docker (tags) / metadata (push) Has been skipped
2026-03-31 15:31:16 +00:00
bb6c26484d BREAKING CHANGE(db): replace StorageManager and CacheDb with a unified smartdata-backed database layer 2026-03-31 15:31:16 +00:00
193a4bb180 v11.23.5
Some checks failed
Docker (tags) / security (push) Failing after 3s
Docker (tags) / test (push) Has been skipped
Docker (tags) / release (push) Has been skipped
Docker (tags) / metadata (push) Has been skipped
2026-03-31 11:51:46 +00:00
0d9e6a4925 fix(config): correct VPN mandatory flag default handling in route config manager 2026-03-31 11:51:45 +00:00
ece9e46be9 v11.23.4
Some checks failed
Docker (tags) / security (push) Failing after 3s
Docker (tags) / test (push) Has been skipped
Docker (tags) / release (push) Has been skipped
Docker (tags) / metadata (push) Has been skipped
2026-03-31 11:41:44 +00:00
918390a6a4 fix(deps): bump @push.rocks/smartvpn to 1.17.1 2026-03-31 11:41:44 +00:00
51 changed files with 1494 additions and 1695 deletions

View File

@@ -1,5 +1,24 @@
# Changelog
## 2026-03-31 - 12.0.0 - BREAKING CHANGE(db)
replace StorageManager and CacheDb with a unified smartdata-backed database layer
- introduces DcRouterDb with embedded LocalSmartDb or external MongoDB support via dbConfig
- migrates persisted routes, API tokens, VPN data, certificates, remote ingress, VLAN mappings, RADIUS accounting, and cache records to smartdata document classes
- removes StorageManager and CacheDb modules and renames configuration from cacheConfig to dbConfig
- updates certificate, security, remote ingress, VPN, and RADIUS components to read and write through document models
## 2026-03-31 - 11.23.5 - fix(config)
correct VPN mandatory flag default handling in route config manager
- Changes the VPN mandatory check so it only applies when explicitly set to true, matching the updated default behavior of false.
- Prevents routes from being treated as VPN-mandatory when the setting is omitted.
## 2026-03-31 - 11.23.4 - fix(deps)
bump @push.rocks/smartvpn to 1.17.1
- Updates the @push.rocks/smartvpn dependency from 1.16.5 to 1.17.1.
## 2026-03-31 - 11.23.3 - fix(ts_web)
update appstate to import interfaces from source TypeScript module path

View File

@@ -1,7 +1,7 @@
{
"name": "@serve.zone/dcrouter",
"private": false,
"version": "11.23.3",
"version": "12.0.0",
"description": "A multifaceted routing service handling mail and SMS delivery functions.",
"type": "module",
"exports": {
@@ -59,7 +59,7 @@
"@push.rocks/smartrx": "^3.0.10",
"@push.rocks/smartstate": "^2.3.0",
"@push.rocks/smartunique": "^3.0.9",
"@push.rocks/smartvpn": "1.16.5",
"@push.rocks/smartvpn": "1.17.1",
"@push.rocks/taskbuffer": "^8.0.2",
"@serve.zone/catalog": "^2.9.0",
"@serve.zone/interfaces": "^5.3.0",

10
pnpm-lock.yaml generated
View File

@@ -96,8 +96,8 @@ importers:
specifier: ^3.0.9
version: 3.0.9
'@push.rocks/smartvpn':
specifier: 1.16.5
version: 1.16.5
specifier: 1.17.1
version: 1.17.1
'@push.rocks/taskbuffer':
specifier: ^8.0.2
version: 8.0.2
@@ -1339,8 +1339,8 @@ packages:
'@push.rocks/smartversion@3.0.5':
resolution: {integrity: sha512-8MZSo1yqyaKxKq0Q5N188l4un++9GFWVbhCAX5mXJwewZHn97ujffTeL+eOQYpWFTEpUhaq1QhL4NhqObBCt1Q==}
'@push.rocks/smartvpn@1.16.5':
resolution: {integrity: sha512-wUau/Ad18p36AeIF5R33S45WEM78R7Y4SZSkWdxMdvKNIqSfn1mhf4Zd57iAtxvq+2iO246xfifBrATZWfjPeQ==}
'@push.rocks/smartvpn@1.17.1':
resolution: {integrity: sha512-oTOxNUrh+doL9AocgPnMbcYZKrWJhCeuqNotu1RfiteIV9DDdznvA+cl3nOgxD/ImUYrFPz6PUp5BEMogWcS8Q==}
'@push.rocks/smartwatch@6.4.0':
resolution: {integrity: sha512-KDswRgE/siBmZRCsRA07MtW5oF4c9uQEBkwTGPIWneHzksbCDsvs/7agKFEL7WnNifLNwo8w1K1qoiVWkX1fvw==}
@@ -6622,7 +6622,7 @@ snapshots:
'@types/semver': 7.7.1
semver: 7.7.4
'@push.rocks/smartvpn@1.16.5':
'@push.rocks/smartvpn@1.17.1':
dependencies:
'@push.rocks/smartnftables': 1.1.0
'@push.rocks/smartpath': 6.0.0

View File

@@ -1,120 +1,84 @@
# DCRouter Storage Overview
DCRouter uses two complementary storage systems: **StorageManager** for configuration and state, and **CacheDb** for time-limited cached data.
DCRouter uses a **unified database layer** backed by `@push.rocks/smartdata` for all persistent data. All data is stored as typed document classes in a single database.
## StorageManager (Key-Value Store)
## Database Modes
A lightweight, pluggable key-value store for configuration, credentials, and runtime state. Data is persisted as flat JSON files on disk by default.
### Default Path
```
~/.serve.zone/dcrouter/storage/
```
Configurable via `options.storage.fsPath` or `options.baseDir`.
### Backends
```typescript
// Filesystem (default)
storage: { fsPath: '/var/lib/dcrouter/data' }
// Custom (Redis, S3, etc.)
storage: {
readFunction: async (key) => await redis.get(key),
writeFunction: async (key, value) => await redis.set(key, value),
}
// In-memory (omit storage config — data lost on restart)
```
### What's Stored
| Prefix | Contents | Managed By |
|--------|----------|------------|
| `/vpn/server-keys` | VPN server Noise + WireGuard keypairs | `VpnManager` |
| `/vpn/clients/{clientId}` | VPN client registrations (keys, tags, description, assigned IP) | `VpnManager` |
| `/config-api/routes/{uuid}.json` | Programmatic routes (created via OpsServer API) | `RouteConfigManager` |
| `/config-api/tokens/{uuid}.json` | API tokens (hashed secrets, scopes, expiry) | `ApiTokenManager` |
| `/config-api/overrides/{routeName}.json` | Hardcoded route overrides (enable/disable) | `RouteConfigManager` |
| `/email/bounces/suppression-list.json` | Email bounce suppression list | `smartmta` |
| `/certs/*` | TLS certificates and ACME state | `SmartAcme` (via `StorageBackedCertManager`) |
### API
```typescript
// Read/write JSON
await storageManager.getJSON<T>(key);
await storageManager.setJSON(key, value);
// Raw string read/write
await storageManager.get(key);
await storageManager.set(key, value);
// List keys by prefix
await storageManager.list('/vpn/clients/');
// Delete
await storageManager.delete(key);
```
## CacheDb (Embedded MongoDB)
An embedded MongoDB-compatible database (via `@push.rocks/smartdb` + `@push.rocks/smartdata`) for cached data with automatic TTL-based cleanup.
### Default Path
### Embedded Mode (default)
When no external MongoDB URL is provided, DCRouter starts an embedded `LocalSmartDb` (Rust-based MongoDB-compatible engine) via `@push.rocks/smartdb`.
```
~/.serve.zone/dcrouter/tsmdb/
```
Configurable via `options.cacheConfig.storagePath`.
### What's Cached
| Document Type | Default TTL | Purpose |
|--------------|-------------|---------|
| `CachedEmail` | 30 days | Email metadata cache for dashboard display |
| `CachedIPReputation` | 1 day | IP reputation lookup results (DNSBL checks) |
### Configuration
### External Mode
Connect to any MongoDB-compatible database by providing a connection URL.
```typescript
cacheConfig: {
enabled: true, // default: true
storagePath: '~/.serve.zone/dcrouter/tsmdb', // default
dbName: 'dcrouter', // default
cleanupIntervalHours: 1, // how often to purge expired records
ttlConfig: {
emails: 30, // days
ipReputation: 1, // days
bounces: 30, // days (reserved)
dkimKeys: 90, // days (reserved)
suppression: 30, // days (reserved)
},
dbConfig: {
mongoDbUrl: 'mongodb://host:27017',
dbName: 'dcrouter',
}
```
### How It Works
1. `CacheDb` starts a `LocalSmartDb` instance (embedded MongoDB process)
2. `smartdata` connects to it via Unix socket
3. Document classes (`CachedEmail`, `CachedIPReputation`) are decorated with `@Collection` and use `smartdata` ORM
4. `CacheCleaner` runs on a timer, purging records older than their configured TTL
### Disabling
For development or lightweight deployments, disable the cache to avoid starting a MongoDB process:
## Configuration
```typescript
cacheConfig: { enabled: false }
dbConfig: {
enabled: true, // default: true
mongoDbUrl: undefined, // default: embedded LocalSmartDb
storagePath: '~/.serve.zone/dcrouter/tsmdb', // default (embedded mode only)
dbName: 'dcrouter', // default
cleanupIntervalHours: 1, // TTL cleanup interval
}
```
## When to Use Which
## Document Classes
| Use Case | System | Why |
|----------|--------|-----|
| VPN keys, API tokens, routes, certs | **StorageManager** | Small JSON blobs, key-value access, no queries needed |
| Email metadata, IP reputation | **CacheDb** | Time-series data, TTL expiry, potential for queries/aggregation |
| Runtime state (connected clients, metrics) | **Neither** | In-memory only, rebuilt on startup |
All data is stored as smartdata document classes in `ts/db/documents/`.
| Document Class | Collection | Unique Key | Purpose |
|---|---|---|---|
| `StoredRouteDoc` | storedRoutes | `id` | Programmatic routes (created via API) |
| `RouteOverrideDoc` | routeOverrides | `routeName` | Hardcoded route enable/disable overrides |
| `ApiTokenDoc` | apiTokens | `id` | API tokens (hashed secrets, scopes, expiry) |
| `VpnServerKeysDoc` | vpnServerKeys | `configId` (singleton) | VPN server Noise + WireGuard keypairs |
| `VpnClientDoc` | vpnClients | `clientId` | VPN client registrations |
| `AcmeCertDoc` | acmeCerts | `domainName` | ACME certificates and keys |
| `ProxyCertDoc` | proxyCerts | `domain` | SmartProxy TLS certificates |
| `CertBackoffDoc` | certBackoff | `domain` | Per-domain cert provision backoff state |
| `RemoteIngressEdgeDoc` | remoteIngressEdges | `id` | Edge node registrations |
| `VlanMappingsDoc` | vlanMappings | `configId` (singleton) | MAC-to-VLAN mapping table |
| `AccountingSessionDoc` | accountingSessions | `sessionId` | RADIUS accounting sessions |
| `CachedEmail` | cachedEmails | `id` | Email metadata (TTL: 30 days) |
| `CachedIPReputation` | cachedIPReputation | `ipAddress` | IP reputation results (TTL: 24 hours) |
## Architecture
```
DcRouterDb (singleton)
├── LocalSmartDb (embedded, Rust) ─── or ─── External MongoDB
└── SmartdataDb (ORM)
└── @Collection(() => getDb())
├── StoredRouteDoc
├── RouteOverrideDoc
├── ApiTokenDoc
├── VpnServerKeysDoc / VpnClientDoc
├── AcmeCertDoc / ProxyCertDoc / CertBackoffDoc
├── RemoteIngressEdgeDoc
├── VlanMappingsDoc / AccountingSessionDoc
├── CachedEmail (TTL)
└── CachedIPReputation (TTL)
```
### TTL Cleanup
`CacheCleaner` runs on a configurable interval (default: 1 hour) and removes expired documents where `expiresAt < now()`.
## Disabling
For tests or lightweight deployments without persistence:
```typescript
dbConfig: { enabled: false }
```

View File

@@ -130,7 +130,7 @@ tap.test('DcRouter class - Email config with domains and routes', async () => {
contactEmail: 'test@example.com'
},
opsServerPort: 3104,
cacheConfig: {
dbConfig: {
enabled: false,
}
};

View File

@@ -10,7 +10,7 @@ tap.test('should NOT instantiate DNS server when dnsNsDomains is not set', async
routes: []
},
opsServerPort: 3100,
cacheConfig: { enabled: false }
dbConfig: { enabled: false }
});
await dcRouter.start();

View File

@@ -10,7 +10,7 @@ tap.test('should start DCRouter with OpsServer', async () => {
testDcRouter = new DcRouter({
// Minimal config for testing
opsServerPort: 3102,
cacheConfig: { enabled: false },
dbConfig: { enabled: false },
});
await testDcRouter.start();

View File

@@ -10,7 +10,7 @@ tap.test('should start DCRouter with OpsServer', async () => {
testDcRouter = new DcRouter({
// Minimal config for testing
opsServerPort: 3101,
cacheConfig: { enabled: false },
dbConfig: { enabled: false },
});
await testDcRouter.start();

View File

@@ -10,7 +10,7 @@ tap.test('should start DCRouter with OpsServer', async () => {
testDcRouter = new DcRouter({
// Minimal config for testing
opsServerPort: 3103,
cacheConfig: { enabled: false },
dbConfig: { enabled: false },
});
await testDcRouter.start();

View File

@@ -1,289 +0,0 @@
import { tap, expect } from '@git.zone/tstest/tapbundle';
import * as plugins from '../ts/plugins.js';
import * as paths from '../ts/paths.js';
import { StorageManager } from '../ts/storage/classes.storagemanager.js';
import { promises as fs } from 'fs';
import * as path from 'path';
// Test data
const testData = {
string: 'Hello, World!',
json: { name: 'test', value: 42, nested: { data: true } },
largeString: 'x'.repeat(10000)
};
tap.test('Storage Manager - Memory Backend', async () => {
// Create StorageManager without config (defaults to memory)
const storage = new StorageManager();
// Test basic get/set
await storage.set('/test/key', testData.string);
const value = await storage.get('/test/key');
expect(value).toEqual(testData.string);
// Test JSON helpers
await storage.setJSON('/test/json', testData.json);
const jsonValue = await storage.getJSON('/test/json');
expect(jsonValue).toEqual(testData.json);
// Test exists
expect(await storage.exists('/test/key')).toEqual(true);
expect(await storage.exists('/nonexistent')).toEqual(false);
// Test delete
await storage.delete('/test/key');
expect(await storage.exists('/test/key')).toEqual(false);
// Test list
await storage.set('/items/1', 'one');
await storage.set('/items/2', 'two');
await storage.set('/other/3', 'three');
const items = await storage.list('/items');
expect(items.length).toEqual(2);
expect(items).toContain('/items/1');
expect(items).toContain('/items/2');
// Verify memory backend
expect(storage.getBackend()).toEqual('memory');
});
tap.test('Storage Manager - Filesystem Backend', async () => {
const testDir = path.join(paths.dataDir, '.test-storage');
// Clean up test directory if it exists
try {
await fs.rm(testDir, { recursive: true, force: true });
} catch {}
// Create StorageManager with filesystem path
const storage = new StorageManager({ fsPath: testDir });
// Test basic operations
await storage.set('/test/file', testData.string);
const value = await storage.get('/test/file');
expect(value).toEqual(testData.string);
// Verify file exists on disk
const filePath = path.join(testDir, 'test', 'file');
const fileExists = await fs.access(filePath).then(() => true).catch(() => false);
expect(fileExists).toEqual(true);
// Test atomic writes (temp file should not exist)
const tempPath = filePath + '.tmp';
const tempExists = await fs.access(tempPath).then(() => true).catch(() => false);
expect(tempExists).toEqual(false);
// Test nested paths
await storage.set('/deeply/nested/path/to/file', testData.largeString);
const nestedValue = await storage.get('/deeply/nested/path/to/file');
expect(nestedValue).toEqual(testData.largeString);
// Test list with filesystem
await storage.set('/fs/items/a', 'alpha');
await storage.set('/fs/items/b', 'beta');
await storage.set('/fs/other/c', 'gamma');
// Filesystem backend now properly supports list
const fsItems = await storage.list('/fs/items');
expect(fsItems.length).toEqual(2); // Should find both items
// Clean up
await fs.rm(testDir, { recursive: true, force: true });
});
tap.test('Storage Manager - Custom Function Backend', async () => {
// Create in-memory storage for custom functions
const customStore = new Map<string, string>();
const storage = new StorageManager({
readFunction: async (key: string) => {
return customStore.get(key) || null;
},
writeFunction: async (key: string, value: string) => {
customStore.set(key, value);
}
});
// Test basic operations
await storage.set('/custom/key', testData.string);
expect(customStore.has('/custom/key')).toEqual(true);
const value = await storage.get('/custom/key');
expect(value).toEqual(testData.string);
// Test that delete sets empty value (as per implementation)
await storage.delete('/custom/key');
expect(customStore.get('/custom/key')).toEqual('');
// Verify custom backend (filesystem is implemented as custom backend internally)
expect(storage.getBackend()).toEqual('custom');
});
tap.test('Storage Manager - Key Validation', async () => {
const storage = new StorageManager();
// Test key normalization
await storage.set('test/key', 'value1'); // Missing leading slash
const value1 = await storage.get('/test/key');
expect(value1).toEqual('value1');
// Test dangerous path elements are removed
await storage.set('/test/../danger/key', 'value2');
const value2 = await storage.get('/test/danger/key'); // .. is removed, not the whole path segment
expect(value2).toEqual('value2');
// Test multiple slashes are normalized
await storage.set('/test///multiple////slashes', 'value3');
const value3 = await storage.get('/test/multiple/slashes');
expect(value3).toEqual('value3');
// Test invalid keys throw errors
let emptyKeyError: Error | null = null;
try {
await storage.set('', 'value');
} catch (error) {
emptyKeyError = error as Error;
}
expect(emptyKeyError).toBeTruthy();
expect(emptyKeyError?.message).toEqual('Storage key must be a non-empty string');
let nullKeyError: Error | null = null;
try {
await storage.set(null as any, 'value');
} catch (error) {
nullKeyError = error as Error;
}
expect(nullKeyError).toBeTruthy();
expect(nullKeyError?.message).toEqual('Storage key must be a non-empty string');
});
tap.test('Storage Manager - Concurrent Access', async () => {
const storage = new StorageManager();
const promises: Promise<void>[] = [];
// Simulate concurrent writes
for (let i = 0; i < 100; i++) {
promises.push(storage.set(`/concurrent/key${i}`, `value${i}`));
}
await Promise.all(promises);
// Verify all writes succeeded
for (let i = 0; i < 100; i++) {
const value = await storage.get(`/concurrent/key${i}`);
expect(value).toEqual(`value${i}`);
}
// Test concurrent reads
const readPromises: Promise<string | null>[] = [];
for (let i = 0; i < 100; i++) {
readPromises.push(storage.get(`/concurrent/key${i}`));
}
const results = await Promise.all(readPromises);
for (let i = 0; i < 100; i++) {
expect(results[i]).toEqual(`value${i}`);
}
});
tap.test('Storage Manager - Backend Priority', async () => {
const testDir = path.join(paths.dataDir, '.test-storage-priority');
// Test that custom functions take priority over fsPath
let warningLogged = false;
const originalWarn = console.warn;
console.warn = (message: string) => {
if (message.includes('Using custom read/write functions')) {
warningLogged = true;
}
};
const storage = new StorageManager({
fsPath: testDir,
readFunction: async () => 'custom-value',
writeFunction: async () => {}
});
console.warn = originalWarn;
expect(warningLogged).toEqual(true);
expect(storage.getBackend()).toEqual('custom'); // Custom functions take priority
// Clean up
try {
await fs.rm(testDir, { recursive: true, force: true });
} catch {}
});
tap.test('Storage Manager - Error Handling', async () => {
// Test filesystem errors
const storage = new StorageManager({
readFunction: async () => {
throw new Error('Read error');
},
writeFunction: async () => {
throw new Error('Write error');
}
});
// Read errors should return null
const value = await storage.get('/error/key');
expect(value).toEqual(null);
// Write errors should propagate
let writeError: Error | null = null;
try {
await storage.set('/error/key', 'value');
} catch (error) {
writeError = error as Error;
}
expect(writeError).toBeTruthy();
expect(writeError?.message).toEqual('Write error');
// Test JSON parse errors
const jsonStorage = new StorageManager({
readFunction: async () => 'invalid json',
writeFunction: async () => {}
});
// Test JSON parse errors
let jsonError: Error | null = null;
try {
await jsonStorage.getJSON('/invalid/json');
} catch (error) {
jsonError = error as Error;
}
expect(jsonError).toBeTruthy();
expect(jsonError?.message).toContain('JSON');
});
tap.test('Storage Manager - List Operations', async () => {
const storage = new StorageManager();
// Populate storage with hierarchical data
await storage.set('/app/config/database', 'db-config');
await storage.set('/app/config/cache', 'cache-config');
await storage.set('/app/data/users/1', 'user1');
await storage.set('/app/data/users/2', 'user2');
await storage.set('/app/logs/error.log', 'errors');
// List root
const rootItems = await storage.list('/');
expect(rootItems.length).toBeGreaterThanOrEqual(5);
// List specific paths
const configItems = await storage.list('/app/config');
expect(configItems.length).toEqual(2);
expect(configItems).toContain('/app/config/database');
expect(configItems).toContain('/app/config/cache');
const userItems = await storage.list('/app/data/users');
expect(userItems.length).toEqual(2);
// List non-existent path
const emptyList = await storage.list('/nonexistent/path');
expect(emptyList.length).toEqual(0);
});
export default tap.start();

View File

@@ -49,8 +49,8 @@ const devRouter = new DcRouter({
{ clientId: 'admin-desktop', serverDefinedClientTags: ['admin'], description: 'Admin workstation' },
],
},
// Disable cache/mongo for dev
cacheConfig: { enabled: false },
// Disable db/mongo for dev
dbConfig: { enabled: false },
});
console.log('Starting DcRouter in development mode...');

View File

@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@serve.zone/dcrouter',
version: '11.23.3',
version: '12.0.0',
description: 'A multifaceted routing service handling mail and SMS delivery functions.'
}

View File

@@ -1,155 +0,0 @@
import * as plugins from '../plugins.js';
import { logger } from '../logger.js';
import { defaultTsmDbPath } from '../paths.js';
/**
* Configuration options for CacheDb
*/
export interface ICacheDbOptions {
/** Base storage path for TsmDB data (default: ~/.serve.zone/dcrouter/tsmdb) */
storagePath?: string;
/** Database name (default: dcrouter) */
dbName?: string;
/** Enable debug logging */
debug?: boolean;
}
/**
* CacheDb - Wrapper around LocalSmartDb and smartdata
*
* Provides persistent caching using smartdata as the ORM layer
* and LocalSmartDb as the embedded database engine.
*/
export class CacheDb {
private static instance: CacheDb | null = null;
private localSmartDb!: plugins.smartdb.LocalSmartDb;
private smartdataDb!: plugins.smartdata.SmartdataDb;
private options: Required<ICacheDbOptions>;
private isStarted: boolean = false;
constructor(options: ICacheDbOptions = {}) {
this.options = {
storagePath: options.storagePath || defaultTsmDbPath,
dbName: options.dbName || 'dcrouter',
debug: options.debug || false,
};
}
/**
* Get or create the singleton instance
*/
public static getInstance(options?: ICacheDbOptions): CacheDb {
if (!CacheDb.instance) {
CacheDb.instance = new CacheDb(options);
}
return CacheDb.instance;
}
/**
* Reset the singleton instance (useful for testing)
*/
public static resetInstance(): void {
CacheDb.instance = null;
}
/**
* Start the cache database
* - Initializes LocalSmartDb with file persistence
* - Connects smartdata to the LocalSmartDb via Unix socket
*/
public async start(): Promise<void> {
if (this.isStarted) {
logger.log('warn', 'CacheDb already started');
return;
}
try {
// Ensure storage directory exists
await plugins.fsUtils.ensureDir(this.options.storagePath);
// Create LocalSmartDb instance
this.localSmartDb = new plugins.smartdb.LocalSmartDb({
folderPath: this.options.storagePath,
});
// Start LocalSmartDb and get connection info
const connectionInfo = await this.localSmartDb.start();
if (this.options.debug) {
logger.log('debug', `LocalSmartDb started with URI: ${connectionInfo.connectionUri}`);
}
// Initialize smartdata with the connection URI
this.smartdataDb = new plugins.smartdata.SmartdataDb({
mongoDbUrl: connectionInfo.connectionUri,
mongoDbName: this.options.dbName,
});
await this.smartdataDb.init();
this.isStarted = true;
logger.log('info', `CacheDb started at ${this.options.storagePath}`);
} catch (error: unknown) {
logger.log('error', `Failed to start CacheDb: ${(error as Error).message}`);
throw error;
}
}
/**
* Stop the cache database
*/
public async stop(): Promise<void> {
if (!this.isStarted) {
return;
}
try {
// Close smartdata connection
if (this.smartdataDb) {
await this.smartdataDb.close();
}
// Stop LocalSmartDb
if (this.localSmartDb) {
await this.localSmartDb.stop();
}
this.isStarted = false;
logger.log('info', 'CacheDb stopped');
} catch (error: unknown) {
logger.log('error', `Error stopping CacheDb: ${(error as Error).message}`);
throw error;
}
}
/**
* Get the smartdata database instance
*/
public getDb(): plugins.smartdata.SmartdataDb {
if (!this.isStarted) {
throw new Error('CacheDb not started. Call start() first.');
}
return this.smartdataDb;
}
/**
* Check if the database is ready
*/
public isReady(): boolean {
return this.isStarted;
}
/**
* Get the storage path
*/
public getStoragePath(): string {
return this.options.storagePath;
}
/**
* Get the database name
*/
public getDbName(): string {
return this.options.dbName;
}
}

View File

@@ -1,2 +0,0 @@
export * from './classes.cached.email.js';
export * from './classes.cached.ip.reputation.js';

View File

@@ -1,5 +1,5 @@
import { logger } from './logger.js';
import type { StorageManager } from './storage/index.js';
import { CertBackoffDoc } from './db/index.js';
interface IBackoffEntry {
failures: number;
@@ -10,54 +10,68 @@ interface IBackoffEntry {
/**
* Manages certificate provisioning scheduling with:
* - Per-domain exponential backoff persisted in StorageManager
* - Per-domain exponential backoff persisted via CertBackoffDoc
*
* Note: Serial stagger queue was removed — smartacme v9 handles
* concurrency, per-domain dedup, and rate limiting internally.
*/
export class CertProvisionScheduler {
private storageManager: StorageManager;
private maxBackoffHours: number;
// In-memory backoff cache (mirrors storage for fast lookups)
private backoffCache = new Map<string, IBackoffEntry>();
constructor(
storageManager: StorageManager,
options?: { maxBackoffHours?: number }
) {
this.storageManager = storageManager;
this.maxBackoffHours = options?.maxBackoffHours ?? 24;
}
/**
* Storage key for a domain's backoff entry
* Sanitized domain key for storage lookups
*/
private backoffKey(domain: string): string {
const clean = domain.replace(/\*/g, '_wildcard_').replace(/[^a-zA-Z0-9._-]/g, '_');
return `/cert-backoff/${clean}`;
private sanitizeDomain(domain: string): string {
return domain.replace(/\*/g, '_wildcard_').replace(/[^a-zA-Z0-9._-]/g, '_');
}
/**
* Load backoff entry from storage (with in-memory cache)
* Load backoff entry from database (with in-memory cache)
*/
private async loadBackoff(domain: string): Promise<IBackoffEntry | null> {
const cached = this.backoffCache.get(domain);
if (cached) return cached;
const entry = await this.storageManager.getJSON<IBackoffEntry>(this.backoffKey(domain));
if (entry) {
const sanitized = this.sanitizeDomain(domain);
const doc = await CertBackoffDoc.findByDomain(sanitized);
if (doc) {
const entry: IBackoffEntry = {
failures: doc.failures,
lastFailure: doc.lastFailure,
retryAfter: doc.retryAfter,
lastError: doc.lastError,
};
this.backoffCache.set(domain, entry);
return entry;
}
return entry;
return null;
}
/**
* Save backoff entry to both cache and storage
* Save backoff entry to both cache and database
*/
private async saveBackoff(domain: string, entry: IBackoffEntry): Promise<void> {
this.backoffCache.set(domain, entry);
await this.storageManager.setJSON(this.backoffKey(domain), entry);
const sanitized = this.sanitizeDomain(domain);
let doc = await CertBackoffDoc.findByDomain(sanitized);
if (!doc) {
doc = new CertBackoffDoc();
doc.domain = sanitized;
}
doc.failures = entry.failures;
doc.lastFailure = entry.lastFailure;
doc.retryAfter = entry.retryAfter;
doc.lastError = entry.lastError || '';
await doc.save();
}
/**
@@ -107,9 +121,13 @@ export class CertProvisionScheduler {
async clearBackoff(domain: string): Promise<void> {
this.backoffCache.delete(domain);
try {
await this.storageManager.delete(this.backoffKey(domain));
const sanitized = this.sanitizeDomain(domain);
const doc = await CertBackoffDoc.findByDomain(sanitized);
if (doc) {
await doc.delete();
}
} catch {
// Ignore delete errors (key may not exist)
// Ignore delete errors (doc may not exist)
}
}

View File

@@ -11,12 +11,10 @@ import {
type IEmailDomainConfig,
} from '@push.rocks/smartmta';
import { logger } from './logger.js';
// Import storage manager
import { StorageManager, type IStorageConfig } from './storage/index.js';
import { StorageBackedCertManager } from './classes.storage-cert-manager.js';
import { CertProvisionScheduler } from './classes.cert-provision-scheduler.js';
// Import cache system
import { CacheDb, CacheCleaner, type ICacheDbOptions } from './cache/index.js';
// Import unified database
import { DcRouterDb, type IDcRouterDbConfig, CacheCleaner, ProxyCertDoc, AcmeCertDoc } from './db/index.js';
import { OpsServer } from './opsserver/index.js';
import { MetricsManager } from './monitoring/index.js';
@@ -122,37 +120,23 @@ export interface IDcRouterOptions {
/** Other DNS providers can be added here */
};
/** Storage configuration */
storage?: IStorageConfig;
/**
* Cache database configuration using smartdata and LocalTsmDb
* Provides persistent caching for emails, IP reputation, bounces, etc.
* Unified database configuration.
* All persistent data (config, certs, VPN, cache, etc.) is stored via smartdata.
* If mongoDbUrl is provided, connects to external MongoDB.
* Otherwise, starts an embedded LocalSmartDb automatically.
*/
cacheConfig?: {
/** Enable cache database (default: true) */
dbConfig?: {
/** Enable database (default: true). Set to false in tests to skip DB startup. */
enabled?: boolean;
/** Storage path for TsmDB data (default: ~/.serve.zone/dcrouter/tsmdb) */
/** External MongoDB connection URL. If absent, uses embedded LocalSmartDb. */
mongoDbUrl?: string;
/** Storage path for embedded database data (default: ~/.serve.zone/dcrouter/tsmdb) */
storagePath?: string;
/** Database name (default: dcrouter) */
dbName?: string;
/** Default TTL in days for cached items (default: 30) */
defaultTTLDays?: number;
/** Cleanup interval in hours (default: 1) */
/** Cache cleanup interval in hours (default: 1) */
cleanupIntervalHours?: number;
/** TTL configuration per data type (in days) */
ttlConfig?: {
/** Email cache TTL (default: 30 days) */
emails?: number;
/** IP reputation cache TTL (default: 1 day) */
ipReputation?: number;
/** Bounce records TTL (default: 30 days) */
bounces?: number;
/** DKIM keys TTL (default: 90 days) */
dkimKeys?: number;
/** Suppression list TTL (default: 30 days, can be permanent) */
suppression?: number;
};
};
/**
@@ -248,12 +232,20 @@ export class DcRouter {
public dnsServer?: plugins.smartdns.dnsServerMod.DnsServer;
public emailServer?: UnifiedEmailServer;
public radiusServer?: RadiusServer;
public storageManager: StorageManager;
public opsServer!: OpsServer;
public metricsManager?: MetricsManager;
// Cache system (smartdata + LocalTsmDb)
public cacheDb?: CacheDb;
// Compatibility shim for smartmta's DkimManager which calls dcRouter.storageManager.set()
public storageManager: any = {
get: async (_key: string) => null,
set: async (_key: string, _value: string) => {
// DKIM keys from smartmta — logged but not yet migrated to smartdata
logger.log('debug', `storageManager.set() called (compat shim) for key: ${_key}`);
},
};
// Unified database (smartdata + LocalSmartDb or external MongoDB)
public dcRouterDb?: DcRouterDb;
public cacheCleaner?: CacheCleaner;
// Remote Ingress
@@ -312,16 +304,6 @@ export class DcRouter {
// Resolve all data paths from baseDir
this.resolvedPaths = paths.resolvePaths(this.options.baseDir);
// Default storage to filesystem if not configured
if (!this.options.storage) {
this.options.storage = {
fsPath: this.resolvedPaths.defaultStoragePath,
};
}
// Initialize storage manager
this.storageManager = new StorageManager(this.options.storage);
// Initialize service manager and register all services
this.serviceManager = new plugins.taskbuffer.ServiceManager({
name: 'dcrouter',
@@ -350,23 +332,23 @@ export class DcRouter {
.withRetry({ maxRetries: 0 }),
);
// CacheDb: optional, no dependencies
if (this.options.cacheConfig?.enabled !== false) {
// DcRouterDb: optional, no dependencies — unified database for all persistence
if (this.options.dbConfig?.enabled !== false) {
this.serviceManager.addService(
new plugins.taskbuffer.Service('CacheDb')
new plugins.taskbuffer.Service('DcRouterDb')
.optional()
.withStart(async () => {
await this.setupCacheDb();
await this.setupDcRouterDb();
})
.withStop(async () => {
if (this.cacheCleaner) {
this.cacheCleaner.stop();
this.cacheCleaner = undefined;
}
if (this.cacheDb) {
await this.cacheDb.stop();
CacheDb.resetInstance();
this.cacheDb = undefined;
if (this.dcRouterDb) {
await this.dcRouterDb.stop();
DcRouterDb.resetInstance();
this.dcRouterDb = undefined;
}
})
.withRetry({ maxRetries: 2, baseDelayMs: 1000, maxDelayMs: 5000 }),
@@ -391,10 +373,10 @@ export class DcRouter {
.withRetry({ maxRetries: 1, baseDelayMs: 1000 }),
);
// SmartProxy: critical, depends on CacheDb (if enabled)
// SmartProxy: critical, depends on DcRouterDb (if enabled)
const smartProxyDeps: string[] = [];
if (this.options.cacheConfig?.enabled !== false) {
smartProxyDeps.push('CacheDb');
if (this.options.dbConfig?.enabled !== false) {
smartProxyDeps.push('DcRouterDb');
}
this.serviceManager.addService(
new plugins.taskbuffer.Service('SmartProxy')
@@ -455,36 +437,38 @@ export class DcRouter {
);
}
// ConfigManagers: optional, depends on SmartProxy
this.serviceManager.addService(
new plugins.taskbuffer.Service('ConfigManagers')
.optional()
.dependsOn('SmartProxy')
.withStart(async () => {
this.routeConfigManager = new RouteConfigManager(
this.storageManager,
() => this.getConstructorRoutes(),
() => this.smartProxy,
() => this.options.http3,
this.options.vpnConfig?.enabled
? (tags?: string[]) => {
if (tags?.length && this.vpnManager) {
return this.vpnManager.getClientIpsForServerDefinedTags(tags);
// ConfigManagers: optional, depends on SmartProxy + DcRouterDb
// Requires DcRouterDb to be enabled (document classes need the database)
if (this.options.dbConfig?.enabled !== false) {
this.serviceManager.addService(
new plugins.taskbuffer.Service('ConfigManagers')
.optional()
.dependsOn('SmartProxy', 'DcRouterDb')
.withStart(async () => {
this.routeConfigManager = new RouteConfigManager(
() => this.getConstructorRoutes(),
() => this.smartProxy,
() => this.options.http3,
this.options.vpnConfig?.enabled
? (tags?: string[]) => {
if (tags?.length && this.vpnManager) {
return this.vpnManager.getClientIpsForServerDefinedTags(tags);
}
return [this.options.vpnConfig?.subnet || '10.8.0.0/24'];
}
return [this.options.vpnConfig?.subnet || '10.8.0.0/24'];
}
: undefined,
);
this.apiTokenManager = new ApiTokenManager(this.storageManager);
await this.apiTokenManager.initialize();
await this.routeConfigManager.initialize();
})
.withStop(async () => {
this.routeConfigManager = undefined;
this.apiTokenManager = undefined;
})
.withRetry({ maxRetries: 2, baseDelayMs: 1000 }),
);
: undefined,
);
this.apiTokenManager = new ApiTokenManager();
await this.apiTokenManager.initialize();
await this.routeConfigManager.initialize();
})
.withStop(async () => {
this.routeConfigManager = undefined;
this.apiTokenManager = undefined;
})
.withRetry({ maxRetries: 2, baseDelayMs: 1000 }),
);
}
// Email Server: optional, depends on SmartProxy
if (this.options.emailConfig) {
@@ -695,14 +679,9 @@ export class DcRouter {
logger.log('info', `Remote Ingress: tunnel port=${this.options.remoteIngressConfig.tunnelPort || 8443}, edges=${edgeCount} registered/${connectedCount} connected`);
}
// Storage summary
if (this.storageManager && this.options.storage) {
logger.log('info', `Storage: path=${this.options.storage.fsPath || 'default'}`);
}
// Cache database summary
if (this.cacheDb) {
logger.log('info', `Cache Database: storage=${this.cacheDb.getStoragePath()}, db=${this.cacheDb.getDbName()}, cleaner=${this.cacheCleaner?.isActive() ? 'active' : 'inactive'} (${(this.options.cacheConfig?.cleanupIntervalHours || 1)}h interval)`);
// Database summary
if (this.dcRouterDb) {
logger.log('info', `Database: ${this.dcRouterDb.isEmbedded() ? 'embedded' : 'external'}, db=${this.dcRouterDb.getDbName()}, cleaner=${this.cacheCleaner?.isActive() ? 'active' : 'inactive'} (${(this.options.dbConfig?.cleanupIntervalHours || 1)}h interval)`);
}
// Service status summary from ServiceManager
@@ -723,31 +702,32 @@ export class DcRouter {
}
/**
* Set up the cache database (smartdata + LocalTsmDb)
* Set up the unified database (smartdata + LocalSmartDb or external MongoDB)
*/
private async setupCacheDb(): Promise<void> {
logger.log('info', 'Setting up CacheDb...');
private async setupDcRouterDb(): Promise<void> {
logger.log('info', 'Setting up DcRouterDb...');
const cacheConfig = this.options.cacheConfig || {};
const dbConfig = this.options.dbConfig || {};
// Initialize CacheDb singleton
this.cacheDb = CacheDb.getInstance({
storagePath: cacheConfig.storagePath || this.resolvedPaths.defaultTsmDbPath,
dbName: cacheConfig.dbName || 'dcrouter',
// Initialize DcRouterDb singleton
this.dcRouterDb = DcRouterDb.getInstance({
mongoDbUrl: dbConfig.mongoDbUrl,
storagePath: dbConfig.storagePath || this.resolvedPaths.defaultTsmDbPath,
dbName: dbConfig.dbName || 'dcrouter',
debug: false,
});
await this.cacheDb.start();
await this.dcRouterDb.start();
// Start the cache cleaner
const cleanupIntervalMs = (cacheConfig.cleanupIntervalHours || 1) * 60 * 60 * 1000;
this.cacheCleaner = new CacheCleaner(this.cacheDb, {
// Start the cache cleaner for TTL-based document cleanup
const cleanupIntervalMs = (dbConfig.cleanupIntervalHours || 1) * 60 * 60 * 1000;
this.cacheCleaner = new CacheCleaner(this.dcRouterDb, {
intervalMs: cleanupIntervalMs,
verbose: false,
});
this.cacheCleaner.start();
logger.log('info', `CacheDb initialized at ${this.cacheDb.getStoragePath()}`);
logger.log('info', `DcRouterDb ready (${this.dcRouterDb.isEmbedded() ? 'embedded' : 'external'})`);
}
/**
@@ -850,14 +830,11 @@ export class DcRouter {
acme: acmeConfig,
certStore: {
loadAll: async () => {
const keys = await this.storageManager.list('/proxy-certs/');
const docs = await ProxyCertDoc.findAll();
const certs: Array<{ domain: string; publicKey: string; privateKey: string; ca?: string }> = [];
for (const key of keys) {
const data = await this.storageManager.getJSON(key);
if (data) {
certs.push(data);
loadedCertEntries.push({ domain: data.domain, publicKey: data.publicKey, validUntil: data.validUntil, validFrom: data.validFrom });
}
for (const doc of docs) {
certs.push({ domain: doc.domain, publicKey: doc.publicKey, privateKey: doc.privateKey, ca: doc.ca });
loadedCertEntries.push({ domain: doc.domain, publicKey: doc.publicKey, validUntil: doc.validUntil, validFrom: doc.validFrom });
}
return certs;
},
@@ -869,18 +846,29 @@ export class DcRouter {
validUntil = new Date(x509.validTo).getTime();
validFrom = new Date(x509.validFrom).getTime();
} catch { /* PEM parsing failed */ }
await this.storageManager.setJSON(`/proxy-certs/${domain}`, {
domain, publicKey, privateKey, ca, validUntil, validFrom,
});
let doc = await ProxyCertDoc.findByDomain(domain);
if (!doc) {
doc = new ProxyCertDoc();
doc.domain = domain;
}
doc.publicKey = publicKey;
doc.privateKey = privateKey;
doc.ca = ca || '';
doc.validUntil = validUntil || 0;
doc.validFrom = validFrom || 0;
await doc.save();
},
remove: async (domain: string) => {
await this.storageManager.delete(`/proxy-certs/${domain}`);
const doc = await ProxyCertDoc.findByDomain(domain);
if (doc) {
await doc.delete();
}
},
},
};
// Initialize cert provision scheduler
this.certProvisionScheduler = new CertProvisionScheduler(this.storageManager);
this.certProvisionScheduler = new CertProvisionScheduler();
// If we have DNS challenge handlers, create SmartAcme instance and wire certProvisionFunction
// Note: SmartAcme.start() is NOT called here — it runs as a separate optional service
@@ -895,7 +883,7 @@ export class DcRouter {
}
this.smartAcme = new plugins.smartacme.SmartAcme({
accountEmail: acmeConfig?.accountEmail || this.options.tls?.contactEmail || 'admin@example.com',
certManager: new StorageBackedCertManager(this.storageManager),
certManager: new StorageBackedCertManager(),
environment: 'production',
challengeHandlers: challengeHandlers,
challengePriority: ['dns-01'],
@@ -1037,16 +1025,16 @@ export class DcRouter {
issuedAt = new Date(entry.validFrom).toISOString();
}
// Try SmartAcme /certs/ metadata as secondary source
// Try SmartAcme AcmeCertDoc metadata as secondary source
if (!expiryDate) {
try {
const cleanDomain = entry.domain.replace(/^\*\.?/, '');
const certMeta = await this.storageManager.getJSON(`/certs/${cleanDomain}`);
if (certMeta?.validUntil) {
expiryDate = new Date(certMeta.validUntil).toISOString();
const certDoc = await AcmeCertDoc.findByDomain(cleanDomain);
if (certDoc?.validUntil) {
expiryDate = new Date(certDoc.validUntil).toISOString();
}
if (certMeta?.created && !issuedAt) {
issuedAt = new Date(certMeta.created).toISOString();
if (certDoc?.created && !issuedAt) {
issuedAt = new Date(certDoc.created).toISOString();
}
} catch { /* no metadata available */ }
}
@@ -2030,7 +2018,7 @@ export class DcRouter {
logger.log('info', 'Setting up Remote Ingress hub...');
// Initialize the edge registration manager
this.remoteIngressManager = new RemoteIngressManager(this.storageManager);
this.remoteIngressManager = new RemoteIngressManager();
await this.remoteIngressManager.initialize();
// Pass current routes so the manager can derive edge ports from remoteIngress-tagged routes
@@ -2056,7 +2044,7 @@ export class DcRouter {
// Priority 2: Existing cert from SmartProxy cert store for hubDomain
if (!tlsConfig && riCfg.hubDomain) {
try {
const stored = await this.storageManager.getJSON(`/proxy-certs/${riCfg.hubDomain}`);
const stored = await ProxyCertDoc.findByDomain(riCfg.hubDomain);
if (stored?.publicKey && stored?.privateKey) {
tlsConfig = { certPem: stored.publicKey, keyPem: stored.privateKey };
logger.log('info', `Using stored ACME cert for RemoteIngress tunnel TLS: ${riCfg.hubDomain}`);
@@ -2090,7 +2078,7 @@ export class DcRouter {
logger.log('info', 'Setting up VPN server...');
this.vpnManager = new VpnManager(this.storageManager, {
this.vpnManager = new VpnManager({
subnet: this.options.vpnConfig.subnet,
wgListenPort: this.options.vpnConfig.wgListenPort,
dns: this.options.vpnConfig.dns,
@@ -2180,7 +2168,7 @@ export class DcRouter {
logger.log('info', 'Setting up RADIUS server...');
this.radiusServer = new RadiusServer(this.options.radiusConfig, this.storageManager);
this.radiusServer = new RadiusServer(this.options.radiusConfig);
await this.radiusServer.start();
logger.log('info', `RADIUS server started on ports ${this.options.radiusConfig.authPort || 1812} (auth) and ${this.options.radiusConfig.acctPort || 1813} (acct)`);

View File

@@ -1,46 +1,58 @@
import * as plugins from './plugins.js';
import { StorageManager } from './storage/index.js';
import { AcmeCertDoc } from './db/index.js';
/**
* ICertManager implementation backed by StorageManager.
* Persists SmartAcme certificates under a /certs/ key prefix so they
* ICertManager implementation backed by smartdata document classes.
* Persists SmartAcme certificates via AcmeCertDoc so they
* survive process restarts without re-hitting ACME.
*/
export class StorageBackedCertManager implements plugins.smartacme.ICertManager {
private keyPrefix = '/certs/';
constructor(private storageManager: StorageManager) {}
constructor() {}
async init(): Promise<void> {}
async retrieveCertificate(domainName: string): Promise<plugins.smartacme.Cert | null> {
const data = await this.storageManager.getJSON(this.keyPrefix + domainName);
if (!data) return null;
return new plugins.smartacme.Cert(data);
}
async storeCertificate(cert: plugins.smartacme.Cert): Promise<void> {
await this.storageManager.setJSON(this.keyPrefix + cert.domainName, {
id: cert.id,
domainName: cert.domainName,
created: cert.created,
privateKey: cert.privateKey,
publicKey: cert.publicKey,
csr: cert.csr,
validUntil: cert.validUntil,
const doc = await AcmeCertDoc.findByDomain(domainName);
if (!doc) return null;
return new plugins.smartacme.Cert({
id: doc.id,
domainName: doc.domainName,
created: doc.created,
privateKey: doc.privateKey,
publicKey: doc.publicKey,
csr: doc.csr,
validUntil: doc.validUntil,
});
}
async storeCertificate(cert: plugins.smartacme.Cert): Promise<void> {
let doc = await AcmeCertDoc.findByDomain(cert.domainName);
if (!doc) {
doc = new AcmeCertDoc();
doc.domainName = cert.domainName;
}
doc.id = cert.id;
doc.created = cert.created;
doc.privateKey = cert.privateKey;
doc.publicKey = cert.publicKey;
doc.csr = cert.csr;
doc.validUntil = cert.validUntil;
await doc.save();
}
async deleteCertificate(domainName: string): Promise<void> {
await this.storageManager.delete(this.keyPrefix + domainName);
const doc = await AcmeCertDoc.findByDomain(domainName);
if (doc) {
await doc.delete();
}
}
async close(): Promise<void> {}
async wipe(): Promise<void> {
const keys = await this.storageManager.list(this.keyPrefix);
for (const key of keys) {
await this.storageManager.delete(key);
const docs = await AcmeCertDoc.findAll();
for (const doc of docs) {
await doc.delete();
}
}
}

View File

@@ -1,19 +1,18 @@
import * as plugins from '../plugins.js';
import { logger } from '../logger.js';
import type { StorageManager } from '../storage/index.js';
import { ApiTokenDoc } from '../db/index.js';
import type {
IStoredApiToken,
IApiTokenInfo,
TApiTokenScope,
} from '../../ts_interfaces/data/route-management.js';
const TOKENS_PREFIX = '/config-api/tokens/';
const TOKEN_PREFIX_STR = 'dcr_';
export class ApiTokenManager {
private tokens = new Map<string, IStoredApiToken>();
constructor(private storageManager: StorageManager) {}
constructor() {}
public async initialize(): Promise<void> {
await this.loadTokens();
@@ -117,7 +116,8 @@ export class ApiTokenManager {
if (!this.tokens.has(id)) return false;
const token = this.tokens.get(id)!;
this.tokens.delete(id);
await this.storageManager.delete(`${TOKENS_PREFIX}${id}.json`);
const doc = await ApiTokenDoc.findById(id);
if (doc) await doc.delete();
logger.log('info', `API token '${token.name}' revoked (id: ${id})`);
return true;
}
@@ -157,17 +157,48 @@ export class ApiTokenManager {
// =========================================================================
private async loadTokens(): Promise<void> {
const keys = await this.storageManager.list(TOKENS_PREFIX);
for (const key of keys) {
if (!key.endsWith('.json')) continue;
const stored = await this.storageManager.getJSON<IStoredApiToken>(key);
if (stored?.id) {
this.tokens.set(stored.id, stored);
const docs = await ApiTokenDoc.findAll();
for (const doc of docs) {
if (doc.id) {
this.tokens.set(doc.id, {
id: doc.id,
name: doc.name,
tokenHash: doc.tokenHash,
scopes: doc.scopes,
createdAt: doc.createdAt,
expiresAt: doc.expiresAt,
lastUsedAt: doc.lastUsedAt,
createdBy: doc.createdBy,
enabled: doc.enabled,
});
}
}
}
private async persistToken(stored: IStoredApiToken): Promise<void> {
await this.storageManager.setJSON(`${TOKENS_PREFIX}${stored.id}.json`, stored);
const existing = await ApiTokenDoc.findById(stored.id);
if (existing) {
existing.name = stored.name;
existing.tokenHash = stored.tokenHash;
existing.scopes = stored.scopes;
existing.createdAt = stored.createdAt;
existing.expiresAt = stored.expiresAt;
existing.lastUsedAt = stored.lastUsedAt;
existing.createdBy = stored.createdBy;
existing.enabled = stored.enabled;
await existing.save();
} else {
const doc = new ApiTokenDoc();
doc.id = stored.id;
doc.name = stored.name;
doc.tokenHash = stored.tokenHash;
doc.scopes = stored.scopes;
doc.createdAt = stored.createdAt;
doc.expiresAt = stored.expiresAt;
doc.lastUsedAt = stored.lastUsedAt;
doc.createdBy = stored.createdBy;
doc.enabled = stored.enabled;
await doc.save();
}
}
}

View File

@@ -1,6 +1,6 @@
import * as plugins from '../plugins.js';
import { logger } from '../logger.js';
import type { StorageManager } from '../storage/index.js';
import { StoredRouteDoc, RouteOverrideDoc } from '../db/index.js';
import type {
IStoredRoute,
IRouteOverride,
@@ -10,16 +10,12 @@ import type {
import type { IDcRouterRouteConfig } from '../../ts_interfaces/data/remoteingress.js';
import { type IHttp3Config, augmentRouteWithHttp3 } from '../http3/index.js';
const ROUTES_PREFIX = '/config-api/routes/';
const OVERRIDES_PREFIX = '/config-api/overrides/';
export class RouteConfigManager {
private storedRoutes = new Map<string, IStoredRoute>();
private overrides = new Map<string, IRouteOverride>();
private warnings: IRouteWarning[] = [];
constructor(
private storageManager: StorageManager,
private getHardcodedRoutes: () => plugins.smartproxy.IRouteConfig[],
private getSmartProxy: () => plugins.smartproxy.SmartProxy | undefined,
private getHttp3Config?: () => IHttp3Config | undefined,
@@ -127,7 +123,8 @@ export class RouteConfigManager {
public async deleteRoute(id: string): Promise<boolean> {
if (!this.storedRoutes.has(id)) return false;
this.storedRoutes.delete(id);
await this.storageManager.delete(`${ROUTES_PREFIX}${id}.json`);
const doc = await StoredRouteDoc.findById(id);
if (doc) await doc.delete();
await this.applyRoutes();
return true;
}
@@ -148,7 +145,20 @@ export class RouteConfigManager {
updatedBy,
};
this.overrides.set(routeName, override);
await this.storageManager.setJSON(`${OVERRIDES_PREFIX}${routeName}.json`, override);
const existingDoc = await RouteOverrideDoc.findByRouteName(routeName);
if (existingDoc) {
existingDoc.enabled = override.enabled;
existingDoc.updatedAt = override.updatedAt;
existingDoc.updatedBy = override.updatedBy;
await existingDoc.save();
} else {
const doc = new RouteOverrideDoc();
doc.routeName = override.routeName;
doc.enabled = override.enabled;
doc.updatedAt = override.updatedAt;
doc.updatedBy = override.updatedBy;
await doc.save();
}
this.computeWarnings();
await this.applyRoutes();
}
@@ -156,7 +166,8 @@ export class RouteConfigManager {
public async removeOverride(routeName: string): Promise<boolean> {
if (!this.overrides.has(routeName)) return false;
this.overrides.delete(routeName);
await this.storageManager.delete(`${OVERRIDES_PREFIX}${routeName}.json`);
const doc = await RouteOverrideDoc.findByRouteName(routeName);
if (doc) await doc.delete();
this.computeWarnings();
await this.applyRoutes();
return true;
@@ -167,12 +178,17 @@ export class RouteConfigManager {
// =========================================================================
private async loadStoredRoutes(): Promise<void> {
const keys = await this.storageManager.list(ROUTES_PREFIX);
for (const key of keys) {
if (!key.endsWith('.json')) continue;
const stored = await this.storageManager.getJSON<IStoredRoute>(key);
if (stored?.id) {
this.storedRoutes.set(stored.id, stored);
const docs = await StoredRouteDoc.findAll();
for (const doc of docs) {
if (doc.id) {
this.storedRoutes.set(doc.id, {
id: doc.id,
route: doc.route,
enabled: doc.enabled,
createdAt: doc.createdAt,
updatedAt: doc.updatedAt,
createdBy: doc.createdBy,
});
}
}
if (this.storedRoutes.size > 0) {
@@ -181,12 +197,15 @@ export class RouteConfigManager {
}
private async loadOverrides(): Promise<void> {
const keys = await this.storageManager.list(OVERRIDES_PREFIX);
for (const key of keys) {
if (!key.endsWith('.json')) continue;
const override = await this.storageManager.getJSON<IRouteOverride>(key);
if (override?.routeName) {
this.overrides.set(override.routeName, override);
const docs = await RouteOverrideDoc.findAll();
for (const doc of docs) {
if (doc.routeName) {
this.overrides.set(doc.routeName, {
routeName: doc.routeName,
enabled: doc.enabled,
updatedAt: doc.updatedAt,
updatedBy: doc.updatedBy,
});
}
}
if (this.overrides.size > 0) {
@@ -195,7 +214,23 @@ export class RouteConfigManager {
}
private async persistRoute(stored: IStoredRoute): Promise<void> {
await this.storageManager.setJSON(`${ROUTES_PREFIX}${stored.id}.json`, stored);
const existingDoc = await StoredRouteDoc.findById(stored.id);
if (existingDoc) {
existingDoc.route = stored.route;
existingDoc.enabled = stored.enabled;
existingDoc.updatedAt = stored.updatedAt;
existingDoc.createdBy = stored.createdBy;
await existingDoc.save();
} else {
const doc = new StoredRouteDoc();
doc.id = stored.id;
doc.route = stored.route;
doc.enabled = stored.enabled;
doc.createdAt = stored.createdAt;
doc.updatedAt = stored.updatedAt;
doc.createdBy = stored.createdBy;
await doc.save();
}
}
// =========================================================================
@@ -261,7 +296,7 @@ export class RouteConfigManager {
const dcRoute = route as IDcRouterRouteConfig;
if (!dcRoute.vpn?.enabled) return route;
const allowList = vpnAllowList(dcRoute.vpn.allowedServerDefinedClientTags);
const mandatory = dcRoute.vpn.mandatory !== false; // defaults to true
const mandatory = dcRoute.vpn.mandatory === true; // defaults to false
return {
...route,
security: {

View File

@@ -1,6 +1,6 @@
import * as plugins from '../plugins.js';
import { logger } from '../logger.js';
import { CacheDb } from './classes.cachedb.js';
import { DcRouterDb } from './classes.dcrouter-db.js';
// Import document classes for cleanup
import { CachedEmail } from './documents/classes.cached.email.js';
@@ -26,10 +26,10 @@ export class CacheCleaner {
private cleanupInterval: ReturnType<typeof setInterval> | null = null;
private isRunning: boolean = false;
private options: Required<ICacheCleanerOptions>;
private cacheDb: CacheDb;
private dcRouterDb: DcRouterDb;
constructor(cacheDb: CacheDb, options: ICacheCleanerOptions = {}) {
this.cacheDb = cacheDb;
constructor(dcRouterDb: DcRouterDb, options: ICacheCleanerOptions = {}) {
this.dcRouterDb = dcRouterDb;
this.options = {
intervalMs: options.intervalMs || 60 * 60 * 1000, // 1 hour default
verbose: options.verbose || false,
@@ -86,8 +86,8 @@ export class CacheCleaner {
* Run a single cleanup cycle
*/
public async runCleanup(): Promise<void> {
if (!this.cacheDb.isReady()) {
logger.log('warn', 'CacheDb not ready, skipping cleanup');
if (!this.dcRouterDb.isReady()) {
logger.log('warn', 'DcRouterDb not ready, skipping cleanup');
return;
}

View File

@@ -0,0 +1,179 @@
import * as plugins from '../plugins.js';
import { logger } from '../logger.js';
import { defaultTsmDbPath } from '../paths.js';
/**
* Configuration options for the unified DCRouter database
*/
export interface IDcRouterDbConfig {
/** External MongoDB connection URL. If absent, uses embedded LocalSmartDb. */
mongoDbUrl?: string;
/** Storage path for embedded LocalSmartDb data (default: ~/.serve.zone/dcrouter/tsmdb) */
storagePath?: string;
/** Database name (default: dcrouter) */
dbName?: string;
/** Enable debug logging */
debug?: boolean;
}
/**
* DcRouterDb - Unified database layer for DCRouter
*
* Replaces both StorageManager (flat-file key-value) and CacheDb (embedded MongoDB).
* All data is stored as smartdata document classes in a single database.
*
* Two modes:
* - **Embedded** (default): Spawns a LocalSmartDb (Rust-based MongoDB-compatible engine)
* - **External**: Connects to a provided MongoDB URL
*/
export class DcRouterDb {
private static instance: DcRouterDb | null = null;
private localSmartDb: plugins.smartdb.LocalSmartDb | null = null;
private smartdataDb!: plugins.smartdata.SmartdataDb;
private options: Required<IDcRouterDbConfig>;
private isStarted: boolean = false;
constructor(options: IDcRouterDbConfig = {}) {
this.options = {
mongoDbUrl: options.mongoDbUrl || '',
storagePath: options.storagePath || defaultTsmDbPath,
dbName: options.dbName || 'dcrouter',
debug: options.debug || false,
};
}
/**
* Get or create the singleton instance
*/
public static getInstance(options?: IDcRouterDbConfig): DcRouterDb {
if (!DcRouterDb.instance) {
DcRouterDb.instance = new DcRouterDb(options);
}
return DcRouterDb.instance;
}
/**
* Reset the singleton instance (useful for testing)
*/
public static resetInstance(): void {
DcRouterDb.instance = null;
}
/**
* Start the database
* - If mongoDbUrl is provided, connects directly to external MongoDB
* - Otherwise, starts an embedded LocalSmartDb instance
*/
public async start(): Promise<void> {
if (this.isStarted) {
logger.log('warn', 'DcRouterDb already started');
return;
}
try {
let connectionUri: string;
if (this.options.mongoDbUrl) {
// External MongoDB mode
connectionUri = this.options.mongoDbUrl;
logger.log('info', `DcRouterDb connecting to external MongoDB`);
} else {
// Embedded LocalSmartDb mode
await plugins.fsUtils.ensureDir(this.options.storagePath);
this.localSmartDb = new plugins.smartdb.LocalSmartDb({
folderPath: this.options.storagePath,
});
const connectionInfo = await this.localSmartDb.start();
connectionUri = connectionInfo.connectionUri;
if (this.options.debug) {
logger.log('debug', `LocalSmartDb started with URI: ${connectionUri}`);
}
logger.log('info', `DcRouterDb started embedded instance at ${this.options.storagePath}`);
}
// Initialize smartdata ORM
this.smartdataDb = new plugins.smartdata.SmartdataDb({
mongoDbUrl: connectionUri,
mongoDbName: this.options.dbName,
});
await this.smartdataDb.init();
this.isStarted = true;
logger.log('info', `DcRouterDb ready (db: ${this.options.dbName})`);
} catch (error: unknown) {
logger.log('error', `Failed to start DcRouterDb: ${(error as Error).message}`);
throw error;
}
}
/**
* Stop the database
*/
public async stop(): Promise<void> {
if (!this.isStarted) {
return;
}
try {
// Close smartdata connection
if (this.smartdataDb) {
await this.smartdataDb.close();
}
// Stop embedded LocalSmartDb if running
if (this.localSmartDb) {
await this.localSmartDb.stop();
this.localSmartDb = null;
}
this.isStarted = false;
logger.log('info', 'DcRouterDb stopped');
} catch (error: unknown) {
logger.log('error', `Error stopping DcRouterDb: ${(error as Error).message}`);
throw error;
}
}
/**
* Get the smartdata database instance for @Collection decorators
*/
public getDb(): plugins.smartdata.SmartdataDb {
if (!this.isStarted) {
throw new Error('DcRouterDb not started. Call start() first.');
}
return this.smartdataDb;
}
/**
* Check if the database is ready
*/
public isReady(): boolean {
return this.isStarted;
}
/**
* Whether running in embedded mode (LocalSmartDb) vs external MongoDB
*/
public isEmbedded(): boolean {
return !this.options.mongoDbUrl;
}
/**
* Get the storage path (only relevant for embedded mode)
*/
public getStoragePath(): string {
return this.options.storagePath;
}
/**
* Get the database name
*/
public getDbName(): string {
return this.options.dbName;
}
}

View File

@@ -0,0 +1,106 @@
import * as plugins from '../../plugins.js';
import { DcRouterDb } from '../classes.dcrouter-db.js';
const getDb = () => DcRouterDb.getInstance().getDb();
@plugins.smartdata.Collection(() => getDb())
export class AccountingSessionDoc extends plugins.smartdata.SmartDataDbDoc<AccountingSessionDoc, AccountingSessionDoc> {
@plugins.smartdata.unI()
@plugins.smartdata.svDb()
public sessionId!: string;
@plugins.smartdata.svDb()
public username!: string;
@plugins.smartdata.svDb()
public macAddress!: string;
@plugins.smartdata.svDb()
public nasIpAddress!: string;
@plugins.smartdata.svDb()
public nasPort!: number;
@plugins.smartdata.svDb()
public nasPortType!: string;
@plugins.smartdata.svDb()
public nasIdentifier!: string;
@plugins.smartdata.svDb()
public vlanId!: number;
@plugins.smartdata.svDb()
public framedIpAddress!: string;
@plugins.smartdata.svDb()
public calledStationId!: string;
@plugins.smartdata.svDb()
public callingStationId!: string;
@plugins.smartdata.svDb()
public startTime!: number;
@plugins.smartdata.svDb()
public endTime!: number;
@plugins.smartdata.svDb()
public lastUpdateTime!: number;
@plugins.smartdata.index()
@plugins.smartdata.svDb()
public status!: 'active' | 'stopped' | 'terminated';
@plugins.smartdata.svDb()
public terminateCause!: string;
@plugins.smartdata.svDb()
public inputOctets!: number;
@plugins.smartdata.svDb()
public outputOctets!: number;
@plugins.smartdata.svDb()
public inputPackets!: number;
@plugins.smartdata.svDb()
public outputPackets!: number;
@plugins.smartdata.svDb()
public sessionTime!: number;
@plugins.smartdata.svDb()
public serviceType!: string;
constructor() {
super();
}
public static async findBySessionId(sessionId: string): Promise<AccountingSessionDoc | null> {
return await AccountingSessionDoc.getInstance({ sessionId });
}
public static async findActive(): Promise<AccountingSessionDoc[]> {
return await AccountingSessionDoc.getInstances({ status: 'active' });
}
public static async findByUsername(username: string): Promise<AccountingSessionDoc[]> {
return await AccountingSessionDoc.getInstances({ username });
}
public static async findByNas(nasIpAddress: string): Promise<AccountingSessionDoc[]> {
return await AccountingSessionDoc.getInstances({ nasIpAddress });
}
public static async findByVlan(vlanId: number): Promise<AccountingSessionDoc[]> {
return await AccountingSessionDoc.getInstances({ vlanId });
}
public static async findStoppedBefore(cutoffTime: number): Promise<AccountingSessionDoc[]> {
return await AccountingSessionDoc.getInstances({
status: { $in: ['stopped', 'terminated'] } as any,
endTime: { $lt: cutoffTime, $gt: 0 } as any,
});
}
}

View File

@@ -0,0 +1,41 @@
import * as plugins from '../../plugins.js';
import { DcRouterDb } from '../classes.dcrouter-db.js';
const getDb = () => DcRouterDb.getInstance().getDb();
@plugins.smartdata.Collection(() => getDb())
export class AcmeCertDoc extends plugins.smartdata.SmartDataDbDoc<AcmeCertDoc, AcmeCertDoc> {
@plugins.smartdata.unI()
@plugins.smartdata.svDb()
public domainName!: string;
@plugins.smartdata.svDb()
public id!: string;
@plugins.smartdata.svDb()
public created!: number;
@plugins.smartdata.svDb()
public privateKey!: string;
@plugins.smartdata.svDb()
public publicKey!: string;
@plugins.smartdata.svDb()
public csr!: string;
@plugins.smartdata.svDb()
public validUntil!: number;
constructor() {
super();
}
public static async findByDomain(domainName: string): Promise<AcmeCertDoc | null> {
return await AcmeCertDoc.getInstance({ domainName });
}
public static async findAll(): Promise<AcmeCertDoc[]> {
return await AcmeCertDoc.getInstances({});
}
}

View File

@@ -0,0 +1,56 @@
import * as plugins from '../../plugins.js';
import { DcRouterDb } from '../classes.dcrouter-db.js';
import type { TApiTokenScope } from '../../../ts_interfaces/data/route-management.js';
const getDb = () => DcRouterDb.getInstance().getDb();
@plugins.smartdata.Collection(() => getDb())
export class ApiTokenDoc extends plugins.smartdata.SmartDataDbDoc<ApiTokenDoc, ApiTokenDoc> {
@plugins.smartdata.unI()
@plugins.smartdata.svDb()
public id!: string;
@plugins.smartdata.svDb()
public name: string = '';
@plugins.smartdata.svDb()
public tokenHash!: string;
@plugins.smartdata.svDb()
public scopes!: TApiTokenScope[];
@plugins.smartdata.svDb()
public createdAt!: number;
@plugins.smartdata.svDb()
public expiresAt!: number | null;
@plugins.smartdata.svDb()
public lastUsedAt!: number | null;
@plugins.smartdata.svDb()
public createdBy!: string;
@plugins.smartdata.svDb()
public enabled!: boolean;
constructor() {
super();
}
public static async findById(id: string): Promise<ApiTokenDoc | null> {
return await ApiTokenDoc.getInstance({ id });
}
public static async findByTokenHash(tokenHash: string): Promise<ApiTokenDoc | null> {
return await ApiTokenDoc.getInstance({ tokenHash });
}
public static async findAll(): Promise<ApiTokenDoc[]> {
return await ApiTokenDoc.getInstances({});
}
public static async findEnabled(): Promise<ApiTokenDoc[]> {
return await ApiTokenDoc.getInstances({ enabled: true });
}
}

View File

@@ -1,6 +1,6 @@
import * as plugins from '../../plugins.js';
import { CachedDocument, TTL } from '../classes.cached.document.js';
import { CacheDb } from '../classes.cachedb.js';
import { DcRouterDb } from '../classes.dcrouter-db.js';
/**
* Email status in the cache
@@ -10,7 +10,7 @@ export type TCachedEmailStatus = 'pending' | 'processing' | 'delivered' | 'faile
/**
* Helper to get the smartdata database instance
*/
const getDb = () => CacheDb.getInstance().getDb();
const getDb = () => DcRouterDb.getInstance().getDb();
/**
* CachedEmail - Stores email queue items in the cache

View File

@@ -1,11 +1,11 @@
import * as plugins from '../../plugins.js';
import { CachedDocument, TTL } from '../classes.cached.document.js';
import { CacheDb } from '../classes.cachedb.js';
import { DcRouterDb } from '../classes.dcrouter-db.js';
/**
* Helper to get the smartdata database instance
*/
const getDb = () => CacheDb.getInstance().getDb();
const getDb = () => DcRouterDb.getInstance().getDb();
/**
* IP reputation result data

View File

@@ -0,0 +1,35 @@
import * as plugins from '../../plugins.js';
import { DcRouterDb } from '../classes.dcrouter-db.js';
const getDb = () => DcRouterDb.getInstance().getDb();
@plugins.smartdata.Collection(() => getDb())
export class CertBackoffDoc extends plugins.smartdata.SmartDataDbDoc<CertBackoffDoc, CertBackoffDoc> {
@plugins.smartdata.unI()
@plugins.smartdata.svDb()
public domain!: string;
@plugins.smartdata.svDb()
public failures!: number;
@plugins.smartdata.svDb()
public lastFailure!: string;
@plugins.smartdata.svDb()
public retryAfter!: string;
@plugins.smartdata.svDb()
public lastError!: string;
constructor() {
super();
}
public static async findByDomain(domain: string): Promise<CertBackoffDoc | null> {
return await CertBackoffDoc.getInstance({ domain });
}
public static async findAll(): Promise<CertBackoffDoc[]> {
return await CertBackoffDoc.getInstances({});
}
}

View File

@@ -0,0 +1,38 @@
import * as plugins from '../../plugins.js';
import { DcRouterDb } from '../classes.dcrouter-db.js';
const getDb = () => DcRouterDb.getInstance().getDb();
@plugins.smartdata.Collection(() => getDb())
export class ProxyCertDoc extends plugins.smartdata.SmartDataDbDoc<ProxyCertDoc, ProxyCertDoc> {
@plugins.smartdata.unI()
@plugins.smartdata.svDb()
public domain!: string;
@plugins.smartdata.svDb()
public publicKey!: string;
@plugins.smartdata.svDb()
public privateKey!: string;
@plugins.smartdata.svDb()
public ca!: string;
@plugins.smartdata.svDb()
public validUntil!: number;
@plugins.smartdata.svDb()
public validFrom!: number;
constructor() {
super();
}
public static async findByDomain(domain: string): Promise<ProxyCertDoc | null> {
return await ProxyCertDoc.getInstance({ domain });
}
public static async findAll(): Promise<ProxyCertDoc[]> {
return await ProxyCertDoc.getInstances({});
}
}

View File

@@ -0,0 +1,54 @@
import * as plugins from '../../plugins.js';
import { DcRouterDb } from '../classes.dcrouter-db.js';
const getDb = () => DcRouterDb.getInstance().getDb();
@plugins.smartdata.Collection(() => getDb())
export class RemoteIngressEdgeDoc extends plugins.smartdata.SmartDataDbDoc<RemoteIngressEdgeDoc, RemoteIngressEdgeDoc> {
@plugins.smartdata.unI()
@plugins.smartdata.svDb()
public id!: string;
@plugins.smartdata.svDb()
public name: string = '';
@plugins.smartdata.svDb()
public secret!: string;
@plugins.smartdata.svDb()
public listenPorts!: number[];
@plugins.smartdata.svDb()
public listenPortsUdp!: number[];
@plugins.smartdata.svDb()
public enabled!: boolean;
@plugins.smartdata.svDb()
public autoDerivePorts!: boolean;
@plugins.smartdata.svDb()
public tags!: string[];
@plugins.smartdata.svDb()
public createdAt!: number;
@plugins.smartdata.svDb()
public updatedAt!: number;
constructor() {
super();
}
public static async findById(id: string): Promise<RemoteIngressEdgeDoc | null> {
return await RemoteIngressEdgeDoc.getInstance({ id });
}
public static async findAll(): Promise<RemoteIngressEdgeDoc[]> {
return await RemoteIngressEdgeDoc.getInstances({});
}
public static async findEnabled(): Promise<RemoteIngressEdgeDoc[]> {
return await RemoteIngressEdgeDoc.getInstances({ enabled: true });
}
}

View File

@@ -0,0 +1,32 @@
import * as plugins from '../../plugins.js';
import { DcRouterDb } from '../classes.dcrouter-db.js';
const getDb = () => DcRouterDb.getInstance().getDb();
@plugins.smartdata.Collection(() => getDb())
export class RouteOverrideDoc extends plugins.smartdata.SmartDataDbDoc<RouteOverrideDoc, RouteOverrideDoc> {
@plugins.smartdata.unI()
@plugins.smartdata.svDb()
public routeName!: string;
@plugins.smartdata.svDb()
public enabled!: boolean;
@plugins.smartdata.svDb()
public updatedAt!: number;
@plugins.smartdata.svDb()
public updatedBy!: string;
constructor() {
super();
}
public static async findByRouteName(routeName: string): Promise<RouteOverrideDoc | null> {
return await RouteOverrideDoc.getInstance({ routeName });
}
public static async findAll(): Promise<RouteOverrideDoc[]> {
return await RouteOverrideDoc.getInstances({});
}
}

View File

@@ -0,0 +1,38 @@
import * as plugins from '../../plugins.js';
import { DcRouterDb } from '../classes.dcrouter-db.js';
const getDb = () => DcRouterDb.getInstance().getDb();
@plugins.smartdata.Collection(() => getDb())
export class StoredRouteDoc extends plugins.smartdata.SmartDataDbDoc<StoredRouteDoc, StoredRouteDoc> {
@plugins.smartdata.unI()
@plugins.smartdata.svDb()
public id!: string;
@plugins.smartdata.svDb()
public route!: plugins.smartproxy.IRouteConfig;
@plugins.smartdata.svDb()
public enabled!: boolean;
@plugins.smartdata.svDb()
public createdAt!: number;
@plugins.smartdata.svDb()
public updatedAt!: number;
@plugins.smartdata.svDb()
public createdBy!: string;
constructor() {
super();
}
public static async findById(id: string): Promise<StoredRouteDoc | null> {
return await StoredRouteDoc.getInstance({ id });
}
public static async findAll(): Promise<StoredRouteDoc[]> {
return await StoredRouteDoc.getInstances({});
}
}

View File

@@ -0,0 +1,32 @@
import * as plugins from '../../plugins.js';
import { DcRouterDb } from '../classes.dcrouter-db.js';
const getDb = () => DcRouterDb.getInstance().getDb();
export interface IMacVlanMapping {
mac: string;
vlan: number;
description?: string;
enabled: boolean;
createdAt: number;
updatedAt: number;
}
@plugins.smartdata.Collection(() => getDb())
export class VlanMappingsDoc extends plugins.smartdata.SmartDataDbDoc<VlanMappingsDoc, VlanMappingsDoc> {
@plugins.smartdata.unI()
@plugins.smartdata.svDb()
public configId: string = 'vlan-mappings';
@plugins.smartdata.svDb()
public mappings!: IMacVlanMapping[];
constructor() {
super();
this.mappings = [];
}
public static async load(): Promise<VlanMappingsDoc | null> {
return await VlanMappingsDoc.getInstance({ configId: 'vlan-mappings' });
}
}

View File

@@ -0,0 +1,57 @@
import * as plugins from '../../plugins.js';
import { DcRouterDb } from '../classes.dcrouter-db.js';
const getDb = () => DcRouterDb.getInstance().getDb();
@plugins.smartdata.Collection(() => getDb())
export class VpnClientDoc extends plugins.smartdata.SmartDataDbDoc<VpnClientDoc, VpnClientDoc> {
@plugins.smartdata.unI()
@plugins.smartdata.svDb()
public clientId!: string;
@plugins.smartdata.svDb()
public enabled!: boolean;
@plugins.smartdata.svDb()
public serverDefinedClientTags?: string[];
@plugins.smartdata.svDb()
public description?: string;
@plugins.smartdata.svDb()
public assignedIp?: string;
@plugins.smartdata.svDb()
public noisePublicKey!: string;
@plugins.smartdata.svDb()
public wgPublicKey!: string;
@plugins.smartdata.svDb()
public wgPrivateKey?: string;
@plugins.smartdata.svDb()
public createdAt!: number;
@plugins.smartdata.svDb()
public updatedAt!: number;
@plugins.smartdata.svDb()
public expiresAt?: string;
constructor() {
super();
}
public static async findByClientId(clientId: string): Promise<VpnClientDoc | null> {
return await VpnClientDoc.getInstance({ clientId });
}
public static async findAll(): Promise<VpnClientDoc[]> {
return await VpnClientDoc.getInstances({});
}
public static async findEnabled(): Promise<VpnClientDoc[]> {
return await VpnClientDoc.getInstances({ enabled: true });
}
}

View File

@@ -0,0 +1,31 @@
import * as plugins from '../../plugins.js';
import { DcRouterDb } from '../classes.dcrouter-db.js';
const getDb = () => DcRouterDb.getInstance().getDb();
@plugins.smartdata.Collection(() => getDb())
export class VpnServerKeysDoc extends plugins.smartdata.SmartDataDbDoc<VpnServerKeysDoc, VpnServerKeysDoc> {
@plugins.smartdata.unI()
@plugins.smartdata.svDb()
public configId: string = 'vpn-server-keys';
@plugins.smartdata.svDb()
public noisePrivateKey!: string;
@plugins.smartdata.svDb()
public noisePublicKey!: string;
@plugins.smartdata.svDb()
public wgPrivateKey!: string;
@plugins.smartdata.svDb()
public wgPublicKey!: string;
constructor() {
super();
}
public static async load(): Promise<VpnServerKeysDoc | null> {
return await VpnServerKeysDoc.getInstance({ configId: 'vpn-server-keys' });
}
}

24
ts/db/documents/index.ts Normal file
View File

@@ -0,0 +1,24 @@
// Cached/TTL document classes
export * from './classes.cached.email.js';
export * from './classes.cached.ip.reputation.js';
// Config document classes
export * from './classes.stored-route.doc.js';
export * from './classes.route-override.doc.js';
export * from './classes.api-token.doc.js';
// VPN document classes
export * from './classes.vpn-server-keys.doc.js';
export * from './classes.vpn-client.doc.js';
// Certificate document classes
export * from './classes.acme-cert.doc.js';
export * from './classes.proxy-cert.doc.js';
export * from './classes.cert-backoff.doc.js';
// Remote ingress document classes
export * from './classes.remote-ingress-edge.doc.js';
// RADIUS document classes
export * from './classes.vlan-mappings.doc.js';
export * from './classes.accounting-session.doc.js';

View File

@@ -1,6 +1,10 @@
// Core cache infrastructure
export * from './classes.cachedb.js';
// Unified database manager
export * from './classes.dcrouter-db.js';
// TTL base class and constants
export * from './classes.cached.document.js';
// Cache cleaner
export * from './classes.cache.cleaner.js';
// Document classes

View File

@@ -1,6 +1,7 @@
import * as plugins from '../../plugins.js';
import type { OpsServer } from '../classes.opsserver.js';
import * as interfaces from '../../../ts_interfaces/index.js';
import { AcmeCertDoc, ProxyCertDoc } from '../../db/index.js';
export class CertificateHandler {
constructor(private opsServerRef: OpsServer) {
@@ -187,30 +188,28 @@ export class CertificateHandler {
}
}
// Check persisted cert data from StorageManager
// Check persisted cert data from smartdata document classes
if (status === 'unknown') {
const cleanDomain = domain.replace(/^\*\.?/, '');
let certData = await dcRouter.storageManager.getJSON(`/certs/${cleanDomain}`);
if (!certData) {
// Also check certStore path (proxy-certs)
certData = await dcRouter.storageManager.getJSON(`/proxy-certs/${domain}`);
}
if (certData?.validUntil) {
expiryDate = new Date(certData.validUntil).toISOString();
if (certData.created) {
issuedAt = new Date(certData.created).toISOString();
const acmeDoc = await AcmeCertDoc.findByDomain(cleanDomain);
const proxyDoc = !acmeDoc ? await ProxyCertDoc.findByDomain(domain) : null;
if (acmeDoc?.validUntil) {
expiryDate = new Date(acmeDoc.validUntil).toISOString();
if (acmeDoc.created) {
issuedAt = new Date(acmeDoc.created).toISOString();
}
issuer = 'smartacme-dns-01';
} else if (certData?.publicKey) {
} else if (proxyDoc?.publicKey) {
// certStore has the cert — parse PEM for expiry
try {
const x509 = new plugins.crypto.X509Certificate(certData.publicKey);
const x509 = new plugins.crypto.X509Certificate(proxyDoc.publicKey);
expiryDate = new Date(x509.validTo).toISOString();
issuedAt = new Date(x509.validFrom).toISOString();
} catch { /* PEM parsing failed */ }
status = 'valid';
issuer = 'cert-store';
} else if (certData) {
} else if (acmeDoc || proxyDoc) {
status = 'valid';
issuer = 'cert-store';
}
@@ -366,18 +365,17 @@ export class CertificateHandler {
const dcRouter = this.opsServerRef.dcRouterRef;
const cleanDomain = domain.replace(/^\*\.?/, '');
// Delete from all known storage paths
const paths = [
`/proxy-certs/${domain}`,
`/proxy-certs/${cleanDomain}`,
`/certs/${cleanDomain}`,
];
// Delete from smartdata document classes
const acmeDoc = await AcmeCertDoc.findByDomain(cleanDomain);
if (acmeDoc) {
await acmeDoc.delete();
}
for (const path of paths) {
try {
await dcRouter.storageManager.delete(path);
} catch {
// Path may not exist — ignore
// Try both original domain and clean domain for proxy certs
for (const d of [domain, cleanDomain]) {
const proxyDoc = await ProxyCertDoc.findByDomain(d);
if (proxyDoc) {
await proxyDoc.delete();
}
}
@@ -408,43 +406,41 @@ export class CertificateHandler {
};
message?: string;
}> {
const dcRouter = this.opsServerRef.dcRouterRef;
const cleanDomain = domain.replace(/^\*\.?/, '');
// Try SmartAcme /certs/ path first (has full ICert fields)
let certData = await dcRouter.storageManager.getJSON(`/certs/${cleanDomain}`);
if (certData && certData.publicKey && certData.privateKey) {
// Try AcmeCertDoc first (has full ICert fields)
const acmeDoc = await AcmeCertDoc.findByDomain(cleanDomain);
if (acmeDoc && acmeDoc.publicKey && acmeDoc.privateKey) {
return {
success: true,
cert: {
id: certData.id || plugins.crypto.randomUUID(),
domainName: certData.domainName || domain,
created: certData.created || Date.now(),
validUntil: certData.validUntil || 0,
privateKey: certData.privateKey,
publicKey: certData.publicKey,
csr: certData.csr || '',
id: acmeDoc.id || plugins.crypto.randomUUID(),
domainName: acmeDoc.domainName || domain,
created: acmeDoc.created || Date.now(),
validUntil: acmeDoc.validUntil || 0,
privateKey: acmeDoc.privateKey,
publicKey: acmeDoc.publicKey,
csr: acmeDoc.csr || '',
},
};
}
// Fallback: try /proxy-certs/ with original domain
certData = await dcRouter.storageManager.getJSON(`/proxy-certs/${domain}`);
if (!certData || !certData.publicKey) {
// Try with clean domain
certData = await dcRouter.storageManager.getJSON(`/proxy-certs/${cleanDomain}`);
// Fallback: try ProxyCertDoc with original domain, then clean domain
let proxyDoc = await ProxyCertDoc.findByDomain(domain);
if (!proxyDoc || !proxyDoc.publicKey) {
proxyDoc = await ProxyCertDoc.findByDomain(cleanDomain);
}
if (certData && certData.publicKey && certData.privateKey) {
if (proxyDoc && proxyDoc.publicKey && proxyDoc.privateKey) {
return {
success: true,
cert: {
id: plugins.crypto.randomUUID(),
domainName: domain,
created: certData.validFrom || Date.now(),
validUntil: certData.validUntil || 0,
privateKey: certData.privateKey,
publicKey: certData.publicKey,
created: proxyDoc.validFrom || Date.now(),
validUntil: proxyDoc.validUntil || 0,
privateKey: proxyDoc.privateKey,
publicKey: proxyDoc.publicKey,
csr: '',
},
};
@@ -476,26 +472,32 @@ export class CertificateHandler {
const dcRouter = this.opsServerRef.dcRouterRef;
const cleanDomain = cert.domainName.replace(/^\*\.?/, '');
// Save to /certs/ (SmartAcme-compatible path)
await dcRouter.storageManager.setJSON(`/certs/${cleanDomain}`, {
id: cert.id,
domainName: cert.domainName,
created: cert.created,
validUntil: cert.validUntil,
privateKey: cert.privateKey,
publicKey: cert.publicKey,
csr: cert.csr || '',
});
// Save to AcmeCertDoc (SmartAcme-compatible)
let acmeDoc = await AcmeCertDoc.findByDomain(cleanDomain);
if (!acmeDoc) {
acmeDoc = new AcmeCertDoc();
acmeDoc.domainName = cleanDomain;
}
acmeDoc.id = cert.id;
acmeDoc.created = cert.created;
acmeDoc.validUntil = cert.validUntil;
acmeDoc.privateKey = cert.privateKey;
acmeDoc.publicKey = cert.publicKey;
acmeDoc.csr = cert.csr || '';
await acmeDoc.save();
// Also save to /proxy-certs/ (proxy-cert format)
await dcRouter.storageManager.setJSON(`/proxy-certs/${cert.domainName}`, {
domain: cert.domainName,
publicKey: cert.publicKey,
privateKey: cert.privateKey,
ca: undefined,
validUntil: cert.validUntil,
validFrom: cert.created,
});
// Also save to ProxyCertDoc (proxy-cert format)
let proxyDoc = await ProxyCertDoc.findByDomain(cert.domainName);
if (!proxyDoc) {
proxyDoc = new ProxyCertDoc();
proxyDoc.domain = cert.domainName;
}
proxyDoc.publicKey = cert.publicKey;
proxyDoc.privateKey = cert.privateKey;
proxyDoc.ca = '';
proxyDoc.validUntil = cert.validUntil;
proxyDoc.validFrom = cert.created;
await proxyDoc.save();
// Update in-memory status map
dcRouter.certificateStatusMap.set(cert.domainName, {

View File

@@ -33,11 +33,9 @@ export class ConfigHandler {
const resolvedPaths = dcRouter.resolvedPaths;
// --- System ---
const storageBackend: 'filesystem' | 'custom' | 'memory' = opts.storage?.readFunction
const storageBackend: 'filesystem' | 'custom' | 'memory' = opts.dbConfig?.mongoDbUrl
? 'custom'
: opts.storage?.fsPath
? 'filesystem'
: 'memory';
: 'filesystem';
// Resolve proxy IPs: fall back to SmartProxy's runtime proxyIPs if not in opts
let proxyIps = opts.proxyIps || [];
@@ -55,7 +53,7 @@ export class ConfigHandler {
proxyIps,
uptime: Math.floor(process.uptime()),
storageBackend,
storagePath: opts.storage?.fsPath || null,
storagePath: opts.dbConfig?.storagePath || resolvedPaths.defaultTsmDbPath,
};
// --- SmartProxy ---
@@ -151,15 +149,15 @@ export class ConfigHandler {
keyPath: opts.tls?.keyPath || null,
};
// --- Cache ---
const cacheConfig = opts.cacheConfig;
// --- Database ---
const dbConfig = opts.dbConfig;
const cache: interfaces.requests.IConfigData['cache'] = {
enabled: cacheConfig?.enabled !== false,
storagePath: cacheConfig?.storagePath || resolvedPaths.defaultTsmDbPath,
dbName: cacheConfig?.dbName || 'dcrouter',
defaultTTLDays: cacheConfig?.defaultTTLDays || 30,
cleanupIntervalHours: cacheConfig?.cleanupIntervalHours || 1,
ttlConfig: cacheConfig?.ttlConfig ? { ...cacheConfig.ttlConfig } as Record<string, number> : {},
enabled: dbConfig?.enabled !== false,
storagePath: dbConfig?.storagePath || resolvedPaths.defaultTsmDbPath,
dbName: dbConfig?.dbName || 'dcrouter',
defaultTTLDays: 30,
cleanupIntervalHours: dbConfig?.cleanupIntervalHours || 1,
ttlConfig: {},
};
// --- RADIUS ---
@@ -185,7 +183,8 @@ export class ConfigHandler {
tlsMode = 'custom';
} else if (riCfg?.hubDomain) {
try {
const stored = await dcRouter.storageManager.getJSON(`/proxy-certs/${riCfg.hubDomain}`);
const { ProxyCertDoc } = await import('../../db/index.js');
const stored = await ProxyCertDoc.findByDomain(riCfg.hubDomain);
if (stored?.publicKey && stored?.privateKey) {
tlsMode = 'acme';
}

View File

@@ -34,7 +34,6 @@ export function resolvePaths(baseDir?: string) {
dcrouterHomeDir: root,
dataDir: resolvedDataDir,
defaultTsmDbPath: plugins.path.join(root, 'tsmdb'),
defaultStoragePath: plugins.path.join(root, 'storage'),
dnsRecordsDir: plugins.path.join(resolvedDataDir, 'dns'),
};
}

View File

@@ -1,6 +1,6 @@
import * as plugins from '../plugins.js';
import { logger } from '../logger.js';
import type { StorageManager } from '../storage/index.js';
import { AccountingSessionDoc } from '../db/index.js';
/**
* RADIUS accounting session
@@ -84,8 +84,6 @@ export interface IAccountingSummary {
* Accounting manager configuration
*/
export interface IAccountingManagerConfig {
/** Storage key prefix */
storagePrefix?: string;
/** Session retention period in days (default: 30) */
retentionDays?: number;
/** Enable detailed session logging */
@@ -106,7 +104,6 @@ export interface IAccountingManagerConfig {
export class AccountingManager {
private activeSessions: Map<string, IAccountingSession> = new Map();
private config: Required<IAccountingManagerConfig>;
private storageManager?: StorageManager;
private staleSessionSweepTimer?: ReturnType<typeof setInterval>;
// Counters for statistics
@@ -118,24 +115,20 @@ export class AccountingManager {
interimUpdatesReceived: 0,
};
constructor(config?: IAccountingManagerConfig, storageManager?: StorageManager) {
constructor(config?: IAccountingManagerConfig) {
this.config = {
storagePrefix: config?.storagePrefix ?? '/radius/accounting',
retentionDays: config?.retentionDays ?? 30,
detailedLogging: config?.detailedLogging ?? false,
maxActiveSessions: config?.maxActiveSessions ?? 10000,
staleSessionTimeoutHours: config?.staleSessionTimeoutHours ?? 24,
};
this.storageManager = storageManager;
}
/**
* Initialize the accounting manager
*/
async initialize(): Promise<void> {
if (this.storageManager) {
await this.loadActiveSessions();
}
await this.loadActiveSessions();
// Start periodic sweep to evict stale sessions (every 15 minutes)
this.staleSessionSweepTimer = setInterval(() => {
@@ -176,9 +169,7 @@ export class AccountingManager {
session.endTime = Date.now();
session.sessionTime = Math.floor((session.endTime - session.startTime) / 1000);
if (this.storageManager) {
this.archiveSession(session).catch(() => {});
}
this.persistSession(session).catch(() => {});
this.activeSessions.delete(sessionId);
swept++;
@@ -250,9 +241,7 @@ export class AccountingManager {
}
// Persist session
if (this.storageManager) {
await this.persistSession(session);
}
await this.persistSession(session);
}
/**
@@ -298,9 +287,7 @@ export class AccountingManager {
}
// Update persisted session
if (this.storageManager) {
await this.persistSession(session);
}
await this.persistSession(session);
}
/**
@@ -353,10 +340,8 @@ export class AccountingManager {
logger.log('info', `Accounting Stop: session=${data.sessionId}, duration=${session.sessionTime}s, in=${session.inputOctets}, out=${session.outputOctets}`);
}
// Archive the session
if (this.storageManager) {
await this.archiveSession(session);
}
// Update status in the database (single collection, no active->archive move needed)
await this.persistSession(session);
// Remove from active sessions
this.activeSessions.delete(data.sessionId);
@@ -493,23 +478,16 @@ export class AccountingManager {
* Clean up old archived sessions based on retention policy
*/
async cleanupOldSessions(): Promise<number> {
if (!this.storageManager) {
return 0;
}
const cutoffTime = Date.now() - this.config.retentionDays * 24 * 60 * 60 * 1000;
let deletedCount = 0;
try {
const keys = await this.storageManager.list(`${this.config.storagePrefix}/archive/`);
const oldDocs = await AccountingSessionDoc.findStoppedBefore(cutoffTime);
for (const key of keys) {
for (const doc of oldDocs) {
try {
const session = await this.storageManager.getJSON<IAccountingSession>(key);
if (session && session.endTime > 0 && session.endTime < cutoffTime) {
await this.storageManager.delete(key);
deletedCount++;
}
await doc.delete();
deletedCount++;
} catch (error) {
// Ignore individual errors
}
@@ -552,9 +530,7 @@ export class AccountingManager {
session.terminateCause = 'SessionEvicted';
session.endTime = Date.now();
if (this.storageManager) {
await this.archiveSession(session);
}
await this.persistSession(session);
this.activeSessions.delete(sessionId);
logger.log('warn', `Evicted session ${sessionId} due to capacity limit`);
@@ -562,25 +538,38 @@ export class AccountingManager {
}
/**
* Load active sessions from storage
* Load active sessions from database
*/
private async loadActiveSessions(): Promise<void> {
if (!this.storageManager) {
return;
}
try {
const keys = await this.storageManager.list(`${this.config.storagePrefix}/active/`);
const docs = await AccountingSessionDoc.findActive();
for (const key of keys) {
try {
const session = await this.storageManager.getJSON<IAccountingSession>(key);
if (session && session.status === 'active') {
this.activeSessions.set(session.sessionId, session);
}
} catch (error) {
// Ignore individual errors
}
for (const doc of docs) {
const session: IAccountingSession = {
sessionId: doc.sessionId,
username: doc.username,
macAddress: doc.macAddress,
nasIpAddress: doc.nasIpAddress,
nasPort: doc.nasPort,
nasPortType: doc.nasPortType,
nasIdentifier: doc.nasIdentifier,
vlanId: doc.vlanId,
framedIpAddress: doc.framedIpAddress,
calledStationId: doc.calledStationId,
callingStationId: doc.callingStationId,
startTime: doc.startTime,
endTime: doc.endTime,
lastUpdateTime: doc.lastUpdateTime,
status: doc.status,
terminateCause: doc.terminateCause,
inputOctets: doc.inputOctets,
outputOctets: doc.outputOctets,
inputPackets: doc.inputPackets,
outputPackets: doc.outputPackets,
sessionTime: doc.sessionTime,
serviceType: doc.serviceType,
};
this.activeSessions.set(session.sessionId, session);
}
} catch (error: unknown) {
logger.log('warn', `Failed to load active sessions: ${(error as Error).message}`);
@@ -588,70 +577,59 @@ export class AccountingManager {
}
/**
* Persist a session to storage
* Persist a session to the database (create or update)
*/
private async persistSession(session: IAccountingSession): Promise<void> {
if (!this.storageManager) {
return;
}
const key = `${this.config.storagePrefix}/active/${session.sessionId}.json`;
try {
await this.storageManager.setJSON(key, session);
let doc = await AccountingSessionDoc.findBySessionId(session.sessionId);
if (!doc) {
doc = new AccountingSessionDoc();
}
Object.assign(doc, session);
await doc.save();
} catch (error: unknown) {
logger.log('error', `Failed to persist session ${session.sessionId}: ${(error as Error).message}`);
}
}
/**
* Archive a completed session
*/
private async archiveSession(session: IAccountingSession): Promise<void> {
if (!this.storageManager) {
return;
}
try {
// Remove from active
const activeKey = `${this.config.storagePrefix}/active/${session.sessionId}.json`;
await this.storageManager.delete(activeKey);
// Add to archive with date-based path
const date = new Date(session.endTime);
const archiveKey = `${this.config.storagePrefix}/archive/${date.getFullYear()}/${String(date.getMonth() + 1).padStart(2, '0')}/${String(date.getDate()).padStart(2, '0')}/${session.sessionId}.json`;
await this.storageManager.setJSON(archiveKey, session);
} catch (error: unknown) {
logger.log('error', `Failed to archive session ${session.sessionId}: ${(error as Error).message}`);
}
}
/**
* Get archived sessions for a time period
* Get archived (stopped/terminated) sessions for a time period
*/
private async getArchivedSessions(startTime: number, endTime: number): Promise<IAccountingSession[]> {
if (!this.storageManager) {
return [];
}
const sessions: IAccountingSession[] = [];
try {
const keys = await this.storageManager.list(`${this.config.storagePrefix}/archive/`);
const docs = await AccountingSessionDoc.getInstances({
status: { $in: ['stopped', 'terminated'] } as any,
endTime: { $gt: 0, $gte: startTime } as any,
startTime: { $lte: endTime } as any,
});
for (const key of keys) {
try {
const session = await this.storageManager.getJSON<IAccountingSession>(key);
if (
session &&
session.endTime > 0 &&
session.startTime <= endTime &&
session.endTime >= startTime
) {
sessions.push(session);
}
} catch (error) {
// Ignore individual errors
}
for (const doc of docs) {
sessions.push({
sessionId: doc.sessionId,
username: doc.username,
macAddress: doc.macAddress,
nasIpAddress: doc.nasIpAddress,
nasPort: doc.nasPort,
nasPortType: doc.nasPortType,
nasIdentifier: doc.nasIdentifier,
vlanId: doc.vlanId,
framedIpAddress: doc.framedIpAddress,
calledStationId: doc.calledStationId,
callingStationId: doc.callingStationId,
startTime: doc.startTime,
endTime: doc.endTime,
lastUpdateTime: doc.lastUpdateTime,
status: doc.status,
terminateCause: doc.terminateCause,
inputOctets: doc.inputOctets,
outputOctets: doc.outputOctets,
inputPackets: doc.inputPackets,
outputPackets: doc.outputPackets,
sessionTime: doc.sessionTime,
serviceType: doc.serviceType,
});
}
} catch (error: unknown) {
logger.log('warn', `Failed to get archived sessions: ${(error as Error).message}`);

View File

@@ -1,6 +1,5 @@
import * as plugins from '../plugins.js';
import { logger } from '../logger.js';
import type { StorageManager } from '../storage/index.js';
import { VlanManager, type IMacVlanMapping, type IVlanManagerConfig } from './classes.vlan.manager.js';
import { AccountingManager, type IAccountingSession, type IAccountingManagerConfig } from './classes.accounting.manager.js';
@@ -92,7 +91,6 @@ export class RadiusServer {
private vlanManager: VlanManager;
private accountingManager: AccountingManager;
private config: IRadiusServerConfig;
private storageManager?: StorageManager;
private clientSecrets: Map<string, string> = new Map();
private running: boolean = false;
@@ -105,20 +103,19 @@ export class RadiusServer {
startTime: 0,
};
constructor(config: IRadiusServerConfig, storageManager?: StorageManager) {
constructor(config: IRadiusServerConfig) {
this.config = {
authPort: config.authPort ?? 1812,
acctPort: config.acctPort ?? 1813,
bindAddress: config.bindAddress ?? '0.0.0.0',
...config,
};
this.storageManager = storageManager;
// Initialize VLAN manager
this.vlanManager = new VlanManager(config.vlanAssignment, storageManager);
this.vlanManager = new VlanManager(config.vlanAssignment);
// Initialize accounting manager
this.accountingManager = new AccountingManager(config.accounting, storageManager);
this.accountingManager = new AccountingManager(config.accounting);
}
/**

View File

@@ -1,6 +1,6 @@
import * as plugins from '../plugins.js';
import { logger } from '../logger.js';
import type { StorageManager } from '../storage/index.js';
import { VlanMappingsDoc } from '../db/index.js';
/**
* MAC address to VLAN mapping
@@ -42,8 +42,6 @@ export interface IVlanManagerConfig {
defaultVlan?: number;
/** Whether to allow unknown MACs (assign default VLAN) or reject */
allowUnknownMacs?: boolean;
/** Storage key prefix for persistence */
storagePrefix?: string;
}
/**
@@ -56,27 +54,22 @@ export interface IVlanManagerConfig {
export class VlanManager {
private mappings: Map<string, IMacVlanMapping> = new Map();
private config: Required<IVlanManagerConfig>;
private storageManager?: StorageManager;
// Cache for normalized MAC lookups
private normalizedMacCache: Map<string, string> = new Map();
constructor(config?: IVlanManagerConfig, storageManager?: StorageManager) {
constructor(config?: IVlanManagerConfig) {
this.config = {
defaultVlan: config?.defaultVlan ?? 1,
allowUnknownMacs: config?.allowUnknownMacs ?? true,
storagePrefix: config?.storagePrefix ?? '/radius/vlan-mappings',
};
this.storageManager = storageManager;
}
/**
* Initialize the VLAN manager and load persisted mappings
*/
async initialize(): Promise<void> {
if (this.storageManager) {
await this.loadMappings();
}
await this.loadMappings();
logger.log('info', `VlanManager initialized with ${this.mappings.size} mappings, default VLAN: ${this.config.defaultVlan}`);
}
@@ -157,10 +150,8 @@ export class VlanManager {
this.mappings.set(normalizedMac, fullMapping);
// Persist to storage
if (this.storageManager) {
await this.saveMappings();
}
// Persist to database
await this.saveMappings();
logger.log('info', `VLAN mapping ${existingMapping ? 'updated' : 'added'}: ${normalizedMac} -> VLAN ${mapping.vlan}`);
return fullMapping;
@@ -173,7 +164,7 @@ export class VlanManager {
const normalizedMac = this.normalizeMac(mac);
const removed = this.mappings.delete(normalizedMac);
if (removed && this.storageManager) {
if (removed) {
await this.saveMappings();
logger.log('info', `VLAN mapping removed: ${normalizedMac}`);
}
@@ -333,39 +324,36 @@ export class VlanManager {
}
/**
* Load mappings from storage
* Load mappings from database
*/
private async loadMappings(): Promise<void> {
if (!this.storageManager) {
return;
}
try {
const data = await this.storageManager.getJSON<IMacVlanMapping[]>(this.config.storagePrefix);
if (data && Array.isArray(data)) {
for (const mapping of data) {
const doc = await VlanMappingsDoc.load();
if (doc && Array.isArray(doc.mappings)) {
for (const mapping of doc.mappings) {
this.mappings.set(this.normalizeMac(mapping.mac), mapping);
}
logger.log('info', `Loaded ${data.length} VLAN mappings from storage`);
logger.log('info', `Loaded ${doc.mappings.length} VLAN mappings from database`);
}
} catch (error: unknown) {
logger.log('warn', `Failed to load VLAN mappings from storage: ${(error as Error).message}`);
logger.log('warn', `Failed to load VLAN mappings from database: ${(error as Error).message}`);
}
}
/**
* Save mappings to storage
* Save mappings to database
*/
private async saveMappings(): Promise<void> {
if (!this.storageManager) {
return;
}
try {
const mappings = Array.from(this.mappings.values());
await this.storageManager.setJSON(this.config.storagePrefix, mappings);
let doc = await VlanMappingsDoc.load();
if (!doc) {
doc = new VlanMappingsDoc();
}
doc.mappings = mappings;
await doc.save();
} catch (error: unknown) {
logger.log('error', `Failed to save VLAN mappings to storage: ${(error as Error).message}`);
logger.log('error', `Failed to save VLAN mappings to database: ${(error as Error).message}`);
}
}
}

View File

@@ -6,7 +6,7 @@
* - VLAN assignment based on MAC addresses
* - OUI (vendor prefix) pattern matching for device categorization
* - RADIUS accounting for session tracking and billing
* - Integration with StorageManager for persistence
* - Integration with smartdata document classes for persistence
*/
export * from './classes.radius.server.js';

View File

@@ -1,8 +1,6 @@
import * as plugins from '../plugins.js';
import type { StorageManager } from '../storage/classes.storagemanager.js';
import type { IRemoteIngress, IDcRouterRouteConfig } from '../../ts_interfaces/data/remoteingress.js';
const STORAGE_PREFIX = '/remote-ingress/';
import { RemoteIngressEdgeDoc } from '../db/index.js';
/**
* Flatten a port range (number | number[] | Array<{from, to}>) to a sorted unique number array.
@@ -27,33 +25,40 @@ function extractPorts(portRange: number | Array<number | { from: number; to: num
/**
* Manages CRUD for remote ingress edge registrations.
* Persists edge configs via StorageManager and provides
* Persists edge configs via smartdata document classes and provides
* the allowed edges list for the Rust hub.
*/
export class RemoteIngressManager {
private storageManager: StorageManager;
private edges: Map<string, IRemoteIngress> = new Map();
private routes: IDcRouterRouteConfig[] = [];
constructor(storageManager: StorageManager) {
this.storageManager = storageManager;
constructor() {
}
/**
* Load all edge registrations from storage into memory.
* Load all edge registrations from the database into memory.
*/
public async initialize(): Promise<void> {
const keys = await this.storageManager.list(STORAGE_PREFIX);
for (const key of keys) {
const edge = await this.storageManager.getJSON<IRemoteIngress>(key);
if (edge) {
// Migration: old edges without autoDerivePorts default to true
if ((edge as any).autoDerivePorts === undefined) {
edge.autoDerivePorts = true;
await this.storageManager.setJSON(key, edge);
}
this.edges.set(edge.id, edge);
const docs = await RemoteIngressEdgeDoc.findAll();
for (const doc of docs) {
// Migration: old edges without autoDerivePorts default to true
if ((doc as any).autoDerivePorts === undefined) {
doc.autoDerivePorts = true;
await doc.save();
}
const edge: IRemoteIngress = {
id: doc.id,
name: doc.name,
secret: doc.secret,
listenPorts: doc.listenPorts,
listenPortsUdp: doc.listenPortsUdp,
enabled: doc.enabled,
autoDerivePorts: doc.autoDerivePorts,
tags: doc.tags,
createdAt: doc.createdAt,
updatedAt: doc.updatedAt,
};
this.edges.set(edge.id, edge);
}
}
@@ -189,7 +194,9 @@ export class RemoteIngressManager {
updatedAt: now,
};
await this.storageManager.setJSON(`${STORAGE_PREFIX}${id}`, edge);
const doc = new RemoteIngressEdgeDoc();
Object.assign(doc, edge);
await doc.save();
this.edges.set(id, edge);
return edge;
}
@@ -233,7 +240,11 @@ export class RemoteIngressManager {
if (updates.tags !== undefined) edge.tags = updates.tags;
edge.updatedAt = Date.now();
await this.storageManager.setJSON(`${STORAGE_PREFIX}${id}`, edge);
const doc = await RemoteIngressEdgeDoc.findById(id);
if (doc) {
Object.assign(doc, edge);
await doc.save();
}
this.edges.set(id, edge);
return edge;
}
@@ -245,7 +256,10 @@ export class RemoteIngressManager {
if (!this.edges.has(id)) {
return false;
}
await this.storageManager.delete(`${STORAGE_PREFIX}${id}`);
const doc = await RemoteIngressEdgeDoc.findById(id);
if (doc) {
await doc.delete();
}
this.edges.delete(id);
return true;
}
@@ -262,7 +276,11 @@ export class RemoteIngressManager {
edge.secret = plugins.crypto.randomBytes(32).toString('hex');
edge.updatedAt = Date.now();
await this.storageManager.setJSON(`${STORAGE_PREFIX}${id}`, edge);
const doc = await RemoteIngressEdgeDoc.findById(id);
if (doc) {
Object.assign(doc, edge);
await doc.save();
}
this.edges.set(id, edge);
return edge.secret;
}

View File

@@ -1,8 +1,8 @@
import * as plugins from '../plugins.js';
import * as paths from '../paths.js';
import { logger } from '../logger.js';
import { SecurityLogger, SecurityLogLevel, SecurityEventType } from './classes.securitylogger.js';
import { LRUCache } from 'lru-cache';
import { CachedIPReputation } from '../db/documents/classes.cached.ip.reputation.js';
/**
* Reputation check result information
@@ -52,7 +52,7 @@ export interface IIPReputationOptions {
highRiskThreshold?: number; // Score below this is high risk
mediumRiskThreshold?: number; // Score below this is medium risk
lowRiskThreshold?: number; // Score below this is low risk
enableLocalCache?: boolean; // Whether to persist cache to disk (default: true)
enableLocalCache?: boolean; // Whether to persist cache to database (default: true)
enableDNSBL?: boolean; // Whether to use DNSBL checks (default: true)
enableIPInfo?: boolean; // Whether to use IP info service (default: true)
}
@@ -64,10 +64,7 @@ export class IPReputationChecker {
private static instance: IPReputationChecker | undefined;
private reputationCache: LRUCache<string, IReputationResult>;
private options: Required<IIPReputationOptions>;
private storageManager?: any; // StorageManager instance
private saveCacheTimer: ReturnType<typeof setTimeout> | null = null;
private static readonly SAVE_CACHE_DEBOUNCE_MS = 30_000;
// Default DNSBL servers
private static readonly DEFAULT_DNSBL_SERVERS = [
'zen.spamhaus.org', // Spamhaus
@@ -75,13 +72,13 @@ export class IPReputationChecker {
'b.barracudacentral.org', // Barracuda
'spam.dnsbl.sorbs.net', // SORBS
'dnsbl.sorbs.net', // SORBS (expanded)
'cbl.abuseat.org', // Composite Blocking List
'cbl.abuseat.org', // Composite Blocking List
'xbl.spamhaus.org', // Spamhaus XBL
'pbl.spamhaus.org', // Spamhaus PBL
'dnsbl-1.uceprotect.net', // UCEPROTECT
'psbl.surriel.com' // PSBL
];
// Default options
private static readonly DEFAULT_OPTIONS: Required<IIPReputationOptions> = {
maxCacheSize: 10000,
@@ -94,54 +91,40 @@ export class IPReputationChecker {
enableDNSBL: true,
enableIPInfo: true
};
/**
* Constructor for IPReputationChecker
* @param options Configuration options
* @param storageManager Optional StorageManager instance for persistence
*/
constructor(options: IIPReputationOptions = {}, storageManager?: any) {
constructor(options: IIPReputationOptions = {}) {
// Merge with default options
this.options = {
...IPReputationChecker.DEFAULT_OPTIONS,
...options
};
this.storageManager = storageManager;
// If no storage manager provided, log warning
if (!storageManager && this.options.enableLocalCache) {
logger.log('warn',
'⚠️ WARNING: IPReputationChecker initialized without StorageManager.\n' +
' IP reputation cache will only be stored to filesystem.\n' +
' Consider passing a StorageManager instance for better storage flexibility.'
);
}
// Initialize reputation cache
this.reputationCache = new LRUCache<string, IReputationResult>({
max: this.options.maxCacheSize,
ttl: this.options.cacheTTL, // Cache TTL
});
// Load cache from disk if enabled
// Load persisted reputations into in-memory cache
if (this.options.enableLocalCache) {
// Fire and forget the load operation
this.loadCache().catch((error: unknown) => {
this.loadCacheFromDb().catch((error: unknown) => {
logger.log('error', `Failed to load IP reputation cache during initialization: ${(error as Error).message}`);
});
}
}
/**
* Get the singleton instance of the checker
* @param options Configuration options
* @param storageManager Optional StorageManager instance for persistence
* @returns Singleton instance
*/
public static getInstance(options: IIPReputationOptions = {}, storageManager?: any): IPReputationChecker {
public static getInstance(options: IIPReputationOptions = {}): IPReputationChecker {
if (!IPReputationChecker.instance) {
IPReputationChecker.instance = new IPReputationChecker(options, storageManager);
IPReputationChecker.instance = new IPReputationChecker(options);
}
return IPReputationChecker.instance;
}
@@ -150,12 +133,6 @@ export class IPReputationChecker {
* Reset the singleton instance (for shutdown/testing)
*/
public static resetInstance(): void {
if (IPReputationChecker.instance) {
if (IPReputationChecker.instance.saveCacheTimer) {
clearTimeout(IPReputationChecker.instance.saveCacheTimer);
IPReputationChecker.instance.saveCacheTimer = null;
}
}
IPReputationChecker.instance = undefined;
}
@@ -171,8 +148,8 @@ export class IPReputationChecker {
logger.log('warn', `Invalid IP address format: ${ip}`);
return this.createErrorResult(ip, 'Invalid IP address format');
}
// Check cache first
// Check in-memory LRU cache first (fast path)
const cachedResult = this.reputationCache.get(ip);
if (cachedResult) {
logger.log('info', `Using cached reputation data for IP ${ip}`, {
@@ -181,7 +158,7 @@ export class IPReputationChecker {
});
return cachedResult;
}
// Initialize empty result
const result: IReputationResult = {
score: 100, // Start with perfect score
@@ -191,51 +168,53 @@ export class IPReputationChecker {
isVPN: false,
timestamp: Date.now()
};
// Check IP against DNS blacklists if enabled
if (this.options.enableDNSBL) {
const dnsblResult = await this.checkDNSBL(ip);
// Update result with DNSBL information
result.score -= dnsblResult.listCount * 10; // Subtract 10 points per blacklist
result.isSpam = dnsblResult.listCount > 0;
result.blacklists = dnsblResult.lists;
}
// Get additional IP information if enabled
if (this.options.enableIPInfo) {
const ipInfo = await this.getIPInfo(ip);
// Update result with IP info
result.country = ipInfo.country;
result.asn = ipInfo.asn;
result.org = ipInfo.org;
// Adjust score based on IP type
if (ipInfo.type === IPType.PROXY || ipInfo.type === IPType.TOR || ipInfo.type === IPType.VPN) {
result.score -= 30; // Subtract 30 points for proxies, Tor, VPNs
// Set proxy flags
result.isProxy = ipInfo.type === IPType.PROXY;
result.isTor = ipInfo.type === IPType.TOR;
result.isVPN = ipInfo.type === IPType.VPN;
}
}
// Ensure score is between 0 and 100
result.score = Math.max(0, Math.min(100, result.score));
// Update cache with result
// Update in-memory LRU cache
this.reputationCache.set(ip, result);
// Schedule debounced cache save if enabled
// Persist to database if enabled (fire and forget)
if (this.options.enableLocalCache) {
this.debouncedSaveCache();
this.persistReputationToDb(ip, result).catch((error: unknown) => {
logger.log('error', `Failed to persist IP reputation for ${ip}: ${(error as Error).message}`);
});
}
// Log the reputation check
this.logReputationCheck(ip, result);
return result;
} catch (error: unknown) {
logger.log('error', `Error checking IP reputation for ${ip}: ${(error as Error).message}`, {
@@ -246,7 +225,7 @@ export class IPReputationChecker {
return this.createErrorResult(ip, (error as Error).message);
}
}
/**
* Check an IP against DNS blacklists
* @param ip IP address to check
@@ -259,7 +238,7 @@ export class IPReputationChecker {
try {
// Reverse the IP for DNSBL queries
const reversedIP = this.reverseIP(ip);
const results = await Promise.allSettled(
this.options.dnsblServers.map(async (server) => {
try {
@@ -274,14 +253,14 @@ export class IPReputationChecker {
}
})
);
// Extract successful lookups (listed in DNSBL)
const lists = results
.filter((result): result is PromiseFulfilledResult<string> =>
.filter((result): result is PromiseFulfilledResult<string> =>
result.status === 'fulfilled' && result.value !== null
)
.map(result => result.value);
return {
listCount: lists.length,
lists
@@ -294,7 +273,7 @@ export class IPReputationChecker {
};
}
}
/**
* Get information about an IP address
* @param ip IP address to check
@@ -309,16 +288,16 @@ export class IPReputationChecker {
try {
// In a real implementation, this would use an IP data service API
// For this implementation, we'll use a simplified approach
// Check if it's a known Tor exit node (simplified)
const isTor = ip.startsWith('171.25.') || ip.startsWith('185.220.') || ip.startsWith('95.216.');
// Check if it's a known VPN (simplified)
const isVPN = ip.startsWith('185.156.') || ip.startsWith('37.120.');
// Check if it's a known proxy (simplified)
const isProxy = ip.startsWith('34.92.') || ip.startsWith('34.206.');
// Determine IP type
let type = IPType.UNKNOWN;
if (isTor) {
@@ -341,7 +320,7 @@ export class IPReputationChecker {
type = IPType.RESIDENTIAL;
}
}
// Return the information
return {
country: this.determineCountry(ip), // Simplified, would use geolocation service
@@ -356,7 +335,7 @@ export class IPReputationChecker {
};
}
}
/**
* Simplified method to determine country from IP
* In a real implementation, this would use a geolocation database or service
@@ -371,7 +350,7 @@ export class IPReputationChecker {
if (ip.startsWith('171.')) return 'DE';
return 'XX'; // Unknown
}
/**
* Simplified method to determine organization from IP
* In a real implementation, this would use an IP-to-org database or service
@@ -387,7 +366,7 @@ export class IPReputationChecker {
if (ip.startsWith('185.220.')) return 'Tor Exit Node';
return 'Unknown';
}
/**
* Reverse an IP address for DNSBL lookups (e.g., 1.2.3.4 -> 4.3.2.1)
* @param ip IP address to reverse
@@ -396,7 +375,7 @@ export class IPReputationChecker {
private reverseIP(ip: string): string {
return ip.split('.').reverse().join('.');
}
/**
* Create an error result for when reputation check fails
* @param ip IP address
@@ -414,7 +393,7 @@ export class IPReputationChecker {
error: errorMessage
};
}
/**
* Validate IP address format
* @param ip IP address to validate
@@ -425,7 +404,7 @@ export class IPReputationChecker {
const ipv4Pattern = /^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/;
return ipv4Pattern.test(ip);
}
/**
* Log reputation check to security logger
* @param ip IP address
@@ -439,7 +418,7 @@ export class IPReputationChecker {
} else if (result.score < this.options.mediumRiskThreshold) {
logLevel = SecurityLogLevel.INFO;
}
// Log the check
SecurityLogger.getInstance().logEvent({
level: logLevel,
@@ -458,131 +437,76 @@ export class IPReputationChecker {
success: !result.isSpam
});
}
/**
* Schedule a debounced cache save (at most once per SAVE_CACHE_DEBOUNCE_MS)
*/
private debouncedSaveCache(): void {
if (this.saveCacheTimer) {
return; // already scheduled
}
this.saveCacheTimer = setTimeout(() => {
this.saveCacheTimer = null;
this.saveCache().catch((error: unknown) => {
logger.log('error', `Failed to save IP reputation cache: ${(error as Error).message}`);
});
}, IPReputationChecker.SAVE_CACHE_DEBOUNCE_MS);
}
/**
* Save cache to disk or storage manager
* Persist a single IP reputation result to the database via CachedIPReputation
*/
private async saveCache(): Promise<void> {
private async persistReputationToDb(ip: string, result: IReputationResult): Promise<void> {
try {
// Convert cache entries to serializable array
const entries = Array.from(this.reputationCache.entries()).map(([ip, data]) => ({
ip,
data
}));
// Only save if we have entries
if (entries.length === 0) {
return;
}
const cacheData = JSON.stringify(entries);
// Save to storage manager if available
if (this.storageManager) {
await this.storageManager.set('/security/ip-reputation-cache.json', cacheData);
logger.log('info', `Saved ${entries.length} IP reputation cache entries to StorageManager`);
const data = {
score: result.score,
isSpam: result.isSpam,
isProxy: result.isProxy,
isTor: result.isTor,
isVPN: result.isVPN,
country: result.country,
asn: result.asn,
org: result.org,
blacklists: result.blacklists,
};
const existing = await CachedIPReputation.findByIP(ip);
if (existing) {
existing.updateReputation(data);
await existing.save();
} else {
// Fall back to filesystem
const cacheDir = plugins.path.join(paths.dataDir, 'security');
plugins.fsUtils.ensureDirSync(cacheDir);
const cacheFile = plugins.path.join(cacheDir, 'ip_reputation_cache.json');
plugins.fsUtils.toFsSync(cacheData, cacheFile);
logger.log('info', `Saved ${entries.length} IP reputation cache entries to disk`);
const doc = CachedIPReputation.fromReputationData(ip, data);
await doc.save();
}
} catch (error: unknown) {
logger.log('error', `Failed to save IP reputation cache: ${(error as Error).message}`);
logger.log('error', `Failed to persist IP reputation for ${ip}: ${(error as Error).message}`);
}
}
/**
* Load cache from disk or storage manager
* Load persisted reputations from CachedIPReputation documents into the in-memory LRU cache
*/
private async loadCache(): Promise<void> {
private async loadCacheFromDb(): Promise<void> {
try {
let cacheData: string | null = null;
let fromFilesystem = false;
// Try to load from storage manager first
if (this.storageManager) {
try {
cacheData = await this.storageManager.get('/security/ip-reputation-cache.json');
if (!cacheData) {
// Check if data exists in filesystem and migrate it
const cacheFile = plugins.path.join(paths.dataDir, 'security', 'ip_reputation_cache.json');
if (plugins.fs.existsSync(cacheFile)) {
logger.log('info', 'Migrating IP reputation cache from filesystem to StorageManager');
cacheData = plugins.fs.readFileSync(cacheFile, 'utf8');
fromFilesystem = true;
// Migrate to storage manager
await this.storageManager.set('/security/ip-reputation-cache.json', cacheData);
logger.log('info', 'IP reputation cache migrated to StorageManager successfully');
// Optionally delete the old file after successful migration
try {
plugins.fs.unlinkSync(cacheFile);
logger.log('info', 'Old cache file removed after migration');
} catch (deleteError) {
logger.log('warn', `Could not delete old cache file: ${(deleteError as Error).message}`);
}
}
}
} catch (error: unknown) {
logger.log('error', `Error loading from StorageManager: ${(error as Error).message}`);
}
} else {
// No storage manager, load from filesystem
const cacheFile = plugins.path.join(paths.dataDir, 'security', 'ip_reputation_cache.json');
if (plugins.fs.existsSync(cacheFile)) {
cacheData = plugins.fs.readFileSync(cacheFile, 'utf8');
fromFilesystem = true;
const docs = await CachedIPReputation.getInstances({});
let loadedCount = 0;
for (const doc of docs) {
// Skip expired documents
if (doc.isExpired()) {
continue;
}
const result: IReputationResult = {
score: doc.score,
isSpam: doc.isSpam,
isProxy: doc.isProxy,
isTor: doc.isTor,
isVPN: doc.isVPN,
country: doc.country || undefined,
asn: doc.asn || undefined,
org: doc.org || undefined,
blacklists: doc.blacklists || [],
timestamp: doc.lastAccessedAt?.getTime() ?? doc.createdAt?.getTime() ?? Date.now(),
};
this.reputationCache.set(doc.ipAddress, result);
loadedCount++;
}
// Parse and restore cache if data was found
if (cacheData) {
const entries = JSON.parse(cacheData);
// Validate and filter entries
const now = Date.now();
const validEntries = entries.filter(entry => {
const age = now - entry.data.timestamp;
return age < this.options.cacheTTL; // Only load entries that haven't expired
});
// Restore cache
for (const entry of validEntries) {
this.reputationCache.set(entry.ip, entry.data);
}
const source = fromFilesystem ? 'disk' : 'StorageManager';
logger.log('info', `Loaded ${validEntries.length} IP reputation cache entries from ${source}`);
if (loadedCount > 0) {
logger.log('info', `Loaded ${loadedCount} IP reputation cache entries from database`);
}
} catch (error: unknown) {
logger.log('error', `Failed to load IP reputation cache: ${(error as Error).message}`);
logger.log('error', `Failed to load IP reputation cache from database: ${(error as Error).message}`);
}
}
/**
* Get the risk level for a reputation score
* @param score Reputation score (0-100)
@@ -599,21 +523,4 @@ export class IPReputationChecker {
return 'trusted';
}
}
/**
* Update the storage manager after instantiation
* This is useful when the storage manager is not available at construction time
* @param storageManager The StorageManager instance to use
*/
public updateStorageManager(storageManager: any): void {
this.storageManager = storageManager;
logger.log('info', 'IPReputationChecker storage manager updated');
// If cache is enabled and we have entries, save them to the new storage manager
if (this.options.enableLocalCache && this.reputationCache.size > 0) {
this.saveCache().catch((error: unknown) => {
logger.log('error', `Failed to save cache to new storage manager: ${(error as Error).message}`);
});
}
}
}
}

View File

@@ -1,404 +0,0 @@
import * as plugins from '../plugins.js';
import { logger } from '../logger.js';
// Promisify filesystem operations
const readFile = plugins.util.promisify(plugins.fs.readFile);
const writeFile = plugins.util.promisify(plugins.fs.writeFile);
const unlink = plugins.util.promisify(plugins.fs.unlink);
const rename = plugins.util.promisify(plugins.fs.rename);
const readdir = plugins.util.promisify(plugins.fs.readdir);
/**
* Storage configuration interface
*/
export interface IStorageConfig {
/** Filesystem path for storage */
fsPath?: string;
/** Custom read function */
readFunction?: (key: string) => Promise<string | null>;
/** Custom write function */
writeFunction?: (key: string, value: string) => Promise<void>;
}
/**
* Storage backend type
*/
export type StorageBackend = 'filesystem' | 'custom' | 'memory';
/**
* Central storage manager for DcRouter
* Provides unified key-value storage with multiple backend support
*/
export class StorageManager {
private static readonly MAX_MEMORY_ENTRIES = 10_000;
private backend: StorageBackend;
private memoryStore: Map<string, string> = new Map();
private config: IStorageConfig;
private fsBasePath?: string;
constructor(config?: IStorageConfig) {
this.config = config || {};
// Check if both fsPath and custom functions are provided
if (config?.fsPath && (config?.readFunction || config?.writeFunction)) {
console.warn(
'⚠️ WARNING: Both fsPath and custom read/write functions are configured.\n' +
' Using custom read/write functions. fsPath will be ignored.'
);
}
// Determine backend based on configuration
if (config?.readFunction && config?.writeFunction) {
this.backend = 'custom';
} else if (config?.fsPath) {
// Set up internal read/write functions for filesystem
this.backend = 'custom'; // Use custom backend with internal functions
this.fsBasePath = plugins.path.resolve(config.fsPath);
this.ensureDirectory(this.fsBasePath);
// Set up internal filesystem read/write functions
this.config.readFunction = (key: string): Promise<string | null> => this.fsRead(key);
this.config.writeFunction = async (key: string, value: string) => {
await this.fsWrite(key, value);
};
} else {
this.backend = 'memory';
this.showMemoryWarning();
}
logger.log('info', `StorageManager initialized with ${this.backend} backend`);
}
/**
* Show warning when using memory backend
*/
private showMemoryWarning(): void {
console.warn(
'⚠️ WARNING: StorageManager is using in-memory storage.\n' +
' Data will be lost when the process restarts.\n' +
' Configure storage.fsPath or storage functions for persistence.'
);
}
/**
* Ensure directory exists for filesystem backend
*/
private async ensureDirectory(dirPath: string): Promise<void> {
try {
await plugins.fsUtils.ensureDir(dirPath);
} catch (error: unknown) {
logger.log('error', `Failed to create storage directory: ${(error as Error).message}`);
throw error;
}
}
/**
* Validate and sanitize storage key
*/
private validateKey(key: string): string {
if (!key || typeof key !== 'string') {
throw new Error('Storage key must be a non-empty string');
}
// Ensure key starts with /
if (!key.startsWith('/')) {
key = '/' + key;
}
// Remove any dangerous path elements
key = key.replace(/\.\./g, '').replace(/\/+/g, '/');
return key;
}
/**
* Convert key to filesystem path
*/
private keyToPath(key: string): string {
if (!this.fsBasePath) {
throw new Error('Filesystem base path not configured');
}
// Remove leading slash and convert to path
const relativePath = key.substring(1);
return plugins.path.join(this.fsBasePath, relativePath);
}
/**
* Internal filesystem read function
*/
private async fsRead(key: string): Promise<string | null> {
const filePath = this.keyToPath(key);
try {
const content = await readFile(filePath, 'utf8');
return content;
} catch (error: unknown) {
if ((error as any).code === 'ENOENT') {
return null;
}
throw error;
}
}
/**
* Internal filesystem write function
*/
private async fsWrite(key: string, value: string): Promise<void> {
const filePath = this.keyToPath(key);
const dir = plugins.path.dirname(filePath);
// Ensure directory exists
await plugins.fsUtils.ensureDir(dir);
// Write atomically with temp file
const tempPath = `${filePath}.tmp`;
await writeFile(tempPath, value, 'utf8');
await rename(tempPath, filePath);
}
/**
* Get value by key
*/
async get(key: string): Promise<string | null> {
key = this.validateKey(key);
try {
switch (this.backend) {
case 'custom': {
if (!this.config.readFunction) {
throw new Error('Read function not configured');
}
try {
return await this.config.readFunction(key);
} catch (error) {
// Assume null if read fails (key doesn't exist)
return null;
}
}
case 'memory': {
return this.memoryStore.get(key) || null;
}
default:
throw new Error(`Unknown backend: ${this.backend}`);
}
} catch (error: unknown) {
logger.log('error', `Storage get error for key ${key}: ${(error as Error).message}`);
throw error;
}
}
/**
* Set value by key
*/
async set(key: string, value: string): Promise<void> {
key = this.validateKey(key);
if (typeof value !== 'string') {
throw new Error('Storage value must be a string');
}
try {
switch (this.backend) {
case 'filesystem': {
const filePath = this.keyToPath(key);
const dirPath = plugins.path.dirname(filePath);
// Ensure directory exists
await plugins.fsUtils.ensureDir(dirPath);
// Write atomically
const tempPath = filePath + '.tmp';
await writeFile(tempPath, value, 'utf8');
await rename(tempPath, filePath);
break;
}
case 'custom': {
if (!this.config.writeFunction) {
throw new Error('Write function not configured');
}
await this.config.writeFunction(key, value);
break;
}
case 'memory': {
this.memoryStore.set(key, value);
// Evict oldest entries if memory store exceeds limit
while (this.memoryStore.size > StorageManager.MAX_MEMORY_ENTRIES) {
const firstKey = this.memoryStore.keys().next().value!;
this.memoryStore.delete(firstKey);
}
break;
}
default:
throw new Error(`Unknown backend: ${this.backend}`);
}
} catch (error: unknown) {
logger.log('error', `Storage set error for key ${key}: ${(error as Error).message}`);
throw error;
}
}
/**
* Delete value by key
*/
async delete(key: string): Promise<void> {
key = this.validateKey(key);
try {
switch (this.backend) {
case 'filesystem': {
const filePath = this.keyToPath(key);
try {
await unlink(filePath);
} catch (error: unknown) {
if ((error as any).code !== 'ENOENT') {
throw error;
}
}
break;
}
case 'custom': {
// Try to delete by setting empty value
if (this.config.writeFunction) {
await this.config.writeFunction(key, '');
}
break;
}
case 'memory': {
this.memoryStore.delete(key);
break;
}
default:
throw new Error(`Unknown backend: ${this.backend}`);
}
} catch (error: unknown) {
logger.log('error', `Storage delete error for key ${key}: ${(error as Error).message}`);
throw error;
}
}
/**
* List keys by prefix
*/
async list(prefix?: string): Promise<string[]> {
prefix = prefix ? this.validateKey(prefix) : '/';
try {
switch (this.backend) {
case 'custom': {
// If we have fsBasePath, this is actually filesystem backend
if (this.fsBasePath) {
const basePath = this.keyToPath(prefix);
const keys: string[] = [];
const walkDir = async (dir: string, baseDir: string): Promise<void> => {
try {
const entries = await readdir(dir, { withFileTypes: true });
for (const entry of entries) {
const fullPath = plugins.path.join(dir, entry.name);
if (entry.isDirectory()) {
await walkDir(fullPath, baseDir);
} else if (entry.isFile()) {
// Convert path back to key
const relativePath = plugins.path.relative(this.fsBasePath!, fullPath);
const key = '/' + relativePath.replace(/\\/g, '/');
if (key.startsWith(prefix)) {
keys.push(key);
}
}
}
} catch (error: unknown) {
if ((error as any).code !== 'ENOENT') {
throw error;
}
}
};
await walkDir(basePath, basePath);
return keys.sort();
} else {
// True custom backends need to implement their own listing
logger.log('warn', 'List operation not supported for custom backend');
return [];
}
}
case 'memory': {
const keys: string[] = [];
for (const key of this.memoryStore.keys()) {
if (key.startsWith(prefix)) {
keys.push(key);
}
}
return keys.sort();
}
default:
throw new Error(`Unknown backend: ${this.backend}`);
}
} catch (error: unknown) {
logger.log('error', `Storage list error for prefix ${prefix}: ${(error as Error).message}`);
throw error;
}
}
/**
* Check if key exists
*/
async exists(key: string): Promise<boolean> {
key = this.validateKey(key);
try {
const value = await this.get(key);
return value !== null;
} catch (error) {
return false;
}
}
/**
* Get storage backend type
*/
getBackend(): StorageBackend {
// If we're using custom backend with fsBasePath, report it as filesystem
if (this.backend === 'custom' && this.fsBasePath) {
return 'filesystem' as StorageBackend;
}
return this.backend;
}
/**
* JSON helper: Get and parse JSON value
*/
async getJSON<T = any>(key: string): Promise<T | null> {
const value = await this.get(key);
if (value === null || value.trim() === '') {
return null;
}
try {
return JSON.parse(value) as T;
} catch (error: unknown) {
logger.log('error', `Failed to parse JSON for key ${key}: ${(error as Error).message}`);
throw error;
}
}
/**
* JSON helper: Set value as JSON
*/
async setJSON(key: string, value: any): Promise<void> {
const jsonString = JSON.stringify(value, null, 2);
await this.set(key, jsonString);
}
}

View File

@@ -1,2 +0,0 @@
// Storage module exports
export * from './classes.storagemanager.js';

View File

@@ -1,9 +1,6 @@
import * as plugins from '../plugins.js';
import { logger } from '../logger.js';
import type { StorageManager } from '../storage/classes.storagemanager.js';
const STORAGE_PREFIX_KEYS = '/vpn/server-keys';
const STORAGE_PREFIX_CLIENTS = '/vpn/clients/';
import { VpnServerKeysDoc, VpnClientDoc } from '../db/index.js';
export interface IVpnManagerConfig {
/** VPN subnet CIDR (default: '10.8.0.0/24') */
@@ -35,43 +32,17 @@ export interface IVpnManagerConfig {
getClientAllowedIPs?: (clientTags: string[]) => Promise<string[]>;
}
interface IPersistedServerKeys {
noisePrivateKey: string;
noisePublicKey: string;
wgPrivateKey: string;
wgPublicKey: string;
}
interface IPersistedClient {
clientId: string;
enabled: boolean;
serverDefinedClientTags?: string[];
description?: string;
assignedIp?: string;
noisePublicKey: string;
wgPublicKey: string;
/** WireGuard private key — stored so exports and QR codes produce valid configs */
wgPrivateKey?: string;
createdAt: number;
updatedAt: number;
expiresAt?: string;
/** @deprecated Legacy field — migrated to serverDefinedClientTags on load */
tags?: string[];
}
/**
* Manages the SmartVPN server lifecycle and VPN client CRUD.
* Persists server keys and client registrations via StorageManager.
* Persists server keys and client registrations via smartdata document classes.
*/
export class VpnManager {
private storageManager: StorageManager;
private config: IVpnManagerConfig;
private vpnServer?: plugins.smartvpn.VpnServer;
private clients: Map<string, IPersistedClient> = new Map();
private serverKeys?: IPersistedServerKeys;
private clients: Map<string, VpnClientDoc> = new Map();
private serverKeys?: VpnServerKeysDoc;
constructor(storageManager: StorageManager, config: IVpnManagerConfig) {
this.storageManager = storageManager;
constructor(config: IVpnManagerConfig) {
this.config = config;
}
@@ -204,22 +175,21 @@ export class VpnManager {
}
// Persist client entry (including WG private key for export/QR)
const persisted: IPersistedClient = {
clientId: bundle.entry.clientId,
enabled: bundle.entry.enabled ?? true,
serverDefinedClientTags: bundle.entry.serverDefinedClientTags,
description: bundle.entry.description,
assignedIp: bundle.entry.assignedIp,
noisePublicKey: bundle.entry.publicKey,
wgPublicKey: bundle.entry.wgPublicKey || '',
wgPrivateKey: bundle.secrets?.wgPrivateKey
|| bundle.wireguardConfig?.match(/PrivateKey\s*=\s*(.+)/)?.[1]?.trim(),
createdAt: Date.now(),
updatedAt: Date.now(),
expiresAt: bundle.entry.expiresAt,
};
this.clients.set(persisted.clientId, persisted);
await this.persistClient(persisted);
const doc = new VpnClientDoc();
doc.clientId = bundle.entry.clientId;
doc.enabled = bundle.entry.enabled ?? true;
doc.serverDefinedClientTags = bundle.entry.serverDefinedClientTags;
doc.description = bundle.entry.description;
doc.assignedIp = bundle.entry.assignedIp;
doc.noisePublicKey = bundle.entry.publicKey;
doc.wgPublicKey = bundle.entry.wgPublicKey || '';
doc.wgPrivateKey = bundle.secrets?.wgPrivateKey
|| bundle.wireguardConfig?.match(/PrivateKey\s*=\s*(.+)/)?.[1]?.trim();
doc.createdAt = Date.now();
doc.updatedAt = Date.now();
doc.expiresAt = bundle.entry.expiresAt;
this.clients.set(doc.clientId, doc);
await this.persistClient(doc);
this.config.onClientChanged?.();
return bundle;
@@ -233,15 +203,18 @@ export class VpnManager {
throw new Error('VPN server not running');
}
await this.vpnServer.removeClient(clientId);
const doc = this.clients.get(clientId);
this.clients.delete(clientId);
await this.storageManager.delete(`${STORAGE_PREFIX_CLIENTS}${clientId}`);
if (doc) {
await doc.delete();
}
this.config.onClientChanged?.();
}
/**
* List all registered clients (without secrets).
*/
public listClients(): IPersistedClient[] {
public listClients(): VpnClientDoc[] {
return [...this.clients.values()];
}
@@ -407,8 +380,8 @@ export class VpnManager {
// ── Private helpers ────────────────────────────────────────────────────
private async loadOrGenerateServerKeys(): Promise<IPersistedServerKeys> {
const stored = await this.storageManager.getJSON<IPersistedServerKeys>(STORAGE_PREFIX_KEYS);
private async loadOrGenerateServerKeys(): Promise<VpnServerKeysDoc> {
const stored = await VpnServerKeysDoc.load();
if (stored?.noisePrivateKey && stored?.wgPrivateKey) {
logger.log('info', 'Loaded VPN server keys from storage');
return stored;
@@ -424,38 +397,34 @@ export class VpnManager {
const wgKeys = await tempServer.generateWgKeypair();
tempServer.stop();
const keys: IPersistedServerKeys = {
noisePrivateKey: noiseKeys.privateKey,
noisePublicKey: noiseKeys.publicKey,
wgPrivateKey: wgKeys.privateKey,
wgPublicKey: wgKeys.publicKey,
};
const doc = stored || new VpnServerKeysDoc();
doc.noisePrivateKey = noiseKeys.privateKey;
doc.noisePublicKey = noiseKeys.publicKey;
doc.wgPrivateKey = wgKeys.privateKey;
doc.wgPublicKey = wgKeys.publicKey;
await doc.save();
await this.storageManager.setJSON(STORAGE_PREFIX_KEYS, keys);
logger.log('info', 'Generated and persisted new VPN server keys');
return keys;
return doc;
}
private async loadPersistedClients(): Promise<void> {
const keys = await this.storageManager.list(STORAGE_PREFIX_CLIENTS);
for (const key of keys) {
const client = await this.storageManager.getJSON<IPersistedClient>(key);
if (client) {
// Migrate legacy `tags` → `serverDefinedClientTags`
if (!client.serverDefinedClientTags && client.tags) {
client.serverDefinedClientTags = client.tags;
delete client.tags;
await this.persistClient(client);
}
this.clients.set(client.clientId, client);
const docs = await VpnClientDoc.findAll();
for (const doc of docs) {
// Migrate legacy `tags` → `serverDefinedClientTags`
if (!doc.serverDefinedClientTags && (doc as any).tags) {
doc.serverDefinedClientTags = (doc as any).tags;
(doc as any).tags = undefined;
await doc.save();
}
this.clients.set(doc.clientId, doc);
}
if (this.clients.size > 0) {
logger.log('info', `Loaded ${this.clients.size} persisted VPN client(s)`);
}
}
private async persistClient(client: IPersistedClient): Promise<void> {
await this.storageManager.setJSON(`${STORAGE_PREFIX_CLIENTS}${client.clientId}`, client);
private async persistClient(client: VpnClientDoc): Promise<void> {
await client.save();
}
}

View File

@@ -87,11 +87,11 @@ export function getOciContainerConfig(): IDcRouterOptions {
} as IDcRouterOptions['emailConfig'];
}
// Cache config
// DB config
const cacheEnabled = process.env.DCROUTER_CACHE_ENABLED;
if (cacheEnabled !== undefined) {
options.cacheConfig = {
...options.cacheConfig,
options.dbConfig = {
...options.dbConfig,
enabled: cacheEnabled === 'true',
};
}

View File

@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@serve.zone/dcrouter',
version: '11.23.3',
version: '12.0.0',
description: 'A multifaceted routing service handling mail and SMS delivery functions.'
}