serve.zone/dcrouter
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

BREAKING CHANGE(db): replace StorageManager and CacheDb with a unified smartdata-backed database layer

Browse Source
This commit is contained in:
Jürgen Kunz
2026-03-31 15:31:16 +00:00
parent 193a4bb180
commit bb6c26484d
49 changed files with 1475 additions and 1687 deletions
Download Patch File Download Diff File Expand all files Collapse all files

299
ts/security/classes.ipreputationchecker.ts
View File

@@ -1,8 +1,8 @@
import * as plugins from '../plugins.js';
import * as paths from '../paths.js';
import { logger } from '../logger.js';
import { SecurityLogger, SecurityLogLevel, SecurityEventType } from './classes.securitylogger.js';
import { LRUCache } from 'lru-cache';
import { CachedIPReputation } from '../db/documents/classes.cached.ip.reputation.js';
/**
* Reputation check result information
@@ -52,7 +52,7 @@ export interface IIPReputationOptions {
highRiskThreshold?: number; // Score below this is high risk
mediumRiskThreshold?: number; // Score below this is medium risk
lowRiskThreshold?: number; // Score below this is low risk
enableLocalCache?: boolean; // Whether to persist cache to disk (default: true)
enableLocalCache?: boolean; // Whether to persist cache to database (default: true)
enableDNSBL?: boolean; // Whether to use DNSBL checks (default: true)
enableIPInfo?: boolean; // Whether to use IP info service (default: true)
}
@@ -64,10 +64,7 @@ export class IPReputationChecker {
private static instance: IPReputationChecker | undefined;
private reputationCache: LRUCache<string, IReputationResult>;
private options: Required<IIPReputationOptions>;
private storageManager?: any; // StorageManager instance
private saveCacheTimer: ReturnType<typeof setTimeout> | null = null;
private static readonly SAVE_CACHE_DEBOUNCE_MS = 30_000;
// Default DNSBL servers
private static readonly DEFAULT_DNSBL_SERVERS = [
'zen.spamhaus.org', // Spamhaus
@@ -75,13 +72,13 @@ export class IPReputationChecker {
'b.barracudacentral.org', // Barracuda
'spam.dnsbl.sorbs.net', // SORBS
'dnsbl.sorbs.net', // SORBS (expanded)
'cbl.abuseat.org', // Composite Blocking List
'cbl.abuseat.org', // Composite Blocking List
'xbl.spamhaus.org', // Spamhaus XBL
'pbl.spamhaus.org', // Spamhaus PBL
'dnsbl-1.uceprotect.net', // UCEPROTECT
'psbl.surriel.com' // PSBL
];
// Default options
private static readonly DEFAULT_OPTIONS: Required<IIPReputationOptions> = {
maxCacheSize: 10000,
@@ -94,54 +91,40 @@ export class IPReputationChecker {
enableDNSBL: true,
enableIPInfo: true
};
/**
* Constructor for IPReputationChecker
* @param options Configuration options
* @param storageManager Optional StorageManager instance for persistence
*/
constructor(options: IIPReputationOptions = {}, storageManager?: any) {
constructor(options: IIPReputationOptions = {}) {
// Merge with default options
this.options = {
...IPReputationChecker.DEFAULT_OPTIONS,
...options
};
this.storageManager = storageManager;
// If no storage manager provided, log warning
if (!storageManager && this.options.enableLocalCache) {
logger.log('warn',
'⚠️ WARNING: IPReputationChecker initialized without StorageManager.\n' +
' IP reputation cache will only be stored to filesystem.\n' +
' Consider passing a StorageManager instance for better storage flexibility.'
);
}
// Initialize reputation cache
this.reputationCache = new LRUCache<string, IReputationResult>({
max: this.options.maxCacheSize,
ttl: this.options.cacheTTL, // Cache TTL
});
// Load cache from disk if enabled
// Load persisted reputations into in-memory cache
if (this.options.enableLocalCache) {
// Fire and forget the load operation
this.loadCache().catch((error: unknown) => {
this.loadCacheFromDb().catch((error: unknown) => {
logger.log('error', `Failed to load IP reputation cache during initialization: ${(error as Error).message}`);
});
}
}
/**
* Get the singleton instance of the checker
* @param options Configuration options
* @param storageManager Optional StorageManager instance for persistence
* @returns Singleton instance
*/
public static getInstance(options: IIPReputationOptions = {}, storageManager?: any): IPReputationChecker {
public static getInstance(options: IIPReputationOptions = {}): IPReputationChecker {
if (!IPReputationChecker.instance) {
IPReputationChecker.instance = new IPReputationChecker(options, storageManager);
IPReputationChecker.instance = new IPReputationChecker(options);
}
return IPReputationChecker.instance;
}
@@ -150,12 +133,6 @@ export class IPReputationChecker {
* Reset the singleton instance (for shutdown/testing)
*/
public static resetInstance(): void {
if (IPReputationChecker.instance) {
if (IPReputationChecker.instance.saveCacheTimer) {
clearTimeout(IPReputationChecker.instance.saveCacheTimer);
IPReputationChecker.instance.saveCacheTimer = null;
}
}
IPReputationChecker.instance = undefined;
}
@@ -171,8 +148,8 @@ export class IPReputationChecker {
logger.log('warn', `Invalid IP address format: ${ip}`);
return this.createErrorResult(ip, 'Invalid IP address format');
}
// Check cache first
// Check in-memory LRU cache first (fast path)
const cachedResult = this.reputationCache.get(ip);
if (cachedResult) {
logger.log('info', `Using cached reputation data for IP ${ip}`, {
@@ -181,7 +158,7 @@ export class IPReputationChecker {
});
return cachedResult;
}
// Initialize empty result
const result: IReputationResult = {
score: 100, // Start with perfect score
@@ -191,51 +168,53 @@ export class IPReputationChecker {
isVPN: false,
timestamp: Date.now()
};
// Check IP against DNS blacklists if enabled
if (this.options.enableDNSBL) {
const dnsblResult = await this.checkDNSBL(ip);
// Update result with DNSBL information
result.score -= dnsblResult.listCount * 10; // Subtract 10 points per blacklist
result.isSpam = dnsblResult.listCount > 0;
result.blacklists = dnsblResult.lists;
}
// Get additional IP information if enabled
if (this.options.enableIPInfo) {
const ipInfo = await this.getIPInfo(ip);
// Update result with IP info
result.country = ipInfo.country;
result.asn = ipInfo.asn;
result.org = ipInfo.org;
// Adjust score based on IP type
if (ipInfo.type === IPType.PROXY || ipInfo.type === IPType.TOR || ipInfo.type === IPType.VPN) {
result.score -= 30; // Subtract 30 points for proxies, Tor, VPNs
// Set proxy flags
result.isProxy = ipInfo.type === IPType.PROXY;
result.isTor = ipInfo.type === IPType.TOR;
result.isVPN = ipInfo.type === IPType.VPN;
}
}
// Ensure score is between 0 and 100
result.score = Math.max(0, Math.min(100, result.score));
// Update cache with result
// Update in-memory LRU cache
this.reputationCache.set(ip, result);
// Schedule debounced cache save if enabled
// Persist to database if enabled (fire and forget)
if (this.options.enableLocalCache) {
this.debouncedSaveCache();
this.persistReputationToDb(ip, result).catch((error: unknown) => {
logger.log('error', `Failed to persist IP reputation for ${ip}: ${(error as Error).message}`);
});
}
// Log the reputation check
this.logReputationCheck(ip, result);
return result;
} catch (error: unknown) {
logger.log('error', `Error checking IP reputation for ${ip}: ${(error as Error).message}`, {
@@ -246,7 +225,7 @@ export class IPReputationChecker {
return this.createErrorResult(ip, (error as Error).message);
}
}
/**
* Check an IP against DNS blacklists
* @param ip IP address to check
@@ -259,7 +238,7 @@ export class IPReputationChecker {
try {
// Reverse the IP for DNSBL queries
const reversedIP = this.reverseIP(ip);
const results = await Promise.allSettled(
this.options.dnsblServers.map(async (server) => {
try {
@@ -274,14 +253,14 @@ export class IPReputationChecker {
}
})
);
// Extract successful lookups (listed in DNSBL)
const lists = results
.filter((result): result is PromiseFulfilledResult<string> =>
.filter((result): result is PromiseFulfilledResult<string> =>
result.status === 'fulfilled' && result.value !== null
)
.map(result => result.value);
return {
listCount: lists.length,
lists
@@ -294,7 +273,7 @@ export class IPReputationChecker {
};
}
}
/**
* Get information about an IP address
* @param ip IP address to check
@@ -309,16 +288,16 @@ export class IPReputationChecker {
try {
// In a real implementation, this would use an IP data service API
// For this implementation, we'll use a simplified approach
// Check if it's a known Tor exit node (simplified)
const isTor = ip.startsWith('171.25.') || ip.startsWith('185.220.') || ip.startsWith('95.216.');
// Check if it's a known VPN (simplified)
const isVPN = ip.startsWith('185.156.') || ip.startsWith('37.120.');
// Check if it's a known proxy (simplified)
const isProxy = ip.startsWith('34.92.') || ip.startsWith('34.206.');
// Determine IP type
let type = IPType.UNKNOWN;
if (isTor) {
@@ -341,7 +320,7 @@ export class IPReputationChecker {
type = IPType.RESIDENTIAL;
}
}
// Return the information
return {
country: this.determineCountry(ip), // Simplified, would use geolocation service
@@ -356,7 +335,7 @@ export class IPReputationChecker {
};
}
}
/**
* Simplified method to determine country from IP
* In a real implementation, this would use a geolocation database or service
@@ -371,7 +350,7 @@ export class IPReputationChecker {
if (ip.startsWith('171.')) return 'DE';
return 'XX'; // Unknown
}
/**
* Simplified method to determine organization from IP
* In a real implementation, this would use an IP-to-org database or service
@@ -387,7 +366,7 @@ export class IPReputationChecker {
if (ip.startsWith('185.220.')) return 'Tor Exit Node';
return 'Unknown';
}
/**
* Reverse an IP address for DNSBL lookups (e.g., 1.2.3.4 -> 4.3.2.1)
* @param ip IP address to reverse
@@ -396,7 +375,7 @@ export class IPReputationChecker {
private reverseIP(ip: string): string {
return ip.split('.').reverse().join('.');
}
/**
* Create an error result for when reputation check fails
* @param ip IP address
@@ -414,7 +393,7 @@ export class IPReputationChecker {
error: errorMessage
};
}
/**
* Validate IP address format
* @param ip IP address to validate
@@ -425,7 +404,7 @@ export class IPReputationChecker {
const ipv4Pattern = /^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/;
return ipv4Pattern.test(ip);
}
/**
* Log reputation check to security logger
* @param ip IP address
@@ -439,7 +418,7 @@ export class IPReputationChecker {
} else if (result.score < this.options.mediumRiskThreshold) {
logLevel = SecurityLogLevel.INFO;
}
// Log the check
SecurityLogger.getInstance().logEvent({
level: logLevel,
@@ -458,131 +437,76 @@ export class IPReputationChecker {
success: !result.isSpam
});
}
/**
* Schedule a debounced cache save (at most once per SAVE_CACHE_DEBOUNCE_MS)
*/
private debouncedSaveCache(): void {
if (this.saveCacheTimer) {
return; // already scheduled
}
this.saveCacheTimer = setTimeout(() => {
this.saveCacheTimer = null;
this.saveCache().catch((error: unknown) => {
logger.log('error', `Failed to save IP reputation cache: ${(error as Error).message}`);
});
}, IPReputationChecker.SAVE_CACHE_DEBOUNCE_MS);
}
/**
* Save cache to disk or storage manager
* Persist a single IP reputation result to the database via CachedIPReputation
*/
private async saveCache(): Promise<void> {
private async persistReputationToDb(ip: string, result: IReputationResult): Promise<void> {
try {
// Convert cache entries to serializable array
const entries = Array.from(this.reputationCache.entries()).map(([ip, data]) => ({
ip,
data
}));
// Only save if we have entries
if (entries.length === 0) {
return;
}
const cacheData = JSON.stringify(entries);
// Save to storage manager if available
if (this.storageManager) {
await this.storageManager.set('/security/ip-reputation-cache.json', cacheData);
logger.log('info', `Saved ${entries.length} IP reputation cache entries to StorageManager`);
const data = {
score: result.score,
isSpam: result.isSpam,
isProxy: result.isProxy,
isTor: result.isTor,
isVPN: result.isVPN,
country: result.country,
asn: result.asn,
org: result.org,
blacklists: result.blacklists,
};
const existing = await CachedIPReputation.findByIP(ip);
if (existing) {
existing.updateReputation(data);
await existing.save();
} else {
// Fall back to filesystem
const cacheDir = plugins.path.join(paths.dataDir, 'security');
plugins.fsUtils.ensureDirSync(cacheDir);
const cacheFile = plugins.path.join(cacheDir, 'ip_reputation_cache.json');
plugins.fsUtils.toFsSync(cacheData, cacheFile);
logger.log('info', `Saved ${entries.length} IP reputation cache entries to disk`);
const doc = CachedIPReputation.fromReputationData(ip, data);
await doc.save();
}
} catch (error: unknown) {
logger.log('error', `Failed to save IP reputation cache: ${(error as Error).message}`);
logger.log('error', `Failed to persist IP reputation for ${ip}: ${(error as Error).message}`);
}
}
/**
* Load cache from disk or storage manager
* Load persisted reputations from CachedIPReputation documents into the in-memory LRU cache
*/
private async loadCache(): Promise<void> {
private async loadCacheFromDb(): Promise<void> {
try {
let cacheData: string | null = null;
let fromFilesystem = false;
// Try to load from storage manager first
if (this.storageManager) {
try {
cacheData = await this.storageManager.get('/security/ip-reputation-cache.json');
if (!cacheData) {
// Check if data exists in filesystem and migrate it
const cacheFile = plugins.path.join(paths.dataDir, 'security', 'ip_reputation_cache.json');
if (plugins.fs.existsSync(cacheFile)) {
logger.log('info', 'Migrating IP reputation cache from filesystem to StorageManager');
cacheData = plugins.fs.readFileSync(cacheFile, 'utf8');
fromFilesystem = true;
// Migrate to storage manager
await this.storageManager.set('/security/ip-reputation-cache.json', cacheData);
logger.log('info', 'IP reputation cache migrated to StorageManager successfully');
// Optionally delete the old file after successful migration
try {
plugins.fs.unlinkSync(cacheFile);
logger.log('info', 'Old cache file removed after migration');
} catch (deleteError) {
logger.log('warn', `Could not delete old cache file: ${(deleteError as Error).message}`);
}
}
}
} catch (error: unknown) {
logger.log('error', `Error loading from StorageManager: ${(error as Error).message}`);
}
} else {
// No storage manager, load from filesystem
const cacheFile = plugins.path.join(paths.dataDir, 'security', 'ip_reputation_cache.json');
if (plugins.fs.existsSync(cacheFile)) {
cacheData = plugins.fs.readFileSync(cacheFile, 'utf8');
fromFilesystem = true;
const docs = await CachedIPReputation.getInstances({});
let loadedCount = 0;
for (const doc of docs) {
// Skip expired documents
if (doc.isExpired()) {
continue;
}
const result: IReputationResult = {
score: doc.score,
isSpam: doc.isSpam,
isProxy: doc.isProxy,
isTor: doc.isTor,
isVPN: doc.isVPN,
country: doc.country || undefined,
asn: doc.asn || undefined,
org: doc.org || undefined,
blacklists: doc.blacklists || [],
timestamp: doc.lastAccessedAt?.getTime() ?? doc.createdAt?.getTime() ?? Date.now(),
};
this.reputationCache.set(doc.ipAddress, result);
loadedCount++;
}
// Parse and restore cache if data was found
if (cacheData) {
const entries = JSON.parse(cacheData);
// Validate and filter entries
const now = Date.now();
const validEntries = entries.filter(entry => {
const age = now - entry.data.timestamp;
return age < this.options.cacheTTL; // Only load entries that haven't expired
});
// Restore cache
for (const entry of validEntries) {
this.reputationCache.set(entry.ip, entry.data);
}
const source = fromFilesystem ? 'disk' : 'StorageManager';
logger.log('info', `Loaded ${validEntries.length} IP reputation cache entries from ${source}`);
if (loadedCount > 0) {
logger.log('info', `Loaded ${loadedCount} IP reputation cache entries from database`);
}
} catch (error: unknown) {
logger.log('error', `Failed to load IP reputation cache: ${(error as Error).message}`);
logger.log('error', `Failed to load IP reputation cache from database: ${(error as Error).message}`);
}
}
/**
* Get the risk level for a reputation score
* @param score Reputation score (0-100)
@@ -599,21 +523,4 @@ export class IPReputationChecker {
return 'trusted';
}
}
/**
* Update the storage manager after instantiation
* This is useful when the storage manager is not available at construction time
* @param storageManager The StorageManager instance to use
*/
public updateStorageManager(storageManager: any): void {
this.storageManager = storageManager;
logger.log('info', 'IPReputationChecker storage manager updated');
// If cache is enabled and we have entries, save them to the new storage manager
if (this.options.enableLocalCache && this.reputationCache.size > 0) {
this.saveCache().catch((error: unknown) => {
logger.log('error', `Failed to save cache to new storage manager: ${(error as Error).message}`);
});
}
}
}
}