1137 lines
36 KiB
TypeScript
1137 lines
36 KiB
TypeScript
import * as plugins from '../plugins.js';
|
|
import * as paths from '../paths.js';
|
|
import { logger } from '../logger.js';
|
|
|
|
/**
|
|
* Domain reputation metrics
|
|
*/
|
|
export interface IDomainReputationMetrics {
|
|
/** Domain being monitored */
|
|
domain: string;
|
|
/** Date the metrics were last updated */
|
|
lastUpdated: Date;
|
|
/** Sending volume metrics */
|
|
volume: {
|
|
/** Total emails sent in the tracking period */
|
|
sent: number;
|
|
/** Delivered emails (excluding bounces) */
|
|
delivered: number;
|
|
/** Hard bounces */
|
|
hardBounces: number;
|
|
/** Soft bounces */
|
|
softBounces: number;
|
|
/** Daily sending volume for the last 30 days */
|
|
dailySendVolume: Record<string, number>;
|
|
};
|
|
/** Engagement metrics */
|
|
engagement: {
|
|
/** Number of opens */
|
|
opens: number;
|
|
/** Number of clicks */
|
|
clicks: number;
|
|
/** Calculated open rate (percentage) */
|
|
openRate: number;
|
|
/** Calculated click rate (percentage) */
|
|
clickRate: number;
|
|
/** Click-to-open rate (percentage) */
|
|
clickToOpenRate: number;
|
|
};
|
|
/** Complaint metrics */
|
|
complaints: {
|
|
/** Number of spam complaints */
|
|
total: number;
|
|
/** Complaint rate (percentage) */
|
|
rate: number;
|
|
/** Domains with highest complaint rates */
|
|
topDomains: Array<{ domain: string; rate: number; count: number }>;
|
|
};
|
|
/** Authentication metrics */
|
|
authentication: {
|
|
/** Percentage of emails with valid SPF */
|
|
spfPassRate: number;
|
|
/** Percentage of emails with valid DKIM */
|
|
dkimPassRate: number;
|
|
/** Percentage of emails with valid DMARC */
|
|
dmarcPassRate: number;
|
|
/** Authentication failures */
|
|
failures: Array<{ type: string; domain: string; count: number }>;
|
|
};
|
|
/** Blocklist status */
|
|
blocklist: {
|
|
/** Current blocklist status */
|
|
listed: boolean;
|
|
/** Blocklists the domain is on, if any */
|
|
activeListings: Array<{ list: string; listedSince: Date }>;
|
|
/** Recent delistings */
|
|
recentDelistings: Array<{ list: string; listedFrom: Date; listedTo: Date }>;
|
|
};
|
|
/** Inbox placement estimates */
|
|
inboxPlacement: {
|
|
/** Overall inbox placement rate estimate */
|
|
overall: number;
|
|
/** Inbox placement rates by major provider */
|
|
providers: Record<string, number>;
|
|
};
|
|
/** Historical reputation scores */
|
|
historical: {
|
|
/** Reputation scores for the last 30 days */
|
|
reputationScores: Record<string, number>;
|
|
/** Trends in key metrics */
|
|
trends: {
|
|
/** Open rate trend (positive or negative percentage) */
|
|
openRate: number;
|
|
/** Complaint rate trend */
|
|
complaintRate: number;
|
|
/** Bounce rate trend */
|
|
bounceRate: number;
|
|
/** Spam listing trend */
|
|
spamListings: number;
|
|
};
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Configuration for reputation monitoring
|
|
*/
|
|
export interface IReputationMonitorConfig {
|
|
/** Whether monitoring is enabled */
|
|
enabled?: boolean;
|
|
/** Domains to monitor */
|
|
domains?: string[];
|
|
/** How frequently to update metrics (ms) */
|
|
updateFrequency?: number;
|
|
/** Endpoints for external data sources */
|
|
dataSources?: {
|
|
/** Spam list monitoring service */
|
|
spamLists?: string[];
|
|
/** Deliverability monitoring service endpoint */
|
|
deliverabilityMonitor?: string;
|
|
};
|
|
/** Alerting thresholds */
|
|
alertThresholds?: {
|
|
/** Minimum safe reputation score */
|
|
minReputationScore?: number;
|
|
/** Maximum acceptable complaint rate */
|
|
maxComplaintRate?: number;
|
|
/** Maximum acceptable bounce rate */
|
|
maxBounceRate?: number;
|
|
/** Minimum acceptable open rate */
|
|
minOpenRate?: number;
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Reputation score components
|
|
*/
|
|
interface IReputationComponents {
|
|
/** Engagement score (0-100) */
|
|
engagement: number;
|
|
/** Complaint score (0-100) */
|
|
complaints: number;
|
|
/** Authentication score (0-100) */
|
|
authentication: number;
|
|
/** Volume stability score (0-100) */
|
|
volumeStability: number;
|
|
/** Infrastructure score (0-100) */
|
|
infrastructure: number;
|
|
/** Blocklist score (0-100) */
|
|
blocklist: number;
|
|
}
|
|
|
|
/**
|
|
* Default configuration
|
|
*/
|
|
const DEFAULT_CONFIG: Required<IReputationMonitorConfig> = {
|
|
enabled: true,
|
|
domains: [],
|
|
updateFrequency: 24 * 60 * 60 * 1000, // Daily
|
|
dataSources: {
|
|
spamLists: [
|
|
'zen.spamhaus.org',
|
|
'bl.spamcop.net',
|
|
'dnsbl.sorbs.net',
|
|
'b.barracudacentral.org'
|
|
],
|
|
deliverabilityMonitor: null
|
|
},
|
|
alertThresholds: {
|
|
minReputationScore: 70,
|
|
maxComplaintRate: 0.1, // 0.1%
|
|
maxBounceRate: 5, // 5%
|
|
minOpenRate: 15 // 15%
|
|
}
|
|
};
|
|
|
|
/**
|
|
* Class for monitoring and tracking sender reputation for domains
|
|
*/
|
|
export class SenderReputationMonitor {
|
|
private static instance: SenderReputationMonitor;
|
|
private config: Required<IReputationMonitorConfig>;
|
|
private reputationData: Map<string, IDomainReputationMetrics> = new Map();
|
|
private updateTimer: NodeJS.Timeout = null;
|
|
private isInitialized: boolean = false;
|
|
|
|
/**
|
|
* Constructor for SenderReputationMonitor
|
|
* @param config Configuration options
|
|
*/
|
|
constructor(config: IReputationMonitorConfig = {}) {
|
|
// Merge with default config
|
|
this.config = {
|
|
...DEFAULT_CONFIG,
|
|
...config,
|
|
dataSources: {
|
|
...DEFAULT_CONFIG.dataSources,
|
|
...config.dataSources
|
|
},
|
|
alertThresholds: {
|
|
...DEFAULT_CONFIG.alertThresholds,
|
|
...config.alertThresholds
|
|
}
|
|
};
|
|
|
|
// Initialize
|
|
this.initialize();
|
|
}
|
|
|
|
/**
|
|
* Get the singleton instance
|
|
* @param config Configuration options
|
|
* @returns Singleton instance
|
|
*/
|
|
public static getInstance(config: IReputationMonitorConfig = {}): SenderReputationMonitor {
|
|
if (!SenderReputationMonitor.instance) {
|
|
SenderReputationMonitor.instance = new SenderReputationMonitor(config);
|
|
}
|
|
return SenderReputationMonitor.instance;
|
|
}
|
|
|
|
/**
|
|
* Initialize the reputation monitor
|
|
*/
|
|
private initialize(): void {
|
|
if (this.isInitialized) return;
|
|
|
|
try {
|
|
// Only load data if not running in a test environment
|
|
const isTestEnvironment = process.env.NODE_ENV === 'test' || !!process.env.JEST_WORKER_ID;
|
|
|
|
if (!isTestEnvironment) {
|
|
// Load existing reputation data
|
|
this.loadReputationData();
|
|
}
|
|
|
|
// Initialize data for any new domains
|
|
for (const domain of this.config.domains) {
|
|
if (!this.reputationData.has(domain)) {
|
|
this.initializeDomainData(domain);
|
|
}
|
|
}
|
|
|
|
// Schedule updates if enabled and not in test environment
|
|
if (this.config.enabled && !isTestEnvironment) {
|
|
this.scheduleUpdates();
|
|
}
|
|
|
|
this.isInitialized = true;
|
|
logger.log('info', `Sender Reputation Monitor initialized for ${this.config.domains.length} domains`);
|
|
} catch (error) {
|
|
logger.log('error', `Failed to initialize Sender Reputation Monitor: ${error.message}`, {
|
|
stack: error.stack
|
|
});
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Initialize reputation data for a new domain
|
|
* @param domain Domain to initialize
|
|
*/
|
|
private initializeDomainData(domain: string): void {
|
|
// Create new domain reputation metrics with default values
|
|
const newMetrics: IDomainReputationMetrics = {
|
|
domain,
|
|
lastUpdated: new Date(),
|
|
volume: {
|
|
sent: 0,
|
|
delivered: 0,
|
|
hardBounces: 0,
|
|
softBounces: 0,
|
|
dailySendVolume: {}
|
|
},
|
|
engagement: {
|
|
opens: 0,
|
|
clicks: 0,
|
|
openRate: 0,
|
|
clickRate: 0,
|
|
clickToOpenRate: 0
|
|
},
|
|
complaints: {
|
|
total: 0,
|
|
rate: 0,
|
|
topDomains: []
|
|
},
|
|
authentication: {
|
|
spfPassRate: 100, // Assume perfect initially
|
|
dkimPassRate: 100,
|
|
dmarcPassRate: 100,
|
|
failures: []
|
|
},
|
|
blocklist: {
|
|
listed: false,
|
|
activeListings: [],
|
|
recentDelistings: []
|
|
},
|
|
inboxPlacement: {
|
|
overall: 95, // Start with optimistic estimate
|
|
providers: {
|
|
gmail: 95,
|
|
outlook: 95,
|
|
yahoo: 95,
|
|
aol: 95,
|
|
other: 95
|
|
}
|
|
},
|
|
historical: {
|
|
reputationScores: {},
|
|
trends: {
|
|
openRate: 0,
|
|
complaintRate: 0,
|
|
bounceRate: 0,
|
|
spamListings: 0
|
|
}
|
|
}
|
|
};
|
|
|
|
// Generate some initial historical data points
|
|
const today = new Date();
|
|
for (let i = 0; i < 30; i++) {
|
|
const date = new Date(today);
|
|
date.setDate(date.getDate() - i);
|
|
const dateKey = date.toISOString().split('T')[0];
|
|
newMetrics.historical.reputationScores[dateKey] = 95; // Default good score
|
|
newMetrics.volume.dailySendVolume[dateKey] = 0;
|
|
}
|
|
|
|
// Save the new metrics
|
|
this.reputationData.set(domain, newMetrics);
|
|
logger.log('info', `Initialized reputation data for domain ${domain}`);
|
|
}
|
|
|
|
/**
|
|
* Schedule regular reputation data updates
|
|
*/
|
|
private scheduleUpdates(): void {
|
|
if (this.updateTimer) {
|
|
clearTimeout(this.updateTimer);
|
|
}
|
|
|
|
this.updateTimer = setTimeout(async () => {
|
|
await this.updateAllDomainMetrics();
|
|
this.scheduleUpdates(); // Reschedule for next update
|
|
}, this.config.updateFrequency);
|
|
|
|
logger.log('info', `Scheduled reputation updates every ${this.config.updateFrequency / (60 * 60 * 1000)} hours`);
|
|
}
|
|
|
|
/**
|
|
* Update metrics for all monitored domains
|
|
*/
|
|
private async updateAllDomainMetrics(): Promise<void> {
|
|
if (!this.config.enabled) return;
|
|
|
|
logger.log('info', 'Starting reputation metrics update for all domains');
|
|
|
|
for (const domain of this.config.domains) {
|
|
try {
|
|
await this.updateDomainMetrics(domain);
|
|
logger.log('info', `Updated reputation metrics for ${domain}`);
|
|
} catch (error) {
|
|
logger.log('error', `Error updating metrics for ${domain}: ${error.message}`, {
|
|
stack: error.stack
|
|
});
|
|
}
|
|
}
|
|
|
|
// Save all updated data
|
|
this.saveReputationData();
|
|
|
|
logger.log('info', 'Completed reputation metrics update for all domains');
|
|
}
|
|
|
|
/**
|
|
* Update reputation metrics for a specific domain
|
|
* @param domain Domain to update
|
|
*/
|
|
private async updateDomainMetrics(domain: string): Promise<void> {
|
|
const metrics = this.reputationData.get(domain);
|
|
if (!metrics) {
|
|
logger.log('warn', `No reputation data found for domain ${domain}`);
|
|
return;
|
|
}
|
|
|
|
try {
|
|
// Update last updated timestamp
|
|
metrics.lastUpdated = new Date();
|
|
|
|
// Check blocklist status
|
|
await this.checkBlocklistStatus(domain, metrics);
|
|
|
|
// Update historical data
|
|
this.updateHistoricalData(metrics);
|
|
|
|
// Calculate current reputation score
|
|
const reputationScore = this.calculateReputationScore(metrics);
|
|
|
|
// Save current reputation score to historical data
|
|
const today = new Date().toISOString().split('T')[0];
|
|
metrics.historical.reputationScores[today] = reputationScore;
|
|
|
|
// Calculate trends
|
|
this.calculateTrends(metrics);
|
|
|
|
// Check alert thresholds
|
|
this.checkAlertThresholds(metrics);
|
|
} catch (error) {
|
|
logger.log('error', `Error in updateDomainMetrics for ${domain}: ${error.message}`, {
|
|
stack: error.stack
|
|
});
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Check domain blocklist status
|
|
* @param domain Domain to check
|
|
* @param metrics Metrics to update
|
|
*/
|
|
private async checkBlocklistStatus(domain: string, metrics: IDomainReputationMetrics): Promise<void> {
|
|
// Skip DNS lookups in test environment
|
|
const isTestEnvironment = process.env.NODE_ENV === 'test' || !!process.env.JEST_WORKER_ID;
|
|
if (isTestEnvironment || !this.config.dataSources.spamLists?.length) {
|
|
return;
|
|
}
|
|
|
|
const previouslyListed = metrics.blocklist.listed;
|
|
const previousListings = new Set(metrics.blocklist.activeListings.map(l => l.list));
|
|
|
|
// Store current listings to detect changes
|
|
const currentListings: Array<{ list: string; listedSince: Date }> = [];
|
|
|
|
// Check each blocklist
|
|
for (const list of this.config.dataSources.spamLists) {
|
|
try {
|
|
const isListed = await this.checkDomainOnBlocklist(domain, list);
|
|
|
|
if (isListed) {
|
|
// If already known to be listed on this one, keep the original listing date
|
|
const existingListing = metrics.blocklist.activeListings.find(l => l.list === list);
|
|
if (existingListing) {
|
|
currentListings.push(existingListing);
|
|
} else {
|
|
// New listing
|
|
currentListings.push({
|
|
list,
|
|
listedSince: new Date()
|
|
});
|
|
}
|
|
}
|
|
} catch (error) {
|
|
logger.log('warn', `Error checking ${domain} on blocklist ${list}: ${error.message}`);
|
|
}
|
|
}
|
|
|
|
// Update active listings
|
|
metrics.blocklist.activeListings = currentListings;
|
|
metrics.blocklist.listed = currentListings.length > 0;
|
|
|
|
// Check for delistings
|
|
if (previouslyListed) {
|
|
const currentListsSet = new Set(currentListings.map(l => l.list));
|
|
|
|
// Convert Set to Array for compatibility with older JS versions
|
|
Array.from(previousListings).forEach(list => {
|
|
if (!currentListsSet.has(list)) {
|
|
// This list no longer contains the domain - it was delisted
|
|
const previousListing = metrics.blocklist.activeListings.find(l => l.list === list);
|
|
|
|
if (previousListing) {
|
|
metrics.blocklist.recentDelistings.push({
|
|
list,
|
|
listedFrom: previousListing.listedSince,
|
|
listedTo: new Date()
|
|
});
|
|
}
|
|
}
|
|
});
|
|
|
|
// Keep only recent delistings (last 90 days)
|
|
const ninetyDaysAgo = new Date();
|
|
ninetyDaysAgo.setDate(ninetyDaysAgo.getDate() - 90);
|
|
|
|
metrics.blocklist.recentDelistings = metrics.blocklist.recentDelistings
|
|
.filter(d => d.listedTo > ninetyDaysAgo);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Check if a domain is on a specific blocklist
|
|
* @param domain Domain to check
|
|
* @param list Blocklist to check
|
|
* @returns Whether the domain is listed
|
|
*/
|
|
private async checkDomainOnBlocklist(domain: string, list: string): Promise<boolean> {
|
|
try {
|
|
// Look up the domain in the blocklist (simplified)
|
|
if (list === 'zen.spamhaus.org') {
|
|
// For Spamhaus and similar lists, we check the domain MX IPs
|
|
const mxRecords = await plugins.dns.promises.resolveMx(domain);
|
|
|
|
if (mxRecords && mxRecords.length > 0) {
|
|
// Check the primary MX record
|
|
const primaryMx = mxRecords.sort((a, b) => a.priority - b.priority)[0].exchange;
|
|
|
|
// Resolve IP addresses for the MX
|
|
const ips = await plugins.dns.promises.resolve(primaryMx);
|
|
|
|
// Check the first IP
|
|
if (ips.length > 0) {
|
|
const ip = ips[0];
|
|
const reversedIp = ip.split('.').reverse().join('.');
|
|
const lookupDomain = `${reversedIp}.${list}`;
|
|
|
|
try {
|
|
await plugins.dns.promises.resolve(lookupDomain);
|
|
return true; // Listed
|
|
} catch (err) {
|
|
if (err.code === 'ENOTFOUND') {
|
|
return false; // Not listed
|
|
}
|
|
throw err; // Other error
|
|
}
|
|
}
|
|
}
|
|
return false;
|
|
} else {
|
|
// For domain-based blocklists
|
|
const lookupDomain = `${domain}.${list}`;
|
|
try {
|
|
await plugins.dns.promises.resolve(lookupDomain);
|
|
return true; // Listed
|
|
} catch (err) {
|
|
if (err.code === 'ENOTFOUND') {
|
|
return false; // Not listed
|
|
}
|
|
throw err; // Other error
|
|
}
|
|
}
|
|
} catch (error) {
|
|
logger.log('warn', `Error checking blocklist status for ${domain} on ${list}: ${error.message}`);
|
|
return false; // Assume not listed on error
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Update historical data in metrics
|
|
* @param metrics Metrics to update
|
|
*/
|
|
private updateHistoricalData(metrics: IDomainReputationMetrics): void {
|
|
// Keep only the last 30 days of data
|
|
const dates = Object.keys(metrics.historical.reputationScores)
|
|
.sort((a, b) => b.localeCompare(a)); // Sort descending
|
|
|
|
if (dates.length > 30) {
|
|
const daysToKeep = dates.slice(0, 30);
|
|
const newScores: Record<string, number> = {};
|
|
|
|
for (const day of daysToKeep) {
|
|
newScores[day] = metrics.historical.reputationScores[day];
|
|
}
|
|
|
|
metrics.historical.reputationScores = newScores;
|
|
}
|
|
|
|
// Same for daily send volume
|
|
const volumeDates = Object.keys(metrics.volume.dailySendVolume)
|
|
.sort((a, b) => b.localeCompare(a));
|
|
|
|
if (volumeDates.length > 30) {
|
|
const daysToKeep = volumeDates.slice(0, 30);
|
|
const newVolume: Record<string, number> = {};
|
|
|
|
for (const day of daysToKeep) {
|
|
newVolume[day] = metrics.volume.dailySendVolume[day];
|
|
}
|
|
|
|
metrics.volume.dailySendVolume = newVolume;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Calculate reputation score from metrics
|
|
* @param metrics Domain reputation metrics
|
|
* @returns Reputation score (0-100)
|
|
*/
|
|
private calculateReputationScore(metrics: IDomainReputationMetrics): number {
|
|
// Calculate component scores
|
|
const components: IReputationComponents = {
|
|
engagement: this.calculateEngagementScore(metrics),
|
|
complaints: this.calculateComplaintScore(metrics),
|
|
authentication: this.calculateAuthenticationScore(metrics),
|
|
volumeStability: this.calculateVolumeStabilityScore(metrics),
|
|
infrastructure: this.calculateInfrastructureScore(metrics),
|
|
blocklist: this.calculateBlocklistScore(metrics)
|
|
};
|
|
|
|
// Apply weights to components
|
|
const weightedScore =
|
|
components.engagement * 0.25 +
|
|
components.complaints * 0.25 +
|
|
components.authentication * 0.2 +
|
|
components.volumeStability * 0.1 +
|
|
components.infrastructure * 0.1 +
|
|
components.blocklist * 0.1;
|
|
|
|
// Round to 2 decimal places
|
|
return Math.round(weightedScore * 100) / 100;
|
|
}
|
|
|
|
/**
|
|
* Calculate engagement component score
|
|
* @param metrics Domain metrics
|
|
* @returns Engagement score (0-100)
|
|
*/
|
|
private calculateEngagementScore(metrics: IDomainReputationMetrics): number {
|
|
const openRate = metrics.engagement.openRate;
|
|
const clickRate = metrics.engagement.clickRate;
|
|
|
|
// Benchmark open and click rates
|
|
// <5% open rate = poor (score: 0-30)
|
|
// 5-15% = average (score: 30-70)
|
|
// >15% = good (score: 70-100)
|
|
let openScore = 0;
|
|
if (openRate < 5) {
|
|
openScore = openRate * 6; // 0-30 scale
|
|
} else if (openRate < 15) {
|
|
openScore = 30 + (openRate - 5) * 4; // 30-70 scale
|
|
} else {
|
|
openScore = 70 + Math.min(30, (openRate - 15) * 2); // 70-100 scale
|
|
}
|
|
|
|
// Similarly for click rate
|
|
let clickScore = 0;
|
|
if (clickRate < 1) {
|
|
clickScore = clickRate * 30; // 0-30 scale
|
|
} else if (clickRate < 5) {
|
|
clickScore = 30 + (clickRate - 1) * 10; // 30-70 scale
|
|
} else {
|
|
clickScore = 70 + Math.min(30, (clickRate - 5) * 6); // 70-100 scale
|
|
}
|
|
|
|
// Combine with 60% weight to open rate, 40% to click rate
|
|
return (openScore * 0.6 + clickScore * 0.4);
|
|
}
|
|
|
|
/**
|
|
* Calculate complaint component score
|
|
* @param metrics Domain metrics
|
|
* @returns Complaint score (0-100)
|
|
*/
|
|
private calculateComplaintScore(metrics: IDomainReputationMetrics): number {
|
|
const complaintRate = metrics.complaints.rate;
|
|
|
|
// Industry standard: complaint rate should be under 0.1%
|
|
// 0% = perfect (score: 100)
|
|
// 0.1% = threshold (score: 70)
|
|
// 0.5% = problematic (score: 30)
|
|
// 1%+ = critical (score: 0)
|
|
|
|
if (complaintRate === 0) return 100;
|
|
if (complaintRate >= 1) return 0;
|
|
|
|
if (complaintRate < 0.1) {
|
|
// 0-0.1% maps to 100-70
|
|
return 100 - (complaintRate / 0.1) * 30;
|
|
} else if (complaintRate < 0.5) {
|
|
// 0.1-0.5% maps to 70-30
|
|
return 70 - ((complaintRate - 0.1) / 0.4) * 40;
|
|
} else {
|
|
// 0.5-1% maps to 30-0
|
|
return 30 - ((complaintRate - 0.5) / 0.5) * 30;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Calculate authentication component score
|
|
* @param metrics Domain metrics
|
|
* @returns Authentication score (0-100)
|
|
*/
|
|
private calculateAuthenticationScore(metrics: IDomainReputationMetrics): number {
|
|
const spfRate = metrics.authentication.spfPassRate;
|
|
const dkimRate = metrics.authentication.dkimPassRate;
|
|
const dmarcRate = metrics.authentication.dmarcPassRate;
|
|
|
|
// Weight SPF, DKIM, and DMARC
|
|
return (spfRate * 0.3 + dkimRate * 0.3 + dmarcRate * 0.4);
|
|
}
|
|
|
|
/**
|
|
* Calculate volume stability component score
|
|
* @param metrics Domain metrics
|
|
* @returns Volume stability score (0-100)
|
|
*/
|
|
private calculateVolumeStabilityScore(metrics: IDomainReputationMetrics): number {
|
|
const volumes = Object.values(metrics.volume.dailySendVolume);
|
|
|
|
if (volumes.length < 2) return 100; // Not enough data
|
|
|
|
// Calculate coefficient of variation (stdev / mean)
|
|
const mean = volumes.reduce((sum, v) => sum + v, 0) / volumes.length;
|
|
if (mean === 0) return 100; // No sending activity
|
|
|
|
const variance = volumes.reduce((sum, v) => sum + Math.pow(v - mean, 2), 0) / volumes.length;
|
|
const stdev = Math.sqrt(variance);
|
|
const cv = stdev / mean;
|
|
|
|
// Convert to score: lower CV means more stability
|
|
// CV < 0.1 is very stable (score: 90-100)
|
|
// CV < 0.5 is normal (score: 60-90)
|
|
// CV < 1.0 is somewhat unstable (score: 30-60)
|
|
// CV >= 1.0 is unstable (score: 0-30)
|
|
|
|
if (cv < 0.1) {
|
|
return 90 + (1 - cv / 0.1) * 10;
|
|
} else if (cv < 0.5) {
|
|
return 60 + (1 - (cv - 0.1) / 0.4) * 30;
|
|
} else if (cv < 1.0) {
|
|
return 30 + (1 - (cv - 0.5) / 0.5) * 30;
|
|
} else {
|
|
return Math.max(0, 30 - (cv - 1.0) * 10);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Calculate infrastructure component score
|
|
* @param metrics Domain metrics
|
|
* @returns Infrastructure score (0-100)
|
|
*/
|
|
private calculateInfrastructureScore(metrics: IDomainReputationMetrics): number {
|
|
// This is a placeholder; in reality, this would be based on:
|
|
// - IP reputation
|
|
// - Reverse DNS configuration
|
|
// - IP warming status
|
|
// - Historical IP behavior
|
|
|
|
// For now, assume good infrastructure
|
|
return 90;
|
|
}
|
|
|
|
/**
|
|
* Calculate blocklist component score
|
|
* @param metrics Domain metrics
|
|
* @returns Blocklist score (0-100)
|
|
*/
|
|
private calculateBlocklistScore(metrics: IDomainReputationMetrics): number {
|
|
// If currently listed on any blocklist, score is heavily impacted
|
|
if (metrics.blocklist.listed) {
|
|
// Number of active listings determines severity
|
|
const listingCount = metrics.blocklist.activeListings.length;
|
|
if (listingCount >= 3) return 0; // Critical: listed on 3+ lists
|
|
if (listingCount === 2) return 20; // Severe: listed on 2 lists
|
|
return 40; // Serious: listed on 1 list
|
|
}
|
|
|
|
// If recently delisted, some penalty still applies
|
|
if (metrics.blocklist.recentDelistings.length > 0) {
|
|
// Check how recent the delistings are
|
|
const now = new Date();
|
|
const mostRecent = metrics.blocklist.recentDelistings
|
|
.reduce((latest, delisting) =>
|
|
delisting.listedTo > latest ? delisting.listedTo : latest,
|
|
new Date(0));
|
|
|
|
const daysSinceDelisting = Math.floor(
|
|
(now.getTime() - mostRecent.getTime()) / (24 * 60 * 60 * 1000)
|
|
);
|
|
|
|
// Score improves as time passes since delisting
|
|
if (daysSinceDelisting < 7) return 60; // Delisted within last week
|
|
if (daysSinceDelisting < 30) return 80; // Delisted within last month
|
|
return 90; // Delisted over a month ago
|
|
}
|
|
|
|
// Never listed
|
|
return 100;
|
|
}
|
|
|
|
/**
|
|
* Calculate trend metrics
|
|
* @param metrics Domain metrics to update
|
|
*/
|
|
private calculateTrends(metrics: IDomainReputationMetrics): void {
|
|
// Get dates in descending order
|
|
const dates = Object.keys(metrics.historical.reputationScores)
|
|
.sort((a, b) => b.localeCompare(a));
|
|
|
|
if (dates.length < 7) {
|
|
// Not enough data for trends
|
|
metrics.historical.trends = {
|
|
openRate: 0,
|
|
complaintRate: 0,
|
|
bounceRate: 0,
|
|
spamListings: 0
|
|
};
|
|
return;
|
|
}
|
|
|
|
// Calculate trends over past 7 days compared to previous 7 days
|
|
const current7Days = dates.slice(0, 7);
|
|
const previous7Days = dates.slice(7, 14);
|
|
|
|
if (previous7Days.length < 7) {
|
|
// Not enough historical data
|
|
return;
|
|
}
|
|
|
|
// Calculate averages for the periods
|
|
const currentReputation = current7Days.reduce(
|
|
(sum, date) => sum + metrics.historical.reputationScores[date], 0
|
|
) / current7Days.length;
|
|
|
|
const previousReputation = previous7Days.reduce(
|
|
(sum, date) => sum + metrics.historical.reputationScores[date], 0
|
|
) / previous7Days.length;
|
|
|
|
// Calculate percent change
|
|
const reputationChange = ((currentReputation - previousReputation) / previousReputation) * 100;
|
|
|
|
// For now, use reputation change for all trends (in a real implementation
|
|
// we would calculate each metric's trend separately)
|
|
metrics.historical.trends = {
|
|
openRate: reputationChange,
|
|
complaintRate: -reputationChange, // Inverse for complaint rate (negative is good)
|
|
bounceRate: -reputationChange, // Inverse for bounce rate
|
|
spamListings: -reputationChange // Inverse for spam listings
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Check if metrics exceed alert thresholds
|
|
* @param metrics Domain metrics to check
|
|
*/
|
|
private checkAlertThresholds(metrics: IDomainReputationMetrics): void {
|
|
const thresholds = this.config.alertThresholds;
|
|
const today = new Date().toISOString().split('T')[0];
|
|
const todayScore = metrics.historical.reputationScores[today] || 0;
|
|
|
|
// Check reputation score
|
|
if (todayScore < thresholds.minReputationScore) {
|
|
this.sendAlert(metrics.domain, 'reputation_score', {
|
|
score: todayScore,
|
|
threshold: thresholds.minReputationScore
|
|
});
|
|
}
|
|
|
|
// Check complaint rate
|
|
if (metrics.complaints.rate > thresholds.maxComplaintRate) {
|
|
this.sendAlert(metrics.domain, 'complaint_rate', {
|
|
rate: metrics.complaints.rate,
|
|
threshold: thresholds.maxComplaintRate
|
|
});
|
|
}
|
|
|
|
// Check bounce rate
|
|
const bounceRate = (metrics.volume.hardBounces + metrics.volume.softBounces) /
|
|
Math.max(1, metrics.volume.sent) * 100;
|
|
|
|
if (bounceRate > thresholds.maxBounceRate) {
|
|
this.sendAlert(metrics.domain, 'bounce_rate', {
|
|
rate: bounceRate,
|
|
threshold: thresholds.maxBounceRate
|
|
});
|
|
}
|
|
|
|
// Check open rate
|
|
if (metrics.engagement.openRate < thresholds.minOpenRate) {
|
|
this.sendAlert(metrics.domain, 'open_rate', {
|
|
rate: metrics.engagement.openRate,
|
|
threshold: thresholds.minOpenRate
|
|
});
|
|
}
|
|
|
|
// Check blocklist status
|
|
if (metrics.blocklist.listed) {
|
|
this.sendAlert(metrics.domain, 'blocklist', {
|
|
lists: metrics.blocklist.activeListings.map(l => l.list)
|
|
});
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Send an alert for a reputation issue
|
|
* @param domain Domain with the issue
|
|
* @param alertType Type of alert
|
|
* @param data Alert data
|
|
*/
|
|
private sendAlert(domain: string, alertType: string, data: any): void {
|
|
logger.log('warn', `Reputation alert for ${domain}: ${alertType}`, data);
|
|
|
|
// In a real implementation, this would send alerts via email,
|
|
// notification systems, webhooks, etc.
|
|
}
|
|
|
|
/**
|
|
* Record a send event for domain reputation tracking
|
|
* @param domain The domain sending the email
|
|
* @param event Event details
|
|
*/
|
|
public recordSendEvent(domain: string, event: {
|
|
type: 'sent' | 'delivered' | 'bounce' | 'complaint' | 'open' | 'click';
|
|
count?: number;
|
|
hardBounce?: boolean;
|
|
receivingDomain?: string;
|
|
}): void {
|
|
// Ensure we have metrics for this domain
|
|
if (!this.reputationData.has(domain)) {
|
|
this.initializeDomainData(domain);
|
|
}
|
|
|
|
const metrics = this.reputationData.get(domain);
|
|
const count = event.count || 1;
|
|
const today = new Date().toISOString().split('T')[0];
|
|
|
|
// Update metrics based on event type
|
|
switch (event.type) {
|
|
case 'sent':
|
|
metrics.volume.sent += count;
|
|
// Update daily send volume
|
|
metrics.volume.dailySendVolume[today] =
|
|
(metrics.volume.dailySendVolume[today] || 0) + count;
|
|
break;
|
|
|
|
case 'delivered':
|
|
metrics.volume.delivered += count;
|
|
break;
|
|
|
|
case 'bounce':
|
|
if (event.hardBounce) {
|
|
metrics.volume.hardBounces += count;
|
|
} else {
|
|
metrics.volume.softBounces += count;
|
|
}
|
|
break;
|
|
|
|
case 'complaint':
|
|
metrics.complaints.total += count;
|
|
|
|
// Track by receiving domain
|
|
if (event.receivingDomain) {
|
|
const domainIndex = metrics.complaints.topDomains.findIndex(
|
|
d => d.domain === event.receivingDomain
|
|
);
|
|
|
|
if (domainIndex >= 0) {
|
|
metrics.complaints.topDomains[domainIndex].count += count;
|
|
metrics.complaints.topDomains[domainIndex].rate =
|
|
metrics.complaints.topDomains[domainIndex].count / Math.max(1, metrics.volume.sent);
|
|
} else {
|
|
metrics.complaints.topDomains.push({
|
|
domain: event.receivingDomain,
|
|
count,
|
|
rate: count / Math.max(1, metrics.volume.sent)
|
|
});
|
|
}
|
|
|
|
// Sort by count
|
|
metrics.complaints.topDomains.sort((a, b) => b.count - a.count);
|
|
|
|
// Keep only top 10
|
|
if (metrics.complaints.topDomains.length > 10) {
|
|
metrics.complaints.topDomains = metrics.complaints.topDomains.slice(0, 10);
|
|
}
|
|
}
|
|
|
|
// Update overall complaint rate
|
|
metrics.complaints.rate =
|
|
metrics.complaints.total / Math.max(1, metrics.volume.sent);
|
|
break;
|
|
|
|
case 'open':
|
|
metrics.engagement.opens += count;
|
|
metrics.engagement.openRate =
|
|
metrics.engagement.opens / Math.max(1, metrics.volume.delivered);
|
|
break;
|
|
|
|
case 'click':
|
|
metrics.engagement.clicks += count;
|
|
metrics.engagement.clickRate =
|
|
metrics.engagement.clicks / Math.max(1, metrics.volume.delivered);
|
|
metrics.engagement.clickToOpenRate =
|
|
metrics.engagement.clicks / Math.max(1, metrics.engagement.opens);
|
|
break;
|
|
}
|
|
|
|
// Update last updated timestamp
|
|
metrics.lastUpdated = new Date();
|
|
|
|
// Save data periodically (not after every event to avoid excessive I/O)
|
|
// Skip in test environment
|
|
const isTestEnvironment = process.env.NODE_ENV === 'test' || !!process.env.JEST_WORKER_ID;
|
|
if (!isTestEnvironment && Math.random() < 0.01) { // ~1% chance to save on each event
|
|
this.saveReputationData();
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get reputation data for a domain
|
|
* @param domain Domain to get data for
|
|
* @returns Reputation data
|
|
*/
|
|
public getReputationData(domain: string): IDomainReputationMetrics | null {
|
|
return this.reputationData.get(domain) || null;
|
|
}
|
|
|
|
/**
|
|
* Get summary reputation data for all domains
|
|
* @returns Summary data for all domains
|
|
*/
|
|
public getReputationSummary(): Array<{
|
|
domain: string;
|
|
score: number;
|
|
status: 'excellent' | 'good' | 'fair' | 'poor' | 'critical';
|
|
listed: boolean;
|
|
trend: number;
|
|
}> {
|
|
return Array.from(this.reputationData.entries())
|
|
.map(([domain, metrics]) => {
|
|
const today = new Date().toISOString().split('T')[0];
|
|
const score = metrics.historical.reputationScores[today] || 0;
|
|
|
|
// Determine status based on score
|
|
let status: 'excellent' | 'good' | 'fair' | 'poor' | 'critical';
|
|
if (score >= 90) status = 'excellent';
|
|
else if (score >= 75) status = 'good';
|
|
else if (score >= 60) status = 'fair';
|
|
else if (score >= 40) status = 'poor';
|
|
else status = 'critical';
|
|
|
|
return {
|
|
domain,
|
|
score,
|
|
status,
|
|
listed: metrics.blocklist.listed,
|
|
trend: metrics.historical.trends.openRate // Use open rate trend as overall trend
|
|
};
|
|
})
|
|
.sort((a, b) => b.score - a.score); // Sort by score descending
|
|
}
|
|
|
|
/**
|
|
* Add a domain to monitor
|
|
* @param domain Domain to monitor
|
|
*/
|
|
public addDomain(domain: string): void {
|
|
if (this.config.domains.includes(domain)) {
|
|
logger.log('info', `Domain ${domain} is already being monitored`);
|
|
return;
|
|
}
|
|
|
|
this.config.domains.push(domain);
|
|
this.initializeDomainData(domain);
|
|
this.saveReputationData();
|
|
|
|
logger.log('info', `Added ${domain} to reputation monitoring`);
|
|
}
|
|
|
|
/**
|
|
* Remove a domain from monitoring
|
|
* @param domain Domain to remove
|
|
*/
|
|
public removeDomain(domain: string): void {
|
|
const index = this.config.domains.indexOf(domain);
|
|
if (index === -1) {
|
|
logger.log('info', `Domain ${domain} is not being monitored`);
|
|
return;
|
|
}
|
|
|
|
this.config.domains.splice(index, 1);
|
|
this.reputationData.delete(domain);
|
|
this.saveReputationData();
|
|
|
|
logger.log('info', `Removed ${domain} from reputation monitoring`);
|
|
}
|
|
|
|
/**
|
|
* Load reputation data from storage
|
|
*/
|
|
private loadReputationData(): void {
|
|
// Skip loading in test environment to prevent file system race conditions
|
|
const isTestEnvironment = process.env.NODE_ENV === 'test' || !!process.env.JEST_WORKER_ID;
|
|
if (isTestEnvironment) {
|
|
return;
|
|
}
|
|
|
|
try {
|
|
const reputationDir = plugins.path.join(paths.dataDir, 'reputation');
|
|
plugins.smartfile.fs.ensureDirSync(reputationDir);
|
|
|
|
const dataFile = plugins.path.join(reputationDir, 'domain_reputation.json');
|
|
|
|
if (plugins.fs.existsSync(dataFile)) {
|
|
const data = plugins.fs.readFileSync(dataFile, 'utf8');
|
|
const reputationEntries = JSON.parse(data);
|
|
|
|
for (const entry of reputationEntries) {
|
|
// Restore Date objects
|
|
entry.lastUpdated = new Date(entry.lastUpdated);
|
|
|
|
for (const listing of entry.blocklist.activeListings) {
|
|
listing.listedSince = new Date(listing.listedSince);
|
|
}
|
|
|
|
for (const delisting of entry.blocklist.recentDelistings) {
|
|
delisting.listedFrom = new Date(delisting.listedFrom);
|
|
delisting.listedTo = new Date(delisting.listedTo);
|
|
}
|
|
|
|
this.reputationData.set(entry.domain, entry);
|
|
}
|
|
|
|
logger.log('info', `Loaded reputation data for ${this.reputationData.size} domains`);
|
|
}
|
|
} catch (error) {
|
|
logger.log('error', `Failed to load reputation data: ${error.message}`, {
|
|
stack: error.stack
|
|
});
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Save reputation data to storage
|
|
*/
|
|
private saveReputationData(): void {
|
|
// Skip saving in test environment to prevent file system race conditions
|
|
const isTestEnvironment = process.env.NODE_ENV === 'test' || !!process.env.JEST_WORKER_ID;
|
|
if (isTestEnvironment) {
|
|
return;
|
|
}
|
|
|
|
try {
|
|
const reputationDir = plugins.path.join(paths.dataDir, 'reputation');
|
|
plugins.smartfile.fs.ensureDirSync(reputationDir);
|
|
|
|
const dataFile = plugins.path.join(reputationDir, 'domain_reputation.json');
|
|
const reputationEntries = Array.from(this.reputationData.values());
|
|
|
|
plugins.smartfile.memory.toFsSync(
|
|
JSON.stringify(reputationEntries, null, 2),
|
|
dataFile
|
|
);
|
|
|
|
logger.log('debug', `Saved reputation data for ${reputationEntries.length} domains`);
|
|
} catch (error) {
|
|
logger.log('error', `Failed to save reputation data: ${error.message}`, {
|
|
stack: error.stack
|
|
});
|
|
}
|
|
}
|
|
} |