serve.zone/platformservice
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
Philipp Kunz 2025-05-07 20:20:17 +00:00
parent f6377d1973
commit 630e911589
40 changed files with 8745 additions and 333 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -19,3 +19,4 @@ dist_*/
# custom
**/.claude/settings.local.json
data/

2
package.json
View File

@ -28,7 +28,6 @@
"@api.global/typedserver": "^3.0.74",
"@api.global/typedsocket": "^3.0.0",
"@apiclient.xyz/cloudflare": "^6.4.1",
"@apiclient.xyz/letterxpress": "^1.0.22",
"@push.rocks/projectinfo": "^5.0.1",
"@push.rocks/qenv": "^6.1.0",
"@push.rocks/smartacme": "^7.3.3",
@ -47,6 +46,7 @@
"@serve.zone/interfaces": "^5.0.4",
"@tsclass/tsclass": "^9.2.0",
"@types/mailparser": "^3.4.6",
"ip": "^2.0.1",
"lru-cache": "^11.1.0",
"mailauth": "^4.8.4",
"mailparser": "^3.6.9",

177
pnpm-lock.yaml generated
View File

@ -20,9 +20,6 @@ importers:
'@apiclient.xyz/cloudflare':
specifier: ^6.4.1
version: 6.4.1
'@apiclient.xyz/letterxpress':
specifier: ^1.0.22
version: 1.0.22(typescript@5.7.3)
'@push.rocks/projectinfo':
specifier: ^5.0.1
version: 5.0.2
@ -77,6 +74,9 @@ importers:
'@types/mailparser':
specifier: ^3.4.6
version: 3.4.6
ip:
specifier: ^2.0.1
version: 2.0.1
lru-cache:
specifier: ^11.1.0
version: 11.1.0
@ -129,9 +129,6 @@ packages:
'@apiclient.xyz/cloudflare@6.4.1':
resolution: {integrity: sha512-RYFphnbunjK+Imq/3ynIQpAvIGBJ38kqSZ2nrpTm26zsBIxW7S6xEe3zhXfVMtUIgC99OL3Xr/SGXl3CNBwCug==}
'@apiclient.xyz/letterxpress@1.0.22':
resolution: {integrity: sha512-7kGu/8TBpO4NRnal1PVAkC7X+TJSZf8Uczbc6JpCznh4up7e3Rhyg8Aqp7w0MnnmIxLjpZgqMsOElCY/qbFncA==}
'@aws-crypto/crc32@5.2.0':
resolution: {integrity: sha512-nLbCWqQNgUiwwtFsen1AdzAtvuLRsQS8rYgMuxCrdKf9kOssamGLuPwyTY9wyYblNr9+1XM8v6zoDTPPSIeANg==}
engines: {node: '>=16.0.0'}
@ -323,18 +320,12 @@ packages:
'@design.estate/dees-comms@1.0.27':
resolution: {integrity: sha512-GvzTUwkV442LD60T08iqSoqvhA02Mou5lFvvqBPc4yBUiU7cZISqBx+76xvMgMIEI9Dx9JfTl4/2nW8MoVAanw==}
'@design.estate/dees-document@1.6.9':
resolution: {integrity: sha512-QcmyXfB3QQccsmHbex2Hk6xv2qwVmliVqiOhp5XNE6P7rROnvsDRWB68M892ZStIPihEotqZXZnDVjM38YVS/A==}
'@design.estate/dees-domtools@2.3.2':
resolution: {integrity: sha512-RfXR2t67M9kaCoF6CBkKJtVdsdp6p1O7S1OaWjrs8V0S3277ch4bSYfO+8f+QYweXKkI6Tr2PKaq3PIlwFSC1g==}
'@design.estate/dees-element@2.0.42':
resolution: {integrity: sha512-1PzHP6q/PtSiu4P0nCxjSeHtRHn62zoSouMy8JFW2h29FT/CSDVaTUAUqYqnvwE/U98aLNivWTmerZitDF7kBQ==}
'@design.estate/dees-wcctools@1.0.90':
resolution: {integrity: sha512-EHYWHiOe+P261e9fBbOBmkD7lIsOpD+tu4VZQr20oc8vhsFjeUGJqYeBm/Ghwg+Gck/dto+K9zyJNIyQ642cEw==}
'@esbuild/aix-ppc64@0.24.2':
resolution: {integrity: sha512-thpVCb/rhxE/BnMLQ7GReQLLN8q9qbHmI55F4489/ByVg2aQaQ6kbcLb6FHkocZzQhxc4gx0sCk0tJkKBFzDhA==}
engines: {node: '>=18'}
@ -1623,9 +1614,6 @@ packages:
'@types/ping@0.4.4':
resolution: {integrity: sha512-ifvo6w2f5eJYlXm+HiVx67iJe8WZp87sfa683nlqED5Vnt9Z93onkokNoWqOG21EaE8fMxyKPobE+mkPEyxsdw==}
'@types/qrcode@1.5.5':
resolution: {integrity: sha512-CdfBi/e3Qk+3Z/fXYShipBT13OJ2fDO2Q2w5CIP5anLTLIndQG9z6P1cnm+8zCWSpm5dnxMFd/uREtb0EXuQzg==}
'@types/qs@6.9.18':
resolution: {integrity: sha512-kK7dgTYDyGqS+e2Q4aK9X3D7q234CIZ1Bv0q/7Z5IwRDoADNU81xXJK/YVyLbLTZCoIwUoDoffFeF+p/eIklAA==}
@ -1977,10 +1965,6 @@ packages:
camel-case@3.0.0:
resolution: {integrity: sha1-yjw2iKTpzzpM2nd9xNy8cTJJz3M=}
camelcase@5.3.1:
resolution: {integrity: sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==}
engines: {node: '>=6'}
camelcase@6.3.0:
resolution: {integrity: sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==}
engines: {node: '>=10'}
@ -2041,9 +2025,6 @@ packages:
resolution: {integrity: sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==}
engines: {node: '>=8'}
cliui@6.0.0:
resolution: {integrity: sha512-t6wbgtoCXvAzst7QgXxJYqPt0usEfbgQdftEPbLL/cvv6HPE5VgvqCuAIDR0NgU52ds6rFwqrgakNLrHEjCbrQ==}
cliui@8.0.1:
resolution: {integrity: sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==}
engines: {node: '>=12'}
@ -2224,10 +2205,6 @@ packages:
supports-color:
optional: true
decamelize@1.2.0:
resolution: {integrity: sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=}
engines: {node: '>=0.10.0'}
decode-named-character-reference@1.1.0:
resolution: {integrity: sha512-Wy+JTSbFThEOXQIR2L6mxJvEs+veIzpmqD7ynWxMXGpnk3smkHQOp6forLdHsKpAMW9iJpaBBIxz285t1n1C3w==}
@ -2311,9 +2288,6 @@ packages:
resolution: {integrity: sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==}
engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}
dijkstrajs@1.0.3:
resolution: {integrity: sha512-qiSlmBq9+BCdCA/L46dw8Uy93mloxsPSbwnm5yrKn2vMPiy8KyAskTF6zuV/j5BMsmOGZDPs7KjU+mjb670kfA==}
dir-glob@3.0.1:
resolution: {integrity: sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==}
engines: {node: '>=8'}
@ -2912,6 +2886,9 @@ packages:
resolution: {integrity: sha512-fOCG6lhoKKakwv+C6KdsOnGvgXnmgfmp0myi3bcNwj3qfwPAxRKWEuFhvEFF7ceYIz6+1jRZ+yguLFAmUNPEfw==}
engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
ip@2.0.1:
resolution: {integrity: sha512-lJUL9imLTNi1ZfXT+DU6rBBdbiKGBuay9B6xGSPVjUeQwaH1RIGqef8RZkUtHioLmSNpPR5M4HVKJGm1j8FWVQ==}
ipaddr.js@1.9.1:
resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==}
engines: {node: '>= 0.10'}
@ -3851,10 +3828,6 @@ packages:
resolution: {integrity: sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==}
engines: {node: '>=8'}
pngjs@5.0.0:
resolution: {integrity: sha512-40QW5YalBNfQo5yRYmiw7Yz6TKKVr3h6970B2YE+3fQpsWcrbj1PzJgxeJ19DRQjhMbKPIuMY8rFaXc8moolVw==}
engines: {node: '>=10.13.0'}
pretty-format@29.7.0:
resolution: {integrity: sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==}
engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}
@ -3939,11 +3912,6 @@ packages:
resolution: {integrity: sha512-pMpnA0qRdFp32b1sJl1wOJNxZLQ2cbQx+k6tjNtZ8CpvVhNqEPRgivZ2WOUev2YMajecdH7ctUPDvEe87nariQ==}
engines: {node: '>=6.0.0'}
qrcode@1.5.4:
resolution: {integrity: sha512-1ca71Zgiu6ORjHqFBDpnSMTR2ReToX4l1Au1VFLyVeBTFavzQnv5JxMFr3ukHVKpSrSA2MCk0lNJSykjUfz7Zg==}
engines: {node: '>=10.13.0'}
hasBin: true
qs@6.13.0:
resolution: {integrity: sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==}
engines: {node: '>=0.6'}
@ -4030,9 +3998,6 @@ packages:
resolution: {integrity: sha1-jGStX9MNqxyXbiNE/+f3kqam30I=}
engines: {node: '>=0.10.0'}
require-main-filename@2.0.0:
resolution: {integrity: sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==}
resolve-alpn@1.2.1:
resolution: {integrity: sha512-0a1F4l73/ZFZOakJnQ3FvkJ2+gSTQWz/r2KE5OdDY0TxPm5h4GkqkWWfM47T7HsbnOtcJVEF4epCVy6u7Q3K+g==}
@ -4117,9 +4082,6 @@ packages:
resolution: {integrity: sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==}
engines: {node: '>= 0.8.0'}
set-blocking@2.0.0:
resolution: {integrity: sha1-BF+XgtARrppoA93TgrJDkrPYkPc=}
set-function-length@1.2.2:
resolution: {integrity: sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==}
engines: {node: '>= 0.4'}
@ -4561,9 +4523,6 @@ packages:
whatwg-url@5.0.0:
resolution: {integrity: sha1-lmRU6HZUYuN2RNNib2dCzotwll0=}
which-module@2.0.1:
resolution: {integrity: sha512-iBdZ57RDvnOR9AGBhML2vFZf7h8vmBjhoaZqODJBFWHVtKkDmKuHai3cx5PgVMrX5YDNp27AofYbAwctSS+vhQ==}
which@2.0.2:
resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==}
engines: {node: '>= 8'}
@ -4660,9 +4619,6 @@ packages:
resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==}
engines: {node: '>=0.4'}
y18n@4.0.3:
resolution: {integrity: sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ==}
y18n@5.0.8:
resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==}
engines: {node: '>=10'}
@ -4670,18 +4626,10 @@ packages:
yallist@4.0.0:
resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==}
yargs-parser@18.1.3:
resolution: {integrity: sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==}
engines: {node: '>=6'}
yargs-parser@21.1.1:
resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==}
engines: {node: '>=12'}
yargs@15.4.1:
resolution: {integrity: sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A==}
engines: {node: '>=8'}
yargs@17.7.2:
resolution: {integrity: sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==}
engines: {node: '>=12'}
@ -4798,25 +4746,6 @@ snapshots:
transitivePeerDependencies:
- encoding
'@apiclient.xyz/letterxpress@1.0.22(typescript@5.7.3)':
dependencies:
'@design.estate/dees-document': 1.6.9(typescript@5.7.3)
'@push.rocks/smartbuffer': 3.0.5
'@push.rocks/smarthash': 3.0.4
'@push.rocks/smartpromise': 4.2.3
'@push.rocks/smartrequest': 2.1.0
'@push.rocks/smartrx': 3.0.10
'@tsclass/tsclass': 5.0.0
transitivePeerDependencies:
- '@nuxt/kit'
- bare-buffer
- bufferutil
- react
- supports-color
- typescript
- utf-8-validate
- vue
'@aws-crypto/crc32@5.2.0':
dependencies:
'@aws-crypto/util': 5.2.0
@ -5389,31 +5318,6 @@ snapshots:
'@push.rocks/smartdelay': 3.0.5
broadcast-channel: 7.1.0
'@design.estate/dees-document@1.6.9(typescript@5.7.3)':
dependencies:
'@design.estate/dees-domtools': 2.3.2
'@design.estate/dees-element': 2.0.42
'@design.estate/dees-wcctools': 1.0.90
'@git.zone/tsrun': 1.3.3
'@push.rocks/smartfile': 11.2.0
'@push.rocks/smartjson': 5.0.20
'@push.rocks/smartpath': 5.0.18
'@push.rocks/smartpdf': 3.2.2(typescript@5.7.3)
'@push.rocks/smarttime': 4.1.1
'@tsclass/tsclass': 4.4.4
'@types/node': 22.15.15
'@types/qrcode': 1.5.5
qrcode: 1.5.4
transitivePeerDependencies:
- '@nuxt/kit'
- bare-buffer
- bufferutil
- react
- supports-color
- typescript
- utf-8-validate
- vue
'@design.estate/dees-domtools@2.3.2':
dependencies:
'@api.global/typedrequest': 3.1.10
@ -5452,18 +5356,6 @@ snapshots:
- supports-color
- vue
'@design.estate/dees-wcctools@1.0.90':
dependencies:
'@design.estate/dees-domtools': 2.3.2
'@design.estate/dees-element': 2.0.42
'@push.rocks/smartdelay': 3.0.5
lit: 3.3.0
transitivePeerDependencies:
- '@nuxt/kit'
- react
- supports-color
- vue
'@esbuild/aix-ppc64@0.24.2':
optional: true
@ -5720,10 +5612,8 @@ snapshots:
'@push.rocks/taskbuffer': 3.1.7
transitivePeerDependencies:
- '@nuxt/kit'
- bufferutil
- react
- supports-color
- utf-8-validate
- vue
'@hapi/bourne@3.0.0': {}
@ -6085,7 +5975,6 @@ snapshots:
- '@aws-sdk/credential-providers'
- '@mongodb-js/zstd'
- '@nuxt/kit'
- aws-crt
- encoding
- gcp-metadata
- kerberos
@ -6496,7 +6385,6 @@ snapshots:
- '@aws-sdk/credential-providers'
- '@mongodb-js/zstd'
- '@nuxt/kit'
- aws-crt
- bufferutil
- encoding
- gcp-metadata
@ -7550,10 +7438,6 @@ snapshots:
'@types/ping@0.4.4': {}
'@types/qrcode@1.5.5':
dependencies:
'@types/node': 22.15.15
'@types/qs@6.9.18': {}
'@types/randomatic@3.1.5': {}
@ -7971,8 +7855,6 @@ snapshots:
no-case: 2.3.2
upper-case: 1.1.3
camelcase@5.3.1: {}
camelcase@6.3.0: {}
ccount@2.0.1: {}
@ -8026,12 +7908,6 @@ snapshots:
dependencies:
restore-cursor: 3.1.0
cliui@6.0.0:
dependencies:
string-width: 4.2.3
strip-ansi: 6.0.1
wrap-ansi: 6.2.0
cliui@8.0.1:
dependencies:
string-width: 4.2.3
@ -8189,8 +8065,6 @@ snapshots:
dependencies:
ms: 2.1.3
decamelize@1.2.0: {}
decode-named-character-reference@1.1.0:
dependencies:
character-entities: 2.0.2
@ -8257,8 +8131,6 @@ snapshots:
diff-sequences@29.6.3: {}
dijkstrajs@1.0.3: {}
dir-glob@3.0.1:
dependencies:
path-type: 4.0.0
@ -9047,6 +8919,8 @@ snapshots:
ip-regex@5.0.0: {}
ip@2.0.1: {}
ipaddr.js@1.9.1: {}
ipaddr.js@2.2.0: {}
@ -10163,8 +10037,6 @@ snapshots:
dependencies:
find-up: 4.1.0
pngjs@5.0.0: {}
pretty-format@29.7.0:
dependencies:
'@jest/schemas': 29.6.3
@ -10281,12 +10153,6 @@ snapshots:
pvutils@1.1.3: {}
qrcode@1.5.4:
dependencies:
dijkstrajs: 1.0.3
pngjs: 5.0.0
yargs: 15.4.1
qs@6.13.0:
dependencies:
side-channel: 1.1.0
@ -10402,8 +10268,6 @@ snapshots:
require-directory@2.1.1: {}
require-main-filename@2.0.0: {}
resolve-alpn@1.2.1: {}
resolve-from@4.0.0: {}
@ -10510,8 +10374,6 @@ snapshots:
transitivePeerDependencies:
- supports-color
set-blocking@2.0.0: {}
set-function-length@1.2.2:
dependencies:
define-data-property: 1.1.4
@ -10982,8 +10844,6 @@ snapshots:
tr46: 0.0.3
webidl-conversions: 3.0.1
which-module@2.0.1: {}
which@2.0.2:
dependencies:
isexe: 2.0.0
@ -11062,33 +10922,12 @@ snapshots:
xtend@4.0.2: {}
y18n@4.0.3: {}
y18n@5.0.8: {}
yallist@4.0.0: {}
yargs-parser@18.1.3:
dependencies:
camelcase: 5.3.1
decamelize: 1.2.0
yargs-parser@21.1.1: {}
yargs@15.4.1:
dependencies:
cliui: 6.0.0
decamelize: 1.2.0
find-up: 4.1.0
get-caller-file: 2.0.5
require-directory: 2.1.1
require-main-filename: 2.0.0
set-blocking: 2.0.0
string-width: 4.2.3
which-module: 2.0.1
y18n: 4.0.3
yargs-parser: 18.1.3
yargs@17.7.2:
dependencies:
cliui: 8.0.1

74
readme.plan.md
View File

@ -13,24 +13,24 @@ The platformservice now has a robust email system with:
### 1. Performance Optimization
- [ ] Replace setTimeout-based DNS cache with proper LRU cache implementation
- [ ] Implement rate limiting for outbound emails
- [x] Replace setTimeout-based DNS cache with proper LRU cache implementation
- [x] Implement rate limiting for outbound emails
- [ ] Add bulk email handling with batching capabilities
- [ ] Optimize template rendering for high-volume scenarios
### 2. Security Enhancements
- [ ] Implement DMARC policy checking and enforcement
- [ ] Add SPF validation for incoming emails
- [ ] Enhance logging for security-related events
- [ ] Add IP reputation checking for inbound emails
- [ ] Implement content scanning for potentially malicious payloads
- [x] Implement DMARC policy checking and enforcement
- [x] Add SPF validation for incoming emails
- [x] Enhance logging for security-related events
- [x] Add IP reputation checking for inbound emails
- [x] Implement content scanning for potentially malicious payloads
### 3. Deliverability Improvements
- [ ] Implement bounce handling and feedback loop processing
- [ ] Add automated IP warmup capabilities
- [ ] Develop sender reputation monitoring
- [x] Implement bounce handling and feedback loop processing
- [x] Add automated IP warmup capabilities
- [x] Develop sender reputation monitoring
- [ ] Create domain rotation for high-volume sending
### 4. Advanced Templating
@ -62,16 +62,56 @@ The platformservice now has a robust email system with:
- [ ] Create end-to-end testing of complete email journeys
- [ ] Add spam testing and deliverability scoring
## Implementation Strategy
## Implementation Progress
1. Begin with security enhancements to ensure the system is as secure as possible
2. Focus on deliverability improvements to maximize email delivery success
### Completed Enhancements
1. **Performance Optimization**
- Replaced setTimeout-based DNS cache with LRU cache for more efficient and reliable caching
- Implemented advanced rate limiting with token bucket algorithm for outbound emails
2. **Security Enhancements**
- Added comprehensive security logging system for email-related security events
- Created a centralized SecurityLogger with event categorization and filtering
- Implemented DMARC policy checking and enforcement for improved email authentication
- Added SPF validation for incoming emails with proper record parsing and verification
- Implemented IP reputation checking for inbound emails with DNSBL integration
- Added detection for suspicious IPs (proxies, VPNs, Tor exit nodes)
- Implemented configurable throttling/rejection for low-reputation IPs
- Implemented content scanning for malicious payloads with pattern matching
- Added detection for phishing, spam, malware indicators, executable attachments
- Created quarantine capabilities for suspicious emails with configurable thresholds
- Implemented macro detection in Office document attachments
3. **Deliverability Improvements**
- Implemented bounce handling with detection and categorization of different bounce types
- Created suppression list management to prevent sending to known bad addresses
- Added exponential backoff retry strategy for soft bounces
- Implemented automated IP warmup capabilities:
- Created configurable warmup stages with progressive volume increases
- Added multiple allocation policies (balanced, round robin, dedicated domain)
- Implemented daily and hourly sending limits with tracking
- Added persistence for warmup state between service restarts
- Developed comprehensive sender reputation monitoring:
- Implemented tracking of key deliverability metrics (bounces, complaints, opens, etc.)
- Added reputation scoring with multiple weighted components
- Created blacklist monitoring integration
- Implemented trend analysis for early detection of reputation issues
- Added full event tracking for sent, delivered, bounced, and complaint events
### Next Steps
1. Continue with security enhancements:
- ✅ Added IP reputation checking for inbound emails with DNS blacklist integration and caching
- ✅ Implemented content scanning for potentially malicious payloads with pattern matching and threat scoring
2. Further deliverability improvements:
- ✅ Added automated IP warmup capabilities with configurable stages and allocation policies
- ✅ Developed sender reputation monitoring with bounce tracking and metric calculation
3. Implement analytics and monitoring to gain visibility into performance
4. Add advanced templating features to enhance email capabilities
5. Optimize performance for scale
6. Expand integrations to increase flexibility
Each enhancement should be implemented incrementally with comprehensive testing to ensure reliability and backward compatibility. Focus on maintaining the clean separation of concerns that's already established in the codebase.
Each enhancement is being implemented incrementally with comprehensive testing to ensure reliability and backward compatibility, while maintaining the clean separation of concerns established in the codebase.
## Success Metrics

61
test/test.base.ts Normal file
View File

@ -0,0 +1,61 @@
import { tap, expect } from '@push.rocks/tapbundle';
import * as plugins from '../ts/plugins.js';
import * as paths from '../ts/paths.js';
import { SenderReputationMonitor } from '../ts/deliverability/classes.senderreputationmonitor.js';
import { IPWarmupManager } from '../ts/deliverability/classes.ipwarmupmanager.js';
/**
* Basic test to check if our integrated classes work correctly
*/
tap.test('verify that SenderReputationMonitor and IPWarmupManager are functioning', async () => {
// Create instances of both classes
const reputationMonitor = SenderReputationMonitor.getInstance({
enabled: true,
domains: ['example.com']
});
const ipWarmupManager = IPWarmupManager.getInstance({
enabled: true,
ipAddresses: ['192.168.1.1', '192.168.1.2'],
targetDomains: ['example.com']
});
// Test SenderReputationMonitor
reputationMonitor.recordSendEvent('example.com', { type: 'sent', count: 100 });
reputationMonitor.recordSendEvent('example.com', { type: 'delivered', count: 95 });
const reputationData = reputationMonitor.getReputationData('example.com');
expect(reputationData).to.not.be.null;
const summary = reputationMonitor.getReputationSummary();
expect(summary.length).to.be.at.least(1);
// Add and remove domains
reputationMonitor.addDomain('test.com');
reputationMonitor.removeDomain('test.com');
// Test IPWarmupManager
ipWarmupManager.setActiveAllocationPolicy('balanced');
const bestIP = ipWarmupManager.getBestIPForSending({
from: 'test@example.com',
to: ['recipient@test.com'],
domain: 'example.com'
});
if (bestIP) {
ipWarmupManager.recordSend(bestIP);
const canSendMore = ipWarmupManager.canSendMoreToday(bestIP);
expect(typeof canSendMore).to.equal('boolean');
}
const stageCount = ipWarmupManager.getStageCount();
expect(stageCount).to.be.greaterThan(0);
});
// Final clean-up test
tap.test('clean up after tests', async () => {
// No-op - just to make sure everything is cleaned up properly
});
export default tap.start();

193
test/test.bouncemanager.ts Normal file
View File

@ -0,0 +1,193 @@
import { tap, expect } from '@push.rocks/tapbundle';
import * as plugins from '../ts/plugins.js';
import { SzPlatformService } from '../ts/platformservice.js';
import { BounceManager, BounceType, BounceCategory } from '../ts/email/classes.bouncemanager.js';
/**
* Test the BounceManager class
*/
tap.test('BounceManager - should be instantiable', async () => {
const bounceManager = new BounceManager();
expect(bounceManager).toBeTruthy();
});
tap.test('BounceManager - should process basic bounce categories', async () => {
const bounceManager = new BounceManager();
// Test hard bounce detection
const hardBounce = await bounceManager.processBounce({
recipient: 'invalid@example.com',
sender: 'sender@example.com',
smtpResponse: 'user unknown',
domain: 'example.com'
});
expect(hardBounce.bounceCategory).toEqual(BounceCategory.HARD);
// Test soft bounce detection
const softBounce = await bounceManager.processBounce({
recipient: 'valid@example.com',
sender: 'sender@example.com',
smtpResponse: 'server unavailable',
domain: 'example.com'
});
expect(softBounce.bounceCategory).toEqual(BounceCategory.SOFT);
// Test auto-response detection
const autoResponse = await bounceManager.processBounce({
recipient: 'away@example.com',
sender: 'sender@example.com',
smtpResponse: 'auto-reply: out of office',
domain: 'example.com'
});
expect(autoResponse.bounceCategory).toEqual(BounceCategory.AUTO_RESPONSE);
});
tap.test('BounceManager - should add and check suppression list entries', async () => {
const bounceManager = new BounceManager();
// Add to suppression list permanently
bounceManager.addToSuppressionList('permanent@example.com', 'Test hard bounce', undefined);
// Add to suppression list temporarily (5 seconds)
const expireTime = Date.now() + 5000;
bounceManager.addToSuppressionList('temporary@example.com', 'Test soft bounce', expireTime);
// Check suppression status
expect(bounceManager.isEmailSuppressed('permanent@example.com')).toEqual(true);
expect(bounceManager.isEmailSuppressed('temporary@example.com')).toEqual(true);
expect(bounceManager.isEmailSuppressed('notsuppressed@example.com')).toEqual(false);
// Get suppression info
const info = bounceManager.getSuppressionInfo('permanent@example.com');
expect(info).toBeTruthy();
expect(info.reason).toEqual('Test hard bounce');
expect(info.expiresAt).toBeUndefined();
// Verify temporary suppression info
const tempInfo = bounceManager.getSuppressionInfo('temporary@example.com');
expect(tempInfo).toBeTruthy();
expect(tempInfo.reason).toEqual('Test soft bounce');
expect(tempInfo.expiresAt).toEqual(expireTime);
// Wait for expiration (6 seconds)
await new Promise(resolve => setTimeout(resolve, 6000));
// Verify permanent suppression is still active
expect(bounceManager.isEmailSuppressed('permanent@example.com')).toEqual(true);
// Verify temporary suppression has expired
expect(bounceManager.isEmailSuppressed('temporary@example.com')).toEqual(false);
});
tap.test('BounceManager - should process SMTP failures correctly', async () => {
const bounceManager = new BounceManager();
const result = await bounceManager.processSmtpFailure(
'recipient@example.com',
'550 5.1.1 User unknown',
{
sender: 'sender@example.com',
statusCode: '550'
}
);
expect(result.bounceType).toEqual(BounceType.INVALID_RECIPIENT);
expect(result.bounceCategory).toEqual(BounceCategory.HARD);
// Check that the email was added to the suppression list
expect(bounceManager.isEmailSuppressed('recipient@example.com')).toEqual(true);
});
tap.test('BounceManager - should process bounce emails correctly', async () => {
const bounceManager = new BounceManager();
// Create a mock bounce email as Smartmail
const bounceEmail = new plugins.smartmail.Smartmail({
from: 'mailer-daemon@example.com',
subject: 'Mail delivery failed: returning message to sender',
body: `
This message was created automatically by mail delivery software.
A message that you sent could not be delivered to one or more of its recipients.
The following address(es) failed:
recipient@example.com
mailbox is full
------ This is a copy of the message, including all the headers. ------
Original-Recipient: rfc822;recipient@example.com
Final-Recipient: rfc822;recipient@example.com
Status: 5.2.2
diagnostic-code: smtp; 552 5.2.2 Mailbox full
`,
creationObjectRef: {}
});
const result = await bounceManager.processBounceEmail(bounceEmail);
expect(result).toBeTruthy();
expect(result.bounceType).toEqual(BounceType.MAILBOX_FULL);
expect(result.bounceCategory).toEqual(BounceCategory.HARD);
expect(result.recipient).toEqual('recipient@example.com');
});
tap.test('BounceManager - should handle retries for soft bounces', async () => {
const bounceManager = new BounceManager({
retryStrategy: {
maxRetries: 2,
initialDelay: 100, // 100ms for test
maxDelay: 1000,
backoffFactor: 2
}
});
// First attempt
const result1 = await bounceManager.processBounce({
recipient: 'retry@example.com',
sender: 'sender@example.com',
bounceType: BounceType.SERVER_UNAVAILABLE,
bounceCategory: BounceCategory.SOFT,
domain: 'example.com'
});
// Email should be suppressed temporarily
expect(bounceManager.isEmailSuppressed('retry@example.com')).toEqual(true);
expect(result1.retryCount).toEqual(1);
expect(result1.nextRetryTime).toBeGreaterThan(Date.now());
// Second attempt
const result2 = await bounceManager.processBounce({
recipient: 'retry@example.com',
sender: 'sender@example.com',
bounceType: BounceType.SERVER_UNAVAILABLE,
bounceCategory: BounceCategory.SOFT,
domain: 'example.com',
retryCount: 1
});
expect(result2.retryCount).toEqual(2);
// Third attempt (should convert to hard bounce)
const result3 = await bounceManager.processBounce({
recipient: 'retry@example.com',
sender: 'sender@example.com',
bounceType: BounceType.SERVER_UNAVAILABLE,
bounceCategory: BounceCategory.SOFT,
domain: 'example.com',
retryCount: 2
});
// Should now be a hard bounce after max retries
expect(result3.bounceCategory).toEqual(BounceCategory.HARD);
// Email should be suppressed permanently
expect(bounceManager.isEmailSuppressed('retry@example.com')).toEqual(true);
const info = bounceManager.getSuppressionInfo('retry@example.com');
expect(info.expiresAt).toBeUndefined(); // Permanent
});
export default tap.start();

261
test/test.contentscanner.ts Normal file
View File

@ -0,0 +1,261 @@
import { tap, expect } from '@push.rocks/tapbundle';
import { ContentScanner, ThreatCategory } from '../ts/security/classes.contentscanner.js';
import { Email } from '../ts/mta/classes.email.js';
// Test instantiation
tap.test('ContentScanner - should be instantiable', async () => {
const scanner = ContentScanner.getInstance({
scanBody: true,
scanSubject: true,
scanAttachments: true
});
expect(scanner).toBeTruthy();
});
// Test singleton pattern
tap.test('ContentScanner - should use singleton pattern', async () => {
const scanner1 = ContentScanner.getInstance();
const scanner2 = ContentScanner.getInstance();
// Both instances should be the same object
expect(scanner1 === scanner2).toEqual(true);
});
// Test clean email can be correctly distinguished from high-risk email
tap.test('ContentScanner - should distinguish between clean and suspicious emails', async () => {
// Create an instance with a higher minimum threat score
const scanner = new ContentScanner({
minThreatScore: 50 // Higher threshold to consider clean
});
// Create a truly clean email with no potentially sensitive data patterns
const cleanEmail = new Email({
from: 'sender@example.com',
to: 'recipient@example.com',
subject: 'Project Update',
text: 'The project is on track. Let me know if you have questions.',
html: '<p>The project is on track. Let me know if you have questions.</p>'
});
// Create a highly suspicious email
const suspiciousEmail = new Email({
from: 'admin@bank-fake.com',
to: 'victim@example.com',
subject: 'URGENT: Your account needs verification now!',
text: 'Click here to verify your account or it will be suspended: https://bit.ly/12345',
html: '<p>Click here to verify your account or it will be suspended: <a href="https://bit.ly/12345">click here</a></p>'
});
// Test both emails
const cleanResult = await scanner.scanEmail(cleanEmail);
const suspiciousResult = await scanner.scanEmail(suspiciousEmail);
console.log('Clean vs Suspicious results:', {
cleanScore: cleanResult.threatScore,
suspiciousScore: suspiciousResult.threatScore
});
// Verify the scanner can distinguish between them
// Suspicious email should have a significantly higher score
expect(suspiciousResult.threatScore > cleanResult.threatScore + 40).toEqual(true);
// Verify clean email scans all expected elements
expect(cleanResult.scannedElements.length > 0).toEqual(true);
});
// Test phishing detection in subject
tap.test('ContentScanner - should detect phishing in subject', async () => {
// Create a dedicated scanner for this test
const scanner = new ContentScanner({
scanSubject: true,
scanBody: true,
scanAttachments: false,
customRules: []
});
const email = new Email({
from: 'security@bank-account-verify.com',
to: 'victim@example.com',
subject: 'URGENT: Verify your bank account details immediately',
text: 'Your account will be suspended. Please verify your details.',
html: '<p>Your account will be suspended. Please verify your details.</p>'
});
const result = await scanner.scanEmail(email);
console.log('Phishing email scan result:', result);
// We only care that it detected something suspicious
expect(result.threatScore >= 20).toEqual(true);
// Check if any threat was detected (specific type may vary)
expect(result.threatType).toBeTruthy();
});
// Test malware indicators in body
tap.test('ContentScanner - should detect malware indicators in body', async () => {
const scanner = ContentScanner.getInstance();
const email = new Email({
from: 'invoice@company.com',
to: 'recipient@example.com',
subject: 'Your invoice',
text: 'Please see the attached invoice. You need to enable macros to view this document properly.',
html: '<p>Please see the attached invoice. You need to enable macros to view this document properly.</p>'
});
const result = await scanner.scanEmail(email);
expect(result.isClean).toEqual(false);
expect(result.threatType === ThreatCategory.MALWARE || result.threatType).toBeTruthy();
expect(result.threatScore >= 30).toEqual(true);
});
// Test suspicious link detection
tap.test('ContentScanner - should detect suspicious links', async () => {
const scanner = ContentScanner.getInstance();
const email = new Email({
from: 'newsletter@example.com',
to: 'recipient@example.com',
subject: 'Weekly Newsletter',
text: 'Check our latest offer at https://bit.ly/2x3F5 and https://t.co/abc123',
html: '<p>Check our latest offer at <a href="https://bit.ly/2x3F5">here</a> and <a href="https://t.co/abc123">here</a></p>'
});
const result = await scanner.scanEmail(email);
expect(result.isClean).toEqual(false);
expect(result.threatType).toEqual(ThreatCategory.SUSPICIOUS_LINK);
expect(result.threatScore >= 30).toEqual(true);
});
// Test script injection detection
tap.test('ContentScanner - should detect script injection', async () => {
const scanner = ContentScanner.getInstance();
const email = new Email({
from: 'newsletter@example.com',
to: 'recipient@example.com',
subject: 'Newsletter',
text: 'Check our website',
html: '<p>Check our website</p><script>document.cookie="session="+localStorage.getItem("token");</script>'
});
const result = await scanner.scanEmail(email);
expect(result.isClean).toEqual(false);
expect(result.threatType).toEqual(ThreatCategory.XSS);
expect(result.threatScore >= 40).toEqual(true);
});
// Test executable attachment detection
tap.test('ContentScanner - should detect executable attachments', async () => {
const scanner = ContentScanner.getInstance();
const email = new Email({
from: 'sender@example.com',
to: 'recipient@example.com',
subject: 'Software Update',
text: 'Please install the attached software update.',
attachments: [{
filename: 'update.exe',
content: Buffer.from('MZ...fake executable content...'),
contentType: 'application/octet-stream'
}]
});
const result = await scanner.scanEmail(email);
expect(result.isClean).toEqual(false);
expect(result.threatType).toEqual(ThreatCategory.EXECUTABLE);
expect(result.threatScore >= 70).toEqual(true);
});
// Test macro document detection
tap.test('ContentScanner - should detect macro documents', async () => {
// Create a mock Office document with macro indicators
const fakeDocContent = Buffer.from('Document content...vbaProject.bin...Auto_Open...DocumentOpen...Microsoft VBA...');
const scanner = ContentScanner.getInstance();
const email = new Email({
from: 'sender@example.com',
to: 'recipient@example.com',
subject: 'Financial Report',
text: 'Please review the attached financial report.',
attachments: [{
filename: 'report.docm',
content: fakeDocContent,
contentType: 'application/vnd.ms-word.document.macroEnabled.12'
}]
});
const result = await scanner.scanEmail(email);
expect(result.isClean).toEqual(false);
expect(result.threatType).toEqual(ThreatCategory.MALICIOUS_MACRO);
expect(result.threatScore >= 60).toEqual(true);
});
// Test compound threat detection (multiple indicators)
tap.test('ContentScanner - should detect compound threats', async () => {
const scanner = ContentScanner.getInstance();
const email = new Email({
from: 'security@bank-verify.com',
to: 'victim@example.com',
subject: 'URGENT: Verify your account details immediately',
text: 'Your account will be suspended unless you verify your details at https://bit.ly/2x3F5',
html: '<p>Your account will be suspended unless you verify your details <a href="https://bit.ly/2x3F5">here</a>.</p>',
attachments: [{
filename: 'verification.exe',
content: Buffer.from('MZ...fake executable content...'),
contentType: 'application/octet-stream'
}]
});
const result = await scanner.scanEmail(email);
expect(result.isClean).toEqual(false);
expect(result.threatScore > 70).toEqual(true); // Should have a high score due to multiple threats
});
// Test custom rules
tap.test('ContentScanner - should apply custom rules', async () => {
// Create a scanner with custom rules
const scanner = new ContentScanner({
customRules: [
{
pattern: /CUSTOM_PATTERN_FOR_TESTING/,
type: ThreatCategory.CUSTOM_RULE,
score: 50,
description: 'Custom pattern detected'
}
]
});
const email = new Email({
from: 'sender@example.com',
to: 'recipient@example.com',
subject: 'Test Custom Rule',
text: 'This message contains CUSTOM_PATTERN_FOR_TESTING that should be detected.'
});
const result = await scanner.scanEmail(email);
expect(result.isClean).toEqual(false);
expect(result.threatType).toEqual(ThreatCategory.CUSTOM_RULE);
expect(result.threatScore >= 50).toEqual(true);
});
// Test threat level classification
tap.test('ContentScanner - should classify threat levels correctly', async () => {
expect(ContentScanner.getThreatLevel(10)).toEqual('none');
expect(ContentScanner.getThreatLevel(25)).toEqual('low');
expect(ContentScanner.getThreatLevel(50)).toEqual('medium');
expect(ContentScanner.getThreatLevel(80)).toEqual('high');
});
export default tap.start();

51
test/test.deliverability.ts Normal file
View File

@ -0,0 +1,51 @@
import { tap, expect } from '@push.rocks/tapbundle';
import * as plugins from '../ts/plugins.js';
import * as paths from '../ts/paths.js';
// Import the components we want to test
import { SenderReputationMonitor } from '../ts/deliverability/classes.senderreputationmonitor.js';
import { IPWarmupManager } from '../ts/deliverability/classes.ipwarmupmanager.js';
// Ensure test directories exist
paths.ensureDirectories();
// Test SenderReputationMonitor functionality
tap.test('SenderReputationMonitor should track sending events', async () => {
// Initialize monitor with test domain
const monitor = SenderReputationMonitor.getInstance({
enabled: true,
domains: ['test-domain.com']
});
// Record some events
monitor.recordSendEvent('test-domain.com', { type: 'sent', count: 100 });
monitor.recordSendEvent('test-domain.com', { type: 'delivered', count: 95 });
// Get domain metrics
const metrics = monitor.getReputationData('test-domain.com');
// Verify metrics were recorded
if (metrics) {
expect(metrics.volume.sent).toEqual(100);
expect(metrics.volume.delivered).toEqual(95);
}
});
// Test IPWarmupManager functionality
tap.test('IPWarmupManager should handle IP allocation policies', async () => {
// Initialize warmup manager
const manager = IPWarmupManager.getInstance({
enabled: true,
ipAddresses: ['192.168.1.1', '192.168.1.2'],
targetDomains: ['test-domain.com']
});
// Set allocation policy
manager.setActiveAllocationPolicy('balanced');
// Verify allocation methods work
const canSend = manager.canSendMoreToday('192.168.1.1');
expect(typeof canSend).toEqual('boolean');
});
export default tap.start();

199
test/test.emailauth.ts Normal file
View File

@ -0,0 +1,199 @@
import { tap, expect } from '@push.rocks/tapbundle';
import { SzPlatformService } from '../ts/platformservice.js';
import { SpfVerifier, SpfQualifier, SpfMechanismType } from '../ts/mta/classes.spfverifier.js';
import { DmarcVerifier, DmarcPolicy, DmarcAlignment } from '../ts/mta/classes.dmarcverifier.js';
import { Email } from '../ts/mta/classes.email.js';
/**
* Test email authentication systems: SPF and DMARC
*/
// Setup platform service for testing
let platformService: SzPlatformService;
tap.test('Setup test environment', async () => {
platformService = new SzPlatformService();
await platformService.init('test');
expect(platformService.mtaService).toBeTruthy();
});
// SPF Verifier Tests
tap.test('SPF Verifier - should parse SPF record', async () => {
const spfVerifier = new SpfVerifier(platformService.mtaService);
// Test valid SPF record parsing
const record = 'v=spf1 a mx ip4:192.168.0.1/24 include:example.org ~all';
const parsedRecord = spfVerifier.parseSpfRecord(record);
expect(parsedRecord).toBeTruthy();
expect(parsedRecord.version).toEqual('spf1');
expect(parsedRecord.mechanisms.length).toEqual(5);
// Check specific mechanisms
expect(parsedRecord.mechanisms[0].type).toEqual(SpfMechanismType.A);
expect(parsedRecord.mechanisms[0].qualifier).toEqual(SpfQualifier.PASS);
expect(parsedRecord.mechanisms[1].type).toEqual(SpfMechanismType.MX);
expect(parsedRecord.mechanisms[1].qualifier).toEqual(SpfQualifier.PASS);
expect(parsedRecord.mechanisms[2].type).toEqual(SpfMechanismType.IP4);
expect(parsedRecord.mechanisms[2].value).toEqual('192.168.0.1/24');
expect(parsedRecord.mechanisms[3].type).toEqual(SpfMechanismType.INCLUDE);
expect(parsedRecord.mechanisms[3].value).toEqual('example.org');
expect(parsedRecord.mechanisms[4].type).toEqual(SpfMechanismType.ALL);
expect(parsedRecord.mechanisms[4].qualifier).toEqual(SpfQualifier.SOFTFAIL);
// Test invalid record
const invalidRecord = 'not-a-spf-record';
const invalidParsed = spfVerifier.parseSpfRecord(invalidRecord);
expect(invalidParsed).toBeNull();
});
// DMARC Verifier Tests
tap.test('DMARC Verifier - should parse DMARC record', async () => {
const dmarcVerifier = new DmarcVerifier(platformService.mtaService);
// Test valid DMARC record parsing
const record = 'v=DMARC1; p=reject; sp=quarantine; pct=50; adkim=s; aspf=r; rua=mailto:dmarc@example.com';
const parsedRecord = dmarcVerifier.parseDmarcRecord(record);
expect(parsedRecord).toBeTruthy();
expect(parsedRecord.version).toEqual('DMARC1');
expect(parsedRecord.policy).toEqual(DmarcPolicy.REJECT);
expect(parsedRecord.subdomainPolicy).toEqual(DmarcPolicy.QUARANTINE);
expect(parsedRecord.pct).toEqual(50);
expect(parsedRecord.adkim).toEqual(DmarcAlignment.STRICT);
expect(parsedRecord.aspf).toEqual(DmarcAlignment.RELAXED);
expect(parsedRecord.reportUriAggregate).toContain('dmarc@example.com');
// Test invalid record
const invalidRecord = 'not-a-dmarc-record';
const invalidParsed = dmarcVerifier.parseDmarcRecord(invalidRecord);
expect(invalidParsed).toBeNull();
});
tap.test('DMARC Verifier - should verify DMARC alignment', async () => {
const dmarcVerifier = new DmarcVerifier(platformService.mtaService);
// Test email domains with DMARC alignment
const email = new Email({
from: 'sender@example.com',
to: 'recipient@example.net',
subject: 'Test DMARC alignment',
text: 'This is a test email'
});
// Test when both SPF and DKIM pass with alignment
const dmarcResult = await dmarcVerifier.verify(
email,
{ domain: 'example.com', result: true }, // SPF - aligned and passed
{ domain: 'example.com', result: true } // DKIM - aligned and passed
);
expect(dmarcResult).toBeTruthy();
expect(dmarcResult.spfPassed).toEqual(true);
expect(dmarcResult.dkimPassed).toEqual(true);
expect(dmarcResult.spfDomainAligned).toEqual(true);
expect(dmarcResult.dkimDomainAligned).toEqual(true);
expect(dmarcResult.action).toEqual('pass');
// Test when neither SPF nor DKIM is aligned
const dmarcResult2 = await dmarcVerifier.verify(
email,
{ domain: 'differentdomain.com', result: true }, // SPF - passed but not aligned
{ domain: 'anotherdomain.com', result: true } // DKIM - passed but not aligned
);
expect(dmarcResult2).toBeTruthy();
expect(dmarcResult2.spfPassed).toEqual(true);
expect(dmarcResult2.dkimPassed).toEqual(true);
expect(dmarcResult2.spfDomainAligned).toEqual(false);
expect(dmarcResult2.dkimDomainAligned).toEqual(false);
// Since there's no DMARC record in test environment, we expect "none" policy
expect(dmarcResult2.policyEvaluated).toEqual(DmarcPolicy.NONE);
});
tap.test('DMARC Verifier - should apply policy correctly', async () => {
const dmarcVerifier = new DmarcVerifier(platformService.mtaService);
// Create test email
const email = new Email({
from: 'sender@example.com',
to: 'recipient@example.net',
subject: 'Test DMARC policy application',
text: 'This is a test email'
});
// Test pass action
const passResult = {
hasDmarc: true,
spfDomainAligned: true,
dkimDomainAligned: true,
spfPassed: true,
dkimPassed: true,
policyEvaluated: DmarcPolicy.NONE,
actualPolicy: DmarcPolicy.NONE,
appliedPercentage: 100,
action: 'pass',
details: 'DMARC passed'
};
const passApplied = dmarcVerifier.applyPolicy(email, passResult);
expect(passApplied).toEqual(true);
expect(email.mightBeSpam).toEqual(false);
expect(email.headers['X-DMARC-Result']).toEqual('DMARC passed');
// Test quarantine action
const quarantineResult = {
hasDmarc: true,
spfDomainAligned: false,
dkimDomainAligned: false,
spfPassed: false,
dkimPassed: false,
policyEvaluated: DmarcPolicy.QUARANTINE,
actualPolicy: DmarcPolicy.QUARANTINE,
appliedPercentage: 100,
action: 'quarantine',
details: 'DMARC failed, policy=quarantine'
};
// Reset email spam flag
email.mightBeSpam = false;
email.headers = {};
const quarantineApplied = dmarcVerifier.applyPolicy(email, quarantineResult);
expect(quarantineApplied).toEqual(true);
expect(email.mightBeSpam).toEqual(true);
expect(email.headers['X-Spam-Flag']).toEqual('YES');
expect(email.headers['X-DMARC-Result']).toEqual('DMARC failed, policy=quarantine');
// Test reject action
const rejectResult = {
hasDmarc: true,
spfDomainAligned: false,
dkimDomainAligned: false,
spfPassed: false,
dkimPassed: false,
policyEvaluated: DmarcPolicy.REJECT,
actualPolicy: DmarcPolicy.REJECT,
appliedPercentage: 100,
action: 'reject',
details: 'DMARC failed, policy=reject'
};
// Reset email spam flag
email.mightBeSpam = false;
email.headers = {};
const rejectApplied = dmarcVerifier.applyPolicy(email, rejectResult);
expect(rejectApplied).toEqual(false);
expect(email.mightBeSpam).toEqual(true);
});
tap.test('Cleanup test environment', async () => {
await platformService.stop();
});
export default tap.start();

175
test/test.ipreputationchecker.ts Normal file
View File

@ -0,0 +1,175 @@
import { tap, expect } from '@push.rocks/tapbundle';
import { IPReputationChecker, ReputationThreshold, IPType } from '../ts/security/classes.ipreputationchecker.js';
import * as plugins from '../ts/plugins.js';
// Mock for dns lookup
const originalDnsResolve = plugins.dns.promises.resolve;
let mockDnsResolveImpl: (hostname: string) => Promise<string[]> = async () => ['127.0.0.1'];
// Setup mock DNS resolver
plugins.dns.promises.resolve = async (hostname: string) => {
return mockDnsResolveImpl(hostname);
};
// Test instantiation
tap.test('IPReputationChecker - should be instantiable', async () => {
const checker = IPReputationChecker.getInstance({
enableDNSBL: false,
enableIPInfo: false,
enableLocalCache: false
});
expect(checker).toBeTruthy();
});
// Test singleton pattern
tap.test('IPReputationChecker - should use singleton pattern', async () => {
const checker1 = IPReputationChecker.getInstance();
const checker2 = IPReputationChecker.getInstance();
// Both instances should be the same object
expect(checker1 === checker2).toEqual(true);
});
// Test IP validation
tap.test('IPReputationChecker - should validate IP address format', async () => {
const checker = IPReputationChecker.getInstance({
enableDNSBL: false,
enableIPInfo: false,
enableLocalCache: false
});
// Valid IP should work
const result = await checker.checkReputation('192.168.1.1');
expect(result.score).toBeGreaterThan(0);
expect(result.error).toBeUndefined();
// Invalid IP should fail with error
const invalidResult = await checker.checkReputation('invalid.ip');
expect(invalidResult.error).toBeTruthy();
});
// Test DNSBL lookups
tap.test('IPReputationChecker - should check IP against DNSBL', async () => {
try {
// Setup mock implementation for DNSBL
mockDnsResolveImpl = async (hostname: string) => {
// Listed in DNSBL if IP contains 2
if (hostname.includes('2.1.168.192') && hostname.includes('zen.spamhaus.org')) {
return ['127.0.0.2'];
}
throw { code: 'ENOTFOUND' };
};
// Create a new instance with specific settings for this test
const testInstance = new IPReputationChecker({
dnsblServers: ['zen.spamhaus.org'],
enableIPInfo: false,
enableLocalCache: false,
maxCacheSize: 1 // Small cache for testing
});
// Clean IP should have good score
const cleanResult = await testInstance.checkReputation('192.168.1.1');
expect(cleanResult.isSpam).toEqual(false);
expect(cleanResult.score).toEqual(100);
// Blacklisted IP should have reduced score
const blacklistedResult = await testInstance.checkReputation('192.168.1.2');
expect(blacklistedResult.isSpam).toEqual(true);
expect(blacklistedResult.score < 100).toEqual(true); // Less than 100
expect(blacklistedResult.blacklists).toBeTruthy();
expect((blacklistedResult.blacklists || []).length > 0).toEqual(true);
} catch (err) {
console.error('Test error:', err);
throw err;
}
});
// Test caching behavior
tap.test('IPReputationChecker - should cache reputation results', async () => {
// Create a fresh instance for this test
const testInstance = new IPReputationChecker({
enableIPInfo: false,
enableLocalCache: false,
maxCacheSize: 10 // Small cache for testing
});
// Check that first look performs a lookup and second uses cache
const ip = '192.168.1.10';
// First check should add to cache
const result1 = await testInstance.checkReputation(ip);
expect(result1).toBeTruthy();
// Manually verify it's in cache - access private member for testing
const hasInCache = (testInstance as any).reputationCache.has(ip);
expect(hasInCache).toEqual(true);
// Call again, should use cache
const result2 = await testInstance.checkReputation(ip);
expect(result2).toBeTruthy();
// Results should be identical
expect(result1.score).toEqual(result2.score);
});
// Test risk level classification
tap.test('IPReputationChecker - should classify risk levels correctly', async () => {
expect(IPReputationChecker.getRiskLevel(10)).toEqual('high');
expect(IPReputationChecker.getRiskLevel(30)).toEqual('medium');
expect(IPReputationChecker.getRiskLevel(60)).toEqual('low');
expect(IPReputationChecker.getRiskLevel(90)).toEqual('trusted');
});
// Test IP type detection
tap.test('IPReputationChecker - should detect special IP types', async () => {
const testInstance = new IPReputationChecker({
enableDNSBL: false,
enableIPInfo: true,
enableLocalCache: false,
maxCacheSize: 5 // Small cache for testing
});
// Test Tor exit node detection
const torResult = await testInstance.checkReputation('171.25.1.1');
expect(torResult.isTor).toEqual(true);
expect(torResult.score < 90).toEqual(true);
// Test VPN detection
const vpnResult = await testInstance.checkReputation('185.156.1.1');
expect(vpnResult.isVPN).toEqual(true);
expect(vpnResult.score < 90).toEqual(true);
// Test proxy detection
const proxyResult = await testInstance.checkReputation('34.92.1.1');
expect(proxyResult.isProxy).toEqual(true);
expect(proxyResult.score < 90).toEqual(true);
});
// Test error handling
tap.test('IPReputationChecker - should handle DNS lookup errors gracefully', async () => {
// Setup mock implementation to simulate error
mockDnsResolveImpl = async () => {
throw new Error('DNS server error');
};
const checker = IPReputationChecker.getInstance({
dnsblServers: ['zen.spamhaus.org'],
enableIPInfo: false,
enableLocalCache: false,
maxCacheSize: 300 // Force new instance
});
// Should return a result despite errors
const result = await checker.checkReputation('192.168.1.1');
expect(result.score).toEqual(100); // No blacklist hits found due to error
expect(result.isSpam).toEqual(false);
});
// Restore original implementation at the end
tap.test('Cleanup - restore mocks', async () => {
plugins.dns.promises.resolve = originalDnsResolve;
});
export default tap.start();

300
test/test.ipwarmupmanager.ts Normal file
View File

@ -0,0 +1,300 @@
import { tap, expect } from '@push.rocks/tapbundle';
import * as plugins from '../ts/plugins.js';
import * as paths from '../ts/paths.js';
import { IPWarmupManager } from '../ts/deliverability/classes.ipwarmupmanager.js';
// Cleanup any temporary test data
const cleanupTestData = () => {
const warmupDataPath = plugins.path.join(paths.dataDir, 'warmup');
if (plugins.fs.existsSync(warmupDataPath)) {
plugins.smartfile.memory.unlinkDir(warmupDataPath);
}
};
// Helper to reset the singleton instance between tests
const resetSingleton = () => {
// @ts-ignore - accessing private static field for testing
IPWarmupManager._instance = null;
};
// Before running any tests
tap.test('setup', async () => {
cleanupTestData();
});
// Test initialization of IPWarmupManager
tap.test('should initialize IPWarmupManager with default settings', async () => {
resetSingleton();
const ipWarmupManager = IPWarmupManager.getInstance();
expect(ipWarmupManager).to.be.an('object');
expect(ipWarmupManager.getBestIPForSending).to.be.a('function');
expect(ipWarmupManager.canSendMoreToday).to.be.a('function');
expect(ipWarmupManager.getStageCount).to.be.a('function');
expect(ipWarmupManager.setActiveAllocationPolicy).to.be.a('function');
});
// Test initialization with custom settings
tap.test('should initialize IPWarmupManager with custom settings', async () => {
resetSingleton();
const ipWarmupManager = IPWarmupManager.getInstance({
enabled: true,
ipAddresses: ['192.168.1.1', '192.168.1.2'],
targetDomains: ['example.com', 'test.com'],
fallbackPercentage: 75
});
// Test setting allocation policy
ipWarmupManager.setActiveAllocationPolicy('roundRobin');
// Get best IP for sending
const bestIP = ipWarmupManager.getBestIPForSending({
from: 'test@example.com',
to: ['recipient@test.com'],
domain: 'example.com'
});
// Check if we can send more today
const canSendMore = ipWarmupManager.canSendMoreToday('192.168.1.1');
// Check stage count
const stageCount = ipWarmupManager.getStageCount();
expect(stageCount).to.be.a('number');
});
// Test IP allocation policies
tap.test('should allocate IPs using balanced policy', async () => {
resetSingleton();
const ipWarmupManager = IPWarmupManager.getInstance({
enabled: true,
ipAddresses: ['192.168.1.1', '192.168.1.2', '192.168.1.3'],
targetDomains: ['example.com', 'test.com'],
allocationPolicy: 'balanced'
});
ipWarmupManager.setActiveAllocationPolicy('balanced');
// Use getBestIPForSending multiple times and check if all IPs are used
const usedIPs = new Set();
for (let i = 0; i < 30; i++) {
const ip = ipWarmupManager.getBestIPForSending({
from: 'test@example.com',
to: ['recipient@test.com'],
domain: 'example.com'
});
if (ip) usedIPs.add(ip);
}
// We should use at least 2 different IPs with balanced policy
expect(usedIPs.size).to.be.at.least(2);
});
// Test round robin allocation policy
tap.test('should allocate IPs using round robin policy', async () => {
resetSingleton();
const ipWarmupManager = IPWarmupManager.getInstance({
enabled: true,
ipAddresses: ['192.168.1.1', '192.168.1.2', '192.168.1.3'],
targetDomains: ['example.com', 'test.com'],
allocationPolicy: 'roundRobin'
});
ipWarmupManager.setActiveAllocationPolicy('roundRobin');
// First few IPs should rotate through the available IPs
const firstIP = ipWarmupManager.getBestIPForSending({
from: 'test@example.com',
to: ['recipient@test.com'],
domain: 'example.com'
});
const secondIP = ipWarmupManager.getBestIPForSending({
from: 'test@example.com',
to: ['recipient@test.com'],
domain: 'example.com'
});
const thirdIP = ipWarmupManager.getBestIPForSending({
from: 'test@example.com',
to: ['recipient@test.com'],
domain: 'example.com'
});
// Round robin should give us different IPs for consecutive calls
expect(firstIP).to.not.equal(secondIP);
// Fourth call should cycle back to first IP
const fourthIP = ipWarmupManager.getBestIPForSending({
from: 'test@example.com',
to: ['recipient@test.com'],
domain: 'example.com'
});
expect(fourthIP).to.equal(firstIP);
});
// Test dedicated domain allocation policy
tap.test('should allocate IPs using dedicated domain policy', async () => {
resetSingleton();
const ipWarmupManager = IPWarmupManager.getInstance({
enabled: true,
ipAddresses: ['192.168.1.1', '192.168.1.2', '192.168.1.3'],
targetDomains: ['example.com', 'test.com', 'other.com'],
allocationPolicy: 'dedicatedDomain'
});
ipWarmupManager.setActiveAllocationPolicy('dedicatedDomain');
// Map domains to IPs
ipWarmupManager.mapDomainToIP('example.com', '192.168.1.1');
ipWarmupManager.mapDomainToIP('test.com', '192.168.1.2');
ipWarmupManager.mapDomainToIP('other.com', '192.168.1.3');
// Each domain should get its dedicated IP
const exampleIP = ipWarmupManager.getBestIPForSending({
from: 'test@example.com',
to: ['recipient@gmail.com'],
domain: 'example.com'
});
const testIP = ipWarmupManager.getBestIPForSending({
from: 'test@test.com',
to: ['recipient@gmail.com'],
domain: 'test.com'
});
const otherIP = ipWarmupManager.getBestIPForSending({
from: 'test@other.com',
to: ['recipient@gmail.com'],
domain: 'other.com'
});
expect(exampleIP).to.equal('192.168.1.1');
expect(testIP).to.equal('192.168.1.2');
expect(otherIP).to.equal('192.168.1.3');
});
// Test daily sending limits
tap.test('should enforce daily sending limits', async () => {
resetSingleton();
const ipWarmupManager = IPWarmupManager.getInstance({
enabled: true,
ipAddresses: ['192.168.1.1'],
targetDomains: ['example.com'],
allocationPolicy: 'balanced'
});
// Override the warmup stage for testing
// @ts-ignore - accessing private method for testing
ipWarmupManager.warmupStatus.set('192.168.1.1', {
isActive: true,
currentStage: 0,
startDate: new Date(),
dailySendCount: 0,
hourlySendCount: {}
});
// Set a very low daily limit for testing
// @ts-ignore - accessing private method for testing
ipWarmupManager.warmupStages = [
{ dailyLimit: 5, duration: 5, hourlyPercentage: { min: 0, max: 40 } }
];
// First 5 sends should succeed
for (let i = 0; i < 5; i++) {
const ip = ipWarmupManager.getBestIPForSending({
from: 'test@example.com',
to: ['recipient@test.com'],
domain: 'example.com'
});
expect(ip).to.equal('192.168.1.1');
ipWarmupManager.recordSend(ip);
}
// 6th send should not get an IP due to daily limit
const sixthIP = ipWarmupManager.getBestIPForSending({
from: 'test@example.com',
to: ['recipient@test.com'],
domain: 'example.com'
});
expect(sixthIP).to.be.null;
});
// Test recording sends
tap.test('should record send events correctly', async () => {
resetSingleton();
const ipWarmupManager = IPWarmupManager.getInstance({
enabled: true,
ipAddresses: ['192.168.1.1', '192.168.1.2'],
targetDomains: ['example.com'],
});
// Set allocation policy
ipWarmupManager.setActiveAllocationPolicy('balanced');
// Get an IP for sending
const ip = ipWarmupManager.getBestIPForSending({
from: 'test@example.com',
to: ['recipient@test.com'],
domain: 'example.com'
});
// If we got an IP, record some sends
if (ip) {
// Record a few sends
for (let i = 0; i < 5; i++) {
ipWarmupManager.recordSend(ip);
}
// Check if we can still send more
const canSendMore = ipWarmupManager.canSendMoreToday(ip);
expect(canSendMore).to.be.a('boolean');
}
});
// Test that DedicatedDomainPolicy assigns IPs correctly
tap.test('should assign IPs using dedicated domain policy', async () => {
resetSingleton();
const ipWarmupManager = IPWarmupManager.getInstance({
enabled: true,
ipAddresses: ['192.168.1.1', '192.168.1.2', '192.168.1.3'],
targetDomains: ['example.com', 'test.com', 'other.com']
});
// Set allocation policy to dedicated domains
ipWarmupManager.setActiveAllocationPolicy('dedicated');
// Check allocation by querying for different domains
const ip1 = ipWarmupManager.getBestIPForSending({
from: 'test@example.com',
to: ['recipient@test.com'],
domain: 'example.com'
});
const ip2 = ipWarmupManager.getBestIPForSending({
from: 'test@test.com',
to: ['recipient@test.com'],
domain: 'test.com'
});
// If we got IPs, they should be consistently assigned
if (ip1 && ip2) {
// Requesting the same domain again should return the same IP
const ip1again = ipWarmupManager.getBestIPForSending({
from: 'another@example.com',
to: ['recipient@test.com'],
domain: 'example.com'
});
expect(ip1again).to.equal(ip1);
}
});
// After all tests, clean up
tap.test('cleanup', async () => {
cleanupTestData();
});
export default tap.start();

62
test/test.minimal.ts Normal file
View File

@ -0,0 +1,62 @@
import { tap } from '@push.rocks/tapbundle';
import * as plugins from '../ts/plugins.js';
import * as paths from '../ts/paths.js';
import { SenderReputationMonitor } from '../ts/deliverability/classes.senderreputationmonitor.js';
import { IPWarmupManager } from '../ts/deliverability/classes.ipwarmupmanager.js';
/**
* Basic test to check if our integrated classes work correctly
*/
tap.test('verify that SenderReputationMonitor and IPWarmupManager are functioning', async (tools) => {
// Create instances of both classes
const reputationMonitor = SenderReputationMonitor.getInstance({
enabled: true,
domains: ['example.com']
});
const ipWarmupManager = IPWarmupManager.getInstance({
enabled: true,
ipAddresses: ['192.168.1.1', '192.168.1.2'],
targetDomains: ['example.com']
});
// Test SenderReputationMonitor
reputationMonitor.recordSendEvent('example.com', { type: 'sent', count: 100 });
reputationMonitor.recordSendEvent('example.com', { type: 'delivered', count: 95 });
const reputationData = reputationMonitor.getReputationData('example.com');
const summary = reputationMonitor.getReputationSummary();
// Basic checks
tools.ok(reputationData, 'Got reputation data');
tools.ok(summary.length > 0, 'Got reputation summary');
// Add and remove domains
reputationMonitor.addDomain('test.com');
reputationMonitor.removeDomain('test.com');
// Test IPWarmupManager
ipWarmupManager.setActiveAllocationPolicy('balanced');
const bestIP = ipWarmupManager.getBestIPForSending({
from: 'test@example.com',
to: ['recipient@test.com'],
domain: 'example.com'
});
if (bestIP) {
ipWarmupManager.recordSend(bestIP);
const canSendMore = ipWarmupManager.canSendMoreToday(bestIP);
tools.ok(canSendMore !== undefined, 'Can check if sending more is allowed');
}
const stageCount = ipWarmupManager.getStageCount();
tools.ok(stageCount > 0, 'Got stage count');
});
// Final clean-up test
tap.test('clean up after tests', async () => {
// No-op - just to make sure everything is cleaned up properly
});
export default tap.start();

137
test/test.ratelimiter.ts Normal file
View File

@ -0,0 +1,137 @@
import { tap, expect } from '@push.rocks/tapbundle';
import { RateLimiter } from '../ts/mta/classes.ratelimiter.js';
tap.test('RateLimiter - should be instantiable', async () => {
const limiter = new RateLimiter({
maxPerPeriod: 10,
periodMs: 1000,
perKey: true
});
expect(limiter).toBeTruthy();
});
tap.test('RateLimiter - should allow requests within rate limit', async () => {
const limiter = new RateLimiter({
maxPerPeriod: 5,
periodMs: 1000,
perKey: true
});
// Should allow 5 requests
for (let i = 0; i < 5; i++) {
expect(limiter.isAllowed('test')).toEqual(true);
}
// 6th request should be denied
expect(limiter.isAllowed('test')).toEqual(false);
});
tap.test('RateLimiter - should enforce per-key limits', async () => {
const limiter = new RateLimiter({
maxPerPeriod: 3,
periodMs: 1000,
perKey: true
});
// Should allow 3 requests for key1
for (let i = 0; i < 3; i++) {
expect(limiter.isAllowed('key1')).toEqual(true);
}
// 4th request for key1 should be denied
expect(limiter.isAllowed('key1')).toEqual(false);
// But key2 should still be allowed
expect(limiter.isAllowed('key2')).toEqual(true);
});
tap.test('RateLimiter - should refill tokens over time', async () => {
const limiter = new RateLimiter({
maxPerPeriod: 2,
periodMs: 100, // Short period for testing
perKey: true
});
// Use all tokens
expect(limiter.isAllowed('test')).toEqual(true);
expect(limiter.isAllowed('test')).toEqual(true);
expect(limiter.isAllowed('test')).toEqual(false);
// Wait for refill
await new Promise(resolve => setTimeout(resolve, 150));
// Should have tokens again
expect(limiter.isAllowed('test')).toEqual(true);
});
tap.test('RateLimiter - should support burst allowance', async () => {
const limiter = new RateLimiter({
maxPerPeriod: 2,
periodMs: 100,
perKey: true,
burstTokens: 2, // Allow 2 extra tokens for bursts
initialTokens: 4 // Start with max + burst tokens
});
// Should allow 4 requests (2 regular + 2 burst)
for (let i = 0; i < 4; i++) {
expect(limiter.isAllowed('test')).toEqual(true);
}
// 5th request should be denied
expect(limiter.isAllowed('test')).toEqual(false);
// Wait for refill
await new Promise(resolve => setTimeout(resolve, 150));
// Should have 2 tokens again (rate-limited to normal max, not burst)
expect(limiter.isAllowed('test')).toEqual(true);
expect(limiter.isAllowed('test')).toEqual(true);
// 3rd request after refill should fail (only normal max is refilled, not burst)
expect(limiter.isAllowed('test')).toEqual(false);
});
tap.test('RateLimiter - should return correct stats', async () => {
const limiter = new RateLimiter({
maxPerPeriod: 10,
periodMs: 1000,
perKey: true
});
// Make some requests
limiter.isAllowed('test');
limiter.isAllowed('test');
limiter.isAllowed('test');
// Get stats
const stats = limiter.getStats('test');
expect(stats.remaining).toEqual(7);
expect(stats.limit).toEqual(10);
expect(stats.allowed).toEqual(3);
expect(stats.denied).toEqual(0);
});
tap.test('RateLimiter - should reset limits', async () => {
const limiter = new RateLimiter({
maxPerPeriod: 3,
periodMs: 1000,
perKey: true
});
// Use all tokens
expect(limiter.isAllowed('test')).toEqual(true);
expect(limiter.isAllowed('test')).toEqual(true);
expect(limiter.isAllowed('test')).toEqual(true);
expect(limiter.isAllowed('test')).toEqual(false);
// Reset
limiter.reset('test');
// Should have tokens again
expect(limiter.isAllowed('test')).toEqual(true);
});
export default tap.start();

238
test/test.reputationmonitor.ts Normal file
View File

@ -0,0 +1,238 @@
import { tap, expect } from '@push.rocks/tapbundle';
import * as plugins from '../ts/plugins.js';
import * as paths from '../ts/paths.js';
import { SenderReputationMonitor } from '../ts/deliverability/classes.senderreputationmonitor.js';
// Cleanup any temporary test data
const cleanupTestData = () => {
const reputationDataPath = plugins.path.join(paths.dataDir, 'reputation');
if (plugins.fs.existsSync(reputationDataPath)) {
plugins.smartfile.memory.unlinkDir(reputationDataPath);
}
};
// Helper to reset the singleton instance between tests
const resetSingleton = () => {
// @ts-ignore - accessing private static field for testing
SenderReputationMonitor._instance = null;
};
// Before running any tests
tap.test('setup', async () => {
cleanupTestData();
});
// Test initialization of SenderReputationMonitor
tap.test('should initialize SenderReputationMonitor with default settings', async () => {
resetSingleton();
const reputationMonitor = SenderReputationMonitor.getInstance();
expect(reputationMonitor).to.be.an('object');
// Check if the object has the expected methods
expect(reputationMonitor.recordSendEvent).to.be.a('function');
expect(reputationMonitor.getReputationData).to.be.a('function');
expect(reputationMonitor.getReputationSummary).to.be.a('function');
});
// Test initialization with custom settings
tap.test('should initialize SenderReputationMonitor with custom settings', async () => {
resetSingleton();
const reputationMonitor = SenderReputationMonitor.getInstance({
enabled: true,
domains: ['example.com', 'test.com'],
updateFrequency: 12 * 60 * 60 * 1000, // 12 hours
alertThresholds: {
minReputationScore: 80,
maxComplaintRate: 0.05
}
});
// Test adding domains
reputationMonitor.addDomain('newdomain.com');
// Test retrieving reputation data
const data = reputationMonitor.getReputationData('example.com');
expect(data).to.be.an('object');
expect(data.domain).to.equal('example.com');
});
// Test recording and tracking send events
tap.test('should record send events and update metrics', async () => {
resetSingleton();
const reputationMonitor = SenderReputationMonitor.getInstance({
enabled: true,
domains: ['example.com']
});
// Record a series of events
reputationMonitor.recordSendEvent('example.com', { type: 'sent', count: 100 });
reputationMonitor.recordSendEvent('example.com', { type: 'delivered', count: 95 });
reputationMonitor.recordSendEvent('example.com', { type: 'bounce', hardBounce: true, count: 3 });
reputationMonitor.recordSendEvent('example.com', { type: 'bounce', hardBounce: false, count: 2 });
reputationMonitor.recordSendEvent('example.com', { type: 'complaint', count: 1 });
// Check metrics
const metrics = reputationMonitor.getReputationData('example.com');
expect(metrics).to.be.an('object');
expect(metrics.volume.sent).to.equal(100);
expect(metrics.volume.delivered).to.equal(95);
expect(metrics.volume.hardBounces).to.equal(3);
expect(metrics.volume.softBounces).to.equal(2);
expect(metrics.complaints.total).to.equal(1);
});
// Test reputation score calculation
tap.test('should calculate reputation scores correctly', async () => {
resetSingleton();
const reputationMonitor = SenderReputationMonitor.getInstance({
enabled: true,
domains: ['high.com', 'medium.com', 'low.com']
});
// Record events for different domains
reputationMonitor.recordSendEvent('high.com', { type: 'sent', count: 1000 });
reputationMonitor.recordSendEvent('high.com', { type: 'delivered', count: 990 });
reputationMonitor.recordSendEvent('high.com', { type: 'open', count: 500 });
reputationMonitor.recordSendEvent('medium.com', { type: 'sent', count: 1000 });
reputationMonitor.recordSendEvent('medium.com', { type: 'delivered', count: 950 });
reputationMonitor.recordSendEvent('medium.com', { type: 'open', count: 300 });
reputationMonitor.recordSendEvent('low.com', { type: 'sent', count: 1000 });
reputationMonitor.recordSendEvent('low.com', { type: 'delivered', count: 850 });
reputationMonitor.recordSendEvent('low.com', { type: 'open', count: 100 });
// Get reputation summary
const summary = reputationMonitor.getReputationSummary();
expect(summary).to.be.an('array');
expect(summary.length).to.be.at.least(3);
// Check that domains are included in the summary
const domains = summary.map(item => item.domain);
expect(domains).to.include('high.com');
expect(domains).to.include('medium.com');
expect(domains).to.include('low.com');
});
// Test adding and removing domains
tap.test('should add and remove domains for monitoring', async () => {
resetSingleton();
const reputationMonitor = SenderReputationMonitor.getInstance({
enabled: true,
domains: ['example.com']
});
// Add a new domain
reputationMonitor.addDomain('newdomain.com');
// Record data for the new domain
reputationMonitor.recordSendEvent('newdomain.com', { type: 'sent', count: 50 });
// Check that data was recorded for the new domain
const metrics = reputationMonitor.getReputationData('newdomain.com');
expect(metrics).to.be.an('object');
expect(metrics.volume.sent).to.equal(50);
// Remove a domain
reputationMonitor.removeDomain('newdomain.com');
// Check that data is no longer available
const removedMetrics = reputationMonitor.getReputationData('newdomain.com');
expect(removedMetrics).to.be.null;
});
// Test handling open and click events
tap.test('should track engagement metrics correctly', async () => {
resetSingleton();
const reputationMonitor = SenderReputationMonitor.getInstance({
enabled: true,
domains: ['example.com']
});
// Record basic sending metrics
reputationMonitor.recordSendEvent('example.com', { type: 'sent', count: 1000 });
reputationMonitor.recordSendEvent('example.com', { type: 'delivered', count: 950 });
// Record engagement events
reputationMonitor.recordSendEvent('example.com', { type: 'open', count: 500 });
reputationMonitor.recordSendEvent('example.com', { type: 'click', count: 250 });
// Check engagement metrics
const metrics = reputationMonitor.getReputationData('example.com');
expect(metrics).to.be.an('object');
expect(metrics.engagement.opens).to.equal(500);
expect(metrics.engagement.clicks).to.equal(250);
expect(metrics.engagement.openRate).to.be.a('number');
expect(metrics.engagement.clickRate).to.be.a('number');
});
// Test historical data tracking
tap.test('should store historical reputation data', async () => {
resetSingleton();
const reputationMonitor = SenderReputationMonitor.getInstance({
enabled: true,
domains: ['example.com']
});
// Record events over multiple days
const today = new Date();
// Record data
reputationMonitor.recordSendEvent('example.com', { type: 'sent', count: 1000 });
reputationMonitor.recordSendEvent('example.com', { type: 'delivered', count: 950 });
// Get metrics data
const metrics = reputationMonitor.getReputationData('example.com');
// Check that historical data exists
expect(metrics.historical).to.be.an('object');
expect(metrics.historical.reputationScores).to.be.an('object');
// Check that daily send volume is tracked
expect(metrics.volume.dailySendVolume).to.be.an('object');
const todayStr = today.toISOString().split('T')[0];
expect(metrics.volume.dailySendVolume[todayStr]).to.equal(1000);
});
// Test event recording for different event types
tap.test('should correctly handle different event types', async () => {
resetSingleton();
const reputationMonitor = SenderReputationMonitor.getInstance({
enabled: true,
domains: ['example.com']
});
// Record different types of events
reputationMonitor.recordSendEvent('example.com', { type: 'sent', count: 100 });
reputationMonitor.recordSendEvent('example.com', { type: 'delivered', count: 95 });
reputationMonitor.recordSendEvent('example.com', { type: 'bounce', hardBounce: true, count: 3 });
reputationMonitor.recordSendEvent('example.com', { type: 'bounce', hardBounce: false, count: 2 });
reputationMonitor.recordSendEvent('example.com', { type: 'complaint', receivingDomain: 'gmail.com', count: 1 });
reputationMonitor.recordSendEvent('example.com', { type: 'open', count: 50 });
reputationMonitor.recordSendEvent('example.com', { type: 'click', count: 25 });
// Check metrics for different event types
const metrics = reputationMonitor.getReputationData('example.com');
// Check volume metrics
expect(metrics.volume.sent).to.equal(100);
expect(metrics.volume.delivered).to.equal(95);
expect(metrics.volume.hardBounces).to.equal(3);
expect(metrics.volume.softBounces).to.equal(2);
// Check complaint metrics
expect(metrics.complaints.total).to.equal(1);
expect(metrics.complaints.topDomains[0].domain).to.equal('gmail.com');
// Check engagement metrics
expect(metrics.engagement.opens).to.equal(50);
expect(metrics.engagement.clicks).to.equal(25);
});
// After all tests, clean up
tap.test('cleanup', async () => {
cleanupTestData();
});
export default tap.start();

896
ts/deliverability/classes.ipwarmupmanager.ts Normal file
View File

@ -0,0 +1,896 @@
import * as plugins from '../plugins.js';
import * as paths from '../paths.js';
import { logger } from '../logger.js';
import { LRUCache } from 'lru-cache';
/**
* Represents a single stage in the warmup process
*/
export interface IWarmupStage {
/** Stage number (1-based) */
stage: number;
/** Maximum daily email volume for this stage */
maxDailyVolume: number;
/** Duration of this stage in days */
durationDays: number;
/** Target engagement metrics for this stage */
targetMetrics?: {
/** Minimum open rate (percentage) */
minOpenRate?: number;
/** Maximum bounce rate (percentage) */
maxBounceRate?: number;
/** Maximum spam complaint rate (percentage) */
maxComplaintRate?: number;
};
}
/**
* Configuration for IP warmup process
*/
export interface IIPWarmupConfig {
/** Whether the warmup is enabled */
enabled?: boolean;
/** List of IP addresses to warm up */
ipAddresses?: string[];
/** Target domains to warm up (e.g. your sending domains) */
targetDomains?: string[];
/** Warmup stages defining volume and duration */
stages?: IWarmupStage[];
/** Date when warmup process started */
startDate?: Date;
/** Default hourly distribution for sending (percentage of daily volume per hour) */
hourlyDistribution?: number[];
/** Whether to automatically advance stages based on metrics */
autoAdvanceStages?: boolean;
/** Whether to suspend warmup if metrics decline */
suspendOnMetricDecline?: boolean;
/** Percentage of traffic to send through fallback provider during warmup */
fallbackPercentage?: number;
/** Whether to prioritize engaged subscribers during warmup */
prioritizeEngagedSubscribers?: boolean;
}
/**
* Status for a specific IP's warmup process
*/
export interface IIPWarmupStatus {
/** IP address being warmed up */
ipAddress: string;
/** Current warmup stage */
currentStage: number;
/** Start date of the warmup process */
startDate: Date;
/** Start date of the current stage */
currentStageStartDate: Date;
/** Target completion date for entire warmup */
targetCompletionDate: Date;
/** Daily volume allocation for current stage */
currentDailyAllocation: number;
/** Emails sent in current stage */
sentInCurrentStage: number;
/** Total emails sent during warmup process */
totalSent: number;
/** Whether the warmup is currently active */
isActive: boolean;
/** Daily statistics for the past week */
dailyStats: Array<{
/** Date of the statistics */
date: string;
/** Number of emails sent */
sent: number;
/** Number of emails opened */
opened: number;
/** Number of bounces */
bounces: number;
/** Number of spam complaints */
complaints: number;
}>;
/** Current metrics */
metrics: {
/** Open rate percentage */
openRate: number;
/** Bounce rate percentage */
bounceRate: number;
/** Complaint rate percentage */
complaintRate: number;
};
}
/**
* Defines methods for a policy used to allocate emails to different IPs
*/
export interface IIPAllocationPolicy {
/** Name of the policy */
name: string;
/**
* Allocate an IP address for sending an email
* @param availableIPs List of available IP addresses
* @param emailInfo Information about the email being sent
* @returns The IP to use, or null if no IP is available
*/
allocateIP(
availableIPs: Array<{ ip: string; priority: number; capacity: number }>,
emailInfo: {
from: string;
to: string[];
domain: string;
isTransactional: boolean;
isWarmup: boolean;
}
): string | null;
}
/**
* Default IP warmup configuration with industry standard stages
*/
const DEFAULT_WARMUP_CONFIG: Required<IIPWarmupConfig> = {
enabled: true,
ipAddresses: [],
targetDomains: [],
stages: [
{ stage: 1, maxDailyVolume: 50, durationDays: 2, targetMetrics: { maxBounceRate: 8, minOpenRate: 15 } },
{ stage: 2, maxDailyVolume: 100, durationDays: 2, targetMetrics: { maxBounceRate: 7, minOpenRate: 18 } },
{ stage: 3, maxDailyVolume: 500, durationDays: 3, targetMetrics: { maxBounceRate: 6, minOpenRate: 20 } },
{ stage: 4, maxDailyVolume: 1000, durationDays: 3, targetMetrics: { maxBounceRate: 5, minOpenRate: 20 } },
{ stage: 5, maxDailyVolume: 5000, durationDays: 5, targetMetrics: { maxBounceRate: 3, minOpenRate: 22 } },
{ stage: 6, maxDailyVolume: 10000, durationDays: 5, targetMetrics: { maxBounceRate: 2, minOpenRate: 25 } },
{ stage: 7, maxDailyVolume: 20000, durationDays: 5, targetMetrics: { maxBounceRate: 1, minOpenRate: 25 } },
{ stage: 8, maxDailyVolume: 50000, durationDays: 5, targetMetrics: { maxBounceRate: 1, minOpenRate: 25 } },
],
startDate: new Date(),
// Default hourly distribution (percentage per hour, sums to 100%)
hourlyDistribution: [
1, 1, 1, 1, 1, 2, 3, 5, 7, 8, 10, 11,
10, 9, 8, 6, 5, 4, 3, 2, 1, 1, 1, 0
],
autoAdvanceStages: true,
suspendOnMetricDecline: true,
fallbackPercentage: 50,
prioritizeEngagedSubscribers: true
};
/**
* Manages the IP warming process for new sending IPs
*/
export class IPWarmupManager {
private static instance: IPWarmupManager;
private config: Required<IIPWarmupConfig>;
private warmupStatuses: Map<string, IIPWarmupStatus> = new Map();
private dailySendCounts: Map<string, number> = new Map();
private hourlySendCounts: Map<string, number[]> = new Map();
private isInitialized: boolean = false;
private allocationPolicies: Map<string, IIPAllocationPolicy> = new Map();
private activePolicy: string = 'balanced';
/**
* Constructor for IPWarmupManager
* @param config Warmup configuration
*/
constructor(config: IIPWarmupConfig = {}) {
this.config = {
...DEFAULT_WARMUP_CONFIG,
...config,
stages: config.stages || [...DEFAULT_WARMUP_CONFIG.stages]
};
// Register default allocation policies
this.registerAllocationPolicy('balanced', new BalancedAllocationPolicy());
this.registerAllocationPolicy('roundRobin', new RoundRobinAllocationPolicy());
this.registerAllocationPolicy('dedicated', new DedicatedDomainPolicy());
this.initialize();
}
/**
* Get the singleton instance of IPWarmupManager
* @param config Warmup configuration
* @returns Singleton instance
*/
public static getInstance(config: IIPWarmupConfig = {}): IPWarmupManager {
if (!IPWarmupManager.instance) {
IPWarmupManager.instance = new IPWarmupManager(config);
}
return IPWarmupManager.instance;
}
/**
* Initialize the warmup manager
*/
private initialize(): void {
if (this.isInitialized) return;
try {
// Load warmup statuses from storage
this.loadWarmupStatuses();
// Initialize any new IPs that might have been added to config
for (const ip of this.config.ipAddresses) {
if (!this.warmupStatuses.has(ip)) {
this.initializeIPWarmup(ip);
}
}
// Initialize daily and hourly counters
const today = new Date().toISOString().split('T')[0];
for (const ip of this.config.ipAddresses) {
this.dailySendCounts.set(ip, 0);
this.hourlySendCounts.set(ip, Array(24).fill(0));
}
// Schedule daily reset of counters
this.scheduleDailyReset();
// Schedule daily evaluation of warmup progress
this.scheduleDailyEvaluation();
this.isInitialized = true;
logger.log('info', `IP Warmup Manager initialized with ${this.config.ipAddresses.length} IPs`);
} catch (error) {
logger.log('error', `Failed to initialize IP Warmup Manager: ${error.message}`, {
stack: error.stack
});
}
}
/**
* Initialize warmup status for a new IP address
* @param ipAddress IP address to initialize
*/
private initializeIPWarmup(ipAddress: string): void {
const startDate = new Date();
let targetCompletionDate = new Date(startDate);
// Calculate target completion date based on stages
let totalDays = 0;
for (const stage of this.config.stages) {
totalDays += stage.durationDays;
}
targetCompletionDate.setDate(targetCompletionDate.getDate() + totalDays);
const warmupStatus: IIPWarmupStatus = {
ipAddress,
currentStage: 1,
startDate,
currentStageStartDate: new Date(),
targetCompletionDate,
currentDailyAllocation: this.config.stages[0].maxDailyVolume,
sentInCurrentStage: 0,
totalSent: 0,
isActive: true,
dailyStats: [],
metrics: {
openRate: 0,
bounceRate: 0,
complaintRate: 0
}
};
this.warmupStatuses.set(ipAddress, warmupStatus);
this.saveWarmupStatuses();
logger.log('info', `Initialized warmup for IP ${ipAddress}`, {
currentStage: 1,
targetCompletion: targetCompletionDate.toISOString().split('T')[0]
});
}
/**
* Schedule daily reset of send counters
*/
private scheduleDailyReset(): void {
// Calculate time until midnight
const now = new Date();
const tomorrow = new Date(now);
tomorrow.setDate(tomorrow.getDate() + 1);
tomorrow.setHours(0, 0, 0, 0);
const timeUntilMidnight = tomorrow.getTime() - now.getTime();
// Schedule reset
setTimeout(() => {
this.resetDailyCounts();
// Reschedule for next day
this.scheduleDailyReset();
}, timeUntilMidnight);
logger.log('info', `Scheduled daily counter reset in ${Math.floor(timeUntilMidnight / 60000)} minutes`);
}
/**
* Reset daily send counters
*/
private resetDailyCounts(): void {
for (const ip of this.config.ipAddresses) {
// Save yesterday's count to history before resetting
const status = this.warmupStatuses.get(ip);
if (status) {
const yesterday = new Date();
yesterday.setDate(yesterday.getDate() - 1);
// Update daily stats with yesterday's data
const sentCount = this.dailySendCounts.get(ip) || 0;
status.dailyStats.push({
date: yesterday.toISOString().split('T')[0],
sent: sentCount,
opened: Math.floor(sentCount * status.metrics.openRate / 100),
bounces: Math.floor(sentCount * status.metrics.bounceRate / 100),
complaints: Math.floor(sentCount * status.metrics.complaintRate / 100)
});
// Keep only the last 7 days of stats
if (status.dailyStats.length > 7) {
status.dailyStats.shift();
}
}
// Reset counters for today
this.dailySendCounts.set(ip, 0);
this.hourlySendCounts.set(ip, Array(24).fill(0));
}
// Save updated statuses
this.saveWarmupStatuses();
logger.log('info', 'Daily send counters reset');
}
/**
* Schedule daily evaluation of warmup progress
*/
private scheduleDailyEvaluation(): void {
// Calculate time until 1 AM (do evaluation after midnight)
const now = new Date();
const evaluationTime = new Date(now);
evaluationTime.setDate(evaluationTime.getDate() + 1);
evaluationTime.setHours(1, 0, 0, 0);
const timeUntilEvaluation = evaluationTime.getTime() - now.getTime();
// Schedule evaluation
setTimeout(() => {
this.evaluateWarmupProgress();
// Reschedule for next day
this.scheduleDailyEvaluation();
}, timeUntilEvaluation);
logger.log('info', `Scheduled daily warmup evaluation in ${Math.floor(timeUntilEvaluation / 60000)} minutes`);
}
/**
* Evaluate warmup progress and possibly advance stages
*/
private evaluateWarmupProgress(): void {
if (!this.config.autoAdvanceStages) {
logger.log('info', 'Auto-advance stages is disabled, skipping evaluation');
return;
}
// Convert entries to array for compatibility with older JS versions
Array.from(this.warmupStatuses.entries()).forEach(([ip, status]) => {
if (!status.isActive) return;
// Check if current stage duration has elapsed
const currentStage = this.config.stages[status.currentStage - 1];
const now = new Date();
const daysSinceStageStart = Math.floor(
(now.getTime() - status.currentStageStartDate.getTime()) / (24 * 60 * 60 * 1000)
);
if (daysSinceStageStart >= currentStage.durationDays) {
// Check if metrics meet requirements for advancing
const metricsOK = this.checkStageMetrics(status, currentStage);
if (metricsOK) {
// Advance to next stage if not at the final stage
if (status.currentStage < this.config.stages.length) {
this.advanceToNextStage(ip);
} else {
logger.log('info', `IP ${ip} has completed the warmup process`);
}
} else if (this.config.suspendOnMetricDecline) {
// Suspend warmup if metrics don't meet requirements
status.isActive = false;
logger.log('warn', `Suspended warmup for IP ${ip} due to poor metrics`, {
openRate: status.metrics.openRate,
bounceRate: status.metrics.bounceRate,
complaintRate: status.metrics.complaintRate
});
} else {
// Extend current stage if metrics don't meet requirements
logger.log('info', `Extending stage ${status.currentStage} for IP ${ip} due to metrics not meeting requirements`);
}
}
});
// Save updated statuses
this.saveWarmupStatuses();
}
/**
* Check if the current metrics meet the requirements for the stage
* @param status Warmup status to check
* @param stage Stage to check against
* @returns Whether metrics meet requirements
*/
private checkStageMetrics(status: IIPWarmupStatus, stage: IWarmupStage): boolean {
// If no target metrics specified, assume met
if (!stage.targetMetrics) return true;
const metrics = status.metrics;
let meetsRequirements = true;
// Check each metric against requirements
if (stage.targetMetrics.minOpenRate !== undefined &&
metrics.openRate < stage.targetMetrics.minOpenRate) {
meetsRequirements = false;
logger.log('info', `Open rate ${metrics.openRate}% below target ${stage.targetMetrics.minOpenRate}% for IP ${status.ipAddress}`);
}
if (stage.targetMetrics.maxBounceRate !== undefined &&
metrics.bounceRate > stage.targetMetrics.maxBounceRate) {
meetsRequirements = false;
logger.log('info', `Bounce rate ${metrics.bounceRate}% above target ${stage.targetMetrics.maxBounceRate}% for IP ${status.ipAddress}`);
}
if (stage.targetMetrics.maxComplaintRate !== undefined &&
metrics.complaintRate > stage.targetMetrics.maxComplaintRate) {
meetsRequirements = false;
logger.log('info', `Complaint rate ${metrics.complaintRate}% above target ${stage.targetMetrics.maxComplaintRate}% for IP ${status.ipAddress}`);
}
return meetsRequirements;
}
/**
* Advance IP to the next warmup stage
* @param ipAddress IP address to advance
*/
private advanceToNextStage(ipAddress: string): void {
const status = this.warmupStatuses.get(ipAddress);
if (!status) return;
// Store metrics for the completed stage
const completedStage = status.currentStage;
// Advance to next stage
status.currentStage++;
status.currentStageStartDate = new Date();
status.sentInCurrentStage = 0;
// Update allocation
const newStage = this.config.stages[status.currentStage - 1];
status.currentDailyAllocation = newStage.maxDailyVolume;
logger.log('info', `Advanced IP ${ipAddress} to warmup stage ${status.currentStage}`, {
previousStage: completedStage,
newDailyLimit: status.currentDailyAllocation,
durationDays: newStage.durationDays
});
}
/**
* Get warmup status for all IPs or a specific IP
* @param ipAddress Optional specific IP to get status for
* @returns Warmup status information
*/
public getWarmupStatus(ipAddress?: string): IIPWarmupStatus | Map<string, IIPWarmupStatus> {
if (ipAddress) {
return this.warmupStatuses.get(ipAddress);
}
return this.warmupStatuses;
}
/**
* Add a new IP address to the warmup process
* @param ipAddress IP address to add
*/
public addIPToWarmup(ipAddress: string): void {
if (this.config.ipAddresses.includes(ipAddress)) {
logger.log('info', `IP ${ipAddress} is already in warmup`);
return;
}
// Add to configuration
this.config.ipAddresses.push(ipAddress);
// Initialize warmup
this.initializeIPWarmup(ipAddress);
// Initialize counters
this.dailySendCounts.set(ipAddress, 0);
this.hourlySendCounts.set(ipAddress, Array(24).fill(0));
logger.log('info', `Added IP ${ipAddress} to warmup process`);
}
/**
* Remove an IP address from the warmup process
* @param ipAddress IP address to remove
*/
public removeIPFromWarmup(ipAddress: string): void {
const index = this.config.ipAddresses.indexOf(ipAddress);
if (index === -1) {
logger.log('info', `IP ${ipAddress} is not in warmup`);
return;
}
// Remove from configuration
this.config.ipAddresses.splice(index, 1);
// Remove from statuses and counters
this.warmupStatuses.delete(ipAddress);
this.dailySendCounts.delete(ipAddress);
this.hourlySendCounts.delete(ipAddress);
this.saveWarmupStatuses();
logger.log('info', `Removed IP ${ipAddress} from warmup process`);
}
/**
* Update metrics for an IP address
* @param ipAddress IP address to update
* @param metrics New metrics
*/
public updateMetrics(
ipAddress: string,
metrics: { openRate?: number; bounceRate?: number; complaintRate?: number }
): void {
const status = this.warmupStatuses.get(ipAddress);
if (!status) {
logger.log('warn', `Cannot update metrics for IP ${ipAddress} - not in warmup`);
return;
}
// Update metrics
if (metrics.openRate !== undefined) {
status.metrics.openRate = metrics.openRate;
}
if (metrics.bounceRate !== undefined) {
status.metrics.bounceRate = metrics.bounceRate;
}
if (metrics.complaintRate !== undefined) {
status.metrics.complaintRate = metrics.complaintRate;
}
this.saveWarmupStatuses();
logger.log('info', `Updated metrics for IP ${ipAddress}`, {
openRate: status.metrics.openRate,
bounceRate: status.metrics.bounceRate,
complaintRate: status.metrics.complaintRate
});
}
/**
* Record a send event for an IP address
* @param ipAddress IP address used for sending
*/
public recordSend(ipAddress: string): void {
if (!this.config.ipAddresses.includes(ipAddress)) {
logger.log('warn', `Cannot record send for IP ${ipAddress} - not in warmup`);
return;
}
// Increment daily counter
const currentCount = this.dailySendCounts.get(ipAddress) || 0;
this.dailySendCounts.set(ipAddress, currentCount + 1);
// Increment hourly counter
const hourlyCount = this.hourlySendCounts.get(ipAddress) || Array(24).fill(0);
const currentHour = new Date().getHours();
hourlyCount[currentHour]++;
this.hourlySendCounts.set(ipAddress, hourlyCount);
// Update warmup status
const status = this.warmupStatuses.get(ipAddress);
if (status) {
status.sentInCurrentStage++;
status.totalSent++;
}
}
/**
* Check if an IP can send more emails today
* @param ipAddress IP address to check
* @returns Whether the IP can send more emails
*/
public canSendMoreToday(ipAddress: string): boolean {
if (!this.config.enabled) return true;
if (!this.config.ipAddresses.includes(ipAddress)) {
// If not in warmup, assume it can send
return true;
}
const status = this.warmupStatuses.get(ipAddress);
if (!status || !status.isActive) {
return false;
}
const currentCount = this.dailySendCounts.get(ipAddress) || 0;
return currentCount < status.currentDailyAllocation;
}
/**
* Check if an IP can send more emails in the current hour
* @param ipAddress IP address to check
* @returns Whether the IP can send more emails this hour
*/
public canSendMoreThisHour(ipAddress: string): boolean {
if (!this.config.enabled) return true;
if (!this.config.ipAddresses.includes(ipAddress)) {
// If not in warmup, assume it can send
return true;
}
const status = this.warmupStatuses.get(ipAddress);
if (!status || !status.isActive) {
return false;
}
const currentDailyLimit = status.currentDailyAllocation;
const currentHour = new Date().getHours();
const hourlyAllocation = Math.ceil((currentDailyLimit * this.config.hourlyDistribution[currentHour]) / 100);
const hourlyCount = this.hourlySendCounts.get(ipAddress) || Array(24).fill(0);
const currentHourCount = hourlyCount[currentHour];
return currentHourCount < hourlyAllocation;
}
/**
* Get the best IP to use for sending an email
* @param emailInfo Information about the email being sent
* @returns The best IP to use, or null if no suitable IP is available
*/
public getBestIPForSending(emailInfo: {
from: string;
to: string[];
domain: string;
isTransactional?: boolean;
}): string | null {
// If warmup is disabled, return null (caller will use default IP)
if (!this.config.enabled || this.config.ipAddresses.length === 0) {
return null;
}
// Prepare information for allocation policy
const availableIPs = this.config.ipAddresses
.filter(ip => this.canSendMoreToday(ip) && this.canSendMoreThisHour(ip))
.map(ip => {
const status = this.warmupStatuses.get(ip);
return {
ip,
priority: status ? status.currentStage : 1,
capacity: status ? (status.currentDailyAllocation - (this.dailySendCounts.get(ip) || 0)) : 0
};
});
// Use the active allocation policy to determine the best IP
const policy = this.allocationPolicies.get(this.activePolicy);
if (!policy) {
logger.log('warn', `No allocation policy named ${this.activePolicy} found`);
return null;
}
return policy.allocateIP(availableIPs, {
...emailInfo,
isTransactional: emailInfo.isTransactional || false,
isWarmup: true
});
}
/**
* Register a new IP allocation policy
* @param name Policy name
* @param policy Policy implementation
*/
public registerAllocationPolicy(name: string, policy: IIPAllocationPolicy): void {
this.allocationPolicies.set(name, policy);
logger.log('info', `Registered IP allocation policy: ${name}`);
}
/**
* Set the active IP allocation policy
* @param name Policy name
*/
public setActiveAllocationPolicy(name: string): void {
if (!this.allocationPolicies.has(name)) {
logger.log('warn', `No allocation policy named ${name} found`);
return;
}
this.activePolicy = name;
logger.log('info', `Set active IP allocation policy to ${name}`);
}
/**
* Get the total number of stages in the warmup process
* @returns Number of stages
*/
public getStageCount(): number {
return this.config.stages.length;
}
/**
* Load warmup statuses from storage
*/
private loadWarmupStatuses(): void {
try {
const warmupDir = plugins.path.join(paths.dataDir, 'warmup');
plugins.smartfile.fs.ensureDirSync(warmupDir);
const statusFile = plugins.path.join(warmupDir, 'ip_warmup_status.json');
if (plugins.fs.existsSync(statusFile)) {
const data = plugins.fs.readFileSync(statusFile, 'utf8');
const statuses = JSON.parse(data);
for (const status of statuses) {
// Restore date objects
status.startDate = new Date(status.startDate);
status.currentStageStartDate = new Date(status.currentStageStartDate);
status.targetCompletionDate = new Date(status.targetCompletionDate);
this.warmupStatuses.set(status.ipAddress, status);
}
logger.log('info', `Loaded ${this.warmupStatuses.size} IP warmup statuses from storage`);
}
} catch (error) {
logger.log('error', `Failed to load warmup statuses: ${error.message}`, {
stack: error.stack
});
}
}
/**
* Save warmup statuses to storage
*/
private saveWarmupStatuses(): void {
try {
const warmupDir = plugins.path.join(paths.dataDir, 'warmup');
plugins.smartfile.fs.ensureDirSync(warmupDir);
const statusFile = plugins.path.join(warmupDir, 'ip_warmup_status.json');
const statuses = Array.from(this.warmupStatuses.values());
plugins.smartfile.memory.toFsSync(
JSON.stringify(statuses, null, 2),
statusFile
);
logger.log('debug', `Saved ${statuses.length} IP warmup statuses to storage`);
} catch (error) {
logger.log('error', `Failed to save warmup statuses: ${error.message}`, {
stack: error.stack
});
}
}
}
/**
* Policy that balances traffic across IPs based on stage and capacity
*/
class BalancedAllocationPolicy implements IIPAllocationPolicy {
name = 'balanced';
allocateIP(
availableIPs: Array<{ ip: string; priority: number; capacity: number }>,
emailInfo: {
from: string;
to: string[];
domain: string;
isTransactional: boolean;
isWarmup: boolean;
}
): string | null {
if (availableIPs.length === 0) return null;
// Sort IPs by priority (prefer higher stage IPs) and capacity
const sortedIPs = [...availableIPs].sort((a, b) => {
// First by priority (descending)
if (b.priority !== a.priority) {
return b.priority - a.priority;
}
// Then by remaining capacity (descending)
return b.capacity - a.capacity;
});
// Prioritize higher-stage IPs for transactional emails
if (emailInfo.isTransactional) {
return sortedIPs[0].ip;
}
// For marketing emails, spread across IPs with preference for higher stages
// Use weighted random selection based on stage
const totalWeight = sortedIPs.reduce((sum, ip) => sum + ip.priority, 0);
let randomPoint = Math.random() * totalWeight;
for (const ip of sortedIPs) {
randomPoint -= ip.priority;
if (randomPoint <= 0) {
return ip.ip;
}
}
// Fallback to the highest priority IP
return sortedIPs[0].ip;
}
}
/**
* Policy that rotates through IPs in a round-robin fashion
*/
class RoundRobinAllocationPolicy implements IIPAllocationPolicy {
name = 'roundRobin';
private lastIndex = -1;
allocateIP(
availableIPs: Array<{ ip: string; priority: number; capacity: number }>,
emailInfo: {
from: string;
to: string[];
domain: string;
isTransactional: boolean;
isWarmup: boolean;
}
): string | null {
if (availableIPs.length === 0) return null;
// Sort by capacity to ensure even distribution
const sortedIPs = [...availableIPs].sort((a, b) => b.capacity - a.capacity);
// Move to next IP
this.lastIndex = (this.lastIndex + 1) % sortedIPs.length;
return sortedIPs[this.lastIndex].ip;
}
}
/**
* Policy that dedicates specific IPs to specific domains
*/
class DedicatedDomainPolicy implements IIPAllocationPolicy {
name = 'dedicated';
private domainAssignments: Map<string, string> = new Map();
allocateIP(
availableIPs: Array<{ ip: string; priority: number; capacity: number }>,
emailInfo: {
from: string;
to: string[];
domain: string;
isTransactional: boolean;
isWarmup: boolean;
}
): string | null {
if (availableIPs.length === 0) return null;
// Check if we have a dedicated IP for this domain
if (this.domainAssignments.has(emailInfo.domain)) {
const dedicatedIP = this.domainAssignments.get(emailInfo.domain);
// Check if the dedicated IP is in the available list
const isAvailable = availableIPs.some(ip => ip.ip === dedicatedIP);
if (isAvailable) {
return dedicatedIP;
}
}
// If not, assign one and save the assignment
const sortedIPs = [...availableIPs].sort((a, b) => b.capacity - a.capacity);
const assignedIP = sortedIPs[0].ip;
this.domainAssignments.set(emailInfo.domain, assignedIP);
return assignedIP;
}
}

1116
ts/deliverability/classes.senderreputationmonitor.ts Normal file
View File

File diff suppressed because it is too large Load Diff

13
ts/deliverability/index.ts Normal file
View File

@ -0,0 +1,13 @@
export {
IPWarmupManager,
type IIPWarmupConfig,
type IWarmupStage,
type IIPWarmupStatus,
type IIPAllocationPolicy
} from './classes.ipwarmupmanager.js';
export {
SenderReputationMonitor,
type IDomainReputationMetrics,
type IReputationMonitorConfig
} from './classes.senderreputationmonitor.js';

902
ts/email/classes.bouncemanager.ts Normal file
View File

@ -0,0 +1,902 @@
import * as plugins from '../plugins.js';
import * as paths from '../paths.js';
import { logger } from '../logger.js';
import { SecurityLogger, SecurityLogLevel, SecurityEventType } from '../security/index.js';
import { LRUCache } from 'lru-cache';
/**
* Bounce types for categorizing the reasons for bounces
*/
export enum BounceType {
// Hard bounces (permanent failures)
INVALID_RECIPIENT = 'invalid_recipient',
DOMAIN_NOT_FOUND = 'domain_not_found',
MAILBOX_FULL = 'mailbox_full',
MAILBOX_INACTIVE = 'mailbox_inactive',
BLOCKED = 'blocked',
SPAM_RELATED = 'spam_related',
POLICY_RELATED = 'policy_related',
// Soft bounces (temporary failures)
SERVER_UNAVAILABLE = 'server_unavailable',
TEMPORARY_FAILURE = 'temporary_failure',
QUOTA_EXCEEDED = 'quota_exceeded',
NETWORK_ERROR = 'network_error',
TIMEOUT = 'timeout',
// Special cases
AUTO_RESPONSE = 'auto_response',
CHALLENGE_RESPONSE = 'challenge_response',
UNKNOWN = 'unknown'
}
/**
* Hard vs soft bounce classification
*/
export enum BounceCategory {
HARD = 'hard',
SOFT = 'soft',
AUTO_RESPONSE = 'auto_response',
UNKNOWN = 'unknown'
}
/**
* Bounce data structure
*/
export interface BounceRecord {
id: string;
originalEmailId?: string;
recipient: string;
sender: string;
domain: string;
subject?: string;
bounceType: BounceType;
bounceCategory: BounceCategory;
timestamp: number;
smtpResponse?: string;
diagnosticCode?: string;
statusCode?: string;
headers?: Record<string, string>;
processed: boolean;
retryCount?: number;
nextRetryTime?: number;
}
/**
* Email bounce patterns to identify bounce types in SMTP responses and bounce messages
*/
const BOUNCE_PATTERNS = {
// Hard bounce patterns
[BounceType.INVALID_RECIPIENT]: [
/no such user/i,
/user unknown/i,
/does not exist/i,
/invalid recipient/i,
/unknown recipient/i,
/no mailbox/i,
/user not found/i,
/recipient address rejected/i,
/550 5\.1\.1/i
],
[BounceType.DOMAIN_NOT_FOUND]: [
/domain not found/i,
/unknown domain/i,
/no such domain/i,
/host not found/i,
/domain invalid/i,
/550 5\.1\.2/i
],
[BounceType.MAILBOX_FULL]: [
/mailbox full/i,
/over quota/i,
/quota exceeded/i,
/552 5\.2\.2/i
],
[BounceType.MAILBOX_INACTIVE]: [
/mailbox disabled/i,
/mailbox inactive/i,
/account disabled/i,
/mailbox not active/i,
/account suspended/i
],
[BounceType.BLOCKED]: [
/blocked/i,
/rejected/i,
/denied/i,
/blacklisted/i,
/prohibited/i,
/refused/i,
/550 5\.7\./i
],
[BounceType.SPAM_RELATED]: [
/spam/i,
/bulk mail/i,
/content rejected/i,
/message rejected/i,
/550 5\.7\.1/i
],
// Soft bounce patterns
[BounceType.SERVER_UNAVAILABLE]: [
/server unavailable/i,
/service unavailable/i,
/try again later/i,
/try later/i,
/451 4\.3\./i,
/421 4\.3\./i
],
[BounceType.TEMPORARY_FAILURE]: [
/temporary failure/i,
/temporary error/i,
/temporary problem/i,
/try again/i,
/451 4\./i
],
[BounceType.QUOTA_EXCEEDED]: [
/quota temporarily exceeded/i,
/mailbox temporarily full/i,
/452 4\.2\.2/i
],
[BounceType.NETWORK_ERROR]: [
/network error/i,
/connection error/i,
/connection timed out/i,
/routing error/i,
/421 4\.4\./i
],
[BounceType.TIMEOUT]: [
/timed out/i,
/timeout/i,
/450 4\.4\.2/i
],
// Auto-responses
[BounceType.AUTO_RESPONSE]: [
/auto[- ]reply/i,
/auto[- ]response/i,
/vacation/i,
/out of office/i,
/away from office/i,
/on vacation/i,
/automatic reply/i
],
[BounceType.CHALLENGE_RESPONSE]: [
/challenge[- ]response/i,
/verify your email/i,
/confirm your email/i,
/email verification/i
]
};
/**
* Retry strategy configuration for soft bounces
*/
interface RetryStrategy {
maxRetries: number;
initialDelay: number; // milliseconds
maxDelay: number; // milliseconds
backoffFactor: number;
}
/**
* Manager for handling email bounces
*/
export class BounceManager {
// Retry strategy with exponential backoff
private retryStrategy: RetryStrategy = {
maxRetries: 5,
initialDelay: 15 * 60 * 1000, // 15 minutes
maxDelay: 24 * 60 * 60 * 1000, // 24 hours
backoffFactor: 2
};
// Store of bounced emails
private bounceStore: BounceRecord[] = [];
// Cache of recently bounced email addresses to avoid sending to known bad addresses
private bounceCache: LRUCache<string, {
lastBounce: number;
count: number;
type: BounceType;
category: BounceCategory;
}>;
// Suppression list for addresses that should not receive emails
private suppressionList: Map<string, {
reason: string;
timestamp: number;
expiresAt?: number; // undefined means permanent
}> = new Map();
constructor(options?: {
retryStrategy?: Partial<RetryStrategy>;
maxCacheSize?: number;
cacheTTL?: number;
}) {
// Set retry strategy with defaults
if (options?.retryStrategy) {
this.retryStrategy = {
...this.retryStrategy,
...options.retryStrategy
};
}
// Initialize bounce cache with LRU (least recently used) caching
this.bounceCache = new LRUCache<string, any>({
max: options?.maxCacheSize || 10000,
ttl: options?.cacheTTL || 30 * 24 * 60 * 60 * 1000, // 30 days default
});
// Load suppression list from storage
this.loadSuppressionList();
}
/**
* Process a bounce notification
* @param bounceData Bounce data to process
* @returns Processed bounce record
*/
public async processBounce(bounceData: Partial<BounceRecord>): Promise<BounceRecord> {
try {
// Add required fields if missing
const bounce: BounceRecord = {
id: bounceData.id || plugins.uuid.v4(),
recipient: bounceData.recipient,
sender: bounceData.sender,
domain: bounceData.domain || bounceData.recipient.split('@')[1],
subject: bounceData.subject,
bounceType: bounceData.bounceType || BounceType.UNKNOWN,
bounceCategory: bounceData.bounceCategory || BounceCategory.UNKNOWN,
timestamp: bounceData.timestamp || Date.now(),
smtpResponse: bounceData.smtpResponse,
diagnosticCode: bounceData.diagnosticCode,
statusCode: bounceData.statusCode,
headers: bounceData.headers,
processed: false,
originalEmailId: bounceData.originalEmailId,
retryCount: bounceData.retryCount || 0,
nextRetryTime: bounceData.nextRetryTime
};
// Determine bounce type and category if not provided
if (!bounceData.bounceType || bounceData.bounceType === BounceType.UNKNOWN) {
const bounceInfo = this.detectBounceType(
bounce.smtpResponse || '',
bounce.diagnosticCode || '',
bounce.statusCode || ''
);
bounce.bounceType = bounceInfo.type;
bounce.bounceCategory = bounceInfo.category;
}
// Process the bounce based on category
switch (bounce.bounceCategory) {
case BounceCategory.HARD:
// Handle hard bounce - add to suppression list
await this.handleHardBounce(bounce);
break;
case BounceCategory.SOFT:
// Handle soft bounce - schedule retry if eligible
await this.handleSoftBounce(bounce);
break;
case BounceCategory.AUTO_RESPONSE:
// Handle auto-response - typically no action needed
logger.log('info', `Auto-response detected for ${bounce.recipient}`);
break;
default:
// Unknown bounce type - log for investigation
logger.log('warn', `Unknown bounce type for ${bounce.recipient}`, {
bounceType: bounce.bounceType,
smtpResponse: bounce.smtpResponse
});
break;
}
// Store the bounce record
bounce.processed = true;
this.bounceStore.push(bounce);
// Update the bounce cache
this.updateBounceCache(bounce);
// Log the bounce
logger.log(
bounce.bounceCategory === BounceCategory.HARD ? 'warn' : 'info',
`Email bounce processed: ${bounce.bounceCategory} bounce for ${bounce.recipient}`,
{
bounceType: bounce.bounceType,
domain: bounce.domain,
category: bounce.bounceCategory
}
);
// Enhanced security logging
SecurityLogger.getInstance().logEvent({
level: bounce.bounceCategory === BounceCategory.HARD
? SecurityLogLevel.WARN
: SecurityLogLevel.INFO,
type: SecurityEventType.EMAIL_VALIDATION,
message: `Email bounce detected: ${bounce.bounceCategory} bounce for recipient`,
domain: bounce.domain,
details: {
recipient: bounce.recipient,
bounceType: bounce.bounceType,
smtpResponse: bounce.smtpResponse,
diagnosticCode: bounce.diagnosticCode,
statusCode: bounce.statusCode
},
success: false
});
return bounce;
} catch (error) {
logger.log('error', `Error processing bounce: ${error.message}`, {
error: error.message,
bounceData
});
throw error;
}
}
/**
* Process an SMTP failure as a bounce
* @param recipient Recipient email
* @param smtpResponse SMTP error response
* @param options Additional options
* @returns Processed bounce record
*/
public async processSmtpFailure(
recipient: string,
smtpResponse: string,
options: {
sender?: string;
originalEmailId?: string;
statusCode?: string;
headers?: Record<string, string>;
} = {}
): Promise<BounceRecord> {
// Create bounce data from SMTP failure
const bounceData: Partial<BounceRecord> = {
recipient,
sender: options.sender || '',
domain: recipient.split('@')[1],
smtpResponse,
statusCode: options.statusCode,
headers: options.headers,
originalEmailId: options.originalEmailId,
timestamp: Date.now()
};
// Process as a regular bounce
return this.processBounce(bounceData);
}
/**
* Process a bounce notification email
* @param bounceEmail The email containing bounce information
* @returns Processed bounce record or null if not a bounce
*/
public async processBounceEmail(bounceEmail: plugins.smartmail.Smartmail<any>): Promise<BounceRecord | null> {
try {
// Check if this is a bounce notification
const subject = bounceEmail.getSubject();
const body = bounceEmail.getBody();
// Check for common bounce notification subject patterns
const isBounceSubject = /mail delivery|delivery (failed|status|notification)|failure notice|returned mail|undeliverable|delivery problem/i.test(subject);
if (!isBounceSubject) {
// Not a bounce notification based on subject
return null;
}
// Extract original recipient from the body or headers
let recipient = '';
let originalMessageId = '';
// Extract recipient from common bounce formats
const recipientMatch = body.match(/(?:failed recipient|to[:=]\s*|recipient:|delivery failed:)\s*<?([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})>?/i);
if (recipientMatch && recipientMatch[1]) {
recipient = recipientMatch[1];
}
// Extract diagnostic code
let diagnosticCode = '';
const diagnosticMatch = body.match(/diagnostic(?:-|\\s+)code:\s*(.+)(?:\n|$)/i);
if (diagnosticMatch && diagnosticMatch[1]) {
diagnosticCode = diagnosticMatch[1].trim();
}
// Extract SMTP status code
let statusCode = '';
const statusMatch = body.match(/status(?:-|\\s+)code:\s*([0-9.]+)/i);
if (statusMatch && statusMatch[1]) {
statusCode = statusMatch[1].trim();
}
// If recipient not found in standard patterns, try DSN (Delivery Status Notification) format
if (!recipient) {
// Look for DSN format with Original-Recipient or Final-Recipient fields
const originalRecipientMatch = body.match(/original-recipient:.*?([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/i);
const finalRecipientMatch = body.match(/final-recipient:.*?([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/i);
if (originalRecipientMatch && originalRecipientMatch[1]) {
recipient = originalRecipientMatch[1];
} else if (finalRecipientMatch && finalRecipientMatch[1]) {
recipient = finalRecipientMatch[1];
}
}
// If still no recipient, can't process as bounce
if (!recipient) {
logger.log('warn', 'Could not extract recipient from bounce notification', {
subject,
sender: bounceEmail.options.from
});
return null;
}
// Extract original message ID if available
const messageIdMatch = body.match(/original[ -]message[ -]id:[ \t]*<?([^>]+)>?/i);
if (messageIdMatch && messageIdMatch[1]) {
originalMessageId = messageIdMatch[1].trim();
}
// Create bounce data
const bounceData: Partial<BounceRecord> = {
recipient,
sender: bounceEmail.options.from,
domain: recipient.split('@')[1],
subject: bounceEmail.getSubject(),
diagnosticCode,
statusCode,
timestamp: Date.now(),
headers: {}
};
// Process as a regular bounce
return this.processBounce(bounceData);
} catch (error) {
logger.log('error', `Error processing bounce email: ${error.message}`);
return null;
}
}
/**
* Handle a hard bounce by adding to suppression list
* @param bounce The bounce record
*/
private async handleHardBounce(bounce: BounceRecord): Promise<void> {
// Add to suppression list permanently (no expiry)
this.addToSuppressionList(bounce.recipient, `Hard bounce: ${bounce.bounceType}`, undefined);
// Increment bounce count in cache
this.updateBounceCache(bounce);
// Save to permanent storage
this.saveBounceRecord(bounce);
// Log hard bounce for monitoring
logger.log('warn', `Hard bounce for ${bounce.recipient}: ${bounce.bounceType}`, {
domain: bounce.domain,
smtpResponse: bounce.smtpResponse,
diagnosticCode: bounce.diagnosticCode
});
}
/**
* Handle a soft bounce by scheduling a retry if eligible
* @param bounce The bounce record
*/
private async handleSoftBounce(bounce: BounceRecord): Promise<void> {
// Check if we've exceeded max retries
if (bounce.retryCount >= this.retryStrategy.maxRetries) {
logger.log('warn', `Max retries exceeded for ${bounce.recipient}, treating as hard bounce`);
// Convert to hard bounce after max retries
bounce.bounceCategory = BounceCategory.HARD;
await this.handleHardBounce(bounce);
return;
}
// Calculate next retry time with exponential backoff
const delay = Math.min(
this.retryStrategy.initialDelay * Math.pow(this.retryStrategy.backoffFactor, bounce.retryCount),
this.retryStrategy.maxDelay
);
bounce.retryCount++;
bounce.nextRetryTime = Date.now() + delay;
// Add to suppression list temporarily (with expiry)
this.addToSuppressionList(
bounce.recipient,
`Soft bounce: ${bounce.bounceType}`,
bounce.nextRetryTime
);
// Log the retry schedule
logger.log('info', `Scheduled retry ${bounce.retryCount} for ${bounce.recipient} at ${new Date(bounce.nextRetryTime).toISOString()}`, {
bounceType: bounce.bounceType,
retryCount: bounce.retryCount,
nextRetry: bounce.nextRetryTime
});
}
/**
* Add an email address to the suppression list
* @param email Email address to suppress
* @param reason Reason for suppression
* @param expiresAt Expiration timestamp (undefined for permanent)
*/
public addToSuppressionList(
email: string,
reason: string,
expiresAt?: number
): void {
this.suppressionList.set(email.toLowerCase(), {
reason,
timestamp: Date.now(),
expiresAt
});
this.saveSuppressionList();
logger.log('info', `Added ${email} to suppression list`, {
reason,
expiresAt: expiresAt ? new Date(expiresAt).toISOString() : 'permanent'
});
}
/**
* Remove an email address from the suppression list
* @param email Email address to remove
*/
public removeFromSuppressionList(email: string): void {
const wasRemoved = this.suppressionList.delete(email.toLowerCase());
if (wasRemoved) {
this.saveSuppressionList();
logger.log('info', `Removed ${email} from suppression list`);
}
}
/**
* Check if an email is on the suppression list
* @param email Email address to check
* @returns Whether the email is suppressed
*/
public isEmailSuppressed(email: string): boolean {
const lowercaseEmail = email.toLowerCase();
const suppression = this.suppressionList.get(lowercaseEmail);
if (!suppression) {
return false;
}
// Check if suppression has expired
if (suppression.expiresAt && Date.now() > suppression.expiresAt) {
this.suppressionList.delete(lowercaseEmail);
this.saveSuppressionList();
return false;
}
return true;
}
/**
* Get suppression information for an email
* @param email Email address to check
* @returns Suppression information or null if not suppressed
*/
public getSuppressionInfo(email: string): {
reason: string;
timestamp: number;
expiresAt?: number;
} | null {
const lowercaseEmail = email.toLowerCase();
const suppression = this.suppressionList.get(lowercaseEmail);
if (!suppression) {
return null;
}
// Check if suppression has expired
if (suppression.expiresAt && Date.now() > suppression.expiresAt) {
this.suppressionList.delete(lowercaseEmail);
this.saveSuppressionList();
return null;
}
return suppression;
}
/**
* Save suppression list to disk
*/
private saveSuppressionList(): void {
try {
const suppressionData = JSON.stringify(Array.from(this.suppressionList.entries()));
plugins.smartfile.memory.toFsSync(
suppressionData,
plugins.path.join(paths.dataDir, 'emails', 'suppression_list.json')
);
} catch (error) {
logger.log('error', `Failed to save suppression list: ${error.message}`);
}
}
/**
* Load suppression list from disk
*/
private loadSuppressionList(): void {
try {
const suppressionPath = plugins.path.join(paths.dataDir, 'emails', 'suppression_list.json');
if (plugins.fs.existsSync(suppressionPath)) {
const data = plugins.fs.readFileSync(suppressionPath, 'utf8');
const entries = JSON.parse(data);
this.suppressionList = new Map(entries);
// Clean expired entries
const now = Date.now();
let expiredCount = 0;
for (const [email, info] of this.suppressionList.entries()) {
if (info.expiresAt && now > info.expiresAt) {
this.suppressionList.delete(email);
expiredCount++;
}
}
if (expiredCount > 0) {
logger.log('info', `Cleaned ${expiredCount} expired entries from suppression list`);
this.saveSuppressionList();
}
logger.log('info', `Loaded ${this.suppressionList.size} entries from suppression list`);
}
} catch (error) {
logger.log('error', `Failed to load suppression list: ${error.message}`);
}
}
/**
* Save bounce record to disk
* @param bounce Bounce record to save
*/
private saveBounceRecord(bounce: BounceRecord): void {
try {
const bounceData = JSON.stringify(bounce);
const bouncePath = plugins.path.join(
paths.dataDir,
'emails',
'bounces',
`${bounce.id}.json`
);
// Ensure directory exists
const bounceDir = plugins.path.join(paths.dataDir, 'emails', 'bounces');
plugins.smartfile.fs.ensureDirSync(bounceDir);
plugins.smartfile.memory.toFsSync(bounceData, bouncePath);
} catch (error) {
logger.log('error', `Failed to save bounce record: ${error.message}`);
}
}
/**
* Update bounce cache with new bounce information
* @param bounce Bounce record to update cache with
*/
private updateBounceCache(bounce: BounceRecord): void {
const email = bounce.recipient.toLowerCase();
const existing = this.bounceCache.get(email);
if (existing) {
// Update existing cache entry
existing.lastBounce = bounce.timestamp;
existing.count++;
existing.type = bounce.bounceType;
existing.category = bounce.bounceCategory;
} else {
// Create new cache entry
this.bounceCache.set(email, {
lastBounce: bounce.timestamp,
count: 1,
type: bounce.bounceType,
category: bounce.bounceCategory
});
}
}
/**
* Check bounce history for an email address
* @param email Email address to check
* @returns Bounce information or null if no bounces
*/
public getBounceInfo(email: string): {
lastBounce: number;
count: number;
type: BounceType;
category: BounceCategory;
} | null {
return this.bounceCache.get(email.toLowerCase()) || null;
}
/**
* Analyze SMTP response and diagnostic codes to determine bounce type
* @param smtpResponse SMTP response string
* @param diagnosticCode Diagnostic code from bounce
* @param statusCode Status code from bounce
* @returns Detected bounce type and category
*/
private detectBounceType(
smtpResponse: string,
diagnosticCode: string,
statusCode: string
): {
type: BounceType;
category: BounceCategory;
} {
// Combine all text for comprehensive pattern matching
const fullText = `${smtpResponse} ${diagnosticCode} ${statusCode}`.toLowerCase();
// Check for auto-responses first
if (this.matchesPattern(fullText, BounceType.AUTO_RESPONSE) ||
this.matchesPattern(fullText, BounceType.CHALLENGE_RESPONSE)) {
return {
type: BounceType.AUTO_RESPONSE,
category: BounceCategory.AUTO_RESPONSE
};
}
// Check for hard bounces
for (const bounceType of [
BounceType.INVALID_RECIPIENT,
BounceType.DOMAIN_NOT_FOUND,
BounceType.MAILBOX_FULL,
BounceType.MAILBOX_INACTIVE,
BounceType.BLOCKED,
BounceType.SPAM_RELATED,
BounceType.POLICY_RELATED
]) {
if (this.matchesPattern(fullText, bounceType)) {
return {
type: bounceType,
category: BounceCategory.HARD
};
}
}
// Check for soft bounces
for (const bounceType of [
BounceType.SERVER_UNAVAILABLE,
BounceType.TEMPORARY_FAILURE,
BounceType.QUOTA_EXCEEDED,
BounceType.NETWORK_ERROR,
BounceType.TIMEOUT
]) {
if (this.matchesPattern(fullText, bounceType)) {
return {
type: bounceType,
category: BounceCategory.SOFT
};
}
}
// Handle DSN (Delivery Status Notification) status codes
if (statusCode) {
// Format: class.subject.detail
const parts = statusCode.split('.');
if (parts.length >= 2) {
const statusClass = parts[0];
const statusSubject = parts[1];
// 5.X.X is permanent failure (hard bounce)
if (statusClass === '5') {
// Try to determine specific type based on subject
if (statusSubject === '1') {
return { type: BounceType.INVALID_RECIPIENT, category: BounceCategory.HARD };
} else if (statusSubject === '2') {
return { type: BounceType.MAILBOX_FULL, category: BounceCategory.HARD };
} else if (statusSubject === '7') {
return { type: BounceType.BLOCKED, category: BounceCategory.HARD };
} else {
return { type: BounceType.UNKNOWN, category: BounceCategory.HARD };
}
}
// 4.X.X is temporary failure (soft bounce)
if (statusClass === '4') {
// Try to determine specific type based on subject
if (statusSubject === '2') {
return { type: BounceType.QUOTA_EXCEEDED, category: BounceCategory.SOFT };
} else if (statusSubject === '3') {
return { type: BounceType.SERVER_UNAVAILABLE, category: BounceCategory.SOFT };
} else if (statusSubject === '4') {
return { type: BounceType.NETWORK_ERROR, category: BounceCategory.SOFT };
} else {
return { type: BounceType.TEMPORARY_FAILURE, category: BounceCategory.SOFT };
}
}
}
}
// Default to unknown
return {
type: BounceType.UNKNOWN,
category: BounceCategory.UNKNOWN
};
}
/**
* Check if text matches any pattern for a bounce type
* @param text Text to check against patterns
* @param bounceType Bounce type to get patterns for
* @returns Whether the text matches any pattern
*/
private matchesPattern(text: string, bounceType: BounceType): boolean {
const patterns = BOUNCE_PATTERNS[bounceType];
if (!patterns) {
return false;
}
for (const pattern of patterns) {
if (pattern.test(text)) {
return true;
}
}
return false;
}
/**
* Get all known hard bounced addresses
* @returns Array of hard bounced email addresses
*/
public getHardBouncedAddresses(): string[] {
const hardBounced: string[] = [];
for (const [email, info] of this.bounceCache.entries()) {
if (info.category === BounceCategory.HARD) {
hardBounced.push(email);
}
}
return hardBounced;
}
/**
* Get suppression list
* @returns Array of suppressed email addresses
*/
public getSuppressionList(): string[] {
return Array.from(this.suppressionList.keys());
}
/**
* Clear old bounce records (for maintenance)
* @param olderThan Timestamp to remove records older than
* @returns Number of records removed
*/
public clearOldBounceRecords(olderThan: number): number {
let removed = 0;
this.bounceStore = this.bounceStore.filter(bounce => {
if (bounce.timestamp < olderThan) {
removed++;
return false;
}
return true;
});
return removed;
}
}

67
ts/email/classes.connector.mta.ts
View File

@ -31,15 +31,42 @@ export class MtaConnector {
toAddresses: string | string[],
options: any = {}
): Promise<string> {
// Check if recipients are on the suppression list
const recipients = Array.isArray(toAddresses)
? toAddresses
: toAddresses.split(',').map(addr => addr.trim());
// Filter out suppressed recipients
const validRecipients = [];
const suppressedRecipients = [];
for (const recipient of recipients) {
if (this.emailRef.bounceManager.isEmailSuppressed(recipient)) {
suppressedRecipients.push(recipient);
} else {
validRecipients.push(recipient);
}
}
// Log suppressed recipients
if (suppressedRecipients.length > 0) {
logger.log('warn', `Skipping ${suppressedRecipients.length} suppressed recipients`, {
suppressedRecipients
});
}
// If all recipients are suppressed, throw error
if (validRecipients.length === 0) {
throw new Error('All recipients are on the suppression list');
}
// Continue with valid recipients
try {
// Process recipients
const toArray = Array.isArray(toAddresses)
? toAddresses
: toAddresses.split(',').map(addr => addr.trim());
// Use filtered recipients - already an array, no need for toArray
// Add recipients to smartmail if they're not already added
if (!smartmail.options.to || smartmail.options.to.length === 0) {
for (const recipient of toArray) {
for (const recipient of validRecipients) {
smartmail.addRecipient(recipient);
}
}
@ -57,15 +84,15 @@ export class MtaConnector {
const mimeEmail = await smartmail.toMimeFormat(smartmail.options.creationObjectRef);
// Parse the MIME email to create an MTA Email
return this.sendMimeEmail(mimeEmail, toArray);
return this.sendMimeEmail(mimeEmail, validRecipients);
} catch (mimeError) {
logger.log('warn', `Failed to use MIME format, falling back to direct conversion: ${mimeError.message}`);
// Fall back to direct conversion
return this.sendDirectEmail(smartmail, toArray);
return this.sendDirectEmail(smartmail, validRecipients);
}
} else {
// Use direct conversion
return this.sendDirectEmail(smartmail, toArray);
return this.sendDirectEmail(smartmail, validRecipients);
}
} catch (error) {
logger.log('error', `Failed to send email via MTA: ${error.message}`, {
@ -73,6 +100,30 @@ export class MtaConnector {
provider: 'mta',
error: error.message
});
// Check if this is a bounce-related error
if (error.message.includes('550') || // Rejected
error.message.includes('551') || // User not local
error.message.includes('552') || // Mailbox full
error.message.includes('553') || // Bad mailbox name
error.message.includes('554') || // Transaction failed
error.message.includes('does not exist') ||
error.message.includes('unknown user') ||
error.message.includes('invalid recipient')) {
// Process as a bounce
for (const recipient of validRecipients) {
await this.emailRef.bounceManager.processSmtpFailure(
recipient,
error.message,
{
sender: smartmail.options.from,
statusCode: error.message.match(/\b([45]\d{2})\b/) ? error.message.match(/\b([45]\d{2})\b/)[1] : undefined
}
);
}
}
throw error;
}
}

5
ts/email/classes.emailservice.ts
View File

@ -5,6 +5,7 @@ import { RuleManager } from './classes.rulemanager.js';
import { ApiManager } from './classes.apimanager.js';
import { TemplateManager } from './classes.templatemanager.js';
import { EmailValidator } from './classes.emailvalidator.js';
import { BounceManager } from './classes.bouncemanager.js';
import { logger } from '../logger.js';
import type { SzPlatformService } from '../platformservice.js';
@ -44,6 +45,7 @@ export class EmailService {
public ruleManager: RuleManager;
public templateManager: TemplateManager;
public emailValidator: EmailValidator;
public bounceManager: BounceManager;
// configuration
private config: IEmailConstructorOptions;
@ -62,6 +64,9 @@ export class EmailService {
// Initialize validator
this.emailValidator = new EmailValidator();
// Initialize bounce manager
this.bounceManager = new BounceManager();
// Initialize template manager
this.templateManager = new TemplateManager(this.config.templateConfig);

38
ts/email/classes.emailvalidator.ts
View File

@ -1,5 +1,6 @@
import * as plugins from '../plugins.js';
import { logger } from '../logger.js';
import { LRUCache } from 'lru-cache';
export interface IEmailValidationResult {
isValid: boolean;
@ -21,10 +22,28 @@ export interface IEmailValidationResult {
*/
export class EmailValidator {
private validator: plugins.smartmail.EmailAddressValidator;
private dnsCache: Map<string, any> = new Map();
private dnsCache: LRUCache<string, string[]>;
constructor() {
constructor(options?: {
maxCacheSize?: number;
cacheTTL?: number;
}) {
this.validator = new plugins.smartmail.EmailAddressValidator();
// Initialize LRU cache for DNS records
this.dnsCache = new LRUCache<string, string[]>({
// Default to 1000 entries (reasonable for most applications)
max: options?.maxCacheSize || 1000,
// Default TTL of 1 hour (DNS records don't change frequently)
ttl: options?.cacheTTL || 60 * 60 * 1000,
// Optional cache monitoring
allowStale: false,
updateAgeOnGet: true,
// Add logging for cache events in production environments
disposeAfter: (value, key) => {
logger.log('debug', `DNS cache entry expired for domain: ${key}`);
},
});
}
/**
@ -162,19 +181,20 @@ export class EmailValidator {
* @returns Array of MX records
*/
private async getMxRecords(domain: string): Promise<string[]> {
if (this.dnsCache.has(domain)) {
return this.dnsCache.get(domain);
// Check cache first
const cachedRecords = this.dnsCache.get(domain);
if (cachedRecords) {
logger.log('debug', `Using cached MX records for domain: ${domain}`);
return cachedRecords;
}
try {
// Use smartmail's getMxRecords method
const records = await this.validator.getMxRecords(domain);
this.dnsCache.set(domain, records);
// Cache expires after 1 hour
setTimeout(() => {
this.dnsCache.delete(domain);
}, 60 * 60 * 1000);
// Store in cache (TTL is handled by the LRU cache configuration)
this.dnsCache.set(domain, records);
logger.log('debug', `Cached MX records for domain: ${domain}`);
return records;
} catch (error) {

18
ts/email/index.ts
View File

@ -1,3 +1,19 @@
import { EmailService } from './classes.emailservice.js';
import { BounceManager, BounceType, BounceCategory } from './classes.bouncemanager.js';
import { EmailValidator } from './classes.emailvalidator.js';
import { TemplateManager } from './classes.templatemanager.js';
import { RuleManager } from './classes.rulemanager.js';
import { ApiManager } from './classes.apimanager.js';
import { MtaConnector } from './classes.connector.mta.js';
export { EmailService as Email };
export {
EmailService as Email,
BounceManager,
BounceType,
BounceCategory,
EmailValidator,
TemplateManager,
RuleManager,
ApiManager,
MtaConnector
};

41
ts/letter/classes.letterservice.ts
View File

@ -1,41 +0,0 @@
import type { SzPlatformService } from '../platformservice.js';
import * as plugins from '../plugins.js';
export interface ILetterConstructorOptions {
letterxpressUser: string;
letterxpressToken: string;
}
export class LetterService {
public platformServiceRef: SzPlatformService;
public options: ILetterConstructorOptions;
public letterxpressAccount: plugins.letterxpress.LetterXpressAccount;
public typedrouter = new plugins.typedrequest.TypedRouter();
constructor(platformServiceRefArg: SzPlatformService, optionsArg: ILetterConstructorOptions) {
this.platformServiceRef = platformServiceRefArg;
this.options = optionsArg;
this.platformServiceRef.typedrouter.addTypedRouter(this.typedrouter);
this.typedrouter.addTypedHandler<
plugins.servezoneInterfaces.platformservice.letter.IRequest_SendLetter
>(new plugins.typedrequest.TypedHandler('sendLetter', async dataArg => {
if(dataArg.needsCover) {
}
return {
processId: '',
}
}));
}
public async start() {
this.letterxpressAccount = new plugins.letterxpress.LetterXpressAccount({
username: this.options.letterxpressUser,
apiKey: this.options.letterxpressToken,
});
await this.letterxpressAccount.start();
}
public async stop() {}
}

1
ts/letter/index.ts
View File

@ -1 +0,0 @@
export * from './classes.letterservice.js';

102
ts/mta/classes.dkimverifier.ts
View File

@ -1,6 +1,7 @@
import * as plugins from '../plugins.js';
import { MtaService } from './classes.mta.js';
import { logger } from '../logger.js';
import { SecurityLogger, SecurityLogLevel, SecurityEventType } from '../security/index.js';
/**
* Result of a DKIM verification
@ -80,10 +81,34 @@ export class DKIMVerifier {
});
logger.log(isValid ? 'info' : 'warn', `DKIM Verification using mailauth: ${isValid ? 'pass' : 'fail'} for domain ${dkimResult.domain}`);
// Enhanced security logging
SecurityLogger.getInstance().logEvent({
level: isValid ? SecurityLogLevel.INFO : SecurityLogLevel.WARN,
type: SecurityEventType.DKIM,
message: `DKIM verification ${isValid ? 'passed' : 'failed'} for domain ${dkimResult.domain}`,
details: {
selector: dkimResult.selector,
signatureFields: dkimResult.signature,
result: dkimResult.status.result
},
domain: dkimResult.domain,
success: isValid
});
return result;
}
} catch (mailauthError) {
logger.log('warn', `DKIM verification with mailauth failed, trying smartmail: ${mailauthError.message}`);
// Enhanced security logging
SecurityLogger.getInstance().logEvent({
level: SecurityLogLevel.WARN,
type: SecurityEventType.DKIM,
message: `DKIM verification with mailauth failed, trying smartmail fallback`,
details: { error: mailauthError.message },
success: false
});
}
// Fall back to smartmail for verification
@ -167,6 +192,20 @@ export class DKIMVerifier {
});
logger.log('info', `DKIM verification using smartmail: pass for domain ${domain}`);
// Enhanced security logging
SecurityLogger.getInstance().logEvent({
level: SecurityLogLevel.INFO,
type: SecurityEventType.DKIM,
message: `DKIM verification passed for domain ${domain} using fallback verification`,
details: {
selector,
signatureFields
},
domain,
success: true
});
return result;
} else {
// Missing domain or selector
@ -185,6 +224,17 @@ export class DKIMVerifier {
});
logger.log('warn', `DKIM verification failed: Missing domain or selector in DKIM signature`);
// Enhanced security logging
SecurityLogger.getInstance().logEvent({
level: SecurityLogLevel.WARN,
type: SecurityEventType.DKIM,
message: `DKIM verification failed: Missing domain or selector in signature`,
details: { domain, selector, signatureFields },
domain: domain || 'unknown',
success: false
});
return result;
}
} catch (error) {
@ -200,11 +250,30 @@ export class DKIMVerifier {
});
logger.log('error', `DKIM verification error: ${error.message}`);
// Enhanced security logging
SecurityLogger.getInstance().logEvent({
level: SecurityLogLevel.ERROR,
type: SecurityEventType.DKIM,
message: `DKIM verification error during processing`,
details: { error: error.message },
success: false
});
return result;
}
} catch (error) {
logger.log('error', `DKIM verification failed with unexpected error: ${error.message}`);
// Enhanced security logging for unexpected errors
SecurityLogger.getInstance().logEvent({
level: SecurityLogLevel.ERROR,
type: SecurityEventType.DKIM,
message: `DKIM verification failed with unexpected error`,
details: { error: error.message },
success: false
});
return {
isValid: false,
status: 'temperror',
@ -241,6 +310,17 @@ export class DKIMVerifier {
if (!txtRecords || txtRecords.length === 0) {
logger.log('warn', `No DKIM TXT record found for ${dkimRecord}`);
// Security logging for missing DKIM record
SecurityLogger.getInstance().logEvent({
level: SecurityLogLevel.WARN,
type: SecurityEventType.DKIM,
message: `No DKIM TXT record found for ${dkimRecord}`,
domain,
success: false,
details: { selector }
});
return null;
}
@ -256,9 +336,31 @@ export class DKIMVerifier {
}
logger.log('warn', `No valid DKIM public key found in TXT records for ${dkimRecord}`);
// Security logging for invalid DKIM key
SecurityLogger.getInstance().logEvent({
level: SecurityLogLevel.WARN,
type: SecurityEventType.DKIM,
message: `No valid DKIM public key found in TXT records`,
domain,
success: false,
details: { dkimRecord, selector }
});
return null;
} catch (error) {
logger.log('error', `Error fetching DKIM key: ${error.message}`);
// Security logging for DKIM key fetch error
SecurityLogger.getInstance().logEvent({
level: SecurityLogLevel.ERROR,
type: SecurityEventType.DKIM,
message: `Error fetching DKIM key for domain`,
domain,
success: false,
details: { error: error.message, selector, dkimRecord: `${selector}._domainkey.${domain}` }
});
return null;
}
}

475
ts/mta/classes.dmarcverifier.ts Normal file
View File

@ -0,0 +1,475 @@
import * as plugins from '../plugins.js';
import { logger } from '../logger.js';
import { SecurityLogger, SecurityLogLevel, SecurityEventType } from '../security/index.js';
import type { MtaService } from './classes.mta.js';
import type { Email } from './classes.email.js';
import type { IDnsVerificationResult } from './classes.dnsmanager.js';
/**
* DMARC policy types
*/
export enum DmarcPolicy {
NONE = 'none',
QUARANTINE = 'quarantine',
REJECT = 'reject'
}
/**
* DMARC alignment modes
*/
export enum DmarcAlignment {
RELAXED = 'r',
STRICT = 's'
}
/**
* DMARC record fields
*/
export interface DmarcRecord {
// Required fields
version: string;
policy: DmarcPolicy;
// Optional fields
subdomainPolicy?: DmarcPolicy;
pct?: number;
adkim?: DmarcAlignment;
aspf?: DmarcAlignment;
reportInterval?: number;
failureOptions?: string;
reportUriAggregate?: string[];
reportUriForensic?: string[];
}
/**
* DMARC verification result
*/
export interface DmarcResult {
hasDmarc: boolean;
record?: DmarcRecord;
spfDomainAligned: boolean;
dkimDomainAligned: boolean;
spfPassed: boolean;
dkimPassed: boolean;
policyEvaluated: DmarcPolicy;
actualPolicy: DmarcPolicy;
appliedPercentage: number;
action: 'pass' | 'quarantine' | 'reject';
details: string;
error?: string;
}
/**
* Class for verifying and enforcing DMARC policies
*/
export class DmarcVerifier {
private mtaRef: MtaService;
constructor(mtaRefArg: MtaService) {
this.mtaRef = mtaRefArg;
}
/**
* Parse a DMARC record from a TXT record string
* @param record DMARC TXT record string
* @returns Parsed DMARC record or null if invalid
*/
public parseDmarcRecord(record: string): DmarcRecord | null {
if (!record.startsWith('v=DMARC1')) {
return null;
}
try {
// Initialize record with default values
const dmarcRecord: DmarcRecord = {
version: 'DMARC1',
policy: DmarcPolicy.NONE,
pct: 100,
adkim: DmarcAlignment.RELAXED,
aspf: DmarcAlignment.RELAXED
};
// Split the record into tag/value pairs
const parts = record.split(';').map(part => part.trim());
for (const part of parts) {
if (!part || !part.includes('=')) continue;
const [tag, value] = part.split('=').map(p => p.trim());
// Process based on tag
switch (tag.toLowerCase()) {
case 'v':
dmarcRecord.version = value;
break;
case 'p':
dmarcRecord.policy = value as DmarcPolicy;
break;
case 'sp':
dmarcRecord.subdomainPolicy = value as DmarcPolicy;
break;
case 'pct':
const pctValue = parseInt(value, 10);
if (!isNaN(pctValue) && pctValue >= 0 && pctValue <= 100) {
dmarcRecord.pct = pctValue;
}
break;
case 'adkim':
dmarcRecord.adkim = value as DmarcAlignment;
break;
case 'aspf':
dmarcRecord.aspf = value as DmarcAlignment;
break;
case 'ri':
const interval = parseInt(value, 10);
if (!isNaN(interval) && interval > 0) {
dmarcRecord.reportInterval = interval;
}
break;
case 'fo':
dmarcRecord.failureOptions = value;
break;
case 'rua':
dmarcRecord.reportUriAggregate = value.split(',').map(uri => {
if (uri.startsWith('mailto:')) {
return uri.substring(7).trim();
}
return uri.trim();
});
break;
case 'ruf':
dmarcRecord.reportUriForensic = value.split(',').map(uri => {
if (uri.startsWith('mailto:')) {
return uri.substring(7).trim();
}
return uri.trim();
});
break;
}
}
// Ensure subdomain policy is set if not explicitly provided
if (!dmarcRecord.subdomainPolicy) {
dmarcRecord.subdomainPolicy = dmarcRecord.policy;
}
return dmarcRecord;
} catch (error) {
logger.log('error', `Error parsing DMARC record: ${error.message}`, {
record,
error: error.message
});
return null;
}
}
/**
* Check if domains are aligned according to DMARC policy
* @param headerDomain Domain from header (From)
* @param authDomain Domain from authentication (SPF, DKIM)
* @param alignment Alignment mode
* @returns Whether the domains are aligned
*/
private isDomainAligned(
headerDomain: string,
authDomain: string,
alignment: DmarcAlignment
): boolean {
if (!headerDomain || !authDomain) {
return false;
}
// For strict alignment, domains must match exactly
if (alignment === DmarcAlignment.STRICT) {
return headerDomain.toLowerCase() === authDomain.toLowerCase();
}
// For relaxed alignment, the authenticated domain must be a subdomain of the header domain
// or the same as the header domain
const headerParts = headerDomain.toLowerCase().split('.');
const authParts = authDomain.toLowerCase().split('.');
// Ensures we have at least two parts (domain and TLD)
if (headerParts.length < 2 || authParts.length < 2) {
return false;
}
// Get organizational domain (last two parts)
const headerOrgDomain = headerParts.slice(-2).join('.');
const authOrgDomain = authParts.slice(-2).join('.');
return headerOrgDomain === authOrgDomain;
}
/**
* Extract domain from an email address
* @param email Email address
* @returns Domain part of the email
*/
private getDomainFromEmail(email: string): string {
if (!email) return '';
// Handle name + email format: "John Doe <john@example.com>"
const matches = email.match(/<([^>]+)>/);
const address = matches ? matches[1] : email;
const parts = address.split('@');
return parts.length > 1 ? parts[1] : '';
}
/**
* Check if DMARC verification should be applied based on percentage
* @param record DMARC record
* @returns Whether DMARC verification should be applied
*/
private shouldApplyDmarc(record: DmarcRecord): boolean {
if (record.pct === undefined || record.pct === 100) {
return true;
}
// Apply DMARC randomly based on percentage
const random = Math.floor(Math.random() * 100) + 1;
return random <= record.pct;
}
/**
* Determine the action to take based on DMARC policy
* @param policy DMARC policy
* @returns Action to take
*/
private determineAction(policy: DmarcPolicy): 'pass' | 'quarantine' | 'reject' {
switch (policy) {
case DmarcPolicy.REJECT:
return 'reject';
case DmarcPolicy.QUARANTINE:
return 'quarantine';
case DmarcPolicy.NONE:
default:
return 'pass';
}
}
/**
* Verify DMARC for an incoming email
* @param email Email to verify
* @param spfResult SPF verification result
* @param dkimResult DKIM verification result
* @returns DMARC verification result
*/
public async verify(
email: Email,
spfResult: { domain: string; result: boolean },
dkimResult: { domain: string; result: boolean }
): Promise<DmarcResult> {
const securityLogger = SecurityLogger.getInstance();
// Initialize result
const result: DmarcResult = {
hasDmarc: false,
spfDomainAligned: false,
dkimDomainAligned: false,
spfPassed: spfResult.result,
dkimPassed: dkimResult.result,
policyEvaluated: DmarcPolicy.NONE,
actualPolicy: DmarcPolicy.NONE,
appliedPercentage: 100,
action: 'pass',
details: 'DMARC not configured'
};
try {
// Extract From domain
const fromHeader = email.getFromEmail();
const fromDomain = this.getDomainFromEmail(fromHeader);
if (!fromDomain) {
result.error = 'Invalid From domain';
return result;
}
// Check alignment
result.spfDomainAligned = this.isDomainAligned(
fromDomain,
spfResult.domain,
DmarcAlignment.RELAXED
);
result.dkimDomainAligned = this.isDomainAligned(
fromDomain,
dkimResult.domain,
DmarcAlignment.RELAXED
);
// Lookup DMARC record
const dmarcVerificationResult = await this.mtaRef.dnsManager.verifyDmarcRecord(fromDomain);
// If DMARC record exists and is valid
if (dmarcVerificationResult.found && dmarcVerificationResult.valid) {
result.hasDmarc = true;
// Parse DMARC record
const parsedRecord = this.parseDmarcRecord(dmarcVerificationResult.value);
if (parsedRecord) {
result.record = parsedRecord;
result.actualPolicy = parsedRecord.policy;
result.appliedPercentage = parsedRecord.pct || 100;
// Override alignment modes if specified in record
if (parsedRecord.adkim) {
result.dkimDomainAligned = this.isDomainAligned(
fromDomain,
dkimResult.domain,
parsedRecord.adkim
);
}
if (parsedRecord.aspf) {
result.spfDomainAligned = this.isDomainAligned(
fromDomain,
spfResult.domain,
parsedRecord.aspf
);
}
// Determine DMARC compliance
const spfAligned = result.spfPassed && result.spfDomainAligned;
const dkimAligned = result.dkimPassed && result.dkimDomainAligned;
// Email passes DMARC if either SPF or DKIM passes with alignment
const dmarcPass = spfAligned || dkimAligned;
// Use record percentage to determine if policy should be applied
const applyPolicy = this.shouldApplyDmarc(parsedRecord);
if (!dmarcPass) {
// DMARC failed, apply policy
result.policyEvaluated = applyPolicy ? parsedRecord.policy : DmarcPolicy.NONE;
result.action = this.determineAction(result.policyEvaluated);
result.details = `DMARC failed: SPF aligned=${spfAligned}, DKIM aligned=${dkimAligned}, policy=${result.policyEvaluated}`;
} else {
result.policyEvaluated = DmarcPolicy.NONE;
result.action = 'pass';
result.details = `DMARC passed: SPF aligned=${spfAligned}, DKIM aligned=${dkimAligned}`;
}
} else {
result.error = 'Invalid DMARC record format';
result.details = 'DMARC record invalid';
}
} else {
// No DMARC record found or invalid
result.details = dmarcVerificationResult.error || 'No DMARC record found';
}
// Log the DMARC verification
securityLogger.logEvent({
level: result.action === 'pass' ? SecurityLogLevel.INFO : SecurityLogLevel.WARN,
type: SecurityEventType.DMARC,
message: result.details,
domain: fromDomain,
details: {
fromDomain,
spfDomain: spfResult.domain,
dkimDomain: dkimResult.domain,
spfPassed: result.spfPassed,
dkimPassed: result.dkimPassed,
spfAligned: result.spfDomainAligned,
dkimAligned: result.dkimDomainAligned,
dmarcPolicy: result.policyEvaluated,
action: result.action
},
success: result.action === 'pass'
});
return result;
} catch (error) {
logger.log('error', `Error verifying DMARC: ${error.message}`, {
error: error.message,
emailId: email.getMessageId()
});
result.error = `DMARC verification error: ${error.message}`;
// Log error
securityLogger.logEvent({
level: SecurityLogLevel.ERROR,
type: SecurityEventType.DMARC,
message: `DMARC verification failed with error`,
details: {
error: error.message,
emailId: email.getMessageId()
},
success: false
});
return result;
}
}
/**
* Apply DMARC policy to an email
* @param email Email to apply policy to
* @param dmarcResult DMARC verification result
* @returns Whether the email should be accepted
*/
public applyPolicy(email: Email, dmarcResult: DmarcResult): boolean {
// Apply action based on DMARC verification result
switch (dmarcResult.action) {
case 'reject':
// Reject the email
email.mightBeSpam = true;
logger.log('warn', `Email rejected due to DMARC policy: ${dmarcResult.details}`, {
emailId: email.getMessageId(),
from: email.getFromEmail(),
subject: email.subject
});
return false;
case 'quarantine':
// Quarantine the email (mark as spam)
email.mightBeSpam = true;
// Add spam header
if (!email.headers['X-Spam-Flag']) {
email.headers['X-Spam-Flag'] = 'YES';
}
// Add DMARC reason header
email.headers['X-DMARC-Result'] = dmarcResult.details;
logger.log('warn', `Email quarantined due to DMARC policy: ${dmarcResult.details}`, {
emailId: email.getMessageId(),
from: email.getFromEmail(),
subject: email.subject
});
return true;
case 'pass':
default:
// Accept the email
// Add DMARC result header for information
email.headers['X-DMARC-Result'] = dmarcResult.details;
return true;
}
}
/**
* End-to-end DMARC verification and policy application
* This method should be called after SPF and DKIM verification
* @param email Email to verify
* @param spfResult SPF verification result
* @param dkimResult DKIM verification result
* @returns Whether the email should be accepted
*/
public async verifyAndApply(
email: Email,
spfResult: { domain: string; result: boolean },
dkimResult: { domain: string; result: boolean }
): Promise<boolean> {
// Verify DMARC
const dmarcResult = await this.verify(email, spfResult, dkimResult);
// Apply DMARC policy
return this.applyPolicy(email, dmarcResult);
}
}

57
ts/mta/classes.email.ts
View File

@ -38,6 +38,8 @@ export class Email {
mightBeSpam: boolean;
priority: 'high' | 'normal' | 'low';
variables: Record<string, any>;
private envelopeFrom: string;
private messageId: string;
// Static validator instance for reuse
private static emailValidator: EmailValidator;
@ -89,6 +91,12 @@ export class Email {
// Set template variables
this.variables = options.variables || {};
// Initialize envelope from (defaults to the from address)
this.envelopeFrom = this.from;
// Generate message ID if not provided
this.messageId = `<${Date.now()}.${Math.random().toString(36).substring(2, 15)}@${this.getFromDomain() || 'localhost'}>`;
}
/**
@ -468,6 +476,53 @@ export class Email {
return smartmail;
}
/**
* Get the from email address
* @returns The from email address
*/
public getFromEmail(): string {
return this.from;
}
/**
* Get the message ID
* @returns The message ID
*/
public getMessageId(): string {
return this.messageId;
}
/**
* Set a custom message ID
* @param id The message ID to set
* @returns This instance for method chaining
*/
public setMessageId(id: string): this {
this.messageId = id;
return this;
}
/**
* Get the envelope from address (return-path)
* @returns The envelope from address
*/
public getEnvelopeFrom(): string {
return this.envelopeFrom;
}
/**
* Set the envelope from address (return-path)
* @param address The envelope from address to set
* @returns This instance for method chaining
*/
public setEnvelopeFrom(address: string): this {
if (!this.isValidEmail(address)) {
throw new Error(`Invalid envelope from address: ${address}`);
}
this.envelopeFrom = address;
return this;
}
/**
* Creates an RFC822 compliant email string
* @param variables Optional template variables to apply
@ -491,6 +546,8 @@ export class Email {
result += `Subject: ${processedSubject}\r\n`;
result += `Date: ${new Date().toUTCString()}\r\n`;
result += `Message-ID: ${this.messageId}\r\n`;
result += `Return-Path: <${this.envelopeFrom}>\r\n`;
// Add custom headers
for (const [key, value] of Object.entries(this.headers)) {

81
ts/mta/classes.emailsendjob.ts
View File

@ -160,6 +160,9 @@ export class EmailSendJob {
this.deliveryInfo.deliveryTime = new Date();
this.log(`Email delivered successfully to ${currentMx}`);
// Record delivery for sender reputation monitoring
this.recordDeliveryEvent('delivered');
// Save successful email record
await this.saveSuccess();
return DeliveryStatus.DELIVERED;
@ -262,7 +265,35 @@ export class EmailSendJob {
this.log(`Connecting to ${mxServer}:25`);
setCommandTimeout();
this.socket = plugins.net.connect(25, mxServer);
// Check if IP warmup is enabled and get an IP to use
let localAddress: string | undefined = undefined;
if (this.mtaRef.config.outbound?.warmup?.enabled) {
const warmupManager = this.mtaRef.getIPWarmupManager();
if (warmupManager) {
const fromDomain = this.email.getFromDomain();
const bestIP = warmupManager.getBestIPForSending({
from: this.email.from,
to: this.email.getAllRecipients(),
domain: fromDomain,
isTransactional: this.email.priority === 'high'
});
if (bestIP) {
this.log(`Using warmed-up IP ${bestIP} for sending`);
localAddress = bestIP;
// Record the send for warm-up tracking
warmupManager.recordSend(bestIP);
}
}
}
// Connect with specified local address if available
this.socket = plugins.net.connect({
port: 25,
host: mxServer,
localAddress
});
this.socket.on('error', (err) => {
this.log(`Socket error: ${err.message}`);
@ -461,6 +492,54 @@ export class EmailSendJob {
return message;
}
/**
* Record an event for sender reputation monitoring
* @param eventType Type of event
* @param isHardBounce Whether the event is a hard bounce (for bounce events)
*/
private recordDeliveryEvent(
eventType: 'sent' | 'delivered' | 'bounce' | 'complaint',
isHardBounce: boolean = false
): void {
try {
// Check if reputation monitoring is enabled
if (!this.mtaRef.config.outbound?.reputation?.enabled) {
return;
}
const reputationMonitor = this.mtaRef.getReputationMonitor();
if (!reputationMonitor) {
return;
}
// Get domain from sender
const domain = this.email.getFromDomain();
if (!domain) {
return;
}
// Determine receiving domain for complaint tracking
let receivingDomain = null;
if (eventType === 'complaint' && this.email.to.length > 0) {
const recipient = this.email.to[0];
const parts = recipient.split('@');
if (parts.length === 2) {
receivingDomain = parts[1];
}
}
// Record the event
reputationMonitor.recordSendEvent(domain, {
type: eventType,
count: 1,
hardBounce: isHardBounce,
receivingDomain
});
} catch (error) {
this.log(`Error recording delivery event: ${error.message}`);
}
}
/**
* Send a command to the SMTP server and wait for the expected response
*/

518
ts/mta/classes.mta.ts
View File

@ -5,9 +5,15 @@ import { Email } from './classes.email.js';
import { EmailSendJob, DeliveryStatus } from './classes.emailsendjob.js';
import { DKIMCreator } from './classes.dkimcreator.js';
import { DKIMVerifier } from './classes.dkimverifier.js';
import { SpfVerifier } from './classes.spfverifier.js';
import { DmarcVerifier } from './classes.dmarcverifier.js';
import { SMTPServer, type ISmtpServerOptions } from './classes.smtpserver.js';
import { DNSManager } from './classes.dnsmanager.js';
import { ApiManager } from './classes.apimanager.js';
import { RateLimiter, type IRateLimitConfig } from './classes.ratelimiter.js';
import { ContentScanner } from '../security/classes.contentscanner.js';
import { IPWarmupManager } from '../deliverability/classes.ipwarmupmanager.js';
import { SenderReputationMonitor } from '../deliverability/classes.senderreputationmonitor.js';
import type { SzPlatformService } from '../platformservice.js';
/**
@ -57,6 +63,33 @@ export interface IMtaConfig {
/** Whether to apply per domain (vs globally) */
perDomain?: boolean;
};
/** IP warmup configuration */
warmup?: {
/** Whether IP warmup is enabled */
enabled?: boolean;
/** IP addresses to warm up */
ipAddresses?: string[];
/** Target domains to warm up */
targetDomains?: string[];
/** Allocation policy to use */
allocationPolicy?: string;
/** Fallback percentage for ESP routing during warmup */
fallbackPercentage?: number;
};
/** Reputation monitoring configuration */
reputation?: {
/** Whether reputation monitoring is enabled */
enabled?: boolean;
/** How frequently to update metrics (ms) */
updateFrequency?: number;
/** Alert thresholds */
alertThresholds?: {
/** Minimum acceptable reputation score */
minReputationScore?: number;
/** Maximum acceptable complaint rate */
maxComplaintRate?: number;
};
};
};
/** Security settings */
security?: {
@ -66,10 +99,26 @@ export interface IMtaConfig {
verifyDkim?: boolean;
/** Whether to verify SPF on inbound */
verifySpf?: boolean;
/** Whether to verify DMARC on inbound */
verifyDmarc?: boolean;
/** Whether to enforce DMARC policy */
enforceDmarc?: boolean;
/** Whether to use TLS for outbound when available */
useTls?: boolean;
/** Whether to require valid certificates */
requireValidCerts?: boolean;
/** Log level for email security events */
securityLogLevel?: 'info' | 'warn' | 'error';
/** Whether to check IP reputation for inbound emails */
checkIPReputation?: boolean;
/** Whether to scan content for malicious payloads */
scanContent?: boolean;
/** Action to take when malicious content is detected */
maliciousContentAction?: 'tag' | 'quarantine' | 'reject';
/** Minimum threat score to trigger action */
threatScoreThreshold?: number;
/** Whether to reject connections from high-risk IPs */
rejectHighRiskIPs?: boolean;
};
/** Domains configuration */
domains?: {
@ -121,6 +170,18 @@ interface MtaStats {
expiresAt: Date;
daysUntilExpiry: number;
};
warmupInfo?: {
enabled: boolean;
activeIPs: number;
inWarmupPhase: number;
completedWarmup: number;
};
reputationInfo?: {
enabled: boolean;
monitoredDomains: number;
averageScore: number;
domainsWithIssues: number;
};
}
/**
@ -130,6 +191,11 @@ export class MtaService {
/** Reference to the platform service */
public platformServiceRef: SzPlatformService;
// Get access to the email service and bounce manager
private get emailService() {
return this.platformServiceRef.emailService;
}
/** SMTP server instance */
public server: SMTPServer;
@ -139,6 +205,12 @@ export class MtaService {
/** DKIM verifier for validating incoming emails */
public dkimVerifier: DKIMVerifier;
/** SPF verifier for validating incoming emails */
public spfVerifier: SpfVerifier;
/** DMARC verifier for email authentication policy enforcement */
public dmarcVerifier: DmarcVerifier;
/** DNS manager for handling DNS records */
public dnsManager: DNSManager;
@ -151,17 +223,20 @@ export class MtaService {
/** Email queue processing state */
private queueProcessing = false;
/** Rate limiters for outbound emails */
private rateLimiters: Map<string, {
tokens: number;
lastRefill: number;
}> = new Map();
/** Rate limiter for outbound emails */
private rateLimiter: RateLimiter;
/** IP warmup manager for controlled scaling of new IPs */
private ipWarmupManager: IPWarmupManager;
/** Sender reputation monitor for tracking domain reputation */
private reputationMonitor: SenderReputationMonitor;
/** Certificate cache */
private certificate: Certificate = null;
/** MTA configuration */
private config: IMtaConfig;
public config: IMtaConfig;
/** Stats for monitoring */
private stats: MtaStats;
@ -191,9 +266,46 @@ export class MtaService {
this.dkimVerifier = new DKIMVerifier(this);
this.dnsManager = new DNSManager(this);
this.apiManager = new ApiManager();
// Initialize authentication verifiers
this.spfVerifier = new SpfVerifier(this);
this.dmarcVerifier = new DmarcVerifier(this);
// Initialize SMTP rule engine
this.smtpRuleEngine = new plugins.smartrule.SmartRule<Email>();
// Initialize rate limiter with config
const rateLimitConfig = this.config.outbound?.rateLimit;
this.rateLimiter = new RateLimiter({
maxPerPeriod: rateLimitConfig?.maxPerPeriod || 100,
periodMs: rateLimitConfig?.periodMs || 60000,
perKey: rateLimitConfig?.perDomain || true,
burstTokens: 5 // Allow small bursts
});
// Initialize IP warmup manager
const warmupConfig = this.config.outbound?.warmup;
this.ipWarmupManager = IPWarmupManager.getInstance({
enabled: warmupConfig?.enabled || false,
ipAddresses: warmupConfig?.ipAddresses || [],
targetDomains: warmupConfig?.targetDomains || [],
fallbackPercentage: warmupConfig?.fallbackPercentage || 50
});
// Set active allocation policy if specified
if (warmupConfig?.allocationPolicy) {
this.ipWarmupManager.setActiveAllocationPolicy(warmupConfig.allocationPolicy);
}
// Initialize sender reputation monitor
const reputationConfig = this.config.outbound?.reputation;
this.reputationMonitor = SenderReputationMonitor.getInstance({
enabled: reputationConfig?.enabled || false,
domains: this.config.domains?.local || [],
updateFrequency: reputationConfig?.updateFrequency || 24 * 60 * 60 * 1000,
alertThresholds: reputationConfig?.alertThresholds || {}
});
// Initialize stats
this.stats = {
startTime: new Date(),
@ -234,14 +346,37 @@ export class MtaService {
maxPerPeriod: 100,
periodMs: 60000, // 1 minute
perDomain: true
},
warmup: {
enabled: false,
ipAddresses: [],
targetDomains: [],
allocationPolicy: 'balanced',
fallbackPercentage: 50
},
reputation: {
enabled: false,
updateFrequency: 24 * 60 * 60 * 1000, // Daily
alertThresholds: {
minReputationScore: 70,
maxComplaintRate: 0.1 // 0.1%
}
}
},
security: {
useDkim: true,
verifyDkim: true,
verifySpf: true,
verifyDmarc: true,
enforceDmarc: true,
useTls: true,
requireValidCerts: false
requireValidCerts: false,
securityLogLevel: 'warn',
checkIPReputation: true,
scanContent: true,
maliciousContentAction: 'tag',
threatScoreThreshold: 50,
rejectHighRiskIPs: false
},
domains: {
local: ['lossless.one'],
@ -393,6 +528,14 @@ export class MtaService {
// Update stats
this.stats.queueSize = this.emailQueue.size;
// Record 'sent' event for sender reputation monitoring
if (this.config.outbound?.reputation?.enabled) {
const fromDomain = email.getFromDomain();
if (fromDomain) {
this.reputationMonitor.recordSendEvent(fromDomain, { type: 'sent' });
}
}
console.log(`Email added to queue: ${id}`);
return id;
@ -413,18 +556,62 @@ export class MtaService {
throw new Error('MTA service is not running');
}
// Apply SMTP rule engine decisions
try {
await this.smtpRuleEngine.makeDecision(email);
} catch (err) {
console.error('Error executing SMTP rules:', err);
}
try {
console.log(`Processing incoming email from ${email.from} to ${email.to}`);
// Update stats
this.stats.emailsReceived++;
// Apply SMTP rule engine decisions
try {
await this.smtpRuleEngine.makeDecision(email);
} catch (err) {
console.error('Error executing SMTP rules:', err);
}
// Scan for malicious content if enabled
if (this.config.security?.scanContent !== false) {
const contentScanner = ContentScanner.getInstance();
const scanResult = await contentScanner.scanEmail(email);
// Log the scan result
console.log(`Content scan result for email ${email.getMessageId()}: score=${scanResult.threatScore}, isClean=${scanResult.isClean}`);
// Take action based on the scan result and configuration
if (!scanResult.isClean) {
const threatScoreThreshold = this.config.security?.threatScoreThreshold || 50;
// Check if the threat score exceeds the threshold
if (scanResult.threatScore >= threatScoreThreshold) {
const action = this.config.security?.maliciousContentAction || 'tag';
switch (action) {
case 'reject':
// Reject the email
console.log(`Rejecting email from ${email.from} due to malicious content: ${scanResult.threatType} (score: ${scanResult.threatScore})`);
return false;
case 'quarantine':
// Save to quarantine folder instead of regular processing
await this.saveToQuarantine(email, scanResult);
return true;
case 'tag':
default:
// Tag the email by modifying subject and adding headers
email.subject = `[SUSPICIOUS] ${email.subject}`;
email.addHeader('X-Content-Scanned', 'True');
email.addHeader('X-Threat-Type', scanResult.threatType || 'unknown');
email.addHeader('X-Threat-Score', scanResult.threatScore.toString());
email.addHeader('X-Threat-Details', scanResult.threatDetails || 'Suspicious content detected');
email.mightBeSpam = true;
console.log(`Tagged email from ${email.from} with suspicious content: ${scanResult.threatType} (score: ${scanResult.threatScore})`);
break;
}
}
}
}
// Check if the recipient domain is local
const recipientDomain = email.to[0].split('@')[1];
const isLocalDomain = this.isLocalDomain(recipientDomain);
@ -444,6 +631,55 @@ export class MtaService {
return false;
}
}
/**
* Save a suspicious email to quarantine
* @param email The email to quarantine
* @param scanResult The scan result
*/
private async saveToQuarantine(email: Email, scanResult: any): Promise<void> {
try {
// Create quarantine directory if it doesn't exist
const quarantinePath = plugins.path.join(paths.dataDir, 'emails', 'quarantine');
plugins.smartfile.fs.ensureDirSync(quarantinePath);
// Generate a filename with timestamp and details
const timestamp = Date.now();
const safeFrom = email.from.replace(/[^a-zA-Z0-9]/g, '_');
const filename = `${timestamp}_${safeFrom}_${scanResult.threatScore}.eml`;
// Save the email
const emailContent = email.toRFC822String();
const filePath = plugins.path.join(quarantinePath, filename);
plugins.smartfile.memory.toFsSync(emailContent, filePath);
// Save scan metadata alongside the email
const metadataPath = plugins.path.join(quarantinePath, `${filename}.meta.json`);
const metadata = {
timestamp,
from: email.from,
to: email.to,
subject: email.subject,
messageId: email.getMessageId(),
scanResult: {
threatType: scanResult.threatType,
threatDetails: scanResult.threatDetails,
threatScore: scanResult.threatScore,
scannedElements: scanResult.scannedElements
}
};
plugins.smartfile.memory.toFsSync(
JSON.stringify(metadata, null, 2),
metadataPath
);
console.log(`Email quarantined: ${filePath}`);
} catch (error) {
console.error('Error saving email to quarantine:', error);
}
}
/**
* Check if a domain is local
@ -456,6 +692,14 @@ export class MtaService {
* Save an email to a local mailbox
*/
private async saveToLocalMailbox(email: Email): Promise<void> {
// Check if this is a bounce notification
const isBounceNotification = this.isBounceNotification(email);
if (isBounceNotification) {
await this.processBounceNotification(email);
return;
}
// Simplified implementation - in a real system, this would store to a user's mailbox
const mailboxPath = plugins.path.join(paths.receivedEmailsDir, 'local');
plugins.smartfile.fs.ensureDirSync(mailboxPath);
@ -470,6 +714,77 @@ export class MtaService {
console.log(`Email saved to local mailbox: ${filename}`);
}
/**
* Check if an email is a bounce notification
*/
private isBounceNotification(email: Email): boolean {
// Check subject for bounce-related keywords
const subject = email.subject?.toLowerCase() || '';
if (
subject.includes('mail delivery') ||
subject.includes('delivery failed') ||
subject.includes('undeliverable') ||
subject.includes('delivery status') ||
subject.includes('failure notice') ||
subject.includes('returned mail') ||
subject.includes('delivery problem')
) {
return true;
}
// Check sender address for common bounced email addresses
const from = email.from.toLowerCase();
if (
from.includes('mailer-daemon') ||
from.includes('postmaster') ||
from.includes('mail-delivery') ||
from.includes('bounces')
) {
return true;
}
return false;
}
/**
* Process a bounce notification
*/
private async processBounceNotification(email: Email): Promise<void> {
try {
console.log(`Processing bounce notification from ${email.from}`);
// Convert to Smartmail for bounce processing
const smartmail = await email.toSmartmail();
// If we have a bounce manager available, process it
if (this.emailService?.bounceManager) {
const bounceResult = await this.emailService.bounceManager.processBounceEmail(smartmail);
if (bounceResult) {
console.log(`Processed bounce for recipient: ${bounceResult.recipient}, type: ${bounceResult.bounceType}`);
} else {
console.log('Could not extract bounce information from email');
}
} else {
console.log('Bounce manager not available, saving bounce notification for later processing');
// Save to bounces directory for later processing
const bouncesPath = plugins.path.join(paths.dataDir, 'emails', 'bounces');
plugins.smartfile.fs.ensureDirSync(bouncesPath);
const emailContent = email.toRFC822String();
const filename = `${Date.now()}_bounce.eml`;
plugins.smartfile.memory.toFsSync(
emailContent,
plugins.path.join(bouncesPath, filename)
);
}
} catch (error) {
console.error('Error processing bounce notification:', error);
}
}
/**
* Start processing the email queue
@ -572,6 +887,17 @@ export class MtaService {
this.stats.emailsFailed++;
console.log(`Email ${entry.id} failed permanently: ${entry.error.message}`);
// Record bounce event for reputation monitoring
if (this.config.outbound?.reputation?.enabled) {
const domain = entry.email.getFromDomain();
if (domain) {
this.reputationMonitor.recordSendEvent(domain, {
type: 'bounce',
hardBounce: true
});
}
}
// Remove from queue
this.emailQueue.delete(entry.id);
} else if (status === DeliveryStatus.DEFERRED) {
@ -587,6 +913,17 @@ export class MtaService {
// Remove from queue
this.emailQueue.delete(entry.id);
} else {
// Record soft bounce for reputation monitoring
if (this.config.outbound?.reputation?.enabled) {
const domain = entry.email.getFromDomain();
if (domain) {
this.reputationMonitor.recordSendEvent(domain, {
type: 'bounce',
hardBounce: false
});
}
}
// Schedule retry
const delay = this.calculateRetryDelay(entry.attempts);
entry.nextAttempt = new Date(Date.now() + delay);
@ -602,9 +939,33 @@ export class MtaService {
if (entry.attempts >= this.config.outbound.retries.max) {
entry.status = DeliveryStatus.FAILED;
this.stats.emailsFailed++;
// Record bounce event for reputation monitoring after max retries
if (this.config.outbound?.reputation?.enabled) {
const domain = entry.email.getFromDomain();
if (domain) {
this.reputationMonitor.recordSendEvent(domain, {
type: 'bounce',
hardBounce: true
});
}
}
this.emailQueue.delete(entry.id);
} else {
entry.status = DeliveryStatus.DEFERRED;
// Record soft bounce for reputation monitoring
if (this.config.outbound?.reputation?.enabled) {
const domain = entry.email.getFromDomain();
if (domain) {
this.reputationMonitor.recordSendEvent(domain, {
type: 'bounce',
hardBounce: false
});
}
}
const delay = this.calculateRetryDelay(entry.attempts);
entry.nextAttempt = new Date(Date.now() + delay);
}
@ -635,42 +996,11 @@ export class MtaService {
* Check if an email can be sent under rate limits
*/
private checkRateLimit(email: Email): boolean {
const config = this.config.outbound.rateLimit;
if (!config || !config.maxPerPeriod) {
return true; // No rate limit configured
}
// Get the appropriate domain key
const domainKey = email.getFromDomain();
// Determine which limiter to use
const key = config.perDomain ? email.getFromDomain() : 'global';
// Initialize limiter if needed
if (!this.rateLimiters.has(key)) {
this.rateLimiters.set(key, {
tokens: config.maxPerPeriod,
lastRefill: Date.now()
});
}
const limiter = this.rateLimiters.get(key);
// Refill tokens based on time elapsed
const now = Date.now();
const elapsedMs = now - limiter.lastRefill;
const tokensToAdd = Math.floor(elapsedMs / config.periodMs) * config.maxPerPeriod;
if (tokensToAdd > 0) {
limiter.tokens = Math.min(config.maxPerPeriod, limiter.tokens + tokensToAdd);
limiter.lastRefill = now - (elapsedMs % config.periodMs);
}
// Check if we have tokens available
if (limiter.tokens > 0) {
limiter.tokens--;
return true;
} else {
console.log(`Rate limit exceeded for ${key}`);
return false;
}
// Check if sending is allowed under rate limits
return this.rateLimiter.consume(domainKey);
}
/**
@ -974,10 +1304,24 @@ export class MtaService {
}
}
/**
* Get the IP warmup manager
*/
public getIPWarmupManager(): IPWarmupManager {
return this.ipWarmupManager;
}
/**
* Get the sender reputation monitor
*/
public getReputationMonitor(): SenderReputationMonitor {
return this.reputationMonitor;
}
/**
* Get MTA service statistics
*/
public getStats(): MtaStats {
public getStats(): MtaStats & { rateLimiting?: any } {
// Update queue size
this.stats.queueSize = this.emailQueue.size;
@ -995,6 +1339,80 @@ export class MtaService {
};
}
return { ...this.stats };
// Add rate limiting stats
const statsWithRateLimiting = {
...this.stats,
rateLimiting: {
global: this.rateLimiter.getStats('global')
}
};
// Add warmup information if enabled
if (this.config.outbound?.warmup?.enabled) {
const warmupStatuses = this.ipWarmupManager.getWarmupStatus() as Map<string, any>;
let activeIPs = 0;
let inWarmupPhase = 0;
let completedWarmup = 0;
warmupStatuses.forEach(status => {
activeIPs++;
if (status.isActive) {
if (status.currentStage < this.ipWarmupManager.getStageCount()) {
inWarmupPhase++;
} else {
completedWarmup++;
}
}
});
statsWithRateLimiting.warmupInfo = {
enabled: true,
activeIPs,
inWarmupPhase,
completedWarmup
};
} else {
statsWithRateLimiting.warmupInfo = {
enabled: false,
activeIPs: 0,
inWarmupPhase: 0,
completedWarmup: 0
};
}
// Add reputation metrics if enabled
if (this.config.outbound?.reputation?.enabled) {
const reputationSummary = this.reputationMonitor.getReputationSummary();
// Calculate average reputation score
const avgScore = reputationSummary.length > 0
? reputationSummary.reduce((sum, domain) => sum + domain.score, 0) / reputationSummary.length
: 0;
// Count domains with issues
const domainsWithIssues = reputationSummary.filter(
domain => domain.status === 'poor' || domain.status === 'critical' || domain.listed
).length;
statsWithRateLimiting.reputationInfo = {
enabled: true,
monitoredDomains: reputationSummary.length,
averageScore: avgScore,
domainsWithIssues
};
} else {
statsWithRateLimiting.reputationInfo = {
enabled: false,
monitoredDomains: 0,
averageScore: 0,
domainsWithIssues: 0
};
}
// Clean up old rate limiter buckets to prevent memory leaks
this.rateLimiter.cleanup();
return statsWithRateLimiting;
}
}

281
ts/mta/classes.ratelimiter.ts Normal file
View File

@ -0,0 +1,281 @@
import { logger } from '../logger.js';
/**
* Configuration options for rate limiter
*/
export interface IRateLimitConfig {
/** Maximum tokens per period */
maxPerPeriod: number;
/** Time period in milliseconds */
periodMs: number;
/** Whether to apply per domain/key (vs globally) */
perKey: boolean;
/** Initial token count (defaults to max) */
initialTokens?: number;
/** Grace tokens to allow occasional bursts */
burstTokens?: number;
/** Apply global limit in addition to per-key limits */
useGlobalLimit?: boolean;
}
/**
* Token bucket for an individual key
*/
interface TokenBucket {
/** Current number of tokens */
tokens: number;
/** Last time tokens were refilled */
lastRefill: number;
/** Total allowed requests */
allowed: number;
/** Total denied requests */
denied: number;
}
/**
* Rate limiter using token bucket algorithm
* Provides more sophisticated rate limiting with burst handling
*/
export class RateLimiter {
/** Rate limit configuration */
private config: IRateLimitConfig;
/** Token buckets per key */
private buckets: Map<string, TokenBucket> = new Map();
/** Global bucket for non-keyed rate limiting */
private globalBucket: TokenBucket;
/**
* Create a new rate limiter
* @param config Rate limiter configuration
*/
constructor(config: IRateLimitConfig) {
// Set defaults
this.config = {
maxPerPeriod: config.maxPerPeriod,
periodMs: config.periodMs,
perKey: config.perKey ?? true,
initialTokens: config.initialTokens ?? config.maxPerPeriod,
burstTokens: config.burstTokens ?? 0,
useGlobalLimit: config.useGlobalLimit ?? false
};
// Initialize global bucket
this.globalBucket = {
tokens: this.config.initialTokens,
lastRefill: Date.now(),
allowed: 0,
denied: 0
};
// Log initialization
logger.log('info', `Rate limiter initialized: ${this.config.maxPerPeriod} per ${this.config.periodMs}ms${this.config.perKey ? ' per key' : ''}`);
}
/**
* Check if a request is allowed under rate limits
* @param key Key to check rate limit for (e.g. domain, user, IP)
* @param cost Token cost (defaults to 1)
* @returns Whether the request is allowed
*/
public isAllowed(key: string = 'global', cost: number = 1): boolean {
// If using global bucket directly, just check that
if (key === 'global' || !this.config.perKey) {
return this.checkBucket(this.globalBucket, cost);
}
// Get the key-specific bucket
const bucket = this.getBucket(key);
// If we also need to check global limit
if (this.config.useGlobalLimit) {
// Both key bucket and global bucket must have tokens
return this.checkBucket(bucket, cost) && this.checkBucket(this.globalBucket, cost);
} else {
// Only need to check the key-specific bucket
return this.checkBucket(bucket, cost);
}
}
/**
* Check if a bucket has enough tokens and consume them
* @param bucket The token bucket to check
* @param cost Token cost
* @returns Whether tokens were consumed
*/
private checkBucket(bucket: TokenBucket, cost: number): boolean {
// Refill tokens based on elapsed time
this.refillBucket(bucket);
// Check if we have enough tokens
if (bucket.tokens >= cost) {
// Use tokens
bucket.tokens -= cost;
bucket.allowed++;
return true;
} else {
// Rate limit exceeded
bucket.denied++;
return false;
}
}
/**
* Consume tokens for a request (if available)
* @param key Key to consume tokens for
* @param cost Token cost (defaults to 1)
* @returns Whether tokens were consumed
*/
public consume(key: string = 'global', cost: number = 1): boolean {
const isAllowed = this.isAllowed(key, cost);
return isAllowed;
}
/**
* Get the remaining tokens for a key
* @param key Key to check
* @returns Number of remaining tokens
*/
public getRemainingTokens(key: string = 'global'): number {
const bucket = this.getBucket(key);
this.refillBucket(bucket);
return bucket.tokens;
}
/**
* Get stats for a specific key
* @param key Key to get stats for
* @returns Rate limit statistics
*/
public getStats(key: string = 'global'): {
remaining: number;
limit: number;
resetIn: number;
allowed: number;
denied: number;
} {
const bucket = this.getBucket(key);
this.refillBucket(bucket);
// Calculate time until next token
const resetIn = bucket.tokens < this.config.maxPerPeriod ?
Math.ceil(this.config.periodMs / this.config.maxPerPeriod) :
0;
return {
remaining: bucket.tokens,
limit: this.config.maxPerPeriod,
resetIn,
allowed: bucket.allowed,
denied: bucket.denied
};
}
/**
* Get or create a token bucket for a key
* @param key The rate limit key
* @returns Token bucket
*/
private getBucket(key: string): TokenBucket {
if (!this.config.perKey || key === 'global') {
return this.globalBucket;
}
if (!this.buckets.has(key)) {
// Create new bucket
this.buckets.set(key, {
tokens: this.config.initialTokens,
lastRefill: Date.now(),
allowed: 0,
denied: 0
});
}
return this.buckets.get(key);
}
/**
* Refill tokens in a bucket based on elapsed time
* @param bucket Token bucket to refill
*/
private refillBucket(bucket: TokenBucket): void {
const now = Date.now();
const elapsedMs = now - bucket.lastRefill;
// Calculate how many tokens to add
const rate = this.config.maxPerPeriod / this.config.periodMs;
const tokensToAdd = elapsedMs * rate;
if (tokensToAdd >= 0.1) { // Allow for partial token refills
// Add tokens, but don't exceed the normal maximum (without burst)
// This ensures burst tokens are only used for bursts and don't refill
const normalMax = this.config.maxPerPeriod;
bucket.tokens = Math.min(
// Don't exceed max + burst
this.config.maxPerPeriod + (this.config.burstTokens || 0),
// Don't exceed normal max when refilling
Math.min(normalMax, bucket.tokens + tokensToAdd)
);
// Update last refill time
bucket.lastRefill = now;
}
}
/**
* Reset rate limits for a specific key
* @param key Key to reset
*/
public reset(key: string = 'global'): void {
if (key === 'global' || !this.config.perKey) {
this.globalBucket.tokens = this.config.initialTokens;
this.globalBucket.lastRefill = Date.now();
} else if (this.buckets.has(key)) {
const bucket = this.buckets.get(key);
bucket.tokens = this.config.initialTokens;
bucket.lastRefill = Date.now();
}
}
/**
* Reset all rate limiters
*/
public resetAll(): void {
this.globalBucket.tokens = this.config.initialTokens;
this.globalBucket.lastRefill = Date.now();
for (const bucket of this.buckets.values()) {
bucket.tokens = this.config.initialTokens;
bucket.lastRefill = Date.now();
}
}
/**
* Cleanup old buckets to prevent memory leaks
* @param maxAge Maximum age in milliseconds
*/
public cleanup(maxAge: number = 24 * 60 * 60 * 1000): void {
const now = Date.now();
let removed = 0;
for (const [key, bucket] of this.buckets.entries()) {
if (now - bucket.lastRefill > maxAge) {
this.buckets.delete(key);
removed++;
}
}
if (removed > 0) {
logger.log('debug', `Cleaned up ${removed} stale rate limit buckets`);
}
}
}

350
ts/mta/classes.smtpserver.ts
View File

@ -3,6 +3,13 @@ import * as paths from '../paths.js';
import { Email } from './classes.email.js';
import type { MtaService } from './classes.mta.js';
import { logger } from '../logger.js';
import {
SecurityLogger,
SecurityLogLevel,
SecurityEventType,
IPReputationChecker,
ReputationThreshold
} from '../security/index.js';
export interface ISmtpServerOptions {
port: number;
@ -53,8 +60,10 @@ export class SMTPServer {
});
}
private handleNewConnection(socket: plugins.net.Socket): void {
console.log(`New connection from ${socket.remoteAddress}:${socket.remotePort}`);
private async handleNewConnection(socket: plugins.net.Socket): Promise<void> {
const clientIp = socket.remoteAddress;
const clientPort = socket.remotePort;
console.log(`New connection from ${clientIp}:${clientPort}`);
// Initialize a new session
this.sessions.set(socket, {
@ -66,6 +75,68 @@ export class SMTPServer {
useTLS: false,
connectionEnded: false
});
// Check IP reputation
try {
if (this.mtaRef.config.security?.checkIPReputation !== false && clientIp) {
const reputationChecker = IPReputationChecker.getInstance();
const reputation = await reputationChecker.checkReputation(clientIp);
// Log the reputation check
SecurityLogger.getInstance().logEvent({
level: reputation.score < ReputationThreshold.HIGH_RISK
? SecurityLogLevel.WARN
: SecurityLogLevel.INFO,
type: SecurityEventType.IP_REPUTATION,
message: `IP reputation checked for new SMTP connection: score=${reputation.score}`,
ipAddress: clientIp,
details: {
clientPort,
score: reputation.score,
isSpam: reputation.isSpam,
isProxy: reputation.isProxy,
isTor: reputation.isTor,
isVPN: reputation.isVPN,
country: reputation.country,
blacklists: reputation.blacklists,
socketId: socket.remotePort.toString() + socket.remoteFamily
}
});
// Handle high-risk IPs - add delay or reject based on score
if (reputation.score < ReputationThreshold.HIGH_RISK) {
// For high-risk connections, add an artificial delay to slow down potential spam
const delayMs = Math.min(5000, Math.max(1000, (ReputationThreshold.HIGH_RISK - reputation.score) * 100));
await new Promise(resolve => setTimeout(resolve, delayMs));
if (reputation.score < 5) {
// Very high risk - can optionally reject the connection
if (this.mtaRef.config.security?.rejectHighRiskIPs) {
this.sendResponse(socket, `554 Transaction failed - IP is on spam blocklist`);
socket.destroy();
return;
}
}
}
}
} catch (error) {
logger.log('error', `Error checking IP reputation: ${error.message}`, {
ip: clientIp,
error: error.message
});
}
// Log the connection as a security event
SecurityLogger.getInstance().logEvent({
level: SecurityLogLevel.INFO,
type: SecurityEventType.CONNECTION,
message: `New SMTP connection established`,
ipAddress: clientIp,
details: {
clientPort,
socketId: socket.remotePort.toString() + socket.remoteFamily
}
});
// Send greeting
this.sendResponse(socket, `220 ${this.hostname} ESMTP Service Ready`);
@ -75,21 +146,69 @@ export class SMTPServer {
});
socket.on('end', () => {
console.log(`Connection ended from ${socket.remoteAddress}:${socket.remotePort}`);
const clientIp = socket.remoteAddress;
const clientPort = socket.remotePort;
console.log(`Connection ended from ${clientIp}:${clientPort}`);
const session = this.sessions.get(socket);
if (session) {
session.connectionEnded = true;
// Log connection end as security event
SecurityLogger.getInstance().logEvent({
level: SecurityLogLevel.INFO,
type: SecurityEventType.CONNECTION,
message: `SMTP connection ended normally`,
ipAddress: clientIp,
details: {
clientPort,
state: SmtpState[session.state],
from: session.mailFrom || 'not set'
}
});
}
});
socket.on('error', (err) => {
const clientIp = socket.remoteAddress;
const clientPort = socket.remotePort;
console.error(`Socket error: ${err.message}`);
// Log connection error as security event
SecurityLogger.getInstance().logEvent({
level: SecurityLogLevel.WARN,
type: SecurityEventType.CONNECTION,
message: `SMTP connection error`,
ipAddress: clientIp,
details: {
clientPort,
error: err.message,
errorCode: (err as any).code,
from: this.sessions.get(socket)?.mailFrom || 'not set'
}
});
this.sessions.delete(socket);
socket.destroy();
});
socket.on('close', () => {
console.log(`Connection closed from ${socket.remoteAddress}:${socket.remotePort}`);
const clientIp = socket.remoteAddress;
const clientPort = socket.remotePort;
console.log(`Connection closed from ${clientIp}:${clientPort}`);
// Log connection closure as security event
SecurityLogger.getInstance().logEvent({
level: SecurityLogLevel.INFO,
type: SecurityEventType.CONNECTION,
message: `SMTP connection closed`,
ipAddress: clientIp,
details: {
clientPort,
sessionEnded: this.sessions.get(socket)?.connectionEnded || false
}
});
this.sessions.delete(socket);
});
}
@ -358,33 +477,165 @@ export class SMTPServer {
// Prepare headers for DKIM verification results
const customHeaders: Record<string, string> = {};
// Verifying the email with enhanced DKIM verification
try {
const verificationResult = await this.mtaRef.dkimVerifier.verify(session.emailData, {
useCache: true,
returnDetails: false
});
// Authentication results
let dkimResult = { domain: '', result: false };
let spfResult = { domain: '', result: false };
// Check security configuration
const securityConfig = this.mtaRef.config.security || {};
// 1. Verify DKIM signature if enabled
if (securityConfig.verifyDkim !== false) {
try {
const verificationResult = await this.mtaRef.dkimVerifier.verify(session.emailData, {
useCache: true,
returnDetails: false
});
mightBeSpam = !verificationResult.isValid;
if (!verificationResult.isValid) {
logger.log('warn', `DKIM verification failed for incoming email: ${verificationResult.errorMessage || 'Unknown error'}`);
} else {
logger.log('info', `DKIM verification passed for incoming email from domain ${verificationResult.domain}`);
dkimResult.result = verificationResult.isValid;
dkimResult.domain = verificationResult.domain || '';
if (!verificationResult.isValid) {
logger.log('warn', `DKIM verification failed for incoming email: ${verificationResult.errorMessage || 'Unknown error'}`);
// Enhanced security logging
SecurityLogger.getInstance().logEvent({
level: SecurityLogLevel.WARN,
type: SecurityEventType.DKIM,
message: `DKIM verification failed for incoming email`,
domain: verificationResult.domain || session.mailFrom.split('@')[1],
details: {
error: verificationResult.errorMessage || 'Unknown error',
status: verificationResult.status,
selector: verificationResult.selector,
senderIP: socket.remoteAddress
},
ipAddress: socket.remoteAddress,
success: false
});
} else {
logger.log('info', `DKIM verification passed for incoming email from domain ${verificationResult.domain}`);
// Enhanced security logging
SecurityLogger.getInstance().logEvent({
level: SecurityLogLevel.INFO,
type: SecurityEventType.DKIM,
message: `DKIM verification passed for incoming email`,
domain: verificationResult.domain,
details: {
selector: verificationResult.selector,
status: verificationResult.status,
senderIP: socket.remoteAddress
},
ipAddress: socket.remoteAddress,
success: true
});
}
// Store verification results in headers
if (verificationResult.domain) {
customHeaders['X-DKIM-Domain'] = verificationResult.domain;
}
customHeaders['X-DKIM-Status'] = verificationResult.status || 'unknown';
customHeaders['X-DKIM-Result'] = verificationResult.isValid ? 'pass' : 'fail';
} catch (error) {
logger.log('error', `Failed to verify DKIM signature: ${error.message}`);
customHeaders['X-DKIM-Status'] = 'error';
customHeaders['X-DKIM-Result'] = 'error';
}
}
// 2. Verify SPF if enabled
if (securityConfig.verifySpf !== false) {
try {
// Get the client IP and hostname
const clientIp = socket.remoteAddress || '127.0.0.1';
const clientHostname = session.clientHostname || 'localhost';
// Store verification results in headers
if (verificationResult.domain) {
customHeaders['X-DKIM-Domain'] = verificationResult.domain;
// Parse the email to get envelope from
const parsedEmail = await plugins.mailparser.simpleParser(session.emailData);
// Create a temporary Email object for SPF verification
const tempEmail = new Email({
from: parsedEmail.from?.value[0].address || session.mailFrom,
to: session.rcptTo[0],
subject: "Temporary Email for SPF Verification",
text: "This is a temporary email for SPF verification"
});
// Set envelope from for SPF verification
tempEmail.setEnvelopeFrom(session.mailFrom);
// Verify SPF
const spfVerified = await this.mtaRef.spfVerifier.verifyAndApply(
tempEmail,
clientIp,
clientHostname
);
// Update SPF result
spfResult.result = spfVerified;
spfResult.domain = session.mailFrom.split('@')[1] || '';
// Copy SPF headers from the temp email
if (tempEmail.headers['Received-SPF']) {
customHeaders['Received-SPF'] = tempEmail.headers['Received-SPF'];
}
// Set spam flag if SPF fails badly
if (tempEmail.mightBeSpam) {
mightBeSpam = true;
}
} catch (error) {
logger.log('error', `Failed to verify SPF: ${error.message}`);
customHeaders['Received-SPF'] = `error (${error.message})`;
}
}
// 3. Verify DMARC if enabled
if (securityConfig.verifyDmarc !== false) {
try {
// Parse the email again
const parsedEmail = await plugins.mailparser.simpleParser(session.emailData);
customHeaders['X-DKIM-Status'] = verificationResult.status || 'unknown';
customHeaders['X-DKIM-Result'] = verificationResult.isValid ? 'pass' : 'fail';
} catch (error) {
logger.log('error', `Failed to verify DKIM signature: ${error.message}`);
mightBeSpam = true;
customHeaders['X-DKIM-Status'] = 'error';
customHeaders['X-DKIM-Result'] = 'error';
// Create a temporary Email object for DMARC verification
const tempEmail = new Email({
from: parsedEmail.from?.value[0].address || session.mailFrom,
to: session.rcptTo[0],
subject: "Temporary Email for DMARC Verification",
text: "This is a temporary email for DMARC verification"
});
// Verify DMARC
const dmarcResult = await this.mtaRef.dmarcVerifier.verify(
tempEmail,
spfResult,
dkimResult
);
// Apply DMARC policy
const dmarcPassed = this.mtaRef.dmarcVerifier.applyPolicy(tempEmail, dmarcResult);
// Add DMARC result to headers
if (tempEmail.headers['X-DMARC-Result']) {
customHeaders['X-DMARC-Result'] = tempEmail.headers['X-DMARC-Result'];
}
// Add Authentication-Results header combining all authentication results
customHeaders['Authentication-Results'] = `${this.mtaRef.config.smtp.hostname}; ` +
`spf=${spfResult.result ? 'pass' : 'fail'} smtp.mailfrom=${session.mailFrom}; ` +
`dkim=${dkimResult.result ? 'pass' : 'fail'} header.d=${dkimResult.domain || 'unknown'}; ` +
`dmarc=${dmarcPassed ? 'pass' : 'fail'} header.from=${tempEmail.getFromDomain()}`;
// Set spam flag if DMARC fails
if (tempEmail.mightBeSpam) {
mightBeSpam = true;
}
} catch (error) {
logger.log('error', `Failed to verify DMARC: ${error.message}`);
customHeaders['X-DMARC-Result'] = `error (${error.message})`;
}
}
try {
@ -411,15 +662,62 @@ export class SMTPServer {
attachments: email.attachments.length,
mightBeSpam: email.mightBeSpam
});
// Enhanced security logging for received email
SecurityLogger.getInstance().logEvent({
level: mightBeSpam ? SecurityLogLevel.WARN : SecurityLogLevel.INFO,
type: mightBeSpam ? SecurityEventType.SPAM : SecurityEventType.EMAIL_VALIDATION,
message: `Email received and ${mightBeSpam ? 'flagged as potential spam' : 'validated successfully'}`,
domain: email.from.split('@')[1],
ipAddress: socket.remoteAddress,
details: {
from: email.from,
subject: email.subject,
recipientCount: email.getAllRecipients().length,
attachmentCount: email.attachments.length,
hasAttachments: email.hasAttachments(),
dkimStatus: customHeaders['X-DKIM-Result'] || 'unknown'
},
success: !mightBeSpam
});
// Process or forward the email via MTA service
try {
await this.mtaRef.processIncomingEmail(email);
} catch (err) {
console.error('Error in MTA processing of incoming email:', err);
// Log processing errors
SecurityLogger.getInstance().logEvent({
level: SecurityLogLevel.ERROR,
type: SecurityEventType.EMAIL_VALIDATION,
message: `Error processing incoming email`,
domain: email.from.split('@')[1],
ipAddress: socket.remoteAddress,
details: {
error: err.message,
from: email.from,
stack: err.stack
},
success: false
});
}
} catch (error) {
console.error('Error parsing email:', error);
// Log parsing errors
SecurityLogger.getInstance().logEvent({
level: SecurityLogLevel.ERROR,
type: SecurityEventType.EMAIL_VALIDATION,
message: `Error parsing incoming email`,
ipAddress: socket.remoteAddress,
details: {
error: error.message,
sender: session.mailFrom,
stack: error.stack
},
success: false
});
}
}

599
ts/mta/classes.spfverifier.ts Normal file
View File

@ -0,0 +1,599 @@
import * as plugins from '../plugins.js';
import { logger } from '../logger.js';
import { SecurityLogger, SecurityLogLevel, SecurityEventType } from '../security/index.js';
import type { MtaService } from './classes.mta.js';
import type { Email } from './classes.email.js';
import type { IDnsVerificationResult } from './classes.dnsmanager.js';
/**
* SPF result qualifiers
*/
export enum SpfQualifier {
PASS = '+',
NEUTRAL = '?',
SOFTFAIL = '~',
FAIL = '-'
}
/**
* SPF mechanism types
*/
export enum SpfMechanismType {
ALL = 'all',
INCLUDE = 'include',
A = 'a',
MX = 'mx',
IP4 = 'ip4',
IP6 = 'ip6',
EXISTS = 'exists',
REDIRECT = 'redirect',
EXP = 'exp'
}
/**
* SPF mechanism definition
*/
export interface SpfMechanism {
qualifier: SpfQualifier;
type: SpfMechanismType;
value?: string;
}
/**
* SPF record parsed data
*/
export interface SpfRecord {
version: string;
mechanisms: SpfMechanism[];
modifiers: Record<string, string>;
}
/**
* SPF verification result
*/
export interface SpfResult {
result: 'pass' | 'neutral' | 'softfail' | 'fail' | 'temperror' | 'permerror' | 'none';
explanation?: string;
domain: string;
ip: string;
record?: string;
error?: string;
}
/**
* Maximum lookup limit for SPF records (prevent infinite loops)
*/
const MAX_SPF_LOOKUPS = 10;
/**
* Class for verifying SPF records
*/
export class SpfVerifier {
private mtaRef: MtaService;
private lookupCount: number = 0;
constructor(mtaRefArg: MtaService) {
this.mtaRef = mtaRefArg;
}
/**
* Parse SPF record from TXT record
* @param record SPF TXT record
* @returns Parsed SPF record or null if invalid
*/
public parseSpfRecord(record: string): SpfRecord | null {
if (!record.startsWith('v=spf1')) {
return null;
}
try {
const spfRecord: SpfRecord = {
version: 'spf1',
mechanisms: [],
modifiers: {}
};
// Split into terms
const terms = record.split(' ').filter(term => term.length > 0);
// Skip version term
for (let i = 1; i < terms.length; i++) {
const term = terms[i];
// Check if it's a modifier (name=value)
if (term.includes('=')) {
const [name, value] = term.split('=');
spfRecord.modifiers[name] = value;
continue;
}
// Parse as mechanism
let qualifier = SpfQualifier.PASS; // Default is +
let mechanismText = term;
// Check for qualifier
if (term.startsWith('+') || term.startsWith('-') ||
term.startsWith('~') || term.startsWith('?')) {
qualifier = term[0] as SpfQualifier;
mechanismText = term.substring(1);
}
// Parse mechanism type and value
const colonIndex = mechanismText.indexOf(':');
let type: SpfMechanismType;
let value: string | undefined;
if (colonIndex !== -1) {
type = mechanismText.substring(0, colonIndex) as SpfMechanismType;
value = mechanismText.substring(colonIndex + 1);
} else {
type = mechanismText as SpfMechanismType;
}
spfRecord.mechanisms.push({ qualifier, type, value });
}
return spfRecord;
} catch (error) {
logger.log('error', `Error parsing SPF record: ${error.message}`, {
record,
error: error.message
});
return null;
}
}
/**
* Check if IP is in CIDR range
* @param ip IP address to check
* @param cidr CIDR range
* @returns Whether the IP is in the CIDR range
*/
private isIpInCidr(ip: string, cidr: string): boolean {
try {
const ipAddress = plugins.ip.Address4.parse(ip);
return ipAddress.isInSubnet(new plugins.ip.Address4(cidr));
} catch (error) {
// Try IPv6
try {
const ipAddress = plugins.ip.Address6.parse(ip);
return ipAddress.isInSubnet(new plugins.ip.Address6(cidr));
} catch (e) {
return false;
}
}
}
/**
* Check if a domain has the specified IP in its A or AAAA records
* @param domain Domain to check
* @param ip IP address to check
* @returns Whether the domain resolves to the IP
*/
private async isDomainResolvingToIp(domain: string, ip: string): Promise<boolean> {
try {
// First try IPv4
const ipv4Addresses = await plugins.dns.promises.resolve4(domain);
if (ipv4Addresses.includes(ip)) {
return true;
}
// Then try IPv6
const ipv6Addresses = await plugins.dns.promises.resolve6(domain);
if (ipv6Addresses.includes(ip)) {
return true;
}
return false;
} catch (error) {
return false;
}
}
/**
* Verify SPF for a given email with IP and helo domain
* @param email Email to verify
* @param ip Sender IP address
* @param heloDomain HELO/EHLO domain used by sender
* @returns SPF verification result
*/
public async verify(
email: Email,
ip: string,
heloDomain: string
): Promise<SpfResult> {
const securityLogger = SecurityLogger.getInstance();
// Reset lookup count
this.lookupCount = 0;
// Get domain from envelope from (return-path)
const domain = email.getEnvelopeFrom().split('@')[1] || '';
if (!domain) {
return {
result: 'permerror',
explanation: 'No envelope from domain',
domain: '',
ip
};
}
try {
// Look up SPF record
const spfVerificationResult = await this.mtaRef.dnsManager.verifySpfRecord(domain);
if (!spfVerificationResult.found) {
return {
result: 'none',
explanation: 'No SPF record found',
domain,
ip
};
}
if (!spfVerificationResult.valid) {
return {
result: 'permerror',
explanation: 'Invalid SPF record',
domain,
ip,
record: spfVerificationResult.value
};
}
// Parse SPF record
const spfRecord = this.parseSpfRecord(spfVerificationResult.value);
if (!spfRecord) {
return {
result: 'permerror',
explanation: 'Failed to parse SPF record',
domain,
ip,
record: spfVerificationResult.value
};
}
// Check SPF record
const result = await this.checkSpfRecord(spfRecord, domain, ip);
// Log the result
const spfLogLevel = result.result === 'pass' ?
SecurityLogLevel.INFO :
(result.result === 'fail' ? SecurityLogLevel.WARN : SecurityLogLevel.INFO);
securityLogger.logEvent({
level: spfLogLevel,
type: SecurityEventType.SPF,
message: `SPF ${result.result} for ${domain} from IP ${ip}`,
domain,
details: {
ip,
heloDomain,
result: result.result,
explanation: result.explanation,
record: spfVerificationResult.value
},
success: result.result === 'pass'
});
return {
...result,
domain,
ip,
record: spfVerificationResult.value
};
} catch (error) {
// Log error
logger.log('error', `SPF verification error: ${error.message}`, {
domain,
ip,
error: error.message
});
securityLogger.logEvent({
level: SecurityLogLevel.ERROR,
type: SecurityEventType.SPF,
message: `SPF verification error for ${domain}`,
domain,
details: {
ip,
error: error.message
},
success: false
});
return {
result: 'temperror',
explanation: `Error verifying SPF: ${error.message}`,
domain,
ip,
error: error.message
};
}
}
/**
* Check SPF record against IP address
* @param spfRecord Parsed SPF record
* @param domain Domain being checked
* @param ip IP address to check
* @returns SPF result
*/
private async checkSpfRecord(
spfRecord: SpfRecord,
domain: string,
ip: string
): Promise<SpfResult> {
// Check for 'redirect' modifier
if (spfRecord.modifiers.redirect) {
this.lookupCount++;
if (this.lookupCount > MAX_SPF_LOOKUPS) {
return {
result: 'permerror',
explanation: 'Too many DNS lookups',
domain,
ip
};
}
// Handle redirect
const redirectDomain = spfRecord.modifiers.redirect;
const redirectResult = await this.mtaRef.dnsManager.verifySpfRecord(redirectDomain);
if (!redirectResult.found || !redirectResult.valid) {
return {
result: 'permerror',
explanation: `Invalid redirect to ${redirectDomain}`,
domain,
ip
};
}
const redirectRecord = this.parseSpfRecord(redirectResult.value);
if (!redirectRecord) {
return {
result: 'permerror',
explanation: `Failed to parse redirect record from ${redirectDomain}`,
domain,
ip
};
}
return this.checkSpfRecord(redirectRecord, redirectDomain, ip);
}
// Check each mechanism in order
for (const mechanism of spfRecord.mechanisms) {
let matched = false;
switch (mechanism.type) {
case SpfMechanismType.ALL:
matched = true;
break;
case SpfMechanismType.IP4:
if (mechanism.value) {
matched = this.isIpInCidr(ip, mechanism.value);
}
break;
case SpfMechanismType.IP6:
if (mechanism.value) {
matched = this.isIpInCidr(ip, mechanism.value);
}
break;
case SpfMechanismType.A:
this.lookupCount++;
if (this.lookupCount > MAX_SPF_LOOKUPS) {
return {
result: 'permerror',
explanation: 'Too many DNS lookups',
domain,
ip
};
}
// Check if domain has A/AAAA record matching IP
const checkDomain = mechanism.value || domain;
matched = await this.isDomainResolvingToIp(checkDomain, ip);
break;
case SpfMechanismType.MX:
this.lookupCount++;
if (this.lookupCount > MAX_SPF_LOOKUPS) {
return {
result: 'permerror',
explanation: 'Too many DNS lookups',
domain,
ip
};
}
// Check MX records
const mxDomain = mechanism.value || domain;
try {
const mxRecords = await plugins.dns.promises.resolveMx(mxDomain);
for (const mx of mxRecords) {
// Check if this MX record's IP matches
const mxMatches = await this.isDomainResolvingToIp(mx.exchange, ip);
if (mxMatches) {
matched = true;
break;
}
}
} catch (error) {
// No MX records or error
matched = false;
}
break;
case SpfMechanismType.INCLUDE:
if (!mechanism.value) {
continue;
}
this.lookupCount++;
if (this.lookupCount > MAX_SPF_LOOKUPS) {
return {
result: 'permerror',
explanation: 'Too many DNS lookups',
domain,
ip
};
}
// Check included domain's SPF record
const includeDomain = mechanism.value;
const includeResult = await this.mtaRef.dnsManager.verifySpfRecord(includeDomain);
if (!includeResult.found || !includeResult.valid) {
continue; // Skip this mechanism
}
const includeRecord = this.parseSpfRecord(includeResult.value);
if (!includeRecord) {
continue; // Skip this mechanism
}
// Recursively check the included SPF record
const includeCheck = await this.checkSpfRecord(includeRecord, includeDomain, ip);
// Include mechanism matches if the result is "pass"
matched = includeCheck.result === 'pass';
break;
case SpfMechanismType.EXISTS:
if (!mechanism.value) {
continue;
}
this.lookupCount++;
if (this.lookupCount > MAX_SPF_LOOKUPS) {
return {
result: 'permerror',
explanation: 'Too many DNS lookups',
domain,
ip
};
}
// Check if domain exists (has any A record)
try {
await plugins.dns.promises.resolve(mechanism.value, 'A');
matched = true;
} catch (error) {
matched = false;
}
break;
}
// If this mechanism matched, return its result
if (matched) {
switch (mechanism.qualifier) {
case SpfQualifier.PASS:
return {
result: 'pass',
explanation: `Matched ${mechanism.type}${mechanism.value ? ':' + mechanism.value : ''}`,
domain,
ip
};
case SpfQualifier.FAIL:
return {
result: 'fail',
explanation: `Matched ${mechanism.type}${mechanism.value ? ':' + mechanism.value : ''}`,
domain,
ip
};
case SpfQualifier.SOFTFAIL:
return {
result: 'softfail',
explanation: `Matched ${mechanism.type}${mechanism.value ? ':' + mechanism.value : ''}`,
domain,
ip
};
case SpfQualifier.NEUTRAL:
return {
result: 'neutral',
explanation: `Matched ${mechanism.type}${mechanism.value ? ':' + mechanism.value : ''}`,
domain,
ip
};
}
}
}
// If no mechanism matched, default to neutral
return {
result: 'neutral',
explanation: 'No matching mechanism found',
domain,
ip
};
}
/**
* Check if email passes SPF verification
* @param email Email to verify
* @param ip Sender IP address
* @param heloDomain HELO/EHLO domain used by sender
* @returns Whether email passes SPF
*/
public async verifyAndApply(
email: Email,
ip: string,
heloDomain: string
): Promise<boolean> {
const result = await this.verify(email, ip, heloDomain);
// Add headers
email.headers['Received-SPF'] = `${result.result} (${result.domain}: ${result.explanation}) client-ip=${ip}; envelope-from=${email.getEnvelopeFrom()}; helo=${heloDomain};`;
// Apply policy based on result
switch (result.result) {
case 'fail':
// Fail - mark as spam
email.mightBeSpam = true;
logger.log('warn', `SPF failed for ${result.domain} from ${ip}: ${result.explanation}`);
return false;
case 'softfail':
// Soft fail - accept but mark as suspicious
email.mightBeSpam = true;
logger.log('info', `SPF softfailed for ${result.domain} from ${ip}: ${result.explanation}`);
return true;
case 'neutral':
case 'none':
// Neutral or none - accept but note in headers
logger.log('info', `SPF ${result.result} for ${result.domain} from ${ip}: ${result.explanation}`);
return true;
case 'pass':
// Pass - accept
logger.log('info', `SPF passed for ${result.domain} from ${ip}: ${result.explanation}`);
return true;
case 'temperror':
case 'permerror':
// Temporary or permanent error - log but accept
logger.log('error', `SPF error for ${result.domain} from ${ip}: ${result.explanation}`);
return true;
default:
return true;
}
}
}

3
ts/mta/index.ts
View File

@ -1,7 +1,10 @@
export * from './classes.dkimcreator.js';
export * from './classes.emailsignjob.js';
export * from './classes.dkimverifier.js';
export * from './classes.dmarcverifier.js';
export * from './classes.spfverifier.js';
export * from './classes.mta.js';
export * from './classes.smtpserver.js';
export * from './classes.emailsendjob.js';
export * from './classes.email.js';
export * from './classes.ratelimiter.js';

7
ts/paths.ts
View File

@ -6,7 +6,12 @@ export const packageDir = plugins.path.join(
plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url),
'../'
);
export const dataDir = plugins.path.join(baseDir, 'data');
// Configure data directory with environment variable or default to .nogit/data
const DEFAULT_DATA_PATH = '.nogit/data';
export const dataDir = process.env.DATA_DIR
? process.env.DATA_DIR
: plugins.path.join(baseDir, DEFAULT_DATA_PATH);
// MTA directories
export const keysDir = plugins.path.join(dataDir, 'keys');

6
ts/platformservice.ts
View File

@ -3,7 +3,6 @@ import * as paths from './paths.js';
import { PlatformServiceDb } from './classes.platformservicedb.js'
import { EmailService } from './email/classes.emailservice.js';
import { SmsService } from './sms/classes.smsservice.js';
import { LetterService } from './letter/classes.letterservice.js';
import { MtaService } from './mta/classes.mta.js';
export class SzPlatformService {
@ -16,7 +15,6 @@ export class SzPlatformService {
// SubServices
public emailService: EmailService;
public letterService: LetterService;
public mtaService: MtaService;
public smsService: SmsService;
@ -26,10 +24,6 @@ export class SzPlatformService {
// lets start the sub services
this.emailService = new EmailService(this);
this.letterService = new LetterService(this, {
letterxpressUser: await this.serviceQenv.getEnvVarOnDemand('LETTER_API_USER'),
letterxpressToken: await this.serviceQenv.getEnvVarOnDemand('LETTER_API_TOKEN')
});
this.mtaService = new MtaService(this);
this.smsService = new SmsService(this, {
apiGatewayApiToken: await this.serviceQenv.getEnvVarOnDemand('SMS_API_TOKEN'),

4
ts/plugins.ts
View File

@ -57,11 +57,9 @@ export { projectinfo, qenv, smartacme, smartdata, smartdns, smartfile, smartlog,
// apiclient.xyz scope
import * as cloudflare from '@apiclient.xyz/cloudflare';
import * as letterxpress from '@apiclient.xyz/letterxpress';
export {
cloudflare,
letterxpress,
}
// tsclass scope
@ -76,10 +74,12 @@ import * as mailauth from 'mailauth';
import { dkimSign } from 'mailauth/lib/dkim/sign.js';
import mailparser from 'mailparser';
import * as uuid from 'uuid';
import * as ip from 'ip';
export {
mailauth,
dkimSign,
mailparser,
uuid,
ip,
}

739
ts/security/classes.contentscanner.ts Normal file
View File

@ -0,0 +1,739 @@
import * as plugins from '../plugins.js';
import * as paths from '../paths.js';
import { logger } from '../logger.js';
import { Email } from '../mta/classes.email.js';
import type { IAttachment } from '../mta/classes.email.js';
import { SecurityLogger, SecurityLogLevel, SecurityEventType } from './classes.securitylogger.js';
import { LRUCache } from 'lru-cache';
/**
* Scan result information
*/
export interface IScanResult {
isClean: boolean; // Whether the content is clean (no threats detected)
threatType?: string; // Type of threat if detected
threatDetails?: string; // Details about the detected threat
threatScore: number; // 0 (clean) to 100 (definitely malicious)
scannedElements: string[]; // What was scanned (subject, body, attachments, etc.)
timestamp: number; // When this scan was performed
}
/**
* Options for content scanner configuration
*/
export interface IContentScannerOptions {
maxCacheSize?: number; // Maximum number of entries to cache
cacheTTL?: number; // TTL for cache entries in ms
scanSubject?: boolean; // Whether to scan email subjects
scanBody?: boolean; // Whether to scan email bodies
scanAttachments?: boolean; // Whether to scan attachments
maxAttachmentSizeToScan?: number; // Max size of attachments to scan in bytes
scanAttachmentNames?: boolean; // Whether to scan attachment filenames
blockExecutables?: boolean; // Whether to block executable attachments
blockMacros?: boolean; // Whether to block documents with macros
customRules?: Array<{ // Custom scanning rules
pattern: string | RegExp; // Pattern to match
type: string; // Type of threat
score: number; // Threat score
description: string; // Description of the threat
}>;
minThreatScore?: number; // Minimum score to consider content as a threat
highThreatScore?: number; // Score above which content is considered high threat
}
/**
* Threat categories
*/
export enum ThreatCategory {
SPAM = 'spam',
PHISHING = 'phishing',
MALWARE = 'malware',
EXECUTABLE = 'executable',
SUSPICIOUS_LINK = 'suspicious_link',
MALICIOUS_MACRO = 'malicious_macro',
XSS = 'xss',
SENSITIVE_DATA = 'sensitive_data',
BLACKLISTED_CONTENT = 'blacklisted_content',
CUSTOM_RULE = 'custom_rule'
}
/**
* Content Scanner for detecting malicious email content
*/
export class ContentScanner {
private static instance: ContentScanner;
private scanCache: LRUCache<string, IScanResult>;
private options: Required<IContentScannerOptions>;
// Predefined patterns for common threats
private static readonly MALICIOUS_PATTERNS = {
// Phishing patterns
phishing: [
/(?:verify|confirm|update|login).*(?:account|password|details)/i,
/urgent.*(?:action|attention|required)/i,
/(?:paypal|apple|microsoft|amazon|google|bank).*(?:verify|confirm|suspend)/i,
/your.*(?:account).*(?:suspended|compromised|locked)/i,
/\b(?:password reset|security alert|security notice)\b/i
],
// Spam indicators
spam: [
/\b(?:viagra|cialis|enlargement|diet pill|lose weight fast|cheap meds)\b/i,
/\b(?:million dollars|lottery winner|prize claim|inheritance|rich widow)\b/i,
/\b(?:earn from home|make money fast|earn \$\d{3,}\/day)\b/i,
/\b(?:limited time offer|act now|exclusive deal|only \d+ left)\b/i,
/\b(?:forex|stock tip|investment opportunity|cryptocurrency|bitcoin)\b/i
],
// Malware indicators in text
malware: [
/(?:attached file|see attachment).*(?:invoice|receipt|statement|document)/i,
/open.*(?:the attached|this attachment)/i,
/(?:enable|allow).*(?:macros|content|editing)/i,
/download.*(?:attachment|file|document)/i,
/\b(?:ransomware protection|virus alert|malware detected)\b/i
],
// Suspicious links
suspiciousLinks: [
/https?:\/\/bit\.ly\//i,
/https?:\/\/goo\.gl\//i,
/https?:\/\/t\.co\//i,
/https?:\/\/tinyurl\.com\//i,
/https?:\/\/(?:\d{1,3}\.){3}\d{1,3}/i, // IP address URLs
/https?:\/\/.*\.(?:xyz|top|club|gq|cf)\//i, // Suspicious TLDs
/(?:login|account|signin|auth).*\.(?!gov|edu|com|org|net)\w+\.\w+/i, // Login pages on unusual domains
],
// XSS and script injection
scriptInjection: [
/<script.*>.*<\/script>/is,
/javascript:/i,
/on(?:click|load|mouse|error|focus|blur)=".*"/i,
/document\.(?:cookie|write|location)/i,
/eval\s*\(/i
],
// Sensitive data patterns
sensitiveData: [
/\b(?:\d{3}-\d{2}-\d{4}|\d{9})\b/, // SSN
/\b\d{13,16}\b/, // Credit card numbers
/\b(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{4})\b/ // Possible Base64
]
};
// Common executable extensions
private static readonly EXECUTABLE_EXTENSIONS = [
'.exe', '.dll', '.bat', '.cmd', '.msi', '.js', '.vbs', '.ps1',
'.sh', '.jar', '.py', '.com', '.scr', '.pif', '.hta', '.cpl',
'.reg', '.vba', '.lnk', '.wsf', '.msi', '.msp', '.mst'
];
// Document formats that may contain macros
private static readonly MACRO_DOCUMENT_EXTENSIONS = [
'.doc', '.docm', '.xls', '.xlsm', '.ppt', '.pptm', '.dotm', '.xlsb', '.ppam', '.potm'
];
/**
* Default options for the content scanner
*/
private static readonly DEFAULT_OPTIONS: Required<IContentScannerOptions> = {
maxCacheSize: 10000,
cacheTTL: 24 * 60 * 60 * 1000, // 24 hours
scanSubject: true,
scanBody: true,
scanAttachments: true,
maxAttachmentSizeToScan: 10 * 1024 * 1024, // 10MB
scanAttachmentNames: true,
blockExecutables: true,
blockMacros: true,
customRules: [],
minThreatScore: 30, // Minimum score to consider content as a threat
highThreatScore: 70 // Score above which content is considered high threat
};
/**
* Constructor for the ContentScanner
* @param options Configuration options
*/
constructor(options: IContentScannerOptions = {}) {
// Merge with default options
this.options = {
...ContentScanner.DEFAULT_OPTIONS,
...options
};
// Initialize cache
this.scanCache = new LRUCache<string, IScanResult>({
max: this.options.maxCacheSize,
ttl: this.options.cacheTTL,
});
logger.log('info', 'ContentScanner initialized');
}
/**
* Get the singleton instance of the scanner
* @param options Configuration options
* @returns Singleton scanner instance
*/
public static getInstance(options: IContentScannerOptions = {}): ContentScanner {
if (!ContentScanner.instance) {
ContentScanner.instance = new ContentScanner(options);
}
return ContentScanner.instance;
}
/**
* Scan an email for malicious content
* @param email The email to scan
* @returns Scan result
*/
public async scanEmail(email: Email): Promise<IScanResult> {
try {
// Generate a cache key from the email
const cacheKey = this.generateCacheKey(email);
// Check cache first
const cachedResult = this.scanCache.get(cacheKey);
if (cachedResult) {
logger.log('info', `Using cached scan result for email ${email.getMessageId()}`);
return cachedResult;
}
// Initialize scan result
const result: IScanResult = {
isClean: true,
threatScore: 0,
scannedElements: [],
timestamp: Date.now()
};
// List of scan promises
const scanPromises: Array<Promise<void>> = [];
// Scan subject
if (this.options.scanSubject && email.subject) {
scanPromises.push(this.scanSubject(email.subject, result));
}
// Scan body content
if (this.options.scanBody) {
if (email.text) {
scanPromises.push(this.scanTextContent(email.text, result));
}
if (email.html) {
scanPromises.push(this.scanHtmlContent(email.html, result));
}
}
// Scan attachments
if (this.options.scanAttachments && email.attachments && email.attachments.length > 0) {
for (const attachment of email.attachments) {
scanPromises.push(this.scanAttachment(attachment, result));
}
}
// Run all scans in parallel
await Promise.all(scanPromises);
// Determine if the email is clean based on threat score
result.isClean = result.threatScore < this.options.minThreatScore;
// Save to cache
this.scanCache.set(cacheKey, result);
// Log high threat findings
if (result.threatScore >= this.options.highThreatScore) {
this.logHighThreatFound(email, result);
} else if (!result.isClean) {
this.logThreatFound(email, result);
}
return result;
} catch (error) {
logger.log('error', `Error scanning email: ${error.message}`, {
messageId: email.getMessageId(),
error: error.stack
});
// Return a safe default with error indication
return {
isClean: true, // Let it pass if scanner fails (configure as desired)
threatScore: 0,
scannedElements: ['error'],
timestamp: Date.now(),
threatType: 'scan_error',
threatDetails: `Scan error: ${error.message}`
};
}
}
/**
* Generate a cache key from an email
* @param email The email to generate a key for
* @returns Cache key
*/
private generateCacheKey(email: Email): string {
// Use message ID if available
if (email.getMessageId()) {
return `email:${email.getMessageId()}`;
}
// Fallback to a hash of key content
const contentToHash = [
email.from,
email.subject || '',
email.text?.substring(0, 1000) || '',
email.html?.substring(0, 1000) || '',
email.attachments?.length || 0
].join(':');
return `email:${plugins.crypto.createHash('sha256').update(contentToHash).digest('hex')}`;
}
/**
* Scan email subject for threats
* @param subject The subject to scan
* @param result The scan result to update
*/
private async scanSubject(subject: string, result: IScanResult): Promise<void> {
result.scannedElements.push('subject');
// Check against phishing patterns
for (const pattern of ContentScanner.MALICIOUS_PATTERNS.phishing) {
if (pattern.test(subject)) {
result.threatScore += 25;
result.threatType = ThreatCategory.PHISHING;
result.threatDetails = `Subject contains potential phishing indicators: ${subject}`;
return;
}
}
// Check against spam patterns
for (const pattern of ContentScanner.MALICIOUS_PATTERNS.spam) {
if (pattern.test(subject)) {
result.threatScore += 15;
result.threatType = ThreatCategory.SPAM;
result.threatDetails = `Subject contains potential spam indicators: ${subject}`;
return;
}
}
// Check custom rules
for (const rule of this.options.customRules) {
const pattern = rule.pattern instanceof RegExp ? rule.pattern : new RegExp(rule.pattern, 'i');
if (pattern.test(subject)) {
result.threatScore += rule.score;
result.threatType = rule.type;
result.threatDetails = rule.description;
return;
}
}
}
/**
* Scan plain text content for threats
* @param text The text content to scan
* @param result The scan result to update
*/
private async scanTextContent(text: string, result: IScanResult): Promise<void> {
result.scannedElements.push('text');
// Check suspicious links
for (const pattern of ContentScanner.MALICIOUS_PATTERNS.suspiciousLinks) {
if (pattern.test(text)) {
result.threatScore += 20;
if (!result.threatType || result.threatScore > (result.threatType === ThreatCategory.SUSPICIOUS_LINK ? 0 : 20)) {
result.threatType = ThreatCategory.SUSPICIOUS_LINK;
result.threatDetails = `Text contains suspicious links`;
}
}
}
// Check phishing
for (const pattern of ContentScanner.MALICIOUS_PATTERNS.phishing) {
if (pattern.test(text)) {
result.threatScore += 25;
if (!result.threatType || result.threatScore > (result.threatType === ThreatCategory.PHISHING ? 0 : 25)) {
result.threatType = ThreatCategory.PHISHING;
result.threatDetails = `Text contains potential phishing indicators`;
}
}
}
// Check spam
for (const pattern of ContentScanner.MALICIOUS_PATTERNS.spam) {
if (pattern.test(text)) {
result.threatScore += 15;
if (!result.threatType || result.threatScore > (result.threatType === ThreatCategory.SPAM ? 0 : 15)) {
result.threatType = ThreatCategory.SPAM;
result.threatDetails = `Text contains potential spam indicators`;
}
}
}
// Check malware indicators
for (const pattern of ContentScanner.MALICIOUS_PATTERNS.malware) {
if (pattern.test(text)) {
result.threatScore += 30;
if (!result.threatType || result.threatScore > (result.threatType === ThreatCategory.MALWARE ? 0 : 30)) {
result.threatType = ThreatCategory.MALWARE;
result.threatDetails = `Text contains potential malware indicators`;
}
}
}
// Check sensitive data
for (const pattern of ContentScanner.MALICIOUS_PATTERNS.sensitiveData) {
if (pattern.test(text)) {
result.threatScore += 25;
if (!result.threatType || result.threatScore > (result.threatType === ThreatCategory.SENSITIVE_DATA ? 0 : 25)) {
result.threatType = ThreatCategory.SENSITIVE_DATA;
result.threatDetails = `Text contains potentially sensitive data patterns`;
}
}
}
// Check custom rules
for (const rule of this.options.customRules) {
const pattern = rule.pattern instanceof RegExp ? rule.pattern : new RegExp(rule.pattern, 'i');
if (pattern.test(text)) {
result.threatScore += rule.score;
if (!result.threatType || result.threatScore > 20) {
result.threatType = rule.type;
result.threatDetails = rule.description;
}
}
}
}
/**
* Scan HTML content for threats
* @param html The HTML content to scan
* @param result The scan result to update
*/
private async scanHtmlContent(html: string, result: IScanResult): Promise<void> {
result.scannedElements.push('html');
// Check for script injection
for (const pattern of ContentScanner.MALICIOUS_PATTERNS.scriptInjection) {
if (pattern.test(html)) {
result.threatScore += 40;
if (!result.threatType || result.threatType !== ThreatCategory.XSS) {
result.threatType = ThreatCategory.XSS;
result.threatDetails = `HTML contains potentially malicious script content`;
}
}
}
// Extract text content from HTML for further scanning
const textContent = this.extractTextFromHtml(html);
if (textContent) {
// We'll leverage the text scanning but not double-count threat score
const tempResult: IScanResult = {
isClean: true,
threatScore: 0,
scannedElements: [],
timestamp: Date.now()
};
await this.scanTextContent(textContent, tempResult);
// Only add additional threat types if they're more severe
if (tempResult.threatType && tempResult.threatScore > 0) {
// Add half of the text content score to avoid double counting
result.threatScore += Math.floor(tempResult.threatScore / 2);
// Adopt the threat type if more severe or no existing type
if (!result.threatType || tempResult.threatScore > result.threatScore) {
result.threatType = tempResult.threatType;
result.threatDetails = tempResult.threatDetails;
}
}
}
// Extract and check links from HTML
const links = this.extractLinksFromHtml(html);
if (links.length > 0) {
// Check for suspicious links
let suspiciousLinks = 0;
for (const link of links) {
for (const pattern of ContentScanner.MALICIOUS_PATTERNS.suspiciousLinks) {
if (pattern.test(link)) {
suspiciousLinks++;
break;
}
}
}
if (suspiciousLinks > 0) {
// Add score based on percentage of suspicious links
const suspiciousPercentage = (suspiciousLinks / links.length) * 100;
const additionalScore = Math.min(40, Math.floor(suspiciousPercentage / 2.5));
result.threatScore += additionalScore;
if (!result.threatType || additionalScore > 20) {
result.threatType = ThreatCategory.SUSPICIOUS_LINK;
result.threatDetails = `HTML contains ${suspiciousLinks} suspicious links out of ${links.length} total links`;
}
}
}
}
/**
* Scan an attachment for threats
* @param attachment The attachment to scan
* @param result The scan result to update
*/
private async scanAttachment(attachment: IAttachment, result: IScanResult): Promise<void> {
const filename = attachment.filename.toLowerCase();
result.scannedElements.push(`attachment:${filename}`);
// Skip large attachments if configured
if (attachment.content && attachment.content.length > this.options.maxAttachmentSizeToScan) {
logger.log('info', `Skipping scan of large attachment: ${filename} (${attachment.content.length} bytes)`);
return;
}
// Check filename for executable extensions
if (this.options.blockExecutables) {
for (const ext of ContentScanner.EXECUTABLE_EXTENSIONS) {
if (filename.endsWith(ext)) {
result.threatScore += 70; // High score for executable attachments
result.threatType = ThreatCategory.EXECUTABLE;
result.threatDetails = `Attachment has a potentially dangerous extension: ${filename}`;
return; // No need to scan contents if filename already flagged
}
}
}
// Check for Office documents with macros
if (this.options.blockMacros) {
for (const ext of ContentScanner.MACRO_DOCUMENT_EXTENSIONS) {
if (filename.endsWith(ext)) {
// For Office documents, check if they contain macros
// This is a simplified check - a real implementation would use specialized libraries
// to detect macros in Office documents
if (attachment.content && this.likelyContainsMacros(attachment)) {
result.threatScore += 60;
result.threatType = ThreatCategory.MALICIOUS_MACRO;
result.threatDetails = `Attachment appears to contain macros: ${filename}`;
return;
}
}
}
}
// Perform basic content analysis if we have content buffer
if (attachment.content) {
// Convert to string for scanning, with a limit to prevent memory issues
const textContent = this.extractTextFromBuffer(attachment.content);
if (textContent) {
// Scan for malicious patterns in attachment content
for (const category in ContentScanner.MALICIOUS_PATTERNS) {
const patterns = ContentScanner.MALICIOUS_PATTERNS[category];
for (const pattern of patterns) {
if (pattern.test(textContent)) {
result.threatScore += 30;
if (!result.threatType) {
result.threatType = this.mapCategoryToThreatType(category);
result.threatDetails = `Attachment content contains suspicious patterns: ${filename}`;
}
break;
}
}
}
}
// Check for PE headers (Windows executables)
if (attachment.content.length > 64 &&
attachment.content[0] === 0x4D &&
attachment.content[1] === 0x5A) { // 'MZ' header
result.threatScore += 80;
result.threatType = ThreatCategory.EXECUTABLE;
result.threatDetails = `Attachment contains executable code: ${filename}`;
}
}
}
/**
* Extract links from HTML content
* @param html HTML content
* @returns Array of extracted links
*/
private extractLinksFromHtml(html: string): string[] {
const links: string[] = [];
// Simple regex-based extraction - a real implementation might use a proper HTML parser
const matches = html.match(/href=["'](https?:\/\/[^"']+)["']/gi);
if (matches) {
for (const match of matches) {
const linkMatch = match.match(/href=["'](https?:\/\/[^"']+)["']/i);
if (linkMatch && linkMatch[1]) {
links.push(linkMatch[1]);
}
}
}
return links;
}
/**
* Extract plain text from HTML
* @param html HTML content
* @returns Extracted text
*/
private extractTextFromHtml(html: string): string {
// Remove HTML tags and decode entities - simplified version
return html
.replace(/<style[^>]*>.*?<\/style>/gs, '')
.replace(/<script[^>]*>.*?<\/script>/gs, '')
.replace(/<[^>]+>/g, ' ')
.replace(/&nbsp;/g, ' ')
.replace(/&lt;/g, '<')
.replace(/&gt;/g, '>')
.replace(/&amp;/g, '&')
.replace(/&quot;/g, '"')
.replace(/&apos;/g, "'")
.replace(/\s+/g, ' ')
.trim();
}
/**
* Extract text from a binary buffer for scanning
* @param buffer Binary content
* @returns Extracted text (may be partial)
*/
private extractTextFromBuffer(buffer: Buffer): string {
try {
// Limit the amount we convert to avoid memory issues
const sampleSize = Math.min(buffer.length, 100 * 1024); // 100KB max sample
const sample = buffer.slice(0, sampleSize);
// Try to convert to string, filtering out non-printable chars
return sample.toString('utf8')
.replace(/[\x00-\x09\x0B-\x1F\x7F-\x9F]/g, '') // Remove control chars
.replace(/\uFFFD/g, ''); // Remove replacement char
} catch (error) {
logger.log('warn', `Error extracting text from buffer: ${error.message}`);
return '';
}
}
/**
* Check if an Office document likely contains macros
* This is a simplified check - real implementation would use specialized libraries
* @param attachment The attachment to check
* @returns Whether the file likely contains macros
*/
private likelyContainsMacros(attachment: IAttachment): boolean {
// Simple heuristic: look for VBA/macro related strings
// This is a simplified approach and not comprehensive
const content = this.extractTextFromBuffer(attachment.content);
const macroIndicators = [
/vbaProject\.bin/i,
/Microsoft VBA/i,
/\bVBA\b/,
/Auto_Open/i,
/AutoExec/i,
/DocumentOpen/i,
/AutoOpen/i,
/\bExecute\(/i,
/\bShell\(/i,
/\bCreateObject\(/i
];
for (const indicator of macroIndicators) {
if (indicator.test(content)) {
return true;
}
}
return false;
}
/**
* Map a pattern category to a threat type
* @param category The pattern category
* @returns The corresponding threat type
*/
private mapCategoryToThreatType(category: string): string {
switch (category) {
case 'phishing': return ThreatCategory.PHISHING;
case 'spam': return ThreatCategory.SPAM;
case 'malware': return ThreatCategory.MALWARE;
case 'suspiciousLinks': return ThreatCategory.SUSPICIOUS_LINK;
case 'scriptInjection': return ThreatCategory.XSS;
case 'sensitiveData': return ThreatCategory.SENSITIVE_DATA;
default: return ThreatCategory.BLACKLISTED_CONTENT;
}
}
/**
* Log a high threat finding to the security logger
* @param email The email containing the threat
* @param result The scan result
*/
private logHighThreatFound(email: Email, result: IScanResult): void {
SecurityLogger.getInstance().logEvent({
level: SecurityLogLevel.ERROR,
type: SecurityEventType.MALWARE,
message: `High threat content detected in email from ${email.from} to ${email.to.join(', ')}`,
details: {
messageId: email.getMessageId(),
threatType: result.threatType,
threatDetails: result.threatDetails,
threatScore: result.threatScore,
scannedElements: result.scannedElements,
subject: email.subject
},
success: false,
domain: email.getFromDomain()
});
}
/**
* Log a threat finding to the security logger
* @param email The email containing the threat
* @param result The scan result
*/
private logThreatFound(email: Email, result: IScanResult): void {
SecurityLogger.getInstance().logEvent({
level: SecurityLogLevel.WARN,
type: SecurityEventType.SPAM,
message: `Suspicious content detected in email from ${email.from} to ${email.to.join(', ')}`,
details: {
messageId: email.getMessageId(),
threatType: result.threatType,
threatDetails: result.threatDetails,
threatScore: result.threatScore,
scannedElements: result.scannedElements,
subject: email.subject
},
success: false,
domain: email.getFromDomain()
});
}
/**
* Get threat level description based on score
* @param score Threat score
* @returns Threat level description
*/
public static getThreatLevel(score: number): 'none' | 'low' | 'medium' | 'high' {
if (score < 20) {
return 'none';
} else if (score < 40) {
return 'low';
} else if (score < 70) {
return 'medium';
} else {
return 'high';
}
}
}

513
ts/security/classes.ipreputationchecker.ts Normal file
View File

@ -0,0 +1,513 @@
import * as plugins from '../plugins.js';
import * as paths from '../paths.js';
import { logger } from '../logger.js';
import { SecurityLogger, SecurityLogLevel, SecurityEventType } from './classes.securitylogger.js';
import { LRUCache } from 'lru-cache';
/**
* Reputation check result information
*/
export interface IReputationResult {
score: number; // 0 (worst) to 100 (best)
isSpam: boolean; // true if the IP is known for spam
isProxy: boolean; // true if the IP is a known proxy
isTor: boolean; // true if the IP is a known Tor exit node
isVPN: boolean; // true if the IP is a known VPN
country?: string; // Country code (if available)
asn?: string; // Autonomous System Number (if available)
org?: string; // Organization name (if available)
blacklists?: string[]; // Names of blacklists that include this IP
timestamp: number; // When this result was created/retrieved
error?: string; // Error message if check failed
}
/**
* Reputation threshold scores
*/
export enum ReputationThreshold {
HIGH_RISK = 20, // Score below this is considered high risk
MEDIUM_RISK = 50, // Score below this is considered medium risk
LOW_RISK = 80 // Score below this is considered low risk (but not trusted)
}
/**
* IP type classifications
*/
export enum IPType {
RESIDENTIAL = 'residential',
DATACENTER = 'datacenter',
PROXY = 'proxy',
TOR = 'tor',
VPN = 'vpn',
UNKNOWN = 'unknown'
}
/**
* Options for the IP Reputation Checker
*/
export interface IIPReputationOptions {
maxCacheSize?: number; // Maximum number of IPs to cache (default: 10000)
cacheTTL?: number; // TTL for cache entries in ms (default: 24 hours)
dnsblServers?: string[]; // List of DNSBL servers to check
highRiskThreshold?: number; // Score below this is high risk
mediumRiskThreshold?: number; // Score below this is medium risk
lowRiskThreshold?: number; // Score below this is low risk
enableLocalCache?: boolean; // Whether to persist cache to disk (default: true)
enableDNSBL?: boolean; // Whether to use DNSBL checks (default: true)
enableIPInfo?: boolean; // Whether to use IP info service (default: true)
}
/**
* Class for checking IP reputation of inbound email senders
*/
export class IPReputationChecker {
private static instance: IPReputationChecker;
private reputationCache: LRUCache<string, IReputationResult>;
private options: Required<IIPReputationOptions>;
// Default DNSBL servers
private static readonly DEFAULT_DNSBL_SERVERS = [
'zen.spamhaus.org', // Spamhaus
'bl.spamcop.net', // SpamCop
'b.barracudacentral.org', // Barracuda
'spam.dnsbl.sorbs.net', // SORBS
'dnsbl.sorbs.net', // SORBS (expanded)
'cbl.abuseat.org', // Composite Blocking List
'xbl.spamhaus.org', // Spamhaus XBL
'pbl.spamhaus.org', // Spamhaus PBL
'dnsbl-1.uceprotect.net', // UCEPROTECT
'psbl.surriel.com' // PSBL
];
// Default options
private static readonly DEFAULT_OPTIONS: Required<IIPReputationOptions> = {
maxCacheSize: 10000,
cacheTTL: 24 * 60 * 60 * 1000, // 24 hours
dnsblServers: IPReputationChecker.DEFAULT_DNSBL_SERVERS,
highRiskThreshold: ReputationThreshold.HIGH_RISK,
mediumRiskThreshold: ReputationThreshold.MEDIUM_RISK,
lowRiskThreshold: ReputationThreshold.LOW_RISK,
enableLocalCache: true,
enableDNSBL: true,
enableIPInfo: true
};
/**
* Constructor for IPReputationChecker
* @param options Configuration options
*/
constructor(options: IIPReputationOptions = {}) {
// Merge with default options
this.options = {
...IPReputationChecker.DEFAULT_OPTIONS,
...options
};
// Initialize reputation cache
this.reputationCache = new LRUCache<string, IReputationResult>({
max: this.options.maxCacheSize,
ttl: this.options.cacheTTL, // Cache TTL
});
// Load cache from disk if enabled
if (this.options.enableLocalCache) {
this.loadCache();
}
}
/**
* Get the singleton instance of the checker
* @param options Configuration options
* @returns Singleton instance
*/
public static getInstance(options: IIPReputationOptions = {}): IPReputationChecker {
if (!IPReputationChecker.instance) {
IPReputationChecker.instance = new IPReputationChecker(options);
}
return IPReputationChecker.instance;
}
/**
* Check an IP address's reputation
* @param ip IP address to check
* @returns Reputation check result
*/
public async checkReputation(ip: string): Promise<IReputationResult> {
try {
// Validate IP address format
if (!this.isValidIPAddress(ip)) {
logger.log('warn', `Invalid IP address format: ${ip}`);
return this.createErrorResult(ip, 'Invalid IP address format');
}
// Check cache first
const cachedResult = this.reputationCache.get(ip);
if (cachedResult) {
logger.log('info', `Using cached reputation data for IP ${ip}`, {
score: cachedResult.score,
isSpam: cachedResult.isSpam
});
return cachedResult;
}
// Initialize empty result
const result: IReputationResult = {
score: 100, // Start with perfect score
isSpam: false,
isProxy: false,
isTor: false,
isVPN: false,
timestamp: Date.now()
};
// Check IP against DNS blacklists if enabled
if (this.options.enableDNSBL) {
const dnsblResult = await this.checkDNSBL(ip);
// Update result with DNSBL information
result.score -= dnsblResult.listCount * 10; // Subtract 10 points per blacklist
result.isSpam = dnsblResult.listCount > 0;
result.blacklists = dnsblResult.lists;
}
// Get additional IP information if enabled
if (this.options.enableIPInfo) {
const ipInfo = await this.getIPInfo(ip);
// Update result with IP info
result.country = ipInfo.country;
result.asn = ipInfo.asn;
result.org = ipInfo.org;
// Adjust score based on IP type
if (ipInfo.type === IPType.PROXY || ipInfo.type === IPType.TOR || ipInfo.type === IPType.VPN) {
result.score -= 30; // Subtract 30 points for proxies, Tor, VPNs
// Set proxy flags
result.isProxy = ipInfo.type === IPType.PROXY;
result.isTor = ipInfo.type === IPType.TOR;
result.isVPN = ipInfo.type === IPType.VPN;
}
}
// Ensure score is between 0 and 100
result.score = Math.max(0, Math.min(100, result.score));
// Update cache with result
this.reputationCache.set(ip, result);
// Save cache if enabled
if (this.options.enableLocalCache) {
this.saveCache();
}
// Log the reputation check
this.logReputationCheck(ip, result);
return result;
} catch (error) {
logger.log('error', `Error checking IP reputation for ${ip}: ${error.message}`, {
ip,
stack: error.stack
});
return this.createErrorResult(ip, error.message);
}
}
/**
* Check an IP against DNS blacklists
* @param ip IP address to check
* @returns DNSBL check results
*/
private async checkDNSBL(ip: string): Promise<{
listCount: number;
lists: string[];
}> {
try {
// Reverse the IP for DNSBL queries
const reversedIP = this.reverseIP(ip);
const results = await Promise.allSettled(
this.options.dnsblServers.map(async (server) => {
try {
const lookupDomain = `${reversedIP}.${server}`;
await plugins.dns.promises.resolve(lookupDomain);
return server; // IP is listed in this DNSBL
} catch (error) {
if (error.code === 'ENOTFOUND') {
return null; // IP is not listed in this DNSBL
}
throw error; // Other error
}
})
);
// Extract successful lookups (listed in DNSBL)
const lists = results
.filter((result): result is PromiseFulfilledResult<string> =>
result.status === 'fulfilled' && result.value !== null
)
.map(result => result.value);
return {
listCount: lists.length,
lists
};
} catch (error) {
logger.log('error', `Error checking DNSBL for ${ip}: ${error.message}`);
return {
listCount: 0,
lists: []
};
}
}
/**
* Get information about an IP address
* @param ip IP address to check
* @returns IP information
*/
private async getIPInfo(ip: string): Promise<{
country?: string;
asn?: string;
org?: string;
type: IPType;
}> {
try {
// In a real implementation, this would use an IP data service API
// For this implementation, we'll use a simplified approach
// Check if it's a known Tor exit node (simplified)
const isTor = ip.startsWith('171.25.') || ip.startsWith('185.220.') || ip.startsWith('95.216.');
// Check if it's a known VPN (simplified)
const isVPN = ip.startsWith('185.156.') || ip.startsWith('37.120.');
// Check if it's a known proxy (simplified)
const isProxy = ip.startsWith('34.92.') || ip.startsWith('34.206.');
// Determine IP type
let type = IPType.UNKNOWN;
if (isTor) {
type = IPType.TOR;
} else if (isVPN) {
type = IPType.VPN;
} else if (isProxy) {
type = IPType.PROXY;
} else {
// Simple datacenters detection (major cloud providers)
if (
ip.startsWith('13.') || // AWS
ip.startsWith('35.') || // Google Cloud
ip.startsWith('52.') || // AWS
ip.startsWith('34.') || // Google Cloud
ip.startsWith('104.') // Various providers
) {
type = IPType.DATACENTER;
} else {
type = IPType.RESIDENTIAL;
}
}
// Return the information
return {
country: this.determineCountry(ip), // Simplified, would use geolocation service
asn: 'AS12345', // Simplified, would look up real ASN
org: this.determineOrg(ip), // Simplified, would use real org data
type
};
} catch (error) {
logger.log('error', `Error getting IP info for ${ip}: ${error.message}`);
return {
type: IPType.UNKNOWN
};
}
}
/**
* Simplified method to determine country from IP
* In a real implementation, this would use a geolocation database or service
* @param ip IP address
* @returns Country code
*/
private determineCountry(ip: string): string {
// Simplified mapping for demo purposes
if (ip.startsWith('13.') || ip.startsWith('52.')) return 'US';
if (ip.startsWith('35.') || ip.startsWith('34.')) return 'US';
if (ip.startsWith('185.')) return 'NL';
if (ip.startsWith('171.')) return 'DE';
return 'XX'; // Unknown
}
/**
* Simplified method to determine organization from IP
* In a real implementation, this would use an IP-to-org database or service
* @param ip IP address
* @returns Organization name
*/
private determineOrg(ip: string): string {
// Simplified mapping for demo purposes
if (ip.startsWith('13.') || ip.startsWith('52.')) return 'Amazon AWS';
if (ip.startsWith('35.') || ip.startsWith('34.')) return 'Google Cloud';
if (ip.startsWith('185.156.')) return 'NordVPN';
if (ip.startsWith('37.120.')) return 'ExpressVPN';
if (ip.startsWith('185.220.')) return 'Tor Exit Node';
return 'Unknown';
}
/**
* Reverse an IP address for DNSBL lookups (e.g., 1.2.3.4 -> 4.3.2.1)
* @param ip IP address to reverse
* @returns Reversed IP for DNSBL queries
*/
private reverseIP(ip: string): string {
return ip.split('.').reverse().join('.');
}
/**
* Create an error result for when reputation check fails
* @param ip IP address
* @param errorMessage Error message
* @returns Error result
*/
private createErrorResult(ip: string, errorMessage: string): IReputationResult {
return {
score: 50, // Neutral score for errors
isSpam: false,
isProxy: false,
isTor: false,
isVPN: false,
timestamp: Date.now(),
error: errorMessage
};
}
/**
* Validate IP address format
* @param ip IP address to validate
* @returns Whether the IP is valid
*/
private isValidIPAddress(ip: string): boolean {
// IPv4 regex pattern
const ipv4Pattern = /^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/;
return ipv4Pattern.test(ip);
}
/**
* Log reputation check to security logger
* @param ip IP address
* @param result Reputation result
*/
private logReputationCheck(ip: string, result: IReputationResult): void {
// Determine log level based on reputation score
let logLevel = SecurityLogLevel.INFO;
if (result.score < this.options.highRiskThreshold) {
logLevel = SecurityLogLevel.WARN;
} else if (result.score < this.options.mediumRiskThreshold) {
logLevel = SecurityLogLevel.INFO;
}
// Log the check
SecurityLogger.getInstance().logEvent({
level: logLevel,
type: SecurityEventType.IP_REPUTATION,
message: `IP reputation check ${result.isSpam ? 'flagged spam' : 'completed'} for ${ip}`,
ipAddress: ip,
details: {
score: result.score,
isSpam: result.isSpam,
isProxy: result.isProxy,
isTor: result.isTor,
isVPN: result.isVPN,
country: result.country,
blacklists: result.blacklists
},
success: !result.isSpam
});
}
/**
* Save cache to disk
*/
private saveCache(): void {
try {
// Convert cache entries to serializable array
const entries = Array.from(this.reputationCache.entries()).map(([ip, data]) => ({
ip,
data
}));
// Only save if we have entries
if (entries.length === 0) {
return;
}
// Ensure directory exists
const cacheDir = plugins.path.join(paths.dataDir, 'security');
plugins.smartfile.fs.ensureDirSync(cacheDir);
// Save to file
const cacheFile = plugins.path.join(cacheDir, 'ip_reputation_cache.json');
plugins.smartfile.memory.toFsSync(
JSON.stringify(entries),
cacheFile
);
logger.log('info', `Saved ${entries.length} IP reputation cache entries to disk`);
} catch (error) {
logger.log('error', `Failed to save IP reputation cache: ${error.message}`);
}
}
/**
* Load cache from disk
*/
private loadCache(): void {
try {
const cacheFile = plugins.path.join(paths.dataDir, 'security', 'ip_reputation_cache.json');
// Check if file exists
if (!plugins.fs.existsSync(cacheFile)) {
return;
}
// Read and parse cache
const cacheData = plugins.fs.readFileSync(cacheFile, 'utf8');
const entries = JSON.parse(cacheData);
// Validate and filter entries
const now = Date.now();
const validEntries = entries.filter(entry => {
const age = now - entry.data.timestamp;
return age < this.options.cacheTTL; // Only load entries that haven't expired
});
// Restore cache
for (const entry of validEntries) {
this.reputationCache.set(entry.ip, entry.data);
}
logger.log('info', `Loaded ${validEntries.length} IP reputation cache entries from disk`);
} catch (error) {
logger.log('error', `Failed to load IP reputation cache: ${error.message}`);
}
}
/**
* Get the risk level for a reputation score
* @param score Reputation score (0-100)
* @returns Risk level description
*/
public static getRiskLevel(score: number): 'high' | 'medium' | 'low' | 'trusted' {
if (score < ReputationThreshold.HIGH_RISK) {
return 'high';
} else if (score < ReputationThreshold.MEDIUM_RISK) {
return 'medium';
} else if (score < ReputationThreshold.LOW_RISK) {
return 'low';
} else {
return 'trusted';
}
}
}

294
ts/security/classes.securitylogger.ts Normal file
View File

@ -0,0 +1,294 @@
import * as plugins from '../plugins.js';
import { logger } from '../logger.js';
/**
* Log level for security events
*/
export enum SecurityLogLevel {
INFO = 'info',
WARN = 'warn',
ERROR = 'error',
CRITICAL = 'critical'
}
/**
* Security event types for categorization
*/
export enum SecurityEventType {
AUTHENTICATION = 'authentication',
ACCESS_CONTROL = 'access_control',
EMAIL_VALIDATION = 'email_validation',
DKIM = 'dkim',
SPF = 'spf',
DMARC = 'dmarc',
RATE_LIMIT = 'rate_limit',
SPAM = 'spam',
MALWARE = 'malware',
CONNECTION = 'connection',
DATA_EXPOSURE = 'data_exposure',
CONFIGURATION = 'configuration',
IP_REPUTATION = 'ip_reputation'
}
/**
* Security event interface
*/
export interface ISecurityEvent {
timestamp: number;
level: SecurityLogLevel;
type: SecurityEventType;
message: string;
details?: any;
ipAddress?: string;
userId?: string;
sessionId?: string;
emailId?: string;
domain?: string;
action?: string;
result?: string;
success?: boolean;
}
/**
* Security logger for enhanced security monitoring
*/
export class SecurityLogger {
private static instance: SecurityLogger;
private securityEvents: ISecurityEvent[] = [];
private maxEventHistory: number;
private enableNotifications: boolean;
private constructor(options?: {
maxEventHistory?: number;
enableNotifications?: boolean;
}) {
this.maxEventHistory = options?.maxEventHistory || 1000;
this.enableNotifications = options?.enableNotifications || false;
}
/**
* Get singleton instance
*/
public static getInstance(options?: {
maxEventHistory?: number;
enableNotifications?: boolean;
}): SecurityLogger {
if (!SecurityLogger.instance) {
SecurityLogger.instance = new SecurityLogger(options);
}
return SecurityLogger.instance;
}
/**
* Log a security event
* @param event The security event to log
*/
public logEvent(event: Omit<ISecurityEvent, 'timestamp'>): void {
const fullEvent: ISecurityEvent = {
...event,
timestamp: Date.now()
};
// Store in memory buffer
this.securityEvents.push(fullEvent);
// Trim history if needed
if (this.securityEvents.length > this.maxEventHistory) {
this.securityEvents.shift();
}
// Log to regular logger with appropriate level
switch (event.level) {
case SecurityLogLevel.INFO:
logger.log('info', `[SECURITY:${event.type}] ${event.message}`, event.details);
break;
case SecurityLogLevel.WARN:
logger.log('warn', `[SECURITY:${event.type}] ${event.message}`, event.details);
break;
case SecurityLogLevel.ERROR:
case SecurityLogLevel.CRITICAL:
logger.log('error', `[SECURITY:${event.type}] ${event.message}`, event.details);
// Send notification for critical events if enabled
if (event.level === SecurityLogLevel.CRITICAL && this.enableNotifications) {
this.sendNotification(fullEvent);
}
break;
}
}
/**
* Get recent security events
* @param limit Maximum number of events to return
* @param filter Filter for specific event types
* @returns Recent security events
*/
public getRecentEvents(limit: number = 100, filter?: {
level?: SecurityLogLevel;
type?: SecurityEventType;
fromTimestamp?: number;
toTimestamp?: number;
}): ISecurityEvent[] {
let filteredEvents = this.securityEvents;
// Apply filters
if (filter) {
if (filter.level) {
filteredEvents = filteredEvents.filter(event => event.level === filter.level);
}
if (filter.type) {
filteredEvents = filteredEvents.filter(event => event.type === filter.type);
}
if (filter.fromTimestamp) {
filteredEvents = filteredEvents.filter(event => event.timestamp >= filter.fromTimestamp);
}
if (filter.toTimestamp) {
filteredEvents = filteredEvents.filter(event => event.timestamp <= filter.toTimestamp);
}
}
// Return most recent events up to limit
return filteredEvents
.sort((a, b) => b.timestamp - a.timestamp)
.slice(0, limit);
}
/**
* Get events by security level
* @param level The security level to filter by
* @param limit Maximum number of events to return
* @returns Security events matching the level
*/
public getEventsByLevel(level: SecurityLogLevel, limit: number = 100): ISecurityEvent[] {
return this.getRecentEvents(limit, { level });
}
/**
* Get events by security type
* @param type The event type to filter by
* @param limit Maximum number of events to return
* @returns Security events matching the type
*/
public getEventsByType(type: SecurityEventType, limit: number = 100): ISecurityEvent[] {
return this.getRecentEvents(limit, { type });
}
/**
* Get security events for a specific IP address
* @param ipAddress The IP address to filter by
* @param limit Maximum number of events to return
* @returns Security events for the IP address
*/
public getEventsByIP(ipAddress: string, limit: number = 100): ISecurityEvent[] {
return this.securityEvents
.filter(event => event.ipAddress === ipAddress)
.sort((a, b) => b.timestamp - a.timestamp)
.slice(0, limit);
}
/**
* Get security events for a specific domain
* @param domain The domain to filter by
* @param limit Maximum number of events to return
* @returns Security events for the domain
*/
public getEventsByDomain(domain: string, limit: number = 100): ISecurityEvent[] {
return this.securityEvents
.filter(event => event.domain === domain)
.sort((a, b) => b.timestamp - a.timestamp)
.slice(0, limit);
}
/**
* Send a notification for critical security events
* @param event The security event to notify about
* @private
*/
private sendNotification(event: ISecurityEvent): void {
// In a production environment, this would integrate with a notification service
// For now, we'll just log that we would send a notification
logger.log('error', `[SECURITY NOTIFICATION] ${event.message}`, {
...event,
notificationSent: true
});
// Future integration with alerting systems would go here
}
/**
* Clear event history
*/
public clearEvents(): void {
this.securityEvents = [];
}
/**
* Get statistical summary of security events
* @param timeWindow Optional time window in milliseconds
* @returns Summary of security events
*/
public getEventsSummary(timeWindow?: number): {
total: number;
byLevel: Record<SecurityLogLevel, number>;
byType: Record<SecurityEventType, number>;
topIPs: Array<{ ip: string; count: number }>;
topDomains: Array<{ domain: string; count: number }>;
} {
// Filter by time window if provided
let events = this.securityEvents;
if (timeWindow) {
const cutoff = Date.now() - timeWindow;
events = events.filter(e => e.timestamp >= cutoff);
}
// Count by level
const byLevel = Object.values(SecurityLogLevel).reduce((acc, level) => {
acc[level] = events.filter(e => e.level === level).length;
return acc;
}, {} as Record<SecurityLogLevel, number>);
// Count by type
const byType = Object.values(SecurityEventType).reduce((acc, type) => {
acc[type] = events.filter(e => e.type === type).length;
return acc;
}, {} as Record<SecurityEventType, number>);
// Count by IP
const ipCounts = new Map<string, number>();
events.forEach(e => {
if (e.ipAddress) {
ipCounts.set(e.ipAddress, (ipCounts.get(e.ipAddress) || 0) + 1);
}
});
// Count by domain
const domainCounts = new Map<string, number>();
events.forEach(e => {
if (e.domain) {
domainCounts.set(e.domain, (domainCounts.get(e.domain) || 0) + 1);
}
});
// Sort and limit top entries
const topIPs = Array.from(ipCounts.entries())
.map(([ip, count]) => ({ ip, count }))
.sort((a, b) => b.count - a.count)
.slice(0, 10);
const topDomains = Array.from(domainCounts.entries())
.map(([domain, count]) => ({ domain, count }))
.sort((a, b) => b.count - a.count)
.slice(0, 10);
return {
total: events.length,
byLevel,
byType,
topIPs,
topDomains
};
}
}

21
ts/security/index.ts Normal file
View File

@ -0,0 +1,21 @@
export {
SecurityLogger,
SecurityLogLevel,
SecurityEventType,
type ISecurityEvent
} from './classes.securitylogger.js';
export {
IPReputationChecker,
ReputationThreshold,
IPType,
type IReputationResult,
type IIPReputationOptions
} from './classes.ipreputationchecker.js';
export {
ContentScanner,
ThreatCategory,
type IScanResult,
type IContentScannerOptions
} from './classes.contentscanner.js';