update

.gitignore

@@ -17,4 +17,5 @@ node_modules/
 dist/
 dist_*/
 
-# custom
+# custom
+**/.claude/settings.local.json

package.json

@@ -20,7 +20,8 @@
     "@git.zone/tsrun": "^1.2.8",
     "@git.zone/tstest": "^1.0.88",
     "@git.zone/tswatch": "^2.0.1",
-    "@push.rocks/tapbundle": "^6.0.3"
+    "@push.rocks/tapbundle": "^6.0.3",
+    "@types/node": "^22.15.14"
   },
   "dependencies": {
     "@api.global/typedrequest": "^3.0.19",
@@ -29,22 +30,23 @@
     "@apiclient.xyz/cloudflare": "^6.4.1",
     "@apiclient.xyz/letterxpress": "^1.0.22",
     "@push.rocks/projectinfo": "^5.0.1",
-    "@push.rocks/qenv": "^6.0.5",
+    "@push.rocks/qenv": "^6.1.0",
+    "@push.rocks/smartacme": "^7.3.3",
     "@push.rocks/smartdata": "^5.15.1",
     "@push.rocks/smartdns": "^6.2.2",
     "@push.rocks/smartfile": "^11.0.4",
     "@push.rocks/smartlog": "^3.0.3",
-    "@push.rocks/smartmail": "^1.0.24",
+    "@push.rocks/smartmail": "^2.0.1",
     "@push.rocks/smartpath": "^5.0.5",
     "@push.rocks/smartpromise": "^4.0.3",
-    "@push.rocks/smartproxy": "^10.0.2",
+    "@push.rocks/smartproxy": "^10.2.0",
     "@push.rocks/smartrequest": "^2.1.0",
     "@push.rocks/smartrule": "^2.0.1",
     "@push.rocks/smartrx": "^3.0.10",
     "@push.rocks/smartstate": "^2.0.0",
     "@serve.zone/interfaces": "^5.0.4",
-    "@tsclass/tsclass": "^9.1.0",
+    "@tsclass/tsclass": "^9.2.0",
-    "@types/mailparser": "^3.4.5",
+    "@types/mailparser": "^3.4.6",
     "mailauth": "^4.8.4",
     "mailparser": "^3.6.9",
     "uuid": "^11.1.0"

readme.plan.md

@@ -1,91 +1,59 @@
- # DcRouter Implementation Plan
+# Plan for Improving Smartmail Integration
 
- This document outlines the plan for developing **DcRouter**, a central routing
+## Current State Analysis
- component for the platform service. DcRouter will provide:
-   - HTTP/HTTPS reverse proxy with TLS (ACME) support
-   - TCP/SNI-based proxying
-   - Mail Transfer Agent (MTA) for inbound/outbound SMTP
-   - Rule-based routing for web and mail traffic
-   - DNS-based dynamic routing
-   - Configuration, persistence, logging, and monitoring
 
- ## 1. Read and Summarize Dependencies
+The platformservice currently uses @push.rocks/smartmail version 2.0.1, primarily for email handling in:
- We will leverage existing modules; key points from their READMEs:
+- Email service via connector.mta.ts
- - **@push.rocks/smartproxy**: high-performance HTTP/HTTPS reverse proxy,
+- Template management 
-   TLS termination, ACME HTTP-01, WebSockets, low-level port forwarding (nftables),
+- API endpoints for sending emails
-   TCP/SNI proxy with dynamic routing and IP filtering.
- - **@push.rocks/smartmail**: unified mail data structures and utilities.
- - **mailparser**: parse raw SMTP messages into structured objects.
- - **mailauth**: SPF/DKIM/DMARC validation and signing (DKIM).
- - **@api.global/typedserver** / **typedsocket**: typed HTTP and TCP server abstractions.
- - **@push.rocks/smartrule**: rule engine for dynamic decision-making on traffic.
- - **@push.rocks/smartdns**: DNS resolution and caching for routing decisions.
- - **@push.rocks/smartdata**: persistent key/value or document store for configs.
- - **@push.rocks/smartlog** and **@push.rocks/smartrx**: logging and metrics/event tracking.
- - **@push.rocks/qenv**: environment-based configuration loading.
- - **ts/mta**: built-in MTA implementation (MtaService, SMTPServer, EmailSendJob, DKIMCreator/Verifier, DNSManager, ApiManager, Email class).
 
- ## 2. High-Level Architecture
+## Identified Integration Improvements
- DcRouter will instantiate and coordinate:
- 1. **HTTP(S) Proxy Layer**
-    - Use `NetworkProxy` and `Port80Handler` from smartproxy
-    - Manage virtual hosts, certificates, redirects
- 2. **TCP/SNI Proxy Layer (SmartProxy)**
-    - Use `SmartProxy` from `@push.rocks/smartproxy` to:
-      - Forward TCP connections by port or port ranges to backends (nftables-like port forwarding).
-      - Route TLS connections based on SNI to different target IPs/ports (SNI bridge).
-      - Integrate HTTP reverse proxy on select ports via `NetworkProxyBridge`.
-      - Manage ACME certificates with built-in `Port80Handler` and `CertProvisioner`.
-      - Apply IP filtering, connection rate limiting, timeouts, and detailed logging.
- 3. **SMTP/MTA Layer**
-    - Leverage `MtaService` from `ts/mta`, coordinating subcomponents:
-      - `SMTPServer` (classes.smtpserver): SMTP listener on ports 25/587 with STARTTLS/TLS
-      - `Email` and `EmailSendJob`: queue-based outbound sending with retries and rate limiting
-      - `DKIMCreator` / `DKIMVerifier`: automatic key management, signing, and verification
-      - `DNSManager`: generate and update DNS records for DKIM/SPF/MTA
-      - `ApiManager`: certificate provisioning and renewal via typedrequest/typedsocket
-    - Integrate inbound processing (mailparser, SPF/DKIM validation) and routing rules
- 4. **Rule Engine**
-    - Use smartrule to define routing rules for HTTP domains and mail addresses
-    - Support wildcard matching, priority, and dynamic updates
- 5. **DNS Integration**
-    - Use smartdns to resolve backend targets by name, with caching
- 6. **Persistence & Config**
-    - Load environment config via qenv
-    - Store dynamic state (rules, domains, MTA settings) in smartdata
- 7. **Logging & Metrics**
-    - Instrument all layers with smartlog and smartrx
- 8. **Management API / CLI**
-    - Expose REST/CLI commands via typedserver for: add/remove domains, rules,
-      view status, reload config, etc.
 
- ## 3. Feature Breakdown and Milestones
+### 1. Update smartmail to Latest Version
- - [ ] A. Core Router scaffolding (`DcRouter` class)
- - [ ] B. HTTP/HTTPS proxy integration
- - [ ] C. SmartProxy Integration
-   - [ ] C1. Define `IPortProxySettings` with `fromPort`, `toPort`, `domainConfigs`, and global port ranges
-   - [ ] C2. Configure `domainConfigs` to mix port forwarding, SNI routing, and `useNetworkProxy` flags
-   - [ ] C3. Initialize and start `SmartProxy`, including `Port80Handler` for ACME and `CertProvisioner`
-   - [ ] C4. Wire SmartProxy events (e.g., certificate issuance, connection logging) into DcRouter
-   - [ ] C5. Support runtime updates to `domainConfigs` for dynamic routing changes
- - [ ] D. MTA Integration (ts/mta)
-  - [ ] D1. Scaffold integration of `MtaService` into `DcRouter`
-  - [ ] D2. Configure and start `SMTPServer` with TLS/STARTTLS on ports 25/587
-  - [ ] D3. Wire inbound email handling via `processIncomingEmail` and rule engine
-  - [ ] D4. Implement outbound queue processing (`EmailSendJob`), retries, rate limiting
-  - [ ] D5. Manage DKIM keys and DNS records (`DKIMCreator`, `DNSManager`)
-  - [ ] D6. Provision and auto-renew certificates (`ApiManager`)
- - [ ] E. Rule engine wiring for HTTP and SMTP
- - [ ] F. DNS-based dynamic resolution
- - [ ] G. Persistence layer for configs and state
- - [ ] H. Logging/metrics instrumentation
- - [ ] I. CLI and REST API endpoints
- - [ ] J. Automated tests for each layer
- - [ ] K. Documentation and examples in README
 
- ## 4. Next Steps
+- [x] Update package.json to use the latest smartmail version
- 1. Review and refine this plan.
+- [ ] Handle any breaking API changes during the upgrade
- 2. Begin implementation starting with the core scaffolding (A).
- 3. Iterate through features in milestone order.
 
- _Last updated: 2025-05-04_
+### 2. Email Validation Improvements
+
+- [ ] Implement advanced email validation using EmailAddressValidator 
+- [ ] Add MX record checking for outbound emails
+- [ ] Integrate domain reputation checking for spam prevention
+
+### 3. Template System Enhancement
+
+- [ ] Refactor TemplateManager to leverage smartmail's templating capabilities fully
+- [ ] Add support for typed email templates with proper interfaces
+- [ ] Create a comprehensive template catalog with standardized formats
+
+### 4. MIME Handling Improvements
+
+- [ ] Utilize smartmail's MIME conversion capabilities in the MTA connector
+- [ ] Streamline attachment handling between smartmail and internal Email class
+- [ ] Handle content encoding more efficiently
+
+### 5. Integration with MTA Service
+
+- [ ] Refactor Email class to extend or delegate to smartmail.Smartmail
+- [ ] Align the interfaces between internal Email and Smartmail classes
+- [ ] Simplify conversion between MTA Email format and Smartmail
+
+### 6. Enhanced Email Processing
+
+- [ ] Add proper DKIM signature verification using smartmail capabilities
+- [ ] Implement proper email tagging and categorization
+- [ ] Better handling of incoming emails with smart parsing
+
+### 7. Testing & Documentation
+
+- [ ] Create comprehensive tests for the smartmail integration
+- [ ] Document the integration patterns for future reference
+- [ ] Create example implementations for common use cases
+
+## Implementation Strategy
+
+1. First focus on non-breaking improvements (validation, template enhancement)
+2. Then tackle the deeper integration with MTA service
+3. Finally implement advanced features (MIME handling, DKIM)
+
+Each change should be made incrementally with thorough testing to ensure no disruption to the existing email functionality.

ts/dcrouter/classes.dcr.sz.connector.ts

@@ -0,0 +1,15 @@
+import * as plugins from '../plugins.js';
+import type DcRouter from './classes.dcrouter.js';
+
+export class SzDcRouterConnector {
+  public qenv: plugins.qenv.Qenv;
+  public dcRouterRef: DcRouter;
+  constructor(dcRouterRef: DcRouter) {
+    this.dcRouterRef = dcRouterRef;
+    this.dcRouterRef.options.platformServiceInstance?.serviceQenv || new plugins.qenv.Qenv('./', '.nogit/');
+  }
+
+  public async getEnvVarOnDemand(varName: string): Promise<string> {
+    return this.qenv.getEnvVarOnDemand(varName) || '';
+  }
+}

ts/dcrouter/classes.dcrouter.ts

@@ -1,48 +1,136 @@
 import * as plugins from '../plugins.js';
 import * as paths from '../paths.js';
+import { SzDcRouterConnector } from './classes.dcr.sz.connector.js';
 
+import type { SzPlatformService } from '../platformservice.js';
 import { type IMtaConfig, MtaService } from '../mta/classes.mta.js';
 
+// Types are referenced via plugins.smartproxy.*
+
 export interface IDcRouterOptions {
-  portProxyConfig?: plugins.smartproxy.IPortProxySettings;
+  platformServiceInstance?: SzPlatformService;
+
+  /** SmartProxy (TCP/SNI) configuration */
+  smartProxyOptions?: plugins.smartproxy.ISmartProxyOptions;
+  /** Reverse proxy host configurations for HTTP(S) layer */
+  reverseProxyConfigs?: plugins.smartproxy.IReverseProxyConfig[];
+  /** MTA (SMTP) service configuration */
   mtaConfig?: IMtaConfig;
+  /** DNS server configuration */
   dnsServerConfig?: plugins.smartdns.IDnsServerOptions;
 }
 
 /**
  * DcRouter can be run on ingress and egress to and from a datacenter site.
  */
+/**
+ * Context passed to HTTP routing rules
+ */
+/**
+ * Context passed to port proxy (SmartProxy) routing rules
+ */
+export interface PortProxyRuleContext {
+  proxy: plugins.smartproxy.SmartProxy;
+  configs: plugins.smartproxy.IPortProxySettings['domainConfigs'];
+}
 export class DcRouter {
-  private options: IDcRouterOptions;
+  public szDcRouterConnector = new SzDcRouterConnector(this);
-  public smartProxy: plugins.smartproxy.SmartProxy;
+  public options: IDcRouterOptions;
-  public mta: MtaService;
+  public smartProxy?: plugins.smartproxy.SmartProxy;
-  public dnsServer: plugins.smartdns.DnsServer;
+  public mta?: MtaService;
+  public dnsServer?: plugins.smartdns.DnsServer;
+  /** SMTP rule engine */
+  public smtpRuleEngine?: plugins.smartrule.SmartRule<any>;
   constructor(optionsArg: IDcRouterOptions) {
     this.options = optionsArg;
-    if (this.options.portProxyConfig) {
-      this.smartProxy = new plugins.smartproxy.SmartProxy(this.options.portProxyConfig);
-    }
-    if (this.options.mtaConfig) {
-      this.mta = new MtaService(null, this.options.mtaConfig);
-    }
   }
 
   public async start() {
+
+    // TCP/SNI proxy (SmartProxy)
+    if (this.options.smartProxyOptions) {
+      // Lets setup smartacme
+      let certProvisionFunction: plugins.smartproxy.ISmartProxyOptions['certProvisionFunction'];
+      if (true) {
+        const smartAcmeInstance = new plugins.smartacme.SmartAcme({
+          accountEmail: this.options.smartProxyOptions.acme.accountEmail,
+          certManager: new plugins.smartacme.certmanagers.MongoCertManager({
+            mongoDbUrl: await this.szDcRouterConnector.getEnvVarOnDemand('MONGO_DB_URL'),
+            mongoDbUser: await this.szDcRouterConnector.getEnvVarOnDemand('MONGO_DB_USER'),
+            mongoDbPass: await this.szDcRouterConnector.getEnvVarOnDemand('MONGO_DB_PASS'),
+            mongoDbName: await this.szDcRouterConnector.getEnvVarOnDemand('MONGO_DB_NAME'),
+          }),
+          environment: 'production',
+          accountPrivateKey: await this.szDcRouterConnector.getEnvVarOnDemand('ACME_ACCOUNT_PRIVATE_KEY'),
+          challengeHandlers: [
+            new plugins.smartacme.handlers.Dns01Handler(new plugins.cloudflare.CloudflareAccount('')) // TODO
+          ],
+
+        });
+        certProvisionFunction = async (domainArg) => {
+          const domainSupported = await smartAcmeInstance.challengeHandlers[0].checkWetherDomainIsSupported(domainArg);
+          if (!domainSupported) {
+            return 'http01';
+          }
+          return smartAcmeInstance.getCertificateForDomain(domainArg);
+        };
+      }
+
+      this.smartProxy = new plugins.smartproxy.SmartProxy(this.options.smartProxyOptions);
+      // Initialize SMTP rule engine from MTA service if available
+      if (this.mta) {
+        this.smtpRuleEngine = this.mta.smtpRuleEngine;
+      }
+    }
+    // MTA service
+    if (this.options.mtaConfig) {
+      this.mta = new MtaService(null, this.options.mtaConfig);
+    }
+    // DNS server
+    if (this.options.dnsServerConfig) {
+      this.dnsServer = new plugins.smartdns.DnsServer(this.options.dnsServerConfig);
+    }
+
+
+
+    // Start SmartProxy if configured
     if (this.smartProxy) {
       await this.smartProxy.start();
     }
+    // Start MTA service if configured
     if (this.mta) {
       await this.mta.start();
     }
+    // Start DNS server if configured
+    if (this.dnsServer) {
+      await this.dnsServer.start();
+    }
   }
 
   public async stop() {
+    // Stop SmartProxy
     if (this.smartProxy) {
       await this.smartProxy.stop();
     }
+    // Stop MTA service
     if (this.mta) {
       await this.mta.stop();
     }
+    // Stop DNS server
+    if (this.dnsServer) {
+      await this.dnsServer.stop();
+    }
+  }
+
+  /**
+   * Register an SMTP routing rule
+   */
+  public addSmtpRule(
+    priority: number,
+    check: (email: any) => Promise<any>,
+    action: (email: any) => Promise<any>
+  ): void {
+    this.smtpRuleEngine?.createRule(priority, check, action);
   }
 }
 

ts/dcrouter/index.ts

@@ -0,0 +1 @@
+export * from './classes.dcrouter.js';

ts/email/classes.apimanager.ts

@@ -19,7 +19,7 @@ export class ApiManager {
    */
   private registerApiEndpoints() {
     // Register the SendEmail endpoint
-    this.typedRouter.addTypedHandler<plugins.servezoneInterfaces.platformservice.mta.IRequest_SendEmail>(
+    this.typedRouter.addTypedHandler<plugins.servezoneInterfaces.platformservice.mta.IReq_SendEmail>(
       new plugins.typedrequest.TypedHandler('sendEmail', async (requestData) => {
         const mailToSend = new plugins.smartmail.Smartmail({
           body: requestData.body,

ts/email/classes.connector.mta.ts

@@ -27,7 +27,7 @@ export class MtaConnector {
    * @param options Additional options
    */
   public async sendEmail(
-    smartmail: plugins.smartmail.Smartmail<>,
+    smartmail: plugins.smartmail.Smartmail<any>, // TODO: look at type
     toAddresses: string | string[],
     options: any = {}
   ): Promise<string> {
@@ -82,7 +82,7 @@ export class MtaConnector {
    * For MTA, this would handle an email already received by the SMTP server
    * @param emailData The raw email data or identifier
    */
-  public async receiveEmail(emailData: string): Promise<plugins.smartmail.Smartmail<>> {
+  public async receiveEmail(emailData: string): Promise<plugins.smartmail.Smartmail<any>> {
     try {
       // In a real implementation, this would retrieve an email from the MTA storage
       // For now, we can use a simplified approach:
@@ -97,7 +97,7 @@ export class MtaConnector {
         body: parsedEmail.html || parsedEmail.text || '',
         creationObjectRef: {
           From: parsedEmail.from?.text || '',
-          To: parsedEmail.to?.text || '',
+          To: parsedEmail.to,
           Subject: parsedEmail.subject || ''
         }
       });

ts/mta/classes.mta.ts

@@ -168,6 +168,9 @@ export class MtaService {
   
   /** Whether the service is currently running */
   private running = false;
+  
+  /** SMTP rule engine for incoming emails */
+  public smtpRuleEngine: plugins.smartrule.SmartRule<Email>;
 
   /**
    * Initialize the MTA service
@@ -188,6 +191,8 @@ export class MtaService {
     this.dkimVerifier = new DKIMVerifier(this);
     this.dnsManager = new DNSManager(this);
     this.apiManager = new ApiManager();
+    // Initialize SMTP rule engine
+    this.smtpRuleEngine = new plugins.smartrule.SmartRule<Email>();
     
     // Initialize stats
     this.stats = {
@@ -408,6 +413,12 @@ export class MtaService {
       throw new Error('MTA service is not running');
     }
     
+    // Apply SMTP rule engine decisions
+    try {
+      await this.smtpRuleEngine.makeDecision(email);
+    } catch (err) {
+      console.error('Error executing SMTP rules:', err);
+    }
     try {
       console.log(`Processing incoming email from ${email.from} to ${email.to}`);
       

ts/mta/classes.smtpserver.ts

@@ -301,7 +301,7 @@ export class SMTPServer {
     this.sessions.delete(socket);
   }
 
-  private processEmailData(socket: plugins.net.Socket | plugins.tls.TLSSocket, data: string): void {
+  private async processEmailData(socket: plugins.net.Socket | plugins.tls.TLSSocket, data: string): Promise<void> {
     const session = this.sessions.get(socket);
     if (!session) return;
 
@@ -384,8 +384,12 @@ export class SMTPServer {
         mightBeSpam: email.mightBeSpam
       });
 
-      // Process or forward the email as needed
+      // Process or forward the email via MTA service
-      // this.mtaRef.processIncomingEmail(email); // You could add this method to your MTA service
+      try {
+        await this.mtaRef.processIncomingEmail(email);
+      } catch (err) {
+        console.error('Error in MTA processing of incoming email:', err);
+      }
     } catch (error) {
       console.error('Error parsing email:', error);
     }

ts/plugins.ts

@@ -40,6 +40,7 @@ export {
 // @push.rocks scope
 import * as projectinfo from '@push.rocks/projectinfo';
 import * as qenv from '@push.rocks/qenv';
+import * as smartacme from '@push.rocks/smartacme';
 import * as smartdata from '@push.rocks/smartdata';
 import * as smartdns from '@push.rocks/smartdns';
 import * as smartfile from '@push.rocks/smartfile';
@@ -52,12 +53,14 @@ import * as smartrequest from '@push.rocks/smartrequest';
 import * as smartrule from '@push.rocks/smartrule';
 import * as smartrx from '@push.rocks/smartrx';
 
-export { projectinfo, qenv, smartdata, smartdns, smartfile, smartlog, smartmail, smartpath, smartproxy, smartpromise, smartrequest, smartrule, smartrx };
+export { projectinfo, qenv, smartacme, smartdata, smartdns, smartfile, smartlog, smartmail, smartpath, smartproxy, smartpromise, smartrequest, smartrule, smartrx };
 
 // apiclient.xyz scope
+import * as cloudflare from '@apiclient.xyz/cloudflare';
 import * as letterxpress from '@apiclient.xyz/letterxpress';
 
 export {
+  cloudflare,
   letterxpress,
 }
 

ts_web/index.ts

@@ -0,0 +1 @@
+console.log('minidash')