From c46ceccb6c52135e2088576c537c2dba73900229 Mon Sep 17 00:00:00 2001 From: Juergen Kunz Date: Wed, 26 Nov 2025 12:16:50 +0000 Subject: [PATCH] update --- readme.hints.md | 30 +- ts/classes/caddy-log-receiver.ts | 417 +++++++++++++ ts/classes/caddy.ts | 572 ++++++++++++++++++ ts/classes/httpserver.ts | 190 ++++++ ts/classes/onebox.ts | 15 + ts/classes/registry.ts | 10 + ts/classes/reverseproxy.ts | 563 ++++------------- ui/src/app/app.routes.ts | 7 + ui/src/app/core/services/api.service.ts | 11 + .../services/network-log-stream.service.ts | 187 ++++++ ui/src/app/core/types/api.types.ts | 52 ++ .../app/features/network/network.component.ts | 388 ++++++++++++ .../components/layout/layout.component.ts | 1 + 13 files changed, 1970 insertions(+), 473 deletions(-) create mode 100644 ts/classes/caddy-log-receiver.ts create mode 100644 ts/classes/caddy.ts create mode 100644 ui/src/app/core/services/network-log-stream.service.ts create mode 100644 ui/src/app/features/network/network.component.ts diff --git a/readme.hints.md b/readme.hints.md index 1c0c434..4b18697 100644 --- a/readme.hints.md +++ b/readme.hints.md @@ -44,22 +44,30 @@ ts/database/ Migration 8 converted certificate storage from file paths to PEM content. -## Reverse Proxy SNI Support (November 2025) +## Reverse Proxy (November 2025 - Caddy) -The HTTPS reverse proxy now uses Node.js `https.createServer()` with SNI support: -- Uses Deno's Node.js compatibility layer for `node:https` module -- Implements `server.addContext(hostname, {cert, key})` for per-domain certificates -- Dynamic certificate addition via `addCertificate()` without server restart -- HTTP-to-HTTPS redirect when certificate exists for domain -- Wildcard pattern support (e.g., `*.bleu.de` covers `sub.bleu.de`) +The reverse proxy now uses **Caddy** for production-grade reverse proxying with native SNI support, HTTP/2, HTTP/3, and WebSocket handling. + +**Architecture:** +- Caddy binary downloaded to `.nogit/caddy` on first run (v2.10.2) +- Caddy process managed by `CaddyManager` class +- Configuration pushed dynamically via Caddy Admin API (port 2019) +- Automatic HTTPS disabled - certificates managed externally via SmartACME +- Zero-downtime configuration updates **Key files:** -- `ts/classes/reverseproxy.ts` - SNI-enabled HTTPS server -- `ts/classes/services.ts` - Dynamic route updates on service start/stop +- `ts/classes/caddy.ts` - CaddyManager class for binary and Admin API +- `ts/classes/reverseproxy.ts` - Delegates to CaddyManager **Certificate workflow:** 1. `CertRequirementManager` creates requirements for domains 2. Daemon processes requirements via `certmanager.ts` 3. Certificates stored in database (PEM content) -4. `reverseProxy.addCertificate()` dynamically adds SNI context -5. HTTP requests redirect to HTTPS when cert exists +4. `reverseProxy.addCertificate()` writes PEM files to `.nogit/certs/` and updates Caddy config +5. Caddy serves TLS with the loaded certificates + +**Configuration:** +- Dev mode: HTTP on 8080, HTTPS on 8443 +- Production: HTTP on 80, HTTPS on 443 +- Admin API: localhost:2019 (not exposed externally) +- Automatic HTTPS disabled to prevent Caddy from binding to default ports diff --git a/ts/classes/caddy-log-receiver.ts b/ts/classes/caddy-log-receiver.ts new file mode 100644 index 0000000..34bb1b8 --- /dev/null +++ b/ts/classes/caddy-log-receiver.ts @@ -0,0 +1,417 @@ +/** + * Caddy Log Receiver for Onebox + * + * TCP server that receives access logs from Caddy and broadcasts them to WebSocket clients. + * Supports per-client filtering by domain and adaptive sampling at high volume. + */ + +import { logger } from '../logging.ts'; +import { getErrorMessage } from '../utils/error.ts'; + +/** + * Filter applied to a WebSocket client's log stream + */ +export interface ILogFilter { + domain?: string; + service?: string; + sampleRate?: number; // 0.01 to 1.0, default 1.0 +} + +/** + * Caddy access log entry structure (from Caddy JSON format) + */ +export interface ICaddyAccessLog { + ts: number; + level?: string; + logger?: string; + msg?: string; + request: { + remote_ip: string; + remote_port?: string; + client_ip?: string; + proto: string; + method: string; + host: string; + uri: string; + headers?: Record; + tls?: { + resumed: boolean; + version: number; + cipher_suite: number; + proto: string; + server_name: string; + }; + }; + bytes_read?: number; + user_id?: string; + duration: number; + size: number; + status: number; + resp_headers?: Record; +} + +/** + * WebSocket client with filter + */ +interface ILogClient { + id: string; + ws: WebSocket; + filter: ILogFilter; +} + +/** + * CaddyLogReceiver - TCP server for Caddy access logs + */ +export class CaddyLogReceiver { + private server: Deno.TcpListener | null = null; + private clients: Map = new Map(); + private port: number; + private running = false; + private connections: Set = new Set(); + + // Adaptive sampling state + private logCountWindow: number[] = []; // timestamps of recent logs + private windowSize = 1000; // track last 1 second + private currentSampleRate = 1.0; + private logCounter = 0; + + // Ring buffer for recent logs (for late-joining clients) + private recentLogs: ICaddyAccessLog[] = []; + private maxRecentLogs = 100; + + constructor(port = 9999) { + this.port = port; + } + + /** + * Start the TCP server + */ + async start(): Promise { + if (this.running) { + logger.warn('CaddyLogReceiver is already running'); + return; + } + + try { + this.server = Deno.listen({ port: this.port, transport: 'tcp' }); + this.running = true; + logger.success(`CaddyLogReceiver started on TCP port ${this.port}`); + + // Start accepting connections in background + this.acceptConnections(); + } catch (error) { + logger.error(`Failed to start CaddyLogReceiver: ${getErrorMessage(error)}`); + throw error; + } + } + + /** + * Accept incoming TCP connections from Caddy + */ + private async acceptConnections(): Promise { + if (!this.server) return; + + try { + for await (const conn of this.server) { + this.connections.add(conn); + this.handleConnection(conn); + } + } catch (error) { + if (this.running) { + logger.error(`CaddyLogReceiver accept error: ${getErrorMessage(error)}`); + } + } + } + + /** + * Handle a single TCP connection from Caddy + */ + private async handleConnection(conn: Deno.TcpConn): Promise { + const remoteAddr = conn.remoteAddr as Deno.NetAddr; + logger.debug(`CaddyLogReceiver: Connection from ${remoteAddr.hostname}:${remoteAddr.port}`); + + const reader = conn.readable.getReader(); + const decoder = new TextDecoder(); + let buffer = ''; + + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + buffer += decoder.decode(value, { stream: true }); + + // Process complete lines (Caddy sends newline-delimited JSON) + const lines = buffer.split('\n'); + buffer = lines.pop() || ''; // Keep incomplete line in buffer + + for (const line of lines) { + if (line.trim()) { + this.processLogLine(line); + } + } + } + } catch (error) { + if (this.running) { + logger.debug(`CaddyLogReceiver connection closed: ${getErrorMessage(error)}`); + } + } finally { + this.connections.delete(conn); + try { + conn.close(); + } catch { + // Already closed + } + } + } + + /** + * Process a single log line from Caddy + */ + private processLogLine(line: string): void { + try { + const log = JSON.parse(line) as ICaddyAccessLog; + + // Only process access logs (check for http.log.access or just access, or any log with request/status) + const isAccessLog = log.logger === 'http.log.access' || + log.logger === 'access' || + (log.request && typeof log.status === 'number'); + if (!isAccessLog) { + logger.debug(`CaddyLogReceiver: Skipping non-access log: ${log.logger || 'unknown'}`); + return; + } + + // Update adaptive sampling + this.updateSampling(); + + // Apply global sampling (skip if randomly filtered out) + if (this.currentSampleRate < 1.0 && Math.random() > this.currentSampleRate) { + return; + } + + logger.debug(`CaddyLogReceiver: Access log received - ${log.request?.method} ${log.request?.host}${log.request?.uri} (status: ${log.status})`); + + // Add to recent logs buffer + this.recentLogs.push(log); + if (this.recentLogs.length > this.maxRecentLogs) { + this.recentLogs.shift(); + } + + // Broadcast to WebSocket clients (log how many clients) + logger.debug(`CaddyLogReceiver: Broadcasting to ${this.clients.size} clients`); + this.broadcast(log); + } catch (error) { + logger.debug(`Failed to parse Caddy log line: ${getErrorMessage(error)}`); + } + } + + /** + * Update adaptive sampling rate based on log volume + */ + private updateSampling(): void { + const now = Date.now(); + + // Add current timestamp + this.logCountWindow.push(now); + + // Remove timestamps older than 1 second + const cutoff = now - this.windowSize; + while (this.logCountWindow.length > 0 && this.logCountWindow[0] < cutoff) { + this.logCountWindow.shift(); + } + + // Calculate logs per second + const logsPerSecond = this.logCountWindow.length; + + // Adjust sample rate based on volume + if (logsPerSecond > 1000) { + this.currentSampleRate = 0.01; // 1% + } else if (logsPerSecond > 500) { + this.currentSampleRate = 0.1; // 10% + } else if (logsPerSecond > 100) { + this.currentSampleRate = 0.5; // 50% + } else { + this.currentSampleRate = 1.0; // 100% + } + } + + /** + * Broadcast a log entry to all connected WebSocket clients + */ + private broadcast(log: ICaddyAccessLog): void { + const message = JSON.stringify({ + type: 'access_log', + data: { + ts: log.ts, + request: { + remote_ip: log.request.remote_ip, + method: log.request.method, + host: log.request.host, + uri: log.request.uri, + proto: log.request.proto, + }, + status: log.status, + duration: log.duration, + size: log.size, + }, + timestamp: Date.now(), + }); + + for (const [clientId, client] of this.clients) { + try { + // Apply client-specific filter + if (!this.matchesFilter(log, client.filter)) { + continue; + } + + // Apply client-specific sample rate + if (client.filter.sampleRate && client.filter.sampleRate < 1.0) { + if (Math.random() > client.filter.sampleRate) { + continue; + } + } + + if (client.ws.readyState === WebSocket.OPEN) { + client.ws.send(message); + } else { + // Remove dead clients + this.clients.delete(clientId); + } + } catch { + this.clients.delete(clientId); + } + } + } + + /** + * Check if a log entry matches a client's filter + */ + private matchesFilter(log: ICaddyAccessLog, filter: ILogFilter): boolean { + // Domain filter + if (filter.domain) { + const logHost = log.request.host.toLowerCase(); + const filterDomain = filter.domain.toLowerCase(); + + // Match exact domain or subdomain + if (logHost !== filterDomain && !logHost.endsWith(`.${filterDomain}`)) { + return false; + } + } + + return true; + } + + /** + * Add a WebSocket client to receive logs + */ + addClient(clientId: string, ws: WebSocket, filter: ILogFilter = {}): void { + this.clients.set(clientId, { id: clientId, ws, filter }); + logger.debug(`CaddyLogReceiver: Added client ${clientId} (${this.clients.size} total)`); + + // Send recent logs to new client + for (const log of this.recentLogs) { + if (this.matchesFilter(log, filter)) { + try { + ws.send( + JSON.stringify({ + type: 'access_log', + data: { + ts: log.ts, + request: { + remote_ip: log.request.remote_ip, + method: log.request.method, + host: log.request.host, + uri: log.request.uri, + proto: log.request.proto, + }, + status: log.status, + duration: log.duration, + size: log.size, + }, + timestamp: Date.now(), + }), + ); + } catch { + // Client disconnected + } + } + } + } + + /** + * Remove a WebSocket client + */ + removeClient(clientId: string): void { + if (this.clients.delete(clientId)) { + logger.debug(`CaddyLogReceiver: Removed client ${clientId} (${this.clients.size} remaining)`); + } + } + + /** + * Update a client's filter + */ + updateClientFilter(clientId: string, filter: ILogFilter): void { + const client = this.clients.get(clientId); + if (client) { + client.filter = filter; + logger.debug(`CaddyLogReceiver: Updated filter for client ${clientId}`); + } + } + + /** + * Stop the TCP server + */ + async stop(): Promise { + if (!this.running) { + return; + } + + this.running = false; + + // Close all connections + for (const conn of this.connections) { + try { + conn.close(); + } catch { + // Already closed + } + } + this.connections.clear(); + + // Close server + if (this.server) { + try { + this.server.close(); + } catch { + // Already closed + } + this.server = null; + } + + // Clear clients + this.clients.clear(); + + logger.info('CaddyLogReceiver stopped'); + } + + /** + * Get current stats + */ + getStats(): { + running: boolean; + port: number; + clients: number; + connections: number; + sampleRate: number; + recentLogsCount: number; + } { + return { + running: this.running, + port: this.port, + clients: this.clients.size, + connections: this.connections.size, + sampleRate: this.currentSampleRate, + recentLogsCount: this.recentLogs.length, + }; + } +} diff --git a/ts/classes/caddy.ts b/ts/classes/caddy.ts new file mode 100644 index 0000000..122aa7b --- /dev/null +++ b/ts/classes/caddy.ts @@ -0,0 +1,572 @@ +/** + * Caddy Manager for Onebox + * + * Manages Caddy binary download, process lifecycle, and Admin API configuration. + * Caddy is used as the reverse proxy with native SNI support. + */ + +import { logger } from '../logging.ts'; +import { getErrorMessage } from '../utils/error.ts'; + +const CADDY_VERSION = '2.10.2'; +const CADDY_DOWNLOAD_URL = `https://github.com/caddyserver/caddy/releases/download/v${CADDY_VERSION}/caddy_${CADDY_VERSION}_linux_amd64.tar.gz`; + +export interface ICaddyRoute { + domain: string; + upstream: string; // e.g., "10.0.1.40:80" +} + +export interface ICaddyCertificate { + domain: string; + certPath: string; + keyPath: string; +} + +interface ICaddyLoggingConfig { + logs: { + [name: string]: { + writer: { + output: string; + address?: string; + dial_timeout?: string; + soft_start?: boolean; + }; + encoder?: { format: string }; + level?: string; + include?: string[]; + }; + }; +} + +interface ICaddyConfig { + admin: { + listen: string; + }; + logging?: ICaddyLoggingConfig; + apps: { + http: { + servers: { + [key: string]: { + listen: string[]; + routes: ICaddyRouteConfig[]; + automatic_https?: { + disable?: boolean; + disable_redirects?: boolean; + }; + logs?: { + default_logger_name: string; + }; + }; + }; + }; + tls?: { + automation?: { + policies: Array<{ issuers: never[] }>; + }; + certificates?: { + load_files?: Array<{ + certificate: string; + key: string; + tags?: string[]; + }>; + }; + }; + }; +} + +interface ICaddyRouteConfig { + match: Array<{ host: string[] }>; + handle: Array<{ + handler: string; + upstreams?: Array<{ dial: string }>; + routes?: ICaddyRouteConfig[]; + }>; + terminal?: boolean; +} + +export class CaddyManager { + private process: Deno.ChildProcess | null = null; + private binaryPath: string; + private certsDir: string; + private adminUrl: string; + private httpPort: number; + private httpsPort: number; + private logReceiverPort: number; + private loggingEnabled: boolean; + private routes: Map = new Map(); + private certificates: Map = new Map(); + + constructor(options?: { + binaryPath?: string; + certsDir?: string; + adminPort?: number; + httpPort?: number; + httpsPort?: number; + logReceiverPort?: number; + loggingEnabled?: boolean; + }) { + this.binaryPath = options?.binaryPath || './.nogit/caddy'; + this.certsDir = options?.certsDir || './.nogit/certs'; + this.adminUrl = `http://localhost:${options?.adminPort || 2019}`; + this.httpPort = options?.httpPort || 8080; + this.httpsPort = options?.httpsPort || 8443; + this.logReceiverPort = options?.logReceiverPort || 9999; + this.loggingEnabled = options?.loggingEnabled ?? true; + } + + /** + * Update listening ports (must call reloadConfig after if running) + */ + setPorts(httpPort: number, httpsPort: number): void { + this.httpPort = httpPort; + this.httpsPort = httpsPort; + } + + /** + * Ensure Caddy binary is downloaded and executable + */ + async ensureBinary(): Promise { + try { + // Check if binary exists + try { + const stat = await Deno.stat(this.binaryPath); + if (stat.isFile) { + // Verify it's executable by checking version + const cmd = new Deno.Command(this.binaryPath, { + args: ['version'], + stdout: 'piped', + stderr: 'piped', + }); + const result = await cmd.output(); + if (result.success) { + const version = new TextDecoder().decode(result.stdout).trim(); + logger.info(`Caddy binary found: ${version}`); + return; + } + } + } catch { + // Binary doesn't exist, need to download + } + + logger.info(`Downloading Caddy v${CADDY_VERSION}...`); + + // Create directory if needed + const dir = this.binaryPath.substring(0, this.binaryPath.lastIndexOf('/')); + await Deno.mkdir(dir, { recursive: true }); + + // Download tar.gz + const response = await fetch(CADDY_DOWNLOAD_URL); + if (!response.ok) { + throw new Error(`Failed to download Caddy: ${response.status} ${response.statusText}`); + } + + const tarGzPath = `${this.binaryPath}.tar.gz`; + const data = new Uint8Array(await response.arrayBuffer()); + await Deno.writeFile(tarGzPath, data); + + // Extract using tar command + const extractCmd = new Deno.Command('tar', { + args: ['-xzf', tarGzPath, '-C', dir, 'caddy'], + stdout: 'piped', + stderr: 'piped', + }); + const extractResult = await extractCmd.output(); + if (!extractResult.success) { + throw new Error(`Failed to extract Caddy: ${new TextDecoder().decode(extractResult.stderr)}`); + } + + // Clean up tar.gz + await Deno.remove(tarGzPath); + + // Make executable + await Deno.chmod(this.binaryPath, 0o755); + + logger.success(`Caddy v${CADDY_VERSION} downloaded to ${this.binaryPath}`); + } catch (error) { + logger.error(`Failed to ensure Caddy binary: ${getErrorMessage(error)}`); + throw error; + } + } + + /** + * Start Caddy process with minimal config, then configure via Admin API + */ + async start(): Promise { + if (this.process) { + logger.warn('Caddy is already running'); + return; + } + + try { + // Create certs directory + await Deno.mkdir(this.certsDir, { recursive: true }); + + logger.info('Starting Caddy server...'); + + // Start Caddy with blank config - Admin API is available immediately + // We'll push the full configuration via Admin API after it's ready + const cmd = new Deno.Command(this.binaryPath, { + args: ['run'], + stdin: 'null', + stdout: 'piped', + stderr: 'piped', + }); + + this.process = cmd.spawn(); + + // Start log readers (non-blocking) + this.readProcessOutput(); + + // Wait for Admin API to be ready + await this.waitForReady(); + + // Now configure via Admin API with current routes and certificates + await this.reloadConfig(); + + logger.success(`Caddy started (HTTP: ${this.httpPort}, HTTPS: ${this.httpsPort}, Admin: ${this.adminUrl})`); + } catch (error) { + logger.error(`Failed to start Caddy: ${getErrorMessage(error)}`); + throw error; + } + } + + /** + * Read process stdout/stderr and log + */ + private async readProcessOutput(): Promise { + if (!this.process) return; + + // Read stderr (Caddy logs to stderr by default) + const stderrReader = this.process.stderr.getReader(); + (async () => { + try { + while (true) { + const { done, value } = await stderrReader.read(); + if (done) break; + const text = new TextDecoder().decode(value).trim(); + if (text) { + // Parse Caddy's JSON log format or just log as-is + for (const line of text.split('\n')) { + if (line.includes('"level":"error"')) { + logger.error(`[Caddy] ${line}`); + } else if (line.includes('"level":"warn"')) { + logger.warn(`[Caddy] ${line}`); + } else { + logger.debug(`[Caddy] ${line}`); + } + } + } + } + } catch { + // Process ended + } + })(); + } + + /** + * Wait for Caddy to be ready by polling admin API + */ + private async waitForReady(maxAttempts = 50, intervalMs = 100): Promise { + for (let i = 0; i < maxAttempts; i++) { + try { + const response = await fetch(`${this.adminUrl}/config/`); + if (response.ok) { + return; + } + } catch { + // Not ready yet + } + await new Promise((resolve) => setTimeout(resolve, intervalMs)); + } + throw new Error('Caddy failed to start within timeout'); + } + + /** + * Stop Caddy process + */ + async stop(): Promise { + if (!this.process) { + return; + } + + try { + logger.info('Stopping Caddy...'); + + // Try graceful shutdown via API first + try { + await fetch(`${this.adminUrl}/stop`, { method: 'POST' }); + // Wait for process to exit + await Promise.race([ + this.process.status, + new Promise((resolve) => setTimeout(resolve, 5000)), + ]); + } catch { + // API not available, kill directly + } + + // Force kill if still running + try { + this.process.kill('SIGTERM'); + } catch { + // Already dead + } + + this.process = null; + logger.info('Caddy stopped'); + } catch (error) { + logger.error(`Failed to stop Caddy: ${getErrorMessage(error)}`); + } + } + + /** + * Check if Caddy is healthy + */ + async isHealthy(): Promise { + try { + const response = await fetch(`${this.adminUrl}/config/`); + return response.ok; + } catch { + return false; + } + } + + /** + * Build Caddy JSON configuration from current routes and certificates + */ + private buildConfig(): ICaddyConfig { + const routes: ICaddyRouteConfig[] = []; + + // Add routes + for (const [domain, route] of this.routes) { + routes.push({ + match: [{ host: [domain] }], + handle: [ + { + handler: 'reverse_proxy', + upstreams: [{ dial: route.upstream }], + }, + ], + terminal: true, + }); + } + + // Build certificate load_files + const loadFiles: Array<{ certificate: string; key: string; tags?: string[] }> = []; + for (const [domain, cert] of this.certificates) { + loadFiles.push({ + certificate: cert.certPath, + key: cert.keyPath, + tags: [domain], + }); + } + + const config: ICaddyConfig = { + admin: { + listen: this.adminUrl.replace('http://', ''), + }, + apps: { + http: { + servers: { + main: { + listen: [`:${this.httpPort}`, `:${this.httpsPort}`], + routes, + // Disable automatic HTTPS to prevent Caddy from trying to bind to port 80/443 + automatic_https: { + disable: true, + }, + }, + }, + }, + }, + }; + + // Add access logging configuration if enabled + if (this.loggingEnabled) { + config.logging = { + logs: { + access: { + writer: { + output: 'net', + address: `tcp/localhost:${this.logReceiverPort}`, + dial_timeout: '5s', + soft_start: true, // Continue even if log receiver is down + }, + encoder: { format: 'json' }, + level: 'INFO', + include: ['http.log.access'], + }, + }, + }; + + // Associate server with access logger + config.apps.http.servers.main.logs = { + default_logger_name: 'access', + }; + } + + // Add TLS config if we have certificates + if (loadFiles.length > 0) { + config.apps.tls = { + automation: { + // Disable automatic HTTPS - we manage certs ourselves + policies: [{ issuers: [] }], + }, + certificates: { + load_files: loadFiles, + }, + }; + } + + return config; + } + + /** + * Reload Caddy configuration via Admin API + */ + async reloadConfig(): Promise { + if (!this.process) { + logger.warn('Caddy not running, cannot reload config'); + return; + } + + const config = this.buildConfig(); + + try { + const response = await fetch(`${this.adminUrl}/load`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(config), + }); + + if (!response.ok) { + const text = await response.text(); + throw new Error(`Failed to reload Caddy config: ${response.status} ${text}`); + } + + logger.debug('Caddy configuration reloaded'); + } catch (error) { + logger.error(`Failed to reload Caddy config: ${getErrorMessage(error)}`); + throw error; + } + } + + /** + * Add or update a route + */ + async addRoute(domain: string, upstream: string): Promise { + this.routes.set(domain, { domain, upstream }); + + if (this.process) { + await this.reloadConfig(); + } + + logger.success(`Added Caddy route: ${domain} -> ${upstream}`); + } + + /** + * Remove a route + */ + async removeRoute(domain: string): Promise { + if (this.routes.delete(domain)) { + if (this.process) { + await this.reloadConfig(); + } + logger.success(`Removed Caddy route: ${domain}`); + } + } + + /** + * Add or update a TLS certificate + * Writes PEM files to disk and updates config + */ + async addCertificate(domain: string, certPem: string, keyPem: string): Promise { + // Write PEM files + const certPath = `${this.certsDir}/${domain}.crt`; + const keyPath = `${this.certsDir}/${domain}.key`; + + await Deno.mkdir(this.certsDir, { recursive: true }); + await Deno.writeTextFile(certPath, certPem); + await Deno.writeTextFile(keyPath, keyPem); + + // Use absolute paths for Caddy + const absoluteCertPath = await Deno.realPath(certPath); + const absoluteKeyPath = await Deno.realPath(keyPath); + + this.certificates.set(domain, { + domain, + certPath: absoluteCertPath, + keyPath: absoluteKeyPath, + }); + + if (this.process) { + await this.reloadConfig(); + } + + logger.success(`Added TLS certificate for ${domain}`); + } + + /** + * Remove a TLS certificate + */ + async removeCertificate(domain: string): Promise { + const cert = this.certificates.get(domain); + if (cert) { + this.certificates.delete(domain); + + // Remove files + try { + await Deno.remove(cert.certPath); + await Deno.remove(cert.keyPath); + } catch { + // Files may not exist + } + + if (this.process) { + await this.reloadConfig(); + } + + logger.success(`Removed TLS certificate for ${domain}`); + } + } + + /** + * Get all current routes + */ + getRoutes(): ICaddyRoute[] { + return Array.from(this.routes.values()); + } + + /** + * Get all current certificates + */ + getCertificates(): ICaddyCertificate[] { + return Array.from(this.certificates.values()); + } + + /** + * Clear all routes and certificates (useful for reload from database) + */ + clear(): void { + this.routes.clear(); + this.certificates.clear(); + } + + /** + * Get status + */ + getStatus(): { + running: boolean; + httpPort: number; + httpsPort: number; + routes: number; + certificates: number; + } { + return { + running: this.process !== null, + httpPort: this.httpPort, + httpsPort: this.httpsPort, + routes: this.routes.size, + certificates: this.certificates.size, + }; + } +} diff --git a/ts/classes/httpserver.ts b/ts/classes/httpserver.ts index 370eb72..bfb2007 100644 --- a/ts/classes/httpserver.ts +++ b/ts/classes/httpserver.ts @@ -83,6 +83,11 @@ export class OneboxHttpServer { return this.handleLogStreamUpgrade(req, serviceName); } + // Network access logs WebSocket + if (path === '/api/network/logs/stream' && req.headers.get('upgrade') === 'websocket') { + return this.handleNetworkLogStreamUpgrade(req, new URL(req.url)); + } + // Docker Registry v2 Token endpoint (for OCI authentication) if (path === '/v2/token') { return await this.handleRegistryTokenRequest(req, url); @@ -291,6 +296,11 @@ export class OneboxHttpServer { } else if (path.match(/^\/api\/services\/[^/]+\/platform-resources$/) && method === 'GET') { const serviceName = path.split('/')[3]; return await this.handleGetServicePlatformResourcesRequest(serviceName); + // Network endpoints + } else if (path === '/api/network/targets' && method === 'GET') { + return await this.handleGetNetworkTargetsRequest(); + } else if (path === '/api/network/stats' && method === 'GET') { + return await this.handleGetNetworkStatsRequest(); } else { return this.jsonResponse({ success: false, error: 'Not found' }, 404); } @@ -995,6 +1005,186 @@ export class OneboxHttpServer { return response; } + /** + * Handle WebSocket upgrade for network access log streaming + */ + private handleNetworkLogStreamUpgrade(req: Request, url: URL): Response { + const { socket, response } = Deno.upgradeWebSocket(req); + + // Extract filter from query params + const filterDomain = url.searchParams.get('domain'); + + // Generate unique client ID + const clientId = crypto.randomUUID(); + + socket.onopen = () => { + logger.info(`Network log stream WebSocket connected (client: ${clientId})`); + + // Register with CaddyLogReceiver + const filter = filterDomain ? { domain: filterDomain } : {}; + this.oneboxRef.caddyLogReceiver.addClient(clientId, socket, filter); + + // Send initial connection message + socket.send(JSON.stringify({ + type: 'connected', + clientId, + filter, + })); + }; + + socket.onmessage = (event) => { + try { + const message = JSON.parse(event.data); + + // Handle filter updates from client + if (message.type === 'set_filter') { + const newFilter = { + domain: message.domain || undefined, + sampleRate: message.sampleRate || undefined, + }; + this.oneboxRef.caddyLogReceiver.updateClientFilter(clientId, newFilter); + + socket.send(JSON.stringify({ + type: 'filter_updated', + filter: newFilter, + })); + } + } catch (error) { + logger.debug(`Network log stream message parse error: ${getErrorMessage(error)}`); + } + }; + + socket.onclose = () => { + logger.info(`Network log stream WebSocket closed (client: ${clientId})`); + this.oneboxRef.caddyLogReceiver.removeClient(clientId); + }; + + socket.onerror = (error) => { + logger.error(`Network log stream WebSocket error: ${error}`); + this.oneboxRef.caddyLogReceiver.removeClient(clientId); + }; + + return response; + } + + // ============ Network Endpoints ============ + + /** + * Get all traffic targets (services, registry, platform services) + */ + private async handleGetNetworkTargetsRequest(): Promise { + try { + const targets: Array<{ + type: 'service' | 'registry' | 'platform'; + name: string; + domain: string | null; + targetHost: string; + targetPort: number; + status: string; + }> = []; + + // Add services + const services = this.oneboxRef.services.listServices(); + for (const service of services) { + targets.push({ + type: 'service', + name: service.name, + domain: service.domain || null, + targetHost: service.containerIP || 'unknown', + targetPort: service.port || 80, + status: service.status, + }); + } + + // Add registry if running + const registryStatus = this.oneboxRef.registry.getStatus(); + if (registryStatus.running) { + targets.push({ + type: 'registry', + name: 'onebox-registry', + domain: null, // Registry is internal + targetHost: 'localhost', + targetPort: registryStatus.port, + status: 'running', + }); + } + + // Add platform services + const platformServices = this.oneboxRef.platformServices.getAllPlatformServices(); + for (const ps of platformServices) { + // Get provider info for display name + const provider = this.oneboxRef.platformServices.getProvider(ps.type); + targets.push({ + type: 'platform', + name: provider?.displayName || ps.type, + domain: null, // Platform services are internal + targetHost: 'localhost', + targetPort: this.getPlatformServicePort(ps.type), + status: ps.status, + }); + } + + return this.jsonResponse({ success: true, data: targets }); + } catch (error) { + logger.error(`Failed to get network targets: ${getErrorMessage(error)}`); + return this.jsonResponse({ + success: false, + error: getErrorMessage(error) || 'Failed to get network targets', + }, 500); + } + } + + /** + * Get default port for a platform service type + */ + private getPlatformServicePort(type: TPlatformServiceType): number { + const ports: Record = { + mongodb: 27017, + minio: 9000, + redis: 6379, + postgresql: 5432, + rabbitmq: 5672, + }; + return ports[type] || 0; + } + + /** + * Get Caddy/network stats + */ + private async handleGetNetworkStatsRequest(): Promise { + try { + const proxyStatus = this.oneboxRef.reverseProxy.getStatus(); + const logReceiverStats = this.oneboxRef.caddyLogReceiver.getStats(); + + return this.jsonResponse({ + success: true, + data: { + proxy: { + running: proxyStatus.running, + httpPort: proxyStatus.httpPort, + httpsPort: proxyStatus.httpsPort, + routes: proxyStatus.routes, + certificates: proxyStatus.certificates, + }, + logReceiver: { + running: logReceiverStats.running, + port: logReceiverStats.port, + clients: logReceiverStats.clients, + connections: logReceiverStats.connections, + sampleRate: logReceiverStats.sampleRate, + recentLogsCount: logReceiverStats.recentLogsCount, + }, + }, + }); + } catch (error) { + logger.error(`Failed to get network stats: ${getErrorMessage(error)}`); + return this.jsonResponse({ + success: false, + error: getErrorMessage(error) || 'Failed to get network stats', + }, 500); + } + } + /** * Broadcast message to all connected WebSocket clients */ diff --git a/ts/classes/onebox.ts b/ts/classes/onebox.ts index 08a617d..18a11bf 100644 --- a/ts/classes/onebox.ts +++ b/ts/classes/onebox.ts @@ -19,6 +19,7 @@ import { CloudflareDomainSync } from './cloudflare-sync.ts'; import { CertRequirementManager } from './cert-requirement-manager.ts'; import { RegistryManager } from './registry.ts'; import { PlatformServicesManager } from './platform-services/index.ts'; +import { CaddyLogReceiver } from './caddy-log-receiver.ts'; export class Onebox { public database: OneboxDatabase; @@ -34,6 +35,7 @@ export class Onebox { public certRequirementManager: CertRequirementManager; public registry: RegistryManager; public platformServices: PlatformServicesManager; + public caddyLogReceiver: CaddyLogReceiver; private initialized = false; @@ -62,6 +64,9 @@ export class Onebox { // Initialize platform services manager this.platformServices = new PlatformServicesManager(this); + + // Initialize Caddy log receiver + this.caddyLogReceiver = new CaddyLogReceiver(9999); } /** @@ -80,6 +85,13 @@ export class Onebox { // Initialize Docker await this.docker.init(); + // Start Caddy log receiver BEFORE reverse proxy (so Caddy can connect to it) + try { + await this.caddyLogReceiver.start(); + } catch (error) { + logger.warn(`Failed to start Caddy log receiver: ${getErrorMessage(error)}`); + } + // Initialize Reverse Proxy await this.reverseProxy.init(); @@ -284,6 +296,9 @@ export class Onebox { // Stop reverse proxy if running await this.reverseProxy.stop(); + // Stop Caddy log receiver + await this.caddyLogReceiver.stop(); + // Close database this.database.close(); diff --git a/ts/classes/registry.ts b/ts/classes/registry.ts index 272a222..795821e 100644 --- a/ts/classes/registry.ts +++ b/ts/classes/registry.ts @@ -289,6 +289,16 @@ export class RegistryManager { return token; } + /** + * Get the registry status + */ + getStatus(): { running: boolean; port: number } { + return { + running: this.isInitialized, + port: this.options.port || 4000, + }; + } + /** * Get the registry base URL */ diff --git a/ts/classes/reverseproxy.ts b/ts/classes/reverseproxy.ts index c62df23..54169b9 100644 --- a/ts/classes/reverseproxy.ts +++ b/ts/classes/reverseproxy.ts @@ -1,17 +1,14 @@ /** * Reverse Proxy for Onebox * - * HTTP/HTTPS reverse proxy with SNI support for multi-domain TLS - * Uses Node.js https module for SNI via Deno's Node compatibility layer + * Delegates to Caddy for production-grade reverse proxy with native SNI support, + * HTTP/2, WebSocket proxying, and zero-downtime configuration updates. */ import { logger } from '../logging.ts'; import { getErrorMessage } from '../utils/error.ts'; import { OneboxDatabase } from './database.ts'; -import * as nodeHttps from 'node:https'; -import * as nodeHttp from 'node:http'; -import type { IncomingMessage, ServerResponse } from 'node:http'; -import { Buffer } from 'node:buffer'; +import { CaddyManager } from './caddy.ts'; interface IProxyRoute { domain: string; @@ -21,33 +18,30 @@ interface IProxyRoute { containerID?: string; } -interface ITlsConfig { - domain: string; - certPem: string; // Certificate PEM content - keyPem: string; // Private key PEM content -} - export class OneboxReverseProxy { private oneboxRef: any; private database: OneboxDatabase; + private caddy: CaddyManager; private routes: Map = new Map(); - private httpServer: Deno.HttpServer | null = null; - private httpsServer: nodeHttps.Server | null = null; // Node.js HTTPS server for SNI support - private httpPort = 80; - private httpsPort = 443; - private tlsConfigs: Map = new Map(); + private httpPort = 8080; // Default to dev ports (will be overridden if production) + private httpsPort = 8443; constructor(oneboxRef: any) { this.oneboxRef = oneboxRef; this.database = oneboxRef.database; + this.caddy = new CaddyManager({ + httpPort: this.httpPort, + httpsPort: this.httpsPort, + }); } /** - * Initialize reverse proxy + * Initialize reverse proxy - ensures Caddy binary is available */ async init(): Promise { try { - logger.info('Reverse proxy initialized'); + await this.caddy.ensureBinary(); + logger.info('Reverse proxy initialized (Caddy)'); } catch (error) { logger.error(`Failed to initialize reverse proxy: ${getErrorMessage(error)}`); throw error; @@ -55,415 +49,50 @@ export class OneboxReverseProxy { } /** - * Start the HTTP reverse proxy server + * Start the HTTP/HTTPS reverse proxy server + * Caddy handles both HTTP and HTTPS on the configured ports */ async startHttp(port?: number): Promise { - if (this.httpServer) { - logger.warn('HTTP reverse proxy already running'); - return; - } - if (port) { this.httpPort = port; + this.caddy.setPorts(this.httpPort, this.httpsPort); } try { - logger.info(`Starting HTTP reverse proxy on port ${this.httpPort}...`); - - this.httpServer = Deno.serve( - { - port: this.httpPort, - hostname: '0.0.0.0', - onListen: ({ hostname, port }) => { - logger.success(`HTTP reverse proxy listening on http://${hostname}:${port}`); - }, - }, - (req) => this.handleRequest(req, false) - ); - - logger.success(`HTTP reverse proxy started on port ${this.httpPort}`); + // Start Caddy (handles both HTTP and HTTPS) + await this.caddy.start(); + logger.success(`Reverse proxy started on port ${this.httpPort} (Caddy)`); } catch (error) { - logger.error(`Failed to start HTTP reverse proxy: ${getErrorMessage(error)}`); + logger.error(`Failed to start reverse proxy: ${getErrorMessage(error)}`); throw error; } } /** - * Start the HTTPS reverse proxy server with SNI support - * Uses Node.js https.createServer() + addContext() for per-domain certificates + * Start HTTPS - Caddy already handles HTTPS when started + * This method exists for interface compatibility */ async startHttps(port?: number): Promise { - if (this.httpsServer) { - logger.warn('HTTPS reverse proxy already running'); - return; - } - if (port) { this.httpsPort = port; + this.caddy.setPorts(this.httpPort, this.httpsPort); } - - try { - // Check if we have any TLS configs - if (this.tlsConfigs.size === 0) { - logger.info('No TLS certificates configured, skipping HTTPS server'); - return; - } - - logger.info(`Starting HTTPS reverse proxy on port ${this.httpsPort} with SNI support...`); - - // Get the first certificate as default (required for server creation) - const defaultConfig = Array.from(this.tlsConfigs.values())[0]; - - // Create HTTPS server with Node.js for SNI support - this.httpsServer = nodeHttps.createServer( - { - cert: defaultConfig.certPem, - key: defaultConfig.keyPem, - }, - (req, res) => this.handleNodeRequest(req, res, true) - ); - - // Add SNI contexts for each domain - for (const [domain, config] of this.tlsConfigs) { - this.httpsServer.addContext(domain, { - cert: config.certPem, - key: config.keyPem, - }); - // Also add wildcard pattern for subdomains - this.httpsServer.addContext(`*.${domain}`, { - cert: config.certPem, - key: config.keyPem, - }); - logger.info(`Added SNI context for ${domain} and *.${domain}`); - } - - // Start listening - await new Promise((resolve, reject) => { - this.httpsServer!.listen(this.httpsPort, '0.0.0.0', () => { - logger.success(`HTTPS reverse proxy listening on https://0.0.0.0:${this.httpsPort}`); - resolve(); - }); - this.httpsServer!.on('error', reject); - }); - - logger.success(`HTTPS reverse proxy started on port ${this.httpsPort} with ${this.tlsConfigs.size} certificates`); - } catch (error) { - logger.error(`Failed to start HTTPS reverse proxy: ${getErrorMessage(error)}`); - // Don't throw - HTTPS is optional - logger.warn('Continuing without HTTPS support'); + // Caddy handles both HTTP and HTTPS together + // If already running, just log and optionally reload with new port + const status = this.caddy.getStatus(); + if (status.running) { + logger.info(`HTTPS already running on port ${this.httpsPort} via Caddy`); + } else { + await this.caddy.start(); } } /** - * Handle Node.js HTTP request and convert to fetch-style handling - */ - private handleNodeRequest( - req: IncomingMessage, - res: ServerResponse, - isHttps: boolean - ): void { - // Collect request body - const chunks: Buffer[] = []; - - req.on('data', (chunk: Buffer) => { - chunks.push(chunk); - }); - - req.on('end', async () => { - try { - const body = chunks.length > 0 ? Buffer.concat(chunks) : undefined; - - // Build URL from Node.js request - const protocol = isHttps ? 'https' : 'http'; - const host = req.headers.host || 'localhost'; - const url = new URL(req.url || '/', `${protocol}://${host}`); - - // Convert Node.js headers to Headers - const headers = new Headers(); - for (const [key, value] of Object.entries(req.headers)) { - if (value) { - if (Array.isArray(value)) { - value.forEach(v => headers.append(key, v)); - } else { - headers.set(key, value); - } - } - } - - // Create fetch-style Request - const fetchRequest = new Request(url.toString(), { - method: req.method || 'GET', - headers, - body: body && req.method !== 'GET' && req.method !== 'HEAD' ? body : undefined, - }); - - // Use existing handleRequest logic - const response = await this.handleRequest(fetchRequest, isHttps); - - // Send response back via Node.js ServerResponse - res.statusCode = response.status; - res.statusMessage = response.statusText; - - // Copy response headers - response.headers.forEach((value, key) => { - res.setHeader(key, value); - }); - - // Send response body - if (response.body) { - const reader = response.body.getReader(); - const pump = async () => { - const { done, value } = await reader.read(); - if (done) { - res.end(); - return; - } - res.write(value); - await pump(); - }; - await pump(); - } else { - res.end(); - } - } catch (error) { - logger.error(`Node request handler error: ${getErrorMessage(error)}`); - res.statusCode = 502; - res.end('Bad Gateway'); - } - }); - - req.on('error', (error) => { - logger.error(`Node request error: ${getErrorMessage(error)}`); - res.statusCode = 502; - res.end('Bad Gateway'); - }); - } - - /** - * Stop all reverse proxy servers + * Stop the reverse proxy */ async stop(): Promise { - const promises: Promise[] = []; - - if (this.httpServer) { - promises.push(this.httpServer.shutdown()); - this.httpServer = null; - logger.info('HTTP reverse proxy stopped'); - } - - if (this.httpsServer) { - // Node.js server uses close() which accepts a callback - promises.push(new Promise((resolve, reject) => { - this.httpsServer!.close((err) => { - if (err) reject(err); - else resolve(); - }); - })); - this.httpsServer = null; - logger.info('HTTPS reverse proxy stopped'); - } - - await Promise.all(promises); - } - - /** - * Check if we have a certificate for a domain (exact or wildcard match) - */ - private hasCertificateForDomain(host: string): boolean { - if (this.tlsConfigs.has(host)) return true; - // Check wildcard: *.example.com covers sub.example.com - const parts = host.split('.'); - if (parts.length >= 2) { - const rootDomain = parts.slice(-2).join('.'); - if (this.tlsConfigs.has(rootDomain)) return true; - } - return false; - } - - /** - * Handle incoming HTTP/HTTPS request - */ - private async handleRequest(req: Request, isHttps: boolean): Promise { - const url = new URL(req.url); - const host = req.headers.get('host')?.split(':')[0] || ''; - - logger.debug(`Proxy request: ${req.method} ${host}${url.pathname}`); - - // HTTP to HTTPS redirect when certificate exists - if (!isHttps && this.httpsServer !== null && this.hasCertificateForDomain(host)) { - const httpsUrl = `https://${host}${url.pathname}${url.search}`; - logger.debug(`Redirecting HTTP to HTTPS: ${httpsUrl}`); - return Response.redirect(httpsUrl, 301); - } - - // Find matching route - const route = this.routes.get(host); - - if (!route) { - logger.debug(`No route found for host: ${host}`); - return new Response('Service not found', { - status: 404, - headers: { 'Content-Type': 'text/plain' }, - }); - } - - // Check if this is a WebSocket upgrade request - const upgrade = req.headers.get('upgrade')?.toLowerCase(); - if (upgrade === 'websocket') { - return await this.handleWebSocketUpgrade(req, route, isHttps); - } - - try { - // Build target URL - const targetUrl = `http://${route.targetHost}:${route.targetPort}${url.pathname}${url.search}`; - - logger.debug(`Proxying to: ${targetUrl}`); - - // Forward request to target - const targetReq = new Request(targetUrl, { - method: req.method, - headers: this.forwardHeaders(req.headers, host, isHttps), - body: req.body, - }); - - const response = await fetch(targetReq); - - // Forward response back to client - return new Response(response.body, { - status: response.status, - statusText: response.statusText, - headers: this.filterResponseHeaders(response.headers), - }); - } catch (error) { - logger.error(`Proxy error for ${host}: ${getErrorMessage(error)}`); - return new Response('Bad Gateway', { - status: 502, - headers: { 'Content-Type': 'text/plain' }, - }); - } - } - - /** - * Handle WebSocket upgrade and proxy connection - */ - private async handleWebSocketUpgrade( - req: Request, - route: IProxyRoute, - isHttps: boolean - ): Promise { - try { - const url = new URL(req.url); - const targetUrl = `ws://${route.targetHost}:${route.targetPort}${url.pathname}${url.search}`; - - logger.info(`WebSocket upgrade: ${url.host} -> ${targetUrl}`); - - // Upgrade the client connection - const { socket: clientSocket, response } = Deno.upgradeWebSocket(req); - - // Connect to backend WebSocket - const backendSocket = new WebSocket(targetUrl); - - // Proxy messages from client to backend - clientSocket.onmessage = (e) => { - if (backendSocket.readyState === WebSocket.OPEN) { - backendSocket.send(e.data); - } - }; - - // Proxy messages from backend to client - backendSocket.onmessage = (e) => { - if (clientSocket.readyState === WebSocket.OPEN) { - clientSocket.send(e.data); - } - }; - - // Handle client close - clientSocket.onclose = () => { - logger.debug(`Client WebSocket closed for ${url.host}`); - backendSocket.close(); - }; - - // Handle backend close - backendSocket.onclose = () => { - logger.debug(`Backend WebSocket closed for ${targetUrl}`); - clientSocket.close(); - }; - - // Handle errors - clientSocket.onerror = (e) => { - logger.error(`Client WebSocket error: ${e}`); - backendSocket.close(); - }; - - backendSocket.onerror = (e) => { - logger.error(`Backend WebSocket error: ${e}`); - clientSocket.close(); - }; - - return response; - } catch (error) { - logger.error(`WebSocket upgrade error: ${getErrorMessage(error)}`); - return new Response('WebSocket Upgrade Failed', { - status: 500, - headers: { 'Content-Type': 'text/plain' }, - }); - } - } - - /** - * Forward request headers to target, filtering out problematic ones - */ - private forwardHeaders(headers: Headers, originalHost: string, isHttps: boolean): Headers { - const forwarded = new Headers(); - - // Copy most headers - for (const [key, value] of headers.entries()) { - // Skip headers that should not be forwarded - if ( - key.toLowerCase() === 'host' || - key.toLowerCase() === 'connection' || - key.toLowerCase() === 'keep-alive' || - key.toLowerCase() === 'proxy-authenticate' || - key.toLowerCase() === 'proxy-authorization' || - key.toLowerCase() === 'te' || - key.toLowerCase() === 'trailers' || - key.toLowerCase() === 'transfer-encoding' || - key.toLowerCase() === 'upgrade' - ) { - continue; - } - - forwarded.set(key, value); - } - - // Add X-Forwarded headers - forwarded.set('X-Forwarded-For', headers.get('x-forwarded-for') || 'unknown'); - forwarded.set('X-Forwarded-Host', originalHost); - forwarded.set('X-Forwarded-Proto', isHttps ? 'https' : 'http'); - - return forwarded; - } - - /** - * Filter response headers - */ - private filterResponseHeaders(headers: Headers): Headers { - const filtered = new Headers(); - - for (const [key, value] of headers.entries()) { - // Skip problematic headers - if ( - key.toLowerCase() === 'connection' || - key.toLowerCase() === 'keep-alive' || - key.toLowerCase() === 'transfer-encoding' - ) { - continue; - } - - filtered.set(key, value); - } - - return filtered; + await this.caddy.stop(); + logger.info('Reverse proxy stopped'); } /** @@ -477,21 +106,24 @@ export class OneboxReverseProxy { throw new Error(`Service not found or has no container: ${serviceId}`); } - // Get container IP from Docker network, fallback to Docker DNS name + // Get container IP from Docker network let targetHost = 'localhost'; try { const containerIP = await this.oneboxRef.docker.getContainerIP(service.containerID); if (containerIP) { targetHost = containerIP; } else { - // Use Docker internal DNS name as fallback - targetHost = `onebox-${service.name}`; - logger.info(`Using Docker DNS name for ${service.name}: ${targetHost}`); + // Caddy runs on host, so we need the actual IP + // Try getting task IP from Swarm + const taskIP = await this.getSwarmTaskIP(service.containerID); + if (taskIP) { + targetHost = taskIP; + } else { + logger.warn(`Could not resolve IP for ${service.name}, using localhost`); + } } } catch (error) { logger.warn(`Could not resolve container IP for ${service.name}: ${getErrorMessage(error)}`); - // Fall back to Docker internal DNS name - targetHost = `onebox-${service.name}`; } const route: IProxyRoute = { @@ -503,18 +135,57 @@ export class OneboxReverseProxy { }; this.routes.set(domain, route); - logger.success(`Added proxy route: ${domain} -> ${targetHost}:${targetPort}`); + + // Add route to Caddy + const upstream = `${targetHost}:${targetPort}`; + await this.caddy.addRoute(domain, upstream); + + logger.success(`Added proxy route: ${domain} -> ${upstream}`); } catch (error) { logger.error(`Failed to add route for ${domain}: ${getErrorMessage(error)}`); throw error; } } + /** + * Get IP address for a Swarm task + */ + private async getSwarmTaskIP(containerIdOrTaskId: string): Promise { + try { + // Try to get task details from Swarm + const docker = this.oneboxRef.docker; + + // First, try to find the task by inspecting the container + const containerInfo = await docker.inspectContainer(containerIdOrTaskId); + if (containerInfo?.NetworkSettings?.Networks) { + // Get IP from the overlay network + for (const [networkName, networkInfo] of Object.entries(containerInfo.NetworkSettings.Networks)) { + if (networkName.includes('onebox') && (networkInfo as any).IPAddress) { + return (networkInfo as any).IPAddress; + } + } + // Fall back to any network + for (const networkInfo of Object.values(containerInfo.NetworkSettings.Networks)) { + if ((networkInfo as any).IPAddress) { + return (networkInfo as any).IPAddress; + } + } + } + return null; + } catch { + return null; + } + } + /** * Remove a route */ removeRoute(domain: string): void { if (this.routes.delete(domain)) { + // Remove from Caddy (async but we don't wait) + this.caddy.removeRoute(domain).catch((error) => { + logger.error(`Failed to remove Caddy route for ${domain}: ${getErrorMessage(error)}`); + }); logger.success(`Removed proxy route: ${domain}`); } else { logger.warn(`Route not found: ${domain}`); @@ -535,7 +206,9 @@ export class OneboxReverseProxy { try { logger.info('Reloading proxy routes...'); + // Clear local and Caddy routes this.routes.clear(); + this.caddy.clear(); const services = this.database.getAllServices(); @@ -553,46 +226,25 @@ export class OneboxReverseProxy { } /** - * Add TLS certificate for a domain (using PEM content) - * Dynamically adds SNI context if HTTPS server is already running + * Add TLS certificate for a domain + * Writes PEM files to disk for Caddy to load */ - addCertificate(domain: string, certPem: string, keyPem: string): void { + async addCertificate(domain: string, certPem: string, keyPem: string): Promise { if (!certPem || !keyPem) { logger.warn(`Cannot add certificate for ${domain}: missing PEM content`); return; } - this.tlsConfigs.set(domain, { - domain, - certPem, - keyPem, - }); - - logger.success(`Added TLS certificate for ${domain}`); - - // Dynamically add SNI context if HTTPS server is already running - if (this.httpsServer) { - this.httpsServer.addContext(domain, { - cert: certPem, - key: keyPem, - }); - this.httpsServer.addContext(`*.${domain}`, { - cert: certPem, - key: keyPem, - }); - logger.success(`Added SNI context for ${domain} and *.${domain}`); - } + await this.caddy.addCertificate(domain, certPem, keyPem); } /** * Remove TLS certificate for a domain */ removeCertificate(domain: string): void { - if (this.tlsConfigs.delete(domain)) { - logger.success(`Removed TLS certificate for ${domain}`); - } else { - logger.warn(`Certificate not found for domain: ${domain}`); - } + this.caddy.removeCertificate(domain).catch((error) => { + logger.error(`Failed to remove certificate for ${domain}: ${getErrorMessage(error)}`); + }); } /** @@ -602,33 +254,18 @@ export class OneboxReverseProxy { try { logger.info('Reloading TLS certificates from database...'); - this.tlsConfigs.clear(); - const certificates = this.database.getAllSSLCertificates(); for (const cert of certificates) { // Use fullchainPem for the cert (includes intermediates) and keyPem for the key if (cert.domain && cert.fullchainPem && cert.keyPem) { - this.addCertificate(cert.domain, cert.fullchainPem, cert.keyPem); + await this.caddy.addCertificate(cert.domain, cert.fullchainPem, cert.keyPem); } else { logger.warn(`Skipping certificate for ${cert.domain}: missing PEM content`); } } - logger.success(`Loaded ${this.tlsConfigs.size} TLS certificates`); - - // Restart HTTPS server if it was running (needed for full reload) - if (this.httpsServer) { - logger.info('Restarting HTTPS server with new certificates...'); - await new Promise((resolve, reject) => { - this.httpsServer!.close((err) => { - if (err) reject(err); - else resolve(); - }); - }); - this.httpsServer = null; - await this.startHttps(); - } + logger.success(`Loaded ${this.caddy.getCertificates().length} TLS certificates`); } catch (error) { logger.error(`Failed to reload certificates: ${getErrorMessage(error)}`); throw error; @@ -639,17 +276,19 @@ export class OneboxReverseProxy { * Get status of reverse proxy */ getStatus() { + const caddyStatus = this.caddy.getStatus(); return { http: { - running: this.httpServer !== null, - port: this.httpPort, + running: caddyStatus.running, + port: caddyStatus.httpPort, }, https: { - running: this.httpsServer !== null, - port: this.httpsPort, - certificates: this.tlsConfigs.size, + running: caddyStatus.running, + port: caddyStatus.httpsPort, + certificates: caddyStatus.certificates, }, - routes: this.routes.size, + routes: caddyStatus.routes, + backend: 'caddy', }; } } diff --git a/ui/src/app/app.routes.ts b/ui/src/app/app.routes.ts index 18542d6..f2e8952 100644 --- a/ui/src/app/app.routes.ts +++ b/ui/src/app/app.routes.ts @@ -77,6 +77,13 @@ export const routes: Routes = [ loadComponent: () => import('./features/dns/dns.component').then((m) => m.DnsComponent), }, + { + path: 'network', + loadComponent: () => + import('./features/network/network.component').then( + (m) => m.NetworkComponent + ), + }, { path: 'registries', loadComponent: () => diff --git a/ui/src/app/core/services/api.service.ts b/ui/src/app/core/services/api.service.ts index f88e1c7..19acb7b 100644 --- a/ui/src/app/core/services/api.service.ts +++ b/ui/src/app/core/services/api.service.ts @@ -20,6 +20,8 @@ import { IPlatformService, IPlatformResource, TPlatformServiceType, + INetworkTarget, + INetworkStats, } from '../types/api.types'; @Injectable({ providedIn: 'root' }) @@ -178,4 +180,13 @@ export class ApiService { async getServicePlatformResources(serviceName: string): Promise> { return firstValueFrom(this.http.get>(`/api/services/${serviceName}/platform-resources`)); } + + // Network + async getNetworkTargets(): Promise> { + return firstValueFrom(this.http.get>('/api/network/targets')); + } + + async getNetworkStats(): Promise> { + return firstValueFrom(this.http.get>('/api/network/stats')); + } } diff --git a/ui/src/app/core/services/network-log-stream.service.ts b/ui/src/app/core/services/network-log-stream.service.ts new file mode 100644 index 0000000..b70cef2 --- /dev/null +++ b/ui/src/app/core/services/network-log-stream.service.ts @@ -0,0 +1,187 @@ +import { Injectable, signal } from '@angular/core'; +import type { ICaddyAccessLog, INetworkLogMessage } from '../types/api.types'; + +export interface INetworkLogStreamState { + connected: boolean; + error: string | null; + clientId: string | null; +} + +export interface INetworkLogFilter { + domain?: string; + sampleRate?: number; +} + +@Injectable({ providedIn: 'root' }) +export class NetworkLogStreamService { + private ws: WebSocket | null = null; + private reconnectAttempts = 0; + private maxReconnectAttempts = 5; + private reconnectTimeout: ReturnType | null = null; + + // Signals for reactive state + state = signal({ + connected: false, + error: null, + clientId: null, + }); + + logs = signal([]); + isStreaming = signal(false); + filter = signal(null); + + /** + * Connect to network log stream + */ + connect(initialFilter?: INetworkLogFilter): void { + // Disconnect any existing stream + this.disconnect(); + + this.isStreaming.set(true); + this.logs.set([]); + this.filter.set(initialFilter || null); + this.state.set({ + connected: false, + error: null, + clientId: null, + }); + + if (typeof window === 'undefined') return; + + const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'; + const host = window.location.host; + let url = `${protocol}//${host}/api/network/logs/stream`; + + // Add initial filter as query params + if (initialFilter?.domain) { + url += `?domain=${encodeURIComponent(initialFilter.domain)}`; + } + + try { + this.ws = new WebSocket(url); + + this.ws.onopen = () => { + this.reconnectAttempts = 0; + }; + + this.ws.onmessage = (event) => { + try { + const message = JSON.parse(event.data) as INetworkLogMessage; + + if (message.type === 'connected') { + this.state.set({ + connected: true, + error: null, + clientId: message.clientId || null, + }); + if (message.filter) { + this.filter.set(message.filter); + } + return; + } + + if (message.type === 'filter_updated') { + this.filter.set(message.filter || null); + return; + } + + if (message.type === 'access_log' && message.data) { + this.logs.update((lines) => { + const newLines = [...lines, message.data!]; + // Keep last 500 logs to prevent memory issues + if (newLines.length > 500) { + return newLines.slice(-500); + } + return newLines; + }); + } + } catch (error) { + console.error('Failed to parse network log message:', error); + } + }; + + this.ws.onclose = () => { + this.state.update((s) => ({ ...s, connected: false })); + this.ws = null; + + // Auto-reconnect with exponential backoff + if (this.isStreaming() && this.reconnectAttempts < this.maxReconnectAttempts) { + const delay = Math.min(1000 * Math.pow(2, this.reconnectAttempts), 30000); + this.reconnectAttempts++; + this.reconnectTimeout = setTimeout(() => { + this.connect(this.filter() || undefined); + }, delay); + } else { + this.isStreaming.set(false); + } + }; + + this.ws.onerror = () => { + this.state.update((s) => ({ + ...s, + connected: false, + error: 'WebSocket connection failed', + })); + }; + } catch (error) { + this.state.set({ + connected: false, + error: 'Failed to connect to network log stream', + clientId: null, + }); + this.isStreaming.set(false); + } + } + + /** + * Disconnect from log stream + */ + disconnect(): void { + if (this.reconnectTimeout) { + clearTimeout(this.reconnectTimeout); + this.reconnectTimeout = null; + } + + if (this.ws) { + this.ws.close(); + this.ws = null; + } + + this.isStreaming.set(false); + this.reconnectAttempts = 0; + this.state.set({ + connected: false, + error: null, + clientId: null, + }); + } + + /** + * Update filter on existing connection + */ + setFilter(newFilter: INetworkLogFilter | null): void { + this.filter.set(newFilter); + + if (this.ws && this.ws.readyState === WebSocket.OPEN) { + this.ws.send(JSON.stringify({ + type: 'set_filter', + domain: newFilter?.domain, + sampleRate: newFilter?.sampleRate, + })); + } + } + + /** + * Clear logs buffer + */ + clearLogs(): void { + this.logs.set([]); + } + + /** + * Check if connected + */ + isConnected(): boolean { + return this.state().connected; + } +} diff --git a/ui/src/app/core/types/api.types.ts b/ui/src/app/core/types/api.types.ts index 2d1c4e1..b7a3d35 100644 --- a/ui/src/app/core/types/api.types.ts +++ b/ui/src/app/core/types/api.types.ts @@ -236,3 +236,55 @@ export interface IPlatformResource { envVars: Record; createdAt: number; } + +// Network Types +export type TNetworkTargetType = 'service' | 'registry' | 'platform'; + +export interface INetworkTarget { + type: TNetworkTargetType; + name: string; + domain: string | null; + targetHost: string; + targetPort: number; + status: string; +} + +export interface INetworkStats { + proxy: { + running: boolean; + httpPort: number; + httpsPort: number; + routes: number; + certificates: number; + }; + logReceiver: { + running: boolean; + port: number; + clients: number; + connections: number; + sampleRate: number; + recentLogsCount: number; + }; +} + +export interface ICaddyAccessLog { + ts: number; + request: { + remote_ip: string; + method: string; + host: string; + uri: string; + proto: string; + }; + status: number; + duration: number; + size: number; +} + +export interface INetworkLogMessage { + type: 'connected' | 'access_log' | 'filter_updated'; + clientId?: string; + filter?: { domain?: string; sampleRate?: number }; + data?: ICaddyAccessLog; + timestamp: number; +} diff --git a/ui/src/app/features/network/network.component.ts b/ui/src/app/features/network/network.component.ts new file mode 100644 index 0000000..b25e6b2 --- /dev/null +++ b/ui/src/app/features/network/network.component.ts @@ -0,0 +1,388 @@ +import { Component, inject, signal, OnInit, OnDestroy, ViewChild, ElementRef, effect } from '@angular/core'; +import { ApiService } from '../../core/services/api.service'; +import { ToastService } from '../../core/services/toast.service'; +import { NetworkLogStreamService } from '../../core/services/network-log-stream.service'; +import type { INetworkTarget, INetworkStats, ICaddyAccessLog } from '../../core/types/api.types'; +import { + CardComponent, + CardHeaderComponent, + CardTitleComponent, + CardDescriptionComponent, + CardContentComponent, +} from '../../ui/card/card.component'; +import { ButtonComponent } from '../../ui/button/button.component'; +import { BadgeComponent } from '../../ui/badge/badge.component'; +import { SkeletonComponent } from '../../ui/skeleton/skeleton.component'; +import { + TableComponent, + TableHeaderComponent, + TableBodyComponent, + TableRowComponent, + TableHeadComponent, + TableCellComponent, +} from '../../ui/table/table.component'; + +@Component({ + selector: 'app-network', + standalone: true, + imports: [ + CardComponent, + CardHeaderComponent, + CardTitleComponent, + CardDescriptionComponent, + CardContentComponent, + ButtonComponent, + BadgeComponent, + SkeletonComponent, + TableComponent, + TableHeaderComponent, + TableBodyComponent, + TableRowComponent, + TableHeadComponent, + TableCellComponent, + ], + template: ` +
+ +
+
+

Network

+

Traffic targets and access logs

+
+ +
+ + @if (loading() && !stats()) { + +
+ @for (_ of [1,2,3,4]; track $index) { + + + + + + + + + } +
+ } @else if (stats()) { + +
+ + + Proxy Status + + + + + + + {{ stats()!.proxy.running ? 'Running' : 'Stopped' }} + + + + + + + Routes + + + + + +
{{ stats()!.proxy.routes }}
+ + + + + + Certificates + + + + + +
{{ stats()!.proxy.certificates }}
+ + + + + + Targets + + + + + +
{{ targets().length }}
+ + +
+ } + + + + + Traffic Targets + Services, registry, and platform services with their routing info. Click to filter logs. + + + @if (targets().length === 0 && !loading()) { +

No traffic targets configured

+ } @else { + + + + Type + Name + Domain + Target + Status + + + + @for (target of targets(); track target.name) { + + + {{ target.type }} + + {{ target.name }} + + @if (target.domain) { + {{ target.domain }} + } @else { + - + } + + {{ target.targetHost }}:{{ target.targetPort }} + + {{ target.status }} + + + } + + + } + + + + + + +
+ Access Logs + + @if (networkLogStream.isStreaming()) { + + + + + + Live streaming + @if (activeFilter()) { + - filtered by {{ activeFilter() }} + } + + } @else { + Real-time Caddy access logs + } + +
+
+ @if (activeFilter()) { + + } + @if (networkLogStream.isStreaming()) { + + } @else { + + } + +
+ + +
+ @if (networkLogStream.state().error) { +

Error: {{ networkLogStream.state().error }}

+ } @else if (networkLogStream.logs().length > 0) { + @for (log of networkLogStream.logs(); track $index) { +
+ {{ formatLog(log) }} +
+ } + } @else if (networkLogStream.isStreaming()) { +

Waiting for access logs...

+ } @else { +

Click "Stream" to start live access log streaming

+ } +
+ + +
+ `, +}) +export class NetworkComponent implements OnInit, OnDestroy { + private api = inject(ApiService); + private toast = inject(ToastService); + networkLogStream = inject(NetworkLogStreamService); + + @ViewChild('logContainer') logContainer!: ElementRef; + + targets = signal([]); + stats = signal(null); + loading = signal(false); + activeFilter = signal(null); + + constructor() { + // Auto-scroll when new logs arrive + effect(() => { + const logs = this.networkLogStream.logs(); + if (logs.length > 0 && this.logContainer?.nativeElement) { + setTimeout(() => { + const container = this.logContainer.nativeElement; + container.scrollTop = container.scrollHeight; + }, 0); + } + }); + } + + ngOnInit(): void { + this.loadData(); + } + + ngOnDestroy(): void { + this.networkLogStream.disconnect(); + } + + async loadData(): Promise { + this.loading.set(true); + try { + const [targetsResponse, statsResponse] = await Promise.all([ + this.api.getNetworkTargets(), + this.api.getNetworkStats(), + ]); + + if (targetsResponse.success && targetsResponse.data) { + this.targets.set(targetsResponse.data); + } + + if (statsResponse.success && statsResponse.data) { + this.stats.set(statsResponse.data); + } + } catch (err) { + this.toast.error('Failed to load network data'); + } finally { + this.loading.set(false); + } + } + + onTargetClick(target: INetworkTarget): void { + if (target.domain) { + this.activeFilter.set(target.domain); + this.networkLogStream.setFilter({ domain: target.domain }); + + // Start streaming if not already + if (!this.networkLogStream.isStreaming()) { + this.startLogStream(); + } + } + } + + clearFilter(): void { + this.activeFilter.set(null); + this.networkLogStream.setFilter(null); + } + + startLogStream(): void { + const filter = this.activeFilter() ? { domain: this.activeFilter()! } : undefined; + this.networkLogStream.connect(filter); + } + + stopLogStream(): void { + this.networkLogStream.disconnect(); + } + + clearLogs(): void { + this.networkLogStream.clearLogs(); + } + + getTypeVariant(type: string): 'default' | 'secondary' | 'outline' { + switch (type) { + case 'service': return 'default'; + case 'registry': return 'secondary'; + case 'platform': return 'outline'; + default: return 'secondary'; + } + } + + getStatusVariant(status: string): 'success' | 'destructive' | 'warning' | 'secondary' { + switch (status) { + case 'running': return 'success'; + case 'stopped': return 'secondary'; + case 'failed': return 'destructive'; + case 'starting': + case 'stopping': return 'warning'; + default: return 'secondary'; + } + } + + getLogClass(status: number): string { + if (status >= 500) return 'text-red-400'; + if (status >= 400) return 'text-yellow-400'; + if (status >= 300) return 'text-blue-400'; + return 'text-green-400'; + } + + formatLog(log: ICaddyAccessLog): string { + const time = new Date(log.ts * 1000).toLocaleTimeString(); + const duration = log.duration < 1 ? `${(log.duration * 1000).toFixed(1)}ms` : `${log.duration.toFixed(2)}s`; + const size = this.formatBytes(log.size); + const method = log.request.method.padEnd(7); + const status = String(log.status).padStart(3); + const host = log.request.host.substring(0, 30).padEnd(30); + const uri = log.request.uri.substring(0, 40); + + return `${time} ${status} ${method} ${host} ${uri.padEnd(40)} ${duration.padStart(8)} ${size.padStart(8)} ${log.request.remote_ip}`; + } + + formatBytes(bytes: number): string { + if (bytes === 0) return '0 B'; + const k = 1024; + const sizes = ['B', 'KB', 'MB', 'GB']; + const i = Math.floor(Math.log(bytes) / Math.log(k)); + return `${(bytes / Math.pow(k, i)).toFixed(1)} ${sizes[i]}`; + } +} diff --git a/ui/src/app/shared/components/layout/layout.component.ts b/ui/src/app/shared/components/layout/layout.component.ts index af5f60c..bd056dd 100644 --- a/ui/src/app/shared/components/layout/layout.component.ts +++ b/ui/src/app/shared/components/layout/layout.component.ts @@ -118,6 +118,7 @@ export class LayoutComponent { navItems: NavItem[] = [ { label: 'Dashboard', path: '/dashboard', icon: 'home' }, { label: 'Services', path: '/services', icon: 'server' }, + { label: 'Network', path: '/network', icon: 'activity' }, { label: 'Registries', path: '/registries', icon: 'database' }, { label: 'Tokens', path: '/tokens', icon: 'key' }, { label: 'DNS', path: '/dns', icon: 'globe' },