Compare commits

...

30 Commits

Author SHA1 Message Date
jkunz dd0cd479d5 v4.17.0
Default (tags) / security (push) Failing after 1s
Default (tags) / test (push) Failing after 0s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-04-26 12:09:58 +00:00
jkunz e709e40404 feat(core): add performance profiles, transport observability, and edge stream budget controls 2026-04-26 12:09:58 +00:00
jkunz 5304bbb486 v4.15.3
Default (tags) / security (push) Failing after 1s
Default (tags) / test (push) Failing after 0s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-03-27 11:34:31 +00:00
jkunz ac993dd5a3 fix(core): harden UDP session handling, QUIC control message validation, and bridge process cleanup 2026-03-27 11:34:31 +00:00
jkunz 0b2a83ddb6 v4.15.2
Default (tags) / security (push) Failing after 0s
Default (tags) / test (push) Failing after 0s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-03-26 17:06:28 +00:00
jkunz 3c5ea6bdc5 fix(readme): adjust tunnel diagram alignment in the README 2026-03-26 17:06:28 +00:00
jkunz 3dea43400b v4.15.1
Default (tags) / security (push) Failing after 1s
Default (tags) / test (push) Failing after 0s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-03-26 17:05:38 +00:00
jkunz 8fa3d414dd fix(readme): clarify unified runtime configuration and firewall update behavior 2026-03-26 17:05:38 +00:00
jkunz 1a62c52d24 v4.15.0
Default (tags) / security (push) Failing after 1s
Default (tags) / test (push) Failing after 0s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-03-26 16:39:53 +00:00
jkunz e9a08bdd0f feat(edge,hub): add hub-controlled nftables firewall configuration for remote ingress edges 2026-03-26 16:39:53 +00:00
jkunz c2c9dd195d v4.14.3
Default (tags) / security (push) Failing after 1s
Default (tags) / test (push) Failing after 1s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-03-26 07:49:44 +00:00
jkunz fb6e9c54ad fix(docs): refresh project metadata and README to reflect current ingress tunnel capabilities 2026-03-26 07:49:44 +00:00
jkunz ac22617849 v4.14.2
Default (tags) / security (push) Failing after 1s
Default (tags) / test (push) Failing after 0s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-03-26 07:10:15 +00:00
jkunz e5a91f298c fix(hub-core): improve stream shutdown handling and connection cleanup in hub and edge 2026-03-26 07:10:15 +00:00
jkunz 5e93710c42 v4.14.1
Default (tags) / security (push) Failing after 0s
Default (tags) / test (push) Failing after 0s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-03-21 19:49:05 +00:00
jkunz 331b5c8d3f fix(remoteingress edge/hub crash recovery): prevent duplicate crash recovery listeners and reset saved runtime state on shutdown 2026-03-21 19:49:05 +00:00
jkunz bf3418d0ed v4.14.0
Default (tags) / security (push) Failing after 1s
Default (tags) / test (push) Failing after 0s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-03-20 00:51:37 +00:00
jkunz 6d5e6f60f8 feat(quic): add QUIC stability test coverage and bridge logging for hub and edge 2026-03-20 00:51:37 +00:00
jkunz de8922148e v4.13.2
Default (tags) / security (push) Failing after 1s
Default (tags) / test (push) Failing after 1s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-03-20 00:11:34 +00:00
jkunz e84eecf82c fix(remoteingress-core): preserve reconnected edge entries during disconnect cleanup 2026-03-20 00:11:34 +00:00
jkunz c7641853cf v4.13.1
Default (tags) / security (push) Failing after 0s
Default (tags) / test (push) Failing after 0s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-03-19 23:28:38 +00:00
jkunz 6e2025db3e fix(remoteingress-core): default edge transport mode to QUIC with fallback 2026-03-19 23:28:38 +00:00
jkunz 693031ecdd v4.13.0
Default (tags) / security (push) Failing after 0s
Default (tags) / test (push) Failing after 1s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-03-19 14:43:42 +00:00
jkunz a2cdadc5e3 feat(docs): document TCP and UDP tunneling over TLS and QUIC 2026-03-19 14:43:42 +00:00
jkunz 948032fc9e v4.12.1
Default (tags) / security (push) Failing after 0s
Default (tags) / test (push) Failing after 1s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-03-19 14:09:32 +00:00
jkunz a400945371 fix(remoteingress-core): send PROXY v2 headers for UDP upstream sessions and expire idle UDP sessions 2026-03-19 14:09:32 +00:00
jkunz bc89e49f39 v4.12.0
Default (tags) / security (push) Failing after 4s
Default (tags) / test (push) Failing after 1s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-03-19 12:19:58 +00:00
jkunz 2087567f15 feat(remoteingress-core): add UDP tunneling over QUIC datagrams and expand transport-specific test coverage 2026-03-19 12:19:58 +00:00
jkunz bfa88f8d76 v4.11.0
Default (tags) / security (push) Failing after 0s
Default (tags) / test (push) Failing after 0s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-03-19 12:02:41 +00:00
jkunz a96b4ba84a feat(remoteingress-core): add UDP tunneling support between edge and hub 2026-03-19 12:02:41 +00:00
26 changed files with 4522 additions and 1554 deletions
+14 -14
View File
@@ -11,26 +11,26 @@
"githost": "code.foss.global",
"gitscope": "serve.zone",
"gitrepo": "remoteingress",
"description": "Provides a service for creating private tunnels and reaching private clusters from the outside, facilitating secure remote access as part of the @serve.zone stack.",
"description": "Edge ingress tunnel for DcRouter - tunnels TCP and UDP traffic from the network edge to SmartProxy over TLS or QUIC, preserving client IP via PROXY protocol.",
"npmPackagename": "@serve.zone/remoteingress",
"license": "MIT",
"projectDomain": "serve.zone",
"keywords": [
"remote access",
"private tunnels",
"network security",
"TLS encryption",
"connector",
"ingress tunnel",
"network edge",
"PROXY protocol",
"multiplexed tunnel",
"TCP proxy",
"TLS tunnel",
"QUIC transport",
"UDP tunneling",
"serve.zone stack",
"private clusters access",
"public access management",
"TypeScript application",
"node.js package",
"secure communications",
"TLS/SSL certificates",
"development tools",
"software development",
"private network integration"
"TypeScript",
"Rust",
"SmartProxy",
"DcRouter",
"flow control"
]
},
"release": {
+1 -1
View File
@@ -1,7 +1,7 @@
{
"json.schemas": [
{
"fileMatch": ["/npmextra.json"],
"fileMatch": ["/.smartconfig.json"],
"schema": {
"type": "object",
"properties": {
+116 -1
View File
@@ -1,5 +1,120 @@
# Changelog
## 2026-04-26 - 4.17.0 - feat(core)
add performance profiles, transport observability, and edge stream budget controls
- introduce configurable performance profiles and effective per-edge limits for stream concurrency, flow-control windows, and QUIC datagram buffers
- expose hub-side edge status for transport mode, fallback usage, flow-control, queue depths, traffic counters, and UDP session metrics
- enforce edge-side stream admission limits before spawning client tunnel tasks and make TCP/TLS window sizing honor edge memory budgets under high concurrency
- increase QUIC datagram receive buffer configurability and improve hub-side QUIC UDP session tracking and idle pruning
- update hub APIs and documentation to support performance configuration and clarify quicWithFallback as the default edge transport
## 2026-04-26 - 4.16.0 - feat(performance)
add remote ingress performance controls and runtime observability
- add performance profiles and configurable stream/window budgets for hub and edge connections
- expose per-edge transport, flow-control, queue, traffic, and UDP status from hub status
- enforce edge-side stream admission before spawning client tunnel tasks
- make TCP/TLS flow control honor an edge-level memory budget under high concurrency
- increase QUIC datagram receive buffers and prune idle hub-side QUIC UDP sessions
## 2026-03-27 - 4.15.3 - fix(core)
harden UDP session handling, QUIC control message validation, and bridge process cleanup
- cap UDP session creation and drop excess datagrams with warnings to prevent unbounded session growth
- periodically prune closed datagram sessions on the hub and reject oversized QUIC control messages to avoid resource exhaustion
- clean up spawned edge and hub bridge processes on startup failure, remove listeners on stop, and avoid restarting after shutdown during backoff
## 2026-03-26 - 4.15.2 - fix(readme)
adjust tunnel diagram alignment in the README
- Improves formatting consistency in the Hub/Edge topology diagram.
## 2026-03-26 - 4.15.1 - fix(readme)
clarify unified runtime configuration and firewall update behavior
- Updates the architecture and feature descriptions to reflect that ports, firewall rules, and rate limits are pushed together in a single config update
- Clarifies that firewall configuration is delivered via FRAME_CONFIG on handshake and subsequent updates, with atomic full-rule replacement at the edge
- Simplifies and reorganizes README wording around edge and hub responsibilities without changing implementation behavior
## 2026-03-26 - 4.15.0 - feat(edge,hub)
add hub-controlled nftables firewall configuration for remote ingress edges
- add firewallConfig support to allowed edge definitions, handshake responses, and runtime config updates
- emit firewallConfigUpdated events from the Rust bridge and edge runtime when firewall settings change
- initialize SmartNftables on edges, apply blocked IPs, rate limits, and custom rules, and clean up nftables rules on stop
- document centralized firewall management, root requirements, and new edge events in the README
## 2026-03-26 - 4.14.3 - fix(docs)
refresh project metadata and README to reflect current ingress tunnel capabilities
- update package metadata description and keywords to better describe edge ingress, TLS/QUIC transport, and SmartProxy integration
- revise README terminology, API docs, and feature list to document crash recovery, bindAddress support, and current event names
- improve README formatting and examples for architecture, wire protocol, QoS, and token usage
## 2026-03-26 - 4.14.2 - fix(hub-core)
improve stream shutdown handling and connection cleanup in hub and edge
- Cancel edge upload loops immediately when the hub closes a stream instead of waiting for the window stall timeout.
- Reduce stalled stream timeouts from 120s to 55s to detect broken connections faster.
- Allow hub writer tasks to shut down gracefully before aborting to avoid unnecessary TCP resets.
- Enable TCP keepalive on hub upstream connections to detect silent SmartProxy failures.
- Remove leaked QUIC UDP session entries when setup fails or sessions end.
- Rename npmextra.json to .smartconfig.json and update package packaging references.
## 2026-03-21 - 4.14.1 - fix(remoteingress edge/hub crash recovery)
prevent duplicate crash recovery listeners and reset saved runtime state on shutdown
- Removes existing exit listeners before re-registering crash recovery handlers for edge and hub processes.
- Clears saved edge and hub configuration on stop to avoid stale restart state.
- Resets orphaned edge status intervals and restarts periodic status logging after successful crash recovery.
## 2026-03-20 - 4.14.0 - feat(quic)
add QUIC stability test coverage and bridge logging for hub and edge
- adds a long-running QUIC stability test with periodic echo probes and disconnect detection
- enables prefixed bridge logging for RemoteIngressHub and RemoteIngressEdge to improve runtime diagnostics
## 2026-03-20 - 4.13.2 - fix(remoteingress-core)
preserve reconnected edge entries during disconnect cleanup
- Guard edge removal so disconnect handlers only delete entries whose cancel token is already cancelled
- Prevents stale TCP and QUIC disconnect paths from removing a newer connection after an edge reconnects
## 2026-03-19 - 4.13.1 - fix(remoteingress-core)
default edge transport mode to QUIC with fallback
- Changes the default transport mode in edge connections from TCP/TLS to QUIC with fallback when no transport mode is explicitly configured.
## 2026-03-19 - 4.13.0 - feat(docs)
document TCP and UDP tunneling over TLS and QUIC
- update package description to reflect TCP and UDP support and TLS or QUIC transports
- refresh README architecture, features, and usage examples for UDP forwarding, QUIC transport, and PROXY protocol v1/v2 support
## 2026-03-19 - 4.12.1 - fix(remoteingress-core)
send PROXY v2 headers for UDP upstream sessions and expire idle UDP sessions
- Adds periodic idle UDP session expiry in edge tunnel and QUIC loops, including UDP close signaling for expired tunnel sessions.
- Sends the PROXY v2 header as the first datagram for UDP upstream connections in both standard and QUIC hub paths.
- Updates the UDP node test server to ignore the initial PROXY v2 datagram per source before echoing payload traffic.
## 2026-03-19 - 4.12.0 - feat(remoteingress-core)
add UDP tunneling over QUIC datagrams and expand transport-specific test coverage
- Implement QUIC datagram-based UDP forwarding on both edge and hub, including session setup, payload routing, and listener cleanup
- Enable QUIC datagram receive buffers in client and server transport configuration
- Add UDP-over-QUIC tests and clarify existing test names to distinguish TCP/TLS, UDP/TLS, and QUIC scenarios
## 2026-03-19 - 4.11.0 - feat(remoteingress-core)
add UDP tunneling support between edge and hub
- extend edge and hub handshake/config updates with UDP listen ports
- add UDP tunnel frame types and PROXY protocol v2 header helpers in the protocol crate
- introduce UDP session management on the edge and upstream UDP forwarding on the hub
- add Node.js integration tests covering UDP echo and concurrent datagrams
- expose UDP listen port configuration in the TypeScript hub API
## 2026-03-19 - 4.10.0 - feat(core,edge,hub,transport)
add QUIC tunnel transport support with optional edge transport selection
@@ -419,4 +534,4 @@ Core updates and fixes.
## 2024-03-24 - 1.0.1 - core
Core updates and fixes.
- fix(core): update
- fix(core): update
+21
View File
@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2024 Task Venture Capital GmbH
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
+11 -10
View File
@@ -1,8 +1,8 @@
{
"name": "@serve.zone/remoteingress",
"version": "4.10.0",
"version": "4.17.0",
"private": false,
"description": "Edge ingress tunnel for DcRouter - accepts incoming TCP connections at network edge and tunnels them to DcRouter SmartProxy preserving client IP via PROXY protocol v1.",
"description": "Edge ingress tunnel for DcRouter - tunnels TCP and UDP traffic from the network edge to SmartProxy over TLS or QUIC, preserving client IP via PROXY protocol.",
"main": "dist_ts/index.js",
"typings": "dist_ts/index.d.ts",
"type": "module",
@@ -14,17 +14,18 @@
"buildDocs": "(tsdoc)"
},
"devDependencies": {
"@git.zone/tsbuild": "^4.1.2",
"@git.zone/tsbundle": "^2.8.3",
"@git.zone/tsrun": "^2.0.1",
"@git.zone/tsrust": "^1.3.0",
"@git.zone/tstest": "^3.1.8",
"@git.zone/tsbuild": "^4.4.0",
"@git.zone/tsbundle": "^2.10.0",
"@git.zone/tsrun": "^2.0.2",
"@git.zone/tsrust": "^1.3.2",
"@git.zone/tstest": "^3.6.0",
"@push.rocks/tapbundle": "^6.0.3",
"@types/node": "^25.3.0"
"@types/node": "^25.5.0"
},
"dependencies": {
"@push.rocks/qenv": "^6.1.3",
"@push.rocks/smartrust": "^1.2.1"
"@push.rocks/smartnftables": "^1.0.1",
"@push.rocks/smartrust": "^1.3.2"
},
"repository": {
"type": "git",
@@ -47,7 +48,7 @@
"dist_rust/**/*",
"assets/**/*",
"cli.js",
"npmextra.json",
".smartconfig.json",
"readme.md"
],
"keywords": [
+1561 -1175
View File
File diff suppressed because it is too large Load Diff
+268 -169
View File
@@ -1,6 +1,6 @@
# @serve.zone/remoteingress
Edge ingress tunnel for DcRouter — accepts incoming TCP connections at the network edge and tunnels them over a single encrypted TLS connection to a DcRouter SmartProxy instance, preserving the original client IP via PROXY protocol v1.
Edge ingress tunnel for DcRouter — tunnels **TCP and UDP** traffic from the network edge to a private DcRouter/SmartProxy cluster over encrypted TLS or QUIC connections, preserving the original client IP via PROXY protocol. Includes **hub-controlled nftables firewall** for IP blocking, rate limiting, and custom firewall rules applied directly at the edge.
## Issue Reporting and Security
@@ -12,48 +12,54 @@ For reporting bugs, issues, or security vulnerabilities, please visit [community
pnpm install @serve.zone/remoteingress
```
## 🏗️ Architecture
## Architecture
`@serve.zone/remoteingress` uses a **Hub/Edge** topology with a high-performance Rust core and a TypeScript API surface:
```
┌─────────────────────┐ TLS Tunnel ┌─────────────────────┐
│ Network Edge │ ◄══════════════════════════► │ Private Cluster │
(multiplexed frames +
RemoteIngressEdge shared-secret auth) │ RemoteIngressHub
Accepts client TCP │ │ Forwards to
connections on │ │ SmartProxy on
hub-assigned ports │ │ local ports
TLS or QUIC Tunnel
┌─────────────────────┐ ◄══════════════════════════► ┌─────────────────────┐
Network EdgeTCP+TLS: frame mux Private Cluster
│ QUIC: native streams │
RemoteIngressEdge │ UDP: QUIC datagrams │ RemoteIngressHub
│ │
• TCP/UDP listeners│ ◄─── FRAME_CONFIG pushes ─── │ • Port assignments
│ • nftables firewall│ ports + firewall rules │ • Firewall config │
│ • Rate limiting │ at any time │ • Rate limit rules │
└─────────────────────┘ └─────────────────────┘
▲ │
│ TCP from end users
│ TCP + UDP from end users ▼
Internet DcRouter / SmartProxy
```
| Component | Role |
|-----------|------|
| **RemoteIngressEdge** | Deployed at the network edge (e.g. a VPS or cloud instance). Listens on ports assigned by the hub, accepts raw TCP connections, and multiplexes them over a single TLS tunnel to the hub. Ports are hot-reloadable — the hub can change them at runtime. |
| **RemoteIngressHub** | Deployed alongside DcRouter/SmartProxy in a private cluster. Accepts edge connections, demuxes streams, and forwards each to SmartProxy with a PROXY protocol v1 header so the real client IP is preserved. Controls which ports each edge listens on. |
| **RemoteIngressEdge** | Deployed at the network edge (VPS, cloud instance). Runs as root. Listens on hub-assigned TCP/UDP ports, tunnels traffic to the hub, and applies hub-pushed nftables rules (IP blocking, rate limiting). All config is hot-reloadable at runtime. |
| **RemoteIngressHub** | Deployed alongside DcRouter/SmartProxy in a private cluster. Accepts edge connections, demuxes streams/datagrams, and forwards each to SmartProxy with PROXY protocol headers so the real client IP is preserved. Pushes all edge config (ports, firewall) via a single API. |
| **Rust Binary** (`remoteingress-bin`) | The performance-critical networking core. Managed via `@push.rocks/smartrust` RustBridge IPC — you never interact with it directly. Cross-compiled for `linux/amd64` and `linux/arm64`. |
### Key Features
### Key Features
- 🔒 **TLS-encrypted tunnel** between edge and hub (auto-generated self-signed cert or bring your own)
- 🔀 **Multiplexed streams** — thousands of client connections flow over a single tunnel
- 🌐 **PROXY protocol v1** — SmartProxy sees the real client IP, not the tunnel IP
- 🔑 **Shared-secret authentication** — edges must present valid credentials to connect
- 🎫 **Connection tokens**encode all connection details into a single opaque string
- 📡 **STUN-based public IP discovery**the edge automatically discovers its public IP via Cloudflare STUN
- 🔄 **Auto-reconnect** with exponential backoff if the tunnel drops
- 🎛️ **Dynamic port configuration**the hub assigns listen ports per edge and can hot-reload them at runtime via `FRAME_CONFIG` frames
- 📣 **Event-driven**both Hub and Edge extend `EventEmitter` for real-time monitoring
- **Rust core** — all frame encoding, TLS, and TCP proxying happen in native code for maximum throughput
- 🎚️ **3-tier QoS**control frames, normal data, and sustained (elephant flow) traffic each get their own priority queue
- 📊 **Adaptive flow control**per-stream windows scale with active stream count to prevent memory overuse
- **Dual transport** — choose between TCP+TLS (frame-multiplexed) or QUIC (native stream multiplexing, zero head-of-line blocking)
- **TCP + UDP tunneling** — tunnel any TCP connection or UDP datagram through the same edge/hub pair
- **PROXY protocol v1 & v2** — SmartProxy sees the real client IP for both TCP (v1 text) and UDP (v2 binary)
- **Hub-controlled firewall** — push nftables rules (IP blocking, rate limiting, custom firewall rules) to edges as part of the same config update that assigns ports — powered by `@push.rocks/smartnftables`
- **Multiplexed streams** — thousands of concurrent TCP connections over a single tunnel
- **QUIC datagrams** — UDP traffic forwarded via QUIC unreliable datagrams for lowest possible latency
- **Shared-secret authentication** — edges must present valid credentials to connect
- **Connection tokens** — encode all connection details into a single opaque base64url string
- **STUN-based public IP discovery** — edges automatically discover their public IP via Cloudflare STUN
- **Auto-reconnect** with exponential backoff if the tunnel drops
- **Dynamic runtime configuration** — the hub pushes ports, firewall rules, and rate limits to edges at any time via a single `updateAllowedEdges()` call
- **Event-driven** — both Hub and Edge extend `EventEmitter` for real-time monitoring
- **3-tier QoS** — control frames, normal data, and sustained (elephant flow) traffic each get their own priority queue
- **Adaptive flow control** — per-stream windows scale with active stream count to prevent memory overuse
- **UDP session management** — automatic session tracking with 60s idle timeout and cleanup
- **Crash recovery** — automatic restart with exponential backoff if the Rust binary crashes unexpectedly
## 🚀 Usage
## Usage
Both classes are imported from the package and communicate with the Rust binary under the hood. All you need to do is configure and start them.
Both classes are imported from the package and communicate with the Rust binary under the hood.
### Setting Up the Hub (Private Cluster Side)
@@ -63,32 +69,34 @@ import { RemoteIngressHub } from '@serve.zone/remoteingress';
const hub = new RemoteIngressHub();
// Listen for events
hub.on('edgeConnected', ({ edgeId }) => {
console.log(`Edge ${edgeId} connected`);
});
hub.on('edgeDisconnected', ({ edgeId }) => {
console.log(`Edge ${edgeId} disconnected`);
});
hub.on('streamOpened', ({ edgeId, streamId }) => {
console.log(`Stream ${streamId} opened from edge ${edgeId}`);
});
hub.on('streamClosed', ({ edgeId, streamId }) => {
console.log(`Stream ${streamId} closed from edge ${edgeId}`);
});
hub.on('edgeConnected', ({ edgeId }) => console.log(`Edge ${edgeId} connected`));
hub.on('edgeDisconnected', ({ edgeId }) => console.log(`Edge ${edgeId} disconnected`));
hub.on('streamOpened', ({ edgeId, streamId }) => console.log(`Stream ${streamId} from ${edgeId}`));
hub.on('streamClosed', ({ edgeId, streamId }) => console.log(`Stream ${streamId} closed`));
// Start the hub — it will listen for incoming edge TLS connections
// Start the hub — listens for edge connections on both TCP and QUIC (same port)
await hub.start({
tunnelPort: 8443, // port edges connect to (default: 8443)
targetHost: '127.0.0.1', // SmartProxy host to forward streams to (default: 127.0.0.1)
targetHost: '127.0.0.1', // SmartProxy host to forward traffic to
});
// Register which edges are allowed to connect, including their listen ports
// Register allowed edges with TCP and UDP listen ports + firewall config
await hub.updateAllowedEdges([
{
id: 'edge-nyc-01',
secret: 'supersecrettoken1',
listenPorts: [80, 443], // ports the edge should listen on
stunIntervalSecs: 300, // STUN discovery interval (default: 300)
listenPorts: [80, 443], // TCP ports the edge should listen on
listenPortsUdp: [53, 51820], // UDP ports (e.g., DNS, WireGuard)
stunIntervalSecs: 300,
firewallConfig: {
blockedIps: ['192.168.1.100', '10.0.0.0/8'],
rateLimits: [
{ id: 'http-rate', port: 80, protocol: 'tcp', rate: '100/second', perSourceIP: true },
],
rules: [
{ id: 'allow-ssh', direction: 'input', action: 'accept', sourceIP: '10.0.0.0/24', destPort: 22, protocol: 'tcp' },
],
},
},
{
id: 'edge-fra-02',
@@ -97,38 +105,35 @@ await hub.updateAllowedEdges([
},
]);
// Dynamically update ports for a connected edge — changes are pushed instantly
// Dynamically update ports and firewall — changes are pushed instantly to connected edges
await hub.updateAllowedEdges([
{
id: 'edge-nyc-01',
secret: 'supersecrettoken1',
listenPorts: [80, 443, 8443], // added port 8443 — edge picks it up in real time
listenPorts: [80, 443, 8443], // added TCP port 8443
listenPortsUdp: [53], // removed WireGuard UDP port
firewallConfig: {
blockedIps: ['192.168.1.100', '10.0.0.0/8', '203.0.113.50'], // added new blocked IP
rateLimits: [
{ id: 'http-rate', port: 80, protocol: 'tcp', rate: '200/second', perSourceIP: true },
],
},
},
]);
// Check status at any time
// Check status
const status = await hub.getStatus();
console.log(status);
// {
// running: true,
// tunnelPort: 8443,
// connectedEdges: [
// { edgeId: 'edge-nyc-01', connectedAt: 1700000000, activeStreams: 12 }
// ]
// }
// { running: true, tunnelPort: 8443, connectedEdges: [...] }
// Graceful shutdown
await hub.stop();
```
### Setting Up the Edge (Network Edge Side)
The edge can be configured in two ways: with an **opaque connection token** (recommended) or with explicit config fields.
The edge connects via **QUIC with TCP+TLS fallback** by default. Edges run as **root** so they can bind to privileged ports and apply nftables firewall rules.
#### Option A: Connection Token (Recommended)
A single token encodes all connection details — ideal for provisioning edges at scale:
```typescript
import { RemoteIngressEdge } from '@serve.zone/remoteingress';
@@ -137,111 +142,159 @@ const edge = new RemoteIngressEdge();
edge.on('tunnelConnected', () => console.log('Tunnel established'));
edge.on('tunnelDisconnected', () => console.log('Tunnel lost — will auto-reconnect'));
edge.on('publicIpDiscovered', ({ ip }) => console.log(`Public IP: ${ip}`));
edge.on('portsAssigned', ({ listenPorts }) => console.log(`Listening on ports: ${listenPorts}`));
edge.on('portsUpdated', ({ listenPorts }) => console.log(`Ports updated: ${listenPorts}`));
edge.on('portsAssigned', ({ listenPorts }) => console.log(`TCP ports: ${listenPorts}`));
edge.on('firewallConfigUpdated', () => console.log('Firewall rules applied'));
// Single token contains hubHost, hubPort, edgeId, and secret
await edge.start({
token: 'eyJoIjoiaHViLmV4YW1wbGUuY29tIiwicCI6ODQ0MywiZSI6ImVkZ2UtbnljLTAxIiwicyI6InN1cGVyc2VjcmV0dG9rZW4xIn0',
token: 'eyJoIjoiaHViLmV4YW1wbGUuY29tIiwi...',
});
```
#### Option B: Explicit Config
#### Option B: Explicit Config with QUIC Transport
```typescript
import { RemoteIngressEdge } from '@serve.zone/remoteingress';
const edge = new RemoteIngressEdge();
edge.on('tunnelConnected', () => console.log('Tunnel established'));
edge.on('tunnelDisconnected', () => console.log('Tunnel lost — will auto-reconnect'));
edge.on('publicIpDiscovered', ({ ip }) => console.log(`Public IP: ${ip}`));
edge.on('portsAssigned', ({ listenPorts }) => console.log(`Listening on ports: ${listenPorts}`));
edge.on('portsUpdated', ({ listenPorts }) => console.log(`Ports updated: ${listenPorts}`));
await edge.start({
hubHost: 'hub.example.com', // hostname or IP of the hub
hubPort: 8443, // must match hub's tunnelPort (default: 8443)
edgeId: 'edge-nyc-01', // unique edge identifier
secret: 'supersecrettoken1', // must match the hub's allowed edge secret
hubHost: 'hub.example.com',
hubPort: 8443,
edgeId: 'edge-nyc-01',
secret: 'supersecrettoken1',
transportMode: 'quic', // 'tcpTls' | 'quic' | 'quicWithFallback' (default)
});
// Check status at any time
const edgeStatus = await edge.getStatus();
console.log(edgeStatus);
// {
// running: true,
// connected: true,
// publicIp: '203.0.113.42',
// activeStreams: 5,
// listenPorts: [80, 443]
// }
// { running: true, connected: true, publicIp: '203.0.113.42', activeStreams: 5, listenPorts: [80, 443] }
// Graceful shutdown
await edge.stop();
```
### 🎫 Connection Tokens
#### Transport Modes
Connection tokens let you distribute a single opaque string instead of four separate config values. The hub operator generates the token; the edge operator just pastes it in.
| Mode | Description |
|------|-------------|
| `'tcpTls'` | Single TLS connection with frame-based multiplexing. Universal compatibility. |
| `'quic'` | QUIC with native stream multiplexing. Eliminates head-of-line blocking. Uses QUIC datagrams for UDP traffic. |
| `'quicWithFallback'` | **Default.** Tries QUIC first (5s timeout), falls back to TCP+TLS if UDP is blocked by the network. |
### Connection Tokens
Encode all connection details into a single opaque string for easy distribution:
```typescript
import { encodeConnectionToken, decodeConnectionToken } from '@serve.zone/remoteingress';
// Hub side: generate a token for a new edge
// Hub operator generates a token
const token = encodeConnectionToken({
hubHost: 'hub.example.com',
hubPort: 8443,
edgeId: 'edge-nyc-01',
secret: 'supersecrettoken1',
});
console.log(token);
// => 'eyJoIjoiaHViLmV4YW1wbGUuY29tIiwi...'
// Edge side: inspect a token (optional — start() does this automatically)
// Edge operator decodes (optional — start() does this automatically)
const data = decodeConnectionToken(token);
console.log(data);
// {
// hubHost: 'hub.example.com',
// hubPort: 8443,
// edgeId: 'edge-nyc-01',
// secret: 'supersecrettoken1'
// }
// { hubHost: 'hub.example.com', hubPort: 8443, edgeId: 'edge-nyc-01', secret: '...' }
```
Tokens are base64url-encoded (URL-safe, no padding) — safe to pass as environment variables, CLI arguments, or store in config files.
Tokens are base64url-encoded — safe for environment variables, CLI arguments, and config files.
## 📖 API Reference
## 🔥 Firewall Config
The `firewallConfig` field in `updateAllowedEdges()` works exactly like `listenPorts` — it travels in the same `FRAME_CONFIG` frame, is delivered on initial handshake and on every subsequent update, and is applied atomically at the edge using `@push.rocks/smartnftables`. Each update fully replaces the previous ruleset.
Since edges run as root, the rules are applied directly to the Linux kernel via nftables. If the edge isn't root or nftables is unavailable, it logs a warning and continues — the tunnel works fine, just without kernel-level firewall rules.
### Config Structure
```typescript
interface IFirewallConfig {
blockedIps?: string[]; // IPs or CIDRs to block (e.g., '1.2.3.4', '10.0.0.0/8')
rateLimits?: IFirewallRateLimit[];
rules?: IFirewallRule[];
}
interface IFirewallRateLimit {
id: string; // unique identifier for this rate limit
port: number; // port to rate-limit
protocol?: 'tcp' | 'udp'; // default: both
rate: string; // e.g., '100/second', '1000/minute'
burst?: number; // burst allowance
perSourceIP?: boolean; // per-client rate limiting (recommended)
}
interface IFirewallRule {
id: string; // unique identifier for this rule
direction: 'input' | 'output' | 'forward';
action: 'accept' | 'drop' | 'reject';
sourceIP?: string; // source IP or CIDR
destPort?: number; // destination port
protocol?: 'tcp' | 'udp';
comment?: string;
}
```
### Example: Rate Limiting + IP Blocking
```typescript
await hub.updateAllowedEdges([
{
id: 'edge-nyc-01',
secret: 'secret',
listenPorts: [80, 443],
firewallConfig: {
// Block known bad actors
blockedIps: ['198.51.100.0/24', '203.0.113.50'],
// Rate limit HTTP traffic per source IP
rateLimits: [
{ id: 'http', port: 80, protocol: 'tcp', rate: '100/second', burst: 50, perSourceIP: true },
{ id: 'https', port: 443, protocol: 'tcp', rate: '200/second', burst: 100, perSourceIP: true },
],
// Allow monitoring from trusted subnet
rules: [
{ id: 'monitoring', direction: 'input', action: 'accept', sourceIP: '10.0.0.0/24', destPort: 9090, protocol: 'tcp', comment: 'Prometheus scraping' },
],
},
},
]);
```
## API Reference
### `RemoteIngressHub`
| Method / Property | Description |
|-------------------|-------------|
| `start(config?)` | Spawns the Rust binary and starts the tunnel listener. Config: `{ tunnelPort?: number, targetHost?: string }` |
| `stop()` | Gracefully shuts down the hub and kills the Rust process. |
| `updateAllowedEdges(edges)` | Dynamically update which edges are authorized and what ports they listen on. Each edge: `{ id: string, secret: string, listenPorts?: number[], stunIntervalSecs?: number }`. If ports change for a connected edge, the update is pushed immediately via a `FRAME_CONFIG` frame. |
| `getStatus()` | Returns current hub status including connected edges and active stream counts. |
| `start(config?)` | Start the hub. Config: `{ tunnelPort?, targetHost?, tls?: { certPem?, keyPem? } }`. Listens on both TCP and UDP (QUIC) on the tunnel port. |
| `stop()` | Graceful shutdown. |
| `updateAllowedEdges(edges)` | Set authorized edges. Each: `{ id, secret, listenPorts?, listenPortsUdp?, stunIntervalSecs?, firewallConfig? }`. Port and firewall changes are pushed to connected edges in real time. |
| `getStatus()` | Returns `{ running, tunnelPort, connectedEdges: [...] }`. |
| `running` | `boolean` — whether the Rust binary is alive. |
**Events:** `edgeConnected`, `edgeDisconnected`, `streamOpened`, `streamClosed`
**Events:** `edgeConnected`, `edgeDisconnected`, `streamOpened`, `streamClosed`, `crashRecovered`, `crashRecoveryFailed`
### `RemoteIngressEdge`
| Method / Property | Description |
|-------------------|-------------|
| `start(config)` | Spawns the Rust binary and connects to the hub. Accepts `{ token: string }` or `IEdgeConfig`. Listen ports are received from the hub during handshake. |
| `stop()` | Gracefully shuts down the edge and kills the Rust process. |
| `getStatus()` | Returns current edge status including connection state, public IP, listen ports, and active streams. |
| `start(config)` | Connect to hub. Accepts `{ token }` or `{ hubHost, hubPort, edgeId, secret, bindAddress?, transportMode? }`. |
| `stop()` | Graceful shutdown. Cleans up all nftables rules. |
| `getStatus()` | Returns `{ running, connected, publicIp, activeStreams, listenPorts }`. |
| `running` | `boolean` — whether the Rust binary is alive. |
**Events:** `tunnelConnected`, `tunnelDisconnected`, `publicIpDiscovered`, `portsAssigned`, `portsUpdated`
**Events:** `tunnelConnected`, `tunnelDisconnected`, `publicIpDiscovered`, `portsAssigned`, `portsUpdated`, `firewallConfigUpdated`, `crashRecovered`, `crashRecoveryFailed`
### Token Utilities
| Function | Description |
|----------|-------------|
| `encodeConnectionToken(data)` | Encodes `IConnectionTokenData` into a base64url token string. |
| `decodeConnectionToken(token)` | Decodes a token back into `IConnectionTokenData`. Throws on malformed or incomplete tokens. |
| `encodeConnectionToken(data)` | Encodes connection info into a base64url token. |
| `decodeConnectionToken(token)` | Decodes a token. Throws on malformed input. |
### Interfaces
@@ -249,6 +302,10 @@ Tokens are base64url-encoded (URL-safe, no padding) — safe to pass as environm
interface IHubConfig {
tunnelPort?: number; // default: 8443
targetHost?: string; // default: '127.0.0.1'
tls?: {
certPem?: string; // PEM-encoded TLS certificate
keyPem?: string; // PEM-encoded TLS private key
};
}
interface IEdgeConfig {
@@ -256,6 +313,8 @@ interface IEdgeConfig {
hubPort?: number; // default: 8443
edgeId: string;
secret: string;
bindAddress?: string;
transportMode?: 'tcpTls' | 'quic' | 'quicWithFallback';
}
interface IConnectionTokenData {
@@ -266,9 +325,11 @@ interface IConnectionTokenData {
}
```
## 🔌 Wire Protocol
## Wire Protocol
The tunnel uses a custom binary frame protocol over TLS:
### TCP+TLS Transport (Frame Protocol)
The tunnel uses a custom binary frame protocol over a single TLS connection:
```
[stream_id: 4 bytes BE][type: 1 byte][length: 4 bytes BE][payload: N bytes]
@@ -276,111 +337,149 @@ The tunnel uses a custom binary frame protocol over TLS:
| Frame Type | Value | Direction | Purpose |
|------------|-------|-----------|---------|
| `OPEN` | `0x01` | Edge → Hub | Open a new stream; payload is PROXY v1 header |
| `DATA` | `0x02` | Edge → Hub | Client data flowing upstream |
| `CLOSE` | `0x03` | Edge → Hub | Client closed the connection |
| `DATA_BACK` | `0x04` | Hub → Edge | Response data flowing downstream |
| `CLOSE_BACK` | `0x05` | Hub → Edge | Upstream (SmartProxy) closed the connection |
| `CONFIG` | `0x06` | Hub → Edge | Runtime configuration update (e.g. port changes); payload is JSON |
| `PING` | `0x07` | Hub → Edge | Heartbeat probe (sent every 15s) |
| `OPEN` | `0x01` | Edge → Hub | Open TCP stream; payload is PROXY v1 header |
| `DATA` | `0x02` | Edge → Hub | Client data (upload) |
| `CLOSE` | `0x03` | Edge → Hub | Client closed connection |
| `DATA_BACK` | `0x04` | Hub → Edge | Response data (download) |
| `CLOSE_BACK` | `0x05` | Hub → Edge | Upstream closed connection |
| `CONFIG` | `0x06` | Hub → Edge | Runtime config update (JSON: ports + firewall config) |
| `PING` | `0x07` | Hub → Edge | Heartbeat probe (every 15s) |
| `PONG` | `0x08` | Edge → Hub | Heartbeat response |
| `WINDOW_UPDATE` | `0x09` | Edge → Hub | Per-stream flow control: edge consumed N bytes, hub can send more |
| `WINDOW_UPDATE_BACK` | `0x0A` | Hub → Edge | Per-stream flow control: hub consumed N bytes, edge can send more |
| `WINDOW_UPDATE` | `0x09` | Edge → Hub | Flow control: edge consumed N bytes |
| `WINDOW_UPDATE_BACK` | `0x0A` | Hub → Edge | Flow control: hub consumed N bytes |
| `UDP_OPEN` | `0x0B` | Edge → Hub | Open UDP session; payload is PROXY v2 header |
| `UDP_DATA` | `0x0C` | Edge → Hub | UDP datagram (upload) |
| `UDP_DATA_BACK` | `0x0D` | Hub → Edge | UDP datagram (download) |
| `UDP_CLOSE` | `0x0E` | Either | Close UDP session |
Max payload size per frame: **16 MB**. Stream IDs are 32-bit unsigned integers.
### QUIC Transport
When using QUIC, the frame protocol is replaced by native QUIC primitives:
- **TCP connections:** Each tunneled TCP connection gets its own QUIC bidirectional stream. No framing overhead.
- **UDP datagrams:** Forwarded via QUIC unreliable datagrams (RFC 9221). Format: `[session_id: 4 bytes][payload]`. Session open uses magic byte `0xFF`: `[session_id: 4][0xFF][PROXY v2 header]`.
- **Control channel:** First QUIC bidirectional stream carries auth handshake + config updates using `[type: 1][length: 4][payload]` format.
### Handshake Sequence
1. Edge opens a TLS connection to the hub
1. Edge opens a TLS or QUIC connection to the hub
2. Edge sends: `EDGE <edgeId> <secret>\n`
3. Hub verifies credentials (constant-time comparison) and responds with JSON: `{"listenPorts":[...],"stunIntervalSecs":300}\n`
4. Edge starts TCP listeners on the assigned ports
5. Frame protocol begins — `OPEN`/`DATA`/`CLOSE` frames flow in both directions
6. Hub can push `CONFIG` frames at any time to update the edge's listen ports
3. Hub verifies credentials (constant-time comparison) and responds with JSON:
`{"listenPorts":[...],"listenPortsUdp":[...],"stunIntervalSecs":300,"firewallConfig":{...}}\n`
4. Edge starts TCP and UDP listeners on the assigned ports
5. Edge applies firewall config via nftables (if present and running as root)
6. Data flows — TCP frames/QUIC streams for TCP traffic, UDP frames/QUIC datagrams for UDP traffic
## 🎚️ QoS & Flow Control
## QoS & Flow Control
The tunnel multiplexer uses a **3-tier priority system** and **per-stream flow control** to ensure fair bandwidth sharing across thousands of concurrent streams.
### Priority Tiers (TCP+TLS Transport)
### Priority Tiers
All outbound frames are queued into one of three priority levels:
| Tier | Queue | Frames | Behavior |
|------|-------|--------|----------|
| 🔴 **Control** (highest) | `ctrl_queue` | PING, PONG, WINDOW_UPDATE, OPEN, CLOSE, CONFIG | Always drained first. Never delayed. |
| 🟡 **Data** (normal) | `data_queue` | DATA, DATA_BACK from normal streams | Drained when ctrl is empty. Gated at 64 buffered items for backpressure. |
| 🟢 **Sustained** (lowest) | `sustained_queue` | DATA, DATA_BACK from elephant flows | Drained freely when ctrl+data are empty. Otherwise guaranteed **1 MB/s** via forced drain every second. |
This prevents large bulk transfers (e.g. git clones, file downloads) from starving interactive traffic and ensures `WINDOW_UPDATE` frames are never delayed — which would cause flow control deadlocks.
| Tier | Frames | Behavior |
|------|--------|----------|
| **Control** | PING, PONG, WINDOW_UPDATE, OPEN, CLOSE, CONFIG | Always drained first. Never delayed. |
| **Data** | DATA/DATA_BACK from normal streams, UDP frames | Drained when control queue is empty. |
| **Sustained** | DATA/DATA_BACK from elephant flows | Lowest priority with guaranteed **1 MB/s** drain rate. |
### Sustained Stream Classification
A stream is automatically classified as **sustained** (elephant flow) when:
- It has been active for **>10 seconds**, AND
- Its average throughput exceeds **20 Mbit/s** (2.5 MB/s)
A TCP stream is classified as **sustained** (elephant flow) when:
- Active for **>10 seconds**, AND
- Average throughput exceeds **20 Mbit/s** (2.5 MB/s)
Once classified, the stream's flow control window is locked to the **1 MB floor** and its data frames move to the lowest-priority queue. Classification is one-way — a stream never gets promoted back to normal.
Once classified, its flow control window locks to 1 MB and data frames move to the lowest-priority queue.
### Adaptive Per-Stream Windows
Each stream has a send window that limits bytes-in-flight. The window size adapts to the number of active streams using a shared **200 MB memory budget**:
Each TCP stream has a send window from a shared **200 MB budget**:
| Active Streams | Window per Stream |
|---|---|
| 150 | 4 MB (maximum) |
| 51100 | Scales down (4 MB → 2 MB) |
| 51200 | Scales down (4 MB → 1 MB) |
| 200+ | 1 MB (floor) |
The consumer sends `WINDOW_UPDATE` frames after processing data, allowing the producer to send more. This prevents any single stream from consuming unbounded memory and provides natural backpressure.
UDP traffic uses no flow control — datagrams are fire-and-forget, matching UDP semantics.
## 💡 Example Scenarios
## Example Scenarios
### 1. Expose a Private Kubernetes Cluster to the Internet
### 1. 🌐 Expose a Private Cluster to the Internet
Deploy an Edge on a public VPS, point your DNS to the VPS IP. The Edge tunnels all traffic to the Hub running inside the cluster, which hands it off to SmartProxy/DcRouter. Your cluster stays fully private — no public-facing ports needed.
Deploy an Edge on a public VPS, point DNS to its IP. The Edge tunnels all TCP and UDP traffic to the Hub running inside your private cluster. No public ports needed on the cluster.
### 2. Multi-Region Edge Ingress
### 2. 🗺️ Multi-Region Edge Ingress
Run multiple Edges in different geographic regions (NYC, Frankfurt, Tokyo) all connecting to a single Hub. Use GeoDNS to route users to their nearest Edge. The Hub sees the real client IPs via PROXY protocol regardless of which edge they connected through.
Run Edges in NYC, Frankfurt, and Tokyo all connecting to a single Hub. Use GeoDNS to route users to their nearest Edge. PROXY protocol ensures the Hub sees real client IPs regardless of which Edge they entered through.
### 3. Secure API Exposure
### 3. 📡 UDP Forwarding (DNS, Gaming, VoIP)
Your backend runs on a private network with no direct internet access. An Edge on a minimal cloud instance acts as the only public entry point. TLS tunnel + shared-secret auth ensure only your authorized Edge can forward traffic.
### 4. Token-Based Edge Provisioning
Generate connection tokens on the hub side and distribute them to edge operators. Each edge only needs a single token string to connect — no manual configuration of host, port, ID, and secret.
Configure UDP listen ports alongside TCP ports. DNS queries, game server traffic, or VoIP packets are tunneled through the same edge/hub connection and forwarded to SmartProxy with a PROXY v2 binary header preserving the client's real IP.
```typescript
// Hub operator generates token
await hub.updateAllowedEdges([
{
id: 'edge-nyc-01',
secret: 'secret',
listenPorts: [80, 443], // TCP
listenPortsUdp: [53, 27015], // DNS + game server
},
]);
```
### 4. 🚀 QUIC Transport for Low-Latency
Use QUIC transport to eliminate head-of-line blocking — a lost packet on one stream doesn't stall others. QUIC also enables 0-RTT reconnection and connection migration.
```typescript
await edge.start({
hubHost: 'hub.example.com',
hubPort: 8443,
edgeId: 'edge-01',
secret: 'secret',
transportMode: 'quicWithFallback', // try QUIC, fall back to TLS if UDP blocked
});
```
### 5. 🔑 Token-Based Edge Provisioning
Generate connection tokens on the hub side and distribute them to edge operators:
```typescript
import { encodeConnectionToken, RemoteIngressEdge } from '@serve.zone/remoteingress';
const token = encodeConnectionToken({
hubHost: 'hub.prod.example.com',
hubPort: 8443,
edgeId: 'edge-tokyo-01',
secret: 'generated-secret-abc123',
});
// Send `token` to the edge operator via secure channel
// Send `token` to the edge operator a single string is all they need
// Edge operator starts with just the token
const edge = new RemoteIngressEdge();
await edge.start({ token });
```
### 5. Dynamic Port Management
### 6. 🛡️ Centralized Firewall Management
The hub controls which ports each edge listens on. Ports can be changed at runtime without restarting the edge — the hub pushes a `CONFIG` frame and the edge hot-reloads its TCP listeners.
Push firewall rules from the hub to all your edge nodes. Block bad actors, rate-limit abusive traffic, and whitelist trusted subnets — all from a single control plane:
```typescript
// Initially assign ports 80 and 443
await hub.updateAllowedEdges([
{ id: 'edge-nyc-01', secret: 'secret', listenPorts: [80, 443] },
]);
// Later, add port 8080 — the connected edge picks it up instantly
await hub.updateAllowedEdges([
{ id: 'edge-nyc-01', secret: 'secret', listenPorts: [80, 443, 8080] },
{
id: 'edge-nyc-01',
secret: 'secret',
listenPorts: [80, 443],
firewallConfig: {
blockedIps: ['198.51.100.0/24'],
rateLimits: [
{ id: 'https', port: 443, protocol: 'tcp', rate: '500/second', perSourceIP: true, burst: 100 },
],
rules: [
{ id: 'allow-monitoring', direction: 'input', action: 'accept', sourceIP: '10.0.0.0/8', destPort: 9090, protocol: 'tcp' },
],
},
},
]);
// Firewall rules are applied at the edge via nftables within seconds
```
## License and Legal Information
@@ -316,6 +316,12 @@ async fn handle_request(
serde_json::json!({ "listenPorts": listen_ports }),
);
}
EdgeEvent::FirewallConfigUpdated { firewall_config } => {
send_event(
"firewallConfigUpdated",
serde_json::json!({ "firewallConfig": firewall_config }),
);
}
}
}
});
+483 -44
View File
@@ -3,7 +3,7 @@ use std::sync::atomic::{AtomicU32, Ordering};
use std::sync::Arc;
use std::time::Duration;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::{TcpListener, TcpStream};
use tokio::net::{TcpListener, TcpStream, UdpSocket};
use tokio::sync::{mpsc, Mutex, Notify, RwLock};
use tokio::task::JoinHandle;
use tokio::time::{Instant, sleep_until};
@@ -13,8 +13,10 @@ use serde::{Deserialize, Serialize};
use bytes::Bytes;
use remoteingress_protocol::*;
use crate::performance::EffectivePerformanceConfig;
use crate::transport::TransportMode;
use crate::transport::quic as quic_transport;
use crate::udp_session::{UdpSessionKey, UdpSessionManager};
type EdgeTlsStream = tokio_rustls::client::TlsStream<TcpStream>;
@@ -35,6 +37,9 @@ struct EdgeStreamState {
send_window: Arc<AtomicU32>,
/// Notifier to wake the client reader when the window opens.
window_notify: Arc<Notify>,
/// Per-stream cancellation token — cancelled on FRAME_CLOSE_BACK to promptly
/// terminate the upload loop instead of waiting for the window stall timeout.
cancel_token: CancellationToken,
}
/// Edge configuration (hub-host + credentials only; ports come from hub).
@@ -49,7 +54,7 @@ pub struct EdgeConfig {
/// Useful for testing on localhost where edge and upstream share the same machine.
#[serde(default)]
pub bind_address: Option<String>,
/// Transport mode for the tunnel connection (defaults to TcpTls).
/// Transport mode for the tunnel connection (defaults to QuicWithFallback).
#[serde(default)]
pub transport_mode: Option<TransportMode>,
}
@@ -59,19 +64,70 @@ pub struct EdgeConfig {
#[serde(rename_all = "camelCase")]
struct HandshakeConfig {
listen_ports: Vec<u16>,
#[serde(default)]
listen_ports_udp: Vec<u16>,
#[serde(default = "default_stun_interval")]
stun_interval_secs: u64,
#[serde(default)]
firewall_config: Option<serde_json::Value>,
#[serde(default)]
performance: EffectivePerformanceConfig,
}
fn default_stun_interval() -> u64 {
300
}
fn try_reserve_stream(active_streams: &AtomicU32, max_streams: usize) -> bool {
let max_streams = max_streams.min(u32::MAX as usize) as u32;
loop {
let current = active_streams.load(Ordering::Relaxed);
if current >= max_streams {
return false;
}
if active_streams
.compare_exchange_weak(current, current + 1, Ordering::Relaxed, Ordering::Relaxed)
.is_ok()
{
return true;
}
}
}
fn release_stream(active_streams: &AtomicU32) {
loop {
let current = active_streams.load(Ordering::Relaxed);
if current == 0 {
break;
}
if active_streams
.compare_exchange_weak(current, current - 1, Ordering::Relaxed, Ordering::Relaxed)
.is_ok()
{
break;
}
}
}
fn transport_mode_wire_name(mode: TransportMode) -> &'static str {
match mode {
TransportMode::TcpTls => "tcpTls",
TransportMode::Quic => "quic",
TransportMode::QuicWithFallback => "quicWithFallback",
}
}
/// Runtime config update received from hub via FRAME_CONFIG.
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
struct ConfigUpdate {
listen_ports: Vec<u16>,
#[serde(default)]
listen_ports_udp: Vec<u16>,
#[serde(default)]
firewall_config: Option<serde_json::Value>,
#[serde(default)]
performance: EffectivePerformanceConfig,
}
/// Events emitted by the edge.
@@ -88,6 +144,8 @@ pub enum EdgeEvent {
PortsAssigned { listen_ports: Vec<u16> },
#[serde(rename_all = "camelCase")]
PortsUpdated { listen_ports: Vec<u16> },
#[serde(rename_all = "camelCase")]
FirewallConfigUpdated { firewall_config: serde_json::Value },
}
/// Edge status response.
@@ -215,7 +273,7 @@ async fn edge_main_loop(
let mut backoff_ms: u64 = 1000;
let max_backoff_ms: u64 = 30000;
let transport_mode = config.transport_mode.unwrap_or(TransportMode::TcpTls);
let transport_mode = config.transport_mode.unwrap_or(TransportMode::QuicWithFallback);
// Build TLS config ONCE outside the reconnect loop — preserves session
// cache across reconnections for TLS session resumption (saves 1 RTT).
@@ -344,7 +402,8 @@ enum EdgeLoopResult {
}
/// Process a single frame received from the hub side of the tunnel.
/// Handles FRAME_DATA_BACK, FRAME_WINDOW_UPDATE_BACK, FRAME_CLOSE_BACK, FRAME_CONFIG, FRAME_PING.
/// Handles FRAME_DATA_BACK, FRAME_WINDOW_UPDATE_BACK, FRAME_CLOSE_BACK, FRAME_CONFIG, FRAME_PING,
/// and UDP frames: FRAME_UDP_DATA_BACK, FRAME_UDP_CLOSE.
async fn handle_edge_frame(
frame: Frame,
tunnel_io: &mut remoteingress_protocol::TunnelIo<EdgeTlsStream>,
@@ -355,11 +414,15 @@ async fn handle_edge_frame(
tunnel_data_tx: &mpsc::Sender<Bytes>,
tunnel_sustained_tx: &mpsc::Sender<Bytes>,
port_listeners: &mut HashMap<u16, JoinHandle<()>>,
udp_listeners: &mut HashMap<u16, JoinHandle<()>>,
active_streams: &Arc<AtomicU32>,
next_stream_id: &Arc<AtomicU32>,
edge_id: &str,
connection_token: &CancellationToken,
bind_address: &str,
udp_sessions: &Arc<Mutex<UdpSessionManager>>,
udp_sockets: &Arc<Mutex<HashMap<u16, Arc<UdpSocket>>>>,
performance: &mut EffectivePerformanceConfig,
) -> EdgeFrameAction {
match frame.frame_type {
FRAME_DATA_BACK => {
@@ -380,8 +443,8 @@ async fn handle_edge_frame(
let writers = client_writers.lock().await;
if let Some(state) = writers.get(&frame.stream_id) {
let prev = state.send_window.fetch_add(increment, Ordering::Release);
if prev + increment > MAX_WINDOW_SIZE {
state.send_window.store(MAX_WINDOW_SIZE, Ordering::Release);
if prev + increment > performance.max_stream_window_bytes {
state.send_window.store(performance.max_stream_window_bytes, Ordering::Release);
}
state.window_notify.notify_one();
}
@@ -390,11 +453,16 @@ async fn handle_edge_frame(
}
FRAME_CLOSE_BACK => {
let mut writers = client_writers.lock().await;
writers.remove(&frame.stream_id);
if let Some(state) = writers.remove(&frame.stream_id) {
// Cancel the stream's token so the upload loop exits promptly
// instead of waiting for the window stall timeout.
state.cancel_token.cancel();
}
}
FRAME_CONFIG => {
if let Ok(update) = serde_json::from_slice::<ConfigUpdate>(&frame.payload) {
log::info!("Config update from hub: ports {:?}", update.listen_ports);
log::info!("Config update from hub: ports {:?}, udp {:?}", update.listen_ports, update.listen_ports_udp);
*performance = update.performance.clone();
*listen_ports.write().await = update.listen_ports.clone();
let _ = event_tx.try_send(EdgeEvent::PortsUpdated {
listen_ports: update.listen_ports.clone(),
@@ -411,13 +479,46 @@ async fn handle_edge_frame(
edge_id,
connection_token,
bind_address,
performance,
);
apply_udp_port_config(
&update.listen_ports_udp,
udp_listeners,
tunnel_writer_tx,
tunnel_data_tx,
udp_sessions,
udp_sockets,
next_stream_id,
connection_token,
bind_address,
);
if let Some(fw_config) = update.firewall_config {
let _ = event_tx.try_send(EdgeEvent::FirewallConfigUpdated {
firewall_config: fw_config,
});
}
}
}
FRAME_PING => {
// Queue PONG directly — no channel round-trip, guaranteed delivery
tunnel_io.queue_ctrl(encode_frame(0, FRAME_PONG, &[]));
}
FRAME_UDP_DATA_BACK => {
// Dispatch return UDP datagram to the original client
let mut sessions = udp_sessions.lock().await;
if let Some(session) = sessions.get_by_stream_id(frame.stream_id) {
let client_addr = session.client_addr;
let dest_port = session.dest_port;
let sockets = udp_sockets.lock().await;
if let Some(socket) = sockets.get(&dest_port) {
let _ = socket.send_to(&frame.payload, client_addr).await;
}
}
}
FRAME_UDP_CLOSE => {
let mut sessions = udp_sessions.lock().await;
sessions.remove_by_stream_id(frame.stream_id);
}
_ => {
log::warn!("Unexpected frame type {} from hub", frame.frame_type);
}
@@ -470,7 +571,13 @@ async fn connect_to_hub_and_run(
};
// Send auth line (we own the whole stream — no split)
let auth_line = format!("EDGE {} {}\n", config.edge_id, config.secret);
let requested_transport = config.transport_mode.unwrap_or(TransportMode::QuicWithFallback);
let auth_line = format!(
"EDGE {} {} {}\n",
config.edge_id,
config.secret,
transport_mode_wire_name(requested_transport),
);
if tls_stream.write_all(auth_line.as_bytes()).await.is_err() {
return EdgeLoopResult::Reconnect("auth_write_failed".to_string());
}
@@ -509,6 +616,7 @@ async fn connect_to_hub_and_run(
return EdgeLoopResult::Reconnect(format!("handshake_invalid: {}", e));
}
};
let mut performance = handshake.performance.clone();
log::info!(
"Handshake from hub: ports {:?}, stun_interval {}s",
@@ -526,6 +634,13 @@ async fn connect_to_hub_and_run(
listen_ports: handshake.listen_ports.clone(),
});
// Emit firewall config if present in handshake
if let Some(fw_config) = handshake.firewall_config {
let _ = event_tx.try_send(EdgeEvent::FirewallConfigUpdated {
firewall_config: fw_config,
});
}
// Start STUN discovery
let stun_interval = handshake.stun_interval_secs;
let public_ip_clone = public_ip.clone();
@@ -579,6 +694,25 @@ async fn connect_to_hub_and_run(
&config.edge_id,
connection_token,
bind_address,
&performance,
);
// UDP session manager + listeners
let udp_sessions: Arc<Mutex<UdpSessionManager>> =
Arc::new(Mutex::new(UdpSessionManager::new(Duration::from_secs(60))));
let udp_sockets: Arc<Mutex<HashMap<u16, Arc<UdpSocket>>>> =
Arc::new(Mutex::new(HashMap::new()));
let mut udp_listeners: HashMap<u16, JoinHandle<()>> = HashMap::new();
apply_udp_port_config(
&handshake.listen_ports_udp,
&mut udp_listeners,
&tunnel_ctrl_tx,
&tunnel_data_tx,
&udp_sessions,
&udp_sockets,
next_stream_id,
connection_token,
bind_address,
);
// Single-owner I/O engine — no tokio::io::split, no mutex
@@ -588,8 +722,23 @@ async fn connect_to_hub_and_run(
let liveness_timeout_dur = Duration::from_secs(45);
let mut last_activity = Instant::now();
let mut liveness_deadline = Box::pin(sleep_until(last_activity + liveness_timeout_dur));
let mut next_udp_expiry = Instant::now() + Duration::from_secs(30);
let result = 'io_loop: loop {
// Expire idle UDP sessions periodically
if Instant::now() >= next_udp_expiry {
let mut sessions = udp_sessions.lock().await;
let expired = sessions.expire_idle();
for sid in &expired {
let close_frame = encode_frame(*sid, FRAME_UDP_CLOSE, &[]);
let _ = tunnel_data_tx.try_send(close_frame);
}
if !expired.is_empty() {
log::debug!("Expired {} idle UDP sessions", expired.len());
}
next_udp_expiry = Instant::now() + Duration::from_secs(30);
}
// Drain any buffered frames
loop {
let frame = match tunnel_io.try_parse_frame() {
@@ -605,7 +754,8 @@ async fn connect_to_hub_and_run(
if let EdgeFrameAction::Disconnect(reason) = handle_edge_frame(
frame, &mut tunnel_io, &client_writers, listen_ports, event_tx,
&tunnel_writer_tx, &tunnel_data_tx, &tunnel_sustained_tx, &mut port_listeners,
active_streams, next_stream_id, &config.edge_id, connection_token, bind_address,
&mut udp_listeners, active_streams, next_stream_id, &config.edge_id,
connection_token, bind_address, &udp_sessions, &udp_sockets, &mut performance,
).await {
break 'io_loop EdgeLoopResult::Reconnect(reason);
}
@@ -623,7 +773,8 @@ async fn connect_to_hub_and_run(
if let EdgeFrameAction::Disconnect(reason) = handle_edge_frame(
frame, &mut tunnel_io, &client_writers, listen_ports, event_tx,
&tunnel_writer_tx, &tunnel_data_tx, &tunnel_sustained_tx, &mut port_listeners,
active_streams, next_stream_id, &config.edge_id, connection_token, bind_address,
&mut udp_listeners, active_streams, next_stream_id, &config.edge_id,
connection_token, bind_address, &udp_sessions, &udp_sockets, &mut performance,
).await {
break EdgeLoopResult::Reconnect(reason);
}
@@ -661,6 +812,9 @@ async fn connect_to_hub_and_run(
for (_, h) in port_listeners.drain() {
h.abort();
}
for (_, h) in udp_listeners.drain() {
h.abort();
}
// Graceful TLS shutdown: send close_notify so the hub sees a clean disconnect.
// Stream handlers are already cancelled, so no new data is being produced.
@@ -686,6 +840,7 @@ fn apply_port_config(
edge_id: &str,
connection_token: &CancellationToken,
bind_address: &str,
performance: &EffectivePerformanceConfig,
) {
let new_set: std::collections::HashSet<u16> = new_ports.iter().copied().collect();
let old_set: std::collections::HashSet<u16> = port_listeners.keys().copied().collect();
@@ -708,6 +863,7 @@ fn apply_port_config(
let next_stream_id = next_stream_id.clone();
let edge_id = edge_id.to_string();
let port_token = connection_token.child_token();
let performance = performance.clone();
let bind_addr = bind_address.to_string();
let handle = tokio::spawn(async move {
@@ -743,8 +899,12 @@ fn apply_port_config(
let edge_id = edge_id.clone();
let client_token = port_token.child_token();
active_streams.fetch_add(1, Ordering::Relaxed);
if !try_reserve_stream(&active_streams, performance.max_streams_per_edge) {
log::warn!("Rejecting client on port {}: max streams ({}) reached", port, performance.max_streams_per_edge);
continue;
}
let performance = performance.clone();
tokio::spawn(async move {
handle_client_connection(
client_stream,
@@ -758,20 +918,10 @@ fn apply_port_config(
client_writers,
client_token,
Arc::clone(&active_streams),
performance,
)
.await;
// Saturating decrement: prevent underflow when
// edge_main_loop's store(0) races with task cleanup.
loop {
let current = active_streams.load(Ordering::Relaxed);
if current == 0 { break; }
if active_streams.compare_exchange_weak(
current, current - 1,
Ordering::Relaxed, Ordering::Relaxed,
).is_ok() {
break;
}
}
release_stream(&active_streams);
});
}
Err(e) => {
@@ -790,6 +940,110 @@ fn apply_port_config(
}
}
/// Apply UDP port configuration: bind UdpSockets for added ports, abort removed ports.
fn apply_udp_port_config(
new_ports: &[u16],
udp_listeners: &mut HashMap<u16, JoinHandle<()>>,
tunnel_ctrl_tx: &mpsc::Sender<Bytes>,
tunnel_data_tx: &mpsc::Sender<Bytes>,
udp_sessions: &Arc<Mutex<UdpSessionManager>>,
udp_sockets: &Arc<Mutex<HashMap<u16, Arc<UdpSocket>>>>,
next_stream_id: &Arc<AtomicU32>,
connection_token: &CancellationToken,
bind_address: &str,
) {
let new_set: std::collections::HashSet<u16> = new_ports.iter().copied().collect();
let old_set: std::collections::HashSet<u16> = udp_listeners.keys().copied().collect();
// Remove ports no longer needed
for &port in old_set.difference(&new_set) {
if let Some(handle) = udp_listeners.remove(&port) {
log::info!("Stopping UDP listener on port {}", port);
handle.abort();
}
// Remove socket from shared map
let sockets = udp_sockets.clone();
tokio::spawn(async move {
sockets.lock().await.remove(&port);
});
}
// Add new ports
for &port in new_set.difference(&old_set) {
let tunnel_ctrl_tx = tunnel_ctrl_tx.clone();
let tunnel_data_tx = tunnel_data_tx.clone();
let udp_sessions = udp_sessions.clone();
let udp_sockets = udp_sockets.clone();
let next_stream_id = next_stream_id.clone();
let port_token = connection_token.child_token();
let bind_addr = bind_address.to_string();
let handle = tokio::spawn(async move {
let socket = match UdpSocket::bind((bind_addr.as_str(), port)).await {
Ok(s) => Arc::new(s),
Err(e) => {
log::error!("Failed to bind UDP port {}: {}", port, e);
return;
}
};
log::info!("Listening on UDP port {}", port);
// Register socket in shared map for return traffic
udp_sockets.lock().await.insert(port, socket.clone());
let mut buf = vec![0u8; 65536]; // max UDP datagram size
loop {
tokio::select! {
recv_result = socket.recv_from(&mut buf) => {
match recv_result {
Ok((len, client_addr)) => {
let key = UdpSessionKey { client_addr, dest_port: port };
let mut sessions = udp_sessions.lock().await;
let stream_id = if let Some(session) = sessions.get_mut(&key) {
session.stream_id
} else {
// New session — allocate stream_id and send UDP_OPEN
let sid = next_stream_id.fetch_add(1, Ordering::Relaxed);
if sessions.insert(key, sid).is_none() {
log::warn!("UDP session limit reached, dropping datagram from {}", client_addr);
continue;
}
let client_ip = client_addr.ip().to_string();
let client_port = client_addr.port();
let proxy_header = build_proxy_v2_header_from_str(
&client_ip, "0.0.0.0", client_port, port,
ProxyV2Transport::Udp,
);
let open_frame = encode_frame(sid, FRAME_UDP_OPEN, &proxy_header);
let _ = tunnel_ctrl_tx.try_send(open_frame);
log::debug!("New UDP session {} from {} -> port {}", sid, client_addr, port);
sid
};
drop(sessions); // release lock before sending
// Send datagram through tunnel
let data_frame = encode_frame(stream_id, FRAME_UDP_DATA, &buf[..len]);
let _ = tunnel_data_tx.try_send(data_frame);
}
Err(e) => {
log::error!("UDP recv error on port {}: {}", port, e);
}
}
}
_ = port_token.cancelled() => {
log::info!("UDP port {} listener cancelled", port);
break;
}
}
}
});
udp_listeners.insert(port, handle);
}
}
async fn handle_client_connection(
client_stream: TcpStream,
client_addr: std::net::SocketAddr,
@@ -802,6 +1056,7 @@ async fn handle_client_connection(
client_writers: Arc<Mutex<HashMap<u32, EdgeStreamState>>>,
client_token: CancellationToken,
active_streams: Arc<AtomicU32>,
performance: EffectivePerformanceConfig,
) {
let client_ip = client_addr.ip().to_string();
let client_port = client_addr.port();
@@ -825,9 +1080,12 @@ async fn handle_client_connection(
// streams due to channel overflow — backpressure slows streams, never kills them.
let (back_tx, mut back_rx) = mpsc::unbounded_channel::<Bytes>();
// Adaptive initial window: scale with current stream count to keep total in-flight
// data within the 200MB budget. Prevents burst flooding when many streams open.
let initial_window = remoteingress_protocol::compute_window_for_stream_count(
// data within the configured edge budget. Prevents burst flooding when many streams open.
let initial_window = remoteingress_protocol::compute_window_for_limits(
active_streams.load(Ordering::Relaxed),
performance.total_window_budget_bytes,
performance.min_stream_window_bytes,
performance.max_stream_window_bytes,
);
let send_window = Arc::new(AtomicU32::new(initial_window));
let window_notify = Arc::new(Notify::new());
@@ -837,6 +1095,7 @@ async fn handle_client_connection(
back_tx,
send_window: Arc::clone(&send_window),
window_notify: Arc::clone(&window_notify),
cancel_token: client_token.clone(),
});
}
@@ -863,8 +1122,11 @@ async fn handle_client_connection(
// effective window shrinks to match current demand (fewer streams
// = larger window, more streams = smaller window per stream).
consumed_since_update += len;
let adaptive_window = remoteingress_protocol::compute_window_for_stream_count(
let adaptive_window = remoteingress_protocol::compute_window_for_limits(
active_streams_h2c.load(Ordering::Relaxed),
performance.total_window_budget_bytes,
performance.min_stream_window_bytes,
performance.max_stream_window_bytes,
);
let threshold = adaptive_window / 2;
if consumed_since_update >= threshold {
@@ -918,8 +1180,8 @@ async fn handle_client_connection(
tokio::select! {
_ = notified => continue,
_ = client_token.cancelled() => break,
_ = tokio::time::sleep(Duration::from_secs(120)) => {
log::warn!("Stream {} upload stalled (window empty for 120s)", stream_id);
_ = tokio::time::sleep(Duration::from_secs(55)) => {
log::warn!("Stream {} upload stalled (window empty for 55s)", stream_id);
break;
}
}
@@ -1078,7 +1340,13 @@ async fn connect_to_hub_and_run_quic_with_connection(
};
// Auth handshake on control stream (same protocol as TCP+TLS)
let auth_line = format!("EDGE {} {}\n", config.edge_id, config.secret);
let requested_transport = config.transport_mode.unwrap_or(TransportMode::QuicWithFallback);
let auth_line = format!(
"EDGE {} {} {}\n",
config.edge_id,
config.secret,
transport_mode_wire_name(requested_transport),
);
if let Err(e) = ctrl_send.write_all(auth_line.as_bytes()).await {
return EdgeLoopResult::Reconnect(format!("quic_auth_write_failed: {}", e));
}
@@ -1110,6 +1378,7 @@ async fn connect_to_hub_and_run_quic_with_connection(
return EdgeLoopResult::Reconnect(format!("quic_handshake_invalid: {}", e));
}
};
let mut performance = handshake.performance.clone();
log::info!(
"QUIC handshake from hub: ports {:?}, stun_interval {}s",
@@ -1126,6 +1395,13 @@ async fn connect_to_hub_and_run_quic_with_connection(
listen_ports: handshake.listen_ports.clone(),
});
// Emit firewall config if present in handshake
if let Some(fw_config) = handshake.firewall_config {
let _ = event_tx.try_send(EdgeEvent::FirewallConfigUpdated {
firewall_config: fw_config,
});
}
// Start STUN discovery
let stun_interval = handshake.stun_interval_secs;
let public_ip_clone = public_ip.clone();
@@ -1166,11 +1442,39 @@ async fn connect_to_hub_and_run_quic_with_connection(
&config.edge_id,
connection_token,
bind_address,
&performance,
);
// Monitor control stream for config updates, and connection health.
// Also handle shutdown signals.
// UDP listeners for QUIC transport — uses QUIC datagrams for low-latency forwarding.
let udp_sessions_quic: Arc<Mutex<UdpSessionManager>> =
Arc::new(Mutex::new(UdpSessionManager::new(Duration::from_secs(60))));
let udp_sockets_quic: Arc<Mutex<HashMap<u16, Arc<UdpSocket>>>> =
Arc::new(Mutex::new(HashMap::new()));
let mut udp_listeners_quic: HashMap<u16, JoinHandle<()>> = HashMap::new();
apply_udp_port_config_quic(
&handshake.listen_ports_udp,
&mut udp_listeners_quic,
&quic_conn,
&udp_sessions_quic,
&udp_sockets_quic,
next_stream_id,
connection_token,
bind_address,
);
// Monitor control stream for config updates, connection health, and QUIC datagrams.
let mut next_udp_expiry_quic = Instant::now() + Duration::from_secs(30);
let result = 'quic_loop: loop {
// Expire idle UDP sessions periodically
if Instant::now() >= next_udp_expiry_quic {
let mut sessions = udp_sessions_quic.lock().await;
let expired = sessions.expire_idle();
if !expired.is_empty() {
log::debug!("Expired {} idle QUIC UDP sessions", expired.len());
}
next_udp_expiry_quic = Instant::now() + Duration::from_secs(30);
}
tokio::select! {
// Read control messages from hub
ctrl_msg = quic_transport::read_ctrl_message(&mut ctrl_recv) => {
@@ -1180,6 +1484,7 @@ async fn connect_to_hub_and_run_quic_with_connection(
quic_transport::CTRL_CONFIG => {
if let Ok(update) = serde_json::from_slice::<ConfigUpdate>(&payload) {
log::info!("QUIC config update from hub: ports {:?}", update.listen_ports);
performance = update.performance.clone();
*listen_ports.write().await = update.listen_ports.clone();
let _ = event_tx.try_send(EdgeEvent::PortsUpdated {
listen_ports: update.listen_ports.clone(),
@@ -1193,6 +1498,17 @@ async fn connect_to_hub_and_run_quic_with_connection(
&config.edge_id,
connection_token,
bind_address,
&performance,
);
apply_udp_port_config_quic(
&update.listen_ports_udp,
&mut udp_listeners_quic,
&quic_conn,
&udp_sessions_quic,
&udp_sockets_quic,
next_stream_id,
connection_token,
bind_address,
);
}
}
@@ -1224,6 +1540,30 @@ async fn connect_to_hub_and_run_quic_with_connection(
}
}
}
// Receive QUIC datagrams (UDP return traffic from hub)
datagram = quic_conn.read_datagram() => {
match datagram {
Ok(data) => {
// Format: [session_id:4][payload:N]
if data.len() >= 4 {
let session_id = u32::from_be_bytes([data[0], data[1], data[2], data[3]]);
let payload = &data[4..];
let mut sessions = udp_sessions_quic.lock().await;
if let Some(session) = sessions.get_by_stream_id(session_id) {
let client_addr = session.client_addr;
let dest_port = session.dest_port;
let sockets = udp_sockets_quic.lock().await;
if let Some(socket) = sockets.get(&dest_port) {
let _ = socket.send_to(payload, client_addr).await;
}
}
}
}
Err(e) => {
log::debug!("QUIC datagram recv error: {}", e);
}
}
}
// QUIC connection closed
reason = quic_conn.closed() => {
log::info!("QUIC connection closed: {}", reason);
@@ -1245,6 +1585,9 @@ async fn connect_to_hub_and_run_quic_with_connection(
for (_, h) in port_listeners.drain() {
h.abort();
}
for (_, h) in udp_listeners_quic.drain() {
h.abort();
}
// Graceful QUIC close
quic_conn.close(quinn::VarInt::from_u32(0), b"shutdown");
@@ -1262,6 +1605,7 @@ fn apply_port_config_quic(
edge_id: &str,
connection_token: &CancellationToken,
bind_address: &str,
performance: &EffectivePerformanceConfig,
) {
let new_set: std::collections::HashSet<u16> = new_ports.iter().copied().collect();
let old_set: std::collections::HashSet<u16> = port_listeners.keys().copied().collect();
@@ -1282,6 +1626,7 @@ fn apply_port_config_quic(
let _edge_id = edge_id.to_string();
let port_token = connection_token.child_token();
let bind_addr = bind_address.to_string();
let performance = performance.clone();
let handle = tokio::spawn(async move {
let listener = match TcpListener::bind((bind_addr.as_str(), port)).await {
@@ -1310,7 +1655,10 @@ fn apply_port_config_quic(
let active_streams = active_streams.clone();
let client_token = port_token.child_token();
active_streams.fetch_add(1, Ordering::Relaxed);
if !try_reserve_stream(&active_streams, performance.max_streams_per_edge) {
log::warn!("Rejecting QUIC client on port {}: max streams ({}) reached", port, performance.max_streams_per_edge);
continue;
}
tokio::spawn(async move {
handle_client_connection_quic(
@@ -1321,17 +1669,7 @@ fn apply_port_config_quic(
quic_conn,
client_token,
).await;
// Saturating decrement
loop {
let current = active_streams.load(Ordering::Relaxed);
if current == 0 { break; }
if active_streams.compare_exchange_weak(
current, current - 1,
Ordering::Relaxed, Ordering::Relaxed,
).is_ok() {
break;
}
}
release_stream(&active_streams);
});
}
Err(e) => {
@@ -1353,6 +1691,107 @@ fn apply_port_config_quic(
/// Handle a single client connection via QUIC transport.
/// Opens a new QUIC bidirectional stream, sends the PROXY header,
/// then bidirectionally copies data between the client TCP socket and the QUIC stream.
/// Apply UDP port config for QUIC transport: bind UdpSockets that send via QUIC datagrams.
fn apply_udp_port_config_quic(
new_ports: &[u16],
udp_listeners: &mut HashMap<u16, JoinHandle<()>>,
quic_conn: &quinn::Connection,
udp_sessions: &Arc<Mutex<UdpSessionManager>>,
udp_sockets: &Arc<Mutex<HashMap<u16, Arc<UdpSocket>>>>,
next_stream_id: &Arc<AtomicU32>,
connection_token: &CancellationToken,
bind_address: &str,
) {
let new_set: std::collections::HashSet<u16> = new_ports.iter().copied().collect();
let old_set: std::collections::HashSet<u16> = udp_listeners.keys().copied().collect();
for &port in old_set.difference(&new_set) {
if let Some(handle) = udp_listeners.remove(&port) {
log::info!("Stopping QUIC UDP listener on port {}", port);
handle.abort();
}
let sockets = udp_sockets.clone();
tokio::spawn(async move { sockets.lock().await.remove(&port); });
}
for &port in new_set.difference(&old_set) {
let quic_conn = quic_conn.clone();
let udp_sessions = udp_sessions.clone();
let udp_sockets = udp_sockets.clone();
let next_stream_id = next_stream_id.clone();
let port_token = connection_token.child_token();
let bind_addr = bind_address.to_string();
let handle = tokio::spawn(async move {
let socket = match UdpSocket::bind((bind_addr.as_str(), port)).await {
Ok(s) => Arc::new(s),
Err(e) => {
log::error!("Failed to bind QUIC UDP port {}: {}", port, e);
return;
}
};
log::info!("Listening on UDP port {} (QUIC datagram transport)", port);
udp_sockets.lock().await.insert(port, socket.clone());
let mut buf = vec![0u8; 65536];
loop {
tokio::select! {
recv_result = socket.recv_from(&mut buf) => {
match recv_result {
Ok((len, client_addr)) => {
let key = UdpSessionKey { client_addr, dest_port: port };
let mut sessions = udp_sessions.lock().await;
let stream_id = if let Some(session) = sessions.get_mut(&key) {
session.stream_id
} else {
// New session — send PROXY v2 header via control-style datagram
let sid = next_stream_id.fetch_add(1, Ordering::Relaxed);
if sessions.insert(key, sid).is_none() {
log::warn!("QUIC UDP session limit reached, dropping datagram from {}", client_addr);
continue;
}
let client_ip = client_addr.ip().to_string();
let client_port = client_addr.port();
let proxy_header = build_proxy_v2_header_from_str(
&client_ip, "0.0.0.0", client_port, port,
ProxyV2Transport::Udp,
);
// Send OPEN as a QUIC datagram: [session_id:4][0xFF magic:1][proxy_header:28]
let mut open_buf = Vec::with_capacity(4 + 1 + proxy_header.len());
open_buf.extend_from_slice(&sid.to_be_bytes());
open_buf.push(0xFF); // magic byte to distinguish OPEN from DATA
open_buf.extend_from_slice(&proxy_header);
let _ = quic_conn.send_datagram(open_buf.into());
log::debug!("New QUIC UDP session {} from {} -> port {}", sid, client_addr, port);
sid
};
drop(sessions);
// Send datagram: [session_id:4][payload:N]
let mut dgram = Vec::with_capacity(4 + len);
dgram.extend_from_slice(&stream_id.to_be_bytes());
dgram.extend_from_slice(&buf[..len]);
let _ = quic_conn.send_datagram(dgram.into());
}
Err(e) => {
log::error!("QUIC UDP recv error on port {}: {}", port, e);
}
}
}
_ = port_token.cancelled() => {
log::info!("QUIC UDP port {} listener cancelled", port);
break;
}
}
}
});
udp_listeners.insert(port, handle);
}
}
async fn handle_client_connection_quic(
client_stream: TcpStream,
client_addr: std::net::SocketAddr,
File diff suppressed because it is too large Load Diff
@@ -2,5 +2,7 @@ pub mod hub;
pub mod edge;
pub mod stun;
pub mod transport;
pub mod udp_session;
pub mod performance;
pub use remoteingress_protocol as protocol;
@@ -0,0 +1,130 @@
use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
pub enum PerformanceProfile {
Balanced,
Throughput,
HighConcurrency,
}
impl Default for PerformanceProfile {
fn default() -> Self {
PerformanceProfile::Balanced
}
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct PerformanceConfig {
#[serde(default)]
pub profile: Option<PerformanceProfile>,
#[serde(default)]
pub max_streams_per_edge: Option<usize>,
#[serde(default)]
pub total_window_budget_bytes: Option<u64>,
#[serde(default)]
pub min_stream_window_bytes: Option<u32>,
#[serde(default)]
pub max_stream_window_bytes: Option<u32>,
#[serde(default)]
pub sustained_stream_window_bytes: Option<u32>,
#[serde(default)]
pub quic_datagram_receive_buffer_bytes: Option<usize>,
}
#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct EffectivePerformanceConfig {
pub profile: PerformanceProfile,
pub max_streams_per_edge: usize,
pub total_window_budget_bytes: u64,
pub min_stream_window_bytes: u32,
pub max_stream_window_bytes: u32,
pub sustained_stream_window_bytes: u32,
pub quic_datagram_receive_buffer_bytes: usize,
}
impl Default for EffectivePerformanceConfig {
fn default() -> Self {
PerformanceConfig::default().effective()
}
}
impl PerformanceConfig {
pub fn effective(&self) -> EffectivePerformanceConfig {
let profile = self.profile.unwrap_or_default();
let defaults = profile_defaults(profile);
EffectivePerformanceConfig {
profile,
max_streams_per_edge: self.max_streams_per_edge.unwrap_or(defaults.max_streams_per_edge),
total_window_budget_bytes: self.total_window_budget_bytes.unwrap_or(defaults.total_window_budget_bytes),
min_stream_window_bytes: self.min_stream_window_bytes.unwrap_or(defaults.min_stream_window_bytes),
max_stream_window_bytes: self.max_stream_window_bytes.unwrap_or(defaults.max_stream_window_bytes),
sustained_stream_window_bytes: self.sustained_stream_window_bytes.unwrap_or(defaults.sustained_stream_window_bytes),
quic_datagram_receive_buffer_bytes: self
.quic_datagram_receive_buffer_bytes
.unwrap_or(defaults.quic_datagram_receive_buffer_bytes),
}
}
pub fn merge(global: Option<&PerformanceConfig>, edge: Option<&PerformanceConfig>) -> PerformanceConfig {
let mut merged = global.cloned().unwrap_or_default();
if let Some(edge) = edge {
if edge.profile.is_some() {
merged.profile = edge.profile;
}
if edge.max_streams_per_edge.is_some() {
merged.max_streams_per_edge = edge.max_streams_per_edge;
}
if edge.total_window_budget_bytes.is_some() {
merged.total_window_budget_bytes = edge.total_window_budget_bytes;
}
if edge.min_stream_window_bytes.is_some() {
merged.min_stream_window_bytes = edge.min_stream_window_bytes;
}
if edge.max_stream_window_bytes.is_some() {
merged.max_stream_window_bytes = edge.max_stream_window_bytes;
}
if edge.sustained_stream_window_bytes.is_some() {
merged.sustained_stream_window_bytes = edge.sustained_stream_window_bytes;
}
if edge.quic_datagram_receive_buffer_bytes.is_some() {
merged.quic_datagram_receive_buffer_bytes = edge.quic_datagram_receive_buffer_bytes;
}
}
merged
}
}
fn profile_defaults(profile: PerformanceProfile) -> EffectivePerformanceConfig {
match profile {
PerformanceProfile::Balanced => EffectivePerformanceConfig {
profile,
max_streams_per_edge: 1024,
total_window_budget_bytes: 256 * 1024 * 1024,
min_stream_window_bytes: 128 * 1024,
max_stream_window_bytes: 8 * 1024 * 1024,
sustained_stream_window_bytes: 512 * 1024,
quic_datagram_receive_buffer_bytes: 4 * 1024 * 1024,
},
PerformanceProfile::Throughput => EffectivePerformanceConfig {
profile,
max_streams_per_edge: 512,
total_window_budget_bytes: 512 * 1024 * 1024,
min_stream_window_bytes: 256 * 1024,
max_stream_window_bytes: 32 * 1024 * 1024,
sustained_stream_window_bytes: 2 * 1024 * 1024,
quic_datagram_receive_buffer_bytes: 8 * 1024 * 1024,
},
PerformanceProfile::HighConcurrency => EffectivePerformanceConfig {
profile,
max_streams_per_edge: 4096,
total_window_budget_bytes: 256 * 1024 * 1024,
min_stream_window_bytes: 64 * 1024,
max_stream_window_bytes: 4 * 1024 * 1024,
sustained_stream_window_bytes: 256 * 1024,
quic_datagram_receive_buffer_bytes: 16 * 1024 * 1024,
},
}
}
@@ -4,9 +4,9 @@ use serde::{Deserialize, Serialize};
/// Transport mode for the tunnel connection between edge and hub.
///
/// - `TcpTls`: TCP + TLS with frame-based multiplexing via TunnelIo (default).
/// - `TcpTls`: TCP + TLS with frame-based multiplexing via TunnelIo.
/// - `Quic`: QUIC with native stream multiplexing (one QUIC stream per tunneled connection).
/// - `QuicWithFallback`: Try QUIC first, fall back to TCP+TLS if UDP is blocked.
/// - `QuicWithFallback`: Try QUIC first, fall back to TCP+TLS if UDP is blocked. This is the edge runtime default.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
pub enum TransportMode {
@@ -1,5 +1,7 @@
use std::sync::Arc;
const DEFAULT_DATAGRAM_RECEIVE_BUFFER_SIZE: usize = 4 * 1024 * 1024;
/// QUIC control stream message types (reuses frame type constants for consistency).
pub const CTRL_CONFIG: u8 = 0x06;
pub const CTRL_PING: u8 = 0x07;
@@ -11,6 +13,13 @@ pub const CTRL_HEADER_SIZE: usize = 5;
/// Build a quinn ClientConfig that skips server certificate verification
/// (auth is via shared secret, same as the TCP+TLS path).
pub fn build_quic_client_config() -> quinn::ClientConfig {
build_quic_client_config_with_limits(1024, DEFAULT_DATAGRAM_RECEIVE_BUFFER_SIZE)
}
pub fn build_quic_client_config_with_limits(
max_concurrent_bidi_streams: u32,
datagram_receive_buffer_size: usize,
) -> quinn::ClientConfig {
let mut tls_config = rustls::ClientConfig::builder()
.dangerous()
.with_custom_certificate_verifier(Arc::new(NoCertVerifier))
@@ -28,9 +37,9 @@ pub fn build_quic_client_config() -> quinn::ClientConfig {
transport.max_idle_timeout(Some(
quinn::IdleTimeout::try_from(std::time::Duration::from_secs(45)).unwrap(),
));
// Match MAX_STREAMS_PER_EDGE (1024) from hub.rs.
// Default is 100 which is too low for high-concurrency tunneling.
transport.max_concurrent_bidi_streams(1024u32.into());
transport.max_concurrent_bidi_streams(max_concurrent_bidi_streams.into());
// Enable QUIC datagrams (RFC 9221) for low-latency UDP tunneling.
transport.datagram_receive_buffer_size(Some(datagram_receive_buffer_size));
let mut client_config = quinn::ClientConfig::new(Arc::new(quic_config));
client_config.transport_config(Arc::new(transport));
@@ -40,6 +49,18 @@ pub fn build_quic_client_config() -> quinn::ClientConfig {
/// Build a quinn ServerConfig from the same TLS server config used for TCP+TLS.
pub fn build_quic_server_config(
tls_server_config: rustls::ServerConfig,
) -> Result<quinn::ServerConfig, Box<dyn std::error::Error + Send + Sync>> {
build_quic_server_config_with_limits(
tls_server_config,
1024,
DEFAULT_DATAGRAM_RECEIVE_BUFFER_SIZE,
)
}
pub fn build_quic_server_config_with_limits(
tls_server_config: rustls::ServerConfig,
max_concurrent_bidi_streams: u32,
datagram_receive_buffer_size: usize,
) -> Result<quinn::ServerConfig, Box<dyn std::error::Error + Send + Sync>> {
let quic_config = quinn::crypto::rustls::QuicServerConfig::try_from(tls_server_config)?;
@@ -48,7 +69,8 @@ pub fn build_quic_server_config(
transport.max_idle_timeout(Some(
quinn::IdleTimeout::try_from(std::time::Duration::from_secs(45)).unwrap(),
));
transport.max_concurrent_bidi_streams(1024u32.into());
transport.max_concurrent_bidi_streams(max_concurrent_bidi_streams.into());
transport.datagram_receive_buffer_size(Some(datagram_receive_buffer_size));
let mut server_config = quinn::ServerConfig::with_crypto(Arc::new(quic_config));
server_config.transport_config(Arc::new(transport));
@@ -73,6 +95,11 @@ pub async fn write_ctrl_message(
Ok(())
}
/// Maximum size for a QUIC control message payload (64 KB).
/// Control messages (CONFIG, PING, PONG) are small; this guards against
/// a malicious peer sending a crafted length field to trigger OOM.
const MAX_CTRL_MESSAGE_SIZE: usize = 65536;
/// Read a control message from a QUIC recv stream.
/// Returns (msg_type, payload). Returns None on EOF.
pub async fn read_ctrl_message(
@@ -90,6 +117,12 @@ pub async fn read_ctrl_message(
}
let msg_type = header[0];
let len = u32::from_be_bytes([header[1], header[2], header[3], header[4]]) as usize;
if len > MAX_CTRL_MESSAGE_SIZE {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
format!("control message too large: {} bytes (max {})", len, MAX_CTRL_MESSAGE_SIZE),
));
}
let mut payload = vec![0u8; len];
if len > 0 {
recv.read_exact(&mut payload).await.map_err(|e| {
@@ -0,0 +1,245 @@
use std::collections::HashMap;
use std::net::SocketAddr;
use tokio::time::Instant;
/// Key identifying a unique UDP "session" (one client endpoint talking to one destination port).
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct UdpSessionKey {
pub client_addr: SocketAddr,
pub dest_port: u16,
}
/// A single UDP session tracked by the edge.
pub struct UdpSession {
pub stream_id: u32,
pub client_addr: SocketAddr,
pub dest_port: u16,
pub last_activity: Instant,
}
/// Manages UDP sessions with idle timeout expiry and a maximum session count.
pub struct UdpSessionManager {
/// Forward map: session key → session data.
sessions: HashMap<UdpSessionKey, UdpSession>,
/// Reverse map: stream_id → session key (for dispatching return traffic).
by_stream_id: HashMap<u32, UdpSessionKey>,
/// Idle timeout duration.
idle_timeout: std::time::Duration,
/// Maximum number of concurrent sessions (prevents unbounded growth from floods).
max_sessions: usize,
}
impl UdpSessionManager {
pub fn new(idle_timeout: std::time::Duration) -> Self {
Self::with_max_sessions(idle_timeout, 65536)
}
pub fn with_max_sessions(idle_timeout: std::time::Duration, max_sessions: usize) -> Self {
Self {
sessions: HashMap::new(),
by_stream_id: HashMap::new(),
idle_timeout,
max_sessions,
}
}
/// Look up an existing session by key. Updates last_activity on hit.
pub fn get_mut(&mut self, key: &UdpSessionKey) -> Option<&mut UdpSession> {
let session = self.sessions.get_mut(key)?;
session.last_activity = Instant::now();
Some(session)
}
/// Look up a session's client address by stream_id (for return traffic).
pub fn client_addr_for_stream(&self, stream_id: u32) -> Option<SocketAddr> {
let key = self.by_stream_id.get(&stream_id)?;
self.sessions.get(key).map(|s| s.client_addr)
}
/// Look up a session by stream_id. Updates last_activity on hit.
pub fn get_by_stream_id(&mut self, stream_id: u32) -> Option<&mut UdpSession> {
let key = self.by_stream_id.get(&stream_id)?;
let session = self.sessions.get_mut(key)?;
session.last_activity = Instant::now();
Some(session)
}
/// Insert a new session. Returns `None` if the session limit has been reached.
pub fn insert(&mut self, key: UdpSessionKey, stream_id: u32) -> Option<&mut UdpSession> {
// Allow re-insertion of existing keys (update), but reject truly new sessions at capacity
if !self.sessions.contains_key(&key) && self.sessions.len() >= self.max_sessions {
return None;
}
let session = UdpSession {
stream_id,
client_addr: key.client_addr,
dest_port: key.dest_port,
last_activity: Instant::now(),
};
self.by_stream_id.insert(stream_id, key);
Some(self.sessions.entry(key).or_insert(session))
}
/// Remove a session by stream_id.
pub fn remove_by_stream_id(&mut self, stream_id: u32) -> Option<UdpSession> {
if let Some(key) = self.by_stream_id.remove(&stream_id) {
self.sessions.remove(&key)
} else {
None
}
}
/// Expire idle sessions. Returns the stream_ids of expired sessions.
pub fn expire_idle(&mut self) -> Vec<u32> {
let now = Instant::now();
let timeout = self.idle_timeout;
let expired_keys: Vec<UdpSessionKey> = self
.sessions
.iter()
.filter(|(_, s)| now.duration_since(s.last_activity) >= timeout)
.map(|(k, _)| *k)
.collect();
let mut expired_ids = Vec::with_capacity(expired_keys.len());
for key in expired_keys {
if let Some(session) = self.sessions.remove(&key) {
self.by_stream_id.remove(&session.stream_id);
expired_ids.push(session.stream_id);
}
}
expired_ids
}
/// Number of active sessions.
pub fn len(&self) -> usize {
self.sessions.len()
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::time::Duration;
fn addr(port: u16) -> SocketAddr {
SocketAddr::from(([127, 0, 0, 1], port))
}
#[test]
fn test_insert_and_lookup() {
let mut mgr = UdpSessionManager::new(Duration::from_secs(60));
let key = UdpSessionKey { client_addr: addr(5000), dest_port: 53 };
assert!(mgr.insert(key, 1).is_some());
assert_eq!(mgr.len(), 1);
assert!(mgr.get_mut(&key).is_some());
assert_eq!(mgr.get_mut(&key).unwrap().stream_id, 1);
}
#[test]
fn test_client_addr_for_stream() {
let mut mgr = UdpSessionManager::new(Duration::from_secs(60));
let key = UdpSessionKey { client_addr: addr(5000), dest_port: 53 };
assert!(mgr.insert(key, 42).is_some());
assert_eq!(mgr.client_addr_for_stream(42), Some(addr(5000)));
assert_eq!(mgr.client_addr_for_stream(99), None);
}
#[test]
fn test_remove_by_stream_id() {
let mut mgr = UdpSessionManager::new(Duration::from_secs(60));
let key = UdpSessionKey { client_addr: addr(5000), dest_port: 53 };
assert!(mgr.insert(key, 1).is_some());
let removed = mgr.remove_by_stream_id(1);
assert!(removed.is_some());
assert_eq!(mgr.len(), 0);
assert!(mgr.get_mut(&key).is_none());
assert_eq!(mgr.client_addr_for_stream(1), None);
}
#[test]
fn test_remove_nonexistent() {
let mut mgr = UdpSessionManager::new(Duration::from_secs(60));
assert!(mgr.remove_by_stream_id(999).is_none());
}
#[tokio::test]
async fn test_expire_idle() {
let mut mgr = UdpSessionManager::new(Duration::from_millis(50));
let key1 = UdpSessionKey { client_addr: addr(5000), dest_port: 53 };
let key2 = UdpSessionKey { client_addr: addr(5001), dest_port: 53 };
assert!(mgr.insert(key1, 1).is_some());
assert!(mgr.insert(key2, 2).is_some());
// Nothing expired yet
assert!(mgr.expire_idle().is_empty());
assert_eq!(mgr.len(), 2);
// Wait for timeout
tokio::time::sleep(Duration::from_millis(60)).await;
let expired = mgr.expire_idle();
assert_eq!(expired.len(), 2);
assert_eq!(mgr.len(), 0);
}
#[tokio::test]
async fn test_activity_prevents_expiry() {
let mut mgr = UdpSessionManager::new(Duration::from_millis(100));
let key = UdpSessionKey { client_addr: addr(5000), dest_port: 53 };
assert!(mgr.insert(key, 1).is_some());
// Touch session at 50ms (before 100ms timeout)
tokio::time::sleep(Duration::from_millis(50)).await;
mgr.get_mut(&key); // refreshes last_activity
// At 80ms from last touch, should still be alive
tokio::time::sleep(Duration::from_millis(80)).await;
assert!(mgr.expire_idle().is_empty());
assert_eq!(mgr.len(), 1);
// Wait for full timeout from last activity
tokio::time::sleep(Duration::from_millis(30)).await;
let expired = mgr.expire_idle();
assert_eq!(expired.len(), 1);
}
#[test]
fn test_multiple_sessions_same_client_different_ports() {
let mut mgr = UdpSessionManager::new(Duration::from_secs(60));
let key1 = UdpSessionKey { client_addr: addr(5000), dest_port: 53 };
let key2 = UdpSessionKey { client_addr: addr(5000), dest_port: 443 };
assert!(mgr.insert(key1, 1).is_some());
assert!(mgr.insert(key2, 2).is_some());
assert_eq!(mgr.len(), 2);
assert_eq!(mgr.get_mut(&key1).unwrap().stream_id, 1);
assert_eq!(mgr.get_mut(&key2).unwrap().stream_id, 2);
}
#[test]
fn test_max_sessions_limit() {
let mut mgr = UdpSessionManager::with_max_sessions(Duration::from_secs(60), 2);
let key1 = UdpSessionKey { client_addr: addr(5000), dest_port: 53 };
let key2 = UdpSessionKey { client_addr: addr(5001), dest_port: 53 };
let key3 = UdpSessionKey { client_addr: addr(5002), dest_port: 53 };
assert!(mgr.insert(key1, 1).is_some());
assert!(mgr.insert(key2, 2).is_some());
// Third insert should be rejected (at capacity)
assert!(mgr.insert(key3, 3).is_none());
assert_eq!(mgr.len(), 2);
// Re-inserting an existing key should succeed (update, not new)
assert!(mgr.insert(key1, 1).is_some());
assert_eq!(mgr.len(), 2);
// After removing one, a new insert should succeed
mgr.remove_by_stream_id(1);
assert_eq!(mgr.len(), 1);
assert!(mgr.insert(key3, 3).is_some());
assert_eq!(mgr.len(), 2);
}
}
+207 -30
View File
@@ -19,6 +19,12 @@ pub const FRAME_PONG: u8 = 0x08; // Edge -> Hub: heartbeat response
pub const FRAME_WINDOW_UPDATE: u8 = 0x09; // Edge -> Hub: per-stream flow control
pub const FRAME_WINDOW_UPDATE_BACK: u8 = 0x0A; // Hub -> Edge: per-stream flow control
// UDP tunnel frame types
pub const FRAME_UDP_OPEN: u8 = 0x0B; // Edge -> Hub: open UDP session (payload: PROXY v2 header)
pub const FRAME_UDP_DATA: u8 = 0x0C; // Edge -> Hub: UDP datagram
pub const FRAME_UDP_DATA_BACK: u8 = 0x0D; // Hub -> Edge: UDP datagram
pub const FRAME_UDP_CLOSE: u8 = 0x0E; // Either direction: close UDP session
// Frame header size: 4 (stream_id) + 1 (type) + 4 (length) = 9 bytes
pub const FRAME_HEADER_SIZE: usize = 9;
@@ -26,12 +32,18 @@ pub const FRAME_HEADER_SIZE: usize = 9;
pub const MAX_PAYLOAD_SIZE: u32 = 16 * 1024 * 1024;
// Per-stream flow control constants
/// Initial (and maximum) per-stream window size (4 MB).
pub const INITIAL_STREAM_WINDOW: u32 = 4 * 1024 * 1024;
/// Send WINDOW_UPDATE after consuming this many bytes (half the initial window).
/// Default maximum per-stream window size (8 MB).
pub const INITIAL_STREAM_WINDOW: u32 = 8 * 1024 * 1024;
/// Minimum safe window size used when strict budget pressure requires going below the configured floor.
pub const ABSOLUTE_MIN_STREAM_WINDOW: u32 = 16 * 1024;
/// Default total TCP/TLS flow-control budget per edge connection (256 MB).
pub const DEFAULT_TOTAL_WINDOW_BUDGET: u64 = 256 * 1024 * 1024;
/// Default preferred minimum stream window (128 KB). The total budget still wins above this.
pub const DEFAULT_MIN_STREAM_WINDOW: u32 = 128 * 1024;
/// Send WINDOW_UPDATE after consuming this many bytes when no dynamic window is available.
pub const WINDOW_UPDATE_THRESHOLD: u32 = INITIAL_STREAM_WINDOW / 2;
/// Maximum window size to prevent overflow.
pub const MAX_WINDOW_SIZE: u32 = 4 * 1024 * 1024;
pub const MAX_WINDOW_SIZE: u32 = 32 * 1024 * 1024;
// Sustained stream classification constants
/// Throughput threshold for sustained classification (2.5 MB/s = 20 Mbit/s).
@@ -49,11 +61,37 @@ pub fn encode_window_update(stream_id: u32, frame_type: u8, increment: u32) -> B
}
/// Compute the target per-stream window size based on the number of active streams.
/// Total memory budget is ~200MB shared across all streams. Up to 50 streams get the
/// full 4MB window; above that the window scales down to a 1MB floor at 200+ streams.
/// The total budget is authoritative: the configured minimum is a preference, not
/// permission to exceed the edge-level memory budget under very high concurrency.
pub fn compute_window_for_stream_count(active: u32) -> u32 {
let per_stream = (200 * 1024 * 1024u64) / (active.max(1) as u64);
per_stream.clamp(1 * 1024 * 1024, INITIAL_STREAM_WINDOW as u64) as u32
compute_window_for_limits(
active,
DEFAULT_TOTAL_WINDOW_BUDGET,
DEFAULT_MIN_STREAM_WINDOW,
INITIAL_STREAM_WINDOW,
)
}
pub fn compute_window_for_limits(
active: u32,
total_budget_bytes: u64,
min_window_bytes: u32,
max_window_bytes: u32,
) -> u32 {
let active = active.max(1) as u64;
let max_window = max_window_bytes.max(ABSOLUTE_MIN_STREAM_WINDOW);
let preferred_min = min_window_bytes
.max(ABSOLUTE_MIN_STREAM_WINDOW)
.min(max_window);
let per_stream_budget = total_budget_bytes
.max(ABSOLUTE_MIN_STREAM_WINDOW as u64)
/ active;
let bounded = per_stream_budget.min(max_window as u64);
if bounded >= preferred_min as u64 {
bounded as u32
} else {
bounded.max(ABSOLUTE_MIN_STREAM_WINDOW as u64) as u32
}
}
/// Decode a WINDOW_UPDATE payload into a byte increment. Returns None if payload is malformed.
@@ -107,6 +145,76 @@ pub fn build_proxy_v1_header(
)
}
/// PROXY protocol v2 signature (12 bytes).
pub const PROXY_V2_SIGNATURE: [u8; 12] = [
0x0D, 0x0A, 0x0D, 0x0A, 0x00, 0x0D, 0x0A, 0x51, 0x55, 0x49, 0x54, 0x0A,
];
/// Transport protocol for PROXY v2 header.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ProxyV2Transport {
/// TCP (STREAM) — byte 13 low nibble = 0x1
Tcp,
/// UDP (DGRAM) — byte 13 low nibble = 0x2
Udp,
}
/// Build a PROXY protocol v2 binary header for IPv4.
///
/// Returns a 28-byte header:
/// - 12B signature
/// - 1B version (0x2) + command (0x1 = PROXY)
/// - 1B address family (0x1 = AF_INET) + transport (0x1 = TCP, 0x2 = UDP)
/// - 2B address block length (0x000C = 12)
/// - 4B source IPv4 address
/// - 4B destination IPv4 address
/// - 2B source port
/// - 2B destination port
pub fn build_proxy_v2_header(
src_ip: &std::net::Ipv4Addr,
dst_ip: &std::net::Ipv4Addr,
src_port: u16,
dst_port: u16,
transport: ProxyV2Transport,
) -> Bytes {
let mut buf = BytesMut::with_capacity(28);
// Signature (12 bytes)
buf.put_slice(&PROXY_V2_SIGNATURE);
// Version 2 + PROXY command
buf.put_u8(0x21);
// AF_INET (0x1) + transport
let transport_nibble = match transport {
ProxyV2Transport::Tcp => 0x1,
ProxyV2Transport::Udp => 0x2,
};
buf.put_u8(0x10 | transport_nibble);
// Address block length: 12 bytes for IPv4
buf.put_u16(12);
// Source address (4 bytes, network byte order)
buf.put_slice(&src_ip.octets());
// Destination address (4 bytes, network byte order)
buf.put_slice(&dst_ip.octets());
// Source port (2 bytes, network byte order)
buf.put_u16(src_port);
// Destination port (2 bytes, network byte order)
buf.put_u16(dst_port);
buf.freeze()
}
/// Build a PROXY protocol v2 binary header from string IP addresses.
/// Falls back to 0.0.0.0 if parsing fails.
pub fn build_proxy_v2_header_from_str(
src_ip: &str,
dst_ip: &str,
src_port: u16,
dst_port: u16,
transport: ProxyV2Transport,
) -> Bytes {
let src: std::net::Ipv4Addr = src_ip.parse().unwrap_or(std::net::Ipv4Addr::UNSPECIFIED);
let dst: std::net::Ipv4Addr = dst_ip.parse().unwrap_or(std::net::Ipv4Addr::UNSPECIFIED);
build_proxy_v2_header(&src, &dst, src_port, dst_port, transport)
}
/// Stateful async frame reader that yields `Frame` values from an `AsyncRead`.
pub struct FrameReader<R> {
reader: R,
@@ -231,6 +339,13 @@ pub struct TunnelIo<S> {
write: WriteState,
}
#[derive(Debug, Clone, Copy, Default)]
pub struct TunnelQueueDepths {
pub ctrl: usize,
pub data: usize,
pub sustained: usize,
}
impl<S: AsyncRead + AsyncWrite + Unpin> TunnelIo<S> {
pub fn new(stream: S, initial_data: Vec<u8>) -> Self {
let read_pos = initial_data.len();
@@ -270,6 +385,14 @@ impl<S: AsyncRead + AsyncWrite + Unpin> TunnelIo<S> {
self.write.sustained_queue.push_back(frame);
}
pub fn queue_depths(&self) -> TunnelQueueDepths {
TunnelQueueDepths {
ctrl: self.write.ctrl_queue.len(),
data: self.write.data_queue.len(),
sustained: self.write.sustained_queue.len(),
}
}
/// Try to parse a complete frame from the read buffer.
/// Uses a parse_pos cursor to avoid drain() on every frame.
pub fn try_parse_frame(&mut self) -> Option<Result<Frame, std::io::Error>> {
@@ -604,6 +727,62 @@ mod tests {
assert_eq!(header, "PROXY TCP4 1.2.3.4 5.6.7.8 12345 443\r\n");
}
#[test]
fn test_proxy_v2_header_tcp4() {
let src = "198.51.100.10".parse().unwrap();
let dst = "203.0.113.25".parse().unwrap();
let header = build_proxy_v2_header(&src, &dst, 54321, 8443, ProxyV2Transport::Tcp);
assert_eq!(header.len(), 28);
// Signature
assert_eq!(&header[0..12], &PROXY_V2_SIGNATURE);
// Version 2 + PROXY command
assert_eq!(header[12], 0x21);
// AF_INET + STREAM (TCP)
assert_eq!(header[13], 0x11);
// Address length = 12
assert_eq!(u16::from_be_bytes([header[14], header[15]]), 12);
// Source IP: 198.51.100.10
assert_eq!(&header[16..20], &[198, 51, 100, 10]);
// Dest IP: 203.0.113.25
assert_eq!(&header[20..24], &[203, 0, 113, 25]);
// Source port: 54321
assert_eq!(u16::from_be_bytes([header[24], header[25]]), 54321);
// Dest port: 8443
assert_eq!(u16::from_be_bytes([header[26], header[27]]), 8443);
}
#[test]
fn test_proxy_v2_header_udp4() {
let src = "10.0.0.1".parse().unwrap();
let dst = "10.0.0.2".parse().unwrap();
let header = build_proxy_v2_header(&src, &dst, 12345, 53, ProxyV2Transport::Udp);
assert_eq!(header.len(), 28);
assert_eq!(header[12], 0x21); // v2, PROXY
assert_eq!(header[13], 0x12); // AF_INET + DGRAM (UDP)
assert_eq!(&header[16..20], &[10, 0, 0, 1]); // src
assert_eq!(&header[20..24], &[10, 0, 0, 2]); // dst
assert_eq!(u16::from_be_bytes([header[24], header[25]]), 12345);
assert_eq!(u16::from_be_bytes([header[26], header[27]]), 53);
}
#[test]
fn test_proxy_v2_header_from_str() {
let header = build_proxy_v2_header_from_str("1.2.3.4", "5.6.7.8", 1000, 443, ProxyV2Transport::Tcp);
assert_eq!(header.len(), 28);
assert_eq!(&header[16..20], &[1, 2, 3, 4]);
assert_eq!(&header[20..24], &[5, 6, 7, 8]);
}
#[test]
fn test_proxy_v2_header_from_str_invalid_ip() {
let header = build_proxy_v2_header_from_str("not-an-ip", "also-not", 1000, 443, ProxyV2Transport::Udp);
assert_eq!(header.len(), 28);
// Falls back to 0.0.0.0
assert_eq!(&header[16..20], &[0, 0, 0, 0]);
assert_eq!(&header[20..24], &[0, 0, 0, 0]);
assert_eq!(header[13], 0x12); // UDP
}
#[tokio::test]
async fn test_frame_reader() {
let frame1 = encode_frame(1, FRAME_OPEN, b"PROXY TCP4 1.2.3.4 5.6.7.8 1234 443\r\n");
@@ -778,7 +957,7 @@ mod tests {
#[test]
fn test_adaptive_window_zero_streams() {
// 0 streams treated as 1: 200MB/1 -> clamped to 4MB max
// 0 streams treated as 1: budget/1 -> clamped to max
assert_eq!(compute_window_for_stream_count(0), INITIAL_STREAM_WINDOW);
}
@@ -788,47 +967,44 @@ mod tests {
}
#[test]
fn test_adaptive_window_50_streams_full() {
// 200MB/50 = 4MB = exactly INITIAL_STREAM_WINDOW
assert_eq!(compute_window_for_stream_count(50), INITIAL_STREAM_WINDOW);
fn test_adaptive_window_32_streams_full() {
// 256MB/32 = 8MB = exactly INITIAL_STREAM_WINDOW
assert_eq!(compute_window_for_stream_count(32), INITIAL_STREAM_WINDOW);
}
#[test]
fn test_adaptive_window_51_streams_starts_scaling() {
// 200MB/51 < 4MB — first value below max
let w = compute_window_for_stream_count(51);
fn test_adaptive_window_33_streams_starts_scaling() {
// 256MB/33 < 8MB — first value below max
let w = compute_window_for_stream_count(33);
assert!(w < INITIAL_STREAM_WINDOW);
assert_eq!(w, (200 * 1024 * 1024u64 / 51) as u32);
assert_eq!(w, (DEFAULT_TOTAL_WINDOW_BUDGET / 33) as u32);
}
#[test]
fn test_adaptive_window_100_streams() {
// 200MB/100 = 2MB
assert_eq!(compute_window_for_stream_count(100), 2 * 1024 * 1024);
assert_eq!(compute_window_for_stream_count(100), (DEFAULT_TOTAL_WINDOW_BUDGET / 100) as u32);
}
#[test]
fn test_adaptive_window_200_streams_at_floor() {
// 200MB/200 = 1MB = exactly the floor
assert_eq!(compute_window_for_stream_count(200), 1 * 1024 * 1024);
fn test_adaptive_window_200_streams_uses_budget() {
assert_eq!(compute_window_for_stream_count(200), (DEFAULT_TOTAL_WINDOW_BUDGET / 200) as u32);
}
#[test]
fn test_adaptive_window_500_streams_clamped() {
// 200MB/500 = 0.4MB -> clamped up to 1MB floor
assert_eq!(compute_window_for_stream_count(500), 1 * 1024 * 1024);
fn test_adaptive_window_500_streams_stays_under_budget() {
assert_eq!(compute_window_for_stream_count(500), (DEFAULT_TOTAL_WINDOW_BUDGET / 500) as u32);
}
#[test]
fn test_adaptive_window_max_u32() {
// Extreme: u32::MAX streams -> tiny value -> clamped to 1MB
assert_eq!(compute_window_for_stream_count(u32::MAX), 1 * 1024 * 1024);
// Extreme: u32::MAX streams -> tiny value -> clamped to absolute minimum.
assert_eq!(compute_window_for_stream_count(u32::MAX), ABSOLUTE_MIN_STREAM_WINDOW);
}
#[test]
fn test_adaptive_window_monotonically_decreasing() {
let mut prev = compute_window_for_stream_count(1);
for n in [2, 10, 50, 51, 100, 200, 500, 1000] {
for n in [2, 10, 32, 33, 100, 200, 500, 1000] {
let w = compute_window_for_stream_count(n);
assert!(w <= prev, "window increased from {} to {} at n={}", prev, w, n);
prev = w;
@@ -837,11 +1013,12 @@ mod tests {
#[test]
fn test_adaptive_window_total_budget_bounded() {
// active x per_stream_window should never exceed 200MB (+ clamp overhead for high N)
for n in [1, 10, 50, 100, 200] {
// active x per_stream_window should never exceed the configured budget while the
// budget can still provide at least the absolute minimum per stream.
for n in [1, 10, 32, 33, 100, 200, 500, 1000] {
let w = compute_window_for_stream_count(n);
let total = w as u64 * n as u64;
assert!(total <= 200 * 1024 * 1024, "total {}MB exceeds budget at n={}", total / (1024*1024), n);
assert!(total <= DEFAULT_TOTAL_WINDOW_BUDGET, "total {}MB exceeds budget at n={}", total / (1024*1024), n);
}
}
+14 -9
View File
@@ -315,16 +315,21 @@ let echoServer: TrackingServer;
let hubPort: number;
let edgePort: number;
tap.test('setup: start echo server and tunnel', async () => {
tap.test('TCP/TLS setup: start TCP echo server and TCP+TLS tunnel', async () => {
[hubPort, edgePort] = await findFreePorts(2);
echoServer = await startEchoServer(edgePort, '127.0.0.2');
tunnel = await startTunnel(edgePort, hubPort);
expect(tunnel.hub.running).toBeTrue();
const hubStatus = await tunnel.hub.getStatus();
expect(hubStatus.connectedEdges.length).toBeGreaterThanOrEqual(1);
const edgeStatus = hubStatus.connectedEdges[0];
expect(['quic', 'tcpTls'].includes(edgeStatus.transportMode)).toEqual(true);
expect(edgeStatus.performance.maxStreamsPerEdge).toBeGreaterThanOrEqual(1024);
});
tap.test('single stream: 32MB transfer exceeding initial 4MB window (multiple refills)', async () => {
tap.test('TCP/TLS: single TCP stream 32MB transfer exceeding initial 4MB window', async () => {
const size = 32 * 1024 * 1024;
const data = crypto.randomBytes(size);
const expectedHash = sha256(data);
@@ -335,7 +340,7 @@ tap.test('single stream: 32MB transfer exceeding initial 4MB window (multiple re
expect(sha256(received)).toEqual(expectedHash);
});
tap.test('200 concurrent streams with 64KB each', async () => {
tap.test('TCP/TLS: 200 concurrent TCP streams x 64KB each', async () => {
const streamCount = 200;
const payloadSize = 64 * 1024;
@@ -355,7 +360,7 @@ tap.test('200 concurrent streams with 64KB each', async () => {
expect(failures.length).toEqual(0);
});
tap.test('512 concurrent streams at minimum window boundary (16KB each)', async () => {
tap.test('TCP/TLS: 512 concurrent TCP streams at minimum window boundary (16KB each)', async () => {
const streamCount = 512;
const payloadSize = 16 * 1024;
@@ -375,7 +380,7 @@ tap.test('512 concurrent streams at minimum window boundary (16KB each)', async
expect(failures.length).toEqual(0);
});
tap.test('asymmetric transfer: 4KB request -> 4MB response', async () => {
tap.test('TCP/TLS: asymmetric TCP transfer 4KB request -> 4MB response', async () => {
// Swap to large-response server
await forceCloseServer(echoServer);
const responseSize = 4 * 1024 * 1024; // 4 MB
@@ -392,7 +397,7 @@ tap.test('asymmetric transfer: 4KB request -> 4MB response', async () => {
}
});
tap.test('100 streams x 1MB each (100MB total exceeding 200MB budget)', async () => {
tap.test('TCP/TLS: 100 TCP streams x 1MB each (100MB total exceeding 200MB budget)', async () => {
const streamCount = 100;
const payloadSize = 1 * 1024 * 1024;
@@ -412,7 +417,7 @@ tap.test('100 streams x 1MB each (100MB total exceeding 200MB budget)', async ()
expect(failures.length).toEqual(0);
});
tap.test('active stream counter tracks concurrent connections', async () => {
tap.test('TCP/TLS: active TCP stream counter tracks concurrent connections', async () => {
const N = 50;
// Open N connections and keep them alive (send data but don't close)
@@ -445,7 +450,7 @@ tap.test('active stream counter tracks concurrent connections', async () => {
}
});
tap.test('50 streams x 2MB each (forces multiple window refills per stream)', async () => {
tap.test('TCP/TLS: 50 TCP streams x 2MB each (forces multiple window refills)', async () => {
// At 50 concurrent streams: adaptive window = 200MB/50 = 4MB per stream
// Each stream sends 2MB → needs ~3 WINDOW_UPDATE refill cycles per stream
const streamCount = 50;
@@ -467,7 +472,7 @@ tap.test('50 streams x 2MB each (forces multiple window refills per stream)', as
expect(failures.length).toEqual(0);
});
tap.test('teardown: stop tunnel and echo server', async () => {
tap.test('TCP/TLS teardown: stop tunnel and TCP echo server', async () => {
await tunnel.cleanup();
await forceCloseServer(echoServer);
});
+7 -7
View File
@@ -231,7 +231,7 @@ let edgePort: number;
// Tests
// ---------------------------------------------------------------------------
tap.test('setup: start throttled tunnel (100 Mbit/s)', async () => {
tap.test('TCP/TLS setup: start throttled TCP+TLS tunnel (100 Mbit/s)', async () => {
[hubPort, proxyPort, edgePort] = await findFreePorts(3);
echoServer = await startEchoServer(edgePort, '127.0.0.2');
@@ -271,7 +271,7 @@ tap.test('setup: start throttled tunnel (100 Mbit/s)', async () => {
expect(status.connected).toBeTrue();
});
tap.test('throttled: 5 streams x 20MB each through 100Mbit tunnel', async () => {
tap.test('TCP/TLS throttled: 5 TCP streams x 20MB each through 100Mbit tunnel', async () => {
const streamCount = 5;
const payloadSize = 20 * 1024 * 1024; // 20MB per stream = 100MB total round-trip
@@ -293,7 +293,7 @@ tap.test('throttled: 5 streams x 20MB each through 100Mbit tunnel', async () =>
expect(status.connected).toBeTrue();
});
tap.test('throttled: slow consumer with 20MB does not kill other streams', async () => {
tap.test('TCP/TLS throttled: slow TCP consumer with 20MB does not kill other streams', async () => {
// Open a connection that creates download-direction backpressure:
// send 20MB but DON'T read the response — client TCP receive buffer fills
const slowSock = net.createConnection({ host: '127.0.0.1', port: edgePort });
@@ -326,7 +326,7 @@ tap.test('throttled: slow consumer with 20MB does not kill other streams', async
slowSock.destroy();
});
tap.test('throttled: rapid churn — 3 x 20MB long + 50 x 1MB short streams', async () => {
tap.test('TCP/TLS throttled: rapid churn — 3 x 20MB long + 50 x 1MB short TCP streams', async () => {
// 3 long streams (20MB each) running alongside 50 short streams (1MB each)
const longPayload = crypto.randomBytes(20 * 1024 * 1024);
const longHash = sha256(longPayload);
@@ -360,7 +360,7 @@ tap.test('throttled: rapid churn — 3 x 20MB long + 50 x 1MB short streams', as
expect(status.connected).toBeTrue();
});
tap.test('throttled: 3 burst waves of 5 streams x 20MB each', async () => {
tap.test('TCP/TLS throttled: 3 burst waves of 5 TCP streams x 20MB each', async () => {
for (let wave = 0; wave < 3; wave++) {
const streamCount = 5;
const payloadSize = 20 * 1024 * 1024; // 20MB per stream = 100MB per wave
@@ -382,7 +382,7 @@ tap.test('throttled: 3 burst waves of 5 streams x 20MB each', async () => {
}
});
tap.test('throttled: tunnel still works after all load tests', async () => {
tap.test('TCP/TLS throttled: TCP tunnel still works after all load tests', async () => {
const data = crypto.randomBytes(1024);
const hash = sha256(data);
const received = await sendAndReceive(edgePort, data, 30000);
@@ -392,7 +392,7 @@ tap.test('throttled: tunnel still works after all load tests', async () => {
expect(status.connected).toBeTrue();
});
tap.test('teardown: stop tunnel', async () => {
tap.test('TCP/TLS teardown: stop throttled tunnel', async () => {
await edge.stop();
await hub.stop();
if (throttle) await throttle.close();
+228
View File
@@ -0,0 +1,228 @@
import { expect, tap } from '@push.rocks/tapbundle';
import * as net from 'net';
import * as crypto from 'crypto';
import { RemoteIngressHub, RemoteIngressEdge } from '../ts/index.js';
// ---------------------------------------------------------------------------
// Helpers (same patterns as test.quic.node.ts)
// ---------------------------------------------------------------------------
async function findFreePorts(count: number): Promise<number[]> {
const servers: net.Server[] = [];
const ports: number[] = [];
for (let i = 0; i < count; i++) {
const server = net.createServer();
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve));
ports.push((server.address() as net.AddressInfo).port);
servers.push(server);
}
await Promise.all(servers.map((s) => new Promise<void>((resolve) => s.close(() => resolve()))));
return ports;
}
type TrackingServer = net.Server & { destroyAll: () => void };
function startEchoServer(port: number, host: string): Promise<TrackingServer> {
return new Promise((resolve, reject) => {
const connections = new Set<net.Socket>();
const server = net.createServer((socket) => {
connections.add(socket);
socket.on('close', () => connections.delete(socket));
let proxyHeaderParsed = false;
let pendingBuf = Buffer.alloc(0);
socket.on('data', (data: Buffer) => {
if (!proxyHeaderParsed) {
pendingBuf = Buffer.concat([pendingBuf, data]);
const idx = pendingBuf.indexOf('\r\n');
if (idx !== -1) {
proxyHeaderParsed = true;
const remainder = pendingBuf.subarray(idx + 2);
if (remainder.length > 0) socket.write(remainder);
}
return;
}
socket.write(data);
});
socket.on('error', () => {});
}) as TrackingServer;
server.destroyAll = () => {
for (const conn of connections) conn.destroy();
connections.clear();
};
server.on('error', reject);
server.listen(port, host, () => resolve(server));
});
}
async function forceCloseServer(server: TrackingServer): Promise<void> {
server.destroyAll();
await new Promise<void>((resolve) => server.close(() => resolve()));
}
function sendAndReceive(port: number, data: Buffer, timeoutMs = 30000): Promise<Buffer> {
return new Promise((resolve, reject) => {
const chunks: Buffer[] = [];
let totalReceived = 0;
const expectedLength = data.length;
let settled = false;
const client = net.createConnection({ host: '127.0.0.1', port }, () => {
client.write(data);
client.end();
});
const timer = setTimeout(() => {
if (!settled) {
settled = true;
client.destroy();
reject(new Error(`Timeout after ${timeoutMs}ms — received ${totalReceived}/${expectedLength} bytes`));
}
}, timeoutMs);
client.on('data', (chunk: Buffer) => {
chunks.push(chunk);
totalReceived += chunk.length;
if (totalReceived >= expectedLength && !settled) {
settled = true;
clearTimeout(timer);
client.destroy();
resolve(Buffer.concat(chunks));
}
});
client.on('end', () => {
if (!settled) {
settled = true;
clearTimeout(timer);
resolve(Buffer.concat(chunks));
}
});
client.on('error', (err) => {
if (!settled) {
settled = true;
clearTimeout(timer);
reject(err);
}
});
});
}
function sha256(buf: Buffer): string {
return crypto.createHash('sha256').update(buf).digest('hex');
}
// ---------------------------------------------------------------------------
// QUIC Long-Running Stability Test — 2 minutes
// ---------------------------------------------------------------------------
let hub: RemoteIngressHub;
let edge: RemoteIngressEdge;
let echoServer: TrackingServer;
let hubPort: number;
let edgePort: number;
let disconnectCount = 0;
tap.test('QUIC stability setup: start echo server and QUIC tunnel', async () => {
[hubPort, edgePort] = await findFreePorts(2);
echoServer = await startEchoServer(edgePort, '127.0.0.2');
hub = new RemoteIngressHub();
edge = new RemoteIngressEdge();
await hub.start({
tunnelPort: hubPort,
targetHost: '127.0.0.2',
});
await hub.updateAllowedEdges([
{ id: 'test-edge', secret: 'test-secret', listenPorts: [edgePort] },
]);
const connectedPromise = new Promise<void>((resolve, reject) => {
const timeout = setTimeout(() => reject(new Error('QUIC edge did not connect within 10s')), 10000);
edge.once('tunnelConnected', () => {
clearTimeout(timeout);
resolve();
});
});
// Track disconnects — any disconnect during the test is a failure signal
edge.on('tunnelDisconnected', () => {
disconnectCount++;
console.log(`[STABILITY] Unexpected tunnel disconnect #${disconnectCount}`);
});
await edge.start({
hubHost: '127.0.0.1',
hubPort,
edgeId: 'test-edge',
secret: 'test-secret',
bindAddress: '127.0.0.1',
transportMode: 'quic',
});
await connectedPromise;
await new Promise((resolve) => setTimeout(resolve, 500));
const status = await edge.getStatus();
expect(status.connected).toBeTrue();
});
tap.test('QUIC stability: tunnel stays alive for 30s with periodic echo probes', async () => {
const testDurationMs = 30_000; // 30 seconds
const probeIntervalMs = 5_000; // probe every 5 seconds
const startTime = Date.now();
let probeCount = 0;
let failedProbes = 0;
while (Date.now() - startTime < testDurationMs) {
probeCount++;
const elapsed = Math.round((Date.now() - startTime) / 1000);
// Verify edge still reports connected
const status = await edge.getStatus();
if (!status.connected) {
throw new Error(`Tunnel disconnected at ${elapsed}s (probe #${probeCount})`);
}
// Send a 4KB echo probe through the tunnel
const data = crypto.randomBytes(4096);
const hash = sha256(data);
try {
const received = await sendAndReceive(edgePort, data, 10000);
if (received.length !== 4096 || sha256(received) !== hash) {
failedProbes++;
console.log(`[STABILITY] Probe #${probeCount} at ${elapsed}s: data mismatch`);
} else {
console.log(`[STABILITY] Probe #${probeCount} at ${elapsed}s: OK`);
}
} catch (err) {
failedProbes++;
console.log(`[STABILITY] Probe #${probeCount} at ${elapsed}s: FAILED — ${err}`);
}
// Wait for next probe interval
const remaining = testDurationMs - (Date.now() - startTime);
if (remaining > 0) {
await new Promise((resolve) => setTimeout(resolve, Math.min(probeIntervalMs, remaining)));
}
}
console.log(`[STABILITY] Completed: ${probeCount} probes, ${failedProbes} failures, ${disconnectCount} disconnects`);
expect(failedProbes).toEqual(0);
expect(disconnectCount).toEqual(0);
// Final status check
const finalStatus = await edge.getStatus();
expect(finalStatus.connected).toBeTrue();
});
tap.test('QUIC stability teardown', async () => {
await edge.stop();
await hub.stop();
await forceCloseServer(echoServer);
});
export default tap.start();
+17 -9
View File
@@ -176,7 +176,7 @@ let echoServer: TrackingServer;
let hubPort: number;
let edgePort: number;
tap.test('QUIC setup: start echo server and QUIC tunnel', async () => {
tap.test('QUIC setup: start TCP echo server and QUIC tunnel', async () => {
[hubPort, edgePort] = await findFreePorts(2);
echoServer = await startEchoServer(edgePort, '127.0.0.2');
@@ -185,9 +185,17 @@ tap.test('QUIC setup: start echo server and QUIC tunnel', async () => {
expect(tunnel.hub.running).toBeTrue();
const status = await tunnel.edge.getStatus();
expect(status.connected).toBeTrue();
const hubStatus = await tunnel.hub.getStatus();
expect(hubStatus.connectedEdges.length).toBeGreaterThanOrEqual(1);
const edgeStatus = hubStatus.connectedEdges[0];
expect(edgeStatus.transportMode).toEqual('quic');
expect(edgeStatus.fallbackUsed).toEqual(false);
expect(edgeStatus.performance.profile).toEqual('balanced');
expect(edgeStatus.flowControl.applies).toEqual(false);
expect(edgeStatus.traffic.streamsOpenedTotal).toEqual(0);
});
tap.test('QUIC: single stream echo — 1KB', async () => {
tap.test('QUIC: single TCP stream echo — 1KB', async () => {
const data = crypto.randomBytes(1024);
const hash = sha256(data);
const received = await sendAndReceive(edgePort, data, 10000);
@@ -195,7 +203,7 @@ tap.test('QUIC: single stream echo — 1KB', async () => {
expect(sha256(received)).toEqual(hash);
});
tap.test('QUIC: single stream echo — 1MB', async () => {
tap.test('QUIC: single TCP stream echo — 1MB', async () => {
const size = 1024 * 1024;
const data = crypto.randomBytes(size);
const hash = sha256(data);
@@ -204,7 +212,7 @@ tap.test('QUIC: single stream echo — 1MB', async () => {
expect(sha256(received)).toEqual(hash);
});
tap.test('QUIC: single stream echo — 16MB', async () => {
tap.test('QUIC: single TCP stream echo — 16MB', async () => {
const size = 16 * 1024 * 1024;
const data = crypto.randomBytes(size);
const hash = sha256(data);
@@ -213,7 +221,7 @@ tap.test('QUIC: single stream echo — 16MB', async () => {
expect(sha256(received)).toEqual(hash);
});
tap.test('QUIC: 10 concurrent streams x 1MB each', async () => {
tap.test('QUIC: 10 concurrent TCP streams x 1MB each', async () => {
const streamCount = 10;
const payloadSize = 1024 * 1024;
@@ -232,7 +240,7 @@ tap.test('QUIC: 10 concurrent streams x 1MB each', async () => {
expect(failures.length).toEqual(0);
});
tap.test('QUIC: 50 concurrent streams x 64KB each', async () => {
tap.test('QUIC: 50 concurrent TCP streams x 64KB each', async () => {
const streamCount = 50;
const payloadSize = 64 * 1024;
@@ -251,7 +259,7 @@ tap.test('QUIC: 50 concurrent streams x 64KB each', async () => {
expect(failures.length).toEqual(0);
});
tap.test('QUIC: 200 concurrent streams x 16KB each', async () => {
tap.test('QUIC: 200 concurrent TCP streams x 16KB each', async () => {
const streamCount = 200;
const payloadSize = 16 * 1024;
@@ -270,12 +278,12 @@ tap.test('QUIC: 200 concurrent streams x 16KB each', async () => {
expect(failures.length).toEqual(0);
});
tap.test('QUIC: tunnel still connected after all tests', async () => {
tap.test('QUIC: TCP tunnel still connected after all tests', async () => {
const status = await tunnel.edge.getStatus();
expect(status.connected).toBeTrue();
});
tap.test('QUIC teardown: stop tunnel and echo server', async () => {
tap.test('QUIC teardown: stop TCP tunnel and echo server', async () => {
await tunnel.cleanup();
await forceCloseServer(echoServer);
});
+284
View File
@@ -0,0 +1,284 @@
import { expect, tap } from '@push.rocks/tapbundle';
import * as dgram from 'dgram';
import * as net from 'net';
import * as crypto from 'crypto';
import { RemoteIngressHub, RemoteIngressEdge } from '../ts/index.js';
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
async function findFreePorts(count: number): Promise<number[]> {
const servers: net.Server[] = [];
const ports: number[] = [];
for (let i = 0; i < count; i++) {
const server = net.createServer();
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve));
ports.push((server.address() as net.AddressInfo).port);
servers.push(server);
}
await Promise.all(servers.map((s) => new Promise<void>((resolve) => s.close(() => resolve()))));
return ports;
}
/**
* Start a UDP echo server that:
* 1. Receives the first datagram (PROXY v2 header 28 bytes) and discards it
* 2. Echoes all subsequent datagrams back to the sender
*/
function startUdpEchoServer(port: number, host: string): Promise<dgram.Socket> {
return new Promise((resolve, reject) => {
const server = dgram.createSocket('udp4');
// Track which source endpoints have sent their PROXY v2 header.
// The hub sends a 28-byte PROXY v2 header as the first datagram per session.
const seenSources = new Set<string>();
server.on('message', (msg, rinfo) => {
const sourceKey = `${rinfo.address}:${rinfo.port}`;
if (!seenSources.has(sourceKey)) {
seenSources.add(sourceKey);
// First datagram from this source is the PROXY v2 header — skip it
return;
}
// Echo back
server.send(msg, rinfo.port, rinfo.address);
});
server.on('error', reject);
server.bind(port, host, () => resolve(server));
});
}
/**
* Send a UDP datagram through the tunnel and wait for the echo response.
*/
function udpSendAndReceive(
port: number,
data: Buffer,
timeoutMs = 10000,
): Promise<Buffer> {
return new Promise((resolve, reject) => {
const client = dgram.createSocket('udp4');
let settled = false;
const timer = setTimeout(() => {
if (!settled) {
settled = true;
client.close();
reject(new Error(`UDP timeout after ${timeoutMs}ms`));
}
}, timeoutMs);
client.on('message', (msg) => {
if (!settled) {
settled = true;
clearTimeout(timer);
client.close();
resolve(msg);
}
});
client.on('error', (err) => {
if (!settled) {
settled = true;
clearTimeout(timer);
client.close();
reject(err);
}
});
client.send(data, port, '127.0.0.1');
});
}
// ---------------------------------------------------------------------------
// Test state
// ---------------------------------------------------------------------------
let hub: RemoteIngressHub;
let edge: RemoteIngressEdge;
let echoServer: dgram.Socket;
let hubPort: number;
let edgeUdpPort: number;
// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------
tap.test('UDP/TLS setup: start UDP echo server and TCP+TLS tunnel with UDP ports', async () => {
[hubPort, edgeUdpPort] = await findFreePorts(2);
// Start UDP echo server on upstream (127.0.0.2)
echoServer = await startUdpEchoServer(edgeUdpPort, '127.0.0.2');
hub = new RemoteIngressHub();
edge = new RemoteIngressEdge();
await hub.start({ tunnelPort: hubPort, targetHost: '127.0.0.2' });
await hub.updateAllowedEdges([
{ id: 'test-edge', secret: 'test-secret', listenPorts: [], listenPortsUdp: [edgeUdpPort] },
]);
const connectedPromise = new Promise<void>((resolve, reject) => {
const timeout = setTimeout(() => reject(new Error('Edge did not connect within 10s')), 10000);
edge.once('tunnelConnected', () => {
clearTimeout(timeout);
resolve();
});
});
await edge.start({
hubHost: '127.0.0.1',
hubPort,
edgeId: 'test-edge',
secret: 'test-secret',
bindAddress: '127.0.0.1',
});
await connectedPromise;
// Wait for UDP listener to bind
await new Promise((resolve) => setTimeout(resolve, 500));
const status = await edge.getStatus();
expect(status.connected).toBeTrue();
});
tap.test('UDP/TLS: single UDP datagram echo — 64 bytes', async () => {
const data = crypto.randomBytes(64);
const received = await udpSendAndReceive(edgeUdpPort, data, 5000);
expect(received.length).toEqual(64);
expect(Buffer.compare(received, data)).toEqual(0);
});
tap.test('UDP/TLS: single UDP datagram echo — 1KB', async () => {
const data = crypto.randomBytes(1024);
const received = await udpSendAndReceive(edgeUdpPort, data, 5000);
expect(received.length).toEqual(1024);
expect(Buffer.compare(received, data)).toEqual(0);
});
tap.test('UDP/TLS: 10 sequential UDP datagrams', async () => {
for (let i = 0; i < 10; i++) {
const data = crypto.randomBytes(128);
const received = await udpSendAndReceive(edgeUdpPort, data, 5000);
expect(received.length).toEqual(128);
expect(Buffer.compare(received, data)).toEqual(0);
}
});
tap.test('UDP/TLS: 10 concurrent UDP datagrams from different source ports', async () => {
const promises = Array.from({ length: 10 }, () => {
const data = crypto.randomBytes(256);
return udpSendAndReceive(edgeUdpPort, data, 5000).then((received) => ({
sizeOk: received.length === 256,
dataOk: Buffer.compare(received, data) === 0,
}));
});
const results = await Promise.all(promises);
const failures = results.filter((r) => !r.sizeOk || !r.dataOk);
expect(failures.length).toEqual(0);
});
tap.test('UDP/TLS: tunnel still connected after UDP tests', async () => {
const status = await edge.getStatus();
expect(status.connected).toBeTrue();
});
tap.test('UDP/TLS teardown: stop tunnel and UDP echo server', async () => {
await edge.stop();
await hub.stop();
await new Promise<void>((resolve) => echoServer.close(() => resolve()));
});
// ---------------------------------------------------------------------------
// QUIC transport UDP tests
// ---------------------------------------------------------------------------
let quicHub: RemoteIngressHub;
let quicEdge: RemoteIngressEdge;
let quicEchoServer: dgram.Socket;
let quicHubPort: number;
let quicEdgeUdpPort: number;
tap.test('UDP/QUIC setup: start UDP echo server and QUIC tunnel with UDP ports', async () => {
[quicHubPort, quicEdgeUdpPort] = await findFreePorts(2);
quicEchoServer = await startUdpEchoServer(quicEdgeUdpPort, '127.0.0.2');
quicHub = new RemoteIngressHub();
quicEdge = new RemoteIngressEdge();
await quicHub.start({ tunnelPort: quicHubPort, targetHost: '127.0.0.2' });
await quicHub.updateAllowedEdges([
{ id: 'test-edge', secret: 'test-secret', listenPorts: [], listenPortsUdp: [quicEdgeUdpPort] },
]);
const connectedPromise = new Promise<void>((resolve, reject) => {
const timeout = setTimeout(() => reject(new Error('QUIC edge did not connect within 10s')), 10000);
quicEdge.once('tunnelConnected', () => {
clearTimeout(timeout);
resolve();
});
});
await quicEdge.start({
hubHost: '127.0.0.1',
hubPort: quicHubPort,
edgeId: 'test-edge',
secret: 'test-secret',
bindAddress: '127.0.0.1',
transportMode: 'quic',
});
await connectedPromise;
await new Promise((resolve) => setTimeout(resolve, 500));
const status = await quicEdge.getStatus();
expect(status.connected).toBeTrue();
});
tap.test('UDP/QUIC: single UDP datagram echo — 64 bytes', async () => {
const data = crypto.randomBytes(64);
const received = await udpSendAndReceive(quicEdgeUdpPort, data, 5000);
expect(received.length).toEqual(64);
expect(Buffer.compare(received, data)).toEqual(0);
});
tap.test('UDP/QUIC: single UDP datagram echo — 1KB', async () => {
const data = crypto.randomBytes(1024);
const received = await udpSendAndReceive(quicEdgeUdpPort, data, 5000);
expect(received.length).toEqual(1024);
expect(Buffer.compare(received, data)).toEqual(0);
});
tap.test('UDP/QUIC: 10 sequential UDP datagrams', async () => {
for (let i = 0; i < 10; i++) {
const data = crypto.randomBytes(128);
const received = await udpSendAndReceive(quicEdgeUdpPort, data, 5000);
expect(received.length).toEqual(128);
expect(Buffer.compare(received, data)).toEqual(0);
}
});
tap.test('UDP/QUIC: 10 concurrent UDP datagrams', async () => {
const promises = Array.from({ length: 10 }, () => {
const data = crypto.randomBytes(256);
return udpSendAndReceive(quicEdgeUdpPort, data, 5000).then((received) => ({
sizeOk: received.length === 256,
dataOk: Buffer.compare(received, data) === 0,
}));
});
const results = await Promise.all(promises);
const failures = results.filter((r) => !r.sizeOk || !r.dataOk);
expect(failures.length).toEqual(0);
});
tap.test('UDP/QUIC teardown: stop QUIC tunnel and UDP echo server', async () => {
await quicEdge.stop();
await quicHub.stop();
await new Promise<void>((resolve) => quicEchoServer.close(() => resolve()));
});
export default tap.start();
+2 -2
View File
@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@serve.zone/remoteingress',
version: '4.10.0',
description: 'Edge ingress tunnel for DcRouter - accepts incoming TCP connections at network edge and tunnels them to DcRouter SmartProxy preserving client IP via PROXY protocol v1.'
version: '4.17.0',
description: 'Edge ingress tunnel for DcRouter - tunnels TCP and UDP traffic from the network edge to SmartProxy over TLS or QUIC, preserving client IP via PROXY protocol.'
}
+150 -9
View File
@@ -1,6 +1,7 @@
import * as plugins from './plugins.js';
import { EventEmitter } from 'events';
import { decodeConnectionToken } from './classes.token.js';
import type { IFirewallConfig } from './classes.remoteingresshub.js';
// Command map for the edge side of remoteingress-bin
type TEdgeCommands = {
@@ -55,6 +56,7 @@ export class RemoteIngressEdge extends EventEmitter {
private restartBackoffMs = 1000;
private restartAttempts = 0;
private statusInterval: ReturnType<typeof setInterval> | undefined;
private nft: InstanceType<typeof plugins.smartnftables.SmartNftables> | null = null;
constructor() {
super();
@@ -79,6 +81,15 @@ export class RemoteIngressEdge extends EventEmitter {
plugins.path.join(packageDir, 'rust', 'target', 'debug', 'remoteingress-bin'),
],
searchSystemPath: false,
logger: {
log: (level: string, message: string) => {
if (level === 'error') {
console.error(`[RemoteIngressEdge] ${message}`);
} else {
console.log(`[RemoteIngressEdge] ${message}`);
}
},
},
});
// Forward events from Rust binary
@@ -101,6 +112,83 @@ export class RemoteIngressEdge extends EventEmitter {
console.log(`[RemoteIngressEdge] Ports updated by hub: ${data.listenPorts.join(', ')}`);
this.emit('portsUpdated', data);
});
this.bridge.on('management:firewallConfigUpdated', (data: { firewallConfig: IFirewallConfig }) => {
console.log(`[RemoteIngressEdge] Firewall config updated from hub`);
this.applyFirewallConfig(data.firewallConfig);
this.emit('firewallConfigUpdated', data);
});
}
/**
* Initialize the nftables manager. Fails gracefully if not running as root.
*/
private async initNft(): Promise<void> {
try {
this.nft = new plugins.smartnftables.SmartNftables({
tableName: 'remoteingress',
dryRun: false,
});
await this.nft.initialize();
console.log('[RemoteIngressEdge] SmartNftables initialized');
} catch (err) {
console.warn(`[RemoteIngressEdge] Failed to initialize nftables (not root?): ${err}`);
this.nft = null;
}
}
/**
* Apply firewall configuration received from the hub.
* Performs a full replacement: cleans up existing rules, then applies the new config.
*/
private async applyFirewallConfig(config: IFirewallConfig): Promise<void> {
if (!this.nft) {
return;
}
try {
// Full cleanup and reinitialize to replace all rules atomically
await this.nft.cleanup();
await this.nft.initialize();
// Apply blocked IPs
if (config.blockedIps && config.blockedIps.length > 0) {
for (const ip of config.blockedIps) {
await this.nft.firewall.blockIP(ip);
}
console.log(`[RemoteIngressEdge] Blocked ${config.blockedIps.length} IPs`);
}
// Apply rate limits
if (config.rateLimits && config.rateLimits.length > 0) {
for (const rl of config.rateLimits) {
await this.nft.rateLimit.addRateLimit(rl.id, {
port: rl.port,
protocol: rl.protocol,
rate: rl.rate,
burst: rl.burst,
perSourceIP: rl.perSourceIP,
});
}
console.log(`[RemoteIngressEdge] Applied ${config.rateLimits.length} rate limits`);
}
// Apply firewall rules
if (config.rules && config.rules.length > 0) {
for (const rule of config.rules) {
await this.nft.firewall.addRule(rule.id, {
direction: rule.direction,
action: rule.action,
sourceIP: rule.sourceIP,
destPort: rule.destPort,
protocol: rule.protocol,
comment: rule.comment,
});
}
console.log(`[RemoteIngressEdge] Applied ${config.rules.length} firewall rules`);
}
} catch (err) {
console.error(`[RemoteIngressEdge] Failed to apply firewall config: ${err}`);
}
}
/**
@@ -130,22 +218,33 @@ export class RemoteIngressEdge extends EventEmitter {
throw new Error('Failed to spawn remoteingress-bin');
}
// Register crash recovery handler
// Register crash recovery handler (remove first to avoid duplicates)
this.bridge.removeListener('exit', this.handleCrashRecovery);
this.bridge.on('exit', this.handleCrashRecovery);
await this.bridge.sendCommand('startEdge', {
hubHost: edgeConfig.hubHost,
hubPort: edgeConfig.hubPort ?? 8443,
edgeId: edgeConfig.edgeId,
secret: edgeConfig.secret,
...(edgeConfig.bindAddress ? { bindAddress: edgeConfig.bindAddress } : {}),
...(edgeConfig.transportMode ? { transportMode: edgeConfig.transportMode } : {}),
});
try {
await this.bridge.sendCommand('startEdge', {
hubHost: edgeConfig.hubHost,
hubPort: edgeConfig.hubPort ?? 8443,
edgeId: edgeConfig.edgeId,
secret: edgeConfig.secret,
...(edgeConfig.bindAddress ? { bindAddress: edgeConfig.bindAddress } : {}),
...(edgeConfig.transportMode ? { transportMode: edgeConfig.transportMode } : {}),
});
} catch (err) {
// Clean up the spawned process to avoid orphaning it
this.bridge.removeListener('exit', this.handleCrashRecovery);
this.bridge.kill();
throw err;
}
this.started = true;
this.restartAttempts = 0;
this.restartBackoffMs = 1000;
// Initialize nftables (graceful degradation if not root)
await this.initNft();
// Start periodic status logging
this.statusInterval = setInterval(async () => {
try {
@@ -170,6 +269,15 @@ export class RemoteIngressEdge extends EventEmitter {
clearInterval(this.statusInterval);
this.statusInterval = undefined;
}
// Clean up nftables rules before stopping
if (this.nft) {
try {
await this.nft.cleanup();
} catch (err) {
console.warn(`[RemoteIngressEdge] nftables cleanup error: ${err}`);
}
this.nft = null;
}
if (this.started) {
try {
await this.bridge.sendCommand('stopEdge', {} as Record<string, never>);
@@ -180,6 +288,10 @@ export class RemoteIngressEdge extends EventEmitter {
this.bridge.kill();
this.started = false;
}
this.savedConfig = null;
// Remove all listeners to prevent memory buildup
this.bridge.removeAllListeners();
this.removeAllListeners();
}
/**
@@ -211,6 +323,12 @@ export class RemoteIngressEdge extends EventEmitter {
this.started = false;
// Clear orphaned status interval from previous run
if (this.statusInterval) {
clearInterval(this.statusInterval);
this.statusInterval = undefined;
}
if (this.restartAttempts >= MAX_RESTART_ATTEMPTS) {
console.error('[RemoteIngressEdge] Max restart attempts reached, giving up');
this.emit('crashRecoveryFailed');
@@ -218,6 +336,10 @@ export class RemoteIngressEdge extends EventEmitter {
}
await new Promise(resolve => setTimeout(resolve, this.restartBackoffMs));
// Re-check after backoff — stop() may have been called during the wait
if (this.stopping || !this.savedConfig) {
return;
}
this.restartBackoffMs = Math.min(this.restartBackoffMs * 2, MAX_RESTART_BACKOFF_MS);
this.restartAttempts++;
@@ -228,6 +350,7 @@ export class RemoteIngressEdge extends EventEmitter {
return;
}
this.bridge.removeListener('exit', this.handleCrashRecovery);
this.bridge.on('exit', this.handleCrashRecovery);
await this.bridge.sendCommand('startEdge', {
@@ -242,6 +365,24 @@ export class RemoteIngressEdge extends EventEmitter {
this.started = true;
this.restartAttempts = 0;
this.restartBackoffMs = 1000;
// Re-initialize nftables (hub will re-push config via handshake)
await this.initNft();
// Restart periodic status logging
this.statusInterval = setInterval(async () => {
try {
const status = await this.getStatus();
console.log(
`[RemoteIngressEdge] Status: connected=${status.connected}, ` +
`streams=${status.activeStreams}, ports=[${status.listenPorts.join(',')}], ` +
`publicIp=${status.publicIp ?? 'unknown'}`
);
} catch {
// Bridge may be shutting down
}
}, 60_000);
console.log('[RemoteIngressEdge] Successfully recovered from crash');
this.emit('crashRecovered');
} catch (err) {
+135 -21
View File
@@ -9,11 +9,12 @@ type THubCommands = {
};
startHub: {
params: {
tunnelPort: number;
targetHost?: string;
tlsCertPem?: string;
tlsKeyPem?: string;
};
tunnelPort: number;
targetHost?: string;
tlsCertPem?: string;
tlsKeyPem?: string;
performance?: IPerformanceConfig;
};
result: { started: boolean };
};
stopHub: {
@@ -22,7 +23,7 @@ type THubCommands = {
};
updateAllowedEdges: {
params: {
edges: Array<{ id: string; secret: string; listenPorts?: number[]; stunIntervalSecs?: number }>;
edges: Array<{ id: string; secret: string; listenPorts?: number[]; listenPortsUdp?: number[]; stunIntervalSecs?: number; firewallConfig?: IFirewallConfig; performance?: IPerformanceConfig }>;
};
result: { updated: boolean };
};
@@ -31,16 +32,48 @@ type THubCommands = {
result: {
running: boolean;
tunnelPort: number;
connectedEdges: Array<{
edgeId: string;
connectedAt: number;
activeStreams: number;
peerAddr: string;
}>;
connectedEdges: Array<{
edgeId: string;
connectedAt: number;
activeStreams: number;
peerAddr: string;
transportMode: 'tcpTls' | 'quic' | 'quicWithFallback';
fallbackUsed: boolean;
performance: IEffectivePerformanceConfig;
flowControl: IFlowControlStatus;
queues: IQueueStatus;
traffic: ITrafficStatus;
udp: IUdpStatus;
}>;
};
};
};
export interface IFirewallRateLimit {
id: string;
port: number;
protocol?: 'tcp' | 'udp';
rate: string;
burst?: number;
perSourceIP?: boolean;
}
export interface IFirewallRule {
id: string;
direction: 'input' | 'output' | 'forward';
action: 'accept' | 'drop' | 'reject';
sourceIP?: string;
destPort?: number;
protocol?: 'tcp' | 'udp';
comment?: string;
}
export interface IFirewallConfig {
blockedIps?: string[];
rateLimits?: IFirewallRateLimit[];
rules?: IFirewallRule[];
}
export interface IHubConfig {
tunnelPort?: number;
targetHost?: string;
@@ -48,9 +81,61 @@ export interface IHubConfig {
certPem?: string;
keyPem?: string;
};
performance?: IPerformanceConfig;
}
type TAllowedEdge = { id: string; secret: string; listenPorts?: number[]; stunIntervalSecs?: number };
export type TPerformanceProfile = 'balanced' | 'throughput' | 'highConcurrency';
export interface IPerformanceConfig {
profile?: TPerformanceProfile;
maxStreamsPerEdge?: number;
totalWindowBudgetBytes?: number;
minStreamWindowBytes?: number;
maxStreamWindowBytes?: number;
sustainedStreamWindowBytes?: number;
quicDatagramReceiveBufferBytes?: number;
}
export interface IEffectivePerformanceConfig {
profile: TPerformanceProfile;
maxStreamsPerEdge: number;
totalWindowBudgetBytes: number;
minStreamWindowBytes: number;
maxStreamWindowBytes: number;
sustainedStreamWindowBytes: number;
quicDatagramReceiveBufferBytes: number;
}
export interface IFlowControlStatus {
applies: boolean;
currentWindowBytes: number;
minWindowBytes: number;
maxWindowBytes: number;
totalWindowBudgetBytes: number;
estimatedInFlightBytes: number;
stalledStreams: number;
}
export interface IQueueStatus {
ctrlQueueDepth: number;
dataQueueDepth: number;
sustainedQueueDepth: number;
}
export interface ITrafficStatus {
bytesIn: number;
bytesOut: number;
streamsOpenedTotal: number;
streamsClosedTotal: number;
rejectedStreams: number;
}
export interface IUdpStatus {
activeSessions: number;
droppedDatagrams: number;
}
type TAllowedEdge = { id: string; secret: string; listenPorts?: number[]; listenPortsUdp?: number[]; stunIntervalSecs?: number; firewallConfig?: IFirewallConfig; performance?: IPerformanceConfig };
const MAX_RESTART_ATTEMPTS = 10;
const MAX_RESTART_BACKOFF_MS = 30_000;
@@ -87,6 +172,15 @@ export class RemoteIngressHub extends EventEmitter {
plugins.path.join(packageDir, 'rust', 'target', 'debug', 'remoteingress-bin'),
],
searchSystemPath: false,
logger: {
log: (level: string, message: string) => {
if (level === 'error') {
console.error(`[RemoteIngressHub] ${message}`);
} else {
console.log(`[RemoteIngressHub] ${message}`);
}
},
},
});
// Forward events from Rust binary
@@ -118,16 +212,25 @@ export class RemoteIngressHub extends EventEmitter {
throw new Error('Failed to spawn remoteingress-bin');
}
// Register crash recovery handler
// Register crash recovery handler (remove first to avoid duplicates)
this.bridge.removeListener('exit', this.handleCrashRecovery);
this.bridge.on('exit', this.handleCrashRecovery);
await this.bridge.sendCommand('startHub', {
tunnelPort: config.tunnelPort ?? 8443,
targetHost: config.targetHost ?? '127.0.0.1',
...(config.tls?.certPem && config.tls?.keyPem
? { tlsCertPem: config.tls.certPem, tlsKeyPem: config.tls.keyPem }
: {}),
});
try {
await this.bridge.sendCommand('startHub', {
tunnelPort: config.tunnelPort ?? 8443,
targetHost: config.targetHost ?? '127.0.0.1',
...(config.performance ? { performance: config.performance } : {}),
...(config.tls?.certPem && config.tls?.keyPem
? { tlsCertPem: config.tls.certPem, tlsKeyPem: config.tls.keyPem }
: {}),
});
} catch (err) {
// Clean up the spawned process to avoid orphaning it
this.bridge.removeListener('exit', this.handleCrashRecovery);
this.bridge.kill();
throw err;
}
this.started = true;
this.restartAttempts = 0;
@@ -149,6 +252,11 @@ export class RemoteIngressHub extends EventEmitter {
this.bridge.kill();
this.started = false;
}
this.savedConfig = null;
this.savedEdges = [];
// Remove all listeners to prevent memory buildup
this.bridge.removeAllListeners();
this.removeAllListeners();
}
/**
@@ -195,6 +303,10 @@ export class RemoteIngressHub extends EventEmitter {
}
await new Promise(resolve => setTimeout(resolve, this.restartBackoffMs));
// Re-check after backoff — stop() may have been called during the wait
if (this.stopping || !this.savedConfig) {
return;
}
this.restartBackoffMs = Math.min(this.restartBackoffMs * 2, MAX_RESTART_BACKOFF_MS);
this.restartAttempts++;
@@ -205,6 +317,7 @@ export class RemoteIngressHub extends EventEmitter {
return;
}
this.bridge.removeListener('exit', this.handleCrashRecovery);
this.bridge.on('exit', this.handleCrashRecovery);
const config = this.savedConfig;
@@ -214,6 +327,7 @@ export class RemoteIngressHub extends EventEmitter {
...(config.tls?.certPem && config.tls?.keyPem
? { tlsCertPem: config.tls.certPem, tlsKeyPem: config.tls.keyPem }
: {}),
...(config.performance ? { performance: config.performance } : {}),
});
// Restore allowed edges
+2 -1
View File
@@ -3,5 +3,6 @@ import * as path from 'path';
export { path };
// @push.rocks scope
import * as smartnftables from '@push.rocks/smartnftables';
import * as smartrust from '@push.rocks/smartrust';
export { smartrust };
export { smartnftables, smartrust };
+2 -1
View File
@@ -6,7 +6,8 @@
"module": "NodeNext",
"moduleResolution": "NodeNext",
"esModuleInterop": true,
"verbatimModuleSyntax": true
"verbatimModuleSyntax": true,
"types": ["node"]
},
"exclude": [
"dist_*/**/*.d.ts"