Compare commits
62 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 9922c3b020 | |||
| 38cde37cff | |||
| 64572827e5 | |||
| c4e26198b9 | |||
| 0b5d72de28 | |||
| e8431c0174 | |||
| d57d6395dd | |||
| 2e5ceeaf5c | |||
| 1979910f6f | |||
| edfad2dffe | |||
| d907943ae5 | |||
| 4bfb1244fc | |||
| e31c3421a6 | |||
| de8422966a | |||
| a87e9578eb | |||
| b851bc7994 | |||
| 1284bb5b73 | |||
| 1afd0e5347 | |||
| 96e7ab00cf | |||
| 17d1a795cd | |||
| 982f648928 | |||
| 3a2a060a85 | |||
| e0c469147e | |||
| 0fdcdf566e | |||
| a808d4c9de | |||
| f8a0171ef3 | |||
| 1d59a48648 | |||
| af2ec11a2d | |||
| b6e66a7fa6 | |||
| 1391b39601 | |||
| e813c2f044 | |||
| 0b8c1f0b57 | |||
| a63dbf2502 | |||
| 4b95a3c999 | |||
| 51ab32f6c3 | |||
| ed52520d50 | |||
| a08011d2da | |||
| 679b247c8a | |||
| 32f9845495 | |||
| c0e1daa0e4 | |||
| fd511c8a5c | |||
| c490e35a8f | |||
| 579e553da0 | |||
| a8ee0b33d7 | |||
| 43e320a36d | |||
| 6ac4b37532 | |||
| f456b0ba4f | |||
| 69530f73aa | |||
| 207b4a5cec | |||
| 761551596b | |||
| cf2d32bfe7 | |||
| 4e9041c6a7 | |||
| 86d4e9889a | |||
| 45a2811f3e | |||
| d6a07c28a0 | |||
| 56a14aa7c5 | |||
| 417f62e646 | |||
| bda82f32ca | |||
| 4b06cb1b24 | |||
| 1aae4b8c8e | |||
| 3474e8c310 | |||
| 3df20df2a1 |
209
changelog.md
209
changelog.md
@@ -1,5 +1,214 @@
|
|||||||
# Changelog
|
# Changelog
|
||||||
|
|
||||||
|
## 2026-03-17 - 4.8.6 - fix(remoteingress-core)
|
||||||
|
initialize disconnect reason only when set in hub loop break paths
|
||||||
|
|
||||||
|
- Replace the default "unknown" disconnect reason with an explicitly assigned string and document that all hub loop exits set it before use
|
||||||
|
- Add an allow attribute for unused assignments to avoid warnings around the deferred initialization pattern
|
||||||
|
|
||||||
|
## 2026-03-17 - 4.8.5 - fix(repo)
|
||||||
|
no changes to commit
|
||||||
|
|
||||||
|
|
||||||
|
## 2026-03-17 - 4.8.4 - fix(remoteingress-core)
|
||||||
|
prevent stream stalls by guaranteeing flow-control updates and avoiding bounded per-stream channel overflows
|
||||||
|
|
||||||
|
- Replace bounded per-stream data channels with unbounded channels on edge and hub, relying on existing WINDOW_UPDATE flow control to limit bytes in flight
|
||||||
|
- Use awaited sends for FRAME_WINDOW_UPDATE and FRAME_WINDOW_UPDATE_BACK so updates are not dropped and streams do not deadlock under backpressure
|
||||||
|
- Clean up stream state when channel receivers have already exited instead of closing active streams because a bounded queue filled
|
||||||
|
|
||||||
|
## 2026-03-17 - 4.8.3 - fix(protocol,edge)
|
||||||
|
optimize tunnel frame handling and zero-copy uploads in edge I/O
|
||||||
|
|
||||||
|
- extract hub frame processing into a shared edge handler to remove duplicated tunnel logic
|
||||||
|
- add zero-copy frame header encoding and read payloads directly into framed buffers for client-to-hub uploads
|
||||||
|
- refactor TunnelIo read/write state to avoid unsafe queue access and reduce buffer churn with incremental parsing
|
||||||
|
|
||||||
|
## 2026-03-17 - 4.8.2 - fix(rust-edge)
|
||||||
|
refactor tunnel I/O to preserve TLS state and prioritize control frames
|
||||||
|
|
||||||
|
- replace split TLS handling with a single-owner TunnelIo to avoid handshake and buffered read corruption
|
||||||
|
- prioritize control frames over data frames to prevent WINDOW_UPDATE starvation and flow-control deadlocks
|
||||||
|
- improve tunnel reliability with incremental frame parsing, liveness/error events, and corrupt frame header logging
|
||||||
|
|
||||||
|
## 2026-03-17 - 4.8.1 - fix(remoteingress-core)
|
||||||
|
remove tunnel writer timeouts from edge and hub buffered writes
|
||||||
|
|
||||||
|
- Drops the 30 second timeout wrapper around writer.write_all and writer.flush in both edge and hub tunnel writers.
|
||||||
|
- Updates error logging to report write failures without referring to stalled writes.
|
||||||
|
|
||||||
|
## 2026-03-17 - 4.8.0 - feat(events)
|
||||||
|
include disconnect reasons in edge and hub management events
|
||||||
|
|
||||||
|
- Add reason fields to tunnelDisconnected and edgeDisconnected events emitted from the Rust core and binary bridge
|
||||||
|
- Propagate specific disconnect causes such as EOF, liveness timeout, writer failure, handshake failure, and hub cancellation
|
||||||
|
- Update TypeScript edge and hub classes to log and forward disconnect reason data
|
||||||
|
- Extend serialization tests to cover the new reason fields
|
||||||
|
|
||||||
|
## 2026-03-17 - 4.7.2 - fix(remoteingress-core)
|
||||||
|
add tunnel write timeouts and scale initial stream windows by active stream count
|
||||||
|
|
||||||
|
- Wrap tunnel frame writes and flushes in a 30-second timeout on both edge and hub to detect stalled writers and trigger faster reconnect or cleanup.
|
||||||
|
- Compute each stream's initial send window from the current active stream count instead of using a fixed window to keep total in-flight data within the 32MB budget.
|
||||||
|
|
||||||
|
## 2026-03-17 - 4.7.1 - fix(remoteingress-core)
|
||||||
|
improve tunnel failure detection and reconnect handling
|
||||||
|
|
||||||
|
- Enable TCP keepalive on edge and hub connections to detect silent network failures sooner
|
||||||
|
- Trigger immediate reconnect or disconnect when tunnel writer tasks fail instead of waiting for liveness timeouts
|
||||||
|
- Prevent active stream counter underflow during concurrent connection cleanup
|
||||||
|
|
||||||
|
## 2026-03-16 - 4.7.0 - feat(edge,protocol,test)
|
||||||
|
add configurable edge bind address and expand flow-control test coverage
|
||||||
|
|
||||||
|
- adds an optional bindAddress configuration for edge TCP listeners, defaulting to 0.0.0.0 when not provided
|
||||||
|
- passes bindAddress through the TypeScript edge client and Rust edge runtime so local test setups can bind to localhost
|
||||||
|
- adds protocol unit tests for adaptive stream window sizing and window update frame encoding/decoding
|
||||||
|
- introduces end-to-end flow-control tests and updates the test script to build before running tests
|
||||||
|
|
||||||
|
## 2026-03-16 - 4.6.1 - fix(remoteingress-core)
|
||||||
|
avoid spurious tunnel disconnect events and increase control channel capacity
|
||||||
|
|
||||||
|
- Emit TunnelDisconnected only after an established connection is actually lost, preventing false disconnect events during failed reconnect attempts.
|
||||||
|
- Increase edge and hub control-channel buffer sizes from 64 to 256 to better prioritize control frames under load.
|
||||||
|
|
||||||
|
## 2026-03-16 - 4.6.0 - feat(remoteingress-core)
|
||||||
|
add adaptive per-stream flow control based on active stream counts
|
||||||
|
|
||||||
|
- Track active stream counts on edge and hub connections to size per-stream flow control windows dynamically.
|
||||||
|
- Cap WINDOW_UPDATE increments and read sizes to the adaptive window so bandwidth is shared more evenly across concurrent streams.
|
||||||
|
- Apply the adaptive logic to both upload and download paths on edge and hub stream handlers.
|
||||||
|
|
||||||
|
## 2026-03-16 - 4.5.12 - fix(remoteingress-core)
|
||||||
|
improve tunnel liveness handling and enable TCP keepalive for accepted client sockets
|
||||||
|
|
||||||
|
- Avoid disconnecting edges when PING or PONG frames cannot be queued because the control channel is temporarily full.
|
||||||
|
- Enable TCP_NODELAY and TCP keepalive on accepted client connections to help detect stale or dropped clients.
|
||||||
|
|
||||||
|
## 2026-03-16 - 4.5.11 - fix(repo)
|
||||||
|
no changes to commit
|
||||||
|
|
||||||
|
|
||||||
|
## 2026-03-16 - 4.5.10 - fix(remoteingress-core)
|
||||||
|
guard zero-window reads to avoid false EOF handling on stalled streams
|
||||||
|
|
||||||
|
- Prevent upload and download loops from calling read on an empty buffer when flow-control window remains at 0 after stall timeout
|
||||||
|
- Log a warning and close the affected stream instead of misinterpreting Ok(0) as end-of-file
|
||||||
|
|
||||||
|
## 2026-03-16 - 4.5.9 - fix(remoteingress-core)
|
||||||
|
delay stream close until downstream response draining finishes to prevent truncated transfers
|
||||||
|
|
||||||
|
- Waits for the hub-to-client download task to finish before sending the stream CLOSE frame
|
||||||
|
- Prevents upstream reads from being cancelled mid-response during asymmetric transfers such as git fetch
|
||||||
|
- Retains the existing timeout so stalled downloads still clean up safely
|
||||||
|
|
||||||
|
## 2026-03-16 - 4.5.8 - fix(remoteingress-core)
|
||||||
|
ensure upstream writes cancel promptly and reliably deliver CLOSE_BACK frames
|
||||||
|
|
||||||
|
- listen for stream cancellation while waiting on upstream write timeouts so FRAME_CLOSE does not block for up to 60 seconds
|
||||||
|
- replace try_send with send().await when emitting CLOSE_BACK frames to avoid silently dropping close notifications when the data channel is full
|
||||||
|
|
||||||
|
## 2026-03-16 - 4.5.7 - fix(remoteingress-core)
|
||||||
|
improve tunnel reconnect and frame write efficiency
|
||||||
|
|
||||||
|
- Reuse the TLS connector across edge reconnections to preserve session resumption state and reduce reconnect latency.
|
||||||
|
- Buffer hub and edge frame writes to coalesce small control and data frames into fewer TLS records and syscalls while still flushing each frame promptly.
|
||||||
|
|
||||||
|
## 2026-03-16 - 4.5.6 - fix(remoteingress-core)
|
||||||
|
disable Nagle's algorithm on edge, hub, and upstream TCP sockets to reduce control-frame latency
|
||||||
|
|
||||||
|
- Enable TCP_NODELAY on the edge connection to the hub for faster PING/PONG and WINDOW_UPDATE delivery
|
||||||
|
- Apply TCP_NODELAY on accepted hub streams before TLS handling
|
||||||
|
- Enable TCP_NODELAY on SmartProxy upstream connections before sending the PROXY header
|
||||||
|
|
||||||
|
## 2026-03-16 - 4.5.5 - fix(remoteingress-core)
|
||||||
|
wait for hub-to-client draining before cleanup and reliably send close frames
|
||||||
|
|
||||||
|
- switch CLOSE frame delivery on the data channel from try_send to send().await to avoid dropping it when the channel is full
|
||||||
|
- delay stream cleanup until the hub-to-client task finishes or times out so large downstream responses continue after upload EOF
|
||||||
|
- add a bounded 5-minute wait for download draining to prevent premature termination of asymmetric transfers such as git fetch
|
||||||
|
|
||||||
|
## 2026-03-15 - 4.5.4 - fix(remoteingress-core)
|
||||||
|
preserve stream close ordering and add flow-control stall timeouts
|
||||||
|
|
||||||
|
- Send CLOSE and CLOSE_BACK frames on the data channel so they arrive after the final stream data frames.
|
||||||
|
- Log and abort stalled upload and download paths when flow-control windows stay empty for 120 seconds.
|
||||||
|
- Apply a 60-second timeout when writing buffered stream data to the upstream connection to prevent hung streams.
|
||||||
|
|
||||||
|
## 2026-03-15 - 4.5.3 - fix(remoteingress-core)
|
||||||
|
prioritize control frames over data in edge and hub tunnel writers
|
||||||
|
|
||||||
|
- Split tunnel/frame writers into separate control and data channels in edge and hub
|
||||||
|
- Use biased select loops so PING, PONG, WINDOW_UPDATE, OPEN, and CLOSE frames are sent before data frames
|
||||||
|
- Route stream data through dedicated data channels while keeping OPEN, CLOSE, and flow-control updates on control channels to prevent keepalive starvation under load
|
||||||
|
|
||||||
|
## 2026-03-15 - 4.5.2 - fix(remoteingress-core)
|
||||||
|
improve stream flow control retries and increase channel buffer capacity
|
||||||
|
|
||||||
|
- increase per-stream mpsc channel capacity from 128 to 256 on both edge and hub paths
|
||||||
|
- only reset accumulated window update bytes after a successful try_send to avoid dropping flow-control credits when the update channel is busy
|
||||||
|
|
||||||
|
## 2026-03-15 - 4.5.1 - fix(protocol)
|
||||||
|
increase per-stream flow control window and channel buffers to improve high-RTT throughput
|
||||||
|
|
||||||
|
- raise the initial stream window from 256 KB to 4 MB to allow more in-flight data per stream
|
||||||
|
- increase edge and hub mpsc channel capacities from 16 to 128 to better absorb throughput under flow control
|
||||||
|
|
||||||
|
## 2026-03-15 - 4.5.0 - feat(remoteingress-core)
|
||||||
|
add per-stream flow control for edge and hub tunnel data transfer
|
||||||
|
|
||||||
|
- introduce WINDOW_UPDATE frame types and protocol helpers for per-stream flow control
|
||||||
|
- track per-stream send windows on both edge and hub to limit reads based on available capacity
|
||||||
|
- send window updates after downstream writes to reduce channel pressure during large transfers
|
||||||
|
|
||||||
|
## 2026-03-15 - 4.4.1 - fix(remoteingress-core)
|
||||||
|
prevent stream data loss by applying backpressure and closing saturated channels
|
||||||
|
|
||||||
|
- replace non-blocking frame writes with awaited sends in per-stream tasks so large transfers respect backpressure instead of dropping data
|
||||||
|
- close and remove streams when back-channel or data channels fill up to avoid TCP stream corruption from silently dropped frames
|
||||||
|
|
||||||
|
## 2026-03-03 - 4.4.0 - feat(remoteingress)
|
||||||
|
add heartbeat PING/PONG and liveness timeouts; implement fast-reconnect/backoff reset and JS crash-recovery auto-restart
|
||||||
|
|
||||||
|
- protocol: add FRAME_PING and FRAME_PONG and unit tests for ping/pong frames
|
||||||
|
- edge (Rust): reset backoff after successful connection, respond to PING with PONG, track liveness via deadline and reconnect on timeout, use Duration/Instant helpers
|
||||||
|
- hub (Rust): send periodic PING to edges, handle PONGs, enforce liveness timeout and disconnect inactive edges, use tokio interval and time utilities
|
||||||
|
- ts: RemoteIngressEdge and RemoteIngressHub: add crash-recovery auto-restart with exponential backoff and max attempts, save/restore config and allowed edges, register/remove exit handlers, ensure stop() marks stopping and cleans up listeners
|
||||||
|
- minor API/typing: introduce TAllowedEdge alias and persist allowed edges for restart recovery
|
||||||
|
|
||||||
|
## 2026-02-26 - 4.3.0 - feat(hub)
|
||||||
|
add optional TLS certificate/key support to hub start config and bridge
|
||||||
|
|
||||||
|
- TypeScript: add tls.certPem and tls.keyPem to IHubConfig and include tlsCertPem/tlsKeyPem in startHub bridge command when both are provided
|
||||||
|
- TypeScript: extend startHub params with tlsCertPem and tlsKeyPem and conditionally send them
|
||||||
|
- Rust: change HubConfig serde attributes for tls_cert_pem and tls_key_pem from skip to default so absent PEM fields deserialize as None
|
||||||
|
- Enables optional provisioning of TLS certificate and key to the hub when provided from the JS side
|
||||||
|
|
||||||
|
## 2026-02-26 - 4.2.0 - feat(core)
|
||||||
|
expose edge peer address in hub events and migrate writers to channel-based, non-blocking framing with stream limits and timeouts
|
||||||
|
|
||||||
|
- Add peerAddr to ConnectedEdgeStatus and HubEvent::EdgeConnected and surface it to the TS frontend event (management:edgeConnected).
|
||||||
|
- Replace Arc<Mutex<WriteHalf>> writers with dedicated mpsc channel writer tasks in both hub and edge crates to serialize writes off the main tasks.
|
||||||
|
- Use non-blocking try_send for data frames to avoid head-of-line blocking and drop frames with warnings when channels are full.
|
||||||
|
- Introduce MAX_STREAMS_PER_EDGE semaphore to limit concurrent streams per edge and reject excess opens with a CLOSE_BACK frame.
|
||||||
|
- Add a 10s timeout when connecting to SmartProxy to avoid hanging connections.
|
||||||
|
- Ensure writer tasks are aborted on shutdown/cleanup and propagate cancellation tokens appropriately.
|
||||||
|
|
||||||
|
## 2026-02-26 - 4.1.0 - feat(remoteingress-bin)
|
||||||
|
use mimalloc as the global allocator to reduce memory overhead and improve allocation performance
|
||||||
|
|
||||||
|
- added mimalloc = "0.1" dependency to rust/crates/remoteingress-bin/Cargo.toml
|
||||||
|
- registered mimalloc as the #[global_allocator] in rust/crates/remoteingress-bin/src/main.rs
|
||||||
|
- updated Cargo.lock with libmimalloc-sys and mimalloc package entries
|
||||||
|
|
||||||
|
## 2026-02-26 - 4.0.1 - fix(hub)
|
||||||
|
cancel per-stream tokens on stream close and avoid duplicate StreamClosed events; bump @types/node devDependency to ^25.3.0
|
||||||
|
|
||||||
|
- Add CancellationToken to per-stream entries so each stream can be cancelled independently.
|
||||||
|
- Ensure StreamClosed event is only emitted when a stream was actually present (guards against duplicate events).
|
||||||
|
- Cancel the stream-specific token on FRAME_CLOSE to stop associated tasks and free resources.
|
||||||
|
- DevDependency bump: @types/node updated from ^25.2.3 to ^25.3.0.
|
||||||
|
|
||||||
## 2026-02-19 - 4.0.0 - BREAKING CHANGE(remoteingress-core)
|
## 2026-02-19 - 4.0.0 - BREAKING CHANGE(remoteingress-core)
|
||||||
add cancellation tokens and cooperative shutdown; switch event channels to bounded mpsc and improve cleanup
|
add cancellation tokens and cooperative shutdown; switch event channels to bounded mpsc and improve cleanup
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@serve.zone/remoteingress",
|
"name": "@serve.zone/remoteingress",
|
||||||
"version": "4.0.0",
|
"version": "4.8.6",
|
||||||
"private": false,
|
"private": false,
|
||||||
"description": "Edge ingress tunnel for DcRouter - accepts incoming TCP connections at network edge and tunnels them to DcRouter SmartProxy preserving client IP via PROXY protocol v1.",
|
"description": "Edge ingress tunnel for DcRouter - accepts incoming TCP connections at network edge and tunnels them to DcRouter SmartProxy preserving client IP via PROXY protocol v1.",
|
||||||
"main": "dist_ts/index.js",
|
"main": "dist_ts/index.js",
|
||||||
@@ -9,7 +9,7 @@
|
|||||||
"author": "Task Venture Capital GmbH",
|
"author": "Task Venture Capital GmbH",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"test": "(tstest test/ --verbose --logfile --timeout 60)",
|
"test": "(pnpm run build && tstest test/ --verbose --logfile --timeout 60)",
|
||||||
"build": "(tsbuild tsfolders --allowimplicitany && tsrust)",
|
"build": "(tsbuild tsfolders --allowimplicitany && tsrust)",
|
||||||
"buildDocs": "(tsdoc)"
|
"buildDocs": "(tsdoc)"
|
||||||
},
|
},
|
||||||
@@ -20,7 +20,7 @@
|
|||||||
"@git.zone/tsrust": "^1.3.0",
|
"@git.zone/tsrust": "^1.3.0",
|
||||||
"@git.zone/tstest": "^3.1.8",
|
"@git.zone/tstest": "^3.1.8",
|
||||||
"@push.rocks/tapbundle": "^6.0.3",
|
"@push.rocks/tapbundle": "^6.0.3",
|
||||||
"@types/node": "^25.2.3"
|
"@types/node": "^25.3.0"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@push.rocks/qenv": "^6.1.3",
|
"@push.rocks/qenv": "^6.1.3",
|
||||||
|
|||||||
66
pnpm-lock.yaml
generated
66
pnpm-lock.yaml
generated
@@ -34,8 +34,8 @@ importers:
|
|||||||
specifier: ^6.0.3
|
specifier: ^6.0.3
|
||||||
version: 6.0.3(socks@2.8.7)
|
version: 6.0.3(socks@2.8.7)
|
||||||
'@types/node':
|
'@types/node':
|
||||||
specifier: ^25.2.3
|
specifier: ^25.3.0
|
||||||
version: 25.2.3
|
version: 25.3.0
|
||||||
|
|
||||||
packages:
|
packages:
|
||||||
|
|
||||||
@@ -1501,8 +1501,8 @@ packages:
|
|||||||
'@types/node@22.19.11':
|
'@types/node@22.19.11':
|
||||||
resolution: {integrity: sha512-BH7YwL6rA93ReqeQS1c4bsPpcfOmJasG+Fkr6Y59q83f9M1WcBRHR2vM+P9eOisYRcN3ujQoiZY8uk5W+1WL8w==}
|
resolution: {integrity: sha512-BH7YwL6rA93ReqeQS1c4bsPpcfOmJasG+Fkr6Y59q83f9M1WcBRHR2vM+P9eOisYRcN3ujQoiZY8uk5W+1WL8w==}
|
||||||
|
|
||||||
'@types/node@25.2.3':
|
'@types/node@25.3.0':
|
||||||
resolution: {integrity: sha512-m0jEgYlYz+mDJZ2+F4v8D1AyQb+QzsNqRuI7xg1VQX/KlKS0qT9r1Mo16yo5F/MtifXFgaofIFsdFMox2SxIbQ==}
|
resolution: {integrity: sha512-4K3bqJpXpqfg2XKGK9bpDTc6xO/xoUP/RBWS7AtRMug6zZFaRekiLzjVtAoZMquxoAbzBvy5nxQ7veS5eYzf8A==}
|
||||||
|
|
||||||
'@types/parse5@6.0.3':
|
'@types/parse5@6.0.3':
|
||||||
resolution: {integrity: sha512-SuT16Q1K51EAVPz1K29DJ/sXjhSQ0zjvsypYJ6tlwVsRV9jwW5Adq2ch8Dq8kDBCkYnELS7N7VNCSB5nC56t/g==}
|
resolution: {integrity: sha512-SuT16Q1K51EAVPz1K29DJ/sXjhSQ0zjvsypYJ6tlwVsRV9jwW5Adq2ch8Dq8kDBCkYnELS7N7VNCSB5nC56t/g==}
|
||||||
@@ -3986,8 +3986,8 @@ packages:
|
|||||||
undici-types@6.21.0:
|
undici-types@6.21.0:
|
||||||
resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==}
|
resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==}
|
||||||
|
|
||||||
undici-types@7.16.0:
|
undici-types@7.18.2:
|
||||||
resolution: {integrity: sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==}
|
resolution: {integrity: sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==}
|
||||||
|
|
||||||
unified@11.0.5:
|
unified@11.0.5:
|
||||||
resolution: {integrity: sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA==}
|
resolution: {integrity: sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA==}
|
||||||
@@ -5178,7 +5178,7 @@ snapshots:
|
|||||||
'@jest/schemas': 29.6.3
|
'@jest/schemas': 29.6.3
|
||||||
'@types/istanbul-lib-coverage': 2.0.6
|
'@types/istanbul-lib-coverage': 2.0.6
|
||||||
'@types/istanbul-reports': 3.0.4
|
'@types/istanbul-reports': 3.0.4
|
||||||
'@types/node': 25.2.3
|
'@types/node': 25.3.0
|
||||||
'@types/yargs': 17.0.35
|
'@types/yargs': 17.0.35
|
||||||
chalk: 4.1.2
|
chalk: 4.1.2
|
||||||
|
|
||||||
@@ -6736,14 +6736,14 @@ snapshots:
|
|||||||
|
|
||||||
'@types/accepts@1.3.7':
|
'@types/accepts@1.3.7':
|
||||||
dependencies:
|
dependencies:
|
||||||
'@types/node': 25.2.3
|
'@types/node': 25.3.0
|
||||||
|
|
||||||
'@types/babel__code-frame@7.27.0': {}
|
'@types/babel__code-frame@7.27.0': {}
|
||||||
|
|
||||||
'@types/body-parser@1.19.6':
|
'@types/body-parser@1.19.6':
|
||||||
dependencies:
|
dependencies:
|
||||||
'@types/connect': 3.4.38
|
'@types/connect': 3.4.38
|
||||||
'@types/node': 25.2.3
|
'@types/node': 25.3.0
|
||||||
|
|
||||||
'@types/buffer-json@2.0.3': {}
|
'@types/buffer-json@2.0.3': {}
|
||||||
|
|
||||||
@@ -6760,17 +6760,17 @@ snapshots:
|
|||||||
|
|
||||||
'@types/clean-css@4.2.11':
|
'@types/clean-css@4.2.11':
|
||||||
dependencies:
|
dependencies:
|
||||||
'@types/node': 25.2.3
|
'@types/node': 25.3.0
|
||||||
source-map: 0.6.1
|
source-map: 0.6.1
|
||||||
|
|
||||||
'@types/co-body@6.1.3':
|
'@types/co-body@6.1.3':
|
||||||
dependencies:
|
dependencies:
|
||||||
'@types/node': 25.2.3
|
'@types/node': 25.3.0
|
||||||
'@types/qs': 6.14.0
|
'@types/qs': 6.14.0
|
||||||
|
|
||||||
'@types/connect@3.4.38':
|
'@types/connect@3.4.38':
|
||||||
dependencies:
|
dependencies:
|
||||||
'@types/node': 25.2.3
|
'@types/node': 25.3.0
|
||||||
|
|
||||||
'@types/content-disposition@0.5.9': {}
|
'@types/content-disposition@0.5.9': {}
|
||||||
|
|
||||||
@@ -6781,11 +6781,11 @@ snapshots:
|
|||||||
'@types/connect': 3.4.38
|
'@types/connect': 3.4.38
|
||||||
'@types/express': 5.0.6
|
'@types/express': 5.0.6
|
||||||
'@types/keygrip': 1.0.6
|
'@types/keygrip': 1.0.6
|
||||||
'@types/node': 25.2.3
|
'@types/node': 25.3.0
|
||||||
|
|
||||||
'@types/cors@2.8.19':
|
'@types/cors@2.8.19':
|
||||||
dependencies:
|
dependencies:
|
||||||
'@types/node': 25.2.3
|
'@types/node': 25.3.0
|
||||||
|
|
||||||
'@types/debounce@1.2.4': {}
|
'@types/debounce@1.2.4': {}
|
||||||
|
|
||||||
@@ -6797,7 +6797,7 @@ snapshots:
|
|||||||
|
|
||||||
'@types/express-serve-static-core@5.1.1':
|
'@types/express-serve-static-core@5.1.1':
|
||||||
dependencies:
|
dependencies:
|
||||||
'@types/node': 25.2.3
|
'@types/node': 25.3.0
|
||||||
'@types/qs': 6.14.0
|
'@types/qs': 6.14.0
|
||||||
'@types/range-parser': 1.2.7
|
'@types/range-parser': 1.2.7
|
||||||
'@types/send': 1.2.1
|
'@types/send': 1.2.1
|
||||||
@@ -6811,7 +6811,7 @@ snapshots:
|
|||||||
'@types/fs-extra@11.0.4':
|
'@types/fs-extra@11.0.4':
|
||||||
dependencies:
|
dependencies:
|
||||||
'@types/jsonfile': 6.1.4
|
'@types/jsonfile': 6.1.4
|
||||||
'@types/node': 25.2.3
|
'@types/node': 25.3.0
|
||||||
|
|
||||||
'@types/hast@3.0.4':
|
'@types/hast@3.0.4':
|
||||||
dependencies:
|
dependencies:
|
||||||
@@ -6845,7 +6845,7 @@ snapshots:
|
|||||||
|
|
||||||
'@types/jsonfile@6.1.4':
|
'@types/jsonfile@6.1.4':
|
||||||
dependencies:
|
dependencies:
|
||||||
'@types/node': 25.2.3
|
'@types/node': 25.3.0
|
||||||
|
|
||||||
'@types/keygrip@1.0.6': {}
|
'@types/keygrip@1.0.6': {}
|
||||||
|
|
||||||
@@ -6862,7 +6862,7 @@ snapshots:
|
|||||||
'@types/http-errors': 2.0.5
|
'@types/http-errors': 2.0.5
|
||||||
'@types/keygrip': 1.0.6
|
'@types/keygrip': 1.0.6
|
||||||
'@types/koa-compose': 3.2.9
|
'@types/koa-compose': 3.2.9
|
||||||
'@types/node': 25.2.3
|
'@types/node': 25.3.0
|
||||||
|
|
||||||
'@types/mdast@4.0.4':
|
'@types/mdast@4.0.4':
|
||||||
dependencies:
|
dependencies:
|
||||||
@@ -6876,19 +6876,19 @@ snapshots:
|
|||||||
|
|
||||||
'@types/mute-stream@0.0.4':
|
'@types/mute-stream@0.0.4':
|
||||||
dependencies:
|
dependencies:
|
||||||
'@types/node': 25.2.3
|
'@types/node': 25.3.0
|
||||||
|
|
||||||
'@types/node-forge@1.3.14':
|
'@types/node-forge@1.3.14':
|
||||||
dependencies:
|
dependencies:
|
||||||
'@types/node': 25.2.3
|
'@types/node': 25.3.0
|
||||||
|
|
||||||
'@types/node@22.19.11':
|
'@types/node@22.19.11':
|
||||||
dependencies:
|
dependencies:
|
||||||
undici-types: 6.21.0
|
undici-types: 6.21.0
|
||||||
|
|
||||||
'@types/node@25.2.3':
|
'@types/node@25.3.0':
|
||||||
dependencies:
|
dependencies:
|
||||||
undici-types: 7.16.0
|
undici-types: 7.18.2
|
||||||
|
|
||||||
'@types/parse5@6.0.3': {}
|
'@types/parse5@6.0.3': {}
|
||||||
|
|
||||||
@@ -6904,18 +6904,18 @@ snapshots:
|
|||||||
|
|
||||||
'@types/s3rver@3.7.4':
|
'@types/s3rver@3.7.4':
|
||||||
dependencies:
|
dependencies:
|
||||||
'@types/node': 25.2.3
|
'@types/node': 25.3.0
|
||||||
|
|
||||||
'@types/semver@7.7.1': {}
|
'@types/semver@7.7.1': {}
|
||||||
|
|
||||||
'@types/send@1.2.1':
|
'@types/send@1.2.1':
|
||||||
dependencies:
|
dependencies:
|
||||||
'@types/node': 25.2.3
|
'@types/node': 25.3.0
|
||||||
|
|
||||||
'@types/serve-static@2.2.0':
|
'@types/serve-static@2.2.0':
|
||||||
dependencies:
|
dependencies:
|
||||||
'@types/http-errors': 2.0.5
|
'@types/http-errors': 2.0.5
|
||||||
'@types/node': 25.2.3
|
'@types/node': 25.3.0
|
||||||
|
|
||||||
'@types/sinon-chai@3.2.12':
|
'@types/sinon-chai@3.2.12':
|
||||||
dependencies:
|
dependencies:
|
||||||
@@ -6934,11 +6934,11 @@ snapshots:
|
|||||||
|
|
||||||
'@types/tar-stream@3.1.4':
|
'@types/tar-stream@3.1.4':
|
||||||
dependencies:
|
dependencies:
|
||||||
'@types/node': 25.2.3
|
'@types/node': 25.3.0
|
||||||
|
|
||||||
'@types/through2@2.0.41':
|
'@types/through2@2.0.41':
|
||||||
dependencies:
|
dependencies:
|
||||||
'@types/node': 25.2.3
|
'@types/node': 25.3.0
|
||||||
|
|
||||||
'@types/triple-beam@1.3.5': {}
|
'@types/triple-beam@1.3.5': {}
|
||||||
|
|
||||||
@@ -6966,11 +6966,11 @@ snapshots:
|
|||||||
|
|
||||||
'@types/ws@7.4.7':
|
'@types/ws@7.4.7':
|
||||||
dependencies:
|
dependencies:
|
||||||
'@types/node': 25.2.3
|
'@types/node': 25.3.0
|
||||||
|
|
||||||
'@types/ws@8.18.1':
|
'@types/ws@8.18.1':
|
||||||
dependencies:
|
dependencies:
|
||||||
'@types/node': 25.2.3
|
'@types/node': 25.3.0
|
||||||
|
|
||||||
'@types/yargs-parser@21.0.3': {}
|
'@types/yargs-parser@21.0.3': {}
|
||||||
|
|
||||||
@@ -6980,7 +6980,7 @@ snapshots:
|
|||||||
|
|
||||||
'@types/yauzl@2.10.3':
|
'@types/yauzl@2.10.3':
|
||||||
dependencies:
|
dependencies:
|
||||||
'@types/node': 25.2.3
|
'@types/node': 25.3.0
|
||||||
optional: true
|
optional: true
|
||||||
|
|
||||||
'@ungap/structured-clone@1.3.0': {}
|
'@ungap/structured-clone@1.3.0': {}
|
||||||
@@ -7585,7 +7585,7 @@ snapshots:
|
|||||||
engine.io@6.6.4:
|
engine.io@6.6.4:
|
||||||
dependencies:
|
dependencies:
|
||||||
'@types/cors': 2.8.19
|
'@types/cors': 2.8.19
|
||||||
'@types/node': 25.2.3
|
'@types/node': 25.3.0
|
||||||
accepts: 1.3.8
|
accepts: 1.3.8
|
||||||
base64id: 2.0.0
|
base64id: 2.0.0
|
||||||
cookie: 0.7.2
|
cookie: 0.7.2
|
||||||
@@ -8299,7 +8299,7 @@ snapshots:
|
|||||||
jest-util@29.7.0:
|
jest-util@29.7.0:
|
||||||
dependencies:
|
dependencies:
|
||||||
'@jest/types': 29.6.3
|
'@jest/types': 29.6.3
|
||||||
'@types/node': 25.2.3
|
'@types/node': 25.3.0
|
||||||
chalk: 4.1.2
|
chalk: 4.1.2
|
||||||
ci-info: 3.9.0
|
ci-info: 3.9.0
|
||||||
graceful-fs: 4.2.11
|
graceful-fs: 4.2.11
|
||||||
@@ -9807,7 +9807,7 @@ snapshots:
|
|||||||
|
|
||||||
undici-types@6.21.0: {}
|
undici-types@6.21.0: {}
|
||||||
|
|
||||||
undici-types@7.16.0: {}
|
undici-types@7.18.2: {}
|
||||||
|
|
||||||
unified@11.0.5:
|
unified@11.0.5:
|
||||||
dependencies:
|
dependencies:
|
||||||
|
|||||||
35
rust/Cargo.lock
generated
35
rust/Cargo.lock
generated
@@ -327,6 +327,16 @@ version = "0.2.182"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112"
|
checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "libmimalloc-sys"
|
||||||
|
version = "0.1.44"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "667f4fec20f29dfc6bc7357c582d91796c169ad7e2fce709468aefeb2c099870"
|
||||||
|
dependencies = [
|
||||||
|
"cc",
|
||||||
|
"libc",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lock_api"
|
name = "lock_api"
|
||||||
version = "0.4.14"
|
version = "0.4.14"
|
||||||
@@ -348,6 +358,15 @@ version = "2.8.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
|
checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "mimalloc"
|
||||||
|
version = "0.1.48"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e1ee66a4b64c74f4ef288bcbb9192ad9c3feaad75193129ac8509af543894fd8"
|
||||||
|
dependencies = [
|
||||||
|
"libmimalloc-sys",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mio"
|
name = "mio"
|
||||||
version = "1.1.1"
|
version = "1.1.1"
|
||||||
@@ -519,6 +538,7 @@ dependencies = [
|
|||||||
"clap",
|
"clap",
|
||||||
"env_logger",
|
"env_logger",
|
||||||
"log",
|
"log",
|
||||||
|
"mimalloc",
|
||||||
"remoteingress-core",
|
"remoteingress-core",
|
||||||
"remoteingress-protocol",
|
"remoteingress-protocol",
|
||||||
"rustls",
|
"rustls",
|
||||||
@@ -538,6 +558,7 @@ dependencies = [
|
|||||||
"rustls-pemfile",
|
"rustls-pemfile",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
|
"socket2 0.5.10",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tokio-rustls",
|
"tokio-rustls",
|
||||||
"tokio-util",
|
"tokio-util",
|
||||||
@@ -547,7 +568,9 @@ dependencies = [
|
|||||||
name = "remoteingress-protocol"
|
name = "remoteingress-protocol"
|
||||||
version = "2.0.0"
|
version = "2.0.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"log",
|
||||||
"tokio",
|
"tokio",
|
||||||
|
"tokio-util",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -681,6 +704,16 @@ version = "1.15.1"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
|
checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "socket2"
|
||||||
|
version = "0.5.10"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
"windows-sys 0.52.0",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "socket2"
|
name = "socket2"
|
||||||
version = "0.6.2"
|
version = "0.6.2"
|
||||||
@@ -745,7 +778,7 @@ dependencies = [
|
|||||||
"parking_lot",
|
"parking_lot",
|
||||||
"pin-project-lite",
|
"pin-project-lite",
|
||||||
"signal-hook-registry",
|
"signal-hook-registry",
|
||||||
"socket2",
|
"socket2 0.6.2",
|
||||||
"tokio-macros",
|
"tokio-macros",
|
||||||
"windows-sys 0.61.2",
|
"windows-sys 0.61.2",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -17,3 +17,4 @@ serde_json = "1"
|
|||||||
log = "0.4"
|
log = "0.4"
|
||||||
env_logger = "0.11"
|
env_logger = "0.11"
|
||||||
rustls = { version = "0.23", default-features = false, features = ["ring"] }
|
rustls = { version = "0.23", default-features = false, features = ["ring"] }
|
||||||
|
mimalloc = "0.1"
|
||||||
|
|||||||
@@ -1,3 +1,6 @@
|
|||||||
|
#[global_allocator]
|
||||||
|
static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
|
||||||
|
|
||||||
use clap::Parser;
|
use clap::Parser;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
@@ -164,16 +167,16 @@ async fn handle_request(
|
|||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
while let Some(event) = event_rx.recv().await {
|
while let Some(event) = event_rx.recv().await {
|
||||||
match &event {
|
match &event {
|
||||||
HubEvent::EdgeConnected { edge_id } => {
|
HubEvent::EdgeConnected { edge_id, peer_addr } => {
|
||||||
send_event(
|
send_event(
|
||||||
"edgeConnected",
|
"edgeConnected",
|
||||||
serde_json::json!({ "edgeId": edge_id }),
|
serde_json::json!({ "edgeId": edge_id, "peerAddr": peer_addr }),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
HubEvent::EdgeDisconnected { edge_id } => {
|
HubEvent::EdgeDisconnected { edge_id, reason } => {
|
||||||
send_event(
|
send_event(
|
||||||
"edgeDisconnected",
|
"edgeDisconnected",
|
||||||
serde_json::json!({ "edgeId": edge_id }),
|
serde_json::json!({ "edgeId": edge_id, "reason": reason }),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
HubEvent::StreamOpened {
|
HubEvent::StreamOpened {
|
||||||
@@ -292,8 +295,8 @@ async fn handle_request(
|
|||||||
EdgeEvent::TunnelConnected => {
|
EdgeEvent::TunnelConnected => {
|
||||||
send_event("tunnelConnected", serde_json::json!({}));
|
send_event("tunnelConnected", serde_json::json!({}));
|
||||||
}
|
}
|
||||||
EdgeEvent::TunnelDisconnected => {
|
EdgeEvent::TunnelDisconnected { reason } => {
|
||||||
send_event("tunnelDisconnected", serde_json::json!({}));
|
send_event("tunnelDisconnected", serde_json::json!({ "reason": reason }));
|
||||||
}
|
}
|
||||||
EdgeEvent::PublicIpDiscovered { ip } => {
|
EdgeEvent::PublicIpDiscovered { ip } => {
|
||||||
send_event(
|
send_event(
|
||||||
|
|||||||
@@ -14,3 +14,4 @@ serde_json = "1"
|
|||||||
log = "0.4"
|
log = "0.4"
|
||||||
rustls-pemfile = "2"
|
rustls-pemfile = "2"
|
||||||
tokio-util = "0.7"
|
tokio-util = "0.7"
|
||||||
|
socket2 = "0.5"
|
||||||
|
|||||||
@@ -1,16 +1,39 @@
|
|||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::sync::atomic::{AtomicU32, Ordering};
|
use std::sync::atomic::{AtomicU32, Ordering};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader};
|
use std::time::Duration;
|
||||||
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||||
use tokio::net::{TcpListener, TcpStream};
|
use tokio::net::{TcpListener, TcpStream};
|
||||||
use tokio::sync::{mpsc, Mutex, RwLock};
|
use tokio::sync::{mpsc, Mutex, Notify, RwLock};
|
||||||
use tokio::task::JoinHandle;
|
use tokio::task::JoinHandle;
|
||||||
|
use tokio::time::{Instant, sleep_until};
|
||||||
use tokio_rustls::TlsConnector;
|
use tokio_rustls::TlsConnector;
|
||||||
use tokio_util::sync::CancellationToken;
|
use tokio_util::sync::CancellationToken;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
use remoteingress_protocol::*;
|
use remoteingress_protocol::*;
|
||||||
|
|
||||||
|
type EdgeTlsStream = tokio_rustls::client::TlsStream<TcpStream>;
|
||||||
|
|
||||||
|
/// Result of processing a frame (shared with hub.rs pattern).
|
||||||
|
#[allow(dead_code)]
|
||||||
|
enum EdgeFrameAction {
|
||||||
|
Continue,
|
||||||
|
Disconnect(String),
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Per-stream state tracked in the edge's client_writers map.
|
||||||
|
struct EdgeStreamState {
|
||||||
|
/// Unbounded channel to deliver FRAME_DATA_BACK payloads to the hub_to_client task.
|
||||||
|
/// Unbounded because flow control (WINDOW_UPDATE) already limits bytes-in-flight.
|
||||||
|
back_tx: mpsc::UnboundedSender<Vec<u8>>,
|
||||||
|
/// Send window for FRAME_DATA (upload direction).
|
||||||
|
/// Decremented by the client reader, incremented by FRAME_WINDOW_UPDATE_BACK from hub.
|
||||||
|
send_window: Arc<AtomicU32>,
|
||||||
|
/// Notifier to wake the client reader when the window opens.
|
||||||
|
window_notify: Arc<Notify>,
|
||||||
|
}
|
||||||
|
|
||||||
/// Edge configuration (hub-host + credentials only; ports come from hub).
|
/// Edge configuration (hub-host + credentials only; ports come from hub).
|
||||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||||
#[serde(rename_all = "camelCase")]
|
#[serde(rename_all = "camelCase")]
|
||||||
@@ -19,6 +42,10 @@ pub struct EdgeConfig {
|
|||||||
pub hub_port: u16,
|
pub hub_port: u16,
|
||||||
pub edge_id: String,
|
pub edge_id: String,
|
||||||
pub secret: String,
|
pub secret: String,
|
||||||
|
/// Optional bind address for TCP listeners (defaults to "0.0.0.0").
|
||||||
|
/// Useful for testing on localhost where edge and upstream share the same machine.
|
||||||
|
#[serde(default)]
|
||||||
|
pub bind_address: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Handshake config received from hub after authentication.
|
/// Handshake config received from hub after authentication.
|
||||||
@@ -47,7 +74,8 @@ struct ConfigUpdate {
|
|||||||
#[serde(tag = "type")]
|
#[serde(tag = "type")]
|
||||||
pub enum EdgeEvent {
|
pub enum EdgeEvent {
|
||||||
TunnelConnected,
|
TunnelConnected,
|
||||||
TunnelDisconnected,
|
#[serde(rename_all = "camelCase")]
|
||||||
|
TunnelDisconnected { reason: String },
|
||||||
#[serde(rename_all = "camelCase")]
|
#[serde(rename_all = "camelCase")]
|
||||||
PublicIpDiscovered { ip: String },
|
PublicIpDiscovered { ip: String },
|
||||||
#[serde(rename_all = "camelCase")]
|
#[serde(rename_all = "camelCase")]
|
||||||
@@ -181,6 +209,14 @@ async fn edge_main_loop(
|
|||||||
let mut backoff_ms: u64 = 1000;
|
let mut backoff_ms: u64 = 1000;
|
||||||
let max_backoff_ms: u64 = 30000;
|
let max_backoff_ms: u64 = 30000;
|
||||||
|
|
||||||
|
// Build TLS config ONCE outside the reconnect loop — preserves session
|
||||||
|
// cache across reconnections for TLS session resumption (saves 1 RTT).
|
||||||
|
let tls_config = rustls::ClientConfig::builder()
|
||||||
|
.dangerous()
|
||||||
|
.with_custom_certificate_verifier(Arc::new(NoCertVerifier))
|
||||||
|
.with_no_client_auth();
|
||||||
|
let connector = TlsConnector::from(Arc::new(tls_config));
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
// Create a per-connection child token
|
// Create a per-connection child token
|
||||||
let connection_token = cancel_token.child_token();
|
let connection_token = cancel_token.child_token();
|
||||||
@@ -196,14 +232,31 @@ async fn edge_main_loop(
|
|||||||
&listen_ports,
|
&listen_ports,
|
||||||
&mut shutdown_rx,
|
&mut shutdown_rx,
|
||||||
&connection_token,
|
&connection_token,
|
||||||
|
&connector,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
// Cancel connection token to kill all orphaned tasks from this cycle
|
// Cancel connection token to kill all orphaned tasks from this cycle
|
||||||
connection_token.cancel();
|
connection_token.cancel();
|
||||||
|
|
||||||
|
// Reset backoff after a successful connection for fast reconnect
|
||||||
|
let was_connected = *connected.read().await;
|
||||||
|
if was_connected {
|
||||||
|
backoff_ms = 1000;
|
||||||
|
log::info!("Was connected; resetting backoff to {}ms for fast reconnect", backoff_ms);
|
||||||
|
}
|
||||||
|
|
||||||
*connected.write().await = false;
|
*connected.write().await = false;
|
||||||
let _ = event_tx.try_send(EdgeEvent::TunnelDisconnected);
|
// Extract reason for disconnect event
|
||||||
|
let reason = match &result {
|
||||||
|
EdgeLoopResult::Reconnect(r) => r.clone(),
|
||||||
|
EdgeLoopResult::Shutdown => "shutdown".to_string(),
|
||||||
|
};
|
||||||
|
// Only emit disconnect event on actual disconnection, not on failed reconnects.
|
||||||
|
// Failed reconnects never reach line 335 (handshake success), so was_connected is false.
|
||||||
|
if was_connected {
|
||||||
|
let _ = event_tx.try_send(EdgeEvent::TunnelDisconnected { reason: reason.clone() });
|
||||||
|
}
|
||||||
active_streams.store(0, Ordering::Relaxed);
|
active_streams.store(0, Ordering::Relaxed);
|
||||||
// Reset stream ID counter for next connection cycle
|
// Reset stream ID counter for next connection cycle
|
||||||
next_stream_id.store(1, Ordering::Relaxed);
|
next_stream_id.store(1, Ordering::Relaxed);
|
||||||
@@ -211,10 +264,10 @@ async fn edge_main_loop(
|
|||||||
|
|
||||||
match result {
|
match result {
|
||||||
EdgeLoopResult::Shutdown => break,
|
EdgeLoopResult::Shutdown => break,
|
||||||
EdgeLoopResult::Reconnect => {
|
EdgeLoopResult::Reconnect(_) => {
|
||||||
log::info!("Reconnecting in {}ms...", backoff_ms);
|
log::info!("Reconnecting in {}ms...", backoff_ms);
|
||||||
tokio::select! {
|
tokio::select! {
|
||||||
_ = tokio::time::sleep(std::time::Duration::from_millis(backoff_ms)) => {}
|
_ = tokio::time::sleep(Duration::from_millis(backoff_ms)) => {}
|
||||||
_ = cancel_token.cancelled() => break,
|
_ = cancel_token.cancelled() => break,
|
||||||
_ = shutdown_rx.recv() => break,
|
_ = shutdown_rx.recv() => break,
|
||||||
}
|
}
|
||||||
@@ -226,7 +279,87 @@ async fn edge_main_loop(
|
|||||||
|
|
||||||
enum EdgeLoopResult {
|
enum EdgeLoopResult {
|
||||||
Shutdown,
|
Shutdown,
|
||||||
Reconnect,
|
Reconnect(String), // reason for disconnection
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Process a single frame received from the hub side of the tunnel.
|
||||||
|
/// Handles FRAME_DATA_BACK, FRAME_WINDOW_UPDATE_BACK, FRAME_CLOSE_BACK, FRAME_CONFIG, FRAME_PING.
|
||||||
|
async fn handle_edge_frame(
|
||||||
|
frame: Frame,
|
||||||
|
tunnel_io: &mut remoteingress_protocol::TunnelIo<EdgeTlsStream>,
|
||||||
|
client_writers: &Arc<Mutex<HashMap<u32, EdgeStreamState>>>,
|
||||||
|
listen_ports: &Arc<RwLock<Vec<u16>>>,
|
||||||
|
event_tx: &mpsc::Sender<EdgeEvent>,
|
||||||
|
tunnel_writer_tx: &mpsc::Sender<Vec<u8>>,
|
||||||
|
tunnel_data_tx: &mpsc::Sender<Vec<u8>>,
|
||||||
|
port_listeners: &mut HashMap<u16, JoinHandle<()>>,
|
||||||
|
active_streams: &Arc<AtomicU32>,
|
||||||
|
next_stream_id: &Arc<AtomicU32>,
|
||||||
|
edge_id: &str,
|
||||||
|
connection_token: &CancellationToken,
|
||||||
|
bind_address: &str,
|
||||||
|
) -> EdgeFrameAction {
|
||||||
|
match frame.frame_type {
|
||||||
|
FRAME_DATA_BACK => {
|
||||||
|
// Dispatch to per-stream unbounded channel. Flow control (WINDOW_UPDATE)
|
||||||
|
// limits bytes-in-flight, so the channel won't grow unbounded. send() only
|
||||||
|
// fails if the receiver is dropped (hub_to_client task already exited).
|
||||||
|
let mut writers = client_writers.lock().await;
|
||||||
|
if let Some(state) = writers.get(&frame.stream_id) {
|
||||||
|
if state.back_tx.send(frame.payload).is_err() {
|
||||||
|
// Receiver dropped — hub_to_client task already exited, clean up
|
||||||
|
writers.remove(&frame.stream_id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
FRAME_WINDOW_UPDATE_BACK => {
|
||||||
|
if let Some(increment) = decode_window_update(&frame.payload) {
|
||||||
|
if increment > 0 {
|
||||||
|
let writers = client_writers.lock().await;
|
||||||
|
if let Some(state) = writers.get(&frame.stream_id) {
|
||||||
|
let prev = state.send_window.fetch_add(increment, Ordering::Release);
|
||||||
|
if prev + increment > MAX_WINDOW_SIZE {
|
||||||
|
state.send_window.store(MAX_WINDOW_SIZE, Ordering::Release);
|
||||||
|
}
|
||||||
|
state.window_notify.notify_one();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
FRAME_CLOSE_BACK => {
|
||||||
|
let mut writers = client_writers.lock().await;
|
||||||
|
writers.remove(&frame.stream_id);
|
||||||
|
}
|
||||||
|
FRAME_CONFIG => {
|
||||||
|
if let Ok(update) = serde_json::from_slice::<ConfigUpdate>(&frame.payload) {
|
||||||
|
log::info!("Config update from hub: ports {:?}", update.listen_ports);
|
||||||
|
*listen_ports.write().await = update.listen_ports.clone();
|
||||||
|
let _ = event_tx.try_send(EdgeEvent::PortsUpdated {
|
||||||
|
listen_ports: update.listen_ports.clone(),
|
||||||
|
});
|
||||||
|
apply_port_config(
|
||||||
|
&update.listen_ports,
|
||||||
|
port_listeners,
|
||||||
|
tunnel_writer_tx,
|
||||||
|
tunnel_data_tx,
|
||||||
|
client_writers,
|
||||||
|
active_streams,
|
||||||
|
next_stream_id,
|
||||||
|
edge_id,
|
||||||
|
connection_token,
|
||||||
|
bind_address,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
FRAME_PING => {
|
||||||
|
// Queue PONG directly — no channel round-trip, guaranteed delivery
|
||||||
|
tunnel_io.queue_ctrl(encode_frame(0, FRAME_PONG, &[]));
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
log::warn!("Unexpected frame type {} from hub", frame.frame_type);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
EdgeFrameAction::Continue
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn connect_to_hub_and_run(
|
async fn connect_to_hub_and_run(
|
||||||
@@ -239,63 +372,78 @@ async fn connect_to_hub_and_run(
|
|||||||
listen_ports: &Arc<RwLock<Vec<u16>>>,
|
listen_ports: &Arc<RwLock<Vec<u16>>>,
|
||||||
shutdown_rx: &mut mpsc::Receiver<()>,
|
shutdown_rx: &mut mpsc::Receiver<()>,
|
||||||
connection_token: &CancellationToken,
|
connection_token: &CancellationToken,
|
||||||
|
connector: &TlsConnector,
|
||||||
) -> EdgeLoopResult {
|
) -> EdgeLoopResult {
|
||||||
// Build TLS connector that skips cert verification (auth is via secret)
|
|
||||||
let tls_config = rustls::ClientConfig::builder()
|
|
||||||
.dangerous()
|
|
||||||
.with_custom_certificate_verifier(Arc::new(NoCertVerifier))
|
|
||||||
.with_no_client_auth();
|
|
||||||
|
|
||||||
let connector = TlsConnector::from(Arc::new(tls_config));
|
|
||||||
|
|
||||||
let addr = format!("{}:{}", config.hub_host, config.hub_port);
|
let addr = format!("{}:{}", config.hub_host, config.hub_port);
|
||||||
let tcp = match TcpStream::connect(&addr).await {
|
let tcp = match TcpStream::connect(&addr).await {
|
||||||
Ok(s) => s,
|
Ok(s) => {
|
||||||
|
// Disable Nagle's algorithm for low-latency control frames (PING/PONG, WINDOW_UPDATE)
|
||||||
|
let _ = s.set_nodelay(true);
|
||||||
|
// TCP keepalive detects silent network failures (NAT timeout, path change)
|
||||||
|
// faster than the 45s application-level liveness timeout.
|
||||||
|
let ka = socket2::TcpKeepalive::new()
|
||||||
|
.with_time(Duration::from_secs(30));
|
||||||
|
#[cfg(target_os = "linux")]
|
||||||
|
let ka = ka.with_interval(Duration::from_secs(10));
|
||||||
|
let _ = socket2::SockRef::from(&s).set_tcp_keepalive(&ka);
|
||||||
|
s
|
||||||
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
log::error!("Failed to connect to hub at {}: {}", addr, e);
|
log::error!("Failed to connect to hub at {}: {}", addr, e);
|
||||||
return EdgeLoopResult::Reconnect;
|
return EdgeLoopResult::Reconnect(format!("tcp_connect_failed: {}", e));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let server_name = rustls::pki_types::ServerName::try_from(config.hub_host.clone())
|
let server_name = rustls::pki_types::ServerName::try_from(config.hub_host.clone())
|
||||||
.unwrap_or_else(|_| rustls::pki_types::ServerName::try_from("remoteingress-hub".to_string()).unwrap());
|
.unwrap_or_else(|_| rustls::pki_types::ServerName::try_from("remoteingress-hub".to_string()).unwrap());
|
||||||
|
|
||||||
let tls_stream = match connector.connect(server_name, tcp).await {
|
let mut tls_stream = match connector.connect(server_name, tcp).await {
|
||||||
Ok(s) => s,
|
Ok(s) => s,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
log::error!("TLS handshake failed: {}", e);
|
log::error!("TLS handshake failed: {}", e);
|
||||||
return EdgeLoopResult::Reconnect;
|
return EdgeLoopResult::Reconnect(format!("tls_handshake_failed: {}", e));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let (read_half, mut write_half) = tokio::io::split(tls_stream);
|
// Send auth line (we own the whole stream — no split)
|
||||||
|
|
||||||
// Send auth line
|
|
||||||
let auth_line = format!("EDGE {} {}\n", config.edge_id, config.secret);
|
let auth_line = format!("EDGE {} {}\n", config.edge_id, config.secret);
|
||||||
if write_half.write_all(auth_line.as_bytes()).await.is_err() {
|
if tls_stream.write_all(auth_line.as_bytes()).await.is_err() {
|
||||||
return EdgeLoopResult::Reconnect;
|
return EdgeLoopResult::Reconnect("auth_write_failed".to_string());
|
||||||
|
}
|
||||||
|
if tls_stream.flush().await.is_err() {
|
||||||
|
return EdgeLoopResult::Reconnect("auth_flush_failed".to_string());
|
||||||
}
|
}
|
||||||
|
|
||||||
// Read handshake response line from hub (JSON with initial config)
|
// Read handshake line byte-by-byte (no BufReader — into_inner corrupts TLS state)
|
||||||
let mut buf_reader = BufReader::new(read_half);
|
let mut handshake_bytes = Vec::with_capacity(512);
|
||||||
let mut handshake_line = String::new();
|
let mut byte = [0u8; 1];
|
||||||
match buf_reader.read_line(&mut handshake_line).await {
|
loop {
|
||||||
Ok(0) => {
|
match tls_stream.read_exact(&mut byte).await {
|
||||||
log::error!("Hub rejected connection (EOF before handshake)");
|
Ok(_) => {
|
||||||
return EdgeLoopResult::Reconnect;
|
handshake_bytes.push(byte[0]);
|
||||||
}
|
if byte[0] == b'\n' { break; }
|
||||||
Ok(_) => {}
|
if handshake_bytes.len() > 8192 {
|
||||||
Err(e) => {
|
return EdgeLoopResult::Reconnect("handshake_too_long".to_string());
|
||||||
log::error!("Failed to read handshake response: {}", e);
|
}
|
||||||
return EdgeLoopResult::Reconnect;
|
}
|
||||||
|
Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => {
|
||||||
|
log::error!("Hub rejected connection (EOF before handshake)");
|
||||||
|
return EdgeLoopResult::Reconnect("hub_rejected_eof".to_string());
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
log::error!("Failed to read handshake response: {}", e);
|
||||||
|
return EdgeLoopResult::Reconnect(format!("handshake_read_failed: {}", e));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
let handshake_line = String::from_utf8_lossy(&handshake_bytes);
|
||||||
|
|
||||||
let handshake: HandshakeConfig = match serde_json::from_str(handshake_line.trim()) {
|
let handshake: HandshakeConfig = match serde_json::from_str(handshake_line.trim()) {
|
||||||
Ok(h) => h,
|
Ok(h) => h,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
log::error!("Invalid handshake response: {}", e);
|
log::error!("Invalid handshake response: {}", e);
|
||||||
return EdgeLoopResult::Reconnect;
|
return EdgeLoopResult::Reconnect(format!("handshake_invalid: {}", e));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -336,95 +484,110 @@ async fn connect_to_hub_and_run(
|
|||||||
_ = stun_token.cancelled() => break,
|
_ = stun_token.cancelled() => break,
|
||||||
}
|
}
|
||||||
tokio::select! {
|
tokio::select! {
|
||||||
_ = tokio::time::sleep(std::time::Duration::from_secs(stun_interval)) => {}
|
_ = tokio::time::sleep(Duration::from_secs(stun_interval)) => {}
|
||||||
_ = stun_token.cancelled() => break,
|
_ = stun_token.cancelled() => break,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
// Client socket map: stream_id -> sender for writing data back to client
|
// Client socket map: stream_id -> per-stream state (back channel + flow control)
|
||||||
let client_writers: Arc<Mutex<HashMap<u32, mpsc::Sender<Vec<u8>>>>> =
|
let client_writers: Arc<Mutex<HashMap<u32, EdgeStreamState>>> =
|
||||||
Arc::new(Mutex::new(HashMap::new()));
|
Arc::new(Mutex::new(HashMap::new()));
|
||||||
|
|
||||||
// Shared tunnel writer
|
// QoS dual-channel: ctrl frames have priority over data frames.
|
||||||
let tunnel_writer = Arc::new(Mutex::new(write_half));
|
// Stream handlers send through these channels → TunnelIo drains them.
|
||||||
|
let (tunnel_ctrl_tx, mut tunnel_ctrl_rx) = mpsc::channel::<Vec<u8>>(256);
|
||||||
|
let (tunnel_data_tx, mut tunnel_data_rx) = mpsc::channel::<Vec<u8>>(4096);
|
||||||
|
let tunnel_writer_tx = tunnel_ctrl_tx.clone();
|
||||||
|
|
||||||
// Start TCP listeners for initial ports (hot-reloadable)
|
// Start TCP listeners for initial ports
|
||||||
let mut port_listeners: HashMap<u16, JoinHandle<()>> = HashMap::new();
|
let mut port_listeners: HashMap<u16, JoinHandle<()>> = HashMap::new();
|
||||||
|
let bind_address = config.bind_address.as_deref().unwrap_or("0.0.0.0");
|
||||||
apply_port_config(
|
apply_port_config(
|
||||||
&handshake.listen_ports,
|
&handshake.listen_ports,
|
||||||
&mut port_listeners,
|
&mut port_listeners,
|
||||||
&tunnel_writer,
|
&tunnel_writer_tx,
|
||||||
|
&tunnel_data_tx,
|
||||||
&client_writers,
|
&client_writers,
|
||||||
active_streams,
|
active_streams,
|
||||||
next_stream_id,
|
next_stream_id,
|
||||||
&config.edge_id,
|
&config.edge_id,
|
||||||
connection_token,
|
connection_token,
|
||||||
|
bind_address,
|
||||||
);
|
);
|
||||||
|
|
||||||
// Read frames from hub
|
// Single-owner I/O engine — no tokio::io::split, no mutex
|
||||||
let mut frame_reader = FrameReader::new(buf_reader);
|
let mut tunnel_io = remoteingress_protocol::TunnelIo::new(tls_stream, Vec::new());
|
||||||
let result = loop {
|
|
||||||
tokio::select! {
|
let liveness_timeout_dur = Duration::from_secs(45);
|
||||||
frame_result = frame_reader.next_frame() => {
|
let mut last_activity = Instant::now();
|
||||||
match frame_result {
|
let mut liveness_deadline = Box::pin(sleep_until(last_activity + liveness_timeout_dur));
|
||||||
Ok(Some(frame)) => {
|
|
||||||
match frame.frame_type {
|
let result = 'io_loop: loop {
|
||||||
FRAME_DATA_BACK => {
|
// Drain any buffered frames
|
||||||
let writers = client_writers.lock().await;
|
loop {
|
||||||
if let Some(tx) = writers.get(&frame.stream_id) {
|
let frame = match tunnel_io.try_parse_frame() {
|
||||||
let _ = tx.send(frame.payload).await;
|
Some(Ok(f)) => f,
|
||||||
}
|
Some(Err(e)) => {
|
||||||
}
|
log::error!("Hub frame error: {}", e);
|
||||||
FRAME_CLOSE_BACK => {
|
break 'io_loop EdgeLoopResult::Reconnect(format!("hub_frame_error: {}", e));
|
||||||
let mut writers = client_writers.lock().await;
|
}
|
||||||
writers.remove(&frame.stream_id);
|
None => break,
|
||||||
}
|
};
|
||||||
FRAME_CONFIG => {
|
last_activity = Instant::now();
|
||||||
if let Ok(update) = serde_json::from_slice::<ConfigUpdate>(&frame.payload) {
|
liveness_deadline.as_mut().reset(last_activity + liveness_timeout_dur);
|
||||||
log::info!("Config update from hub: ports {:?}", update.listen_ports);
|
if let EdgeFrameAction::Disconnect(reason) = handle_edge_frame(
|
||||||
*listen_ports.write().await = update.listen_ports.clone();
|
frame, &mut tunnel_io, &client_writers, listen_ports, event_tx,
|
||||||
let _ = event_tx.try_send(EdgeEvent::PortsUpdated {
|
&tunnel_writer_tx, &tunnel_data_tx, &mut port_listeners,
|
||||||
listen_ports: update.listen_ports.clone(),
|
active_streams, next_stream_id, &config.edge_id, connection_token, bind_address,
|
||||||
});
|
).await {
|
||||||
apply_port_config(
|
break 'io_loop EdgeLoopResult::Reconnect(reason);
|
||||||
&update.listen_ports,
|
}
|
||||||
&mut port_listeners,
|
}
|
||||||
&tunnel_writer,
|
|
||||||
&client_writers,
|
// Poll I/O: write(ctrl→data), flush, read, channels, timers
|
||||||
active_streams,
|
let event = std::future::poll_fn(|cx| {
|
||||||
next_stream_id,
|
tunnel_io.poll_step(cx, &mut tunnel_ctrl_rx, &mut tunnel_data_rx, &mut liveness_deadline, connection_token)
|
||||||
&config.edge_id,
|
}).await;
|
||||||
connection_token,
|
|
||||||
);
|
match event {
|
||||||
}
|
remoteingress_protocol::TunnelEvent::Frame(frame) => {
|
||||||
}
|
last_activity = Instant::now();
|
||||||
_ => {
|
liveness_deadline.as_mut().reset(last_activity + liveness_timeout_dur);
|
||||||
log::warn!("Unexpected frame type {} from hub", frame.frame_type);
|
if let EdgeFrameAction::Disconnect(reason) = handle_edge_frame(
|
||||||
}
|
frame, &mut tunnel_io, &client_writers, listen_ports, event_tx,
|
||||||
}
|
&tunnel_writer_tx, &tunnel_data_tx, &mut port_listeners,
|
||||||
}
|
active_streams, next_stream_id, &config.edge_id, connection_token, bind_address,
|
||||||
Ok(None) => {
|
).await {
|
||||||
log::info!("Hub disconnected (EOF)");
|
break EdgeLoopResult::Reconnect(reason);
|
||||||
break EdgeLoopResult::Reconnect;
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
log::error!("Hub frame error: {}", e);
|
|
||||||
break EdgeLoopResult::Reconnect;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
_ = connection_token.cancelled() => {
|
remoteingress_protocol::TunnelEvent::Eof => {
|
||||||
log::info!("Connection cancelled");
|
log::info!("Hub disconnected (EOF)");
|
||||||
break EdgeLoopResult::Shutdown;
|
break EdgeLoopResult::Reconnect("hub_eof".to_string());
|
||||||
}
|
}
|
||||||
_ = shutdown_rx.recv() => {
|
remoteingress_protocol::TunnelEvent::ReadError(e) => {
|
||||||
|
log::error!("Hub frame read error: {}", e);
|
||||||
|
break EdgeLoopResult::Reconnect(format!("hub_frame_error: {}", e));
|
||||||
|
}
|
||||||
|
remoteingress_protocol::TunnelEvent::WriteError(e) => {
|
||||||
|
log::error!("Tunnel write error: {}", e);
|
||||||
|
break EdgeLoopResult::Reconnect(format!("tunnel_write_error: {}", e));
|
||||||
|
}
|
||||||
|
remoteingress_protocol::TunnelEvent::LivenessTimeout => {
|
||||||
|
log::warn!("Hub liveness timeout (no frames for {}s), reconnecting", liveness_timeout_dur.as_secs());
|
||||||
|
break EdgeLoopResult::Reconnect("liveness_timeout".to_string());
|
||||||
|
}
|
||||||
|
remoteingress_protocol::TunnelEvent::Cancelled => {
|
||||||
|
if shutdown_rx.try_recv().is_ok() {
|
||||||
|
break EdgeLoopResult::Shutdown;
|
||||||
|
}
|
||||||
break EdgeLoopResult::Shutdown;
|
break EdgeLoopResult::Shutdown;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
// Cancel connection token to propagate to all child tasks BEFORE aborting
|
// Cleanup
|
||||||
connection_token.cancel();
|
connection_token.cancel();
|
||||||
stun_handle.abort();
|
stun_handle.abort();
|
||||||
for (_, h) in port_listeners.drain() {
|
for (_, h) in port_listeners.drain() {
|
||||||
@@ -438,12 +601,14 @@ async fn connect_to_hub_and_run(
|
|||||||
fn apply_port_config(
|
fn apply_port_config(
|
||||||
new_ports: &[u16],
|
new_ports: &[u16],
|
||||||
port_listeners: &mut HashMap<u16, JoinHandle<()>>,
|
port_listeners: &mut HashMap<u16, JoinHandle<()>>,
|
||||||
tunnel_writer: &Arc<Mutex<tokio::io::WriteHalf<tokio_rustls::client::TlsStream<TcpStream>>>>,
|
tunnel_ctrl_tx: &mpsc::Sender<Vec<u8>>,
|
||||||
client_writers: &Arc<Mutex<HashMap<u32, mpsc::Sender<Vec<u8>>>>>,
|
tunnel_data_tx: &mpsc::Sender<Vec<u8>>,
|
||||||
|
client_writers: &Arc<Mutex<HashMap<u32, EdgeStreamState>>>,
|
||||||
active_streams: &Arc<AtomicU32>,
|
active_streams: &Arc<AtomicU32>,
|
||||||
next_stream_id: &Arc<AtomicU32>,
|
next_stream_id: &Arc<AtomicU32>,
|
||||||
edge_id: &str,
|
edge_id: &str,
|
||||||
connection_token: &CancellationToken,
|
connection_token: &CancellationToken,
|
||||||
|
bind_address: &str,
|
||||||
) {
|
) {
|
||||||
let new_set: std::collections::HashSet<u16> = new_ports.iter().copied().collect();
|
let new_set: std::collections::HashSet<u16> = new_ports.iter().copied().collect();
|
||||||
let old_set: std::collections::HashSet<u16> = port_listeners.keys().copied().collect();
|
let old_set: std::collections::HashSet<u16> = port_listeners.keys().copied().collect();
|
||||||
@@ -458,15 +623,17 @@ fn apply_port_config(
|
|||||||
|
|
||||||
// Add new ports
|
// Add new ports
|
||||||
for &port in new_set.difference(&old_set) {
|
for &port in new_set.difference(&old_set) {
|
||||||
let tunnel_writer = tunnel_writer.clone();
|
let tunnel_ctrl_tx = tunnel_ctrl_tx.clone();
|
||||||
|
let tunnel_data_tx = tunnel_data_tx.clone();
|
||||||
let client_writers = client_writers.clone();
|
let client_writers = client_writers.clone();
|
||||||
let active_streams = active_streams.clone();
|
let active_streams = active_streams.clone();
|
||||||
let next_stream_id = next_stream_id.clone();
|
let next_stream_id = next_stream_id.clone();
|
||||||
let edge_id = edge_id.to_string();
|
let edge_id = edge_id.to_string();
|
||||||
let port_token = connection_token.child_token();
|
let port_token = connection_token.child_token();
|
||||||
|
|
||||||
|
let bind_addr = bind_address.to_string();
|
||||||
let handle = tokio::spawn(async move {
|
let handle = tokio::spawn(async move {
|
||||||
let listener = match TcpListener::bind(("0.0.0.0", port)).await {
|
let listener = match TcpListener::bind((bind_addr.as_str(), port)).await {
|
||||||
Ok(l) => l,
|
Ok(l) => l,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
log::error!("Failed to bind port {}: {}", port, e);
|
log::error!("Failed to bind port {}: {}", port, e);
|
||||||
@@ -480,8 +647,18 @@ fn apply_port_config(
|
|||||||
accept_result = listener.accept() => {
|
accept_result = listener.accept() => {
|
||||||
match accept_result {
|
match accept_result {
|
||||||
Ok((client_stream, client_addr)) => {
|
Ok((client_stream, client_addr)) => {
|
||||||
|
// TCP keepalive detects dead clients that disappear without FIN.
|
||||||
|
// Without this, zombie streams accumulate and never get cleaned up.
|
||||||
|
let _ = client_stream.set_nodelay(true);
|
||||||
|
let ka = socket2::TcpKeepalive::new()
|
||||||
|
.with_time(Duration::from_secs(60));
|
||||||
|
#[cfg(target_os = "linux")]
|
||||||
|
let ka = ka.with_interval(Duration::from_secs(60));
|
||||||
|
let _ = socket2::SockRef::from(&client_stream).set_tcp_keepalive(&ka);
|
||||||
|
|
||||||
let stream_id = next_stream_id.fetch_add(1, Ordering::Relaxed);
|
let stream_id = next_stream_id.fetch_add(1, Ordering::Relaxed);
|
||||||
let tunnel_writer = tunnel_writer.clone();
|
let tunnel_ctrl_tx = tunnel_ctrl_tx.clone();
|
||||||
|
let tunnel_data_tx = tunnel_data_tx.clone();
|
||||||
let client_writers = client_writers.clone();
|
let client_writers = client_writers.clone();
|
||||||
let active_streams = active_streams.clone();
|
let active_streams = active_streams.clone();
|
||||||
let edge_id = edge_id.clone();
|
let edge_id = edge_id.clone();
|
||||||
@@ -496,12 +673,25 @@ fn apply_port_config(
|
|||||||
stream_id,
|
stream_id,
|
||||||
port,
|
port,
|
||||||
&edge_id,
|
&edge_id,
|
||||||
tunnel_writer,
|
tunnel_ctrl_tx,
|
||||||
|
tunnel_data_tx,
|
||||||
client_writers,
|
client_writers,
|
||||||
client_token,
|
client_token,
|
||||||
|
Arc::clone(&active_streams),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
active_streams.fetch_sub(1, Ordering::Relaxed);
|
// Saturating decrement: prevent underflow when
|
||||||
|
// edge_main_loop's store(0) races with task cleanup.
|
||||||
|
loop {
|
||||||
|
let current = active_streams.load(Ordering::Relaxed);
|
||||||
|
if current == 0 { break; }
|
||||||
|
if active_streams.compare_exchange_weak(
|
||||||
|
current, current - 1,
|
||||||
|
Ordering::Relaxed, Ordering::Relaxed,
|
||||||
|
).is_ok() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
@@ -526,9 +716,11 @@ async fn handle_client_connection(
|
|||||||
stream_id: u32,
|
stream_id: u32,
|
||||||
dest_port: u16,
|
dest_port: u16,
|
||||||
edge_id: &str,
|
edge_id: &str,
|
||||||
tunnel_writer: Arc<Mutex<tokio::io::WriteHalf<tokio_rustls::client::TlsStream<TcpStream>>>>,
|
tunnel_ctrl_tx: mpsc::Sender<Vec<u8>>,
|
||||||
client_writers: Arc<Mutex<HashMap<u32, mpsc::Sender<Vec<u8>>>>>,
|
tunnel_data_tx: mpsc::Sender<Vec<u8>>,
|
||||||
|
client_writers: Arc<Mutex<HashMap<u32, EdgeStreamState>>>,
|
||||||
client_token: CancellationToken,
|
client_token: CancellationToken,
|
||||||
|
active_streams: Arc<AtomicU32>,
|
||||||
) {
|
) {
|
||||||
let client_ip = client_addr.ip().to_string();
|
let client_ip = client_addr.ip().to_string();
|
||||||
let client_port = client_addr.port();
|
let client_port = client_addr.port();
|
||||||
@@ -536,36 +728,74 @@ async fn handle_client_connection(
|
|||||||
// Determine edge IP (use 0.0.0.0 as placeholder — hub doesn't use it for routing)
|
// Determine edge IP (use 0.0.0.0 as placeholder — hub doesn't use it for routing)
|
||||||
let edge_ip = "0.0.0.0";
|
let edge_ip = "0.0.0.0";
|
||||||
|
|
||||||
// Send OPEN frame with PROXY v1 header
|
// Send OPEN frame with PROXY v1 header via control channel
|
||||||
let proxy_header = build_proxy_v1_header(&client_ip, edge_ip, client_port, dest_port);
|
let proxy_header = build_proxy_v1_header(&client_ip, edge_ip, client_port, dest_port);
|
||||||
let open_frame = encode_frame(stream_id, FRAME_OPEN, proxy_header.as_bytes());
|
let open_frame = encode_frame(stream_id, FRAME_OPEN, proxy_header.as_bytes());
|
||||||
{
|
if tunnel_ctrl_tx.send(open_frame).await.is_err() {
|
||||||
let mut w = tunnel_writer.lock().await;
|
return;
|
||||||
if w.write_all(&open_frame).await.is_err() {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Set up channel for data coming back from hub
|
// Per-stream unbounded back-channel. Flow control (WINDOW_UPDATE) limits
|
||||||
let (back_tx, mut back_rx) = mpsc::channel::<Vec<u8>>(256);
|
// bytes-in-flight, so this won't grow unbounded. Unbounded avoids killing
|
||||||
|
// streams due to channel overflow — backpressure slows streams, never kills them.
|
||||||
|
let (back_tx, mut back_rx) = mpsc::unbounded_channel::<Vec<u8>>();
|
||||||
|
// Adaptive initial window: scale with current stream count to keep total in-flight
|
||||||
|
// data within the 32MB budget. Prevents burst flooding when many streams open.
|
||||||
|
let initial_window = remoteingress_protocol::compute_window_for_stream_count(
|
||||||
|
active_streams.load(Ordering::Relaxed),
|
||||||
|
);
|
||||||
|
let send_window = Arc::new(AtomicU32::new(initial_window));
|
||||||
|
let window_notify = Arc::new(Notify::new());
|
||||||
{
|
{
|
||||||
let mut writers = client_writers.lock().await;
|
let mut writers = client_writers.lock().await;
|
||||||
writers.insert(stream_id, back_tx);
|
writers.insert(stream_id, EdgeStreamState {
|
||||||
|
back_tx,
|
||||||
|
send_window: Arc::clone(&send_window),
|
||||||
|
window_notify: Arc::clone(&window_notify),
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
let (mut client_read, mut client_write) = client_stream.into_split();
|
let (mut client_read, mut client_write) = client_stream.into_split();
|
||||||
|
|
||||||
// Task: hub -> client
|
// Task: hub -> client (download direction)
|
||||||
|
// After writing to client TCP, send WINDOW_UPDATE to hub so it can send more
|
||||||
let hub_to_client_token = client_token.clone();
|
let hub_to_client_token = client_token.clone();
|
||||||
let hub_to_client = tokio::spawn(async move {
|
let wu_tx = tunnel_ctrl_tx.clone();
|
||||||
|
let active_streams_h2c = Arc::clone(&active_streams);
|
||||||
|
let mut hub_to_client = tokio::spawn(async move {
|
||||||
|
let mut consumed_since_update: u32 = 0;
|
||||||
loop {
|
loop {
|
||||||
tokio::select! {
|
tokio::select! {
|
||||||
data = back_rx.recv() => {
|
data = back_rx.recv() => {
|
||||||
match data {
|
match data {
|
||||||
Some(data) => {
|
Some(data) => {
|
||||||
|
let len = data.len() as u32;
|
||||||
if client_write.write_all(&data).await.is_err() {
|
if client_write.write_all(&data).await.is_err() {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
// Track consumption for adaptive flow control.
|
||||||
|
// The increment is capped to the adaptive window so the sender's
|
||||||
|
// effective window shrinks to match current demand (fewer streams
|
||||||
|
// = larger window, more streams = smaller window per stream).
|
||||||
|
consumed_since_update += len;
|
||||||
|
let adaptive_window = remoteingress_protocol::compute_window_for_stream_count(
|
||||||
|
active_streams_h2c.load(Ordering::Relaxed),
|
||||||
|
);
|
||||||
|
let threshold = adaptive_window / 2;
|
||||||
|
if consumed_since_update >= threshold {
|
||||||
|
let increment = consumed_since_update.min(adaptive_window);
|
||||||
|
let frame = encode_window_update(stream_id, FRAME_WINDOW_UPDATE, increment);
|
||||||
|
// Use send().await for guaranteed delivery — dropping WINDOW_UPDATEs
|
||||||
|
// causes permanent flow stalls. Safe: runs in per-stream task, not main loop.
|
||||||
|
tokio::select! {
|
||||||
|
result = wu_tx.send(frame) => {
|
||||||
|
if result.is_ok() {
|
||||||
|
consumed_since_update -= increment;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ = hub_to_client_token.cancelled() => break,
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
None => break,
|
None => break,
|
||||||
}
|
}
|
||||||
@@ -573,20 +803,63 @@ async fn handle_client_connection(
|
|||||||
_ = hub_to_client_token.cancelled() => break,
|
_ = hub_to_client_token.cancelled() => break,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// Send final window update for any remaining consumed bytes
|
||||||
|
if consumed_since_update > 0 {
|
||||||
|
let frame = encode_window_update(stream_id, FRAME_WINDOW_UPDATE, consumed_since_update);
|
||||||
|
let _ = wu_tx.send(frame).await;
|
||||||
|
}
|
||||||
let _ = client_write.shutdown().await;
|
let _ = client_write.shutdown().await;
|
||||||
});
|
});
|
||||||
|
|
||||||
// Task: client -> hub
|
// Task: client -> hub (upload direction) with per-stream flow control.
|
||||||
let mut buf = vec![0u8; 32768];
|
// Zero-copy: read payload directly after the header, then prepend header.
|
||||||
|
let mut buf = vec![0u8; FRAME_HEADER_SIZE + 32768];
|
||||||
loop {
|
loop {
|
||||||
|
// Wait for send window to have capacity (with stall timeout).
|
||||||
|
// Safe pattern: register notified BEFORE checking the condition
|
||||||
|
// to avoid missing a notify_one that fires between load and select.
|
||||||
|
loop {
|
||||||
|
let notified = window_notify.notified();
|
||||||
|
tokio::pin!(notified);
|
||||||
|
notified.as_mut().enable();
|
||||||
|
let w = send_window.load(Ordering::Acquire);
|
||||||
|
if w > 0 { break; }
|
||||||
|
tokio::select! {
|
||||||
|
_ = notified => continue,
|
||||||
|
_ = client_token.cancelled() => break,
|
||||||
|
_ = tokio::time::sleep(Duration::from_secs(120)) => {
|
||||||
|
log::warn!("Stream {} upload stalled (window empty for 120s)", stream_id);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if client_token.is_cancelled() { break; }
|
||||||
|
|
||||||
|
// Limit read size to available window.
|
||||||
|
// IMPORTANT: if window is 0 (stall timeout fired), we must NOT
|
||||||
|
// read into an empty buffer — read(&mut buf[..0]) returns Ok(0)
|
||||||
|
// which would be falsely interpreted as EOF.
|
||||||
|
let w = send_window.load(Ordering::Acquire) as usize;
|
||||||
|
if w == 0 {
|
||||||
|
log::warn!("Stream {} upload: window still 0 after stall timeout, closing", stream_id);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
// Adaptive: cap read to current per-stream target window
|
||||||
|
let adaptive_cap = remoteingress_protocol::compute_window_for_stream_count(
|
||||||
|
active_streams.load(Ordering::Relaxed),
|
||||||
|
) as usize;
|
||||||
|
let max_read = w.min(32768).min(adaptive_cap);
|
||||||
|
|
||||||
tokio::select! {
|
tokio::select! {
|
||||||
read_result = client_read.read(&mut buf) => {
|
read_result = client_read.read(&mut buf[FRAME_HEADER_SIZE..FRAME_HEADER_SIZE + max_read]) => {
|
||||||
match read_result {
|
match read_result {
|
||||||
Ok(0) => break,
|
Ok(0) => break,
|
||||||
Ok(n) => {
|
Ok(n) => {
|
||||||
let data_frame = encode_frame(stream_id, FRAME_DATA, &buf[..n]);
|
send_window.fetch_sub(n as u32, Ordering::Release);
|
||||||
let mut w = tunnel_writer.lock().await;
|
encode_frame_header(&mut buf, stream_id, FRAME_DATA, n);
|
||||||
if w.write_all(&data_frame).await.is_err() {
|
let data_frame = buf[..FRAME_HEADER_SIZE + n].to_vec();
|
||||||
|
if tunnel_data_tx.send(data_frame).await.is_err() {
|
||||||
|
log::warn!("Stream {} data channel closed, closing", stream_id);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -597,19 +870,29 @@ async fn handle_client_connection(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Send CLOSE frame (only if not cancelled)
|
// Wait for the download task (hub → client) to finish BEFORE sending CLOSE.
|
||||||
|
// Upload EOF (client done sending) does NOT mean the response is done.
|
||||||
|
// For asymmetric transfers like git fetch (small request, large response),
|
||||||
|
// the response is still streaming when the upload finishes.
|
||||||
|
// Sending CLOSE before the response finishes would cause the hub to cancel
|
||||||
|
// the upstream reader mid-response, truncating the data.
|
||||||
|
let _ = tokio::time::timeout(
|
||||||
|
Duration::from_secs(300), // 5 min max wait for download to finish
|
||||||
|
&mut hub_to_client,
|
||||||
|
).await;
|
||||||
|
|
||||||
|
// NOW send CLOSE — the response has been fully delivered (or timed out).
|
||||||
if !client_token.is_cancelled() {
|
if !client_token.is_cancelled() {
|
||||||
let close_frame = encode_frame(stream_id, FRAME_CLOSE, &[]);
|
let close_frame = encode_frame(stream_id, FRAME_CLOSE, &[]);
|
||||||
let mut w = tunnel_writer.lock().await;
|
let _ = tunnel_data_tx.send(close_frame).await;
|
||||||
let _ = w.write_all(&close_frame).await;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Cleanup
|
// Clean up
|
||||||
{
|
{
|
||||||
let mut writers = client_writers.lock().await;
|
let mut writers = client_writers.lock().await;
|
||||||
writers.remove(&stream_id);
|
writers.remove(&stream_id);
|
||||||
}
|
}
|
||||||
hub_to_client.abort();
|
hub_to_client.abort(); // No-op if already finished; safety net if timeout fired
|
||||||
let _ = edge_id; // used for logging context
|
let _ = edge_id; // used for logging context
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -641,6 +924,7 @@ mod tests {
|
|||||||
hub_port: 9999,
|
hub_port: 9999,
|
||||||
edge_id: "e1".to_string(),
|
edge_id: "e1".to_string(),
|
||||||
secret: "sec".to_string(),
|
secret: "sec".to_string(),
|
||||||
|
bind_address: None,
|
||||||
};
|
};
|
||||||
let json = serde_json::to_string(&config).unwrap();
|
let json = serde_json::to_string(&config).unwrap();
|
||||||
let back: EdgeConfig = serde_json::from_str(&json).unwrap();
|
let back: EdgeConfig = serde_json::from_str(&json).unwrap();
|
||||||
@@ -712,9 +996,10 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_edge_event_tunnel_disconnected() {
|
fn test_edge_event_tunnel_disconnected() {
|
||||||
let event = EdgeEvent::TunnelDisconnected;
|
let event = EdgeEvent::TunnelDisconnected { reason: "hub_eof".to_string() };
|
||||||
let json = serde_json::to_value(&event).unwrap();
|
let json = serde_json::to_value(&event).unwrap();
|
||||||
assert_eq!(json["type"], "tunnelDisconnected");
|
assert_eq!(json["type"], "tunnelDisconnected");
|
||||||
|
assert_eq!(json["reason"], "hub_eof");
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -756,6 +1041,7 @@ mod tests {
|
|||||||
hub_port: 8443,
|
hub_port: 8443,
|
||||||
edge_id: "test-edge".to_string(),
|
edge_id: "test-edge".to_string(),
|
||||||
secret: "test-secret".to_string(),
|
secret: "test-secret".to_string(),
|
||||||
|
bind_address: None,
|
||||||
});
|
});
|
||||||
let status = edge.get_status().await;
|
let status = edge.get_status().await;
|
||||||
assert!(!status.running);
|
assert!(!status.running);
|
||||||
@@ -772,6 +1058,7 @@ mod tests {
|
|||||||
hub_port: 8443,
|
hub_port: 8443,
|
||||||
edge_id: "e".to_string(),
|
edge_id: "e".to_string(),
|
||||||
secret: "s".to_string(),
|
secret: "s".to_string(),
|
||||||
|
bind_address: None,
|
||||||
});
|
});
|
||||||
let rx1 = edge.take_event_rx().await;
|
let rx1 = edge.take_event_rx().await;
|
||||||
assert!(rx1.is_some());
|
assert!(rx1.is_some());
|
||||||
@@ -786,6 +1073,7 @@ mod tests {
|
|||||||
hub_port: 8443,
|
hub_port: 8443,
|
||||||
edge_id: "e".to_string(),
|
edge_id: "e".to_string(),
|
||||||
secret: "s".to_string(),
|
secret: "s".to_string(),
|
||||||
|
bind_address: None,
|
||||||
});
|
});
|
||||||
edge.stop().await; // should not panic
|
edge.stop().await; // should not panic
|
||||||
let status = edge.get_status().await;
|
let status = edge.get_status().await;
|
||||||
|
|||||||
@@ -1,23 +1,50 @@
|
|||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader};
|
use std::sync::atomic::{AtomicU32, Ordering};
|
||||||
|
use std::time::Duration;
|
||||||
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||||
use tokio::net::{TcpListener, TcpStream};
|
use tokio::net::{TcpListener, TcpStream};
|
||||||
use tokio::sync::{mpsc, Mutex, RwLock};
|
use tokio::sync::{mpsc, Mutex, Notify, RwLock, Semaphore};
|
||||||
|
use tokio::time::{interval, sleep_until, Instant};
|
||||||
use tokio_rustls::TlsAcceptor;
|
use tokio_rustls::TlsAcceptor;
|
||||||
use tokio_util::sync::CancellationToken;
|
use tokio_util::sync::CancellationToken;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
use remoteingress_protocol::*;
|
use remoteingress_protocol::*;
|
||||||
|
|
||||||
|
type HubTlsStream = tokio_rustls::server::TlsStream<TcpStream>;
|
||||||
|
|
||||||
|
/// Result of processing a frame.
|
||||||
|
#[allow(dead_code)]
|
||||||
|
enum FrameAction {
|
||||||
|
Continue,
|
||||||
|
Disconnect(String),
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Per-stream state tracked in the hub's stream map.
|
||||||
|
struct HubStreamState {
|
||||||
|
/// Unbounded channel to deliver FRAME_DATA payloads to the upstream writer task.
|
||||||
|
/// Unbounded because flow control (WINDOW_UPDATE) already limits bytes-in-flight.
|
||||||
|
/// A bounded channel would kill streams instead of applying backpressure.
|
||||||
|
data_tx: mpsc::UnboundedSender<Vec<u8>>,
|
||||||
|
/// Cancellation token for this stream.
|
||||||
|
cancel_token: CancellationToken,
|
||||||
|
/// Send window for FRAME_DATA_BACK (download direction).
|
||||||
|
/// Decremented by the upstream reader, incremented by FRAME_WINDOW_UPDATE from edge.
|
||||||
|
send_window: Arc<AtomicU32>,
|
||||||
|
/// Notifier to wake the upstream reader when the window opens.
|
||||||
|
window_notify: Arc<Notify>,
|
||||||
|
}
|
||||||
|
|
||||||
/// Hub configuration.
|
/// Hub configuration.
|
||||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||||
#[serde(rename_all = "camelCase")]
|
#[serde(rename_all = "camelCase")]
|
||||||
pub struct HubConfig {
|
pub struct HubConfig {
|
||||||
pub tunnel_port: u16,
|
pub tunnel_port: u16,
|
||||||
pub target_host: Option<String>,
|
pub target_host: Option<String>,
|
||||||
#[serde(skip)]
|
#[serde(default)]
|
||||||
pub tls_cert_pem: Option<String>,
|
pub tls_cert_pem: Option<String>,
|
||||||
#[serde(skip)]
|
#[serde(default)]
|
||||||
pub tls_key_pem: Option<String>,
|
pub tls_key_pem: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -65,6 +92,7 @@ pub struct ConnectedEdgeStatus {
|
|||||||
pub edge_id: String,
|
pub edge_id: String,
|
||||||
pub connected_at: u64,
|
pub connected_at: u64,
|
||||||
pub active_streams: usize,
|
pub active_streams: usize,
|
||||||
|
pub peer_addr: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Events emitted by the hub.
|
/// Events emitted by the hub.
|
||||||
@@ -73,9 +101,9 @@ pub struct ConnectedEdgeStatus {
|
|||||||
#[serde(tag = "type")]
|
#[serde(tag = "type")]
|
||||||
pub enum HubEvent {
|
pub enum HubEvent {
|
||||||
#[serde(rename_all = "camelCase")]
|
#[serde(rename_all = "camelCase")]
|
||||||
EdgeConnected { edge_id: String },
|
EdgeConnected { edge_id: String, peer_addr: String },
|
||||||
#[serde(rename_all = "camelCase")]
|
#[serde(rename_all = "camelCase")]
|
||||||
EdgeDisconnected { edge_id: String },
|
EdgeDisconnected { edge_id: String, reason: String },
|
||||||
#[serde(rename_all = "camelCase")]
|
#[serde(rename_all = "camelCase")]
|
||||||
StreamOpened { edge_id: String, stream_id: u32 },
|
StreamOpened { edge_id: String, stream_id: u32 },
|
||||||
#[serde(rename_all = "camelCase")]
|
#[serde(rename_all = "camelCase")]
|
||||||
@@ -105,7 +133,8 @@ pub struct TunnelHub {
|
|||||||
|
|
||||||
struct ConnectedEdgeInfo {
|
struct ConnectedEdgeInfo {
|
||||||
connected_at: u64,
|
connected_at: u64,
|
||||||
active_streams: Arc<Mutex<HashMap<u32, mpsc::Sender<Vec<u8>>>>>,
|
peer_addr: String,
|
||||||
|
edge_stream_count: Arc<AtomicU32>,
|
||||||
config_tx: mpsc::Sender<EdgeConfigUpdate>,
|
config_tx: mpsc::Sender<EdgeConfigUpdate>,
|
||||||
#[allow(dead_code)] // kept alive for Drop — cancels child tokens when edge is removed
|
#[allow(dead_code)] // kept alive for Drop — cancels child tokens when edge is removed
|
||||||
cancel_token: CancellationToken,
|
cancel_token: CancellationToken,
|
||||||
@@ -171,11 +200,11 @@ impl TunnelHub {
|
|||||||
|
|
||||||
let mut connected = Vec::new();
|
let mut connected = Vec::new();
|
||||||
for (id, info) in edges.iter() {
|
for (id, info) in edges.iter() {
|
||||||
let streams = info.active_streams.lock().await;
|
|
||||||
connected.push(ConnectedEdgeStatus {
|
connected.push(ConnectedEdgeStatus {
|
||||||
edge_id: id.clone(),
|
edge_id: id.clone(),
|
||||||
connected_at: info.connected_at,
|
connected_at: info.connected_at,
|
||||||
active_streams: streams.len(),
|
active_streams: info.edge_stream_count.load(Ordering::Relaxed) as usize,
|
||||||
|
peer_addr: info.peer_addr.clone(),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -218,9 +247,10 @@ impl TunnelHub {
|
|||||||
let event_tx = event_tx.clone();
|
let event_tx = event_tx.clone();
|
||||||
let target = target_host.clone();
|
let target = target_host.clone();
|
||||||
let edge_token = hub_token.child_token();
|
let edge_token = hub_token.child_token();
|
||||||
|
let peer_addr = addr.ip().to_string();
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
if let Err(e) = handle_edge_connection(
|
if let Err(e) = handle_edge_connection(
|
||||||
stream, acceptor, allowed, connected, event_tx, target, edge_token,
|
stream, acceptor, allowed, connected, event_tx, target, edge_token, peer_addr,
|
||||||
).await {
|
).await {
|
||||||
log::error!("Edge connection error: {}", e);
|
log::error!("Edge connection error: {}", e);
|
||||||
}
|
}
|
||||||
@@ -264,6 +294,293 @@ impl Drop for TunnelHub {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Maximum concurrent streams per edge connection.
|
||||||
|
const MAX_STREAMS_PER_EDGE: usize = 1024;
|
||||||
|
|
||||||
|
/// Process a single frame received from the edge side of the tunnel.
|
||||||
|
/// Handles FRAME_OPEN, FRAME_DATA, FRAME_WINDOW_UPDATE, FRAME_CLOSE, and FRAME_PONG.
|
||||||
|
async fn handle_hub_frame(
|
||||||
|
frame: Frame,
|
||||||
|
tunnel_io: &mut remoteingress_protocol::TunnelIo<HubTlsStream>,
|
||||||
|
streams: &mut HashMap<u32, HubStreamState>,
|
||||||
|
stream_semaphore: &Arc<Semaphore>,
|
||||||
|
edge_stream_count: &Arc<AtomicU32>,
|
||||||
|
edge_id: &str,
|
||||||
|
event_tx: &mpsc::Sender<HubEvent>,
|
||||||
|
ctrl_tx: &mpsc::Sender<Vec<u8>>,
|
||||||
|
data_tx: &mpsc::Sender<Vec<u8>>,
|
||||||
|
target_host: &str,
|
||||||
|
edge_token: &CancellationToken,
|
||||||
|
cleanup_tx: &mpsc::Sender<u32>,
|
||||||
|
) -> FrameAction {
|
||||||
|
match frame.frame_type {
|
||||||
|
FRAME_OPEN => {
|
||||||
|
// A4: Check stream limit before processing
|
||||||
|
let permit = match stream_semaphore.clone().try_acquire_owned() {
|
||||||
|
Ok(p) => p,
|
||||||
|
Err(_) => {
|
||||||
|
log::warn!("Edge {} exceeded max streams ({}), rejecting stream {}",
|
||||||
|
edge_id, MAX_STREAMS_PER_EDGE, frame.stream_id);
|
||||||
|
let close_frame = encode_frame(frame.stream_id, FRAME_CLOSE_BACK, &[]);
|
||||||
|
tunnel_io.queue_ctrl(close_frame);
|
||||||
|
return FrameAction::Continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// Payload is PROXY v1 header line
|
||||||
|
let proxy_header = String::from_utf8_lossy(&frame.payload).to_string();
|
||||||
|
|
||||||
|
// Parse destination port from PROXY header
|
||||||
|
let dest_port = parse_dest_port_from_proxy(&proxy_header).unwrap_or(443);
|
||||||
|
|
||||||
|
let stream_id = frame.stream_id;
|
||||||
|
let cleanup = cleanup_tx.clone();
|
||||||
|
let writer_tx = ctrl_tx.clone(); // control: CLOSE_BACK, WINDOW_UPDATE_BACK
|
||||||
|
let data_writer_tx = data_tx.clone(); // data: DATA_BACK
|
||||||
|
let target = target_host.to_string();
|
||||||
|
let stream_token = edge_token.child_token();
|
||||||
|
|
||||||
|
let _ = event_tx.try_send(HubEvent::StreamOpened {
|
||||||
|
edge_id: edge_id.to_string(),
|
||||||
|
stream_id,
|
||||||
|
});
|
||||||
|
|
||||||
|
// Create channel for data from edge to this stream
|
||||||
|
let (stream_data_tx, mut stream_data_rx) = mpsc::unbounded_channel::<Vec<u8>>();
|
||||||
|
// Adaptive initial window: scale with current stream count
|
||||||
|
// to keep total in-flight data within the 32MB budget.
|
||||||
|
let initial_window = compute_window_for_stream_count(
|
||||||
|
edge_stream_count.load(Ordering::Relaxed),
|
||||||
|
);
|
||||||
|
let send_window = Arc::new(AtomicU32::new(initial_window));
|
||||||
|
let window_notify = Arc::new(Notify::new());
|
||||||
|
streams.insert(stream_id, HubStreamState {
|
||||||
|
data_tx: stream_data_tx,
|
||||||
|
cancel_token: stream_token.clone(),
|
||||||
|
send_window: Arc::clone(&send_window),
|
||||||
|
window_notify: Arc::clone(&window_notify),
|
||||||
|
});
|
||||||
|
|
||||||
|
// Spawn task: connect to SmartProxy, send PROXY header, pipe data
|
||||||
|
let stream_counter = Arc::clone(edge_stream_count);
|
||||||
|
tokio::spawn(async move {
|
||||||
|
let _permit = permit; // hold semaphore permit until stream completes
|
||||||
|
stream_counter.fetch_add(1, Ordering::Relaxed);
|
||||||
|
|
||||||
|
let result = async {
|
||||||
|
// A2: Connect to SmartProxy with timeout
|
||||||
|
let mut upstream = tokio::time::timeout(
|
||||||
|
Duration::from_secs(10),
|
||||||
|
TcpStream::connect((target.as_str(), dest_port)),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|_| -> Box<dyn std::error::Error + Send + Sync> {
|
||||||
|
format!("connect to SmartProxy {}:{} timed out (10s)", target, dest_port).into()
|
||||||
|
})??;
|
||||||
|
|
||||||
|
upstream.set_nodelay(true)?;
|
||||||
|
upstream.write_all(proxy_header.as_bytes()).await?;
|
||||||
|
|
||||||
|
let (mut up_read, mut up_write) =
|
||||||
|
upstream.into_split();
|
||||||
|
|
||||||
|
// Forward data from edge (via channel) to SmartProxy
|
||||||
|
// After writing to upstream, send WINDOW_UPDATE_BACK to edge
|
||||||
|
let writer_token = stream_token.clone();
|
||||||
|
let wub_tx = writer_tx.clone();
|
||||||
|
let stream_counter_w = Arc::clone(&stream_counter);
|
||||||
|
let writer_for_edge_data = tokio::spawn(async move {
|
||||||
|
let mut consumed_since_update: u32 = 0;
|
||||||
|
loop {
|
||||||
|
tokio::select! {
|
||||||
|
data = stream_data_rx.recv() => {
|
||||||
|
match data {
|
||||||
|
Some(data) => {
|
||||||
|
let len = data.len() as u32;
|
||||||
|
// Check cancellation alongside the write so we respond
|
||||||
|
// promptly to FRAME_CLOSE instead of blocking up to 60s.
|
||||||
|
let write_result = tokio::select! {
|
||||||
|
r = tokio::time::timeout(
|
||||||
|
Duration::from_secs(60),
|
||||||
|
up_write.write_all(&data),
|
||||||
|
) => r,
|
||||||
|
_ = writer_token.cancelled() => break,
|
||||||
|
};
|
||||||
|
match write_result {
|
||||||
|
Ok(Ok(())) => {}
|
||||||
|
Ok(Err(_)) => break,
|
||||||
|
Err(_) => {
|
||||||
|
log::warn!("Stream {} write to upstream timed out (60s)", stream_id);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Track consumption for adaptive flow control.
|
||||||
|
// Increment capped to adaptive window to limit per-stream in-flight data.
|
||||||
|
consumed_since_update += len;
|
||||||
|
let adaptive_window = remoteingress_protocol::compute_window_for_stream_count(
|
||||||
|
stream_counter_w.load(Ordering::Relaxed),
|
||||||
|
);
|
||||||
|
let threshold = adaptive_window / 2;
|
||||||
|
if consumed_since_update >= threshold {
|
||||||
|
let increment = consumed_since_update.min(adaptive_window);
|
||||||
|
let frame = encode_window_update(stream_id, FRAME_WINDOW_UPDATE_BACK, increment);
|
||||||
|
// Use send().await for guaranteed delivery — dropping WINDOW_UPDATEs
|
||||||
|
// causes permanent flow stalls. Safe: runs in per-stream task, not main loop.
|
||||||
|
tokio::select! {
|
||||||
|
result = wub_tx.send(frame) => {
|
||||||
|
if result.is_ok() {
|
||||||
|
consumed_since_update -= increment;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ = writer_token.cancelled() => break,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None => break,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ = writer_token.cancelled() => break,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Send final window update for remaining consumed bytes
|
||||||
|
if consumed_since_update > 0 {
|
||||||
|
let frame = encode_window_update(stream_id, FRAME_WINDOW_UPDATE_BACK, consumed_since_update);
|
||||||
|
let _ = wub_tx.send(frame).await;
|
||||||
|
}
|
||||||
|
let _ = up_write.shutdown().await;
|
||||||
|
});
|
||||||
|
|
||||||
|
// Forward data from SmartProxy back to edge via writer channel
|
||||||
|
// with per-stream flow control (check send_window before reading).
|
||||||
|
// Zero-copy: read payload directly after the header, then prepend header.
|
||||||
|
let mut buf = vec![0u8; FRAME_HEADER_SIZE + 32768];
|
||||||
|
loop {
|
||||||
|
// Wait for send window to have capacity (with stall timeout).
|
||||||
|
// Safe pattern: register notified BEFORE checking the condition
|
||||||
|
// to avoid missing a notify_one that fires between load and select.
|
||||||
|
loop {
|
||||||
|
let notified = window_notify.notified();
|
||||||
|
tokio::pin!(notified);
|
||||||
|
notified.as_mut().enable();
|
||||||
|
let w = send_window.load(Ordering::Acquire);
|
||||||
|
if w > 0 { break; }
|
||||||
|
tokio::select! {
|
||||||
|
_ = notified => continue,
|
||||||
|
_ = stream_token.cancelled() => break,
|
||||||
|
_ = tokio::time::sleep(Duration::from_secs(120)) => {
|
||||||
|
log::warn!("Stream {} download stalled (window empty for 120s)", stream_id);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if stream_token.is_cancelled() { break; }
|
||||||
|
|
||||||
|
// Limit read size to available window.
|
||||||
|
// IMPORTANT: if window is 0 (stall timeout fired), we must NOT
|
||||||
|
// read into an empty buffer — read(&mut buf[..0]) returns Ok(0)
|
||||||
|
// which would be falsely interpreted as EOF.
|
||||||
|
let w = send_window.load(Ordering::Acquire) as usize;
|
||||||
|
if w == 0 {
|
||||||
|
log::warn!("Stream {} download: window still 0 after stall timeout, closing", stream_id);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
// Adaptive: cap read to current per-stream target window
|
||||||
|
let adaptive_cap = remoteingress_protocol::compute_window_for_stream_count(
|
||||||
|
stream_counter.load(Ordering::Relaxed),
|
||||||
|
) as usize;
|
||||||
|
let max_read = w.min(32768).min(adaptive_cap);
|
||||||
|
|
||||||
|
tokio::select! {
|
||||||
|
read_result = up_read.read(&mut buf[FRAME_HEADER_SIZE..FRAME_HEADER_SIZE + max_read]) => {
|
||||||
|
match read_result {
|
||||||
|
Ok(0) => break,
|
||||||
|
Ok(n) => {
|
||||||
|
send_window.fetch_sub(n as u32, Ordering::Release);
|
||||||
|
encode_frame_header(&mut buf, stream_id, FRAME_DATA_BACK, n);
|
||||||
|
let frame = buf[..FRAME_HEADER_SIZE + n].to_vec();
|
||||||
|
if data_writer_tx.send(frame).await.is_err() {
|
||||||
|
log::warn!("Stream {} data channel closed, closing", stream_id);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(_) => break,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ = stream_token.cancelled() => break,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Send CLOSE_BACK via DATA channel (must arrive AFTER last DATA_BACK).
|
||||||
|
// Use send().await to guarantee delivery (try_send silently drops if full).
|
||||||
|
if !stream_token.is_cancelled() {
|
||||||
|
let close_frame = encode_frame(stream_id, FRAME_CLOSE_BACK, &[]);
|
||||||
|
let _ = data_writer_tx.send(close_frame).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
writer_for_edge_data.abort();
|
||||||
|
Ok::<(), Box<dyn std::error::Error + Send + Sync>>(())
|
||||||
|
}
|
||||||
|
.await;
|
||||||
|
|
||||||
|
if let Err(e) = result {
|
||||||
|
log::error!("Stream {} error: {}", stream_id, e);
|
||||||
|
// Send CLOSE_BACK via DATA channel on error (must arrive after any DATA_BACK).
|
||||||
|
// Use send().await to guarantee delivery.
|
||||||
|
if !stream_token.is_cancelled() {
|
||||||
|
let close_frame = encode_frame(stream_id, FRAME_CLOSE_BACK, &[]);
|
||||||
|
let _ = data_writer_tx.send(close_frame).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Signal main loop to remove stream from the map
|
||||||
|
let _ = cleanup.send(stream_id).await;
|
||||||
|
stream_counter.fetch_sub(1, Ordering::Relaxed);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
FRAME_DATA => {
|
||||||
|
// Dispatch to per-stream unbounded channel. Flow control (WINDOW_UPDATE)
|
||||||
|
// limits bytes-in-flight, so the channel won't grow unbounded. send() only
|
||||||
|
// fails if the receiver is dropped (stream handler already exited).
|
||||||
|
if let Some(state) = streams.get(&frame.stream_id) {
|
||||||
|
if state.data_tx.send(frame.payload).is_err() {
|
||||||
|
// Receiver dropped — stream handler already exited, clean up
|
||||||
|
streams.remove(&frame.stream_id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
FRAME_WINDOW_UPDATE => {
|
||||||
|
// Edge consumed data — increase our send window for this stream
|
||||||
|
if let Some(increment) = decode_window_update(&frame.payload) {
|
||||||
|
if increment > 0 {
|
||||||
|
if let Some(state) = streams.get(&frame.stream_id) {
|
||||||
|
let prev = state.send_window.fetch_add(increment, Ordering::Release);
|
||||||
|
if prev + increment > MAX_WINDOW_SIZE {
|
||||||
|
state.send_window.store(MAX_WINDOW_SIZE, Ordering::Release);
|
||||||
|
}
|
||||||
|
state.window_notify.notify_one();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
FRAME_CLOSE => {
|
||||||
|
if let Some(state) = streams.remove(&frame.stream_id) {
|
||||||
|
state.cancel_token.cancel();
|
||||||
|
let _ = event_tx.try_send(HubEvent::StreamClosed {
|
||||||
|
edge_id: edge_id.to_string(),
|
||||||
|
stream_id: frame.stream_id,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
FRAME_PONG => {
|
||||||
|
log::debug!("Received PONG from edge {}", edge_id);
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
log::warn!("Unexpected frame type {} from edge", frame.frame_type);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
FrameAction::Continue
|
||||||
|
}
|
||||||
|
|
||||||
/// Handle a single edge connection: authenticate, then enter frame loop.
|
/// Handle a single edge connection: authenticate, then enter frame loop.
|
||||||
async fn handle_edge_connection(
|
async fn handle_edge_connection(
|
||||||
stream: TcpStream,
|
stream: TcpStream,
|
||||||
@@ -273,14 +590,35 @@ async fn handle_edge_connection(
|
|||||||
event_tx: mpsc::Sender<HubEvent>,
|
event_tx: mpsc::Sender<HubEvent>,
|
||||||
target_host: String,
|
target_host: String,
|
||||||
edge_token: CancellationToken,
|
edge_token: CancellationToken,
|
||||||
|
peer_addr: String,
|
||||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
let tls_stream = acceptor.accept(stream).await?;
|
// Disable Nagle's algorithm for low-latency control frames (PING/PONG, WINDOW_UPDATE)
|
||||||
let (read_half, mut write_half) = tokio::io::split(tls_stream);
|
stream.set_nodelay(true)?;
|
||||||
let mut buf_reader = BufReader::new(read_half);
|
// TCP keepalive detects silent network failures (NAT timeout, path change)
|
||||||
|
// faster than the 45s application-level liveness timeout.
|
||||||
|
let ka = socket2::TcpKeepalive::new()
|
||||||
|
.with_time(Duration::from_secs(30));
|
||||||
|
#[cfg(target_os = "linux")]
|
||||||
|
let ka = ka.with_interval(Duration::from_secs(10));
|
||||||
|
let _ = socket2::SockRef::from(&stream).set_tcp_keepalive(&ka);
|
||||||
|
let mut tls_stream = acceptor.accept(stream).await?;
|
||||||
|
|
||||||
// Read auth line: "EDGE <edgeId> <secret>\n"
|
// Byte-by-byte auth line reading (no BufReader).
|
||||||
let mut auth_line = String::new();
|
// Auth line: "EDGE <edgeId> <secret>\n"
|
||||||
buf_reader.read_line(&mut auth_line).await?;
|
let mut auth_buf = Vec::with_capacity(512);
|
||||||
|
loop {
|
||||||
|
let mut byte = [0u8; 1];
|
||||||
|
tls_stream.read_exact(&mut byte).await?;
|
||||||
|
if byte[0] == b'\n' {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
auth_buf.push(byte[0]);
|
||||||
|
if auth_buf.len() > 4096 {
|
||||||
|
return Err("auth line too long".into());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let auth_line = String::from_utf8(auth_buf)
|
||||||
|
.map_err(|_| "auth line not valid UTF-8")?;
|
||||||
let auth_line = auth_line.trim();
|
let auth_line = auth_line.trim();
|
||||||
|
|
||||||
let parts: Vec<&str> = auth_line.splitn(3, ' ').collect();
|
let parts: Vec<&str> = auth_line.splitn(3, ' ').collect();
|
||||||
@@ -307,9 +645,10 @@ async fn handle_edge_connection(
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
log::info!("Edge {} authenticated", edge_id);
|
log::info!("Edge {} authenticated from {}", edge_id, peer_addr);
|
||||||
let _ = event_tx.try_send(HubEvent::EdgeConnected {
|
let _ = event_tx.try_send(HubEvent::EdgeConnected {
|
||||||
edge_id: edge_id.clone(),
|
edge_id: edge_id.clone(),
|
||||||
|
peer_addr: peer_addr.clone(),
|
||||||
});
|
});
|
||||||
|
|
||||||
// Send handshake response with initial config before frame protocol begins
|
// Send handshake response with initial config before frame protocol begins
|
||||||
@@ -319,11 +658,15 @@ async fn handle_edge_connection(
|
|||||||
};
|
};
|
||||||
let mut handshake_json = serde_json::to_string(&handshake)?;
|
let mut handshake_json = serde_json::to_string(&handshake)?;
|
||||||
handshake_json.push('\n');
|
handshake_json.push('\n');
|
||||||
write_half.write_all(handshake_json.as_bytes()).await?;
|
tls_stream.write_all(handshake_json.as_bytes()).await?;
|
||||||
|
tls_stream.flush().await?;
|
||||||
|
|
||||||
// Track this edge
|
// Track this edge
|
||||||
let streams: Arc<Mutex<HashMap<u32, mpsc::Sender<Vec<u8>>>>> =
|
let mut streams: HashMap<u32, HubStreamState> = HashMap::new();
|
||||||
Arc::new(Mutex::new(HashMap::new()));
|
// Per-edge active stream counter for adaptive flow control
|
||||||
|
let edge_stream_count = Arc::new(AtomicU32::new(0));
|
||||||
|
// Cleanup channel: spawned stream tasks send stream_id here when done
|
||||||
|
let (cleanup_tx, mut cleanup_rx) = mpsc::channel::<u32>(256);
|
||||||
let now = std::time::SystemTime::now()
|
let now = std::time::SystemTime::now()
|
||||||
.duration_since(std::time::UNIX_EPOCH)
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
.unwrap_or_default()
|
.unwrap_or_default()
|
||||||
@@ -338,18 +681,21 @@ async fn handle_edge_connection(
|
|||||||
edge_id.clone(),
|
edge_id.clone(),
|
||||||
ConnectedEdgeInfo {
|
ConnectedEdgeInfo {
|
||||||
connected_at: now,
|
connected_at: now,
|
||||||
active_streams: streams.clone(),
|
peer_addr,
|
||||||
|
edge_stream_count: edge_stream_count.clone(),
|
||||||
config_tx,
|
config_tx,
|
||||||
cancel_token: edge_token.clone(),
|
cancel_token: edge_token.clone(),
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Shared writer for sending frames back to edge
|
// QoS dual-channel: ctrl frames have priority over data frames.
|
||||||
let write_half = Arc::new(Mutex::new(write_half));
|
// Stream handlers send through these channels -> TunnelIo drains them.
|
||||||
|
let (ctrl_tx, mut ctrl_rx) = mpsc::channel::<Vec<u8>>(256);
|
||||||
|
let (data_tx, mut data_rx) = mpsc::channel::<Vec<u8>>(4096);
|
||||||
|
|
||||||
// Spawn task to forward config updates as FRAME_CONFIG frames
|
// Spawn task to forward config updates as FRAME_CONFIG frames
|
||||||
let config_writer = write_half.clone();
|
let config_writer_tx = ctrl_tx.clone();
|
||||||
let config_edge_id = edge_id.clone();
|
let config_edge_id = edge_id.clone();
|
||||||
let config_token = edge_token.clone();
|
let config_token = edge_token.clone();
|
||||||
let config_handle = tokio::spawn(async move {
|
let config_handle = tokio::spawn(async move {
|
||||||
@@ -360,8 +706,7 @@ async fn handle_edge_connection(
|
|||||||
Some(update) => {
|
Some(update) => {
|
||||||
if let Ok(payload) = serde_json::to_vec(&update) {
|
if let Ok(payload) = serde_json::to_vec(&update) {
|
||||||
let frame = encode_frame(0, FRAME_CONFIG, &payload);
|
let frame = encode_frame(0, FRAME_CONFIG, &payload);
|
||||||
let mut w = config_writer.lock().await;
|
if config_writer_tx.send(frame).await.is_err() {
|
||||||
if w.write_all(&frame).await.is_err() {
|
|
||||||
log::error!("Failed to send config update to edge {}", config_edge_id);
|
log::error!("Failed to send config update to edge {}", config_edge_id);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@@ -376,155 +721,104 @@ async fn handle_edge_connection(
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
// Frame reading loop
|
// A4: Semaphore to limit concurrent streams per edge
|
||||||
let mut frame_reader = FrameReader::new(buf_reader);
|
let stream_semaphore = Arc::new(Semaphore::new(MAX_STREAMS_PER_EDGE));
|
||||||
|
|
||||||
loop {
|
// Heartbeat: periodic PING and liveness timeout
|
||||||
tokio::select! {
|
let ping_interval_dur = Duration::from_secs(15);
|
||||||
frame_result = frame_reader.next_frame() => {
|
let liveness_timeout_dur = Duration::from_secs(45);
|
||||||
match frame_result {
|
let mut ping_ticker = interval(ping_interval_dur);
|
||||||
Ok(Some(frame)) => {
|
ping_ticker.tick().await; // consume the immediate first tick
|
||||||
match frame.frame_type {
|
let mut last_activity = Instant::now();
|
||||||
FRAME_OPEN => {
|
let mut liveness_deadline = Box::pin(sleep_until(last_activity + liveness_timeout_dur));
|
||||||
// Payload is PROXY v1 header line
|
|
||||||
let proxy_header = String::from_utf8_lossy(&frame.payload).to_string();
|
|
||||||
|
|
||||||
// Parse destination port from PROXY header
|
// Single-owner I/O engine — no tokio::io::split, no mutex
|
||||||
let dest_port = parse_dest_port_from_proxy(&proxy_header).unwrap_or(443);
|
let mut tunnel_io = remoteingress_protocol::TunnelIo::new(tls_stream, Vec::new());
|
||||||
|
|
||||||
let stream_id = frame.stream_id;
|
// Assigned in every break path of the hub_loop before use at the end.
|
||||||
let edge_id_clone = edge_id.clone();
|
#[allow(unused_assignments)]
|
||||||
let event_tx_clone = event_tx.clone();
|
let mut disconnect_reason = String::new();
|
||||||
let streams_clone = streams.clone();
|
|
||||||
let writer_clone = write_half.clone();
|
|
||||||
let target = target_host.clone();
|
|
||||||
let stream_token = edge_token.child_token();
|
|
||||||
|
|
||||||
let _ = event_tx.try_send(HubEvent::StreamOpened {
|
'hub_loop: loop {
|
||||||
edge_id: edge_id.clone(),
|
// Drain completed stream cleanups from spawned tasks
|
||||||
stream_id,
|
while let Ok(stream_id) = cleanup_rx.try_recv() {
|
||||||
});
|
if streams.remove(&stream_id).is_some() {
|
||||||
|
let _ = event_tx.try_send(HubEvent::StreamClosed {
|
||||||
|
edge_id: edge_id.clone(),
|
||||||
|
stream_id,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Create channel for data from edge to this stream
|
// Drain any buffered frames
|
||||||
let (data_tx, mut data_rx) = mpsc::channel::<Vec<u8>>(256);
|
loop {
|
||||||
{
|
let frame = match tunnel_io.try_parse_frame() {
|
||||||
let mut s = streams.lock().await;
|
Some(Ok(f)) => f,
|
||||||
s.insert(stream_id, data_tx);
|
Some(Err(e)) => {
|
||||||
}
|
log::error!("Edge {} frame error: {}", edge_id, e);
|
||||||
|
disconnect_reason = format!("edge_frame_error: {}", e);
|
||||||
|
break 'hub_loop;
|
||||||
|
}
|
||||||
|
None => break,
|
||||||
|
};
|
||||||
|
last_activity = Instant::now();
|
||||||
|
liveness_deadline.as_mut().reset(last_activity + liveness_timeout_dur);
|
||||||
|
if let FrameAction::Disconnect(reason) = handle_hub_frame(
|
||||||
|
frame, &mut tunnel_io, &mut streams, &stream_semaphore, &edge_stream_count,
|
||||||
|
&edge_id, &event_tx, &ctrl_tx, &data_tx, &target_host, &edge_token,
|
||||||
|
&cleanup_tx,
|
||||||
|
).await {
|
||||||
|
disconnect_reason = reason;
|
||||||
|
break 'hub_loop;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Spawn task: connect to SmartProxy, send PROXY header, pipe data
|
// Poll I/O: write(ctrl->data), flush, read, channels, timers
|
||||||
tokio::spawn(async move {
|
let event = std::future::poll_fn(|cx| {
|
||||||
let result = async {
|
// Queue PING if ticker fires
|
||||||
let mut upstream =
|
if ping_ticker.poll_tick(cx).is_ready() {
|
||||||
TcpStream::connect((target.as_str(), dest_port)).await?;
|
tunnel_io.queue_ctrl(encode_frame(0, FRAME_PING, &[]));
|
||||||
upstream.write_all(proxy_header.as_bytes()).await?;
|
}
|
||||||
|
tunnel_io.poll_step(cx, &mut ctrl_rx, &mut data_rx, &mut liveness_deadline, &edge_token)
|
||||||
|
}).await;
|
||||||
|
|
||||||
let (mut up_read, mut up_write) =
|
match event {
|
||||||
upstream.into_split();
|
remoteingress_protocol::TunnelEvent::Frame(frame) => {
|
||||||
|
last_activity = Instant::now();
|
||||||
// Forward data from edge (via channel) to SmartProxy
|
liveness_deadline.as_mut().reset(last_activity + liveness_timeout_dur);
|
||||||
let writer_token = stream_token.clone();
|
if let FrameAction::Disconnect(reason) = handle_hub_frame(
|
||||||
let writer_for_edge_data = tokio::spawn(async move {
|
frame, &mut tunnel_io, &mut streams, &stream_semaphore, &edge_stream_count,
|
||||||
loop {
|
&edge_id, &event_tx, &ctrl_tx, &data_tx, &target_host, &edge_token,
|
||||||
tokio::select! {
|
&cleanup_tx,
|
||||||
data = data_rx.recv() => {
|
).await {
|
||||||
match data {
|
disconnect_reason = reason;
|
||||||
Some(data) => {
|
break;
|
||||||
if up_write.write_all(&data).await.is_err() {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
None => break,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
_ = writer_token.cancelled() => break,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
let _ = up_write.shutdown().await;
|
|
||||||
});
|
|
||||||
|
|
||||||
// Forward data from SmartProxy back to edge
|
|
||||||
let mut buf = vec![0u8; 32768];
|
|
||||||
loop {
|
|
||||||
tokio::select! {
|
|
||||||
read_result = up_read.read(&mut buf) => {
|
|
||||||
match read_result {
|
|
||||||
Ok(0) => break,
|
|
||||||
Ok(n) => {
|
|
||||||
let frame =
|
|
||||||
encode_frame(stream_id, FRAME_DATA_BACK, &buf[..n]);
|
|
||||||
let mut w = writer_clone.lock().await;
|
|
||||||
if w.write_all(&frame).await.is_err() {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Err(_) => break,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
_ = stream_token.cancelled() => break,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Send CLOSE_BACK to edge (only if not cancelled)
|
|
||||||
if !stream_token.is_cancelled() {
|
|
||||||
let close_frame = encode_frame(stream_id, FRAME_CLOSE_BACK, &[]);
|
|
||||||
let mut w = writer_clone.lock().await;
|
|
||||||
let _ = w.write_all(&close_frame).await;
|
|
||||||
}
|
|
||||||
|
|
||||||
writer_for_edge_data.abort();
|
|
||||||
Ok::<(), Box<dyn std::error::Error + Send + Sync>>(())
|
|
||||||
}
|
|
||||||
.await;
|
|
||||||
|
|
||||||
if let Err(e) = result {
|
|
||||||
log::error!("Stream {} error: {}", stream_id, e);
|
|
||||||
// Send CLOSE_BACK on error (only if not cancelled)
|
|
||||||
if !stream_token.is_cancelled() {
|
|
||||||
let close_frame = encode_frame(stream_id, FRAME_CLOSE_BACK, &[]);
|
|
||||||
let mut w = writer_clone.lock().await;
|
|
||||||
let _ = w.write_all(&close_frame).await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Clean up stream
|
|
||||||
{
|
|
||||||
let mut s = streams_clone.lock().await;
|
|
||||||
s.remove(&stream_id);
|
|
||||||
}
|
|
||||||
let _ = event_tx_clone.try_send(HubEvent::StreamClosed {
|
|
||||||
edge_id: edge_id_clone,
|
|
||||||
stream_id,
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
|
||||||
FRAME_DATA => {
|
|
||||||
let s = streams.lock().await;
|
|
||||||
if let Some(tx) = s.get(&frame.stream_id) {
|
|
||||||
let _ = tx.send(frame.payload).await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
FRAME_CLOSE => {
|
|
||||||
let mut s = streams.lock().await;
|
|
||||||
s.remove(&frame.stream_id);
|
|
||||||
}
|
|
||||||
_ => {
|
|
||||||
log::warn!("Unexpected frame type {} from edge", frame.frame_type);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok(None) => {
|
|
||||||
log::info!("Edge {} disconnected (EOF)", edge_id);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
log::error!("Edge {} frame error: {}", edge_id, e);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
_ = edge_token.cancelled() => {
|
remoteingress_protocol::TunnelEvent::Eof => {
|
||||||
|
log::info!("Edge {} disconnected (EOF)", edge_id);
|
||||||
|
disconnect_reason = "edge_eof".to_string();
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
remoteingress_protocol::TunnelEvent::ReadError(e) => {
|
||||||
|
log::error!("Edge {} frame error: {}", edge_id, e);
|
||||||
|
disconnect_reason = format!("edge_frame_error: {}", e);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
remoteingress_protocol::TunnelEvent::WriteError(e) => {
|
||||||
|
log::error!("Tunnel write error to edge {}: {}", edge_id, e);
|
||||||
|
disconnect_reason = format!("tunnel_write_error: {}", e);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
remoteingress_protocol::TunnelEvent::LivenessTimeout => {
|
||||||
|
log::warn!("Edge {} liveness timeout (no frames for {}s), disconnecting",
|
||||||
|
edge_id, liveness_timeout_dur.as_secs());
|
||||||
|
disconnect_reason = "liveness_timeout".to_string();
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
remoteingress_protocol::TunnelEvent::Cancelled => {
|
||||||
log::info!("Edge {} cancelled by hub", edge_id);
|
log::info!("Edge {} cancelled by hub", edge_id);
|
||||||
|
disconnect_reason = "cancelled_by_hub".to_string();
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -539,6 +833,7 @@ async fn handle_edge_connection(
|
|||||||
}
|
}
|
||||||
let _ = event_tx.try_send(HubEvent::EdgeDisconnected {
|
let _ = event_tx.try_send(HubEvent::EdgeDisconnected {
|
||||||
edge_id: edge_id.clone(),
|
edge_id: edge_id.clone(),
|
||||||
|
reason: disconnect_reason,
|
||||||
});
|
});
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -749,20 +1044,24 @@ mod tests {
|
|||||||
fn test_hub_event_edge_connected_serialize() {
|
fn test_hub_event_edge_connected_serialize() {
|
||||||
let event = HubEvent::EdgeConnected {
|
let event = HubEvent::EdgeConnected {
|
||||||
edge_id: "edge-1".to_string(),
|
edge_id: "edge-1".to_string(),
|
||||||
|
peer_addr: "203.0.113.5".to_string(),
|
||||||
};
|
};
|
||||||
let json = serde_json::to_value(&event).unwrap();
|
let json = serde_json::to_value(&event).unwrap();
|
||||||
assert_eq!(json["type"], "edgeConnected");
|
assert_eq!(json["type"], "edgeConnected");
|
||||||
assert_eq!(json["edgeId"], "edge-1");
|
assert_eq!(json["edgeId"], "edge-1");
|
||||||
|
assert_eq!(json["peerAddr"], "203.0.113.5");
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_hub_event_edge_disconnected_serialize() {
|
fn test_hub_event_edge_disconnected_serialize() {
|
||||||
let event = HubEvent::EdgeDisconnected {
|
let event = HubEvent::EdgeDisconnected {
|
||||||
edge_id: "edge-2".to_string(),
|
edge_id: "edge-2".to_string(),
|
||||||
|
reason: "liveness_timeout".to_string(),
|
||||||
};
|
};
|
||||||
let json = serde_json::to_value(&event).unwrap();
|
let json = serde_json::to_value(&event).unwrap();
|
||||||
assert_eq!(json["type"], "edgeDisconnected");
|
assert_eq!(json["type"], "edgeDisconnected");
|
||||||
assert_eq!(json["edgeId"], "edge-2");
|
assert_eq!(json["edgeId"], "edge-2");
|
||||||
|
assert_eq!(json["reason"], "liveness_timeout");
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -4,4 +4,9 @@ version = "2.0.0"
|
|||||||
edition = "2021"
|
edition = "2021"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
tokio = { version = "1", features = ["io-util"] }
|
tokio = { version = "1", features = ["io-util", "sync", "time"] }
|
||||||
|
tokio-util = "0.7"
|
||||||
|
log = "0.4"
|
||||||
|
|
||||||
|
[dev-dependencies]
|
||||||
|
tokio = { version = "1", features = ["io-util", "macros", "rt"] }
|
||||||
|
|||||||
@@ -1,4 +1,8 @@
|
|||||||
use tokio::io::{AsyncRead, AsyncReadExt};
|
use std::collections::VecDeque;
|
||||||
|
use std::future::Future;
|
||||||
|
use std::pin::Pin;
|
||||||
|
use std::task::{Context, Poll};
|
||||||
|
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, ReadBuf};
|
||||||
|
|
||||||
// Frame type constants
|
// Frame type constants
|
||||||
pub const FRAME_OPEN: u8 = 0x01;
|
pub const FRAME_OPEN: u8 = 0x01;
|
||||||
@@ -7,6 +11,10 @@ pub const FRAME_CLOSE: u8 = 0x03;
|
|||||||
pub const FRAME_DATA_BACK: u8 = 0x04;
|
pub const FRAME_DATA_BACK: u8 = 0x04;
|
||||||
pub const FRAME_CLOSE_BACK: u8 = 0x05;
|
pub const FRAME_CLOSE_BACK: u8 = 0x05;
|
||||||
pub const FRAME_CONFIG: u8 = 0x06; // Hub -> Edge: configuration update
|
pub const FRAME_CONFIG: u8 = 0x06; // Hub -> Edge: configuration update
|
||||||
|
pub const FRAME_PING: u8 = 0x07; // Hub -> Edge: heartbeat probe
|
||||||
|
pub const FRAME_PONG: u8 = 0x08; // Edge -> Hub: heartbeat response
|
||||||
|
pub const FRAME_WINDOW_UPDATE: u8 = 0x09; // Edge -> Hub: per-stream flow control
|
||||||
|
pub const FRAME_WINDOW_UPDATE_BACK: u8 = 0x0A; // Hub -> Edge: per-stream flow control
|
||||||
|
|
||||||
// Frame header size: 4 (stream_id) + 1 (type) + 4 (length) = 9 bytes
|
// Frame header size: 4 (stream_id) + 1 (type) + 4 (length) = 9 bytes
|
||||||
pub const FRAME_HEADER_SIZE: usize = 9;
|
pub const FRAME_HEADER_SIZE: usize = 9;
|
||||||
@@ -14,6 +22,37 @@ pub const FRAME_HEADER_SIZE: usize = 9;
|
|||||||
// Maximum payload size (16 MB)
|
// Maximum payload size (16 MB)
|
||||||
pub const MAX_PAYLOAD_SIZE: u32 = 16 * 1024 * 1024;
|
pub const MAX_PAYLOAD_SIZE: u32 = 16 * 1024 * 1024;
|
||||||
|
|
||||||
|
// Per-stream flow control constants
|
||||||
|
/// Initial per-stream window size (4 MB). Sized for full throughput at high RTT:
|
||||||
|
/// at 100ms RTT, this sustains ~40 MB/s per stream.
|
||||||
|
pub const INITIAL_STREAM_WINDOW: u32 = 4 * 1024 * 1024;
|
||||||
|
/// Send WINDOW_UPDATE after consuming this many bytes (half the initial window).
|
||||||
|
pub const WINDOW_UPDATE_THRESHOLD: u32 = INITIAL_STREAM_WINDOW / 2;
|
||||||
|
/// Maximum window size to prevent overflow.
|
||||||
|
pub const MAX_WINDOW_SIZE: u32 = 16 * 1024 * 1024;
|
||||||
|
|
||||||
|
/// Encode a WINDOW_UPDATE frame for a specific stream.
|
||||||
|
pub fn encode_window_update(stream_id: u32, frame_type: u8, increment: u32) -> Vec<u8> {
|
||||||
|
encode_frame(stream_id, frame_type, &increment.to_be_bytes())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Compute the target per-stream window size based on the number of active streams.
|
||||||
|
/// Total memory budget is ~32MB shared across all streams. As more streams are active,
|
||||||
|
/// each gets a smaller window. This adapts to current demand — few streams get high
|
||||||
|
/// throughput, many streams save memory and reduce control frame pressure.
|
||||||
|
pub fn compute_window_for_stream_count(active: u32) -> u32 {
|
||||||
|
let per_stream = (32 * 1024 * 1024u64) / (active.max(1) as u64);
|
||||||
|
per_stream.clamp(64 * 1024, INITIAL_STREAM_WINDOW as u64) as u32
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Decode a WINDOW_UPDATE payload into a byte increment. Returns None if payload is malformed.
|
||||||
|
pub fn decode_window_update(payload: &[u8]) -> Option<u32> {
|
||||||
|
if payload.len() != 4 {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Some(u32::from_be_bytes([payload[0], payload[1], payload[2], payload[3]]))
|
||||||
|
}
|
||||||
|
|
||||||
/// A single multiplexed frame.
|
/// A single multiplexed frame.
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct Frame {
|
pub struct Frame {
|
||||||
@@ -33,6 +72,16 @@ pub fn encode_frame(stream_id: u32, frame_type: u8, payload: &[u8]) -> Vec<u8> {
|
|||||||
buf
|
buf
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Write a frame header into `buf[0..FRAME_HEADER_SIZE]`.
|
||||||
|
/// The caller must ensure payload is already at `buf[FRAME_HEADER_SIZE..FRAME_HEADER_SIZE + payload_len]`.
|
||||||
|
/// This enables zero-copy encoding: read directly into `buf[FRAME_HEADER_SIZE..]`, then
|
||||||
|
/// prepend the header without copying the payload.
|
||||||
|
pub fn encode_frame_header(buf: &mut [u8], stream_id: u32, frame_type: u8, payload_len: usize) {
|
||||||
|
buf[0..4].copy_from_slice(&stream_id.to_be_bytes());
|
||||||
|
buf[4] = frame_type;
|
||||||
|
buf[5..9].copy_from_slice(&(payload_len as u32).to_be_bytes());
|
||||||
|
}
|
||||||
|
|
||||||
/// Build a PROXY protocol v1 header line.
|
/// Build a PROXY protocol v1 header line.
|
||||||
/// Format: `PROXY TCP4 <client_ip> <edge_ip> <client_port> <dest_port>\r\n`
|
/// Format: `PROXY TCP4 <client_ip> <edge_ip> <client_port> <dest_port>\r\n`
|
||||||
pub fn build_proxy_v1_header(
|
pub fn build_proxy_v1_header(
|
||||||
@@ -85,9 +134,13 @@ impl<R: AsyncRead + Unpin> FrameReader<R> {
|
|||||||
]);
|
]);
|
||||||
|
|
||||||
if length > MAX_PAYLOAD_SIZE {
|
if length > MAX_PAYLOAD_SIZE {
|
||||||
|
log::error!(
|
||||||
|
"CORRUPT FRAME HEADER: raw={:02x?} stream_id={} type=0x{:02x} length={}",
|
||||||
|
self.header_buf, stream_id, frame_type, length
|
||||||
|
);
|
||||||
return Err(std::io::Error::new(
|
return Err(std::io::Error::new(
|
||||||
std::io::ErrorKind::InvalidData,
|
std::io::ErrorKind::InvalidData,
|
||||||
format!("frame payload too large: {} bytes", length),
|
format!("frame payload too large: {} bytes (header={:02x?})", length, self.header_buf),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -109,10 +162,303 @@ impl<R: AsyncRead + Unpin> FrameReader<R> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// TunnelIo: single-owner I/O multiplexer for the TLS tunnel connection
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/// Events produced by the TunnelIo event loop.
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub enum TunnelEvent {
|
||||||
|
/// A complete frame was read from the remote side.
|
||||||
|
Frame(Frame),
|
||||||
|
/// The remote side closed the connection (EOF).
|
||||||
|
Eof,
|
||||||
|
/// A read error occurred.
|
||||||
|
ReadError(std::io::Error),
|
||||||
|
/// A write error occurred.
|
||||||
|
WriteError(std::io::Error),
|
||||||
|
/// No frames received for the liveness timeout duration.
|
||||||
|
LivenessTimeout,
|
||||||
|
/// The cancellation token was triggered.
|
||||||
|
Cancelled,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Write state extracted into a sub-struct so the borrow checker can see
|
||||||
|
/// disjoint field access between `self.write` and `self.stream`.
|
||||||
|
struct WriteState {
|
||||||
|
ctrl_queue: VecDeque<Vec<u8>>, // PONG, WINDOW_UPDATE, CLOSE, OPEN — always first
|
||||||
|
data_queue: VecDeque<Vec<u8>>, // DATA, DATA_BACK — only when ctrl is empty
|
||||||
|
offset: usize, // progress within current frame being written
|
||||||
|
flush_needed: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WriteState {
|
||||||
|
fn has_work(&self) -> bool {
|
||||||
|
!self.ctrl_queue.is_empty() || !self.data_queue.is_empty()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Single-owner I/O engine for the tunnel TLS connection.
|
||||||
|
///
|
||||||
|
/// Owns the TLS stream directly — no `tokio::io::split()`, no mutex.
|
||||||
|
/// Uses two priority write queues: ctrl frames (PONG, WINDOW_UPDATE, CLOSE, OPEN)
|
||||||
|
/// are ALWAYS written before data frames (DATA, DATA_BACK). This prevents
|
||||||
|
/// WINDOW_UPDATE starvation that causes flow control deadlocks.
|
||||||
|
pub struct TunnelIo<S> {
|
||||||
|
stream: S,
|
||||||
|
// Read state: accumulate bytes, parse frames incrementally
|
||||||
|
read_buf: Vec<u8>,
|
||||||
|
read_pos: usize,
|
||||||
|
parse_pos: usize,
|
||||||
|
// Write state: extracted sub-struct for safe disjoint borrows
|
||||||
|
write: WriteState,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<S: AsyncRead + AsyncWrite + Unpin> TunnelIo<S> {
|
||||||
|
pub fn new(stream: S, initial_data: Vec<u8>) -> Self {
|
||||||
|
let read_pos = initial_data.len();
|
||||||
|
let mut read_buf = initial_data;
|
||||||
|
if read_buf.capacity() < 65536 {
|
||||||
|
read_buf.reserve(65536 - read_buf.len());
|
||||||
|
}
|
||||||
|
Self {
|
||||||
|
stream,
|
||||||
|
read_buf,
|
||||||
|
read_pos,
|
||||||
|
parse_pos: 0,
|
||||||
|
write: WriteState {
|
||||||
|
ctrl_queue: VecDeque::new(),
|
||||||
|
data_queue: VecDeque::new(),
|
||||||
|
offset: 0,
|
||||||
|
flush_needed: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Queue a high-priority control frame (PONG, WINDOW_UPDATE, CLOSE, OPEN).
|
||||||
|
pub fn queue_ctrl(&mut self, frame: Vec<u8>) {
|
||||||
|
self.write.ctrl_queue.push_back(frame);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Queue a lower-priority data frame (DATA, DATA_BACK).
|
||||||
|
pub fn queue_data(&mut self, frame: Vec<u8>) {
|
||||||
|
self.write.data_queue.push_back(frame);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Try to parse a complete frame from the read buffer.
|
||||||
|
/// Uses a parse_pos cursor to avoid drain() on every frame.
|
||||||
|
pub fn try_parse_frame(&mut self) -> Option<Result<Frame, std::io::Error>> {
|
||||||
|
let available = self.read_pos - self.parse_pos;
|
||||||
|
if available < FRAME_HEADER_SIZE {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
let base = self.parse_pos;
|
||||||
|
let stream_id = u32::from_be_bytes([
|
||||||
|
self.read_buf[base], self.read_buf[base + 1],
|
||||||
|
self.read_buf[base + 2], self.read_buf[base + 3],
|
||||||
|
]);
|
||||||
|
let frame_type = self.read_buf[base + 4];
|
||||||
|
let length = u32::from_be_bytes([
|
||||||
|
self.read_buf[base + 5], self.read_buf[base + 6],
|
||||||
|
self.read_buf[base + 7], self.read_buf[base + 8],
|
||||||
|
]);
|
||||||
|
|
||||||
|
if length > MAX_PAYLOAD_SIZE {
|
||||||
|
let header = [
|
||||||
|
self.read_buf[base], self.read_buf[base + 1],
|
||||||
|
self.read_buf[base + 2], self.read_buf[base + 3],
|
||||||
|
self.read_buf[base + 4], self.read_buf[base + 5],
|
||||||
|
self.read_buf[base + 6], self.read_buf[base + 7],
|
||||||
|
self.read_buf[base + 8],
|
||||||
|
];
|
||||||
|
log::error!(
|
||||||
|
"CORRUPT FRAME HEADER: raw={:02x?} stream_id={} type=0x{:02x} length={}",
|
||||||
|
header, stream_id, frame_type, length
|
||||||
|
);
|
||||||
|
return Some(Err(std::io::Error::new(
|
||||||
|
std::io::ErrorKind::InvalidData,
|
||||||
|
format!("frame payload too large: {} bytes (header={:02x?})", length, header),
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
|
let total_frame_size = FRAME_HEADER_SIZE + length as usize;
|
||||||
|
if available < total_frame_size {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
let payload = self.read_buf[base + FRAME_HEADER_SIZE..base + total_frame_size].to_vec();
|
||||||
|
self.parse_pos += total_frame_size;
|
||||||
|
|
||||||
|
// Compact when parse_pos > half the data to reclaim memory
|
||||||
|
if self.parse_pos > self.read_pos / 2 && self.parse_pos > 0 {
|
||||||
|
self.read_buf.drain(..self.parse_pos);
|
||||||
|
self.read_pos -= self.parse_pos;
|
||||||
|
self.parse_pos = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
Some(Ok(Frame { stream_id, frame_type, payload }))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Poll-based I/O step. Returns Ready on events, Pending when idle.
|
||||||
|
///
|
||||||
|
/// Order: write(ctrl→data) → flush → read → channels → timers
|
||||||
|
pub fn poll_step(
|
||||||
|
&mut self,
|
||||||
|
cx: &mut Context<'_>,
|
||||||
|
ctrl_rx: &mut tokio::sync::mpsc::Receiver<Vec<u8>>,
|
||||||
|
data_rx: &mut tokio::sync::mpsc::Receiver<Vec<u8>>,
|
||||||
|
liveness_deadline: &mut Pin<Box<tokio::time::Sleep>>,
|
||||||
|
cancel_token: &tokio_util::sync::CancellationToken,
|
||||||
|
) -> Poll<TunnelEvent> {
|
||||||
|
// 1. WRITE: drain ctrl queue first, then data queue.
|
||||||
|
// TLS poll_write writes plaintext to session buffer (always Ready).
|
||||||
|
// Batch up to 16 frames per poll cycle.
|
||||||
|
// Safe: `self.write` and `self.stream` are disjoint fields.
|
||||||
|
let mut writes = 0;
|
||||||
|
while self.write.has_work() && writes < 16 {
|
||||||
|
let from_ctrl = !self.write.ctrl_queue.is_empty();
|
||||||
|
let frame = if from_ctrl {
|
||||||
|
self.write.ctrl_queue.front().unwrap()
|
||||||
|
} else {
|
||||||
|
self.write.data_queue.front().unwrap()
|
||||||
|
};
|
||||||
|
let remaining = &frame[self.write.offset..];
|
||||||
|
|
||||||
|
match Pin::new(&mut self.stream).poll_write(cx, remaining) {
|
||||||
|
Poll::Ready(Ok(0)) => {
|
||||||
|
return Poll::Ready(TunnelEvent::WriteError(
|
||||||
|
std::io::Error::new(std::io::ErrorKind::WriteZero, "write zero"),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Poll::Ready(Ok(n)) => {
|
||||||
|
self.write.offset += n;
|
||||||
|
self.write.flush_needed = true;
|
||||||
|
if self.write.offset >= frame.len() {
|
||||||
|
if from_ctrl { self.write.ctrl_queue.pop_front(); }
|
||||||
|
else { self.write.data_queue.pop_front(); }
|
||||||
|
self.write.offset = 0;
|
||||||
|
writes += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Poll::Ready(Err(e)) => return Poll::Ready(TunnelEvent::WriteError(e)),
|
||||||
|
Poll::Pending => break,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. FLUSH: push encrypted data from TLS session to TCP.
|
||||||
|
if self.write.flush_needed {
|
||||||
|
match Pin::new(&mut self.stream).poll_flush(cx) {
|
||||||
|
Poll::Ready(Ok(())) => self.write.flush_needed = false,
|
||||||
|
Poll::Ready(Err(e)) => return Poll::Ready(TunnelEvent::WriteError(e)),
|
||||||
|
Poll::Pending => {} // TCP waker will notify us
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. READ: drain stream until Pending to ensure the TCP waker is always registered.
|
||||||
|
// Without this loop, a Ready return with partial frame data would consume
|
||||||
|
// the waker without re-registering it, causing the task to sleep until a
|
||||||
|
// timer or channel wakes it (potentially 15+ seconds of lost reads).
|
||||||
|
loop {
|
||||||
|
// Compact if needed to make room for reads
|
||||||
|
if self.parse_pos > 0 && self.read_buf.len() - self.read_pos < 32768 {
|
||||||
|
self.read_buf.drain(..self.parse_pos);
|
||||||
|
self.read_pos -= self.parse_pos;
|
||||||
|
self.parse_pos = 0;
|
||||||
|
}
|
||||||
|
if self.read_buf.len() < self.read_pos + 32768 {
|
||||||
|
self.read_buf.resize(self.read_pos + 32768, 0);
|
||||||
|
}
|
||||||
|
let mut rbuf = ReadBuf::new(&mut self.read_buf[self.read_pos..]);
|
||||||
|
match Pin::new(&mut self.stream).poll_read(cx, &mut rbuf) {
|
||||||
|
Poll::Ready(Ok(())) => {
|
||||||
|
let n = rbuf.filled().len();
|
||||||
|
if n == 0 {
|
||||||
|
return Poll::Ready(TunnelEvent::Eof);
|
||||||
|
}
|
||||||
|
self.read_pos += n;
|
||||||
|
if let Some(result) = self.try_parse_frame() {
|
||||||
|
return match result {
|
||||||
|
Ok(frame) => Poll::Ready(TunnelEvent::Frame(frame)),
|
||||||
|
Err(e) => Poll::Ready(TunnelEvent::ReadError(e)),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
// Partial data — loop to call poll_read again so the TCP
|
||||||
|
// waker is re-registered when it finally returns Pending.
|
||||||
|
}
|
||||||
|
Poll::Ready(Err(e)) => return Poll::Ready(TunnelEvent::ReadError(e)),
|
||||||
|
Poll::Pending => break,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. CHANNELS: drain ctrl into ctrl_queue, data into data_queue.
|
||||||
|
let mut got_new = false;
|
||||||
|
loop {
|
||||||
|
match ctrl_rx.poll_recv(cx) {
|
||||||
|
Poll::Ready(Some(frame)) => { self.write.ctrl_queue.push_back(frame); got_new = true; }
|
||||||
|
Poll::Ready(None) => {
|
||||||
|
return Poll::Ready(TunnelEvent::WriteError(
|
||||||
|
std::io::Error::new(std::io::ErrorKind::BrokenPipe, "ctrl channel closed"),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Poll::Pending => break,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
loop {
|
||||||
|
match data_rx.poll_recv(cx) {
|
||||||
|
Poll::Ready(Some(frame)) => { self.write.data_queue.push_back(frame); got_new = true; }
|
||||||
|
Poll::Ready(None) => {
|
||||||
|
return Poll::Ready(TunnelEvent::WriteError(
|
||||||
|
std::io::Error::new(std::io::ErrorKind::BrokenPipe, "data channel closed"),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Poll::Pending => break,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 5. TIMERS
|
||||||
|
if liveness_deadline.as_mut().poll(cx).is_ready() {
|
||||||
|
return Poll::Ready(TunnelEvent::LivenessTimeout);
|
||||||
|
}
|
||||||
|
if cancel_token.is_cancelled() {
|
||||||
|
return Poll::Ready(TunnelEvent::Cancelled);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 6. SELF-WAKE: only when we have frames AND flush is done.
|
||||||
|
// If flush is pending, the TCP write-readiness waker will notify us.
|
||||||
|
// If we got new channel frames, wake to write them.
|
||||||
|
if got_new || (!self.write.flush_needed && self.write.has_work()) {
|
||||||
|
cx.waker().wake_by_ref();
|
||||||
|
}
|
||||||
|
|
||||||
|
Poll::Pending
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn into_inner(self) -> S {
|
||||||
|
self.stream
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_encode_frame_header() {
|
||||||
|
let payload = b"hello";
|
||||||
|
let mut buf = vec![0u8; FRAME_HEADER_SIZE + payload.len()];
|
||||||
|
buf[FRAME_HEADER_SIZE..].copy_from_slice(payload);
|
||||||
|
encode_frame_header(&mut buf, 42, FRAME_DATA, payload.len());
|
||||||
|
assert_eq!(buf, encode_frame(42, FRAME_DATA, payload));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_encode_frame_header_empty_payload() {
|
||||||
|
let mut buf = vec![0u8; FRAME_HEADER_SIZE];
|
||||||
|
encode_frame_header(&mut buf, 99, FRAME_CLOSE, 0);
|
||||||
|
assert_eq!(buf, encode_frame(99, FRAME_CLOSE, &[]));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_encode_frame() {
|
fn test_encode_frame() {
|
||||||
let data = b"hello";
|
let data = b"hello";
|
||||||
@@ -261,6 +607,8 @@ mod tests {
|
|||||||
FRAME_DATA_BACK,
|
FRAME_DATA_BACK,
|
||||||
FRAME_CLOSE_BACK,
|
FRAME_CLOSE_BACK,
|
||||||
FRAME_CONFIG,
|
FRAME_CONFIG,
|
||||||
|
FRAME_PING,
|
||||||
|
FRAME_PONG,
|
||||||
];
|
];
|
||||||
|
|
||||||
let mut data = Vec::new();
|
let mut data = Vec::new();
|
||||||
@@ -293,4 +641,149 @@ mod tests {
|
|||||||
assert_eq!(frame.frame_type, FRAME_CLOSE);
|
assert_eq!(frame.frame_type, FRAME_CLOSE);
|
||||||
assert!(frame.payload.is_empty());
|
assert!(frame.payload.is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_encode_frame_ping_pong() {
|
||||||
|
// PING: stream_id=0, empty payload (control frame)
|
||||||
|
let ping = encode_frame(0, FRAME_PING, &[]);
|
||||||
|
assert_eq!(ping[4], FRAME_PING);
|
||||||
|
assert_eq!(&ping[0..4], &0u32.to_be_bytes());
|
||||||
|
assert_eq!(ping.len(), FRAME_HEADER_SIZE);
|
||||||
|
|
||||||
|
// PONG: stream_id=0, empty payload (control frame)
|
||||||
|
let pong = encode_frame(0, FRAME_PONG, &[]);
|
||||||
|
assert_eq!(pong[4], FRAME_PONG);
|
||||||
|
assert_eq!(&pong[0..4], &0u32.to_be_bytes());
|
||||||
|
assert_eq!(pong.len(), FRAME_HEADER_SIZE);
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- compute_window_for_stream_count tests ---
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_adaptive_window_zero_streams() {
|
||||||
|
// 0 streams treated as 1: 32MB/1 = 32MB → clamped to 4MB max
|
||||||
|
assert_eq!(compute_window_for_stream_count(0), INITIAL_STREAM_WINDOW);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_adaptive_window_one_stream() {
|
||||||
|
// 32MB/1 = 32MB → clamped to 4MB max
|
||||||
|
assert_eq!(compute_window_for_stream_count(1), INITIAL_STREAM_WINDOW);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_adaptive_window_at_max_boundary() {
|
||||||
|
// 32MB/8 = 4MB = exactly INITIAL_STREAM_WINDOW
|
||||||
|
assert_eq!(compute_window_for_stream_count(8), INITIAL_STREAM_WINDOW);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_adaptive_window_just_below_max() {
|
||||||
|
// 32MB/9 = 3,728,270 — first value below INITIAL_STREAM_WINDOW
|
||||||
|
let w = compute_window_for_stream_count(9);
|
||||||
|
assert!(w < INITIAL_STREAM_WINDOW);
|
||||||
|
assert_eq!(w, (32 * 1024 * 1024u64 / 9) as u32);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_adaptive_window_16_streams() {
|
||||||
|
// 32MB/16 = 2MB
|
||||||
|
assert_eq!(compute_window_for_stream_count(16), 2 * 1024 * 1024);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_adaptive_window_100_streams() {
|
||||||
|
// 32MB/100 = 335,544 bytes (~327KB)
|
||||||
|
let w = compute_window_for_stream_count(100);
|
||||||
|
assert_eq!(w, (32 * 1024 * 1024u64 / 100) as u32);
|
||||||
|
assert!(w > 64 * 1024); // above floor
|
||||||
|
assert!(w < INITIAL_STREAM_WINDOW as u32); // below ceiling
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_adaptive_window_200_streams() {
|
||||||
|
// 32MB/200 = 167,772 bytes (~163KB), above 64KB floor
|
||||||
|
let w = compute_window_for_stream_count(200);
|
||||||
|
assert_eq!(w, (32 * 1024 * 1024u64 / 200) as u32);
|
||||||
|
assert!(w > 64 * 1024);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_adaptive_window_500_streams() {
|
||||||
|
// 32MB/500 = 67,108 bytes (~65.5KB), just above 64KB floor
|
||||||
|
let w = compute_window_for_stream_count(500);
|
||||||
|
assert_eq!(w, (32 * 1024 * 1024u64 / 500) as u32);
|
||||||
|
assert!(w > 64 * 1024);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_adaptive_window_at_min_boundary() {
|
||||||
|
// 32MB/512 = 65,536 = exactly 64KB floor
|
||||||
|
assert_eq!(compute_window_for_stream_count(512), 64 * 1024);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_adaptive_window_below_min_clamped() {
|
||||||
|
// 32MB/513 = 65,408 → clamped up to 64KB
|
||||||
|
assert_eq!(compute_window_for_stream_count(513), 64 * 1024);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_adaptive_window_1000_streams() {
|
||||||
|
// 32MB/1000 = 33,554 → clamped to 64KB
|
||||||
|
assert_eq!(compute_window_for_stream_count(1000), 64 * 1024);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_adaptive_window_max_u32() {
|
||||||
|
// Extreme: u32::MAX streams → tiny value → clamped to 64KB
|
||||||
|
assert_eq!(compute_window_for_stream_count(u32::MAX), 64 * 1024);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_adaptive_window_monotonically_decreasing() {
|
||||||
|
// Window should decrease (or stay same) as stream count increases
|
||||||
|
let mut prev = compute_window_for_stream_count(1);
|
||||||
|
for n in [2, 5, 10, 50, 100, 200, 500, 512, 1000] {
|
||||||
|
let w = compute_window_for_stream_count(n);
|
||||||
|
assert!(w <= prev, "window increased from {} to {} at n={}", prev, w, n);
|
||||||
|
prev = w;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_adaptive_window_total_budget_bounded() {
|
||||||
|
// active × per_stream_window should never exceed 32MB (+ clamp overhead for high N)
|
||||||
|
for n in [1, 10, 50, 100, 200, 500] {
|
||||||
|
let w = compute_window_for_stream_count(n);
|
||||||
|
let total = w as u64 * n as u64;
|
||||||
|
assert!(total <= 32 * 1024 * 1024, "total {}MB exceeds budget at n={}", total / (1024*1024), n);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- encode/decode window_update roundtrip ---
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_window_update_roundtrip() {
|
||||||
|
for &increment in &[0u32, 1, 64 * 1024, INITIAL_STREAM_WINDOW, MAX_WINDOW_SIZE, u32::MAX] {
|
||||||
|
let frame = encode_window_update(42, FRAME_WINDOW_UPDATE, increment);
|
||||||
|
assert_eq!(frame[4], FRAME_WINDOW_UPDATE);
|
||||||
|
let decoded = decode_window_update(&frame[FRAME_HEADER_SIZE..]);
|
||||||
|
assert_eq!(decoded, Some(increment));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_window_update_back_roundtrip() {
|
||||||
|
let frame = encode_window_update(7, FRAME_WINDOW_UPDATE_BACK, 1234567);
|
||||||
|
assert_eq!(frame[4], FRAME_WINDOW_UPDATE_BACK);
|
||||||
|
assert_eq!(decode_window_update(&frame[FRAME_HEADER_SIZE..]), Some(1234567));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_decode_window_update_malformed() {
|
||||||
|
assert_eq!(decode_window_update(&[]), None);
|
||||||
|
assert_eq!(decode_window_update(&[0, 0, 0]), None);
|
||||||
|
assert_eq!(decode_window_update(&[0, 0, 0, 0, 0]), None);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
475
test/test.flowcontrol.node.ts
Normal file
475
test/test.flowcontrol.node.ts
Normal file
@@ -0,0 +1,475 @@
|
|||||||
|
import { expect, tap } from '@push.rocks/tapbundle';
|
||||||
|
import * as net from 'net';
|
||||||
|
import * as crypto from 'crypto';
|
||||||
|
import { RemoteIngressHub, RemoteIngressEdge } from '../ts/index.js';
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Helpers
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/** Find N free ports by binding to port 0 and collecting OS-assigned ports. */
|
||||||
|
async function findFreePorts(count: number): Promise<number[]> {
|
||||||
|
const servers: net.Server[] = [];
|
||||||
|
const ports: number[] = [];
|
||||||
|
for (let i = 0; i < count; i++) {
|
||||||
|
const server = net.createServer();
|
||||||
|
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve));
|
||||||
|
ports.push((server.address() as net.AddressInfo).port);
|
||||||
|
servers.push(server);
|
||||||
|
}
|
||||||
|
await Promise.all(servers.map((s) => new Promise<void>((resolve) => s.close(() => resolve()))));
|
||||||
|
return ports;
|
||||||
|
}
|
||||||
|
|
||||||
|
type TrackingServer = net.Server & { destroyAll: () => void };
|
||||||
|
|
||||||
|
/** Start a TCP echo server that tracks connections for force-close. */
|
||||||
|
function startEchoServer(port: number, host: string): Promise<TrackingServer> {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const connections = new Set<net.Socket>();
|
||||||
|
const server = net.createServer((socket) => {
|
||||||
|
connections.add(socket);
|
||||||
|
socket.on('close', () => connections.delete(socket));
|
||||||
|
|
||||||
|
// Skip PROXY protocol v1 header line before echoing
|
||||||
|
let proxyHeaderParsed = false;
|
||||||
|
let pendingBuf = Buffer.alloc(0);
|
||||||
|
socket.on('data', (data: Buffer) => {
|
||||||
|
if (!proxyHeaderParsed) {
|
||||||
|
pendingBuf = Buffer.concat([pendingBuf, data]);
|
||||||
|
const idx = pendingBuf.indexOf('\r\n');
|
||||||
|
if (idx !== -1) {
|
||||||
|
proxyHeaderParsed = true;
|
||||||
|
const remainder = pendingBuf.subarray(idx + 2);
|
||||||
|
if (remainder.length > 0) {
|
||||||
|
socket.write(remainder);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
socket.write(data);
|
||||||
|
});
|
||||||
|
socket.on('error', () => {});
|
||||||
|
}) as TrackingServer;
|
||||||
|
|
||||||
|
server.destroyAll = () => {
|
||||||
|
for (const conn of connections) conn.destroy();
|
||||||
|
connections.clear();
|
||||||
|
};
|
||||||
|
|
||||||
|
server.on('error', reject);
|
||||||
|
server.listen(port, host, () => resolve(server));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Start a server that sends a large response immediately on first data received.
|
||||||
|
* Does NOT wait for end (the tunnel protocol has no half-close).
|
||||||
|
* On receiving first data chunk after PROXY header, sends responseSize bytes then closes.
|
||||||
|
*/
|
||||||
|
function startLargeResponseServer(port: number, host: string, responseSize: number): Promise<TrackingServer> {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const connections = new Set<net.Socket>();
|
||||||
|
const server = net.createServer((socket) => {
|
||||||
|
connections.add(socket);
|
||||||
|
socket.on('close', () => connections.delete(socket));
|
||||||
|
|
||||||
|
let proxyHeaderParsed = false;
|
||||||
|
let pendingBuf = Buffer.alloc(0);
|
||||||
|
let responseSent = false;
|
||||||
|
|
||||||
|
socket.on('data', (data: Buffer) => {
|
||||||
|
if (!proxyHeaderParsed) {
|
||||||
|
pendingBuf = Buffer.concat([pendingBuf, data]);
|
||||||
|
const idx = pendingBuf.indexOf('\r\n');
|
||||||
|
if (idx !== -1) {
|
||||||
|
proxyHeaderParsed = true;
|
||||||
|
const remainder = pendingBuf.subarray(idx + 2);
|
||||||
|
if (remainder.length > 0 && !responseSent) {
|
||||||
|
responseSent = true;
|
||||||
|
sendLargeResponse(socket, responseSize);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!responseSent) {
|
||||||
|
responseSent = true;
|
||||||
|
sendLargeResponse(socket, responseSize);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
socket.on('error', () => {});
|
||||||
|
}) as TrackingServer;
|
||||||
|
|
||||||
|
server.destroyAll = () => {
|
||||||
|
for (const conn of connections) conn.destroy();
|
||||||
|
connections.clear();
|
||||||
|
};
|
||||||
|
|
||||||
|
server.on('error', reject);
|
||||||
|
server.listen(port, host, () => resolve(server));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function sendLargeResponse(socket: net.Socket, totalBytes: number) {
|
||||||
|
const chunkSize = 32 * 1024;
|
||||||
|
let sent = 0;
|
||||||
|
const writeChunk = () => {
|
||||||
|
while (sent < totalBytes) {
|
||||||
|
const toWrite = Math.min(chunkSize, totalBytes - sent);
|
||||||
|
// Use a deterministic pattern for verification
|
||||||
|
const chunk = Buffer.alloc(toWrite, (sent % 256) & 0xff);
|
||||||
|
const canContinue = socket.write(chunk);
|
||||||
|
sent += toWrite;
|
||||||
|
if (!canContinue) {
|
||||||
|
socket.once('drain', writeChunk);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
socket.end();
|
||||||
|
};
|
||||||
|
writeChunk();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Force-close a server: destroy all connections, then close. */
|
||||||
|
async function forceCloseServer(server: TrackingServer): Promise<void> {
|
||||||
|
server.destroyAll();
|
||||||
|
await new Promise<void>((resolve) => server.close(() => resolve()));
|
||||||
|
}
|
||||||
|
|
||||||
|
interface TestTunnel {
|
||||||
|
hub: RemoteIngressHub;
|
||||||
|
edge: RemoteIngressEdge;
|
||||||
|
edgePort: number;
|
||||||
|
cleanup: () => Promise<void>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Start a full hub + edge tunnel.
|
||||||
|
* Edge binds to 127.0.0.1, upstream server binds to 127.0.0.2.
|
||||||
|
* Hub targetHost = 127.0.0.2 so hub -> upstream doesn't loop back to edge.
|
||||||
|
*/
|
||||||
|
async function startTunnel(edgePort: number, hubPort: number): Promise<TestTunnel> {
|
||||||
|
const hub = new RemoteIngressHub();
|
||||||
|
const edge = new RemoteIngressEdge();
|
||||||
|
|
||||||
|
await hub.start({
|
||||||
|
tunnelPort: hubPort,
|
||||||
|
targetHost: '127.0.0.2',
|
||||||
|
});
|
||||||
|
|
||||||
|
await hub.updateAllowedEdges([
|
||||||
|
{ id: 'test-edge', secret: 'test-secret', listenPorts: [edgePort] },
|
||||||
|
]);
|
||||||
|
|
||||||
|
const connectedPromise = new Promise<void>((resolve, reject) => {
|
||||||
|
const timeout = setTimeout(() => reject(new Error('Edge did not connect within 10s')), 10000);
|
||||||
|
edge.once('tunnelConnected', () => {
|
||||||
|
clearTimeout(timeout);
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
await edge.start({
|
||||||
|
hubHost: '127.0.0.1',
|
||||||
|
hubPort,
|
||||||
|
edgeId: 'test-edge',
|
||||||
|
secret: 'test-secret',
|
||||||
|
bindAddress: '127.0.0.1',
|
||||||
|
});
|
||||||
|
|
||||||
|
await connectedPromise;
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 500));
|
||||||
|
|
||||||
|
return {
|
||||||
|
hub,
|
||||||
|
edge,
|
||||||
|
edgePort,
|
||||||
|
cleanup: async () => {
|
||||||
|
await edge.stop();
|
||||||
|
await hub.stop();
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Send data through the tunnel and collect the echoed response.
|
||||||
|
*/
|
||||||
|
function sendAndReceive(port: number, data: Buffer, timeoutMs = 30000): Promise<Buffer> {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const chunks: Buffer[] = [];
|
||||||
|
let totalReceived = 0;
|
||||||
|
const expectedLength = data.length;
|
||||||
|
let settled = false;
|
||||||
|
|
||||||
|
const client = net.createConnection({ host: '127.0.0.1', port }, () => {
|
||||||
|
client.write(data);
|
||||||
|
client.end();
|
||||||
|
});
|
||||||
|
|
||||||
|
const timer = setTimeout(() => {
|
||||||
|
if (!settled) {
|
||||||
|
settled = true;
|
||||||
|
client.destroy();
|
||||||
|
reject(new Error(`Timeout after ${timeoutMs}ms — received ${totalReceived}/${expectedLength} bytes`));
|
||||||
|
}
|
||||||
|
}, timeoutMs);
|
||||||
|
|
||||||
|
client.on('data', (chunk: Buffer) => {
|
||||||
|
chunks.push(chunk);
|
||||||
|
totalReceived += chunk.length;
|
||||||
|
if (totalReceived >= expectedLength && !settled) {
|
||||||
|
settled = true;
|
||||||
|
clearTimeout(timer);
|
||||||
|
client.destroy();
|
||||||
|
resolve(Buffer.concat(chunks));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
client.on('end', () => {
|
||||||
|
if (!settled) {
|
||||||
|
settled = true;
|
||||||
|
clearTimeout(timer);
|
||||||
|
resolve(Buffer.concat(chunks));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
client.on('error', (err) => {
|
||||||
|
if (!settled) {
|
||||||
|
settled = true;
|
||||||
|
clearTimeout(timer);
|
||||||
|
reject(err);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Connect to the tunnel, send a small request, and collect a large response.
|
||||||
|
* Does NOT call end() — the tunnel has no half-close.
|
||||||
|
* Instead, collects until expectedResponseSize bytes arrive.
|
||||||
|
*/
|
||||||
|
function sendAndReceiveLarge(
|
||||||
|
port: number,
|
||||||
|
data: Buffer,
|
||||||
|
expectedResponseSize: number,
|
||||||
|
timeoutMs = 60000,
|
||||||
|
): Promise<Buffer> {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const chunks: Buffer[] = [];
|
||||||
|
let totalReceived = 0;
|
||||||
|
let settled = false;
|
||||||
|
|
||||||
|
const client = net.createConnection({ host: '127.0.0.1', port }, () => {
|
||||||
|
client.write(data);
|
||||||
|
// Do NOT call client.end() — the server will respond immediately
|
||||||
|
// and the tunnel CLOSE will happen when the download finishes
|
||||||
|
});
|
||||||
|
|
||||||
|
const timer = setTimeout(() => {
|
||||||
|
if (!settled) {
|
||||||
|
settled = true;
|
||||||
|
client.destroy();
|
||||||
|
reject(new Error(`Timeout after ${timeoutMs}ms — received ${totalReceived}/${expectedResponseSize} bytes`));
|
||||||
|
}
|
||||||
|
}, timeoutMs);
|
||||||
|
|
||||||
|
client.on('data', (chunk: Buffer) => {
|
||||||
|
chunks.push(chunk);
|
||||||
|
totalReceived += chunk.length;
|
||||||
|
if (totalReceived >= expectedResponseSize && !settled) {
|
||||||
|
settled = true;
|
||||||
|
clearTimeout(timer);
|
||||||
|
client.destroy();
|
||||||
|
resolve(Buffer.concat(chunks));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
client.on('end', () => {
|
||||||
|
if (!settled) {
|
||||||
|
settled = true;
|
||||||
|
clearTimeout(timer);
|
||||||
|
resolve(Buffer.concat(chunks));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
client.on('error', (err) => {
|
||||||
|
if (!settled) {
|
||||||
|
settled = true;
|
||||||
|
clearTimeout(timer);
|
||||||
|
reject(err);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function sha256(buf: Buffer): string {
|
||||||
|
return crypto.createHash('sha256').update(buf).digest('hex');
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Tests
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
let tunnel: TestTunnel;
|
||||||
|
let echoServer: TrackingServer;
|
||||||
|
let hubPort: number;
|
||||||
|
let edgePort: number;
|
||||||
|
|
||||||
|
tap.test('setup: start echo server and tunnel', async () => {
|
||||||
|
[hubPort, edgePort] = await findFreePorts(2);
|
||||||
|
|
||||||
|
echoServer = await startEchoServer(edgePort, '127.0.0.2');
|
||||||
|
tunnel = await startTunnel(edgePort, hubPort);
|
||||||
|
|
||||||
|
expect(tunnel.hub.running).toBeTrue();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('single stream: 32MB transfer exceeding initial 4MB window', async () => {
|
||||||
|
const size = 32 * 1024 * 1024;
|
||||||
|
const data = crypto.randomBytes(size);
|
||||||
|
const expectedHash = sha256(data);
|
||||||
|
|
||||||
|
const received = await sendAndReceive(edgePort, data, 60000);
|
||||||
|
|
||||||
|
expect(received.length).toEqual(size);
|
||||||
|
expect(sha256(received)).toEqual(expectedHash);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('200 concurrent streams with 64KB each', async () => {
|
||||||
|
const streamCount = 200;
|
||||||
|
const payloadSize = 64 * 1024;
|
||||||
|
|
||||||
|
const promises = Array.from({ length: streamCount }, () => {
|
||||||
|
const data = crypto.randomBytes(payloadSize);
|
||||||
|
const hash = sha256(data);
|
||||||
|
return sendAndReceive(edgePort, data, 30000).then((received) => ({
|
||||||
|
sent: hash,
|
||||||
|
received: sha256(received),
|
||||||
|
sizeOk: received.length === payloadSize,
|
||||||
|
}));
|
||||||
|
});
|
||||||
|
|
||||||
|
const results = await Promise.all(promises);
|
||||||
|
const failures = results.filter((r) => !r.sizeOk || r.sent !== r.received);
|
||||||
|
|
||||||
|
expect(failures.length).toEqual(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('512 concurrent streams at minimum window boundary (16KB each)', async () => {
|
||||||
|
const streamCount = 512;
|
||||||
|
const payloadSize = 16 * 1024;
|
||||||
|
|
||||||
|
const promises = Array.from({ length: streamCount }, () => {
|
||||||
|
const data = crypto.randomBytes(payloadSize);
|
||||||
|
const hash = sha256(data);
|
||||||
|
return sendAndReceive(edgePort, data, 60000).then((received) => ({
|
||||||
|
sent: hash,
|
||||||
|
received: sha256(received),
|
||||||
|
sizeOk: received.length === payloadSize,
|
||||||
|
}));
|
||||||
|
});
|
||||||
|
|
||||||
|
const results = await Promise.all(promises);
|
||||||
|
const failures = results.filter((r) => !r.sizeOk || r.sent !== r.received);
|
||||||
|
|
||||||
|
expect(failures.length).toEqual(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('asymmetric transfer: 4KB request -> 4MB response', async () => {
|
||||||
|
// Swap to large-response server
|
||||||
|
await forceCloseServer(echoServer);
|
||||||
|
const responseSize = 4 * 1024 * 1024; // 4 MB
|
||||||
|
const largeServer = await startLargeResponseServer(edgePort, '127.0.0.2', responseSize);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const requestData = crypto.randomBytes(4 * 1024); // 4 KB
|
||||||
|
const received = await sendAndReceiveLarge(edgePort, requestData, responseSize, 60000);
|
||||||
|
expect(received.length).toEqual(responseSize);
|
||||||
|
} finally {
|
||||||
|
// Always restore echo server even on failure
|
||||||
|
await forceCloseServer(largeServer);
|
||||||
|
echoServer = await startEchoServer(edgePort, '127.0.0.2');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('100 streams x 1MB each (100MB total exceeding 32MB budget)', async () => {
|
||||||
|
const streamCount = 100;
|
||||||
|
const payloadSize = 1 * 1024 * 1024;
|
||||||
|
|
||||||
|
const promises = Array.from({ length: streamCount }, () => {
|
||||||
|
const data = crypto.randomBytes(payloadSize);
|
||||||
|
const hash = sha256(data);
|
||||||
|
return sendAndReceive(edgePort, data, 120000).then((received) => ({
|
||||||
|
sent: hash,
|
||||||
|
received: sha256(received),
|
||||||
|
sizeOk: received.length === payloadSize,
|
||||||
|
}));
|
||||||
|
});
|
||||||
|
|
||||||
|
const results = await Promise.all(promises);
|
||||||
|
const failures = results.filter((r) => !r.sizeOk || r.sent !== r.received);
|
||||||
|
|
||||||
|
expect(failures.length).toEqual(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('active stream counter tracks concurrent connections', async () => {
|
||||||
|
const N = 50;
|
||||||
|
|
||||||
|
// Open N connections and keep them alive (send data but don't close)
|
||||||
|
const sockets: net.Socket[] = [];
|
||||||
|
const connectPromises = Array.from({ length: N }, () => {
|
||||||
|
return new Promise<net.Socket>((resolve, reject) => {
|
||||||
|
const sock = net.createConnection({ host: '127.0.0.1', port: edgePort }, () => {
|
||||||
|
resolve(sock);
|
||||||
|
});
|
||||||
|
sock.on('error', () => {});
|
||||||
|
setTimeout(() => reject(new Error('connect timeout')), 5000);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
const connected = await Promise.all(connectPromises);
|
||||||
|
sockets.push(...connected);
|
||||||
|
|
||||||
|
// Brief delay for stream registration to propagate
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 500));
|
||||||
|
|
||||||
|
// Verify the edge reports >= N active streams.
|
||||||
|
// This counter is the input to compute_window_for_stream_count(),
|
||||||
|
// so its accuracy determines whether adaptive window sizing is correct.
|
||||||
|
const status = await tunnel.edge.getStatus();
|
||||||
|
expect(status.activeStreams).toBeGreaterThanOrEqual(N);
|
||||||
|
|
||||||
|
// Clean up: destroy all sockets (the tunnel's 300s stream timeout will handle cleanup)
|
||||||
|
for (const sock of sockets) {
|
||||||
|
sock.destroy();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('50 streams x 2MB each (forces multiple window refills per stream)', async () => {
|
||||||
|
// At 50 concurrent streams: adaptive window = 32MB/50 = 655KB per stream
|
||||||
|
// Each stream sends 2MB → needs ~3 WINDOW_UPDATE refill cycles per stream
|
||||||
|
const streamCount = 50;
|
||||||
|
const payloadSize = 2 * 1024 * 1024;
|
||||||
|
|
||||||
|
const promises = Array.from({ length: streamCount }, () => {
|
||||||
|
const data = crypto.randomBytes(payloadSize);
|
||||||
|
const hash = sha256(data);
|
||||||
|
return sendAndReceive(edgePort, data, 120000).then((received) => ({
|
||||||
|
sent: hash,
|
||||||
|
received: sha256(received),
|
||||||
|
sizeOk: received.length === payloadSize,
|
||||||
|
}));
|
||||||
|
});
|
||||||
|
|
||||||
|
const results = await Promise.all(promises);
|
||||||
|
const failures = results.filter((r) => !r.sizeOk || r.sent !== r.received);
|
||||||
|
|
||||||
|
expect(failures.length).toEqual(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('teardown: stop tunnel and echo server', async () => {
|
||||||
|
await tunnel.cleanup();
|
||||||
|
await forceCloseServer(echoServer);
|
||||||
|
});
|
||||||
|
|
||||||
|
export default tap.start();
|
||||||
@@ -3,6 +3,6 @@
|
|||||||
*/
|
*/
|
||||||
export const commitinfo = {
|
export const commitinfo = {
|
||||||
name: '@serve.zone/remoteingress',
|
name: '@serve.zone/remoteingress',
|
||||||
version: '4.0.0',
|
version: '4.8.6',
|
||||||
description: 'Edge ingress tunnel for DcRouter - accepts incoming TCP connections at network edge and tunnels them to DcRouter SmartProxy preserving client IP via PROXY protocol v1.'
|
description: 'Edge ingress tunnel for DcRouter - accepts incoming TCP connections at network edge and tunnels them to DcRouter SmartProxy preserving client IP via PROXY protocol v1.'
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ type TEdgeCommands = {
|
|||||||
hubPort: number;
|
hubPort: number;
|
||||||
edgeId: string;
|
edgeId: string;
|
||||||
secret: string;
|
secret: string;
|
||||||
|
bindAddress?: string;
|
||||||
};
|
};
|
||||||
result: { started: boolean };
|
result: { started: boolean };
|
||||||
};
|
};
|
||||||
@@ -38,11 +39,19 @@ export interface IEdgeConfig {
|
|||||||
hubPort?: number;
|
hubPort?: number;
|
||||||
edgeId: string;
|
edgeId: string;
|
||||||
secret: string;
|
secret: string;
|
||||||
|
bindAddress?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const MAX_RESTART_ATTEMPTS = 10;
|
||||||
|
const MAX_RESTART_BACKOFF_MS = 30_000;
|
||||||
|
|
||||||
export class RemoteIngressEdge extends EventEmitter {
|
export class RemoteIngressEdge extends EventEmitter {
|
||||||
private bridge: InstanceType<typeof plugins.smartrust.RustBridge<TEdgeCommands>>;
|
private bridge: InstanceType<typeof plugins.smartrust.RustBridge<TEdgeCommands>>;
|
||||||
private started = false;
|
private started = false;
|
||||||
|
private stopping = false;
|
||||||
|
private savedConfig: IEdgeConfig | null = null;
|
||||||
|
private restartBackoffMs = 1000;
|
||||||
|
private restartAttempts = 0;
|
||||||
private statusInterval: ReturnType<typeof setInterval> | undefined;
|
private statusInterval: ReturnType<typeof setInterval> | undefined;
|
||||||
|
|
||||||
constructor() {
|
constructor() {
|
||||||
@@ -74,8 +83,10 @@ export class RemoteIngressEdge extends EventEmitter {
|
|||||||
this.bridge.on('management:tunnelConnected', () => {
|
this.bridge.on('management:tunnelConnected', () => {
|
||||||
this.emit('tunnelConnected');
|
this.emit('tunnelConnected');
|
||||||
});
|
});
|
||||||
this.bridge.on('management:tunnelDisconnected', () => {
|
this.bridge.on('management:tunnelDisconnected', (data: { reason?: string }) => {
|
||||||
this.emit('tunnelDisconnected');
|
const reason = data?.reason ?? 'unknown';
|
||||||
|
console.log(`[RemoteIngressEdge] Tunnel disconnected: ${reason}`);
|
||||||
|
this.emit('tunnelDisconnected', data);
|
||||||
});
|
});
|
||||||
this.bridge.on('management:publicIpDiscovered', (data: { ip: string }) => {
|
this.bridge.on('management:publicIpDiscovered', (data: { ip: string }) => {
|
||||||
this.emit('publicIpDiscovered', data);
|
this.emit('publicIpDiscovered', data);
|
||||||
@@ -109,19 +120,28 @@ export class RemoteIngressEdge extends EventEmitter {
|
|||||||
edgeConfig = config;
|
edgeConfig = config;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
this.savedConfig = edgeConfig;
|
||||||
|
this.stopping = false;
|
||||||
|
|
||||||
const spawned = await this.bridge.spawn();
|
const spawned = await this.bridge.spawn();
|
||||||
if (!spawned) {
|
if (!spawned) {
|
||||||
throw new Error('Failed to spawn remoteingress-bin');
|
throw new Error('Failed to spawn remoteingress-bin');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Register crash recovery handler
|
||||||
|
this.bridge.on('exit', this.handleCrashRecovery);
|
||||||
|
|
||||||
await this.bridge.sendCommand('startEdge', {
|
await this.bridge.sendCommand('startEdge', {
|
||||||
hubHost: edgeConfig.hubHost,
|
hubHost: edgeConfig.hubHost,
|
||||||
hubPort: edgeConfig.hubPort ?? 8443,
|
hubPort: edgeConfig.hubPort ?? 8443,
|
||||||
edgeId: edgeConfig.edgeId,
|
edgeId: edgeConfig.edgeId,
|
||||||
secret: edgeConfig.secret,
|
secret: edgeConfig.secret,
|
||||||
|
...(edgeConfig.bindAddress ? { bindAddress: edgeConfig.bindAddress } : {}),
|
||||||
});
|
});
|
||||||
|
|
||||||
this.started = true;
|
this.started = true;
|
||||||
|
this.restartAttempts = 0;
|
||||||
|
this.restartBackoffMs = 1000;
|
||||||
|
|
||||||
// Start periodic status logging
|
// Start periodic status logging
|
||||||
this.statusInterval = setInterval(async () => {
|
this.statusInterval = setInterval(async () => {
|
||||||
@@ -142,6 +162,7 @@ export class RemoteIngressEdge extends EventEmitter {
|
|||||||
* Stop the edge and kill the Rust process.
|
* Stop the edge and kill the Rust process.
|
||||||
*/
|
*/
|
||||||
public async stop(): Promise<void> {
|
public async stop(): Promise<void> {
|
||||||
|
this.stopping = true;
|
||||||
if (this.statusInterval) {
|
if (this.statusInterval) {
|
||||||
clearInterval(this.statusInterval);
|
clearInterval(this.statusInterval);
|
||||||
this.statusInterval = undefined;
|
this.statusInterval = undefined;
|
||||||
@@ -152,6 +173,7 @@ export class RemoteIngressEdge extends EventEmitter {
|
|||||||
} catch {
|
} catch {
|
||||||
// Process may already be dead
|
// Process may already be dead
|
||||||
}
|
}
|
||||||
|
this.bridge.removeListener('exit', this.handleCrashRecovery);
|
||||||
this.bridge.kill();
|
this.bridge.kill();
|
||||||
this.started = false;
|
this.started = false;
|
||||||
}
|
}
|
||||||
@@ -170,4 +192,56 @@ export class RemoteIngressEdge extends EventEmitter {
|
|||||||
public get running(): boolean {
|
public get running(): boolean {
|
||||||
return this.bridge.running;
|
return this.bridge.running;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Handle unexpected Rust binary crash — auto-restart with backoff.
|
||||||
|
*/
|
||||||
|
private handleCrashRecovery = async (code: number | null, signal: string | null) => {
|
||||||
|
if (this.stopping || !this.started || !this.savedConfig) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
console.error(
|
||||||
|
`[RemoteIngressEdge] Rust binary crashed (code=${code}, signal=${signal}), ` +
|
||||||
|
`attempt ${this.restartAttempts + 1}/${MAX_RESTART_ATTEMPTS}`
|
||||||
|
);
|
||||||
|
|
||||||
|
this.started = false;
|
||||||
|
|
||||||
|
if (this.restartAttempts >= MAX_RESTART_ATTEMPTS) {
|
||||||
|
console.error('[RemoteIngressEdge] Max restart attempts reached, giving up');
|
||||||
|
this.emit('crashRecoveryFailed');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await new Promise(resolve => setTimeout(resolve, this.restartBackoffMs));
|
||||||
|
this.restartBackoffMs = Math.min(this.restartBackoffMs * 2, MAX_RESTART_BACKOFF_MS);
|
||||||
|
this.restartAttempts++;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const spawned = await this.bridge.spawn();
|
||||||
|
if (!spawned) {
|
||||||
|
console.error('[RemoteIngressEdge] Failed to respawn binary');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.bridge.on('exit', this.handleCrashRecovery);
|
||||||
|
|
||||||
|
await this.bridge.sendCommand('startEdge', {
|
||||||
|
hubHost: this.savedConfig.hubHost,
|
||||||
|
hubPort: this.savedConfig.hubPort ?? 8443,
|
||||||
|
edgeId: this.savedConfig.edgeId,
|
||||||
|
secret: this.savedConfig.secret,
|
||||||
|
...(this.savedConfig.bindAddress ? { bindAddress: this.savedConfig.bindAddress } : {}),
|
||||||
|
});
|
||||||
|
|
||||||
|
this.started = true;
|
||||||
|
this.restartAttempts = 0;
|
||||||
|
this.restartBackoffMs = 1000;
|
||||||
|
console.log('[RemoteIngressEdge] Successfully recovered from crash');
|
||||||
|
this.emit('crashRecovered');
|
||||||
|
} catch (err) {
|
||||||
|
console.error(`[RemoteIngressEdge] Crash recovery failed: ${err}`);
|
||||||
|
}
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,6 +11,8 @@ type THubCommands = {
|
|||||||
params: {
|
params: {
|
||||||
tunnelPort: number;
|
tunnelPort: number;
|
||||||
targetHost?: string;
|
targetHost?: string;
|
||||||
|
tlsCertPem?: string;
|
||||||
|
tlsKeyPem?: string;
|
||||||
};
|
};
|
||||||
result: { started: boolean };
|
result: { started: boolean };
|
||||||
};
|
};
|
||||||
@@ -33,6 +35,7 @@ type THubCommands = {
|
|||||||
edgeId: string;
|
edgeId: string;
|
||||||
connectedAt: number;
|
connectedAt: number;
|
||||||
activeStreams: number;
|
activeStreams: number;
|
||||||
|
peerAddr: string;
|
||||||
}>;
|
}>;
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
@@ -41,11 +44,25 @@ type THubCommands = {
|
|||||||
export interface IHubConfig {
|
export interface IHubConfig {
|
||||||
tunnelPort?: number;
|
tunnelPort?: number;
|
||||||
targetHost?: string;
|
targetHost?: string;
|
||||||
|
tls?: {
|
||||||
|
certPem?: string;
|
||||||
|
keyPem?: string;
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type TAllowedEdge = { id: string; secret: string; listenPorts?: number[]; stunIntervalSecs?: number };
|
||||||
|
|
||||||
|
const MAX_RESTART_ATTEMPTS = 10;
|
||||||
|
const MAX_RESTART_BACKOFF_MS = 30_000;
|
||||||
|
|
||||||
export class RemoteIngressHub extends EventEmitter {
|
export class RemoteIngressHub extends EventEmitter {
|
||||||
private bridge: InstanceType<typeof plugins.smartrust.RustBridge<THubCommands>>;
|
private bridge: InstanceType<typeof plugins.smartrust.RustBridge<THubCommands>>;
|
||||||
private started = false;
|
private started = false;
|
||||||
|
private stopping = false;
|
||||||
|
private savedConfig: IHubConfig | null = null;
|
||||||
|
private savedEdges: TAllowedEdge[] = [];
|
||||||
|
private restartBackoffMs = 1000;
|
||||||
|
private restartAttempts = 0;
|
||||||
|
|
||||||
constructor() {
|
constructor() {
|
||||||
super();
|
super();
|
||||||
@@ -73,10 +90,12 @@ export class RemoteIngressHub extends EventEmitter {
|
|||||||
});
|
});
|
||||||
|
|
||||||
// Forward events from Rust binary
|
// Forward events from Rust binary
|
||||||
this.bridge.on('management:edgeConnected', (data: { edgeId: string }) => {
|
this.bridge.on('management:edgeConnected', (data: { edgeId: string; peerAddr: string }) => {
|
||||||
this.emit('edgeConnected', data);
|
this.emit('edgeConnected', data);
|
||||||
});
|
});
|
||||||
this.bridge.on('management:edgeDisconnected', (data: { edgeId: string }) => {
|
this.bridge.on('management:edgeDisconnected', (data: { edgeId: string; reason?: string }) => {
|
||||||
|
const reason = data?.reason ?? 'unknown';
|
||||||
|
console.log(`[RemoteIngressHub] Edge ${data.edgeId} disconnected: ${reason}`);
|
||||||
this.emit('edgeDisconnected', data);
|
this.emit('edgeDisconnected', data);
|
||||||
});
|
});
|
||||||
this.bridge.on('management:streamOpened', (data: { edgeId: string; streamId: number }) => {
|
this.bridge.on('management:streamOpened', (data: { edgeId: string; streamId: number }) => {
|
||||||
@@ -91,29 +110,42 @@ export class RemoteIngressHub extends EventEmitter {
|
|||||||
* Start the hub — spawns the Rust binary and starts the tunnel server.
|
* Start the hub — spawns the Rust binary and starts the tunnel server.
|
||||||
*/
|
*/
|
||||||
public async start(config: IHubConfig = {}): Promise<void> {
|
public async start(config: IHubConfig = {}): Promise<void> {
|
||||||
|
this.savedConfig = config;
|
||||||
|
this.stopping = false;
|
||||||
|
|
||||||
const spawned = await this.bridge.spawn();
|
const spawned = await this.bridge.spawn();
|
||||||
if (!spawned) {
|
if (!spawned) {
|
||||||
throw new Error('Failed to spawn remoteingress-bin');
|
throw new Error('Failed to spawn remoteingress-bin');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Register crash recovery handler
|
||||||
|
this.bridge.on('exit', this.handleCrashRecovery);
|
||||||
|
|
||||||
await this.bridge.sendCommand('startHub', {
|
await this.bridge.sendCommand('startHub', {
|
||||||
tunnelPort: config.tunnelPort ?? 8443,
|
tunnelPort: config.tunnelPort ?? 8443,
|
||||||
targetHost: config.targetHost ?? '127.0.0.1',
|
targetHost: config.targetHost ?? '127.0.0.1',
|
||||||
|
...(config.tls?.certPem && config.tls?.keyPem
|
||||||
|
? { tlsCertPem: config.tls.certPem, tlsKeyPem: config.tls.keyPem }
|
||||||
|
: {}),
|
||||||
});
|
});
|
||||||
|
|
||||||
this.started = true;
|
this.started = true;
|
||||||
|
this.restartAttempts = 0;
|
||||||
|
this.restartBackoffMs = 1000;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Stop the hub and kill the Rust process.
|
* Stop the hub and kill the Rust process.
|
||||||
*/
|
*/
|
||||||
public async stop(): Promise<void> {
|
public async stop(): Promise<void> {
|
||||||
|
this.stopping = true;
|
||||||
if (this.started) {
|
if (this.started) {
|
||||||
try {
|
try {
|
||||||
await this.bridge.sendCommand('stopHub', {} as Record<string, never>);
|
await this.bridge.sendCommand('stopHub', {} as Record<string, never>);
|
||||||
} catch {
|
} catch {
|
||||||
// Process may already be dead
|
// Process may already be dead
|
||||||
}
|
}
|
||||||
|
this.bridge.removeListener('exit', this.handleCrashRecovery);
|
||||||
this.bridge.kill();
|
this.bridge.kill();
|
||||||
this.started = false;
|
this.started = false;
|
||||||
}
|
}
|
||||||
@@ -122,7 +154,8 @@ export class RemoteIngressHub extends EventEmitter {
|
|||||||
/**
|
/**
|
||||||
* Update the list of allowed edges that can connect to this hub.
|
* Update the list of allowed edges that can connect to this hub.
|
||||||
*/
|
*/
|
||||||
public async updateAllowedEdges(edges: Array<{ id: string; secret: string; listenPorts?: number[]; stunIntervalSecs?: number }>): Promise<void> {
|
public async updateAllowedEdges(edges: TAllowedEdge[]): Promise<void> {
|
||||||
|
this.savedEdges = edges;
|
||||||
await this.bridge.sendCommand('updateAllowedEdges', { edges });
|
await this.bridge.sendCommand('updateAllowedEdges', { edges });
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -139,4 +172,62 @@ export class RemoteIngressHub extends EventEmitter {
|
|||||||
public get running(): boolean {
|
public get running(): boolean {
|
||||||
return this.bridge.running;
|
return this.bridge.running;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Handle unexpected Rust binary crash — auto-restart with backoff.
|
||||||
|
*/
|
||||||
|
private handleCrashRecovery = async (code: number | null, signal: string | null) => {
|
||||||
|
if (this.stopping || !this.started || !this.savedConfig) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
console.error(
|
||||||
|
`[RemoteIngressHub] Rust binary crashed (code=${code}, signal=${signal}), ` +
|
||||||
|
`attempt ${this.restartAttempts + 1}/${MAX_RESTART_ATTEMPTS}`
|
||||||
|
);
|
||||||
|
|
||||||
|
this.started = false;
|
||||||
|
|
||||||
|
if (this.restartAttempts >= MAX_RESTART_ATTEMPTS) {
|
||||||
|
console.error('[RemoteIngressHub] Max restart attempts reached, giving up');
|
||||||
|
this.emit('crashRecoveryFailed');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await new Promise(resolve => setTimeout(resolve, this.restartBackoffMs));
|
||||||
|
this.restartBackoffMs = Math.min(this.restartBackoffMs * 2, MAX_RESTART_BACKOFF_MS);
|
||||||
|
this.restartAttempts++;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const spawned = await this.bridge.spawn();
|
||||||
|
if (!spawned) {
|
||||||
|
console.error('[RemoteIngressHub] Failed to respawn binary');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.bridge.on('exit', this.handleCrashRecovery);
|
||||||
|
|
||||||
|
const config = this.savedConfig;
|
||||||
|
await this.bridge.sendCommand('startHub', {
|
||||||
|
tunnelPort: config.tunnelPort ?? 8443,
|
||||||
|
targetHost: config.targetHost ?? '127.0.0.1',
|
||||||
|
...(config.tls?.certPem && config.tls?.keyPem
|
||||||
|
? { tlsCertPem: config.tls.certPem, tlsKeyPem: config.tls.keyPem }
|
||||||
|
: {}),
|
||||||
|
});
|
||||||
|
|
||||||
|
// Restore allowed edges
|
||||||
|
if (this.savedEdges.length > 0) {
|
||||||
|
await this.bridge.sendCommand('updateAllowedEdges', { edges: this.savedEdges });
|
||||||
|
}
|
||||||
|
|
||||||
|
this.started = true;
|
||||||
|
this.restartAttempts = 0;
|
||||||
|
this.restartBackoffMs = 1000;
|
||||||
|
console.log('[RemoteIngressHub] Successfully recovered from crash');
|
||||||
|
this.emit('crashRecovered');
|
||||||
|
} catch (err) {
|
||||||
|
console.error(`[RemoteIngressHub] Crash recovery failed: ${err}`);
|
||||||
|
}
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user