feat(core): add performance profiles, transport observability, and edge stream budget controls
This commit is contained in:
+19
-1
@@ -1,5 +1,23 @@
|
|||||||
# Changelog
|
# Changelog
|
||||||
|
|
||||||
|
## 2026-04-26 - 4.17.0 - feat(core)
|
||||||
|
add performance profiles, transport observability, and edge stream budget controls
|
||||||
|
|
||||||
|
- introduce configurable performance profiles and effective per-edge limits for stream concurrency, flow-control windows, and QUIC datagram buffers
|
||||||
|
- expose hub-side edge status for transport mode, fallback usage, flow-control, queue depths, traffic counters, and UDP session metrics
|
||||||
|
- enforce edge-side stream admission limits before spawning client tunnel tasks and make TCP/TLS window sizing honor edge memory budgets under high concurrency
|
||||||
|
- increase QUIC datagram receive buffer configurability and improve hub-side QUIC UDP session tracking and idle pruning
|
||||||
|
- update hub APIs and documentation to support performance configuration and clarify quicWithFallback as the default edge transport
|
||||||
|
|
||||||
|
## 2026-04-26 - 4.16.0 - feat(performance)
|
||||||
|
add remote ingress performance controls and runtime observability
|
||||||
|
|
||||||
|
- add performance profiles and configurable stream/window budgets for hub and edge connections
|
||||||
|
- expose per-edge transport, flow-control, queue, traffic, and UDP status from hub status
|
||||||
|
- enforce edge-side stream admission before spawning client tunnel tasks
|
||||||
|
- make TCP/TLS flow control honor an edge-level memory budget under high concurrency
|
||||||
|
- increase QUIC datagram receive buffers and prune idle hub-side QUIC UDP sessions
|
||||||
|
|
||||||
## 2026-03-27 - 4.15.3 - fix(core)
|
## 2026-03-27 - 4.15.3 - fix(core)
|
||||||
harden UDP session handling, QUIC control message validation, and bridge process cleanup
|
harden UDP session handling, QUIC control message validation, and bridge process cleanup
|
||||||
|
|
||||||
@@ -516,4 +534,4 @@ Core updates and fixes.
|
|||||||
## 2024-03-24 - 1.0.1 - core
|
## 2024-03-24 - 1.0.1 - core
|
||||||
Core updates and fixes.
|
Core updates and fixes.
|
||||||
|
|
||||||
- fix(core): update
|
- fix(core): update
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@serve.zone/remoteingress",
|
"name": "@serve.zone/remoteingress",
|
||||||
"version": "4.15.3",
|
"version": "4.16.0",
|
||||||
"private": false,
|
"private": false,
|
||||||
"description": "Edge ingress tunnel for DcRouter - tunnels TCP and UDP traffic from the network edge to SmartProxy over TLS or QUIC, preserving client IP via PROXY protocol.",
|
"description": "Edge ingress tunnel for DcRouter - tunnels TCP and UDP traffic from the network edge to SmartProxy over TLS or QUIC, preserving client IP via PROXY protocol.",
|
||||||
"main": "dist_ts/index.js",
|
"main": "dist_ts/index.js",
|
||||||
|
|||||||
@@ -130,7 +130,7 @@ await hub.stop();
|
|||||||
|
|
||||||
### Setting Up the Edge (Network Edge Side)
|
### Setting Up the Edge (Network Edge Side)
|
||||||
|
|
||||||
The edge can connect via **TCP+TLS** (default) or **QUIC** transport. Edges run as **root** so they can bind to privileged ports and apply nftables firewall rules.
|
The edge connects via **QUIC with TCP+TLS fallback** by default. Edges run as **root** so they can bind to privileged ports and apply nftables firewall rules.
|
||||||
|
|
||||||
#### Option A: Connection Token (Recommended)
|
#### Option A: Connection Token (Recommended)
|
||||||
|
|
||||||
@@ -162,7 +162,7 @@ await edge.start({
|
|||||||
hubPort: 8443,
|
hubPort: 8443,
|
||||||
edgeId: 'edge-nyc-01',
|
edgeId: 'edge-nyc-01',
|
||||||
secret: 'supersecrettoken1',
|
secret: 'supersecrettoken1',
|
||||||
transportMode: 'quic', // 'tcpTls' (default) | 'quic' | 'quicWithFallback'
|
transportMode: 'quic', // 'tcpTls' | 'quic' | 'quicWithFallback' (default)
|
||||||
});
|
});
|
||||||
|
|
||||||
const edgeStatus = await edge.getStatus();
|
const edgeStatus = await edge.getStatus();
|
||||||
@@ -175,9 +175,9 @@ await edge.stop();
|
|||||||
|
|
||||||
| Mode | Description |
|
| Mode | Description |
|
||||||
|------|-------------|
|
|------|-------------|
|
||||||
| `'tcpTls'` | **Default.** Single TLS connection with frame-based multiplexing. Universal compatibility. |
|
| `'tcpTls'` | Single TLS connection with frame-based multiplexing. Universal compatibility. |
|
||||||
| `'quic'` | QUIC with native stream multiplexing. Eliminates head-of-line blocking. Uses QUIC datagrams for UDP traffic. |
|
| `'quic'` | QUIC with native stream multiplexing. Eliminates head-of-line blocking. Uses QUIC datagrams for UDP traffic. |
|
||||||
| `'quicWithFallback'` | Tries QUIC first (5s timeout), falls back to TCP+TLS if UDP is blocked by the network. |
|
| `'quicWithFallback'` | **Default.** Tries QUIC first (5s timeout), falls back to TCP+TLS if UDP is blocked by the network. |
|
||||||
|
|
||||||
### Connection Tokens
|
### Connection Tokens
|
||||||
|
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ use serde::{Deserialize, Serialize};
|
|||||||
|
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
use remoteingress_protocol::*;
|
use remoteingress_protocol::*;
|
||||||
|
use crate::performance::EffectivePerformanceConfig;
|
||||||
use crate::transport::TransportMode;
|
use crate::transport::TransportMode;
|
||||||
use crate::transport::quic as quic_transport;
|
use crate::transport::quic as quic_transport;
|
||||||
use crate::udp_session::{UdpSessionKey, UdpSessionManager};
|
use crate::udp_session::{UdpSessionKey, UdpSessionManager};
|
||||||
@@ -53,7 +54,7 @@ pub struct EdgeConfig {
|
|||||||
/// Useful for testing on localhost where edge and upstream share the same machine.
|
/// Useful for testing on localhost where edge and upstream share the same machine.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub bind_address: Option<String>,
|
pub bind_address: Option<String>,
|
||||||
/// Transport mode for the tunnel connection (defaults to TcpTls).
|
/// Transport mode for the tunnel connection (defaults to QuicWithFallback).
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub transport_mode: Option<TransportMode>,
|
pub transport_mode: Option<TransportMode>,
|
||||||
}
|
}
|
||||||
@@ -69,12 +70,53 @@ struct HandshakeConfig {
|
|||||||
stun_interval_secs: u64,
|
stun_interval_secs: u64,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
firewall_config: Option<serde_json::Value>,
|
firewall_config: Option<serde_json::Value>,
|
||||||
|
#[serde(default)]
|
||||||
|
performance: EffectivePerformanceConfig,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn default_stun_interval() -> u64 {
|
fn default_stun_interval() -> u64 {
|
||||||
300
|
300
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn try_reserve_stream(active_streams: &AtomicU32, max_streams: usize) -> bool {
|
||||||
|
let max_streams = max_streams.min(u32::MAX as usize) as u32;
|
||||||
|
loop {
|
||||||
|
let current = active_streams.load(Ordering::Relaxed);
|
||||||
|
if current >= max_streams {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if active_streams
|
||||||
|
.compare_exchange_weak(current, current + 1, Ordering::Relaxed, Ordering::Relaxed)
|
||||||
|
.is_ok()
|
||||||
|
{
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn release_stream(active_streams: &AtomicU32) {
|
||||||
|
loop {
|
||||||
|
let current = active_streams.load(Ordering::Relaxed);
|
||||||
|
if current == 0 {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if active_streams
|
||||||
|
.compare_exchange_weak(current, current - 1, Ordering::Relaxed, Ordering::Relaxed)
|
||||||
|
.is_ok()
|
||||||
|
{
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn transport_mode_wire_name(mode: TransportMode) -> &'static str {
|
||||||
|
match mode {
|
||||||
|
TransportMode::TcpTls => "tcpTls",
|
||||||
|
TransportMode::Quic => "quic",
|
||||||
|
TransportMode::QuicWithFallback => "quicWithFallback",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Runtime config update received from hub via FRAME_CONFIG.
|
/// Runtime config update received from hub via FRAME_CONFIG.
|
||||||
#[derive(Debug, Clone, Deserialize)]
|
#[derive(Debug, Clone, Deserialize)]
|
||||||
#[serde(rename_all = "camelCase")]
|
#[serde(rename_all = "camelCase")]
|
||||||
@@ -84,6 +126,8 @@ struct ConfigUpdate {
|
|||||||
listen_ports_udp: Vec<u16>,
|
listen_ports_udp: Vec<u16>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
firewall_config: Option<serde_json::Value>,
|
firewall_config: Option<serde_json::Value>,
|
||||||
|
#[serde(default)]
|
||||||
|
performance: EffectivePerformanceConfig,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Events emitted by the edge.
|
/// Events emitted by the edge.
|
||||||
@@ -378,6 +422,7 @@ async fn handle_edge_frame(
|
|||||||
bind_address: &str,
|
bind_address: &str,
|
||||||
udp_sessions: &Arc<Mutex<UdpSessionManager>>,
|
udp_sessions: &Arc<Mutex<UdpSessionManager>>,
|
||||||
udp_sockets: &Arc<Mutex<HashMap<u16, Arc<UdpSocket>>>>,
|
udp_sockets: &Arc<Mutex<HashMap<u16, Arc<UdpSocket>>>>,
|
||||||
|
performance: &mut EffectivePerformanceConfig,
|
||||||
) -> EdgeFrameAction {
|
) -> EdgeFrameAction {
|
||||||
match frame.frame_type {
|
match frame.frame_type {
|
||||||
FRAME_DATA_BACK => {
|
FRAME_DATA_BACK => {
|
||||||
@@ -398,8 +443,8 @@ async fn handle_edge_frame(
|
|||||||
let writers = client_writers.lock().await;
|
let writers = client_writers.lock().await;
|
||||||
if let Some(state) = writers.get(&frame.stream_id) {
|
if let Some(state) = writers.get(&frame.stream_id) {
|
||||||
let prev = state.send_window.fetch_add(increment, Ordering::Release);
|
let prev = state.send_window.fetch_add(increment, Ordering::Release);
|
||||||
if prev + increment > MAX_WINDOW_SIZE {
|
if prev + increment > performance.max_stream_window_bytes {
|
||||||
state.send_window.store(MAX_WINDOW_SIZE, Ordering::Release);
|
state.send_window.store(performance.max_stream_window_bytes, Ordering::Release);
|
||||||
}
|
}
|
||||||
state.window_notify.notify_one();
|
state.window_notify.notify_one();
|
||||||
}
|
}
|
||||||
@@ -417,6 +462,7 @@ async fn handle_edge_frame(
|
|||||||
FRAME_CONFIG => {
|
FRAME_CONFIG => {
|
||||||
if let Ok(update) = serde_json::from_slice::<ConfigUpdate>(&frame.payload) {
|
if let Ok(update) = serde_json::from_slice::<ConfigUpdate>(&frame.payload) {
|
||||||
log::info!("Config update from hub: ports {:?}, udp {:?}", update.listen_ports, update.listen_ports_udp);
|
log::info!("Config update from hub: ports {:?}, udp {:?}", update.listen_ports, update.listen_ports_udp);
|
||||||
|
*performance = update.performance.clone();
|
||||||
*listen_ports.write().await = update.listen_ports.clone();
|
*listen_ports.write().await = update.listen_ports.clone();
|
||||||
let _ = event_tx.try_send(EdgeEvent::PortsUpdated {
|
let _ = event_tx.try_send(EdgeEvent::PortsUpdated {
|
||||||
listen_ports: update.listen_ports.clone(),
|
listen_ports: update.listen_ports.clone(),
|
||||||
@@ -433,6 +479,7 @@ async fn handle_edge_frame(
|
|||||||
edge_id,
|
edge_id,
|
||||||
connection_token,
|
connection_token,
|
||||||
bind_address,
|
bind_address,
|
||||||
|
performance,
|
||||||
);
|
);
|
||||||
apply_udp_port_config(
|
apply_udp_port_config(
|
||||||
&update.listen_ports_udp,
|
&update.listen_ports_udp,
|
||||||
@@ -524,7 +571,13 @@ async fn connect_to_hub_and_run(
|
|||||||
};
|
};
|
||||||
|
|
||||||
// Send auth line (we own the whole stream — no split)
|
// Send auth line (we own the whole stream — no split)
|
||||||
let auth_line = format!("EDGE {} {}\n", config.edge_id, config.secret);
|
let requested_transport = config.transport_mode.unwrap_or(TransportMode::QuicWithFallback);
|
||||||
|
let auth_line = format!(
|
||||||
|
"EDGE {} {} {}\n",
|
||||||
|
config.edge_id,
|
||||||
|
config.secret,
|
||||||
|
transport_mode_wire_name(requested_transport),
|
||||||
|
);
|
||||||
if tls_stream.write_all(auth_line.as_bytes()).await.is_err() {
|
if tls_stream.write_all(auth_line.as_bytes()).await.is_err() {
|
||||||
return EdgeLoopResult::Reconnect("auth_write_failed".to_string());
|
return EdgeLoopResult::Reconnect("auth_write_failed".to_string());
|
||||||
}
|
}
|
||||||
@@ -563,6 +616,7 @@ async fn connect_to_hub_and_run(
|
|||||||
return EdgeLoopResult::Reconnect(format!("handshake_invalid: {}", e));
|
return EdgeLoopResult::Reconnect(format!("handshake_invalid: {}", e));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
let mut performance = handshake.performance.clone();
|
||||||
|
|
||||||
log::info!(
|
log::info!(
|
||||||
"Handshake from hub: ports {:?}, stun_interval {}s",
|
"Handshake from hub: ports {:?}, stun_interval {}s",
|
||||||
@@ -640,6 +694,7 @@ async fn connect_to_hub_and_run(
|
|||||||
&config.edge_id,
|
&config.edge_id,
|
||||||
connection_token,
|
connection_token,
|
||||||
bind_address,
|
bind_address,
|
||||||
|
&performance,
|
||||||
);
|
);
|
||||||
|
|
||||||
// UDP session manager + listeners
|
// UDP session manager + listeners
|
||||||
@@ -700,7 +755,7 @@ async fn connect_to_hub_and_run(
|
|||||||
frame, &mut tunnel_io, &client_writers, listen_ports, event_tx,
|
frame, &mut tunnel_io, &client_writers, listen_ports, event_tx,
|
||||||
&tunnel_writer_tx, &tunnel_data_tx, &tunnel_sustained_tx, &mut port_listeners,
|
&tunnel_writer_tx, &tunnel_data_tx, &tunnel_sustained_tx, &mut port_listeners,
|
||||||
&mut udp_listeners, active_streams, next_stream_id, &config.edge_id,
|
&mut udp_listeners, active_streams, next_stream_id, &config.edge_id,
|
||||||
connection_token, bind_address, &udp_sessions, &udp_sockets,
|
connection_token, bind_address, &udp_sessions, &udp_sockets, &mut performance,
|
||||||
).await {
|
).await {
|
||||||
break 'io_loop EdgeLoopResult::Reconnect(reason);
|
break 'io_loop EdgeLoopResult::Reconnect(reason);
|
||||||
}
|
}
|
||||||
@@ -719,7 +774,7 @@ async fn connect_to_hub_and_run(
|
|||||||
frame, &mut tunnel_io, &client_writers, listen_ports, event_tx,
|
frame, &mut tunnel_io, &client_writers, listen_ports, event_tx,
|
||||||
&tunnel_writer_tx, &tunnel_data_tx, &tunnel_sustained_tx, &mut port_listeners,
|
&tunnel_writer_tx, &tunnel_data_tx, &tunnel_sustained_tx, &mut port_listeners,
|
||||||
&mut udp_listeners, active_streams, next_stream_id, &config.edge_id,
|
&mut udp_listeners, active_streams, next_stream_id, &config.edge_id,
|
||||||
connection_token, bind_address, &udp_sessions, &udp_sockets,
|
connection_token, bind_address, &udp_sessions, &udp_sockets, &mut performance,
|
||||||
).await {
|
).await {
|
||||||
break EdgeLoopResult::Reconnect(reason);
|
break EdgeLoopResult::Reconnect(reason);
|
||||||
}
|
}
|
||||||
@@ -785,6 +840,7 @@ fn apply_port_config(
|
|||||||
edge_id: &str,
|
edge_id: &str,
|
||||||
connection_token: &CancellationToken,
|
connection_token: &CancellationToken,
|
||||||
bind_address: &str,
|
bind_address: &str,
|
||||||
|
performance: &EffectivePerformanceConfig,
|
||||||
) {
|
) {
|
||||||
let new_set: std::collections::HashSet<u16> = new_ports.iter().copied().collect();
|
let new_set: std::collections::HashSet<u16> = new_ports.iter().copied().collect();
|
||||||
let old_set: std::collections::HashSet<u16> = port_listeners.keys().copied().collect();
|
let old_set: std::collections::HashSet<u16> = port_listeners.keys().copied().collect();
|
||||||
@@ -807,6 +863,7 @@ fn apply_port_config(
|
|||||||
let next_stream_id = next_stream_id.clone();
|
let next_stream_id = next_stream_id.clone();
|
||||||
let edge_id = edge_id.to_string();
|
let edge_id = edge_id.to_string();
|
||||||
let port_token = connection_token.child_token();
|
let port_token = connection_token.child_token();
|
||||||
|
let performance = performance.clone();
|
||||||
|
|
||||||
let bind_addr = bind_address.to_string();
|
let bind_addr = bind_address.to_string();
|
||||||
let handle = tokio::spawn(async move {
|
let handle = tokio::spawn(async move {
|
||||||
@@ -842,8 +899,12 @@ fn apply_port_config(
|
|||||||
let edge_id = edge_id.clone();
|
let edge_id = edge_id.clone();
|
||||||
let client_token = port_token.child_token();
|
let client_token = port_token.child_token();
|
||||||
|
|
||||||
active_streams.fetch_add(1, Ordering::Relaxed);
|
if !try_reserve_stream(&active_streams, performance.max_streams_per_edge) {
|
||||||
|
log::warn!("Rejecting client on port {}: max streams ({}) reached", port, performance.max_streams_per_edge);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let performance = performance.clone();
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
handle_client_connection(
|
handle_client_connection(
|
||||||
client_stream,
|
client_stream,
|
||||||
@@ -857,20 +918,10 @@ fn apply_port_config(
|
|||||||
client_writers,
|
client_writers,
|
||||||
client_token,
|
client_token,
|
||||||
Arc::clone(&active_streams),
|
Arc::clone(&active_streams),
|
||||||
|
performance,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
// Saturating decrement: prevent underflow when
|
release_stream(&active_streams);
|
||||||
// edge_main_loop's store(0) races with task cleanup.
|
|
||||||
loop {
|
|
||||||
let current = active_streams.load(Ordering::Relaxed);
|
|
||||||
if current == 0 { break; }
|
|
||||||
if active_streams.compare_exchange_weak(
|
|
||||||
current, current - 1,
|
|
||||||
Ordering::Relaxed, Ordering::Relaxed,
|
|
||||||
).is_ok() {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
@@ -1005,6 +1056,7 @@ async fn handle_client_connection(
|
|||||||
client_writers: Arc<Mutex<HashMap<u32, EdgeStreamState>>>,
|
client_writers: Arc<Mutex<HashMap<u32, EdgeStreamState>>>,
|
||||||
client_token: CancellationToken,
|
client_token: CancellationToken,
|
||||||
active_streams: Arc<AtomicU32>,
|
active_streams: Arc<AtomicU32>,
|
||||||
|
performance: EffectivePerformanceConfig,
|
||||||
) {
|
) {
|
||||||
let client_ip = client_addr.ip().to_string();
|
let client_ip = client_addr.ip().to_string();
|
||||||
let client_port = client_addr.port();
|
let client_port = client_addr.port();
|
||||||
@@ -1028,9 +1080,12 @@ async fn handle_client_connection(
|
|||||||
// streams due to channel overflow — backpressure slows streams, never kills them.
|
// streams due to channel overflow — backpressure slows streams, never kills them.
|
||||||
let (back_tx, mut back_rx) = mpsc::unbounded_channel::<Bytes>();
|
let (back_tx, mut back_rx) = mpsc::unbounded_channel::<Bytes>();
|
||||||
// Adaptive initial window: scale with current stream count to keep total in-flight
|
// Adaptive initial window: scale with current stream count to keep total in-flight
|
||||||
// data within the 200MB budget. Prevents burst flooding when many streams open.
|
// data within the configured edge budget. Prevents burst flooding when many streams open.
|
||||||
let initial_window = remoteingress_protocol::compute_window_for_stream_count(
|
let initial_window = remoteingress_protocol::compute_window_for_limits(
|
||||||
active_streams.load(Ordering::Relaxed),
|
active_streams.load(Ordering::Relaxed),
|
||||||
|
performance.total_window_budget_bytes,
|
||||||
|
performance.min_stream_window_bytes,
|
||||||
|
performance.max_stream_window_bytes,
|
||||||
);
|
);
|
||||||
let send_window = Arc::new(AtomicU32::new(initial_window));
|
let send_window = Arc::new(AtomicU32::new(initial_window));
|
||||||
let window_notify = Arc::new(Notify::new());
|
let window_notify = Arc::new(Notify::new());
|
||||||
@@ -1067,8 +1122,11 @@ async fn handle_client_connection(
|
|||||||
// effective window shrinks to match current demand (fewer streams
|
// effective window shrinks to match current demand (fewer streams
|
||||||
// = larger window, more streams = smaller window per stream).
|
// = larger window, more streams = smaller window per stream).
|
||||||
consumed_since_update += len;
|
consumed_since_update += len;
|
||||||
let adaptive_window = remoteingress_protocol::compute_window_for_stream_count(
|
let adaptive_window = remoteingress_protocol::compute_window_for_limits(
|
||||||
active_streams_h2c.load(Ordering::Relaxed),
|
active_streams_h2c.load(Ordering::Relaxed),
|
||||||
|
performance.total_window_budget_bytes,
|
||||||
|
performance.min_stream_window_bytes,
|
||||||
|
performance.max_stream_window_bytes,
|
||||||
);
|
);
|
||||||
let threshold = adaptive_window / 2;
|
let threshold = adaptive_window / 2;
|
||||||
if consumed_since_update >= threshold {
|
if consumed_since_update >= threshold {
|
||||||
@@ -1282,7 +1340,13 @@ async fn connect_to_hub_and_run_quic_with_connection(
|
|||||||
};
|
};
|
||||||
|
|
||||||
// Auth handshake on control stream (same protocol as TCP+TLS)
|
// Auth handshake on control stream (same protocol as TCP+TLS)
|
||||||
let auth_line = format!("EDGE {} {}\n", config.edge_id, config.secret);
|
let requested_transport = config.transport_mode.unwrap_or(TransportMode::QuicWithFallback);
|
||||||
|
let auth_line = format!(
|
||||||
|
"EDGE {} {} {}\n",
|
||||||
|
config.edge_id,
|
||||||
|
config.secret,
|
||||||
|
transport_mode_wire_name(requested_transport),
|
||||||
|
);
|
||||||
if let Err(e) = ctrl_send.write_all(auth_line.as_bytes()).await {
|
if let Err(e) = ctrl_send.write_all(auth_line.as_bytes()).await {
|
||||||
return EdgeLoopResult::Reconnect(format!("quic_auth_write_failed: {}", e));
|
return EdgeLoopResult::Reconnect(format!("quic_auth_write_failed: {}", e));
|
||||||
}
|
}
|
||||||
@@ -1314,6 +1378,7 @@ async fn connect_to_hub_and_run_quic_with_connection(
|
|||||||
return EdgeLoopResult::Reconnect(format!("quic_handshake_invalid: {}", e));
|
return EdgeLoopResult::Reconnect(format!("quic_handshake_invalid: {}", e));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
let mut performance = handshake.performance.clone();
|
||||||
|
|
||||||
log::info!(
|
log::info!(
|
||||||
"QUIC handshake from hub: ports {:?}, stun_interval {}s",
|
"QUIC handshake from hub: ports {:?}, stun_interval {}s",
|
||||||
@@ -1377,6 +1442,7 @@ async fn connect_to_hub_and_run_quic_with_connection(
|
|||||||
&config.edge_id,
|
&config.edge_id,
|
||||||
connection_token,
|
connection_token,
|
||||||
bind_address,
|
bind_address,
|
||||||
|
&performance,
|
||||||
);
|
);
|
||||||
|
|
||||||
// UDP listeners for QUIC transport — uses QUIC datagrams for low-latency forwarding.
|
// UDP listeners for QUIC transport — uses QUIC datagrams for low-latency forwarding.
|
||||||
@@ -1418,6 +1484,7 @@ async fn connect_to_hub_and_run_quic_with_connection(
|
|||||||
quic_transport::CTRL_CONFIG => {
|
quic_transport::CTRL_CONFIG => {
|
||||||
if let Ok(update) = serde_json::from_slice::<ConfigUpdate>(&payload) {
|
if let Ok(update) = serde_json::from_slice::<ConfigUpdate>(&payload) {
|
||||||
log::info!("QUIC config update from hub: ports {:?}", update.listen_ports);
|
log::info!("QUIC config update from hub: ports {:?}", update.listen_ports);
|
||||||
|
performance = update.performance.clone();
|
||||||
*listen_ports.write().await = update.listen_ports.clone();
|
*listen_ports.write().await = update.listen_ports.clone();
|
||||||
let _ = event_tx.try_send(EdgeEvent::PortsUpdated {
|
let _ = event_tx.try_send(EdgeEvent::PortsUpdated {
|
||||||
listen_ports: update.listen_ports.clone(),
|
listen_ports: update.listen_ports.clone(),
|
||||||
@@ -1431,6 +1498,7 @@ async fn connect_to_hub_and_run_quic_with_connection(
|
|||||||
&config.edge_id,
|
&config.edge_id,
|
||||||
connection_token,
|
connection_token,
|
||||||
bind_address,
|
bind_address,
|
||||||
|
&performance,
|
||||||
);
|
);
|
||||||
apply_udp_port_config_quic(
|
apply_udp_port_config_quic(
|
||||||
&update.listen_ports_udp,
|
&update.listen_ports_udp,
|
||||||
@@ -1537,6 +1605,7 @@ fn apply_port_config_quic(
|
|||||||
edge_id: &str,
|
edge_id: &str,
|
||||||
connection_token: &CancellationToken,
|
connection_token: &CancellationToken,
|
||||||
bind_address: &str,
|
bind_address: &str,
|
||||||
|
performance: &EffectivePerformanceConfig,
|
||||||
) {
|
) {
|
||||||
let new_set: std::collections::HashSet<u16> = new_ports.iter().copied().collect();
|
let new_set: std::collections::HashSet<u16> = new_ports.iter().copied().collect();
|
||||||
let old_set: std::collections::HashSet<u16> = port_listeners.keys().copied().collect();
|
let old_set: std::collections::HashSet<u16> = port_listeners.keys().copied().collect();
|
||||||
@@ -1557,6 +1626,7 @@ fn apply_port_config_quic(
|
|||||||
let _edge_id = edge_id.to_string();
|
let _edge_id = edge_id.to_string();
|
||||||
let port_token = connection_token.child_token();
|
let port_token = connection_token.child_token();
|
||||||
let bind_addr = bind_address.to_string();
|
let bind_addr = bind_address.to_string();
|
||||||
|
let performance = performance.clone();
|
||||||
|
|
||||||
let handle = tokio::spawn(async move {
|
let handle = tokio::spawn(async move {
|
||||||
let listener = match TcpListener::bind((bind_addr.as_str(), port)).await {
|
let listener = match TcpListener::bind((bind_addr.as_str(), port)).await {
|
||||||
@@ -1585,7 +1655,10 @@ fn apply_port_config_quic(
|
|||||||
let active_streams = active_streams.clone();
|
let active_streams = active_streams.clone();
|
||||||
let client_token = port_token.child_token();
|
let client_token = port_token.child_token();
|
||||||
|
|
||||||
active_streams.fetch_add(1, Ordering::Relaxed);
|
if !try_reserve_stream(&active_streams, performance.max_streams_per_edge) {
|
||||||
|
log::warn!("Rejecting QUIC client on port {}: max streams ({}) reached", port, performance.max_streams_per_edge);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
handle_client_connection_quic(
|
handle_client_connection_quic(
|
||||||
@@ -1596,17 +1669,7 @@ fn apply_port_config_quic(
|
|||||||
quic_conn,
|
quic_conn,
|
||||||
client_token,
|
client_token,
|
||||||
).await;
|
).await;
|
||||||
// Saturating decrement
|
release_stream(&active_streams);
|
||||||
loop {
|
|
||||||
let current = active_streams.load(Ordering::Relaxed);
|
|
||||||
if current == 0 { break; }
|
|
||||||
if active_streams.compare_exchange_weak(
|
|
||||||
current, current - 1,
|
|
||||||
Ordering::Relaxed, Ordering::Relaxed,
|
|
||||||
).is_ok() {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::sync::atomic::{AtomicU32, Ordering};
|
use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||||
use tokio::net::{TcpListener, TcpStream, UdpSocket};
|
use tokio::net::{TcpListener, TcpStream, UdpSocket};
|
||||||
@@ -12,6 +12,8 @@ use serde::{Deserialize, Serialize};
|
|||||||
|
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
use remoteingress_protocol::*;
|
use remoteingress_protocol::*;
|
||||||
|
use crate::performance::{EffectivePerformanceConfig, PerformanceConfig};
|
||||||
|
use crate::transport::TransportMode;
|
||||||
use crate::transport::quic as quic_transport;
|
use crate::transport::quic as quic_transport;
|
||||||
|
|
||||||
type HubTlsStream = tokio_rustls::server::TlsStream<TcpStream>;
|
type HubTlsStream = tokio_rustls::server::TlsStream<TcpStream>;
|
||||||
@@ -56,6 +58,8 @@ pub struct HubConfig {
|
|||||||
pub tls_cert_pem: Option<String>,
|
pub tls_cert_pem: Option<String>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub tls_key_pem: Option<String>,
|
pub tls_key_pem: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub performance: Option<PerformanceConfig>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for HubConfig {
|
impl Default for HubConfig {
|
||||||
@@ -65,6 +69,7 @@ impl Default for HubConfig {
|
|||||||
target_host: Some("127.0.0.1".to_string()),
|
target_host: Some("127.0.0.1".to_string()),
|
||||||
tls_cert_pem: None,
|
tls_cert_pem: None,
|
||||||
tls_key_pem: None,
|
tls_key_pem: None,
|
||||||
|
performance: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -82,6 +87,8 @@ pub struct AllowedEdge {
|
|||||||
pub stun_interval_secs: Option<u64>,
|
pub stun_interval_secs: Option<u64>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub firewall_config: Option<serde_json::Value>,
|
pub firewall_config: Option<serde_json::Value>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub performance: Option<PerformanceConfig>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Handshake response sent to edge after authentication.
|
/// Handshake response sent to edge after authentication.
|
||||||
@@ -94,6 +101,7 @@ struct HandshakeResponse {
|
|||||||
stun_interval_secs: u64,
|
stun_interval_secs: u64,
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
firewall_config: Option<serde_json::Value>,
|
firewall_config: Option<serde_json::Value>,
|
||||||
|
performance: EffectivePerformanceConfig,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Configuration update pushed to a connected edge at runtime.
|
/// Configuration update pushed to a connected edge at runtime.
|
||||||
@@ -105,6 +113,44 @@ pub struct EdgeConfigUpdate {
|
|||||||
pub listen_ports_udp: Vec<u16>,
|
pub listen_ports_udp: Vec<u16>,
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
pub firewall_config: Option<serde_json::Value>,
|
pub firewall_config: Option<serde_json::Value>,
|
||||||
|
pub performance: EffectivePerformanceConfig,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct FlowControlStatus {
|
||||||
|
pub applies: bool,
|
||||||
|
pub current_window_bytes: u32,
|
||||||
|
pub min_window_bytes: u32,
|
||||||
|
pub max_window_bytes: u32,
|
||||||
|
pub total_window_budget_bytes: u64,
|
||||||
|
pub estimated_in_flight_bytes: u64,
|
||||||
|
pub stalled_streams: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct QueueStatus {
|
||||||
|
pub ctrl_queue_depth: u64,
|
||||||
|
pub data_queue_depth: u64,
|
||||||
|
pub sustained_queue_depth: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct TrafficStatus {
|
||||||
|
pub bytes_in: u64,
|
||||||
|
pub bytes_out: u64,
|
||||||
|
pub streams_opened_total: u64,
|
||||||
|
pub streams_closed_total: u64,
|
||||||
|
pub rejected_streams: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct UdpStatus {
|
||||||
|
pub active_sessions: u64,
|
||||||
|
pub dropped_datagrams: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Runtime status of a connected edge.
|
/// Runtime status of a connected edge.
|
||||||
@@ -115,6 +161,13 @@ pub struct ConnectedEdgeStatus {
|
|||||||
pub connected_at: u64,
|
pub connected_at: u64,
|
||||||
pub active_streams: usize,
|
pub active_streams: usize,
|
||||||
pub peer_addr: String,
|
pub peer_addr: String,
|
||||||
|
pub transport_mode: TransportMode,
|
||||||
|
pub fallback_used: bool,
|
||||||
|
pub performance: EffectivePerformanceConfig,
|
||||||
|
pub flow_control: FlowControlStatus,
|
||||||
|
pub queues: QueueStatus,
|
||||||
|
pub traffic: TrafficStatus,
|
||||||
|
pub udp: UdpStatus,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Events emitted by the hub.
|
/// Events emitted by the hub.
|
||||||
@@ -157,11 +210,30 @@ struct ConnectedEdgeInfo {
|
|||||||
connected_at: u64,
|
connected_at: u64,
|
||||||
peer_addr: String,
|
peer_addr: String,
|
||||||
edge_stream_count: Arc<AtomicU32>,
|
edge_stream_count: Arc<AtomicU32>,
|
||||||
|
transport_mode: TransportMode,
|
||||||
|
fallback_used: bool,
|
||||||
|
performance: EffectivePerformanceConfig,
|
||||||
|
metrics: Arc<EdgeRuntimeMetrics>,
|
||||||
config_tx: mpsc::Sender<EdgeConfigUpdate>,
|
config_tx: mpsc::Sender<EdgeConfigUpdate>,
|
||||||
/// Used to cancel the old connection when an edge reconnects.
|
/// Used to cancel the old connection when an edge reconnects.
|
||||||
cancel_token: CancellationToken,
|
cancel_token: CancellationToken,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct EdgeRuntimeMetrics {
|
||||||
|
streams_opened_total: AtomicU64,
|
||||||
|
streams_closed_total: AtomicU64,
|
||||||
|
rejected_streams: AtomicU64,
|
||||||
|
bytes_in: AtomicU64,
|
||||||
|
bytes_out: AtomicU64,
|
||||||
|
stalled_streams: AtomicU64,
|
||||||
|
dropped_datagrams: AtomicU64,
|
||||||
|
active_udp_sessions: AtomicU64,
|
||||||
|
ctrl_queue_depth: AtomicU64,
|
||||||
|
data_queue_depth: AtomicU64,
|
||||||
|
sustained_queue_depth: AtomicU64,
|
||||||
|
}
|
||||||
|
|
||||||
impl TunnelHub {
|
impl TunnelHub {
|
||||||
pub fn new(config: HubConfig) -> Self {
|
pub fn new(config: HubConfig) -> Self {
|
||||||
let (event_tx, event_rx) = mpsc::channel(1024);
|
let (event_tx, event_rx) = mpsc::channel(1024);
|
||||||
@@ -185,6 +257,7 @@ impl TunnelHub {
|
|||||||
/// Update the list of allowed edges.
|
/// Update the list of allowed edges.
|
||||||
/// For any currently-connected edge whose ports changed, push a config update.
|
/// For any currently-connected edge whose ports changed, push a config update.
|
||||||
pub async fn update_allowed_edges(&self, edges: Vec<AllowedEdge>) {
|
pub async fn update_allowed_edges(&self, edges: Vec<AllowedEdge>) {
|
||||||
|
let global_performance = self.config.read().await.performance.clone();
|
||||||
let mut map = self.allowed_edges.write().await;
|
let mut map = self.allowed_edges.write().await;
|
||||||
|
|
||||||
// Build new map
|
// Build new map
|
||||||
@@ -201,7 +274,8 @@ impl TunnelHub {
|
|||||||
let config_changed = match map.get(&edge.id) {
|
let config_changed = match map.get(&edge.id) {
|
||||||
Some(old) => old.listen_ports != edge.listen_ports
|
Some(old) => old.listen_ports != edge.listen_ports
|
||||||
|| old.listen_ports_udp != edge.listen_ports_udp
|
|| old.listen_ports_udp != edge.listen_ports_udp
|
||||||
|| old.firewall_config != edge.firewall_config,
|
|| old.firewall_config != edge.firewall_config
|
||||||
|
|| old.performance != edge.performance,
|
||||||
None => true, // newly allowed edge that's already connected
|
None => true, // newly allowed edge that's already connected
|
||||||
};
|
};
|
||||||
if config_changed {
|
if config_changed {
|
||||||
@@ -209,6 +283,10 @@ impl TunnelHub {
|
|||||||
listen_ports: edge.listen_ports.clone(),
|
listen_ports: edge.listen_ports.clone(),
|
||||||
listen_ports_udp: edge.listen_ports_udp.clone(),
|
listen_ports_udp: edge.listen_ports_udp.clone(),
|
||||||
firewall_config: edge.firewall_config.clone(),
|
firewall_config: edge.firewall_config.clone(),
|
||||||
|
performance: PerformanceConfig::merge(
|
||||||
|
global_performance.as_ref(),
|
||||||
|
edge.performance.as_ref(),
|
||||||
|
).effective(),
|
||||||
};
|
};
|
||||||
let _ = info.config_tx.try_send(update);
|
let _ = info.config_tx.try_send(update);
|
||||||
}
|
}
|
||||||
@@ -226,11 +304,50 @@ impl TunnelHub {
|
|||||||
|
|
||||||
let mut connected = Vec::new();
|
let mut connected = Vec::new();
|
||||||
for (id, info) in edges.iter() {
|
for (id, info) in edges.iter() {
|
||||||
|
let active_streams = info.edge_stream_count.load(Ordering::Relaxed);
|
||||||
|
let flow_window = if info.transport_mode == TransportMode::TcpTls {
|
||||||
|
compute_window_for_limits(
|
||||||
|
active_streams,
|
||||||
|
info.performance.total_window_budget_bytes,
|
||||||
|
info.performance.min_stream_window_bytes,
|
||||||
|
info.performance.max_stream_window_bytes,
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
};
|
||||||
connected.push(ConnectedEdgeStatus {
|
connected.push(ConnectedEdgeStatus {
|
||||||
edge_id: id.clone(),
|
edge_id: id.clone(),
|
||||||
connected_at: info.connected_at,
|
connected_at: info.connected_at,
|
||||||
active_streams: info.edge_stream_count.load(Ordering::Relaxed) as usize,
|
active_streams: active_streams as usize,
|
||||||
peer_addr: info.peer_addr.clone(),
|
peer_addr: info.peer_addr.clone(),
|
||||||
|
transport_mode: info.transport_mode,
|
||||||
|
fallback_used: info.fallback_used,
|
||||||
|
performance: info.performance.clone(),
|
||||||
|
flow_control: FlowControlStatus {
|
||||||
|
applies: info.transport_mode == TransportMode::TcpTls,
|
||||||
|
current_window_bytes: flow_window,
|
||||||
|
min_window_bytes: info.performance.min_stream_window_bytes,
|
||||||
|
max_window_bytes: info.performance.max_stream_window_bytes,
|
||||||
|
total_window_budget_bytes: info.performance.total_window_budget_bytes,
|
||||||
|
estimated_in_flight_bytes: flow_window as u64 * active_streams as u64,
|
||||||
|
stalled_streams: info.metrics.stalled_streams.load(Ordering::Relaxed),
|
||||||
|
},
|
||||||
|
queues: QueueStatus {
|
||||||
|
ctrl_queue_depth: info.metrics.ctrl_queue_depth.load(Ordering::Relaxed),
|
||||||
|
data_queue_depth: info.metrics.data_queue_depth.load(Ordering::Relaxed),
|
||||||
|
sustained_queue_depth: info.metrics.sustained_queue_depth.load(Ordering::Relaxed),
|
||||||
|
},
|
||||||
|
traffic: TrafficStatus {
|
||||||
|
bytes_in: info.metrics.bytes_in.load(Ordering::Relaxed),
|
||||||
|
bytes_out: info.metrics.bytes_out.load(Ordering::Relaxed),
|
||||||
|
streams_opened_total: info.metrics.streams_opened_total.load(Ordering::Relaxed),
|
||||||
|
streams_closed_total: info.metrics.streams_closed_total.load(Ordering::Relaxed),
|
||||||
|
rejected_streams: info.metrics.rejected_streams.load(Ordering::Relaxed),
|
||||||
|
},
|
||||||
|
udp: UdpStatus {
|
||||||
|
active_sessions: info.metrics.active_udp_sessions.load(Ordering::Relaxed),
|
||||||
|
dropped_datagrams: info.metrics.dropped_datagrams.load(Ordering::Relaxed),
|
||||||
|
},
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -249,9 +366,14 @@ impl TunnelHub {
|
|||||||
|
|
||||||
let listener = TcpListener::bind(("0.0.0.0", config.tunnel_port)).await?;
|
let listener = TcpListener::bind(("0.0.0.0", config.tunnel_port)).await?;
|
||||||
log::info!("Hub listening on TCP port {}", config.tunnel_port);
|
log::info!("Hub listening on TCP port {}", config.tunnel_port);
|
||||||
|
let effective_performance = config.performance.clone().unwrap_or_default().effective();
|
||||||
|
|
||||||
// Start QUIC endpoint on the same port (UDP)
|
// Start QUIC endpoint on the same port (UDP)
|
||||||
let quic_endpoint = match quic_transport::build_quic_server_config(tls_config) {
|
let quic_endpoint = match quic_transport::build_quic_server_config_with_limits(
|
||||||
|
tls_config,
|
||||||
|
effective_performance.max_streams_per_edge.min(u32::MAX as usize) as u32,
|
||||||
|
effective_performance.quic_datagram_receive_buffer_bytes,
|
||||||
|
) {
|
||||||
Ok(quic_server_config) => {
|
Ok(quic_server_config) => {
|
||||||
let bind_addr: std::net::SocketAddr = ([0, 0, 0, 0], config.tunnel_port).into();
|
let bind_addr: std::net::SocketAddr = ([0, 0, 0, 0], config.tunnel_port).into();
|
||||||
match quinn::Endpoint::server(quic_server_config, bind_addr) {
|
match quinn::Endpoint::server(quic_server_config, bind_addr) {
|
||||||
@@ -280,6 +402,7 @@ impl TunnelHub {
|
|||||||
let event_tx = self.event_tx.clone();
|
let event_tx = self.event_tx.clone();
|
||||||
let target_host = config.target_host.unwrap_or_else(|| "127.0.0.1".to_string());
|
let target_host = config.target_host.unwrap_or_else(|| "127.0.0.1".to_string());
|
||||||
let hub_token = self.cancel_token.clone();
|
let hub_token = self.cancel_token.clone();
|
||||||
|
let hub_performance = config.performance.clone();
|
||||||
|
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
// Spawn QUIC acceptor as a separate task
|
// Spawn QUIC acceptor as a separate task
|
||||||
@@ -289,6 +412,7 @@ impl TunnelHub {
|
|||||||
let event_tx_q = event_tx.clone();
|
let event_tx_q = event_tx.clone();
|
||||||
let target_q = target_host.clone();
|
let target_q = target_host.clone();
|
||||||
let hub_token_q = hub_token.clone();
|
let hub_token_q = hub_token.clone();
|
||||||
|
let performance_q = hub_performance.clone();
|
||||||
Some(tokio::spawn(async move {
|
Some(tokio::spawn(async move {
|
||||||
loop {
|
loop {
|
||||||
tokio::select! {
|
tokio::select! {
|
||||||
@@ -301,6 +425,7 @@ impl TunnelHub {
|
|||||||
let target = target_q.clone();
|
let target = target_q.clone();
|
||||||
let edge_token = hub_token_q.child_token();
|
let edge_token = hub_token_q.child_token();
|
||||||
let peer_addr = incoming.remote_address().ip().to_string();
|
let peer_addr = incoming.remote_address().ip().to_string();
|
||||||
|
let performance = performance_q.clone();
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
// Accept the QUIC connection
|
// Accept the QUIC connection
|
||||||
let quic_conn = match incoming.await {
|
let quic_conn = match incoming.await {
|
||||||
@@ -310,8 +435,8 @@ impl TunnelHub {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
if let Err(e) = handle_edge_connection_quic(
|
if let Err(e) = handle_edge_connection_quic(
|
||||||
quic_conn, allowed, connected, event_tx, target, edge_token, peer_addr,
|
quic_conn, allowed, connected, event_tx, target, edge_token, peer_addr, performance,
|
||||||
).await {
|
).await {
|
||||||
log::error!("QUIC edge connection error: {}", e);
|
log::error!("QUIC edge connection error: {}", e);
|
||||||
}
|
}
|
||||||
@@ -345,9 +470,10 @@ impl TunnelHub {
|
|||||||
let target = target_host.clone();
|
let target = target_host.clone();
|
||||||
let edge_token = hub_token.child_token();
|
let edge_token = hub_token.child_token();
|
||||||
let peer_addr = addr.ip().to_string();
|
let peer_addr = addr.ip().to_string();
|
||||||
|
let performance = hub_performance.clone();
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
if let Err(e) = handle_edge_connection(
|
if let Err(e) = handle_edge_connection(
|
||||||
stream, acceptor, allowed, connected, event_tx, target, edge_token, peer_addr,
|
stream, acceptor, allowed, connected, event_tx, target, edge_token, peer_addr, performance,
|
||||||
).await {
|
).await {
|
||||||
log::error!("Edge connection error: {}", e);
|
log::error!("Edge connection error: {}", e);
|
||||||
}
|
}
|
||||||
@@ -390,15 +516,21 @@ impl TunnelHub {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn parse_requested_transport_mode(value: Option<&str>) -> Option<TransportMode> {
|
||||||
|
match value {
|
||||||
|
Some("tcpTls") => Some(TransportMode::TcpTls),
|
||||||
|
Some("quic") => Some(TransportMode::Quic),
|
||||||
|
Some("quicWithFallback") => Some(TransportMode::QuicWithFallback),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl Drop for TunnelHub {
|
impl Drop for TunnelHub {
|
||||||
fn drop(&mut self) {
|
fn drop(&mut self) {
|
||||||
self.cancel_token.cancel();
|
self.cancel_token.cancel();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Maximum concurrent streams per edge connection.
|
|
||||||
const MAX_STREAMS_PER_EDGE: usize = 1024;
|
|
||||||
|
|
||||||
/// Process a single frame received from the edge side of the tunnel.
|
/// Process a single frame received from the edge side of the tunnel.
|
||||||
/// Handles FRAME_OPEN, FRAME_DATA, FRAME_WINDOW_UPDATE, FRAME_CLOSE, and FRAME_PONG.
|
/// Handles FRAME_OPEN, FRAME_DATA, FRAME_WINDOW_UPDATE, FRAME_CLOSE, and FRAME_PONG.
|
||||||
async fn handle_hub_frame(
|
async fn handle_hub_frame(
|
||||||
@@ -416,6 +548,8 @@ async fn handle_hub_frame(
|
|||||||
target_host: &str,
|
target_host: &str,
|
||||||
edge_token: &CancellationToken,
|
edge_token: &CancellationToken,
|
||||||
cleanup_tx: &mpsc::Sender<u32>,
|
cleanup_tx: &mpsc::Sender<u32>,
|
||||||
|
performance: &EffectivePerformanceConfig,
|
||||||
|
metrics: &Arc<EdgeRuntimeMetrics>,
|
||||||
) -> FrameAction {
|
) -> FrameAction {
|
||||||
match frame.frame_type {
|
match frame.frame_type {
|
||||||
FRAME_OPEN => {
|
FRAME_OPEN => {
|
||||||
@@ -423,8 +557,9 @@ async fn handle_hub_frame(
|
|||||||
let permit = match stream_semaphore.clone().try_acquire_owned() {
|
let permit = match stream_semaphore.clone().try_acquire_owned() {
|
||||||
Ok(p) => p,
|
Ok(p) => p,
|
||||||
Err(_) => {
|
Err(_) => {
|
||||||
|
metrics.rejected_streams.fetch_add(1, Ordering::Relaxed);
|
||||||
log::warn!("Edge {} exceeded max streams ({}), rejecting stream {}",
|
log::warn!("Edge {} exceeded max streams ({}), rejecting stream {}",
|
||||||
edge_id, MAX_STREAMS_PER_EDGE, frame.stream_id);
|
edge_id, performance.max_streams_per_edge, frame.stream_id);
|
||||||
let close_frame = encode_frame(frame.stream_id, FRAME_CLOSE_BACK, &[]);
|
let close_frame = encode_frame(frame.stream_id, FRAME_CLOSE_BACK, &[]);
|
||||||
tunnel_io.queue_ctrl(close_frame);
|
tunnel_io.queue_ctrl(close_frame);
|
||||||
return FrameAction::Continue;
|
return FrameAction::Continue;
|
||||||
@@ -444,6 +579,8 @@ async fn handle_hub_frame(
|
|||||||
let sustained_writer_tx = sustained_tx.clone(); // sustained: DATA_BACK from elephant flows
|
let sustained_writer_tx = sustained_tx.clone(); // sustained: DATA_BACK from elephant flows
|
||||||
let target = target_host.to_string();
|
let target = target_host.to_string();
|
||||||
let stream_token = edge_token.child_token();
|
let stream_token = edge_token.child_token();
|
||||||
|
let active_after_open = edge_stream_count.fetch_add(1, Ordering::Relaxed) + 1;
|
||||||
|
metrics.streams_opened_total.fetch_add(1, Ordering::Relaxed);
|
||||||
|
|
||||||
let _ = event_tx.try_send(HubEvent::StreamOpened {
|
let _ = event_tx.try_send(HubEvent::StreamOpened {
|
||||||
edge_id: edge_id.to_string(),
|
edge_id: edge_id.to_string(),
|
||||||
@@ -453,9 +590,12 @@ async fn handle_hub_frame(
|
|||||||
// Create channel for data from edge to this stream
|
// Create channel for data from edge to this stream
|
||||||
let (stream_data_tx, mut stream_data_rx) = mpsc::unbounded_channel::<Bytes>();
|
let (stream_data_tx, mut stream_data_rx) = mpsc::unbounded_channel::<Bytes>();
|
||||||
// Adaptive initial window: scale with current stream count
|
// Adaptive initial window: scale with current stream count
|
||||||
// to keep total in-flight data within the 200MB budget.
|
// to keep total in-flight data within the configured edge budget.
|
||||||
let initial_window = compute_window_for_stream_count(
|
let initial_window = compute_window_for_limits(
|
||||||
edge_stream_count.load(Ordering::Relaxed),
|
active_after_open,
|
||||||
|
performance.total_window_budget_bytes,
|
||||||
|
performance.min_stream_window_bytes,
|
||||||
|
performance.max_stream_window_bytes,
|
||||||
);
|
);
|
||||||
let send_window = Arc::new(AtomicU32::new(initial_window));
|
let send_window = Arc::new(AtomicU32::new(initial_window));
|
||||||
let window_notify = Arc::new(Notify::new());
|
let window_notify = Arc::new(Notify::new());
|
||||||
@@ -468,9 +608,10 @@ async fn handle_hub_frame(
|
|||||||
|
|
||||||
// Spawn task: connect to SmartProxy, send PROXY header, pipe data
|
// Spawn task: connect to SmartProxy, send PROXY header, pipe data
|
||||||
let stream_counter = Arc::clone(edge_stream_count);
|
let stream_counter = Arc::clone(edge_stream_count);
|
||||||
|
let stream_metrics = metrics.clone();
|
||||||
|
let stream_performance = performance.clone();
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
let _permit = permit; // hold semaphore permit until stream completes
|
let _permit = permit; // hold semaphore permit until stream completes
|
||||||
stream_counter.fetch_add(1, Ordering::Relaxed);
|
|
||||||
|
|
||||||
let result = async {
|
let result = async {
|
||||||
// A2: Connect to SmartProxy with timeout
|
// A2: Connect to SmartProxy with timeout
|
||||||
@@ -528,8 +669,11 @@ async fn handle_hub_frame(
|
|||||||
// Track consumption for adaptive flow control.
|
// Track consumption for adaptive flow control.
|
||||||
// Increment capped to adaptive window to limit per-stream in-flight data.
|
// Increment capped to adaptive window to limit per-stream in-flight data.
|
||||||
consumed_since_update += len;
|
consumed_since_update += len;
|
||||||
let adaptive_window = remoteingress_protocol::compute_window_for_stream_count(
|
let adaptive_window = remoteingress_protocol::compute_window_for_limits(
|
||||||
stream_counter_w.load(Ordering::Relaxed),
|
stream_counter_w.load(Ordering::Relaxed),
|
||||||
|
stream_performance.total_window_budget_bytes,
|
||||||
|
stream_performance.min_stream_window_bytes,
|
||||||
|
stream_performance.max_stream_window_bytes,
|
||||||
);
|
);
|
||||||
let threshold = adaptive_window / 2;
|
let threshold = adaptive_window / 2;
|
||||||
if consumed_since_update >= threshold {
|
if consumed_since_update >= threshold {
|
||||||
@@ -584,9 +728,10 @@ async fn handle_hub_frame(
|
|||||||
tokio::select! {
|
tokio::select! {
|
||||||
_ = notified => continue,
|
_ = notified => continue,
|
||||||
_ = stream_token.cancelled() => break,
|
_ = stream_token.cancelled() => break,
|
||||||
_ = tokio::time::sleep(Duration::from_secs(55)) => {
|
_ = tokio::time::sleep(Duration::from_secs(55)) => {
|
||||||
log::warn!("Stream {} download stalled (window empty for 55s)", stream_id);
|
stream_metrics.stalled_streams.fetch_add(1, Ordering::Relaxed);
|
||||||
break;
|
log::warn!("Stream {} download stalled (window empty for 55s)", stream_id);
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -613,6 +758,7 @@ async fn handle_hub_frame(
|
|||||||
let frame = Bytes::copy_from_slice(&buf[..FRAME_HEADER_SIZE + n]);
|
let frame = Bytes::copy_from_slice(&buf[..FRAME_HEADER_SIZE + n]);
|
||||||
// Sustained classification: >2.5 MB/s for >10 seconds
|
// Sustained classification: >2.5 MB/s for >10 seconds
|
||||||
dl_bytes_sent += n as u64;
|
dl_bytes_sent += n as u64;
|
||||||
|
stream_metrics.bytes_out.fetch_add(n as u64, Ordering::Relaxed);
|
||||||
if !is_sustained {
|
if !is_sustained {
|
||||||
let elapsed = dl_start.elapsed().as_secs();
|
let elapsed = dl_start.elapsed().as_secs();
|
||||||
if elapsed >= remoteingress_protocol::SUSTAINED_MIN_DURATION_SECS
|
if elapsed >= remoteingress_protocol::SUSTAINED_MIN_DURATION_SECS
|
||||||
@@ -677,6 +823,7 @@ async fn handle_hub_frame(
|
|||||||
_ = cleanup.send(stream_id) => {}
|
_ = cleanup.send(stream_id) => {}
|
||||||
_ = stream_token.cancelled() => {}
|
_ = stream_token.cancelled() => {}
|
||||||
}
|
}
|
||||||
|
stream_metrics.streams_closed_total.fetch_add(1, Ordering::Relaxed);
|
||||||
stream_counter.fetch_sub(1, Ordering::Relaxed);
|
stream_counter.fetch_sub(1, Ordering::Relaxed);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -685,6 +832,7 @@ async fn handle_hub_frame(
|
|||||||
// limits bytes-in-flight, so the channel won't grow unbounded. send() only
|
// limits bytes-in-flight, so the channel won't grow unbounded. send() only
|
||||||
// fails if the receiver is dropped (stream handler already exited).
|
// fails if the receiver is dropped (stream handler already exited).
|
||||||
if let Some(state) = streams.get(&frame.stream_id) {
|
if let Some(state) = streams.get(&frame.stream_id) {
|
||||||
|
metrics.bytes_in.fetch_add(frame.payload.len() as u64, Ordering::Relaxed);
|
||||||
if state.data_tx.send(frame.payload).is_err() {
|
if state.data_tx.send(frame.payload).is_err() {
|
||||||
// Receiver dropped — stream handler already exited, clean up
|
// Receiver dropped — stream handler already exited, clean up
|
||||||
streams.remove(&frame.stream_id);
|
streams.remove(&frame.stream_id);
|
||||||
@@ -697,8 +845,8 @@ async fn handle_hub_frame(
|
|||||||
if increment > 0 {
|
if increment > 0 {
|
||||||
if let Some(state) = streams.get(&frame.stream_id) {
|
if let Some(state) = streams.get(&frame.stream_id) {
|
||||||
let prev = state.send_window.fetch_add(increment, Ordering::Release);
|
let prev = state.send_window.fetch_add(increment, Ordering::Release);
|
||||||
if prev + increment > MAX_WINDOW_SIZE {
|
if prev + increment > performance.max_stream_window_bytes {
|
||||||
state.send_window.store(MAX_WINDOW_SIZE, Ordering::Release);
|
state.send_window.store(performance.max_stream_window_bytes, Ordering::Release);
|
||||||
}
|
}
|
||||||
state.window_notify.notify_one();
|
state.window_notify.notify_one();
|
||||||
}
|
}
|
||||||
@@ -733,6 +881,7 @@ async fn handle_hub_frame(
|
|||||||
data_tx: udp_tx,
|
data_tx: udp_tx,
|
||||||
cancel_token: session_token.clone(),
|
cancel_token: session_token.clone(),
|
||||||
});
|
});
|
||||||
|
metrics.active_udp_sessions.fetch_add(1, Ordering::Relaxed);
|
||||||
|
|
||||||
// Spawn upstream UDP forwarder
|
// Spawn upstream UDP forwarder
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
@@ -767,6 +916,7 @@ async fn handle_hub_frame(
|
|||||||
Ok(len) => {
|
Ok(len) => {
|
||||||
let frame = encode_frame(stream_id, FRAME_UDP_DATA_BACK, &buf[..len]);
|
let frame = encode_frame(stream_id, FRAME_UDP_DATA_BACK, &buf[..len]);
|
||||||
if data_writer_tx.try_send(frame).is_err() {
|
if data_writer_tx.try_send(frame).is_err() {
|
||||||
|
// Return datagrams may be dropped under pressure.
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -800,18 +950,22 @@ async fn handle_hub_frame(
|
|||||||
}
|
}
|
||||||
|
|
||||||
recv_handle.abort();
|
recv_handle.abort();
|
||||||
|
// active_udp_sessions is decremented by the FRAME_UDP_CLOSE path or connection cleanup.
|
||||||
log::debug!("UDP session {} closed for edge {}", stream_id, edge_id_str);
|
log::debug!("UDP session {} closed for edge {}", stream_id, edge_id_str);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
FRAME_UDP_DATA => {
|
FRAME_UDP_DATA => {
|
||||||
// Forward datagram to upstream
|
// Forward datagram to upstream
|
||||||
if let Some(state) = udp_sessions.get(&frame.stream_id) {
|
if let Some(state) = udp_sessions.get(&frame.stream_id) {
|
||||||
let _ = state.data_tx.try_send(frame.payload);
|
if state.data_tx.try_send(frame.payload).is_err() {
|
||||||
|
metrics.dropped_datagrams.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
FRAME_UDP_CLOSE => {
|
FRAME_UDP_CLOSE => {
|
||||||
if let Some(state) = udp_sessions.remove(&frame.stream_id) {
|
if let Some(state) = udp_sessions.remove(&frame.stream_id) {
|
||||||
state.cancel_token.cancel();
|
state.cancel_token.cancel();
|
||||||
|
metrics.active_udp_sessions.fetch_sub(1, Ordering::Relaxed);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
_ => {
|
_ => {
|
||||||
@@ -831,6 +985,7 @@ async fn handle_edge_connection(
|
|||||||
target_host: String,
|
target_host: String,
|
||||||
edge_token: CancellationToken,
|
edge_token: CancellationToken,
|
||||||
peer_addr: String,
|
peer_addr: String,
|
||||||
|
hub_performance: Option<PerformanceConfig>,
|
||||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
// Disable Nagle's algorithm for low-latency control frames (PING/PONG, WINDOW_UPDATE)
|
// Disable Nagle's algorithm for low-latency control frames (PING/PONG, WINDOW_UPDATE)
|
||||||
stream.set_nodelay(true)?;
|
stream.set_nodelay(true)?;
|
||||||
@@ -861,29 +1016,38 @@ async fn handle_edge_connection(
|
|||||||
.map_err(|_| "auth line not valid UTF-8")?;
|
.map_err(|_| "auth line not valid UTF-8")?;
|
||||||
let auth_line = auth_line.trim();
|
let auth_line = auth_line.trim();
|
||||||
|
|
||||||
let parts: Vec<&str> = auth_line.splitn(3, ' ').collect();
|
let parts: Vec<&str> = auth_line.split_whitespace().collect();
|
||||||
if parts.len() != 3 || parts[0] != "EDGE" {
|
if parts.len() < 3 || parts[0] != "EDGE" {
|
||||||
return Err("invalid auth line".into());
|
return Err("invalid auth line".into());
|
||||||
}
|
}
|
||||||
|
|
||||||
let edge_id = parts[1].to_string();
|
let edge_id = parts[1].to_string();
|
||||||
let secret = parts[2];
|
let secret = parts[2];
|
||||||
|
let requested_transport = parse_requested_transport_mode(parts.get(3).copied());
|
||||||
|
let fallback_used = requested_transport == Some(TransportMode::QuicWithFallback);
|
||||||
|
|
||||||
// Verify credentials and extract edge config
|
// Verify credentials and extract edge config
|
||||||
let (listen_ports, listen_ports_udp, stun_interval_secs, firewall_config) = {
|
let (listen_ports, listen_ports_udp, stun_interval_secs, firewall_config, edge_performance) = {
|
||||||
let edges = allowed.read().await;
|
let edges = allowed.read().await;
|
||||||
match edges.get(&edge_id) {
|
match edges.get(&edge_id) {
|
||||||
Some(edge) => {
|
Some(edge) => {
|
||||||
if !constant_time_eq(secret.as_bytes(), edge.secret.as_bytes()) {
|
if !constant_time_eq(secret.as_bytes(), edge.secret.as_bytes()) {
|
||||||
return Err(format!("invalid secret for edge {}", edge_id).into());
|
return Err(format!("invalid secret for edge {}", edge_id).into());
|
||||||
}
|
}
|
||||||
(edge.listen_ports.clone(), edge.listen_ports_udp.clone(), edge.stun_interval_secs.unwrap_or(300), edge.firewall_config.clone())
|
(
|
||||||
|
edge.listen_ports.clone(),
|
||||||
|
edge.listen_ports_udp.clone(),
|
||||||
|
edge.stun_interval_secs.unwrap_or(300),
|
||||||
|
edge.firewall_config.clone(),
|
||||||
|
edge.performance.clone(),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
None => {
|
None => {
|
||||||
return Err(format!("unknown edge {}", edge_id).into());
|
return Err(format!("unknown edge {}", edge_id).into());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
let performance = PerformanceConfig::merge(hub_performance.as_ref(), edge_performance.as_ref()).effective();
|
||||||
|
|
||||||
log::info!("Edge {} authenticated from {}", edge_id, peer_addr);
|
log::info!("Edge {} authenticated from {}", edge_id, peer_addr);
|
||||||
let _ = event_tx.try_send(HubEvent::EdgeConnected {
|
let _ = event_tx.try_send(HubEvent::EdgeConnected {
|
||||||
@@ -897,6 +1061,7 @@ async fn handle_edge_connection(
|
|||||||
listen_ports_udp: listen_ports_udp.clone(),
|
listen_ports_udp: listen_ports_udp.clone(),
|
||||||
stun_interval_secs,
|
stun_interval_secs,
|
||||||
firewall_config,
|
firewall_config,
|
||||||
|
performance: performance.clone(),
|
||||||
};
|
};
|
||||||
let mut handshake_json = serde_json::to_string(&handshake)?;
|
let mut handshake_json = serde_json::to_string(&handshake)?;
|
||||||
handshake_json.push('\n');
|
handshake_json.push('\n');
|
||||||
@@ -908,6 +1073,7 @@ async fn handle_edge_connection(
|
|||||||
let mut udp_sessions: HashMap<u32, HubUdpSessionState> = HashMap::new();
|
let mut udp_sessions: HashMap<u32, HubUdpSessionState> = HashMap::new();
|
||||||
// Per-edge active stream counter for adaptive flow control
|
// Per-edge active stream counter for adaptive flow control
|
||||||
let edge_stream_count = Arc::new(AtomicU32::new(0));
|
let edge_stream_count = Arc::new(AtomicU32::new(0));
|
||||||
|
let metrics = Arc::new(EdgeRuntimeMetrics::default());
|
||||||
// Cleanup channel: spawned stream tasks send stream_id here when done
|
// Cleanup channel: spawned stream tasks send stream_id here when done
|
||||||
let (cleanup_tx, mut cleanup_rx) = mpsc::channel::<u32>(256);
|
let (cleanup_tx, mut cleanup_rx) = mpsc::channel::<u32>(256);
|
||||||
let now = std::time::SystemTime::now()
|
let now = std::time::SystemTime::now()
|
||||||
@@ -933,6 +1099,10 @@ async fn handle_edge_connection(
|
|||||||
connected_at: now,
|
connected_at: now,
|
||||||
peer_addr,
|
peer_addr,
|
||||||
edge_stream_count: edge_stream_count.clone(),
|
edge_stream_count: edge_stream_count.clone(),
|
||||||
|
transport_mode: TransportMode::TcpTls,
|
||||||
|
fallback_used,
|
||||||
|
performance: performance.clone(),
|
||||||
|
metrics: metrics.clone(),
|
||||||
config_tx,
|
config_tx,
|
||||||
cancel_token: edge_token.clone(),
|
cancel_token: edge_token.clone(),
|
||||||
},
|
},
|
||||||
@@ -973,7 +1143,7 @@ async fn handle_edge_connection(
|
|||||||
});
|
});
|
||||||
|
|
||||||
// A4: Semaphore to limit concurrent streams per edge
|
// A4: Semaphore to limit concurrent streams per edge
|
||||||
let stream_semaphore = Arc::new(Semaphore::new(MAX_STREAMS_PER_EDGE));
|
let stream_semaphore = Arc::new(Semaphore::new(performance.max_streams_per_edge));
|
||||||
|
|
||||||
// Heartbeat: periodic PING and liveness timeout
|
// Heartbeat: periodic PING and liveness timeout
|
||||||
let ping_interval_dur = Duration::from_secs(15);
|
let ping_interval_dur = Duration::from_secs(15);
|
||||||
@@ -1019,7 +1189,7 @@ async fn handle_edge_connection(
|
|||||||
frame, &mut tunnel_io, &mut streams, &mut udp_sessions,
|
frame, &mut tunnel_io, &mut streams, &mut udp_sessions,
|
||||||
&stream_semaphore, &edge_stream_count,
|
&stream_semaphore, &edge_stream_count,
|
||||||
&edge_id, &event_tx, &ctrl_tx, &data_tx, &sustained_tx, &target_host, &edge_token,
|
&edge_id, &event_tx, &ctrl_tx, &data_tx, &sustained_tx, &target_host, &edge_token,
|
||||||
&cleanup_tx,
|
&cleanup_tx, &performance, &metrics,
|
||||||
).await {
|
).await {
|
||||||
disconnect_reason = reason;
|
disconnect_reason = reason;
|
||||||
break 'hub_loop;
|
break 'hub_loop;
|
||||||
@@ -1032,6 +1202,10 @@ async fn handle_edge_connection(
|
|||||||
if ping_ticker.poll_tick(cx).is_ready() {
|
if ping_ticker.poll_tick(cx).is_ready() {
|
||||||
tunnel_io.queue_ctrl(encode_frame(0, FRAME_PING, &[]));
|
tunnel_io.queue_ctrl(encode_frame(0, FRAME_PING, &[]));
|
||||||
}
|
}
|
||||||
|
let depths = tunnel_io.queue_depths();
|
||||||
|
metrics.ctrl_queue_depth.store(depths.ctrl as u64, Ordering::Relaxed);
|
||||||
|
metrics.data_queue_depth.store(depths.data as u64, Ordering::Relaxed);
|
||||||
|
metrics.sustained_queue_depth.store(depths.sustained as u64, Ordering::Relaxed);
|
||||||
tunnel_io.poll_step(cx, &mut ctrl_rx, &mut data_rx, &mut sustained_rx, &mut liveness_deadline, &edge_token)
|
tunnel_io.poll_step(cx, &mut ctrl_rx, &mut data_rx, &mut sustained_rx, &mut liveness_deadline, &edge_token)
|
||||||
}).await;
|
}).await;
|
||||||
|
|
||||||
@@ -1043,7 +1217,7 @@ async fn handle_edge_connection(
|
|||||||
frame, &mut tunnel_io, &mut streams, &mut udp_sessions,
|
frame, &mut tunnel_io, &mut streams, &mut udp_sessions,
|
||||||
&stream_semaphore, &edge_stream_count,
|
&stream_semaphore, &edge_stream_count,
|
||||||
&edge_id, &event_tx, &ctrl_tx, &data_tx, &sustained_tx, &target_host, &edge_token,
|
&edge_id, &event_tx, &ctrl_tx, &data_tx, &sustained_tx, &target_host, &edge_token,
|
||||||
&cleanup_tx,
|
&cleanup_tx, &performance, &metrics,
|
||||||
).await {
|
).await {
|
||||||
disconnect_reason = reason;
|
disconnect_reason = reason;
|
||||||
break;
|
break;
|
||||||
@@ -1201,6 +1375,7 @@ async fn handle_edge_connection_quic(
|
|||||||
target_host: String,
|
target_host: String,
|
||||||
edge_token: CancellationToken,
|
edge_token: CancellationToken,
|
||||||
peer_addr: String,
|
peer_addr: String,
|
||||||
|
hub_performance: Option<PerformanceConfig>,
|
||||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
log::info!("QUIC edge connection from {}", peer_addr);
|
log::info!("QUIC edge connection from {}", peer_addr);
|
||||||
|
|
||||||
@@ -1229,8 +1404,8 @@ async fn handle_edge_connection_quic(
|
|||||||
.map_err(|_| "QUIC auth line not valid UTF-8")?;
|
.map_err(|_| "QUIC auth line not valid UTF-8")?;
|
||||||
let auth_line = auth_line.trim();
|
let auth_line = auth_line.trim();
|
||||||
|
|
||||||
let parts: Vec<&str> = auth_line.splitn(3, ' ').collect();
|
let parts: Vec<&str> = auth_line.split_whitespace().collect();
|
||||||
if parts.len() != 3 || parts[0] != "EDGE" {
|
if parts.len() < 3 || parts[0] != "EDGE" {
|
||||||
return Err("invalid QUIC auth line".into());
|
return Err("invalid QUIC auth line".into());
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1238,18 +1413,25 @@ async fn handle_edge_connection_quic(
|
|||||||
let secret = parts[2];
|
let secret = parts[2];
|
||||||
|
|
||||||
// Verify credentials
|
// Verify credentials
|
||||||
let (listen_ports, listen_ports_udp, stun_interval_secs, firewall_config) = {
|
let (listen_ports, listen_ports_udp, stun_interval_secs, firewall_config, edge_performance) = {
|
||||||
let edges = allowed.read().await;
|
let edges = allowed.read().await;
|
||||||
match edges.get(&edge_id) {
|
match edges.get(&edge_id) {
|
||||||
Some(edge) => {
|
Some(edge) => {
|
||||||
if !constant_time_eq(secret.as_bytes(), edge.secret.as_bytes()) {
|
if !constant_time_eq(secret.as_bytes(), edge.secret.as_bytes()) {
|
||||||
return Err(format!("invalid secret for edge {}", edge_id).into());
|
return Err(format!("invalid secret for edge {}", edge_id).into());
|
||||||
}
|
}
|
||||||
(edge.listen_ports.clone(), edge.listen_ports_udp.clone(), edge.stun_interval_secs.unwrap_or(300), edge.firewall_config.clone())
|
(
|
||||||
|
edge.listen_ports.clone(),
|
||||||
|
edge.listen_ports_udp.clone(),
|
||||||
|
edge.stun_interval_secs.unwrap_or(300),
|
||||||
|
edge.firewall_config.clone(),
|
||||||
|
edge.performance.clone(),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
None => return Err(format!("unknown edge {}", edge_id).into()),
|
None => return Err(format!("unknown edge {}", edge_id).into()),
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
let performance = PerformanceConfig::merge(hub_performance.as_ref(), edge_performance.as_ref()).effective();
|
||||||
|
|
||||||
log::info!("QUIC edge {} authenticated from {}", edge_id, peer_addr);
|
log::info!("QUIC edge {} authenticated from {}", edge_id, peer_addr);
|
||||||
let _ = event_tx.try_send(HubEvent::EdgeConnected {
|
let _ = event_tx.try_send(HubEvent::EdgeConnected {
|
||||||
@@ -1263,6 +1445,7 @@ async fn handle_edge_connection_quic(
|
|||||||
listen_ports_udp: listen_ports_udp.clone(),
|
listen_ports_udp: listen_ports_udp.clone(),
|
||||||
stun_interval_secs,
|
stun_interval_secs,
|
||||||
firewall_config,
|
firewall_config,
|
||||||
|
performance: performance.clone(),
|
||||||
};
|
};
|
||||||
let mut handshake_json = serde_json::to_string(&handshake)?;
|
let mut handshake_json = serde_json::to_string(&handshake)?;
|
||||||
handshake_json.push('\n');
|
handshake_json.push('\n');
|
||||||
@@ -1271,6 +1454,7 @@ async fn handle_edge_connection_quic(
|
|||||||
|
|
||||||
// Track this edge
|
// Track this edge
|
||||||
let edge_stream_count = Arc::new(AtomicU32::new(0));
|
let edge_stream_count = Arc::new(AtomicU32::new(0));
|
||||||
|
let metrics = Arc::new(EdgeRuntimeMetrics::default());
|
||||||
let now = std::time::SystemTime::now()
|
let now = std::time::SystemTime::now()
|
||||||
.duration_since(std::time::UNIX_EPOCH)
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
.unwrap_or_default()
|
.unwrap_or_default()
|
||||||
@@ -1290,13 +1474,17 @@ async fn handle_edge_connection_quic(
|
|||||||
connected_at: now,
|
connected_at: now,
|
||||||
peer_addr,
|
peer_addr,
|
||||||
edge_stream_count: edge_stream_count.clone(),
|
edge_stream_count: edge_stream_count.clone(),
|
||||||
|
transport_mode: TransportMode::Quic,
|
||||||
|
fallback_used: false,
|
||||||
|
performance: performance.clone(),
|
||||||
|
metrics: metrics.clone(),
|
||||||
config_tx,
|
config_tx,
|
||||||
cancel_token: edge_token.clone(),
|
cancel_token: edge_token.clone(),
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
let stream_semaphore = Arc::new(Semaphore::new(MAX_STREAMS_PER_EDGE));
|
let stream_semaphore = Arc::new(Semaphore::new(performance.max_streams_per_edge));
|
||||||
|
|
||||||
// Spawn task to accept data streams (tunneled client connections)
|
// Spawn task to accept data streams (tunneled client connections)
|
||||||
let data_stream_conn = quic_conn.clone();
|
let data_stream_conn = quic_conn.clone();
|
||||||
@@ -1305,6 +1493,7 @@ async fn handle_edge_connection_quic(
|
|||||||
let data_event_tx = event_tx.clone();
|
let data_event_tx = event_tx.clone();
|
||||||
let data_semaphore = stream_semaphore.clone();
|
let data_semaphore = stream_semaphore.clone();
|
||||||
let data_stream_count = edge_stream_count.clone();
|
let data_stream_count = edge_stream_count.clone();
|
||||||
|
let data_metrics = metrics.clone();
|
||||||
let data_token = edge_token.clone();
|
let data_token = edge_token.clone();
|
||||||
let data_handle = tokio::spawn(async move {
|
let data_handle = tokio::spawn(async move {
|
||||||
let mut stream_id_counter: u32 = 0;
|
let mut stream_id_counter: u32 = 0;
|
||||||
@@ -1317,6 +1506,7 @@ async fn handle_edge_connection_quic(
|
|||||||
let permit = match data_semaphore.clone().try_acquire_owned() {
|
let permit = match data_semaphore.clone().try_acquire_owned() {
|
||||||
Ok(p) => p,
|
Ok(p) => p,
|
||||||
Err(_) => {
|
Err(_) => {
|
||||||
|
data_metrics.rejected_streams.fetch_add(1, Ordering::Relaxed);
|
||||||
log::warn!("QUIC edge {} exceeded max streams, rejecting", data_edge_id);
|
log::warn!("QUIC edge {} exceeded max streams, rejecting", data_edge_id);
|
||||||
// Drop the streams to reject
|
// Drop the streams to reject
|
||||||
drop(quic_send);
|
drop(quic_send);
|
||||||
@@ -1331,21 +1521,24 @@ async fn handle_edge_connection_quic(
|
|||||||
let edge_id = data_edge_id.clone();
|
let edge_id = data_edge_id.clone();
|
||||||
let event_tx = data_event_tx.clone();
|
let event_tx = data_event_tx.clone();
|
||||||
let stream_count = data_stream_count.clone();
|
let stream_count = data_stream_count.clone();
|
||||||
|
let stream_metrics = data_metrics.clone();
|
||||||
let stream_token = data_token.child_token();
|
let stream_token = data_token.child_token();
|
||||||
|
|
||||||
let _ = event_tx.try_send(HubEvent::StreamOpened {
|
let _ = event_tx.try_send(HubEvent::StreamOpened {
|
||||||
edge_id: edge_id.clone(),
|
edge_id: edge_id.clone(),
|
||||||
stream_id,
|
stream_id,
|
||||||
});
|
});
|
||||||
|
stream_metrics.streams_opened_total.fetch_add(1, Ordering::Relaxed);
|
||||||
|
|
||||||
stream_count.fetch_add(1, Ordering::Relaxed);
|
stream_count.fetch_add(1, Ordering::Relaxed);
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
let _permit = permit;
|
let _permit = permit;
|
||||||
handle_quic_stream(
|
handle_quic_stream(
|
||||||
quic_send, quic_recv, stream_id,
|
quic_send, quic_recv, stream_id,
|
||||||
&target, &edge_id, stream_token,
|
&target, &edge_id, stream_token, stream_metrics.clone(),
|
||||||
).await;
|
).await;
|
||||||
stream_count.fetch_sub(1, Ordering::Relaxed);
|
stream_count.fetch_sub(1, Ordering::Relaxed);
|
||||||
|
stream_metrics.streams_closed_total.fetch_add(1, Ordering::Relaxed);
|
||||||
let _ = event_tx.try_send(HubEvent::StreamClosed {
|
let _ = event_tx.try_send(HubEvent::StreamClosed {
|
||||||
edge_id,
|
edge_id,
|
||||||
stream_id,
|
stream_id,
|
||||||
@@ -1364,7 +1557,7 @@ async fn handle_edge_connection_quic(
|
|||||||
});
|
});
|
||||||
|
|
||||||
// UDP sessions for QUIC datagram transport
|
// UDP sessions for QUIC datagram transport
|
||||||
let quic_udp_sessions: Arc<Mutex<HashMap<u32, mpsc::Sender<Bytes>>>> =
|
let quic_udp_sessions: Arc<Mutex<HashMap<u32, (mpsc::Sender<Bytes>, Instant)>>> =
|
||||||
Arc::new(Mutex::new(HashMap::new()));
|
Arc::new(Mutex::new(HashMap::new()));
|
||||||
|
|
||||||
// Spawn QUIC datagram receiver task
|
// Spawn QUIC datagram receiver task
|
||||||
@@ -1373,6 +1566,7 @@ async fn handle_edge_connection_quic(
|
|||||||
let dgram_target = target_host.clone();
|
let dgram_target = target_host.clone();
|
||||||
let dgram_edge_id = edge_id.clone();
|
let dgram_edge_id = edge_id.clone();
|
||||||
let dgram_token = edge_token.clone();
|
let dgram_token = edge_token.clone();
|
||||||
|
let dgram_metrics = metrics.clone();
|
||||||
let dgram_handle = tokio::spawn(async move {
|
let dgram_handle = tokio::spawn(async move {
|
||||||
let mut cleanup_interval = tokio::time::interval(Duration::from_secs(30));
|
let mut cleanup_interval = tokio::time::interval(Duration::from_secs(30));
|
||||||
cleanup_interval.tick().await; // consume initial tick
|
cleanup_interval.tick().await; // consume initial tick
|
||||||
@@ -1380,8 +1574,12 @@ async fn handle_edge_connection_quic(
|
|||||||
tokio::select! {
|
tokio::select! {
|
||||||
// Periodic sweep: prune sessions whose task has exited (receiver dropped)
|
// Periodic sweep: prune sessions whose task has exited (receiver dropped)
|
||||||
_ = cleanup_interval.tick() => {
|
_ = cleanup_interval.tick() => {
|
||||||
|
let now = Instant::now();
|
||||||
let mut s = dgram_sessions.lock().await;
|
let mut s = dgram_sessions.lock().await;
|
||||||
s.retain(|_id, tx| !tx.is_closed());
|
s.retain(|_id, (tx, last_activity)| {
|
||||||
|
!tx.is_closed() && now.duration_since(*last_activity) < Duration::from_secs(60)
|
||||||
|
});
|
||||||
|
dgram_metrics.active_udp_sessions.store(s.len() as u64, Ordering::Relaxed);
|
||||||
}
|
}
|
||||||
datagram = dgram_conn.read_datagram() => {
|
datagram = dgram_conn.read_datagram() => {
|
||||||
match datagram {
|
match datagram {
|
||||||
@@ -1408,10 +1606,12 @@ async fn handle_edge_connection_quic(
|
|||||||
let (tx, mut rx) = mpsc::channel::<Bytes>(256);
|
let (tx, mut rx) = mpsc::channel::<Bytes>(256);
|
||||||
let proxy_v2_data: Vec<u8> = proxy_data.to_vec();
|
let proxy_v2_data: Vec<u8> = proxy_data.to_vec();
|
||||||
let cleanup_sessions = sessions.clone();
|
let cleanup_sessions = sessions.clone();
|
||||||
|
let session_metrics = dgram_metrics.clone();
|
||||||
|
|
||||||
{
|
{
|
||||||
let mut s = sessions.lock().await;
|
let mut s = sessions.lock().await;
|
||||||
s.insert(session_id, tx);
|
s.insert(session_id, (tx, Instant::now()));
|
||||||
|
dgram_metrics.active_udp_sessions.store(s.len() as u64, Ordering::Relaxed);
|
||||||
}
|
}
|
||||||
|
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
@@ -1419,20 +1619,26 @@ async fn handle_edge_connection_quic(
|
|||||||
Ok(s) => Arc::new(s),
|
Ok(s) => Arc::new(s),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
log::error!("QUIC UDP session {} bind failed: {}", session_id, e);
|
log::error!("QUIC UDP session {} bind failed: {}", session_id, e);
|
||||||
cleanup_sessions.lock().await.remove(&session_id);
|
let mut s = cleanup_sessions.lock().await;
|
||||||
|
s.remove(&session_id);
|
||||||
|
session_metrics.active_udp_sessions.store(s.len() as u64, Ordering::Relaxed);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
if let Err(e) = upstream.connect((target.as_str(), dest_port)).await {
|
if let Err(e) = upstream.connect((target.as_str(), dest_port)).await {
|
||||||
log::error!("QUIC UDP session {} connect failed: {}", session_id, e);
|
log::error!("QUIC UDP session {} connect failed: {}", session_id, e);
|
||||||
cleanup_sessions.lock().await.remove(&session_id);
|
let mut s = cleanup_sessions.lock().await;
|
||||||
|
s.remove(&session_id);
|
||||||
|
session_metrics.active_udp_sessions.store(s.len() as u64, Ordering::Relaxed);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Send PROXY v2 header as first datagram so SmartProxy knows the original client
|
// Send PROXY v2 header as first datagram so SmartProxy knows the original client
|
||||||
if let Err(e) = upstream.send(&proxy_v2_data).await {
|
if let Err(e) = upstream.send(&proxy_v2_data).await {
|
||||||
log::error!("QUIC UDP session {} failed to send PROXY v2 header: {}", session_id, e);
|
log::error!("QUIC UDP session {} failed to send PROXY v2 header: {}", session_id, e);
|
||||||
cleanup_sessions.lock().await.remove(&session_id);
|
let mut s = cleanup_sessions.lock().await;
|
||||||
|
s.remove(&session_id);
|
||||||
|
session_metrics.active_udp_sessions.store(s.len() as u64, Ordering::Relaxed);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1476,16 +1682,23 @@ async fn handle_edge_connection_quic(
|
|||||||
}
|
}
|
||||||
recv_handle.abort();
|
recv_handle.abort();
|
||||||
// Clean up session entry to prevent memory leak
|
// Clean up session entry to prevent memory leak
|
||||||
cleanup_sessions.lock().await.remove(&session_id);
|
let mut s = cleanup_sessions.lock().await;
|
||||||
|
s.remove(&session_id);
|
||||||
|
session_metrics.active_udp_sessions.store(s.len() as u64, Ordering::Relaxed);
|
||||||
});
|
});
|
||||||
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Regular data datagram — forward to upstream
|
// Regular data datagram — forward to upstream
|
||||||
let sessions = dgram_sessions.lock().await;
|
let mut sessions = dgram_sessions.lock().await;
|
||||||
if let Some(tx) = sessions.get(&session_id) {
|
if let Some((tx, last_activity)) = sessions.get_mut(&session_id) {
|
||||||
let _ = tx.try_send(Bytes::copy_from_slice(payload));
|
*last_activity = Instant::now();
|
||||||
|
if tx.try_send(Bytes::copy_from_slice(payload)).is_err() {
|
||||||
|
dgram_metrics.dropped_datagrams.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
dgram_metrics.dropped_datagrams.fetch_add(1, Ordering::Relaxed);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
@@ -1595,6 +1808,7 @@ async fn handle_quic_stream(
|
|||||||
target_host: &str,
|
target_host: &str,
|
||||||
_edge_id: &str,
|
_edge_id: &str,
|
||||||
stream_token: CancellationToken,
|
stream_token: CancellationToken,
|
||||||
|
metrics: Arc<EdgeRuntimeMetrics>,
|
||||||
) {
|
) {
|
||||||
// Read PROXY header from the beginning of the stream
|
// Read PROXY header from the beginning of the stream
|
||||||
let proxy_header = match quic_transport::read_proxy_header(&mut quic_recv).await {
|
let proxy_header = match quic_transport::read_proxy_header(&mut quic_recv).await {
|
||||||
@@ -1640,6 +1854,7 @@ async fn handle_quic_stream(
|
|||||||
|
|
||||||
// Task: QUIC -> upstream (edge data to SmartProxy)
|
// Task: QUIC -> upstream (edge data to SmartProxy)
|
||||||
let writer_token = stream_token.clone();
|
let writer_token = stream_token.clone();
|
||||||
|
let writer_metrics = metrics.clone();
|
||||||
let mut writer_task = tokio::spawn(async move {
|
let mut writer_task = tokio::spawn(async move {
|
||||||
let mut buf = vec![0u8; 32768];
|
let mut buf = vec![0u8; 32768];
|
||||||
loop {
|
loop {
|
||||||
@@ -1647,6 +1862,7 @@ async fn handle_quic_stream(
|
|||||||
read_result = quic_recv.read(&mut buf) => {
|
read_result = quic_recv.read(&mut buf) => {
|
||||||
match read_result {
|
match read_result {
|
||||||
Ok(Some(n)) => {
|
Ok(Some(n)) => {
|
||||||
|
writer_metrics.bytes_in.fetch_add(n as u64, Ordering::Relaxed);
|
||||||
let write_result = tokio::select! {
|
let write_result = tokio::select! {
|
||||||
r = tokio::time::timeout(
|
r = tokio::time::timeout(
|
||||||
Duration::from_secs(60),
|
Duration::from_secs(60),
|
||||||
@@ -1678,6 +1894,7 @@ async fn handle_quic_stream(
|
|||||||
match read_result {
|
match read_result {
|
||||||
Ok(0) => break,
|
Ok(0) => break,
|
||||||
Ok(n) => {
|
Ok(n) => {
|
||||||
|
metrics.bytes_out.fetch_add(n as u64, Ordering::Relaxed);
|
||||||
if quic_send.write_all(&buf[..n]).await.is_err() {
|
if quic_send.write_all(&buf[..n]).await.is_err() {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@@ -1806,6 +2023,7 @@ mod tests {
|
|||||||
listen_ports_udp: vec![],
|
listen_ports_udp: vec![],
|
||||||
stun_interval_secs: 300,
|
stun_interval_secs: 300,
|
||||||
firewall_config: None,
|
firewall_config: None,
|
||||||
|
performance: EffectivePerformanceConfig::default(),
|
||||||
};
|
};
|
||||||
let json = serde_json::to_value(&resp).unwrap();
|
let json = serde_json::to_value(&resp).unwrap();
|
||||||
assert_eq!(json["listenPorts"], serde_json::json!([443, 8080]));
|
assert_eq!(json["listenPorts"], serde_json::json!([443, 8080]));
|
||||||
@@ -1821,6 +2039,7 @@ mod tests {
|
|||||||
listen_ports: vec![80, 443],
|
listen_ports: vec![80, 443],
|
||||||
listen_ports_udp: vec![53],
|
listen_ports_udp: vec![53],
|
||||||
firewall_config: None,
|
firewall_config: None,
|
||||||
|
performance: EffectivePerformanceConfig::default(),
|
||||||
};
|
};
|
||||||
let json = serde_json::to_value(&update).unwrap();
|
let json = serde_json::to_value(&update).unwrap();
|
||||||
assert_eq!(json["listenPorts"], serde_json::json!([80, 443]));
|
assert_eq!(json["listenPorts"], serde_json::json!([80, 443]));
|
||||||
|
|||||||
@@ -3,5 +3,6 @@ pub mod edge;
|
|||||||
pub mod stun;
|
pub mod stun;
|
||||||
pub mod transport;
|
pub mod transport;
|
||||||
pub mod udp_session;
|
pub mod udp_session;
|
||||||
|
pub mod performance;
|
||||||
|
|
||||||
pub use remoteingress_protocol as protocol;
|
pub use remoteingress_protocol as protocol;
|
||||||
|
|||||||
@@ -0,0 +1,130 @@
|
|||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub enum PerformanceProfile {
|
||||||
|
Balanced,
|
||||||
|
Throughput,
|
||||||
|
HighConcurrency,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for PerformanceProfile {
|
||||||
|
fn default() -> Self {
|
||||||
|
PerformanceProfile::Balanced
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize, Serialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct PerformanceConfig {
|
||||||
|
#[serde(default)]
|
||||||
|
pub profile: Option<PerformanceProfile>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub max_streams_per_edge: Option<usize>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub total_window_budget_bytes: Option<u64>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub min_stream_window_bytes: Option<u32>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub max_stream_window_bytes: Option<u32>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub sustained_stream_window_bytes: Option<u32>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub quic_datagram_receive_buffer_bytes: Option<usize>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct EffectivePerformanceConfig {
|
||||||
|
pub profile: PerformanceProfile,
|
||||||
|
pub max_streams_per_edge: usize,
|
||||||
|
pub total_window_budget_bytes: u64,
|
||||||
|
pub min_stream_window_bytes: u32,
|
||||||
|
pub max_stream_window_bytes: u32,
|
||||||
|
pub sustained_stream_window_bytes: u32,
|
||||||
|
pub quic_datagram_receive_buffer_bytes: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for EffectivePerformanceConfig {
|
||||||
|
fn default() -> Self {
|
||||||
|
PerformanceConfig::default().effective()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PerformanceConfig {
|
||||||
|
pub fn effective(&self) -> EffectivePerformanceConfig {
|
||||||
|
let profile = self.profile.unwrap_or_default();
|
||||||
|
let defaults = profile_defaults(profile);
|
||||||
|
EffectivePerformanceConfig {
|
||||||
|
profile,
|
||||||
|
max_streams_per_edge: self.max_streams_per_edge.unwrap_or(defaults.max_streams_per_edge),
|
||||||
|
total_window_budget_bytes: self.total_window_budget_bytes.unwrap_or(defaults.total_window_budget_bytes),
|
||||||
|
min_stream_window_bytes: self.min_stream_window_bytes.unwrap_or(defaults.min_stream_window_bytes),
|
||||||
|
max_stream_window_bytes: self.max_stream_window_bytes.unwrap_or(defaults.max_stream_window_bytes),
|
||||||
|
sustained_stream_window_bytes: self.sustained_stream_window_bytes.unwrap_or(defaults.sustained_stream_window_bytes),
|
||||||
|
quic_datagram_receive_buffer_bytes: self
|
||||||
|
.quic_datagram_receive_buffer_bytes
|
||||||
|
.unwrap_or(defaults.quic_datagram_receive_buffer_bytes),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn merge(global: Option<&PerformanceConfig>, edge: Option<&PerformanceConfig>) -> PerformanceConfig {
|
||||||
|
let mut merged = global.cloned().unwrap_or_default();
|
||||||
|
if let Some(edge) = edge {
|
||||||
|
if edge.profile.is_some() {
|
||||||
|
merged.profile = edge.profile;
|
||||||
|
}
|
||||||
|
if edge.max_streams_per_edge.is_some() {
|
||||||
|
merged.max_streams_per_edge = edge.max_streams_per_edge;
|
||||||
|
}
|
||||||
|
if edge.total_window_budget_bytes.is_some() {
|
||||||
|
merged.total_window_budget_bytes = edge.total_window_budget_bytes;
|
||||||
|
}
|
||||||
|
if edge.min_stream_window_bytes.is_some() {
|
||||||
|
merged.min_stream_window_bytes = edge.min_stream_window_bytes;
|
||||||
|
}
|
||||||
|
if edge.max_stream_window_bytes.is_some() {
|
||||||
|
merged.max_stream_window_bytes = edge.max_stream_window_bytes;
|
||||||
|
}
|
||||||
|
if edge.sustained_stream_window_bytes.is_some() {
|
||||||
|
merged.sustained_stream_window_bytes = edge.sustained_stream_window_bytes;
|
||||||
|
}
|
||||||
|
if edge.quic_datagram_receive_buffer_bytes.is_some() {
|
||||||
|
merged.quic_datagram_receive_buffer_bytes = edge.quic_datagram_receive_buffer_bytes;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
merged
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn profile_defaults(profile: PerformanceProfile) -> EffectivePerformanceConfig {
|
||||||
|
match profile {
|
||||||
|
PerformanceProfile::Balanced => EffectivePerformanceConfig {
|
||||||
|
profile,
|
||||||
|
max_streams_per_edge: 1024,
|
||||||
|
total_window_budget_bytes: 256 * 1024 * 1024,
|
||||||
|
min_stream_window_bytes: 128 * 1024,
|
||||||
|
max_stream_window_bytes: 8 * 1024 * 1024,
|
||||||
|
sustained_stream_window_bytes: 512 * 1024,
|
||||||
|
quic_datagram_receive_buffer_bytes: 4 * 1024 * 1024,
|
||||||
|
},
|
||||||
|
PerformanceProfile::Throughput => EffectivePerformanceConfig {
|
||||||
|
profile,
|
||||||
|
max_streams_per_edge: 512,
|
||||||
|
total_window_budget_bytes: 512 * 1024 * 1024,
|
||||||
|
min_stream_window_bytes: 256 * 1024,
|
||||||
|
max_stream_window_bytes: 32 * 1024 * 1024,
|
||||||
|
sustained_stream_window_bytes: 2 * 1024 * 1024,
|
||||||
|
quic_datagram_receive_buffer_bytes: 8 * 1024 * 1024,
|
||||||
|
},
|
||||||
|
PerformanceProfile::HighConcurrency => EffectivePerformanceConfig {
|
||||||
|
profile,
|
||||||
|
max_streams_per_edge: 4096,
|
||||||
|
total_window_budget_bytes: 256 * 1024 * 1024,
|
||||||
|
min_stream_window_bytes: 64 * 1024,
|
||||||
|
max_stream_window_bytes: 4 * 1024 * 1024,
|
||||||
|
sustained_stream_window_bytes: 256 * 1024,
|
||||||
|
quic_datagram_receive_buffer_bytes: 16 * 1024 * 1024,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -4,9 +4,9 @@ use serde::{Deserialize, Serialize};
|
|||||||
|
|
||||||
/// Transport mode for the tunnel connection between edge and hub.
|
/// Transport mode for the tunnel connection between edge and hub.
|
||||||
///
|
///
|
||||||
/// - `TcpTls`: TCP + TLS with frame-based multiplexing via TunnelIo (default).
|
/// - `TcpTls`: TCP + TLS with frame-based multiplexing via TunnelIo.
|
||||||
/// - `Quic`: QUIC with native stream multiplexing (one QUIC stream per tunneled connection).
|
/// - `Quic`: QUIC with native stream multiplexing (one QUIC stream per tunneled connection).
|
||||||
/// - `QuicWithFallback`: Try QUIC first, fall back to TCP+TLS if UDP is blocked.
|
/// - `QuicWithFallback`: Try QUIC first, fall back to TCP+TLS if UDP is blocked. This is the edge runtime default.
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)]
|
||||||
#[serde(rename_all = "camelCase")]
|
#[serde(rename_all = "camelCase")]
|
||||||
pub enum TransportMode {
|
pub enum TransportMode {
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
const DEFAULT_DATAGRAM_RECEIVE_BUFFER_SIZE: usize = 4 * 1024 * 1024;
|
||||||
|
|
||||||
/// QUIC control stream message types (reuses frame type constants for consistency).
|
/// QUIC control stream message types (reuses frame type constants for consistency).
|
||||||
pub const CTRL_CONFIG: u8 = 0x06;
|
pub const CTRL_CONFIG: u8 = 0x06;
|
||||||
pub const CTRL_PING: u8 = 0x07;
|
pub const CTRL_PING: u8 = 0x07;
|
||||||
@@ -11,6 +13,13 @@ pub const CTRL_HEADER_SIZE: usize = 5;
|
|||||||
/// Build a quinn ClientConfig that skips server certificate verification
|
/// Build a quinn ClientConfig that skips server certificate verification
|
||||||
/// (auth is via shared secret, same as the TCP+TLS path).
|
/// (auth is via shared secret, same as the TCP+TLS path).
|
||||||
pub fn build_quic_client_config() -> quinn::ClientConfig {
|
pub fn build_quic_client_config() -> quinn::ClientConfig {
|
||||||
|
build_quic_client_config_with_limits(1024, DEFAULT_DATAGRAM_RECEIVE_BUFFER_SIZE)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn build_quic_client_config_with_limits(
|
||||||
|
max_concurrent_bidi_streams: u32,
|
||||||
|
datagram_receive_buffer_size: usize,
|
||||||
|
) -> quinn::ClientConfig {
|
||||||
let mut tls_config = rustls::ClientConfig::builder()
|
let mut tls_config = rustls::ClientConfig::builder()
|
||||||
.dangerous()
|
.dangerous()
|
||||||
.with_custom_certificate_verifier(Arc::new(NoCertVerifier))
|
.with_custom_certificate_verifier(Arc::new(NoCertVerifier))
|
||||||
@@ -28,11 +37,9 @@ pub fn build_quic_client_config() -> quinn::ClientConfig {
|
|||||||
transport.max_idle_timeout(Some(
|
transport.max_idle_timeout(Some(
|
||||||
quinn::IdleTimeout::try_from(std::time::Duration::from_secs(45)).unwrap(),
|
quinn::IdleTimeout::try_from(std::time::Duration::from_secs(45)).unwrap(),
|
||||||
));
|
));
|
||||||
// Match MAX_STREAMS_PER_EDGE (1024) from hub.rs.
|
transport.max_concurrent_bidi_streams(max_concurrent_bidi_streams.into());
|
||||||
// Default is 100 which is too low for high-concurrency tunneling.
|
|
||||||
transport.max_concurrent_bidi_streams(1024u32.into());
|
|
||||||
// Enable QUIC datagrams (RFC 9221) for low-latency UDP tunneling.
|
// Enable QUIC datagrams (RFC 9221) for low-latency UDP tunneling.
|
||||||
transport.datagram_receive_buffer_size(Some(65536));
|
transport.datagram_receive_buffer_size(Some(datagram_receive_buffer_size));
|
||||||
|
|
||||||
let mut client_config = quinn::ClientConfig::new(Arc::new(quic_config));
|
let mut client_config = quinn::ClientConfig::new(Arc::new(quic_config));
|
||||||
client_config.transport_config(Arc::new(transport));
|
client_config.transport_config(Arc::new(transport));
|
||||||
@@ -42,6 +49,18 @@ pub fn build_quic_client_config() -> quinn::ClientConfig {
|
|||||||
/// Build a quinn ServerConfig from the same TLS server config used for TCP+TLS.
|
/// Build a quinn ServerConfig from the same TLS server config used for TCP+TLS.
|
||||||
pub fn build_quic_server_config(
|
pub fn build_quic_server_config(
|
||||||
tls_server_config: rustls::ServerConfig,
|
tls_server_config: rustls::ServerConfig,
|
||||||
|
) -> Result<quinn::ServerConfig, Box<dyn std::error::Error + Send + Sync>> {
|
||||||
|
build_quic_server_config_with_limits(
|
||||||
|
tls_server_config,
|
||||||
|
1024,
|
||||||
|
DEFAULT_DATAGRAM_RECEIVE_BUFFER_SIZE,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn build_quic_server_config_with_limits(
|
||||||
|
tls_server_config: rustls::ServerConfig,
|
||||||
|
max_concurrent_bidi_streams: u32,
|
||||||
|
datagram_receive_buffer_size: usize,
|
||||||
) -> Result<quinn::ServerConfig, Box<dyn std::error::Error + Send + Sync>> {
|
) -> Result<quinn::ServerConfig, Box<dyn std::error::Error + Send + Sync>> {
|
||||||
let quic_config = quinn::crypto::rustls::QuicServerConfig::try_from(tls_server_config)?;
|
let quic_config = quinn::crypto::rustls::QuicServerConfig::try_from(tls_server_config)?;
|
||||||
|
|
||||||
@@ -50,8 +69,8 @@ pub fn build_quic_server_config(
|
|||||||
transport.max_idle_timeout(Some(
|
transport.max_idle_timeout(Some(
|
||||||
quinn::IdleTimeout::try_from(std::time::Duration::from_secs(45)).unwrap(),
|
quinn::IdleTimeout::try_from(std::time::Duration::from_secs(45)).unwrap(),
|
||||||
));
|
));
|
||||||
transport.max_concurrent_bidi_streams(1024u32.into());
|
transport.max_concurrent_bidi_streams(max_concurrent_bidi_streams.into());
|
||||||
transport.datagram_receive_buffer_size(Some(65536));
|
transport.datagram_receive_buffer_size(Some(datagram_receive_buffer_size));
|
||||||
|
|
||||||
let mut server_config = quinn::ServerConfig::with_crypto(Arc::new(quic_config));
|
let mut server_config = quinn::ServerConfig::with_crypto(Arc::new(quic_config));
|
||||||
server_config.transport_config(Arc::new(transport));
|
server_config.transport_config(Arc::new(transport));
|
||||||
|
|||||||
@@ -32,12 +32,18 @@ pub const FRAME_HEADER_SIZE: usize = 9;
|
|||||||
pub const MAX_PAYLOAD_SIZE: u32 = 16 * 1024 * 1024;
|
pub const MAX_PAYLOAD_SIZE: u32 = 16 * 1024 * 1024;
|
||||||
|
|
||||||
// Per-stream flow control constants
|
// Per-stream flow control constants
|
||||||
/// Initial (and maximum) per-stream window size (4 MB).
|
/// Default maximum per-stream window size (8 MB).
|
||||||
pub const INITIAL_STREAM_WINDOW: u32 = 4 * 1024 * 1024;
|
pub const INITIAL_STREAM_WINDOW: u32 = 8 * 1024 * 1024;
|
||||||
/// Send WINDOW_UPDATE after consuming this many bytes (half the initial window).
|
/// Minimum safe window size used when strict budget pressure requires going below the configured floor.
|
||||||
|
pub const ABSOLUTE_MIN_STREAM_WINDOW: u32 = 16 * 1024;
|
||||||
|
/// Default total TCP/TLS flow-control budget per edge connection (256 MB).
|
||||||
|
pub const DEFAULT_TOTAL_WINDOW_BUDGET: u64 = 256 * 1024 * 1024;
|
||||||
|
/// Default preferred minimum stream window (128 KB). The total budget still wins above this.
|
||||||
|
pub const DEFAULT_MIN_STREAM_WINDOW: u32 = 128 * 1024;
|
||||||
|
/// Send WINDOW_UPDATE after consuming this many bytes when no dynamic window is available.
|
||||||
pub const WINDOW_UPDATE_THRESHOLD: u32 = INITIAL_STREAM_WINDOW / 2;
|
pub const WINDOW_UPDATE_THRESHOLD: u32 = INITIAL_STREAM_WINDOW / 2;
|
||||||
/// Maximum window size to prevent overflow.
|
/// Maximum window size to prevent overflow.
|
||||||
pub const MAX_WINDOW_SIZE: u32 = 4 * 1024 * 1024;
|
pub const MAX_WINDOW_SIZE: u32 = 32 * 1024 * 1024;
|
||||||
|
|
||||||
// Sustained stream classification constants
|
// Sustained stream classification constants
|
||||||
/// Throughput threshold for sustained classification (2.5 MB/s = 20 Mbit/s).
|
/// Throughput threshold for sustained classification (2.5 MB/s = 20 Mbit/s).
|
||||||
@@ -55,11 +61,37 @@ pub fn encode_window_update(stream_id: u32, frame_type: u8, increment: u32) -> B
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Compute the target per-stream window size based on the number of active streams.
|
/// Compute the target per-stream window size based on the number of active streams.
|
||||||
/// Total memory budget is ~200MB shared across all streams. Up to 50 streams get the
|
/// The total budget is authoritative: the configured minimum is a preference, not
|
||||||
/// full 4MB window; above that the window scales down to a 1MB floor at 200+ streams.
|
/// permission to exceed the edge-level memory budget under very high concurrency.
|
||||||
pub fn compute_window_for_stream_count(active: u32) -> u32 {
|
pub fn compute_window_for_stream_count(active: u32) -> u32 {
|
||||||
let per_stream = (200 * 1024 * 1024u64) / (active.max(1) as u64);
|
compute_window_for_limits(
|
||||||
per_stream.clamp(1 * 1024 * 1024, INITIAL_STREAM_WINDOW as u64) as u32
|
active,
|
||||||
|
DEFAULT_TOTAL_WINDOW_BUDGET,
|
||||||
|
DEFAULT_MIN_STREAM_WINDOW,
|
||||||
|
INITIAL_STREAM_WINDOW,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn compute_window_for_limits(
|
||||||
|
active: u32,
|
||||||
|
total_budget_bytes: u64,
|
||||||
|
min_window_bytes: u32,
|
||||||
|
max_window_bytes: u32,
|
||||||
|
) -> u32 {
|
||||||
|
let active = active.max(1) as u64;
|
||||||
|
let max_window = max_window_bytes.max(ABSOLUTE_MIN_STREAM_WINDOW);
|
||||||
|
let preferred_min = min_window_bytes
|
||||||
|
.max(ABSOLUTE_MIN_STREAM_WINDOW)
|
||||||
|
.min(max_window);
|
||||||
|
let per_stream_budget = total_budget_bytes
|
||||||
|
.max(ABSOLUTE_MIN_STREAM_WINDOW as u64)
|
||||||
|
/ active;
|
||||||
|
let bounded = per_stream_budget.min(max_window as u64);
|
||||||
|
if bounded >= preferred_min as u64 {
|
||||||
|
bounded as u32
|
||||||
|
} else {
|
||||||
|
bounded.max(ABSOLUTE_MIN_STREAM_WINDOW as u64) as u32
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Decode a WINDOW_UPDATE payload into a byte increment. Returns None if payload is malformed.
|
/// Decode a WINDOW_UPDATE payload into a byte increment. Returns None if payload is malformed.
|
||||||
@@ -307,6 +339,13 @@ pub struct TunnelIo<S> {
|
|||||||
write: WriteState,
|
write: WriteState,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, Default)]
|
||||||
|
pub struct TunnelQueueDepths {
|
||||||
|
pub ctrl: usize,
|
||||||
|
pub data: usize,
|
||||||
|
pub sustained: usize,
|
||||||
|
}
|
||||||
|
|
||||||
impl<S: AsyncRead + AsyncWrite + Unpin> TunnelIo<S> {
|
impl<S: AsyncRead + AsyncWrite + Unpin> TunnelIo<S> {
|
||||||
pub fn new(stream: S, initial_data: Vec<u8>) -> Self {
|
pub fn new(stream: S, initial_data: Vec<u8>) -> Self {
|
||||||
let read_pos = initial_data.len();
|
let read_pos = initial_data.len();
|
||||||
@@ -346,6 +385,14 @@ impl<S: AsyncRead + AsyncWrite + Unpin> TunnelIo<S> {
|
|||||||
self.write.sustained_queue.push_back(frame);
|
self.write.sustained_queue.push_back(frame);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn queue_depths(&self) -> TunnelQueueDepths {
|
||||||
|
TunnelQueueDepths {
|
||||||
|
ctrl: self.write.ctrl_queue.len(),
|
||||||
|
data: self.write.data_queue.len(),
|
||||||
|
sustained: self.write.sustained_queue.len(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Try to parse a complete frame from the read buffer.
|
/// Try to parse a complete frame from the read buffer.
|
||||||
/// Uses a parse_pos cursor to avoid drain() on every frame.
|
/// Uses a parse_pos cursor to avoid drain() on every frame.
|
||||||
pub fn try_parse_frame(&mut self) -> Option<Result<Frame, std::io::Error>> {
|
pub fn try_parse_frame(&mut self) -> Option<Result<Frame, std::io::Error>> {
|
||||||
@@ -910,7 +957,7 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_adaptive_window_zero_streams() {
|
fn test_adaptive_window_zero_streams() {
|
||||||
// 0 streams treated as 1: 200MB/1 -> clamped to 4MB max
|
// 0 streams treated as 1: budget/1 -> clamped to max
|
||||||
assert_eq!(compute_window_for_stream_count(0), INITIAL_STREAM_WINDOW);
|
assert_eq!(compute_window_for_stream_count(0), INITIAL_STREAM_WINDOW);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -920,47 +967,44 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_adaptive_window_50_streams_full() {
|
fn test_adaptive_window_32_streams_full() {
|
||||||
// 200MB/50 = 4MB = exactly INITIAL_STREAM_WINDOW
|
// 256MB/32 = 8MB = exactly INITIAL_STREAM_WINDOW
|
||||||
assert_eq!(compute_window_for_stream_count(50), INITIAL_STREAM_WINDOW);
|
assert_eq!(compute_window_for_stream_count(32), INITIAL_STREAM_WINDOW);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_adaptive_window_51_streams_starts_scaling() {
|
fn test_adaptive_window_33_streams_starts_scaling() {
|
||||||
// 200MB/51 < 4MB — first value below max
|
// 256MB/33 < 8MB — first value below max
|
||||||
let w = compute_window_for_stream_count(51);
|
let w = compute_window_for_stream_count(33);
|
||||||
assert!(w < INITIAL_STREAM_WINDOW);
|
assert!(w < INITIAL_STREAM_WINDOW);
|
||||||
assert_eq!(w, (200 * 1024 * 1024u64 / 51) as u32);
|
assert_eq!(w, (DEFAULT_TOTAL_WINDOW_BUDGET / 33) as u32);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_adaptive_window_100_streams() {
|
fn test_adaptive_window_100_streams() {
|
||||||
// 200MB/100 = 2MB
|
assert_eq!(compute_window_for_stream_count(100), (DEFAULT_TOTAL_WINDOW_BUDGET / 100) as u32);
|
||||||
assert_eq!(compute_window_for_stream_count(100), 2 * 1024 * 1024);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_adaptive_window_200_streams_at_floor() {
|
fn test_adaptive_window_200_streams_uses_budget() {
|
||||||
// 200MB/200 = 1MB = exactly the floor
|
assert_eq!(compute_window_for_stream_count(200), (DEFAULT_TOTAL_WINDOW_BUDGET / 200) as u32);
|
||||||
assert_eq!(compute_window_for_stream_count(200), 1 * 1024 * 1024);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_adaptive_window_500_streams_clamped() {
|
fn test_adaptive_window_500_streams_stays_under_budget() {
|
||||||
// 200MB/500 = 0.4MB -> clamped up to 1MB floor
|
assert_eq!(compute_window_for_stream_count(500), (DEFAULT_TOTAL_WINDOW_BUDGET / 500) as u32);
|
||||||
assert_eq!(compute_window_for_stream_count(500), 1 * 1024 * 1024);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_adaptive_window_max_u32() {
|
fn test_adaptive_window_max_u32() {
|
||||||
// Extreme: u32::MAX streams -> tiny value -> clamped to 1MB
|
// Extreme: u32::MAX streams -> tiny value -> clamped to absolute minimum.
|
||||||
assert_eq!(compute_window_for_stream_count(u32::MAX), 1 * 1024 * 1024);
|
assert_eq!(compute_window_for_stream_count(u32::MAX), ABSOLUTE_MIN_STREAM_WINDOW);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_adaptive_window_monotonically_decreasing() {
|
fn test_adaptive_window_monotonically_decreasing() {
|
||||||
let mut prev = compute_window_for_stream_count(1);
|
let mut prev = compute_window_for_stream_count(1);
|
||||||
for n in [2, 10, 50, 51, 100, 200, 500, 1000] {
|
for n in [2, 10, 32, 33, 100, 200, 500, 1000] {
|
||||||
let w = compute_window_for_stream_count(n);
|
let w = compute_window_for_stream_count(n);
|
||||||
assert!(w <= prev, "window increased from {} to {} at n={}", prev, w, n);
|
assert!(w <= prev, "window increased from {} to {} at n={}", prev, w, n);
|
||||||
prev = w;
|
prev = w;
|
||||||
@@ -969,11 +1013,12 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_adaptive_window_total_budget_bounded() {
|
fn test_adaptive_window_total_budget_bounded() {
|
||||||
// active x per_stream_window should never exceed 200MB (+ clamp overhead for high N)
|
// active x per_stream_window should never exceed the configured budget while the
|
||||||
for n in [1, 10, 50, 100, 200] {
|
// budget can still provide at least the absolute minimum per stream.
|
||||||
|
for n in [1, 10, 32, 33, 100, 200, 500, 1000] {
|
||||||
let w = compute_window_for_stream_count(n);
|
let w = compute_window_for_stream_count(n);
|
||||||
let total = w as u64 * n as u64;
|
let total = w as u64 * n as u64;
|
||||||
assert!(total <= 200 * 1024 * 1024, "total {}MB exceeds budget at n={}", total / (1024*1024), n);
|
assert!(total <= DEFAULT_TOTAL_WINDOW_BUDGET, "total {}MB exceeds budget at n={}", total / (1024*1024), n);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -322,6 +322,11 @@ tap.test('TCP/TLS setup: start TCP echo server and TCP+TLS tunnel', async () =>
|
|||||||
tunnel = await startTunnel(edgePort, hubPort);
|
tunnel = await startTunnel(edgePort, hubPort);
|
||||||
|
|
||||||
expect(tunnel.hub.running).toBeTrue();
|
expect(tunnel.hub.running).toBeTrue();
|
||||||
|
const hubStatus = await tunnel.hub.getStatus();
|
||||||
|
expect(hubStatus.connectedEdges.length).toBeGreaterThanOrEqual(1);
|
||||||
|
const edgeStatus = hubStatus.connectedEdges[0];
|
||||||
|
expect(['quic', 'tcpTls'].includes(edgeStatus.transportMode)).toEqual(true);
|
||||||
|
expect(edgeStatus.performance.maxStreamsPerEdge).toBeGreaterThanOrEqual(1024);
|
||||||
});
|
});
|
||||||
|
|
||||||
tap.test('TCP/TLS: single TCP stream — 32MB transfer exceeding initial 4MB window', async () => {
|
tap.test('TCP/TLS: single TCP stream — 32MB transfer exceeding initial 4MB window', async () => {
|
||||||
|
|||||||
@@ -185,6 +185,14 @@ tap.test('QUIC setup: start TCP echo server and QUIC tunnel', async () => {
|
|||||||
expect(tunnel.hub.running).toBeTrue();
|
expect(tunnel.hub.running).toBeTrue();
|
||||||
const status = await tunnel.edge.getStatus();
|
const status = await tunnel.edge.getStatus();
|
||||||
expect(status.connected).toBeTrue();
|
expect(status.connected).toBeTrue();
|
||||||
|
const hubStatus = await tunnel.hub.getStatus();
|
||||||
|
expect(hubStatus.connectedEdges.length).toBeGreaterThanOrEqual(1);
|
||||||
|
const edgeStatus = hubStatus.connectedEdges[0];
|
||||||
|
expect(edgeStatus.transportMode).toEqual('quic');
|
||||||
|
expect(edgeStatus.fallbackUsed).toEqual(false);
|
||||||
|
expect(edgeStatus.performance.profile).toEqual('balanced');
|
||||||
|
expect(edgeStatus.flowControl.applies).toEqual(false);
|
||||||
|
expect(edgeStatus.traffic.streamsOpenedTotal).toEqual(0);
|
||||||
});
|
});
|
||||||
|
|
||||||
tap.test('QUIC: single TCP stream echo — 1KB', async () => {
|
tap.test('QUIC: single TCP stream echo — 1KB', async () => {
|
||||||
|
|||||||
@@ -3,6 +3,6 @@
|
|||||||
*/
|
*/
|
||||||
export const commitinfo = {
|
export const commitinfo = {
|
||||||
name: '@serve.zone/remoteingress',
|
name: '@serve.zone/remoteingress',
|
||||||
version: '4.15.3',
|
version: '4.17.0',
|
||||||
description: 'Edge ingress tunnel for DcRouter - tunnels TCP and UDP traffic from the network edge to SmartProxy over TLS or QUIC, preserving client IP via PROXY protocol.'
|
description: 'Edge ingress tunnel for DcRouter - tunnels TCP and UDP traffic from the network edge to SmartProxy over TLS or QUIC, preserving client IP via PROXY protocol.'
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,11 +9,12 @@ type THubCommands = {
|
|||||||
};
|
};
|
||||||
startHub: {
|
startHub: {
|
||||||
params: {
|
params: {
|
||||||
tunnelPort: number;
|
tunnelPort: number;
|
||||||
targetHost?: string;
|
targetHost?: string;
|
||||||
tlsCertPem?: string;
|
tlsCertPem?: string;
|
||||||
tlsKeyPem?: string;
|
tlsKeyPem?: string;
|
||||||
};
|
performance?: IPerformanceConfig;
|
||||||
|
};
|
||||||
result: { started: boolean };
|
result: { started: boolean };
|
||||||
};
|
};
|
||||||
stopHub: {
|
stopHub: {
|
||||||
@@ -22,7 +23,7 @@ type THubCommands = {
|
|||||||
};
|
};
|
||||||
updateAllowedEdges: {
|
updateAllowedEdges: {
|
||||||
params: {
|
params: {
|
||||||
edges: Array<{ id: string; secret: string; listenPorts?: number[]; listenPortsUdp?: number[]; stunIntervalSecs?: number; firewallConfig?: IFirewallConfig }>;
|
edges: Array<{ id: string; secret: string; listenPorts?: number[]; listenPortsUdp?: number[]; stunIntervalSecs?: number; firewallConfig?: IFirewallConfig; performance?: IPerformanceConfig }>;
|
||||||
};
|
};
|
||||||
result: { updated: boolean };
|
result: { updated: boolean };
|
||||||
};
|
};
|
||||||
@@ -31,12 +32,19 @@ type THubCommands = {
|
|||||||
result: {
|
result: {
|
||||||
running: boolean;
|
running: boolean;
|
||||||
tunnelPort: number;
|
tunnelPort: number;
|
||||||
connectedEdges: Array<{
|
connectedEdges: Array<{
|
||||||
edgeId: string;
|
edgeId: string;
|
||||||
connectedAt: number;
|
connectedAt: number;
|
||||||
activeStreams: number;
|
activeStreams: number;
|
||||||
peerAddr: string;
|
peerAddr: string;
|
||||||
}>;
|
transportMode: 'tcpTls' | 'quic' | 'quicWithFallback';
|
||||||
|
fallbackUsed: boolean;
|
||||||
|
performance: IEffectivePerformanceConfig;
|
||||||
|
flowControl: IFlowControlStatus;
|
||||||
|
queues: IQueueStatus;
|
||||||
|
traffic: ITrafficStatus;
|
||||||
|
udp: IUdpStatus;
|
||||||
|
}>;
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
@@ -73,9 +81,61 @@ export interface IHubConfig {
|
|||||||
certPem?: string;
|
certPem?: string;
|
||||||
keyPem?: string;
|
keyPem?: string;
|
||||||
};
|
};
|
||||||
|
performance?: IPerformanceConfig;
|
||||||
}
|
}
|
||||||
|
|
||||||
type TAllowedEdge = { id: string; secret: string; listenPorts?: number[]; listenPortsUdp?: number[]; stunIntervalSecs?: number; firewallConfig?: IFirewallConfig };
|
export type TPerformanceProfile = 'balanced' | 'throughput' | 'highConcurrency';
|
||||||
|
|
||||||
|
export interface IPerformanceConfig {
|
||||||
|
profile?: TPerformanceProfile;
|
||||||
|
maxStreamsPerEdge?: number;
|
||||||
|
totalWindowBudgetBytes?: number;
|
||||||
|
minStreamWindowBytes?: number;
|
||||||
|
maxStreamWindowBytes?: number;
|
||||||
|
sustainedStreamWindowBytes?: number;
|
||||||
|
quicDatagramReceiveBufferBytes?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface IEffectivePerformanceConfig {
|
||||||
|
profile: TPerformanceProfile;
|
||||||
|
maxStreamsPerEdge: number;
|
||||||
|
totalWindowBudgetBytes: number;
|
||||||
|
minStreamWindowBytes: number;
|
||||||
|
maxStreamWindowBytes: number;
|
||||||
|
sustainedStreamWindowBytes: number;
|
||||||
|
quicDatagramReceiveBufferBytes: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface IFlowControlStatus {
|
||||||
|
applies: boolean;
|
||||||
|
currentWindowBytes: number;
|
||||||
|
minWindowBytes: number;
|
||||||
|
maxWindowBytes: number;
|
||||||
|
totalWindowBudgetBytes: number;
|
||||||
|
estimatedInFlightBytes: number;
|
||||||
|
stalledStreams: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface IQueueStatus {
|
||||||
|
ctrlQueueDepth: number;
|
||||||
|
dataQueueDepth: number;
|
||||||
|
sustainedQueueDepth: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ITrafficStatus {
|
||||||
|
bytesIn: number;
|
||||||
|
bytesOut: number;
|
||||||
|
streamsOpenedTotal: number;
|
||||||
|
streamsClosedTotal: number;
|
||||||
|
rejectedStreams: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface IUdpStatus {
|
||||||
|
activeSessions: number;
|
||||||
|
droppedDatagrams: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
type TAllowedEdge = { id: string; secret: string; listenPorts?: number[]; listenPortsUdp?: number[]; stunIntervalSecs?: number; firewallConfig?: IFirewallConfig; performance?: IPerformanceConfig };
|
||||||
|
|
||||||
const MAX_RESTART_ATTEMPTS = 10;
|
const MAX_RESTART_ATTEMPTS = 10;
|
||||||
const MAX_RESTART_BACKOFF_MS = 30_000;
|
const MAX_RESTART_BACKOFF_MS = 30_000;
|
||||||
@@ -160,6 +220,7 @@ export class RemoteIngressHub extends EventEmitter {
|
|||||||
await this.bridge.sendCommand('startHub', {
|
await this.bridge.sendCommand('startHub', {
|
||||||
tunnelPort: config.tunnelPort ?? 8443,
|
tunnelPort: config.tunnelPort ?? 8443,
|
||||||
targetHost: config.targetHost ?? '127.0.0.1',
|
targetHost: config.targetHost ?? '127.0.0.1',
|
||||||
|
...(config.performance ? { performance: config.performance } : {}),
|
||||||
...(config.tls?.certPem && config.tls?.keyPem
|
...(config.tls?.certPem && config.tls?.keyPem
|
||||||
? { tlsCertPem: config.tls.certPem, tlsKeyPem: config.tls.keyPem }
|
? { tlsCertPem: config.tls.certPem, tlsKeyPem: config.tls.keyPem }
|
||||||
: {}),
|
: {}),
|
||||||
@@ -266,6 +327,7 @@ export class RemoteIngressHub extends EventEmitter {
|
|||||||
...(config.tls?.certPem && config.tls?.keyPem
|
...(config.tls?.certPem && config.tls?.keyPem
|
||||||
? { tlsCertPem: config.tls.certPem, tlsKeyPem: config.tls.keyPem }
|
? { tlsCertPem: config.tls.certPem, tlsKeyPem: config.tls.keyPem }
|
||||||
: {}),
|
: {}),
|
||||||
|
...(config.performance ? { performance: config.performance } : {}),
|
||||||
});
|
});
|
||||||
|
|
||||||
// Restore allowed edges
|
// Restore allowed edges
|
||||||
|
|||||||
Reference in New Issue
Block a user