Compare commits

..

20 Commits

Author SHA1 Message Date
b1f4181139 v25.11.9
Some checks failed
Default (tags) / security (push) Failing after 1s
Default (tags) / test (push) Failing after 1s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-03-16 09:38:55 +00:00
a1b8d40011 fix(rustproxy-routing): reduce hot-path allocations in routing, metrics, and proxy protocol handling 2026-03-16 09:38:55 +00:00
246b44913e v25.11.8
Some checks failed
Default (tags) / security (push) Failing after 1s
Default (tags) / test (push) Failing after 1s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-03-16 08:58:11 +00:00
b3d4949225 fix(rustproxy-http): prevent premature idle timeouts during streamed HTTP responses and ensure TLS close_notify is sent on dropped connections 2026-03-16 08:58:11 +00:00
0475e6b442 v25.11.7
Some checks failed
Default (tags) / security (push) Failing after 1s
Default (tags) / test (push) Failing after 0s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-03-16 03:01:16 +00:00
8cdb95a853 fix(rustproxy): prevent TLS route reload certificate mismatches and tighten passthrough connection handling 2026-03-16 03:01:16 +00:00
8cefe9d66a v25.11.6
Some checks failed
Default (tags) / security (push) Failing after 0s
Default (tags) / test (push) Failing after 0s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-03-16 02:05:47 +00:00
d5e08c83fc fix(rustproxy-http,rustproxy-passthrough): improve upstream connection cleanup and graceful tunnel shutdown 2026-03-16 02:05:47 +00:00
1247f48856 v25.11.5
Some checks failed
Default (tags) / security (push) Failing after 1s
Default (tags) / test (push) Failing after 1s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-03-16 00:03:10 +00:00
e3bae4c399 fix(repo): no changes to commit 2026-03-16 00:03:10 +00:00
0930f7e10c v25.11.4
Some checks failed
Default (tags) / security (push) Failing after 1s
Default (tags) / test (push) Failing after 1s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-03-15 21:44:32 +00:00
aa9e6dfd94 fix(rustproxy-http): report streamed HTTP and WebSocket bytes per chunk for real-time throughput metrics 2026-03-15 21:44:32 +00:00
211d5cf835 v25.11.3
Some checks failed
Default (tags) / security (push) Failing after 1s
Default (tags) / test (push) Failing after 1s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-03-15 17:00:33 +00:00
2ce1899337 fix(repo): no changes to commit 2026-03-15 17:00:33 +00:00
2e2ffc4485 v25.11.2
Some checks failed
Default (tags) / security (push) Failing after 1s
Default (tags) / test (push) Failing after 1s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-03-15 16:58:41 +00:00
da26816af5 fix(rustproxy-http): avoid reusing HTTP/1 senders during streaming responses and relax HTTP/2 keep-alive timeouts 2026-03-15 16:58:41 +00:00
d598bffec3 v25.11.1
Some checks failed
Default (tags) / security (push) Failing after 1s
Default (tags) / test (push) Failing after 1s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-03-15 16:24:41 +00:00
a9dbccfaff fix(rustproxy-http): keep connection idle tracking alive during streaming and tune HTTP/2 connection lifetimes 2026-03-15 16:24:41 +00:00
386859a2bd v25.11.0
Some checks failed
Default (tags) / security (push) Failing after 1s
Default (tags) / test (push) Failing after 1s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-03-15 16:00:26 +00:00
2b58615d24 feat(rustproxy-http): add HTTP/2 Extended CONNECT WebSocket proxy support 2026-03-15 16:00:26 +00:00
14 changed files with 695 additions and 232 deletions

View File

@@ -1,5 +1,71 @@
# Changelog # Changelog
## 2026-03-16 - 25.11.9 - fix(rustproxy-routing)
reduce hot-path allocations in routing, metrics, and proxy protocol handling
- skip HTTP header map construction unless a route on the current port uses header matching
- reuse computed client IP strings during HTTP route matching to avoid redundant allocations
- optimize per-route and per-IP metric updates with get-first lookups to avoid unnecessary String creation on existing entries
- replace heap-allocated PROXY protocol peek and discard buffers with stack-allocated buffers in the TCP listener
- improve domain matcher case-insensitive wildcard checks while preserving glob fallback behavior
## 2026-03-16 - 25.11.8 - fix(rustproxy-http)
prevent premature idle timeouts during streamed HTTP responses and ensure TLS close_notify is sent on dropped connections
- track active streaming response bodies so the HTTP idle watchdog does not close connections mid-transfer
- add a ShutdownOnDrop wrapper for TLS-terminated HTTP connections to send shutdown on drop and avoid improperly terminated TLS sessions
- apply the shutdown wrapper in passthrough TLS terminate and terminate+reencrypt HTTP handling
## 2026-03-16 - 25.11.7 - fix(rustproxy)
prevent TLS route reload certificate mismatches and tighten passthrough connection handling
- Load updated TLS configs before swapping the route manager so newly visible routes always have their certificates available.
- Add timeouts when peeking initial decrypted data after TLS handshake to avoid leaked idle connections.
- Raise dropped, blocked, unmatched, and errored passthrough connection events from debug to warn for better operational visibility.
## 2026-03-16 - 25.11.6 - fix(rustproxy-http,rustproxy-passthrough)
improve upstream connection cleanup and graceful tunnel shutdown
- Evict pooled HTTP/2 connections when their driver exits and shorten the maximum pooled H2 age to reduce reuse of stale upstream connections.
- Strip hop-by-hop headers from backend responses before forwarding to HTTP/2 clients to avoid invalid H2 response handling.
- Replace immediate task aborts in WebSocket and TCP tunnel watchdogs with cancellation-driven graceful shutdown plus timed fallback aborts.
- Use non-blocking semaphore acquisition in the TCP listener so connection limits do not stall the accept loop for the entire port.
## 2026-03-16 - 25.11.5 - fix(repo)
no changes to commit
## 2026-03-15 - 25.11.4 - fix(rustproxy-http)
report streamed HTTP and WebSocket bytes per chunk for real-time throughput metrics
- Update CountingBody to record bytes immediately on each data frame instead of aggregating until completion or drop
- Record WebSocket tunnel traffic inside both copy loops and remove the final aggregate byte report to keep throughput metrics current
## 2026-03-15 - 25.11.3 - fix(repo)
no changes to commit
## 2026-03-15 - 25.11.2 - fix(rustproxy-http)
avoid reusing HTTP/1 senders during streaming responses and relax HTTP/2 keep-alive timeouts
- Stop returning HTTP/1 senders to the connection pool before upstream response bodies finish streaming to prevent unsafe reuse on active connections.
- Increase HTTP/2 keep-alive timeout from 5 seconds to 30 seconds in proxy connection builders to better support longer-lived backend streams.
- Improves reliability for large streaming payloads and backend fallback request handling.
## 2026-03-15 - 25.11.1 - fix(rustproxy-http)
keep connection idle tracking alive during streaming and tune HTTP/2 connection lifetimes
- Propagate connection activity tracking through HTTP/1, HTTP/2, and WebSocket forwarding so active request and response body streams do not trigger the idle watchdog.
- Update CountingBody to refresh connection activity timestamps while data frames are polled during uploads and downloads.
- Increase pooled HTTP/2 max age and set explicit HTTP/2 connection window sizes to improve long-lived streaming behavior.
## 2026-03-15 - 25.11.0 - feat(rustproxy-http)
add HTTP/2 Extended CONNECT WebSocket proxy support
- Enable HTTP/2 CONNECT protocol support on the Hyper auto connection builder
- Detect WebSocket requests for both HTTP/1 Upgrade and HTTP/2 Extended CONNECT flows
- Translate HTTP/2 WebSocket requests to an HTTP/1.1 backend handshake and return RFC-compliant client responses
## 2026-03-12 - 25.10.7 - fix(rustproxy-http) ## 2026-03-12 - 25.10.7 - fix(rustproxy-http)
remove Host header from HTTP/2 upstream requests while preserving it for HTTP/1 retries remove Host header from HTTP/2 upstream requests while preserving it for HTTP/1 retries

View File

@@ -1,6 +1,6 @@
{ {
"name": "@push.rocks/smartproxy", "name": "@push.rocks/smartproxy",
"version": "25.10.7", "version": "25.11.9",
"private": false, "private": false,
"description": "A powerful proxy package with unified route-based configuration for high traffic management. Features include SSL/TLS support, flexible routing patterns, WebSocket handling, advanced security options, and automatic ACME certificate management.", "description": "A powerful proxy package with unified route-based configuration for high traffic management. Features include SSL/TLS support, flexible routing patterns, WebSocket handling, advanced security options, and automatic ACME certificate management.",
"main": "dist_ts/index.js", "main": "dist_ts/index.js",

View File

@@ -20,6 +20,7 @@ const IDLE_TIMEOUT: Duration = Duration::from_secs(90);
const EVICTION_INTERVAL: Duration = Duration::from_secs(30); const EVICTION_INTERVAL: Duration = Duration::from_secs(30);
/// Maximum age for pooled HTTP/2 connections before proactive eviction. /// Maximum age for pooled HTTP/2 connections before proactive eviction.
/// Prevents staleness from backends that close idle connections (e.g. nginx GOAWAY). /// Prevents staleness from backends that close idle connections (e.g. nginx GOAWAY).
/// 120s is well within typical server GOAWAY windows (nginx: ~60s idle, envoy: ~60s).
const MAX_H2_AGE: Duration = Duration::from_secs(120); const MAX_H2_AGE: Duration = Duration::from_secs(120);
/// Identifies a unique backend endpoint. /// Identifies a unique backend endpoint.

View File

@@ -11,20 +11,26 @@ use rustproxy_metrics::MetricsCollector;
/// Wraps any `http_body::Body` and counts data bytes passing through. /// Wraps any `http_body::Body` and counts data bytes passing through.
/// ///
/// When the body is fully consumed or dropped, accumulated byte counts /// Each chunk is reported to the `MetricsCollector` immediately so that
/// are reported to the `MetricsCollector`. /// the throughput tracker (sampled at 1 Hz) reflects real-time data flow.
/// ///
/// The inner body is pinned on the heap to support `!Unpin` types like `hyper::body::Incoming`. /// The inner body is pinned on the heap to support `!Unpin` types like `hyper::body::Incoming`.
pub struct CountingBody<B> { pub struct CountingBody<B> {
inner: Pin<Box<B>>, inner: Pin<Box<B>>,
counted_bytes: AtomicU64,
metrics: Arc<MetricsCollector>, metrics: Arc<MetricsCollector>,
route_id: Option<String>, route_id: Option<String>,
source_ip: Option<String>, source_ip: Option<String>,
/// Whether we count bytes as "in" (request body) or "out" (response body). /// Whether we count bytes as "in" (request body) or "out" (response body).
direction: Direction, direction: Direction,
/// Whether we've already reported the bytes (to avoid double-reporting on drop). /// Optional connection-level activity tracker. When set, poll_frame updates this
reported: bool, /// to keep the idle watchdog alive during active body streaming (uploads/downloads).
connection_activity: Option<Arc<AtomicU64>>,
/// Start instant for computing elapsed ms for connection_activity.
activity_start: Option<std::time::Instant>,
/// Optional active-request counter. When set, CountingBody increments on creation
/// and decrements on Drop, keeping the HTTP idle watchdog aware that a response
/// body is still streaming (even after the request handler has returned).
active_requests: Option<Arc<AtomicU64>>,
} }
/// Which direction the bytes flow. /// Which direction the bytes flow.
@@ -47,42 +53,46 @@ impl<B> CountingBody<B> {
) -> Self { ) -> Self {
Self { Self {
inner: Box::pin(inner), inner: Box::pin(inner),
counted_bytes: AtomicU64::new(0),
metrics, metrics,
route_id, route_id,
source_ip, source_ip,
direction, direction,
reported: false, connection_activity: None,
activity_start: None,
active_requests: None,
} }
} }
/// Report accumulated bytes to the metrics collector. /// Set the connection-level activity tracker. When set, each data frame
fn report(&mut self) { /// updates this timestamp to prevent the idle watchdog from killing the
if self.reported { /// connection during active body streaming.
return; pub fn with_connection_activity(mut self, activity: Arc<AtomicU64>, start: std::time::Instant) -> Self {
} self.connection_activity = Some(activity);
self.reported = true; self.activity_start = Some(start);
self
}
let bytes = self.counted_bytes.load(Ordering::Relaxed); /// Set the active-request counter for the HTTP idle watchdog.
if bytes == 0 { /// CountingBody increments on creation and decrements on Drop, ensuring the
return; /// idle watchdog sees an "active request" while the response body streams.
} pub fn with_active_requests(mut self, counter: Arc<AtomicU64>) -> Self {
counter.fetch_add(1, Ordering::Relaxed);
self.active_requests = Some(counter);
self
}
/// Report a chunk of bytes immediately to the metrics collector.
#[inline]
fn report_chunk(&self, len: u64) {
let route_id = self.route_id.as_deref(); let route_id = self.route_id.as_deref();
let source_ip = self.source_ip.as_deref(); let source_ip = self.source_ip.as_deref();
match self.direction { match self.direction {
Direction::In => self.metrics.record_bytes(bytes, 0, route_id, source_ip), Direction::In => self.metrics.record_bytes(len, 0, route_id, source_ip),
Direction::Out => self.metrics.record_bytes(0, bytes, route_id, source_ip), Direction::Out => self.metrics.record_bytes(0, len, route_id, source_ip),
} }
} }
} }
impl<B> Drop for CountingBody<B> {
fn drop(&mut self) {
self.report();
}
}
// CountingBody is Unpin because inner is Pin<Box<B>> (always Unpin). // CountingBody is Unpin because inner is Pin<Box<B>> (always Unpin).
impl<B> Unpin for CountingBody<B> {} impl<B> Unpin for CountingBody<B> {}
@@ -102,16 +112,18 @@ where
match this.inner.as_mut().poll_frame(cx) { match this.inner.as_mut().poll_frame(cx) {
Poll::Ready(Some(Ok(frame))) => { Poll::Ready(Some(Ok(frame))) => {
if let Some(data) = frame.data_ref() { if let Some(data) = frame.data_ref() {
this.counted_bytes.fetch_add(data.len() as u64, Ordering::Relaxed); let len = data.len() as u64;
// Report bytes immediately so the 1 Hz throughput sampler sees them
this.report_chunk(len);
// Keep the connection-level idle watchdog alive during body streaming
if let (Some(activity), Some(start)) = (&this.connection_activity, &this.activity_start) {
activity.store(start.elapsed().as_millis() as u64, Ordering::Relaxed);
}
} }
Poll::Ready(Some(Ok(frame))) Poll::Ready(Some(Ok(frame)))
} }
Poll::Ready(Some(Err(e))) => Poll::Ready(Some(Err(e))), Poll::Ready(Some(Err(e))) => Poll::Ready(Some(Err(e))),
Poll::Ready(None) => { Poll::Ready(None) => Poll::Ready(None),
// Body is fully consumed — report now
this.report();
Poll::Ready(None)
}
Poll::Pending => Poll::Pending, Poll::Pending => Poll::Pending,
} }
} }
@@ -124,3 +136,13 @@ where
self.inner.size_hint() self.inner.size_hint()
} }
} }
impl<B> Drop for CountingBody<B> {
fn drop(&mut self) {
// Decrement the active-request counter so the HTTP idle watchdog
// knows this response body is no longer streaming.
if let Some(ref counter) = self.active_requests {
counter.fetch_sub(1, Ordering::Relaxed);
}
}
}

View File

@@ -9,6 +9,7 @@ pub mod protocol_cache;
pub mod proxy_service; pub mod proxy_service;
pub mod request_filter; pub mod request_filter;
pub mod response_filter; pub mod response_filter;
pub mod shutdown_on_drop;
pub mod template; pub mod template;
pub mod upstream_selector; pub mod upstream_selector;

View File

@@ -33,6 +33,18 @@ use crate::request_filter::RequestFilter;
use crate::response_filter::ResponseFilter; use crate::response_filter::ResponseFilter;
use crate::upstream_selector::UpstreamSelector; use crate::upstream_selector::UpstreamSelector;
/// Per-connection context for keeping the idle watchdog alive during body streaming.
/// Passed through the forwarding chain so CountingBody can update the timestamp.
#[derive(Clone)]
struct ConnActivity {
last_activity: Arc<AtomicU64>,
start: std::time::Instant,
/// Active-request counter from handle_io's idle watchdog. When set, CountingBody
/// increments on creation and decrements on Drop, keeping the watchdog aware that
/// a response body is still streaming after the request handler has returned.
active_requests: Option<Arc<AtomicU64>>,
}
/// Default upstream connect timeout (30 seconds). /// Default upstream connect timeout (30 seconds).
const DEFAULT_CONNECT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); const DEFAULT_CONNECT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);
@@ -294,8 +306,9 @@ impl HttpProxyService {
let cn = cancel_inner.clone(); let cn = cancel_inner.clone();
let la = Arc::clone(&la_inner); let la = Arc::clone(&la_inner);
let st = start; let st = start;
let ca = ConnActivity { last_activity: Arc::clone(&la_inner), start, active_requests: Some(Arc::clone(&ar_inner)) };
async move { async move {
let result = svc.handle_request(req, peer, port, cn).await; let result = svc.handle_request(req, peer, port, cn, ca).await;
// Mark request end — update activity timestamp before guard drops // Mark request end — update activity timestamp before guard drops
la.store(st.elapsed().as_millis() as u64, Ordering::Relaxed); la.store(st.elapsed().as_millis() as u64, Ordering::Relaxed);
drop(req_guard); // Explicitly drop to decrement active_requests drop(req_guard); // Explicitly drop to decrement active_requests
@@ -304,8 +317,13 @@ impl HttpProxyService {
}); });
// Auto-detect h1 vs h2 based on ALPN / connection preface. // Auto-detect h1 vs h2 based on ALPN / connection preface.
// serve_connection_with_upgrades supports h1 Upgrade (WebSocket) and h2 CONNECT. // serve_connection_with_upgrades supports h1 Upgrade (WebSocket) and h2 Extended CONNECT (RFC 8441).
let builder = hyper_util::server::conn::auto::Builder::new(hyper_util::rt::TokioExecutor::new()); let mut builder = hyper_util::server::conn::auto::Builder::new(hyper_util::rt::TokioExecutor::new());
// Configure H2 server settings: Extended CONNECT for WebSocket + flow control tuning
builder.http2()
.enable_connect_protocol()
.initial_stream_window_size(2 * 1024 * 1024) // 2MB per stream (vs default 64KB)
.initial_connection_window_size(8 * 1024 * 1024); // 8MB per client connection
let conn = builder.serve_connection_with_upgrades(io, service); let conn = builder.serve_connection_with_upgrades(io, service);
// Pin on the heap — auto::UpgradeableConnection is !Unpin // Pin on the heap — auto::UpgradeableConnection is !Unpin
let mut conn = Box::pin(conn); let mut conn = Box::pin(conn);
@@ -365,6 +383,7 @@ impl HttpProxyService {
peer_addr: std::net::SocketAddr, peer_addr: std::net::SocketAddr,
port: u16, port: u16,
cancel: CancellationToken, cancel: CancellationToken,
conn_activity: ConnActivity,
) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> { ) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
let host = req.headers() let host = req.headers()
.get("host") .get("host")
@@ -380,11 +399,19 @@ impl HttpProxyService {
let path = req.uri().path().to_string(); let path = req.uri().path().to_string();
let method = req.method().clone(); let method = req.method().clone();
// Extract headers for matching // Extract headers for matching — only allocate the HashMap if any route
let headers: HashMap<String, String> = req.headers() // on this port actually uses header matching. Most deployments don't,
.iter() // so this saves ~20-30 String allocations per request.
.map(|(k, v)| (k.to_string(), v.to_str().unwrap_or("").to_string())) let current_rm = self.route_manager.load();
.collect(); let needs_headers = current_rm.any_route_has_headers(port);
let headers: Option<HashMap<String, String>> = if needs_headers {
Some(req.headers()
.iter()
.map(|(k, v)| (k.to_string(), v.to_str().unwrap_or("").to_string()))
.collect())
} else {
None
};
debug!("HTTP {} {} (host: {:?}) from {}", method, path, host, peer_addr); debug!("HTTP {} {} (host: {:?}) from {}", method, path, host, peer_addr);
@@ -395,19 +422,19 @@ impl HttpProxyService {
} }
} }
// Match route // Match route (current_rm already loaded above for headers check)
let ip_string = peer_addr.ip().to_string();
let ctx = rustproxy_routing::MatchContext { let ctx = rustproxy_routing::MatchContext {
port, port,
domain: host.as_deref(), domain: host.as_deref(),
path: Some(&path), path: Some(&path),
client_ip: Some(&peer_addr.ip().to_string()), client_ip: Some(&ip_string),
tls_version: None, tls_version: None,
headers: Some(&headers), headers: headers.as_ref(),
is_tls: false, is_tls: false,
protocol: Some("http"), protocol: Some("http"),
}; };
let current_rm = self.route_manager.load();
let route_match = match current_rm.find_route(&ctx) { let route_match = match current_rm.find_route(&ctx) {
Some(rm) => rm, Some(rm) => rm,
None => { None => {
@@ -417,7 +444,7 @@ impl HttpProxyService {
}; };
let route_id = route_match.route.id.as_deref(); let route_id = route_match.route.id.as_deref();
let ip_str = peer_addr.ip().to_string(); let ip_str = ip_string; // reuse from above (avoid redundant to_string())
self.metrics.record_http_request(); self.metrics.record_http_request();
// Apply request filters (IP check, rate limiting, auth) // Apply request filters (IP check, rate limiting, auth)
@@ -482,16 +509,23 @@ impl HttpProxyService {
let domain_str = host.as_deref().unwrap_or("-"); let domain_str = host.as_deref().unwrap_or("-");
self.upstream_selector.connection_started(&upstream_key); self.upstream_selector.connection_started(&upstream_key);
// Check for WebSocket upgrade // Check for WebSocket upgrade: H1 (Upgrade header) or H2 Extended CONNECT (RFC 8441)
let is_websocket = req.headers() let is_h1_websocket = req.headers()
.get("upgrade") .get("upgrade")
.and_then(|v| v.to_str().ok()) .and_then(|v| v.to_str().ok())
.map(|v| v.eq_ignore_ascii_case("websocket")) .map(|v| v.eq_ignore_ascii_case("websocket"))
.unwrap_or(false); .unwrap_or(false);
if is_websocket { let is_h2_websocket = req.method() == hyper::Method::CONNECT
&& req.extensions()
.get::<hyper::ext::Protocol>()
.map(|p| p.as_str().eq_ignore_ascii_case("websocket"))
.unwrap_or(false);
if is_h1_websocket || is_h2_websocket {
let result = self.handle_websocket_upgrade( let result = self.handle_websocket_upgrade(
req, peer_addr, &upstream, route_match.route, route_id, &upstream_key, cancel, &ip_str, req, peer_addr, &upstream, route_match.route, route_id, &upstream_key, cancel, &ip_str, is_h2_websocket,
if is_h2_websocket { Some(conn_activity.clone()) } else { None },
).await; ).await;
// Note: for WebSocket, connection_ended is called inside // Note: for WebSocket, connection_ended is called inside
// the spawned tunnel task when the connection closes. // the spawned tunnel task when the connection closes.
@@ -632,7 +666,7 @@ impl HttpProxyService {
self.metrics.set_backend_protocol(&upstream_key, "h2"); self.metrics.set_backend_protocol(&upstream_key, "h2");
let result = self.forward_h2_pooled( let result = self.forward_h2_pooled(
sender, parts, body, upstream_headers, &upstream_path, sender, parts, body, upstream_headers, &upstream_path,
route_match.route, route_id, &ip_str, &pool_key, domain_str, route_match.route, route_id, &ip_str, &pool_key, domain_str, &conn_activity,
).await; ).await;
self.upstream_selector.connection_ended(&upstream_key); self.upstream_selector.connection_ended(&upstream_key);
return result; return result;
@@ -771,19 +805,19 @@ impl HttpProxyService {
self.forward_h2_with_fallback( self.forward_h2_with_fallback(
io, parts, body, upstream_headers, &upstream_path, io, parts, body, upstream_headers, &upstream_path,
&upstream, route_match.route, route_id, &ip_str, &final_pool_key, &upstream, route_match.route, route_id, &ip_str, &final_pool_key,
host.clone(), domain_str, host.clone(), domain_str, &conn_activity,
).await ).await
} else { } else {
// Explicit H2 mode: hard-fail on handshake error (preserved behavior) // Explicit H2 mode: hard-fail on handshake error (preserved behavior)
self.forward_h2( self.forward_h2(
io, parts, body, upstream_headers, &upstream_path, io, parts, body, upstream_headers, &upstream_path,
&upstream, route_match.route, route_id, &ip_str, &final_pool_key, domain_str, &upstream, route_match.route, route_id, &ip_str, &final_pool_key, domain_str, &conn_activity,
).await ).await
} }
} else { } else {
self.forward_h1( self.forward_h1(
io, parts, body, upstream_headers, &upstream_path, io, parts, body, upstream_headers, &upstream_path,
&upstream, route_match.route, route_id, &ip_str, &final_pool_key, domain_str, &upstream, route_match.route, route_id, &ip_str, &final_pool_key, domain_str, &conn_activity,
).await ).await
}; };
self.upstream_selector.connection_ended(&upstream_key); self.upstream_selector.connection_ended(&upstream_key);
@@ -806,6 +840,7 @@ impl HttpProxyService {
source_ip: &str, source_ip: &str,
pool_key: &crate::connection_pool::PoolKey, pool_key: &crate::connection_pool::PoolKey,
domain: &str, domain: &str,
conn_activity: &ConnActivity,
) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> { ) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
let backend_key = format!("{}:{}", pool_key.host, pool_key.port); let backend_key = format!("{}:{}", pool_key.host, pool_key.port);
@@ -814,7 +849,7 @@ impl HttpProxyService {
self.metrics.backend_pool_hit(&backend_key); self.metrics.backend_pool_hit(&backend_key);
return self.forward_h1_with_sender( return self.forward_h1_with_sender(
pooled_sender, parts, body, upstream_headers, upstream_path, pooled_sender, parts, body, upstream_headers, upstream_path,
route, route_id, source_ip, pool_key, domain, route, route_id, source_ip, pool_key, domain, conn_activity,
).await; ).await;
} }
@@ -837,7 +872,7 @@ impl HttpProxyService {
} }
}); });
self.forward_h1_with_sender(sender, parts, body, upstream_headers, upstream_path, route, route_id, source_ip, pool_key, domain).await self.forward_h1_with_sender(sender, parts, body, upstream_headers, upstream_path, route, route_id, source_ip, pool_key, domain, conn_activity).await
} }
/// Common H1 forwarding logic used by both fresh and pooled paths. /// Common H1 forwarding logic used by both fresh and pooled paths.
@@ -853,6 +888,7 @@ impl HttpProxyService {
source_ip: &str, source_ip: &str,
pool_key: &crate::connection_pool::PoolKey, pool_key: &crate::connection_pool::PoolKey,
domain: &str, domain: &str,
conn_activity: &ConnActivity,
) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> { ) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
// Always use HTTP/1.1 for h1 backend connections (h2 incoming requests have version HTTP/2.0) // Always use HTTP/1.1 for h1 backend connections (h2 incoming requests have version HTTP/2.0)
let mut upstream_req = Request::builder() let mut upstream_req = Request::builder()
@@ -871,7 +907,7 @@ impl HttpProxyService {
route_id.map(|s| s.to_string()), route_id.map(|s| s.to_string()),
Some(source_ip.to_string()), Some(source_ip.to_string()),
Direction::In, Direction::In,
); ).with_connection_activity(Arc::clone(&conn_activity.last_activity), conn_activity.start);
let boxed_body: BoxBody<Bytes, hyper::Error> = BoxBody::new(counting_req_body); let boxed_body: BoxBody<Bytes, hyper::Error> = BoxBody::new(counting_req_body);
let upstream_req = upstream_req.body(boxed_body).unwrap(); let upstream_req = upstream_req.body(boxed_body).unwrap();
@@ -886,10 +922,17 @@ impl HttpProxyService {
} }
}; };
// Return sender to pool (body streams lazily, sender is reusable once response head is received) // Note: we do NOT return the sender to the pool here because the response body
self.connection_pool.checkin_h1(pool_key.clone(), sender); // hasn't been fully streamed yet. Pooling a sender while its response body is still
// in-flight risks another request being dispatched on the same connection if is_ready()
// momentarily returns true between chunks. The sender is dropped after this scope,
// and the backend connection remains alive via the spawned conn driver task until
// the response body finishes streaming.
// For small/empty responses, the sender could theoretically be reused, but the safety
// of large streaming responses (e.g. 352MB Docker layers) takes priority.
drop(sender);
self.build_streaming_response(upstream_response, route, route_id, source_ip).await self.build_streaming_response(upstream_response, route, route_id, source_ip, conn_activity).await
} }
/// Forward request to backend via HTTP/2 with body streaming (fresh connection). /// Forward request to backend via HTTP/2 with body streaming (fresh connection).
@@ -907,6 +950,7 @@ impl HttpProxyService {
source_ip: &str, source_ip: &str,
pool_key: &crate::connection_pool::PoolKey, pool_key: &crate::connection_pool::PoolKey,
domain: &str, domain: &str,
conn_activity: &ConnActivity,
) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> { ) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
let backend_key = format!("{}:{}", pool_key.host, pool_key.port); let backend_key = format!("{}:{}", pool_key.host, pool_key.port);
let exec = hyper_util::rt::TokioExecutor::new(); let exec = hyper_util::rt::TokioExecutor::new();
@@ -914,9 +958,9 @@ impl HttpProxyService {
h2_builder h2_builder
.timer(hyper_util::rt::TokioTimer::new()) .timer(hyper_util::rt::TokioTimer::new())
.keep_alive_interval(std::time::Duration::from_secs(10)) .keep_alive_interval(std::time::Duration::from_secs(10))
.keep_alive_timeout(std::time::Duration::from_secs(5)) .keep_alive_timeout(std::time::Duration::from_secs(30))
.adaptive_window(true) .initial_stream_window_size(2 * 1024 * 1024)
.initial_stream_window_size(2 * 1024 * 1024); .initial_connection_window_size(16 * 1024 * 1024);
let (sender, conn): ( let (sender, conn): (
hyper::client::conn::http2::SendRequest<BoxBody<Bytes, hyper::Error>>, hyper::client::conn::http2::SendRequest<BoxBody<Bytes, hyper::Error>>,
hyper::client::conn::http2::Connection<TokioIo<BackendStream>, BoxBody<Bytes, hyper::Error>, hyper_util::rt::TokioExecutor>, hyper::client::conn::http2::Connection<TokioIo<BackendStream>, BoxBody<Bytes, hyper::Error>, hyper_util::rt::TokioExecutor>,
@@ -934,15 +978,22 @@ impl HttpProxyService {
} }
}; };
tokio::spawn(async move { // Spawn the H2 connection driver; proactively evict from pool on exit
if let Err(e) = conn.await { // so the next request gets a fresh connection instead of a dead sender.
debug!("HTTP/2 upstream connection error: {}", e); {
} let pool = Arc::clone(&self.connection_pool);
}); let key = pool_key.clone();
tokio::spawn(async move {
if let Err(e) = conn.await {
debug!("HTTP/2 upstream connection error: {}", e);
}
pool.remove_h2(&key);
});
}
// Clone sender for potential pool registration; register only after first request succeeds // Clone sender for potential pool registration; register only after first request succeeds
let sender_for_pool = sender.clone(); let sender_for_pool = sender.clone();
let result = self.forward_h2_with_sender(sender, parts, body, upstream_headers, upstream_path, route, route_id, source_ip, Some(pool_key), domain).await; let result = self.forward_h2_with_sender(sender, parts, body, upstream_headers, upstream_path, route, route_id, source_ip, Some(pool_key), domain, conn_activity).await;
if matches!(&result, Ok(ref resp) if resp.status() != StatusCode::BAD_GATEWAY) { if matches!(&result, Ok(ref resp) if resp.status() != StatusCode::BAD_GATEWAY) {
self.connection_pool.register_h2(pool_key.clone(), sender_for_pool); self.connection_pool.register_h2(pool_key.clone(), sender_for_pool);
} }
@@ -964,6 +1015,7 @@ impl HttpProxyService {
source_ip: &str, source_ip: &str,
pool_key: &crate::connection_pool::PoolKey, pool_key: &crate::connection_pool::PoolKey,
domain: &str, domain: &str,
conn_activity: &ConnActivity,
) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> { ) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
// Save retry state for bodyless requests (cheap: Method is an enum, HeaderMap clones Arc-backed Bytes) // Save retry state for bodyless requests (cheap: Method is an enum, HeaderMap clones Arc-backed Bytes)
let retry_state = if body.is_end_stream() { let retry_state = if body.is_end_stream() {
@@ -974,7 +1026,7 @@ impl HttpProxyService {
let result = self.forward_h2_with_sender( let result = self.forward_h2_with_sender(
sender, parts, body, upstream_headers, upstream_path, sender, parts, body, upstream_headers, upstream_path,
route, route_id, source_ip, Some(pool_key), domain, route, route_id, source_ip, Some(pool_key), domain, conn_activity,
).await; ).await;
// If the request failed (502) and we can retry with an empty body, do so // If the request failed (502) and we can retry with an empty body, do so
@@ -985,7 +1037,7 @@ impl HttpProxyService {
"Stale pooled H2 sender, retrying with fresh connection"); "Stale pooled H2 sender, retrying with fresh connection");
return self.retry_h2_with_fresh_connection( return self.retry_h2_with_fresh_connection(
method, headers, upstream_path, method, headers, upstream_path,
pool_key, route, route_id, source_ip, domain, pool_key, route, route_id, source_ip, domain, conn_activity,
).await; ).await;
} }
} }
@@ -1004,6 +1056,7 @@ impl HttpProxyService {
route_id: Option<&str>, route_id: Option<&str>,
source_ip: &str, source_ip: &str,
domain: &str, domain: &str,
conn_activity: &ConnActivity,
) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> { ) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
let backend_key = format!("{}:{}", pool_key.host, pool_key.port); let backend_key = format!("{}:{}", pool_key.host, pool_key.port);
@@ -1055,9 +1108,9 @@ impl HttpProxyService {
h2_builder h2_builder
.timer(hyper_util::rt::TokioTimer::new()) .timer(hyper_util::rt::TokioTimer::new())
.keep_alive_interval(std::time::Duration::from_secs(10)) .keep_alive_interval(std::time::Duration::from_secs(10))
.keep_alive_timeout(std::time::Duration::from_secs(5)) .keep_alive_timeout(std::time::Duration::from_secs(30))
.adaptive_window(true) .initial_stream_window_size(2 * 1024 * 1024)
.initial_stream_window_size(2 * 1024 * 1024); .initial_connection_window_size(16 * 1024 * 1024);
let (mut sender, conn): ( let (mut sender, conn): (
hyper::client::conn::http2::SendRequest<BoxBody<Bytes, hyper::Error>>, hyper::client::conn::http2::SendRequest<BoxBody<Bytes, hyper::Error>>,
hyper::client::conn::http2::Connection<TokioIo<BackendStream>, BoxBody<Bytes, hyper::Error>, hyper_util::rt::TokioExecutor>, hyper::client::conn::http2::Connection<TokioIo<BackendStream>, BoxBody<Bytes, hyper::Error>, hyper_util::rt::TokioExecutor>,
@@ -1077,11 +1130,17 @@ impl HttpProxyService {
} }
}; };
tokio::spawn(async move { // Spawn the H2 connection driver; proactively evict from pool on exit.
if let Err(e) = conn.await { {
debug!("H2 retry: upstream connection error: {}", e); let pool = Arc::clone(&self.connection_pool);
} let key = pool_key.clone();
}); tokio::spawn(async move {
if let Err(e) = conn.await {
debug!("H2 retry: upstream connection error: {}", e);
}
pool.remove_h2(&key);
});
}
// Build request with empty body using absolute URI for H2 pseudo-headers // Build request with empty body using absolute URI for H2 pseudo-headers
let scheme = if pool_key.use_tls { "https" } else { "http" }; let scheme = if pool_key.use_tls { "https" } else { "http" };
@@ -1108,7 +1167,7 @@ impl HttpProxyService {
Ok(resp) => { Ok(resp) => {
// Register in pool only after request succeeds // Register in pool only after request succeeds
self.connection_pool.register_h2(pool_key.clone(), sender); self.connection_pool.register_h2(pool_key.clone(), sender);
let result = self.build_streaming_response(resp, route, route_id, source_ip).await; let result = self.build_streaming_response(resp, route, route_id, source_ip, conn_activity).await;
// Close the fresh backend connection (opened above) // Close the fresh backend connection (opened above)
self.metrics.backend_connection_closed(&backend_key); self.metrics.backend_connection_closed(&backend_key);
result result
@@ -1144,15 +1203,16 @@ impl HttpProxyService {
pool_key: &crate::connection_pool::PoolKey, pool_key: &crate::connection_pool::PoolKey,
requested_host: Option<String>, requested_host: Option<String>,
domain: &str, domain: &str,
conn_activity: &ConnActivity,
) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> { ) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
let exec = hyper_util::rt::TokioExecutor::new(); let exec = hyper_util::rt::TokioExecutor::new();
let mut h2_builder = hyper::client::conn::http2::Builder::new(exec); let mut h2_builder = hyper::client::conn::http2::Builder::new(exec);
h2_builder h2_builder
.timer(hyper_util::rt::TokioTimer::new()) .timer(hyper_util::rt::TokioTimer::new())
.keep_alive_interval(std::time::Duration::from_secs(10)) .keep_alive_interval(std::time::Duration::from_secs(10))
.keep_alive_timeout(std::time::Duration::from_secs(5)) .keep_alive_timeout(std::time::Duration::from_secs(30))
.adaptive_window(true) .initial_stream_window_size(2 * 1024 * 1024)
.initial_stream_window_size(2 * 1024 * 1024); .initial_connection_window_size(16 * 1024 * 1024);
let handshake_result = tokio::time::timeout( let handshake_result = tokio::time::timeout(
self.connect_timeout, self.connect_timeout,
h2_builder.handshake(io), h2_builder.handshake(io),
@@ -1188,7 +1248,7 @@ impl HttpProxyService {
let fallback_io = TokioIo::new(fallback_backend); let fallback_io = TokioIo::new(fallback_backend);
let result = self.forward_h1( let result = self.forward_h1(
fallback_io, parts, body, upstream_headers, upstream_path, fallback_io, parts, body, upstream_headers, upstream_path,
upstream, route, route_id, source_ip, &h1_pool_key, domain, upstream, route, route_id, source_ip, &h1_pool_key, domain, conn_activity,
).await; ).await;
self.metrics.backend_connection_closed(&bk); self.metrics.backend_connection_closed(&bk);
result result
@@ -1199,11 +1259,17 @@ impl HttpProxyService {
} }
} }
Ok(Ok((mut sender, conn))) => { Ok(Ok((mut sender, conn))) => {
tokio::spawn(async move { // Spawn the H2 connection driver; proactively evict from pool on exit.
if let Err(e) = conn.await { {
debug!("HTTP/2 upstream connection error: {}", e); let pool = Arc::clone(&self.connection_pool);
} let key = pool_key.clone();
}); tokio::spawn(async move {
if let Err(e) = conn.await {
debug!("HTTP/2 upstream connection error: {}", e);
}
pool.remove_h2(&key);
});
}
// Save retry state before consuming parts/body (for bodyless requests like GET) // Save retry state before consuming parts/body (for bodyless requests like GET)
// Clone BEFORE removing Host — H1 fallback needs Host header // Clone BEFORE removing Host — H1 fallback needs Host header
@@ -1236,7 +1302,7 @@ impl HttpProxyService {
route_id.map(|s| s.to_string()), route_id.map(|s| s.to_string()),
Some(source_ip.to_string()), Some(source_ip.to_string()),
Direction::In, Direction::In,
); ).with_connection_activity(Arc::clone(&conn_activity.last_activity), conn_activity.start);
let boxed_body: BoxBody<Bytes, hyper::Error> = BoxBody::new(counting_req_body); let boxed_body: BoxBody<Bytes, hyper::Error> = BoxBody::new(counting_req_body);
let upstream_req = upstream_req.body(boxed_body).unwrap(); let upstream_req = upstream_req.body(boxed_body).unwrap();
@@ -1244,7 +1310,7 @@ impl HttpProxyService {
Ok(upstream_response) => { Ok(upstream_response) => {
// H2 works! Register sender in pool for multiplexed reuse // H2 works! Register sender in pool for multiplexed reuse
self.connection_pool.register_h2(pool_key.clone(), sender); self.connection_pool.register_h2(pool_key.clone(), sender);
self.build_streaming_response(upstream_response, route, route_id, source_ip).await self.build_streaming_response(upstream_response, route, route_id, source_ip, conn_activity).await
} }
Err(e) => { Err(e) => {
// H2 request failed — backend advertises h2 via ALPN but doesn't // H2 request failed — backend advertises h2 via ALPN but doesn't
@@ -1277,7 +1343,7 @@ impl HttpProxyService {
let fallback_io = TokioIo::new(fallback_backend); let fallback_io = TokioIo::new(fallback_backend);
let result = self.forward_h1_empty_body( let result = self.forward_h1_empty_body(
fallback_io, method, headers, upstream_path, fallback_io, method, headers, upstream_path,
route, route_id, source_ip, &h1_pool_key, domain, route, route_id, source_ip, &h1_pool_key, domain, conn_activity,
).await; ).await;
// Close the reconnected backend connection (opened in reconnect_backend) // Close the reconnected backend connection (opened in reconnect_backend)
self.metrics.backend_connection_closed(&bk); self.metrics.backend_connection_closed(&bk);
@@ -1326,7 +1392,7 @@ impl HttpProxyService {
let fallback_io = TokioIo::new(fallback_backend); let fallback_io = TokioIo::new(fallback_backend);
let result = self.forward_h1( let result = self.forward_h1(
fallback_io, parts, body, upstream_headers, upstream_path, fallback_io, parts, body, upstream_headers, upstream_path,
upstream, route, route_id, source_ip, &h1_pool_key, domain, upstream, route, route_id, source_ip, &h1_pool_key, domain, conn_activity,
).await; ).await;
// Close the reconnected backend connection (opened in reconnect_backend) // Close the reconnected backend connection (opened in reconnect_backend)
self.metrics.backend_connection_closed(&bk); self.metrics.backend_connection_closed(&bk);
@@ -1353,6 +1419,7 @@ impl HttpProxyService {
source_ip: &str, source_ip: &str,
pool_key: &crate::connection_pool::PoolKey, pool_key: &crate::connection_pool::PoolKey,
domain: &str, domain: &str,
conn_activity: &ConnActivity,
) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> { ) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
let backend_key = format!("{}:{}", pool_key.host, pool_key.port); let backend_key = format!("{}:{}", pool_key.host, pool_key.port);
let (mut sender, conn): ( let (mut sender, conn): (
@@ -1396,10 +1463,10 @@ impl HttpProxyService {
} }
}; };
// Return sender to pool for keep-alive reuse // Don't pool the sender while response body is still streaming (same safety as forward_h1_with_sender)
self.connection_pool.checkin_h1(pool_key.clone(), sender); drop(sender);
self.build_streaming_response(upstream_response, route, route_id, source_ip).await self.build_streaming_response(upstream_response, route, route_id, source_ip, conn_activity).await
} }
/// Reconnect to a backend (used for H2→H1 fallback). /// Reconnect to a backend (used for H2→H1 fallback).
@@ -1470,6 +1537,7 @@ impl HttpProxyService {
source_ip: &str, source_ip: &str,
pool_key: Option<&crate::connection_pool::PoolKey>, pool_key: Option<&crate::connection_pool::PoolKey>,
domain: &str, domain: &str,
conn_activity: &ConnActivity,
) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> { ) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
// Build absolute URI for H2 pseudo-headers (:scheme, :authority) // Build absolute URI for H2 pseudo-headers (:scheme, :authority)
// Use the requested domain as authority (not backend address) so :authority matches Host header // Use the requested domain as authority (not backend address) so :authority matches Host header
@@ -1498,7 +1566,7 @@ impl HttpProxyService {
route_id.map(|s| s.to_string()), route_id.map(|s| s.to_string()),
Some(source_ip.to_string()), Some(source_ip.to_string()),
Direction::In, Direction::In,
); ).with_connection_activity(Arc::clone(&conn_activity.last_activity), conn_activity.start);
let boxed_body: BoxBody<Bytes, hyper::Error> = BoxBody::new(counting_req_body); let boxed_body: BoxBody<Bytes, hyper::Error> = BoxBody::new(counting_req_body);
let upstream_req = upstream_req.body(boxed_body).unwrap(); let upstream_req = upstream_req.body(boxed_body).unwrap();
@@ -1519,7 +1587,7 @@ impl HttpProxyService {
} }
}; };
self.build_streaming_response(upstream_response, route, route_id, source_ip).await self.build_streaming_response(upstream_response, route, route_id, source_ip, conn_activity).await
} }
/// Build the client-facing response from an upstream response, streaming the body. /// Build the client-facing response from an upstream response, streaming the body.
@@ -1532,6 +1600,7 @@ impl HttpProxyService {
route: &rustproxy_config::RouteConfig, route: &rustproxy_config::RouteConfig,
route_id: Option<&str>, route_id: Option<&str>,
source_ip: &str, source_ip: &str,
conn_activity: &ConnActivity,
) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> { ) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
let (resp_parts, resp_body) = upstream_response.into_parts(); let (resp_parts, resp_body) = upstream_response.into_parts();
@@ -1540,6 +1609,19 @@ impl HttpProxyService {
if let Some(headers) = response.headers_mut() { if let Some(headers) = response.headers_mut() {
*headers = resp_parts.headers; *headers = resp_parts.headers;
// Strip hop-by-hop headers from the backend response.
// RFC 9113 §8.2.2 forbids connection-specific headers in HTTP/2 responses;
// forwarding them from an H1 backend can cause H2 stream resets.
// Mirrors the request-path stripping at the forward methods above.
headers.remove("connection");
headers.remove("keep-alive");
headers.remove("proxy-connection");
headers.remove("transfer-encoding");
headers.remove("te");
headers.remove("trailer");
// Note: "upgrade" is intentionally kept — needed for WebSocket 101 responses.
ResponseFilter::apply_headers(route, headers, None); ResponseFilter::apply_headers(route, headers, None);
} }
@@ -1552,14 +1634,23 @@ impl HttpProxyService {
route_id.map(|s| s.to_string()), route_id.map(|s| s.to_string()),
Some(source_ip.to_string()), Some(source_ip.to_string()),
Direction::Out, Direction::Out,
); ).with_connection_activity(Arc::clone(&conn_activity.last_activity), conn_activity.start);
// Keep active_requests > 0 while the response body streams, so the idle
// watchdog doesn't kill the connection mid-transfer (e.g. during git fetch).
// CountingBody increments on creation and decrements on Drop.
let counting_body = if let Some(ref ar) = conn_activity.active_requests {
counting_body.with_active_requests(Arc::clone(ar))
} else {
counting_body
};
let body: BoxBody<Bytes, hyper::Error> = BoxBody::new(counting_body); let body: BoxBody<Bytes, hyper::Error> = BoxBody::new(counting_body);
Ok(response.body(body).unwrap()) Ok(response.body(body).unwrap())
} }
/// Handle a WebSocket upgrade request. /// Handle a WebSocket upgrade request (H1 Upgrade or H2 Extended CONNECT per RFC 8441).
async fn handle_websocket_upgrade( async fn handle_websocket_upgrade(
&self, &self,
req: Request<Incoming>, req: Request<Incoming>,
@@ -1570,6 +1661,8 @@ impl HttpProxyService {
upstream_key: &str, upstream_key: &str,
cancel: CancellationToken, cancel: CancellationToken,
source_ip: &str, source_ip: &str,
is_h2: bool,
conn_activity: Option<ConnActivity>,
) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> { ) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::io::{AsyncReadExt, AsyncWriteExt};
@@ -1655,9 +1748,11 @@ impl HttpProxyService {
let (parts, _body) = req.into_parts(); let (parts, _body) = req.into_parts();
// H2 Extended CONNECT uses method=CONNECT, but the H1.1 backend expects GET
let backend_method = if is_h2 { "GET" } else { parts.method.as_str() };
let mut raw_request = format!( let mut raw_request = format!(
"{} {} HTTP/1.1\r\n", "{} {} HTTP/1.1\r\n",
parts.method, upstream_path backend_method, upstream_path
); );
// Copy all original headers (preserving the client's Host header). // Copy all original headers (preserving the client's Host header).
@@ -1685,6 +1780,23 @@ impl HttpProxyService {
} }
} }
// H2 Extended CONNECT doesn't carry H1 WebSocket handshake headers;
// inject them so the H1.1 backend can complete the upgrade.
if is_h2 {
if !parts.headers.contains_key("upgrade") {
raw_request.push_str("upgrade: websocket\r\n");
}
if !parts.headers.contains_key("connection") {
raw_request.push_str("connection: Upgrade\r\n");
}
if !parts.headers.contains_key("sec-websocket-version") {
raw_request.push_str("sec-websocket-version: 13\r\n");
}
if !parts.headers.contains_key("sec-websocket-key") {
raw_request.push_str("sec-websocket-key: dGhlIHNhbXBsZSBub25jZQ==\r\n");
}
}
// Add standard reverse-proxy headers (X-Forwarded-*) // Add standard reverse-proxy headers (X-Forwarded-*)
{ {
let original_host = parts.headers.get("host") let original_host = parts.headers.get("host")
@@ -1787,8 +1899,12 @@ impl HttpProxyService {
)); ));
} }
let mut client_resp = Response::builder() // H1: 101 Switching Protocols; H2: 200 OK (RFC 8441 — hyper requires 2xx for Extended CONNECT upgrade)
.status(StatusCode::SWITCHING_PROTOCOLS); let mut client_resp = if is_h2 {
Response::builder().status(StatusCode::OK)
} else {
Response::builder().status(StatusCode::SWITCHING_PROTOCOLS)
};
if let Some(resp_headers) = client_resp.headers_mut() { if let Some(resp_headers) = client_resp.headers_mut() {
for line in response_str.lines().skip(1) { for line in response_str.lines().skip(1) {
@@ -1799,6 +1915,17 @@ impl HttpProxyService {
if let Some((name, value)) = line.split_once(':') { if let Some((name, value)) = line.split_once(':') {
let name = name.trim(); let name = name.trim();
let value = value.trim(); let value = value.trim();
// Skip hop-by-hop headers for H2 (forbidden by RFC 9113 §8.2.2)
if is_h2 {
let name_lower = name.to_lowercase();
if name_lower == "upgrade" || name_lower == "connection"
|| name_lower == "sec-websocket-accept"
|| name_lower == "transfer-encoding"
|| name_lower == "keep-alive"
{
continue;
}
}
if let Ok(header_name) = hyper::header::HeaderName::from_bytes(name.as_bytes()) { if let Ok(header_name) = hyper::header::HeaderName::from_bytes(name.as_bytes()) {
if let Ok(header_value) = hyper::header::HeaderValue::from_str(value) { if let Ok(header_value) = hyper::header::HeaderValue::from_str(value) {
resp_headers.insert(header_name, header_value); resp_headers.insert(header_name, header_value);
@@ -1839,48 +1966,89 @@ impl HttpProxyService {
let last_activity = Arc::new(AtomicU64::new(0)); let last_activity = Arc::new(AtomicU64::new(0));
let start = std::time::Instant::now(); let start = std::time::Instant::now();
// Per-connection cancellation token: the watchdog cancels this instead of
// aborting tasks, so the copy loops can shut down gracefully (TLS close_notify).
let ws_cancel = CancellationToken::new();
// For H2 WebSocket: also update the connection-level activity tracker
// to prevent the idle watchdog from killing the H2 connection
let conn_act_c2u = conn_activity.as_ref().map(|ca| (Arc::clone(&ca.last_activity), ca.start));
let conn_act_u2c = conn_activity.as_ref().map(|ca| (Arc::clone(&ca.last_activity), ca.start));
let la1 = Arc::clone(&last_activity); let la1 = Arc::clone(&last_activity);
let metrics_c2u = Arc::clone(&metrics);
let route_c2u = route_id_owned.clone();
let ip_c2u = source_ip_owned.clone();
let wsc1 = ws_cancel.clone();
let c2u = tokio::spawn(async move { let c2u = tokio::spawn(async move {
let mut buf = vec![0u8; 65536]; let mut buf = vec![0u8; 65536];
let mut total = 0u64; let mut total = 0u64;
loop { loop {
let n = match cr.read(&mut buf).await { let n = tokio::select! {
Ok(0) | Err(_) => break, result = cr.read(&mut buf) => match result {
Ok(n) => n, Ok(0) | Err(_) => break,
Ok(n) => n,
},
_ = wsc1.cancelled() => break,
}; };
if uw.write_all(&buf[..n]).await.is_err() { if uw.write_all(&buf[..n]).await.is_err() {
break; break;
} }
total += n as u64; total += n as u64;
metrics_c2u.record_bytes(n as u64, 0, route_c2u.as_deref(), Some(&ip_c2u));
la1.store(start.elapsed().as_millis() as u64, Ordering::Relaxed); la1.store(start.elapsed().as_millis() as u64, Ordering::Relaxed);
if let Some((ref ca, ca_start)) = conn_act_c2u {
ca.store(ca_start.elapsed().as_millis() as u64, Ordering::Relaxed);
}
} }
let _ = uw.shutdown().await; // Graceful shutdown with timeout (sends TLS close_notify / TCP FIN)
let _ = tokio::time::timeout(
std::time::Duration::from_secs(2),
uw.shutdown(),
).await;
total total
}); });
let la2 = Arc::clone(&last_activity); let la2 = Arc::clone(&last_activity);
let metrics_u2c = Arc::clone(&metrics);
let route_u2c = route_id_owned.clone();
let ip_u2c = source_ip_owned.clone();
let wsc2 = ws_cancel.clone();
let u2c = tokio::spawn(async move { let u2c = tokio::spawn(async move {
let mut buf = vec![0u8; 65536]; let mut buf = vec![0u8; 65536];
let mut total = 0u64; let mut total = 0u64;
loop { loop {
let n = match ur.read(&mut buf).await { let n = tokio::select! {
Ok(0) | Err(_) => break, result = ur.read(&mut buf) => match result {
Ok(n) => n, Ok(0) | Err(_) => break,
Ok(n) => n,
},
_ = wsc2.cancelled() => break,
}; };
if cw.write_all(&buf[..n]).await.is_err() { if cw.write_all(&buf[..n]).await.is_err() {
break; break;
} }
total += n as u64; total += n as u64;
metrics_u2c.record_bytes(0, n as u64, route_u2c.as_deref(), Some(&ip_u2c));
la2.store(start.elapsed().as_millis() as u64, Ordering::Relaxed); la2.store(start.elapsed().as_millis() as u64, Ordering::Relaxed);
if let Some((ref ca, ca_start)) = conn_act_u2c {
ca.store(ca_start.elapsed().as_millis() as u64, Ordering::Relaxed);
}
} }
let _ = cw.shutdown().await; // Graceful shutdown with timeout (sends TLS close_notify / TCP FIN)
let _ = tokio::time::timeout(
std::time::Duration::from_secs(2),
cw.shutdown(),
).await;
total total
}); });
// Watchdog: monitors inactivity, max lifetime, and cancellation // Watchdog: monitors inactivity, max lifetime, and cancellation.
// First cancels the per-connection token for graceful shutdown (close_notify/FIN),
// then falls back to abort if the tasks are stuck (e.g. on a blocked write_all).
let la_watch = Arc::clone(&last_activity); let la_watch = Arc::clone(&last_activity);
let c2u_handle = c2u.abort_handle(); let c2u_abort = c2u.abort_handle();
let u2c_handle = u2c.abort_handle(); let u2c_abort = u2c.abort_handle();
let inactivity_timeout = ws_inactivity_timeout; let inactivity_timeout = ws_inactivity_timeout;
let max_lifetime = ws_max_lifetime; let max_lifetime = ws_max_lifetime;
@@ -1892,8 +2060,6 @@ impl HttpProxyService {
_ = tokio::time::sleep(check_interval) => {} _ = tokio::time::sleep(check_interval) => {}
_ = cancel.cancelled() => { _ = cancel.cancelled() => {
debug!("WebSocket tunnel cancelled by shutdown"); debug!("WebSocket tunnel cancelled by shutdown");
c2u_handle.abort();
u2c_handle.abort();
break; break;
} }
} }
@@ -1901,8 +2067,6 @@ impl HttpProxyService {
// Check max lifetime // Check max lifetime
if start.elapsed() >= max_lifetime { if start.elapsed() >= max_lifetime {
debug!("WebSocket tunnel exceeded max lifetime, closing"); debug!("WebSocket tunnel exceeded max lifetime, closing");
c2u_handle.abort();
u2c_handle.abort();
break; break;
} }
@@ -1912,13 +2076,18 @@ impl HttpProxyService {
let elapsed_since_activity = start.elapsed().as_millis() as u64 - current; let elapsed_since_activity = start.elapsed().as_millis() as u64 - current;
if elapsed_since_activity >= inactivity_timeout.as_millis() as u64 { if elapsed_since_activity >= inactivity_timeout.as_millis() as u64 {
debug!("WebSocket tunnel inactive for {}ms, closing", elapsed_since_activity); debug!("WebSocket tunnel inactive for {}ms, closing", elapsed_since_activity);
c2u_handle.abort();
u2c_handle.abort();
break; break;
} }
} }
last_seen = current; last_seen = current;
} }
// Phase 1: Signal copy loops to exit gracefully (allows close_notify/FIN)
ws_cancel.cancel();
// Phase 2: Wait for graceful shutdown (2s shutdown timeout + 2s margin)
tokio::time::sleep(std::time::Duration::from_secs(4)).await;
// Phase 3: Force-abort if still stuck (e.g. blocked on write_all)
c2u_abort.abort();
u2c_abort.abort();
}); });
let bytes_in = c2u.await.unwrap_or(0); let bytes_in = c2u.await.unwrap_or(0);
@@ -1928,9 +2097,7 @@ impl HttpProxyService {
debug!("WebSocket tunnel closed: {} bytes in, {} bytes out", bytes_in, bytes_out); debug!("WebSocket tunnel closed: {} bytes in, {} bytes out", bytes_in, bytes_out);
upstream_selector.connection_ended(&upstream_key_owned); upstream_selector.connection_ended(&upstream_key_owned);
if let Some(ref rid) = route_id_owned { // Bytes already reported per-chunk in the copy loops above
metrics.record_bytes(bytes_in, bytes_out, Some(rid.as_str()), Some(&source_ip_owned));
}
}); });
let body: BoxBody<Bytes, hyper::Error> = BoxBody::new( let body: BoxBody<Bytes, hyper::Error> = BoxBody::new(

View File

@@ -0,0 +1,90 @@
//! Wrapper that ensures TLS close_notify is sent when the stream is dropped.
//!
//! When hyper drops an HTTP connection (backend error, timeout, normal H2 close),
//! the underlying TLS stream is dropped WITHOUT `shutdown()`. tokio-rustls cannot
//! send `close_notify` in Drop (requires async). This wrapper tracks whether
//! `poll_shutdown` was called and, if not, spawns a background task to send it.
use std::io;
use std::pin::Pin;
use std::task::{Context, Poll};
use tokio::io::{AsyncRead, AsyncWrite, ReadBuf};
/// Wraps an AsyncRead+AsyncWrite stream and ensures `shutdown()` is called when
/// dropped, even if the caller (e.g. hyper) doesn't explicitly shut down.
///
/// This guarantees TLS `close_notify` is sent for TLS-wrapped streams, preventing
/// "GnuTLS recv error (-110): The TLS connection was non-properly terminated" errors.
pub struct ShutdownOnDrop<S: AsyncRead + AsyncWrite + Unpin + Send + 'static> {
inner: Option<S>,
shutdown_called: bool,
}
impl<S: AsyncRead + AsyncWrite + Unpin + Send + 'static> ShutdownOnDrop<S> {
/// Create a new wrapper around the given stream.
pub fn new(stream: S) -> Self {
Self {
inner: Some(stream),
shutdown_called: false,
}
}
}
impl<S: AsyncRead + AsyncWrite + Unpin + Send + 'static> AsyncRead for ShutdownOnDrop<S> {
fn poll_read(
self: Pin<&mut Self>,
cx: &mut Context<'_>,
buf: &mut ReadBuf<'_>,
) -> Poll<io::Result<()>> {
Pin::new(self.get_mut().inner.as_mut().unwrap()).poll_read(cx, buf)
}
}
impl<S: AsyncRead + AsyncWrite + Unpin + Send + 'static> AsyncWrite for ShutdownOnDrop<S> {
fn poll_write(
self: Pin<&mut Self>,
cx: &mut Context<'_>,
buf: &[u8],
) -> Poll<io::Result<usize>> {
Pin::new(self.get_mut().inner.as_mut().unwrap()).poll_write(cx, buf)
}
fn poll_flush(
self: Pin<&mut Self>,
cx: &mut Context<'_>,
) -> Poll<io::Result<()>> {
Pin::new(self.get_mut().inner.as_mut().unwrap()).poll_flush(cx)
}
fn poll_shutdown(
self: Pin<&mut Self>,
cx: &mut Context<'_>,
) -> Poll<io::Result<()>> {
let this = self.get_mut();
let result = Pin::new(this.inner.as_mut().unwrap()).poll_shutdown(cx);
if result.is_ready() {
this.shutdown_called = true;
}
result
}
}
impl<S: AsyncRead + AsyncWrite + Unpin + Send + 'static> Drop for ShutdownOnDrop<S> {
fn drop(&mut self) {
// If shutdown was already called (hyper closed properly), nothing to do.
// If not (hyper dropped without shutdown — e.g. H2 close, error, timeout),
// spawn a background task to send close_notify / TCP FIN.
if !self.shutdown_called {
if let Some(mut stream) = self.inner.take() {
tokio::spawn(async move {
let _ = tokio::time::timeout(
std::time::Duration::from_secs(2),
tokio::io::AsyncWriteExt::shutdown(&mut stream),
).await;
// stream is dropped here — all resources freed
});
}
}
}
}

View File

@@ -266,44 +266,67 @@ impl MetricsCollector {
self.global_pending_tp_in.fetch_add(bytes_in, Ordering::Relaxed); self.global_pending_tp_in.fetch_add(bytes_in, Ordering::Relaxed);
self.global_pending_tp_out.fetch_add(bytes_out, Ordering::Relaxed); self.global_pending_tp_out.fetch_add(bytes_out, Ordering::Relaxed);
// Per-route tracking: use get() first (zero-alloc fast path for existing entries),
// fall back to entry() with to_string() only on the rare first-chunk miss.
if let Some(route_id) = route_id { if let Some(route_id) = route_id {
self.route_bytes_in if let Some(counter) = self.route_bytes_in.get(route_id) {
.entry(route_id.to_string()) counter.fetch_add(bytes_in, Ordering::Relaxed);
.or_insert_with(|| AtomicU64::new(0)) } else {
.fetch_add(bytes_in, Ordering::Relaxed); self.route_bytes_in.entry(route_id.to_string())
self.route_bytes_out .or_insert_with(|| AtomicU64::new(0))
.entry(route_id.to_string()) .fetch_add(bytes_in, Ordering::Relaxed);
.or_insert_with(|| AtomicU64::new(0)) }
.fetch_add(bytes_out, Ordering::Relaxed); if let Some(counter) = self.route_bytes_out.get(route_id) {
counter.fetch_add(bytes_out, Ordering::Relaxed);
} else {
self.route_bytes_out.entry(route_id.to_string())
.or_insert_with(|| AtomicU64::new(0))
.fetch_add(bytes_out, Ordering::Relaxed);
}
// Accumulate into per-route pending throughput counters (lock-free) // Accumulate into per-route pending throughput counters (lock-free)
let entry = self.route_pending_tp if let Some(entry) = self.route_pending_tp.get(route_id) {
.entry(route_id.to_string()) entry.0.fetch_add(bytes_in, Ordering::Relaxed);
.or_insert_with(|| (AtomicU64::new(0), AtomicU64::new(0))); entry.1.fetch_add(bytes_out, Ordering::Relaxed);
entry.0.fetch_add(bytes_in, Ordering::Relaxed); } else {
entry.1.fetch_add(bytes_out, Ordering::Relaxed); let entry = self.route_pending_tp.entry(route_id.to_string())
.or_insert_with(|| (AtomicU64::new(0), AtomicU64::new(0)));
entry.0.fetch_add(bytes_in, Ordering::Relaxed);
entry.1.fetch_add(bytes_out, Ordering::Relaxed);
}
} }
// Per-IP tracking: same get()-first pattern to avoid String allocation on hot path.
if let Some(ip) = source_ip { if let Some(ip) = source_ip {
// Only record per-IP stats if the IP still has active connections. // Only record per-IP stats if the IP still has active connections.
// This prevents orphaned entries when record_bytes races with // This prevents orphaned entries when record_bytes races with
// connection_closed (which evicts all per-IP data on last close). // connection_closed (which evicts all per-IP data on last close).
if self.ip_connections.contains_key(ip) { if self.ip_connections.contains_key(ip) {
self.ip_bytes_in if let Some(counter) = self.ip_bytes_in.get(ip) {
.entry(ip.to_string()) counter.fetch_add(bytes_in, Ordering::Relaxed);
.or_insert_with(|| AtomicU64::new(0)) } else {
.fetch_add(bytes_in, Ordering::Relaxed); self.ip_bytes_in.entry(ip.to_string())
self.ip_bytes_out .or_insert_with(|| AtomicU64::new(0))
.entry(ip.to_string()) .fetch_add(bytes_in, Ordering::Relaxed);
.or_insert_with(|| AtomicU64::new(0)) }
.fetch_add(bytes_out, Ordering::Relaxed); if let Some(counter) = self.ip_bytes_out.get(ip) {
counter.fetch_add(bytes_out, Ordering::Relaxed);
} else {
self.ip_bytes_out.entry(ip.to_string())
.or_insert_with(|| AtomicU64::new(0))
.fetch_add(bytes_out, Ordering::Relaxed);
}
// Accumulate into per-IP pending throughput counters (lock-free) // Accumulate into per-IP pending throughput counters (lock-free)
let entry = self.ip_pending_tp if let Some(entry) = self.ip_pending_tp.get(ip) {
.entry(ip.to_string()) entry.0.fetch_add(bytes_in, Ordering::Relaxed);
.or_insert_with(|| (AtomicU64::new(0), AtomicU64::new(0))); entry.1.fetch_add(bytes_out, Ordering::Relaxed);
entry.0.fetch_add(bytes_in, Ordering::Relaxed); } else {
entry.1.fetch_add(bytes_out, Ordering::Relaxed); let entry = self.ip_pending_tp.entry(ip.to_string())
.or_insert_with(|| (AtomicU64::new(0), AtomicU64::new(0)));
entry.0.fetch_add(bytes_in, Ordering::Relaxed);
entry.1.fetch_add(bytes_out, Ordering::Relaxed);
}
} }
} }
} }

View File

@@ -97,16 +97,25 @@ pub async fn forward_bidirectional_with_timeouts(
let last_activity = Arc::new(AtomicU64::new(0)); let last_activity = Arc::new(AtomicU64::new(0));
let start = std::time::Instant::now(); let start = std::time::Instant::now();
// Per-connection cancellation token: the watchdog cancels this instead of
// aborting tasks, so the copy loops can shut down gracefully (TCP FIN instead
// of RST, TLS close_notify if the stream is TLS-wrapped).
let conn_cancel = CancellationToken::new();
let la1 = Arc::clone(&last_activity); let la1 = Arc::clone(&last_activity);
let initial_len = initial_data.map_or(0u64, |d| d.len() as u64); let initial_len = initial_data.map_or(0u64, |d| d.len() as u64);
let metrics_c2b = metrics.clone(); let metrics_c2b = metrics.clone();
let cc1 = conn_cancel.clone();
let c2b = tokio::spawn(async move { let c2b = tokio::spawn(async move {
let mut buf = vec![0u8; 65536]; let mut buf = vec![0u8; 65536];
let mut total = initial_len; let mut total = initial_len;
loop { loop {
let n = match client_read.read(&mut buf).await { let n = tokio::select! {
Ok(0) | Err(_) => break, result = client_read.read(&mut buf) => match result {
Ok(n) => n, Ok(0) | Err(_) => break,
Ok(n) => n,
},
_ = cc1.cancelled() => break,
}; };
if backend_write.write_all(&buf[..n]).await.is_err() { if backend_write.write_all(&buf[..n]).await.is_err() {
break; break;
@@ -117,19 +126,27 @@ pub async fn forward_bidirectional_with_timeouts(
ctx.collector.record_bytes(n as u64, 0, ctx.route_id.as_deref(), ctx.source_ip.as_deref()); ctx.collector.record_bytes(n as u64, 0, ctx.route_id.as_deref(), ctx.source_ip.as_deref());
} }
} }
let _ = backend_write.shutdown().await; // Graceful shutdown with timeout (sends TCP FIN / TLS close_notify)
let _ = tokio::time::timeout(
std::time::Duration::from_secs(2),
backend_write.shutdown(),
).await;
total total
}); });
let la2 = Arc::clone(&last_activity); let la2 = Arc::clone(&last_activity);
let metrics_b2c = metrics; let metrics_b2c = metrics;
let cc2 = conn_cancel.clone();
let b2c = tokio::spawn(async move { let b2c = tokio::spawn(async move {
let mut buf = vec![0u8; 65536]; let mut buf = vec![0u8; 65536];
let mut total = 0u64; let mut total = 0u64;
loop { loop {
let n = match backend_read.read(&mut buf).await { let n = tokio::select! {
Ok(0) | Err(_) => break, result = backend_read.read(&mut buf) => match result {
Ok(n) => n, Ok(0) | Err(_) => break,
Ok(n) => n,
},
_ = cc2.cancelled() => break,
}; };
if client_write.write_all(&buf[..n]).await.is_err() { if client_write.write_all(&buf[..n]).await.is_err() {
break; break;
@@ -140,14 +157,20 @@ pub async fn forward_bidirectional_with_timeouts(
ctx.collector.record_bytes(0, n as u64, ctx.route_id.as_deref(), ctx.source_ip.as_deref()); ctx.collector.record_bytes(0, n as u64, ctx.route_id.as_deref(), ctx.source_ip.as_deref());
} }
} }
let _ = client_write.shutdown().await; // Graceful shutdown with timeout (sends TCP FIN / TLS close_notify)
let _ = tokio::time::timeout(
std::time::Duration::from_secs(2),
client_write.shutdown(),
).await;
total total
}); });
// Watchdog: inactivity, max lifetime, and cancellation // Watchdog: inactivity, max lifetime, and cancellation.
// First cancels the per-connection token for graceful shutdown (FIN/close_notify),
// then falls back to abort if the tasks are stuck (e.g. on a blocked write_all).
let la_watch = Arc::clone(&last_activity); let la_watch = Arc::clone(&last_activity);
let c2b_handle = c2b.abort_handle(); let c2b_abort = c2b.abort_handle();
let b2c_handle = b2c.abort_handle(); let b2c_abort = b2c.abort_handle();
let watchdog = tokio::spawn(async move { let watchdog = tokio::spawn(async move {
let check_interval = std::time::Duration::from_secs(5); let check_interval = std::time::Duration::from_secs(5);
let mut last_seen = 0u64; let mut last_seen = 0u64;
@@ -155,16 +178,12 @@ pub async fn forward_bidirectional_with_timeouts(
tokio::select! { tokio::select! {
_ = cancel.cancelled() => { _ = cancel.cancelled() => {
debug!("Connection cancelled by shutdown"); debug!("Connection cancelled by shutdown");
c2b_handle.abort();
b2c_handle.abort();
break; break;
} }
_ = tokio::time::sleep(check_interval) => { _ = tokio::time::sleep(check_interval) => {
// Check max lifetime // Check max lifetime
if start.elapsed() >= max_lifetime { if start.elapsed() >= max_lifetime {
debug!("Connection exceeded max lifetime, closing"); debug!("Connection exceeded max lifetime, closing");
c2b_handle.abort();
b2c_handle.abort();
break; break;
} }
@@ -174,8 +193,6 @@ pub async fn forward_bidirectional_with_timeouts(
let elapsed_since_activity = start.elapsed().as_millis() as u64 - current; let elapsed_since_activity = start.elapsed().as_millis() as u64 - current;
if elapsed_since_activity >= inactivity_timeout.as_millis() as u64 { if elapsed_since_activity >= inactivity_timeout.as_millis() as u64 {
debug!("Connection inactive for {}ms, closing", elapsed_since_activity); debug!("Connection inactive for {}ms, closing", elapsed_since_activity);
c2b_handle.abort();
b2c_handle.abort();
break; break;
} }
} }
@@ -183,6 +200,13 @@ pub async fn forward_bidirectional_with_timeouts(
} }
} }
} }
// Phase 1: Signal copy loops to exit gracefully (allows FIN/close_notify)
conn_cancel.cancel();
// Phase 2: Wait for graceful shutdown (2s shutdown timeout + 2s margin)
tokio::time::sleep(std::time::Duration::from_secs(4)).await;
// Phase 3: Force-abort if still stuck (e.g. blocked on write_all)
c2b_abort.abort();
b2c_abort.abort();
}); });
let bytes_in = c2b.await.unwrap_or(0); let bytes_in = c2b.await.unwrap_or(0);

View File

@@ -465,21 +465,19 @@ impl TcpListenerManager {
Ok((stream, peer_addr)) => { Ok((stream, peer_addr)) => {
let ip = peer_addr.ip(); let ip = peer_addr.ip();
// Global connection limit — acquire semaphore permit with timeout // Global connection limit — non-blocking check.
let permit = match tokio::time::timeout( // MUST NOT block the accept loop: a blocking acquire would stall
std::time::Duration::from_secs(5), // ALL connections to this port (not just the one over limit), because
conn_semaphore.clone().acquire_owned(), // listener.accept() is not polled while we await the semaphore.
).await { let permit = match conn_semaphore.clone().try_acquire_owned() {
Ok(Ok(permit)) => permit, Ok(permit) => permit,
Ok(Err(_)) => { Err(tokio::sync::TryAcquireError::NoPermits) => {
// Semaphore closed — shouldn't happen, but be safe warn!("Global connection limit reached, dropping connection from {}", peer_addr);
debug!("Connection semaphore closed, dropping connection from {}", peer_addr);
drop(stream); drop(stream);
continue; continue;
} }
Err(_) => { Err(tokio::sync::TryAcquireError::Closed) => {
// Timeout — global limit reached warn!("Connection semaphore closed, dropping connection from {}", peer_addr);
debug!("Global connection limit reached, dropping connection from {}", peer_addr);
drop(stream); drop(stream);
continue; continue;
} }
@@ -487,7 +485,7 @@ impl TcpListenerManager {
// Check per-IP limits and rate limiting // Check per-IP limits and rate limiting
if !conn_tracker.try_accept(&ip) { if !conn_tracker.try_accept(&ip) {
debug!("Rejected connection from {} (per-IP limit or rate limit)", peer_addr); warn!("Rejected connection from {} (per-IP limit or rate limit)", peer_addr);
drop(stream); drop(stream);
drop(permit); drop(permit);
continue; continue;
@@ -519,7 +517,7 @@ impl TcpListenerManager {
stream, port, peer_addr, rm, m, tc, sa, hp, cc, cn, sr, rc, stream, port, peer_addr, rm, m, tc, sa, hp, cc, cn, sr, rc,
).await; ).await;
if let Err(e) = result { if let Err(e) = result {
debug!("Connection error from {}: {}", peer_addr, e); warn!("Connection error from {}: {}", peer_addr, e);
} }
}); });
} }
@@ -563,8 +561,9 @@ impl TcpListenerManager {
// Non-proxy connections skip the peek entirely (no latency cost). // Non-proxy connections skip the peek entirely (no latency cost).
let mut effective_peer_addr = peer_addr; let mut effective_peer_addr = peer_addr;
if !conn_config.proxy_ips.is_empty() && conn_config.proxy_ips.contains(&peer_addr.ip()) { if !conn_config.proxy_ips.is_empty() && conn_config.proxy_ips.contains(&peer_addr.ip()) {
// Trusted proxy IP — peek for PROXY protocol header // Trusted proxy IP — peek for PROXY protocol header.
let mut proxy_peek = vec![0u8; 256]; // Use stack-allocated buffers (PROXY v1 headers are max ~108 bytes).
let mut proxy_peek = [0u8; 256];
let pn = match tokio::time::timeout( let pn = match tokio::time::timeout(
std::time::Duration::from_millis(conn_config.initial_data_timeout_ms), std::time::Duration::from_millis(conn_config.initial_data_timeout_ms),
stream.peek(&mut proxy_peek), stream.peek(&mut proxy_peek),
@@ -579,9 +578,9 @@ impl TcpListenerManager {
Ok((header, consumed)) => { Ok((header, consumed)) => {
debug!("PROXY protocol: real client {} -> {}", header.source_addr, header.dest_addr); debug!("PROXY protocol: real client {} -> {}", header.source_addr, header.dest_addr);
effective_peer_addr = header.source_addr; effective_peer_addr = header.source_addr;
// Consume the proxy protocol header bytes // Consume the proxy protocol header bytes (stack buffer, max 108 bytes)
let mut discard = vec![0u8; consumed]; let mut discard = [0u8; 128];
stream.read_exact(&mut discard).await?; stream.read_exact(&mut discard[..consumed]).await?;
} }
Err(e) => { Err(e) => {
debug!("Failed to parse PROXY protocol header: {}", e); debug!("Failed to parse PROXY protocol header: {}", e);
@@ -664,7 +663,7 @@ impl TcpListenerManager {
if !rustproxy_http::request_filter::RequestFilter::check_ip_security( if !rustproxy_http::request_filter::RequestFilter::check_ip_security(
security, &peer_addr.ip(), security, &peer_addr.ip(),
) { ) {
debug!("Connection from {} blocked by route security", peer_addr); warn!("Connection from {} blocked by route security", peer_addr);
return Ok(()); return Ok(());
} }
} }
@@ -810,7 +809,7 @@ impl TcpListenerManager {
let route_match = match route_match { let route_match = match route_match {
Some(rm) => rm, Some(rm) => rm,
None => { None => {
debug!("No route matched for port {} domain {:?}", port, domain); warn!("No route matched for port {} domain {:?} from {}", port, domain, peer_addr);
if is_http { if is_http {
// Send a proper HTTP error instead of dropping the connection // Send a proper HTTP error instead of dropping the connection
use tokio::io::AsyncWriteExt; use tokio::io::AsyncWriteExt;
@@ -844,7 +843,7 @@ impl TcpListenerManager {
security, security,
&peer_addr.ip(), &peer_addr.ip(),
) { ) {
debug!("Connection from {} blocked by route security", peer_addr); warn!("Connection from {} blocked by route security", peer_addr);
return Ok(()); return Ok(());
} }
} }
@@ -987,13 +986,18 @@ impl TcpListenerManager {
Err(_) => return Err("TLS handshake timeout".into()), Err(_) => return Err("TLS handshake timeout".into()),
}; };
// Peek at decrypted data to determine if HTTP // Peek at decrypted data to determine if HTTP.
// Timeout prevents connection leak if client completes TLS
// but never sends application data (scanners, health probes, slow-loris).
let mut buf_stream = tokio::io::BufReader::new(tls_stream); let mut buf_stream = tokio::io::BufReader::new(tls_stream);
let peeked = { let peeked = {
use tokio::io::AsyncBufReadExt; use tokio::io::AsyncBufReadExt;
match buf_stream.fill_buf().await { match tokio::time::timeout(
Ok(data) => sni_parser::is_http(data), std::time::Duration::from_millis(conn_config.initial_data_timeout_ms),
Err(_) => false, buf_stream.fill_buf(),
).await {
Ok(Ok(data)) => sni_parser::is_http(data),
Ok(Err(_)) | Err(_) => false,
} }
}; };
@@ -1011,7 +1015,11 @@ impl TcpListenerManager {
"TLS Terminate + HTTP: {} -> {}:{} (domain: {:?})", "TLS Terminate + HTTP: {} -> {}:{} (domain: {:?})",
peer_addr, target_host, target_port, domain peer_addr, target_host, target_port, domain
); );
http_proxy.handle_io(buf_stream, peer_addr, port, cancel.clone()).await; // Wrap in ShutdownOnDrop to ensure TLS close_notify is sent
// even if hyper drops the connection without calling shutdown
// (e.g. H2 close, backend error, idle timeout drain).
let wrapped = rustproxy_http::shutdown_on_drop::ShutdownOnDrop::new(buf_stream);
http_proxy.handle_io(wrapped, peer_addr, port, cancel.clone()).await;
} else { } else {
debug!( debug!(
"TLS Terminate + TCP: {} -> {}:{} (domain: {:?})", "TLS Terminate + TCP: {} -> {}:{} (domain: {:?})",
@@ -1062,13 +1070,18 @@ impl TcpListenerManager {
Err(_) => return Err("TLS handshake timeout".into()), Err(_) => return Err("TLS handshake timeout".into()),
}; };
// Peek at decrypted data to detect protocol // Peek at decrypted data to detect protocol.
// Timeout prevents connection leak if client completes TLS
// but never sends application data (scanners, health probes, slow-loris).
let mut buf_stream = tokio::io::BufReader::new(tls_stream); let mut buf_stream = tokio::io::BufReader::new(tls_stream);
let is_http_data = { let is_http_data = {
use tokio::io::AsyncBufReadExt; use tokio::io::AsyncBufReadExt;
match buf_stream.fill_buf().await { match tokio::time::timeout(
Ok(data) => sni_parser::is_http(data), std::time::Duration::from_millis(conn_config.initial_data_timeout_ms),
Err(_) => false, buf_stream.fill_buf(),
).await {
Ok(Ok(data)) => sni_parser::is_http(data),
Ok(Err(_)) | Err(_) => false,
} }
}; };
@@ -1088,7 +1101,10 @@ impl TcpListenerManager {
"TLS Terminate+Reencrypt + HTTP: {} (domain: {:?})", "TLS Terminate+Reencrypt + HTTP: {} (domain: {:?})",
peer_addr, domain peer_addr, domain
); );
http_proxy.handle_io(buf_stream, peer_addr, port, cancel.clone()).await; // Wrap in ShutdownOnDrop to ensure TLS close_notify is sent
// even if hyper drops the connection without calling shutdown.
let wrapped = rustproxy_http::shutdown_on_drop::ShutdownOnDrop::new(buf_stream);
http_proxy.handle_io(wrapped, peer_addr, port, cancel.clone()).await;
} else { } else {
// Non-HTTP: TLS-to-TLS tunnel (existing behavior for raw TCP protocols) // Non-HTTP: TLS-to-TLS tunnel (existing behavior for raw TCP protocols)
debug!( debug!(
@@ -1396,15 +1412,24 @@ impl TcpListenerManager {
let last_activity = Arc::new(AtomicU64::new(0)); let last_activity = Arc::new(AtomicU64::new(0));
let start = std::time::Instant::now(); let start = std::time::Instant::now();
// Per-connection cancellation token: the watchdog cancels this instead of
// aborting tasks, so the copy loops can shut down gracefully (TLS close_notify
// for terminate/reencrypt mode, TCP FIN for passthrough mode).
let conn_cancel = CancellationToken::new();
let la1 = Arc::clone(&last_activity); let la1 = Arc::clone(&last_activity);
let metrics_c2b = metrics.clone(); let metrics_c2b = metrics.clone();
let cc1 = conn_cancel.clone();
let c2b = tokio::spawn(async move { let c2b = tokio::spawn(async move {
let mut buf = vec![0u8; 65536]; let mut buf = vec![0u8; 65536];
let mut total = 0u64; let mut total = 0u64;
loop { loop {
let n = match client_read.read(&mut buf).await { let n = tokio::select! {
Ok(0) | Err(_) => break, result = client_read.read(&mut buf) => match result {
Ok(n) => n, Ok(0) | Err(_) => break,
Ok(n) => n,
},
_ = cc1.cancelled() => break,
}; };
if backend_write.write_all(&buf[..n]).await.is_err() { if backend_write.write_all(&buf[..n]).await.is_err() {
break; break;
@@ -1418,19 +1443,27 @@ impl TcpListenerManager {
ctx.collector.record_bytes(n as u64, 0, ctx.route_id.as_deref(), ctx.source_ip.as_deref()); ctx.collector.record_bytes(n as u64, 0, ctx.route_id.as_deref(), ctx.source_ip.as_deref());
} }
} }
let _ = backend_write.shutdown().await; // Graceful shutdown with timeout (sends TLS close_notify / TCP FIN)
let _ = tokio::time::timeout(
std::time::Duration::from_secs(2),
backend_write.shutdown(),
).await;
total total
}); });
let la2 = Arc::clone(&last_activity); let la2 = Arc::clone(&last_activity);
let metrics_b2c = metrics; let metrics_b2c = metrics;
let cc2 = conn_cancel.clone();
let b2c = tokio::spawn(async move { let b2c = tokio::spawn(async move {
let mut buf = vec![0u8; 65536]; let mut buf = vec![0u8; 65536];
let mut total = 0u64; let mut total = 0u64;
loop { loop {
let n = match backend_read.read(&mut buf).await { let n = tokio::select! {
Ok(0) | Err(_) => break, result = backend_read.read(&mut buf) => match result {
Ok(n) => n, Ok(0) | Err(_) => break,
Ok(n) => n,
},
_ = cc2.cancelled() => break,
}; };
if client_write.write_all(&buf[..n]).await.is_err() { if client_write.write_all(&buf[..n]).await.is_err() {
break; break;
@@ -1444,14 +1477,20 @@ impl TcpListenerManager {
ctx.collector.record_bytes(0, n as u64, ctx.route_id.as_deref(), ctx.source_ip.as_deref()); ctx.collector.record_bytes(0, n as u64, ctx.route_id.as_deref(), ctx.source_ip.as_deref());
} }
} }
let _ = client_write.shutdown().await; // Graceful shutdown with timeout (sends TLS close_notify / TCP FIN)
let _ = tokio::time::timeout(
std::time::Duration::from_secs(2),
client_write.shutdown(),
).await;
total total
}); });
// Watchdog task: check for inactivity, max lifetime, and cancellation // Watchdog task: check for inactivity, max lifetime, and cancellation.
// First cancels the per-connection token for graceful shutdown (close_notify/FIN),
// then falls back to abort if the tasks are stuck (e.g. on a blocked write_all).
let la_watch = Arc::clone(&last_activity); let la_watch = Arc::clone(&last_activity);
let c2b_handle = c2b.abort_handle(); let c2b_abort = c2b.abort_handle();
let b2c_handle = b2c.abort_handle(); let b2c_abort = b2c.abort_handle();
let watchdog = tokio::spawn(async move { let watchdog = tokio::spawn(async move {
let check_interval = std::time::Duration::from_secs(5); let check_interval = std::time::Duration::from_secs(5);
let mut last_seen = 0u64; let mut last_seen = 0u64;
@@ -1459,16 +1498,12 @@ impl TcpListenerManager {
tokio::select! { tokio::select! {
_ = cancel.cancelled() => { _ = cancel.cancelled() => {
debug!("Split-stream connection cancelled by shutdown"); debug!("Split-stream connection cancelled by shutdown");
c2b_handle.abort();
b2c_handle.abort();
break; break;
} }
_ = tokio::time::sleep(check_interval) => { _ = tokio::time::sleep(check_interval) => {
// Check max lifetime // Check max lifetime
if start.elapsed() >= max_lifetime { if start.elapsed() >= max_lifetime {
debug!("Connection exceeded max lifetime, closing"); debug!("Connection exceeded max lifetime, closing");
c2b_handle.abort();
b2c_handle.abort();
break; break;
} }
@@ -1479,8 +1514,6 @@ impl TcpListenerManager {
let elapsed_since_activity = start.elapsed().as_millis() as u64 - current; let elapsed_since_activity = start.elapsed().as_millis() as u64 - current;
if elapsed_since_activity >= inactivity_timeout.as_millis() as u64 { if elapsed_since_activity >= inactivity_timeout.as_millis() as u64 {
debug!("Connection inactive for {}ms, closing", elapsed_since_activity); debug!("Connection inactive for {}ms, closing", elapsed_since_activity);
c2b_handle.abort();
b2c_handle.abort();
break; break;
} }
} }
@@ -1488,6 +1521,13 @@ impl TcpListenerManager {
} }
} }
} }
// Phase 1: Signal copy loops to exit gracefully (allows close_notify/FIN)
conn_cancel.cancel();
// Phase 2: Wait for graceful shutdown (2s shutdown timeout + 2s margin)
tokio::time::sleep(std::time::Duration::from_secs(4)).await;
// Phase 3: Force-abort if still stuck (e.g. blocked on write_all)
c2b_abort.abort();
b2c_abort.abort();
}); });
let bytes_in = c2b.await.unwrap_or(0); let bytes_in = c2b.await.unwrap_or(0);

View File

@@ -6,25 +6,28 @@
/// - `example.com` exact match /// - `example.com` exact match
/// - `**.example.com` matches any depth of subdomain /// - `**.example.com` matches any depth of subdomain
pub fn domain_matches(pattern: &str, domain: &str) -> bool { pub fn domain_matches(pattern: &str, domain: &str) -> bool {
let pattern = pattern.trim().to_lowercase(); let pattern = pattern.trim();
let domain = domain.trim().to_lowercase(); let domain = domain.trim();
if pattern == "*" { if pattern == "*" {
return true; return true;
} }
if pattern == domain { if pattern.eq_ignore_ascii_case(domain) {
return true; return true;
} }
// Wildcard patterns // Wildcard patterns
if pattern.starts_with("*.") { if pattern.starts_with("*.") || pattern.starts_with("*.") {
let suffix = &pattern[2..]; // e.g., "example.com" let suffix = &pattern[2..]; // e.g., "example.com"
// Match exact parent or any single-level subdomain // Match exact parent or any single-level subdomain
if domain == suffix { if domain.eq_ignore_ascii_case(suffix) {
return true; return true;
} }
if domain.ends_with(&format!(".{}", suffix)) { if domain.len() > suffix.len() + 1
&& domain.as_bytes()[domain.len() - suffix.len() - 1] == b'.'
&& domain[domain.len() - suffix.len()..].eq_ignore_ascii_case(suffix)
{
// Check it's a single level subdomain for `*.` // Check it's a single level subdomain for `*.`
let prefix = &domain[..domain.len() - suffix.len() - 1]; let prefix = &domain[..domain.len() - suffix.len() - 1];
return !prefix.contains('.'); return !prefix.contains('.');
@@ -35,11 +38,22 @@ pub fn domain_matches(pattern: &str, domain: &str) -> bool {
if pattern.starts_with("**.") { if pattern.starts_with("**.") {
let suffix = &pattern[3..]; let suffix = &pattern[3..];
// Match exact parent or any depth of subdomain // Match exact parent or any depth of subdomain
return domain == suffix || domain.ends_with(&format!(".{}", suffix)); if domain.eq_ignore_ascii_case(suffix) {
return true;
}
if domain.len() > suffix.len() + 1
&& domain.as_bytes()[domain.len() - suffix.len() - 1] == b'.'
&& domain[domain.len() - suffix.len()..].eq_ignore_ascii_case(suffix)
{
return true;
}
return false;
} }
// Use glob-match for more complex patterns // Use glob-match for more complex patterns (case-insensitive via lowercasing)
glob_match::glob_match(&pattern, &domain) let pattern_lower = pattern.to_lowercase();
let domain_lower = domain.to_lowercase();
glob_match::glob_match(&pattern_lower, &domain_lower)
} }
/// Check if a domain matches any of the given patterns. /// Check if a domain matches any of the given patterns.

View File

@@ -60,6 +60,16 @@ impl RouteManager {
manager manager
} }
/// Check if any route on the given port uses header matching.
/// Used to skip expensive header HashMap construction when no route needs it.
pub fn any_route_has_headers(&self, port: u16) -> bool {
if let Some(indices) = self.port_index.get(&port) {
indices.iter().any(|&idx| self.routes[idx].route_match.headers.is_some())
} else {
false
}
}
/// Find the best matching route for the given context. /// Find the best matching route for the given context.
pub fn find_route<'a>(&'a self, ctx: &MatchContext<'_>) -> Option<RouteMatchResult<'a>> { pub fn find_route<'a>(&'a self, ctx: &MatchContext<'_>) -> Option<RouteMatchResult<'a>> {
// Get routes for this port // Get routes for this port

View File

@@ -632,15 +632,13 @@ impl RustProxy {
let new_manager = Arc::new(new_manager); let new_manager = Arc::new(new_manager);
self.route_table.store(Arc::clone(&new_manager)); self.route_table.store(Arc::clone(&new_manager));
// Update listener manager // Update listener manager.
// IMPORTANT: TLS configs must be swapped BEFORE the route manager so that
// new routes only become visible after their certs are loaded. The reverse
// order (routes first) creates a window where connections match new routes
// but get the old TLS acceptor, causing cert mismatches.
if let Some(ref mut listener) = self.listener_manager { if let Some(ref mut listener) = self.listener_manager {
listener.update_route_manager(Arc::clone(&new_manager)); // 1. Update TLS configs first (so new certs are available before new routes)
// Cancel connections on routes that were removed or disabled
listener.invalidate_removed_routes(&active_route_ids);
// Prune HTTP proxy caches (rate limiters, regex cache, round-robin counters)
listener.prune_http_proxy_caches(&active_route_ids);
// Update TLS configs
let mut tls_configs = Self::extract_tls_configs(&routes); let mut tls_configs = Self::extract_tls_configs(&routes);
if let Some(ref cm_arc) = self.cert_manager { if let Some(ref cm_arc) = self.cert_manager {
let cm = cm_arc.lock().await; let cm = cm_arc.lock().await;
@@ -661,6 +659,13 @@ impl RustProxy {
} }
listener.set_tls_configs(tls_configs); listener.set_tls_configs(tls_configs);
// 2. Now swap the route manager (new routes become visible with certs already loaded)
listener.update_route_manager(Arc::clone(&new_manager));
// Cancel connections on routes that were removed or disabled
listener.invalidate_removed_routes(&active_route_ids);
// Prune HTTP proxy caches (rate limiters, regex cache, round-robin counters)
listener.prune_http_proxy_caches(&active_route_ids);
// Add new ports // Add new ports
for port in &new_ports { for port in &new_ports {
if !old_ports.contains(port) { if !old_ports.contains(port) {

View File

@@ -3,6 +3,6 @@
*/ */
export const commitinfo = { export const commitinfo = {
name: '@push.rocks/smartproxy', name: '@push.rocks/smartproxy',
version: '25.10.7', version: '25.11.9',
description: 'A powerful proxy package with unified route-based configuration for high traffic management. Features include SSL/TLS support, flexible routing patterns, WebSocket handling, advanced security options, and automatic ACME certificate management.' description: 'A powerful proxy package with unified route-based configuration for high traffic management. Features include SSL/TLS support, flexible routing patterns, WebSocket handling, advanced security options, and automatic ACME certificate management.'
} }