Compare commits

..

8 Commits

Author SHA1 Message Date
0930f7e10c v25.11.4
Some checks failed
Default (tags) / security (push) Failing after 1s
Default (tags) / test (push) Failing after 1s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-03-15 21:44:32 +00:00
aa9e6dfd94 fix(rustproxy-http): report streamed HTTP and WebSocket bytes per chunk for real-time throughput metrics 2026-03-15 21:44:32 +00:00
211d5cf835 v25.11.3
Some checks failed
Default (tags) / security (push) Failing after 1s
Default (tags) / test (push) Failing after 1s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-03-15 17:00:33 +00:00
2ce1899337 fix(repo): no changes to commit 2026-03-15 17:00:33 +00:00
2e2ffc4485 v25.11.2
Some checks failed
Default (tags) / security (push) Failing after 1s
Default (tags) / test (push) Failing after 1s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-03-15 16:58:41 +00:00
da26816af5 fix(rustproxy-http): avoid reusing HTTP/1 senders during streaming responses and relax HTTP/2 keep-alive timeouts 2026-03-15 16:58:41 +00:00
d598bffec3 v25.11.1
Some checks failed
Default (tags) / security (push) Failing after 1s
Default (tags) / test (push) Failing after 1s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-03-15 16:24:41 +00:00
a9dbccfaff fix(rustproxy-http): keep connection idle tracking alive during streaming and tune HTTP/2 connection lifetimes 2026-03-15 16:24:41 +00:00
6 changed files with 145 additions and 75 deletions

View File

@@ -1,5 +1,29 @@
# Changelog # Changelog
## 2026-03-15 - 25.11.4 - fix(rustproxy-http)
report streamed HTTP and WebSocket bytes per chunk for real-time throughput metrics
- Update CountingBody to record bytes immediately on each data frame instead of aggregating until completion or drop
- Record WebSocket tunnel traffic inside both copy loops and remove the final aggregate byte report to keep throughput metrics current
## 2026-03-15 - 25.11.3 - fix(repo)
no changes to commit
## 2026-03-15 - 25.11.2 - fix(rustproxy-http)
avoid reusing HTTP/1 senders during streaming responses and relax HTTP/2 keep-alive timeouts
- Stop returning HTTP/1 senders to the connection pool before upstream response bodies finish streaming to prevent unsafe reuse on active connections.
- Increase HTTP/2 keep-alive timeout from 5 seconds to 30 seconds in proxy connection builders to better support longer-lived backend streams.
- Improves reliability for large streaming payloads and backend fallback request handling.
## 2026-03-15 - 25.11.1 - fix(rustproxy-http)
keep connection idle tracking alive during streaming and tune HTTP/2 connection lifetimes
- Propagate connection activity tracking through HTTP/1, HTTP/2, and WebSocket forwarding so active request and response body streams do not trigger the idle watchdog.
- Update CountingBody to refresh connection activity timestamps while data frames are polled during uploads and downloads.
- Increase pooled HTTP/2 max age and set explicit HTTP/2 connection window sizes to improve long-lived streaming behavior.
## 2026-03-15 - 25.11.0 - feat(rustproxy-http) ## 2026-03-15 - 25.11.0 - feat(rustproxy-http)
add HTTP/2 Extended CONNECT WebSocket proxy support add HTTP/2 Extended CONNECT WebSocket proxy support

View File

@@ -1,6 +1,6 @@
{ {
"name": "@push.rocks/smartproxy", "name": "@push.rocks/smartproxy",
"version": "25.11.0", "version": "25.11.4",
"private": false, "private": false,
"description": "A powerful proxy package with unified route-based configuration for high traffic management. Features include SSL/TLS support, flexible routing patterns, WebSocket handling, advanced security options, and automatic ACME certificate management.", "description": "A powerful proxy package with unified route-based configuration for high traffic management. Features include SSL/TLS support, flexible routing patterns, WebSocket handling, advanced security options, and automatic ACME certificate management.",
"main": "dist_ts/index.js", "main": "dist_ts/index.js",

View File

@@ -20,7 +20,7 @@ const IDLE_TIMEOUT: Duration = Duration::from_secs(90);
const EVICTION_INTERVAL: Duration = Duration::from_secs(30); const EVICTION_INTERVAL: Duration = Duration::from_secs(30);
/// Maximum age for pooled HTTP/2 connections before proactive eviction. /// Maximum age for pooled HTTP/2 connections before proactive eviction.
/// Prevents staleness from backends that close idle connections (e.g. nginx GOAWAY). /// Prevents staleness from backends that close idle connections (e.g. nginx GOAWAY).
const MAX_H2_AGE: Duration = Duration::from_secs(120); const MAX_H2_AGE: Duration = Duration::from_secs(300);
/// Identifies a unique backend endpoint. /// Identifies a unique backend endpoint.
#[derive(Clone, Debug, Hash, Eq, PartialEq)] #[derive(Clone, Debug, Hash, Eq, PartialEq)]

View File

@@ -11,20 +11,22 @@ use rustproxy_metrics::MetricsCollector;
/// Wraps any `http_body::Body` and counts data bytes passing through. /// Wraps any `http_body::Body` and counts data bytes passing through.
/// ///
/// When the body is fully consumed or dropped, accumulated byte counts /// Each chunk is reported to the `MetricsCollector` immediately so that
/// are reported to the `MetricsCollector`. /// the throughput tracker (sampled at 1 Hz) reflects real-time data flow.
/// ///
/// The inner body is pinned on the heap to support `!Unpin` types like `hyper::body::Incoming`. /// The inner body is pinned on the heap to support `!Unpin` types like `hyper::body::Incoming`.
pub struct CountingBody<B> { pub struct CountingBody<B> {
inner: Pin<Box<B>>, inner: Pin<Box<B>>,
counted_bytes: AtomicU64,
metrics: Arc<MetricsCollector>, metrics: Arc<MetricsCollector>,
route_id: Option<String>, route_id: Option<String>,
source_ip: Option<String>, source_ip: Option<String>,
/// Whether we count bytes as "in" (request body) or "out" (response body). /// Whether we count bytes as "in" (request body) or "out" (response body).
direction: Direction, direction: Direction,
/// Whether we've already reported the bytes (to avoid double-reporting on drop). /// Optional connection-level activity tracker. When set, poll_frame updates this
reported: bool, /// to keep the idle watchdog alive during active body streaming (uploads/downloads).
connection_activity: Option<Arc<AtomicU64>>,
/// Start instant for computing elapsed ms for connection_activity.
activity_start: Option<std::time::Instant>,
} }
/// Which direction the bytes flow. /// Which direction the bytes flow.
@@ -47,42 +49,36 @@ impl<B> CountingBody<B> {
) -> Self { ) -> Self {
Self { Self {
inner: Box::pin(inner), inner: Box::pin(inner),
counted_bytes: AtomicU64::new(0),
metrics, metrics,
route_id, route_id,
source_ip, source_ip,
direction, direction,
reported: false, connection_activity: None,
activity_start: None,
} }
} }
/// Report accumulated bytes to the metrics collector. /// Set the connection-level activity tracker. When set, each data frame
fn report(&mut self) { /// updates this timestamp to prevent the idle watchdog from killing the
if self.reported { /// connection during active body streaming.
return; pub fn with_connection_activity(mut self, activity: Arc<AtomicU64>, start: std::time::Instant) -> Self {
} self.connection_activity = Some(activity);
self.reported = true; self.activity_start = Some(start);
self
let bytes = self.counted_bytes.load(Ordering::Relaxed);
if bytes == 0 {
return;
} }
/// Report a chunk of bytes immediately to the metrics collector.
#[inline]
fn report_chunk(&self, len: u64) {
let route_id = self.route_id.as_deref(); let route_id = self.route_id.as_deref();
let source_ip = self.source_ip.as_deref(); let source_ip = self.source_ip.as_deref();
match self.direction { match self.direction {
Direction::In => self.metrics.record_bytes(bytes, 0, route_id, source_ip), Direction::In => self.metrics.record_bytes(len, 0, route_id, source_ip),
Direction::Out => self.metrics.record_bytes(0, bytes, route_id, source_ip), Direction::Out => self.metrics.record_bytes(0, len, route_id, source_ip),
} }
} }
} }
impl<B> Drop for CountingBody<B> {
fn drop(&mut self) {
self.report();
}
}
// CountingBody is Unpin because inner is Pin<Box<B>> (always Unpin). // CountingBody is Unpin because inner is Pin<Box<B>> (always Unpin).
impl<B> Unpin for CountingBody<B> {} impl<B> Unpin for CountingBody<B> {}
@@ -102,16 +98,18 @@ where
match this.inner.as_mut().poll_frame(cx) { match this.inner.as_mut().poll_frame(cx) {
Poll::Ready(Some(Ok(frame))) => { Poll::Ready(Some(Ok(frame))) => {
if let Some(data) = frame.data_ref() { if let Some(data) = frame.data_ref() {
this.counted_bytes.fetch_add(data.len() as u64, Ordering::Relaxed); let len = data.len() as u64;
// Report bytes immediately so the 1 Hz throughput sampler sees them
this.report_chunk(len);
// Keep the connection-level idle watchdog alive during body streaming
if let (Some(activity), Some(start)) = (&this.connection_activity, &this.activity_start) {
activity.store(start.elapsed().as_millis() as u64, Ordering::Relaxed);
}
} }
Poll::Ready(Some(Ok(frame))) Poll::Ready(Some(Ok(frame)))
} }
Poll::Ready(Some(Err(e))) => Poll::Ready(Some(Err(e))), Poll::Ready(Some(Err(e))) => Poll::Ready(Some(Err(e))),
Poll::Ready(None) => { Poll::Ready(None) => Poll::Ready(None),
// Body is fully consumed — report now
this.report();
Poll::Ready(None)
}
Poll::Pending => Poll::Pending, Poll::Pending => Poll::Pending,
} }
} }

View File

@@ -33,6 +33,14 @@ use crate::request_filter::RequestFilter;
use crate::response_filter::ResponseFilter; use crate::response_filter::ResponseFilter;
use crate::upstream_selector::UpstreamSelector; use crate::upstream_selector::UpstreamSelector;
/// Per-connection context for keeping the idle watchdog alive during body streaming.
/// Passed through the forwarding chain so CountingBody can update the timestamp.
#[derive(Clone)]
struct ConnActivity {
last_activity: Arc<AtomicU64>,
start: std::time::Instant,
}
/// Default upstream connect timeout (30 seconds). /// Default upstream connect timeout (30 seconds).
const DEFAULT_CONNECT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); const DEFAULT_CONNECT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);
@@ -294,8 +302,9 @@ impl HttpProxyService {
let cn = cancel_inner.clone(); let cn = cancel_inner.clone();
let la = Arc::clone(&la_inner); let la = Arc::clone(&la_inner);
let st = start; let st = start;
let ca = ConnActivity { last_activity: Arc::clone(&la_inner), start };
async move { async move {
let result = svc.handle_request(req, peer, port, cn).await; let result = svc.handle_request(req, peer, port, cn, ca).await;
// Mark request end — update activity timestamp before guard drops // Mark request end — update activity timestamp before guard drops
la.store(st.elapsed().as_millis() as u64, Ordering::Relaxed); la.store(st.elapsed().as_millis() as u64, Ordering::Relaxed);
drop(req_guard); // Explicitly drop to decrement active_requests drop(req_guard); // Explicitly drop to decrement active_requests
@@ -306,8 +315,11 @@ impl HttpProxyService {
// Auto-detect h1 vs h2 based on ALPN / connection preface. // Auto-detect h1 vs h2 based on ALPN / connection preface.
// serve_connection_with_upgrades supports h1 Upgrade (WebSocket) and h2 Extended CONNECT (RFC 8441). // serve_connection_with_upgrades supports h1 Upgrade (WebSocket) and h2 Extended CONNECT (RFC 8441).
let mut builder = hyper_util::server::conn::auto::Builder::new(hyper_util::rt::TokioExecutor::new()); let mut builder = hyper_util::server::conn::auto::Builder::new(hyper_util::rt::TokioExecutor::new());
// Advertise Extended CONNECT support so H2 clients can initiate WebSocket connections // Configure H2 server settings: Extended CONNECT for WebSocket + flow control tuning
builder.http2().enable_connect_protocol(); builder.http2()
.enable_connect_protocol()
.initial_stream_window_size(2 * 1024 * 1024) // 2MB per stream (vs default 64KB)
.initial_connection_window_size(8 * 1024 * 1024); // 8MB per client connection
let conn = builder.serve_connection_with_upgrades(io, service); let conn = builder.serve_connection_with_upgrades(io, service);
// Pin on the heap — auto::UpgradeableConnection is !Unpin // Pin on the heap — auto::UpgradeableConnection is !Unpin
let mut conn = Box::pin(conn); let mut conn = Box::pin(conn);
@@ -367,6 +379,7 @@ impl HttpProxyService {
peer_addr: std::net::SocketAddr, peer_addr: std::net::SocketAddr,
port: u16, port: u16,
cancel: CancellationToken, cancel: CancellationToken,
conn_activity: ConnActivity,
) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> { ) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
let host = req.headers() let host = req.headers()
.get("host") .get("host")
@@ -500,6 +513,7 @@ impl HttpProxyService {
if is_h1_websocket || is_h2_websocket { if is_h1_websocket || is_h2_websocket {
let result = self.handle_websocket_upgrade( let result = self.handle_websocket_upgrade(
req, peer_addr, &upstream, route_match.route, route_id, &upstream_key, cancel, &ip_str, is_h2_websocket, req, peer_addr, &upstream, route_match.route, route_id, &upstream_key, cancel, &ip_str, is_h2_websocket,
if is_h2_websocket { Some(conn_activity.clone()) } else { None },
).await; ).await;
// Note: for WebSocket, connection_ended is called inside // Note: for WebSocket, connection_ended is called inside
// the spawned tunnel task when the connection closes. // the spawned tunnel task when the connection closes.
@@ -640,7 +654,7 @@ impl HttpProxyService {
self.metrics.set_backend_protocol(&upstream_key, "h2"); self.metrics.set_backend_protocol(&upstream_key, "h2");
let result = self.forward_h2_pooled( let result = self.forward_h2_pooled(
sender, parts, body, upstream_headers, &upstream_path, sender, parts, body, upstream_headers, &upstream_path,
route_match.route, route_id, &ip_str, &pool_key, domain_str, route_match.route, route_id, &ip_str, &pool_key, domain_str, &conn_activity,
).await; ).await;
self.upstream_selector.connection_ended(&upstream_key); self.upstream_selector.connection_ended(&upstream_key);
return result; return result;
@@ -779,19 +793,19 @@ impl HttpProxyService {
self.forward_h2_with_fallback( self.forward_h2_with_fallback(
io, parts, body, upstream_headers, &upstream_path, io, parts, body, upstream_headers, &upstream_path,
&upstream, route_match.route, route_id, &ip_str, &final_pool_key, &upstream, route_match.route, route_id, &ip_str, &final_pool_key,
host.clone(), domain_str, host.clone(), domain_str, &conn_activity,
).await ).await
} else { } else {
// Explicit H2 mode: hard-fail on handshake error (preserved behavior) // Explicit H2 mode: hard-fail on handshake error (preserved behavior)
self.forward_h2( self.forward_h2(
io, parts, body, upstream_headers, &upstream_path, io, parts, body, upstream_headers, &upstream_path,
&upstream, route_match.route, route_id, &ip_str, &final_pool_key, domain_str, &upstream, route_match.route, route_id, &ip_str, &final_pool_key, domain_str, &conn_activity,
).await ).await
} }
} else { } else {
self.forward_h1( self.forward_h1(
io, parts, body, upstream_headers, &upstream_path, io, parts, body, upstream_headers, &upstream_path,
&upstream, route_match.route, route_id, &ip_str, &final_pool_key, domain_str, &upstream, route_match.route, route_id, &ip_str, &final_pool_key, domain_str, &conn_activity,
).await ).await
}; };
self.upstream_selector.connection_ended(&upstream_key); self.upstream_selector.connection_ended(&upstream_key);
@@ -814,6 +828,7 @@ impl HttpProxyService {
source_ip: &str, source_ip: &str,
pool_key: &crate::connection_pool::PoolKey, pool_key: &crate::connection_pool::PoolKey,
domain: &str, domain: &str,
conn_activity: &ConnActivity,
) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> { ) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
let backend_key = format!("{}:{}", pool_key.host, pool_key.port); let backend_key = format!("{}:{}", pool_key.host, pool_key.port);
@@ -822,7 +837,7 @@ impl HttpProxyService {
self.metrics.backend_pool_hit(&backend_key); self.metrics.backend_pool_hit(&backend_key);
return self.forward_h1_with_sender( return self.forward_h1_with_sender(
pooled_sender, parts, body, upstream_headers, upstream_path, pooled_sender, parts, body, upstream_headers, upstream_path,
route, route_id, source_ip, pool_key, domain, route, route_id, source_ip, pool_key, domain, conn_activity,
).await; ).await;
} }
@@ -845,7 +860,7 @@ impl HttpProxyService {
} }
}); });
self.forward_h1_with_sender(sender, parts, body, upstream_headers, upstream_path, route, route_id, source_ip, pool_key, domain).await self.forward_h1_with_sender(sender, parts, body, upstream_headers, upstream_path, route, route_id, source_ip, pool_key, domain, conn_activity).await
} }
/// Common H1 forwarding logic used by both fresh and pooled paths. /// Common H1 forwarding logic used by both fresh and pooled paths.
@@ -861,6 +876,7 @@ impl HttpProxyService {
source_ip: &str, source_ip: &str,
pool_key: &crate::connection_pool::PoolKey, pool_key: &crate::connection_pool::PoolKey,
domain: &str, domain: &str,
conn_activity: &ConnActivity,
) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> { ) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
// Always use HTTP/1.1 for h1 backend connections (h2 incoming requests have version HTTP/2.0) // Always use HTTP/1.1 for h1 backend connections (h2 incoming requests have version HTTP/2.0)
let mut upstream_req = Request::builder() let mut upstream_req = Request::builder()
@@ -879,7 +895,7 @@ impl HttpProxyService {
route_id.map(|s| s.to_string()), route_id.map(|s| s.to_string()),
Some(source_ip.to_string()), Some(source_ip.to_string()),
Direction::In, Direction::In,
); ).with_connection_activity(Arc::clone(&conn_activity.last_activity), conn_activity.start);
let boxed_body: BoxBody<Bytes, hyper::Error> = BoxBody::new(counting_req_body); let boxed_body: BoxBody<Bytes, hyper::Error> = BoxBody::new(counting_req_body);
let upstream_req = upstream_req.body(boxed_body).unwrap(); let upstream_req = upstream_req.body(boxed_body).unwrap();
@@ -894,10 +910,17 @@ impl HttpProxyService {
} }
}; };
// Return sender to pool (body streams lazily, sender is reusable once response head is received) // Note: we do NOT return the sender to the pool here because the response body
self.connection_pool.checkin_h1(pool_key.clone(), sender); // hasn't been fully streamed yet. Pooling a sender while its response body is still
// in-flight risks another request being dispatched on the same connection if is_ready()
// momentarily returns true between chunks. The sender is dropped after this scope,
// and the backend connection remains alive via the spawned conn driver task until
// the response body finishes streaming.
// For small/empty responses, the sender could theoretically be reused, but the safety
// of large streaming responses (e.g. 352MB Docker layers) takes priority.
drop(sender);
self.build_streaming_response(upstream_response, route, route_id, source_ip).await self.build_streaming_response(upstream_response, route, route_id, source_ip, conn_activity).await
} }
/// Forward request to backend via HTTP/2 with body streaming (fresh connection). /// Forward request to backend via HTTP/2 with body streaming (fresh connection).
@@ -915,6 +938,7 @@ impl HttpProxyService {
source_ip: &str, source_ip: &str,
pool_key: &crate::connection_pool::PoolKey, pool_key: &crate::connection_pool::PoolKey,
domain: &str, domain: &str,
conn_activity: &ConnActivity,
) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> { ) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
let backend_key = format!("{}:{}", pool_key.host, pool_key.port); let backend_key = format!("{}:{}", pool_key.host, pool_key.port);
let exec = hyper_util::rt::TokioExecutor::new(); let exec = hyper_util::rt::TokioExecutor::new();
@@ -922,9 +946,9 @@ impl HttpProxyService {
h2_builder h2_builder
.timer(hyper_util::rt::TokioTimer::new()) .timer(hyper_util::rt::TokioTimer::new())
.keep_alive_interval(std::time::Duration::from_secs(10)) .keep_alive_interval(std::time::Duration::from_secs(10))
.keep_alive_timeout(std::time::Duration::from_secs(5)) .keep_alive_timeout(std::time::Duration::from_secs(30))
.adaptive_window(true) .initial_stream_window_size(2 * 1024 * 1024)
.initial_stream_window_size(2 * 1024 * 1024); .initial_connection_window_size(16 * 1024 * 1024);
let (sender, conn): ( let (sender, conn): (
hyper::client::conn::http2::SendRequest<BoxBody<Bytes, hyper::Error>>, hyper::client::conn::http2::SendRequest<BoxBody<Bytes, hyper::Error>>,
hyper::client::conn::http2::Connection<TokioIo<BackendStream>, BoxBody<Bytes, hyper::Error>, hyper_util::rt::TokioExecutor>, hyper::client::conn::http2::Connection<TokioIo<BackendStream>, BoxBody<Bytes, hyper::Error>, hyper_util::rt::TokioExecutor>,
@@ -950,7 +974,7 @@ impl HttpProxyService {
// Clone sender for potential pool registration; register only after first request succeeds // Clone sender for potential pool registration; register only after first request succeeds
let sender_for_pool = sender.clone(); let sender_for_pool = sender.clone();
let result = self.forward_h2_with_sender(sender, parts, body, upstream_headers, upstream_path, route, route_id, source_ip, Some(pool_key), domain).await; let result = self.forward_h2_with_sender(sender, parts, body, upstream_headers, upstream_path, route, route_id, source_ip, Some(pool_key), domain, conn_activity).await;
if matches!(&result, Ok(ref resp) if resp.status() != StatusCode::BAD_GATEWAY) { if matches!(&result, Ok(ref resp) if resp.status() != StatusCode::BAD_GATEWAY) {
self.connection_pool.register_h2(pool_key.clone(), sender_for_pool); self.connection_pool.register_h2(pool_key.clone(), sender_for_pool);
} }
@@ -972,6 +996,7 @@ impl HttpProxyService {
source_ip: &str, source_ip: &str,
pool_key: &crate::connection_pool::PoolKey, pool_key: &crate::connection_pool::PoolKey,
domain: &str, domain: &str,
conn_activity: &ConnActivity,
) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> { ) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
// Save retry state for bodyless requests (cheap: Method is an enum, HeaderMap clones Arc-backed Bytes) // Save retry state for bodyless requests (cheap: Method is an enum, HeaderMap clones Arc-backed Bytes)
let retry_state = if body.is_end_stream() { let retry_state = if body.is_end_stream() {
@@ -982,7 +1007,7 @@ impl HttpProxyService {
let result = self.forward_h2_with_sender( let result = self.forward_h2_with_sender(
sender, parts, body, upstream_headers, upstream_path, sender, parts, body, upstream_headers, upstream_path,
route, route_id, source_ip, Some(pool_key), domain, route, route_id, source_ip, Some(pool_key), domain, conn_activity,
).await; ).await;
// If the request failed (502) and we can retry with an empty body, do so // If the request failed (502) and we can retry with an empty body, do so
@@ -993,7 +1018,7 @@ impl HttpProxyService {
"Stale pooled H2 sender, retrying with fresh connection"); "Stale pooled H2 sender, retrying with fresh connection");
return self.retry_h2_with_fresh_connection( return self.retry_h2_with_fresh_connection(
method, headers, upstream_path, method, headers, upstream_path,
pool_key, route, route_id, source_ip, domain, pool_key, route, route_id, source_ip, domain, conn_activity,
).await; ).await;
} }
} }
@@ -1012,6 +1037,7 @@ impl HttpProxyService {
route_id: Option<&str>, route_id: Option<&str>,
source_ip: &str, source_ip: &str,
domain: &str, domain: &str,
conn_activity: &ConnActivity,
) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> { ) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
let backend_key = format!("{}:{}", pool_key.host, pool_key.port); let backend_key = format!("{}:{}", pool_key.host, pool_key.port);
@@ -1063,9 +1089,9 @@ impl HttpProxyService {
h2_builder h2_builder
.timer(hyper_util::rt::TokioTimer::new()) .timer(hyper_util::rt::TokioTimer::new())
.keep_alive_interval(std::time::Duration::from_secs(10)) .keep_alive_interval(std::time::Duration::from_secs(10))
.keep_alive_timeout(std::time::Duration::from_secs(5)) .keep_alive_timeout(std::time::Duration::from_secs(30))
.adaptive_window(true) .initial_stream_window_size(2 * 1024 * 1024)
.initial_stream_window_size(2 * 1024 * 1024); .initial_connection_window_size(16 * 1024 * 1024);
let (mut sender, conn): ( let (mut sender, conn): (
hyper::client::conn::http2::SendRequest<BoxBody<Bytes, hyper::Error>>, hyper::client::conn::http2::SendRequest<BoxBody<Bytes, hyper::Error>>,
hyper::client::conn::http2::Connection<TokioIo<BackendStream>, BoxBody<Bytes, hyper::Error>, hyper_util::rt::TokioExecutor>, hyper::client::conn::http2::Connection<TokioIo<BackendStream>, BoxBody<Bytes, hyper::Error>, hyper_util::rt::TokioExecutor>,
@@ -1116,7 +1142,7 @@ impl HttpProxyService {
Ok(resp) => { Ok(resp) => {
// Register in pool only after request succeeds // Register in pool only after request succeeds
self.connection_pool.register_h2(pool_key.clone(), sender); self.connection_pool.register_h2(pool_key.clone(), sender);
let result = self.build_streaming_response(resp, route, route_id, source_ip).await; let result = self.build_streaming_response(resp, route, route_id, source_ip, conn_activity).await;
// Close the fresh backend connection (opened above) // Close the fresh backend connection (opened above)
self.metrics.backend_connection_closed(&backend_key); self.metrics.backend_connection_closed(&backend_key);
result result
@@ -1152,15 +1178,16 @@ impl HttpProxyService {
pool_key: &crate::connection_pool::PoolKey, pool_key: &crate::connection_pool::PoolKey,
requested_host: Option<String>, requested_host: Option<String>,
domain: &str, domain: &str,
conn_activity: &ConnActivity,
) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> { ) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
let exec = hyper_util::rt::TokioExecutor::new(); let exec = hyper_util::rt::TokioExecutor::new();
let mut h2_builder = hyper::client::conn::http2::Builder::new(exec); let mut h2_builder = hyper::client::conn::http2::Builder::new(exec);
h2_builder h2_builder
.timer(hyper_util::rt::TokioTimer::new()) .timer(hyper_util::rt::TokioTimer::new())
.keep_alive_interval(std::time::Duration::from_secs(10)) .keep_alive_interval(std::time::Duration::from_secs(10))
.keep_alive_timeout(std::time::Duration::from_secs(5)) .keep_alive_timeout(std::time::Duration::from_secs(30))
.adaptive_window(true) .initial_stream_window_size(2 * 1024 * 1024)
.initial_stream_window_size(2 * 1024 * 1024); .initial_connection_window_size(16 * 1024 * 1024);
let handshake_result = tokio::time::timeout( let handshake_result = tokio::time::timeout(
self.connect_timeout, self.connect_timeout,
h2_builder.handshake(io), h2_builder.handshake(io),
@@ -1196,7 +1223,7 @@ impl HttpProxyService {
let fallback_io = TokioIo::new(fallback_backend); let fallback_io = TokioIo::new(fallback_backend);
let result = self.forward_h1( let result = self.forward_h1(
fallback_io, parts, body, upstream_headers, upstream_path, fallback_io, parts, body, upstream_headers, upstream_path,
upstream, route, route_id, source_ip, &h1_pool_key, domain, upstream, route, route_id, source_ip, &h1_pool_key, domain, conn_activity,
).await; ).await;
self.metrics.backend_connection_closed(&bk); self.metrics.backend_connection_closed(&bk);
result result
@@ -1244,7 +1271,7 @@ impl HttpProxyService {
route_id.map(|s| s.to_string()), route_id.map(|s| s.to_string()),
Some(source_ip.to_string()), Some(source_ip.to_string()),
Direction::In, Direction::In,
); ).with_connection_activity(Arc::clone(&conn_activity.last_activity), conn_activity.start);
let boxed_body: BoxBody<Bytes, hyper::Error> = BoxBody::new(counting_req_body); let boxed_body: BoxBody<Bytes, hyper::Error> = BoxBody::new(counting_req_body);
let upstream_req = upstream_req.body(boxed_body).unwrap(); let upstream_req = upstream_req.body(boxed_body).unwrap();
@@ -1252,7 +1279,7 @@ impl HttpProxyService {
Ok(upstream_response) => { Ok(upstream_response) => {
// H2 works! Register sender in pool for multiplexed reuse // H2 works! Register sender in pool for multiplexed reuse
self.connection_pool.register_h2(pool_key.clone(), sender); self.connection_pool.register_h2(pool_key.clone(), sender);
self.build_streaming_response(upstream_response, route, route_id, source_ip).await self.build_streaming_response(upstream_response, route, route_id, source_ip, conn_activity).await
} }
Err(e) => { Err(e) => {
// H2 request failed — backend advertises h2 via ALPN but doesn't // H2 request failed — backend advertises h2 via ALPN but doesn't
@@ -1285,7 +1312,7 @@ impl HttpProxyService {
let fallback_io = TokioIo::new(fallback_backend); let fallback_io = TokioIo::new(fallback_backend);
let result = self.forward_h1_empty_body( let result = self.forward_h1_empty_body(
fallback_io, method, headers, upstream_path, fallback_io, method, headers, upstream_path,
route, route_id, source_ip, &h1_pool_key, domain, route, route_id, source_ip, &h1_pool_key, domain, conn_activity,
).await; ).await;
// Close the reconnected backend connection (opened in reconnect_backend) // Close the reconnected backend connection (opened in reconnect_backend)
self.metrics.backend_connection_closed(&bk); self.metrics.backend_connection_closed(&bk);
@@ -1334,7 +1361,7 @@ impl HttpProxyService {
let fallback_io = TokioIo::new(fallback_backend); let fallback_io = TokioIo::new(fallback_backend);
let result = self.forward_h1( let result = self.forward_h1(
fallback_io, parts, body, upstream_headers, upstream_path, fallback_io, parts, body, upstream_headers, upstream_path,
upstream, route, route_id, source_ip, &h1_pool_key, domain, upstream, route, route_id, source_ip, &h1_pool_key, domain, conn_activity,
).await; ).await;
// Close the reconnected backend connection (opened in reconnect_backend) // Close the reconnected backend connection (opened in reconnect_backend)
self.metrics.backend_connection_closed(&bk); self.metrics.backend_connection_closed(&bk);
@@ -1361,6 +1388,7 @@ impl HttpProxyService {
source_ip: &str, source_ip: &str,
pool_key: &crate::connection_pool::PoolKey, pool_key: &crate::connection_pool::PoolKey,
domain: &str, domain: &str,
conn_activity: &ConnActivity,
) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> { ) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
let backend_key = format!("{}:{}", pool_key.host, pool_key.port); let backend_key = format!("{}:{}", pool_key.host, pool_key.port);
let (mut sender, conn): ( let (mut sender, conn): (
@@ -1404,10 +1432,10 @@ impl HttpProxyService {
} }
}; };
// Return sender to pool for keep-alive reuse // Don't pool the sender while response body is still streaming (same safety as forward_h1_with_sender)
self.connection_pool.checkin_h1(pool_key.clone(), sender); drop(sender);
self.build_streaming_response(upstream_response, route, route_id, source_ip).await self.build_streaming_response(upstream_response, route, route_id, source_ip, conn_activity).await
} }
/// Reconnect to a backend (used for H2→H1 fallback). /// Reconnect to a backend (used for H2→H1 fallback).
@@ -1478,6 +1506,7 @@ impl HttpProxyService {
source_ip: &str, source_ip: &str,
pool_key: Option<&crate::connection_pool::PoolKey>, pool_key: Option<&crate::connection_pool::PoolKey>,
domain: &str, domain: &str,
conn_activity: &ConnActivity,
) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> { ) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
// Build absolute URI for H2 pseudo-headers (:scheme, :authority) // Build absolute URI for H2 pseudo-headers (:scheme, :authority)
// Use the requested domain as authority (not backend address) so :authority matches Host header // Use the requested domain as authority (not backend address) so :authority matches Host header
@@ -1506,7 +1535,7 @@ impl HttpProxyService {
route_id.map(|s| s.to_string()), route_id.map(|s| s.to_string()),
Some(source_ip.to_string()), Some(source_ip.to_string()),
Direction::In, Direction::In,
); ).with_connection_activity(Arc::clone(&conn_activity.last_activity), conn_activity.start);
let boxed_body: BoxBody<Bytes, hyper::Error> = BoxBody::new(counting_req_body); let boxed_body: BoxBody<Bytes, hyper::Error> = BoxBody::new(counting_req_body);
let upstream_req = upstream_req.body(boxed_body).unwrap(); let upstream_req = upstream_req.body(boxed_body).unwrap();
@@ -1527,7 +1556,7 @@ impl HttpProxyService {
} }
}; };
self.build_streaming_response(upstream_response, route, route_id, source_ip).await self.build_streaming_response(upstream_response, route, route_id, source_ip, conn_activity).await
} }
/// Build the client-facing response from an upstream response, streaming the body. /// Build the client-facing response from an upstream response, streaming the body.
@@ -1540,6 +1569,7 @@ impl HttpProxyService {
route: &rustproxy_config::RouteConfig, route: &rustproxy_config::RouteConfig,
route_id: Option<&str>, route_id: Option<&str>,
source_ip: &str, source_ip: &str,
conn_activity: &ConnActivity,
) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> { ) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
let (resp_parts, resp_body) = upstream_response.into_parts(); let (resp_parts, resp_body) = upstream_response.into_parts();
@@ -1560,7 +1590,7 @@ impl HttpProxyService {
route_id.map(|s| s.to_string()), route_id.map(|s| s.to_string()),
Some(source_ip.to_string()), Some(source_ip.to_string()),
Direction::Out, Direction::Out,
); ).with_connection_activity(Arc::clone(&conn_activity.last_activity), conn_activity.start);
let body: BoxBody<Bytes, hyper::Error> = BoxBody::new(counting_body); let body: BoxBody<Bytes, hyper::Error> = BoxBody::new(counting_body);
@@ -1579,6 +1609,7 @@ impl HttpProxyService {
cancel: CancellationToken, cancel: CancellationToken,
source_ip: &str, source_ip: &str,
is_h2: bool, is_h2: bool,
conn_activity: Option<ConnActivity>,
) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> { ) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::io::{AsyncReadExt, AsyncWriteExt};
@@ -1882,7 +1913,15 @@ impl HttpProxyService {
let last_activity = Arc::new(AtomicU64::new(0)); let last_activity = Arc::new(AtomicU64::new(0));
let start = std::time::Instant::now(); let start = std::time::Instant::now();
// For H2 WebSocket: also update the connection-level activity tracker
// to prevent the idle watchdog from killing the H2 connection
let conn_act_c2u = conn_activity.as_ref().map(|ca| (Arc::clone(&ca.last_activity), ca.start));
let conn_act_u2c = conn_activity.as_ref().map(|ca| (Arc::clone(&ca.last_activity), ca.start));
let la1 = Arc::clone(&last_activity); let la1 = Arc::clone(&last_activity);
let metrics_c2u = Arc::clone(&metrics);
let route_c2u = route_id_owned.clone();
let ip_c2u = source_ip_owned.clone();
let c2u = tokio::spawn(async move { let c2u = tokio::spawn(async move {
let mut buf = vec![0u8; 65536]; let mut buf = vec![0u8; 65536];
let mut total = 0u64; let mut total = 0u64;
@@ -1895,13 +1934,20 @@ impl HttpProxyService {
break; break;
} }
total += n as u64; total += n as u64;
metrics_c2u.record_bytes(n as u64, 0, route_c2u.as_deref(), Some(&ip_c2u));
la1.store(start.elapsed().as_millis() as u64, Ordering::Relaxed); la1.store(start.elapsed().as_millis() as u64, Ordering::Relaxed);
if let Some((ref ca, ca_start)) = conn_act_c2u {
ca.store(ca_start.elapsed().as_millis() as u64, Ordering::Relaxed);
}
} }
let _ = uw.shutdown().await; let _ = uw.shutdown().await;
total total
}); });
let la2 = Arc::clone(&last_activity); let la2 = Arc::clone(&last_activity);
let metrics_u2c = Arc::clone(&metrics);
let route_u2c = route_id_owned.clone();
let ip_u2c = source_ip_owned.clone();
let u2c = tokio::spawn(async move { let u2c = tokio::spawn(async move {
let mut buf = vec![0u8; 65536]; let mut buf = vec![0u8; 65536];
let mut total = 0u64; let mut total = 0u64;
@@ -1914,7 +1960,11 @@ impl HttpProxyService {
break; break;
} }
total += n as u64; total += n as u64;
metrics_u2c.record_bytes(0, n as u64, route_u2c.as_deref(), Some(&ip_u2c));
la2.store(start.elapsed().as_millis() as u64, Ordering::Relaxed); la2.store(start.elapsed().as_millis() as u64, Ordering::Relaxed);
if let Some((ref ca, ca_start)) = conn_act_u2c {
ca.store(ca_start.elapsed().as_millis() as u64, Ordering::Relaxed);
}
} }
let _ = cw.shutdown().await; let _ = cw.shutdown().await;
total total
@@ -1971,9 +2021,7 @@ impl HttpProxyService {
debug!("WebSocket tunnel closed: {} bytes in, {} bytes out", bytes_in, bytes_out); debug!("WebSocket tunnel closed: {} bytes in, {} bytes out", bytes_in, bytes_out);
upstream_selector.connection_ended(&upstream_key_owned); upstream_selector.connection_ended(&upstream_key_owned);
if let Some(ref rid) = route_id_owned { // Bytes already reported per-chunk in the copy loops above
metrics.record_bytes(bytes_in, bytes_out, Some(rid.as_str()), Some(&source_ip_owned));
}
}); });
let body: BoxBody<Bytes, hyper::Error> = BoxBody::new( let body: BoxBody<Bytes, hyper::Error> = BoxBody::new(

View File

@@ -3,6 +3,6 @@
*/ */
export const commitinfo = { export const commitinfo = {
name: '@push.rocks/smartproxy', name: '@push.rocks/smartproxy',
version: '25.11.0', version: '25.11.4',
description: 'A powerful proxy package with unified route-based configuration for high traffic management. Features include SSL/TLS support, flexible routing patterns, WebSocket handling, advanced security options, and automatic ACME certificate management.' description: 'A powerful proxy package with unified route-based configuration for high traffic management. Features include SSL/TLS support, flexible routing patterns, WebSocket handling, advanced security options, and automatic ACME certificate management.'
} }