Compare commits
34 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| c2eacd1b30 | |||
| 1fdbfcf0aa | |||
| 9b184acc8c | |||
| b475968f4e | |||
| 878eab6e88 | |||
| 77abe0804d | |||
| ae0342d018 | |||
| 365981d9cf | |||
| 2cc0ff0030 | |||
| 72935e7ee0 | |||
| 61db285e04 | |||
| d165829022 | |||
| 5e6cf391ab | |||
| 2b1a21c599 | |||
| b8e1c9f3cf | |||
| c65369540c | |||
| 59e108edbd | |||
| 1e2ca68fc7 | |||
| 4c76a9f9f3 | |||
| 8e76c42cea | |||
| b1f4181139 | |||
| a1b8d40011 | |||
| 246b44913e | |||
| b3d4949225 | |||
| 0475e6b442 | |||
| 8cdb95a853 | |||
| 8cefe9d66a | |||
| d5e08c83fc | |||
| 1247f48856 | |||
| e3bae4c399 | |||
| 0930f7e10c | |||
| aa9e6dfd94 | |||
| 211d5cf835 | |||
| 2ce1899337 |
103
changelog.md
103
changelog.md
@@ -1,5 +1,108 @@
|
||||
# Changelog
|
||||
|
||||
## 2026-03-16 - 25.11.19 - fix(rustproxy-http)
|
||||
avoid reusing pooled HTTP/2 connections for requests with bodies to prevent upload flow-control stalls
|
||||
|
||||
- Limit HTTP/2 pool checkout to bodyless requests such as GET, HEAD, and DELETE
|
||||
- Skip re-registering HTTP/2 connections in the pool after requests that send a body
|
||||
- Prevent stalled uploads caused by depleted connection-level flow control windows on reused HTTP/2 connections
|
||||
|
||||
## 2026-03-16 - 25.11.18 - fix(repo)
|
||||
no changes to commit
|
||||
|
||||
|
||||
## 2026-03-16 - 25.11.17 - fix(rustproxy-http)
|
||||
prevent stale HTTP/2 connection drivers from evicting newer pooled connections
|
||||
|
||||
- add generation IDs to pooled HTTP/2 senders so pool removal only affects the matching connection
|
||||
- update HTTP/2 proxy and retry paths to register generation-tagged connections and skip eviction before registration completes
|
||||
|
||||
## 2026-03-16 - 25.11.16 - fix(repo)
|
||||
no changes to commit
|
||||
|
||||
|
||||
## 2026-03-16 - 25.11.15 - fix(rustproxy-http)
|
||||
implement vectored write support for backend streams
|
||||
|
||||
- Add poll_write_vectored forwarding for both plain and TLS backend stream variants
|
||||
- Expose is_write_vectored so the proxy can correctly report vectored write capability
|
||||
|
||||
## 2026-03-16 - 25.11.14 - fix(rustproxy-http)
|
||||
forward vectored write support in ShutdownOnDrop AsyncWrite wrapper
|
||||
|
||||
- Implements poll_write_vectored by delegating to the wrapped writer
|
||||
- Exposes is_write_vectored so the wrapper preserves underlying AsyncWrite capabilities
|
||||
|
||||
## 2026-03-16 - 25.11.13 - fix(rustproxy-http)
|
||||
remove hot-path debug logging from HTTP/1 connection pool hits
|
||||
|
||||
- Stops emitting debug logs when reusing HTTP/1 idle connections in the connection pool.
|
||||
- Keeps pool hit behavior unchanged while reducing overhead on a frequently executed path.
|
||||
|
||||
## 2026-03-16 - 25.11.12 - fix(rustproxy-http)
|
||||
remove connection pool hit logging and keep logging limited to actual failures
|
||||
|
||||
- Removes debug and warning logs for HTTP/2 connection pool hits and age checks.
|
||||
- Keeps pool behavior unchanged while reducing noisy per-request logging in the Rust HTTP proxy layer.
|
||||
|
||||
## 2026-03-16 - 25.11.11 - fix(rustproxy-http)
|
||||
improve HTTP/2 proxy error logging with warning-level connection failures and debug error details
|
||||
|
||||
- Adds debug-formatted error fields to HTTP/2 handshake, retry, fallback, and request failure logs
|
||||
- Promotes upstream HTTP/2 connection error logs from debug to warn to improve operational visibility
|
||||
|
||||
## 2026-03-16 - 25.11.10 - fix(rustproxy-http)
|
||||
validate pooled HTTP/2 connections asynchronously before reuse and evict stale senders
|
||||
|
||||
- Add an async ready() check with a 500ms timeout before reusing pooled HTTP/2 senders to catch GOAWAY/RST states before forwarding requests
|
||||
- Return connection age from the HTTP/2 pool checkout path and log warnings for older pooled connections
|
||||
- Evict pooled HTTP/2 senders when they are closed, exceed max age, fail readiness validation, or time out during readiness checks
|
||||
|
||||
## 2026-03-16 - 25.11.9 - fix(rustproxy-routing)
|
||||
reduce hot-path allocations in routing, metrics, and proxy protocol handling
|
||||
|
||||
- skip HTTP header map construction unless a route on the current port uses header matching
|
||||
- reuse computed client IP strings during HTTP route matching to avoid redundant allocations
|
||||
- optimize per-route and per-IP metric updates with get-first lookups to avoid unnecessary String creation on existing entries
|
||||
- replace heap-allocated PROXY protocol peek and discard buffers with stack-allocated buffers in the TCP listener
|
||||
- improve domain matcher case-insensitive wildcard checks while preserving glob fallback behavior
|
||||
|
||||
## 2026-03-16 - 25.11.8 - fix(rustproxy-http)
|
||||
prevent premature idle timeouts during streamed HTTP responses and ensure TLS close_notify is sent on dropped connections
|
||||
|
||||
- track active streaming response bodies so the HTTP idle watchdog does not close connections mid-transfer
|
||||
- add a ShutdownOnDrop wrapper for TLS-terminated HTTP connections to send shutdown on drop and avoid improperly terminated TLS sessions
|
||||
- apply the shutdown wrapper in passthrough TLS terminate and terminate+reencrypt HTTP handling
|
||||
|
||||
## 2026-03-16 - 25.11.7 - fix(rustproxy)
|
||||
prevent TLS route reload certificate mismatches and tighten passthrough connection handling
|
||||
|
||||
- Load updated TLS configs before swapping the route manager so newly visible routes always have their certificates available.
|
||||
- Add timeouts when peeking initial decrypted data after TLS handshake to avoid leaked idle connections.
|
||||
- Raise dropped, blocked, unmatched, and errored passthrough connection events from debug to warn for better operational visibility.
|
||||
|
||||
## 2026-03-16 - 25.11.6 - fix(rustproxy-http,rustproxy-passthrough)
|
||||
improve upstream connection cleanup and graceful tunnel shutdown
|
||||
|
||||
- Evict pooled HTTP/2 connections when their driver exits and shorten the maximum pooled H2 age to reduce reuse of stale upstream connections.
|
||||
- Strip hop-by-hop headers from backend responses before forwarding to HTTP/2 clients to avoid invalid H2 response handling.
|
||||
- Replace immediate task aborts in WebSocket and TCP tunnel watchdogs with cancellation-driven graceful shutdown plus timed fallback aborts.
|
||||
- Use non-blocking semaphore acquisition in the TCP listener so connection limits do not stall the accept loop for the entire port.
|
||||
|
||||
## 2026-03-16 - 25.11.5 - fix(repo)
|
||||
no changes to commit
|
||||
|
||||
|
||||
## 2026-03-15 - 25.11.4 - fix(rustproxy-http)
|
||||
report streamed HTTP and WebSocket bytes per chunk for real-time throughput metrics
|
||||
|
||||
- Update CountingBody to record bytes immediately on each data frame instead of aggregating until completion or drop
|
||||
- Record WebSocket tunnel traffic inside both copy loops and remove the final aggregate byte report to keep throughput metrics current
|
||||
|
||||
## 2026-03-15 - 25.11.3 - fix(repo)
|
||||
no changes to commit
|
||||
|
||||
|
||||
## 2026-03-15 - 25.11.2 - fix(rustproxy-http)
|
||||
avoid reusing HTTP/1 senders during streaming responses and relax HTTP/2 keep-alive timeouts
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@push.rocks/smartproxy",
|
||||
"version": "25.11.2",
|
||||
"version": "25.11.19",
|
||||
"private": false,
|
||||
"description": "A powerful proxy package with unified route-based configuration for high traffic management. Features include SSL/TLS support, flexible routing patterns, WebSocket handling, advanced security options, and automatic ACME certificate management.",
|
||||
"main": "dist_ts/index.js",
|
||||
|
||||
@@ -4,13 +4,13 @@
|
||||
//! HTTP/2 connections are multiplexed (clone the sender for each request).
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use bytes::Bytes;
|
||||
use dashmap::DashMap;
|
||||
use http_body_util::combinators::BoxBody;
|
||||
use hyper::client::conn::{http1, http2};
|
||||
use tracing::debug;
|
||||
|
||||
/// Maximum idle connections per backend key.
|
||||
const MAX_IDLE_PER_KEY: usize = 16;
|
||||
@@ -20,7 +20,8 @@ const IDLE_TIMEOUT: Duration = Duration::from_secs(90);
|
||||
const EVICTION_INTERVAL: Duration = Duration::from_secs(30);
|
||||
/// Maximum age for pooled HTTP/2 connections before proactive eviction.
|
||||
/// Prevents staleness from backends that close idle connections (e.g. nginx GOAWAY).
|
||||
const MAX_H2_AGE: Duration = Duration::from_secs(300);
|
||||
/// 120s is well within typical server GOAWAY windows (nginx: ~60s idle, envoy: ~60s).
|
||||
const MAX_H2_AGE: Duration = Duration::from_secs(120);
|
||||
|
||||
/// Identifies a unique backend endpoint.
|
||||
#[derive(Clone, Debug, Hash, Eq, PartialEq)]
|
||||
@@ -37,10 +38,13 @@ struct IdleH1 {
|
||||
idle_since: Instant,
|
||||
}
|
||||
|
||||
/// A pooled HTTP/2 sender (multiplexed, Clone-able).
|
||||
/// A pooled HTTP/2 sender (multiplexed, Clone-able) with a generation tag.
|
||||
struct PooledH2 {
|
||||
sender: http2::SendRequest<BoxBody<Bytes, hyper::Error>>,
|
||||
created_at: Instant,
|
||||
/// Unique generation ID. Connection drivers use this to only remove their OWN
|
||||
/// entry, preventing phantom eviction when multiple connections share the same key.
|
||||
generation: u64,
|
||||
}
|
||||
|
||||
/// Backend connection pool.
|
||||
@@ -49,6 +53,8 @@ pub struct ConnectionPool {
|
||||
h1_pool: Arc<DashMap<PoolKey, Vec<IdleH1>>>,
|
||||
/// HTTP/2 multiplexed connections indexed by backend key.
|
||||
h2_pool: Arc<DashMap<PoolKey, PooledH2>>,
|
||||
/// Monotonic generation counter for H2 pool entries.
|
||||
h2_generation: AtomicU64,
|
||||
/// Handle for the background eviction task.
|
||||
eviction_handle: Option<tokio::task::JoinHandle<()>>,
|
||||
}
|
||||
@@ -68,6 +74,7 @@ impl ConnectionPool {
|
||||
Self {
|
||||
h1_pool,
|
||||
h2_pool,
|
||||
h2_generation: AtomicU64::new(0),
|
||||
eviction_handle: Some(eviction_handle),
|
||||
}
|
||||
}
|
||||
@@ -81,7 +88,7 @@ impl ConnectionPool {
|
||||
while let Some(idle) = idles.pop() {
|
||||
// Check if the connection is still alive and ready
|
||||
if idle.idle_since.elapsed() < IDLE_TIMEOUT && idle.sender.is_ready() && !idle.sender.is_closed() {
|
||||
debug!("Pool hit (h1): {}:{}", key.host, key.port);
|
||||
// H1 pool hit — no logging on hot path
|
||||
return Some(idle.sender);
|
||||
}
|
||||
// Stale or closed — drop it
|
||||
@@ -114,40 +121,56 @@ impl ConnectionPool {
|
||||
|
||||
/// Try to get a cloned HTTP/2 sender for the given key.
|
||||
/// HTTP/2 senders are Clone-able (multiplexed), so we clone rather than remove.
|
||||
pub fn checkout_h2(&self, key: &PoolKey) -> Option<http2::SendRequest<BoxBody<Bytes, hyper::Error>>> {
|
||||
pub fn checkout_h2(&self, key: &PoolKey) -> Option<(http2::SendRequest<BoxBody<Bytes, hyper::Error>>, Duration)> {
|
||||
let entry = self.h2_pool.get(key)?;
|
||||
let pooled = entry.value();
|
||||
let age = pooled.created_at.elapsed();
|
||||
|
||||
// Check if the h2 connection is still alive and not too old
|
||||
if pooled.sender.is_closed() || pooled.created_at.elapsed() >= MAX_H2_AGE {
|
||||
if pooled.sender.is_closed() || age >= MAX_H2_AGE {
|
||||
drop(entry);
|
||||
self.h2_pool.remove(key);
|
||||
return None;
|
||||
}
|
||||
|
||||
if pooled.sender.is_ready() {
|
||||
debug!("Pool hit (h2): {}:{}", key.host, key.port);
|
||||
return Some(pooled.sender.clone());
|
||||
return Some((pooled.sender.clone(), age));
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Remove a dead HTTP/2 sender from the pool.
|
||||
/// Remove a dead HTTP/2 sender from the pool (unconditional).
|
||||
/// Called when `send_request` fails to prevent subsequent requests from reusing the stale sender.
|
||||
pub fn remove_h2(&self, key: &PoolKey) {
|
||||
self.h2_pool.remove(key);
|
||||
}
|
||||
|
||||
/// Register an HTTP/2 sender in the pool. Since h2 is multiplexed,
|
||||
/// only one sender per key is stored (it's Clone-able).
|
||||
pub fn register_h2(&self, key: PoolKey, sender: http2::SendRequest<BoxBody<Bytes, hyper::Error>>) {
|
||||
/// Remove an HTTP/2 sender ONLY if the current entry has the expected generation.
|
||||
/// This prevents phantom eviction: when multiple connections share the same key,
|
||||
/// an old connection's driver won't accidentally remove a newer connection's entry.
|
||||
pub fn remove_h2_if_generation(&self, key: &PoolKey, expected_gen: u64) {
|
||||
if let Some(entry) = self.h2_pool.get(key) {
|
||||
if entry.value().generation == expected_gen {
|
||||
drop(entry); // release DashMap ref before remove
|
||||
self.h2_pool.remove(key);
|
||||
}
|
||||
// else: a newer connection replaced ours — don't touch it
|
||||
}
|
||||
}
|
||||
|
||||
/// Register an HTTP/2 sender in the pool. Returns the generation ID for this entry.
|
||||
/// The caller should pass this generation to the connection driver so it can use
|
||||
/// `remove_h2_if_generation` instead of `remove_h2` to avoid phantom eviction.
|
||||
pub fn register_h2(&self, key: PoolKey, sender: http2::SendRequest<BoxBody<Bytes, hyper::Error>>) -> u64 {
|
||||
let gen = self.h2_generation.fetch_add(1, Ordering::Relaxed);
|
||||
if sender.is_closed() {
|
||||
return;
|
||||
return gen;
|
||||
}
|
||||
self.h2_pool.insert(key, PooledH2 {
|
||||
sender,
|
||||
created_at: Instant::now(),
|
||||
generation: gen,
|
||||
});
|
||||
gen
|
||||
}
|
||||
|
||||
/// Background eviction loop — runs every EVICTION_INTERVAL to remove stale connections.
|
||||
|
||||
@@ -11,25 +11,26 @@ use rustproxy_metrics::MetricsCollector;
|
||||
|
||||
/// Wraps any `http_body::Body` and counts data bytes passing through.
|
||||
///
|
||||
/// When the body is fully consumed or dropped, accumulated byte counts
|
||||
/// are reported to the `MetricsCollector`.
|
||||
/// Each chunk is reported to the `MetricsCollector` immediately so that
|
||||
/// the throughput tracker (sampled at 1 Hz) reflects real-time data flow.
|
||||
///
|
||||
/// The inner body is pinned on the heap to support `!Unpin` types like `hyper::body::Incoming`.
|
||||
pub struct CountingBody<B> {
|
||||
inner: Pin<Box<B>>,
|
||||
counted_bytes: AtomicU64,
|
||||
metrics: Arc<MetricsCollector>,
|
||||
route_id: Option<String>,
|
||||
source_ip: Option<String>,
|
||||
/// Whether we count bytes as "in" (request body) or "out" (response body).
|
||||
direction: Direction,
|
||||
/// Whether we've already reported the bytes (to avoid double-reporting on drop).
|
||||
reported: bool,
|
||||
/// Optional connection-level activity tracker. When set, poll_frame updates this
|
||||
/// to keep the idle watchdog alive during active body streaming (uploads/downloads).
|
||||
connection_activity: Option<Arc<AtomicU64>>,
|
||||
/// Start instant for computing elapsed ms for connection_activity.
|
||||
activity_start: Option<std::time::Instant>,
|
||||
/// Optional active-request counter. When set, CountingBody increments on creation
|
||||
/// and decrements on Drop, keeping the HTTP idle watchdog aware that a response
|
||||
/// body is still streaming (even after the request handler has returned).
|
||||
active_requests: Option<Arc<AtomicU64>>,
|
||||
}
|
||||
|
||||
/// Which direction the bytes flow.
|
||||
@@ -52,14 +53,13 @@ impl<B> CountingBody<B> {
|
||||
) -> Self {
|
||||
Self {
|
||||
inner: Box::pin(inner),
|
||||
counted_bytes: AtomicU64::new(0),
|
||||
metrics,
|
||||
route_id,
|
||||
source_ip,
|
||||
direction,
|
||||
reported: false,
|
||||
connection_activity: None,
|
||||
activity_start: None,
|
||||
active_requests: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -72,33 +72,27 @@ impl<B> CountingBody<B> {
|
||||
self
|
||||
}
|
||||
|
||||
/// Report accumulated bytes to the metrics collector.
|
||||
fn report(&mut self) {
|
||||
if self.reported {
|
||||
return;
|
||||
}
|
||||
self.reported = true;
|
||||
|
||||
let bytes = self.counted_bytes.load(Ordering::Relaxed);
|
||||
if bytes == 0 {
|
||||
return;
|
||||
}
|
||||
/// Set the active-request counter for the HTTP idle watchdog.
|
||||
/// CountingBody increments on creation and decrements on Drop, ensuring the
|
||||
/// idle watchdog sees an "active request" while the response body streams.
|
||||
pub fn with_active_requests(mut self, counter: Arc<AtomicU64>) -> Self {
|
||||
counter.fetch_add(1, Ordering::Relaxed);
|
||||
self.active_requests = Some(counter);
|
||||
self
|
||||
}
|
||||
|
||||
/// Report a chunk of bytes immediately to the metrics collector.
|
||||
#[inline]
|
||||
fn report_chunk(&self, len: u64) {
|
||||
let route_id = self.route_id.as_deref();
|
||||
let source_ip = self.source_ip.as_deref();
|
||||
match self.direction {
|
||||
Direction::In => self.metrics.record_bytes(bytes, 0, route_id, source_ip),
|
||||
Direction::Out => self.metrics.record_bytes(0, bytes, route_id, source_ip),
|
||||
Direction::In => self.metrics.record_bytes(len, 0, route_id, source_ip),
|
||||
Direction::Out => self.metrics.record_bytes(0, len, route_id, source_ip),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<B> Drop for CountingBody<B> {
|
||||
fn drop(&mut self) {
|
||||
self.report();
|
||||
}
|
||||
}
|
||||
|
||||
// CountingBody is Unpin because inner is Pin<Box<B>> (always Unpin).
|
||||
impl<B> Unpin for CountingBody<B> {}
|
||||
|
||||
@@ -118,7 +112,9 @@ where
|
||||
match this.inner.as_mut().poll_frame(cx) {
|
||||
Poll::Ready(Some(Ok(frame))) => {
|
||||
if let Some(data) = frame.data_ref() {
|
||||
this.counted_bytes.fetch_add(data.len() as u64, Ordering::Relaxed);
|
||||
let len = data.len() as u64;
|
||||
// Report bytes immediately so the 1 Hz throughput sampler sees them
|
||||
this.report_chunk(len);
|
||||
// Keep the connection-level idle watchdog alive during body streaming
|
||||
if let (Some(activity), Some(start)) = (&this.connection_activity, &this.activity_start) {
|
||||
activity.store(start.elapsed().as_millis() as u64, Ordering::Relaxed);
|
||||
@@ -127,11 +123,7 @@ where
|
||||
Poll::Ready(Some(Ok(frame)))
|
||||
}
|
||||
Poll::Ready(Some(Err(e))) => Poll::Ready(Some(Err(e))),
|
||||
Poll::Ready(None) => {
|
||||
// Body is fully consumed — report now
|
||||
this.report();
|
||||
Poll::Ready(None)
|
||||
}
|
||||
Poll::Ready(None) => Poll::Ready(None),
|
||||
Poll::Pending => Poll::Pending,
|
||||
}
|
||||
}
|
||||
@@ -144,3 +136,13 @@ where
|
||||
self.inner.size_hint()
|
||||
}
|
||||
}
|
||||
|
||||
impl<B> Drop for CountingBody<B> {
|
||||
fn drop(&mut self) {
|
||||
// Decrement the active-request counter so the HTTP idle watchdog
|
||||
// knows this response body is no longer streaming.
|
||||
if let Some(ref counter) = self.active_requests {
|
||||
counter.fetch_sub(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ pub mod protocol_cache;
|
||||
pub mod proxy_service;
|
||||
pub mod request_filter;
|
||||
pub mod response_filter;
|
||||
pub mod shutdown_on_drop;
|
||||
pub mod template;
|
||||
pub mod upstream_selector;
|
||||
|
||||
|
||||
@@ -39,6 +39,10 @@ use crate::upstream_selector::UpstreamSelector;
|
||||
struct ConnActivity {
|
||||
last_activity: Arc<AtomicU64>,
|
||||
start: std::time::Instant,
|
||||
/// Active-request counter from handle_io's idle watchdog. When set, CountingBody
|
||||
/// increments on creation and decrements on Drop, keeping the watchdog aware that
|
||||
/// a response body is still streaming after the request handler has returned.
|
||||
active_requests: Option<Arc<AtomicU64>>,
|
||||
}
|
||||
|
||||
/// Default upstream connect timeout (30 seconds).
|
||||
@@ -105,6 +109,24 @@ impl tokio::io::AsyncWrite for BackendStream {
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_write_vectored(
|
||||
self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
bufs: &[std::io::IoSlice<'_>],
|
||||
) -> Poll<std::io::Result<usize>> {
|
||||
match self.get_mut() {
|
||||
BackendStream::Plain(s) => Pin::new(s).poll_write_vectored(cx, bufs),
|
||||
BackendStream::Tls(s) => Pin::new(s).poll_write_vectored(cx, bufs),
|
||||
}
|
||||
}
|
||||
|
||||
fn is_write_vectored(&self) -> bool {
|
||||
match self {
|
||||
BackendStream::Plain(s) => s.is_write_vectored(),
|
||||
BackendStream::Tls(s) => s.is_write_vectored(),
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
|
||||
match self.get_mut() {
|
||||
BackendStream::Plain(s) => Pin::new(s).poll_flush(cx),
|
||||
@@ -302,7 +324,7 @@ impl HttpProxyService {
|
||||
let cn = cancel_inner.clone();
|
||||
let la = Arc::clone(&la_inner);
|
||||
let st = start;
|
||||
let ca = ConnActivity { last_activity: Arc::clone(&la_inner), start };
|
||||
let ca = ConnActivity { last_activity: Arc::clone(&la_inner), start, active_requests: Some(Arc::clone(&ar_inner)) };
|
||||
async move {
|
||||
let result = svc.handle_request(req, peer, port, cn, ca).await;
|
||||
// Mark request end — update activity timestamp before guard drops
|
||||
@@ -395,11 +417,19 @@ impl HttpProxyService {
|
||||
let path = req.uri().path().to_string();
|
||||
let method = req.method().clone();
|
||||
|
||||
// Extract headers for matching
|
||||
let headers: HashMap<String, String> = req.headers()
|
||||
.iter()
|
||||
.map(|(k, v)| (k.to_string(), v.to_str().unwrap_or("").to_string()))
|
||||
.collect();
|
||||
// Extract headers for matching — only allocate the HashMap if any route
|
||||
// on this port actually uses header matching. Most deployments don't,
|
||||
// so this saves ~20-30 String allocations per request.
|
||||
let current_rm = self.route_manager.load();
|
||||
let needs_headers = current_rm.any_route_has_headers(port);
|
||||
let headers: Option<HashMap<String, String>> = if needs_headers {
|
||||
Some(req.headers()
|
||||
.iter()
|
||||
.map(|(k, v)| (k.to_string(), v.to_str().unwrap_or("").to_string()))
|
||||
.collect())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
debug!("HTTP {} {} (host: {:?}) from {}", method, path, host, peer_addr);
|
||||
|
||||
@@ -410,19 +440,19 @@ impl HttpProxyService {
|
||||
}
|
||||
}
|
||||
|
||||
// Match route
|
||||
// Match route (current_rm already loaded above for headers check)
|
||||
let ip_string = peer_addr.ip().to_string();
|
||||
let ctx = rustproxy_routing::MatchContext {
|
||||
port,
|
||||
domain: host.as_deref(),
|
||||
path: Some(&path),
|
||||
client_ip: Some(&peer_addr.ip().to_string()),
|
||||
client_ip: Some(&ip_string),
|
||||
tls_version: None,
|
||||
headers: Some(&headers),
|
||||
headers: headers.as_ref(),
|
||||
is_tls: false,
|
||||
protocol: Some("http"),
|
||||
};
|
||||
|
||||
let current_rm = self.route_manager.load();
|
||||
let route_match = match current_rm.find_route(&ctx) {
|
||||
Some(rm) => rm,
|
||||
None => {
|
||||
@@ -432,7 +462,7 @@ impl HttpProxyService {
|
||||
};
|
||||
|
||||
let route_id = route_match.route.id.as_deref();
|
||||
let ip_str = peer_addr.ip().to_string();
|
||||
let ip_str = ip_string; // reuse from above (avoid redundant to_string())
|
||||
self.metrics.record_http_request();
|
||||
|
||||
// Apply request filters (IP check, rate limiting, auth)
|
||||
@@ -647,17 +677,48 @@ impl HttpProxyService {
|
||||
h2: use_h2,
|
||||
};
|
||||
|
||||
// H2 pool checkout (H2 senders are Clone and multiplexed)
|
||||
if use_h2 {
|
||||
if let Some(sender) = self.connection_pool.checkout_h2(&pool_key) {
|
||||
self.metrics.backend_pool_hit(&upstream_key);
|
||||
self.metrics.set_backend_protocol(&upstream_key, "h2");
|
||||
let result = self.forward_h2_pooled(
|
||||
sender, parts, body, upstream_headers, &upstream_path,
|
||||
route_match.route, route_id, &ip_str, &pool_key, domain_str, &conn_activity,
|
||||
).await;
|
||||
self.upstream_selector.connection_ended(&upstream_key);
|
||||
return result;
|
||||
// H2 pool checkout — only for bodyless requests (GET/HEAD/DELETE).
|
||||
//
|
||||
// WORKAROUND: Requests with bodies (POST/PUT uploads) always get fresh H2
|
||||
// connections. Reusing a pooled H2 connection after a large upload can stall
|
||||
// forever due to depleted connection-level flow control windows. The h2 crate
|
||||
// has no stall/timeout detection (https://github.com/hyperium/hyper/issues/2899),
|
||||
// and Go/nginx HTTP/2 servers have known issues with connection-level window
|
||||
// replenishment after large transfers (https://github.com/golang/go/issues/16481,
|
||||
// https://github.com/golang/go/issues/56558). A fresh connection guarantees
|
||||
// clean flow control state. The overhead is ~3-5ms for TLS+H2 handshake.
|
||||
//
|
||||
// TODO: Revisit once h2 crate adds flow control stall detection, or once
|
||||
// Go/nginx H2 connection-level window handling is confirmed reliable.
|
||||
if use_h2 && body.is_end_stream() {
|
||||
if let Some((mut sender, age)) = self.connection_pool.checkout_h2(&pool_key) {
|
||||
match tokio::time::timeout(
|
||||
std::time::Duration::from_millis(500),
|
||||
sender.ready(),
|
||||
).await {
|
||||
Ok(Ok(())) => {
|
||||
self.metrics.backend_pool_hit(&upstream_key);
|
||||
self.metrics.set_backend_protocol(&upstream_key, "h2");
|
||||
let result = self.forward_h2_pooled(
|
||||
sender, parts, body, upstream_headers, &upstream_path,
|
||||
route_match.route, route_id, &ip_str, &pool_key, domain_str, &conn_activity,
|
||||
).await;
|
||||
self.upstream_selector.connection_ended(&upstream_key);
|
||||
return result;
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
warn!(backend = %upstream_key, age_secs = age.as_secs(),
|
||||
"Pooled H2 sender failed ready check (GOAWAY/RST): {}, evicting", e);
|
||||
self.connection_pool.remove_h2(&pool_key);
|
||||
// Fall through to fresh connection
|
||||
}
|
||||
Err(_) => {
|
||||
warn!(backend = %upstream_key, age_secs = age.as_secs(),
|
||||
"Pooled H2 sender ready check timed out (500ms), evicting");
|
||||
self.connection_pool.remove_h2(&pool_key);
|
||||
// Fall through to fresh connection
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -955,7 +1016,7 @@ impl HttpProxyService {
|
||||
) = match tokio::time::timeout(self.connect_timeout, h2_builder.handshake(io)).await {
|
||||
Ok(Ok(h)) => h,
|
||||
Ok(Err(e)) => {
|
||||
error!(backend = %backend_key, domain = %domain, error = %e, "Backend H2 handshake failed");
|
||||
error!(backend = %backend_key, domain = %domain, error = %e, error_debug = ?e, "Backend H2 handshake failed");
|
||||
self.metrics.backend_handshake_error(&backend_key);
|
||||
return Ok(error_response(StatusCode::BAD_GATEWAY, "Backend H2 handshake failed"));
|
||||
}
|
||||
@@ -966,17 +1027,35 @@ impl HttpProxyService {
|
||||
}
|
||||
};
|
||||
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = conn.await {
|
||||
debug!("HTTP/2 upstream connection error: {}", e);
|
||||
}
|
||||
});
|
||||
// Shared generation ID: driver reads it after registration sets it.
|
||||
// Uses u64::MAX as sentinel for "not yet registered" (driver waits/skips eviction).
|
||||
let gen_holder = Arc::new(std::sync::atomic::AtomicU64::new(u64::MAX));
|
||||
|
||||
// Clone sender for potential pool registration; register only after first request succeeds
|
||||
// Spawn the H2 connection driver; evict from pool on exit using generation-tagged
|
||||
// removal to prevent phantom eviction when multiple connections share the same key.
|
||||
{
|
||||
let pool = Arc::clone(&self.connection_pool);
|
||||
let key = pool_key.clone();
|
||||
let gen = Arc::clone(&gen_holder);
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = conn.await {
|
||||
warn!("HTTP/2 upstream connection error: {} ({:?})", e, e);
|
||||
}
|
||||
let g = gen.load(std::sync::atomic::Ordering::Relaxed);
|
||||
if g != u64::MAX {
|
||||
pool.remove_h2_if_generation(&key, g);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Only pool the H2 connection if the request had no body.
|
||||
// Requests with bodies (uploads) deplete connection-level flow control windows.
|
||||
let request_had_body = !body.is_end_stream();
|
||||
let sender_for_pool = sender.clone();
|
||||
let result = self.forward_h2_with_sender(sender, parts, body, upstream_headers, upstream_path, route, route_id, source_ip, Some(pool_key), domain, conn_activity).await;
|
||||
if matches!(&result, Ok(ref resp) if resp.status() != StatusCode::BAD_GATEWAY) {
|
||||
self.connection_pool.register_h2(pool_key.clone(), sender_for_pool);
|
||||
if !request_had_body && matches!(&result, Ok(ref resp) if resp.status() != StatusCode::BAD_GATEWAY) {
|
||||
let g = self.connection_pool.register_h2(pool_key.clone(), sender_for_pool);
|
||||
gen_holder.store(g, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
result
|
||||
}
|
||||
@@ -1098,7 +1177,7 @@ impl HttpProxyService {
|
||||
) = match tokio::time::timeout(self.connect_timeout, h2_builder.handshake(io)).await {
|
||||
Ok(Ok(h)) => h,
|
||||
Ok(Err(e)) => {
|
||||
error!(backend = %backend_key, domain = %domain, error = %e, "H2 retry: handshake failed");
|
||||
error!(backend = %backend_key, domain = %domain, error = %e, error_debug = ?e, "H2 retry: handshake failed");
|
||||
self.metrics.backend_handshake_error(&backend_key);
|
||||
self.metrics.backend_connection_closed(&backend_key);
|
||||
return Ok(error_response(StatusCode::BAD_GATEWAY, "Backend H2 retry handshake failed"));
|
||||
@@ -1111,11 +1190,22 @@ impl HttpProxyService {
|
||||
}
|
||||
};
|
||||
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = conn.await {
|
||||
debug!("H2 retry: upstream connection error: {}", e);
|
||||
}
|
||||
});
|
||||
// Spawn the H2 connection driver with generation-tagged eviction.
|
||||
let gen_holder = Arc::new(std::sync::atomic::AtomicU64::new(u64::MAX));
|
||||
{
|
||||
let pool = Arc::clone(&self.connection_pool);
|
||||
let key = pool_key.clone();
|
||||
let gen = Arc::clone(&gen_holder);
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = conn.await {
|
||||
warn!("H2 retry: upstream connection error: {} ({:?})", e, e);
|
||||
}
|
||||
let g = gen.load(std::sync::atomic::Ordering::Relaxed);
|
||||
if g != u64::MAX {
|
||||
pool.remove_h2_if_generation(&key, g);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Build request with empty body using absolute URI for H2 pseudo-headers
|
||||
let scheme = if pool_key.use_tls { "https" } else { "http" };
|
||||
@@ -1141,7 +1231,8 @@ impl HttpProxyService {
|
||||
match sender.send_request(upstream_req).await {
|
||||
Ok(resp) => {
|
||||
// Register in pool only after request succeeds
|
||||
self.connection_pool.register_h2(pool_key.clone(), sender);
|
||||
let g = self.connection_pool.register_h2(pool_key.clone(), sender);
|
||||
gen_holder.store(g, std::sync::atomic::Ordering::Relaxed);
|
||||
let result = self.build_streaming_response(resp, route, route_id, source_ip, conn_activity).await;
|
||||
// Close the fresh backend connection (opened above)
|
||||
self.metrics.backend_connection_closed(&backend_key);
|
||||
@@ -1234,11 +1325,22 @@ impl HttpProxyService {
|
||||
}
|
||||
}
|
||||
Ok(Ok((mut sender, conn))) => {
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = conn.await {
|
||||
debug!("HTTP/2 upstream connection error: {}", e);
|
||||
}
|
||||
});
|
||||
// Spawn the H2 connection driver with generation-tagged eviction.
|
||||
let gen_holder = Arc::new(std::sync::atomic::AtomicU64::new(u64::MAX));
|
||||
{
|
||||
let pool = Arc::clone(&self.connection_pool);
|
||||
let key = pool_key.clone();
|
||||
let gen = Arc::clone(&gen_holder);
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = conn.await {
|
||||
warn!("HTTP/2 upstream connection error: {} ({:?})", e, e);
|
||||
}
|
||||
let g = gen.load(std::sync::atomic::Ordering::Relaxed);
|
||||
if g != u64::MAX {
|
||||
pool.remove_h2_if_generation(&key, g);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Save retry state before consuming parts/body (for bodyless requests like GET)
|
||||
// Clone BEFORE removing Host — H1 fallback needs Host header
|
||||
@@ -1277,8 +1379,12 @@ impl HttpProxyService {
|
||||
|
||||
match sender.send_request(upstream_req).await {
|
||||
Ok(upstream_response) => {
|
||||
// H2 works! Register sender in pool for multiplexed reuse
|
||||
self.connection_pool.register_h2(pool_key.clone(), sender);
|
||||
// Only pool after bodyless requests — uploads deplete connection-level
|
||||
// flow control windows (see comment at pool checkout above).
|
||||
if retry_state.is_some() {
|
||||
let g = self.connection_pool.register_h2(pool_key.clone(), sender);
|
||||
gen_holder.store(g, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
self.build_streaming_response(upstream_response, route, route_id, source_ip, conn_activity).await
|
||||
}
|
||||
Err(e) => {
|
||||
@@ -1289,6 +1395,7 @@ impl HttpProxyService {
|
||||
backend = %bk,
|
||||
domain = %domain,
|
||||
error = %e,
|
||||
error_debug = ?e,
|
||||
"Auto-detect: H2 request failed, falling back to H1"
|
||||
);
|
||||
self.metrics.backend_h2_failure(&bk);
|
||||
@@ -1546,11 +1653,11 @@ impl HttpProxyService {
|
||||
// Evict the dead sender so subsequent requests get fresh connections
|
||||
if let Some(key) = pool_key {
|
||||
let bk = format!("{}:{}", key.host, key.port);
|
||||
error!(backend = %bk, domain = %domain, error = %e, "Backend H2 request failed");
|
||||
error!(backend = %bk, domain = %domain, error = %e, error_debug = ?e, "Backend H2 request failed");
|
||||
self.metrics.backend_request_error(&bk);
|
||||
self.connection_pool.remove_h2(key);
|
||||
} else {
|
||||
error!(domain = %domain, error = %e, "Backend H2 request failed");
|
||||
error!(domain = %domain, error = %e, error_debug = ?e, "Backend H2 request failed");
|
||||
}
|
||||
return Ok(error_response(StatusCode::BAD_GATEWAY, "Backend H2 request failed"));
|
||||
}
|
||||
@@ -1578,6 +1685,19 @@ impl HttpProxyService {
|
||||
|
||||
if let Some(headers) = response.headers_mut() {
|
||||
*headers = resp_parts.headers;
|
||||
|
||||
// Strip hop-by-hop headers from the backend response.
|
||||
// RFC 9113 §8.2.2 forbids connection-specific headers in HTTP/2 responses;
|
||||
// forwarding them from an H1 backend can cause H2 stream resets.
|
||||
// Mirrors the request-path stripping at the forward methods above.
|
||||
headers.remove("connection");
|
||||
headers.remove("keep-alive");
|
||||
headers.remove("proxy-connection");
|
||||
headers.remove("transfer-encoding");
|
||||
headers.remove("te");
|
||||
headers.remove("trailer");
|
||||
// Note: "upgrade" is intentionally kept — needed for WebSocket 101 responses.
|
||||
|
||||
ResponseFilter::apply_headers(route, headers, None);
|
||||
}
|
||||
|
||||
@@ -1592,6 +1712,15 @@ impl HttpProxyService {
|
||||
Direction::Out,
|
||||
).with_connection_activity(Arc::clone(&conn_activity.last_activity), conn_activity.start);
|
||||
|
||||
// Keep active_requests > 0 while the response body streams, so the idle
|
||||
// watchdog doesn't kill the connection mid-transfer (e.g. during git fetch).
|
||||
// CountingBody increments on creation and decrements on Drop.
|
||||
let counting_body = if let Some(ref ar) = conn_activity.active_requests {
|
||||
counting_body.with_active_requests(Arc::clone(ar))
|
||||
} else {
|
||||
counting_body
|
||||
};
|
||||
|
||||
let body: BoxBody<Bytes, hyper::Error> = BoxBody::new(counting_body);
|
||||
|
||||
Ok(response.body(body).unwrap())
|
||||
@@ -1913,59 +2042,89 @@ impl HttpProxyService {
|
||||
let last_activity = Arc::new(AtomicU64::new(0));
|
||||
let start = std::time::Instant::now();
|
||||
|
||||
// Per-connection cancellation token: the watchdog cancels this instead of
|
||||
// aborting tasks, so the copy loops can shut down gracefully (TLS close_notify).
|
||||
let ws_cancel = CancellationToken::new();
|
||||
|
||||
// For H2 WebSocket: also update the connection-level activity tracker
|
||||
// to prevent the idle watchdog from killing the H2 connection
|
||||
let conn_act_c2u = conn_activity.as_ref().map(|ca| (Arc::clone(&ca.last_activity), ca.start));
|
||||
let conn_act_u2c = conn_activity.as_ref().map(|ca| (Arc::clone(&ca.last_activity), ca.start));
|
||||
|
||||
let la1 = Arc::clone(&last_activity);
|
||||
let metrics_c2u = Arc::clone(&metrics);
|
||||
let route_c2u = route_id_owned.clone();
|
||||
let ip_c2u = source_ip_owned.clone();
|
||||
let wsc1 = ws_cancel.clone();
|
||||
let c2u = tokio::spawn(async move {
|
||||
let mut buf = vec![0u8; 65536];
|
||||
let mut total = 0u64;
|
||||
loop {
|
||||
let n = match cr.read(&mut buf).await {
|
||||
Ok(0) | Err(_) => break,
|
||||
Ok(n) => n,
|
||||
let n = tokio::select! {
|
||||
result = cr.read(&mut buf) => match result {
|
||||
Ok(0) | Err(_) => break,
|
||||
Ok(n) => n,
|
||||
},
|
||||
_ = wsc1.cancelled() => break,
|
||||
};
|
||||
if uw.write_all(&buf[..n]).await.is_err() {
|
||||
break;
|
||||
}
|
||||
total += n as u64;
|
||||
metrics_c2u.record_bytes(n as u64, 0, route_c2u.as_deref(), Some(&ip_c2u));
|
||||
la1.store(start.elapsed().as_millis() as u64, Ordering::Relaxed);
|
||||
if let Some((ref ca, ca_start)) = conn_act_c2u {
|
||||
ca.store(ca_start.elapsed().as_millis() as u64, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
let _ = uw.shutdown().await;
|
||||
// Graceful shutdown with timeout (sends TLS close_notify / TCP FIN)
|
||||
let _ = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(2),
|
||||
uw.shutdown(),
|
||||
).await;
|
||||
total
|
||||
});
|
||||
|
||||
let la2 = Arc::clone(&last_activity);
|
||||
let metrics_u2c = Arc::clone(&metrics);
|
||||
let route_u2c = route_id_owned.clone();
|
||||
let ip_u2c = source_ip_owned.clone();
|
||||
let wsc2 = ws_cancel.clone();
|
||||
let u2c = tokio::spawn(async move {
|
||||
let mut buf = vec![0u8; 65536];
|
||||
let mut total = 0u64;
|
||||
loop {
|
||||
let n = match ur.read(&mut buf).await {
|
||||
Ok(0) | Err(_) => break,
|
||||
Ok(n) => n,
|
||||
let n = tokio::select! {
|
||||
result = ur.read(&mut buf) => match result {
|
||||
Ok(0) | Err(_) => break,
|
||||
Ok(n) => n,
|
||||
},
|
||||
_ = wsc2.cancelled() => break,
|
||||
};
|
||||
if cw.write_all(&buf[..n]).await.is_err() {
|
||||
break;
|
||||
}
|
||||
total += n as u64;
|
||||
metrics_u2c.record_bytes(0, n as u64, route_u2c.as_deref(), Some(&ip_u2c));
|
||||
la2.store(start.elapsed().as_millis() as u64, Ordering::Relaxed);
|
||||
if let Some((ref ca, ca_start)) = conn_act_u2c {
|
||||
ca.store(ca_start.elapsed().as_millis() as u64, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
let _ = cw.shutdown().await;
|
||||
// Graceful shutdown with timeout (sends TLS close_notify / TCP FIN)
|
||||
let _ = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(2),
|
||||
cw.shutdown(),
|
||||
).await;
|
||||
total
|
||||
});
|
||||
|
||||
// Watchdog: monitors inactivity, max lifetime, and cancellation
|
||||
// Watchdog: monitors inactivity, max lifetime, and cancellation.
|
||||
// First cancels the per-connection token for graceful shutdown (close_notify/FIN),
|
||||
// then falls back to abort if the tasks are stuck (e.g. on a blocked write_all).
|
||||
let la_watch = Arc::clone(&last_activity);
|
||||
let c2u_handle = c2u.abort_handle();
|
||||
let u2c_handle = u2c.abort_handle();
|
||||
let c2u_abort = c2u.abort_handle();
|
||||
let u2c_abort = u2c.abort_handle();
|
||||
let inactivity_timeout = ws_inactivity_timeout;
|
||||
let max_lifetime = ws_max_lifetime;
|
||||
|
||||
@@ -1977,8 +2136,6 @@ impl HttpProxyService {
|
||||
_ = tokio::time::sleep(check_interval) => {}
|
||||
_ = cancel.cancelled() => {
|
||||
debug!("WebSocket tunnel cancelled by shutdown");
|
||||
c2u_handle.abort();
|
||||
u2c_handle.abort();
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -1986,8 +2143,6 @@ impl HttpProxyService {
|
||||
// Check max lifetime
|
||||
if start.elapsed() >= max_lifetime {
|
||||
debug!("WebSocket tunnel exceeded max lifetime, closing");
|
||||
c2u_handle.abort();
|
||||
u2c_handle.abort();
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -1997,13 +2152,18 @@ impl HttpProxyService {
|
||||
let elapsed_since_activity = start.elapsed().as_millis() as u64 - current;
|
||||
if elapsed_since_activity >= inactivity_timeout.as_millis() as u64 {
|
||||
debug!("WebSocket tunnel inactive for {}ms, closing", elapsed_since_activity);
|
||||
c2u_handle.abort();
|
||||
u2c_handle.abort();
|
||||
break;
|
||||
}
|
||||
}
|
||||
last_seen = current;
|
||||
}
|
||||
// Phase 1: Signal copy loops to exit gracefully (allows close_notify/FIN)
|
||||
ws_cancel.cancel();
|
||||
// Phase 2: Wait for graceful shutdown (2s shutdown timeout + 2s margin)
|
||||
tokio::time::sleep(std::time::Duration::from_secs(4)).await;
|
||||
// Phase 3: Force-abort if still stuck (e.g. blocked on write_all)
|
||||
c2u_abort.abort();
|
||||
u2c_abort.abort();
|
||||
});
|
||||
|
||||
let bytes_in = c2u.await.unwrap_or(0);
|
||||
@@ -2013,9 +2173,7 @@ impl HttpProxyService {
|
||||
debug!("WebSocket tunnel closed: {} bytes in, {} bytes out", bytes_in, bytes_out);
|
||||
|
||||
upstream_selector.connection_ended(&upstream_key_owned);
|
||||
if let Some(ref rid) = route_id_owned {
|
||||
metrics.record_bytes(bytes_in, bytes_out, Some(rid.as_str()), Some(&source_ip_owned));
|
||||
}
|
||||
// Bytes already reported per-chunk in the copy loops above
|
||||
});
|
||||
|
||||
let body: BoxBody<Bytes, hyper::Error> = BoxBody::new(
|
||||
|
||||
102
rust/crates/rustproxy-http/src/shutdown_on_drop.rs
Normal file
102
rust/crates/rustproxy-http/src/shutdown_on_drop.rs
Normal file
@@ -0,0 +1,102 @@
|
||||
//! Wrapper that ensures TLS close_notify is sent when the stream is dropped.
|
||||
//!
|
||||
//! When hyper drops an HTTP connection (backend error, timeout, normal H2 close),
|
||||
//! the underlying TLS stream is dropped WITHOUT `shutdown()`. tokio-rustls cannot
|
||||
//! send `close_notify` in Drop (requires async). This wrapper tracks whether
|
||||
//! `poll_shutdown` was called and, if not, spawns a background task to send it.
|
||||
|
||||
use std::io;
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
|
||||
use tokio::io::{AsyncRead, AsyncWrite, ReadBuf};
|
||||
|
||||
/// Wraps an AsyncRead+AsyncWrite stream and ensures `shutdown()` is called when
|
||||
/// dropped, even if the caller (e.g. hyper) doesn't explicitly shut down.
|
||||
///
|
||||
/// This guarantees TLS `close_notify` is sent for TLS-wrapped streams, preventing
|
||||
/// "GnuTLS recv error (-110): The TLS connection was non-properly terminated" errors.
|
||||
pub struct ShutdownOnDrop<S: AsyncRead + AsyncWrite + Unpin + Send + 'static> {
|
||||
inner: Option<S>,
|
||||
shutdown_called: bool,
|
||||
}
|
||||
|
||||
impl<S: AsyncRead + AsyncWrite + Unpin + Send + 'static> ShutdownOnDrop<S> {
|
||||
/// Create a new wrapper around the given stream.
|
||||
pub fn new(stream: S) -> Self {
|
||||
Self {
|
||||
inner: Some(stream),
|
||||
shutdown_called: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<S: AsyncRead + AsyncWrite + Unpin + Send + 'static> AsyncRead for ShutdownOnDrop<S> {
|
||||
fn poll_read(
|
||||
self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
buf: &mut ReadBuf<'_>,
|
||||
) -> Poll<io::Result<()>> {
|
||||
Pin::new(self.get_mut().inner.as_mut().unwrap()).poll_read(cx, buf)
|
||||
}
|
||||
}
|
||||
|
||||
impl<S: AsyncRead + AsyncWrite + Unpin + Send + 'static> AsyncWrite for ShutdownOnDrop<S> {
|
||||
fn poll_write(
|
||||
self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
buf: &[u8],
|
||||
) -> Poll<io::Result<usize>> {
|
||||
Pin::new(self.get_mut().inner.as_mut().unwrap()).poll_write(cx, buf)
|
||||
}
|
||||
|
||||
fn poll_write_vectored(
|
||||
self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
bufs: &[io::IoSlice<'_>],
|
||||
) -> Poll<io::Result<usize>> {
|
||||
Pin::new(self.get_mut().inner.as_mut().unwrap()).poll_write_vectored(cx, bufs)
|
||||
}
|
||||
|
||||
fn is_write_vectored(&self) -> bool {
|
||||
self.inner.as_ref().unwrap().is_write_vectored()
|
||||
}
|
||||
|
||||
fn poll_flush(
|
||||
self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
) -> Poll<io::Result<()>> {
|
||||
Pin::new(self.get_mut().inner.as_mut().unwrap()).poll_flush(cx)
|
||||
}
|
||||
|
||||
fn poll_shutdown(
|
||||
self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
) -> Poll<io::Result<()>> {
|
||||
let this = self.get_mut();
|
||||
let result = Pin::new(this.inner.as_mut().unwrap()).poll_shutdown(cx);
|
||||
if result.is_ready() {
|
||||
this.shutdown_called = true;
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
impl<S: AsyncRead + AsyncWrite + Unpin + Send + 'static> Drop for ShutdownOnDrop<S> {
|
||||
fn drop(&mut self) {
|
||||
// If shutdown was already called (hyper closed properly), nothing to do.
|
||||
// If not (hyper dropped without shutdown — e.g. H2 close, error, timeout),
|
||||
// spawn a background task to send close_notify / TCP FIN.
|
||||
if !self.shutdown_called {
|
||||
if let Some(mut stream) = self.inner.take() {
|
||||
tokio::spawn(async move {
|
||||
let _ = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(2),
|
||||
tokio::io::AsyncWriteExt::shutdown(&mut stream),
|
||||
).await;
|
||||
// stream is dropped here — all resources freed
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -266,44 +266,67 @@ impl MetricsCollector {
|
||||
self.global_pending_tp_in.fetch_add(bytes_in, Ordering::Relaxed);
|
||||
self.global_pending_tp_out.fetch_add(bytes_out, Ordering::Relaxed);
|
||||
|
||||
// Per-route tracking: use get() first (zero-alloc fast path for existing entries),
|
||||
// fall back to entry() with to_string() only on the rare first-chunk miss.
|
||||
if let Some(route_id) = route_id {
|
||||
self.route_bytes_in
|
||||
.entry(route_id.to_string())
|
||||
.or_insert_with(|| AtomicU64::new(0))
|
||||
.fetch_add(bytes_in, Ordering::Relaxed);
|
||||
self.route_bytes_out
|
||||
.entry(route_id.to_string())
|
||||
.or_insert_with(|| AtomicU64::new(0))
|
||||
.fetch_add(bytes_out, Ordering::Relaxed);
|
||||
if let Some(counter) = self.route_bytes_in.get(route_id) {
|
||||
counter.fetch_add(bytes_in, Ordering::Relaxed);
|
||||
} else {
|
||||
self.route_bytes_in.entry(route_id.to_string())
|
||||
.or_insert_with(|| AtomicU64::new(0))
|
||||
.fetch_add(bytes_in, Ordering::Relaxed);
|
||||
}
|
||||
if let Some(counter) = self.route_bytes_out.get(route_id) {
|
||||
counter.fetch_add(bytes_out, Ordering::Relaxed);
|
||||
} else {
|
||||
self.route_bytes_out.entry(route_id.to_string())
|
||||
.or_insert_with(|| AtomicU64::new(0))
|
||||
.fetch_add(bytes_out, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
// Accumulate into per-route pending throughput counters (lock-free)
|
||||
let entry = self.route_pending_tp
|
||||
.entry(route_id.to_string())
|
||||
.or_insert_with(|| (AtomicU64::new(0), AtomicU64::new(0)));
|
||||
entry.0.fetch_add(bytes_in, Ordering::Relaxed);
|
||||
entry.1.fetch_add(bytes_out, Ordering::Relaxed);
|
||||
if let Some(entry) = self.route_pending_tp.get(route_id) {
|
||||
entry.0.fetch_add(bytes_in, Ordering::Relaxed);
|
||||
entry.1.fetch_add(bytes_out, Ordering::Relaxed);
|
||||
} else {
|
||||
let entry = self.route_pending_tp.entry(route_id.to_string())
|
||||
.or_insert_with(|| (AtomicU64::new(0), AtomicU64::new(0)));
|
||||
entry.0.fetch_add(bytes_in, Ordering::Relaxed);
|
||||
entry.1.fetch_add(bytes_out, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
// Per-IP tracking: same get()-first pattern to avoid String allocation on hot path.
|
||||
if let Some(ip) = source_ip {
|
||||
// Only record per-IP stats if the IP still has active connections.
|
||||
// This prevents orphaned entries when record_bytes races with
|
||||
// connection_closed (which evicts all per-IP data on last close).
|
||||
if self.ip_connections.contains_key(ip) {
|
||||
self.ip_bytes_in
|
||||
.entry(ip.to_string())
|
||||
.or_insert_with(|| AtomicU64::new(0))
|
||||
.fetch_add(bytes_in, Ordering::Relaxed);
|
||||
self.ip_bytes_out
|
||||
.entry(ip.to_string())
|
||||
.or_insert_with(|| AtomicU64::new(0))
|
||||
.fetch_add(bytes_out, Ordering::Relaxed);
|
||||
if let Some(counter) = self.ip_bytes_in.get(ip) {
|
||||
counter.fetch_add(bytes_in, Ordering::Relaxed);
|
||||
} else {
|
||||
self.ip_bytes_in.entry(ip.to_string())
|
||||
.or_insert_with(|| AtomicU64::new(0))
|
||||
.fetch_add(bytes_in, Ordering::Relaxed);
|
||||
}
|
||||
if let Some(counter) = self.ip_bytes_out.get(ip) {
|
||||
counter.fetch_add(bytes_out, Ordering::Relaxed);
|
||||
} else {
|
||||
self.ip_bytes_out.entry(ip.to_string())
|
||||
.or_insert_with(|| AtomicU64::new(0))
|
||||
.fetch_add(bytes_out, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
// Accumulate into per-IP pending throughput counters (lock-free)
|
||||
let entry = self.ip_pending_tp
|
||||
.entry(ip.to_string())
|
||||
.or_insert_with(|| (AtomicU64::new(0), AtomicU64::new(0)));
|
||||
entry.0.fetch_add(bytes_in, Ordering::Relaxed);
|
||||
entry.1.fetch_add(bytes_out, Ordering::Relaxed);
|
||||
if let Some(entry) = self.ip_pending_tp.get(ip) {
|
||||
entry.0.fetch_add(bytes_in, Ordering::Relaxed);
|
||||
entry.1.fetch_add(bytes_out, Ordering::Relaxed);
|
||||
} else {
|
||||
let entry = self.ip_pending_tp.entry(ip.to_string())
|
||||
.or_insert_with(|| (AtomicU64::new(0), AtomicU64::new(0)));
|
||||
entry.0.fetch_add(bytes_in, Ordering::Relaxed);
|
||||
entry.1.fetch_add(bytes_out, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -97,16 +97,25 @@ pub async fn forward_bidirectional_with_timeouts(
|
||||
let last_activity = Arc::new(AtomicU64::new(0));
|
||||
let start = std::time::Instant::now();
|
||||
|
||||
// Per-connection cancellation token: the watchdog cancels this instead of
|
||||
// aborting tasks, so the copy loops can shut down gracefully (TCP FIN instead
|
||||
// of RST, TLS close_notify if the stream is TLS-wrapped).
|
||||
let conn_cancel = CancellationToken::new();
|
||||
|
||||
let la1 = Arc::clone(&last_activity);
|
||||
let initial_len = initial_data.map_or(0u64, |d| d.len() as u64);
|
||||
let metrics_c2b = metrics.clone();
|
||||
let cc1 = conn_cancel.clone();
|
||||
let c2b = tokio::spawn(async move {
|
||||
let mut buf = vec![0u8; 65536];
|
||||
let mut total = initial_len;
|
||||
loop {
|
||||
let n = match client_read.read(&mut buf).await {
|
||||
Ok(0) | Err(_) => break,
|
||||
Ok(n) => n,
|
||||
let n = tokio::select! {
|
||||
result = client_read.read(&mut buf) => match result {
|
||||
Ok(0) | Err(_) => break,
|
||||
Ok(n) => n,
|
||||
},
|
||||
_ = cc1.cancelled() => break,
|
||||
};
|
||||
if backend_write.write_all(&buf[..n]).await.is_err() {
|
||||
break;
|
||||
@@ -117,19 +126,27 @@ pub async fn forward_bidirectional_with_timeouts(
|
||||
ctx.collector.record_bytes(n as u64, 0, ctx.route_id.as_deref(), ctx.source_ip.as_deref());
|
||||
}
|
||||
}
|
||||
let _ = backend_write.shutdown().await;
|
||||
// Graceful shutdown with timeout (sends TCP FIN / TLS close_notify)
|
||||
let _ = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(2),
|
||||
backend_write.shutdown(),
|
||||
).await;
|
||||
total
|
||||
});
|
||||
|
||||
let la2 = Arc::clone(&last_activity);
|
||||
let metrics_b2c = metrics;
|
||||
let cc2 = conn_cancel.clone();
|
||||
let b2c = tokio::spawn(async move {
|
||||
let mut buf = vec![0u8; 65536];
|
||||
let mut total = 0u64;
|
||||
loop {
|
||||
let n = match backend_read.read(&mut buf).await {
|
||||
Ok(0) | Err(_) => break,
|
||||
Ok(n) => n,
|
||||
let n = tokio::select! {
|
||||
result = backend_read.read(&mut buf) => match result {
|
||||
Ok(0) | Err(_) => break,
|
||||
Ok(n) => n,
|
||||
},
|
||||
_ = cc2.cancelled() => break,
|
||||
};
|
||||
if client_write.write_all(&buf[..n]).await.is_err() {
|
||||
break;
|
||||
@@ -140,14 +157,20 @@ pub async fn forward_bidirectional_with_timeouts(
|
||||
ctx.collector.record_bytes(0, n as u64, ctx.route_id.as_deref(), ctx.source_ip.as_deref());
|
||||
}
|
||||
}
|
||||
let _ = client_write.shutdown().await;
|
||||
// Graceful shutdown with timeout (sends TCP FIN / TLS close_notify)
|
||||
let _ = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(2),
|
||||
client_write.shutdown(),
|
||||
).await;
|
||||
total
|
||||
});
|
||||
|
||||
// Watchdog: inactivity, max lifetime, and cancellation
|
||||
// Watchdog: inactivity, max lifetime, and cancellation.
|
||||
// First cancels the per-connection token for graceful shutdown (FIN/close_notify),
|
||||
// then falls back to abort if the tasks are stuck (e.g. on a blocked write_all).
|
||||
let la_watch = Arc::clone(&last_activity);
|
||||
let c2b_handle = c2b.abort_handle();
|
||||
let b2c_handle = b2c.abort_handle();
|
||||
let c2b_abort = c2b.abort_handle();
|
||||
let b2c_abort = b2c.abort_handle();
|
||||
let watchdog = tokio::spawn(async move {
|
||||
let check_interval = std::time::Duration::from_secs(5);
|
||||
let mut last_seen = 0u64;
|
||||
@@ -155,16 +178,12 @@ pub async fn forward_bidirectional_with_timeouts(
|
||||
tokio::select! {
|
||||
_ = cancel.cancelled() => {
|
||||
debug!("Connection cancelled by shutdown");
|
||||
c2b_handle.abort();
|
||||
b2c_handle.abort();
|
||||
break;
|
||||
}
|
||||
_ = tokio::time::sleep(check_interval) => {
|
||||
// Check max lifetime
|
||||
if start.elapsed() >= max_lifetime {
|
||||
debug!("Connection exceeded max lifetime, closing");
|
||||
c2b_handle.abort();
|
||||
b2c_handle.abort();
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -174,8 +193,6 @@ pub async fn forward_bidirectional_with_timeouts(
|
||||
let elapsed_since_activity = start.elapsed().as_millis() as u64 - current;
|
||||
if elapsed_since_activity >= inactivity_timeout.as_millis() as u64 {
|
||||
debug!("Connection inactive for {}ms, closing", elapsed_since_activity);
|
||||
c2b_handle.abort();
|
||||
b2c_handle.abort();
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -183,6 +200,13 @@ pub async fn forward_bidirectional_with_timeouts(
|
||||
}
|
||||
}
|
||||
}
|
||||
// Phase 1: Signal copy loops to exit gracefully (allows FIN/close_notify)
|
||||
conn_cancel.cancel();
|
||||
// Phase 2: Wait for graceful shutdown (2s shutdown timeout + 2s margin)
|
||||
tokio::time::sleep(std::time::Duration::from_secs(4)).await;
|
||||
// Phase 3: Force-abort if still stuck (e.g. blocked on write_all)
|
||||
c2b_abort.abort();
|
||||
b2c_abort.abort();
|
||||
});
|
||||
|
||||
let bytes_in = c2b.await.unwrap_or(0);
|
||||
|
||||
@@ -465,21 +465,19 @@ impl TcpListenerManager {
|
||||
Ok((stream, peer_addr)) => {
|
||||
let ip = peer_addr.ip();
|
||||
|
||||
// Global connection limit — acquire semaphore permit with timeout
|
||||
let permit = match tokio::time::timeout(
|
||||
std::time::Duration::from_secs(5),
|
||||
conn_semaphore.clone().acquire_owned(),
|
||||
).await {
|
||||
Ok(Ok(permit)) => permit,
|
||||
Ok(Err(_)) => {
|
||||
// Semaphore closed — shouldn't happen, but be safe
|
||||
debug!("Connection semaphore closed, dropping connection from {}", peer_addr);
|
||||
// Global connection limit — non-blocking check.
|
||||
// MUST NOT block the accept loop: a blocking acquire would stall
|
||||
// ALL connections to this port (not just the one over limit), because
|
||||
// listener.accept() is not polled while we await the semaphore.
|
||||
let permit = match conn_semaphore.clone().try_acquire_owned() {
|
||||
Ok(permit) => permit,
|
||||
Err(tokio::sync::TryAcquireError::NoPermits) => {
|
||||
warn!("Global connection limit reached, dropping connection from {}", peer_addr);
|
||||
drop(stream);
|
||||
continue;
|
||||
}
|
||||
Err(_) => {
|
||||
// Timeout — global limit reached
|
||||
debug!("Global connection limit reached, dropping connection from {}", peer_addr);
|
||||
Err(tokio::sync::TryAcquireError::Closed) => {
|
||||
warn!("Connection semaphore closed, dropping connection from {}", peer_addr);
|
||||
drop(stream);
|
||||
continue;
|
||||
}
|
||||
@@ -487,7 +485,7 @@ impl TcpListenerManager {
|
||||
|
||||
// Check per-IP limits and rate limiting
|
||||
if !conn_tracker.try_accept(&ip) {
|
||||
debug!("Rejected connection from {} (per-IP limit or rate limit)", peer_addr);
|
||||
warn!("Rejected connection from {} (per-IP limit or rate limit)", peer_addr);
|
||||
drop(stream);
|
||||
drop(permit);
|
||||
continue;
|
||||
@@ -519,7 +517,7 @@ impl TcpListenerManager {
|
||||
stream, port, peer_addr, rm, m, tc, sa, hp, cc, cn, sr, rc,
|
||||
).await;
|
||||
if let Err(e) = result {
|
||||
debug!("Connection error from {}: {}", peer_addr, e);
|
||||
warn!("Connection error from {}: {}", peer_addr, e);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -563,8 +561,9 @@ impl TcpListenerManager {
|
||||
// Non-proxy connections skip the peek entirely (no latency cost).
|
||||
let mut effective_peer_addr = peer_addr;
|
||||
if !conn_config.proxy_ips.is_empty() && conn_config.proxy_ips.contains(&peer_addr.ip()) {
|
||||
// Trusted proxy IP — peek for PROXY protocol header
|
||||
let mut proxy_peek = vec![0u8; 256];
|
||||
// Trusted proxy IP — peek for PROXY protocol header.
|
||||
// Use stack-allocated buffers (PROXY v1 headers are max ~108 bytes).
|
||||
let mut proxy_peek = [0u8; 256];
|
||||
let pn = match tokio::time::timeout(
|
||||
std::time::Duration::from_millis(conn_config.initial_data_timeout_ms),
|
||||
stream.peek(&mut proxy_peek),
|
||||
@@ -579,9 +578,9 @@ impl TcpListenerManager {
|
||||
Ok((header, consumed)) => {
|
||||
debug!("PROXY protocol: real client {} -> {}", header.source_addr, header.dest_addr);
|
||||
effective_peer_addr = header.source_addr;
|
||||
// Consume the proxy protocol header bytes
|
||||
let mut discard = vec![0u8; consumed];
|
||||
stream.read_exact(&mut discard).await?;
|
||||
// Consume the proxy protocol header bytes (stack buffer, max 108 bytes)
|
||||
let mut discard = [0u8; 128];
|
||||
stream.read_exact(&mut discard[..consumed]).await?;
|
||||
}
|
||||
Err(e) => {
|
||||
debug!("Failed to parse PROXY protocol header: {}", e);
|
||||
@@ -664,7 +663,7 @@ impl TcpListenerManager {
|
||||
if !rustproxy_http::request_filter::RequestFilter::check_ip_security(
|
||||
security, &peer_addr.ip(),
|
||||
) {
|
||||
debug!("Connection from {} blocked by route security", peer_addr);
|
||||
warn!("Connection from {} blocked by route security", peer_addr);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
@@ -810,7 +809,7 @@ impl TcpListenerManager {
|
||||
let route_match = match route_match {
|
||||
Some(rm) => rm,
|
||||
None => {
|
||||
debug!("No route matched for port {} domain {:?}", port, domain);
|
||||
warn!("No route matched for port {} domain {:?} from {}", port, domain, peer_addr);
|
||||
if is_http {
|
||||
// Send a proper HTTP error instead of dropping the connection
|
||||
use tokio::io::AsyncWriteExt;
|
||||
@@ -844,7 +843,7 @@ impl TcpListenerManager {
|
||||
security,
|
||||
&peer_addr.ip(),
|
||||
) {
|
||||
debug!("Connection from {} blocked by route security", peer_addr);
|
||||
warn!("Connection from {} blocked by route security", peer_addr);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
@@ -987,13 +986,18 @@ impl TcpListenerManager {
|
||||
Err(_) => return Err("TLS handshake timeout".into()),
|
||||
};
|
||||
|
||||
// Peek at decrypted data to determine if HTTP
|
||||
// Peek at decrypted data to determine if HTTP.
|
||||
// Timeout prevents connection leak if client completes TLS
|
||||
// but never sends application data (scanners, health probes, slow-loris).
|
||||
let mut buf_stream = tokio::io::BufReader::new(tls_stream);
|
||||
let peeked = {
|
||||
use tokio::io::AsyncBufReadExt;
|
||||
match buf_stream.fill_buf().await {
|
||||
Ok(data) => sni_parser::is_http(data),
|
||||
Err(_) => false,
|
||||
match tokio::time::timeout(
|
||||
std::time::Duration::from_millis(conn_config.initial_data_timeout_ms),
|
||||
buf_stream.fill_buf(),
|
||||
).await {
|
||||
Ok(Ok(data)) => sni_parser::is_http(data),
|
||||
Ok(Err(_)) | Err(_) => false,
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1011,7 +1015,11 @@ impl TcpListenerManager {
|
||||
"TLS Terminate + HTTP: {} -> {}:{} (domain: {:?})",
|
||||
peer_addr, target_host, target_port, domain
|
||||
);
|
||||
http_proxy.handle_io(buf_stream, peer_addr, port, cancel.clone()).await;
|
||||
// Wrap in ShutdownOnDrop to ensure TLS close_notify is sent
|
||||
// even if hyper drops the connection without calling shutdown
|
||||
// (e.g. H2 close, backend error, idle timeout drain).
|
||||
let wrapped = rustproxy_http::shutdown_on_drop::ShutdownOnDrop::new(buf_stream);
|
||||
http_proxy.handle_io(wrapped, peer_addr, port, cancel.clone()).await;
|
||||
} else {
|
||||
debug!(
|
||||
"TLS Terminate + TCP: {} -> {}:{} (domain: {:?})",
|
||||
@@ -1062,13 +1070,18 @@ impl TcpListenerManager {
|
||||
Err(_) => return Err("TLS handshake timeout".into()),
|
||||
};
|
||||
|
||||
// Peek at decrypted data to detect protocol
|
||||
// Peek at decrypted data to detect protocol.
|
||||
// Timeout prevents connection leak if client completes TLS
|
||||
// but never sends application data (scanners, health probes, slow-loris).
|
||||
let mut buf_stream = tokio::io::BufReader::new(tls_stream);
|
||||
let is_http_data = {
|
||||
use tokio::io::AsyncBufReadExt;
|
||||
match buf_stream.fill_buf().await {
|
||||
Ok(data) => sni_parser::is_http(data),
|
||||
Err(_) => false,
|
||||
match tokio::time::timeout(
|
||||
std::time::Duration::from_millis(conn_config.initial_data_timeout_ms),
|
||||
buf_stream.fill_buf(),
|
||||
).await {
|
||||
Ok(Ok(data)) => sni_parser::is_http(data),
|
||||
Ok(Err(_)) | Err(_) => false,
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1088,7 +1101,10 @@ impl TcpListenerManager {
|
||||
"TLS Terminate+Reencrypt + HTTP: {} (domain: {:?})",
|
||||
peer_addr, domain
|
||||
);
|
||||
http_proxy.handle_io(buf_stream, peer_addr, port, cancel.clone()).await;
|
||||
// Wrap in ShutdownOnDrop to ensure TLS close_notify is sent
|
||||
// even if hyper drops the connection without calling shutdown.
|
||||
let wrapped = rustproxy_http::shutdown_on_drop::ShutdownOnDrop::new(buf_stream);
|
||||
http_proxy.handle_io(wrapped, peer_addr, port, cancel.clone()).await;
|
||||
} else {
|
||||
// Non-HTTP: TLS-to-TLS tunnel (existing behavior for raw TCP protocols)
|
||||
debug!(
|
||||
@@ -1396,15 +1412,24 @@ impl TcpListenerManager {
|
||||
let last_activity = Arc::new(AtomicU64::new(0));
|
||||
let start = std::time::Instant::now();
|
||||
|
||||
// Per-connection cancellation token: the watchdog cancels this instead of
|
||||
// aborting tasks, so the copy loops can shut down gracefully (TLS close_notify
|
||||
// for terminate/reencrypt mode, TCP FIN for passthrough mode).
|
||||
let conn_cancel = CancellationToken::new();
|
||||
|
||||
let la1 = Arc::clone(&last_activity);
|
||||
let metrics_c2b = metrics.clone();
|
||||
let cc1 = conn_cancel.clone();
|
||||
let c2b = tokio::spawn(async move {
|
||||
let mut buf = vec![0u8; 65536];
|
||||
let mut total = 0u64;
|
||||
loop {
|
||||
let n = match client_read.read(&mut buf).await {
|
||||
Ok(0) | Err(_) => break,
|
||||
Ok(n) => n,
|
||||
let n = tokio::select! {
|
||||
result = client_read.read(&mut buf) => match result {
|
||||
Ok(0) | Err(_) => break,
|
||||
Ok(n) => n,
|
||||
},
|
||||
_ = cc1.cancelled() => break,
|
||||
};
|
||||
if backend_write.write_all(&buf[..n]).await.is_err() {
|
||||
break;
|
||||
@@ -1418,19 +1443,27 @@ impl TcpListenerManager {
|
||||
ctx.collector.record_bytes(n as u64, 0, ctx.route_id.as_deref(), ctx.source_ip.as_deref());
|
||||
}
|
||||
}
|
||||
let _ = backend_write.shutdown().await;
|
||||
// Graceful shutdown with timeout (sends TLS close_notify / TCP FIN)
|
||||
let _ = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(2),
|
||||
backend_write.shutdown(),
|
||||
).await;
|
||||
total
|
||||
});
|
||||
|
||||
let la2 = Arc::clone(&last_activity);
|
||||
let metrics_b2c = metrics;
|
||||
let cc2 = conn_cancel.clone();
|
||||
let b2c = tokio::spawn(async move {
|
||||
let mut buf = vec![0u8; 65536];
|
||||
let mut total = 0u64;
|
||||
loop {
|
||||
let n = match backend_read.read(&mut buf).await {
|
||||
Ok(0) | Err(_) => break,
|
||||
Ok(n) => n,
|
||||
let n = tokio::select! {
|
||||
result = backend_read.read(&mut buf) => match result {
|
||||
Ok(0) | Err(_) => break,
|
||||
Ok(n) => n,
|
||||
},
|
||||
_ = cc2.cancelled() => break,
|
||||
};
|
||||
if client_write.write_all(&buf[..n]).await.is_err() {
|
||||
break;
|
||||
@@ -1444,14 +1477,20 @@ impl TcpListenerManager {
|
||||
ctx.collector.record_bytes(0, n as u64, ctx.route_id.as_deref(), ctx.source_ip.as_deref());
|
||||
}
|
||||
}
|
||||
let _ = client_write.shutdown().await;
|
||||
// Graceful shutdown with timeout (sends TLS close_notify / TCP FIN)
|
||||
let _ = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(2),
|
||||
client_write.shutdown(),
|
||||
).await;
|
||||
total
|
||||
});
|
||||
|
||||
// Watchdog task: check for inactivity, max lifetime, and cancellation
|
||||
// Watchdog task: check for inactivity, max lifetime, and cancellation.
|
||||
// First cancels the per-connection token for graceful shutdown (close_notify/FIN),
|
||||
// then falls back to abort if the tasks are stuck (e.g. on a blocked write_all).
|
||||
let la_watch = Arc::clone(&last_activity);
|
||||
let c2b_handle = c2b.abort_handle();
|
||||
let b2c_handle = b2c.abort_handle();
|
||||
let c2b_abort = c2b.abort_handle();
|
||||
let b2c_abort = b2c.abort_handle();
|
||||
let watchdog = tokio::spawn(async move {
|
||||
let check_interval = std::time::Duration::from_secs(5);
|
||||
let mut last_seen = 0u64;
|
||||
@@ -1459,16 +1498,12 @@ impl TcpListenerManager {
|
||||
tokio::select! {
|
||||
_ = cancel.cancelled() => {
|
||||
debug!("Split-stream connection cancelled by shutdown");
|
||||
c2b_handle.abort();
|
||||
b2c_handle.abort();
|
||||
break;
|
||||
}
|
||||
_ = tokio::time::sleep(check_interval) => {
|
||||
// Check max lifetime
|
||||
if start.elapsed() >= max_lifetime {
|
||||
debug!("Connection exceeded max lifetime, closing");
|
||||
c2b_handle.abort();
|
||||
b2c_handle.abort();
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -1479,8 +1514,6 @@ impl TcpListenerManager {
|
||||
let elapsed_since_activity = start.elapsed().as_millis() as u64 - current;
|
||||
if elapsed_since_activity >= inactivity_timeout.as_millis() as u64 {
|
||||
debug!("Connection inactive for {}ms, closing", elapsed_since_activity);
|
||||
c2b_handle.abort();
|
||||
b2c_handle.abort();
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -1488,6 +1521,13 @@ impl TcpListenerManager {
|
||||
}
|
||||
}
|
||||
}
|
||||
// Phase 1: Signal copy loops to exit gracefully (allows close_notify/FIN)
|
||||
conn_cancel.cancel();
|
||||
// Phase 2: Wait for graceful shutdown (2s shutdown timeout + 2s margin)
|
||||
tokio::time::sleep(std::time::Duration::from_secs(4)).await;
|
||||
// Phase 3: Force-abort if still stuck (e.g. blocked on write_all)
|
||||
c2b_abort.abort();
|
||||
b2c_abort.abort();
|
||||
});
|
||||
|
||||
let bytes_in = c2b.await.unwrap_or(0);
|
||||
|
||||
@@ -6,25 +6,28 @@
|
||||
/// - `example.com` exact match
|
||||
/// - `**.example.com` matches any depth of subdomain
|
||||
pub fn domain_matches(pattern: &str, domain: &str) -> bool {
|
||||
let pattern = pattern.trim().to_lowercase();
|
||||
let domain = domain.trim().to_lowercase();
|
||||
let pattern = pattern.trim();
|
||||
let domain = domain.trim();
|
||||
|
||||
if pattern == "*" {
|
||||
return true;
|
||||
}
|
||||
|
||||
if pattern == domain {
|
||||
if pattern.eq_ignore_ascii_case(domain) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Wildcard patterns
|
||||
if pattern.starts_with("*.") {
|
||||
if pattern.starts_with("*.") || pattern.starts_with("*.") {
|
||||
let suffix = &pattern[2..]; // e.g., "example.com"
|
||||
// Match exact parent or any single-level subdomain
|
||||
if domain == suffix {
|
||||
if domain.eq_ignore_ascii_case(suffix) {
|
||||
return true;
|
||||
}
|
||||
if domain.ends_with(&format!(".{}", suffix)) {
|
||||
if domain.len() > suffix.len() + 1
|
||||
&& domain.as_bytes()[domain.len() - suffix.len() - 1] == b'.'
|
||||
&& domain[domain.len() - suffix.len()..].eq_ignore_ascii_case(suffix)
|
||||
{
|
||||
// Check it's a single level subdomain for `*.`
|
||||
let prefix = &domain[..domain.len() - suffix.len() - 1];
|
||||
return !prefix.contains('.');
|
||||
@@ -35,11 +38,22 @@ pub fn domain_matches(pattern: &str, domain: &str) -> bool {
|
||||
if pattern.starts_with("**.") {
|
||||
let suffix = &pattern[3..];
|
||||
// Match exact parent or any depth of subdomain
|
||||
return domain == suffix || domain.ends_with(&format!(".{}", suffix));
|
||||
if domain.eq_ignore_ascii_case(suffix) {
|
||||
return true;
|
||||
}
|
||||
if domain.len() > suffix.len() + 1
|
||||
&& domain.as_bytes()[domain.len() - suffix.len() - 1] == b'.'
|
||||
&& domain[domain.len() - suffix.len()..].eq_ignore_ascii_case(suffix)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Use glob-match for more complex patterns
|
||||
glob_match::glob_match(&pattern, &domain)
|
||||
// Use glob-match for more complex patterns (case-insensitive via lowercasing)
|
||||
let pattern_lower = pattern.to_lowercase();
|
||||
let domain_lower = domain.to_lowercase();
|
||||
glob_match::glob_match(&pattern_lower, &domain_lower)
|
||||
}
|
||||
|
||||
/// Check if a domain matches any of the given patterns.
|
||||
|
||||
@@ -60,6 +60,16 @@ impl RouteManager {
|
||||
manager
|
||||
}
|
||||
|
||||
/// Check if any route on the given port uses header matching.
|
||||
/// Used to skip expensive header HashMap construction when no route needs it.
|
||||
pub fn any_route_has_headers(&self, port: u16) -> bool {
|
||||
if let Some(indices) = self.port_index.get(&port) {
|
||||
indices.iter().any(|&idx| self.routes[idx].route_match.headers.is_some())
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
/// Find the best matching route for the given context.
|
||||
pub fn find_route<'a>(&'a self, ctx: &MatchContext<'_>) -> Option<RouteMatchResult<'a>> {
|
||||
// Get routes for this port
|
||||
|
||||
@@ -632,15 +632,13 @@ impl RustProxy {
|
||||
let new_manager = Arc::new(new_manager);
|
||||
self.route_table.store(Arc::clone(&new_manager));
|
||||
|
||||
// Update listener manager
|
||||
// Update listener manager.
|
||||
// IMPORTANT: TLS configs must be swapped BEFORE the route manager so that
|
||||
// new routes only become visible after their certs are loaded. The reverse
|
||||
// order (routes first) creates a window where connections match new routes
|
||||
// but get the old TLS acceptor, causing cert mismatches.
|
||||
if let Some(ref mut listener) = self.listener_manager {
|
||||
listener.update_route_manager(Arc::clone(&new_manager));
|
||||
// Cancel connections on routes that were removed or disabled
|
||||
listener.invalidate_removed_routes(&active_route_ids);
|
||||
// Prune HTTP proxy caches (rate limiters, regex cache, round-robin counters)
|
||||
listener.prune_http_proxy_caches(&active_route_ids);
|
||||
|
||||
// Update TLS configs
|
||||
// 1. Update TLS configs first (so new certs are available before new routes)
|
||||
let mut tls_configs = Self::extract_tls_configs(&routes);
|
||||
if let Some(ref cm_arc) = self.cert_manager {
|
||||
let cm = cm_arc.lock().await;
|
||||
@@ -661,6 +659,13 @@ impl RustProxy {
|
||||
}
|
||||
listener.set_tls_configs(tls_configs);
|
||||
|
||||
// 2. Now swap the route manager (new routes become visible with certs already loaded)
|
||||
listener.update_route_manager(Arc::clone(&new_manager));
|
||||
// Cancel connections on routes that were removed or disabled
|
||||
listener.invalidate_removed_routes(&active_route_ids);
|
||||
// Prune HTTP proxy caches (rate limiters, regex cache, round-robin counters)
|
||||
listener.prune_http_proxy_caches(&active_route_ids);
|
||||
|
||||
// Add new ports
|
||||
for port in &new_ports {
|
||||
if !old_ports.contains(port) {
|
||||
|
||||
@@ -3,6 +3,6 @@
|
||||
*/
|
||||
export const commitinfo = {
|
||||
name: '@push.rocks/smartproxy',
|
||||
version: '25.11.2',
|
||||
version: '25.11.19',
|
||||
description: 'A powerful proxy package with unified route-based configuration for high traffic management. Features include SSL/TLS support, flexible routing patterns, WebSocket handling, advanced security options, and automatic ACME certificate management.'
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user