v25.11.18

changelog.md

@@ -1,5 +1,87 @@
 # Changelog
 
+## 2026-03-16 - 25.11.18 - fix(repo)
+no changes to commit
+
+
+## 2026-03-16 - 25.11.17 - fix(rustproxy-http)
+prevent stale HTTP/2 connection drivers from evicting newer pooled connections
+
+- add generation IDs to pooled HTTP/2 senders so pool removal only affects the matching connection
+- update HTTP/2 proxy and retry paths to register generation-tagged connections and skip eviction before registration completes
+
+## 2026-03-16 - 25.11.16 - fix(repo)
+no changes to commit
+
+
+## 2026-03-16 - 25.11.15 - fix(rustproxy-http)
+implement vectored write support for backend streams
+
+- Add poll_write_vectored forwarding for both plain and TLS backend stream variants
+- Expose is_write_vectored so the proxy can correctly report vectored write capability
+
+## 2026-03-16 - 25.11.14 - fix(rustproxy-http)
+forward vectored write support in ShutdownOnDrop AsyncWrite wrapper
+
+- Implements poll_write_vectored by delegating to the wrapped writer
+- Exposes is_write_vectored so the wrapper preserves underlying AsyncWrite capabilities
+
+## 2026-03-16 - 25.11.13 - fix(rustproxy-http)
+remove hot-path debug logging from HTTP/1 connection pool hits
+
+- Stops emitting debug logs when reusing HTTP/1 idle connections in the connection pool.
+- Keeps pool hit behavior unchanged while reducing overhead on a frequently executed path.
+
+## 2026-03-16 - 25.11.12 - fix(rustproxy-http)
+remove connection pool hit logging and keep logging limited to actual failures
+
+- Removes debug and warning logs for HTTP/2 connection pool hits and age checks.
+- Keeps pool behavior unchanged while reducing noisy per-request logging in the Rust HTTP proxy layer.
+
+## 2026-03-16 - 25.11.11 - fix(rustproxy-http)
+improve HTTP/2 proxy error logging with warning-level connection failures and debug error details
+
+- Adds debug-formatted error fields to HTTP/2 handshake, retry, fallback, and request failure logs
+- Promotes upstream HTTP/2 connection error logs from debug to warn to improve operational visibility
+
+## 2026-03-16 - 25.11.10 - fix(rustproxy-http)
+validate pooled HTTP/2 connections asynchronously before reuse and evict stale senders
+
+- Add an async ready() check with a 500ms timeout before reusing pooled HTTP/2 senders to catch GOAWAY/RST states before forwarding requests
+- Return connection age from the HTTP/2 pool checkout path and log warnings for older pooled connections
+- Evict pooled HTTP/2 senders when they are closed, exceed max age, fail readiness validation, or time out during readiness checks
+
+## 2026-03-16 - 25.11.9 - fix(rustproxy-routing)
+reduce hot-path allocations in routing, metrics, and proxy protocol handling
+
+- skip HTTP header map construction unless a route on the current port uses header matching
+- reuse computed client IP strings during HTTP route matching to avoid redundant allocations
+- optimize per-route and per-IP metric updates with get-first lookups to avoid unnecessary String creation on existing entries
+- replace heap-allocated PROXY protocol peek and discard buffers with stack-allocated buffers in the TCP listener
+- improve domain matcher case-insensitive wildcard checks while preserving glob fallback behavior
+
+## 2026-03-16 - 25.11.8 - fix(rustproxy-http)
+prevent premature idle timeouts during streamed HTTP responses and ensure TLS close_notify is sent on dropped connections
+
+- track active streaming response bodies so the HTTP idle watchdog does not close connections mid-transfer
+- add a ShutdownOnDrop wrapper for TLS-terminated HTTP connections to send shutdown on drop and avoid improperly terminated TLS sessions
+- apply the shutdown wrapper in passthrough TLS terminate and terminate+reencrypt HTTP handling
+
+## 2026-03-16 - 25.11.7 - fix(rustproxy)
+prevent TLS route reload certificate mismatches and tighten passthrough connection handling
+
+- Load updated TLS configs before swapping the route manager so newly visible routes always have their certificates available.
+- Add timeouts when peeking initial decrypted data after TLS handshake to avoid leaked idle connections.
+- Raise dropped, blocked, unmatched, and errored passthrough connection events from debug to warn for better operational visibility.
+
+## 2026-03-16 - 25.11.6 - fix(rustproxy-http,rustproxy-passthrough)
+improve upstream connection cleanup and graceful tunnel shutdown
+
+- Evict pooled HTTP/2 connections when their driver exits and shorten the maximum pooled H2 age to reduce reuse of stale upstream connections.
+- Strip hop-by-hop headers from backend responses before forwarding to HTTP/2 clients to avoid invalid H2 response handling.
+- Replace immediate task aborts in WebSocket and TCP tunnel watchdogs with cancellation-driven graceful shutdown plus timed fallback aborts.
+- Use non-blocking semaphore acquisition in the TCP listener so connection limits do not stall the accept loop for the entire port.
+
 ## 2026-03-16 - 25.11.5 - fix(repo)
 no changes to commit
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@push.rocks/smartproxy",
-  "version": "25.11.5",
+  "version": "25.11.18",
   "private": false,
   "description": "A powerful proxy package with unified route-based configuration for high traffic management. Features include SSL/TLS support, flexible routing patterns, WebSocket handling, advanced security options, and automatic ACME certificate management.",
   "main": "dist_ts/index.js",

rust/crates/rustproxy-http/src/connection_pool.rs

@@ -4,13 +4,13 @@
 //! HTTP/2 connections are multiplexed (clone the sender for each request).
 
 use std::sync::Arc;
+use std::sync::atomic::{AtomicU64, Ordering};
 use std::time::{Duration, Instant};
 
 use bytes::Bytes;
 use dashmap::DashMap;
 use http_body_util::combinators::BoxBody;
 use hyper::client::conn::{http1, http2};
-use tracing::debug;
 
 /// Maximum idle connections per backend key.
 const MAX_IDLE_PER_KEY: usize = 16;
@@ -20,7 +20,8 @@ const IDLE_TIMEOUT: Duration = Duration::from_secs(90);
 const EVICTION_INTERVAL: Duration = Duration::from_secs(30);
 /// Maximum age for pooled HTTP/2 connections before proactive eviction.
 /// Prevents staleness from backends that close idle connections (e.g. nginx GOAWAY).
-const MAX_H2_AGE: Duration = Duration::from_secs(300);
+/// 120s is well within typical server GOAWAY windows (nginx: ~60s idle, envoy: ~60s).
+const MAX_H2_AGE: Duration = Duration::from_secs(120);
 
 /// Identifies a unique backend endpoint.
 #[derive(Clone, Debug, Hash, Eq, PartialEq)]
@@ -37,10 +38,13 @@ struct IdleH1 {
     idle_since: Instant,
 }
 
-/// A pooled HTTP/2 sender (multiplexed, Clone-able).
+/// A pooled HTTP/2 sender (multiplexed, Clone-able) with a generation tag.
 struct PooledH2 {
     sender: http2::SendRequest<BoxBody<Bytes, hyper::Error>>,
     created_at: Instant,
+    /// Unique generation ID. Connection drivers use this to only remove their OWN
+    /// entry, preventing phantom eviction when multiple connections share the same key.
+    generation: u64,
 }
 
 /// Backend connection pool.
@@ -49,6 +53,8 @@ pub struct ConnectionPool {
     h1_pool: Arc<DashMap<PoolKey, Vec<IdleH1>>>,
     /// HTTP/2 multiplexed connections indexed by backend key.
     h2_pool: Arc<DashMap<PoolKey, PooledH2>>,
+    /// Monotonic generation counter for H2 pool entries.
+    h2_generation: AtomicU64,
     /// Handle for the background eviction task.
     eviction_handle: Option<tokio::task::JoinHandle<()>>,
 }
@@ -68,6 +74,7 @@ impl ConnectionPool {
         Self {
             h1_pool,
             h2_pool,
+            h2_generation: AtomicU64::new(0),
             eviction_handle: Some(eviction_handle),
         }
     }
@@ -81,7 +88,7 @@ impl ConnectionPool {
         while let Some(idle) = idles.pop() {
             // Check if the connection is still alive and ready
             if idle.idle_since.elapsed() < IDLE_TIMEOUT && idle.sender.is_ready() && !idle.sender.is_closed() {
-                debug!("Pool hit (h1): {}:{}", key.host, key.port);
+                // H1 pool hit — no logging on hot path
                 return Some(idle.sender);
             }
             // Stale or closed — drop it
@@ -114,40 +121,56 @@ impl ConnectionPool {
 
     /// Try to get a cloned HTTP/2 sender for the given key.
     /// HTTP/2 senders are Clone-able (multiplexed), so we clone rather than remove.
-    pub fn checkout_h2(&self, key: &PoolKey) -> Option<http2::SendRequest<BoxBody<Bytes, hyper::Error>>> {
+    pub fn checkout_h2(&self, key: &PoolKey) -> Option<(http2::SendRequest<BoxBody<Bytes, hyper::Error>>, Duration)> {
         let entry = self.h2_pool.get(key)?;
         let pooled = entry.value();
+        let age = pooled.created_at.elapsed();
 
-        // Check if the h2 connection is still alive and not too old
-        if pooled.sender.is_closed() || pooled.created_at.elapsed() >= MAX_H2_AGE {
+        if pooled.sender.is_closed() || age >= MAX_H2_AGE {
             drop(entry);
             self.h2_pool.remove(key);
             return None;
         }
 
         if pooled.sender.is_ready() {
-            debug!("Pool hit (h2): {}:{}", key.host, key.port);
-            return Some(pooled.sender.clone());
+            return Some((pooled.sender.clone(), age));
         }
         None
     }
 
-    /// Remove a dead HTTP/2 sender from the pool.
+    /// Remove a dead HTTP/2 sender from the pool (unconditional).
     /// Called when `send_request` fails to prevent subsequent requests from reusing the stale sender.
     pub fn remove_h2(&self, key: &PoolKey) {
         self.h2_pool.remove(key);
     }
 
-    /// Register an HTTP/2 sender in the pool. Since h2 is multiplexed,
-    /// only one sender per key is stored (it's Clone-able).
-    pub fn register_h2(&self, key: PoolKey, sender: http2::SendRequest<BoxBody<Bytes, hyper::Error>>) {
+    /// Remove an HTTP/2 sender ONLY if the current entry has the expected generation.
+    /// This prevents phantom eviction: when multiple connections share the same key,
+    /// an old connection's driver won't accidentally remove a newer connection's entry.
+    pub fn remove_h2_if_generation(&self, key: &PoolKey, expected_gen: u64) {
+        if let Some(entry) = self.h2_pool.get(key) {
+            if entry.value().generation == expected_gen {
+                drop(entry); // release DashMap ref before remove
+                self.h2_pool.remove(key);
+            }
+            // else: a newer connection replaced ours — don't touch it
+        }
+    }
+
+    /// Register an HTTP/2 sender in the pool. Returns the generation ID for this entry.
+    /// The caller should pass this generation to the connection driver so it can use
+    /// `remove_h2_if_generation` instead of `remove_h2` to avoid phantom eviction.
+    pub fn register_h2(&self, key: PoolKey, sender: http2::SendRequest<BoxBody<Bytes, hyper::Error>>) -> u64 {
+        let gen = self.h2_generation.fetch_add(1, Ordering::Relaxed);
         if sender.is_closed() {
-            return;
+            return gen;
         }
         self.h2_pool.insert(key, PooledH2 {
             sender,
             created_at: Instant::now(),
+            generation: gen,
         });
+        gen
     }
 
     /// Background eviction loop — runs every EVICTION_INTERVAL to remove stale connections.

rust/crates/rustproxy-http/src/counting_body.rs

@@ -27,6 +27,10 @@ pub struct CountingBody<B> {
     connection_activity: Option<Arc<AtomicU64>>,
     /// Start instant for computing elapsed ms for connection_activity.
     activity_start: Option<std::time::Instant>,
+    /// Optional active-request counter. When set, CountingBody increments on creation
+    /// and decrements on Drop, keeping the HTTP idle watchdog aware that a response
+    /// body is still streaming (even after the request handler has returned).
+    active_requests: Option<Arc<AtomicU64>>,
 }
 
 /// Which direction the bytes flow.
@@ -55,6 +59,7 @@ impl<B> CountingBody<B> {
             direction,
             connection_activity: None,
             activity_start: None,
+            active_requests: None,
         }
     }
 
@@ -67,6 +72,15 @@ impl<B> CountingBody<B> {
         self
     }
 
+    /// Set the active-request counter for the HTTP idle watchdog.
+    /// CountingBody increments on creation and decrements on Drop, ensuring the
+    /// idle watchdog sees an "active request" while the response body streams.
+    pub fn with_active_requests(mut self, counter: Arc<AtomicU64>) -> Self {
+        counter.fetch_add(1, Ordering::Relaxed);
+        self.active_requests = Some(counter);
+        self
+    }
+
     /// Report a chunk of bytes immediately to the metrics collector.
     #[inline]
     fn report_chunk(&self, len: u64) {
@@ -122,3 +136,13 @@ where
         self.inner.size_hint()
     }
 }
+
+impl<B> Drop for CountingBody<B> {
+    fn drop(&mut self) {
+        // Decrement the active-request counter so the HTTP idle watchdog
+        // knows this response body is no longer streaming.
+        if let Some(ref counter) = self.active_requests {
+            counter.fetch_sub(1, Ordering::Relaxed);
+        }
+    }
+}

rust/crates/rustproxy-http/src/lib.rs

@@ -9,6 +9,7 @@ pub mod protocol_cache;
 pub mod proxy_service;
 pub mod request_filter;
 pub mod response_filter;
+pub mod shutdown_on_drop;
 pub mod template;
 pub mod upstream_selector;
 

rust/crates/rustproxy-http/src/proxy_service.rs

@@ -39,6 +39,10 @@ use crate::upstream_selector::UpstreamSelector;
 struct ConnActivity {
     last_activity: Arc<AtomicU64>,
     start: std::time::Instant,
+    /// Active-request counter from handle_io's idle watchdog. When set, CountingBody
+    /// increments on creation and decrements on Drop, keeping the watchdog aware that
+    /// a response body is still streaming after the request handler has returned.
+    active_requests: Option<Arc<AtomicU64>>,
 }
 
 /// Default upstream connect timeout (30 seconds).
@@ -105,6 +109,24 @@ impl tokio::io::AsyncWrite for BackendStream {
         }
     }
 
+    fn poll_write_vectored(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+        bufs: &[std::io::IoSlice<'_>],
+    ) -> Poll<std::io::Result<usize>> {
+        match self.get_mut() {
+            BackendStream::Plain(s) => Pin::new(s).poll_write_vectored(cx, bufs),
+            BackendStream::Tls(s) => Pin::new(s).poll_write_vectored(cx, bufs),
+        }
+    }
+
+    fn is_write_vectored(&self) -> bool {
+        match self {
+            BackendStream::Plain(s) => s.is_write_vectored(),
+            BackendStream::Tls(s) => s.is_write_vectored(),
+        }
+    }
+
     fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
         match self.get_mut() {
             BackendStream::Plain(s) => Pin::new(s).poll_flush(cx),
@@ -302,7 +324,7 @@ impl HttpProxyService {
             let cn = cancel_inner.clone();
             let la = Arc::clone(&la_inner);
             let st = start;
-            let ca = ConnActivity { last_activity: Arc::clone(&la_inner), start };
+            let ca = ConnActivity { last_activity: Arc::clone(&la_inner), start, active_requests: Some(Arc::clone(&ar_inner)) };
             async move {
                 let result = svc.handle_request(req, peer, port, cn, ca).await;
                 // Mark request end — update activity timestamp before guard drops
@@ -395,11 +417,19 @@ impl HttpProxyService {
         let path = req.uri().path().to_string();
         let method = req.method().clone();
 
-        // Extract headers for matching
-        let headers: HashMap<String, String> = req.headers()
-            .iter()
-            .map(|(k, v)| (k.to_string(), v.to_str().unwrap_or("").to_string()))
-            .collect();
+        // Extract headers for matching — only allocate the HashMap if any route
+        // on this port actually uses header matching. Most deployments don't,
+        // so this saves ~20-30 String allocations per request.
+        let current_rm = self.route_manager.load();
+        let needs_headers = current_rm.any_route_has_headers(port);
+        let headers: Option<HashMap<String, String>> = if needs_headers {
+            Some(req.headers()
+                .iter()
+                .map(|(k, v)| (k.to_string(), v.to_str().unwrap_or("").to_string()))
+                .collect())
+        } else {
+            None
+        };
 
         debug!("HTTP {} {} (host: {:?}) from {}", method, path, host, peer_addr);
 
@@ -410,19 +440,19 @@ impl HttpProxyService {
             }
         }
 
-        // Match route
+        // Match route (current_rm already loaded above for headers check)
+        let ip_string = peer_addr.ip().to_string();
         let ctx = rustproxy_routing::MatchContext {
             port,
             domain: host.as_deref(),
             path: Some(&path),
-            client_ip: Some(&peer_addr.ip().to_string()),
+            client_ip: Some(&ip_string),
             tls_version: None,
-            headers: Some(&headers),
+            headers: headers.as_ref(),
             is_tls: false,
             protocol: Some("http"),
         };
 
-        let current_rm = self.route_manager.load();
         let route_match = match current_rm.find_route(&ctx) {
             Some(rm) => rm,
             None => {
@@ -432,7 +462,7 @@ impl HttpProxyService {
         };
 
         let route_id = route_match.route.id.as_deref();
-        let ip_str = peer_addr.ip().to_string();
+        let ip_str = ip_string; // reuse from above (avoid redundant to_string())
         self.metrics.record_http_request();
 
         // Apply request filters (IP check, rate limiting, auth)
@@ -647,17 +677,40 @@ impl HttpProxyService {
                 h2: use_h2,
             };
 
-            // H2 pool checkout (H2 senders are Clone and multiplexed)
+            // H2 pool checkout with async readiness validation.
+            // checkout_h2 does synchronous is_closed()/is_ready() checks, but these
+            // reflect cached state — the H2 connection driver (a separate tokio task)
+            // may not have processed a pending GOAWAY/RST yet. The ready().await
+            // forces the runtime to yield, giving the driver a chance to detect failures.
             if use_h2 {
-                if let Some(sender) = self.connection_pool.checkout_h2(&pool_key) {
-                    self.metrics.backend_pool_hit(&upstream_key);
-                    self.metrics.set_backend_protocol(&upstream_key, "h2");
-                    let result = self.forward_h2_pooled(
-                        sender, parts, body, upstream_headers, &upstream_path,
-                        route_match.route, route_id, &ip_str, &pool_key, domain_str, &conn_activity,
-                    ).await;
-                    self.upstream_selector.connection_ended(&upstream_key);
-                    return result;
+                if let Some((mut sender, age)) = self.connection_pool.checkout_h2(&pool_key) {
+                    match tokio::time::timeout(
+                        std::time::Duration::from_millis(500),
+                        sender.ready(),
+                    ).await {
+                        Ok(Ok(())) => {
+                            self.metrics.backend_pool_hit(&upstream_key);
+                            self.metrics.set_backend_protocol(&upstream_key, "h2");
+                            let result = self.forward_h2_pooled(
+                                sender, parts, body, upstream_headers, &upstream_path,
+                                route_match.route, route_id, &ip_str, &pool_key, domain_str, &conn_activity,
+                            ).await;
+                            self.upstream_selector.connection_ended(&upstream_key);
+                            return result;
+                        }
+                        Ok(Err(e)) => {
+                            warn!(backend = %upstream_key, age_secs = age.as_secs(),
+                                "Pooled H2 sender failed ready check (GOAWAY/RST): {}, evicting", e);
+                            self.connection_pool.remove_h2(&pool_key);
+                            // Fall through to fresh connection
+                        }
+                        Err(_) => {
+                            warn!(backend = %upstream_key, age_secs = age.as_secs(),
+                                "Pooled H2 sender ready check timed out (500ms), evicting");
+                            self.connection_pool.remove_h2(&pool_key);
+                            // Fall through to fresh connection
+                        }
+                    }
                 }
             }
         }
@@ -955,7 +1008,7 @@ impl HttpProxyService {
         ) = match tokio::time::timeout(self.connect_timeout, h2_builder.handshake(io)).await {
             Ok(Ok(h)) => h,
             Ok(Err(e)) => {
-                error!(backend = %backend_key, domain = %domain, error = %e, "Backend H2 handshake failed");
+                error!(backend = %backend_key, domain = %domain, error = %e, error_debug = ?e, "Backend H2 handshake failed");
                 self.metrics.backend_handshake_error(&backend_key);
                 return Ok(error_response(StatusCode::BAD_GATEWAY, "Backend H2 handshake failed"));
             }
@@ -966,17 +1019,33 @@ impl HttpProxyService {
             }
         };
 
-        tokio::spawn(async move {
-            if let Err(e) = conn.await {
-                debug!("HTTP/2 upstream connection error: {}", e);
-            }
-        });
+        // Shared generation ID: driver reads it after registration sets it.
+        // Uses u64::MAX as sentinel for "not yet registered" (driver waits/skips eviction).
+        let gen_holder = Arc::new(std::sync::atomic::AtomicU64::new(u64::MAX));
+
+        // Spawn the H2 connection driver; evict from pool on exit using generation-tagged
+        // removal to prevent phantom eviction when multiple connections share the same key.
+        {
+            let pool = Arc::clone(&self.connection_pool);
+            let key = pool_key.clone();
+            let gen = Arc::clone(&gen_holder);
+            tokio::spawn(async move {
+                if let Err(e) = conn.await {
+                    warn!("HTTP/2 upstream connection error: {} ({:?})", e, e);
+                }
+                let g = gen.load(std::sync::atomic::Ordering::Relaxed);
+                if g != u64::MAX {
+                    pool.remove_h2_if_generation(&key, g);
+                }
+            });
+        }
 
         // Clone sender for potential pool registration; register only after first request succeeds
         let sender_for_pool = sender.clone();
         let result = self.forward_h2_with_sender(sender, parts, body, upstream_headers, upstream_path, route, route_id, source_ip, Some(pool_key), domain, conn_activity).await;
         if matches!(&result, Ok(ref resp) if resp.status() != StatusCode::BAD_GATEWAY) {
-            self.connection_pool.register_h2(pool_key.clone(), sender_for_pool);
+            let g = self.connection_pool.register_h2(pool_key.clone(), sender_for_pool);
+            gen_holder.store(g, std::sync::atomic::Ordering::Relaxed);
         }
         result
     }
@@ -1098,7 +1167,7 @@ impl HttpProxyService {
         ) = match tokio::time::timeout(self.connect_timeout, h2_builder.handshake(io)).await {
             Ok(Ok(h)) => h,
             Ok(Err(e)) => {
-                error!(backend = %backend_key, domain = %domain, error = %e, "H2 retry: handshake failed");
+                error!(backend = %backend_key, domain = %domain, error = %e, error_debug = ?e, "H2 retry: handshake failed");
                 self.metrics.backend_handshake_error(&backend_key);
                 self.metrics.backend_connection_closed(&backend_key);
                 return Ok(error_response(StatusCode::BAD_GATEWAY, "Backend H2 retry handshake failed"));
@@ -1111,11 +1180,22 @@ impl HttpProxyService {
             }
         };
 
-        tokio::spawn(async move {
-            if let Err(e) = conn.await {
-                debug!("H2 retry: upstream connection error: {}", e);
-            }
-        });
+        // Spawn the H2 connection driver with generation-tagged eviction.
+        let gen_holder = Arc::new(std::sync::atomic::AtomicU64::new(u64::MAX));
+        {
+            let pool = Arc::clone(&self.connection_pool);
+            let key = pool_key.clone();
+            let gen = Arc::clone(&gen_holder);
+            tokio::spawn(async move {
+                if let Err(e) = conn.await {
+                    warn!("H2 retry: upstream connection error: {} ({:?})", e, e);
+                }
+                let g = gen.load(std::sync::atomic::Ordering::Relaxed);
+                if g != u64::MAX {
+                    pool.remove_h2_if_generation(&key, g);
+                }
+            });
+        }
 
         // Build request with empty body using absolute URI for H2 pseudo-headers
         let scheme = if pool_key.use_tls { "https" } else { "http" };
@@ -1141,7 +1221,8 @@ impl HttpProxyService {
         match sender.send_request(upstream_req).await {
             Ok(resp) => {
                 // Register in pool only after request succeeds
-                self.connection_pool.register_h2(pool_key.clone(), sender);
+                let g = self.connection_pool.register_h2(pool_key.clone(), sender);
+                gen_holder.store(g, std::sync::atomic::Ordering::Relaxed);
                 let result = self.build_streaming_response(resp, route, route_id, source_ip, conn_activity).await;
                 // Close the fresh backend connection (opened above)
                 self.metrics.backend_connection_closed(&backend_key);
@@ -1234,11 +1315,22 @@ impl HttpProxyService {
                 }
             }
             Ok(Ok((mut sender, conn))) => {
-                tokio::spawn(async move {
-                    if let Err(e) = conn.await {
-                        debug!("HTTP/2 upstream connection error: {}", e);
-                    }
-                });
+                // Spawn the H2 connection driver with generation-tagged eviction.
+                let gen_holder = Arc::new(std::sync::atomic::AtomicU64::new(u64::MAX));
+                {
+                    let pool = Arc::clone(&self.connection_pool);
+                    let key = pool_key.clone();
+                    let gen = Arc::clone(&gen_holder);
+                    tokio::spawn(async move {
+                        if let Err(e) = conn.await {
+                            warn!("HTTP/2 upstream connection error: {} ({:?})", e, e);
+                        }
+                        let g = gen.load(std::sync::atomic::Ordering::Relaxed);
+                        if g != u64::MAX {
+                            pool.remove_h2_if_generation(&key, g);
+                        }
+                    });
+                }
 
                 // Save retry state before consuming parts/body (for bodyless requests like GET)
                 // Clone BEFORE removing Host — H1 fallback needs Host header
@@ -1278,7 +1370,8 @@ impl HttpProxyService {
                 match sender.send_request(upstream_req).await {
                     Ok(upstream_response) => {
                         // H2 works! Register sender in pool for multiplexed reuse
-                        self.connection_pool.register_h2(pool_key.clone(), sender);
+                        let g = self.connection_pool.register_h2(pool_key.clone(), sender);
+                        gen_holder.store(g, std::sync::atomic::Ordering::Relaxed);
                         self.build_streaming_response(upstream_response, route, route_id, source_ip, conn_activity).await
                     }
                     Err(e) => {
@@ -1289,6 +1382,7 @@ impl HttpProxyService {
                             backend = %bk,
                             domain = %domain,
                             error = %e,
+                            error_debug = ?e,
                             "Auto-detect: H2 request failed, falling back to H1"
                         );
                         self.metrics.backend_h2_failure(&bk);
@@ -1546,11 +1640,11 @@ impl HttpProxyService {
                 // Evict the dead sender so subsequent requests get fresh connections
                 if let Some(key) = pool_key {
                     let bk = format!("{}:{}", key.host, key.port);
-                    error!(backend = %bk, domain = %domain, error = %e, "Backend H2 request failed");
+                    error!(backend = %bk, domain = %domain, error = %e, error_debug = ?e, "Backend H2 request failed");
                     self.metrics.backend_request_error(&bk);
                     self.connection_pool.remove_h2(key);
                 } else {
-                    error!(domain = %domain, error = %e, "Backend H2 request failed");
+                    error!(domain = %domain, error = %e, error_debug = ?e, "Backend H2 request failed");
                 }
                 return Ok(error_response(StatusCode::BAD_GATEWAY, "Backend H2 request failed"));
             }
@@ -1578,6 +1672,19 @@ impl HttpProxyService {
 
         if let Some(headers) = response.headers_mut() {
             *headers = resp_parts.headers;
+
+            // Strip hop-by-hop headers from the backend response.
+            // RFC 9113 §8.2.2 forbids connection-specific headers in HTTP/2 responses;
+            // forwarding them from an H1 backend can cause H2 stream resets.
+            // Mirrors the request-path stripping at the forward methods above.
+            headers.remove("connection");
+            headers.remove("keep-alive");
+            headers.remove("proxy-connection");
+            headers.remove("transfer-encoding");
+            headers.remove("te");
+            headers.remove("trailer");
+            // Note: "upgrade" is intentionally kept — needed for WebSocket 101 responses.
+
             ResponseFilter::apply_headers(route, headers, None);
         }
 
@@ -1592,6 +1699,15 @@ impl HttpProxyService {
             Direction::Out,
         ).with_connection_activity(Arc::clone(&conn_activity.last_activity), conn_activity.start);
 
+        // Keep active_requests > 0 while the response body streams, so the idle
+        // watchdog doesn't kill the connection mid-transfer (e.g. during git fetch).
+        // CountingBody increments on creation and decrements on Drop.
+        let counting_body = if let Some(ref ar) = conn_activity.active_requests {
+            counting_body.with_active_requests(Arc::clone(ar))
+        } else {
+            counting_body
+        };
+
         let body: BoxBody<Bytes, hyper::Error> = BoxBody::new(counting_body);
 
         Ok(response.body(body).unwrap())
@@ -1913,6 +2029,10 @@ impl HttpProxyService {
             let last_activity = Arc::new(AtomicU64::new(0));
             let start = std::time::Instant::now();
 
+            // Per-connection cancellation token: the watchdog cancels this instead of
+            // aborting tasks, so the copy loops can shut down gracefully (TLS close_notify).
+            let ws_cancel = CancellationToken::new();
+
             // For H2 WebSocket: also update the connection-level activity tracker
             // to prevent the idle watchdog from killing the H2 connection
             let conn_act_c2u = conn_activity.as_ref().map(|ca| (Arc::clone(&ca.last_activity), ca.start));
@@ -1922,13 +2042,17 @@ impl HttpProxyService {
             let metrics_c2u = Arc::clone(&metrics);
             let route_c2u = route_id_owned.clone();
             let ip_c2u = source_ip_owned.clone();
+            let wsc1 = ws_cancel.clone();
             let c2u = tokio::spawn(async move {
                 let mut buf = vec![0u8; 65536];
                 let mut total = 0u64;
                 loop {
-                    let n = match cr.read(&mut buf).await {
-                        Ok(0) | Err(_) => break,
-                        Ok(n) => n,
+                    let n = tokio::select! {
+                        result = cr.read(&mut buf) => match result {
+                            Ok(0) | Err(_) => break,
+                            Ok(n) => n,
+                        },
+                        _ = wsc1.cancelled() => break,
                     };
                     if uw.write_all(&buf[..n]).await.is_err() {
                         break;
@@ -1940,7 +2064,11 @@ impl HttpProxyService {
                         ca.store(ca_start.elapsed().as_millis() as u64, Ordering::Relaxed);
                     }
                 }
-                let _ = uw.shutdown().await;
+                // Graceful shutdown with timeout (sends TLS close_notify / TCP FIN)
+                let _ = tokio::time::timeout(
+                    std::time::Duration::from_secs(2),
+                    uw.shutdown(),
+                ).await;
                 total
             });
 
@@ -1948,13 +2076,17 @@ impl HttpProxyService {
             let metrics_u2c = Arc::clone(&metrics);
             let route_u2c = route_id_owned.clone();
             let ip_u2c = source_ip_owned.clone();
+            let wsc2 = ws_cancel.clone();
             let u2c = tokio::spawn(async move {
                 let mut buf = vec![0u8; 65536];
                 let mut total = 0u64;
                 loop {
-                    let n = match ur.read(&mut buf).await {
-                        Ok(0) | Err(_) => break,
-                        Ok(n) => n,
+                    let n = tokio::select! {
+                        result = ur.read(&mut buf) => match result {
+                            Ok(0) | Err(_) => break,
+                            Ok(n) => n,
+                        },
+                        _ = wsc2.cancelled() => break,
                     };
                     if cw.write_all(&buf[..n]).await.is_err() {
                         break;
@@ -1966,14 +2098,20 @@ impl HttpProxyService {
                         ca.store(ca_start.elapsed().as_millis() as u64, Ordering::Relaxed);
                     }
                 }
-                let _ = cw.shutdown().await;
+                // Graceful shutdown with timeout (sends TLS close_notify / TCP FIN)
+                let _ = tokio::time::timeout(
+                    std::time::Duration::from_secs(2),
+                    cw.shutdown(),
+                ).await;
                 total
             });
 
-            // Watchdog: monitors inactivity, max lifetime, and cancellation
+            // Watchdog: monitors inactivity, max lifetime, and cancellation.
+            // First cancels the per-connection token for graceful shutdown (close_notify/FIN),
+            // then falls back to abort if the tasks are stuck (e.g. on a blocked write_all).
             let la_watch = Arc::clone(&last_activity);
-            let c2u_handle = c2u.abort_handle();
-            let u2c_handle = u2c.abort_handle();
+            let c2u_abort = c2u.abort_handle();
+            let u2c_abort = u2c.abort_handle();
             let inactivity_timeout = ws_inactivity_timeout;
             let max_lifetime = ws_max_lifetime;
 
@@ -1985,8 +2123,6 @@ impl HttpProxyService {
                         _ = tokio::time::sleep(check_interval) => {}
                         _ = cancel.cancelled() => {
                             debug!("WebSocket tunnel cancelled by shutdown");
-                            c2u_handle.abort();
-                            u2c_handle.abort();
                             break;
                         }
                     }
@@ -1994,8 +2130,6 @@ impl HttpProxyService {
                     // Check max lifetime
                     if start.elapsed() >= max_lifetime {
                         debug!("WebSocket tunnel exceeded max lifetime, closing");
-                        c2u_handle.abort();
-                        u2c_handle.abort();
                         break;
                     }
 
@@ -2005,13 +2139,18 @@ impl HttpProxyService {
                         let elapsed_since_activity = start.elapsed().as_millis() as u64 - current;
                         if elapsed_since_activity >= inactivity_timeout.as_millis() as u64 {
                             debug!("WebSocket tunnel inactive for {}ms, closing", elapsed_since_activity);
-                            c2u_handle.abort();
-                            u2c_handle.abort();
                             break;
                         }
                     }
                     last_seen = current;
                 }
+                // Phase 1: Signal copy loops to exit gracefully (allows close_notify/FIN)
+                ws_cancel.cancel();
+                // Phase 2: Wait for graceful shutdown (2s shutdown timeout + 2s margin)
+                tokio::time::sleep(std::time::Duration::from_secs(4)).await;
+                // Phase 3: Force-abort if still stuck (e.g. blocked on write_all)
+                c2u_abort.abort();
+                u2c_abort.abort();
             });
 
             let bytes_in = c2u.await.unwrap_or(0);

rust/crates/rustproxy-http/src/shutdown_on_drop.rs

@@ -0,0 +1,102 @@
+//! Wrapper that ensures TLS close_notify is sent when the stream is dropped.
+//!
+//! When hyper drops an HTTP connection (backend error, timeout, normal H2 close),
+//! the underlying TLS stream is dropped WITHOUT `shutdown()`. tokio-rustls cannot
+//! send `close_notify` in Drop (requires async). This wrapper tracks whether
+//! `poll_shutdown` was called and, if not, spawns a background task to send it.
+
+use std::io;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+
+use tokio::io::{AsyncRead, AsyncWrite, ReadBuf};
+
+/// Wraps an AsyncRead+AsyncWrite stream and ensures `shutdown()` is called when
+/// dropped, even if the caller (e.g. hyper) doesn't explicitly shut down.
+///
+/// This guarantees TLS `close_notify` is sent for TLS-wrapped streams, preventing
+/// "GnuTLS recv error (-110): The TLS connection was non-properly terminated" errors.
+pub struct ShutdownOnDrop<S: AsyncRead + AsyncWrite + Unpin + Send + 'static> {
+    inner: Option<S>,
+    shutdown_called: bool,
+}
+
+impl<S: AsyncRead + AsyncWrite + Unpin + Send + 'static> ShutdownOnDrop<S> {
+    /// Create a new wrapper around the given stream.
+    pub fn new(stream: S) -> Self {
+        Self {
+            inner: Some(stream),
+            shutdown_called: false,
+        }
+    }
+}
+
+impl<S: AsyncRead + AsyncWrite + Unpin + Send + 'static> AsyncRead for ShutdownOnDrop<S> {
+    fn poll_read(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+        buf: &mut ReadBuf<'_>,
+    ) -> Poll<io::Result<()>> {
+        Pin::new(self.get_mut().inner.as_mut().unwrap()).poll_read(cx, buf)
+    }
+}
+
+impl<S: AsyncRead + AsyncWrite + Unpin + Send + 'static> AsyncWrite for ShutdownOnDrop<S> {
+    fn poll_write(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+        buf: &[u8],
+    ) -> Poll<io::Result<usize>> {
+        Pin::new(self.get_mut().inner.as_mut().unwrap()).poll_write(cx, buf)
+    }
+
+    fn poll_write_vectored(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+        bufs: &[io::IoSlice<'_>],
+    ) -> Poll<io::Result<usize>> {
+        Pin::new(self.get_mut().inner.as_mut().unwrap()).poll_write_vectored(cx, bufs)
+    }
+
+    fn is_write_vectored(&self) -> bool {
+        self.inner.as_ref().unwrap().is_write_vectored()
+    }
+
+    fn poll_flush(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+    ) -> Poll<io::Result<()>> {
+        Pin::new(self.get_mut().inner.as_mut().unwrap()).poll_flush(cx)
+    }
+
+    fn poll_shutdown(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+    ) -> Poll<io::Result<()>> {
+        let this = self.get_mut();
+        let result = Pin::new(this.inner.as_mut().unwrap()).poll_shutdown(cx);
+        if result.is_ready() {
+            this.shutdown_called = true;
+        }
+        result
+    }
+}
+
+impl<S: AsyncRead + AsyncWrite + Unpin + Send + 'static> Drop for ShutdownOnDrop<S> {
+    fn drop(&mut self) {
+        // If shutdown was already called (hyper closed properly), nothing to do.
+        // If not (hyper dropped without shutdown — e.g. H2 close, error, timeout),
+        // spawn a background task to send close_notify / TCP FIN.
+        if !self.shutdown_called {
+            if let Some(mut stream) = self.inner.take() {
+                tokio::spawn(async move {
+                    let _ = tokio::time::timeout(
+                        std::time::Duration::from_secs(2),
+                        tokio::io::AsyncWriteExt::shutdown(&mut stream),
+                    ).await;
+                    // stream is dropped here — all resources freed
+                });
+            }
+        }
+    }
+}

rust/crates/rustproxy-metrics/src/collector.rs

@@ -266,44 +266,67 @@ impl MetricsCollector {
         self.global_pending_tp_in.fetch_add(bytes_in, Ordering::Relaxed);
         self.global_pending_tp_out.fetch_add(bytes_out, Ordering::Relaxed);
 
+        // Per-route tracking: use get() first (zero-alloc fast path for existing entries),
+        // fall back to entry() with to_string() only on the rare first-chunk miss.
         if let Some(route_id) = route_id {
-            self.route_bytes_in
-                .entry(route_id.to_string())
-                .or_insert_with(|| AtomicU64::new(0))
-                .fetch_add(bytes_in, Ordering::Relaxed);
-            self.route_bytes_out
-                .entry(route_id.to_string())
-                .or_insert_with(|| AtomicU64::new(0))
-                .fetch_add(bytes_out, Ordering::Relaxed);
+            if let Some(counter) = self.route_bytes_in.get(route_id) {
+                counter.fetch_add(bytes_in, Ordering::Relaxed);
+            } else {
+                self.route_bytes_in.entry(route_id.to_string())
+                    .or_insert_with(|| AtomicU64::new(0))
+                    .fetch_add(bytes_in, Ordering::Relaxed);
+            }
+            if let Some(counter) = self.route_bytes_out.get(route_id) {
+                counter.fetch_add(bytes_out, Ordering::Relaxed);
+            } else {
+                self.route_bytes_out.entry(route_id.to_string())
+                    .or_insert_with(|| AtomicU64::new(0))
+                    .fetch_add(bytes_out, Ordering::Relaxed);
+            }
 
             // Accumulate into per-route pending throughput counters (lock-free)
-            let entry = self.route_pending_tp
-                .entry(route_id.to_string())
-                .or_insert_with(|| (AtomicU64::new(0), AtomicU64::new(0)));
-            entry.0.fetch_add(bytes_in, Ordering::Relaxed);
-            entry.1.fetch_add(bytes_out, Ordering::Relaxed);
+            if let Some(entry) = self.route_pending_tp.get(route_id) {
+                entry.0.fetch_add(bytes_in, Ordering::Relaxed);
+                entry.1.fetch_add(bytes_out, Ordering::Relaxed);
+            } else {
+                let entry = self.route_pending_tp.entry(route_id.to_string())
+                    .or_insert_with(|| (AtomicU64::new(0), AtomicU64::new(0)));
+                entry.0.fetch_add(bytes_in, Ordering::Relaxed);
+                entry.1.fetch_add(bytes_out, Ordering::Relaxed);
+            }
         }
 
+        // Per-IP tracking: same get()-first pattern to avoid String allocation on hot path.
         if let Some(ip) = source_ip {
             // Only record per-IP stats if the IP still has active connections.
             // This prevents orphaned entries when record_bytes races with
             // connection_closed (which evicts all per-IP data on last close).
             if self.ip_connections.contains_key(ip) {
-                self.ip_bytes_in
-                    .entry(ip.to_string())
-                    .or_insert_with(|| AtomicU64::new(0))
-                    .fetch_add(bytes_in, Ordering::Relaxed);
-                self.ip_bytes_out
-                    .entry(ip.to_string())
-                    .or_insert_with(|| AtomicU64::new(0))
-                    .fetch_add(bytes_out, Ordering::Relaxed);
+                if let Some(counter) = self.ip_bytes_in.get(ip) {
+                    counter.fetch_add(bytes_in, Ordering::Relaxed);
+                } else {
+                    self.ip_bytes_in.entry(ip.to_string())
+                        .or_insert_with(|| AtomicU64::new(0))
+                        .fetch_add(bytes_in, Ordering::Relaxed);
+                }
+                if let Some(counter) = self.ip_bytes_out.get(ip) {
+                    counter.fetch_add(bytes_out, Ordering::Relaxed);
+                } else {
+                    self.ip_bytes_out.entry(ip.to_string())
+                        .or_insert_with(|| AtomicU64::new(0))
+                        .fetch_add(bytes_out, Ordering::Relaxed);
+                }
 
                 // Accumulate into per-IP pending throughput counters (lock-free)
-                let entry = self.ip_pending_tp
-                    .entry(ip.to_string())
-                    .or_insert_with(|| (AtomicU64::new(0), AtomicU64::new(0)));
-                entry.0.fetch_add(bytes_in, Ordering::Relaxed);
-                entry.1.fetch_add(bytes_out, Ordering::Relaxed);
+                if let Some(entry) = self.ip_pending_tp.get(ip) {
+                    entry.0.fetch_add(bytes_in, Ordering::Relaxed);
+                    entry.1.fetch_add(bytes_out, Ordering::Relaxed);
+                } else {
+                    let entry = self.ip_pending_tp.entry(ip.to_string())
+                        .or_insert_with(|| (AtomicU64::new(0), AtomicU64::new(0)));
+                    entry.0.fetch_add(bytes_in, Ordering::Relaxed);
+                    entry.1.fetch_add(bytes_out, Ordering::Relaxed);
+                }
             }
         }
     }

rust/crates/rustproxy-passthrough/src/forwarder.rs

@@ -97,16 +97,25 @@ pub async fn forward_bidirectional_with_timeouts(
     let last_activity = Arc::new(AtomicU64::new(0));
     let start = std::time::Instant::now();
 
+    // Per-connection cancellation token: the watchdog cancels this instead of
+    // aborting tasks, so the copy loops can shut down gracefully (TCP FIN instead
+    // of RST, TLS close_notify if the stream is TLS-wrapped).
+    let conn_cancel = CancellationToken::new();
+
     let la1 = Arc::clone(&last_activity);
     let initial_len = initial_data.map_or(0u64, |d| d.len() as u64);
     let metrics_c2b = metrics.clone();
+    let cc1 = conn_cancel.clone();
     let c2b = tokio::spawn(async move {
         let mut buf = vec![0u8; 65536];
         let mut total = initial_len;
         loop {
-            let n = match client_read.read(&mut buf).await {
-                Ok(0) | Err(_) => break,
-                Ok(n) => n,
+            let n = tokio::select! {
+                result = client_read.read(&mut buf) => match result {
+                    Ok(0) | Err(_) => break,
+                    Ok(n) => n,
+                },
+                _ = cc1.cancelled() => break,
             };
             if backend_write.write_all(&buf[..n]).await.is_err() {
                 break;
@@ -117,19 +126,27 @@ pub async fn forward_bidirectional_with_timeouts(
                 ctx.collector.record_bytes(n as u64, 0, ctx.route_id.as_deref(), ctx.source_ip.as_deref());
             }
         }
-        let _ = backend_write.shutdown().await;
+        // Graceful shutdown with timeout (sends TCP FIN / TLS close_notify)
+        let _ = tokio::time::timeout(
+            std::time::Duration::from_secs(2),
+            backend_write.shutdown(),
+        ).await;
         total
     });
 
     let la2 = Arc::clone(&last_activity);
     let metrics_b2c = metrics;
+    let cc2 = conn_cancel.clone();
     let b2c = tokio::spawn(async move {
         let mut buf = vec![0u8; 65536];
         let mut total = 0u64;
         loop {
-            let n = match backend_read.read(&mut buf).await {
-                Ok(0) | Err(_) => break,
-                Ok(n) => n,
+            let n = tokio::select! {
+                result = backend_read.read(&mut buf) => match result {
+                    Ok(0) | Err(_) => break,
+                    Ok(n) => n,
+                },
+                _ = cc2.cancelled() => break,
             };
             if client_write.write_all(&buf[..n]).await.is_err() {
                 break;
@@ -140,14 +157,20 @@ pub async fn forward_bidirectional_with_timeouts(
                 ctx.collector.record_bytes(0, n as u64, ctx.route_id.as_deref(), ctx.source_ip.as_deref());
             }
         }
-        let _ = client_write.shutdown().await;
+        // Graceful shutdown with timeout (sends TCP FIN / TLS close_notify)
+        let _ = tokio::time::timeout(
+            std::time::Duration::from_secs(2),
+            client_write.shutdown(),
+        ).await;
         total
     });
 
-    // Watchdog: inactivity, max lifetime, and cancellation
+    // Watchdog: inactivity, max lifetime, and cancellation.
+    // First cancels the per-connection token for graceful shutdown (FIN/close_notify),
+    // then falls back to abort if the tasks are stuck (e.g. on a blocked write_all).
     let la_watch = Arc::clone(&last_activity);
-    let c2b_handle = c2b.abort_handle();
-    let b2c_handle = b2c.abort_handle();
+    let c2b_abort = c2b.abort_handle();
+    let b2c_abort = b2c.abort_handle();
     let watchdog = tokio::spawn(async move {
         let check_interval = std::time::Duration::from_secs(5);
         let mut last_seen = 0u64;
@@ -155,16 +178,12 @@ pub async fn forward_bidirectional_with_timeouts(
             tokio::select! {
                 _ = cancel.cancelled() => {
                     debug!("Connection cancelled by shutdown");
-                    c2b_handle.abort();
-                    b2c_handle.abort();
                     break;
                 }
                 _ = tokio::time::sleep(check_interval) => {
                     // Check max lifetime
                     if start.elapsed() >= max_lifetime {
                         debug!("Connection exceeded max lifetime, closing");
-                        c2b_handle.abort();
-                        b2c_handle.abort();
                         break;
                     }
 
@@ -174,8 +193,6 @@ pub async fn forward_bidirectional_with_timeouts(
                         let elapsed_since_activity = start.elapsed().as_millis() as u64 - current;
                         if elapsed_since_activity >= inactivity_timeout.as_millis() as u64 {
                             debug!("Connection inactive for {}ms, closing", elapsed_since_activity);
-                            c2b_handle.abort();
-                            b2c_handle.abort();
                             break;
                         }
                     }
@@ -183,6 +200,13 @@ pub async fn forward_bidirectional_with_timeouts(
                 }
             }
         }
+        // Phase 1: Signal copy loops to exit gracefully (allows FIN/close_notify)
+        conn_cancel.cancel();
+        // Phase 2: Wait for graceful shutdown (2s shutdown timeout + 2s margin)
+        tokio::time::sleep(std::time::Duration::from_secs(4)).await;
+        // Phase 3: Force-abort if still stuck (e.g. blocked on write_all)
+        c2b_abort.abort();
+        b2c_abort.abort();
     });
 
     let bytes_in = c2b.await.unwrap_or(0);

rust/crates/rustproxy-passthrough/src/tcp_listener.rs

@@ -465,21 +465,19 @@ impl TcpListenerManager {
                         Ok((stream, peer_addr)) => {
                             let ip = peer_addr.ip();
 
-                            // Global connection limit — acquire semaphore permit with timeout
-                            let permit = match tokio::time::timeout(
-                                std::time::Duration::from_secs(5),
-                                conn_semaphore.clone().acquire_owned(),
-                            ).await {
-                                Ok(Ok(permit)) => permit,
-                                Ok(Err(_)) => {
-                                    // Semaphore closed — shouldn't happen, but be safe
-                                    debug!("Connection semaphore closed, dropping connection from {}", peer_addr);
+                            // Global connection limit — non-blocking check.
+                            // MUST NOT block the accept loop: a blocking acquire would stall
+                            // ALL connections to this port (not just the one over limit), because
+                            // listener.accept() is not polled while we await the semaphore.
+                            let permit = match conn_semaphore.clone().try_acquire_owned() {
+                                Ok(permit) => permit,
+                                Err(tokio::sync::TryAcquireError::NoPermits) => {
+                                    warn!("Global connection limit reached, dropping connection from {}", peer_addr);
                                     drop(stream);
                                     continue;
                                 }
-                                Err(_) => {
-                                    // Timeout — global limit reached
-                                    debug!("Global connection limit reached, dropping connection from {}", peer_addr);
+                                Err(tokio::sync::TryAcquireError::Closed) => {
+                                    warn!("Connection semaphore closed, dropping connection from {}", peer_addr);
                                     drop(stream);
                                     continue;
                                 }
@@ -487,7 +485,7 @@ impl TcpListenerManager {
 
                             // Check per-IP limits and rate limiting
                             if !conn_tracker.try_accept(&ip) {
-                                debug!("Rejected connection from {} (per-IP limit or rate limit)", peer_addr);
+                                warn!("Rejected connection from {} (per-IP limit or rate limit)", peer_addr);
                                 drop(stream);
                                 drop(permit);
                                 continue;
@@ -519,7 +517,7 @@ impl TcpListenerManager {
                                     stream, port, peer_addr, rm, m, tc, sa, hp, cc, cn, sr, rc,
                                 ).await;
                                 if let Err(e) = result {
-                                    debug!("Connection error from {}: {}", peer_addr, e);
+                                    warn!("Connection error from {}: {}", peer_addr, e);
                                 }
                             });
                         }
@@ -563,8 +561,9 @@ impl TcpListenerManager {
         // Non-proxy connections skip the peek entirely (no latency cost).
         let mut effective_peer_addr = peer_addr;
         if !conn_config.proxy_ips.is_empty() && conn_config.proxy_ips.contains(&peer_addr.ip()) {
-            // Trusted proxy IP — peek for PROXY protocol header
-            let mut proxy_peek = vec![0u8; 256];
+            // Trusted proxy IP — peek for PROXY protocol header.
+            // Use stack-allocated buffers (PROXY v1 headers are max ~108 bytes).
+            let mut proxy_peek = [0u8; 256];
             let pn = match tokio::time::timeout(
                 std::time::Duration::from_millis(conn_config.initial_data_timeout_ms),
                 stream.peek(&mut proxy_peek),
@@ -579,9 +578,9 @@ impl TcpListenerManager {
                     Ok((header, consumed)) => {
                         debug!("PROXY protocol: real client {} -> {}", header.source_addr, header.dest_addr);
                         effective_peer_addr = header.source_addr;
-                        // Consume the proxy protocol header bytes
-                        let mut discard = vec![0u8; consumed];
-                        stream.read_exact(&mut discard).await?;
+                        // Consume the proxy protocol header bytes (stack buffer, max 108 bytes)
+                        let mut discard = [0u8; 128];
+                        stream.read_exact(&mut discard[..consumed]).await?;
                     }
                     Err(e) => {
                         debug!("Failed to parse PROXY protocol header: {}", e);
@@ -664,7 +663,7 @@ impl TcpListenerManager {
                             if !rustproxy_http::request_filter::RequestFilter::check_ip_security(
                                 security, &peer_addr.ip(),
                             ) {
-                                debug!("Connection from {} blocked by route security", peer_addr);
+                                warn!("Connection from {} blocked by route security", peer_addr);
                                 return Ok(());
                             }
                         }
@@ -810,7 +809,7 @@ impl TcpListenerManager {
         let route_match = match route_match {
             Some(rm) => rm,
             None => {
-                debug!("No route matched for port {} domain {:?}", port, domain);
+                warn!("No route matched for port {} domain {:?} from {}", port, domain, peer_addr);
                 if is_http {
                     // Send a proper HTTP error instead of dropping the connection
                     use tokio::io::AsyncWriteExt;
@@ -844,7 +843,7 @@ impl TcpListenerManager {
                 security,
                 &peer_addr.ip(),
             ) {
-                debug!("Connection from {} blocked by route security", peer_addr);
+                warn!("Connection from {} blocked by route security", peer_addr);
                 return Ok(());
             }
         }
@@ -987,13 +986,18 @@ impl TcpListenerManager {
                     Err(_) => return Err("TLS handshake timeout".into()),
                 };
 
-                // Peek at decrypted data to determine if HTTP
+                // Peek at decrypted data to determine if HTTP.
+                // Timeout prevents connection leak if client completes TLS
+                // but never sends application data (scanners, health probes, slow-loris).
                 let mut buf_stream = tokio::io::BufReader::new(tls_stream);
                 let peeked = {
                     use tokio::io::AsyncBufReadExt;
-                    match buf_stream.fill_buf().await {
-                        Ok(data) => sni_parser::is_http(data),
-                        Err(_) => false,
+                    match tokio::time::timeout(
+                        std::time::Duration::from_millis(conn_config.initial_data_timeout_ms),
+                        buf_stream.fill_buf(),
+                    ).await {
+                        Ok(Ok(data)) => sni_parser::is_http(data),
+                        Ok(Err(_)) | Err(_) => false,
                     }
                 };
 
@@ -1011,7 +1015,11 @@ impl TcpListenerManager {
                         "TLS Terminate + HTTP: {} -> {}:{} (domain: {:?})",
                         peer_addr, target_host, target_port, domain
                     );
-                    http_proxy.handle_io(buf_stream, peer_addr, port, cancel.clone()).await;
+                    // Wrap in ShutdownOnDrop to ensure TLS close_notify is sent
+                    // even if hyper drops the connection without calling shutdown
+                    // (e.g. H2 close, backend error, idle timeout drain).
+                    let wrapped = rustproxy_http::shutdown_on_drop::ShutdownOnDrop::new(buf_stream);
+                    http_proxy.handle_io(wrapped, peer_addr, port, cancel.clone()).await;
                 } else {
                     debug!(
                         "TLS Terminate + TCP: {} -> {}:{} (domain: {:?})",
@@ -1062,13 +1070,18 @@ impl TcpListenerManager {
                     Err(_) => return Err("TLS handshake timeout".into()),
                 };
 
-                // Peek at decrypted data to detect protocol
+                // Peek at decrypted data to detect protocol.
+                // Timeout prevents connection leak if client completes TLS
+                // but never sends application data (scanners, health probes, slow-loris).
                 let mut buf_stream = tokio::io::BufReader::new(tls_stream);
                 let is_http_data = {
                     use tokio::io::AsyncBufReadExt;
-                    match buf_stream.fill_buf().await {
-                        Ok(data) => sni_parser::is_http(data),
-                        Err(_) => false,
+                    match tokio::time::timeout(
+                        std::time::Duration::from_millis(conn_config.initial_data_timeout_ms),
+                        buf_stream.fill_buf(),
+                    ).await {
+                        Ok(Ok(data)) => sni_parser::is_http(data),
+                        Ok(Err(_)) | Err(_) => false,
                     }
                 };
 
@@ -1088,7 +1101,10 @@ impl TcpListenerManager {
                         "TLS Terminate+Reencrypt + HTTP: {} (domain: {:?})",
                         peer_addr, domain
                     );
-                    http_proxy.handle_io(buf_stream, peer_addr, port, cancel.clone()).await;
+                    // Wrap in ShutdownOnDrop to ensure TLS close_notify is sent
+                    // even if hyper drops the connection without calling shutdown.
+                    let wrapped = rustproxy_http::shutdown_on_drop::ShutdownOnDrop::new(buf_stream);
+                    http_proxy.handle_io(wrapped, peer_addr, port, cancel.clone()).await;
                 } else {
                     // Non-HTTP: TLS-to-TLS tunnel (existing behavior for raw TCP protocols)
                     debug!(
@@ -1396,15 +1412,24 @@ impl TcpListenerManager {
         let last_activity = Arc::new(AtomicU64::new(0));
         let start = std::time::Instant::now();
 
+        // Per-connection cancellation token: the watchdog cancels this instead of
+        // aborting tasks, so the copy loops can shut down gracefully (TLS close_notify
+        // for terminate/reencrypt mode, TCP FIN for passthrough mode).
+        let conn_cancel = CancellationToken::new();
+
         let la1 = Arc::clone(&last_activity);
         let metrics_c2b = metrics.clone();
+        let cc1 = conn_cancel.clone();
         let c2b = tokio::spawn(async move {
             let mut buf = vec![0u8; 65536];
             let mut total = 0u64;
             loop {
-                let n = match client_read.read(&mut buf).await {
-                    Ok(0) | Err(_) => break,
-                    Ok(n) => n,
+                let n = tokio::select! {
+                    result = client_read.read(&mut buf) => match result {
+                        Ok(0) | Err(_) => break,
+                        Ok(n) => n,
+                    },
+                    _ = cc1.cancelled() => break,
                 };
                 if backend_write.write_all(&buf[..n]).await.is_err() {
                     break;
@@ -1418,19 +1443,27 @@ impl TcpListenerManager {
                     ctx.collector.record_bytes(n as u64, 0, ctx.route_id.as_deref(), ctx.source_ip.as_deref());
                 }
             }
-            let _ = backend_write.shutdown().await;
+            // Graceful shutdown with timeout (sends TLS close_notify / TCP FIN)
+            let _ = tokio::time::timeout(
+                std::time::Duration::from_secs(2),
+                backend_write.shutdown(),
+            ).await;
             total
         });
 
         let la2 = Arc::clone(&last_activity);
         let metrics_b2c = metrics;
+        let cc2 = conn_cancel.clone();
         let b2c = tokio::spawn(async move {
             let mut buf = vec![0u8; 65536];
             let mut total = 0u64;
             loop {
-                let n = match backend_read.read(&mut buf).await {
-                    Ok(0) | Err(_) => break,
-                    Ok(n) => n,
+                let n = tokio::select! {
+                    result = backend_read.read(&mut buf) => match result {
+                        Ok(0) | Err(_) => break,
+                        Ok(n) => n,
+                    },
+                    _ = cc2.cancelled() => break,
                 };
                 if client_write.write_all(&buf[..n]).await.is_err() {
                     break;
@@ -1444,14 +1477,20 @@ impl TcpListenerManager {
                     ctx.collector.record_bytes(0, n as u64, ctx.route_id.as_deref(), ctx.source_ip.as_deref());
                 }
             }
-            let _ = client_write.shutdown().await;
+            // Graceful shutdown with timeout (sends TLS close_notify / TCP FIN)
+            let _ = tokio::time::timeout(
+                std::time::Duration::from_secs(2),
+                client_write.shutdown(),
+            ).await;
             total
         });
 
-        // Watchdog task: check for inactivity, max lifetime, and cancellation
+        // Watchdog task: check for inactivity, max lifetime, and cancellation.
+        // First cancels the per-connection token for graceful shutdown (close_notify/FIN),
+        // then falls back to abort if the tasks are stuck (e.g. on a blocked write_all).
         let la_watch = Arc::clone(&last_activity);
-        let c2b_handle = c2b.abort_handle();
-        let b2c_handle = b2c.abort_handle();
+        let c2b_abort = c2b.abort_handle();
+        let b2c_abort = b2c.abort_handle();
         let watchdog = tokio::spawn(async move {
             let check_interval = std::time::Duration::from_secs(5);
             let mut last_seen = 0u64;
@@ -1459,16 +1498,12 @@ impl TcpListenerManager {
                 tokio::select! {
                     _ = cancel.cancelled() => {
                         debug!("Split-stream connection cancelled by shutdown");
-                        c2b_handle.abort();
-                        b2c_handle.abort();
                         break;
                     }
                     _ = tokio::time::sleep(check_interval) => {
                         // Check max lifetime
                         if start.elapsed() >= max_lifetime {
                             debug!("Connection exceeded max lifetime, closing");
-                            c2b_handle.abort();
-                            b2c_handle.abort();
                             break;
                         }
 
@@ -1479,8 +1514,6 @@ impl TcpListenerManager {
                             let elapsed_since_activity = start.elapsed().as_millis() as u64 - current;
                             if elapsed_since_activity >= inactivity_timeout.as_millis() as u64 {
                                 debug!("Connection inactive for {}ms, closing", elapsed_since_activity);
-                                c2b_handle.abort();
-                                b2c_handle.abort();
                                 break;
                             }
                         }
@@ -1488,6 +1521,13 @@ impl TcpListenerManager {
                     }
                 }
             }
+            // Phase 1: Signal copy loops to exit gracefully (allows close_notify/FIN)
+            conn_cancel.cancel();
+            // Phase 2: Wait for graceful shutdown (2s shutdown timeout + 2s margin)
+            tokio::time::sleep(std::time::Duration::from_secs(4)).await;
+            // Phase 3: Force-abort if still stuck (e.g. blocked on write_all)
+            c2b_abort.abort();
+            b2c_abort.abort();
         });
 
         let bytes_in = c2b.await.unwrap_or(0);

rust/crates/rustproxy-routing/src/matchers/domain.rs

@@ -6,25 +6,28 @@
 /// - `example.com` exact match
 /// - `**.example.com` matches any depth of subdomain
 pub fn domain_matches(pattern: &str, domain: &str) -> bool {
-    let pattern = pattern.trim().to_lowercase();
-    let domain = domain.trim().to_lowercase();
+    let pattern = pattern.trim();
+    let domain = domain.trim();
 
     if pattern == "*" {
         return true;
     }
 
-    if pattern == domain {
+    if pattern.eq_ignore_ascii_case(domain) {
         return true;
     }
 
     // Wildcard patterns
-    if pattern.starts_with("*.") {
+    if pattern.starts_with("*.") || pattern.starts_with("*.") {
         let suffix = &pattern[2..]; // e.g., "example.com"
         // Match exact parent or any single-level subdomain
-        if domain == suffix {
+        if domain.eq_ignore_ascii_case(suffix) {
             return true;
         }
-        if domain.ends_with(&format!(".{}", suffix)) {
+        if domain.len() > suffix.len() + 1
+            && domain.as_bytes()[domain.len() - suffix.len() - 1] == b'.'
+            && domain[domain.len() - suffix.len()..].eq_ignore_ascii_case(suffix)
+        {
             // Check it's a single level subdomain for `*.`
             let prefix = &domain[..domain.len() - suffix.len() - 1];
             return !prefix.contains('.');
@@ -35,11 +38,22 @@ pub fn domain_matches(pattern: &str, domain: &str) -> bool {
     if pattern.starts_with("**.") {
         let suffix = &pattern[3..];
         // Match exact parent or any depth of subdomain
-        return domain == suffix || domain.ends_with(&format!(".{}", suffix));
+        if domain.eq_ignore_ascii_case(suffix) {
+            return true;
+        }
+        if domain.len() > suffix.len() + 1
+            && domain.as_bytes()[domain.len() - suffix.len() - 1] == b'.'
+            && domain[domain.len() - suffix.len()..].eq_ignore_ascii_case(suffix)
+        {
+            return true;
+        }
+        return false;
     }
 
-    // Use glob-match for more complex patterns
-    glob_match::glob_match(&pattern, &domain)
+    // Use glob-match for more complex patterns (case-insensitive via lowercasing)
+    let pattern_lower = pattern.to_lowercase();
+    let domain_lower = domain.to_lowercase();
+    glob_match::glob_match(&pattern_lower, &domain_lower)
 }
 
 /// Check if a domain matches any of the given patterns.

rust/crates/rustproxy-routing/src/route_manager.rs

@@ -60,6 +60,16 @@ impl RouteManager {
         manager
     }
 
+    /// Check if any route on the given port uses header matching.
+    /// Used to skip expensive header HashMap construction when no route needs it.
+    pub fn any_route_has_headers(&self, port: u16) -> bool {
+        if let Some(indices) = self.port_index.get(&port) {
+            indices.iter().any(|&idx| self.routes[idx].route_match.headers.is_some())
+        } else {
+            false
+        }
+    }
+
     /// Find the best matching route for the given context.
     pub fn find_route<'a>(&'a self, ctx: &MatchContext<'_>) -> Option<RouteMatchResult<'a>> {
         // Get routes for this port

rust/crates/rustproxy/src/lib.rs

@@ -632,15 +632,13 @@ impl RustProxy {
         let new_manager = Arc::new(new_manager);
         self.route_table.store(Arc::clone(&new_manager));
 
-        // Update listener manager
+        // Update listener manager.
+        // IMPORTANT: TLS configs must be swapped BEFORE the route manager so that
+        // new routes only become visible after their certs are loaded. The reverse
+        // order (routes first) creates a window where connections match new routes
+        // but get the old TLS acceptor, causing cert mismatches.
         if let Some(ref mut listener) = self.listener_manager {
-            listener.update_route_manager(Arc::clone(&new_manager));
-            // Cancel connections on routes that were removed or disabled
-            listener.invalidate_removed_routes(&active_route_ids);
-            // Prune HTTP proxy caches (rate limiters, regex cache, round-robin counters)
-            listener.prune_http_proxy_caches(&active_route_ids);
-
-            // Update TLS configs
+            // 1. Update TLS configs first (so new certs are available before new routes)
             let mut tls_configs = Self::extract_tls_configs(&routes);
             if let Some(ref cm_arc) = self.cert_manager {
                 let cm = cm_arc.lock().await;
@@ -661,6 +659,13 @@ impl RustProxy {
             }
             listener.set_tls_configs(tls_configs);
 
+            // 2. Now swap the route manager (new routes become visible with certs already loaded)
+            listener.update_route_manager(Arc::clone(&new_manager));
+            // Cancel connections on routes that were removed or disabled
+            listener.invalidate_removed_routes(&active_route_ids);
+            // Prune HTTP proxy caches (rate limiters, regex cache, round-robin counters)
+            listener.prune_http_proxy_caches(&active_route_ids);
+
             // Add new ports
             for port in &new_ports {
                 if !old_ports.contains(port) {

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@push.rocks/smartproxy',
-  version: '25.11.5',
+  version: '25.11.18',
   description: 'A powerful proxy package with unified route-based configuration for high traffic management. Features include SSL/TLS support, flexible routing patterns, WebSocket handling, advanced security options, and automatic ACME certificate management.'
 }