v25.10.7

changelog.md

@@ -1,5 +1,55 @@
 # Changelog
 
+## 2026-03-12 - 25.10.7 - fix(rustproxy-http)
+remove Host header from HTTP/2 upstream requests while preserving it for HTTP/1 retries
+
+- strips the Host header before sending HTTP/2 upstream requests so :authority from the URI is used instead
+- avoids 400 responses from nginx caused by sending both Host and :authority headers
+- keeps a cloned header set for bodyless request retries so HTTP/1 fallback still retains the Host header
+
+## 2026-03-12 - 25.10.6 - fix(rustproxy-http)
+use the requested domain as HTTP/2 authority instead of the backend host and port
+
+- build HTTP/2 absolute URIs from the client-facing domain so the :authority pseudo-header matches the Host header
+- remove backend port from generated HTTP/2 request URIs and fall back to the upstream host only when no domain is available
+- apply the authority handling consistently across pooled, inline, and generic upstream request paths
+
+## 2026-03-12 - 25.10.5 - fix(rustproxy-http)
+configure HTTP/2 client builders with a Tokio timer for keep-alive handling
+
+- Adds TokioTimer to all HTTP/2 client builder instances in proxy_service.
+- Ensures configured HTTP/2 keep-alive interval and timeout settings have the required timer runtime support.
+
+## 2026-03-12 - 25.10.4 - fix(rustproxy-http)
+stabilize upstream HTTP/2 forwarding and fallback behavior
+
+- Remove hop-by-hop headers before forwarding requests to HTTP/2 backends to comply with RFC 9113.
+- Use ALPN-enabled TLS configuration whenever HTTP/2 is possible, including explicit H2 connections and retries.
+- Add HTTP/2 handshake timeouts, tuned connection settings, and fallback to HTTP/1 when H2 negotiation times out or fails.
+- Register pooled HTTP/2 senders only after a successful first request to avoid reusing broken connections.
+- Build absolute URIs for HTTP/2 upstream requests so pseudo-headers such as scheme and authority are derived correctly.
+
+## 2026-03-12 - 25.10.3 - fix(rustproxy-http)
+include request domain in backend proxy error and protocol detection logs
+
+- Adds domain context to backend TCP/TLS connect, handshake, request failure, retry, and fallback log entries in the Rust HTTP proxy service.
+- Propagates the resolved host/domain through H1, H2, pooled, and fallback forwarding paths so backend-level diagnostics can be correlated with the original request domain.
+
+## 2026-03-12 - 25.10.2 - fix(repo)
+no code changes to release
+
+
+## 2026-03-12 - 25.10.1 - fix(repo)
+no changes to commit
+
+
+## 2026-03-12 - 25.10.0 - feat(metrics)
+add per-backend connection, error, protocol, and pool metrics with stale backend pruning
+
+- tracks backend connection lifecycle, connect timing, protocol detection, pool hit/miss rates, handshake/request errors, and h2 fallback failures in Rust metrics
+- exposes backend metrics through the TypeScript metrics adapter with backend listings, protocol lookup, and top error summaries
+- prunes backend metrics for backends no longer referenced by active routes, including preserved-port targets expanded across listening ports
+
 ## 2026-03-11 - 25.9.3 - fix(rustproxy-http)
 Evict stale HTTP/2 pooled senders and retry bodyless requests with fresh backend connections to avoid 502s
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@push.rocks/smartproxy",
-  "version": "25.9.3",
+  "version": "25.10.7",
   "private": false,
   "description": "A powerful proxy package with unified route-based configuration for high traffic management. Features include SSL/TLS support, flexible routing patterns, WebSocket handling, advanced security options, and automatic ACME certificate management.",
   "main": "dist_ts/index.js",

rust/crates/rustproxy-http/src/proxy_service.rs

@@ -479,6 +479,7 @@ impl HttpProxyService {
         }
 
         let upstream_key = format!("{}:{}", upstream.host, upstream.port);
+        let domain_str = host.as_deref().unwrap_or("-");
         self.upstream_selector.connection_started(&upstream_key);
 
         // Check for WebSocket upgrade
@@ -538,6 +539,15 @@ impl HttpProxyService {
             }
         }
 
+        // Remove hop-by-hop headers (RFC 9113 §8.2.2 forbids connection-specific headers in H2)
+        upstream_headers.remove("connection");
+        upstream_headers.remove("keep-alive");
+        upstream_headers.remove("proxy-connection");
+        upstream_headers.remove("transfer-encoding");
+        upstream_headers.remove("te");
+        upstream_headers.remove("trailer");
+        upstream_headers.remove("upgrade");
+
         // Add standard reverse-proxy headers (X-Forwarded-*)
         {
             let original_host = host.as_deref().unwrap_or("");
@@ -618,9 +628,11 @@ impl HttpProxyService {
             // H2 pool checkout (H2 senders are Clone and multiplexed)
             if use_h2 {
                 if let Some(sender) = self.connection_pool.checkout_h2(&pool_key) {
+                    self.metrics.backend_pool_hit(&upstream_key);
+                    self.metrics.set_backend_protocol(&upstream_key, "h2");
                     let result = self.forward_h2_pooled(
                         sender, parts, body, upstream_headers, &upstream_path,
-                        route_match.route, route_id, &ip_str, &pool_key,
+                        route_match.route, route_id, &ip_str, &pool_key, domain_str,
                     ).await;
                     self.upstream_selector.connection_ended(&upstream_key);
                     return result;
@@ -629,14 +641,17 @@ impl HttpProxyService {
         }
 
         // --- Fresh connection path ---
-        // Choose TLS config: use ALPN config for auto-detect probe, plain config otherwise
-        let tls_config = if needs_alpn_probe {
+        self.metrics.backend_pool_miss(&upstream_key);
+
+        // Choose TLS config: use ALPN config when H2 is possible (auto-detect probe OR explicit H2)
+        let tls_config = if needs_alpn_probe || use_h2 {
             &self.backend_tls_config_alpn
         } else {
             &self.backend_tls_config
         };
 
         // Establish backend connection
+        let connect_start = std::time::Instant::now();
         let (backend, detected_h2) = if upstream.use_tls {
             match tokio::time::timeout(
                 self.connect_timeout,
@@ -661,25 +676,42 @@ impl HttpProxyService {
                         };
                         self.protocol_cache.insert(cache_key, detected);
 
-                        debug!(
-                            "Auto-detected {} for backend {}:{}",
-                            if is_h2 { "HTTP/2" } else { "HTTP/1.1" },
-                            upstream.host, upstream.port
+                        info!(
+                            backend = %upstream_key,
+                            domain = %domain_str,
+                            protocol = if is_h2 { "h2" } else { "h1" },
+                            connect_time_ms = %connect_start.elapsed().as_millis(),
+                            "Backend protocol detected via ALPN"
                         );
 
                         is_h2
                     } else {
                         use_h2
                     };
+                    self.metrics.backend_connection_opened(&upstream_key, connect_start.elapsed());
+                    self.metrics.set_backend_protocol(&upstream_key, if final_h2 { "h2" } else { "h1" });
                     (BackendStream::Tls(tls), final_h2)
                 }
                 Ok(Err(e)) => {
-                    error!("Failed TLS connect to upstream {}:{}: {}", upstream.host, upstream.port, e);
+                    error!(
+                        backend = %upstream_key,
+                        domain = %domain_str,
+                        connect_time_ms = %connect_start.elapsed().as_millis(),
+                        error = %e,
+                        "Backend TLS connect failed"
+                    );
+                    self.metrics.backend_connect_error(&upstream_key);
                     self.upstream_selector.connection_ended(&upstream_key);
                     return Ok(error_response(StatusCode::BAD_GATEWAY, "Backend TLS unavailable"));
                 }
                 Err(_) => {
-                    error!("Upstream TLS connect timeout for {}:{}", upstream.host, upstream.port);
+                    error!(
+                        backend = %upstream_key,
+                        domain = %domain_str,
+                        connect_time_ms = %connect_start.elapsed().as_millis(),
+                        "Backend TLS connect timeout"
+                    );
+                    self.metrics.backend_connect_error(&upstream_key);
                     self.upstream_selector.connection_ended(&upstream_key);
                     return Ok(error_response(StatusCode::GATEWAY_TIMEOUT, "Backend TLS connect timeout"));
                 }
@@ -694,15 +726,30 @@ impl HttpProxyService {
                     let _ = socket2::SockRef::from(&s).set_tcp_keepalive(
                         &socket2::TcpKeepalive::new().with_time(std::time::Duration::from_secs(60))
                     );
+                    self.metrics.backend_connection_opened(&upstream_key, connect_start.elapsed());
+                    self.metrics.set_backend_protocol(&upstream_key, if use_h2 { "h2" } else { "h1" });
                     (BackendStream::Plain(s), use_h2)
                 }
                 Ok(Err(e)) => {
-                    error!("Failed to connect to upstream {}:{}: {}", upstream.host, upstream.port, e);
+                    error!(
+                        backend = %upstream_key,
+                        domain = %domain_str,
+                        connect_time_ms = %connect_start.elapsed().as_millis(),
+                        error = %e,
+                        "Backend TCP connect failed"
+                    );
+                    self.metrics.backend_connect_error(&upstream_key);
                     self.upstream_selector.connection_ended(&upstream_key);
                     return Ok(error_response(StatusCode::BAD_GATEWAY, "Backend unavailable"));
                 }
                 Err(_) => {
-                    error!("Upstream connect timeout for {}:{}", upstream.host, upstream.port);
+                    error!(
+                        backend = %upstream_key,
+                        domain = %domain_str,
+                        connect_time_ms = %connect_start.elapsed().as_millis(),
+                        "Backend TCP connect timeout"
+                    );
+                    self.metrics.backend_connect_error(&upstream_key);
                     self.upstream_selector.connection_ended(&upstream_key);
                     return Ok(error_response(StatusCode::GATEWAY_TIMEOUT, "Backend connect timeout"));
                 }
@@ -724,22 +771,23 @@ impl HttpProxyService {
                 self.forward_h2_with_fallback(
                     io, parts, body, upstream_headers, &upstream_path,
                     &upstream, route_match.route, route_id, &ip_str, &final_pool_key,
-                    host.clone(),
+                    host.clone(), domain_str,
                 ).await
             } else {
                 // Explicit H2 mode: hard-fail on handshake error (preserved behavior)
                 self.forward_h2(
                     io, parts, body, upstream_headers, &upstream_path,
-                    &upstream, route_match.route, route_id, &ip_str, &final_pool_key,
+                    &upstream, route_match.route, route_id, &ip_str, &final_pool_key, domain_str,
                 ).await
             }
         } else {
             self.forward_h1(
                 io, parts, body, upstream_headers, &upstream_path,
-                &upstream, route_match.route, route_id, &ip_str, &final_pool_key,
+                &upstream, route_match.route, route_id, &ip_str, &final_pool_key, domain_str,
             ).await
         };
         self.upstream_selector.connection_ended(&upstream_key);
+        self.metrics.backend_connection_closed(&upstream_key);
         result
     }
 
@@ -757,12 +805,16 @@ impl HttpProxyService {
         route_id: Option<&str>,
         source_ip: &str,
         pool_key: &crate::connection_pool::PoolKey,
+        domain: &str,
     ) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
+        let backend_key = format!("{}:{}", pool_key.host, pool_key.port);
+
         // Try pooled H1 connection first — avoids TCP+TLS handshake
         if let Some(pooled_sender) = self.connection_pool.checkout_h1(pool_key) {
+            self.metrics.backend_pool_hit(&backend_key);
             return self.forward_h1_with_sender(
                 pooled_sender, parts, body, upstream_headers, upstream_path,
-                route, route_id, source_ip, pool_key,
+                route, route_id, source_ip, pool_key, domain,
             ).await;
         }
 
@@ -773,7 +825,8 @@ impl HttpProxyService {
         ) = match hyper::client::conn::http1::handshake(io).await {
             Ok(h) => h,
             Err(e) => {
-                error!("Upstream handshake failed: {}", e);
+                error!(backend = %backend_key, domain = %domain, error = %e, "Backend H1 handshake failed");
+                self.metrics.backend_handshake_error(&backend_key);
                 return Ok(error_response(StatusCode::BAD_GATEWAY, "Backend handshake failed"));
             }
         };
@@ -784,7 +837,7 @@ impl HttpProxyService {
             }
         });
 
-        self.forward_h1_with_sender(sender, parts, body, upstream_headers, upstream_path, route, route_id, source_ip, pool_key).await
+        self.forward_h1_with_sender(sender, parts, body, upstream_headers, upstream_path, route, route_id, source_ip, pool_key, domain).await
     }
 
     /// Common H1 forwarding logic used by both fresh and pooled paths.
@@ -799,6 +852,7 @@ impl HttpProxyService {
         route_id: Option<&str>,
         source_ip: &str,
         pool_key: &crate::connection_pool::PoolKey,
+        domain: &str,
     ) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
         // Always use HTTP/1.1 for h1 backend connections (h2 incoming requests have version HTTP/2.0)
         let mut upstream_req = Request::builder()
@@ -825,7 +879,9 @@ impl HttpProxyService {
         let upstream_response = match sender.send_request(upstream_req).await {
             Ok(resp) => resp,
             Err(e) => {
-                error!("Upstream request failed: {}", e);
+                let bk = format!("{}:{}", pool_key.host, pool_key.port);
+                error!(backend = %bk, domain = %domain, error = %e, "Backend H1 request failed");
+                self.metrics.backend_request_error(&bk);
                 return Ok(error_response(StatusCode::BAD_GATEWAY, "Backend request failed"));
             }
         };
@@ -850,18 +906,32 @@ impl HttpProxyService {
         route_id: Option<&str>,
         source_ip: &str,
         pool_key: &crate::connection_pool::PoolKey,
+        domain: &str,
     ) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
+        let backend_key = format!("{}:{}", pool_key.host, pool_key.port);
         let exec = hyper_util::rt::TokioExecutor::new();
-        // Explicitly type the handshake with BoxBody for uniform pool type
+        let mut h2_builder = hyper::client::conn::http2::Builder::new(exec);
+        h2_builder
+            .timer(hyper_util::rt::TokioTimer::new())
+            .keep_alive_interval(std::time::Duration::from_secs(10))
+            .keep_alive_timeout(std::time::Duration::from_secs(5))
+            .adaptive_window(true)
+            .initial_stream_window_size(2 * 1024 * 1024);
         let (sender, conn): (
             hyper::client::conn::http2::SendRequest<BoxBody<Bytes, hyper::Error>>,
             hyper::client::conn::http2::Connection<TokioIo<BackendStream>, BoxBody<Bytes, hyper::Error>, hyper_util::rt::TokioExecutor>,
-        ) = match hyper::client::conn::http2::handshake(exec, io).await {
-            Ok(h) => h,
-            Err(e) => {
-                error!("HTTP/2 upstream handshake failed: {}", e);
+        ) = match tokio::time::timeout(self.connect_timeout, h2_builder.handshake(io)).await {
+            Ok(Ok(h)) => h,
+            Ok(Err(e)) => {
+                error!(backend = %backend_key, domain = %domain, error = %e, "Backend H2 handshake failed");
+                self.metrics.backend_handshake_error(&backend_key);
                 return Ok(error_response(StatusCode::BAD_GATEWAY, "Backend H2 handshake failed"));
             }
+            Err(_) => {
+                error!(backend = %backend_key, domain = %domain, "Backend H2 handshake timeout");
+                self.metrics.backend_handshake_error(&backend_key);
+                return Ok(error_response(StatusCode::GATEWAY_TIMEOUT, "Backend H2 handshake timeout"));
+            }
         };
 
         tokio::spawn(async move {
@@ -870,10 +940,13 @@ impl HttpProxyService {
             }
         });
 
-        // Register for multiplexed reuse
-        self.connection_pool.register_h2(pool_key.clone(), sender.clone());
-
-        self.forward_h2_with_sender(sender, parts, body, upstream_headers, upstream_path, route, route_id, source_ip, Some(pool_key)).await
+        // Clone sender for potential pool registration; register only after first request succeeds
+        let sender_for_pool = sender.clone();
+        let result = self.forward_h2_with_sender(sender, parts, body, upstream_headers, upstream_path, route, route_id, source_ip, Some(pool_key), domain).await;
+        if matches!(&result, Ok(ref resp) if resp.status() != StatusCode::BAD_GATEWAY) {
+            self.connection_pool.register_h2(pool_key.clone(), sender_for_pool);
+        }
+        result
     }
 
     /// Forward request using an existing (pooled) HTTP/2 sender.
@@ -890,6 +963,7 @@ impl HttpProxyService {
         route_id: Option<&str>,
         source_ip: &str,
         pool_key: &crate::connection_pool::PoolKey,
+        domain: &str,
     ) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
         // Save retry state for bodyless requests (cheap: Method is an enum, HeaderMap clones Arc-backed Bytes)
         let retry_state = if body.is_end_stream() {
@@ -900,18 +974,18 @@ impl HttpProxyService {
 
         let result = self.forward_h2_with_sender(
             sender, parts, body, upstream_headers, upstream_path,
-            route, route_id, source_ip, Some(pool_key),
+            route, route_id, source_ip, Some(pool_key), domain,
         ).await;
 
         // If the request failed (502) and we can retry with an empty body, do so
         let is_502 = matches!(&result, Ok(resp) if resp.status() == StatusCode::BAD_GATEWAY);
         if is_502 {
             if let Some((method, headers)) = retry_state {
-                warn!("Stale pooled H2 sender for {}:{}, retrying with fresh connection",
-                    pool_key.host, pool_key.port);
+                warn!(backend = %format!("{}:{}", pool_key.host, pool_key.port), domain = %domain,
+                    "Stale pooled H2 sender, retrying with fresh connection");
                 return self.retry_h2_with_fresh_connection(
                     method, headers, upstream_path,
-                    pool_key, route, route_id, source_ip,
+                    pool_key, route, route_id, source_ip, domain,
                 ).await;
             }
         }
@@ -929,20 +1003,26 @@ impl HttpProxyService {
         route: &rustproxy_config::RouteConfig,
         route_id: Option<&str>,
         source_ip: &str,
+        domain: &str,
     ) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
+        let backend_key = format!("{}:{}", pool_key.host, pool_key.port);
+
         // Establish fresh backend connection
+        let retry_connect_start = std::time::Instant::now();
         let backend = if pool_key.use_tls {
             match tokio::time::timeout(
                 self.connect_timeout,
-                connect_tls_backend(&self.backend_tls_config, &pool_key.host, pool_key.port),
+                connect_tls_backend(&self.backend_tls_config_alpn, &pool_key.host, pool_key.port),
             ).await {
                 Ok(Ok(tls)) => BackendStream::Tls(tls),
                 Ok(Err(e)) => {
-                    error!("H2 retry: TLS connect failed for {}:{}: {}", pool_key.host, pool_key.port, e);
+                    error!(backend = %backend_key, domain = %domain, error = %e, "H2 retry: TLS connect failed");
+                    self.metrics.backend_connect_error(&backend_key);
                     return Ok(error_response(StatusCode::BAD_GATEWAY, "Backend unavailable on H2 retry"));
                 }
                 Err(_) => {
-                    error!("H2 retry: TLS connect timeout for {}:{}", pool_key.host, pool_key.port);
+                    error!(backend = %backend_key, domain = %domain, "H2 retry: TLS connect timeout");
+                    self.metrics.backend_connect_error(&backend_key);
                     return Ok(error_response(StatusCode::GATEWAY_TIMEOUT, "Backend timeout on H2 retry"));
                 }
             }
@@ -956,27 +1036,45 @@ impl HttpProxyService {
                     BackendStream::Plain(s)
                 }
                 Ok(Err(e)) => {
-                    error!("H2 retry: connect failed for {}:{}: {}", pool_key.host, pool_key.port, e);
+                    error!(backend = %backend_key, domain = %domain, error = %e, "H2 retry: TCP connect failed");
+                    self.metrics.backend_connect_error(&backend_key);
                     return Ok(error_response(StatusCode::BAD_GATEWAY, "Backend unavailable on H2 retry"));
                 }
                 Err(_) => {
-                    error!("H2 retry: connect timeout for {}:{}", pool_key.host, pool_key.port);
+                    error!(backend = %backend_key, domain = %domain, "H2 retry: TCP connect timeout");
+                    self.metrics.backend_connect_error(&backend_key);
                     return Ok(error_response(StatusCode::GATEWAY_TIMEOUT, "Backend timeout on H2 retry"));
                 }
             }
         };
+        self.metrics.backend_connection_opened(&backend_key, retry_connect_start.elapsed());
 
         let io = TokioIo::new(backend);
         let exec = hyper_util::rt::TokioExecutor::new();
+        let mut h2_builder = hyper::client::conn::http2::Builder::new(exec);
+        h2_builder
+            .timer(hyper_util::rt::TokioTimer::new())
+            .keep_alive_interval(std::time::Duration::from_secs(10))
+            .keep_alive_timeout(std::time::Duration::from_secs(5))
+            .adaptive_window(true)
+            .initial_stream_window_size(2 * 1024 * 1024);
         let (mut sender, conn): (
             hyper::client::conn::http2::SendRequest<BoxBody<Bytes, hyper::Error>>,
             hyper::client::conn::http2::Connection<TokioIo<BackendStream>, BoxBody<Bytes, hyper::Error>, hyper_util::rt::TokioExecutor>,
-        ) = match hyper::client::conn::http2::handshake(exec, io).await {
-            Ok(h) => h,
-            Err(e) => {
-                error!("H2 retry: handshake failed for {}:{}: {}", pool_key.host, pool_key.port, e);
+        ) = match tokio::time::timeout(self.connect_timeout, h2_builder.handshake(io)).await {
+            Ok(Ok(h)) => h,
+            Ok(Err(e)) => {
+                error!(backend = %backend_key, domain = %domain, error = %e, "H2 retry: handshake failed");
+                self.metrics.backend_handshake_error(&backend_key);
+                self.metrics.backend_connection_closed(&backend_key);
                 return Ok(error_response(StatusCode::BAD_GATEWAY, "Backend H2 retry handshake failed"));
             }
+            Err(_) => {
+                error!(backend = %backend_key, domain = %domain, "H2 retry: handshake timeout");
+                self.metrics.backend_handshake_error(&backend_key);
+                self.metrics.backend_connection_closed(&backend_key);
+                return Ok(error_response(StatusCode::GATEWAY_TIMEOUT, "Backend H2 retry handshake timeout"));
+            }
         };
 
         tokio::spawn(async move {
@@ -985,13 +1083,17 @@ impl HttpProxyService {
             }
         });
 
-        // Register fresh sender in pool for future requests
-        self.connection_pool.register_h2(pool_key.clone(), sender.clone());
-
-        // Build request with empty body
+        // Build request with empty body using absolute URI for H2 pseudo-headers
+        let scheme = if pool_key.use_tls { "https" } else { "http" };
+        let authority = if domain != "-" { domain } else { pool_key.host.as_str() };
+        let h2_uri = format!("{}://{}{}", scheme, authority, upstream_path);
         let mut upstream_req = Request::builder()
             .method(method)
-            .uri(upstream_path);
+            .uri(&h2_uri);
+
+        // Remove Host header for H2 — :authority pseudo-header (from URI) is sufficient
+        let mut upstream_headers = upstream_headers;
+        upstream_headers.remove("host");
 
         if let Some(headers) = upstream_req.headers_mut() {
             *headers = upstream_headers;
@@ -1004,11 +1106,18 @@ impl HttpProxyService {
 
         match sender.send_request(upstream_req).await {
             Ok(resp) => {
-                self.build_streaming_response(resp, route, route_id, source_ip).await
+                // Register in pool only after request succeeds
+                self.connection_pool.register_h2(pool_key.clone(), sender);
+                let result = self.build_streaming_response(resp, route, route_id, source_ip).await;
+                // Close the fresh backend connection (opened above)
+                self.metrics.backend_connection_closed(&backend_key);
+                result
             }
             Err(e) => {
-                error!("H2 retry: request failed for {}:{}: {}", pool_key.host, pool_key.port, e);
-                self.connection_pool.remove_h2(pool_key);
+                error!(backend = %backend_key, domain = %domain, error = %e, "H2 retry: request failed");
+                self.metrics.backend_request_error(&backend_key);
+                // Close the fresh backend connection (opened above)
+                self.metrics.backend_connection_closed(&backend_key);
                 Ok(error_response(StatusCode::BAD_GATEWAY, "Backend H2 request failed on retry"))
             }
         }
@@ -1026,7 +1135,7 @@ impl HttpProxyService {
         io: TokioIo<BackendStream>,
         parts: hyper::http::request::Parts,
         body: Incoming,
-        upstream_headers: hyper::HeaderMap,
+        mut upstream_headers: hyper::HeaderMap,
         upstream_path: &str,
         upstream: &crate::upstream_selector::UpstreamSelection,
         route: &rustproxy_config::RouteConfig,
@@ -1034,15 +1143,62 @@ impl HttpProxyService {
         source_ip: &str,
         pool_key: &crate::connection_pool::PoolKey,
         requested_host: Option<String>,
+        domain: &str,
     ) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
         let exec = hyper_util::rt::TokioExecutor::new();
-        let handshake_result: Result<(
-            hyper::client::conn::http2::SendRequest<BoxBody<Bytes, hyper::Error>>,
-            hyper::client::conn::http2::Connection<TokioIo<BackendStream>, BoxBody<Bytes, hyper::Error>, hyper_util::rt::TokioExecutor>,
-        ), hyper::Error> = hyper::client::conn::http2::handshake(exec, io).await;
+        let mut h2_builder = hyper::client::conn::http2::Builder::new(exec);
+        h2_builder
+            .timer(hyper_util::rt::TokioTimer::new())
+            .keep_alive_interval(std::time::Duration::from_secs(10))
+            .keep_alive_timeout(std::time::Duration::from_secs(5))
+            .adaptive_window(true)
+            .initial_stream_window_size(2 * 1024 * 1024);
+        let handshake_result = tokio::time::timeout(
+            self.connect_timeout,
+            h2_builder.handshake(io),
+        ).await;
 
         match handshake_result {
-            Ok((mut sender, conn)) => {
+            Err(_) => {
+                // H2 handshake timed out — fall back to H1
+                let bk = format!("{}:{}", upstream.host, upstream.port);
+                warn!(
+                    backend = %bk,
+                    domain = %domain,
+                    "H2 handshake timeout, falling back to H1"
+                );
+                self.metrics.backend_h2_failure(&bk);
+                self.metrics.backend_handshake_error(&bk);
+
+                let cache_key = crate::protocol_cache::ProtocolCacheKey {
+                    host: upstream.host.clone(),
+                    port: upstream.port,
+                    requested_host: requested_host.clone(),
+                };
+                self.protocol_cache.insert(cache_key, crate::protocol_cache::DetectedProtocol::H1);
+
+                match self.reconnect_backend(upstream, domain).await {
+                    Some(fallback_backend) => {
+                        let h1_pool_key = crate::connection_pool::PoolKey {
+                            host: upstream.host.clone(),
+                            port: upstream.port,
+                            use_tls: upstream.use_tls,
+                            h2: false,
+                        };
+                        let fallback_io = TokioIo::new(fallback_backend);
+                        let result = self.forward_h1(
+                            fallback_io, parts, body, upstream_headers, upstream_path,
+                            upstream, route, route_id, source_ip, &h1_pool_key, domain,
+                        ).await;
+                        self.metrics.backend_connection_closed(&bk);
+                        result
+                    }
+                    None => {
+                        Ok(error_response(StatusCode::BAD_GATEWAY, "Backend unavailable after H2 timeout fallback"))
+                    }
+                }
+            }
+            Ok(Ok((mut sender, conn))) => {
                 tokio::spawn(async move {
                     if let Err(e) = conn.await {
                         debug!("HTTP/2 upstream connection error: {}", e);
@@ -1050,18 +1206,25 @@ impl HttpProxyService {
                 });
 
                 // Save retry state before consuming parts/body (for bodyless requests like GET)
+                // Clone BEFORE removing Host — H1 fallback needs Host header
                 let retry_state = if body.is_end_stream() {
                     Some((parts.method.clone(), upstream_headers.clone()))
                 } else {
                     None
                 };
 
+                // Remove Host header for H2 — :authority pseudo-header (from URI) is sufficient
+                upstream_headers.remove("host");
+
                 // Build and send the h2 request inline (don't register in pool yet —
                 // we need to verify the request actually succeeds first, because some
                 // backends advertise h2 via ALPN but don't speak the h2 binary protocol).
+                let scheme = if upstream.use_tls { "https" } else { "http" };
+                let authority = if domain != "-" { domain } else { upstream.host.as_str() };
+                let h2_uri = format!("{}://{}{}", scheme, authority, upstream_path);
                 let mut upstream_req = Request::builder()
                     .method(parts.method)
-                    .uri(upstream_path);
+                    .uri(&h2_uri);
 
                 if let Some(headers) = upstream_req.headers_mut() {
                     *headers = upstream_headers;
@@ -1086,10 +1249,14 @@ impl HttpProxyService {
                     Err(e) => {
                         // H2 request failed — backend advertises h2 via ALPN but doesn't
                         // actually speak it. Update cache so future requests use H1.
+                        let bk = format!("{}:{}", upstream.host, upstream.port);
                         warn!(
-                            "Auto-detect: H2 request failed for {}:{}, falling back to H1: {}",
-                            upstream.host, upstream.port, e
+                            backend = %bk,
+                            domain = %domain,
+                            error = %e,
+                            "Auto-detect: H2 request failed, falling back to H1"
                         );
+                        self.metrics.backend_h2_failure(&bk);
                         let cache_key = crate::protocol_cache::ProtocolCacheKey {
                             host: upstream.host.clone(),
                             port: upstream.port,
@@ -1099,7 +1266,7 @@ impl HttpProxyService {
 
                         // Retry as H1 for bodyless requests; return 502 for requests with bodies
                         if let Some((method, headers)) = retry_state {
-                            match self.reconnect_backend(upstream).await {
+                            match self.reconnect_backend(upstream, domain).await {
                                 Some(fallback_backend) => {
                                     let h1_pool_key = crate::connection_pool::PoolKey {
                                         host: upstream.host.clone(),
@@ -1108,10 +1275,13 @@ impl HttpProxyService {
                                         h2: false,
                                     };
                                     let fallback_io = TokioIo::new(fallback_backend);
-                                    self.forward_h1_empty_body(
+                                    let result = self.forward_h1_empty_body(
                                         fallback_io, method, headers, upstream_path,
-                                        route, route_id, source_ip, &h1_pool_key,
-                                    ).await
+                                        route, route_id, source_ip, &h1_pool_key, domain,
+                                    ).await;
+                                    // Close the reconnected backend connection (opened in reconnect_backend)
+                                    self.metrics.backend_connection_closed(&bk);
+                                    result
                                 }
                                 None => {
                                     Ok(error_response(StatusCode::BAD_GATEWAY, "Backend unavailable after H2 fallback"))
@@ -1123,13 +1293,18 @@ impl HttpProxyService {
                     }
                 }
             }
-            Err(e) => {
+            Ok(Err(e)) => {
                 // H2 handshake truly failed — fall back to H1
                 // Body is NOT consumed yet, so we can retry the full request.
+                let bk = format!("{}:{}", upstream.host, upstream.port);
                 warn!(
-                    "H2 handshake failed for {}:{}, falling back to H1: {}",
-                    upstream.host, upstream.port, e
+                    backend = %bk,
+                    domain = %domain,
+                    error = %e,
+                    "H2 handshake failed, falling back to H1"
                 );
+                self.metrics.backend_h2_failure(&bk);
+                self.metrics.backend_handshake_error(&bk);
 
                 // Update cache to H1 so subsequent requests skip H2
                 let cache_key = crate::protocol_cache::ProtocolCacheKey {
@@ -1140,7 +1315,7 @@ impl HttpProxyService {
                 self.protocol_cache.insert(cache_key, crate::protocol_cache::DetectedProtocol::H1);
 
                 // Reconnect for H1 (the original io was consumed by the failed h2 handshake)
-                match self.reconnect_backend(upstream).await {
+                match self.reconnect_backend(upstream, domain).await {
                     Some(fallback_backend) => {
                         let h1_pool_key = crate::connection_pool::PoolKey {
                             host: upstream.host.clone(),
@@ -1149,10 +1324,13 @@ impl HttpProxyService {
                             h2: false,
                         };
                         let fallback_io = TokioIo::new(fallback_backend);
-                        self.forward_h1(
+                        let result = self.forward_h1(
                             fallback_io, parts, body, upstream_headers, upstream_path,
-                            upstream, route, route_id, source_ip, &h1_pool_key,
-                        ).await
+                            upstream, route, route_id, source_ip, &h1_pool_key, domain,
+                        ).await;
+                        // Close the reconnected backend connection (opened in reconnect_backend)
+                        self.metrics.backend_connection_closed(&bk);
+                        result
                     }
                     None => {
                         Ok(error_response(StatusCode::BAD_GATEWAY, "Backend unavailable after H2 fallback"))
@@ -1174,14 +1352,17 @@ impl HttpProxyService {
         route_id: Option<&str>,
         source_ip: &str,
         pool_key: &crate::connection_pool::PoolKey,
+        domain: &str,
     ) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
+        let backend_key = format!("{}:{}", pool_key.host, pool_key.port);
         let (mut sender, conn): (
             hyper::client::conn::http1::SendRequest<BoxBody<Bytes, hyper::Error>>,
             hyper::client::conn::http1::Connection<TokioIo<BackendStream>, BoxBody<Bytes, hyper::Error>>,
         ) = match hyper::client::conn::http1::handshake(io).await {
             Ok(h) => h,
             Err(e) => {
-                error!("H1 fallback: handshake failed: {}", e);
+                error!(backend = %backend_key, domain = %domain, error = %e, "H1 fallback: handshake failed");
+                self.metrics.backend_handshake_error(&backend_key);
                 return Ok(error_response(StatusCode::BAD_GATEWAY, "Backend H1 fallback handshake failed"));
             }
         };
@@ -1209,7 +1390,8 @@ impl HttpProxyService {
         let upstream_response = match sender.send_request(upstream_req).await {
             Ok(resp) => resp,
             Err(e) => {
-                error!("H1 fallback: request failed: {}", e);
+                error!(backend = %backend_key, domain = %domain, error = %e, "H1 fallback: request failed");
+                self.metrics.backend_request_error(&backend_key);
                 return Ok(error_response(StatusCode::BAD_GATEWAY, "Backend H1 fallback request failed"));
             }
         };
@@ -1224,19 +1406,27 @@ impl HttpProxyService {
     async fn reconnect_backend(
         &self,
         upstream: &crate::upstream_selector::UpstreamSelection,
+        domain: &str,
     ) -> Option<BackendStream> {
+        let backend_key = format!("{}:{}", upstream.host, upstream.port);
+        let reconnect_start = std::time::Instant::now();
         if upstream.use_tls {
             match tokio::time::timeout(
                 self.connect_timeout,
                 connect_tls_backend(&self.backend_tls_config, &upstream.host, upstream.port),
             ).await {
-                Ok(Ok(tls)) => Some(BackendStream::Tls(tls)),
+                Ok(Ok(tls)) => {
+                    self.metrics.backend_connection_opened(&backend_key, reconnect_start.elapsed());
+                    Some(BackendStream::Tls(tls))
+                }
                 Ok(Err(e)) => {
-                    error!("H1 fallback: TLS reconnect failed for {}:{}: {}", upstream.host, upstream.port, e);
+                    error!(backend = %backend_key, domain = %domain, error = %e, "H1 fallback: TLS reconnect failed");
+                    self.metrics.backend_connect_error(&backend_key);
                     None
                 }
                 Err(_) => {
-                    error!("H1 fallback: TLS reconnect timeout for {}:{}", upstream.host, upstream.port);
+                    error!(backend = %backend_key, domain = %domain, "H1 fallback: TLS reconnect timeout");
+                    self.metrics.backend_connect_error(&backend_key);
                     None
                 }
             }
@@ -1250,14 +1440,17 @@ impl HttpProxyService {
                     let _ = socket2::SockRef::from(&s).set_tcp_keepalive(
                         &socket2::TcpKeepalive::new().with_time(std::time::Duration::from_secs(60))
                     );
+                    self.metrics.backend_connection_opened(&backend_key, reconnect_start.elapsed());
                     Some(BackendStream::Plain(s))
                 }
                 Ok(Err(e)) => {
-                    error!("H1 fallback: reconnect failed for {}:{}: {}", upstream.host, upstream.port, e);
+                    error!(backend = %backend_key, domain = %domain, error = %e, "H1 fallback: TCP reconnect failed");
+                    self.metrics.backend_connect_error(&backend_key);
                     None
                 }
                 Err(_) => {
-                    error!("H1 fallback: reconnect timeout for {}:{}", upstream.host, upstream.port);
+                    error!(backend = %backend_key, domain = %domain, "H1 fallback: TCP reconnect timeout");
+                    self.metrics.backend_connect_error(&backend_key);
                     None
                 }
             }
@@ -1276,10 +1469,23 @@ impl HttpProxyService {
         route_id: Option<&str>,
         source_ip: &str,
         pool_key: Option<&crate::connection_pool::PoolKey>,
+        domain: &str,
     ) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
+        // Build absolute URI for H2 pseudo-headers (:scheme, :authority)
+        // Use the requested domain as authority (not backend address) so :authority matches Host header
+        let scheme = if pool_key.map(|pk| pk.use_tls).unwrap_or(false) { "https" } else { "http" };
+        let authority = if domain != "-" { domain } else {
+            pool_key.map(|pk| pk.host.as_str()).unwrap_or("localhost")
+        };
+        let h2_uri = format!("{}://{}{}", scheme, authority, upstream_path);
         let mut upstream_req = Request::builder()
             .method(parts.method)
-            .uri(upstream_path);
+            .uri(&h2_uri);
+
+        // Remove Host header for H2 — :authority pseudo-header (from URI) is sufficient
+        // Having both Host and :authority causes nginx to return 400
+        let mut upstream_headers = upstream_headers;
+        upstream_headers.remove("host");
 
         if let Some(headers) = upstream_req.headers_mut() {
             *headers = upstream_headers;
@@ -1300,10 +1506,14 @@ impl HttpProxyService {
         let upstream_response = match sender.send_request(upstream_req).await {
             Ok(resp) => resp,
             Err(e) => {
-                error!("HTTP/2 upstream request failed: {}", e);
                 // Evict the dead sender so subsequent requests get fresh connections
                 if let Some(key) = pool_key {
+                    let bk = format!("{}:{}", key.host, key.port);
+                    error!(backend = %bk, domain = %domain, error = %e, "Backend H2 request failed");
+                    self.metrics.backend_request_error(&bk);
                     self.connection_pool.remove_h2(key);
+                } else {
+                    error!(domain = %domain, error = %e, "Backend H2 request failed");
                 }
                 return Ok(error_response(StatusCode::BAD_GATEWAY, "Backend H2 request failed"));
             }

rust/crates/rustproxy-metrics/src/collector.rs

@@ -3,6 +3,7 @@ use serde::{Deserialize, Serialize};
 use std::collections::HashSet;
 use std::sync::atomic::{AtomicU64, Ordering};
 use std::sync::Mutex;
+use std::time::Duration;
 
 use crate::throughput::{ThroughputSample, ThroughputTracker};
 
@@ -20,6 +21,7 @@ pub struct Metrics {
     pub throughput_recent_out_bytes_per_sec: u64,
     pub routes: std::collections::HashMap<String, RouteMetrics>,
     pub ips: std::collections::HashMap<String, IpMetrics>,
+    pub backends: std::collections::HashMap<String, BackendMetrics>,
     pub throughput_history: Vec<ThroughputSample>,
     pub total_http_requests: u64,
     pub http_requests_per_sec: u64,
@@ -52,6 +54,23 @@ pub struct IpMetrics {
     pub throughput_out_bytes_per_sec: u64,
 }
 
+/// Per-backend metrics (keyed by "host:port").
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct BackendMetrics {
+    pub active_connections: u64,
+    pub total_connections: u64,
+    pub protocol: String,
+    pub connect_errors: u64,
+    pub handshake_errors: u64,
+    pub request_errors: u64,
+    pub total_connect_time_us: u64,
+    pub connect_count: u64,
+    pub pool_hits: u64,
+    pub pool_misses: u64,
+    pub h2_failures: u64,
+}
+
 /// Statistics snapshot.
 #[derive(Debug, Clone, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
@@ -69,6 +88,9 @@ const DEFAULT_RETENTION_SECONDS: usize = 3600;
 /// Maximum number of IPs to include in a snapshot (top by active connections).
 const MAX_IPS_IN_SNAPSHOT: usize = 100;
 
+/// Maximum number of backends to include in a snapshot (top by total connections).
+const MAX_BACKENDS_IN_SNAPSHOT: usize = 100;
+
 /// Metrics collector tracking connections and throughput.
 ///
 /// Design: The hot path (`record_bytes`) is entirely lock-free — it only touches
@@ -96,6 +118,19 @@ pub struct MetricsCollector {
     ip_pending_tp: DashMap<String, (AtomicU64, AtomicU64)>,
     ip_throughput: DashMap<String, Mutex<ThroughputTracker>>,
 
+    // ── Per-backend tracking (keyed by "host:port") ──
+    backend_active: DashMap<String, AtomicU64>,
+    backend_total: DashMap<String, AtomicU64>,
+    backend_protocol: DashMap<String, String>,
+    backend_connect_errors: DashMap<String, AtomicU64>,
+    backend_handshake_errors: DashMap<String, AtomicU64>,
+    backend_request_errors: DashMap<String, AtomicU64>,
+    backend_connect_time_us: DashMap<String, AtomicU64>,
+    backend_connect_count: DashMap<String, AtomicU64>,
+    backend_pool_hits: DashMap<String, AtomicU64>,
+    backend_pool_misses: DashMap<String, AtomicU64>,
+    backend_h2_failures: DashMap<String, AtomicU64>,
+
     // ── HTTP request tracking ──
     total_http_requests: AtomicU64,
     pending_http_requests: AtomicU64,
@@ -134,6 +169,17 @@ impl MetricsCollector {
             ip_bytes_out: DashMap::new(),
             ip_pending_tp: DashMap::new(),
             ip_throughput: DashMap::new(),
+            backend_active: DashMap::new(),
+            backend_total: DashMap::new(),
+            backend_protocol: DashMap::new(),
+            backend_connect_errors: DashMap::new(),
+            backend_handshake_errors: DashMap::new(),
+            backend_request_errors: DashMap::new(),
+            backend_connect_time_us: DashMap::new(),
+            backend_connect_count: DashMap::new(),
+            backend_pool_hits: DashMap::new(),
+            backend_pool_misses: DashMap::new(),
+            backend_h2_failures: DashMap::new(),
             total_http_requests: AtomicU64::new(0),
             pending_http_requests: AtomicU64::new(0),
             http_request_throughput: Mutex::new(ThroughputTracker::new(retention_seconds)),
@@ -268,6 +314,113 @@ impl MetricsCollector {
         self.pending_http_requests.fetch_add(1, Ordering::Relaxed);
     }
 
+    // ── Per-backend recording methods ──
+
+    /// Record a successful backend connection with its connect duration.
+    pub fn backend_connection_opened(&self, key: &str, connect_time: Duration) {
+        self.backend_active
+            .entry(key.to_string())
+            .or_insert_with(|| AtomicU64::new(0))
+            .fetch_add(1, Ordering::Relaxed);
+        self.backend_total
+            .entry(key.to_string())
+            .or_insert_with(|| AtomicU64::new(0))
+            .fetch_add(1, Ordering::Relaxed);
+        self.backend_connect_time_us
+            .entry(key.to_string())
+            .or_insert_with(|| AtomicU64::new(0))
+            .fetch_add(connect_time.as_micros() as u64, Ordering::Relaxed);
+        self.backend_connect_count
+            .entry(key.to_string())
+            .or_insert_with(|| AtomicU64::new(0))
+            .fetch_add(1, Ordering::Relaxed);
+    }
+
+    /// Record a backend connection closing.
+    pub fn backend_connection_closed(&self, key: &str) {
+        if let Some(counter) = self.backend_active.get(key) {
+            let val = counter.load(Ordering::Relaxed);
+            if val > 0 {
+                counter.fetch_sub(1, Ordering::Relaxed);
+            }
+        }
+    }
+
+    /// Record a backend connect error (TCP or TLS connect failure/timeout).
+    pub fn backend_connect_error(&self, key: &str) {
+        self.backend_connect_errors
+            .entry(key.to_string())
+            .or_insert_with(|| AtomicU64::new(0))
+            .fetch_add(1, Ordering::Relaxed);
+    }
+
+    /// Record a backend handshake error (H1 or H2 handshake failure).
+    pub fn backend_handshake_error(&self, key: &str) {
+        self.backend_handshake_errors
+            .entry(key.to_string())
+            .or_insert_with(|| AtomicU64::new(0))
+            .fetch_add(1, Ordering::Relaxed);
+    }
+
+    /// Record a backend request error (send_request failure).
+    pub fn backend_request_error(&self, key: &str) {
+        self.backend_request_errors
+            .entry(key.to_string())
+            .or_insert_with(|| AtomicU64::new(0))
+            .fetch_add(1, Ordering::Relaxed);
+    }
+
+    /// Record a connection pool hit for a backend.
+    pub fn backend_pool_hit(&self, key: &str) {
+        self.backend_pool_hits
+            .entry(key.to_string())
+            .or_insert_with(|| AtomicU64::new(0))
+            .fetch_add(1, Ordering::Relaxed);
+    }
+
+    /// Record a connection pool miss for a backend.
+    pub fn backend_pool_miss(&self, key: &str) {
+        self.backend_pool_misses
+            .entry(key.to_string())
+            .or_insert_with(|| AtomicU64::new(0))
+            .fetch_add(1, Ordering::Relaxed);
+    }
+
+    /// Record an H2 failure (h2 attempted but fell back to h1).
+    pub fn backend_h2_failure(&self, key: &str) {
+        self.backend_h2_failures
+            .entry(key.to_string())
+            .or_insert_with(|| AtomicU64::new(0))
+            .fetch_add(1, Ordering::Relaxed);
+    }
+
+    /// Set the protocol in use for a backend ("h1" or "h2").
+    pub fn set_backend_protocol(&self, key: &str, protocol: &str) {
+        self.backend_protocol
+            .entry(key.to_string())
+            .and_modify(|v| {
+                if v != protocol {
+                    *v = protocol.to_string();
+                }
+            })
+            .or_insert_with(|| protocol.to_string());
+    }
+
+    /// Remove per-backend metrics for backends no longer in any route target.
+    pub fn retain_backends(&self, active_backends: &HashSet<String>) {
+        self.backend_active.retain(|k, _| active_backends.contains(k));
+        self.backend_total.retain(|k, _| active_backends.contains(k));
+        self.backend_protocol.retain(|k, _| active_backends.contains(k));
+        self.backend_connect_errors.retain(|k, _| active_backends.contains(k));
+        self.backend_handshake_errors.retain(|k, _| active_backends.contains(k));
+        self.backend_request_errors.retain(|k, _| active_backends.contains(k));
+        self.backend_connect_time_us.retain(|k, _| active_backends.contains(k));
+        self.backend_connect_count.retain(|k, _| active_backends.contains(k));
+        self.backend_pool_hits.retain(|k, _| active_backends.contains(k));
+        self.backend_pool_misses.retain(|k, _| active_backends.contains(k));
+        self.backend_h2_failures.retain(|k, _| active_backends.contains(k));
+    }
+
     /// Take a throughput sample on all trackers (cold path, call at 1Hz or configured interval).
     ///
     /// Drains the lock-free pending counters and feeds the accumulated bytes
@@ -488,6 +641,72 @@ impl MetricsCollector {
             });
         }
 
+        // Collect per-backend metrics, capped at top MAX_BACKENDS_IN_SNAPSHOT by total connections
+        let mut backend_entries: Vec<(String, BackendMetrics)> = Vec::new();
+        for entry in self.backend_total.iter() {
+            let key = entry.key().clone();
+            let total = entry.value().load(Ordering::Relaxed);
+            let active = self.backend_active
+                .get(&key)
+                .map(|c| c.load(Ordering::Relaxed))
+                .unwrap_or(0);
+            let protocol = self.backend_protocol
+                .get(&key)
+                .map(|v| v.value().clone())
+                .unwrap_or_else(|| "unknown".to_string());
+            let connect_errors = self.backend_connect_errors
+                .get(&key)
+                .map(|c| c.load(Ordering::Relaxed))
+                .unwrap_or(0);
+            let handshake_errors = self.backend_handshake_errors
+                .get(&key)
+                .map(|c| c.load(Ordering::Relaxed))
+                .unwrap_or(0);
+            let request_errors = self.backend_request_errors
+                .get(&key)
+                .map(|c| c.load(Ordering::Relaxed))
+                .unwrap_or(0);
+            let total_connect_time_us = self.backend_connect_time_us
+                .get(&key)
+                .map(|c| c.load(Ordering::Relaxed))
+                .unwrap_or(0);
+            let connect_count = self.backend_connect_count
+                .get(&key)
+                .map(|c| c.load(Ordering::Relaxed))
+                .unwrap_or(0);
+            let pool_hits = self.backend_pool_hits
+                .get(&key)
+                .map(|c| c.load(Ordering::Relaxed))
+                .unwrap_or(0);
+            let pool_misses = self.backend_pool_misses
+                .get(&key)
+                .map(|c| c.load(Ordering::Relaxed))
+                .unwrap_or(0);
+            let h2_failures = self.backend_h2_failures
+                .get(&key)
+                .map(|c| c.load(Ordering::Relaxed))
+                .unwrap_or(0);
+
+            backend_entries.push((key, BackendMetrics {
+                active_connections: active,
+                total_connections: total,
+                protocol,
+                connect_errors,
+                handshake_errors,
+                request_errors,
+                total_connect_time_us,
+                connect_count,
+                pool_hits,
+                pool_misses,
+                h2_failures,
+            }));
+        }
+        // Sort by total connections descending, then cap
+        backend_entries.sort_by(|a, b| b.1.total_connections.cmp(&a.1.total_connections));
+        backend_entries.truncate(MAX_BACKENDS_IN_SNAPSHOT);
+
+        let backends: std::collections::HashMap<String, BackendMetrics> = backend_entries.into_iter().collect();
+
         // HTTP request rates
         let (http_rps, http_rps_recent) = self.http_request_throughput
             .lock()
@@ -509,6 +728,7 @@ impl MetricsCollector {
             throughput_recent_out_bytes_per_sec: global_recent_out,
             routes,
             ips,
+            backends,
             throughput_history,
             total_http_requests: self.total_http_requests.load(Ordering::Relaxed),
             http_requests_per_sec: http_rps,
@@ -805,4 +1025,120 @@ mod tests {
         assert_eq!(snapshot.throughput_history[0].bytes_in, 100);
         assert_eq!(snapshot.throughput_history[4].bytes_in, 500);
     }
+
+    #[test]
+    fn test_backend_metrics_basic() {
+        let collector = MetricsCollector::new();
+        let key = "backend1:8080";
+
+        // Open connections with timing
+        collector.backend_connection_opened(key, Duration::from_millis(15));
+        collector.backend_connection_opened(key, Duration::from_millis(25));
+
+        assert_eq!(collector.backend_active.get(key).unwrap().load(Ordering::Relaxed), 2);
+        assert_eq!(collector.backend_total.get(key).unwrap().load(Ordering::Relaxed), 2);
+        assert_eq!(collector.backend_connect_count.get(key).unwrap().load(Ordering::Relaxed), 2);
+        // 15ms + 25ms = 40ms = 40_000us
+        assert_eq!(collector.backend_connect_time_us.get(key).unwrap().load(Ordering::Relaxed), 40_000);
+
+        // Close one
+        collector.backend_connection_closed(key);
+        assert_eq!(collector.backend_active.get(key).unwrap().load(Ordering::Relaxed), 1);
+        // total stays
+        assert_eq!(collector.backend_total.get(key).unwrap().load(Ordering::Relaxed), 2);
+
+        // Record errors
+        collector.backend_connect_error(key);
+        collector.backend_handshake_error(key);
+        collector.backend_request_error(key);
+        collector.backend_h2_failure(key);
+        collector.backend_pool_hit(key);
+        collector.backend_pool_hit(key);
+        collector.backend_pool_miss(key);
+
+        assert_eq!(collector.backend_connect_errors.get(key).unwrap().load(Ordering::Relaxed), 1);
+        assert_eq!(collector.backend_handshake_errors.get(key).unwrap().load(Ordering::Relaxed), 1);
+        assert_eq!(collector.backend_request_errors.get(key).unwrap().load(Ordering::Relaxed), 1);
+        assert_eq!(collector.backend_h2_failures.get(key).unwrap().load(Ordering::Relaxed), 1);
+        assert_eq!(collector.backend_pool_hits.get(key).unwrap().load(Ordering::Relaxed), 2);
+        assert_eq!(collector.backend_pool_misses.get(key).unwrap().load(Ordering::Relaxed), 1);
+
+        // Protocol
+        collector.set_backend_protocol(key, "h1");
+        assert_eq!(collector.backend_protocol.get(key).unwrap().value(), "h1");
+        collector.set_backend_protocol(key, "h2");
+        assert_eq!(collector.backend_protocol.get(key).unwrap().value(), "h2");
+    }
+
+    #[test]
+    fn test_backend_metrics_in_snapshot() {
+        let collector = MetricsCollector::new();
+
+        collector.backend_connection_opened("b1:443", Duration::from_millis(10));
+        collector.backend_connection_opened("b2:8080", Duration::from_millis(20));
+        collector.set_backend_protocol("b1:443", "h2");
+        collector.set_backend_protocol("b2:8080", "h1");
+        collector.backend_connect_error("b1:443");
+
+        let snapshot = collector.snapshot();
+        assert_eq!(snapshot.backends.len(), 2);
+
+        let b1 = snapshot.backends.get("b1:443").unwrap();
+        assert_eq!(b1.active_connections, 1);
+        assert_eq!(b1.total_connections, 1);
+        assert_eq!(b1.protocol, "h2");
+        assert_eq!(b1.connect_errors, 1);
+        assert_eq!(b1.total_connect_time_us, 10_000);
+        assert_eq!(b1.connect_count, 1);
+
+        let b2 = snapshot.backends.get("b2:8080").unwrap();
+        assert_eq!(b2.protocol, "h1");
+        assert_eq!(b2.connect_errors, 0);
+    }
+
+    #[test]
+    fn test_retain_backends_prunes_stale() {
+        let collector = MetricsCollector::new();
+
+        collector.backend_connection_opened("active:443", Duration::from_millis(5));
+        collector.backend_connection_opened("stale:8080", Duration::from_millis(10));
+        collector.set_backend_protocol("active:443", "h1");
+        collector.set_backend_protocol("stale:8080", "h2");
+        collector.backend_connect_error("stale:8080");
+
+        let active = HashSet::from(["active:443".to_string()]);
+        collector.retain_backends(&active);
+
+        // active:443 should still exist
+        assert!(collector.backend_total.get("active:443").is_some());
+        assert!(collector.backend_protocol.get("active:443").is_some());
+
+        // stale:8080 should be fully removed
+        assert!(collector.backend_active.get("stale:8080").is_none());
+        assert!(collector.backend_total.get("stale:8080").is_none());
+        assert!(collector.backend_protocol.get("stale:8080").is_none());
+        assert!(collector.backend_connect_errors.get("stale:8080").is_none());
+        assert!(collector.backend_connect_time_us.get("stale:8080").is_none());
+        assert!(collector.backend_connect_count.get("stale:8080").is_none());
+        assert!(collector.backend_pool_hits.get("stale:8080").is_none());
+        assert!(collector.backend_pool_misses.get("stale:8080").is_none());
+        assert!(collector.backend_h2_failures.get("stale:8080").is_none());
+    }
+
+    #[test]
+    fn test_backend_connection_closed_saturates() {
+        let collector = MetricsCollector::new();
+        let key = "b:80";
+
+        // Close without opening — should not underflow
+        collector.backend_connection_closed(key);
+        // No entry created
+        assert!(collector.backend_active.get(key).is_none());
+
+        // Open one, close two — should saturate at 0
+        collector.backend_connection_opened(key, Duration::from_millis(1));
+        collector.backend_connection_closed(key);
+        collector.backend_connection_closed(key);
+        assert_eq!(collector.backend_active.get(key).unwrap().load(Ordering::Relaxed), 0);
+    }
 }

rust/crates/rustproxy/src/lib.rs

@@ -603,6 +603,31 @@ impl RustProxy {
             .collect();
         self.metrics.retain_routes(&active_route_ids);
 
+        // Prune per-backend metrics for backends no longer in any route target.
+        // For PortSpec::Preserve routes, expand across all listening ports since
+        // the actual runtime port depends on the incoming connection.
+        let listening_ports = self.get_listening_ports();
+        let active_backends: HashSet<String> = routes.iter()
+            .filter_map(|r| r.action.targets.as_ref())
+            .flat_map(|targets| targets.iter())
+            .flat_map(|target| {
+                let hosts: Vec<String> = target.host.to_vec().into_iter().map(|s| s.to_string()).collect();
+                match &target.port {
+                    rustproxy_config::PortSpec::Fixed(p) => {
+                        hosts.into_iter().map(|h| format!("{}:{}", h, p)).collect::<Vec<_>>()
+                    }
+                    _ => {
+                        // Preserve/special: expand across all listening ports
+                        let lp = &listening_ports;
+                        hosts.into_iter()
+                            .flat_map(|h| lp.iter().map(move |p| format!("{}:{}", h, *p)))
+                            .collect::<Vec<_>>()
+                    }
+                }
+            })
+            .collect();
+        self.metrics.retain_backends(&active_backends);
+
         // Atomically swap the route table
         let new_manager = Arc::new(new_manager);
         self.route_table.store(Arc::clone(&new_manager));

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@push.rocks/smartproxy',
-  version: '25.9.3',
+  version: '25.10.7',
   description: 'A powerful proxy package with unified route-based configuration for high traffic management. Features include SSL/TLS support, flexible routing patterns, WebSocket handling, advanced security options, and automatic ACME certificate management.'
 }

ts/proxies/smart-proxy/models/metrics-types.ts

@@ -67,6 +67,13 @@ export interface IMetrics {
     connections(): number;
   };
   
+  // Backend metrics
+  backends: {
+    byBackend(): Map<string, IBackendMetrics>;
+    protocols(): Map<string, string>;
+    topByErrors(limit?: number): Array<{ backend: string; errors: number }>;
+  };
+
   // Performance metrics
   percentiles: {
     connectionDuration(): { p50: number; p95: number; p99: number };
@@ -98,6 +105,21 @@ export interface IMetricsConfig {
   prometheusPrefix: string;        // Default: smartproxy_
 }
 
+/**
+ * Per-backend metrics
+ */
+export interface IBackendMetrics {
+  protocol: string;
+  activeConnections: number;
+  totalConnections: number;
+  connectErrors: number;
+  handshakeErrors: number;
+  requestErrors: number;
+  avgConnectTimeMs: number;
+  poolHitRate: number;
+  h2Failures: number;
+}
+
 /**
  * Internal interface for connection byte tracking
  */

ts/proxies/smart-proxy/rust-metrics-adapter.ts

@@ -1,4 +1,4 @@
-import type { IMetrics, IThroughputData, IThroughputHistoryPoint } from './models/metrics-types.js';
+import type { IMetrics, IBackendMetrics, IThroughputData, IThroughputHistoryPoint } from './models/metrics-types.js';
 import type { RustProxyBridge } from './rust-proxy-bridge.js';
 
 /**
@@ -169,6 +169,55 @@ export class RustMetricsAdapter implements IMetrics {
     },
   };
 
+  public backends = {
+    byBackend: (): Map<string, IBackendMetrics> => {
+      const result = new Map<string, IBackendMetrics>();
+      if (this.cache?.backends) {
+        for (const [key, bm] of Object.entries(this.cache.backends)) {
+          const m = bm as any;
+          const totalTimeUs = m.totalConnectTimeUs ?? 0;
+          const count = m.connectCount ?? 0;
+          const poolHits = m.poolHits ?? 0;
+          const poolMisses = m.poolMisses ?? 0;
+          const poolTotal = poolHits + poolMisses;
+          result.set(key, {
+            protocol: m.protocol ?? 'unknown',
+            activeConnections: m.activeConnections ?? 0,
+            totalConnections: m.totalConnections ?? 0,
+            connectErrors: m.connectErrors ?? 0,
+            handshakeErrors: m.handshakeErrors ?? 0,
+            requestErrors: m.requestErrors ?? 0,
+            avgConnectTimeMs: count > 0 ? (totalTimeUs / count) / 1000 : 0,
+            poolHitRate: poolTotal > 0 ? poolHits / poolTotal : 0,
+            h2Failures: m.h2Failures ?? 0,
+          });
+        }
+      }
+      return result;
+    },
+    protocols: (): Map<string, string> => {
+      const result = new Map<string, string>();
+      if (this.cache?.backends) {
+        for (const [key, bm] of Object.entries(this.cache.backends)) {
+          result.set(key, (bm as any).protocol ?? 'unknown');
+        }
+      }
+      return result;
+    },
+    topByErrors: (limit: number = 10): Array<{ backend: string; errors: number }> => {
+      const result: Array<{ backend: string; errors: number }> = [];
+      if (this.cache?.backends) {
+        for (const [key, bm] of Object.entries(this.cache.backends)) {
+          const m = bm as any;
+          const errors = (m.connectErrors ?? 0) + (m.handshakeErrors ?? 0) + (m.requestErrors ?? 0);
+          if (errors > 0) result.push({ backend: key, errors });
+        }
+      }
+      result.sort((a, b) => b.errors - a.errors);
+      return result.slice(0, limit);
+    },
+  };
+
   public percentiles = {
     connectionDuration: (): { p50: number; p95: number; p99: number } => {
       return { p50: 0, p95: 0, p99: 0 };