v25.11.19

changelog.md

@@ -1,5 +1,50 @@
 # Changelog
 
+## 2026-03-16 - 25.11.19 - fix(rustproxy-http)
+avoid reusing pooled HTTP/2 connections for requests with bodies to prevent upload flow-control stalls
+
+- Limit HTTP/2 pool checkout to bodyless requests such as GET, HEAD, and DELETE
+- Skip re-registering HTTP/2 connections in the pool after requests that send a body
+- Prevent stalled uploads caused by depleted connection-level flow control windows on reused HTTP/2 connections
+
+## 2026-03-16 - 25.11.18 - fix(repo)
+no changes to commit
+
+
+## 2026-03-16 - 25.11.17 - fix(rustproxy-http)
+prevent stale HTTP/2 connection drivers from evicting newer pooled connections
+
+- add generation IDs to pooled HTTP/2 senders so pool removal only affects the matching connection
+- update HTTP/2 proxy and retry paths to register generation-tagged connections and skip eviction before registration completes
+
+## 2026-03-16 - 25.11.16 - fix(repo)
+no changes to commit
+
+
+## 2026-03-16 - 25.11.15 - fix(rustproxy-http)
+implement vectored write support for backend streams
+
+- Add poll_write_vectored forwarding for both plain and TLS backend stream variants
+- Expose is_write_vectored so the proxy can correctly report vectored write capability
+
+## 2026-03-16 - 25.11.14 - fix(rustproxy-http)
+forward vectored write support in ShutdownOnDrop AsyncWrite wrapper
+
+- Implements poll_write_vectored by delegating to the wrapped writer
+- Exposes is_write_vectored so the wrapper preserves underlying AsyncWrite capabilities
+
+## 2026-03-16 - 25.11.13 - fix(rustproxy-http)
+remove hot-path debug logging from HTTP/1 connection pool hits
+
+- Stops emitting debug logs when reusing HTTP/1 idle connections in the connection pool.
+- Keeps pool hit behavior unchanged while reducing overhead on a frequently executed path.
+
+## 2026-03-16 - 25.11.12 - fix(rustproxy-http)
+remove connection pool hit logging and keep logging limited to actual failures
+
+- Removes debug and warning logs for HTTP/2 connection pool hits and age checks.
+- Keeps pool behavior unchanged while reducing noisy per-request logging in the Rust HTTP proxy layer.
+
 ## 2026-03-16 - 25.11.11 - fix(rustproxy-http)
 improve HTTP/2 proxy error logging with warning-level connection failures and debug error details
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@push.rocks/smartproxy",
-  "version": "25.11.11",
+  "version": "25.11.19",
   "private": false,
   "description": "A powerful proxy package with unified route-based configuration for high traffic management. Features include SSL/TLS support, flexible routing patterns, WebSocket handling, advanced security options, and automatic ACME certificate management.",
   "main": "dist_ts/index.js",

rust/crates/rustproxy-http/src/connection_pool.rs

@@ -4,13 +4,13 @@
 //! HTTP/2 connections are multiplexed (clone the sender for each request).
 
 use std::sync::Arc;
+use std::sync::atomic::{AtomicU64, Ordering};
 use std::time::{Duration, Instant};
 
 use bytes::Bytes;
 use dashmap::DashMap;
 use http_body_util::combinators::BoxBody;
 use hyper::client::conn::{http1, http2};
-use tracing::{debug, warn};
 
 /// Maximum idle connections per backend key.
 const MAX_IDLE_PER_KEY: usize = 16;
@@ -38,10 +38,13 @@ struct IdleH1 {
     idle_since: Instant,
 }
 
-/// A pooled HTTP/2 sender (multiplexed, Clone-able).
+/// A pooled HTTP/2 sender (multiplexed, Clone-able) with a generation tag.
 struct PooledH2 {
     sender: http2::SendRequest<BoxBody<Bytes, hyper::Error>>,
     created_at: Instant,
+    /// Unique generation ID. Connection drivers use this to only remove their OWN
+    /// entry, preventing phantom eviction when multiple connections share the same key.
+    generation: u64,
 }
 
 /// Backend connection pool.
@@ -50,6 +53,8 @@ pub struct ConnectionPool {
     h1_pool: Arc<DashMap<PoolKey, Vec<IdleH1>>>,
     /// HTTP/2 multiplexed connections indexed by backend key.
     h2_pool: Arc<DashMap<PoolKey, PooledH2>>,
+    /// Monotonic generation counter for H2 pool entries.
+    h2_generation: AtomicU64,
     /// Handle for the background eviction task.
     eviction_handle: Option<tokio::task::JoinHandle<()>>,
 }
@@ -69,6 +74,7 @@ impl ConnectionPool {
         Self {
             h1_pool,
             h2_pool,
+            h2_generation: AtomicU64::new(0),
             eviction_handle: Some(eviction_handle),
         }
     }
@@ -82,7 +88,7 @@ impl ConnectionPool {
         while let Some(idle) = idles.pop() {
             // Check if the connection is still alive and ready
             if idle.idle_since.elapsed() < IDLE_TIMEOUT && idle.sender.is_ready() && !idle.sender.is_closed() {
-                debug!("Pool hit (h1): {}:{}", key.host, key.port);
+                // H1 pool hit — no logging on hot path
                 return Some(idle.sender);
             }
             // Stale or closed — drop it
@@ -120,42 +126,51 @@ impl ConnectionPool {
         let pooled = entry.value();
         let age = pooled.created_at.elapsed();
 
-        // Check if the h2 connection is still alive and not too old
         if pooled.sender.is_closed() || age >= MAX_H2_AGE {
-            let reason = if pooled.sender.is_closed() { "closed" } else { "max_age" };
-            debug!("Pool evict (h2): {}:{} (reason={}, age={:.1}s)", key.host, key.port, reason, age.as_secs_f64());
             drop(entry);
             self.h2_pool.remove(key);
             return None;
         }
 
         if pooled.sender.is_ready() {
-            if age > Duration::from_secs(30) {
-                warn!("Pool hit (h2): {}:{} — connection age {:.1}s (>30s, may be stale)", key.host, key.port, age.as_secs_f64());
-            } else {
-                debug!("Pool hit (h2): {}:{} (age={:.1}s)", key.host, key.port, age.as_secs_f64());
-            }
             return Some((pooled.sender.clone(), age));
         }
         None
     }
 
-    /// Remove a dead HTTP/2 sender from the pool.
+    /// Remove a dead HTTP/2 sender from the pool (unconditional).
     /// Called when `send_request` fails to prevent subsequent requests from reusing the stale sender.
     pub fn remove_h2(&self, key: &PoolKey) {
         self.h2_pool.remove(key);
     }
 
-    /// Register an HTTP/2 sender in the pool. Since h2 is multiplexed,
-    /// only one sender per key is stored (it's Clone-able).
-    pub fn register_h2(&self, key: PoolKey, sender: http2::SendRequest<BoxBody<Bytes, hyper::Error>>) {
+    /// Remove an HTTP/2 sender ONLY if the current entry has the expected generation.
+    /// This prevents phantom eviction: when multiple connections share the same key,
+    /// an old connection's driver won't accidentally remove a newer connection's entry.
+    pub fn remove_h2_if_generation(&self, key: &PoolKey, expected_gen: u64) {
+        if let Some(entry) = self.h2_pool.get(key) {
+            if entry.value().generation == expected_gen {
+                drop(entry); // release DashMap ref before remove
+                self.h2_pool.remove(key);
+            }
+            // else: a newer connection replaced ours — don't touch it
+        }
+    }
+
+    /// Register an HTTP/2 sender in the pool. Returns the generation ID for this entry.
+    /// The caller should pass this generation to the connection driver so it can use
+    /// `remove_h2_if_generation` instead of `remove_h2` to avoid phantom eviction.
+    pub fn register_h2(&self, key: PoolKey, sender: http2::SendRequest<BoxBody<Bytes, hyper::Error>>) -> u64 {
+        let gen = self.h2_generation.fetch_add(1, Ordering::Relaxed);
         if sender.is_closed() {
-            return;
+            return gen;
         }
         self.h2_pool.insert(key, PooledH2 {
             sender,
             created_at: Instant::now(),
+            generation: gen,
         });
+        gen
     }
 
     /// Background eviction loop — runs every EVICTION_INTERVAL to remove stale connections.

rust/crates/rustproxy-http/src/proxy_service.rs

@@ -109,6 +109,24 @@ impl tokio::io::AsyncWrite for BackendStream {
         }
     }
 
+    fn poll_write_vectored(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+        bufs: &[std::io::IoSlice<'_>],
+    ) -> Poll<std::io::Result<usize>> {
+        match self.get_mut() {
+            BackendStream::Plain(s) => Pin::new(s).poll_write_vectored(cx, bufs),
+            BackendStream::Tls(s) => Pin::new(s).poll_write_vectored(cx, bufs),
+        }
+    }
+
+    fn is_write_vectored(&self) -> bool {
+        match self {
+            BackendStream::Plain(s) => s.is_write_vectored(),
+            BackendStream::Tls(s) => s.is_write_vectored(),
+        }
+    }
+
     fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
         match self.get_mut() {
             BackendStream::Plain(s) => Pin::new(s).poll_flush(cx),
@@ -659,12 +677,20 @@ impl HttpProxyService {
                 h2: use_h2,
             };
 
-            // H2 pool checkout with async readiness validation.
-            // checkout_h2 does synchronous is_closed()/is_ready() checks, but these
-            // reflect cached state — the H2 connection driver (a separate tokio task)
-            // may not have processed a pending GOAWAY/RST yet. The ready().await
-            // forces the runtime to yield, giving the driver a chance to detect failures.
-            if use_h2 {
+            // H2 pool checkout — only for bodyless requests (GET/HEAD/DELETE).
+            //
+            // WORKAROUND: Requests with bodies (POST/PUT uploads) always get fresh H2
+            // connections. Reusing a pooled H2 connection after a large upload can stall
+            // forever due to depleted connection-level flow control windows. The h2 crate
+            // has no stall/timeout detection (https://github.com/hyperium/hyper/issues/2899),
+            // and Go/nginx HTTP/2 servers have known issues with connection-level window
+            // replenishment after large transfers (https://github.com/golang/go/issues/16481,
+            // https://github.com/golang/go/issues/56558). A fresh connection guarantees
+            // clean flow control state. The overhead is ~3-5ms for TLS+H2 handshake.
+            //
+            // TODO: Revisit once h2 crate adds flow control stall detection, or once
+            // Go/nginx H2 connection-level window handling is confirmed reliable.
+            if use_h2 && body.is_end_stream() {
                 if let Some((mut sender, age)) = self.connection_pool.checkout_h2(&pool_key) {
                     match tokio::time::timeout(
                         std::time::Duration::from_millis(500),
@@ -1001,24 +1027,35 @@ impl HttpProxyService {
             }
         };
 
-        // Spawn the H2 connection driver; proactively evict from pool on exit
-        // so the next request gets a fresh connection instead of a dead sender.
+        // Shared generation ID: driver reads it after registration sets it.
+        // Uses u64::MAX as sentinel for "not yet registered" (driver waits/skips eviction).
+        let gen_holder = Arc::new(std::sync::atomic::AtomicU64::new(u64::MAX));
+
+        // Spawn the H2 connection driver; evict from pool on exit using generation-tagged
+        // removal to prevent phantom eviction when multiple connections share the same key.
         {
             let pool = Arc::clone(&self.connection_pool);
             let key = pool_key.clone();
+            let gen = Arc::clone(&gen_holder);
             tokio::spawn(async move {
                 if let Err(e) = conn.await {
                     warn!("HTTP/2 upstream connection error: {} ({:?})", e, e);
                 }
-                pool.remove_h2(&key);
+                let g = gen.load(std::sync::atomic::Ordering::Relaxed);
+                if g != u64::MAX {
+                    pool.remove_h2_if_generation(&key, g);
+                }
             });
         }
 
-        // Clone sender for potential pool registration; register only after first request succeeds
+        // Only pool the H2 connection if the request had no body.
+        // Requests with bodies (uploads) deplete connection-level flow control windows.
+        let request_had_body = !body.is_end_stream();
         let sender_for_pool = sender.clone();
         let result = self.forward_h2_with_sender(sender, parts, body, upstream_headers, upstream_path, route, route_id, source_ip, Some(pool_key), domain, conn_activity).await;
-        if matches!(&result, Ok(ref resp) if resp.status() != StatusCode::BAD_GATEWAY) {
-            self.connection_pool.register_h2(pool_key.clone(), sender_for_pool);
+        if !request_had_body && matches!(&result, Ok(ref resp) if resp.status() != StatusCode::BAD_GATEWAY) {
+            let g = self.connection_pool.register_h2(pool_key.clone(), sender_for_pool);
+            gen_holder.store(g, std::sync::atomic::Ordering::Relaxed);
         }
         result
     }
@@ -1153,15 +1190,20 @@ impl HttpProxyService {
             }
         };
 
-        // Spawn the H2 connection driver; proactively evict from pool on exit.
+        // Spawn the H2 connection driver with generation-tagged eviction.
+        let gen_holder = Arc::new(std::sync::atomic::AtomicU64::new(u64::MAX));
         {
             let pool = Arc::clone(&self.connection_pool);
             let key = pool_key.clone();
+            let gen = Arc::clone(&gen_holder);
             tokio::spawn(async move {
                 if let Err(e) = conn.await {
                     warn!("H2 retry: upstream connection error: {} ({:?})", e, e);
                 }
-                pool.remove_h2(&key);
+                let g = gen.load(std::sync::atomic::Ordering::Relaxed);
+                if g != u64::MAX {
+                    pool.remove_h2_if_generation(&key, g);
+                }
             });
         }
 
@@ -1189,7 +1231,8 @@ impl HttpProxyService {
         match sender.send_request(upstream_req).await {
             Ok(resp) => {
                 // Register in pool only after request succeeds
-                self.connection_pool.register_h2(pool_key.clone(), sender);
+                let g = self.connection_pool.register_h2(pool_key.clone(), sender);
+                gen_holder.store(g, std::sync::atomic::Ordering::Relaxed);
                 let result = self.build_streaming_response(resp, route, route_id, source_ip, conn_activity).await;
                 // Close the fresh backend connection (opened above)
                 self.metrics.backend_connection_closed(&backend_key);
@@ -1282,15 +1325,20 @@ impl HttpProxyService {
                 }
             }
             Ok(Ok((mut sender, conn))) => {
-                // Spawn the H2 connection driver; proactively evict from pool on exit.
+                // Spawn the H2 connection driver with generation-tagged eviction.
+                let gen_holder = Arc::new(std::sync::atomic::AtomicU64::new(u64::MAX));
                 {
                     let pool = Arc::clone(&self.connection_pool);
                     let key = pool_key.clone();
+                    let gen = Arc::clone(&gen_holder);
                     tokio::spawn(async move {
                         if let Err(e) = conn.await {
                             warn!("HTTP/2 upstream connection error: {} ({:?})", e, e);
                         }
-                        pool.remove_h2(&key);
+                        let g = gen.load(std::sync::atomic::Ordering::Relaxed);
+                        if g != u64::MAX {
+                            pool.remove_h2_if_generation(&key, g);
+                        }
                     });
                 }
 
@@ -1331,8 +1379,12 @@ impl HttpProxyService {
 
                 match sender.send_request(upstream_req).await {
                     Ok(upstream_response) => {
-                        // H2 works! Register sender in pool for multiplexed reuse
-                        self.connection_pool.register_h2(pool_key.clone(), sender);
+                        // Only pool after bodyless requests — uploads deplete connection-level
+                        // flow control windows (see comment at pool checkout above).
+                        if retry_state.is_some() {
+                            let g = self.connection_pool.register_h2(pool_key.clone(), sender);
+                            gen_holder.store(g, std::sync::atomic::Ordering::Relaxed);
+                        }
                         self.build_streaming_response(upstream_response, route, route_id, source_ip, conn_activity).await
                     }
                     Err(e) => {

rust/crates/rustproxy-http/src/shutdown_on_drop.rs

@@ -50,6 +50,18 @@ impl<S: AsyncRead + AsyncWrite + Unpin + Send + 'static> AsyncWrite for Shutdown
         Pin::new(self.get_mut().inner.as_mut().unwrap()).poll_write(cx, buf)
     }
 
+    fn poll_write_vectored(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+        bufs: &[io::IoSlice<'_>],
+    ) -> Poll<io::Result<usize>> {
+        Pin::new(self.get_mut().inner.as_mut().unwrap()).poll_write_vectored(cx, bufs)
+    }
+
+    fn is_write_vectored(&self) -> bool {
+        self.inner.as_ref().unwrap().is_write_vectored()
+    }
+
     fn poll_flush(
         self: Pin<&mut Self>,
         cx: &mut Context<'_>,

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@push.rocks/smartproxy',
-  version: '25.11.11',
+  version: '25.11.19',
   description: 'A powerful proxy package with unified route-based configuration for high traffic management. Features include SSL/TLS support, flexible routing patterns, WebSocket handling, advanced security options, and automatic ACME certificate management.'
 }