fix(rustproxy-http): prevent stale HTTP/2 connection drivers from evicting newer pooled connections

rust/crates/rustproxy-http/src/connection_pool.rs

@@ -4,13 +4,13 @@
 //! HTTP/2 connections are multiplexed (clone the sender for each request).
 
 use std::sync::Arc;
+use std::sync::atomic::{AtomicU64, Ordering};
 use std::time::{Duration, Instant};
 
 use bytes::Bytes;
 use dashmap::DashMap;
 use http_body_util::combinators::BoxBody;
 use hyper::client::conn::{http1, http2};
-// No per-request logging in the pool — only log on actual failures (in proxy_service.rs)
 
 /// Maximum idle connections per backend key.
 const MAX_IDLE_PER_KEY: usize = 16;
@@ -38,10 +38,13 @@ struct IdleH1 {
     idle_since: Instant,
 }
 
-/// A pooled HTTP/2 sender (multiplexed, Clone-able).
+/// A pooled HTTP/2 sender (multiplexed, Clone-able) with a generation tag.
 struct PooledH2 {
     sender: http2::SendRequest<BoxBody<Bytes, hyper::Error>>,
     created_at: Instant,
+    /// Unique generation ID. Connection drivers use this to only remove their OWN
+    /// entry, preventing phantom eviction when multiple connections share the same key.
+    generation: u64,
 }
 
 /// Backend connection pool.
@@ -50,6 +53,8 @@ pub struct ConnectionPool {
     h1_pool: Arc<DashMap<PoolKey, Vec<IdleH1>>>,
     /// HTTP/2 multiplexed connections indexed by backend key.
     h2_pool: Arc<DashMap<PoolKey, PooledH2>>,
+    /// Monotonic generation counter for H2 pool entries.
+    h2_generation: AtomicU64,
     /// Handle for the background eviction task.
     eviction_handle: Option<tokio::task::JoinHandle<()>>,
 }
@@ -69,6 +74,7 @@ impl ConnectionPool {
         Self {
             h1_pool,
             h2_pool,
+            h2_generation: AtomicU64::new(0),
             eviction_handle: Some(eviction_handle),
         }
     }
@@ -132,22 +138,39 @@ impl ConnectionPool {
         None
     }
 
-    /// Remove a dead HTTP/2 sender from the pool.
+    /// Remove a dead HTTP/2 sender from the pool (unconditional).
     /// Called when `send_request` fails to prevent subsequent requests from reusing the stale sender.
     pub fn remove_h2(&self, key: &PoolKey) {
         self.h2_pool.remove(key);
     }
 
-    /// Register an HTTP/2 sender in the pool. Since h2 is multiplexed,
-    /// only one sender per key is stored (it's Clone-able).
-    pub fn register_h2(&self, key: PoolKey, sender: http2::SendRequest<BoxBody<Bytes, hyper::Error>>) {
+    /// Remove an HTTP/2 sender ONLY if the current entry has the expected generation.
+    /// This prevents phantom eviction: when multiple connections share the same key,
+    /// an old connection's driver won't accidentally remove a newer connection's entry.
+    pub fn remove_h2_if_generation(&self, key: &PoolKey, expected_gen: u64) {
+        if let Some(entry) = self.h2_pool.get(key) {
+            if entry.value().generation == expected_gen {
+                drop(entry); // release DashMap ref before remove
+                self.h2_pool.remove(key);
+            }
+            // else: a newer connection replaced ours — don't touch it
+        }
+    }
+
+    /// Register an HTTP/2 sender in the pool. Returns the generation ID for this entry.
+    /// The caller should pass this generation to the connection driver so it can use
+    /// `remove_h2_if_generation` instead of `remove_h2` to avoid phantom eviction.
+    pub fn register_h2(&self, key: PoolKey, sender: http2::SendRequest<BoxBody<Bytes, hyper::Error>>) -> u64 {
+        let gen = self.h2_generation.fetch_add(1, Ordering::Relaxed);
         if sender.is_closed() {
-            return;
+            return gen;
         }
         self.h2_pool.insert(key, PooledH2 {
             sender,
             created_at: Instant::now(),
+            generation: gen,
         });
+        gen
     }
 
     /// Background eviction loop — runs every EVICTION_INTERVAL to remove stale connections.