|
|
|
|
@@ -479,6 +479,7 @@ impl HttpProxyService {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
let upstream_key = format!("{}:{}", upstream.host, upstream.port);
|
|
|
|
|
let domain_str = host.as_deref().unwrap_or("-");
|
|
|
|
|
self.upstream_selector.connection_started(&upstream_key);
|
|
|
|
|
|
|
|
|
|
// Check for WebSocket upgrade
|
|
|
|
|
@@ -538,6 +539,15 @@ impl HttpProxyService {
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Remove hop-by-hop headers (RFC 9113 §8.2.2 forbids connection-specific headers in H2)
|
|
|
|
|
upstream_headers.remove("connection");
|
|
|
|
|
upstream_headers.remove("keep-alive");
|
|
|
|
|
upstream_headers.remove("proxy-connection");
|
|
|
|
|
upstream_headers.remove("transfer-encoding");
|
|
|
|
|
upstream_headers.remove("te");
|
|
|
|
|
upstream_headers.remove("trailer");
|
|
|
|
|
upstream_headers.remove("upgrade");
|
|
|
|
|
|
|
|
|
|
// Add standard reverse-proxy headers (X-Forwarded-*)
|
|
|
|
|
{
|
|
|
|
|
let original_host = host.as_deref().unwrap_or("");
|
|
|
|
|
@@ -622,7 +632,7 @@ impl HttpProxyService {
|
|
|
|
|
self.metrics.set_backend_protocol(&upstream_key, "h2");
|
|
|
|
|
let result = self.forward_h2_pooled(
|
|
|
|
|
sender, parts, body, upstream_headers, &upstream_path,
|
|
|
|
|
route_match.route, route_id, &ip_str, &pool_key,
|
|
|
|
|
route_match.route, route_id, &ip_str, &pool_key, domain_str,
|
|
|
|
|
).await;
|
|
|
|
|
self.upstream_selector.connection_ended(&upstream_key);
|
|
|
|
|
return result;
|
|
|
|
|
@@ -633,8 +643,8 @@ impl HttpProxyService {
|
|
|
|
|
// --- Fresh connection path ---
|
|
|
|
|
self.metrics.backend_pool_miss(&upstream_key);
|
|
|
|
|
|
|
|
|
|
// Choose TLS config: use ALPN config for auto-detect probe, plain config otherwise
|
|
|
|
|
let tls_config = if needs_alpn_probe {
|
|
|
|
|
// Choose TLS config: use ALPN config when H2 is possible (auto-detect probe OR explicit H2)
|
|
|
|
|
let tls_config = if needs_alpn_probe || use_h2 {
|
|
|
|
|
&self.backend_tls_config_alpn
|
|
|
|
|
} else {
|
|
|
|
|
&self.backend_tls_config
|
|
|
|
|
@@ -668,6 +678,7 @@ impl HttpProxyService {
|
|
|
|
|
|
|
|
|
|
info!(
|
|
|
|
|
backend = %upstream_key,
|
|
|
|
|
domain = %domain_str,
|
|
|
|
|
protocol = if is_h2 { "h2" } else { "h1" },
|
|
|
|
|
connect_time_ms = %connect_start.elapsed().as_millis(),
|
|
|
|
|
"Backend protocol detected via ALPN"
|
|
|
|
|
@@ -684,6 +695,7 @@ impl HttpProxyService {
|
|
|
|
|
Ok(Err(e)) => {
|
|
|
|
|
error!(
|
|
|
|
|
backend = %upstream_key,
|
|
|
|
|
domain = %domain_str,
|
|
|
|
|
connect_time_ms = %connect_start.elapsed().as_millis(),
|
|
|
|
|
error = %e,
|
|
|
|
|
"Backend TLS connect failed"
|
|
|
|
|
@@ -695,6 +707,7 @@ impl HttpProxyService {
|
|
|
|
|
Err(_) => {
|
|
|
|
|
error!(
|
|
|
|
|
backend = %upstream_key,
|
|
|
|
|
domain = %domain_str,
|
|
|
|
|
connect_time_ms = %connect_start.elapsed().as_millis(),
|
|
|
|
|
"Backend TLS connect timeout"
|
|
|
|
|
);
|
|
|
|
|
@@ -720,6 +733,7 @@ impl HttpProxyService {
|
|
|
|
|
Ok(Err(e)) => {
|
|
|
|
|
error!(
|
|
|
|
|
backend = %upstream_key,
|
|
|
|
|
domain = %domain_str,
|
|
|
|
|
connect_time_ms = %connect_start.elapsed().as_millis(),
|
|
|
|
|
error = %e,
|
|
|
|
|
"Backend TCP connect failed"
|
|
|
|
|
@@ -731,6 +745,7 @@ impl HttpProxyService {
|
|
|
|
|
Err(_) => {
|
|
|
|
|
error!(
|
|
|
|
|
backend = %upstream_key,
|
|
|
|
|
domain = %domain_str,
|
|
|
|
|
connect_time_ms = %connect_start.elapsed().as_millis(),
|
|
|
|
|
"Backend TCP connect timeout"
|
|
|
|
|
);
|
|
|
|
|
@@ -756,19 +771,19 @@ impl HttpProxyService {
|
|
|
|
|
self.forward_h2_with_fallback(
|
|
|
|
|
io, parts, body, upstream_headers, &upstream_path,
|
|
|
|
|
&upstream, route_match.route, route_id, &ip_str, &final_pool_key,
|
|
|
|
|
host.clone(),
|
|
|
|
|
host.clone(), domain_str,
|
|
|
|
|
).await
|
|
|
|
|
} else {
|
|
|
|
|
// Explicit H2 mode: hard-fail on handshake error (preserved behavior)
|
|
|
|
|
self.forward_h2(
|
|
|
|
|
io, parts, body, upstream_headers, &upstream_path,
|
|
|
|
|
&upstream, route_match.route, route_id, &ip_str, &final_pool_key,
|
|
|
|
|
&upstream, route_match.route, route_id, &ip_str, &final_pool_key, domain_str,
|
|
|
|
|
).await
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
self.forward_h1(
|
|
|
|
|
io, parts, body, upstream_headers, &upstream_path,
|
|
|
|
|
&upstream, route_match.route, route_id, &ip_str, &final_pool_key,
|
|
|
|
|
&upstream, route_match.route, route_id, &ip_str, &final_pool_key, domain_str,
|
|
|
|
|
).await
|
|
|
|
|
};
|
|
|
|
|
self.upstream_selector.connection_ended(&upstream_key);
|
|
|
|
|
@@ -790,6 +805,7 @@ impl HttpProxyService {
|
|
|
|
|
route_id: Option<&str>,
|
|
|
|
|
source_ip: &str,
|
|
|
|
|
pool_key: &crate::connection_pool::PoolKey,
|
|
|
|
|
domain: &str,
|
|
|
|
|
) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
|
|
|
|
|
let backend_key = format!("{}:{}", pool_key.host, pool_key.port);
|
|
|
|
|
|
|
|
|
|
@@ -798,7 +814,7 @@ impl HttpProxyService {
|
|
|
|
|
self.metrics.backend_pool_hit(&backend_key);
|
|
|
|
|
return self.forward_h1_with_sender(
|
|
|
|
|
pooled_sender, parts, body, upstream_headers, upstream_path,
|
|
|
|
|
route, route_id, source_ip, pool_key,
|
|
|
|
|
route, route_id, source_ip, pool_key, domain,
|
|
|
|
|
).await;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@@ -809,7 +825,7 @@ impl HttpProxyService {
|
|
|
|
|
) = match hyper::client::conn::http1::handshake(io).await {
|
|
|
|
|
Ok(h) => h,
|
|
|
|
|
Err(e) => {
|
|
|
|
|
error!(backend = %backend_key, error = %e, "Backend H1 handshake failed");
|
|
|
|
|
error!(backend = %backend_key, domain = %domain, error = %e, "Backend H1 handshake failed");
|
|
|
|
|
self.metrics.backend_handshake_error(&backend_key);
|
|
|
|
|
return Ok(error_response(StatusCode::BAD_GATEWAY, "Backend handshake failed"));
|
|
|
|
|
}
|
|
|
|
|
@@ -821,7 +837,7 @@ impl HttpProxyService {
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
self.forward_h1_with_sender(sender, parts, body, upstream_headers, upstream_path, route, route_id, source_ip, pool_key).await
|
|
|
|
|
self.forward_h1_with_sender(sender, parts, body, upstream_headers, upstream_path, route, route_id, source_ip, pool_key, domain).await
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Common H1 forwarding logic used by both fresh and pooled paths.
|
|
|
|
|
@@ -836,6 +852,7 @@ impl HttpProxyService {
|
|
|
|
|
route_id: Option<&str>,
|
|
|
|
|
source_ip: &str,
|
|
|
|
|
pool_key: &crate::connection_pool::PoolKey,
|
|
|
|
|
domain: &str,
|
|
|
|
|
) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
|
|
|
|
|
// Always use HTTP/1.1 for h1 backend connections (h2 incoming requests have version HTTP/2.0)
|
|
|
|
|
let mut upstream_req = Request::builder()
|
|
|
|
|
@@ -863,7 +880,7 @@ impl HttpProxyService {
|
|
|
|
|
Ok(resp) => resp,
|
|
|
|
|
Err(e) => {
|
|
|
|
|
let bk = format!("{}:{}", pool_key.host, pool_key.port);
|
|
|
|
|
error!(backend = %bk, error = %e, "Backend H1 request failed");
|
|
|
|
|
error!(backend = %bk, domain = %domain, error = %e, "Backend H1 request failed");
|
|
|
|
|
self.metrics.backend_request_error(&bk);
|
|
|
|
|
return Ok(error_response(StatusCode::BAD_GATEWAY, "Backend request failed"));
|
|
|
|
|
}
|
|
|
|
|
@@ -889,20 +906,32 @@ impl HttpProxyService {
|
|
|
|
|
route_id: Option<&str>,
|
|
|
|
|
source_ip: &str,
|
|
|
|
|
pool_key: &crate::connection_pool::PoolKey,
|
|
|
|
|
domain: &str,
|
|
|
|
|
) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
|
|
|
|
|
let backend_key = format!("{}:{}", pool_key.host, pool_key.port);
|
|
|
|
|
let exec = hyper_util::rt::TokioExecutor::new();
|
|
|
|
|
// Explicitly type the handshake with BoxBody for uniform pool type
|
|
|
|
|
let mut h2_builder = hyper::client::conn::http2::Builder::new(exec);
|
|
|
|
|
h2_builder
|
|
|
|
|
.timer(hyper_util::rt::TokioTimer::new())
|
|
|
|
|
.keep_alive_interval(std::time::Duration::from_secs(10))
|
|
|
|
|
.keep_alive_timeout(std::time::Duration::from_secs(5))
|
|
|
|
|
.adaptive_window(true)
|
|
|
|
|
.initial_stream_window_size(2 * 1024 * 1024);
|
|
|
|
|
let (sender, conn): (
|
|
|
|
|
hyper::client::conn::http2::SendRequest<BoxBody<Bytes, hyper::Error>>,
|
|
|
|
|
hyper::client::conn::http2::Connection<TokioIo<BackendStream>, BoxBody<Bytes, hyper::Error>, hyper_util::rt::TokioExecutor>,
|
|
|
|
|
) = match hyper::client::conn::http2::handshake(exec, io).await {
|
|
|
|
|
Ok(h) => h,
|
|
|
|
|
Err(e) => {
|
|
|
|
|
error!(backend = %backend_key, error = %e, "Backend H2 handshake failed");
|
|
|
|
|
) = match tokio::time::timeout(self.connect_timeout, h2_builder.handshake(io)).await {
|
|
|
|
|
Ok(Ok(h)) => h,
|
|
|
|
|
Ok(Err(e)) => {
|
|
|
|
|
error!(backend = %backend_key, domain = %domain, error = %e, "Backend H2 handshake failed");
|
|
|
|
|
self.metrics.backend_handshake_error(&backend_key);
|
|
|
|
|
return Ok(error_response(StatusCode::BAD_GATEWAY, "Backend H2 handshake failed"));
|
|
|
|
|
}
|
|
|
|
|
Err(_) => {
|
|
|
|
|
error!(backend = %backend_key, domain = %domain, "Backend H2 handshake timeout");
|
|
|
|
|
self.metrics.backend_handshake_error(&backend_key);
|
|
|
|
|
return Ok(error_response(StatusCode::GATEWAY_TIMEOUT, "Backend H2 handshake timeout"));
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
tokio::spawn(async move {
|
|
|
|
|
@@ -911,10 +940,13 @@ impl HttpProxyService {
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
// Register for multiplexed reuse
|
|
|
|
|
self.connection_pool.register_h2(pool_key.clone(), sender.clone());
|
|
|
|
|
|
|
|
|
|
self.forward_h2_with_sender(sender, parts, body, upstream_headers, upstream_path, route, route_id, source_ip, Some(pool_key)).await
|
|
|
|
|
// Clone sender for potential pool registration; register only after first request succeeds
|
|
|
|
|
let sender_for_pool = sender.clone();
|
|
|
|
|
let result = self.forward_h2_with_sender(sender, parts, body, upstream_headers, upstream_path, route, route_id, source_ip, Some(pool_key), domain).await;
|
|
|
|
|
if matches!(&result, Ok(ref resp) if resp.status() != StatusCode::BAD_GATEWAY) {
|
|
|
|
|
self.connection_pool.register_h2(pool_key.clone(), sender_for_pool);
|
|
|
|
|
}
|
|
|
|
|
result
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Forward request using an existing (pooled) HTTP/2 sender.
|
|
|
|
|
@@ -931,6 +963,7 @@ impl HttpProxyService {
|
|
|
|
|
route_id: Option<&str>,
|
|
|
|
|
source_ip: &str,
|
|
|
|
|
pool_key: &crate::connection_pool::PoolKey,
|
|
|
|
|
domain: &str,
|
|
|
|
|
) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
|
|
|
|
|
// Save retry state for bodyless requests (cheap: Method is an enum, HeaderMap clones Arc-backed Bytes)
|
|
|
|
|
let retry_state = if body.is_end_stream() {
|
|
|
|
|
@@ -941,18 +974,18 @@ impl HttpProxyService {
|
|
|
|
|
|
|
|
|
|
let result = self.forward_h2_with_sender(
|
|
|
|
|
sender, parts, body, upstream_headers, upstream_path,
|
|
|
|
|
route, route_id, source_ip, Some(pool_key),
|
|
|
|
|
route, route_id, source_ip, Some(pool_key), domain,
|
|
|
|
|
).await;
|
|
|
|
|
|
|
|
|
|
// If the request failed (502) and we can retry with an empty body, do so
|
|
|
|
|
let is_502 = matches!(&result, Ok(resp) if resp.status() == StatusCode::BAD_GATEWAY);
|
|
|
|
|
if is_502 {
|
|
|
|
|
if let Some((method, headers)) = retry_state {
|
|
|
|
|
warn!("Stale pooled H2 sender for {}:{}, retrying with fresh connection",
|
|
|
|
|
pool_key.host, pool_key.port);
|
|
|
|
|
warn!(backend = %format!("{}:{}", pool_key.host, pool_key.port), domain = %domain,
|
|
|
|
|
"Stale pooled H2 sender, retrying with fresh connection");
|
|
|
|
|
return self.retry_h2_with_fresh_connection(
|
|
|
|
|
method, headers, upstream_path,
|
|
|
|
|
pool_key, route, route_id, source_ip,
|
|
|
|
|
pool_key, route, route_id, source_ip, domain,
|
|
|
|
|
).await;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
@@ -970,6 +1003,7 @@ impl HttpProxyService {
|
|
|
|
|
route: &rustproxy_config::RouteConfig,
|
|
|
|
|
route_id: Option<&str>,
|
|
|
|
|
source_ip: &str,
|
|
|
|
|
domain: &str,
|
|
|
|
|
) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
|
|
|
|
|
let backend_key = format!("{}:{}", pool_key.host, pool_key.port);
|
|
|
|
|
|
|
|
|
|
@@ -978,16 +1012,16 @@ impl HttpProxyService {
|
|
|
|
|
let backend = if pool_key.use_tls {
|
|
|
|
|
match tokio::time::timeout(
|
|
|
|
|
self.connect_timeout,
|
|
|
|
|
connect_tls_backend(&self.backend_tls_config, &pool_key.host, pool_key.port),
|
|
|
|
|
connect_tls_backend(&self.backend_tls_config_alpn, &pool_key.host, pool_key.port),
|
|
|
|
|
).await {
|
|
|
|
|
Ok(Ok(tls)) => BackendStream::Tls(tls),
|
|
|
|
|
Ok(Err(e)) => {
|
|
|
|
|
error!(backend = %backend_key, error = %e, "H2 retry: TLS connect failed");
|
|
|
|
|
error!(backend = %backend_key, domain = %domain, error = %e, "H2 retry: TLS connect failed");
|
|
|
|
|
self.metrics.backend_connect_error(&backend_key);
|
|
|
|
|
return Ok(error_response(StatusCode::BAD_GATEWAY, "Backend unavailable on H2 retry"));
|
|
|
|
|
}
|
|
|
|
|
Err(_) => {
|
|
|
|
|
error!(backend = %backend_key, "H2 retry: TLS connect timeout");
|
|
|
|
|
error!(backend = %backend_key, domain = %domain, "H2 retry: TLS connect timeout");
|
|
|
|
|
self.metrics.backend_connect_error(&backend_key);
|
|
|
|
|
return Ok(error_response(StatusCode::GATEWAY_TIMEOUT, "Backend timeout on H2 retry"));
|
|
|
|
|
}
|
|
|
|
|
@@ -1002,12 +1036,12 @@ impl HttpProxyService {
|
|
|
|
|
BackendStream::Plain(s)
|
|
|
|
|
}
|
|
|
|
|
Ok(Err(e)) => {
|
|
|
|
|
error!(backend = %backend_key, error = %e, "H2 retry: TCP connect failed");
|
|
|
|
|
error!(backend = %backend_key, domain = %domain, error = %e, "H2 retry: TCP connect failed");
|
|
|
|
|
self.metrics.backend_connect_error(&backend_key);
|
|
|
|
|
return Ok(error_response(StatusCode::BAD_GATEWAY, "Backend unavailable on H2 retry"));
|
|
|
|
|
}
|
|
|
|
|
Err(_) => {
|
|
|
|
|
error!(backend = %backend_key, "H2 retry: TCP connect timeout");
|
|
|
|
|
error!(backend = %backend_key, domain = %domain, "H2 retry: TCP connect timeout");
|
|
|
|
|
self.metrics.backend_connect_error(&backend_key);
|
|
|
|
|
return Ok(error_response(StatusCode::GATEWAY_TIMEOUT, "Backend timeout on H2 retry"));
|
|
|
|
|
}
|
|
|
|
|
@@ -1017,16 +1051,30 @@ impl HttpProxyService {
|
|
|
|
|
|
|
|
|
|
let io = TokioIo::new(backend);
|
|
|
|
|
let exec = hyper_util::rt::TokioExecutor::new();
|
|
|
|
|
let mut h2_builder = hyper::client::conn::http2::Builder::new(exec);
|
|
|
|
|
h2_builder
|
|
|
|
|
.timer(hyper_util::rt::TokioTimer::new())
|
|
|
|
|
.keep_alive_interval(std::time::Duration::from_secs(10))
|
|
|
|
|
.keep_alive_timeout(std::time::Duration::from_secs(5))
|
|
|
|
|
.adaptive_window(true)
|
|
|
|
|
.initial_stream_window_size(2 * 1024 * 1024);
|
|
|
|
|
let (mut sender, conn): (
|
|
|
|
|
hyper::client::conn::http2::SendRequest<BoxBody<Bytes, hyper::Error>>,
|
|
|
|
|
hyper::client::conn::http2::Connection<TokioIo<BackendStream>, BoxBody<Bytes, hyper::Error>, hyper_util::rt::TokioExecutor>,
|
|
|
|
|
) = match hyper::client::conn::http2::handshake(exec, io).await {
|
|
|
|
|
Ok(h) => h,
|
|
|
|
|
Err(e) => {
|
|
|
|
|
error!(backend = %backend_key, error = %e, "H2 retry: handshake failed");
|
|
|
|
|
) = match tokio::time::timeout(self.connect_timeout, h2_builder.handshake(io)).await {
|
|
|
|
|
Ok(Ok(h)) => h,
|
|
|
|
|
Ok(Err(e)) => {
|
|
|
|
|
error!(backend = %backend_key, domain = %domain, error = %e, "H2 retry: handshake failed");
|
|
|
|
|
self.metrics.backend_handshake_error(&backend_key);
|
|
|
|
|
self.metrics.backend_connection_closed(&backend_key);
|
|
|
|
|
return Ok(error_response(StatusCode::BAD_GATEWAY, "Backend H2 retry handshake failed"));
|
|
|
|
|
}
|
|
|
|
|
Err(_) => {
|
|
|
|
|
error!(backend = %backend_key, domain = %domain, "H2 retry: handshake timeout");
|
|
|
|
|
self.metrics.backend_handshake_error(&backend_key);
|
|
|
|
|
self.metrics.backend_connection_closed(&backend_key);
|
|
|
|
|
return Ok(error_response(StatusCode::GATEWAY_TIMEOUT, "Backend H2 retry handshake timeout"));
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
tokio::spawn(async move {
|
|
|
|
|
@@ -1035,13 +1083,17 @@ impl HttpProxyService {
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
// Register fresh sender in pool for future requests
|
|
|
|
|
self.connection_pool.register_h2(pool_key.clone(), sender.clone());
|
|
|
|
|
|
|
|
|
|
// Build request with empty body
|
|
|
|
|
// Build request with empty body using absolute URI for H2 pseudo-headers
|
|
|
|
|
let scheme = if pool_key.use_tls { "https" } else { "http" };
|
|
|
|
|
let authority = if domain != "-" { domain } else { pool_key.host.as_str() };
|
|
|
|
|
let h2_uri = format!("{}://{}{}", scheme, authority, upstream_path);
|
|
|
|
|
let mut upstream_req = Request::builder()
|
|
|
|
|
.method(method)
|
|
|
|
|
.uri(upstream_path);
|
|
|
|
|
.uri(&h2_uri);
|
|
|
|
|
|
|
|
|
|
// Remove Host header for H2 — :authority pseudo-header (from URI) is sufficient
|
|
|
|
|
let mut upstream_headers = upstream_headers;
|
|
|
|
|
upstream_headers.remove("host");
|
|
|
|
|
|
|
|
|
|
if let Some(headers) = upstream_req.headers_mut() {
|
|
|
|
|
*headers = upstream_headers;
|
|
|
|
|
@@ -1054,16 +1106,17 @@ impl HttpProxyService {
|
|
|
|
|
|
|
|
|
|
match sender.send_request(upstream_req).await {
|
|
|
|
|
Ok(resp) => {
|
|
|
|
|
// Register in pool only after request succeeds
|
|
|
|
|
self.connection_pool.register_h2(pool_key.clone(), sender);
|
|
|
|
|
let result = self.build_streaming_response(resp, route, route_id, source_ip).await;
|
|
|
|
|
// Close the fresh backend connection (opened at line 1016 above)
|
|
|
|
|
// Close the fresh backend connection (opened above)
|
|
|
|
|
self.metrics.backend_connection_closed(&backend_key);
|
|
|
|
|
result
|
|
|
|
|
}
|
|
|
|
|
Err(e) => {
|
|
|
|
|
error!(backend = %backend_key, error = %e, "H2 retry: request failed");
|
|
|
|
|
error!(backend = %backend_key, domain = %domain, error = %e, "H2 retry: request failed");
|
|
|
|
|
self.metrics.backend_request_error(&backend_key);
|
|
|
|
|
self.connection_pool.remove_h2(pool_key);
|
|
|
|
|
// Close the fresh backend connection (opened at line 1016 above)
|
|
|
|
|
// Close the fresh backend connection (opened above)
|
|
|
|
|
self.metrics.backend_connection_closed(&backend_key);
|
|
|
|
|
Ok(error_response(StatusCode::BAD_GATEWAY, "Backend H2 request failed on retry"))
|
|
|
|
|
}
|
|
|
|
|
@@ -1082,7 +1135,7 @@ impl HttpProxyService {
|
|
|
|
|
io: TokioIo<BackendStream>,
|
|
|
|
|
parts: hyper::http::request::Parts,
|
|
|
|
|
body: Incoming,
|
|
|
|
|
upstream_headers: hyper::HeaderMap,
|
|
|
|
|
mut upstream_headers: hyper::HeaderMap,
|
|
|
|
|
upstream_path: &str,
|
|
|
|
|
upstream: &crate::upstream_selector::UpstreamSelection,
|
|
|
|
|
route: &rustproxy_config::RouteConfig,
|
|
|
|
|
@@ -1090,15 +1143,62 @@ impl HttpProxyService {
|
|
|
|
|
source_ip: &str,
|
|
|
|
|
pool_key: &crate::connection_pool::PoolKey,
|
|
|
|
|
requested_host: Option<String>,
|
|
|
|
|
domain: &str,
|
|
|
|
|
) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
|
|
|
|
|
let exec = hyper_util::rt::TokioExecutor::new();
|
|
|
|
|
let handshake_result: Result<(
|
|
|
|
|
hyper::client::conn::http2::SendRequest<BoxBody<Bytes, hyper::Error>>,
|
|
|
|
|
hyper::client::conn::http2::Connection<TokioIo<BackendStream>, BoxBody<Bytes, hyper::Error>, hyper_util::rt::TokioExecutor>,
|
|
|
|
|
), hyper::Error> = hyper::client::conn::http2::handshake(exec, io).await;
|
|
|
|
|
let mut h2_builder = hyper::client::conn::http2::Builder::new(exec);
|
|
|
|
|
h2_builder
|
|
|
|
|
.timer(hyper_util::rt::TokioTimer::new())
|
|
|
|
|
.keep_alive_interval(std::time::Duration::from_secs(10))
|
|
|
|
|
.keep_alive_timeout(std::time::Duration::from_secs(5))
|
|
|
|
|
.adaptive_window(true)
|
|
|
|
|
.initial_stream_window_size(2 * 1024 * 1024);
|
|
|
|
|
let handshake_result = tokio::time::timeout(
|
|
|
|
|
self.connect_timeout,
|
|
|
|
|
h2_builder.handshake(io),
|
|
|
|
|
).await;
|
|
|
|
|
|
|
|
|
|
match handshake_result {
|
|
|
|
|
Ok((mut sender, conn)) => {
|
|
|
|
|
Err(_) => {
|
|
|
|
|
// H2 handshake timed out — fall back to H1
|
|
|
|
|
let bk = format!("{}:{}", upstream.host, upstream.port);
|
|
|
|
|
warn!(
|
|
|
|
|
backend = %bk,
|
|
|
|
|
domain = %domain,
|
|
|
|
|
"H2 handshake timeout, falling back to H1"
|
|
|
|
|
);
|
|
|
|
|
self.metrics.backend_h2_failure(&bk);
|
|
|
|
|
self.metrics.backend_handshake_error(&bk);
|
|
|
|
|
|
|
|
|
|
let cache_key = crate::protocol_cache::ProtocolCacheKey {
|
|
|
|
|
host: upstream.host.clone(),
|
|
|
|
|
port: upstream.port,
|
|
|
|
|
requested_host: requested_host.clone(),
|
|
|
|
|
};
|
|
|
|
|
self.protocol_cache.insert(cache_key, crate::protocol_cache::DetectedProtocol::H1);
|
|
|
|
|
|
|
|
|
|
match self.reconnect_backend(upstream, domain).await {
|
|
|
|
|
Some(fallback_backend) => {
|
|
|
|
|
let h1_pool_key = crate::connection_pool::PoolKey {
|
|
|
|
|
host: upstream.host.clone(),
|
|
|
|
|
port: upstream.port,
|
|
|
|
|
use_tls: upstream.use_tls,
|
|
|
|
|
h2: false,
|
|
|
|
|
};
|
|
|
|
|
let fallback_io = TokioIo::new(fallback_backend);
|
|
|
|
|
let result = self.forward_h1(
|
|
|
|
|
fallback_io, parts, body, upstream_headers, upstream_path,
|
|
|
|
|
upstream, route, route_id, source_ip, &h1_pool_key, domain,
|
|
|
|
|
).await;
|
|
|
|
|
self.metrics.backend_connection_closed(&bk);
|
|
|
|
|
result
|
|
|
|
|
}
|
|
|
|
|
None => {
|
|
|
|
|
Ok(error_response(StatusCode::BAD_GATEWAY, "Backend unavailable after H2 timeout fallback"))
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
Ok(Ok((mut sender, conn))) => {
|
|
|
|
|
tokio::spawn(async move {
|
|
|
|
|
if let Err(e) = conn.await {
|
|
|
|
|
debug!("HTTP/2 upstream connection error: {}", e);
|
|
|
|
|
@@ -1106,18 +1206,25 @@ impl HttpProxyService {
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
// Save retry state before consuming parts/body (for bodyless requests like GET)
|
|
|
|
|
// Clone BEFORE removing Host — H1 fallback needs Host header
|
|
|
|
|
let retry_state = if body.is_end_stream() {
|
|
|
|
|
Some((parts.method.clone(), upstream_headers.clone()))
|
|
|
|
|
} else {
|
|
|
|
|
None
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
// Remove Host header for H2 — :authority pseudo-header (from URI) is sufficient
|
|
|
|
|
upstream_headers.remove("host");
|
|
|
|
|
|
|
|
|
|
// Build and send the h2 request inline (don't register in pool yet —
|
|
|
|
|
// we need to verify the request actually succeeds first, because some
|
|
|
|
|
// backends advertise h2 via ALPN but don't speak the h2 binary protocol).
|
|
|
|
|
let scheme = if upstream.use_tls { "https" } else { "http" };
|
|
|
|
|
let authority = if domain != "-" { domain } else { upstream.host.as_str() };
|
|
|
|
|
let h2_uri = format!("{}://{}{}", scheme, authority, upstream_path);
|
|
|
|
|
let mut upstream_req = Request::builder()
|
|
|
|
|
.method(parts.method)
|
|
|
|
|
.uri(upstream_path);
|
|
|
|
|
.uri(&h2_uri);
|
|
|
|
|
|
|
|
|
|
if let Some(headers) = upstream_req.headers_mut() {
|
|
|
|
|
*headers = upstream_headers;
|
|
|
|
|
@@ -1145,6 +1252,7 @@ impl HttpProxyService {
|
|
|
|
|
let bk = format!("{}:{}", upstream.host, upstream.port);
|
|
|
|
|
warn!(
|
|
|
|
|
backend = %bk,
|
|
|
|
|
domain = %domain,
|
|
|
|
|
error = %e,
|
|
|
|
|
"Auto-detect: H2 request failed, falling back to H1"
|
|
|
|
|
);
|
|
|
|
|
@@ -1158,7 +1266,7 @@ impl HttpProxyService {
|
|
|
|
|
|
|
|
|
|
// Retry as H1 for bodyless requests; return 502 for requests with bodies
|
|
|
|
|
if let Some((method, headers)) = retry_state {
|
|
|
|
|
match self.reconnect_backend(upstream).await {
|
|
|
|
|
match self.reconnect_backend(upstream, domain).await {
|
|
|
|
|
Some(fallback_backend) => {
|
|
|
|
|
let h1_pool_key = crate::connection_pool::PoolKey {
|
|
|
|
|
host: upstream.host.clone(),
|
|
|
|
|
@@ -1169,7 +1277,7 @@ impl HttpProxyService {
|
|
|
|
|
let fallback_io = TokioIo::new(fallback_backend);
|
|
|
|
|
let result = self.forward_h1_empty_body(
|
|
|
|
|
fallback_io, method, headers, upstream_path,
|
|
|
|
|
route, route_id, source_ip, &h1_pool_key,
|
|
|
|
|
route, route_id, source_ip, &h1_pool_key, domain,
|
|
|
|
|
).await;
|
|
|
|
|
// Close the reconnected backend connection (opened in reconnect_backend)
|
|
|
|
|
self.metrics.backend_connection_closed(&bk);
|
|
|
|
|
@@ -1185,12 +1293,13 @@ impl HttpProxyService {
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
Err(e) => {
|
|
|
|
|
Ok(Err(e)) => {
|
|
|
|
|
// H2 handshake truly failed — fall back to H1
|
|
|
|
|
// Body is NOT consumed yet, so we can retry the full request.
|
|
|
|
|
let bk = format!("{}:{}", upstream.host, upstream.port);
|
|
|
|
|
warn!(
|
|
|
|
|
backend = %bk,
|
|
|
|
|
domain = %domain,
|
|
|
|
|
error = %e,
|
|
|
|
|
"H2 handshake failed, falling back to H1"
|
|
|
|
|
);
|
|
|
|
|
@@ -1206,7 +1315,7 @@ impl HttpProxyService {
|
|
|
|
|
self.protocol_cache.insert(cache_key, crate::protocol_cache::DetectedProtocol::H1);
|
|
|
|
|
|
|
|
|
|
// Reconnect for H1 (the original io was consumed by the failed h2 handshake)
|
|
|
|
|
match self.reconnect_backend(upstream).await {
|
|
|
|
|
match self.reconnect_backend(upstream, domain).await {
|
|
|
|
|
Some(fallback_backend) => {
|
|
|
|
|
let h1_pool_key = crate::connection_pool::PoolKey {
|
|
|
|
|
host: upstream.host.clone(),
|
|
|
|
|
@@ -1217,7 +1326,7 @@ impl HttpProxyService {
|
|
|
|
|
let fallback_io = TokioIo::new(fallback_backend);
|
|
|
|
|
let result = self.forward_h1(
|
|
|
|
|
fallback_io, parts, body, upstream_headers, upstream_path,
|
|
|
|
|
upstream, route, route_id, source_ip, &h1_pool_key,
|
|
|
|
|
upstream, route, route_id, source_ip, &h1_pool_key, domain,
|
|
|
|
|
).await;
|
|
|
|
|
// Close the reconnected backend connection (opened in reconnect_backend)
|
|
|
|
|
self.metrics.backend_connection_closed(&bk);
|
|
|
|
|
@@ -1243,6 +1352,7 @@ impl HttpProxyService {
|
|
|
|
|
route_id: Option<&str>,
|
|
|
|
|
source_ip: &str,
|
|
|
|
|
pool_key: &crate::connection_pool::PoolKey,
|
|
|
|
|
domain: &str,
|
|
|
|
|
) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
|
|
|
|
|
let backend_key = format!("{}:{}", pool_key.host, pool_key.port);
|
|
|
|
|
let (mut sender, conn): (
|
|
|
|
|
@@ -1251,7 +1361,7 @@ impl HttpProxyService {
|
|
|
|
|
) = match hyper::client::conn::http1::handshake(io).await {
|
|
|
|
|
Ok(h) => h,
|
|
|
|
|
Err(e) => {
|
|
|
|
|
error!(backend = %backend_key, error = %e, "H1 fallback: handshake failed");
|
|
|
|
|
error!(backend = %backend_key, domain = %domain, error = %e, "H1 fallback: handshake failed");
|
|
|
|
|
self.metrics.backend_handshake_error(&backend_key);
|
|
|
|
|
return Ok(error_response(StatusCode::BAD_GATEWAY, "Backend H1 fallback handshake failed"));
|
|
|
|
|
}
|
|
|
|
|
@@ -1280,7 +1390,7 @@ impl HttpProxyService {
|
|
|
|
|
let upstream_response = match sender.send_request(upstream_req).await {
|
|
|
|
|
Ok(resp) => resp,
|
|
|
|
|
Err(e) => {
|
|
|
|
|
error!(backend = %backend_key, error = %e, "H1 fallback: request failed");
|
|
|
|
|
error!(backend = %backend_key, domain = %domain, error = %e, "H1 fallback: request failed");
|
|
|
|
|
self.metrics.backend_request_error(&backend_key);
|
|
|
|
|
return Ok(error_response(StatusCode::BAD_GATEWAY, "Backend H1 fallback request failed"));
|
|
|
|
|
}
|
|
|
|
|
@@ -1296,6 +1406,7 @@ impl HttpProxyService {
|
|
|
|
|
async fn reconnect_backend(
|
|
|
|
|
&self,
|
|
|
|
|
upstream: &crate::upstream_selector::UpstreamSelection,
|
|
|
|
|
domain: &str,
|
|
|
|
|
) -> Option<BackendStream> {
|
|
|
|
|
let backend_key = format!("{}:{}", upstream.host, upstream.port);
|
|
|
|
|
let reconnect_start = std::time::Instant::now();
|
|
|
|
|
@@ -1309,12 +1420,12 @@ impl HttpProxyService {
|
|
|
|
|
Some(BackendStream::Tls(tls))
|
|
|
|
|
}
|
|
|
|
|
Ok(Err(e)) => {
|
|
|
|
|
error!(backend = %backend_key, error = %e, "H1 fallback: TLS reconnect failed");
|
|
|
|
|
error!(backend = %backend_key, domain = %domain, error = %e, "H1 fallback: TLS reconnect failed");
|
|
|
|
|
self.metrics.backend_connect_error(&backend_key);
|
|
|
|
|
None
|
|
|
|
|
}
|
|
|
|
|
Err(_) => {
|
|
|
|
|
error!(backend = %backend_key, "H1 fallback: TLS reconnect timeout");
|
|
|
|
|
error!(backend = %backend_key, domain = %domain, "H1 fallback: TLS reconnect timeout");
|
|
|
|
|
self.metrics.backend_connect_error(&backend_key);
|
|
|
|
|
None
|
|
|
|
|
}
|
|
|
|
|
@@ -1333,12 +1444,12 @@ impl HttpProxyService {
|
|
|
|
|
Some(BackendStream::Plain(s))
|
|
|
|
|
}
|
|
|
|
|
Ok(Err(e)) => {
|
|
|
|
|
error!(backend = %backend_key, error = %e, "H1 fallback: TCP reconnect failed");
|
|
|
|
|
error!(backend = %backend_key, domain = %domain, error = %e, "H1 fallback: TCP reconnect failed");
|
|
|
|
|
self.metrics.backend_connect_error(&backend_key);
|
|
|
|
|
None
|
|
|
|
|
}
|
|
|
|
|
Err(_) => {
|
|
|
|
|
error!(backend = %backend_key, "H1 fallback: TCP reconnect timeout");
|
|
|
|
|
error!(backend = %backend_key, domain = %domain, "H1 fallback: TCP reconnect timeout");
|
|
|
|
|
self.metrics.backend_connect_error(&backend_key);
|
|
|
|
|
None
|
|
|
|
|
}
|
|
|
|
|
@@ -1358,10 +1469,23 @@ impl HttpProxyService {
|
|
|
|
|
route_id: Option<&str>,
|
|
|
|
|
source_ip: &str,
|
|
|
|
|
pool_key: Option<&crate::connection_pool::PoolKey>,
|
|
|
|
|
domain: &str,
|
|
|
|
|
) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
|
|
|
|
|
// Build absolute URI for H2 pseudo-headers (:scheme, :authority)
|
|
|
|
|
// Use the requested domain as authority (not backend address) so :authority matches Host header
|
|
|
|
|
let scheme = if pool_key.map(|pk| pk.use_tls).unwrap_or(false) { "https" } else { "http" };
|
|
|
|
|
let authority = if domain != "-" { domain } else {
|
|
|
|
|
pool_key.map(|pk| pk.host.as_str()).unwrap_or("localhost")
|
|
|
|
|
};
|
|
|
|
|
let h2_uri = format!("{}://{}{}", scheme, authority, upstream_path);
|
|
|
|
|
let mut upstream_req = Request::builder()
|
|
|
|
|
.method(parts.method)
|
|
|
|
|
.uri(upstream_path);
|
|
|
|
|
.uri(&h2_uri);
|
|
|
|
|
|
|
|
|
|
// Remove Host header for H2 — :authority pseudo-header (from URI) is sufficient
|
|
|
|
|
// Having both Host and :authority causes nginx to return 400
|
|
|
|
|
let mut upstream_headers = upstream_headers;
|
|
|
|
|
upstream_headers.remove("host");
|
|
|
|
|
|
|
|
|
|
if let Some(headers) = upstream_req.headers_mut() {
|
|
|
|
|
*headers = upstream_headers;
|
|
|
|
|
@@ -1385,11 +1509,11 @@ impl HttpProxyService {
|
|
|
|
|
// Evict the dead sender so subsequent requests get fresh connections
|
|
|
|
|
if let Some(key) = pool_key {
|
|
|
|
|
let bk = format!("{}:{}", key.host, key.port);
|
|
|
|
|
error!(backend = %bk, error = %e, "Backend H2 request failed");
|
|
|
|
|
error!(backend = %bk, domain = %domain, error = %e, "Backend H2 request failed");
|
|
|
|
|
self.metrics.backend_request_error(&bk);
|
|
|
|
|
self.connection_pool.remove_h2(key);
|
|
|
|
|
} else {
|
|
|
|
|
error!(error = %e, "Backend H2 request failed");
|
|
|
|
|
error!(domain = %domain, error = %e, "Backend H2 request failed");
|
|
|
|
|
}
|
|
|
|
|
return Ok(error_response(StatusCode::BAD_GATEWAY, "Backend H2 request failed"));
|
|
|
|
|
}
|
|
|
|
|
|