feat(remoteingress): add heartbeat PING/PONG and liveness timeouts; implement fast-reconnect/backoff reset and JS crash-recovery auto-restart

rust/crates/remoteingress-core/src/hub.rs

@@ -1,8 +1,10 @@
 use std::collections::HashMap;
 use std::sync::Arc;
+use std::time::Duration;
 use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader};
 use tokio::net::{TcpListener, TcpStream};
 use tokio::sync::{mpsc, Mutex, RwLock, Semaphore};
+use tokio::time::{interval, sleep_until, Instant};
 use tokio_rustls::TlsAcceptor;
 use tokio_util::sync::CancellationToken;
 use serde::{Deserialize, Serialize};
@@ -407,6 +409,14 @@ async fn handle_edge_connection(
     // A4: Semaphore to limit concurrent streams per edge
     let stream_semaphore = Arc::new(Semaphore::new(MAX_STREAMS_PER_EDGE));
 
+    // Heartbeat: periodic PING and liveness timeout
+    let ping_interval_dur = Duration::from_secs(15);
+    let liveness_timeout_dur = Duration::from_secs(45);
+    let mut ping_ticker = interval(ping_interval_dur);
+    ping_ticker.tick().await; // consume the immediate first tick
+    let mut last_activity = Instant::now();
+    let mut liveness_deadline = Box::pin(sleep_until(last_activity + liveness_timeout_dur));
+
     // Frame reading loop
     let mut frame_reader = FrameReader::new(buf_reader);
 
@@ -415,6 +425,10 @@ async fn handle_edge_connection(
             frame_result = frame_reader.next_frame() => {
                 match frame_result {
                     Ok(Some(frame)) => {
+                        // Reset liveness on any received frame
+                        last_activity = Instant::now();
+                        liveness_deadline.as_mut().reset(last_activity + liveness_timeout_dur);
+
                         match frame.frame_type {
                             FRAME_OPEN => {
                                 // A4: Check stream limit before processing
@@ -462,7 +476,7 @@ async fn handle_edge_connection(
                                     let result = async {
                                         // A2: Connect to SmartProxy with timeout
                                         let mut upstream = tokio::time::timeout(
-                                            std::time::Duration::from_secs(10),
+                                            Duration::from_secs(10),
                                             TcpStream::connect((target.as_str(), dest_port)),
                                         )
                                         .await
@@ -571,6 +585,9 @@ async fn handle_edge_connection(
                                     });
                                 }
                             }
+                            FRAME_PONG => {
+                                log::debug!("Received PONG from edge {}", edge_id);
+                            }
                             _ => {
                                 log::warn!("Unexpected frame type {} from edge", frame.frame_type);
                             }
@@ -586,6 +603,19 @@ async fn handle_edge_connection(
                     }
                 }
             }
+            _ = ping_ticker.tick() => {
+                let ping_frame = encode_frame(0, FRAME_PING, &[]);
+                if frame_writer_tx.try_send(ping_frame).is_err() {
+                    log::warn!("Failed to send PING to edge {}, writer channel full/closed", edge_id);
+                    break;
+                }
+                log::trace!("Sent PING to edge {}", edge_id);
+            }
+            _ = &mut liveness_deadline => {
+                log::warn!("Edge {} liveness timeout (no frames for {}s), disconnecting",
+                    edge_id, liveness_timeout_dur.as_secs());
+                break;
+            }
             _ = edge_token.cancelled() => {
                 log::info!("Edge {} cancelled by hub", edge_id);
                 break;