Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| cf2d32bfe7 | |||
| 4e9041c6a7 | |||
| 86d4e9889a | |||
| 45a2811f3e | |||
| d6a07c28a0 | |||
| 56a14aa7c5 |
23
changelog.md
23
changelog.md
@@ -1,5 +1,28 @@
|
|||||||
# Changelog
|
# Changelog
|
||||||
|
|
||||||
|
## 2026-03-15 - 4.4.1 - fix(remoteingress-core)
|
||||||
|
prevent stream data loss by applying backpressure and closing saturated channels
|
||||||
|
|
||||||
|
- replace non-blocking frame writes with awaited sends in per-stream tasks so large transfers respect backpressure instead of dropping data
|
||||||
|
- close and remove streams when back-channel or data channels fill up to avoid TCP stream corruption from silently dropped frames
|
||||||
|
|
||||||
|
## 2026-03-03 - 4.4.0 - feat(remoteingress)
|
||||||
|
add heartbeat PING/PONG and liveness timeouts; implement fast-reconnect/backoff reset and JS crash-recovery auto-restart
|
||||||
|
|
||||||
|
- protocol: add FRAME_PING and FRAME_PONG and unit tests for ping/pong frames
|
||||||
|
- edge (Rust): reset backoff after successful connection, respond to PING with PONG, track liveness via deadline and reconnect on timeout, use Duration/Instant helpers
|
||||||
|
- hub (Rust): send periodic PING to edges, handle PONGs, enforce liveness timeout and disconnect inactive edges, use tokio interval and time utilities
|
||||||
|
- ts: RemoteIngressEdge and RemoteIngressHub: add crash-recovery auto-restart with exponential backoff and max attempts, save/restore config and allowed edges, register/remove exit handlers, ensure stop() marks stopping and cleans up listeners
|
||||||
|
- minor API/typing: introduce TAllowedEdge alias and persist allowed edges for restart recovery
|
||||||
|
|
||||||
|
## 2026-02-26 - 4.3.0 - feat(hub)
|
||||||
|
add optional TLS certificate/key support to hub start config and bridge
|
||||||
|
|
||||||
|
- TypeScript: add tls.certPem and tls.keyPem to IHubConfig and include tlsCertPem/tlsKeyPem in startHub bridge command when both are provided
|
||||||
|
- TypeScript: extend startHub params with tlsCertPem and tlsKeyPem and conditionally send them
|
||||||
|
- Rust: change HubConfig serde attributes for tls_cert_pem and tls_key_pem from skip to default so absent PEM fields deserialize as None
|
||||||
|
- Enables optional provisioning of TLS certificate and key to the hub when provided from the JS side
|
||||||
|
|
||||||
## 2026-02-26 - 4.2.0 - feat(core)
|
## 2026-02-26 - 4.2.0 - feat(core)
|
||||||
expose edge peer address in hub events and migrate writers to channel-based, non-blocking framing with stream limits and timeouts
|
expose edge peer address in hub events and migrate writers to channel-based, non-blocking framing with stream limits and timeouts
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@serve.zone/remoteingress",
|
"name": "@serve.zone/remoteingress",
|
||||||
"version": "4.2.0",
|
"version": "4.4.1",
|
||||||
"private": false,
|
"private": false,
|
||||||
"description": "Edge ingress tunnel for DcRouter - accepts incoming TCP connections at network edge and tunnels them to DcRouter SmartProxy preserving client IP via PROXY protocol v1.",
|
"description": "Edge ingress tunnel for DcRouter - accepts incoming TCP connections at network edge and tunnels them to DcRouter SmartProxy preserving client IP via PROXY protocol v1.",
|
||||||
"main": "dist_ts/index.js",
|
"main": "dist_ts/index.js",
|
||||||
|
|||||||
@@ -1,10 +1,12 @@
|
|||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::sync::atomic::{AtomicU32, Ordering};
|
use std::sync::atomic::{AtomicU32, Ordering};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
use std::time::Duration;
|
||||||
use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader};
|
use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader};
|
||||||
use tokio::net::{TcpListener, TcpStream};
|
use tokio::net::{TcpListener, TcpStream};
|
||||||
use tokio::sync::{mpsc, Mutex, RwLock};
|
use tokio::sync::{mpsc, Mutex, RwLock};
|
||||||
use tokio::task::JoinHandle;
|
use tokio::task::JoinHandle;
|
||||||
|
use tokio::time::{Instant, sleep_until};
|
||||||
use tokio_rustls::TlsConnector;
|
use tokio_rustls::TlsConnector;
|
||||||
use tokio_util::sync::CancellationToken;
|
use tokio_util::sync::CancellationToken;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
@@ -202,6 +204,13 @@ async fn edge_main_loop(
|
|||||||
// Cancel connection token to kill all orphaned tasks from this cycle
|
// Cancel connection token to kill all orphaned tasks from this cycle
|
||||||
connection_token.cancel();
|
connection_token.cancel();
|
||||||
|
|
||||||
|
// Reset backoff after a successful connection for fast reconnect
|
||||||
|
let was_connected = *connected.read().await;
|
||||||
|
if was_connected {
|
||||||
|
backoff_ms = 1000;
|
||||||
|
log::info!("Was connected; resetting backoff to {}ms for fast reconnect", backoff_ms);
|
||||||
|
}
|
||||||
|
|
||||||
*connected.write().await = false;
|
*connected.write().await = false;
|
||||||
let _ = event_tx.try_send(EdgeEvent::TunnelDisconnected);
|
let _ = event_tx.try_send(EdgeEvent::TunnelDisconnected);
|
||||||
active_streams.store(0, Ordering::Relaxed);
|
active_streams.store(0, Ordering::Relaxed);
|
||||||
@@ -214,7 +223,7 @@ async fn edge_main_loop(
|
|||||||
EdgeLoopResult::Reconnect => {
|
EdgeLoopResult::Reconnect => {
|
||||||
log::info!("Reconnecting in {}ms...", backoff_ms);
|
log::info!("Reconnecting in {}ms...", backoff_ms);
|
||||||
tokio::select! {
|
tokio::select! {
|
||||||
_ = tokio::time::sleep(std::time::Duration::from_millis(backoff_ms)) => {}
|
_ = tokio::time::sleep(Duration::from_millis(backoff_ms)) => {}
|
||||||
_ = cancel_token.cancelled() => break,
|
_ = cancel_token.cancelled() => break,
|
||||||
_ = shutdown_rx.recv() => break,
|
_ = shutdown_rx.recv() => break,
|
||||||
}
|
}
|
||||||
@@ -336,7 +345,7 @@ async fn connect_to_hub_and_run(
|
|||||||
_ = stun_token.cancelled() => break,
|
_ = stun_token.cancelled() => break,
|
||||||
}
|
}
|
||||||
tokio::select! {
|
tokio::select! {
|
||||||
_ = tokio::time::sleep(std::time::Duration::from_secs(stun_interval)) => {}
|
_ = tokio::time::sleep(Duration::from_secs(stun_interval)) => {}
|
||||||
_ = stun_token.cancelled() => break,
|
_ = stun_token.cancelled() => break,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -380,6 +389,11 @@ async fn connect_to_hub_and_run(
|
|||||||
connection_token,
|
connection_token,
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// Heartbeat: liveness timeout detects silent hub failures
|
||||||
|
let liveness_timeout_dur = Duration::from_secs(45);
|
||||||
|
let mut last_activity = Instant::now();
|
||||||
|
let mut liveness_deadline = Box::pin(sleep_until(last_activity + liveness_timeout_dur));
|
||||||
|
|
||||||
// Read frames from hub
|
// Read frames from hub
|
||||||
let mut frame_reader = FrameReader::new(buf_reader);
|
let mut frame_reader = FrameReader::new(buf_reader);
|
||||||
let result = loop {
|
let result = loop {
|
||||||
@@ -387,13 +401,20 @@ async fn connect_to_hub_and_run(
|
|||||||
frame_result = frame_reader.next_frame() => {
|
frame_result = frame_reader.next_frame() => {
|
||||||
match frame_result {
|
match frame_result {
|
||||||
Ok(Some(frame)) => {
|
Ok(Some(frame)) => {
|
||||||
|
// Reset liveness on any received frame
|
||||||
|
last_activity = Instant::now();
|
||||||
|
liveness_deadline.as_mut().reset(last_activity + liveness_timeout_dur);
|
||||||
|
|
||||||
match frame.frame_type {
|
match frame.frame_type {
|
||||||
FRAME_DATA_BACK => {
|
FRAME_DATA_BACK => {
|
||||||
// A1: Non-blocking send to prevent head-of-line blocking
|
// Non-blocking send to prevent head-of-line blocking in the main dispatch loop.
|
||||||
let writers = client_writers.lock().await;
|
// If the per-stream channel is full, close the stream rather than silently
|
||||||
|
// dropping data (which would corrupt the TCP stream).
|
||||||
|
let mut writers = client_writers.lock().await;
|
||||||
if let Some(tx) = writers.get(&frame.stream_id) {
|
if let Some(tx) = writers.get(&frame.stream_id) {
|
||||||
if tx.try_send(frame.payload).is_err() {
|
if tx.try_send(frame.payload).is_err() {
|
||||||
log::warn!("Stream {} back-channel full, dropping frame", frame.stream_id);
|
log::warn!("Stream {} back-channel full, closing stream to prevent data corruption", frame.stream_id);
|
||||||
|
writers.remove(&frame.stream_id);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -420,6 +441,14 @@ async fn connect_to_hub_and_run(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
FRAME_PING => {
|
||||||
|
let pong_frame = encode_frame(0, FRAME_PONG, &[]);
|
||||||
|
if tunnel_writer_tx.try_send(pong_frame).is_err() {
|
||||||
|
log::warn!("Failed to send PONG, writer channel full/closed");
|
||||||
|
break EdgeLoopResult::Reconnect;
|
||||||
|
}
|
||||||
|
log::trace!("Received PING from hub, sent PONG");
|
||||||
|
}
|
||||||
_ => {
|
_ => {
|
||||||
log::warn!("Unexpected frame type {} from hub", frame.frame_type);
|
log::warn!("Unexpected frame type {} from hub", frame.frame_type);
|
||||||
}
|
}
|
||||||
@@ -435,6 +464,11 @@ async fn connect_to_hub_and_run(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
_ = &mut liveness_deadline => {
|
||||||
|
log::warn!("Hub liveness timeout (no frames for {}s), reconnecting",
|
||||||
|
liveness_timeout_dur.as_secs());
|
||||||
|
break EdgeLoopResult::Reconnect;
|
||||||
|
}
|
||||||
_ = connection_token.cancelled() => {
|
_ = connection_token.cancelled() => {
|
||||||
log::info!("Connection cancelled");
|
log::info!("Connection cancelled");
|
||||||
break EdgeLoopResult::Shutdown;
|
break EdgeLoopResult::Shutdown;
|
||||||
@@ -604,9 +638,11 @@ async fn handle_client_connection(
|
|||||||
Ok(0) => break,
|
Ok(0) => break,
|
||||||
Ok(n) => {
|
Ok(n) => {
|
||||||
let data_frame = encode_frame(stream_id, FRAME_DATA, &buf[..n]);
|
let data_frame = encode_frame(stream_id, FRAME_DATA, &buf[..n]);
|
||||||
// A5: Use try_send to avoid blocking if writer channel is full
|
// Use send().await for backpressure — this is a per-stream task so
|
||||||
if tunnel_writer_tx.try_send(data_frame).is_err() {
|
// blocking only stalls this stream, not others. Prevents data loss
|
||||||
log::warn!("Stream {} tunnel writer full, closing", stream_id);
|
// for large transfers (e.g. 352MB Docker layers).
|
||||||
|
if tunnel_writer_tx.send(data_frame).await.is_err() {
|
||||||
|
log::warn!("Stream {} tunnel writer closed, closing", stream_id);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,8 +1,10 @@
|
|||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
use std::time::Duration;
|
||||||
use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader};
|
use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader};
|
||||||
use tokio::net::{TcpListener, TcpStream};
|
use tokio::net::{TcpListener, TcpStream};
|
||||||
use tokio::sync::{mpsc, Mutex, RwLock, Semaphore};
|
use tokio::sync::{mpsc, Mutex, RwLock, Semaphore};
|
||||||
|
use tokio::time::{interval, sleep_until, Instant};
|
||||||
use tokio_rustls::TlsAcceptor;
|
use tokio_rustls::TlsAcceptor;
|
||||||
use tokio_util::sync::CancellationToken;
|
use tokio_util::sync::CancellationToken;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
@@ -15,9 +17,9 @@ use remoteingress_protocol::*;
|
|||||||
pub struct HubConfig {
|
pub struct HubConfig {
|
||||||
pub tunnel_port: u16,
|
pub tunnel_port: u16,
|
||||||
pub target_host: Option<String>,
|
pub target_host: Option<String>,
|
||||||
#[serde(skip)]
|
#[serde(default)]
|
||||||
pub tls_cert_pem: Option<String>,
|
pub tls_cert_pem: Option<String>,
|
||||||
#[serde(skip)]
|
#[serde(default)]
|
||||||
pub tls_key_pem: Option<String>,
|
pub tls_key_pem: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -407,6 +409,14 @@ async fn handle_edge_connection(
|
|||||||
// A4: Semaphore to limit concurrent streams per edge
|
// A4: Semaphore to limit concurrent streams per edge
|
||||||
let stream_semaphore = Arc::new(Semaphore::new(MAX_STREAMS_PER_EDGE));
|
let stream_semaphore = Arc::new(Semaphore::new(MAX_STREAMS_PER_EDGE));
|
||||||
|
|
||||||
|
// Heartbeat: periodic PING and liveness timeout
|
||||||
|
let ping_interval_dur = Duration::from_secs(15);
|
||||||
|
let liveness_timeout_dur = Duration::from_secs(45);
|
||||||
|
let mut ping_ticker = interval(ping_interval_dur);
|
||||||
|
ping_ticker.tick().await; // consume the immediate first tick
|
||||||
|
let mut last_activity = Instant::now();
|
||||||
|
let mut liveness_deadline = Box::pin(sleep_until(last_activity + liveness_timeout_dur));
|
||||||
|
|
||||||
// Frame reading loop
|
// Frame reading loop
|
||||||
let mut frame_reader = FrameReader::new(buf_reader);
|
let mut frame_reader = FrameReader::new(buf_reader);
|
||||||
|
|
||||||
@@ -415,6 +425,10 @@ async fn handle_edge_connection(
|
|||||||
frame_result = frame_reader.next_frame() => {
|
frame_result = frame_reader.next_frame() => {
|
||||||
match frame_result {
|
match frame_result {
|
||||||
Ok(Some(frame)) => {
|
Ok(Some(frame)) => {
|
||||||
|
// Reset liveness on any received frame
|
||||||
|
last_activity = Instant::now();
|
||||||
|
liveness_deadline.as_mut().reset(last_activity + liveness_timeout_dur);
|
||||||
|
|
||||||
match frame.frame_type {
|
match frame.frame_type {
|
||||||
FRAME_OPEN => {
|
FRAME_OPEN => {
|
||||||
// A4: Check stream limit before processing
|
// A4: Check stream limit before processing
|
||||||
@@ -462,7 +476,7 @@ async fn handle_edge_connection(
|
|||||||
let result = async {
|
let result = async {
|
||||||
// A2: Connect to SmartProxy with timeout
|
// A2: Connect to SmartProxy with timeout
|
||||||
let mut upstream = tokio::time::timeout(
|
let mut upstream = tokio::time::timeout(
|
||||||
std::time::Duration::from_secs(10),
|
Duration::from_secs(10),
|
||||||
TcpStream::connect((target.as_str(), dest_port)),
|
TcpStream::connect((target.as_str(), dest_port)),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
@@ -506,9 +520,11 @@ async fn handle_edge_connection(
|
|||||||
Ok(n) => {
|
Ok(n) => {
|
||||||
let frame =
|
let frame =
|
||||||
encode_frame(stream_id, FRAME_DATA_BACK, &buf[..n]);
|
encode_frame(stream_id, FRAME_DATA_BACK, &buf[..n]);
|
||||||
// A5: Use try_send to avoid blocking if writer channel is full
|
// Use send().await for backpressure — this is a per-stream task so
|
||||||
if writer_tx.try_send(frame).is_err() {
|
// blocking only stalls this stream, not others. Prevents data loss
|
||||||
log::warn!("Stream {} writer channel full, closing", stream_id);
|
// for large transfers (e.g. 352MB Docker layers).
|
||||||
|
if writer_tx.send(frame).await.is_err() {
|
||||||
|
log::warn!("Stream {} writer channel closed, closing", stream_id);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -553,11 +569,16 @@ async fn handle_edge_connection(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
FRAME_DATA => {
|
FRAME_DATA => {
|
||||||
// A1: Non-blocking send to prevent head-of-line blocking
|
// Non-blocking send to prevent head-of-line blocking in the main dispatch loop.
|
||||||
let s = streams.lock().await;
|
// If the per-stream channel is full, close the stream rather than silently
|
||||||
|
// dropping data (which would corrupt the TCP stream).
|
||||||
|
let mut s = streams.lock().await;
|
||||||
if let Some((tx, _)) = s.get(&frame.stream_id) {
|
if let Some((tx, _)) = s.get(&frame.stream_id) {
|
||||||
if tx.try_send(frame.payload).is_err() {
|
if tx.try_send(frame.payload).is_err() {
|
||||||
log::warn!("Stream {} data channel full, dropping frame", frame.stream_id);
|
log::warn!("Stream {} data channel full, closing stream to prevent data corruption", frame.stream_id);
|
||||||
|
if let Some((_, token)) = s.remove(&frame.stream_id) {
|
||||||
|
token.cancel();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -571,6 +592,9 @@ async fn handle_edge_connection(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
FRAME_PONG => {
|
||||||
|
log::debug!("Received PONG from edge {}", edge_id);
|
||||||
|
}
|
||||||
_ => {
|
_ => {
|
||||||
log::warn!("Unexpected frame type {} from edge", frame.frame_type);
|
log::warn!("Unexpected frame type {} from edge", frame.frame_type);
|
||||||
}
|
}
|
||||||
@@ -586,6 +610,19 @@ async fn handle_edge_connection(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
_ = ping_ticker.tick() => {
|
||||||
|
let ping_frame = encode_frame(0, FRAME_PING, &[]);
|
||||||
|
if frame_writer_tx.try_send(ping_frame).is_err() {
|
||||||
|
log::warn!("Failed to send PING to edge {}, writer channel full/closed", edge_id);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
log::trace!("Sent PING to edge {}", edge_id);
|
||||||
|
}
|
||||||
|
_ = &mut liveness_deadline => {
|
||||||
|
log::warn!("Edge {} liveness timeout (no frames for {}s), disconnecting",
|
||||||
|
edge_id, liveness_timeout_dur.as_secs());
|
||||||
|
break;
|
||||||
|
}
|
||||||
_ = edge_token.cancelled() => {
|
_ = edge_token.cancelled() => {
|
||||||
log::info!("Edge {} cancelled by hub", edge_id);
|
log::info!("Edge {} cancelled by hub", edge_id);
|
||||||
break;
|
break;
|
||||||
|
|||||||
@@ -7,6 +7,8 @@ pub const FRAME_CLOSE: u8 = 0x03;
|
|||||||
pub const FRAME_DATA_BACK: u8 = 0x04;
|
pub const FRAME_DATA_BACK: u8 = 0x04;
|
||||||
pub const FRAME_CLOSE_BACK: u8 = 0x05;
|
pub const FRAME_CLOSE_BACK: u8 = 0x05;
|
||||||
pub const FRAME_CONFIG: u8 = 0x06; // Hub -> Edge: configuration update
|
pub const FRAME_CONFIG: u8 = 0x06; // Hub -> Edge: configuration update
|
||||||
|
pub const FRAME_PING: u8 = 0x07; // Hub -> Edge: heartbeat probe
|
||||||
|
pub const FRAME_PONG: u8 = 0x08; // Edge -> Hub: heartbeat response
|
||||||
|
|
||||||
// Frame header size: 4 (stream_id) + 1 (type) + 4 (length) = 9 bytes
|
// Frame header size: 4 (stream_id) + 1 (type) + 4 (length) = 9 bytes
|
||||||
pub const FRAME_HEADER_SIZE: usize = 9;
|
pub const FRAME_HEADER_SIZE: usize = 9;
|
||||||
@@ -261,6 +263,8 @@ mod tests {
|
|||||||
FRAME_DATA_BACK,
|
FRAME_DATA_BACK,
|
||||||
FRAME_CLOSE_BACK,
|
FRAME_CLOSE_BACK,
|
||||||
FRAME_CONFIG,
|
FRAME_CONFIG,
|
||||||
|
FRAME_PING,
|
||||||
|
FRAME_PONG,
|
||||||
];
|
];
|
||||||
|
|
||||||
let mut data = Vec::new();
|
let mut data = Vec::new();
|
||||||
@@ -293,4 +297,19 @@ mod tests {
|
|||||||
assert_eq!(frame.frame_type, FRAME_CLOSE);
|
assert_eq!(frame.frame_type, FRAME_CLOSE);
|
||||||
assert!(frame.payload.is_empty());
|
assert!(frame.payload.is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_encode_frame_ping_pong() {
|
||||||
|
// PING: stream_id=0, empty payload (control frame)
|
||||||
|
let ping = encode_frame(0, FRAME_PING, &[]);
|
||||||
|
assert_eq!(ping[4], FRAME_PING);
|
||||||
|
assert_eq!(&ping[0..4], &0u32.to_be_bytes());
|
||||||
|
assert_eq!(ping.len(), FRAME_HEADER_SIZE);
|
||||||
|
|
||||||
|
// PONG: stream_id=0, empty payload (control frame)
|
||||||
|
let pong = encode_frame(0, FRAME_PONG, &[]);
|
||||||
|
assert_eq!(pong[4], FRAME_PONG);
|
||||||
|
assert_eq!(&pong[0..4], &0u32.to_be_bytes());
|
||||||
|
assert_eq!(pong.len(), FRAME_HEADER_SIZE);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,6 +3,6 @@
|
|||||||
*/
|
*/
|
||||||
export const commitinfo = {
|
export const commitinfo = {
|
||||||
name: '@serve.zone/remoteingress',
|
name: '@serve.zone/remoteingress',
|
||||||
version: '4.2.0',
|
version: '4.4.1',
|
||||||
description: 'Edge ingress tunnel for DcRouter - accepts incoming TCP connections at network edge and tunnels them to DcRouter SmartProxy preserving client IP via PROXY protocol v1.'
|
description: 'Edge ingress tunnel for DcRouter - accepts incoming TCP connections at network edge and tunnels them to DcRouter SmartProxy preserving client IP via PROXY protocol v1.'
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -40,9 +40,16 @@ export interface IEdgeConfig {
|
|||||||
secret: string;
|
secret: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const MAX_RESTART_ATTEMPTS = 10;
|
||||||
|
const MAX_RESTART_BACKOFF_MS = 30_000;
|
||||||
|
|
||||||
export class RemoteIngressEdge extends EventEmitter {
|
export class RemoteIngressEdge extends EventEmitter {
|
||||||
private bridge: InstanceType<typeof plugins.smartrust.RustBridge<TEdgeCommands>>;
|
private bridge: InstanceType<typeof plugins.smartrust.RustBridge<TEdgeCommands>>;
|
||||||
private started = false;
|
private started = false;
|
||||||
|
private stopping = false;
|
||||||
|
private savedConfig: IEdgeConfig | null = null;
|
||||||
|
private restartBackoffMs = 1000;
|
||||||
|
private restartAttempts = 0;
|
||||||
private statusInterval: ReturnType<typeof setInterval> | undefined;
|
private statusInterval: ReturnType<typeof setInterval> | undefined;
|
||||||
|
|
||||||
constructor() {
|
constructor() {
|
||||||
@@ -109,11 +116,17 @@ export class RemoteIngressEdge extends EventEmitter {
|
|||||||
edgeConfig = config;
|
edgeConfig = config;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
this.savedConfig = edgeConfig;
|
||||||
|
this.stopping = false;
|
||||||
|
|
||||||
const spawned = await this.bridge.spawn();
|
const spawned = await this.bridge.spawn();
|
||||||
if (!spawned) {
|
if (!spawned) {
|
||||||
throw new Error('Failed to spawn remoteingress-bin');
|
throw new Error('Failed to spawn remoteingress-bin');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Register crash recovery handler
|
||||||
|
this.bridge.on('exit', this.handleCrashRecovery);
|
||||||
|
|
||||||
await this.bridge.sendCommand('startEdge', {
|
await this.bridge.sendCommand('startEdge', {
|
||||||
hubHost: edgeConfig.hubHost,
|
hubHost: edgeConfig.hubHost,
|
||||||
hubPort: edgeConfig.hubPort ?? 8443,
|
hubPort: edgeConfig.hubPort ?? 8443,
|
||||||
@@ -122,6 +135,8 @@ export class RemoteIngressEdge extends EventEmitter {
|
|||||||
});
|
});
|
||||||
|
|
||||||
this.started = true;
|
this.started = true;
|
||||||
|
this.restartAttempts = 0;
|
||||||
|
this.restartBackoffMs = 1000;
|
||||||
|
|
||||||
// Start periodic status logging
|
// Start periodic status logging
|
||||||
this.statusInterval = setInterval(async () => {
|
this.statusInterval = setInterval(async () => {
|
||||||
@@ -142,6 +157,7 @@ export class RemoteIngressEdge extends EventEmitter {
|
|||||||
* Stop the edge and kill the Rust process.
|
* Stop the edge and kill the Rust process.
|
||||||
*/
|
*/
|
||||||
public async stop(): Promise<void> {
|
public async stop(): Promise<void> {
|
||||||
|
this.stopping = true;
|
||||||
if (this.statusInterval) {
|
if (this.statusInterval) {
|
||||||
clearInterval(this.statusInterval);
|
clearInterval(this.statusInterval);
|
||||||
this.statusInterval = undefined;
|
this.statusInterval = undefined;
|
||||||
@@ -152,6 +168,7 @@ export class RemoteIngressEdge extends EventEmitter {
|
|||||||
} catch {
|
} catch {
|
||||||
// Process may already be dead
|
// Process may already be dead
|
||||||
}
|
}
|
||||||
|
this.bridge.removeListener('exit', this.handleCrashRecovery);
|
||||||
this.bridge.kill();
|
this.bridge.kill();
|
||||||
this.started = false;
|
this.started = false;
|
||||||
}
|
}
|
||||||
@@ -170,4 +187,55 @@ export class RemoteIngressEdge extends EventEmitter {
|
|||||||
public get running(): boolean {
|
public get running(): boolean {
|
||||||
return this.bridge.running;
|
return this.bridge.running;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Handle unexpected Rust binary crash — auto-restart with backoff.
|
||||||
|
*/
|
||||||
|
private handleCrashRecovery = async (code: number | null, signal: string | null) => {
|
||||||
|
if (this.stopping || !this.started || !this.savedConfig) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
console.error(
|
||||||
|
`[RemoteIngressEdge] Rust binary crashed (code=${code}, signal=${signal}), ` +
|
||||||
|
`attempt ${this.restartAttempts + 1}/${MAX_RESTART_ATTEMPTS}`
|
||||||
|
);
|
||||||
|
|
||||||
|
this.started = false;
|
||||||
|
|
||||||
|
if (this.restartAttempts >= MAX_RESTART_ATTEMPTS) {
|
||||||
|
console.error('[RemoteIngressEdge] Max restart attempts reached, giving up');
|
||||||
|
this.emit('crashRecoveryFailed');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await new Promise(resolve => setTimeout(resolve, this.restartBackoffMs));
|
||||||
|
this.restartBackoffMs = Math.min(this.restartBackoffMs * 2, MAX_RESTART_BACKOFF_MS);
|
||||||
|
this.restartAttempts++;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const spawned = await this.bridge.spawn();
|
||||||
|
if (!spawned) {
|
||||||
|
console.error('[RemoteIngressEdge] Failed to respawn binary');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.bridge.on('exit', this.handleCrashRecovery);
|
||||||
|
|
||||||
|
await this.bridge.sendCommand('startEdge', {
|
||||||
|
hubHost: this.savedConfig.hubHost,
|
||||||
|
hubPort: this.savedConfig.hubPort ?? 8443,
|
||||||
|
edgeId: this.savedConfig.edgeId,
|
||||||
|
secret: this.savedConfig.secret,
|
||||||
|
});
|
||||||
|
|
||||||
|
this.started = true;
|
||||||
|
this.restartAttempts = 0;
|
||||||
|
this.restartBackoffMs = 1000;
|
||||||
|
console.log('[RemoteIngressEdge] Successfully recovered from crash');
|
||||||
|
this.emit('crashRecovered');
|
||||||
|
} catch (err) {
|
||||||
|
console.error(`[RemoteIngressEdge] Crash recovery failed: ${err}`);
|
||||||
|
}
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,6 +11,8 @@ type THubCommands = {
|
|||||||
params: {
|
params: {
|
||||||
tunnelPort: number;
|
tunnelPort: number;
|
||||||
targetHost?: string;
|
targetHost?: string;
|
||||||
|
tlsCertPem?: string;
|
||||||
|
tlsKeyPem?: string;
|
||||||
};
|
};
|
||||||
result: { started: boolean };
|
result: { started: boolean };
|
||||||
};
|
};
|
||||||
@@ -42,11 +44,25 @@ type THubCommands = {
|
|||||||
export interface IHubConfig {
|
export interface IHubConfig {
|
||||||
tunnelPort?: number;
|
tunnelPort?: number;
|
||||||
targetHost?: string;
|
targetHost?: string;
|
||||||
|
tls?: {
|
||||||
|
certPem?: string;
|
||||||
|
keyPem?: string;
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type TAllowedEdge = { id: string; secret: string; listenPorts?: number[]; stunIntervalSecs?: number };
|
||||||
|
|
||||||
|
const MAX_RESTART_ATTEMPTS = 10;
|
||||||
|
const MAX_RESTART_BACKOFF_MS = 30_000;
|
||||||
|
|
||||||
export class RemoteIngressHub extends EventEmitter {
|
export class RemoteIngressHub extends EventEmitter {
|
||||||
private bridge: InstanceType<typeof plugins.smartrust.RustBridge<THubCommands>>;
|
private bridge: InstanceType<typeof plugins.smartrust.RustBridge<THubCommands>>;
|
||||||
private started = false;
|
private started = false;
|
||||||
|
private stopping = false;
|
||||||
|
private savedConfig: IHubConfig | null = null;
|
||||||
|
private savedEdges: TAllowedEdge[] = [];
|
||||||
|
private restartBackoffMs = 1000;
|
||||||
|
private restartAttempts = 0;
|
||||||
|
|
||||||
constructor() {
|
constructor() {
|
||||||
super();
|
super();
|
||||||
@@ -92,29 +108,42 @@ export class RemoteIngressHub extends EventEmitter {
|
|||||||
* Start the hub — spawns the Rust binary and starts the tunnel server.
|
* Start the hub — spawns the Rust binary and starts the tunnel server.
|
||||||
*/
|
*/
|
||||||
public async start(config: IHubConfig = {}): Promise<void> {
|
public async start(config: IHubConfig = {}): Promise<void> {
|
||||||
|
this.savedConfig = config;
|
||||||
|
this.stopping = false;
|
||||||
|
|
||||||
const spawned = await this.bridge.spawn();
|
const spawned = await this.bridge.spawn();
|
||||||
if (!spawned) {
|
if (!spawned) {
|
||||||
throw new Error('Failed to spawn remoteingress-bin');
|
throw new Error('Failed to spawn remoteingress-bin');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Register crash recovery handler
|
||||||
|
this.bridge.on('exit', this.handleCrashRecovery);
|
||||||
|
|
||||||
await this.bridge.sendCommand('startHub', {
|
await this.bridge.sendCommand('startHub', {
|
||||||
tunnelPort: config.tunnelPort ?? 8443,
|
tunnelPort: config.tunnelPort ?? 8443,
|
||||||
targetHost: config.targetHost ?? '127.0.0.1',
|
targetHost: config.targetHost ?? '127.0.0.1',
|
||||||
|
...(config.tls?.certPem && config.tls?.keyPem
|
||||||
|
? { tlsCertPem: config.tls.certPem, tlsKeyPem: config.tls.keyPem }
|
||||||
|
: {}),
|
||||||
});
|
});
|
||||||
|
|
||||||
this.started = true;
|
this.started = true;
|
||||||
|
this.restartAttempts = 0;
|
||||||
|
this.restartBackoffMs = 1000;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Stop the hub and kill the Rust process.
|
* Stop the hub and kill the Rust process.
|
||||||
*/
|
*/
|
||||||
public async stop(): Promise<void> {
|
public async stop(): Promise<void> {
|
||||||
|
this.stopping = true;
|
||||||
if (this.started) {
|
if (this.started) {
|
||||||
try {
|
try {
|
||||||
await this.bridge.sendCommand('stopHub', {} as Record<string, never>);
|
await this.bridge.sendCommand('stopHub', {} as Record<string, never>);
|
||||||
} catch {
|
} catch {
|
||||||
// Process may already be dead
|
// Process may already be dead
|
||||||
}
|
}
|
||||||
|
this.bridge.removeListener('exit', this.handleCrashRecovery);
|
||||||
this.bridge.kill();
|
this.bridge.kill();
|
||||||
this.started = false;
|
this.started = false;
|
||||||
}
|
}
|
||||||
@@ -123,7 +152,8 @@ export class RemoteIngressHub extends EventEmitter {
|
|||||||
/**
|
/**
|
||||||
* Update the list of allowed edges that can connect to this hub.
|
* Update the list of allowed edges that can connect to this hub.
|
||||||
*/
|
*/
|
||||||
public async updateAllowedEdges(edges: Array<{ id: string; secret: string; listenPorts?: number[]; stunIntervalSecs?: number }>): Promise<void> {
|
public async updateAllowedEdges(edges: TAllowedEdge[]): Promise<void> {
|
||||||
|
this.savedEdges = edges;
|
||||||
await this.bridge.sendCommand('updateAllowedEdges', { edges });
|
await this.bridge.sendCommand('updateAllowedEdges', { edges });
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -140,4 +170,62 @@ export class RemoteIngressHub extends EventEmitter {
|
|||||||
public get running(): boolean {
|
public get running(): boolean {
|
||||||
return this.bridge.running;
|
return this.bridge.running;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Handle unexpected Rust binary crash — auto-restart with backoff.
|
||||||
|
*/
|
||||||
|
private handleCrashRecovery = async (code: number | null, signal: string | null) => {
|
||||||
|
if (this.stopping || !this.started || !this.savedConfig) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
console.error(
|
||||||
|
`[RemoteIngressHub] Rust binary crashed (code=${code}, signal=${signal}), ` +
|
||||||
|
`attempt ${this.restartAttempts + 1}/${MAX_RESTART_ATTEMPTS}`
|
||||||
|
);
|
||||||
|
|
||||||
|
this.started = false;
|
||||||
|
|
||||||
|
if (this.restartAttempts >= MAX_RESTART_ATTEMPTS) {
|
||||||
|
console.error('[RemoteIngressHub] Max restart attempts reached, giving up');
|
||||||
|
this.emit('crashRecoveryFailed');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await new Promise(resolve => setTimeout(resolve, this.restartBackoffMs));
|
||||||
|
this.restartBackoffMs = Math.min(this.restartBackoffMs * 2, MAX_RESTART_BACKOFF_MS);
|
||||||
|
this.restartAttempts++;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const spawned = await this.bridge.spawn();
|
||||||
|
if (!spawned) {
|
||||||
|
console.error('[RemoteIngressHub] Failed to respawn binary');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.bridge.on('exit', this.handleCrashRecovery);
|
||||||
|
|
||||||
|
const config = this.savedConfig;
|
||||||
|
await this.bridge.sendCommand('startHub', {
|
||||||
|
tunnelPort: config.tunnelPort ?? 8443,
|
||||||
|
targetHost: config.targetHost ?? '127.0.0.1',
|
||||||
|
...(config.tls?.certPem && config.tls?.keyPem
|
||||||
|
? { tlsCertPem: config.tls.certPem, tlsKeyPem: config.tls.keyPem }
|
||||||
|
: {}),
|
||||||
|
});
|
||||||
|
|
||||||
|
// Restore allowed edges
|
||||||
|
if (this.savedEdges.length > 0) {
|
||||||
|
await this.bridge.sendCommand('updateAllowedEdges', { edges: this.savedEdges });
|
||||||
|
}
|
||||||
|
|
||||||
|
this.started = true;
|
||||||
|
this.restartAttempts = 0;
|
||||||
|
this.restartBackoffMs = 1000;
|
||||||
|
console.log('[RemoteIngressHub] Successfully recovered from crash');
|
||||||
|
this.emit('crashRecovered');
|
||||||
|
} catch (err) {
|
||||||
|
console.error(`[RemoteIngressHub] Crash recovery failed: ${err}`);
|
||||||
|
}
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user