BREAKING CHANGE(core): replace the TypeScript database engine with a Rust-backed embedded server and bridge

rust/crates/rustdb/Cargo.toml

@@ -0,0 +1,38 @@
+[package]
+name = "rustdb"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+authors.workspace = true
+description = "MongoDB-compatible embedded database server with wire protocol support"
+
+[[bin]]
+name = "rustdb"
+path = "src/main.rs"
+
+[lib]
+name = "rustdb"
+path = "src/lib.rs"
+
+[dependencies]
+rustdb-config = { workspace = true }
+rustdb-wire = { workspace = true }
+rustdb-query = { workspace = true }
+rustdb-storage = { workspace = true }
+rustdb-index = { workspace = true }
+rustdb-txn = { workspace = true }
+rustdb-commands = { workspace = true }
+tokio = { workspace = true }
+tokio-util = { workspace = true }
+tracing = { workspace = true }
+tracing-subscriber = { workspace = true }
+clap = { workspace = true }
+anyhow = { workspace = true }
+arc-swap = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+bson = { workspace = true }
+bytes = { workspace = true }
+dashmap = { workspace = true }
+mimalloc = { workspace = true }
+futures-util = { version = "0.3", features = ["sink"] }

rust/crates/rustdb/src/lib.rs

@@ -0,0 +1,213 @@
+pub mod management;
+
+use std::sync::Arc;
+
+use anyhow::Result;
+use dashmap::DashMap;
+use tokio::net::TcpListener;
+#[cfg(unix)]
+use tokio::net::UnixListener;
+use tokio_util::codec::Framed;
+use tokio_util::sync::CancellationToken;
+
+use rustdb_config::{RustDbOptions, StorageType};
+use rustdb_wire::{WireCodec, OP_QUERY};
+use rustdb_wire::{encode_op_msg_response, encode_op_reply_response};
+use rustdb_storage::{StorageAdapter, MemoryStorageAdapter, FileStorageAdapter};
+// IndexEngine is used indirectly via CommandContext
+use rustdb_txn::{TransactionEngine, SessionEngine};
+use rustdb_commands::{CommandRouter, CommandContext};
+
+/// The main RustDb server.
+pub struct RustDb {
+    options: RustDbOptions,
+    ctx: Arc<CommandContext>,
+    router: Arc<CommandRouter>,
+    cancel_token: CancellationToken,
+    listener_handle: Option<tokio::task::JoinHandle<()>>,
+}
+
+impl RustDb {
+    /// Create a new RustDb server with the given options.
+    pub async fn new(options: RustDbOptions) -> Result<Self> {
+        // Create storage adapter
+        let storage: Arc<dyn StorageAdapter> = match options.storage {
+            StorageType::Memory => {
+                let adapter = MemoryStorageAdapter::new();
+                Arc::new(adapter)
+            }
+            StorageType::File => {
+                let path = options
+                    .storage_path
+                    .clone()
+                    .unwrap_or_else(|| "./data".to_string());
+                let adapter = FileStorageAdapter::new(&path);
+                Arc::new(adapter)
+            }
+        };
+
+        // Initialize storage
+        storage.initialize().await?;
+
+        let ctx = Arc::new(CommandContext {
+            storage,
+            indexes: Arc::new(DashMap::new()),
+            transactions: Arc::new(TransactionEngine::new()),
+            sessions: Arc::new(SessionEngine::new(30 * 60 * 1000, 60 * 1000)),
+            cursors: Arc::new(DashMap::new()),
+            start_time: std::time::Instant::now(),
+        });
+
+        let router = Arc::new(CommandRouter::new(ctx.clone()));
+
+        Ok(Self {
+            options,
+            ctx,
+            router,
+            cancel_token: CancellationToken::new(),
+            listener_handle: None,
+        })
+    }
+
+    /// Start listening for connections.
+    pub async fn start(&mut self) -> Result<()> {
+        let cancel = self.cancel_token.clone();
+        let router = self.router.clone();
+
+        if let Some(ref socket_path) = self.options.socket_path {
+            #[cfg(unix)]
+            {
+                // Remove stale socket file
+                let _ = tokio::fs::remove_file(socket_path).await;
+
+                let listener = UnixListener::bind(socket_path)?;
+                let socket_path_clone = socket_path.clone();
+                tracing::info!("RustDb listening on unix:{}", socket_path_clone);
+
+                let handle = tokio::spawn(async move {
+                    loop {
+                        tokio::select! {
+                            _ = cancel.cancelled() => break,
+                            result = listener.accept() => {
+                                match result {
+                                    Ok((stream, _addr)) => {
+                                        let router = router.clone();
+                                        tokio::spawn(async move {
+                                            handle_connection(stream, router).await;
+                                        });
+                                    }
+                                    Err(e) => {
+                                        tracing::error!("Accept error: {}", e);
+                                    }
+                                }
+                            }
+                        }
+                    }
+                });
+                self.listener_handle = Some(handle);
+            }
+            #[cfg(not(unix))]
+            {
+                anyhow::bail!("Unix sockets are not supported on this platform");
+            }
+        } else {
+            let addr = format!("{}:{}", self.options.host, self.options.port);
+            let listener = TcpListener::bind(&addr).await?;
+            tracing::info!("RustDb listening on {}", addr);
+
+            let handle = tokio::spawn(async move {
+                loop {
+                    tokio::select! {
+                        _ = cancel.cancelled() => break,
+                        result = listener.accept() => {
+                            match result {
+                                Ok((stream, _addr)) => {
+                                    let _ = stream.set_nodelay(true);
+                                    let router = router.clone();
+                                    tokio::spawn(async move {
+                                        handle_connection(stream, router).await;
+                                    });
+                                }
+                                Err(e) => {
+                                    tracing::error!("Accept error: {}", e);
+                                }
+                            }
+                        }
+                    }
+                }
+            });
+            self.listener_handle = Some(handle);
+        }
+
+        Ok(())
+    }
+
+    /// Stop the server.
+    pub async fn stop(&mut self) -> Result<()> {
+        self.cancel_token.cancel();
+
+        if let Some(handle) = self.listener_handle.take() {
+            handle.abort();
+            let _ = handle.await;
+        }
+
+        // Close storage (persists if configured)
+        self.ctx.storage.close().await?;
+
+        // Clean up Unix socket file
+        if let Some(ref socket_path) = self.options.socket_path {
+            let _ = tokio::fs::remove_file(socket_path).await;
+        }
+
+        Ok(())
+    }
+
+    /// Get the connection URI.
+    pub fn connection_uri(&self) -> String {
+        self.options.connection_uri()
+    }
+}
+
+/// Handle a single client connection using the wire protocol codec.
+async fn handle_connection<S>(stream: S, router: Arc<CommandRouter>)
+where
+    S: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin,
+{
+    use futures_util::{SinkExt, StreamExt};
+
+    let mut framed = Framed::new(stream, WireCodec);
+
+    while let Some(result) = framed.next().await {
+        match result {
+            Ok(parsed_cmd) => {
+                let request_id = parsed_cmd.request_id;
+                let op_code = parsed_cmd.op_code;
+
+                let response_doc = router.route(&parsed_cmd).await;
+
+                let response_id = next_request_id();
+
+                let response_bytes = if op_code == OP_QUERY {
+                    encode_op_reply_response(request_id, &[response_doc], response_id, 0)
+                } else {
+                    encode_op_msg_response(request_id, &response_doc, response_id)
+                };
+
+                if let Err(e) = framed.send(response_bytes).await {
+                    tracing::debug!("Failed to send response: {}", e);
+                    break;
+                }
+            }
+            Err(e) => {
+                tracing::debug!("Wire protocol error: {}", e);
+                break;
+            }
+        }
+    }
+}
+
+fn next_request_id() -> i32 {
+    use std::sync::atomic::{AtomicI32, Ordering};
+    static COUNTER: AtomicI32 = AtomicI32::new(1);
+    COUNTER.fetch_add(1, Ordering::Relaxed)
+}

rust/crates/rustdb/src/main.rs

@@ -0,0 +1,85 @@
+#[global_allocator]
+static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
+
+use clap::Parser;
+use tracing_subscriber::EnvFilter;
+use anyhow::Result;
+
+use rustdb::RustDb;
+use rustdb::management;
+use rustdb_config::RustDbOptions;
+
+/// RustDb - MongoDB-compatible embedded database server
+#[derive(Parser, Debug)]
+#[command(name = "rustdb", version, about)]
+struct Cli {
+    /// Path to JSON configuration file
+    #[arg(short, long, default_value = "config.json")]
+    config: String,
+
+    /// Log level (trace, debug, info, warn, error)
+    #[arg(short, long, default_value = "info")]
+    log_level: String,
+
+    /// Validate configuration without starting
+    #[arg(long)]
+    validate: bool,
+
+    /// Run in management mode (JSON-over-stdin IPC for TypeScript wrapper)
+    #[arg(long)]
+    management: bool,
+}
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let cli = Cli::parse();
+
+    // Initialize tracing - write to stderr so stdout is reserved for management IPC
+    tracing_subscriber::fmt()
+        .with_writer(std::io::stderr)
+        .with_env_filter(
+            EnvFilter::try_from_default_env()
+                .unwrap_or_else(|_| EnvFilter::new(&cli.log_level)),
+        )
+        .init();
+
+    // Management mode: JSON IPC over stdin/stdout
+    if cli.management {
+        tracing::info!("RustDb starting in management mode...");
+        return management::management_loop().await;
+    }
+
+    tracing::info!("RustDb starting...");
+
+    // Load configuration
+    let options = RustDbOptions::from_file(&cli.config)
+        .map_err(|e| anyhow::anyhow!("Failed to load config '{}': {}", cli.config, e))?;
+
+    // Validate-only mode
+    if cli.validate {
+        match options.validate() {
+            Ok(()) => {
+                tracing::info!("Configuration is valid");
+                return Ok(());
+            }
+            Err(e) => {
+                tracing::error!("Validation error: {}", e);
+                anyhow::bail!("Configuration validation failed: {}", e);
+            }
+        }
+    }
+
+    // Create and start server
+    let mut db = RustDb::new(options).await?;
+    db.start().await?;
+
+    // Wait for shutdown signal
+    tracing::info!("RustDb is running. Press Ctrl+C to stop.");
+    tokio::signal::ctrl_c().await?;
+
+    tracing::info!("Shutdown signal received");
+    db.stop().await?;
+
+    tracing::info!("RustDb shutdown complete");
+    Ok(())
+}

rust/crates/rustdb/src/management.rs

@@ -0,0 +1,240 @@
+use anyhow::Result;
+use serde::{Deserialize, Serialize};
+use tokio::io::{AsyncBufReadExt, BufReader};
+use tracing::{info, error};
+
+use crate::RustDb;
+use rustdb_config::RustDbOptions;
+
+/// A management request from the TypeScript wrapper.
+#[derive(Debug, Deserialize)]
+pub struct ManagementRequest {
+    pub id: String,
+    pub method: String,
+    #[serde(default)]
+    pub params: serde_json::Value,
+}
+
+/// A management response back to the TypeScript wrapper.
+#[derive(Debug, Serialize)]
+pub struct ManagementResponse {
+    pub id: String,
+    pub success: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub result: Option<serde_json::Value>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub error: Option<String>,
+}
+
+/// An unsolicited event from the server to the TypeScript wrapper.
+#[derive(Debug, Serialize)]
+pub struct ManagementEvent {
+    pub event: String,
+    pub data: serde_json::Value,
+}
+
+impl ManagementResponse {
+    fn ok(id: String, result: serde_json::Value) -> Self {
+        Self {
+            id,
+            success: true,
+            result: Some(result),
+            error: None,
+        }
+    }
+
+    fn err(id: String, message: String) -> Self {
+        Self {
+            id,
+            success: false,
+            result: None,
+            error: Some(message),
+        }
+    }
+}
+
+fn send_line(line: &str) {
+    use std::io::Write;
+    let stdout = std::io::stdout();
+    let mut handle = stdout.lock();
+    let _ = handle.write_all(line.as_bytes());
+    let _ = handle.write_all(b"\n");
+    let _ = handle.flush();
+}
+
+fn send_response(response: &ManagementResponse) {
+    match serde_json::to_string(response) {
+        Ok(json) => send_line(&json),
+        Err(e) => error!("Failed to serialize management response: {}", e),
+    }
+}
+
+fn send_event(event: &str, data: serde_json::Value) {
+    let evt = ManagementEvent {
+        event: event.to_string(),
+        data,
+    };
+    match serde_json::to_string(&evt) {
+        Ok(json) => send_line(&json),
+        Err(e) => error!("Failed to serialize management event: {}", e),
+    }
+}
+
+/// Run the management loop, reading JSON commands from stdin and writing responses to stdout.
+pub async fn management_loop() -> Result<()> {
+    let stdin = BufReader::new(tokio::io::stdin());
+    let mut lines = stdin.lines();
+    let mut db: Option<RustDb> = None;
+
+    send_event("ready", serde_json::json!({}));
+
+    loop {
+        let line = match lines.next_line().await {
+            Ok(Some(line)) => line,
+            Ok(None) => {
+                // stdin closed - parent process exited
+                info!("Management stdin closed, shutting down");
+                if let Some(ref mut d) = db {
+                    let _ = d.stop().await;
+                }
+                break;
+            }
+            Err(e) => {
+                error!("Error reading management stdin: {}", e);
+                break;
+            }
+        };
+
+        let line = line.trim().to_string();
+        if line.is_empty() {
+            continue;
+        }
+
+        let request: ManagementRequest = match serde_json::from_str(&line) {
+            Ok(r) => r,
+            Err(e) => {
+                error!("Failed to parse management request: {}", e);
+                send_response(&ManagementResponse::err(
+                    "unknown".to_string(),
+                    format!("Failed to parse request: {}", e),
+                ));
+                continue;
+            }
+        };
+
+        let response = handle_request(&request, &mut db).await;
+        send_response(&response);
+    }
+
+    Ok(())
+}
+
+async fn handle_request(
+    request: &ManagementRequest,
+    db: &mut Option<RustDb>,
+) -> ManagementResponse {
+    let id = request.id.clone();
+
+    match request.method.as_str() {
+        "start" => handle_start(&id, &request.params, db).await,
+        "stop" => handle_stop(&id, db).await,
+        "getStatus" => handle_get_status(&id, db),
+        "getMetrics" => handle_get_metrics(&id, db),
+        _ => ManagementResponse::err(id, format!("Unknown method: {}", request.method)),
+    }
+}
+
+async fn handle_start(
+    id: &str,
+    params: &serde_json::Value,
+    db: &mut Option<RustDb>,
+) -> ManagementResponse {
+    if db.is_some() {
+        return ManagementResponse::err(id.to_string(), "Server is already running".to_string());
+    }
+
+    let config = match params.get("config") {
+        Some(config) => config,
+        None => return ManagementResponse::err(id.to_string(), "Missing 'config' parameter".to_string()),
+    };
+
+    let options: RustDbOptions = match serde_json::from_value(config.clone()) {
+        Ok(o) => o,
+        Err(e) => return ManagementResponse::err(id.to_string(), format!("Invalid config: {}", e)),
+    };
+
+    let connection_uri = options.connection_uri();
+
+    match RustDb::new(options).await {
+        Ok(mut d) => {
+            match d.start().await {
+                Ok(()) => {
+                    send_event("started", serde_json::json!({}));
+                    *db = Some(d);
+                    ManagementResponse::ok(
+                        id.to_string(),
+                        serde_json::json!({ "connectionUri": connection_uri }),
+                    )
+                }
+                Err(e) => {
+                    send_event("error", serde_json::json!({"message": format!("{}", e)}));
+                    ManagementResponse::err(id.to_string(), format!("Failed to start: {}", e))
+                }
+            }
+        }
+        Err(e) => ManagementResponse::err(id.to_string(), format!("Failed to create server: {}", e)),
+    }
+}
+
+async fn handle_stop(
+    id: &str,
+    db: &mut Option<RustDb>,
+) -> ManagementResponse {
+    match db.as_mut() {
+        Some(d) => {
+            match d.stop().await {
+                Ok(()) => {
+                    *db = None;
+                    send_event("stopped", serde_json::json!({}));
+                    ManagementResponse::ok(id.to_string(), serde_json::json!({}))
+                }
+                Err(e) => ManagementResponse::err(id.to_string(), format!("Failed to stop: {}", e)),
+            }
+        }
+        None => ManagementResponse::ok(id.to_string(), serde_json::json!({})),
+    }
+}
+
+fn handle_get_status(
+    id: &str,
+    db: &Option<RustDb>,
+) -> ManagementResponse {
+    match db.as_ref() {
+        Some(_d) => ManagementResponse::ok(
+            id.to_string(),
+            serde_json::json!({
+                "running": true,
+            }),
+        ),
+        None => ManagementResponse::ok(
+            id.to_string(),
+            serde_json::json!({ "running": false }),
+        ),
+    }
+}
+
+fn handle_get_metrics(
+    id: &str,
+    db: &Option<RustDb>,
+) -> ManagementResponse {
+    match db.as_ref() {
+        Some(_d) => ManagementResponse::ok(
+            id.to_string(),
+            serde_json::json!({
+                "connections": 0,
+                "databases": 0,
+            }),
+        ),
+        None => ManagementResponse::err(id.to_string(), "Server is not running".to_string()),
+    }
+}