push.rocks/smartdb
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(smartdb): add operation log APIs, point-in-time revert support, and a web-based debug dashboard

Browse Source
This commit is contained in:
Jürgen Kunz
2026-04-02 17:02:03 +00:00
parent 943302f789
commit d34b8673e1
27 changed files with 3536 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
rust/crates/rustdb-commands/src/context.rs
View File

@@ -3,7 +3,7 @@ use std::sync::Arc;
use bson::Document;
use dashmap::DashMap;
use rustdb_index::IndexEngine;
use rustdb_storage::StorageAdapter;
use rustdb_storage::{OpLog, StorageAdapter};
use rustdb_txn::{SessionEngine, TransactionEngine};
/// Shared command execution context, passed to all handlers.
@@ -20,6 +20,8 @@ pub struct CommandContext {
pub cursors: Arc<DashMap<i64, CursorState>>,
/// Server start time (for uptime reporting).
pub start_time: std::time::Instant,
/// Operation log for point-in-time replay.
pub oplog: Arc<OpLog>,
}
/// State of an open cursor from a find or aggregate command.

11
rust/crates/rustdb-commands/src/handlers/delete_handler.rs
View File

@@ -2,6 +2,7 @@ use std::collections::HashSet;
use bson::{doc, Bson, Document};
use rustdb_query::QueryMatcher;
use rustdb_storage::OpType;
use tracing::debug;
use crate::context::CommandContext;
@@ -171,6 +172,16 @@ async fn delete_matching(
.await
.map_err(|e| CommandError::StorageError(e.to_string()))?;
// Record in oplog.
ctx.oplog.append(
OpType::Delete,
db,
coll,
&id_str,
None,
Some(doc.clone()),
);
// Update index engine.
if let Some(mut engine) = ctx.indexes.get_mut(ns_key) {
engine.on_delete(doc);

13
rust/crates/rustdb-commands/src/handlers/insert_handler.rs
View File

@@ -2,6 +2,7 @@ use std::collections::HashMap;
use bson::{doc, oid::ObjectId, Bson, Document};
use rustdb_index::IndexEngine;
use rustdb_storage::OpType;
use tracing::{debug, warn};
use crate::context::CommandContext;
@@ -63,7 +64,17 @@ pub async fn handle(
// Attempt storage insert.
match ctx.storage.insert_one(db, coll, doc.clone()).await {
Ok(_id_str) => {
Ok(id_str) => {
// Record in oplog.
ctx.oplog.append(
OpType::Insert,
db,
coll,
&id_str,
Some(doc.clone()),
None,
);
// Update index engine.
let mut engine = ctx
.indexes

55
rust/crates/rustdb-commands/src/handlers/update_handler.rs
View File

@@ -3,6 +3,7 @@ use std::collections::HashSet;
use bson::{doc, oid::ObjectId, Bson, Document};
use rustdb_index::IndexEngine;
use rustdb_query::{QueryMatcher, UpdateEngine, sort_documents, apply_projection};
use rustdb_storage::OpType;
use tracing::debug;
use crate::context::CommandContext;
@@ -151,7 +152,17 @@ async fn handle_update(
// Insert the new document.
match ctx.storage.insert_one(db, coll, updated.clone()).await {
Ok(_) => {
Ok(id_str) => {
// Record upsert in oplog as an insert.
ctx.oplog.append(
OpType::Insert,
db,
coll,
&id_str,
Some(updated.clone()),
None,
);
// Update index.
let mut engine = ctx
.indexes
@@ -212,6 +223,16 @@ async fn handle_update(
.await
{
Ok(()) => {
// Record in oplog.
ctx.oplog.append(
OpType::Update,
db,
coll,
&id_str,
Some(updated_doc.clone()),
Some(matched_doc.clone()),
);
// Update index.
if let Some(mut engine) = ctx.indexes.get_mut(&ns_key) {
let _ = engine.on_update(matched_doc, &updated_doc);
@@ -362,6 +383,16 @@ async fn handle_find_and_modify(
let id_str = extract_id_string(doc);
ctx.storage.delete_by_id(db, coll, &id_str).await?;
// Record in oplog.
ctx.oplog.append(
OpType::Delete,
db,
coll,
&id_str,
None,
Some(doc.clone()),
);
// Update index.
if let Some(mut engine) = ctx.indexes.get_mut(&ns_key) {
engine.on_delete(doc);
@@ -418,6 +449,16 @@ async fn handle_find_and_modify(
.update_by_id(db, coll, &id_str, updated_doc.clone())
.await?;
// Record in oplog.
ctx.oplog.append(
OpType::Update,
db,
coll,
&id_str,
Some(updated_doc.clone()),
Some(original_doc.clone()),
);
// Update index.
if let Some(mut engine) = ctx.indexes.get_mut(&ns_key) {
let _ = engine.on_update(&original_doc, &updated_doc);
@@ -464,10 +505,20 @@ async fn handle_find_and_modify(
updated_doc.get("_id").unwrap().clone()
};
ctx.storage
let inserted_id_str = ctx.storage
.insert_one(db, coll, updated_doc.clone())
.await?;
// Record upsert in oplog as an insert.
ctx.oplog.append(
OpType::Insert,
db,
coll,
&inserted_id_str,
Some(updated_doc.clone()),
None,
);
// Update index.
{
let mut engine = ctx

2
rust/crates/rustdb-storage/src/lib.rs
View File

@@ -18,5 +18,5 @@ pub use adapter::StorageAdapter;
pub use error::{StorageError, StorageResult};
pub use file::FileStorageAdapter;
pub use memory::MemoryStorageAdapter;
pub use oplog::{OpLog, OpLogEntry, OpType};
pub use oplog::{OpLog, OpLogEntry, OpLogStats, OpType};
pub use wal::{WalOp, WalRecord, WriteAheadLog};

85
rust/crates/rustdb-storage/src/oplog.rs
View File

@@ -2,6 +2,8 @@
//!
//! The OpLog records every write operation so that changes can be replayed,
//! replicated, or used for change-stream style notifications.
//! Each entry stores both the new and previous document state, enabling
//! point-in-time replay and revert.
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{SystemTime, UNIX_EPOCH};
@@ -33,8 +35,21 @@ pub struct OpLogEntry {
pub collection: String,
/// Document id (hex string).
pub document_id: String,
/// The document snapshot (for insert/update; None for delete).
/// The new document snapshot (for insert/update; None for delete).
pub document: Option<Document>,
/// The previous document snapshot (for update/delete; None for insert).
pub previous_document: Option<Document>,
}
/// Aggregate statistics about the oplog.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct OpLogStats {
pub current_seq: u64,
pub total_entries: usize,
pub oldest_seq: u64,
pub inserts: usize,
pub updates: usize,
pub deletes: usize,
}
/// In-memory operation log.
@@ -61,6 +76,7 @@ impl OpLog {
collection: &str,
document_id: &str,
document: Option<Document>,
previous_document: Option<Document>,
) -> u64 {
let seq = self.next_seq.fetch_add(1, Ordering::SeqCst);
let entry = OpLogEntry {
@@ -74,11 +90,17 @@ impl OpLog {
collection: collection.to_string(),
document_id: document_id.to_string(),
document,
previous_document,
};
self.entries.insert(seq, entry);
seq
}
/// Get a single entry by sequence number.
pub fn get_entry(&self, seq: u64) -> Option<OpLogEntry> {
self.entries.get(&seq).map(|e| e.value().clone())
}
/// Get all entries with sequence number >= `since`.
pub fn entries_since(&self, since: u64) -> Vec<OpLogEntry> {
let mut result: Vec<_> = self
@@ -91,11 +113,72 @@ impl OpLog {
result
}
/// Get entries in range [from_seq, to_seq] inclusive, sorted by seq.
pub fn entries_range(&self, from_seq: u64, to_seq: u64) -> Vec<OpLogEntry> {
let mut result: Vec<_> = self
.entries
.iter()
.filter(|e| {
let k = *e.key();
k >= from_seq && k <= to_seq
})
.map(|e| e.value().clone())
.collect();
result.sort_by_key(|e| e.seq);
result
}
/// Remove all entries with seq > `after_seq` and reset the next_seq counter.
pub fn truncate_after(&self, after_seq: u64) {
let keys_to_remove: Vec<u64> = self
.entries
.iter()
.filter(|e| *e.key() > after_seq)
.map(|e| *e.key())
.collect();
for key in keys_to_remove {
self.entries.remove(&key);
}
self.next_seq.store(after_seq + 1, Ordering::SeqCst);
}
/// Get the current (latest) sequence number. Returns 0 if empty.
pub fn current_seq(&self) -> u64 {
self.next_seq.load(Ordering::SeqCst).saturating_sub(1)
}
/// Get aggregate statistics.
pub fn stats(&self) -> OpLogStats {
let mut inserts = 0usize;
let mut updates = 0usize;
let mut deletes = 0usize;
let mut oldest_seq = u64::MAX;
for entry in self.entries.iter() {
match entry.value().op {
OpType::Insert => inserts += 1,
OpType::Update => updates += 1,
OpType::Delete => deletes += 1,
}
if entry.value().seq < oldest_seq {
oldest_seq = entry.value().seq;
}
}
if oldest_seq == u64::MAX {
oldest_seq = 0;
}
OpLogStats {
current_seq: self.current_seq(),
total_entries: self.entries.len(),
oldest_seq,
inserts,
updates,
deletes,
}
}
/// Clear all entries.
pub fn clear(&self) {
self.entries.clear();

8
rust/crates/rustdb/src/lib.rs
View File

@@ -13,7 +13,7 @@ use tokio_util::sync::CancellationToken;
use rustdb_config::{RustDbOptions, StorageType};
use rustdb_wire::{WireCodec, OP_QUERY};
use rustdb_wire::{encode_op_msg_response, encode_op_reply_response};
use rustdb_storage::{StorageAdapter, MemoryStorageAdapter, FileStorageAdapter};
use rustdb_storage::{StorageAdapter, MemoryStorageAdapter, FileStorageAdapter, OpLog};
// IndexEngine is used indirectly via CommandContext
use rustdb_txn::{TransactionEngine, SessionEngine};
use rustdb_commands::{CommandRouter, CommandContext};
@@ -56,6 +56,7 @@ impl RustDb {
sessions: Arc::new(SessionEngine::new(30 * 60 * 1000, 60 * 1000)),
cursors: Arc::new(DashMap::new()),
start_time: std::time::Instant::now(),
oplog: Arc::new(OpLog::new()),
});
let router = Arc::new(CommandRouter::new(ctx.clone()));
@@ -166,6 +167,11 @@ impl RustDb {
pub fn connection_uri(&self) -> String {
self.options.connection_uri()
}
/// Get a reference to the shared command context (for management IPC access to oplog, storage, etc.).
pub fn ctx(&self) -> &Arc<CommandContext> {
&self.ctx
}
}
/// Handle a single client connection using the wire protocol codec.

338
rust/crates/rustdb/src/management.rs
View File

@@ -139,7 +139,12 @@ async fn handle_request(
"start" => handle_start(&id, &request.params, db).await,
"stop" => handle_stop(&id, db).await,
"getStatus" => handle_get_status(&id, db),
"getMetrics" => handle_get_metrics(&id, db),
"getMetrics" => handle_get_metrics(&id, db).await,
"getOpLog" => handle_get_oplog(&id, &request.params, db),
"getOpLogStats" => handle_get_oplog_stats(&id, db),
"revertToSeq" => handle_revert_to_seq(&id, &request.params, db).await,
"getCollections" => handle_get_collections(&id, &request.params, db).await,
"getDocuments" => handle_get_documents(&id, &request.params, db).await,
_ => ManagementResponse::err(id, format!("Unknown method: {}", request.method)),
}
}
@@ -223,18 +228,333 @@ fn handle_get_status(
}
}
fn handle_get_metrics(
async fn handle_get_metrics(
id: &str,
db: &Option<RustDb>,
) -> ManagementResponse {
match db.as_ref() {
Some(_d) => ManagementResponse::ok(
id.to_string(),
serde_json::json!({
"connections": 0,
"databases": 0,
}),
),
Some(d) => {
let ctx = d.ctx();
let db_list = ctx.storage.list_databases().await.unwrap_or_default();
let mut total_collections = 0u64;
for db_name in &db_list {
if let Ok(colls) = ctx.storage.list_collections(db_name).await {
total_collections += colls.len() as u64;
}
}
let oplog_stats = ctx.oplog.stats();
let uptime_secs = ctx.start_time.elapsed().as_secs();
ManagementResponse::ok(
id.to_string(),
serde_json::json!({
"databases": db_list.len(),
"collections": total_collections,
"oplogEntries": oplog_stats.total_entries,
"oplogCurrentSeq": oplog_stats.current_seq,
"uptimeSeconds": uptime_secs,
}),
)
}
None => ManagementResponse::err(id.to_string(), "Server is not running".to_string()),
}
}
fn handle_get_oplog(
id: &str,
params: &serde_json::Value,
db: &Option<RustDb>,
) -> ManagementResponse {
let d = match db.as_ref() {
Some(d) => d,
None => return ManagementResponse::err(id.to_string(), "Server is not running".to_string()),
};
let ctx = d.ctx();
let since_seq = params.get("sinceSeq").and_then(|v| v.as_u64()).unwrap_or(1);
let limit = params.get("limit").and_then(|v| v.as_u64()).unwrap_or(100) as usize;
let filter_db = params.get("db").and_then(|v| v.as_str());
let filter_coll = params.get("collection").and_then(|v| v.as_str());
let mut entries = ctx.oplog.entries_since(since_seq);
// Apply filters.
if let Some(fdb) = filter_db {
entries.retain(|e| e.db == fdb);
}
if let Some(fcoll) = filter_coll {
entries.retain(|e| e.collection == fcoll);
}
let total = entries.len();
entries.truncate(limit);
// Serialize entries to JSON.
let entries_json: Vec<serde_json::Value> = entries
.iter()
.map(|e| {
let doc_json = e.document.as_ref().map(|d| bson_doc_to_json(d));
let prev_json = e.previous_document.as_ref().map(|d| bson_doc_to_json(d));
serde_json::json!({
"seq": e.seq,
"timestampMs": e.timestamp_ms,
"op": match e.op {
rustdb_storage::OpType::Insert => "insert",
rustdb_storage::OpType::Update => "update",
rustdb_storage::OpType::Delete => "delete",
},
"db": e.db,
"collection": e.collection,
"documentId": e.document_id,
"document": doc_json,
"previousDocument": prev_json,
})
})
.collect();
ManagementResponse::ok(
id.to_string(),
serde_json::json!({
"entries": entries_json,
"currentSeq": ctx.oplog.current_seq(),
"totalEntries": total,
}),
)
}
fn handle_get_oplog_stats(
id: &str,
db: &Option<RustDb>,
) -> ManagementResponse {
let d = match db.as_ref() {
Some(d) => d,
None => return ManagementResponse::err(id.to_string(), "Server is not running".to_string()),
};
let stats = d.ctx().oplog.stats();
ManagementResponse::ok(
id.to_string(),
serde_json::json!({
"currentSeq": stats.current_seq,
"totalEntries": stats.total_entries,
"oldestSeq": stats.oldest_seq,
"entriesByOp": {
"insert": stats.inserts,
"update": stats.updates,
"delete": stats.deletes,
},
}),
)
}
async fn handle_revert_to_seq(
id: &str,
params: &serde_json::Value,
db: &Option<RustDb>,
) -> ManagementResponse {
let d = match db.as_ref() {
Some(d) => d,
None => return ManagementResponse::err(id.to_string(), "Server is not running".to_string()),
};
let target_seq = match params.get("seq").and_then(|v| v.as_u64()) {
Some(s) => s,
None => return ManagementResponse::err(id.to_string(), "Missing 'seq' parameter".to_string()),
};
let dry_run = params.get("dryRun").and_then(|v| v.as_bool()).unwrap_or(false);
let ctx = d.ctx();
let current = ctx.oplog.current_seq();
if target_seq > current {
return ManagementResponse::err(
id.to_string(),
format!("Target seq {} is beyond current seq {}", target_seq, current),
);
}
// Collect entries to revert (from target+1 to current), sorted descending for reverse processing.
let mut entries_to_revert = ctx.oplog.entries_range(target_seq + 1, current);
entries_to_revert.reverse();
if dry_run {
let entries_json: Vec<serde_json::Value> = entries_to_revert
.iter()
.map(|e| {
serde_json::json!({
"seq": e.seq,
"op": match e.op {
rustdb_storage::OpType::Insert => "insert",
rustdb_storage::OpType::Update => "update",
rustdb_storage::OpType::Delete => "delete",
},
"db": e.db,
"collection": e.collection,
"documentId": e.document_id,
})
})
.collect();
return ManagementResponse::ok(
id.to_string(),
serde_json::json!({
"dryRun": true,
"reverted": entries_to_revert.len(),
"entries": entries_json,
}),
);
}
// Execute revert: process each entry in reverse, using storage directly.
let mut reverted = 0u64;
let mut errors: Vec<String> = Vec::new();
for entry in &entries_to_revert {
let result = match entry.op {
rustdb_storage::OpType::Insert => {
// Undo insert -> delete the document.
ctx.storage.delete_by_id(&entry.db, &entry.collection, &entry.document_id).await
}
rustdb_storage::OpType::Update => {
// Undo update -> restore the previous document.
if let Some(ref prev_doc) = entry.previous_document {
ctx.storage
.update_by_id(&entry.db, &entry.collection, &entry.document_id, prev_doc.clone())
.await
} else {
errors.push(format!("seq {}: update entry missing previous_document", entry.seq));
continue;
}
}
rustdb_storage::OpType::Delete => {
// Undo delete -> re-insert the previous document.
if let Some(ref prev_doc) = entry.previous_document {
ctx.storage
.insert_one(&entry.db, &entry.collection, prev_doc.clone())
.await
.map(|_| ())
} else {
errors.push(format!("seq {}: delete entry missing previous_document", entry.seq));
continue;
}
}
};
match result {
Ok(()) => reverted += 1,
Err(e) => errors.push(format!("seq {}: {}", entry.seq, e)),
}
}
// Truncate the oplog to the target sequence.
ctx.oplog.truncate_after(target_seq);
let mut response = serde_json::json!({
"dryRun": false,
"reverted": reverted,
"targetSeq": target_seq,
});
if !errors.is_empty() {
response["errors"] = serde_json::json!(errors);
}
ManagementResponse::ok(id.to_string(), response)
}
async fn handle_get_collections(
id: &str,
params: &serde_json::Value,
db: &Option<RustDb>,
) -> ManagementResponse {
let d = match db.as_ref() {
Some(d) => d,
None => return ManagementResponse::err(id.to_string(), "Server is not running".to_string()),
};
let ctx = d.ctx();
let filter_db = params.get("db").and_then(|v| v.as_str());
let databases = match ctx.storage.list_databases().await {
Ok(dbs) => dbs,
Err(e) => return ManagementResponse::err(id.to_string(), format!("Failed to list databases: {}", e)),
};
let mut collections: Vec<serde_json::Value> = Vec::new();
for db_name in &databases {
if let Some(fdb) = filter_db {
if db_name != fdb {
continue;
}
}
if let Ok(colls) = ctx.storage.list_collections(db_name).await {
for coll_name in colls {
let count = ctx.storage.count(db_name, &coll_name).await.unwrap_or(0);
collections.push(serde_json::json!({
"db": db_name,
"name": coll_name,
"count": count,
}));
}
}
}
ManagementResponse::ok(
id.to_string(),
serde_json::json!({ "collections": collections }),
)
}
async fn handle_get_documents(
id: &str,
params: &serde_json::Value,
db: &Option<RustDb>,
) -> ManagementResponse {
let d = match db.as_ref() {
Some(d) => d,
None => return ManagementResponse::err(id.to_string(), "Server is not running".to_string()),
};
let db_name = match params.get("db").and_then(|v| v.as_str()) {
Some(s) => s,
None => return ManagementResponse::err(id.to_string(), "Missing 'db' parameter".to_string()),
};
let coll_name = match params.get("collection").and_then(|v| v.as_str()) {
Some(s) => s,
None => return ManagementResponse::err(id.to_string(), "Missing 'collection' parameter".to_string()),
};
let limit = params.get("limit").and_then(|v| v.as_u64()).unwrap_or(50) as usize;
let skip = params.get("skip").and_then(|v| v.as_u64()).unwrap_or(0) as usize;
let ctx = d.ctx();
let all_docs = match ctx.storage.find_all(db_name, coll_name).await {
Ok(docs) => docs,
Err(e) => return ManagementResponse::err(id.to_string(), format!("Failed to find documents: {}", e)),
};
let total = all_docs.len();
let docs: Vec<serde_json::Value> = all_docs
.into_iter()
.skip(skip)
.take(limit)
.map(|d| bson_doc_to_json(&d))
.collect();
ManagementResponse::ok(
id.to_string(),
serde_json::json!({
"documents": docs,
"total": total,
}),
)
}
/// Convert a BSON Document to a serde_json::Value.
fn bson_doc_to_json(doc: &bson::Document) -> serde_json::Value {
// Use bson's built-in relaxed extended JSON serialization.
let bson_val = bson::Bson::Document(doc.clone());
bson_val.into_relaxed_extjson()
}