push.rocks/smartdb
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(rustdb-storage): detect stale hint files using data file size metadata and add restart persistence regression tests

Browse Source
This commit is contained in:
Jürgen Kunz
2026-04-05 09:48:10 +00:00
parent 8ebc1bb9e1
commit 22e010c554
10 changed files with 674 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rust/crates/rustdb-storage/src/file.rs
+44 -21
View File
@@ -178,7 +178,8 @@ impl CollectionState {
tracing::warn!("compaction failed for {:?}: {e}", self.coll_dir);
} else {
// Persist hint file after successful compaction to prevent stale hints
if let Err(e) = self.keydir.persist_to_hint_file(&self.hint_path()) {
let current_size = self.data_file_size.load(Ordering::Relaxed);
if let Err(e) = self.keydir.persist_to_hint_file(&self.hint_path(), current_size) {
tracing::warn!("failed to persist hint after compaction for {:?}: {e}", self.coll_dir);
}
}
@@ -257,26 +258,47 @@ impl FileStorageAdapter {
// Try loading from hint file first, fall back to data file scan
let (keydir, dead_bytes, loaded_from_hint) = if hint_path.exists() && data_path.exists() {
match KeyDir::load_from_hint_file(&hint_path) {
Ok(Some(kd)) => {
// Validate hint against actual data file
let hint_valid = kd.validate_against_data_file(&data_path, 16)
.unwrap_or(false);
if hint_valid {
debug!("loaded KeyDir from hint file: {:?}", hint_path);
let file_size = std::fs::metadata(&data_path)
.map(|m| m.len())
.unwrap_or(FILE_HEADER_SIZE as u64);
let live_bytes: u64 = {
let mut total = 0u64;
kd.for_each(|_, e| total += e.record_len as u64);
total
};
let dead = file_size.saturating_sub(FILE_HEADER_SIZE as u64).saturating_sub(live_bytes);
(kd, dead, true)
} else {
tracing::warn!("hint file {:?} is stale, rebuilding from data file", hint_path);
Ok(Some((kd, stored_size))) => {
let actual_size = std::fs::metadata(&data_path)
.map(|m| m.len())
.unwrap_or(0);
// Check if data.rdb changed since the hint was written.
// If stored_size is 0, this is an old-format hint without size tracking.
let size_matches = stored_size > 0 && stored_size == actual_size;
if !size_matches {
// data.rdb size differs from hint snapshot — records were appended
// (inserts, tombstones) after the hint was written. Full scan required
// to pick up tombstones that would otherwise be invisible.
if stored_size == 0 {
debug!("hint file {:?} has no size tracking, rebuilding from data file", hint_path);
} else {
tracing::warn!(
"hint file {:?} is stale: data size changed ({} -> {}), rebuilding",
hint_path, stored_size, actual_size
);
}
let (kd, dead, _stats) = KeyDir::build_from_data_file(&data_path)?;
(kd, dead, false)
} else {
// Size matches — validate entry integrity with spot-checks
let hint_valid = kd.validate_against_data_file(&data_path, 16)
.unwrap_or(false);
if hint_valid {
debug!("loaded KeyDir from hint file: {:?}", hint_path);
let live_bytes: u64 = {
let mut total = 0u64;
kd.for_each(|_, e| total += e.record_len as u64);
total
};
let dead = actual_size.saturating_sub(FILE_HEADER_SIZE as u64).saturating_sub(live_bytes);
(kd, dead, true)
} else {
tracing::warn!("hint file {:?} failed validation, rebuilding from data file", hint_path);
let (kd, dead, _stats) = KeyDir::build_from_data_file(&data_path)?;
(kd, dead, false)
}
}
}
_ => {
@@ -510,10 +532,11 @@ impl StorageAdapter for FileStorageAdapter {
handle.abort();
}
// Persist all KeyDir hint files
// Persist all KeyDir hint files with current data file sizes
for entry in self.collections.iter() {
let state = entry.value();
let _ = state.keydir.persist_to_hint_file(&state.hint_path());
let current_size = state.data_file_size.load(Ordering::Relaxed);
let _ = state.keydir.persist_to_hint_file(&state.hint_path(), current_size);
}
debug!("FileStorageAdapter closed");
Ok(())