feat(storage): add offline data validation and strengthen storage/index integrity checks

rust/crates/rustdb-storage/src/compaction.rs

@@ -253,7 +253,7 @@ mod tests {
         assert!(b_entry.offset > a_entry.offset);
 
         // Verify the compacted file can be used to rebuild KeyDir
-        let (rebuilt, dead) = KeyDir::build_from_data_file(&data_path).unwrap();
+        let (rebuilt, dead, _stats) = KeyDir::build_from_data_file(&data_path).unwrap();
         assert_eq!(rebuilt.len(), 2);
         assert_eq!(dead, 0); // no dead records in compacted file
     }

rust/crates/rustdb-storage/src/file.rs

@@ -21,7 +21,7 @@ use std::sync::Arc;
 use async_trait::async_trait;
 use bson::{doc, oid::ObjectId, Document};
 use dashmap::DashMap;
-use tracing::debug;
+use tracing::{debug, info};
 
 use crate::adapter::StorageAdapter;
 use crate::binary_wal::{BinaryWal, WalOpType};
@@ -83,6 +83,20 @@ impl CollectionState {
             .map_err(|e| StorageError::SerializationError(format!("BSON decode: {e}")))
     }
 
+    /// Ensure a data file has the 64-byte SMARTDB header.
+    /// If the file was just created (empty), writes the header and updates
+    /// the data_file_size counter. Must be called under write_lock.
+    fn ensure_data_header(&self, file: &mut std::fs::File) -> StorageResult<()> {
+        let pos = file.seek(SeekFrom::End(0))?;
+        if pos == 0 {
+            let hdr = FileHeader::new(FileType::Data);
+            file.write_all(&hdr.encode())?;
+            self.data_file_size
+                .fetch_add(FILE_HEADER_SIZE as u64, Ordering::Relaxed);
+        }
+        Ok(())
+    }
+
     /// Append a data record and update the KeyDir. Must be called under write_lock.
     fn append_record(
         &self,
@@ -104,6 +118,7 @@ impl CollectionState {
             .append(true)
             .open(&data_path)?;
 
+        self.ensure_data_header(&mut file)?;
         let offset = file.seek(SeekFrom::End(0))?;
         file.write_all(&encoded)?;
         file.sync_all()?;
@@ -137,6 +152,7 @@ impl CollectionState {
             .append(true)
             .open(&data_path)?;
 
+        self.ensure_data_header(&mut file)?;
         file.write_all(&encoded)?;
         file.sync_all()?;
 
@@ -160,6 +176,11 @@ impl CollectionState {
                 &self.data_file_size,
             ) {
                 tracing::warn!("compaction failed for {:?}: {e}", self.coll_dir);
+            } else {
+                // Persist hint file after successful compaction to prevent stale hints
+                if let Err(e) = self.keydir.persist_to_hint_file(&self.hint_path()) {
+                    tracing::warn!("failed to persist hint after compaction for {:?}: {e}", self.coll_dir);
+                }
             }
         }
     }
@@ -234,33 +255,42 @@ impl FileStorageAdapter {
         let hint_path = coll_dir.join("keydir.hint");
 
         // Try loading from hint file first, fall back to data file scan
-        let (keydir, dead_bytes) = if hint_path.exists() && data_path.exists() {
+        let (keydir, dead_bytes, loaded_from_hint) = if hint_path.exists() && data_path.exists() {
             match KeyDir::load_from_hint_file(&hint_path) {
                 Ok(Some(kd)) => {
-                    debug!("loaded KeyDir from hint file: {:?}", hint_path);
-                    // We don't know dead_bytes from the hint file; estimate from file size
-                    let file_size = std::fs::metadata(&data_path)
-                        .map(|m| m.len())
-                        .unwrap_or(FILE_HEADER_SIZE as u64);
-                    let live_bytes: u64 = {
-                        let mut total = 0u64;
-                        kd.for_each(|_, e| total += e.record_len as u64);
-                        total
-                    };
-                    let dead = file_size.saturating_sub(FILE_HEADER_SIZE as u64).saturating_sub(live_bytes);
-                    (kd, dead)
+                    // Validate hint against actual data file
+                    let hint_valid = kd.validate_against_data_file(&data_path, 16)
+                        .unwrap_or(false);
+                    if hint_valid {
+                        debug!("loaded KeyDir from hint file: {:?}", hint_path);
+                        let file_size = std::fs::metadata(&data_path)
+                            .map(|m| m.len())
+                            .unwrap_or(FILE_HEADER_SIZE as u64);
+                        let live_bytes: u64 = {
+                            let mut total = 0u64;
+                            kd.for_each(|_, e| total += e.record_len as u64);
+                            total
+                        };
+                        let dead = file_size.saturating_sub(FILE_HEADER_SIZE as u64).saturating_sub(live_bytes);
+                        (kd, dead, true)
+                    } else {
+                        tracing::warn!("hint file {:?} is stale, rebuilding from data file", hint_path);
+                        let (kd, dead, _stats) = KeyDir::build_from_data_file(&data_path)?;
+                        (kd, dead, false)
+                    }
                 }
                 _ => {
                     debug!("hint file invalid, rebuilding KeyDir from data file");
-                    KeyDir::build_from_data_file(&data_path)?
+                    let (kd, dead, _stats) = KeyDir::build_from_data_file(&data_path)?;
+                    (kd, dead, false)
                 }
             }
         } else if data_path.exists() {
-            KeyDir::build_from_data_file(&data_path)?
+            let (kd, dead, _stats) = KeyDir::build_from_data_file(&data_path)?;
+            (kd, dead, false)
         } else {
-            (KeyDir::new(), 0)
+            (KeyDir::new(), 0, false)
         };
-
         let doc_count = keydir.len();
         let data_file_size = if data_path.exists() {
             std::fs::metadata(&data_path)?.len()
@@ -268,6 +298,15 @@ impl FileStorageAdapter {
             FILE_HEADER_SIZE as u64
         };
 
+        info!(
+            collection = %coll_dir.display(),
+            documents = doc_count,
+            data_bytes = data_file_size,
+            dead_bytes = dead_bytes,
+            source = if loaded_from_hint { "hint" } else { "scan" },
+            "loaded collection"
+        );
+
         // Initialize WAL and recover
         let wal = BinaryWal::new(wal_path);
         wal.initialize()?;
@@ -275,10 +314,10 @@ impl FileStorageAdapter {
         // Recover uncommitted WAL entries
         let uncommitted = wal.recover()?;
         if !uncommitted.is_empty() {
-            debug!(
-                "recovering {} uncommitted WAL entries for {:?}",
-                uncommitted.len(),
-                coll_dir
+            info!(
+                collection = %coll_dir.display(),
+                entries = uncommitted.len(),
+                "recovering uncommitted WAL entries"
             );
         }
 
@@ -415,15 +454,18 @@ impl FileStorageAdapter {
 impl StorageAdapter for FileStorageAdapter {
     async fn initialize(&self) -> StorageResult<()> {
         std::fs::create_dir_all(&self.base_path)?;
-        debug!("FileStorageAdapter initialized at {:?}", self.base_path);
 
         // Pre-load all existing collections
+        let mut db_count: usize = 0;
         if let Ok(entries) = std::fs::read_dir(&self.base_path) {
             for entry in entries.flatten() {
                 if entry.file_type().map(|ft| ft.is_dir()).unwrap_or(false) {
                     if let Some(db_name) = entry.file_name().to_str() {
                         let db_name = db_name.to_string();
                         if let Ok(colls) = self.list_collection_dirs(&db_name) {
+                            if !colls.is_empty() {
+                                db_count += 1;
+                            }
                             for coll_name in colls {
                                 let _ = self.get_or_init_collection(&db_name, &coll_name);
                             }
@@ -433,6 +475,13 @@ impl StorageAdapter for FileStorageAdapter {
             }
         }
 
+        info!(
+            databases = db_count,
+            collections = self.collections.len(),
+            path = %self.base_path.display(),
+            "FileStorageAdapter initialization complete"
+        );
+
         // Start periodic compaction task (runs every 24 hours)
         {
             let collections = self.collections.clone();

rust/crates/rustdb-storage/src/keydir.rs

@@ -6,7 +6,7 @@
 //! The KeyDir can be rebuilt from a data file scan, or loaded quickly from a
 //! persisted hint file for fast restart.
 
-use std::io::{self, BufReader, BufWriter, Read, Write};
+use std::io::{self, BufReader, BufWriter, Read, Seek, SeekFrom, Write};
 use std::path::Path;
 use std::sync::atomic::{AtomicU64, Ordering};
 
@@ -14,7 +14,7 @@ use dashmap::DashMap;
 
 use crate::error::{StorageError, StorageResult};
 use crate::record::{
-    FileHeader, FileType, RecordScanner, FILE_HEADER_SIZE, FORMAT_VERSION,
+    DataRecord, FileHeader, FileType, RecordScanner, FILE_HEADER_SIZE, FORMAT_VERSION,
 };
 
 // ---------------------------------------------------------------------------
@@ -34,6 +34,23 @@ pub struct KeyDirEntry {
     pub timestamp: u64,
 }
 
+// ---------------------------------------------------------------------------
+// BuildStats — statistics from building KeyDir from a data file scan
+// ---------------------------------------------------------------------------
+
+/// Statistics collected while building a KeyDir from a data file scan.
+#[derive(Debug, Clone, Default)]
+pub struct BuildStats {
+    /// Total records scanned (live + tombstones + superseded).
+    pub total_records_scanned: u64,
+    /// Number of live documents in the final KeyDir.
+    pub live_documents: u64,
+    /// Number of tombstone records encountered.
+    pub tombstones: u64,
+    /// Number of records superseded by a later write for the same key.
+    pub superseded_records: u64,
+}
+
 // ---------------------------------------------------------------------------
 // KeyDir
 // ---------------------------------------------------------------------------
@@ -116,9 +133,9 @@ impl KeyDir {
 
     /// Rebuild the KeyDir by scanning an entire data file.
     /// The file must start with a valid `FileHeader`.
-    /// Returns `(keydir, dead_bytes)` where `dead_bytes` is the total size of
+    /// Returns `(keydir, dead_bytes, stats)` where `dead_bytes` is the total size of
     /// stale records (superseded by later writes or tombstoned).
-    pub fn build_from_data_file(path: &Path) -> StorageResult<(Self, u64)> {
+    pub fn build_from_data_file(path: &Path) -> StorageResult<(Self, u64, BuildStats)> {
         let file = std::fs::File::open(path)?;
         let mut reader = BufReader::new(file);
 
@@ -135,6 +152,7 @@ impl KeyDir {
 
         let keydir = KeyDir::new();
         let mut dead_bytes: u64 = 0;
+        let mut stats = BuildStats::default();
 
         let scanner = RecordScanner::new(reader, FILE_HEADER_SIZE as u64);
         for result in scanner {
@@ -146,7 +164,10 @@ impl KeyDir {
             let key = String::from_utf8(record.key)
                 .map_err(|e| StorageError::CorruptRecord(format!("invalid UTF-8 key: {e}")))?;
 
+            stats.total_records_scanned += 1;
+
             if is_tombstone {
+                stats.tombstones += 1;
                 // Remove from index; the tombstone itself is dead weight
                 if let Some(prev) = keydir.remove(&key) {
                     dead_bytes += prev.record_len as u64;
@@ -162,11 +183,13 @@ impl KeyDir {
                 if let Some(prev) = keydir.insert(key, entry) {
                     // Previous version of same key is now dead
                     dead_bytes += prev.record_len as u64;
+                    stats.superseded_records += 1;
                 }
             }
         }
 
-        Ok((keydir, dead_bytes))
+        stats.live_documents = keydir.len();
+        Ok((keydir, dead_bytes, stats))
     }
 
     // -----------------------------------------------------------------------
@@ -271,6 +294,86 @@ impl KeyDir {
 
         Ok(Some(keydir))
     }
+
+    // -----------------------------------------------------------------------
+    // Hint file validation
+    // -----------------------------------------------------------------------
+
+    /// Validate this KeyDir (loaded from a hint file) against the actual data file.
+    /// Returns `Ok(true)` if the hint appears consistent, `Ok(false)` if a rebuild
+    /// from the data file is recommended.
+    ///
+    /// Checks:
+    /// 1. All entry offsets + record_len fit within the data file size.
+    /// 2. All entry offsets are >= FILE_HEADER_SIZE.
+    /// 3. A random sample of entries is spot-checked by reading the record at
+    ///    the offset and verifying the key matches.
+    pub fn validate_against_data_file(&self, data_path: &Path, sample_size: usize) -> StorageResult<bool> {
+        let file_size = std::fs::metadata(data_path)
+            .map(|m| m.len())
+            .unwrap_or(0);
+
+        if file_size < FILE_HEADER_SIZE as u64 {
+            // Data file is too small to even contain a header
+            return Ok(self.is_empty());
+        }
+
+        // Pass 1: bounds check all entries
+        let mut all_keys: Vec<(String, KeyDirEntry)> = Vec::with_capacity(self.len() as usize);
+        let mut bounds_ok = true;
+        self.for_each(|key, entry| {
+            if entry.offset < FILE_HEADER_SIZE as u64
+                || entry.offset + entry.record_len as u64 > file_size
+            {
+                bounds_ok = false;
+            }
+            all_keys.push((key.to_string(), *entry));
+        });
+
+        if !bounds_ok {
+            return Ok(false);
+        }
+
+        // Pass 2: spot-check a sample of entries by reading records from data.rdb
+        if all_keys.is_empty() {
+            return Ok(true);
+        }
+
+        // Sort by offset for sequential I/O, take first `sample_size` entries
+        all_keys.sort_by_key(|(_, e)| e.offset);
+        let step = if all_keys.len() <= sample_size {
+            1
+        } else {
+            all_keys.len() / sample_size
+        };
+
+        let mut file = std::fs::File::open(data_path)?;
+        let mut checked = 0usize;
+        for (i, (expected_key, entry)) in all_keys.iter().enumerate() {
+            if checked >= sample_size {
+                break;
+            }
+            if i % step != 0 {
+                continue;
+            }
+            // Seek to the entry's offset and try to decode the record
+            file.seek(SeekFrom::Start(entry.offset))?;
+            match DataRecord::decode_from(&mut file) {
+                Ok(Some((record, _disk_size))) => {
+                    let record_key = String::from_utf8_lossy(&record.key);
+                    if record_key != *expected_key {
+                        return Ok(false);
+                    }
+                }
+                Ok(None) | Err(_) => {
+                    return Ok(false);
+                }
+            }
+            checked += 1;
+        }
+
+        Ok(true)
+    }
 }
 
 impl Default for KeyDir {
@@ -372,7 +475,7 @@ mod tests {
             f.write_all(&r3.encode()).unwrap();
         }
 
-        let (kd, dead_bytes) = KeyDir::build_from_data_file(&data_path).unwrap();
+        let (kd, dead_bytes, stats) = KeyDir::build_from_data_file(&data_path).unwrap();
 
         // Only B should be live
         assert_eq!(kd.len(), 1);
@@ -381,6 +484,12 @@ mod tests {
 
         // Dead bytes: r1 (aaa live, then superseded by tombstone) + r3 (tombstone itself)
         assert!(dead_bytes > 0);
+
+        // Stats
+        assert_eq!(stats.total_records_scanned, 3);
+        assert_eq!(stats.live_documents, 1);
+        assert_eq!(stats.tombstones, 1);
+        assert_eq!(stats.superseded_records, 0); // aaa was removed by tombstone, not superseded
     }
 
     #[test]

rust/crates/rustdb-storage/src/lib.rs

@@ -16,13 +16,14 @@ pub mod keydir;
 pub mod memory;
 pub mod oplog;
 pub mod record;
+pub mod validate;
 
 pub use adapter::StorageAdapter;
 pub use binary_wal::{BinaryWal, WalEntry, WalOpType};
 pub use compaction::{compact_data_file, should_compact, CompactionResult};
 pub use error::{StorageError, StorageResult};
 pub use file::FileStorageAdapter;
-pub use keydir::{KeyDir, KeyDirEntry};
+pub use keydir::{BuildStats, KeyDir, KeyDirEntry};
 pub use memory::MemoryStorageAdapter;
 pub use oplog::{OpLog, OpLogEntry, OpLogStats, OpType};
 pub use record::{

rust/crates/rustdb-storage/src/validate.rs

@@ -0,0 +1,324 @@
+//! Data integrity validation for RustDb storage directories.
+//!
+//! Provides offline validation of data files without starting the server.
+//! Checks header magic, record CRC32 checksums, duplicate IDs, and
+//! keydir.hint consistency.
+
+use std::collections::HashMap;
+use std::io::{BufReader, Read};
+use std::path::Path;
+
+use crate::error::{StorageError, StorageResult};
+use crate::keydir::KeyDir;
+use crate::record::{FileHeader, FileType, RecordScanner, FILE_HEADER_SIZE};
+
+/// Result of validating an entire data directory.
+pub struct ValidationReport {
+    pub collections: Vec<CollectionReport>,
+}
+
+/// Result of validating a single collection.
+pub struct CollectionReport {
+    pub db: String,
+    pub collection: String,
+    pub header_valid: bool,
+    pub total_records: u64,
+    pub live_documents: u64,
+    pub tombstones: u64,
+    pub duplicate_ids: Vec<String>,
+    pub checksum_errors: u64,
+    pub decode_errors: u64,
+    pub data_file_size: u64,
+    pub hint_file_exists: bool,
+    pub orphaned_hint_entries: u64,
+    pub errors: Vec<String>,
+}
+
+impl ValidationReport {
+    /// Whether any errors were found across all collections.
+    pub fn has_errors(&self) -> bool {
+        self.collections.iter().any(|c| {
+            !c.header_valid
+                || !c.duplicate_ids.is_empty()
+                || c.checksum_errors > 0
+                || c.decode_errors > 0
+                || c.orphaned_hint_entries > 0
+                || !c.errors.is_empty()
+        })
+    }
+
+    /// Print a human-readable summary to stdout.
+    pub fn print_summary(&self) {
+        println!("=== SmartDB Data Integrity Report ===");
+        println!();
+
+        let mut total_errors = 0u64;
+
+        for report in &self.collections {
+            println!("Database: {}", report.db);
+            println!("  Collection: {}", report.collection);
+            println!(
+                "    Header:       {}",
+                if report.header_valid { "OK" } else { "INVALID" }
+            );
+            println!(
+                "    Records:      {} ({} live, {} tombstones)",
+                report.total_records, report.live_documents, report.tombstones
+            );
+            println!("    Data size:    {} bytes", report.data_file_size);
+
+            if report.duplicate_ids.is_empty() {
+                println!("    Duplicates:   0");
+            } else {
+                let ids_preview: Vec<&str> = report.duplicate_ids.iter().take(5).map(|s| s.as_str()).collect();
+                let suffix = if report.duplicate_ids.len() > 5 {
+                    format!(", ... and {} more", report.duplicate_ids.len() - 5)
+                } else {
+                    String::new()
+                };
+                println!(
+                    "    Duplicates:   {} (ids: {}{})",
+                    report.duplicate_ids.len(),
+                    ids_preview.join(", "),
+                    suffix
+                );
+            }
+
+            if report.checksum_errors > 0 {
+                println!("    CRC errors:   {}", report.checksum_errors);
+            } else {
+                println!("    CRC errors:   0");
+            }
+
+            if report.decode_errors > 0 {
+                println!("    Decode errors: {}", report.decode_errors);
+            }
+
+            if report.hint_file_exists {
+                if report.orphaned_hint_entries > 0 {
+                    println!(
+                        "    Hint file:    STALE ({} orphaned entries)",
+                        report.orphaned_hint_entries
+                    );
+                } else {
+                    println!("    Hint file:    OK");
+                }
+            } else {
+                println!("    Hint file:    absent");
+            }
+
+            for err in &report.errors {
+                println!("    ERROR: {}", err);
+            }
+
+            println!();
+
+            if !report.header_valid { total_errors += 1; }
+            total_errors += report.duplicate_ids.len() as u64;
+            total_errors += report.checksum_errors;
+            total_errors += report.decode_errors;
+            total_errors += report.orphaned_hint_entries;
+            total_errors += report.errors.len() as u64;
+        }
+
+        println!(
+            "Summary: {} collection(s) checked, {} error(s) found.",
+            self.collections.len(),
+            total_errors
+        );
+    }
+}
+
+/// Validate all collections in a data directory.
+///
+/// The directory structure is expected to be:
+/// ```text
+/// {base_path}/{db}/{collection}/data.rdb
+/// ```
+pub fn validate_data_directory(base_path: &str) -> StorageResult<ValidationReport> {
+    let base = Path::new(base_path);
+    if !base.exists() {
+        return Err(StorageError::IoError(std::io::Error::new(
+            std::io::ErrorKind::NotFound,
+            format!("data directory not found: {base_path}"),
+        )));
+    }
+
+    let mut collections = Vec::new();
+
+    // Iterate database directories
+    let entries = std::fs::read_dir(base)?;
+    for entry in entries {
+        let entry = entry?;
+        if !entry.file_type()?.is_dir() {
+            continue;
+        }
+        let db_name = match entry.file_name().to_str() {
+            Some(s) => s.to_string(),
+            None => continue,
+        };
+
+        // Iterate collection directories
+        let db_entries = std::fs::read_dir(entry.path())?;
+        for coll_entry in db_entries {
+            let coll_entry = coll_entry?;
+            if !coll_entry.file_type()?.is_dir() {
+                continue;
+            }
+            let coll_name = match coll_entry.file_name().to_str() {
+                Some(s) => s.to_string(),
+                None => continue,
+            };
+
+            let data_path = coll_entry.path().join("data.rdb");
+            if !data_path.exists() {
+                continue;
+            }
+
+            let report = validate_collection(&db_name, &coll_name, &coll_entry.path());
+            collections.push(report);
+        }
+    }
+
+    // Sort for deterministic output
+    collections.sort_by(|a, b| (&a.db, &a.collection).cmp(&(&b.db, &b.collection)));
+
+    Ok(ValidationReport { collections })
+}
+
+/// Validate a single collection directory.
+fn validate_collection(db: &str, coll: &str, coll_dir: &Path) -> CollectionReport {
+    let data_path = coll_dir.join("data.rdb");
+    let hint_path = coll_dir.join("keydir.hint");
+
+    let mut report = CollectionReport {
+        db: db.to_string(),
+        collection: coll.to_string(),
+        header_valid: false,
+        total_records: 0,
+        live_documents: 0,
+        tombstones: 0,
+        duplicate_ids: Vec::new(),
+        checksum_errors: 0,
+        decode_errors: 0,
+        data_file_size: 0,
+        hint_file_exists: hint_path.exists(),
+        orphaned_hint_entries: 0,
+        errors: Vec::new(),
+    };
+
+    // Get file size
+    match std::fs::metadata(&data_path) {
+        Ok(m) => report.data_file_size = m.len(),
+        Err(e) => {
+            report.errors.push(format!("cannot stat data.rdb: {e}"));
+            return report;
+        }
+    }
+
+    // Open and validate header
+    let file = match std::fs::File::open(&data_path) {
+        Ok(f) => f,
+        Err(e) => {
+            report.errors.push(format!("cannot open data.rdb: {e}"));
+            return report;
+        }
+    };
+    let mut reader = BufReader::new(file);
+
+    let mut hdr_buf = [0u8; FILE_HEADER_SIZE];
+    if let Err(e) = reader.read_exact(&mut hdr_buf) {
+        report.errors.push(format!("cannot read header: {e}"));
+        return report;
+    }
+
+    match FileHeader::decode(&hdr_buf) {
+        Ok(hdr) => {
+            if hdr.file_type != FileType::Data {
+                report.errors.push(format!(
+                    "wrong file type: expected Data, got {:?}",
+                    hdr.file_type
+                ));
+            } else {
+                report.header_valid = true;
+            }
+        }
+        Err(e) => {
+            report.errors.push(format!("invalid header: {e}"));
+            return report;
+        }
+    }
+
+    // Scan all records
+    let mut id_counts: HashMap<String, u64> = HashMap::new();
+    let mut live_ids: std::collections::HashSet<String> = std::collections::HashSet::new();
+    let scanner = RecordScanner::new(reader, FILE_HEADER_SIZE as u64);
+
+    for result in scanner {
+        match result {
+            Ok((_offset, record)) => {
+                report.total_records += 1;
+                let key = String::from_utf8_lossy(&record.key).to_string();
+
+                if record.is_tombstone() {
+                    report.tombstones += 1;
+                    live_ids.remove(&key);
+                } else {
+                    *id_counts.entry(key.clone()).or_insert(0) += 1;
+                    live_ids.insert(key);
+                }
+            }
+            Err(e) => {
+                let err_str = e.to_string();
+                if err_str.contains("checksum") || err_str.contains("Checksum") {
+                    report.checksum_errors += 1;
+                } else {
+                    report.decode_errors += 1;
+                }
+                // Cannot continue scanning after a decode error — the stream position is lost
+                report.errors.push(format!("record decode error: {e}"));
+                break;
+            }
+        }
+    }
+
+    report.live_documents = live_ids.len() as u64;
+
+    // Find duplicates (keys that appeared more than once as live inserts)
+    for (id, count) in &id_counts {
+        if *count > 1 {
+            report.duplicate_ids.push(id.clone());
+        }
+    }
+    report.duplicate_ids.sort();
+
+    // Validate hint file if present
+    if hint_path.exists() {
+        match KeyDir::load_from_hint_file(&hint_path) {
+            Ok(Some(hint_kd)) => {
+                // Check for orphaned entries: keys in hint but not live in data
+                hint_kd.for_each(|key, _entry| {
+                    if !live_ids.contains(key) {
+                        report.orphaned_hint_entries += 1;
+                    }
+                });
+
+                // Also check if hint references offsets beyond file size
+                hint_kd.for_each(|_key, entry| {
+                    if entry.offset + entry.record_len as u64 > report.data_file_size {
+                        report.orphaned_hint_entries += 1;
+                    }
+                });
+            }
+            Ok(None) => {
+                // File existed but was empty or unreadable
+                report.errors.push("hint file exists but is empty".into());
+            }
+            Err(e) => {
+                report.errors.push(format!("hint file decode error: {e}"));
+            }
+        }
+    }
+
+    report
+}