push.rocks/smartdb
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(enterprise): add auth TLS and recovery hardening

Browse Source
This commit is contained in:
Jürgen Kunz
2026-04-29 22:01:43 +00:00
parent 2f3031cfc7
commit ed2c02bcf9
27 changed files with 2369 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rust/crates/rustdb-storage/src/file.rs
+29 -4
View File
@@ -187,6 +187,27 @@ impl CollectionState {
}
}
fn truncate_invalid_tail(
data_path: &PathBuf,
stats: &crate::keydir::BuildStats,
) -> StorageResult<()> {
if stats.invalid_tail_bytes == 0 {
return Ok(());
}
tracing::warn!(
path = %data_path.display(),
valid_data_end = stats.valid_data_end,
invalid_tail_bytes = stats.invalid_tail_bytes,
"truncating invalid data file tail"
);
let file = std::fs::OpenOptions::new().write(true).open(data_path)?;
file.set_len(stats.valid_data_end)?;
file.sync_all()?;
Ok(())
}
// ---------------------------------------------------------------------------
// Collection cache key: "db\0coll"
// ---------------------------------------------------------------------------
@@ -279,7 +300,8 @@ impl FileStorageAdapter {
hint_path, stored_size, actual_size
);
}
let (kd, dead, _stats) = KeyDir::build_from_data_file(&data_path)?;
let (kd, dead, stats) = KeyDir::build_from_data_file(&data_path)?;
truncate_invalid_tail(&data_path, &stats)?;
(kd, dead, false)
} else {
// Size matches — validate entry integrity with spot-checks
@@ -296,19 +318,22 @@ impl FileStorageAdapter {
(kd, dead, true)
} else {
tracing::warn!("hint file {:?} failed validation, rebuilding from data file", hint_path);
let (kd, dead, _stats) = KeyDir::build_from_data_file(&data_path)?;
let (kd, dead, stats) = KeyDir::build_from_data_file(&data_path)?;
truncate_invalid_tail(&data_path, &stats)?;
(kd, dead, false)
}
}
}
_ => {
debug!("hint file invalid, rebuilding KeyDir from data file");
let (kd, dead, _stats) = KeyDir::build_from_data_file(&data_path)?;
let (kd, dead, stats) = KeyDir::build_from_data_file(&data_path)?;
truncate_invalid_tail(&data_path, &stats)?;
(kd, dead, false)
}
}
} else if data_path.exists() {
let (kd, dead, _stats) = KeyDir::build_from_data_file(&data_path)?;
let (kd, dead, stats) = KeyDir::build_from_data_file(&data_path)?;
truncate_invalid_tail(&data_path, &stats)?;
(kd, dead, false)
} else {
(KeyDir::new(), 0, false)
rust/crates/rustdb-storage/src/keydir.rs
+48 -7
View File
@@ -14,7 +14,7 @@ use dashmap::DashMap;
use crate::error::{StorageError, StorageResult};
use crate::record::{
DataRecord, FileHeader, FileType, RecordScanner, FILE_HEADER_SIZE, FORMAT_VERSION,
DataRecord, FileHeader, FileType, FILE_HEADER_SIZE, FORMAT_VERSION,
};
// ---------------------------------------------------------------------------
@@ -49,6 +49,10 @@ pub struct BuildStats {
pub tombstones: u64,
/// Number of records superseded by a later write for the same key.
pub superseded_records: u64,
/// Byte offset immediately after the last valid record.
pub valid_data_end: u64,
/// Number of invalid tail bytes after the last valid record.
pub invalid_tail_bytes: u64,
}
// ---------------------------------------------------------------------------
@@ -137,6 +141,7 @@ impl KeyDir {
/// stale records (superseded by later writes or tombstoned).
pub fn build_from_data_file(path: &Path) -> StorageResult<(Self, u64, BuildStats)> {
let file = std::fs::File::open(path)?;
let file_len = file.metadata()?.len();
let mut reader = BufReader::new(file);
// Read and validate file header
@@ -152,13 +157,49 @@ impl KeyDir {
let keydir = KeyDir::new();
let mut dead_bytes: u64 = 0;
let mut stats = BuildStats::default();
let mut stats = BuildStats {
valid_data_end: FILE_HEADER_SIZE as u64,
..BuildStats::default()
};
let scanner = RecordScanner::new(reader, FILE_HEADER_SIZE as u64);
for result in scanner {
let (offset, record) = result?;
loop {
let record_offset = stats.valid_data_end;
let (record, disk_size) = match DataRecord::decode_from(&mut reader) {
Ok(Some((record, disk_size))) => (record, disk_size),
Ok(None) => {
if file_len > record_offset {
stats.invalid_tail_bytes = file_len - record_offset;
}
break;
}
Err(StorageError::IoError(e)) if e.kind() == io::ErrorKind::UnexpectedEof => {
stats.invalid_tail_bytes = file_len.saturating_sub(record_offset);
break;
}
Err(StorageError::ChecksumMismatch { expected, actual }) => {
tracing::warn!(
path = %path.display(),
offset = record_offset,
"stopping data file scan at checksum mismatch: expected 0x{expected:08X}, got 0x{actual:08X}"
);
stats.invalid_tail_bytes = file_len.saturating_sub(record_offset);
break;
}
Err(StorageError::CorruptRecord(message)) => {
tracing::warn!(
path = %path.display(),
offset = record_offset,
"stopping data file scan at corrupt record: {message}"
);
stats.invalid_tail_bytes = file_len.saturating_sub(record_offset);
break;
}
Err(e) => return Err(e),
};
stats.valid_data_end += disk_size as u64;
let is_tombstone = record.is_tombstone();
let disk_size = record.disk_size() as u32;
let disk_size = disk_size as u32;
let value_len = record.value.len() as u32;
let timestamp = record.timestamp;
let key = String::from_utf8(record.key)
@@ -175,7 +216,7 @@ impl KeyDir {
dead_bytes += disk_size as u64;
} else {
let entry = KeyDirEntry {
offset,
offset: record_offset,
record_len: disk_size,
value_len,
timestamp,