feat: add multi-item ingest and Reed-Solomon parity

- Multi-item ingest: each item gets its own Unix socket, Rust processes
  them sequentially into a single snapshot with separate chunk lists
- Reed-Solomon parity: rs(20,1) erasure coding for pack file groups,
  enabling single-pack-loss recovery via parity reconstruction
- Repair now attempts parity-based recovery for missing pack files
- 16 integration tests + 12 Rust unit tests all pass
This commit is contained in:
2026-03-21 23:46:29 +00:00
parent a5849791d2
commit ca510f4578
10 changed files with 830 additions and 115 deletions

View File

@@ -117,6 +117,43 @@ tap.test('should restore data byte-for-byte', async () => {
expect(restored.equals(expected)).toBeTrue();
});
// ==================== Multi-Item Ingest ====================
tap.test('should ingest multiple items in one snapshot', async () => {
const data1 = Buffer.alloc(64 * 1024, 'item-one-data');
const data2 = Buffer.alloc(32 * 1024, 'item-two-data');
const snapshot = await repo.ingestMulti([
{ stream: stream.Readable.from(data1), name: 'database.sql', type: 'database-dump' },
{ stream: stream.Readable.from(data2), name: 'config.tar', type: 'volume-tar' },
], { tags: { type: 'multi-test' } });
expect(snapshot).toBeTruthy();
expect(snapshot.items.length).toEqual(2);
expect(snapshot.items[0].name).toEqual('database.sql');
expect(snapshot.items[1].name).toEqual('config.tar');
expect(snapshot.items[0].size).toEqual(64 * 1024);
expect(snapshot.items[1].size).toEqual(32 * 1024);
});
tap.test('should restore specific item from multi-item snapshot', async () => {
const snapshots = await repo.listSnapshots({ tags: { type: 'multi-test' } });
expect(snapshots.length).toEqual(1);
const restoreStream = await repo.restore(snapshots[0].id, { item: 'config.tar' });
const chunks: Buffer[] = [];
await new Promise<void>((resolve, reject) => {
restoreStream.on('data', (chunk: Buffer) => chunks.push(chunk));
restoreStream.on('end', resolve);
restoreStream.on('error', reject);
});
const restored = Buffer.concat(chunks);
const expected = Buffer.alloc(32 * 1024, 'item-two-data');
expect(restored.length).toEqual(expected.length);
expect(restored.equals(expected)).toBeTrue();
});
// ==================== Verify ====================
tap.test('should verify repository at quick level', async () => {
@@ -139,8 +176,9 @@ tap.test('should verify repository at full level', async () => {
// ==================== Prune ====================
tap.test('should prune with keepLast=1', async () => {
const snapshotsBefore = await repo.listSnapshots();
const result = await repo.prune({ keepLast: 1 });
expect(result.removedSnapshots).toEqual(1);
expect(result.removedSnapshots).toEqual(snapshotsBefore.length - 1);
expect(result.dryRun).toBeFalse();
// Verify only 1 snapshot remains