feat: add multi-item ingest and Reed-Solomon parity

- Multi-item ingest: each item gets its own Unix socket, Rust processes
  them sequentially into a single snapshot with separate chunk lists
- Reed-Solomon parity: rs(20,1) erasure coding for pack file groups,
  enabling single-pack-loss recovery via parity reconstruction
- Repair now attempts parity-based recovery for missing pack files
- 16 integration tests + 12 Rust unit tests all pass
This commit is contained in:
2026-03-21 23:46:29 +00:00
parent a5849791d2
commit ca510f4578
10 changed files with 830 additions and 115 deletions

View File

@@ -150,24 +150,70 @@ export class ContainerArchive {
/**
* Ingest multiple data streams as a single multi-item snapshot.
* Each item gets its own Unix socket for parallel data transfer.
*/
async ingestMulti(
items: IIngestItem[],
options?: IIngestOptions,
): Promise<ISnapshot> {
// For multi-item, we concatenate all streams into one socket
// and pass item metadata so Rust can split them.
// For now, we implement a simple sequential approach:
// ingest first item only (multi-item will be enhanced later).
if (items.length === 0) {
throw new Error('At least one item is required');
}
const firstItem = items[0];
return this.ingest(firstItem.stream, {
...options,
items: items.map((i) => ({ name: i.name, type: i.type || 'data' })),
});
// Create one socket per item
const sockets: Array<{
socketPath: string;
promise: Promise<void>;
server: plugins.net.Server;
}> = [];
const itemOptions: Array<{
name: string;
type: string;
socketPath: string;
}> = [];
try {
for (const item of items) {
const socketPath = plugins.path.join(
plugins.os.tmpdir(),
`containerarchive-ingest-${Date.now()}-${Math.random().toString(36).slice(2)}.sock`,
);
const { promise, server } = await this.createSocketServer(socketPath, item.stream);
sockets.push({ socketPath, promise, server });
itemOptions.push({
name: item.name,
type: item.type || 'data',
socketPath,
});
}
// Send ingestMulti command to Rust with per-item socket paths
const result = await this.bridge.sendCommand('ingestMulti', {
tags: options?.tags,
items: itemOptions,
});
// Wait for all data transfers
await Promise.all(sockets.map((s) => s.promise));
const snapshot = result.snapshot;
this.ingestComplete.next({
snapshotId: snapshot.id,
originalSize: snapshot.originalSize,
storedSize: snapshot.storedSize,
newChunks: snapshot.newChunks,
reusedChunks: snapshot.reusedChunks,
});
return snapshot;
} finally {
for (const s of sockets) {
s.server.close();
try { plugins.fs.unlinkSync(s.socketPath); } catch {}
}
}
}
/**

View File

@@ -134,6 +134,7 @@ export interface IRepairResult {
indexRebuilt: boolean;
indexedChunks: number;
staleLocksRemoved: number;
packsRepaired: number;
errors: string[];
}
@@ -180,6 +181,13 @@ export type TContainerArchiveCommands = {
},
{ snapshot: ISnapshot }
>;
ingestMulti: ICommandDefinition<
{
tags?: Record<string, string>;
items: Array<{ name: string; type: string; socketPath: string }>;
},
{ snapshot: ISnapshot }
>;
restore: ICommandDefinition<
{
snapshotId: string;