feat: add multi-item ingest and Reed-Solomon parity
- Multi-item ingest: each item gets its own Unix socket, Rust processes them sequentially into a single snapshot with separate chunk lists - Reed-Solomon parity: rs(20,1) erasure coding for pack file groups, enabling single-pack-loss recovery via parity reconstruction - Repair now attempts parity-based recovery for missing pack files - 16 integration tests + 12 Rust unit tests all pass
This commit is contained in:
@@ -150,24 +150,70 @@ export class ContainerArchive {
|
||||
|
||||
/**
|
||||
* Ingest multiple data streams as a single multi-item snapshot.
|
||||
* Each item gets its own Unix socket for parallel data transfer.
|
||||
*/
|
||||
async ingestMulti(
|
||||
items: IIngestItem[],
|
||||
options?: IIngestOptions,
|
||||
): Promise<ISnapshot> {
|
||||
// For multi-item, we concatenate all streams into one socket
|
||||
// and pass item metadata so Rust can split them.
|
||||
// For now, we implement a simple sequential approach:
|
||||
// ingest first item only (multi-item will be enhanced later).
|
||||
if (items.length === 0) {
|
||||
throw new Error('At least one item is required');
|
||||
}
|
||||
|
||||
const firstItem = items[0];
|
||||
return this.ingest(firstItem.stream, {
|
||||
...options,
|
||||
items: items.map((i) => ({ name: i.name, type: i.type || 'data' })),
|
||||
});
|
||||
// Create one socket per item
|
||||
const sockets: Array<{
|
||||
socketPath: string;
|
||||
promise: Promise<void>;
|
||||
server: plugins.net.Server;
|
||||
}> = [];
|
||||
|
||||
const itemOptions: Array<{
|
||||
name: string;
|
||||
type: string;
|
||||
socketPath: string;
|
||||
}> = [];
|
||||
|
||||
try {
|
||||
for (const item of items) {
|
||||
const socketPath = plugins.path.join(
|
||||
plugins.os.tmpdir(),
|
||||
`containerarchive-ingest-${Date.now()}-${Math.random().toString(36).slice(2)}.sock`,
|
||||
);
|
||||
|
||||
const { promise, server } = await this.createSocketServer(socketPath, item.stream);
|
||||
sockets.push({ socketPath, promise, server });
|
||||
itemOptions.push({
|
||||
name: item.name,
|
||||
type: item.type || 'data',
|
||||
socketPath,
|
||||
});
|
||||
}
|
||||
|
||||
// Send ingestMulti command to Rust with per-item socket paths
|
||||
const result = await this.bridge.sendCommand('ingestMulti', {
|
||||
tags: options?.tags,
|
||||
items: itemOptions,
|
||||
});
|
||||
|
||||
// Wait for all data transfers
|
||||
await Promise.all(sockets.map((s) => s.promise));
|
||||
|
||||
const snapshot = result.snapshot;
|
||||
this.ingestComplete.next({
|
||||
snapshotId: snapshot.id,
|
||||
originalSize: snapshot.originalSize,
|
||||
storedSize: snapshot.storedSize,
|
||||
newChunks: snapshot.newChunks,
|
||||
reusedChunks: snapshot.reusedChunks,
|
||||
});
|
||||
|
||||
return snapshot;
|
||||
} finally {
|
||||
for (const s of sockets) {
|
||||
s.server.close();
|
||||
try { plugins.fs.unlinkSync(s.socketPath); } catch {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user