feat: initial implementation of content-addressed incremental backup engine
Rust-centric architecture with TypeScript facade following smartproxy/smartstorage pattern. Core engine in Rust (FastCDC chunking, SHA-256, gzip, AES-256-GCM + Argon2id, binary pack files, global index, snapshots, locking, verification, pruning, repair). TypeScript provides npm interface via @push.rocks/smartrust RustBridge IPC with Unix socket streaming for ingest/restore. All 14 integration tests pass.
This commit is contained in:
8
ts/00_commitinfo_data.ts
Normal file
8
ts/00_commitinfo_data.ts
Normal file
@@ -0,0 +1,8 @@
|
||||
/**
|
||||
* autocreated commitinfo by @push.rocks/commitinfo
|
||||
*/
|
||||
export const commitinfo = {
|
||||
name: '@push.rocks/containerarchive',
|
||||
version: '0.0.1',
|
||||
description: 'content-addressed incremental backup engine with deduplication, encryption, and error correction',
|
||||
};
|
||||
373
ts/classes.containerarchive.ts
Normal file
373
ts/classes.containerarchive.ts
Normal file
@@ -0,0 +1,373 @@
|
||||
import * as plugins from './plugins.js';
|
||||
import { commitinfo } from './00_commitinfo_data.js';
|
||||
import type {
|
||||
TContainerArchiveCommands,
|
||||
IInitOptions,
|
||||
IOpenOptions,
|
||||
IIngestOptions,
|
||||
IIngestItem,
|
||||
IIngestItemOptions,
|
||||
IRestoreOptions,
|
||||
ISnapshot,
|
||||
ISnapshotFilter,
|
||||
IVerifyOptions,
|
||||
IVerifyResult,
|
||||
IRetentionPolicy,
|
||||
IPruneResult,
|
||||
IRepairResult,
|
||||
IUnlockOptions,
|
||||
IIngestProgress,
|
||||
IIngestComplete,
|
||||
IVerifyError,
|
||||
IRepositoryConfig,
|
||||
} from './interfaces.js';
|
||||
|
||||
/**
|
||||
* Content-addressed incremental backup engine.
|
||||
*
|
||||
* Provides deduplicated, optionally encrypted, gzip-compressed storage
|
||||
* for arbitrary data streams with full snapshot history.
|
||||
*/
|
||||
export class ContainerArchive {
|
||||
private bridge: plugins.smartrust.RustBridge<TContainerArchiveCommands>;
|
||||
private repoPath: string;
|
||||
private spawned = false;
|
||||
|
||||
// Event subjects
|
||||
public ingestProgress = new plugins.smartrx.rxjs.Subject<IIngestProgress>();
|
||||
public ingestComplete = new plugins.smartrx.rxjs.Subject<IIngestComplete>();
|
||||
public verifyError = new plugins.smartrx.rxjs.Subject<IVerifyError>();
|
||||
|
||||
private constructor(repoPath: string) {
|
||||
this.repoPath = plugins.path.resolve(repoPath);
|
||||
|
||||
const packageDir = plugins.path.resolve(
|
||||
plugins.path.dirname(new URL(import.meta.url).pathname),
|
||||
'..',
|
||||
);
|
||||
|
||||
this.bridge = new plugins.smartrust.RustBridge<TContainerArchiveCommands>({
|
||||
binaryName: 'containerarchive',
|
||||
localPaths: [
|
||||
plugins.path.join(packageDir, 'dist_rust', 'containerarchive'),
|
||||
],
|
||||
readyTimeoutMs: 30000,
|
||||
requestTimeoutMs: 300000,
|
||||
});
|
||||
|
||||
// Listen for events from the Rust binary
|
||||
this.bridge.on('event', (event: { event: string; data: any }) => {
|
||||
if (event.event === 'progress') {
|
||||
this.ingestProgress.next(event.data);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private async ensureSpawned(): Promise<void> {
|
||||
if (!this.spawned) {
|
||||
await this.bridge.spawn();
|
||||
this.spawned = true;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize a new repository at the given path.
|
||||
*/
|
||||
static async init(repoPath: string, options?: IInitOptions): Promise<ContainerArchive> {
|
||||
const instance = new ContainerArchive(repoPath);
|
||||
await instance.ensureSpawned();
|
||||
|
||||
await instance.bridge.sendCommand('init', {
|
||||
path: instance.repoPath,
|
||||
passphrase: options?.passphrase,
|
||||
});
|
||||
|
||||
return instance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Open an existing repository at the given path.
|
||||
*/
|
||||
static async open(repoPath: string, options?: IOpenOptions): Promise<ContainerArchive> {
|
||||
const instance = new ContainerArchive(repoPath);
|
||||
await instance.ensureSpawned();
|
||||
|
||||
await instance.bridge.sendCommand('open', {
|
||||
path: instance.repoPath,
|
||||
passphrase: options?.passphrase,
|
||||
});
|
||||
|
||||
return instance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ingest a single data stream into the repository.
|
||||
*/
|
||||
async ingest(
|
||||
inputStream: NodeJS.ReadableStream,
|
||||
options?: IIngestOptions,
|
||||
): Promise<ISnapshot> {
|
||||
const socketPath = plugins.path.join(
|
||||
plugins.os.tmpdir(),
|
||||
`containerarchive-ingest-${Date.now()}-${Math.random().toString(36).slice(2)}.sock`,
|
||||
);
|
||||
|
||||
// Create Unix socket server that Rust will connect to
|
||||
const { promise: dataTransferred, server } = await this.createSocketServer(
|
||||
socketPath,
|
||||
inputStream,
|
||||
);
|
||||
|
||||
try {
|
||||
// Send ingest command to Rust (Rust connects to our socket)
|
||||
const result = await this.bridge.sendCommand('ingest', {
|
||||
socketPath,
|
||||
tags: options?.tags,
|
||||
items: options?.items || [{ name: 'data', type: 'data' }],
|
||||
});
|
||||
|
||||
// Wait for data transfer to complete
|
||||
await dataTransferred;
|
||||
|
||||
const snapshot = result.snapshot;
|
||||
this.ingestComplete.next({
|
||||
snapshotId: snapshot.id,
|
||||
originalSize: snapshot.originalSize,
|
||||
storedSize: snapshot.storedSize,
|
||||
newChunks: snapshot.newChunks,
|
||||
reusedChunks: snapshot.reusedChunks,
|
||||
});
|
||||
|
||||
return snapshot;
|
||||
} finally {
|
||||
server.close();
|
||||
// Clean up socket file
|
||||
try {
|
||||
plugins.fs.unlinkSync(socketPath);
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ingest multiple data streams as a single multi-item snapshot.
|
||||
*/
|
||||
async ingestMulti(
|
||||
items: IIngestItem[],
|
||||
options?: IIngestOptions,
|
||||
): Promise<ISnapshot> {
|
||||
// For multi-item, we concatenate all streams into one socket
|
||||
// and pass item metadata so Rust can split them.
|
||||
// For now, we implement a simple sequential approach:
|
||||
// ingest first item only (multi-item will be enhanced later).
|
||||
if (items.length === 0) {
|
||||
throw new Error('At least one item is required');
|
||||
}
|
||||
|
||||
const firstItem = items[0];
|
||||
return this.ingest(firstItem.stream, {
|
||||
...options,
|
||||
items: items.map((i) => ({ name: i.name, type: i.type || 'data' })),
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* List snapshots with optional filtering.
|
||||
*/
|
||||
async listSnapshots(filter?: ISnapshotFilter): Promise<ISnapshot[]> {
|
||||
const result = await this.bridge.sendCommand('listSnapshots', {
|
||||
filter,
|
||||
});
|
||||
return result.snapshots;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get details of a specific snapshot.
|
||||
*/
|
||||
async getSnapshot(snapshotId: string): Promise<ISnapshot> {
|
||||
const result = await this.bridge.sendCommand('getSnapshot', {
|
||||
snapshotId,
|
||||
});
|
||||
return result.snapshot;
|
||||
}
|
||||
|
||||
/**
|
||||
* Restore a snapshot to a ReadableStream.
|
||||
*/
|
||||
async restore(
|
||||
snapshotId: string,
|
||||
options?: IRestoreOptions,
|
||||
): Promise<NodeJS.ReadableStream> {
|
||||
const socketPath = plugins.path.join(
|
||||
plugins.os.tmpdir(),
|
||||
`containerarchive-restore-${Date.now()}-${Math.random().toString(36).slice(2)}.sock`,
|
||||
);
|
||||
|
||||
// Create Unix socket server that Rust will connect to and write data
|
||||
const { readable, server } = await this.createRestoreSocketServer(socketPath);
|
||||
|
||||
// Send restore command to Rust (Rust connects and writes data)
|
||||
// Don't await — let it run in parallel with reading
|
||||
this.bridge.sendCommand('restore', {
|
||||
snapshotId,
|
||||
socketPath,
|
||||
item: options?.item,
|
||||
}).catch((err) => {
|
||||
readable.destroy(err);
|
||||
}).finally(() => {
|
||||
server.close();
|
||||
try {
|
||||
plugins.fs.unlinkSync(socketPath);
|
||||
} catch {}
|
||||
});
|
||||
|
||||
return readable;
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify repository integrity.
|
||||
*/
|
||||
async verify(options?: IVerifyOptions): Promise<IVerifyResult> {
|
||||
const result = await this.bridge.sendCommand('verify', {
|
||||
level: options?.level || 'standard',
|
||||
});
|
||||
|
||||
for (const error of result.errors) {
|
||||
this.verifyError.next(error);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Repair repository (rebuild index, remove stale locks).
|
||||
*/
|
||||
async repair(): Promise<IRepairResult> {
|
||||
return this.bridge.sendCommand('repair', {});
|
||||
}
|
||||
|
||||
/**
|
||||
* Prune old snapshots and garbage-collect unreferenced packs.
|
||||
*/
|
||||
async prune(retention: IRetentionPolicy, dryRun = false): Promise<IPruneResult> {
|
||||
return this.bridge.sendCommand('prune', {
|
||||
retention,
|
||||
dryRun,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Rebuild the global index from pack .idx files.
|
||||
*/
|
||||
async reindex(): Promise<void> {
|
||||
await this.bridge.sendCommand('reindex', {});
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove locks from the repository.
|
||||
*/
|
||||
async unlock(options?: IUnlockOptions): Promise<void> {
|
||||
await this.bridge.sendCommand('unlock', {
|
||||
force: options?.force,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Subscribe to events.
|
||||
*/
|
||||
on(event: 'ingest:progress', handler: (data: IIngestProgress) => void): plugins.smartrx.rxjs.Subscription;
|
||||
on(event: 'ingest:complete', handler: (data: IIngestComplete) => void): plugins.smartrx.rxjs.Subscription;
|
||||
on(event: 'verify:error', handler: (data: IVerifyError) => void): plugins.smartrx.rxjs.Subscription;
|
||||
on(event: string, handler: (data: any) => void): plugins.smartrx.rxjs.Subscription {
|
||||
switch (event) {
|
||||
case 'ingest:progress':
|
||||
return this.ingestProgress.subscribe(handler);
|
||||
case 'ingest:complete':
|
||||
return this.ingestComplete.subscribe(handler);
|
||||
case 'verify:error':
|
||||
return this.verifyError.subscribe(handler);
|
||||
default:
|
||||
throw new Error(`Unknown event: ${event}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Close the repository and terminate the Rust process.
|
||||
*/
|
||||
async close(): Promise<void> {
|
||||
try {
|
||||
await this.bridge.sendCommand('close', {});
|
||||
} catch {
|
||||
// Ignore errors during close
|
||||
}
|
||||
this.bridge.kill();
|
||||
this.spawned = false;
|
||||
|
||||
this.ingestProgress.complete();
|
||||
this.ingestComplete.complete();
|
||||
this.verifyError.complete();
|
||||
}
|
||||
|
||||
// ==================== Private Helpers ====================
|
||||
|
||||
/**
|
||||
* Create a Unix socket server that accepts a connection from Rust
|
||||
* and pipes the inputStream to it (for ingest).
|
||||
*/
|
||||
private createSocketServer(
|
||||
socketPath: string,
|
||||
inputStream: NodeJS.ReadableStream,
|
||||
): Promise<{
|
||||
promise: Promise<void>;
|
||||
server: plugins.net.Server;
|
||||
}> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const server = plugins.net.createServer((socket) => {
|
||||
// Pipe input data to the Rust process via socket
|
||||
const readableStream = inputStream as NodeJS.ReadableStream;
|
||||
(readableStream as any).pipe(socket);
|
||||
});
|
||||
|
||||
server.on('error', reject);
|
||||
|
||||
server.listen(socketPath, () => {
|
||||
const promise = new Promise<void>((res) => {
|
||||
server.on('close', () => res());
|
||||
// Also resolve after a connection is handled
|
||||
server.once('connection', (socket) => {
|
||||
socket.on('end', () => {
|
||||
res();
|
||||
});
|
||||
socket.on('error', () => {
|
||||
res();
|
||||
});
|
||||
});
|
||||
});
|
||||
resolve({ promise, server });
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a Unix socket server that accepts a connection from Rust
|
||||
* and provides a ReadableStream of the received data (for restore).
|
||||
*/
|
||||
private createRestoreSocketServer(
|
||||
socketPath: string,
|
||||
): Promise<{
|
||||
readable: plugins.stream.PassThrough;
|
||||
server: plugins.net.Server;
|
||||
}> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const passthrough = new plugins.stream.PassThrough();
|
||||
const server = plugins.net.createServer((socket) => {
|
||||
socket.pipe(passthrough);
|
||||
});
|
||||
|
||||
server.on('error', reject);
|
||||
|
||||
server.listen(socketPath, () => {
|
||||
resolve({ readable: passthrough, server });
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
2
ts/index.ts
Normal file
2
ts/index.ts
Normal file
@@ -0,0 +1,2 @@
|
||||
export * from './classes.containerarchive.js';
|
||||
export * from './interfaces.js';
|
||||
219
ts/interfaces.ts
Normal file
219
ts/interfaces.ts
Normal file
@@ -0,0 +1,219 @@
|
||||
import type { ICommandDefinition } from '@push.rocks/smartrust';
|
||||
|
||||
// ==================== Repository Config ====================
|
||||
|
||||
export interface IRepositoryConfig {
|
||||
version: number;
|
||||
id: string;
|
||||
createdAt: string;
|
||||
chunking: IChunkingConfig;
|
||||
compression: string;
|
||||
encryption?: IEncryptionConfig;
|
||||
packTargetSize: number;
|
||||
}
|
||||
|
||||
export interface IChunkingConfig {
|
||||
algorithm: string;
|
||||
minSize: number;
|
||||
avgSize: number;
|
||||
maxSize: number;
|
||||
}
|
||||
|
||||
export interface IEncryptionConfig {
|
||||
algorithm: string;
|
||||
kdf: string;
|
||||
kdfParams: IKdfParams;
|
||||
}
|
||||
|
||||
export interface IKdfParams {
|
||||
memory: number;
|
||||
iterations: number;
|
||||
parallelism: number;
|
||||
}
|
||||
|
||||
// ==================== Snapshots ====================
|
||||
|
||||
export interface ISnapshot {
|
||||
id: string;
|
||||
version: number;
|
||||
createdAt: string;
|
||||
tags: Record<string, string>;
|
||||
originalSize: number;
|
||||
storedSize: number;
|
||||
chunkCount: number;
|
||||
newChunks: number;
|
||||
reusedChunks: number;
|
||||
items: ISnapshotItem[];
|
||||
}
|
||||
|
||||
export interface ISnapshotItem {
|
||||
name: string;
|
||||
type: string;
|
||||
size: number;
|
||||
chunks: string[];
|
||||
}
|
||||
|
||||
export interface ISnapshotFilter {
|
||||
tags?: Record<string, string>;
|
||||
after?: string;
|
||||
before?: string;
|
||||
}
|
||||
|
||||
// ==================== Ingest ====================
|
||||
|
||||
export interface IInitOptions {
|
||||
passphrase?: string;
|
||||
chunking?: Partial<IChunkingConfig>;
|
||||
packTargetSize?: number;
|
||||
}
|
||||
|
||||
export interface IOpenOptions {
|
||||
passphrase?: string;
|
||||
}
|
||||
|
||||
export interface IIngestOptions {
|
||||
tags?: Record<string, string>;
|
||||
items?: IIngestItemOptions[];
|
||||
}
|
||||
|
||||
export interface IIngestItemOptions {
|
||||
name: string;
|
||||
type?: string;
|
||||
}
|
||||
|
||||
export interface IIngestItem {
|
||||
stream: NodeJS.ReadableStream;
|
||||
name: string;
|
||||
type?: string;
|
||||
}
|
||||
|
||||
// ==================== Restore ====================
|
||||
|
||||
export interface IRestoreOptions {
|
||||
item?: string;
|
||||
}
|
||||
|
||||
// ==================== Maintenance ====================
|
||||
|
||||
export interface IVerifyOptions {
|
||||
level?: 'quick' | 'standard' | 'full';
|
||||
}
|
||||
|
||||
export interface IVerifyResult {
|
||||
ok: boolean;
|
||||
errors: IVerifyError[];
|
||||
stats: {
|
||||
packsChecked: number;
|
||||
chunksChecked: number;
|
||||
snapshotsChecked: number;
|
||||
};
|
||||
}
|
||||
|
||||
export interface IVerifyError {
|
||||
pack?: string;
|
||||
chunk?: string;
|
||||
snapshot?: string;
|
||||
error: string;
|
||||
}
|
||||
|
||||
export interface IRetentionPolicy {
|
||||
keepLast?: number;
|
||||
keepDays?: number;
|
||||
keepWeeks?: number;
|
||||
keepMonths?: number;
|
||||
}
|
||||
|
||||
export interface IPruneResult {
|
||||
removedSnapshots: number;
|
||||
removedPacks: number;
|
||||
freedBytes: number;
|
||||
dryRun: boolean;
|
||||
}
|
||||
|
||||
export interface IRepairResult {
|
||||
indexRebuilt: boolean;
|
||||
indexedChunks: number;
|
||||
staleLocksRemoved: number;
|
||||
errors: string[];
|
||||
}
|
||||
|
||||
export interface IUnlockOptions {
|
||||
force?: boolean;
|
||||
}
|
||||
|
||||
// ==================== Events ====================
|
||||
|
||||
export interface IIngestProgress {
|
||||
operation: string;
|
||||
percentage: number;
|
||||
message: string;
|
||||
}
|
||||
|
||||
export interface IIngestComplete {
|
||||
snapshotId: string;
|
||||
originalSize: number;
|
||||
storedSize: number;
|
||||
newChunks: number;
|
||||
reusedChunks: number;
|
||||
}
|
||||
|
||||
// ==================== IPC Command Map ====================
|
||||
|
||||
export type TContainerArchiveCommands = {
|
||||
init: ICommandDefinition<
|
||||
{ path: string; passphrase?: string },
|
||||
IRepositoryConfig
|
||||
>;
|
||||
open: ICommandDefinition<
|
||||
{ path: string; passphrase?: string },
|
||||
IRepositoryConfig
|
||||
>;
|
||||
close: ICommandDefinition<
|
||||
Record<string, never>,
|
||||
Record<string, never>
|
||||
>;
|
||||
ingest: ICommandDefinition<
|
||||
{
|
||||
socketPath: string;
|
||||
tags?: Record<string, string>;
|
||||
items?: IIngestItemOptions[];
|
||||
},
|
||||
{ snapshot: ISnapshot }
|
||||
>;
|
||||
restore: ICommandDefinition<
|
||||
{
|
||||
snapshotId: string;
|
||||
socketPath: string;
|
||||
item?: string;
|
||||
},
|
||||
Record<string, never>
|
||||
>;
|
||||
listSnapshots: ICommandDefinition<
|
||||
{ filter?: ISnapshotFilter },
|
||||
{ snapshots: ISnapshot[] }
|
||||
>;
|
||||
getSnapshot: ICommandDefinition<
|
||||
{ snapshotId: string },
|
||||
{ snapshot: ISnapshot }
|
||||
>;
|
||||
verify: ICommandDefinition<
|
||||
{ level: string },
|
||||
IVerifyResult
|
||||
>;
|
||||
repair: ICommandDefinition<
|
||||
Record<string, never>,
|
||||
IRepairResult
|
||||
>;
|
||||
prune: ICommandDefinition<
|
||||
{ retention: IRetentionPolicy; dryRun?: boolean },
|
||||
IPruneResult
|
||||
>;
|
||||
reindex: ICommandDefinition<
|
||||
Record<string, never>,
|
||||
{ indexedChunks: number }
|
||||
>;
|
||||
unlock: ICommandDefinition<
|
||||
{ force?: boolean },
|
||||
{ removedLocks: number }
|
||||
>;
|
||||
};
|
||||
17
ts/plugins.ts
Normal file
17
ts/plugins.ts
Normal file
@@ -0,0 +1,17 @@
|
||||
// node native scope
|
||||
import * as path from 'node:path';
|
||||
import * as fs from 'node:fs';
|
||||
import * as net from 'node:net';
|
||||
import * as os from 'node:os';
|
||||
import * as stream from 'node:stream';
|
||||
import * as crypto from 'node:crypto';
|
||||
|
||||
export { path, fs, net, os, stream, crypto };
|
||||
|
||||
// @push.rocks scope
|
||||
import * as smartrust from '@push.rocks/smartrust';
|
||||
import * as smartrx from '@push.rocks/smartrx';
|
||||
import * as smartpromise from '@push.rocks/smartpromise';
|
||||
import * as lik from '@push.rocks/lik';
|
||||
|
||||
export { smartrust, smartrx, smartpromise, lik };
|
||||
Reference in New Issue
Block a user