fix(rustdb-storage): detect stale hint files using data file size metadata and add restart persistence regression tests

2026-04-05 09:48:10 +00:00
parent 8ebc1bb9e1
commit 22e010c554
10 changed files with 674 additions and 31 deletions
--- a/test/test.delete-persistence.ts
+++ b/test/test.delete-persistence.ts
@@ -0,0 +1,191 @@
+import { expect, tap } from '@git.zone/tstest/tapbundle';
+import * as smartdb from '../ts/index.js';
+import { MongoClient, Db } from 'mongodb';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as os from 'os';
+
+// ---------------------------------------------------------------------------
+// Test: Deletes persist across restart (tombstone + hint staleness detection)
+// Covers: append_tombstone to data.rdb, hint file data_file_size tracking,
+//         stale hint detection on restart
+// ---------------------------------------------------------------------------
+
+let tmpDir: string;
+let localDb: smartdb.LocalSmartDb;
+let client: MongoClient;
+let db: Db;
+
+function makeTmpDir(): string {
+  return fs.mkdtempSync(path.join(os.tmpdir(), 'smartdb-delete-test-'));
+}
+
+function cleanTmpDir(dir: string): void {
+  if (fs.existsSync(dir)) {
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+// ============================================================================
+// Setup
+// ============================================================================
+
+tap.test('setup: start local db and insert documents', async () => {
+  tmpDir = makeTmpDir();
+  localDb = new smartdb.LocalSmartDb({ folderPath: tmpDir });
+  const info = await localDb.start();
+  client = new MongoClient(info.connectionUri, {
+    directConnection: true,
+    serverSelectionTimeoutMS: 5000,
+  });
+  await client.connect();
+  db = client.db('deletetest');
+
+  const coll = db.collection('items');
+  await coll.insertMany([
+    { name: 'keep-1', value: 100 },
+    { name: 'keep-2', value: 200 },
+    { name: 'delete-me', value: 999 },
+    { name: 'keep-3', value: 300 },
+  ]);
+  const count = await coll.countDocuments();
+  expect(count).toEqual(4);
+});
+
+// ============================================================================
+// Delete and verify
+// ============================================================================
+
+tap.test('delete-persistence: delete a document', async () => {
+  const coll = db.collection('items');
+  const result = await coll.deleteOne({ name: 'delete-me' });
+  expect(result.deletedCount).toEqual(1);
+
+  const remaining = await coll.countDocuments();
+  expect(remaining).toEqual(3);
+
+  const deleted = await coll.findOne({ name: 'delete-me' });
+  expect(deleted).toBeNull();
+});
+
+// ============================================================================
+// Graceful restart: delete survives
+// ============================================================================
+
+tap.test('delete-persistence: graceful stop and restart', async () => {
+  await client.close();
+  await localDb.stop(); // graceful — writes hint file
+
+  localDb = new smartdb.LocalSmartDb({ folderPath: tmpDir });
+  const info = await localDb.start();
+  client = new MongoClient(info.connectionUri, {
+    directConnection: true,
+    serverSelectionTimeoutMS: 5000,
+  });
+  await client.connect();
+  db = client.db('deletetest');
+});
+
+tap.test('delete-persistence: deleted doc stays deleted after graceful restart', async () => {
+  const coll = db.collection('items');
+  const count = await coll.countDocuments();
+  expect(count).toEqual(3);
+
+  const deleted = await coll.findOne({ name: 'delete-me' });
+  expect(deleted).toBeNull();
+
+  // The remaining docs are intact
+  const keep1 = await coll.findOne({ name: 'keep-1' });
+  expect(keep1).toBeTruthy();
+  expect(keep1!.value).toEqual(100);
+});
+
+// ============================================================================
+// Simulate ungraceful restart: delete after hint write, then restart
+// The hint file data_file_size check should detect the stale hint
+// ============================================================================
+
+tap.test('delete-persistence: insert and delete more docs, then restart', async () => {
+  const coll = db.collection('items');
+
+  // Insert a new doc
+  await coll.insertOne({ name: 'temporary', value: 777 });
+  expect(await coll.countDocuments()).toEqual(4);
+
+  // Delete it
+  await coll.deleteOne({ name: 'temporary' });
+  expect(await coll.countDocuments()).toEqual(3);
+
+  const gone = await coll.findOne({ name: 'temporary' });
+  expect(gone).toBeNull();
+});
+
+tap.test('delete-persistence: stop and restart again', async () => {
+  await client.close();
+  await localDb.stop();
+
+  localDb = new smartdb.LocalSmartDb({ folderPath: tmpDir });
+  const info = await localDb.start();
+  client = new MongoClient(info.connectionUri, {
+    directConnection: true,
+    serverSelectionTimeoutMS: 5000,
+  });
+  await client.connect();
+  db = client.db('deletetest');
+});
+
+tap.test('delete-persistence: all deletes survived second restart', async () => {
+  const coll = db.collection('items');
+  const count = await coll.countDocuments();
+  expect(count).toEqual(3);
+
+  // Both deletes are permanent
+  expect(await coll.findOne({ name: 'delete-me' })).toBeNull();
+  expect(await coll.findOne({ name: 'temporary' })).toBeNull();
+
+  // Survivors intact
+  const names = (await coll.find({}).toArray()).map(d => d.name).sort();
+  expect(names).toEqual(['keep-1', 'keep-2', 'keep-3']);
+});
+
+// ============================================================================
+// Delete all docs and verify empty after restart
+// ============================================================================
+
+tap.test('delete-persistence: delete all remaining docs', async () => {
+  const coll = db.collection('items');
+  await coll.deleteMany({});
+  expect(await coll.countDocuments()).toEqual(0);
+});
+
+tap.test('delete-persistence: restart with empty collection', async () => {
+  await client.close();
+  await localDb.stop();
+
+  localDb = new smartdb.LocalSmartDb({ folderPath: tmpDir });
+  const info = await localDb.start();
+  client = new MongoClient(info.connectionUri, {
+    directConnection: true,
+    serverSelectionTimeoutMS: 5000,
+  });
+  await client.connect();
+  db = client.db('deletetest');
+});
+
+tap.test('delete-persistence: collection is empty after restart', async () => {
+  const coll = db.collection('items');
+  const count = await coll.countDocuments();
+  expect(count).toEqual(0);
+});
+
+// ============================================================================
+// Cleanup
+// ============================================================================
+
+tap.test('delete-persistence: cleanup', async () => {
+  await client.close();
+  await localDb.stop();
+  cleanTmpDir(tmpDir);
+});
+
+export default tap.start();
--- a/test/test.header-recovery.ts
+++ b/test/test.header-recovery.ts
@@ -0,0 +1,126 @@
+import { expect, tap } from '@git.zone/tstest/tapbundle';
+import * as smartdb from '../ts/index.js';
+import { MongoClient, Db } from 'mongodb';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as os from 'os';
+
+// ---------------------------------------------------------------------------
+// Test: Missing data.rdb header recovery + startup logging
+// Covers: ensure_data_header, BuildStats, info-level startup logging
+// ---------------------------------------------------------------------------
+
+let tmpDir: string;
+let localDb: smartdb.LocalSmartDb;
+let client: MongoClient;
+let db: Db;
+
+function makeTmpDir(): string {
+  return fs.mkdtempSync(path.join(os.tmpdir(), 'smartdb-header-test-'));
+}
+
+function cleanTmpDir(dir: string): void {
+  if (fs.existsSync(dir)) {
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+// ============================================================================
+// Setup: create data, then corrupt it
+// ============================================================================
+
+tap.test('setup: start, insert data, stop', async () => {
+  tmpDir = makeTmpDir();
+  localDb = new smartdb.LocalSmartDb({ folderPath: tmpDir });
+  const info = await localDb.start();
+  client = new MongoClient(info.connectionUri, {
+    directConnection: true,
+    serverSelectionTimeoutMS: 5000,
+  });
+  await client.connect();
+  db = client.db('headertest');
+
+  const coll = db.collection('docs');
+  await coll.insertMany([
+    { key: 'a', val: 1 },
+    { key: 'b', val: 2 },
+    { key: 'c', val: 3 },
+  ]);
+
+  await client.close();
+  await localDb.stop();
+});
+
+// ============================================================================
+// Delete hint file and restart: should rebuild from data.rdb scan
+// ============================================================================
+
+tap.test('header-recovery: delete hint file and restart', async () => {
+  // Find and delete hint files
+  const dbDir = path.join(tmpDir, 'headertest', 'docs');
+  const hintPath = path.join(dbDir, 'keydir.hint');
+  if (fs.existsSync(hintPath)) {
+    fs.unlinkSync(hintPath);
+  }
+
+  localDb = new smartdb.LocalSmartDb({ folderPath: tmpDir });
+  const info = await localDb.start();
+  client = new MongoClient(info.connectionUri, {
+    directConnection: true,
+    serverSelectionTimeoutMS: 5000,
+  });
+  await client.connect();
+  db = client.db('headertest');
+});
+
+tap.test('header-recovery: data intact after hint deletion', async () => {
+  const coll = db.collection('docs');
+  const count = await coll.countDocuments();
+  expect(count).toEqual(3);
+
+  const a = await coll.findOne({ key: 'a' });
+  expect(a!.val).toEqual(1);
+});
+
+// ============================================================================
+// Write new data after restart, stop, restart again
+// ============================================================================
+
+tap.test('header-recovery: write after hint-less restart', async () => {
+  const coll = db.collection('docs');
+  await coll.insertOne({ key: 'd', val: 4 });
+  expect(await coll.countDocuments()).toEqual(4);
+});
+
+tap.test('header-recovery: restart and verify all data', async () => {
+  await client.close();
+  await localDb.stop();
+
+  localDb = new smartdb.LocalSmartDb({ folderPath: tmpDir });
+  const info = await localDb.start();
+  client = new MongoClient(info.connectionUri, {
+    directConnection: true,
+    serverSelectionTimeoutMS: 5000,
+  });
+  await client.connect();
+  db = client.db('headertest');
+
+  const coll = db.collection('docs');
+  const count = await coll.countDocuments();
+  expect(count).toEqual(4);
+
+  const keys = (await coll.find({}).toArray()).map(d => d.key).sort();
+  expect(keys).toEqual(['a', 'b', 'c', 'd']);
+});
+
+// ============================================================================
+// Cleanup
+// ============================================================================
+
+tap.test('header-recovery: cleanup', async () => {
+  await client.close();
+  await localDb.stop();
+  cleanTmpDir(tmpDir);
+});
+
+export default tap.start();
--- a/test/test.stale-sockets.ts
+++ b/test/test.stale-sockets.ts
@@ -0,0 +1,82 @@
+import { expect, tap } from '@git.zone/tstest/tapbundle';
+import * as smartdb from '../ts/index.js';
+import * as fs from 'fs';
+import * as net from 'net';
+import * as path from 'path';
+import * as os from 'os';
+
+// ---------------------------------------------------------------------------
+// Test: Stale socket cleanup on startup
+// Covers: LocalSmartDb.cleanStaleSockets(), isSocketAlive()
+// ---------------------------------------------------------------------------
+
+function makeTmpDir(): string {
+  return fs.mkdtempSync(path.join(os.tmpdir(), 'smartdb-socket-test-'));
+}
+
+function cleanTmpDir(dir: string): void {
+  if (fs.existsSync(dir)) {
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+// ============================================================================
+// Stale socket cleanup: active sockets are preserved
+// ============================================================================
+
+tap.test('stale-sockets: does not remove active sockets', async () => {
+  const tmpDir = makeTmpDir();
+  const activeSocketPath = path.join(os.tmpdir(), `smartdb-active-${Date.now()}.sock`);
+
+  // Create an active socket (server still listening)
+  const activeServer = net.createServer();
+  await new Promise<void>((resolve) => activeServer.listen(activeSocketPath, resolve));
+
+  expect(fs.existsSync(activeSocketPath)).toBeTrue();
+
+  // Start LocalSmartDb — should NOT remove the active socket
+  const localDb = new smartdb.LocalSmartDb({ folderPath: tmpDir });
+  await localDb.start();
+
+  expect(fs.existsSync(activeSocketPath)).toBeTrue();
+
+  // Cleanup
+  await localDb.stop();
+  await new Promise<void>((resolve) => activeServer.close(() => resolve()));
+  try { fs.unlinkSync(activeSocketPath); } catch {}
+  cleanTmpDir(tmpDir);
+});
+
+// ============================================================================
+// Stale socket cleanup: startup works with no stale sockets
+// ============================================================================
+
+tap.test('stale-sockets: startup works cleanly with no stale sockets', async () => {
+  const tmpDir = makeTmpDir();
+  const localDb = new smartdb.LocalSmartDb({ folderPath: tmpDir });
+  const info = await localDb.start();
+  expect(localDb.running).toBeTrue();
+  expect(info.socketPath).toBeTruthy();
+  await localDb.stop();
+  cleanTmpDir(tmpDir);
+});
+
+// ============================================================================
+// Stale socket cleanup: the socket file for the current instance is cleaned on stop
+// ============================================================================
+
+tap.test('stale-sockets: own socket file is removed on stop', async () => {
+  const tmpDir = makeTmpDir();
+  const localDb = new smartdb.LocalSmartDb({ folderPath: tmpDir });
+  const info = await localDb.start();
+
+  expect(fs.existsSync(info.socketPath)).toBeTrue();
+
+  await localDb.stop();
+
+  // Socket file should be gone after graceful stop
+  expect(fs.existsSync(info.socketPath)).toBeFalse();
+  cleanTmpDir(tmpDir);
+});
+
+export default tap.start();
--- a/test/test.unique-index.ts
+++ b/test/test.unique-index.ts
@@ -0,0 +1,180 @@
+import { expect, tap } from '@git.zone/tstest/tapbundle';
+import * as smartdb from '../ts/index.js';
+import { MongoClient, Db } from 'mongodb';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as os from 'os';
+
+// ---------------------------------------------------------------------------
+// Test: Unique index enforcement via wire protocol
+// Covers: unique index pre-check, createIndexes persistence, index restoration
+// ---------------------------------------------------------------------------
+
+let tmpDir: string;
+let localDb: smartdb.LocalSmartDb;
+let client: MongoClient;
+let db: Db;
+
+function makeTmpDir(): string {
+  return fs.mkdtempSync(path.join(os.tmpdir(), 'smartdb-unique-test-'));
+}
+
+function cleanTmpDir(dir: string): void {
+  if (fs.existsSync(dir)) {
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+// ============================================================================
+// Setup
+// ============================================================================
+
+tap.test('setup: start local db', async () => {
+  tmpDir = makeTmpDir();
+  localDb = new smartdb.LocalSmartDb({ folderPath: tmpDir });
+  const info = await localDb.start();
+  client = new MongoClient(info.connectionUri, {
+    directConnection: true,
+    serverSelectionTimeoutMS: 5000,
+  });
+  await client.connect();
+  db = client.db('uniquetest');
+});
+
+// ============================================================================
+// Unique index enforcement on insert
+// ============================================================================
+
+tap.test('unique-index: createIndex with unique: true', async () => {
+  const coll = db.collection('users');
+  await coll.insertOne({ email: 'alice@example.com', name: 'Alice' });
+  const indexName = await coll.createIndex({ email: 1 }, { unique: true });
+  expect(indexName).toBeTruthy();
+});
+
+tap.test('unique-index: reject duplicate on insertOne', async () => {
+  const coll = db.collection('users');
+  let threw = false;
+  try {
+    await coll.insertOne({ email: 'alice@example.com', name: 'Alice2' });
+  } catch (err: any) {
+    threw = true;
+    expect(err.code).toEqual(11000);
+  }
+  expect(threw).toBeTrue();
+
+  // Verify only 1 document exists
+  const count = await coll.countDocuments();
+  expect(count).toEqual(1);
+});
+
+tap.test('unique-index: allow insert with different unique value', async () => {
+  const coll = db.collection('users');
+  await coll.insertOne({ email: 'bob@example.com', name: 'Bob' });
+  const count = await coll.countDocuments();
+  expect(count).toEqual(2);
+});
+
+// ============================================================================
+// Unique index enforcement on update
+// ============================================================================
+
+tap.test('unique-index: reject duplicate on updateOne that changes unique field', async () => {
+  const coll = db.collection('users');
+  let threw = false;
+  try {
+    await coll.updateOne(
+      { email: 'bob@example.com' },
+      { $set: { email: 'alice@example.com' } }
+    );
+  } catch (err: any) {
+    threw = true;
+    expect(err.code).toEqual(11000);
+  }
+  expect(threw).toBeTrue();
+
+  // Bob's email should be unchanged
+  const bob = await coll.findOne({ name: 'Bob' });
+  expect(bob!.email).toEqual('bob@example.com');
+});
+
+tap.test('unique-index: allow update that keeps same unique value', async () => {
+  const coll = db.collection('users');
+  await coll.updateOne(
+    { email: 'bob@example.com' },
+    { $set: { name: 'Robert' } }
+  );
+  const bob = await coll.findOne({ email: 'bob@example.com' });
+  expect(bob!.name).toEqual('Robert');
+});
+
+// ============================================================================
+// Unique index enforcement on upsert
+// ============================================================================
+
+tap.test('unique-index: reject duplicate on upsert insert', async () => {
+  const coll = db.collection('users');
+  let threw = false;
+  try {
+    await coll.updateOne(
+      { email: 'new@example.com' },
+      { $set: { email: 'alice@example.com', name: 'Imposter' } },
+      { upsert: true }
+    );
+  } catch (err: any) {
+    threw = true;
+  }
+  expect(threw).toBeTrue();
+});
+
+// ============================================================================
+// Unique index survives restart (persistence + restoration)
+// ============================================================================
+
+tap.test('unique-index: stop and restart', async () => {
+  await client.close();
+  await localDb.stop();
+
+  localDb = new smartdb.LocalSmartDb({ folderPath: tmpDir });
+  const info = await localDb.start();
+  client = new MongoClient(info.connectionUri, {
+    directConnection: true,
+    serverSelectionTimeoutMS: 5000,
+  });
+  await client.connect();
+  db = client.db('uniquetest');
+});
+
+tap.test('unique-index: enforcement persists after restart', async () => {
+  const coll = db.collection('users');
+
+  // Data should still be there
+  const count = await coll.countDocuments();
+  expect(count).toEqual(2);
+
+  // Unique constraint should still be enforced without calling createIndex again
+  let threw = false;
+  try {
+    await coll.insertOne({ email: 'alice@example.com', name: 'Alice3' });
+  } catch (err: any) {
+    threw = true;
+    expect(err.code).toEqual(11000);
+  }
+  expect(threw).toBeTrue();
+
+  // Count unchanged
+  const countAfter = await coll.countDocuments();
+  expect(countAfter).toEqual(2);
+});
+
+// ============================================================================
+// Cleanup
+// ============================================================================
+
+tap.test('unique-index: cleanup', async () => {
+  await client.close();
+  await localDb.stop();
+  cleanTmpDir(tmpDir);
+});
+
+export default tap.start();