feat(tsmdb): implement TsmDB Mongo-wire-compatible server, add storage/engine modules and reorganize exports

changelog.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## 2026-02-01 - 4.2.0 - feat(tsmdb)
+implement TsmDB Mongo-wire-compatible server, add storage/engine modules and reorganize exports
+
+- Add full TsmDB implementation under ts/ts_tsmdb: wire protocol, server, command router, handlers, engines (Query, Update, Aggregation, Index, Transaction, Session), storage adapters (Memory, File), OpLog, WAL, utils and types.
+- Remove legacy ts/tsmdb implementation and replace with new ts_tsmdb module exports.
+- Introduce ts/ts_mongotools module and move SmartMongo class there; update top-level exports in ts/index.ts to export SmartMongo, tsmdb (from ts_tsmdb) and LocalTsmDb.
+- Add LocalTsmDb convenience class (ts/ts_local) to start a file-backed TsmDB and return a connected MongoClient.
+- Refactor plugin imports into per-module plugins files and add utilities (checksum, persistence, query planner, index engine).
+
 ## 2026-02-01 - 4.1.1 - fix(tsmdb)
 add comprehensive unit tests for tsmdb components: checksum, query planner, index engine, session, and WAL
 

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@push.rocks/smartmongo',
-  version: '4.1.1',
+  version: '4.2.0',
   description: 'A module for creating and managing a local MongoDB instance for testing purposes.'
 }

ts/index.ts

@@ -1,74 +1,14 @@
 import { commitinfo } from './00_commitinfo_data.js';
-import * as plugins from './smartmongo.plugins.js';
+
+// Export SmartMongo from ts_mongotools
+export { SmartMongo } from './ts_mongotools/index.js';
 
 // Export TsmDB module
-export * as tsmdb from './tsmdb/index.js';
+export * as tsmdb from './ts_tsmdb/index.js';
 
-export class SmartMongo {
-  // STATIC
-  public static async createAndStart(replCountArg: number = 1) {
-    const smartMongoInstance = new SmartMongo();
-    await smartMongoInstance.start(replCountArg);
-    return smartMongoInstance;
-  }
+// Export LocalTsmDb from ts_local
+export { LocalTsmDb } from './ts_local/index.js';
+export type { ILocalTsmDbOptions } from './ts_local/index.js';
 
-  // INSTANCE
-  private _readyDeferred = plugins.smartpromise.defer();
-  public readyPromise = this._readyDeferred.promise;
-  public mongoReplicaSet: plugins.mongoPlugin.MongoMemoryReplSet;
-
-  constructor() {}
-
-  public async start(countArg: number = 1) {
-    this.mongoReplicaSet = await plugins.mongoPlugin.MongoMemoryReplSet.create({
-      replSet: { count: countArg },
-      instanceOpts: [
-        {
-          storageEngine: 'wiredTiger',
-        },
-      ],
-    });
-    this._readyDeferred.resolve();
-    console.log(`mongoReplicaSet with ${countArg} replicas started.`);
-    console.log(`@pushrocks/smartmongo version ${commitinfo.version}`);
-  }
-
-  /**
-   * returns a mongo descriptor for modules like
-   * @pushrocks/smartfile.
-   */
-  public async getMongoDescriptor(): Promise<plugins.smartdata.IMongoDescriptor> {
-    await this.readyPromise;
-    return {
-      mongoDbName: `smartmongo_testdatabase`,
-      mongoDbUrl: this.mongoReplicaSet.getUri(),
-    };
-  }
-
-  /**
-   * stops the smartmongo instance
-   * and cleans up after itself
-   */
-  public async stop() {
-    await this.mongoReplicaSet.stop();
-    await this.mongoReplicaSet.cleanup();
-  }
-
-  /**
-   * like stop() but allows you to actually store
-   * the database on disk
-   */
-  public async stopAndDumpToDir(
-    dirArg: string,
-    nameFunctionArg?: (doc: any) => string,
-    emptyDirArg = true,
-  ) {
-    const mongodumpInstance = new plugins.mongodump.MongoDump();
-    const mongodumpTarget = await mongodumpInstance.addMongoTargetByMongoDescriptor(
-      await this.getMongoDescriptor(),
-    );
-    await mongodumpTarget.dumpAllCollectionsToDir(dirArg, nameFunctionArg, emptyDirArg);
-    await mongodumpInstance.stop();
-    await this.stop();
-  }
-}
+// Export commitinfo
+export { commitinfo };

ts/ts_local/classes.localtsmdb.ts

@@ -0,0 +1,138 @@
+import * as plugins from './plugins.js';
+import { TsmdbServer } from '../ts_tsmdb/index.js';
+import type { MongoClient } from 'mongodb';
+
+export interface ILocalTsmDbOptions {
+  folderPath: string;
+  port?: number;
+  host?: string;
+}
+
+/**
+ * LocalTsmDb - Convenience class for local MongoDB-compatible database
+ *
+ * This class wraps TsmdbServer and provides a simple interface for
+ * starting a local file-based MongoDB-compatible server and connecting to it.
+ *
+ * @example
+ * ```typescript
+ * import { LocalTsmDb } from '@push.rocks/smartmongo';
+ *
+ * const db = new LocalTsmDb({ folderPath: './data' });
+ * const client = await db.start();
+ *
+ * // Use the MongoDB client
+ * const collection = client.db('mydb').collection('users');
+ * await collection.insertOne({ name: 'Alice' });
+ *
+ * // When done
+ * await db.stop();
+ * ```
+ */
+export class LocalTsmDb {
+  private options: ILocalTsmDbOptions;
+  private server: TsmdbServer | null = null;
+  private client: MongoClient | null = null;
+
+  constructor(options: ILocalTsmDbOptions) {
+    this.options = options;
+  }
+
+  /**
+   * Find an available port starting from the given port
+   */
+  private async findAvailablePort(startPort = 27017): Promise<number> {
+    return new Promise((resolve, reject) => {
+      const server = plugins.net.createServer();
+      server.listen(startPort, '127.0.0.1', () => {
+        const addr = server.address();
+        const port = typeof addr === 'object' && addr ? addr.port : startPort;
+        server.close(() => resolve(port));
+      });
+      server.on('error', () => {
+        this.findAvailablePort(startPort + 1).then(resolve).catch(reject);
+      });
+    });
+  }
+
+  /**
+   * Start the local TsmDB server and return a connected MongoDB client
+   */
+  async start(): Promise<MongoClient> {
+    if (this.server && this.client) {
+      throw new Error('LocalTsmDb is already running');
+    }
+
+    const port = this.options.port ?? await this.findAvailablePort();
+    const host = this.options.host ?? '127.0.0.1';
+
+    this.server = new TsmdbServer({
+      port,
+      host,
+      storage: 'file',
+      storagePath: this.options.folderPath,
+    });
+    await this.server.start();
+
+    // Dynamically import mongodb to avoid requiring it as a hard dependency
+    const mongodb = await import('mongodb');
+    this.client = new mongodb.MongoClient(this.server.getConnectionUri(), {
+      directConnection: true,
+      serverSelectionTimeoutMS: 5000,
+    });
+    await this.client.connect();
+
+    return this.client;
+  }
+
+  /**
+   * Get the MongoDB client (throws if not started)
+   */
+  getClient(): MongoClient {
+    if (!this.client) {
+      throw new Error('LocalTsmDb is not running. Call start() first.');
+    }
+    return this.client;
+  }
+
+  /**
+   * Get the underlying TsmdbServer instance (throws if not started)
+   */
+  getServer(): TsmdbServer {
+    if (!this.server) {
+      throw new Error('LocalTsmDb is not running. Call start() first.');
+    }
+    return this.server;
+  }
+
+  /**
+   * Get the connection URI
+   */
+  getConnectionUri(): string {
+    if (!this.server) {
+      throw new Error('LocalTsmDb is not running. Call start() first.');
+    }
+    return this.server.getConnectionUri();
+  }
+
+  /**
+   * Check if the server is running
+   */
+  get running(): boolean {
+    return this.server !== null && this.server.running;
+  }
+
+  /**
+   * Stop the local TsmDB server and close the client connection
+   */
+  async stop(): Promise<void> {
+    if (this.client) {
+      await this.client.close();
+      this.client = null;
+    }
+    if (this.server) {
+      await this.server.stop();
+      this.server = null;
+    }
+  }
+}

ts/ts_local/index.ts

@@ -0,0 +1,2 @@
+export { LocalTsmDb } from './classes.localtsmdb.js';
+export type { ILocalTsmDbOptions } from './classes.localtsmdb.js';

ts/ts_local/plugins.ts

@@ -0,0 +1,4 @@
+import * as smartpromise from '@push.rocks/smartpromise';
+import * as net from 'net';
+
+export { smartpromise, net };

ts/ts_mongotools/classes.smartmongo.ts

@@ -0,0 +1,71 @@
+import { commitinfo } from '../00_commitinfo_data.js';
+import * as plugins from './plugins.js';
+
+export class SmartMongo {
+  // STATIC
+  public static async createAndStart(replCountArg: number = 1) {
+    const smartMongoInstance = new SmartMongo();
+    await smartMongoInstance.start(replCountArg);
+    return smartMongoInstance;
+  }
+
+  // INSTANCE
+  private _readyDeferred = plugins.smartpromise.defer();
+  public readyPromise = this._readyDeferred.promise;
+  public mongoReplicaSet: plugins.mongoPlugin.MongoMemoryReplSet;
+
+  constructor() {}
+
+  public async start(countArg: number = 1) {
+    this.mongoReplicaSet = await plugins.mongoPlugin.MongoMemoryReplSet.create({
+      replSet: { count: countArg },
+      instanceOpts: [
+        {
+          storageEngine: 'wiredTiger',
+        },
+      ],
+    });
+    this._readyDeferred.resolve();
+    console.log(`mongoReplicaSet with ${countArg} replicas started.`);
+    console.log(`@pushrocks/smartmongo version ${commitinfo.version}`);
+  }
+
+  /**
+   * returns a mongo descriptor for modules like
+   * @pushrocks/smartfile.
+   */
+  public async getMongoDescriptor(): Promise<plugins.smartdata.IMongoDescriptor> {
+    await this.readyPromise;
+    return {
+      mongoDbName: `smartmongo_testdatabase`,
+      mongoDbUrl: this.mongoReplicaSet.getUri(),
+    };
+  }
+
+  /**
+   * stops the smartmongo instance
+   * and cleans up after itself
+   */
+  public async stop() {
+    await this.mongoReplicaSet.stop();
+    await this.mongoReplicaSet.cleanup();
+  }
+
+  /**
+   * like stop() but allows you to actually store
+   * the database on disk
+   */
+  public async stopAndDumpToDir(
+    dirArg: string,
+    nameFunctionArg?: (doc: any) => string,
+    emptyDirArg = true,
+  ) {
+    const mongodumpInstance = new plugins.mongodump.MongoDump();
+    const mongodumpTarget = await mongodumpInstance.addMongoTargetByMongoDescriptor(
+      await this.getMongoDescriptor(),
+    );
+    await mongodumpTarget.dumpAllCollectionsToDir(dirArg, nameFunctionArg, emptyDirArg);
+    await mongodumpInstance.stop();
+    await this.stop();
+  }
+}

ts/ts_mongotools/index.ts

@@ -0,0 +1,2 @@
+export * from './plugins.js';
+export { SmartMongo } from './classes.smartmongo.js';

ts/ts_tsmdb/engine/AggregationEngine.ts

@@ -1,4 +1,4 @@
-import * as plugins from '../tsmdb.plugins.js';
+import * as plugins from '../plugins.js';
 import type { Document, IStoredDocument, IAggregateOptions } from '../types/interfaces.js';
 
 // Import mingo Aggregator

ts/ts_tsmdb/engine/IndexEngine.ts

@@ -1,4 +1,4 @@
-import * as plugins from '../tsmdb.plugins.js';
+import * as plugins from '../plugins.js';
 import type { IStorageAdapter } from '../storage/IStorageAdapter.js';
 
 // Simple B-Tree implementation for range queries

ts/ts_tsmdb/engine/QueryEngine.ts

@@ -1,4 +1,4 @@
-import * as plugins from '../tsmdb.plugins.js';
+import * as plugins from '../plugins.js';
 import type { Document, IStoredDocument, ISortSpecification, ISortDirection } from '../types/interfaces.js';
 
 // Import mingo Query class

ts/ts_tsmdb/engine/QueryPlanner.ts

@@ -1,4 +1,4 @@
-import * as plugins from '../tsmdb.plugins.js';
+import * as plugins from '../plugins.js';
 import type { Document, IStoredDocument } from '../types/interfaces.js';
 import { IndexEngine } from './IndexEngine.js';
 

ts/ts_tsmdb/engine/SessionEngine.ts

@@ -1,4 +1,4 @@
-import * as plugins from '../tsmdb.plugins.js';
+import * as plugins from '../plugins.js';
 import type { TransactionEngine } from './TransactionEngine.js';
 
 /**

ts/ts_tsmdb/engine/TransactionEngine.ts

@@ -1,4 +1,4 @@
-import * as plugins from '../tsmdb.plugins.js';
+import * as plugins from '../plugins.js';
 import type { IStorageAdapter } from '../storage/IStorageAdapter.js';
 import type { Document, IStoredDocument, ITransactionOptions } from '../types/interfaces.js';
 import { TsmdbTransactionError, TsmdbWriteConflictError } from '../errors/TsmdbErrors.js';

ts/ts_tsmdb/engine/UpdateEngine.ts

@@ -1,4 +1,4 @@
-import * as plugins from '../tsmdb.plugins.js';
+import * as plugins from '../plugins.js';
 import type { Document, IStoredDocument } from '../types/interfaces.js';
 import { QueryEngine } from './QueryEngine.js';
 

ts/ts_tsmdb/index.ts

@@ -2,7 +2,7 @@
 // Use the official MongoDB driver to connect to TsmdbServer
 
 // Re-export plugins for external use
-import * as plugins from './tsmdb.plugins.js';
+import * as plugins from './plugins.js';
 export { plugins };
 
 // Export BSON types for convenience

ts/ts_tsmdb/server/CommandRouter.ts

@@ -1,4 +1,4 @@
-import * as plugins from '../tsmdb.plugins.js';
+import * as plugins from '../plugins.js';
 import type { IStorageAdapter } from '../storage/IStorageAdapter.js';
 import type { IParsedCommand } from './WireProtocol.js';
 import type { TsmdbServer } from './TsmdbServer.js';

ts/ts_tsmdb/server/TsmdbServer.ts

@@ -1,5 +1,5 @@
 import * as net from 'net';
-import * as plugins from '../tsmdb.plugins.js';
+import * as plugins from '../plugins.js';
 import { WireProtocol, OP_QUERY } from './WireProtocol.js';
 import { CommandRouter } from './CommandRouter.js';
 import { MemoryStorageAdapter } from '../storage/MemoryStorageAdapter.js';

ts/ts_tsmdb/server/WireProtocol.ts

@@ -1,4 +1,4 @@
-import * as plugins from '../tsmdb.plugins.js';
+import * as plugins from '../plugins.js';
 
 /**
  * MongoDB Wire Protocol Implementation

ts/ts_tsmdb/server/handlers/AdminHandler.ts

@@ -1,4 +1,4 @@
-import * as plugins from '../../tsmdb.plugins.js';
+import * as plugins from '../../plugins.js';
 import type { ICommandHandler, IHandlerContext } from '../CommandRouter.js';
 import { SessionEngine } from '../../engine/SessionEngine.js';
 

ts/ts_tsmdb/server/handlers/AggregateHandler.ts

@@ -1,4 +1,4 @@
-import * as plugins from '../../tsmdb.plugins.js';
+import * as plugins from '../../plugins.js';
 import type { ICommandHandler, IHandlerContext, ICursorState } from '../CommandRouter.js';
 import { AggregationEngine } from '../../engine/AggregationEngine.js';
 

ts/ts_tsmdb/server/handlers/DeleteHandler.ts

@@ -1,4 +1,4 @@
-import * as plugins from '../../tsmdb.plugins.js';
+import * as plugins from '../../plugins.js';
 import type { ICommandHandler, IHandlerContext } from '../CommandRouter.js';
 import type { IStoredDocument } from '../../types/interfaces.js';
 import { QueryEngine } from '../../engine/QueryEngine.js';

ts/ts_tsmdb/server/handlers/FindHandler.ts

@@ -1,4 +1,4 @@
-import * as plugins from '../../tsmdb.plugins.js';
+import * as plugins from '../../plugins.js';
 import type { ICommandHandler, IHandlerContext, ICursorState } from '../CommandRouter.js';
 import type { IStoredDocument } from '../../types/interfaces.js';
 import { QueryEngine } from '../../engine/QueryEngine.js';

ts/ts_tsmdb/server/handlers/HelloHandler.ts

@@ -1,4 +1,4 @@
-import * as plugins from '../../tsmdb.plugins.js';
+import * as plugins from '../../plugins.js';
 import type { ICommandHandler, IHandlerContext } from '../CommandRouter.js';
 
 /**

ts/ts_tsmdb/server/handlers/IndexHandler.ts

@@ -1,4 +1,4 @@
-import * as plugins from '../../tsmdb.plugins.js';
+import * as plugins from '../../plugins.js';
 import type { ICommandHandler, IHandlerContext } from '../CommandRouter.js';
 import { IndexEngine } from '../../engine/IndexEngine.js';
 

ts/ts_tsmdb/server/handlers/InsertHandler.ts

@@ -1,4 +1,4 @@
-import * as plugins from '../../tsmdb.plugins.js';
+import * as plugins from '../../plugins.js';
 import type { ICommandHandler, IHandlerContext } from '../CommandRouter.js';
 import type { IStoredDocument } from '../../types/interfaces.js';
 

ts/ts_tsmdb/server/handlers/UpdateHandler.ts

@@ -1,4 +1,4 @@
-import * as plugins from '../../tsmdb.plugins.js';
+import * as plugins from '../../plugins.js';
 import type { ICommandHandler, IHandlerContext } from '../CommandRouter.js';
 import type { IStoredDocument } from '../../types/interfaces.js';
 import { QueryEngine } from '../../engine/QueryEngine.js';

ts/ts_tsmdb/storage/FileStorageAdapter.ts

@@ -1,4 +1,4 @@
-import * as plugins from '../tsmdb.plugins.js';
+import * as plugins from '../plugins.js';
 import type { IStorageAdapter } from './IStorageAdapter.js';
 import type { IStoredDocument, IOpLogEntry, Document } from '../types/interfaces.js';
 import { calculateDocumentChecksum, verifyChecksum } from '../utils/checksum.js';

ts/ts_tsmdb/storage/IStorageAdapter.ts

@@ -1,4 +1,4 @@
-import type * as plugins from '../tsmdb.plugins.js';
+import type * as plugins from '../plugins.js';
 import type { IStoredDocument, IOpLogEntry, Document } from '../types/interfaces.js';
 
 /**

ts/ts_tsmdb/storage/MemoryStorageAdapter.ts

@@ -1,4 +1,4 @@
-import * as plugins from '../tsmdb.plugins.js';
+import * as plugins from '../plugins.js';
 import type { IStorageAdapter } from './IStorageAdapter.js';
 import type { IStoredDocument, IOpLogEntry, Document } from '../types/interfaces.js';
 

ts/ts_tsmdb/storage/OpLog.ts

@@ -1,4 +1,4 @@
-import * as plugins from '../tsmdb.plugins.js';
+import * as plugins from '../plugins.js';
 import type { IStorageAdapter } from './IStorageAdapter.js';
 import type { IOpLogEntry, Document, IResumeToken, ChangeStreamOperationType } from '../types/interfaces.js';
 

ts/ts_tsmdb/storage/WAL.ts

@@ -1,4 +1,4 @@
-import * as plugins from '../tsmdb.plugins.js';
+import * as plugins from '../plugins.js';
 import type { Document, IStoredDocument } from '../types/interfaces.js';
 
 /**

ts/ts_tsmdb/types/interfaces.ts

@@ -1,4 +1,4 @@
-import type * as plugins from '../tsmdb.plugins.js';
+import type * as plugins from '../plugins.js';
 
 // ============================================================================
 // Document Types