push.rocks/smartmongo
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

BREAKING CHANGE(storage,engine,server): add session & transaction management, index/query planner, WAL and checksum support; integrate index-accelerated queries and update storage API (findByIds) to enable index optimizations

Browse Source
This commit is contained in:
Jürgen Kunz
2026-02-01 16:02:03 +00:00
parent 12102255c4
commit bd1764159e
19 changed files with 1973 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

109
ts/tsmdb/server/CommandRouter.ts
View File

@@ -2,6 +2,9 @@ import * as plugins from '../tsmdb.plugins.js';
import type { IStorageAdapter } from '../storage/IStorageAdapter.js';
import type { IParsedCommand } from './WireProtocol.js';
import type { TsmdbServer } from './TsmdbServer.js';
import { IndexEngine } from '../engine/IndexEngine.js';
import { TransactionEngine } from '../engine/TransactionEngine.js';
import { SessionEngine } from '../engine/SessionEngine.js';
// Import handlers
import { HelloHandler } from './handlers/HelloHandler.js';
@@ -22,6 +25,16 @@ export interface IHandlerContext {
database: string;
command: plugins.bson.Document;
documentSequences?: Map<string, plugins.bson.Document[]>;
/** Get or create an IndexEngine for a collection */
getIndexEngine: (collName: string) => IndexEngine;
/** Transaction engine instance */
transactionEngine: TransactionEngine;
/** Current transaction ID (if in a transaction) */
txnId?: string;
/** Session ID (from lsid) */
sessionId?: string;
/** Session engine instance */
sessionEngine: SessionEngine;
}
/**
@@ -43,12 +56,54 @@ export class CommandRouter {
private cursors: Map<bigint, ICursorState> = new Map();
private cursorIdCounter: bigint = BigInt(1);
// Index engine cache: db.collection -> IndexEngine
private indexEngines: Map<string, IndexEngine> = new Map();
// Transaction engine (shared across all handlers)
private transactionEngine: TransactionEngine;
// Session engine (shared across all handlers)
private sessionEngine: SessionEngine;
constructor(storage: IStorageAdapter, server: TsmdbServer) {
this.storage = storage;
this.server = server;
this.transactionEngine = new TransactionEngine(storage);
this.sessionEngine = new SessionEngine();
// Link session engine to transaction engine for auto-abort on session expiry
this.sessionEngine.setTransactionEngine(this.transactionEngine);
this.registerHandlers();
}
/**
* Get or create an IndexEngine for a database.collection
*/
getIndexEngine(dbName: string, collName: string): IndexEngine {
const key = `${dbName}.${collName}`;
let engine = this.indexEngines.get(key);
if (!engine) {
engine = new IndexEngine(dbName, collName, this.storage);
this.indexEngines.set(key, engine);
}
return engine;
}
/**
* Clear index engine cache for a collection (used when collection is dropped)
*/
clearIndexEngineCache(dbName: string, collName?: string): void {
if (collName) {
this.indexEngines.delete(`${dbName}.${collName}`);
} else {
// Clear all engines for the database
for (const key of this.indexEngines.keys()) {
if (key.startsWith(`${dbName}.`)) {
this.indexEngines.delete(key);
}
}
}
}
/**
* Register all command handlers
*/
@@ -120,6 +175,29 @@ export class CommandRouter {
async route(parsedCommand: IParsedCommand): Promise<plugins.bson.Document> {
const { commandName, command, database, documentSequences } = parsedCommand;
// Extract session ID from lsid using SessionEngine helper
let sessionId = SessionEngine.extractSessionId(command.lsid);
let txnId: string | undefined;
// If we have a session ID, register/touch the session
if (sessionId) {
this.sessionEngine.getOrCreateSession(sessionId);
}
// Check if this starts a new transaction
if (command.startTransaction && sessionId) {
txnId = this.transactionEngine.startTransaction(sessionId);
this.sessionEngine.startTransaction(sessionId, txnId, command.txnNumber);
} else if (sessionId && this.sessionEngine.isInTransaction(sessionId)) {
// Continue existing transaction
txnId = this.sessionEngine.getTransactionId(sessionId);
// Verify transaction is still active
if (txnId && !this.transactionEngine.isActive(txnId)) {
this.sessionEngine.endTransaction(sessionId);
txnId = undefined;
}
}
// Create handler context
const context: IHandlerContext = {
storage: this.storage,
@@ -127,6 +205,11 @@ export class CommandRouter {
database,
command,
documentSequences,
getIndexEngine: (collName: string) => this.getIndexEngine(database, collName),
transactionEngine: this.transactionEngine,
sessionEngine: this.sessionEngine,
txnId,
sessionId,
};
// Find handler
@@ -164,6 +247,32 @@ export class CommandRouter {
};
}
}
/**
* Close the command router and cleanup resources
*/
close(): void {
// Close session engine (stops cleanup interval, clears sessions)
this.sessionEngine.close();
// Clear cursors
this.cursors.clear();
// Clear index engine cache
this.indexEngines.clear();
}
/**
* Get session engine (for administrative purposes)
*/
getSessionEngine(): SessionEngine {
return this.sessionEngine;
}
/**
* Get transaction engine (for administrative purposes)
*/
getTransactionEngine(): TransactionEngine {
return this.transactionEngine;
}
}
/**

3
ts/tsmdb/server/TsmdbServer.ts
View File

@@ -154,6 +154,9 @@ export class TsmdbServer {
}
this.connections.clear();
// Close command router (cleans up session engine, cursors, etc.)
this.commandRouter.close();
// Close storage
await this.storage.close();

115
ts/tsmdb/server/handlers/AdminHandler.ts
View File

@@ -1,5 +1,6 @@
import * as plugins from '../../tsmdb.plugins.js';
import type { ICommandHandler, IHandlerContext } from '../CommandRouter.js';
import { SessionEngine } from '../../engine/SessionEngine.js';
/**
* AdminHandler - Handles administrative commands
@@ -237,10 +238,12 @@ export class AdminHandler implements ICommandHandler {
* Handle serverStatus command
*/
private async handleServerStatus(context: IHandlerContext): Promise<plugins.bson.Document> {
const { server } = context;
const { server, sessionEngine } = context;
const uptime = server.getUptime();
const connections = server.getConnectionCount();
const sessions = sessionEngine.listSessions();
const sessionsWithTxn = sessionEngine.getSessionsWithTransactions();
return {
ok: 1,
@@ -263,6 +266,26 @@ export class AdminHandler implements ICommandHandler {
totalCreated: connections,
active: connections,
},
logicalSessionRecordCache: {
activeSessionsCount: sessions.length,
sessionsCollectionJobCount: 0,
lastSessionsCollectionJobDurationMillis: 0,
lastSessionsCollectionJobTimestamp: new Date(),
transactionReaperJobCount: 0,
lastTransactionReaperJobDurationMillis: 0,
lastTransactionReaperJobTimestamp: new Date(),
},
transactions: {
retriedCommandsCount: 0,
retriedStatementsCount: 0,
transactionsCollectionWriteCount: 0,
currentActive: sessionsWithTxn.length,
currentInactive: 0,
currentOpen: sessionsWithTxn.length,
totalStarted: sessionsWithTxn.length,
totalCommitted: 0,
totalAborted: 0,
},
network: {
bytesIn: 0,
bytesOut: 0,
@@ -409,6 +432,17 @@ export class AdminHandler implements ICommandHandler {
* Handle endSessions command
*/
private async handleEndSessions(context: IHandlerContext): Promise<plugins.bson.Document> {
const { command, sessionEngine } = context;
// End each session in the array
const sessions = command.endSessions || [];
for (const sessionSpec of sessions) {
const sessionId = SessionEngine.extractSessionId(sessionSpec);
if (sessionId) {
await sessionEngine.endSession(sessionId);
}
}
return { ok: 1 };
}
@@ -416,16 +450,87 @@ export class AdminHandler implements ICommandHandler {
* Handle abortTransaction command
*/
private async handleAbortTransaction(context: IHandlerContext): Promise<plugins.bson.Document> {
// Transactions are not fully supported, but acknowledge the command
return { ok: 1 };
const { transactionEngine, sessionEngine, txnId, sessionId } = context;
if (!txnId) {
return {
ok: 0,
errmsg: 'No transaction started',
code: 251,
codeName: 'NoSuchTransaction',
};
}
try {
await transactionEngine.abortTransaction(txnId);
transactionEngine.endTransaction(txnId);
// Update session state
if (sessionId) {
sessionEngine.endTransaction(sessionId);
}
return { ok: 1 };
} catch (error: any) {
return {
ok: 0,
errmsg: error.message || 'Abort transaction failed',
code: error.code || 1,
codeName: error.codeName || 'UnknownError',
};
}
}
/**
* Handle commitTransaction command
*/
private async handleCommitTransaction(context: IHandlerContext): Promise<plugins.bson.Document> {
// Transactions are not fully supported, but acknowledge the command
return { ok: 1 };
const { transactionEngine, sessionEngine, txnId, sessionId } = context;
if (!txnId) {
return {
ok: 0,
errmsg: 'No transaction started',
code: 251,
codeName: 'NoSuchTransaction',
};
}
try {
await transactionEngine.commitTransaction(txnId);
transactionEngine.endTransaction(txnId);
// Update session state
if (sessionId) {
sessionEngine.endTransaction(sessionId);
}
return { ok: 1 };
} catch (error: any) {
// If commit fails, transaction should be aborted
try {
await transactionEngine.abortTransaction(txnId);
transactionEngine.endTransaction(txnId);
if (sessionId) {
sessionEngine.endTransaction(sessionId);
}
} catch {
// Ignore abort errors
}
if (error.code === 112) {
// Write conflict
return {
ok: 0,
errmsg: error.message || 'Write conflict during commit',
code: 112,
codeName: 'WriteConflict',
};
}
return {
ok: 0,
errmsg: error.message || 'Commit transaction failed',
code: error.code || 1,
codeName: error.codeName || 'UnknownError',
};
}
}
/**

19
ts/tsmdb/server/handlers/DeleteHandler.ts
View File

@@ -1,5 +1,6 @@
import * as plugins from '../../tsmdb.plugins.js';
import type { ICommandHandler, IHandlerContext } from '../CommandRouter.js';
import type { IStoredDocument } from '../../types/interfaces.js';
import { QueryEngine } from '../../engine/QueryEngine.js';
/**
@@ -47,6 +48,8 @@ export class DeleteHandler implements ICommandHandler {
return { ok: 1, n: 0 };
}
const indexEngine = context.getIndexEngine(collection);
for (let i = 0; i < deletes.length; i++) {
const deleteSpec = deletes[i];
const filter = deleteSpec.q || deleteSpec.filter || {};
@@ -56,8 +59,15 @@ export class DeleteHandler implements ICommandHandler {
const deleteAll = limit === 0;
try {
// Get all documents
const documents = await storage.findAll(database, collection);
// Try to use index-accelerated query
const candidateIds = await indexEngine.findCandidateIds(filter);
let documents: IStoredDocument[];
if (candidateIds !== null) {
documents = await storage.findByIds(database, collection, candidateIds);
} else {
documents = await storage.findAll(database, collection);
}
// Apply filter
const matchingDocs = QueryEngine.filter(documents, filter);
@@ -69,6 +79,11 @@ export class DeleteHandler implements ICommandHandler {
// Determine which documents to delete
const docsToDelete = deleteAll ? matchingDocs : matchingDocs.slice(0, 1);
// Update indexes for deleted documents
for (const doc of docsToDelete) {
await indexEngine.onDelete(doc as any);
}
// Delete the documents
const idsToDelete = docsToDelete.map(doc => doc._id);
const deleted = await storage.deleteByIds(database, collection, idsToDelete);

55
ts/tsmdb/server/handlers/FindHandler.ts
View File

@@ -1,5 +1,6 @@
import * as plugins from '../../tsmdb.plugins.js';
import type { ICommandHandler, IHandlerContext, ICursorState } from '../CommandRouter.js';
import type { IStoredDocument } from '../../types/interfaces.js';
import { QueryEngine } from '../../engine/QueryEngine.js';
/**
@@ -45,7 +46,7 @@ export class FindHandler implements ICommandHandler {
* Handle find command
*/
private async handleFind(context: IHandlerContext): Promise<plugins.bson.Document> {
const { storage, database, command } = context;
const { storage, database, command, getIndexEngine } = context;
const collection = command.find;
const filter = command.filter || {};
@@ -70,11 +71,22 @@ export class FindHandler implements ICommandHandler {
};
}
// Get all documents
let documents = await storage.findAll(database, collection);
// Try to use index-accelerated query
const indexEngine = getIndexEngine(collection);
const candidateIds = await indexEngine.findCandidateIds(filter);
// Apply filter
documents = QueryEngine.filter(documents, filter);
let documents: IStoredDocument[];
if (candidateIds !== null) {
// Index hit - fetch only candidate documents
documents = await storage.findByIds(database, collection, candidateIds);
// Still apply filter for any conditions the index couldn't fully satisfy
documents = QueryEngine.filter(documents, filter);
} else {
// No suitable index - full collection scan
documents = await storage.findAll(database, collection);
// Apply filter
documents = QueryEngine.filter(documents, filter);
}
// Apply sort
if (sort) {
@@ -233,7 +245,7 @@ export class FindHandler implements ICommandHandler {
* Handle count command
*/
private async handleCount(context: IHandlerContext): Promise<plugins.bson.Document> {
const { storage, database, command } = context;
const { storage, database, command, getIndexEngine } = context;
const collection = command.count;
const query = command.query || {};
@@ -246,11 +258,20 @@ export class FindHandler implements ICommandHandler {
return { ok: 1, n: 0 };
}
// Get all documents
let documents = await storage.findAll(database, collection);
// Try to use index-accelerated query
const indexEngine = getIndexEngine(collection);
const candidateIds = await indexEngine.findCandidateIds(query);
// Apply filter
documents = QueryEngine.filter(documents, query);
let documents: IStoredDocument[];
if (candidateIds !== null) {
// Index hit - fetch only candidate documents
documents = await storage.findByIds(database, collection, candidateIds);
documents = QueryEngine.filter(documents, query);
} else {
// No suitable index - full collection scan
documents = await storage.findAll(database, collection);
documents = QueryEngine.filter(documents, query);
}
// Apply skip
if (skip > 0) {
@@ -269,7 +290,7 @@ export class FindHandler implements ICommandHandler {
* Handle distinct command
*/
private async handleDistinct(context: IHandlerContext): Promise<plugins.bson.Document> {
const { storage, database, command } = context;
const { storage, database, command, getIndexEngine } = context;
const collection = command.distinct;
const key = command.key;
@@ -290,8 +311,16 @@ export class FindHandler implements ICommandHandler {
return { ok: 1, values: [] };
}
// Get all documents
const documents = await storage.findAll(database, collection);
// Try to use index-accelerated query
const indexEngine = getIndexEngine(collection);
const candidateIds = await indexEngine.findCandidateIds(query);
let documents: IStoredDocument[];
if (candidateIds !== null) {
documents = await storage.findByIds(database, collection, candidateIds);
} else {
documents = await storage.findAll(database, collection);
}
// Get distinct values
const values = QueryEngine.distinct(documents, key, query);

6
ts/tsmdb/server/handlers/InsertHandler.ts
View File

@@ -1,5 +1,6 @@
import * as plugins from '../../tsmdb.plugins.js';
import type { ICommandHandler, IHandlerContext } from '../CommandRouter.js';
import type { IStoredDocument } from '../../types/interfaces.js';
/**
* InsertHandler - Handles insert commands
@@ -42,6 +43,8 @@ export class InsertHandler implements ICommandHandler {
// Ensure collection exists
await storage.createCollection(database, collection);
const indexEngine = context.getIndexEngine(collection);
// Insert documents
for (let i = 0; i < documents.length; i++) {
const doc = documents[i];
@@ -52,6 +55,9 @@ export class InsertHandler implements ICommandHandler {
doc._id = new plugins.bson.ObjectId();
}
// Check index constraints before insert (doc now has _id)
await indexEngine.onInsert(doc as IStoredDocument);
await storage.insertOne(database, collection, doc);
insertedCount++;
} catch (error: any) {

37
ts/tsmdb/server/handlers/UpdateHandler.ts
View File

@@ -1,5 +1,6 @@
import * as plugins from '../../tsmdb.plugins.js';
import type { ICommandHandler, IHandlerContext } from '../CommandRouter.js';
import type { IStoredDocument } from '../../types/interfaces.js';
import { QueryEngine } from '../../engine/QueryEngine.js';
import { UpdateEngine } from '../../engine/UpdateEngine.js';
@@ -69,6 +70,8 @@ export class UpdateHandler implements ICommandHandler {
// Ensure collection exists
await storage.createCollection(database, collection);
const indexEngine = context.getIndexEngine(collection);
for (let i = 0; i < updates.length; i++) {
const updateSpec = updates[i];
const filter = updateSpec.q || updateSpec.filter || {};
@@ -78,8 +81,15 @@ export class UpdateHandler implements ICommandHandler {
const arrayFilters = updateSpec.arrayFilters;
try {
// Get all documents
let documents = await storage.findAll(database, collection);
// Try to use index-accelerated query
const candidateIds = await indexEngine.findCandidateIds(filter);
let documents: IStoredDocument[];
if (candidateIds !== null) {
documents = await storage.findByIds(database, collection, candidateIds);
} else {
documents = await storage.findAll(database, collection);
}
// Apply filter
let matchingDocs = QueryEngine.filter(documents, filter);
@@ -99,6 +109,8 @@ export class UpdateHandler implements ICommandHandler {
Object.assign(updatedDoc, update.$setOnInsert);
}
// Update index for the new document
await indexEngine.onInsert(updatedDoc);
await storage.insertOne(database, collection, updatedDoc);
totalUpserted++;
upserted.push({ index: i, _id: updatedDoc._id });
@@ -113,6 +125,8 @@ export class UpdateHandler implements ICommandHandler {
// Check if document actually changed
const changed = JSON.stringify(doc) !== JSON.stringify(updatedDoc);
if (changed) {
// Update index
await indexEngine.onUpdate(doc as any, updatedDoc);
await storage.updateById(database, collection, doc._id, updatedDoc);
totalModified++;
}
@@ -186,8 +200,17 @@ export class UpdateHandler implements ICommandHandler {
// Ensure collection exists
await storage.createCollection(database, collection);
// Get matching documents
let documents = await storage.findAll(database, collection);
// Try to use index-accelerated query
const indexEngine = context.getIndexEngine(collection);
const candidateIds = await indexEngine.findCandidateIds(query);
let documents: IStoredDocument[];
if (candidateIds !== null) {
documents = await storage.findByIds(database, collection, candidateIds);
} else {
documents = await storage.findAll(database, collection);
}
let matchingDocs = QueryEngine.filter(documents, query);
// Apply sort if specified
@@ -203,6 +226,8 @@ export class UpdateHandler implements ICommandHandler {
return { ok: 1, value: null };
}
// Update index for delete
await indexEngine.onDelete(doc as any);
await storage.deleteById(database, collection, doc._id);
let result = doc;
@@ -231,6 +256,8 @@ export class UpdateHandler implements ICommandHandler {
// Update existing
originalDoc = { ...doc };
resultDoc = UpdateEngine.applyUpdate(doc, update, arrayFilters);
// Update index
await indexEngine.onUpdate(doc as any, resultDoc as any);
await storage.updateById(database, collection, doc._id, resultDoc as any);
} else {
// Upsert
@@ -243,6 +270,8 @@ export class UpdateHandler implements ICommandHandler {
Object.assign(resultDoc, update.$setOnInsert);
}
// Update index for insert
await indexEngine.onInsert(resultDoc as any);
await storage.insertOne(database, collection, resultDoc);
}