push.rocks/smartmongo
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

BREAKING CHANGE(storage,engine,server): add session & transaction management, index/query planner, WAL and checksum support; integrate index-accelerated queries and update storage API (findByIds) to enable index optimizations

Browse Source
This commit is contained in:
Jürgen Kunz
2026-02-01 16:02:03 +00:00
parent 12102255c4
commit bd1764159e
19 changed files with 1973 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

431
ts/tsmdb/engine/IndexEngine.ts
View File

@@ -1,5 +1,89 @@
import * as plugins from '../tsmdb.plugins.js';
import type { IStorageAdapter } from '../storage/IStorageAdapter.js';
// Simple B-Tree implementation for range queries
// Since sorted-btree has ESM/CJS interop issues, we use a simple custom implementation
class SimpleBTree<K, V> {
private entries: Map<string, { key: K; value: V }> = new Map();
private sortedKeys: K[] = [];
private comparator: (a: K, b: K) => number;
constructor(_unused?: undefined, comparator?: (a: K, b: K) => number) {
this.comparator = comparator || ((a: K, b: K) => {
if (a < b) return -1;
if (a > b) return 1;
return 0;
});
}
private keyToString(key: K): string {
return JSON.stringify(key);
}
set(key: K, value: V): boolean {
const keyStr = this.keyToString(key);
const existed = this.entries.has(keyStr);
this.entries.set(keyStr, { key, value });
if (!existed) {
// Insert in sorted order
const idx = this.sortedKeys.findIndex(k => this.comparator(k, key) > 0);
if (idx === -1) {
this.sortedKeys.push(key);
} else {
this.sortedKeys.splice(idx, 0, key);
}
}
return !existed;
}
get(key: K): V | undefined {
const entry = this.entries.get(this.keyToString(key));
return entry?.value;
}
delete(key: K): boolean {
const keyStr = this.keyToString(key);
if (this.entries.has(keyStr)) {
this.entries.delete(keyStr);
const idx = this.sortedKeys.findIndex(k => this.comparator(k, key) === 0);
if (idx !== -1) {
this.sortedKeys.splice(idx, 1);
}
return true;
}
return false;
}
forRange(
lowKey: K | undefined,
highKey: K | undefined,
lowInclusive: boolean,
highInclusive: boolean,
callback: (value: V, key: K) => void
): void {
for (const key of this.sortedKeys) {
// Check low bound
if (lowKey !== undefined) {
const cmp = this.comparator(key, lowKey);
if (cmp < 0) continue;
if (cmp === 0 && !lowInclusive) continue;
}
// Check high bound
if (highKey !== undefined) {
const cmp = this.comparator(key, highKey);
if (cmp > 0) break;
if (cmp === 0 && !highInclusive) break;
}
const entry = this.entries.get(this.keyToString(key));
if (entry) {
callback(entry.value, key);
}
}
}
}
import type {
Document,
IStoredDocument,
@@ -11,7 +95,61 @@ import { TsmdbDuplicateKeyError, TsmdbIndexError } from '../errors/TsmdbErrors.j
import { QueryEngine } from './QueryEngine.js';
/**
* Index data structure for fast lookups
* Comparator for B-Tree that handles mixed types consistently
*/
function indexKeyComparator(a: any, b: any): number {
// Handle null/undefined
if (a === null || a === undefined) {
if (b === null || b === undefined) return 0;
return -1;
}
if (b === null || b === undefined) return 1;
// Handle arrays (compound keys)
if (Array.isArray(a) && Array.isArray(b)) {
for (let i = 0; i < Math.max(a.length, b.length); i++) {
const cmp = indexKeyComparator(a[i], b[i]);
if (cmp !== 0) return cmp;
}
return 0;
}
// Handle ObjectId
if (a instanceof plugins.bson.ObjectId && b instanceof plugins.bson.ObjectId) {
return a.toHexString().localeCompare(b.toHexString());
}
// Handle Date
if (a instanceof Date && b instanceof Date) {
return a.getTime() - b.getTime();
}
// Handle different types - use type ordering (null < number < string < object)
const typeOrder = (v: any): number => {
if (v === null || v === undefined) return 0;
if (typeof v === 'number') return 1;
if (typeof v === 'string') return 2;
if (typeof v === 'boolean') return 3;
if (v instanceof Date) return 4;
if (v instanceof plugins.bson.ObjectId) return 5;
return 6;
};
const typeA = typeOrder(a);
const typeB = typeOrder(b);
if (typeA !== typeB) return typeA - typeB;
// Same type comparison
if (typeof a === 'number') return a - b;
if (typeof a === 'string') return a.localeCompare(b);
if (typeof a === 'boolean') return (a ? 1 : 0) - (b ? 1 : 0);
// Fallback to string comparison
return String(a).localeCompare(String(b));
}
/**
* Index data structure using B-Tree for range queries
*/
interface IIndexData {
name: string;
@@ -19,8 +157,10 @@ interface IIndexData {
unique: boolean;
sparse: boolean;
expireAfterSeconds?: number;
// Map from index key value to document _id(s)
entries: Map<string, Set<string>>;
// B-Tree for ordered index lookups (supports range queries)
btree: SimpleBTree<any, Set<string>>;
// Hash map for fast equality lookups
hashMap: Map<string, Set<string>>;
}
/**
@@ -55,7 +195,8 @@ export class IndexEngine {
unique: indexSpec.unique || false,
sparse: indexSpec.sparse || false,
expireAfterSeconds: indexSpec.expireAfterSeconds,
entries: new Map(),
btree: new SimpleBTree<any, Set<string>>(undefined, indexKeyComparator),
hashMap: new Map(),
};
// Build index entries
@@ -63,10 +204,20 @@ export class IndexEngine {
const keyValue = this.extractKeyValue(doc, indexSpec.key);
if (keyValue !== null || !indexData.sparse) {
const keyStr = JSON.stringify(keyValue);
if (!indexData.entries.has(keyStr)) {
indexData.entries.set(keyStr, new Set());
// Add to hash map
if (!indexData.hashMap.has(keyStr)) {
indexData.hashMap.set(keyStr, new Set());
}
indexData.hashMap.get(keyStr)!.add(doc._id.toHexString());
// Add to B-tree
const existing = indexData.btree.get(keyValue);
if (existing) {
existing.add(doc._id.toHexString());
} else {
indexData.btree.set(keyValue, new Set([doc._id.toHexString()]));
}
indexData.entries.get(keyStr)!.add(doc._id.toHexString());
}
}
@@ -100,7 +251,8 @@ export class IndexEngine {
unique: options?.unique || false,
sparse: options?.sparse || false,
expireAfterSeconds: options?.expireAfterSeconds,
entries: new Map(),
btree: new SimpleBTree<any, Set<string>>(undefined, indexKeyComparator),
hashMap: new Map(),
};
// Build index from existing documents
@@ -115,7 +267,7 @@ export class IndexEngine {
const keyStr = JSON.stringify(keyValue);
if (indexData.unique && indexData.entries.has(keyStr)) {
if (indexData.unique && indexData.hashMap.has(keyStr)) {
throw new TsmdbDuplicateKeyError(
`E11000 duplicate key error index: ${this.dbName}.${this.collName}.$${name}`,
key as Record<string, 1>,
@@ -123,10 +275,19 @@ export class IndexEngine {
);
}
if (!indexData.entries.has(keyStr)) {
indexData.entries.set(keyStr, new Set());
// Add to hash map
if (!indexData.hashMap.has(keyStr)) {
indexData.hashMap.set(keyStr, new Set());
}
indexData.hashMap.get(keyStr)!.add(doc._id.toHexString());
// Add to B-tree
const existing = indexData.btree.get(keyValue);
if (existing) {
existing.add(doc._id.toHexString());
} else {
indexData.btree.set(keyValue, new Set([doc._id.toHexString()]));
}
indexData.entries.get(keyStr)!.add(doc._id.toHexString());
}
// Store index
@@ -213,7 +374,7 @@ export class IndexEngine {
// Check unique constraint
if (indexData.unique) {
const existing = indexData.entries.get(keyStr);
const existing = indexData.hashMap.get(keyStr);
if (existing && existing.size > 0) {
throw new TsmdbDuplicateKeyError(
`E11000 duplicate key error collection: ${this.dbName}.${this.collName} index: ${name}`,
@@ -223,10 +384,19 @@ export class IndexEngine {
}
}
if (!indexData.entries.has(keyStr)) {
indexData.entries.set(keyStr, new Set());
// Add to hash map
if (!indexData.hashMap.has(keyStr)) {
indexData.hashMap.set(keyStr, new Set());
}
indexData.hashMap.get(keyStr)!.add(doc._id.toHexString());
// Add to B-tree
const btreeSet = indexData.btree.get(keyValue);
if (btreeSet) {
btreeSet.add(doc._id.toHexString());
} else {
indexData.btree.set(keyValue, new Set([doc._id.toHexString()]));
}
indexData.entries.get(keyStr)!.add(doc._id.toHexString());
}
}
@@ -245,11 +415,21 @@ export class IndexEngine {
// Remove old entry if key changed
if (oldKeyStr !== newKeyStr) {
if (oldKeyValue !== null || !indexData.sparse) {
const oldSet = indexData.entries.get(oldKeyStr);
if (oldSet) {
oldSet.delete(oldDoc._id.toHexString());
if (oldSet.size === 0) {
indexData.entries.delete(oldKeyStr);
// Remove from hash map
const oldHashSet = indexData.hashMap.get(oldKeyStr);
if (oldHashSet) {
oldHashSet.delete(oldDoc._id.toHexString());
if (oldHashSet.size === 0) {
indexData.hashMap.delete(oldKeyStr);
}
}
// Remove from B-tree
const oldBtreeSet = indexData.btree.get(oldKeyValue);
if (oldBtreeSet) {
oldBtreeSet.delete(oldDoc._id.toHexString());
if (oldBtreeSet.size === 0) {
indexData.btree.delete(oldKeyValue);
}
}
}
@@ -258,7 +438,7 @@ export class IndexEngine {
if (newKeyValue !== null || !indexData.sparse) {
// Check unique constraint
if (indexData.unique) {
const existing = indexData.entries.get(newKeyStr);
const existing = indexData.hashMap.get(newKeyStr);
if (existing && existing.size > 0) {
throw new TsmdbDuplicateKeyError(
`E11000 duplicate key error collection: ${this.dbName}.${this.collName} index: ${name}`,
@@ -268,10 +448,19 @@ export class IndexEngine {
}
}
if (!indexData.entries.has(newKeyStr)) {
indexData.entries.set(newKeyStr, new Set());
// Add to hash map
if (!indexData.hashMap.has(newKeyStr)) {
indexData.hashMap.set(newKeyStr, new Set());
}
indexData.hashMap.get(newKeyStr)!.add(newDoc._id.toHexString());
// Add to B-tree
const newBtreeSet = indexData.btree.get(newKeyValue);
if (newBtreeSet) {
newBtreeSet.add(newDoc._id.toHexString());
} else {
indexData.btree.set(newKeyValue, new Set([newDoc._id.toHexString()]));
}
indexData.entries.get(newKeyStr)!.add(newDoc._id.toHexString());
}
}
}
@@ -291,11 +480,22 @@ export class IndexEngine {
}
const keyStr = JSON.stringify(keyValue);
const set = indexData.entries.get(keyStr);
if (set) {
set.delete(doc._id.toHexString());
if (set.size === 0) {
indexData.entries.delete(keyStr);
// Remove from hash map
const hashSet = indexData.hashMap.get(keyStr);
if (hashSet) {
hashSet.delete(doc._id.toHexString());
if (hashSet.size === 0) {
indexData.hashMap.delete(keyStr);
}
}
// Remove from B-tree
const btreeSet = indexData.btree.get(keyValue);
if (btreeSet) {
btreeSet.delete(doc._id.toHexString());
if (btreeSet.size === 0) {
indexData.btree.delete(keyValue);
}
}
}
@@ -309,8 +509,8 @@ export class IndexEngine {
return null;
}
// Get filter fields
const filterFields = new Set(this.getFilterFields(filter));
// Get filter fields and operators
const filterInfo = this.analyzeFilter(filter);
// Score each index
let bestIndex: { name: string; data: IIndexData } | null = null;
@@ -320,12 +520,21 @@ export class IndexEngine {
const indexFields = Object.keys(indexData.key);
let score = 0;
// Count how many index fields are in the filter
// Count how many index fields can be used
for (const field of indexFields) {
if (filterFields.has(field)) {
score++;
const info = filterInfo.get(field);
if (!info) break;
// Equality is best
if (info.equality) {
score += 2;
} else if (info.range) {
// Range queries can use B-tree
score += 1;
} else if (info.in) {
score += 1.5;
} else {
break; // Index fields must be contiguous
break;
}
}
@@ -344,7 +553,46 @@ export class IndexEngine {
}
/**
* Use index to find candidate document IDs
* Analyze filter to extract field operators
*/
private analyzeFilter(filter: Document): Map<string, { equality: boolean; range: boolean; in: boolean; ops: Record<string, any> }> {
const result = new Map<string, { equality: boolean; range: boolean; in: boolean; ops: Record<string, any> }>();
for (const [key, value] of Object.entries(filter)) {
if (key.startsWith('$')) continue;
const info = { equality: false, range: false, in: false, ops: {} as Record<string, any> };
if (typeof value !== 'object' || value === null || value instanceof plugins.bson.ObjectId || value instanceof Date) {
info.equality = true;
info.ops['$eq'] = value;
} else {
const ops = value as Record<string, any>;
if (ops.$eq !== undefined) {
info.equality = true;
info.ops['$eq'] = ops.$eq;
}
if (ops.$in !== undefined) {
info.in = true;
info.ops['$in'] = ops.$in;
}
if (ops.$gt !== undefined || ops.$gte !== undefined || ops.$lt !== undefined || ops.$lte !== undefined) {
info.range = true;
if (ops.$gt !== undefined) info.ops['$gt'] = ops.$gt;
if (ops.$gte !== undefined) info.ops['$gte'] = ops.$gte;
if (ops.$lt !== undefined) info.ops['$lt'] = ops.$lt;
if (ops.$lte !== undefined) info.ops['$lte'] = ops.$lte;
}
}
result.set(key, info);
}
return result;
}
/**
* Use index to find candidate document IDs (supports range queries with B-tree)
*/
async findCandidateIds(filter: Document): Promise<Set<string> | null> {
await this.initialize();
@@ -352,25 +600,58 @@ export class IndexEngine {
const index = this.selectIndex(filter);
if (!index) return null;
// Try to use the index for equality matches
const filterInfo = this.analyzeFilter(filter);
const indexFields = Object.keys(index.data.key);
const equalityValues: Record<string, any> = {};
for (const field of indexFields) {
const filterValue = this.getFilterValue(filter, field);
if (filterValue === undefined) break;
// For single-field indexes with range queries, use B-tree
if (indexFields.length === 1) {
const field = indexFields[0];
const info = filterInfo.get(field);
// Only use equality matches for index lookup
if (typeof filterValue === 'object' && filterValue !== null) {
if (filterValue.$eq !== undefined) {
equalityValues[field] = filterValue.$eq;
} else if (filterValue.$in !== undefined) {
if (info) {
// Handle equality using hash map (faster)
if (info.equality) {
const keyStr = JSON.stringify(info.ops['$eq']);
return index.data.hashMap.get(keyStr) || new Set();
}
// Handle $in using hash map
if (info.in) {
const results = new Set<string>();
for (const val of info.ops['$in']) {
const keyStr = JSON.stringify(val);
const ids = index.data.hashMap.get(keyStr);
if (ids) {
for (const id of ids) {
results.add(id);
}
}
}
return results;
}
// Handle range queries using B-tree
if (info.range) {
return this.findRangeCandidates(index.data, info.ops);
}
}
} else {
// For compound indexes, use hash map with partial key matching
const equalityValues: Record<string, any> = {};
for (const field of indexFields) {
const info = filterInfo.get(field);
if (!info) break;
if (info.equality) {
equalityValues[field] = info.ops['$eq'];
} else if (info.in) {
// Handle $in with multiple lookups
const results = new Set<string>();
for (const val of filterValue.$in) {
for (const val of info.ops['$in']) {
equalityValues[field] = val;
const keyStr = JSON.stringify(this.buildKeyValue(equalityValues, index.data.key));
const ids = index.data.entries.get(keyStr);
const ids = index.data.hashMap.get(keyStr);
if (ids) {
for (const id of ids) {
results.add(id);
@@ -379,19 +660,57 @@ export class IndexEngine {
}
return results;
} else {
break; // Non-equality operator, stop here
break; // Non-equality/in operator, stop here
}
} else {
equalityValues[field] = filterValue;
}
if (Object.keys(equalityValues).length > 0) {
const keyStr = JSON.stringify(this.buildKeyValue(equalityValues, index.data.key));
return index.data.hashMap.get(keyStr) || new Set();
}
}
if (Object.keys(equalityValues).length === 0) {
return null;
return null;
}
/**
* Find candidates using B-tree range scan
*/
private findRangeCandidates(indexData: IIndexData, ops: Record<string, any>): Set<string> {
const results = new Set<string>();
let lowKey: any = undefined;
let highKey: any = undefined;
let lowInclusive = true;
let highInclusive = true;
if (ops['$gt'] !== undefined) {
lowKey = ops['$gt'];
lowInclusive = false;
}
if (ops['$gte'] !== undefined) {
lowKey = ops['$gte'];
lowInclusive = true;
}
if (ops['$lt'] !== undefined) {
highKey = ops['$lt'];
highInclusive = false;
}
if (ops['$lte'] !== undefined) {
highKey = ops['$lte'];
highInclusive = true;
}
const keyStr = JSON.stringify(this.buildKeyValue(equalityValues, index.data.key));
return index.data.entries.get(keyStr) || new Set();
// Use B-tree range iteration
indexData.btree.forRange(lowKey, highKey, lowInclusive, highInclusive, (value, key) => {
if (value) {
for (const id of value) {
results.add(id);
}
}
});
return results;
}
// ============================================================================

393
ts/tsmdb/engine/QueryPlanner.ts Normal file
View File

@@ -0,0 +1,393 @@
import * as plugins from '../tsmdb.plugins.js';
import type { Document, IStoredDocument } from '../types/interfaces.js';
import { IndexEngine } from './IndexEngine.js';
/**
* Query execution plan types
*/
export type TQueryPlanType = 'IXSCAN' | 'COLLSCAN' | 'FETCH' | 'IXSCAN_RANGE';
/**
* Represents a query execution plan
*/
export interface IQueryPlan {
/** The type of scan used */
type: TQueryPlanType;
/** Index name if using an index */
indexName?: string;
/** Index key specification */
indexKey?: Record<string, 1 | -1 | string>;
/** Whether the query can be fully satisfied by the index */
indexCovering: boolean;
/** Estimated selectivity (0-1, lower is more selective) */
selectivity: number;
/** Whether range operators are used */
usesRange: boolean;
/** Fields used from the index */
indexFieldsUsed: string[];
/** Filter conditions that must be applied post-index lookup */
residualFilter?: Document;
/** Explanation for debugging */
explanation: string;
}
/**
* Filter operator analysis
*/
interface IFilterOperatorInfo {
field: string;
operators: string[];
equality: boolean;
range: boolean;
in: boolean;
exists: boolean;
regex: boolean;
values: Record<string, any>;
}
/**
* QueryPlanner - Analyzes queries and selects optimal execution plans
*/
export class QueryPlanner {
private indexEngine: IndexEngine;
constructor(indexEngine: IndexEngine) {
this.indexEngine = indexEngine;
}
/**
* Generate an execution plan for a query filter
*/
async plan(filter: Document): Promise<IQueryPlan> {
await this.indexEngine['initialize']();
// Empty filter = full collection scan
if (!filter || Object.keys(filter).length === 0) {
return {
type: 'COLLSCAN',
indexCovering: false,
selectivity: 1.0,
usesRange: false,
indexFieldsUsed: [],
explanation: 'No filter specified, full collection scan required',
};
}
// Analyze the filter
const operatorInfo = this.analyzeFilter(filter);
// Get available indexes
const indexes = await this.indexEngine.listIndexes();
// Score each index
let bestPlan: IQueryPlan | null = null;
let bestScore = -1;
for (const index of indexes) {
const plan = this.scoreIndex(index, operatorInfo, filter);
if (plan.selectivity < 1.0) {
const score = this.calculateScore(plan);
if (score > bestScore) {
bestScore = score;
bestPlan = plan;
}
}
}
// If no suitable index found, fall back to collection scan
if (!bestPlan || bestScore <= 0) {
return {
type: 'COLLSCAN',
indexCovering: false,
selectivity: 1.0,
usesRange: false,
indexFieldsUsed: [],
explanation: 'No suitable index found for this query',
};
}
return bestPlan;
}
/**
* Analyze filter to extract operator information per field
*/
private analyzeFilter(filter: Document, prefix = ''): Map<string, IFilterOperatorInfo> {
const result = new Map<string, IFilterOperatorInfo>();
for (const [key, value] of Object.entries(filter)) {
// Skip logical operators at the top level
if (key.startsWith('$')) {
if (key === '$and' && Array.isArray(value)) {
// Merge $and conditions
for (const subFilter of value) {
const subInfo = this.analyzeFilter(subFilter, prefix);
for (const [field, info] of subInfo) {
if (result.has(field)) {
// Merge operators
const existing = result.get(field)!;
existing.operators.push(...info.operators);
existing.equality = existing.equality || info.equality;
existing.range = existing.range || info.range;
existing.in = existing.in || info.in;
Object.assign(existing.values, info.values);
} else {
result.set(field, info);
}
}
}
}
continue;
}
const fullKey = prefix ? `${prefix}.${key}` : key;
const info: IFilterOperatorInfo = {
field: fullKey,
operators: [],
equality: false,
range: false,
in: false,
exists: false,
regex: false,
values: {},
};
if (typeof value !== 'object' || value === null || value instanceof plugins.bson.ObjectId || value instanceof Date) {
// Direct equality
info.equality = true;
info.operators.push('$eq');
info.values['$eq'] = value;
} else if (Array.isArray(value)) {
// Array equality (rare, but possible)
info.equality = true;
info.operators.push('$eq');
info.values['$eq'] = value;
} else {
// Operator object
for (const [op, opValue] of Object.entries(value)) {
if (op.startsWith('$')) {
info.operators.push(op);
info.values[op] = opValue;
switch (op) {
case '$eq':
info.equality = true;
break;
case '$ne':
case '$not':
// These can use indexes but with low selectivity
break;
case '$in':
info.in = true;
break;
case '$nin':
// Can't efficiently use indexes
break;
case '$gt':
case '$gte':
case '$lt':
case '$lte':
info.range = true;
break;
case '$exists':
info.exists = true;
break;
case '$regex':
info.regex = true;
break;
}
} else {
// Nested object - recurse
const nestedInfo = this.analyzeFilter({ [op]: opValue }, fullKey);
for (const [nestedField, nestedFieldInfo] of nestedInfo) {
result.set(nestedField, nestedFieldInfo);
}
}
}
}
if (info.operators.length > 0) {
result.set(fullKey, info);
}
}
return result;
}
/**
* Score an index for the given filter
*/
private scoreIndex(
index: { name: string; key: Record<string, any>; unique?: boolean; sparse?: boolean },
operatorInfo: Map<string, IFilterOperatorInfo>,
filter: Document
): IQueryPlan {
const indexFields = Object.keys(index.key);
const usedFields: string[] = [];
let usesRange = false;
let canUseIndex = true;
let selectivity = 1.0;
let residualFilter: Document | undefined;
// Check each index field in order
for (const field of indexFields) {
const info = operatorInfo.get(field);
if (!info) {
// Index field not in filter - stop here
break;
}
usedFields.push(field);
// Calculate selectivity based on operator
if (info.equality) {
// Equality has high selectivity
selectivity *= 0.01; // Assume 1% match
} else if (info.in) {
// $in selectivity depends on array size
const inValues = info.values['$in'];
if (Array.isArray(inValues)) {
selectivity *= Math.min(0.5, inValues.length * 0.01);
} else {
selectivity *= 0.1;
}
} else if (info.range) {
// Range queries have moderate selectivity
selectivity *= 0.25;
usesRange = true;
// After range, can't use more index fields efficiently
break;
} else if (info.exists) {
// $exists can use sparse indexes
selectivity *= 0.5;
} else {
// Other operators may not be indexable
canUseIndex = false;
break;
}
}
if (!canUseIndex || usedFields.length === 0) {
return {
type: 'COLLSCAN',
indexCovering: false,
selectivity: 1.0,
usesRange: false,
indexFieldsUsed: [],
explanation: `Index ${index.name} cannot be used for this query`,
};
}
// Build residual filter for conditions not covered by index
const coveredFields = new Set(usedFields);
const residualConditions: Record<string, any> = {};
for (const [field, info] of operatorInfo) {
if (!coveredFields.has(field)) {
// This field isn't covered by the index
if (info.equality) {
residualConditions[field] = info.values['$eq'];
} else {
residualConditions[field] = info.values;
}
}
}
if (Object.keys(residualConditions).length > 0) {
residualFilter = residualConditions;
}
// Unique indexes have better selectivity for equality
if (index.unique && usedFields.length === indexFields.length) {
selectivity = Math.min(selectivity, 0.001); // At most 1 document
}
return {
type: usesRange ? 'IXSCAN_RANGE' : 'IXSCAN',
indexName: index.name,
indexKey: index.key,
indexCovering: Object.keys(residualConditions).length === 0,
selectivity,
usesRange,
indexFieldsUsed: usedFields,
residualFilter,
explanation: `Using index ${index.name} on fields [${usedFields.join(', ')}]`,
};
}
/**
* Calculate overall score for a plan (higher is better)
*/
private calculateScore(plan: IQueryPlan): number {
let score = 0;
// Lower selectivity is better (fewer documents to fetch)
score += (1 - plan.selectivity) * 100;
// Index covering queries are best
if (plan.indexCovering) {
score += 50;
}
// More index fields used is better
score += plan.indexFieldsUsed.length * 10;
// Equality scans are better than range scans
if (!plan.usesRange) {
score += 20;
}
return score;
}
/**
* Explain a query - returns detailed plan information
*/
async explain(filter: Document): Promise<{
queryPlanner: {
plannerVersion: number;
namespace: string;
indexFilterSet: boolean;
winningPlan: IQueryPlan;
rejectedPlans: IQueryPlan[];
};
}> {
await this.indexEngine['initialize']();
// Analyze the filter
const operatorInfo = this.analyzeFilter(filter);
// Get available indexes
const indexes = await this.indexEngine.listIndexes();
// Score all indexes
const plans: IQueryPlan[] = [];
for (const index of indexes) {
const plan = this.scoreIndex(index, operatorInfo, filter);
plans.push(plan);
}
// Add collection scan as fallback
plans.push({
type: 'COLLSCAN',
indexCovering: false,
selectivity: 1.0,
usesRange: false,
indexFieldsUsed: [],
explanation: 'Full collection scan',
});
// Sort by score (best first)
plans.sort((a, b) => this.calculateScore(b) - this.calculateScore(a));
return {
queryPlanner: {
plannerVersion: 1,
namespace: `${this.indexEngine['dbName']}.${this.indexEngine['collName']}`,
indexFilterSet: false,
winningPlan: plans[0],
rejectedPlans: plans.slice(1),
},
};
}
}

292
ts/tsmdb/engine/SessionEngine.ts Normal file
View File

@@ -0,0 +1,292 @@
import * as plugins from '../tsmdb.plugins.js';
import type { TransactionEngine } from './TransactionEngine.js';
/**
* Session state
*/
export interface ISession {
/** Session ID (UUID) */
id: string;
/** Timestamp when the session was created */
createdAt: number;
/** Timestamp of the last activity */
lastActivityAt: number;
/** Current transaction ID if any */
txnId?: string;
/** Transaction number for ordering */
txnNumber?: number;
/** Whether the session is in a transaction */
inTransaction: boolean;
/** Session metadata */
metadata?: Record<string, any>;
}
/**
* Session engine options
*/
export interface ISessionEngineOptions {
/** Session timeout in milliseconds (default: 30 minutes) */
sessionTimeoutMs?: number;
/** Interval to check for expired sessions in ms (default: 60 seconds) */
cleanupIntervalMs?: number;
}
/**
* Session engine for managing client sessions
* - Tracks session lifecycle (create, touch, end)
* - Links sessions to transactions
* - Auto-aborts transactions on session expiry
*/
export class SessionEngine {
private sessions: Map<string, ISession> = new Map();
private sessionTimeoutMs: number;
private cleanupInterval?: ReturnType<typeof setInterval>;
private transactionEngine?: TransactionEngine;
constructor(options?: ISessionEngineOptions) {
this.sessionTimeoutMs = options?.sessionTimeoutMs ?? 30 * 60 * 1000; // 30 minutes default
const cleanupIntervalMs = options?.cleanupIntervalMs ?? 60 * 1000; // 1 minute default
// Start cleanup interval
this.cleanupInterval = setInterval(() => {
this.cleanupExpiredSessions();
}, cleanupIntervalMs);
}
/**
* Set the transaction engine to use for auto-abort
*/
setTransactionEngine(engine: TransactionEngine): void {
this.transactionEngine = engine;
}
/**
* Start a new session
*/
startSession(sessionId?: string, metadata?: Record<string, any>): ISession {
const id = sessionId ?? new plugins.bson.UUID().toHexString();
const now = Date.now();
const session: ISession = {
id,
createdAt: now,
lastActivityAt: now,
inTransaction: false,
metadata,
};
this.sessions.set(id, session);
return session;
}
/**
* Get a session by ID
*/
getSession(sessionId: string): ISession | undefined {
const session = this.sessions.get(sessionId);
if (session && this.isSessionExpired(session)) {
// Session expired, clean it up
this.endSession(sessionId);
return undefined;
}
return session;
}
/**
* Touch a session to update last activity time
*/
touchSession(sessionId: string): boolean {
const session = this.sessions.get(sessionId);
if (!session) return false;
if (this.isSessionExpired(session)) {
this.endSession(sessionId);
return false;
}
session.lastActivityAt = Date.now();
return true;
}
/**
* End a session explicitly
* This will also abort any active transaction
*/
async endSession(sessionId: string): Promise<boolean> {
const session = this.sessions.get(sessionId);
if (!session) return false;
// If session has an active transaction, abort it
if (session.inTransaction && session.txnId && this.transactionEngine) {
try {
await this.transactionEngine.abortTransaction(session.txnId);
} catch (e) {
// Ignore abort errors during cleanup
}
}
this.sessions.delete(sessionId);
return true;
}
/**
* Start a transaction in a session
*/
startTransaction(sessionId: string, txnId: string, txnNumber?: number): boolean {
const session = this.sessions.get(sessionId);
if (!session) return false;
if (this.isSessionExpired(session)) {
this.endSession(sessionId);
return false;
}
session.txnId = txnId;
session.txnNumber = txnNumber;
session.inTransaction = true;
session.lastActivityAt = Date.now();
return true;
}
/**
* End a transaction in a session (commit or abort)
*/
endTransaction(sessionId: string): boolean {
const session = this.sessions.get(sessionId);
if (!session) return false;
session.txnId = undefined;
session.txnNumber = undefined;
session.inTransaction = false;
session.lastActivityAt = Date.now();
return true;
}
/**
* Get transaction ID for a session
*/
getTransactionId(sessionId: string): string | undefined {
const session = this.sessions.get(sessionId);
return session?.txnId;
}
/**
* Check if session is in a transaction
*/
isInTransaction(sessionId: string): boolean {
const session = this.sessions.get(sessionId);
return session?.inTransaction ?? false;
}
/**
* Check if a session is expired
*/
isSessionExpired(session: ISession): boolean {
return Date.now() - session.lastActivityAt > this.sessionTimeoutMs;
}
/**
* Cleanup expired sessions
* This is called periodically by the cleanup interval
*/
private async cleanupExpiredSessions(): Promise<void> {
const expiredSessions: string[] = [];
for (const [id, session] of this.sessions) {
if (this.isSessionExpired(session)) {
expiredSessions.push(id);
}
}
// End all expired sessions (this will also abort their transactions)
for (const sessionId of expiredSessions) {
await this.endSession(sessionId);
}
}
/**
* Get all active sessions
*/
listSessions(): ISession[] {
const activeSessions: ISession[] = [];
for (const session of this.sessions.values()) {
if (!this.isSessionExpired(session)) {
activeSessions.push(session);
}
}
return activeSessions;
}
/**
* Get session count
*/
getSessionCount(): number {
return this.sessions.size;
}
/**
* Get sessions with active transactions
*/
getSessionsWithTransactions(): ISession[] {
return this.listSessions().filter(s => s.inTransaction);
}
/**
* Refresh session timeout
*/
refreshSession(sessionId: string): boolean {
return this.touchSession(sessionId);
}
/**
* Close the session engine and cleanup
*/
close(): void {
if (this.cleanupInterval) {
clearInterval(this.cleanupInterval);
this.cleanupInterval = undefined;
}
// Clear all sessions
this.sessions.clear();
}
/**
* Get or create a session for a given session ID
* Useful for handling MongoDB driver session requests
*/
getOrCreateSession(sessionId: string): ISession {
let session = this.getSession(sessionId);
if (!session) {
session = this.startSession(sessionId);
} else {
this.touchSession(sessionId);
}
return session;
}
/**
* Extract session ID from MongoDB lsid (logical session ID)
*/
static extractSessionId(lsid: any): string | undefined {
if (!lsid) return undefined;
// MongoDB session ID format: { id: UUID }
if (lsid.id) {
if (lsid.id instanceof plugins.bson.UUID) {
return lsid.id.toHexString();
}
if (typeof lsid.id === 'string') {
return lsid.id;
}
if (lsid.id.$binary?.base64) {
// Binary UUID format
return Buffer.from(lsid.id.$binary.base64, 'base64').toString('hex');
}
}
return undefined;
}
}