feat(tsmdb): implement TsmDB Mongo-wire-compatible server, add storage/engine modules and reorganize exports
This commit is contained in:
283
ts/ts_tsmdb/engine/AggregationEngine.ts
Normal file
283
ts/ts_tsmdb/engine/AggregationEngine.ts
Normal file
@@ -0,0 +1,283 @@
|
||||
import * as plugins from '../plugins.js';
|
||||
import type { Document, IStoredDocument, IAggregateOptions } from '../types/interfaces.js';
|
||||
|
||||
// Import mingo Aggregator
|
||||
import { Aggregator } from 'mingo';
|
||||
|
||||
/**
|
||||
* Aggregation engine using mingo for MongoDB-compatible aggregation pipeline execution
|
||||
*/
|
||||
export class AggregationEngine {
|
||||
/**
|
||||
* Execute an aggregation pipeline on a collection of documents
|
||||
*/
|
||||
static aggregate(
|
||||
documents: IStoredDocument[],
|
||||
pipeline: Document[],
|
||||
options?: IAggregateOptions
|
||||
): Document[] {
|
||||
if (!pipeline || pipeline.length === 0) {
|
||||
return documents;
|
||||
}
|
||||
|
||||
// Create mingo aggregator with the pipeline
|
||||
const aggregator = new Aggregator(pipeline, {
|
||||
collation: options?.collation as any,
|
||||
});
|
||||
|
||||
// Run the aggregation
|
||||
const result = aggregator.run(documents);
|
||||
|
||||
return Array.isArray(result) ? result : [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute aggregation and return an iterator for lazy evaluation
|
||||
*/
|
||||
static *aggregateIterator(
|
||||
documents: IStoredDocument[],
|
||||
pipeline: Document[],
|
||||
options?: IAggregateOptions
|
||||
): Generator<Document> {
|
||||
const aggregator = new Aggregator(pipeline, {
|
||||
collation: options?.collation as any,
|
||||
});
|
||||
|
||||
// Get the cursor from mingo
|
||||
const cursor = aggregator.stream(documents);
|
||||
|
||||
for (const doc of cursor) {
|
||||
yield doc;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a $lookup stage manually (for cross-collection lookups)
|
||||
* This is used when the lookup references another collection in the same database
|
||||
*/
|
||||
static executeLookup(
|
||||
documents: IStoredDocument[],
|
||||
lookupSpec: {
|
||||
from: string;
|
||||
localField: string;
|
||||
foreignField: string;
|
||||
as: string;
|
||||
},
|
||||
foreignCollection: IStoredDocument[]
|
||||
): Document[] {
|
||||
const { localField, foreignField, as } = lookupSpec;
|
||||
|
||||
return documents.map(doc => {
|
||||
const localValue = this.getNestedValue(doc, localField);
|
||||
const matches = foreignCollection.filter(foreignDoc => {
|
||||
const foreignValue = this.getNestedValue(foreignDoc, foreignField);
|
||||
return this.valuesMatch(localValue, foreignValue);
|
||||
});
|
||||
|
||||
return {
|
||||
...doc,
|
||||
[as]: matches,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a $graphLookup stage manually
|
||||
*/
|
||||
static executeGraphLookup(
|
||||
documents: IStoredDocument[],
|
||||
graphLookupSpec: {
|
||||
from: string;
|
||||
startWith: string | Document;
|
||||
connectFromField: string;
|
||||
connectToField: string;
|
||||
as: string;
|
||||
maxDepth?: number;
|
||||
depthField?: string;
|
||||
restrictSearchWithMatch?: Document;
|
||||
},
|
||||
foreignCollection: IStoredDocument[]
|
||||
): Document[] {
|
||||
const {
|
||||
startWith,
|
||||
connectFromField,
|
||||
connectToField,
|
||||
as,
|
||||
maxDepth = 10,
|
||||
depthField,
|
||||
restrictSearchWithMatch,
|
||||
} = graphLookupSpec;
|
||||
|
||||
return documents.map(doc => {
|
||||
const startValue = typeof startWith === 'string' && startWith.startsWith('$')
|
||||
? this.getNestedValue(doc, startWith.slice(1))
|
||||
: startWith;
|
||||
|
||||
const results: Document[] = [];
|
||||
const visited = new Set<string>();
|
||||
const queue: Array<{ value: any; depth: number }> = [];
|
||||
|
||||
// Initialize with start value(s)
|
||||
const startValues = Array.isArray(startValue) ? startValue : [startValue];
|
||||
for (const val of startValues) {
|
||||
queue.push({ value: val, depth: 0 });
|
||||
}
|
||||
|
||||
while (queue.length > 0) {
|
||||
const { value, depth } = queue.shift()!;
|
||||
if (depth > maxDepth) continue;
|
||||
|
||||
const valueKey = JSON.stringify(value);
|
||||
if (visited.has(valueKey)) continue;
|
||||
visited.add(valueKey);
|
||||
|
||||
// Find matching documents
|
||||
for (const foreignDoc of foreignCollection) {
|
||||
const foreignValue = this.getNestedValue(foreignDoc, connectToField);
|
||||
|
||||
if (this.valuesMatch(value, foreignValue)) {
|
||||
// Check restrictSearchWithMatch
|
||||
if (restrictSearchWithMatch) {
|
||||
const matchQuery = new plugins.mingo.Query(restrictSearchWithMatch);
|
||||
if (!matchQuery.test(foreignDoc)) continue;
|
||||
}
|
||||
|
||||
const resultDoc = depthField
|
||||
? { ...foreignDoc, [depthField]: depth }
|
||||
: { ...foreignDoc };
|
||||
|
||||
// Avoid duplicates in results
|
||||
const docKey = foreignDoc._id.toHexString();
|
||||
if (!results.some(r => r._id?.toHexString?.() === docKey)) {
|
||||
results.push(resultDoc);
|
||||
|
||||
// Add connected values to queue
|
||||
const nextValue = this.getNestedValue(foreignDoc, connectFromField);
|
||||
if (nextValue !== undefined) {
|
||||
const nextValues = Array.isArray(nextValue) ? nextValue : [nextValue];
|
||||
for (const nv of nextValues) {
|
||||
queue.push({ value: nv, depth: depth + 1 });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
...doc,
|
||||
[as]: results,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a $facet stage manually
|
||||
*/
|
||||
static executeFacet(
|
||||
documents: IStoredDocument[],
|
||||
facetSpec: Record<string, Document[]>
|
||||
): Document {
|
||||
const result: Document = {};
|
||||
|
||||
for (const [facetName, pipeline] of Object.entries(facetSpec)) {
|
||||
result[facetName] = this.aggregate(documents, pipeline);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a $unionWith stage
|
||||
*/
|
||||
static executeUnionWith(
|
||||
documents: IStoredDocument[],
|
||||
otherDocuments: IStoredDocument[],
|
||||
pipeline?: Document[]
|
||||
): Document[] {
|
||||
let unionDocs: Document[] = otherDocuments;
|
||||
if (pipeline && pipeline.length > 0) {
|
||||
unionDocs = this.aggregate(otherDocuments, pipeline);
|
||||
}
|
||||
return [...documents, ...unionDocs];
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a $merge stage (output to another collection)
|
||||
* Returns the documents that would be inserted/updated
|
||||
*/
|
||||
static prepareMerge(
|
||||
documents: Document[],
|
||||
mergeSpec: {
|
||||
into: string;
|
||||
on?: string | string[];
|
||||
whenMatched?: 'replace' | 'keepExisting' | 'merge' | 'fail' | Document[];
|
||||
whenNotMatched?: 'insert' | 'discard' | 'fail';
|
||||
}
|
||||
): {
|
||||
toInsert: Document[];
|
||||
toUpdate: Array<{ filter: Document; update: Document }>;
|
||||
onField: string | string[];
|
||||
whenMatched: string | Document[];
|
||||
whenNotMatched: string;
|
||||
} {
|
||||
const onField = mergeSpec.on || '_id';
|
||||
const whenMatched = mergeSpec.whenMatched || 'merge';
|
||||
const whenNotMatched = mergeSpec.whenNotMatched || 'insert';
|
||||
|
||||
return {
|
||||
toInsert: [],
|
||||
toUpdate: [],
|
||||
onField,
|
||||
whenMatched,
|
||||
whenNotMatched,
|
||||
};
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Helper Methods
|
||||
// ============================================================================
|
||||
|
||||
private static getNestedValue(obj: any, path: string): any {
|
||||
const parts = path.split('.');
|
||||
let current = obj;
|
||||
|
||||
for (const part of parts) {
|
||||
if (current === null || current === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
current = current[part];
|
||||
}
|
||||
|
||||
return current;
|
||||
}
|
||||
|
||||
private static valuesMatch(a: any, b: any): boolean {
|
||||
if (a === b) return true;
|
||||
|
||||
// Handle ObjectId comparison
|
||||
if (a instanceof plugins.bson.ObjectId && b instanceof plugins.bson.ObjectId) {
|
||||
return a.equals(b);
|
||||
}
|
||||
|
||||
// Handle array contains check
|
||||
if (Array.isArray(a)) {
|
||||
return a.some(item => this.valuesMatch(item, b));
|
||||
}
|
||||
if (Array.isArray(b)) {
|
||||
return b.some(item => this.valuesMatch(a, item));
|
||||
}
|
||||
|
||||
// Handle Date comparison
|
||||
if (a instanceof Date && b instanceof Date) {
|
||||
return a.getTime() === b.getTime();
|
||||
}
|
||||
|
||||
// Handle object comparison
|
||||
if (typeof a === 'object' && typeof b === 'object' && a !== null && b !== null) {
|
||||
return JSON.stringify(a) === JSON.stringify(b);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
798
ts/ts_tsmdb/engine/IndexEngine.ts
Normal file
798
ts/ts_tsmdb/engine/IndexEngine.ts
Normal file
@@ -0,0 +1,798 @@
|
||||
import * as plugins from '../plugins.js';
|
||||
import type { IStorageAdapter } from '../storage/IStorageAdapter.js';
|
||||
|
||||
// Simple B-Tree implementation for range queries
|
||||
// Since sorted-btree has ESM/CJS interop issues, we use a simple custom implementation
|
||||
class SimpleBTree<K, V> {
|
||||
private entries: Map<string, { key: K; value: V }> = new Map();
|
||||
private sortedKeys: K[] = [];
|
||||
private comparator: (a: K, b: K) => number;
|
||||
|
||||
constructor(_unused?: undefined, comparator?: (a: K, b: K) => number) {
|
||||
this.comparator = comparator || ((a: K, b: K) => {
|
||||
if (a < b) return -1;
|
||||
if (a > b) return 1;
|
||||
return 0;
|
||||
});
|
||||
}
|
||||
|
||||
private keyToString(key: K): string {
|
||||
return JSON.stringify(key);
|
||||
}
|
||||
|
||||
set(key: K, value: V): boolean {
|
||||
const keyStr = this.keyToString(key);
|
||||
const existed = this.entries.has(keyStr);
|
||||
this.entries.set(keyStr, { key, value });
|
||||
|
||||
if (!existed) {
|
||||
// Insert in sorted order
|
||||
const idx = this.sortedKeys.findIndex(k => this.comparator(k, key) > 0);
|
||||
if (idx === -1) {
|
||||
this.sortedKeys.push(key);
|
||||
} else {
|
||||
this.sortedKeys.splice(idx, 0, key);
|
||||
}
|
||||
}
|
||||
return !existed;
|
||||
}
|
||||
|
||||
get(key: K): V | undefined {
|
||||
const entry = this.entries.get(this.keyToString(key));
|
||||
return entry?.value;
|
||||
}
|
||||
|
||||
delete(key: K): boolean {
|
||||
const keyStr = this.keyToString(key);
|
||||
if (this.entries.has(keyStr)) {
|
||||
this.entries.delete(keyStr);
|
||||
const idx = this.sortedKeys.findIndex(k => this.comparator(k, key) === 0);
|
||||
if (idx !== -1) {
|
||||
this.sortedKeys.splice(idx, 1);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
forRange(
|
||||
lowKey: K | undefined,
|
||||
highKey: K | undefined,
|
||||
lowInclusive: boolean,
|
||||
highInclusive: boolean,
|
||||
callback: (value: V, key: K) => void
|
||||
): void {
|
||||
for (const key of this.sortedKeys) {
|
||||
// Check low bound
|
||||
if (lowKey !== undefined) {
|
||||
const cmp = this.comparator(key, lowKey);
|
||||
if (cmp < 0) continue;
|
||||
if (cmp === 0 && !lowInclusive) continue;
|
||||
}
|
||||
|
||||
// Check high bound
|
||||
if (highKey !== undefined) {
|
||||
const cmp = this.comparator(key, highKey);
|
||||
if (cmp > 0) break;
|
||||
if (cmp === 0 && !highInclusive) break;
|
||||
}
|
||||
|
||||
const entry = this.entries.get(this.keyToString(key));
|
||||
if (entry) {
|
||||
callback(entry.value, key);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
import type {
|
||||
Document,
|
||||
IStoredDocument,
|
||||
IIndexSpecification,
|
||||
IIndexInfo,
|
||||
ICreateIndexOptions,
|
||||
} from '../types/interfaces.js';
|
||||
import { TsmdbDuplicateKeyError, TsmdbIndexError } from '../errors/TsmdbErrors.js';
|
||||
import { QueryEngine } from './QueryEngine.js';
|
||||
|
||||
/**
|
||||
* Comparator for B-Tree that handles mixed types consistently
|
||||
*/
|
||||
function indexKeyComparator(a: any, b: any): number {
|
||||
// Handle null/undefined
|
||||
if (a === null || a === undefined) {
|
||||
if (b === null || b === undefined) return 0;
|
||||
return -1;
|
||||
}
|
||||
if (b === null || b === undefined) return 1;
|
||||
|
||||
// Handle arrays (compound keys)
|
||||
if (Array.isArray(a) && Array.isArray(b)) {
|
||||
for (let i = 0; i < Math.max(a.length, b.length); i++) {
|
||||
const cmp = indexKeyComparator(a[i], b[i]);
|
||||
if (cmp !== 0) return cmp;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Handle ObjectId
|
||||
if (a instanceof plugins.bson.ObjectId && b instanceof plugins.bson.ObjectId) {
|
||||
return a.toHexString().localeCompare(b.toHexString());
|
||||
}
|
||||
|
||||
// Handle Date
|
||||
if (a instanceof Date && b instanceof Date) {
|
||||
return a.getTime() - b.getTime();
|
||||
}
|
||||
|
||||
// Handle different types - use type ordering (null < number < string < object)
|
||||
const typeOrder = (v: any): number => {
|
||||
if (v === null || v === undefined) return 0;
|
||||
if (typeof v === 'number') return 1;
|
||||
if (typeof v === 'string') return 2;
|
||||
if (typeof v === 'boolean') return 3;
|
||||
if (v instanceof Date) return 4;
|
||||
if (v instanceof plugins.bson.ObjectId) return 5;
|
||||
return 6;
|
||||
};
|
||||
|
||||
const typeA = typeOrder(a);
|
||||
const typeB = typeOrder(b);
|
||||
if (typeA !== typeB) return typeA - typeB;
|
||||
|
||||
// Same type comparison
|
||||
if (typeof a === 'number') return a - b;
|
||||
if (typeof a === 'string') return a.localeCompare(b);
|
||||
if (typeof a === 'boolean') return (a ? 1 : 0) - (b ? 1 : 0);
|
||||
|
||||
// Fallback to string comparison
|
||||
return String(a).localeCompare(String(b));
|
||||
}
|
||||
|
||||
/**
|
||||
* Index data structure using B-Tree for range queries
|
||||
*/
|
||||
interface IIndexData {
|
||||
name: string;
|
||||
key: Record<string, 1 | -1 | string>;
|
||||
unique: boolean;
|
||||
sparse: boolean;
|
||||
expireAfterSeconds?: number;
|
||||
// B-Tree for ordered index lookups (supports range queries)
|
||||
btree: SimpleBTree<any, Set<string>>;
|
||||
// Hash map for fast equality lookups
|
||||
hashMap: Map<string, Set<string>>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Index engine for managing indexes and query optimization
|
||||
*/
|
||||
export class IndexEngine {
|
||||
private dbName: string;
|
||||
private collName: string;
|
||||
private storage: IStorageAdapter;
|
||||
private indexes: Map<string, IIndexData> = new Map();
|
||||
private initialized = false;
|
||||
|
||||
constructor(dbName: string, collName: string, storage: IStorageAdapter) {
|
||||
this.dbName = dbName;
|
||||
this.collName = collName;
|
||||
this.storage = storage;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize indexes from storage
|
||||
*/
|
||||
async initialize(): Promise<void> {
|
||||
if (this.initialized) return;
|
||||
|
||||
const storedIndexes = await this.storage.getIndexes(this.dbName, this.collName);
|
||||
const documents = await this.storage.findAll(this.dbName, this.collName);
|
||||
|
||||
for (const indexSpec of storedIndexes) {
|
||||
const indexData: IIndexData = {
|
||||
name: indexSpec.name,
|
||||
key: indexSpec.key,
|
||||
unique: indexSpec.unique || false,
|
||||
sparse: indexSpec.sparse || false,
|
||||
expireAfterSeconds: indexSpec.expireAfterSeconds,
|
||||
btree: new SimpleBTree<any, Set<string>>(undefined, indexKeyComparator),
|
||||
hashMap: new Map(),
|
||||
};
|
||||
|
||||
// Build index entries
|
||||
for (const doc of documents) {
|
||||
const keyValue = this.extractKeyValue(doc, indexSpec.key);
|
||||
if (keyValue !== null || !indexData.sparse) {
|
||||
const keyStr = JSON.stringify(keyValue);
|
||||
|
||||
// Add to hash map
|
||||
if (!indexData.hashMap.has(keyStr)) {
|
||||
indexData.hashMap.set(keyStr, new Set());
|
||||
}
|
||||
indexData.hashMap.get(keyStr)!.add(doc._id.toHexString());
|
||||
|
||||
// Add to B-tree
|
||||
const existing = indexData.btree.get(keyValue);
|
||||
if (existing) {
|
||||
existing.add(doc._id.toHexString());
|
||||
} else {
|
||||
indexData.btree.set(keyValue, new Set([doc._id.toHexString()]));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.indexes.set(indexSpec.name, indexData);
|
||||
}
|
||||
|
||||
this.initialized = true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new index
|
||||
*/
|
||||
async createIndex(
|
||||
key: Record<string, 1 | -1 | 'text' | '2dsphere'>,
|
||||
options?: ICreateIndexOptions
|
||||
): Promise<string> {
|
||||
await this.initialize();
|
||||
|
||||
// Generate index name if not provided
|
||||
const name = options?.name || this.generateIndexName(key);
|
||||
|
||||
// Check if index already exists
|
||||
if (this.indexes.has(name)) {
|
||||
return name;
|
||||
}
|
||||
|
||||
// Create index data structure
|
||||
const indexData: IIndexData = {
|
||||
name,
|
||||
key: key as Record<string, 1 | -1 | string>,
|
||||
unique: options?.unique || false,
|
||||
sparse: options?.sparse || false,
|
||||
expireAfterSeconds: options?.expireAfterSeconds,
|
||||
btree: new SimpleBTree<any, Set<string>>(undefined, indexKeyComparator),
|
||||
hashMap: new Map(),
|
||||
};
|
||||
|
||||
// Build index from existing documents
|
||||
const documents = await this.storage.findAll(this.dbName, this.collName);
|
||||
|
||||
for (const doc of documents) {
|
||||
const keyValue = this.extractKeyValue(doc, key);
|
||||
|
||||
if (keyValue === null && indexData.sparse) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const keyStr = JSON.stringify(keyValue);
|
||||
|
||||
if (indexData.unique && indexData.hashMap.has(keyStr)) {
|
||||
throw new TsmdbDuplicateKeyError(
|
||||
`E11000 duplicate key error index: ${this.dbName}.${this.collName}.$${name}`,
|
||||
key as Record<string, 1>,
|
||||
keyValue
|
||||
);
|
||||
}
|
||||
|
||||
// Add to hash map
|
||||
if (!indexData.hashMap.has(keyStr)) {
|
||||
indexData.hashMap.set(keyStr, new Set());
|
||||
}
|
||||
indexData.hashMap.get(keyStr)!.add(doc._id.toHexString());
|
||||
|
||||
// Add to B-tree
|
||||
const existing = indexData.btree.get(keyValue);
|
||||
if (existing) {
|
||||
existing.add(doc._id.toHexString());
|
||||
} else {
|
||||
indexData.btree.set(keyValue, new Set([doc._id.toHexString()]));
|
||||
}
|
||||
}
|
||||
|
||||
// Store index
|
||||
this.indexes.set(name, indexData);
|
||||
await this.storage.saveIndex(this.dbName, this.collName, name, {
|
||||
key,
|
||||
unique: options?.unique,
|
||||
sparse: options?.sparse,
|
||||
expireAfterSeconds: options?.expireAfterSeconds,
|
||||
});
|
||||
|
||||
return name;
|
||||
}
|
||||
|
||||
/**
|
||||
* Drop an index
|
||||
*/
|
||||
async dropIndex(name: string): Promise<void> {
|
||||
await this.initialize();
|
||||
|
||||
if (name === '_id_') {
|
||||
throw new TsmdbIndexError('cannot drop _id index');
|
||||
}
|
||||
|
||||
if (!this.indexes.has(name)) {
|
||||
throw new TsmdbIndexError(`index not found: ${name}`);
|
||||
}
|
||||
|
||||
this.indexes.delete(name);
|
||||
await this.storage.dropIndex(this.dbName, this.collName, name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Drop all indexes except _id
|
||||
*/
|
||||
async dropAllIndexes(): Promise<void> {
|
||||
await this.initialize();
|
||||
|
||||
const names = Array.from(this.indexes.keys()).filter(n => n !== '_id_');
|
||||
for (const name of names) {
|
||||
this.indexes.delete(name);
|
||||
await this.storage.dropIndex(this.dbName, this.collName, name);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* List all indexes
|
||||
*/
|
||||
async listIndexes(): Promise<IIndexInfo[]> {
|
||||
await this.initialize();
|
||||
|
||||
return Array.from(this.indexes.values()).map(idx => ({
|
||||
v: 2,
|
||||
key: idx.key,
|
||||
name: idx.name,
|
||||
unique: idx.unique || undefined,
|
||||
sparse: idx.sparse || undefined,
|
||||
expireAfterSeconds: idx.expireAfterSeconds,
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if an index exists
|
||||
*/
|
||||
async indexExists(name: string): Promise<boolean> {
|
||||
await this.initialize();
|
||||
return this.indexes.has(name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Update index entries after document insert
|
||||
*/
|
||||
async onInsert(doc: IStoredDocument): Promise<void> {
|
||||
await this.initialize();
|
||||
|
||||
for (const [name, indexData] of this.indexes) {
|
||||
const keyValue = this.extractKeyValue(doc, indexData.key);
|
||||
|
||||
if (keyValue === null && indexData.sparse) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const keyStr = JSON.stringify(keyValue);
|
||||
|
||||
// Check unique constraint
|
||||
if (indexData.unique) {
|
||||
const existing = indexData.hashMap.get(keyStr);
|
||||
if (existing && existing.size > 0) {
|
||||
throw new TsmdbDuplicateKeyError(
|
||||
`E11000 duplicate key error collection: ${this.dbName}.${this.collName} index: ${name}`,
|
||||
indexData.key as Record<string, 1>,
|
||||
keyValue
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Add to hash map
|
||||
if (!indexData.hashMap.has(keyStr)) {
|
||||
indexData.hashMap.set(keyStr, new Set());
|
||||
}
|
||||
indexData.hashMap.get(keyStr)!.add(doc._id.toHexString());
|
||||
|
||||
// Add to B-tree
|
||||
const btreeSet = indexData.btree.get(keyValue);
|
||||
if (btreeSet) {
|
||||
btreeSet.add(doc._id.toHexString());
|
||||
} else {
|
||||
indexData.btree.set(keyValue, new Set([doc._id.toHexString()]));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update index entries after document update
|
||||
*/
|
||||
async onUpdate(oldDoc: IStoredDocument, newDoc: IStoredDocument): Promise<void> {
|
||||
await this.initialize();
|
||||
|
||||
for (const [name, indexData] of this.indexes) {
|
||||
const oldKeyValue = this.extractKeyValue(oldDoc, indexData.key);
|
||||
const newKeyValue = this.extractKeyValue(newDoc, indexData.key);
|
||||
const oldKeyStr = JSON.stringify(oldKeyValue);
|
||||
const newKeyStr = JSON.stringify(newKeyValue);
|
||||
|
||||
// Remove old entry if key changed
|
||||
if (oldKeyStr !== newKeyStr) {
|
||||
if (oldKeyValue !== null || !indexData.sparse) {
|
||||
// Remove from hash map
|
||||
const oldHashSet = indexData.hashMap.get(oldKeyStr);
|
||||
if (oldHashSet) {
|
||||
oldHashSet.delete(oldDoc._id.toHexString());
|
||||
if (oldHashSet.size === 0) {
|
||||
indexData.hashMap.delete(oldKeyStr);
|
||||
}
|
||||
}
|
||||
|
||||
// Remove from B-tree
|
||||
const oldBtreeSet = indexData.btree.get(oldKeyValue);
|
||||
if (oldBtreeSet) {
|
||||
oldBtreeSet.delete(oldDoc._id.toHexString());
|
||||
if (oldBtreeSet.size === 0) {
|
||||
indexData.btree.delete(oldKeyValue);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Add new entry
|
||||
if (newKeyValue !== null || !indexData.sparse) {
|
||||
// Check unique constraint
|
||||
if (indexData.unique) {
|
||||
const existing = indexData.hashMap.get(newKeyStr);
|
||||
if (existing && existing.size > 0) {
|
||||
throw new TsmdbDuplicateKeyError(
|
||||
`E11000 duplicate key error collection: ${this.dbName}.${this.collName} index: ${name}`,
|
||||
indexData.key as Record<string, 1>,
|
||||
newKeyValue
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Add to hash map
|
||||
if (!indexData.hashMap.has(newKeyStr)) {
|
||||
indexData.hashMap.set(newKeyStr, new Set());
|
||||
}
|
||||
indexData.hashMap.get(newKeyStr)!.add(newDoc._id.toHexString());
|
||||
|
||||
// Add to B-tree
|
||||
const newBtreeSet = indexData.btree.get(newKeyValue);
|
||||
if (newBtreeSet) {
|
||||
newBtreeSet.add(newDoc._id.toHexString());
|
||||
} else {
|
||||
indexData.btree.set(newKeyValue, new Set([newDoc._id.toHexString()]));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update index entries after document delete
|
||||
*/
|
||||
async onDelete(doc: IStoredDocument): Promise<void> {
|
||||
await this.initialize();
|
||||
|
||||
for (const indexData of this.indexes.values()) {
|
||||
const keyValue = this.extractKeyValue(doc, indexData.key);
|
||||
|
||||
if (keyValue === null && indexData.sparse) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const keyStr = JSON.stringify(keyValue);
|
||||
|
||||
// Remove from hash map
|
||||
const hashSet = indexData.hashMap.get(keyStr);
|
||||
if (hashSet) {
|
||||
hashSet.delete(doc._id.toHexString());
|
||||
if (hashSet.size === 0) {
|
||||
indexData.hashMap.delete(keyStr);
|
||||
}
|
||||
}
|
||||
|
||||
// Remove from B-tree
|
||||
const btreeSet = indexData.btree.get(keyValue);
|
||||
if (btreeSet) {
|
||||
btreeSet.delete(doc._id.toHexString());
|
||||
if (btreeSet.size === 0) {
|
||||
indexData.btree.delete(keyValue);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the best index for a query
|
||||
*/
|
||||
selectIndex(filter: Document): { name: string; data: IIndexData } | null {
|
||||
if (!filter || Object.keys(filter).length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Get filter fields and operators
|
||||
const filterInfo = this.analyzeFilter(filter);
|
||||
|
||||
// Score each index
|
||||
let bestIndex: { name: string; data: IIndexData } | null = null;
|
||||
let bestScore = 0;
|
||||
|
||||
for (const [name, indexData] of this.indexes) {
|
||||
const indexFields = Object.keys(indexData.key);
|
||||
let score = 0;
|
||||
|
||||
// Count how many index fields can be used
|
||||
for (const field of indexFields) {
|
||||
const info = filterInfo.get(field);
|
||||
if (!info) break;
|
||||
|
||||
// Equality is best
|
||||
if (info.equality) {
|
||||
score += 2;
|
||||
} else if (info.range) {
|
||||
// Range queries can use B-tree
|
||||
score += 1;
|
||||
} else if (info.in) {
|
||||
score += 1.5;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Prefer unique indexes
|
||||
if (indexData.unique && score > 0) {
|
||||
score += 0.5;
|
||||
}
|
||||
|
||||
if (score > bestScore) {
|
||||
bestScore = score;
|
||||
bestIndex = { name, data: indexData };
|
||||
}
|
||||
}
|
||||
|
||||
return bestIndex;
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyze filter to extract field operators
|
||||
*/
|
||||
private analyzeFilter(filter: Document): Map<string, { equality: boolean; range: boolean; in: boolean; ops: Record<string, any> }> {
|
||||
const result = new Map<string, { equality: boolean; range: boolean; in: boolean; ops: Record<string, any> }>();
|
||||
|
||||
for (const [key, value] of Object.entries(filter)) {
|
||||
if (key.startsWith('$')) continue;
|
||||
|
||||
const info = { equality: false, range: false, in: false, ops: {} as Record<string, any> };
|
||||
|
||||
if (typeof value !== 'object' || value === null || value instanceof plugins.bson.ObjectId || value instanceof Date) {
|
||||
info.equality = true;
|
||||
info.ops['$eq'] = value;
|
||||
} else {
|
||||
const ops = value as Record<string, any>;
|
||||
if (ops.$eq !== undefined) {
|
||||
info.equality = true;
|
||||
info.ops['$eq'] = ops.$eq;
|
||||
}
|
||||
if (ops.$in !== undefined) {
|
||||
info.in = true;
|
||||
info.ops['$in'] = ops.$in;
|
||||
}
|
||||
if (ops.$gt !== undefined || ops.$gte !== undefined || ops.$lt !== undefined || ops.$lte !== undefined) {
|
||||
info.range = true;
|
||||
if (ops.$gt !== undefined) info.ops['$gt'] = ops.$gt;
|
||||
if (ops.$gte !== undefined) info.ops['$gte'] = ops.$gte;
|
||||
if (ops.$lt !== undefined) info.ops['$lt'] = ops.$lt;
|
||||
if (ops.$lte !== undefined) info.ops['$lte'] = ops.$lte;
|
||||
}
|
||||
}
|
||||
|
||||
result.set(key, info);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Use index to find candidate document IDs (supports range queries with B-tree)
|
||||
*/
|
||||
async findCandidateIds(filter: Document): Promise<Set<string> | null> {
|
||||
await this.initialize();
|
||||
|
||||
const index = this.selectIndex(filter);
|
||||
if (!index) return null;
|
||||
|
||||
const filterInfo = this.analyzeFilter(filter);
|
||||
const indexFields = Object.keys(index.data.key);
|
||||
|
||||
// For single-field indexes with range queries, use B-tree
|
||||
if (indexFields.length === 1) {
|
||||
const field = indexFields[0];
|
||||
const info = filterInfo.get(field);
|
||||
|
||||
if (info) {
|
||||
// Handle equality using hash map (faster)
|
||||
if (info.equality) {
|
||||
const keyStr = JSON.stringify(info.ops['$eq']);
|
||||
return index.data.hashMap.get(keyStr) || new Set();
|
||||
}
|
||||
|
||||
// Handle $in using hash map
|
||||
if (info.in) {
|
||||
const results = new Set<string>();
|
||||
for (const val of info.ops['$in']) {
|
||||
const keyStr = JSON.stringify(val);
|
||||
const ids = index.data.hashMap.get(keyStr);
|
||||
if (ids) {
|
||||
for (const id of ids) {
|
||||
results.add(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
// Handle range queries using B-tree
|
||||
if (info.range) {
|
||||
return this.findRangeCandidates(index.data, info.ops);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// For compound indexes, use hash map with partial key matching
|
||||
const equalityValues: Record<string, any> = {};
|
||||
|
||||
for (const field of indexFields) {
|
||||
const info = filterInfo.get(field);
|
||||
if (!info) break;
|
||||
|
||||
if (info.equality) {
|
||||
equalityValues[field] = info.ops['$eq'];
|
||||
} else if (info.in) {
|
||||
// Handle $in with multiple lookups
|
||||
const results = new Set<string>();
|
||||
for (const val of info.ops['$in']) {
|
||||
equalityValues[field] = val;
|
||||
const keyStr = JSON.stringify(this.buildKeyValue(equalityValues, index.data.key));
|
||||
const ids = index.data.hashMap.get(keyStr);
|
||||
if (ids) {
|
||||
for (const id of ids) {
|
||||
results.add(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
return results;
|
||||
} else {
|
||||
break; // Non-equality/in operator, stop here
|
||||
}
|
||||
}
|
||||
|
||||
if (Object.keys(equalityValues).length > 0) {
|
||||
const keyStr = JSON.stringify(this.buildKeyValue(equalityValues, index.data.key));
|
||||
return index.data.hashMap.get(keyStr) || new Set();
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find candidates using B-tree range scan
|
||||
*/
|
||||
private findRangeCandidates(indexData: IIndexData, ops: Record<string, any>): Set<string> {
|
||||
const results = new Set<string>();
|
||||
|
||||
let lowKey: any = undefined;
|
||||
let highKey: any = undefined;
|
||||
let lowInclusive = true;
|
||||
let highInclusive = true;
|
||||
|
||||
if (ops['$gt'] !== undefined) {
|
||||
lowKey = ops['$gt'];
|
||||
lowInclusive = false;
|
||||
}
|
||||
if (ops['$gte'] !== undefined) {
|
||||
lowKey = ops['$gte'];
|
||||
lowInclusive = true;
|
||||
}
|
||||
if (ops['$lt'] !== undefined) {
|
||||
highKey = ops['$lt'];
|
||||
highInclusive = false;
|
||||
}
|
||||
if (ops['$lte'] !== undefined) {
|
||||
highKey = ops['$lte'];
|
||||
highInclusive = true;
|
||||
}
|
||||
|
||||
// Use B-tree range iteration
|
||||
indexData.btree.forRange(lowKey, highKey, lowInclusive, highInclusive, (value, key) => {
|
||||
if (value) {
|
||||
for (const id of value) {
|
||||
results.add(id);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Helper Methods
|
||||
// ============================================================================
|
||||
|
||||
private generateIndexName(key: Record<string, any>): string {
|
||||
return Object.entries(key)
|
||||
.map(([field, dir]) => `${field}_${dir}`)
|
||||
.join('_');
|
||||
}
|
||||
|
||||
private extractKeyValue(doc: Document, key: Record<string, any>): any {
|
||||
const values: any[] = [];
|
||||
|
||||
for (const field of Object.keys(key)) {
|
||||
const value = QueryEngine.getNestedValue(doc, field);
|
||||
values.push(value === undefined ? null : value);
|
||||
}
|
||||
|
||||
// For single-field index, return the value directly
|
||||
if (values.length === 1) {
|
||||
return values[0];
|
||||
}
|
||||
|
||||
return values;
|
||||
}
|
||||
|
||||
private buildKeyValue(values: Record<string, any>, key: Record<string, any>): any {
|
||||
const result: any[] = [];
|
||||
|
||||
for (const field of Object.keys(key)) {
|
||||
result.push(values[field] !== undefined ? values[field] : null);
|
||||
}
|
||||
|
||||
if (result.length === 1) {
|
||||
return result[0];
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private getFilterFields(filter: Document, prefix = ''): string[] {
|
||||
const fields: string[] = [];
|
||||
|
||||
for (const [key, value] of Object.entries(filter)) {
|
||||
if (key.startsWith('$')) {
|
||||
// Logical operator
|
||||
if (key === '$and' || key === '$or' || key === '$nor') {
|
||||
for (const subFilter of value as Document[]) {
|
||||
fields.push(...this.getFilterFields(subFilter, prefix));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const fullKey = prefix ? `${prefix}.${key}` : key;
|
||||
fields.push(fullKey);
|
||||
|
||||
// Check for nested filters
|
||||
if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
|
||||
const subKeys = Object.keys(value);
|
||||
if (subKeys.length > 0 && !subKeys[0].startsWith('$')) {
|
||||
fields.push(...this.getFilterFields(value, fullKey));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return fields;
|
||||
}
|
||||
|
||||
private getFilterValue(filter: Document, field: string): any {
|
||||
// Handle dot notation
|
||||
const parts = field.split('.');
|
||||
let current: any = filter;
|
||||
|
||||
for (const part of parts) {
|
||||
if (current === null || current === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
current = current[part];
|
||||
}
|
||||
|
||||
return current;
|
||||
}
|
||||
}
|
||||
301
ts/ts_tsmdb/engine/QueryEngine.ts
Normal file
301
ts/ts_tsmdb/engine/QueryEngine.ts
Normal file
@@ -0,0 +1,301 @@
|
||||
import * as plugins from '../plugins.js';
|
||||
import type { Document, IStoredDocument, ISortSpecification, ISortDirection } from '../types/interfaces.js';
|
||||
|
||||
// Import mingo Query class
|
||||
import { Query } from 'mingo';
|
||||
|
||||
/**
|
||||
* Query engine using mingo for MongoDB-compatible query matching
|
||||
*/
|
||||
export class QueryEngine {
|
||||
/**
|
||||
* Filter documents by a MongoDB query filter
|
||||
*/
|
||||
static filter(documents: IStoredDocument[], filter: Document): IStoredDocument[] {
|
||||
if (!filter || Object.keys(filter).length === 0) {
|
||||
return documents;
|
||||
}
|
||||
|
||||
const query = new Query(filter);
|
||||
return documents.filter(doc => query.test(doc));
|
||||
}
|
||||
|
||||
/**
|
||||
* Test if a single document matches a filter
|
||||
*/
|
||||
static matches(document: Document, filter: Document): boolean {
|
||||
if (!filter || Object.keys(filter).length === 0) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const query = new Query(filter);
|
||||
return query.test(document);
|
||||
}
|
||||
|
||||
/**
|
||||
* Find a single document matching the filter
|
||||
*/
|
||||
static findOne(documents: IStoredDocument[], filter: Document): IStoredDocument | null {
|
||||
if (!filter || Object.keys(filter).length === 0) {
|
||||
return documents[0] || null;
|
||||
}
|
||||
|
||||
const query = new Query(filter);
|
||||
for (const doc of documents) {
|
||||
if (query.test(doc)) {
|
||||
return doc;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sort documents by a sort specification
|
||||
*/
|
||||
static sort(documents: IStoredDocument[], sort: ISortSpecification): IStoredDocument[] {
|
||||
if (!sort) {
|
||||
return documents;
|
||||
}
|
||||
|
||||
// Normalize sort specification to array of [field, direction] pairs
|
||||
const sortFields: Array<[string, number]> = [];
|
||||
|
||||
if (Array.isArray(sort)) {
|
||||
for (const [field, direction] of sort) {
|
||||
sortFields.push([field, this.normalizeDirection(direction)]);
|
||||
}
|
||||
} else {
|
||||
for (const [field, direction] of Object.entries(sort)) {
|
||||
sortFields.push([field, this.normalizeDirection(direction)]);
|
||||
}
|
||||
}
|
||||
|
||||
return [...documents].sort((a, b) => {
|
||||
for (const [field, direction] of sortFields) {
|
||||
const aVal = this.getNestedValue(a, field);
|
||||
const bVal = this.getNestedValue(b, field);
|
||||
|
||||
const comparison = this.compareValues(aVal, bVal);
|
||||
if (comparison !== 0) {
|
||||
return comparison * direction;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply projection to documents
|
||||
*/
|
||||
static project(documents: IStoredDocument[], projection: Document): Document[] {
|
||||
if (!projection || Object.keys(projection).length === 0) {
|
||||
return documents;
|
||||
}
|
||||
|
||||
// Determine if this is inclusion or exclusion projection
|
||||
const keys = Object.keys(projection);
|
||||
const hasInclusion = keys.some(k => k !== '_id' && projection[k] === 1);
|
||||
const hasExclusion = keys.some(k => k !== '_id' && projection[k] === 0);
|
||||
|
||||
// Can't mix inclusion and exclusion (except for _id)
|
||||
if (hasInclusion && hasExclusion) {
|
||||
throw new Error('Cannot mix inclusion and exclusion in projection');
|
||||
}
|
||||
|
||||
return documents.map(doc => {
|
||||
if (hasInclusion) {
|
||||
// Inclusion projection
|
||||
const result: Document = {};
|
||||
|
||||
// Handle _id
|
||||
if (projection._id !== 0 && projection._id !== false) {
|
||||
result._id = doc._id;
|
||||
}
|
||||
|
||||
for (const key of keys) {
|
||||
if (key === '_id') continue;
|
||||
if (projection[key] === 1 || projection[key] === true) {
|
||||
const value = this.getNestedValue(doc, key);
|
||||
if (value !== undefined) {
|
||||
this.setNestedValue(result, key, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
} else {
|
||||
// Exclusion projection - start with copy and remove fields
|
||||
const result = { ...doc };
|
||||
|
||||
for (const key of keys) {
|
||||
if (projection[key] === 0 || projection[key] === false) {
|
||||
this.deleteNestedValue(result, key);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get distinct values for a field
|
||||
*/
|
||||
static distinct(documents: IStoredDocument[], field: string, filter?: Document): any[] {
|
||||
let docs = documents;
|
||||
if (filter && Object.keys(filter).length > 0) {
|
||||
docs = this.filter(documents, filter);
|
||||
}
|
||||
|
||||
const values = new Set<any>();
|
||||
for (const doc of docs) {
|
||||
const value = this.getNestedValue(doc, field);
|
||||
if (value !== undefined) {
|
||||
if (Array.isArray(value)) {
|
||||
// For arrays, add each element
|
||||
for (const v of value) {
|
||||
values.add(this.toComparable(v));
|
||||
}
|
||||
} else {
|
||||
values.add(this.toComparable(value));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return Array.from(values);
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize sort direction to 1 or -1
|
||||
*/
|
||||
private static normalizeDirection(direction: ISortDirection): number {
|
||||
if (typeof direction === 'number') {
|
||||
return direction > 0 ? 1 : -1;
|
||||
}
|
||||
if (direction === 'asc' || direction === 'ascending') {
|
||||
return 1;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a nested value from an object using dot notation
|
||||
*/
|
||||
static getNestedValue(obj: any, path: string): any {
|
||||
const parts = path.split('.');
|
||||
let current = obj;
|
||||
|
||||
for (const part of parts) {
|
||||
if (current === null || current === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
if (Array.isArray(current)) {
|
||||
// Handle array access
|
||||
const index = parseInt(part, 10);
|
||||
if (!isNaN(index)) {
|
||||
current = current[index];
|
||||
} else {
|
||||
// Get the field from all array elements
|
||||
return current.map(item => this.getNestedValue(item, part)).flat();
|
||||
}
|
||||
} else {
|
||||
current = current[part];
|
||||
}
|
||||
}
|
||||
|
||||
return current;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set a nested value in an object using dot notation
|
||||
*/
|
||||
private static setNestedValue(obj: any, path: string, value: any): void {
|
||||
const parts = path.split('.');
|
||||
let current = obj;
|
||||
|
||||
for (let i = 0; i < parts.length - 1; i++) {
|
||||
const part = parts[i];
|
||||
if (!(part in current)) {
|
||||
current[part] = {};
|
||||
}
|
||||
current = current[part];
|
||||
}
|
||||
|
||||
current[parts[parts.length - 1]] = value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a nested value from an object using dot notation
|
||||
*/
|
||||
private static deleteNestedValue(obj: any, path: string): void {
|
||||
const parts = path.split('.');
|
||||
let current = obj;
|
||||
|
||||
for (let i = 0; i < parts.length - 1; i++) {
|
||||
const part = parts[i];
|
||||
if (!(part in current)) {
|
||||
return;
|
||||
}
|
||||
current = current[part];
|
||||
}
|
||||
|
||||
delete current[parts[parts.length - 1]];
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare two values for sorting
|
||||
*/
|
||||
private static compareValues(a: any, b: any): number {
|
||||
// Handle undefined/null
|
||||
if (a === undefined && b === undefined) return 0;
|
||||
if (a === undefined) return -1;
|
||||
if (b === undefined) return 1;
|
||||
if (a === null && b === null) return 0;
|
||||
if (a === null) return -1;
|
||||
if (b === null) return 1;
|
||||
|
||||
// Handle ObjectId
|
||||
if (a instanceof plugins.bson.ObjectId && b instanceof plugins.bson.ObjectId) {
|
||||
return a.toHexString().localeCompare(b.toHexString());
|
||||
}
|
||||
|
||||
// Handle dates
|
||||
if (a instanceof Date && b instanceof Date) {
|
||||
return a.getTime() - b.getTime();
|
||||
}
|
||||
|
||||
// Handle numbers
|
||||
if (typeof a === 'number' && typeof b === 'number') {
|
||||
return a - b;
|
||||
}
|
||||
|
||||
// Handle strings
|
||||
if (typeof a === 'string' && typeof b === 'string') {
|
||||
return a.localeCompare(b);
|
||||
}
|
||||
|
||||
// Handle booleans
|
||||
if (typeof a === 'boolean' && typeof b === 'boolean') {
|
||||
return (a ? 1 : 0) - (b ? 1 : 0);
|
||||
}
|
||||
|
||||
// Fall back to string comparison
|
||||
return String(a).localeCompare(String(b));
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a value to a comparable form (for distinct)
|
||||
*/
|
||||
private static toComparable(value: any): any {
|
||||
if (value instanceof plugins.bson.ObjectId) {
|
||||
return value.toHexString();
|
||||
}
|
||||
if (value instanceof Date) {
|
||||
return value.toISOString();
|
||||
}
|
||||
if (typeof value === 'object' && value !== null) {
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
}
|
||||
393
ts/ts_tsmdb/engine/QueryPlanner.ts
Normal file
393
ts/ts_tsmdb/engine/QueryPlanner.ts
Normal file
@@ -0,0 +1,393 @@
|
||||
import * as plugins from '../plugins.js';
|
||||
import type { Document, IStoredDocument } from '../types/interfaces.js';
|
||||
import { IndexEngine } from './IndexEngine.js';
|
||||
|
||||
/**
|
||||
* Query execution plan types
|
||||
*/
|
||||
export type TQueryPlanType = 'IXSCAN' | 'COLLSCAN' | 'FETCH' | 'IXSCAN_RANGE';
|
||||
|
||||
/**
|
||||
* Represents a query execution plan
|
||||
*/
|
||||
export interface IQueryPlan {
|
||||
/** The type of scan used */
|
||||
type: TQueryPlanType;
|
||||
/** Index name if using an index */
|
||||
indexName?: string;
|
||||
/** Index key specification */
|
||||
indexKey?: Record<string, 1 | -1 | string>;
|
||||
/** Whether the query can be fully satisfied by the index */
|
||||
indexCovering: boolean;
|
||||
/** Estimated selectivity (0-1, lower is more selective) */
|
||||
selectivity: number;
|
||||
/** Whether range operators are used */
|
||||
usesRange: boolean;
|
||||
/** Fields used from the index */
|
||||
indexFieldsUsed: string[];
|
||||
/** Filter conditions that must be applied post-index lookup */
|
||||
residualFilter?: Document;
|
||||
/** Explanation for debugging */
|
||||
explanation: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter operator analysis
|
||||
*/
|
||||
interface IFilterOperatorInfo {
|
||||
field: string;
|
||||
operators: string[];
|
||||
equality: boolean;
|
||||
range: boolean;
|
||||
in: boolean;
|
||||
exists: boolean;
|
||||
regex: boolean;
|
||||
values: Record<string, any>;
|
||||
}
|
||||
|
||||
/**
|
||||
* QueryPlanner - Analyzes queries and selects optimal execution plans
|
||||
*/
|
||||
export class QueryPlanner {
|
||||
private indexEngine: IndexEngine;
|
||||
|
||||
constructor(indexEngine: IndexEngine) {
|
||||
this.indexEngine = indexEngine;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate an execution plan for a query filter
|
||||
*/
|
||||
async plan(filter: Document): Promise<IQueryPlan> {
|
||||
await this.indexEngine['initialize']();
|
||||
|
||||
// Empty filter = full collection scan
|
||||
if (!filter || Object.keys(filter).length === 0) {
|
||||
return {
|
||||
type: 'COLLSCAN',
|
||||
indexCovering: false,
|
||||
selectivity: 1.0,
|
||||
usesRange: false,
|
||||
indexFieldsUsed: [],
|
||||
explanation: 'No filter specified, full collection scan required',
|
||||
};
|
||||
}
|
||||
|
||||
// Analyze the filter
|
||||
const operatorInfo = this.analyzeFilter(filter);
|
||||
|
||||
// Get available indexes
|
||||
const indexes = await this.indexEngine.listIndexes();
|
||||
|
||||
// Score each index
|
||||
let bestPlan: IQueryPlan | null = null;
|
||||
let bestScore = -1;
|
||||
|
||||
for (const index of indexes) {
|
||||
const plan = this.scoreIndex(index, operatorInfo, filter);
|
||||
if (plan.selectivity < 1.0) {
|
||||
const score = this.calculateScore(plan);
|
||||
if (score > bestScore) {
|
||||
bestScore = score;
|
||||
bestPlan = plan;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If no suitable index found, fall back to collection scan
|
||||
if (!bestPlan || bestScore <= 0) {
|
||||
return {
|
||||
type: 'COLLSCAN',
|
||||
indexCovering: false,
|
||||
selectivity: 1.0,
|
||||
usesRange: false,
|
||||
indexFieldsUsed: [],
|
||||
explanation: 'No suitable index found for this query',
|
||||
};
|
||||
}
|
||||
|
||||
return bestPlan;
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyze filter to extract operator information per field
|
||||
*/
|
||||
private analyzeFilter(filter: Document, prefix = ''): Map<string, IFilterOperatorInfo> {
|
||||
const result = new Map<string, IFilterOperatorInfo>();
|
||||
|
||||
for (const [key, value] of Object.entries(filter)) {
|
||||
// Skip logical operators at the top level
|
||||
if (key.startsWith('$')) {
|
||||
if (key === '$and' && Array.isArray(value)) {
|
||||
// Merge $and conditions
|
||||
for (const subFilter of value) {
|
||||
const subInfo = this.analyzeFilter(subFilter, prefix);
|
||||
for (const [field, info] of subInfo) {
|
||||
if (result.has(field)) {
|
||||
// Merge operators
|
||||
const existing = result.get(field)!;
|
||||
existing.operators.push(...info.operators);
|
||||
existing.equality = existing.equality || info.equality;
|
||||
existing.range = existing.range || info.range;
|
||||
existing.in = existing.in || info.in;
|
||||
Object.assign(existing.values, info.values);
|
||||
} else {
|
||||
result.set(field, info);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const fullKey = prefix ? `${prefix}.${key}` : key;
|
||||
const info: IFilterOperatorInfo = {
|
||||
field: fullKey,
|
||||
operators: [],
|
||||
equality: false,
|
||||
range: false,
|
||||
in: false,
|
||||
exists: false,
|
||||
regex: false,
|
||||
values: {},
|
||||
};
|
||||
|
||||
if (typeof value !== 'object' || value === null || value instanceof plugins.bson.ObjectId || value instanceof Date) {
|
||||
// Direct equality
|
||||
info.equality = true;
|
||||
info.operators.push('$eq');
|
||||
info.values['$eq'] = value;
|
||||
} else if (Array.isArray(value)) {
|
||||
// Array equality (rare, but possible)
|
||||
info.equality = true;
|
||||
info.operators.push('$eq');
|
||||
info.values['$eq'] = value;
|
||||
} else {
|
||||
// Operator object
|
||||
for (const [op, opValue] of Object.entries(value)) {
|
||||
if (op.startsWith('$')) {
|
||||
info.operators.push(op);
|
||||
info.values[op] = opValue;
|
||||
|
||||
switch (op) {
|
||||
case '$eq':
|
||||
info.equality = true;
|
||||
break;
|
||||
case '$ne':
|
||||
case '$not':
|
||||
// These can use indexes but with low selectivity
|
||||
break;
|
||||
case '$in':
|
||||
info.in = true;
|
||||
break;
|
||||
case '$nin':
|
||||
// Can't efficiently use indexes
|
||||
break;
|
||||
case '$gt':
|
||||
case '$gte':
|
||||
case '$lt':
|
||||
case '$lte':
|
||||
info.range = true;
|
||||
break;
|
||||
case '$exists':
|
||||
info.exists = true;
|
||||
break;
|
||||
case '$regex':
|
||||
info.regex = true;
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
// Nested object - recurse
|
||||
const nestedInfo = this.analyzeFilter({ [op]: opValue }, fullKey);
|
||||
for (const [nestedField, nestedFieldInfo] of nestedInfo) {
|
||||
result.set(nestedField, nestedFieldInfo);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (info.operators.length > 0) {
|
||||
result.set(fullKey, info);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Score an index for the given filter
|
||||
*/
|
||||
private scoreIndex(
|
||||
index: { name: string; key: Record<string, any>; unique?: boolean; sparse?: boolean },
|
||||
operatorInfo: Map<string, IFilterOperatorInfo>,
|
||||
filter: Document
|
||||
): IQueryPlan {
|
||||
const indexFields = Object.keys(index.key);
|
||||
const usedFields: string[] = [];
|
||||
let usesRange = false;
|
||||
let canUseIndex = true;
|
||||
let selectivity = 1.0;
|
||||
let residualFilter: Document | undefined;
|
||||
|
||||
// Check each index field in order
|
||||
for (const field of indexFields) {
|
||||
const info = operatorInfo.get(field);
|
||||
if (!info) {
|
||||
// Index field not in filter - stop here
|
||||
break;
|
||||
}
|
||||
|
||||
usedFields.push(field);
|
||||
|
||||
// Calculate selectivity based on operator
|
||||
if (info.equality) {
|
||||
// Equality has high selectivity
|
||||
selectivity *= 0.01; // Assume 1% match
|
||||
} else if (info.in) {
|
||||
// $in selectivity depends on array size
|
||||
const inValues = info.values['$in'];
|
||||
if (Array.isArray(inValues)) {
|
||||
selectivity *= Math.min(0.5, inValues.length * 0.01);
|
||||
} else {
|
||||
selectivity *= 0.1;
|
||||
}
|
||||
} else if (info.range) {
|
||||
// Range queries have moderate selectivity
|
||||
selectivity *= 0.25;
|
||||
usesRange = true;
|
||||
// After range, can't use more index fields efficiently
|
||||
break;
|
||||
} else if (info.exists) {
|
||||
// $exists can use sparse indexes
|
||||
selectivity *= 0.5;
|
||||
} else {
|
||||
// Other operators may not be indexable
|
||||
canUseIndex = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!canUseIndex || usedFields.length === 0) {
|
||||
return {
|
||||
type: 'COLLSCAN',
|
||||
indexCovering: false,
|
||||
selectivity: 1.0,
|
||||
usesRange: false,
|
||||
indexFieldsUsed: [],
|
||||
explanation: `Index ${index.name} cannot be used for this query`,
|
||||
};
|
||||
}
|
||||
|
||||
// Build residual filter for conditions not covered by index
|
||||
const coveredFields = new Set(usedFields);
|
||||
const residualConditions: Record<string, any> = {};
|
||||
for (const [field, info] of operatorInfo) {
|
||||
if (!coveredFields.has(field)) {
|
||||
// This field isn't covered by the index
|
||||
if (info.equality) {
|
||||
residualConditions[field] = info.values['$eq'];
|
||||
} else {
|
||||
residualConditions[field] = info.values;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (Object.keys(residualConditions).length > 0) {
|
||||
residualFilter = residualConditions;
|
||||
}
|
||||
|
||||
// Unique indexes have better selectivity for equality
|
||||
if (index.unique && usedFields.length === indexFields.length) {
|
||||
selectivity = Math.min(selectivity, 0.001); // At most 1 document
|
||||
}
|
||||
|
||||
return {
|
||||
type: usesRange ? 'IXSCAN_RANGE' : 'IXSCAN',
|
||||
indexName: index.name,
|
||||
indexKey: index.key,
|
||||
indexCovering: Object.keys(residualConditions).length === 0,
|
||||
selectivity,
|
||||
usesRange,
|
||||
indexFieldsUsed: usedFields,
|
||||
residualFilter,
|
||||
explanation: `Using index ${index.name} on fields [${usedFields.join(', ')}]`,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate overall score for a plan (higher is better)
|
||||
*/
|
||||
private calculateScore(plan: IQueryPlan): number {
|
||||
let score = 0;
|
||||
|
||||
// Lower selectivity is better (fewer documents to fetch)
|
||||
score += (1 - plan.selectivity) * 100;
|
||||
|
||||
// Index covering queries are best
|
||||
if (plan.indexCovering) {
|
||||
score += 50;
|
||||
}
|
||||
|
||||
// More index fields used is better
|
||||
score += plan.indexFieldsUsed.length * 10;
|
||||
|
||||
// Equality scans are better than range scans
|
||||
if (!plan.usesRange) {
|
||||
score += 20;
|
||||
}
|
||||
|
||||
return score;
|
||||
}
|
||||
|
||||
/**
|
||||
* Explain a query - returns detailed plan information
|
||||
*/
|
||||
async explain(filter: Document): Promise<{
|
||||
queryPlanner: {
|
||||
plannerVersion: number;
|
||||
namespace: string;
|
||||
indexFilterSet: boolean;
|
||||
winningPlan: IQueryPlan;
|
||||
rejectedPlans: IQueryPlan[];
|
||||
};
|
||||
}> {
|
||||
await this.indexEngine['initialize']();
|
||||
|
||||
// Analyze the filter
|
||||
const operatorInfo = this.analyzeFilter(filter);
|
||||
|
||||
// Get available indexes
|
||||
const indexes = await this.indexEngine.listIndexes();
|
||||
|
||||
// Score all indexes
|
||||
const plans: IQueryPlan[] = [];
|
||||
|
||||
for (const index of indexes) {
|
||||
const plan = this.scoreIndex(index, operatorInfo, filter);
|
||||
plans.push(plan);
|
||||
}
|
||||
|
||||
// Add collection scan as fallback
|
||||
plans.push({
|
||||
type: 'COLLSCAN',
|
||||
indexCovering: false,
|
||||
selectivity: 1.0,
|
||||
usesRange: false,
|
||||
indexFieldsUsed: [],
|
||||
explanation: 'Full collection scan',
|
||||
});
|
||||
|
||||
// Sort by score (best first)
|
||||
plans.sort((a, b) => this.calculateScore(b) - this.calculateScore(a));
|
||||
|
||||
return {
|
||||
queryPlanner: {
|
||||
plannerVersion: 1,
|
||||
namespace: `${this.indexEngine['dbName']}.${this.indexEngine['collName']}`,
|
||||
indexFilterSet: false,
|
||||
winningPlan: plans[0],
|
||||
rejectedPlans: plans.slice(1),
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
292
ts/ts_tsmdb/engine/SessionEngine.ts
Normal file
292
ts/ts_tsmdb/engine/SessionEngine.ts
Normal file
@@ -0,0 +1,292 @@
|
||||
import * as plugins from '../plugins.js';
|
||||
import type { TransactionEngine } from './TransactionEngine.js';
|
||||
|
||||
/**
|
||||
* Session state
|
||||
*/
|
||||
export interface ISession {
|
||||
/** Session ID (UUID) */
|
||||
id: string;
|
||||
/** Timestamp when the session was created */
|
||||
createdAt: number;
|
||||
/** Timestamp of the last activity */
|
||||
lastActivityAt: number;
|
||||
/** Current transaction ID if any */
|
||||
txnId?: string;
|
||||
/** Transaction number for ordering */
|
||||
txnNumber?: number;
|
||||
/** Whether the session is in a transaction */
|
||||
inTransaction: boolean;
|
||||
/** Session metadata */
|
||||
metadata?: Record<string, any>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Session engine options
|
||||
*/
|
||||
export interface ISessionEngineOptions {
|
||||
/** Session timeout in milliseconds (default: 30 minutes) */
|
||||
sessionTimeoutMs?: number;
|
||||
/** Interval to check for expired sessions in ms (default: 60 seconds) */
|
||||
cleanupIntervalMs?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Session engine for managing client sessions
|
||||
* - Tracks session lifecycle (create, touch, end)
|
||||
* - Links sessions to transactions
|
||||
* - Auto-aborts transactions on session expiry
|
||||
*/
|
||||
export class SessionEngine {
|
||||
private sessions: Map<string, ISession> = new Map();
|
||||
private sessionTimeoutMs: number;
|
||||
private cleanupInterval?: ReturnType<typeof setInterval>;
|
||||
private transactionEngine?: TransactionEngine;
|
||||
|
||||
constructor(options?: ISessionEngineOptions) {
|
||||
this.sessionTimeoutMs = options?.sessionTimeoutMs ?? 30 * 60 * 1000; // 30 minutes default
|
||||
const cleanupIntervalMs = options?.cleanupIntervalMs ?? 60 * 1000; // 1 minute default
|
||||
|
||||
// Start cleanup interval
|
||||
this.cleanupInterval = setInterval(() => {
|
||||
this.cleanupExpiredSessions();
|
||||
}, cleanupIntervalMs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the transaction engine to use for auto-abort
|
||||
*/
|
||||
setTransactionEngine(engine: TransactionEngine): void {
|
||||
this.transactionEngine = engine;
|
||||
}
|
||||
|
||||
/**
|
||||
* Start a new session
|
||||
*/
|
||||
startSession(sessionId?: string, metadata?: Record<string, any>): ISession {
|
||||
const id = sessionId ?? new plugins.bson.UUID().toHexString();
|
||||
const now = Date.now();
|
||||
|
||||
const session: ISession = {
|
||||
id,
|
||||
createdAt: now,
|
||||
lastActivityAt: now,
|
||||
inTransaction: false,
|
||||
metadata,
|
||||
};
|
||||
|
||||
this.sessions.set(id, session);
|
||||
return session;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a session by ID
|
||||
*/
|
||||
getSession(sessionId: string): ISession | undefined {
|
||||
const session = this.sessions.get(sessionId);
|
||||
if (session && this.isSessionExpired(session)) {
|
||||
// Session expired, clean it up
|
||||
this.endSession(sessionId);
|
||||
return undefined;
|
||||
}
|
||||
return session;
|
||||
}
|
||||
|
||||
/**
|
||||
* Touch a session to update last activity time
|
||||
*/
|
||||
touchSession(sessionId: string): boolean {
|
||||
const session = this.sessions.get(sessionId);
|
||||
if (!session) return false;
|
||||
|
||||
if (this.isSessionExpired(session)) {
|
||||
this.endSession(sessionId);
|
||||
return false;
|
||||
}
|
||||
|
||||
session.lastActivityAt = Date.now();
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* End a session explicitly
|
||||
* This will also abort any active transaction
|
||||
*/
|
||||
async endSession(sessionId: string): Promise<boolean> {
|
||||
const session = this.sessions.get(sessionId);
|
||||
if (!session) return false;
|
||||
|
||||
// If session has an active transaction, abort it
|
||||
if (session.inTransaction && session.txnId && this.transactionEngine) {
|
||||
try {
|
||||
await this.transactionEngine.abortTransaction(session.txnId);
|
||||
} catch (e) {
|
||||
// Ignore abort errors during cleanup
|
||||
}
|
||||
}
|
||||
|
||||
this.sessions.delete(sessionId);
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Start a transaction in a session
|
||||
*/
|
||||
startTransaction(sessionId: string, txnId: string, txnNumber?: number): boolean {
|
||||
const session = this.sessions.get(sessionId);
|
||||
if (!session) return false;
|
||||
|
||||
if (this.isSessionExpired(session)) {
|
||||
this.endSession(sessionId);
|
||||
return false;
|
||||
}
|
||||
|
||||
session.txnId = txnId;
|
||||
session.txnNumber = txnNumber;
|
||||
session.inTransaction = true;
|
||||
session.lastActivityAt = Date.now();
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* End a transaction in a session (commit or abort)
|
||||
*/
|
||||
endTransaction(sessionId: string): boolean {
|
||||
const session = this.sessions.get(sessionId);
|
||||
if (!session) return false;
|
||||
|
||||
session.txnId = undefined;
|
||||
session.txnNumber = undefined;
|
||||
session.inTransaction = false;
|
||||
session.lastActivityAt = Date.now();
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get transaction ID for a session
|
||||
*/
|
||||
getTransactionId(sessionId: string): string | undefined {
|
||||
const session = this.sessions.get(sessionId);
|
||||
return session?.txnId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if session is in a transaction
|
||||
*/
|
||||
isInTransaction(sessionId: string): boolean {
|
||||
const session = this.sessions.get(sessionId);
|
||||
return session?.inTransaction ?? false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a session is expired
|
||||
*/
|
||||
isSessionExpired(session: ISession): boolean {
|
||||
return Date.now() - session.lastActivityAt > this.sessionTimeoutMs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleanup expired sessions
|
||||
* This is called periodically by the cleanup interval
|
||||
*/
|
||||
private async cleanupExpiredSessions(): Promise<void> {
|
||||
const expiredSessions: string[] = [];
|
||||
|
||||
for (const [id, session] of this.sessions) {
|
||||
if (this.isSessionExpired(session)) {
|
||||
expiredSessions.push(id);
|
||||
}
|
||||
}
|
||||
|
||||
// End all expired sessions (this will also abort their transactions)
|
||||
for (const sessionId of expiredSessions) {
|
||||
await this.endSession(sessionId);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all active sessions
|
||||
*/
|
||||
listSessions(): ISession[] {
|
||||
const activeSessions: ISession[] = [];
|
||||
for (const session of this.sessions.values()) {
|
||||
if (!this.isSessionExpired(session)) {
|
||||
activeSessions.push(session);
|
||||
}
|
||||
}
|
||||
return activeSessions;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get session count
|
||||
*/
|
||||
getSessionCount(): number {
|
||||
return this.sessions.size;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get sessions with active transactions
|
||||
*/
|
||||
getSessionsWithTransactions(): ISession[] {
|
||||
return this.listSessions().filter(s => s.inTransaction);
|
||||
}
|
||||
|
||||
/**
|
||||
* Refresh session timeout
|
||||
*/
|
||||
refreshSession(sessionId: string): boolean {
|
||||
return this.touchSession(sessionId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Close the session engine and cleanup
|
||||
*/
|
||||
close(): void {
|
||||
if (this.cleanupInterval) {
|
||||
clearInterval(this.cleanupInterval);
|
||||
this.cleanupInterval = undefined;
|
||||
}
|
||||
|
||||
// Clear all sessions
|
||||
this.sessions.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get or create a session for a given session ID
|
||||
* Useful for handling MongoDB driver session requests
|
||||
*/
|
||||
getOrCreateSession(sessionId: string): ISession {
|
||||
let session = this.getSession(sessionId);
|
||||
if (!session) {
|
||||
session = this.startSession(sessionId);
|
||||
} else {
|
||||
this.touchSession(sessionId);
|
||||
}
|
||||
return session;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract session ID from MongoDB lsid (logical session ID)
|
||||
*/
|
||||
static extractSessionId(lsid: any): string | undefined {
|
||||
if (!lsid) return undefined;
|
||||
|
||||
// MongoDB session ID format: { id: UUID }
|
||||
if (lsid.id) {
|
||||
if (lsid.id instanceof plugins.bson.UUID) {
|
||||
return lsid.id.toHexString();
|
||||
}
|
||||
if (typeof lsid.id === 'string') {
|
||||
return lsid.id;
|
||||
}
|
||||
if (lsid.id.$binary?.base64) {
|
||||
// Binary UUID format
|
||||
return Buffer.from(lsid.id.$binary.base64, 'base64').toString('hex');
|
||||
}
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
351
ts/ts_tsmdb/engine/TransactionEngine.ts
Normal file
351
ts/ts_tsmdb/engine/TransactionEngine.ts
Normal file
@@ -0,0 +1,351 @@
|
||||
import * as plugins from '../plugins.js';
|
||||
import type { IStorageAdapter } from '../storage/IStorageAdapter.js';
|
||||
import type { Document, IStoredDocument, ITransactionOptions } from '../types/interfaces.js';
|
||||
import { TsmdbTransactionError, TsmdbWriteConflictError } from '../errors/TsmdbErrors.js';
|
||||
|
||||
/**
|
||||
* Transaction state
|
||||
*/
|
||||
export interface ITransactionState {
|
||||
id: string;
|
||||
sessionId: string;
|
||||
startTime: plugins.bson.Timestamp;
|
||||
status: 'active' | 'committed' | 'aborted';
|
||||
readSet: Map<string, Set<string>>; // ns -> document _ids read
|
||||
writeSet: Map<string, Map<string, { op: 'insert' | 'update' | 'delete'; doc?: IStoredDocument; originalDoc?: IStoredDocument }>>; // ns -> _id -> operation
|
||||
snapshots: Map<string, IStoredDocument[]>; // ns -> snapshot of documents
|
||||
}
|
||||
|
||||
/**
|
||||
* Transaction engine for ACID transaction support
|
||||
*/
|
||||
export class TransactionEngine {
|
||||
private storage: IStorageAdapter;
|
||||
private transactions: Map<string, ITransactionState> = new Map();
|
||||
private txnCounter = 0;
|
||||
|
||||
constructor(storage: IStorageAdapter) {
|
||||
this.storage = storage;
|
||||
}
|
||||
|
||||
/**
|
||||
* Start a new transaction
|
||||
*/
|
||||
startTransaction(sessionId: string, options?: ITransactionOptions): string {
|
||||
this.txnCounter++;
|
||||
const txnId = `txn_${sessionId}_${this.txnCounter}`;
|
||||
|
||||
const transaction: ITransactionState = {
|
||||
id: txnId,
|
||||
sessionId,
|
||||
startTime: new plugins.bson.Timestamp({ t: Math.floor(Date.now() / 1000), i: this.txnCounter }),
|
||||
status: 'active',
|
||||
readSet: new Map(),
|
||||
writeSet: new Map(),
|
||||
snapshots: new Map(),
|
||||
};
|
||||
|
||||
this.transactions.set(txnId, transaction);
|
||||
return txnId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a transaction by ID
|
||||
*/
|
||||
getTransaction(txnId: string): ITransactionState | undefined {
|
||||
return this.transactions.get(txnId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a transaction is active
|
||||
*/
|
||||
isActive(txnId: string): boolean {
|
||||
const txn = this.transactions.get(txnId);
|
||||
return txn?.status === 'active';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get or create a snapshot for a namespace
|
||||
*/
|
||||
async getSnapshot(txnId: string, dbName: string, collName: string): Promise<IStoredDocument[]> {
|
||||
const txn = this.transactions.get(txnId);
|
||||
if (!txn || txn.status !== 'active') {
|
||||
throw new TsmdbTransactionError('Transaction is not active');
|
||||
}
|
||||
|
||||
const ns = `${dbName}.${collName}`;
|
||||
if (!txn.snapshots.has(ns)) {
|
||||
const snapshot = await this.storage.createSnapshot(dbName, collName);
|
||||
txn.snapshots.set(ns, snapshot);
|
||||
}
|
||||
|
||||
// Apply transaction writes to snapshot
|
||||
const snapshot = txn.snapshots.get(ns)!;
|
||||
const writes = txn.writeSet.get(ns);
|
||||
|
||||
if (!writes) {
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
// Create a modified view of the snapshot
|
||||
const result: IStoredDocument[] = [];
|
||||
const deletedIds = new Set<string>();
|
||||
const modifiedDocs = new Map<string, IStoredDocument>();
|
||||
|
||||
for (const [idStr, write] of writes) {
|
||||
if (write.op === 'delete') {
|
||||
deletedIds.add(idStr);
|
||||
} else if (write.op === 'update' || write.op === 'insert') {
|
||||
if (write.doc) {
|
||||
modifiedDocs.set(idStr, write.doc);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Add existing documents (not deleted, possibly modified)
|
||||
for (const doc of snapshot) {
|
||||
const idStr = doc._id.toHexString();
|
||||
if (deletedIds.has(idStr)) {
|
||||
continue;
|
||||
}
|
||||
if (modifiedDocs.has(idStr)) {
|
||||
result.push(modifiedDocs.get(idStr)!);
|
||||
modifiedDocs.delete(idStr);
|
||||
} else {
|
||||
result.push(doc);
|
||||
}
|
||||
}
|
||||
|
||||
// Add new documents (inserts)
|
||||
for (const doc of modifiedDocs.values()) {
|
||||
result.push(doc);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Record a read operation
|
||||
*/
|
||||
recordRead(txnId: string, dbName: string, collName: string, docIds: string[]): void {
|
||||
const txn = this.transactions.get(txnId);
|
||||
if (!txn || txn.status !== 'active') return;
|
||||
|
||||
const ns = `${dbName}.${collName}`;
|
||||
if (!txn.readSet.has(ns)) {
|
||||
txn.readSet.set(ns, new Set());
|
||||
}
|
||||
|
||||
const readSet = txn.readSet.get(ns)!;
|
||||
for (const id of docIds) {
|
||||
readSet.add(id);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Record a write operation (insert)
|
||||
*/
|
||||
recordInsert(txnId: string, dbName: string, collName: string, doc: IStoredDocument): void {
|
||||
const txn = this.transactions.get(txnId);
|
||||
if (!txn || txn.status !== 'active') {
|
||||
throw new TsmdbTransactionError('Transaction is not active');
|
||||
}
|
||||
|
||||
const ns = `${dbName}.${collName}`;
|
||||
if (!txn.writeSet.has(ns)) {
|
||||
txn.writeSet.set(ns, new Map());
|
||||
}
|
||||
|
||||
txn.writeSet.get(ns)!.set(doc._id.toHexString(), {
|
||||
op: 'insert',
|
||||
doc,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Record a write operation (update)
|
||||
*/
|
||||
recordUpdate(
|
||||
txnId: string,
|
||||
dbName: string,
|
||||
collName: string,
|
||||
originalDoc: IStoredDocument,
|
||||
updatedDoc: IStoredDocument
|
||||
): void {
|
||||
const txn = this.transactions.get(txnId);
|
||||
if (!txn || txn.status !== 'active') {
|
||||
throw new TsmdbTransactionError('Transaction is not active');
|
||||
}
|
||||
|
||||
const ns = `${dbName}.${collName}`;
|
||||
if (!txn.writeSet.has(ns)) {
|
||||
txn.writeSet.set(ns, new Map());
|
||||
}
|
||||
|
||||
const idStr = originalDoc._id.toHexString();
|
||||
const existing = txn.writeSet.get(ns)!.get(idStr);
|
||||
|
||||
// If we already have a write for this document, update it
|
||||
if (existing) {
|
||||
existing.doc = updatedDoc;
|
||||
} else {
|
||||
txn.writeSet.get(ns)!.set(idStr, {
|
||||
op: 'update',
|
||||
doc: updatedDoc,
|
||||
originalDoc,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Record a write operation (delete)
|
||||
*/
|
||||
recordDelete(txnId: string, dbName: string, collName: string, doc: IStoredDocument): void {
|
||||
const txn = this.transactions.get(txnId);
|
||||
if (!txn || txn.status !== 'active') {
|
||||
throw new TsmdbTransactionError('Transaction is not active');
|
||||
}
|
||||
|
||||
const ns = `${dbName}.${collName}`;
|
||||
if (!txn.writeSet.has(ns)) {
|
||||
txn.writeSet.set(ns, new Map());
|
||||
}
|
||||
|
||||
const idStr = doc._id.toHexString();
|
||||
const existing = txn.writeSet.get(ns)!.get(idStr);
|
||||
|
||||
if (existing && existing.op === 'insert') {
|
||||
// If we inserted and then deleted, just remove the write
|
||||
txn.writeSet.get(ns)!.delete(idStr);
|
||||
} else {
|
||||
txn.writeSet.get(ns)!.set(idStr, {
|
||||
op: 'delete',
|
||||
originalDoc: doc,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Commit a transaction
|
||||
*/
|
||||
async commitTransaction(txnId: string): Promise<void> {
|
||||
const txn = this.transactions.get(txnId);
|
||||
if (!txn) {
|
||||
throw new TsmdbTransactionError('Transaction not found');
|
||||
}
|
||||
if (txn.status !== 'active') {
|
||||
throw new TsmdbTransactionError(`Cannot commit transaction in state: ${txn.status}`);
|
||||
}
|
||||
|
||||
// Check for write conflicts
|
||||
for (const [ns, writes] of txn.writeSet) {
|
||||
const [dbName, collName] = ns.split('.');
|
||||
const ids = Array.from(writes.keys()).map(id => new plugins.bson.ObjectId(id));
|
||||
|
||||
const hasConflicts = await this.storage.hasConflicts(dbName, collName, ids, txn.startTime);
|
||||
if (hasConflicts) {
|
||||
txn.status = 'aborted';
|
||||
throw new TsmdbWriteConflictError();
|
||||
}
|
||||
}
|
||||
|
||||
// Apply all writes
|
||||
for (const [ns, writes] of txn.writeSet) {
|
||||
const [dbName, collName] = ns.split('.');
|
||||
|
||||
for (const [idStr, write] of writes) {
|
||||
switch (write.op) {
|
||||
case 'insert':
|
||||
if (write.doc) {
|
||||
await this.storage.insertOne(dbName, collName, write.doc);
|
||||
}
|
||||
break;
|
||||
case 'update':
|
||||
if (write.doc) {
|
||||
await this.storage.updateById(dbName, collName, new plugins.bson.ObjectId(idStr), write.doc);
|
||||
}
|
||||
break;
|
||||
case 'delete':
|
||||
await this.storage.deleteById(dbName, collName, new plugins.bson.ObjectId(idStr));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
txn.status = 'committed';
|
||||
}
|
||||
|
||||
/**
|
||||
* Abort a transaction
|
||||
*/
|
||||
async abortTransaction(txnId: string): Promise<void> {
|
||||
const txn = this.transactions.get(txnId);
|
||||
if (!txn) {
|
||||
throw new TsmdbTransactionError('Transaction not found');
|
||||
}
|
||||
if (txn.status !== 'active') {
|
||||
// Already committed or aborted, just return
|
||||
return;
|
||||
}
|
||||
|
||||
// Simply discard all buffered writes
|
||||
txn.writeSet.clear();
|
||||
txn.readSet.clear();
|
||||
txn.snapshots.clear();
|
||||
txn.status = 'aborted';
|
||||
}
|
||||
|
||||
/**
|
||||
* End a transaction (cleanup)
|
||||
*/
|
||||
endTransaction(txnId: string): void {
|
||||
this.transactions.delete(txnId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all pending writes for a namespace
|
||||
*/
|
||||
getPendingWrites(txnId: string, dbName: string, collName: string): Map<string, { op: 'insert' | 'update' | 'delete'; doc?: IStoredDocument }> | undefined {
|
||||
const txn = this.transactions.get(txnId);
|
||||
if (!txn) return undefined;
|
||||
|
||||
const ns = `${dbName}.${collName}`;
|
||||
return txn.writeSet.get(ns);
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a callback within a transaction, with automatic retry on conflict
|
||||
*/
|
||||
async withTransaction<T>(
|
||||
sessionId: string,
|
||||
callback: (txnId: string) => Promise<T>,
|
||||
options?: ITransactionOptions & { maxRetries?: number }
|
||||
): Promise<T> {
|
||||
const maxRetries = options?.maxRetries ?? 3;
|
||||
let lastError: Error | undefined;
|
||||
|
||||
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
||||
const txnId = this.startTransaction(sessionId, options);
|
||||
|
||||
try {
|
||||
const result = await callback(txnId);
|
||||
await this.commitTransaction(txnId);
|
||||
this.endTransaction(txnId);
|
||||
return result;
|
||||
} catch (error: any) {
|
||||
await this.abortTransaction(txnId);
|
||||
this.endTransaction(txnId);
|
||||
|
||||
if (error instanceof TsmdbWriteConflictError && attempt < maxRetries - 1) {
|
||||
// Retry on write conflict
|
||||
lastError = error;
|
||||
continue;
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError || new TsmdbTransactionError('Transaction failed after max retries');
|
||||
}
|
||||
}
|
||||
506
ts/ts_tsmdb/engine/UpdateEngine.ts
Normal file
506
ts/ts_tsmdb/engine/UpdateEngine.ts
Normal file
@@ -0,0 +1,506 @@
|
||||
import * as plugins from '../plugins.js';
|
||||
import type { Document, IStoredDocument } from '../types/interfaces.js';
|
||||
import { QueryEngine } from './QueryEngine.js';
|
||||
|
||||
/**
|
||||
* Update engine for MongoDB-compatible update operations
|
||||
*/
|
||||
export class UpdateEngine {
|
||||
/**
|
||||
* Apply an update specification to a document
|
||||
* Returns the updated document or null if no update was applied
|
||||
*/
|
||||
static applyUpdate(document: IStoredDocument, update: Document, arrayFilters?: Document[]): IStoredDocument {
|
||||
// Check if this is an aggregation pipeline update
|
||||
if (Array.isArray(update)) {
|
||||
// Aggregation pipeline updates are not yet supported
|
||||
throw new Error('Aggregation pipeline updates are not yet supported');
|
||||
}
|
||||
|
||||
// Check if this is a replacement (no $ operators at top level)
|
||||
const hasOperators = Object.keys(update).some(k => k.startsWith('$'));
|
||||
|
||||
if (!hasOperators) {
|
||||
// This is a replacement - preserve _id
|
||||
return {
|
||||
_id: document._id,
|
||||
...update,
|
||||
};
|
||||
}
|
||||
|
||||
// Apply update operators
|
||||
const result = this.deepClone(document);
|
||||
|
||||
for (const [operator, operand] of Object.entries(update)) {
|
||||
switch (operator) {
|
||||
case '$set':
|
||||
this.applySet(result, operand);
|
||||
break;
|
||||
case '$unset':
|
||||
this.applyUnset(result, operand);
|
||||
break;
|
||||
case '$inc':
|
||||
this.applyInc(result, operand);
|
||||
break;
|
||||
case '$mul':
|
||||
this.applyMul(result, operand);
|
||||
break;
|
||||
case '$min':
|
||||
this.applyMin(result, operand);
|
||||
break;
|
||||
case '$max':
|
||||
this.applyMax(result, operand);
|
||||
break;
|
||||
case '$rename':
|
||||
this.applyRename(result, operand);
|
||||
break;
|
||||
case '$currentDate':
|
||||
this.applyCurrentDate(result, operand);
|
||||
break;
|
||||
case '$setOnInsert':
|
||||
// Only applied during upsert insert, handled elsewhere
|
||||
break;
|
||||
case '$push':
|
||||
this.applyPush(result, operand, arrayFilters);
|
||||
break;
|
||||
case '$pop':
|
||||
this.applyPop(result, operand);
|
||||
break;
|
||||
case '$pull':
|
||||
this.applyPull(result, operand, arrayFilters);
|
||||
break;
|
||||
case '$pullAll':
|
||||
this.applyPullAll(result, operand);
|
||||
break;
|
||||
case '$addToSet':
|
||||
this.applyAddToSet(result, operand);
|
||||
break;
|
||||
case '$bit':
|
||||
this.applyBit(result, operand);
|
||||
break;
|
||||
default:
|
||||
throw new Error(`Unknown update operator: ${operator}`);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply $setOnInsert for upsert operations
|
||||
*/
|
||||
static applySetOnInsert(document: IStoredDocument, setOnInsert: Document): IStoredDocument {
|
||||
const result = this.deepClone(document);
|
||||
this.applySet(result, setOnInsert);
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Deep clone a document
|
||||
*/
|
||||
private static deepClone(obj: any): any {
|
||||
if (obj === null || typeof obj !== 'object') {
|
||||
return obj;
|
||||
}
|
||||
|
||||
if (obj instanceof plugins.bson.ObjectId) {
|
||||
return new plugins.bson.ObjectId(obj.toHexString());
|
||||
}
|
||||
|
||||
if (obj instanceof Date) {
|
||||
return new Date(obj.getTime());
|
||||
}
|
||||
|
||||
if (obj instanceof plugins.bson.Timestamp) {
|
||||
return new plugins.bson.Timestamp({ t: obj.high, i: obj.low });
|
||||
}
|
||||
|
||||
if (Array.isArray(obj)) {
|
||||
return obj.map(item => this.deepClone(item));
|
||||
}
|
||||
|
||||
const cloned: any = {};
|
||||
for (const key of Object.keys(obj)) {
|
||||
cloned[key] = this.deepClone(obj[key]);
|
||||
}
|
||||
return cloned;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set a nested value
|
||||
*/
|
||||
private static setNestedValue(obj: any, path: string, value: any): void {
|
||||
const parts = path.split('.');
|
||||
let current = obj;
|
||||
|
||||
for (let i = 0; i < parts.length - 1; i++) {
|
||||
const part = parts[i];
|
||||
|
||||
// Handle array index notation
|
||||
const arrayMatch = part.match(/^(\w+)\[(\d+)\]$/);
|
||||
if (arrayMatch) {
|
||||
const [, fieldName, indexStr] = arrayMatch;
|
||||
const index = parseInt(indexStr, 10);
|
||||
if (!(fieldName in current)) {
|
||||
current[fieldName] = [];
|
||||
}
|
||||
if (!current[fieldName][index]) {
|
||||
current[fieldName][index] = {};
|
||||
}
|
||||
current = current[fieldName][index];
|
||||
continue;
|
||||
}
|
||||
|
||||
// Handle numeric index (array positional)
|
||||
const numIndex = parseInt(part, 10);
|
||||
if (!isNaN(numIndex) && Array.isArray(current)) {
|
||||
if (!current[numIndex]) {
|
||||
current[numIndex] = {};
|
||||
}
|
||||
current = current[numIndex];
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!(part in current) || current[part] === null) {
|
||||
current[part] = {};
|
||||
}
|
||||
current = current[part];
|
||||
}
|
||||
|
||||
const lastPart = parts[parts.length - 1];
|
||||
const numIndex = parseInt(lastPart, 10);
|
||||
if (!isNaN(numIndex) && Array.isArray(current)) {
|
||||
current[numIndex] = value;
|
||||
} else {
|
||||
current[lastPart] = value;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a nested value
|
||||
*/
|
||||
private static getNestedValue(obj: any, path: string): any {
|
||||
return QueryEngine.getNestedValue(obj, path);
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a nested value
|
||||
*/
|
||||
private static deleteNestedValue(obj: any, path: string): void {
|
||||
const parts = path.split('.');
|
||||
let current = obj;
|
||||
|
||||
for (let i = 0; i < parts.length - 1; i++) {
|
||||
const part = parts[i];
|
||||
if (!(part in current)) {
|
||||
return;
|
||||
}
|
||||
current = current[part];
|
||||
}
|
||||
|
||||
delete current[parts[parts.length - 1]];
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Field Update Operators
|
||||
// ============================================================================
|
||||
|
||||
private static applySet(doc: any, fields: Document): void {
|
||||
for (const [path, value] of Object.entries(fields)) {
|
||||
this.setNestedValue(doc, path, this.deepClone(value));
|
||||
}
|
||||
}
|
||||
|
||||
private static applyUnset(doc: any, fields: Document): void {
|
||||
for (const path of Object.keys(fields)) {
|
||||
this.deleteNestedValue(doc, path);
|
||||
}
|
||||
}
|
||||
|
||||
private static applyInc(doc: any, fields: Document): void {
|
||||
for (const [path, value] of Object.entries(fields)) {
|
||||
const current = this.getNestedValue(doc, path) || 0;
|
||||
if (typeof current !== 'number') {
|
||||
throw new Error(`Cannot apply $inc to non-numeric field: ${path}`);
|
||||
}
|
||||
this.setNestedValue(doc, path, current + (value as number));
|
||||
}
|
||||
}
|
||||
|
||||
private static applyMul(doc: any, fields: Document): void {
|
||||
for (const [path, value] of Object.entries(fields)) {
|
||||
const current = this.getNestedValue(doc, path) || 0;
|
||||
if (typeof current !== 'number') {
|
||||
throw new Error(`Cannot apply $mul to non-numeric field: ${path}`);
|
||||
}
|
||||
this.setNestedValue(doc, path, current * (value as number));
|
||||
}
|
||||
}
|
||||
|
||||
private static applyMin(doc: any, fields: Document): void {
|
||||
for (const [path, value] of Object.entries(fields)) {
|
||||
const current = this.getNestedValue(doc, path);
|
||||
if (current === undefined || this.compareValues(value, current) < 0) {
|
||||
this.setNestedValue(doc, path, this.deepClone(value));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static applyMax(doc: any, fields: Document): void {
|
||||
for (const [path, value] of Object.entries(fields)) {
|
||||
const current = this.getNestedValue(doc, path);
|
||||
if (current === undefined || this.compareValues(value, current) > 0) {
|
||||
this.setNestedValue(doc, path, this.deepClone(value));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static applyRename(doc: any, fields: Document): void {
|
||||
for (const [oldPath, newPath] of Object.entries(fields)) {
|
||||
const value = this.getNestedValue(doc, oldPath);
|
||||
if (value !== undefined) {
|
||||
this.deleteNestedValue(doc, oldPath);
|
||||
this.setNestedValue(doc, newPath as string, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static applyCurrentDate(doc: any, fields: Document): void {
|
||||
for (const [path, spec] of Object.entries(fields)) {
|
||||
if (spec === true) {
|
||||
this.setNestedValue(doc, path, new Date());
|
||||
} else if (typeof spec === 'object' && spec.$type === 'date') {
|
||||
this.setNestedValue(doc, path, new Date());
|
||||
} else if (typeof spec === 'object' && spec.$type === 'timestamp') {
|
||||
this.setNestedValue(doc, path, new plugins.bson.Timestamp({ t: Math.floor(Date.now() / 1000), i: 0 }));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Array Update Operators
|
||||
// ============================================================================
|
||||
|
||||
private static applyPush(doc: any, fields: Document, arrayFilters?: Document[]): void {
|
||||
for (const [path, spec] of Object.entries(fields)) {
|
||||
let arr = this.getNestedValue(doc, path);
|
||||
if (arr === undefined) {
|
||||
arr = [];
|
||||
this.setNestedValue(doc, path, arr);
|
||||
}
|
||||
if (!Array.isArray(arr)) {
|
||||
throw new Error(`Cannot apply $push to non-array field: ${path}`);
|
||||
}
|
||||
|
||||
if (spec && typeof spec === 'object' && '$each' in spec) {
|
||||
// $push with modifiers
|
||||
let elements = (spec.$each as any[]).map(e => this.deepClone(e));
|
||||
const position = spec.$position as number | undefined;
|
||||
const slice = spec.$slice as number | undefined;
|
||||
const sortSpec = spec.$sort;
|
||||
|
||||
if (position !== undefined) {
|
||||
arr.splice(position, 0, ...elements);
|
||||
} else {
|
||||
arr.push(...elements);
|
||||
}
|
||||
|
||||
if (sortSpec !== undefined) {
|
||||
if (typeof sortSpec === 'number') {
|
||||
arr.sort((a, b) => (a - b) * sortSpec);
|
||||
} else {
|
||||
// Sort by field(s)
|
||||
const entries = Object.entries(sortSpec as Document);
|
||||
arr.sort((a, b) => {
|
||||
for (const [field, dir] of entries) {
|
||||
const av = this.getNestedValue(a, field);
|
||||
const bv = this.getNestedValue(b, field);
|
||||
const cmp = this.compareValues(av, bv) * (dir as number);
|
||||
if (cmp !== 0) return cmp;
|
||||
}
|
||||
return 0;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (slice !== undefined) {
|
||||
if (slice >= 0) {
|
||||
arr.splice(slice);
|
||||
} else {
|
||||
arr.splice(0, arr.length + slice);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Simple push
|
||||
arr.push(this.deepClone(spec));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static applyPop(doc: any, fields: Document): void {
|
||||
for (const [path, direction] of Object.entries(fields)) {
|
||||
const arr = this.getNestedValue(doc, path);
|
||||
if (!Array.isArray(arr)) {
|
||||
throw new Error(`Cannot apply $pop to non-array field: ${path}`);
|
||||
}
|
||||
|
||||
if ((direction as number) === 1) {
|
||||
arr.pop();
|
||||
} else {
|
||||
arr.shift();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static applyPull(doc: any, fields: Document, arrayFilters?: Document[]): void {
|
||||
for (const [path, condition] of Object.entries(fields)) {
|
||||
const arr = this.getNestedValue(doc, path);
|
||||
if (!Array.isArray(arr)) {
|
||||
continue; // Skip if not an array
|
||||
}
|
||||
|
||||
if (typeof condition === 'object' && condition !== null && !Array.isArray(condition)) {
|
||||
// Condition is a query filter
|
||||
const hasOperators = Object.keys(condition).some(k => k.startsWith('$'));
|
||||
if (hasOperators) {
|
||||
// Filter using query operators
|
||||
const remaining = arr.filter(item => !QueryEngine.matches(item, condition));
|
||||
arr.length = 0;
|
||||
arr.push(...remaining);
|
||||
} else {
|
||||
// Match documents with all specified fields
|
||||
const remaining = arr.filter(item => {
|
||||
if (typeof item !== 'object' || item === null) {
|
||||
return true;
|
||||
}
|
||||
return !Object.entries(condition).every(([k, v]) => {
|
||||
const itemVal = this.getNestedValue(item, k);
|
||||
return this.valuesEqual(itemVal, v);
|
||||
});
|
||||
});
|
||||
arr.length = 0;
|
||||
arr.push(...remaining);
|
||||
}
|
||||
} else {
|
||||
// Direct value match
|
||||
const remaining = arr.filter(item => !this.valuesEqual(item, condition));
|
||||
arr.length = 0;
|
||||
arr.push(...remaining);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static applyPullAll(doc: any, fields: Document): void {
|
||||
for (const [path, values] of Object.entries(fields)) {
|
||||
const arr = this.getNestedValue(doc, path);
|
||||
if (!Array.isArray(arr)) {
|
||||
continue;
|
||||
}
|
||||
if (!Array.isArray(values)) {
|
||||
throw new Error(`$pullAll requires an array argument`);
|
||||
}
|
||||
|
||||
const valueSet = new Set(values.map(v => JSON.stringify(v)));
|
||||
const remaining = arr.filter(item => !valueSet.has(JSON.stringify(item)));
|
||||
arr.length = 0;
|
||||
arr.push(...remaining);
|
||||
}
|
||||
}
|
||||
|
||||
private static applyAddToSet(doc: any, fields: Document): void {
|
||||
for (const [path, spec] of Object.entries(fields)) {
|
||||
let arr = this.getNestedValue(doc, path);
|
||||
if (arr === undefined) {
|
||||
arr = [];
|
||||
this.setNestedValue(doc, path, arr);
|
||||
}
|
||||
if (!Array.isArray(arr)) {
|
||||
throw new Error(`Cannot apply $addToSet to non-array field: ${path}`);
|
||||
}
|
||||
|
||||
const existingSet = new Set(arr.map(v => JSON.stringify(v)));
|
||||
|
||||
if (spec && typeof spec === 'object' && '$each' in spec) {
|
||||
for (const item of spec.$each as any[]) {
|
||||
const key = JSON.stringify(item);
|
||||
if (!existingSet.has(key)) {
|
||||
arr.push(this.deepClone(item));
|
||||
existingSet.add(key);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const key = JSON.stringify(spec);
|
||||
if (!existingSet.has(key)) {
|
||||
arr.push(this.deepClone(spec));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static applyBit(doc: any, fields: Document): void {
|
||||
for (const [path, operations] of Object.entries(fields)) {
|
||||
let current = this.getNestedValue(doc, path) || 0;
|
||||
if (typeof current !== 'number') {
|
||||
throw new Error(`Cannot apply $bit to non-numeric field: ${path}`);
|
||||
}
|
||||
|
||||
for (const [op, value] of Object.entries(operations as Document)) {
|
||||
switch (op) {
|
||||
case 'and':
|
||||
current = current & (value as number);
|
||||
break;
|
||||
case 'or':
|
||||
current = current | (value as number);
|
||||
break;
|
||||
case 'xor':
|
||||
current = current ^ (value as number);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
this.setNestedValue(doc, path, current);
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Helper Methods
|
||||
// ============================================================================
|
||||
|
||||
private static compareValues(a: any, b: any): number {
|
||||
if (a === b) return 0;
|
||||
if (a === null || a === undefined) return -1;
|
||||
if (b === null || b === undefined) return 1;
|
||||
|
||||
if (typeof a === 'number' && typeof b === 'number') {
|
||||
return a - b;
|
||||
}
|
||||
|
||||
if (a instanceof Date && b instanceof Date) {
|
||||
return a.getTime() - b.getTime();
|
||||
}
|
||||
|
||||
if (typeof a === 'string' && typeof b === 'string') {
|
||||
return a.localeCompare(b);
|
||||
}
|
||||
|
||||
return String(a).localeCompare(String(b));
|
||||
}
|
||||
|
||||
private static valuesEqual(a: any, b: any): boolean {
|
||||
if (a === b) return true;
|
||||
|
||||
if (a instanceof plugins.bson.ObjectId && b instanceof plugins.bson.ObjectId) {
|
||||
return a.equals(b);
|
||||
}
|
||||
|
||||
if (a instanceof Date && b instanceof Date) {
|
||||
return a.getTime() === b.getTime();
|
||||
}
|
||||
|
||||
if (typeof a === 'object' && typeof b === 'object' && a !== null && b !== null) {
|
||||
return JSON.stringify(a) === JSON.stringify(b);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
181
ts/ts_tsmdb/errors/TsmdbErrors.ts
Normal file
181
ts/ts_tsmdb/errors/TsmdbErrors.ts
Normal file
@@ -0,0 +1,181 @@
|
||||
/**
|
||||
* Base error class for all TsmDB errors
|
||||
* Mirrors MongoDB driver error hierarchy
|
||||
*/
|
||||
export class TsmdbError extends Error {
|
||||
public code?: number;
|
||||
public codeName?: string;
|
||||
|
||||
constructor(message: string, code?: number, codeName?: string) {
|
||||
super(message);
|
||||
this.name = 'TsmdbError';
|
||||
this.code = code;
|
||||
this.codeName = codeName;
|
||||
Object.setPrototypeOf(this, new.target.prototype);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Error thrown during connection issues
|
||||
*/
|
||||
export class TsmdbConnectionError extends TsmdbError {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = 'TsmdbConnectionError';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Error thrown when an operation times out
|
||||
*/
|
||||
export class TsmdbTimeoutError extends TsmdbError {
|
||||
constructor(message: string) {
|
||||
super(message, 50, 'MaxTimeMSExpired');
|
||||
this.name = 'TsmdbTimeoutError';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Error thrown during write operations
|
||||
*/
|
||||
export class TsmdbWriteError extends TsmdbError {
|
||||
public writeErrors?: IWriteError[];
|
||||
public result?: any;
|
||||
|
||||
constructor(message: string, code?: number, writeErrors?: IWriteError[]) {
|
||||
super(message, code);
|
||||
this.name = 'TsmdbWriteError';
|
||||
this.writeErrors = writeErrors;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Error thrown for duplicate key violations
|
||||
*/
|
||||
export class TsmdbDuplicateKeyError extends TsmdbWriteError {
|
||||
public keyPattern?: Record<string, 1>;
|
||||
public keyValue?: Record<string, any>;
|
||||
|
||||
constructor(message: string, keyPattern?: Record<string, 1>, keyValue?: Record<string, any>) {
|
||||
super(message, 11000);
|
||||
this.name = 'TsmdbDuplicateKeyError';
|
||||
this.codeName = 'DuplicateKey';
|
||||
this.keyPattern = keyPattern;
|
||||
this.keyValue = keyValue;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Error thrown for bulk write failures
|
||||
*/
|
||||
export class TsmdbBulkWriteError extends TsmdbError {
|
||||
public writeErrors: IWriteError[];
|
||||
public result: any;
|
||||
|
||||
constructor(message: string, writeErrors: IWriteError[], result: any) {
|
||||
super(message, 65);
|
||||
this.name = 'TsmdbBulkWriteError';
|
||||
this.writeErrors = writeErrors;
|
||||
this.result = result;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Error thrown during transaction operations
|
||||
*/
|
||||
export class TsmdbTransactionError extends TsmdbError {
|
||||
constructor(message: string, code?: number) {
|
||||
super(message, code);
|
||||
this.name = 'TsmdbTransactionError';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Error thrown when a transaction is aborted due to conflict
|
||||
*/
|
||||
export class TsmdbWriteConflictError extends TsmdbTransactionError {
|
||||
constructor(message: string = 'Write conflict during transaction') {
|
||||
super(message, 112);
|
||||
this.name = 'TsmdbWriteConflictError';
|
||||
this.codeName = 'WriteConflict';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Error thrown for invalid arguments
|
||||
*/
|
||||
export class TsmdbArgumentError extends TsmdbError {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = 'TsmdbArgumentError';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Error thrown when an operation is not supported
|
||||
*/
|
||||
export class TsmdbNotSupportedError extends TsmdbError {
|
||||
constructor(message: string) {
|
||||
super(message, 115);
|
||||
this.name = 'TsmdbNotSupportedError';
|
||||
this.codeName = 'CommandNotSupported';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Error thrown when cursor is exhausted or closed
|
||||
*/
|
||||
export class TsmdbCursorError extends TsmdbError {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = 'TsmdbCursorError';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Error thrown when a namespace (database.collection) is invalid
|
||||
*/
|
||||
export class TsmdbNamespaceError extends TsmdbError {
|
||||
constructor(message: string) {
|
||||
super(message, 73);
|
||||
this.name = 'TsmdbNamespaceError';
|
||||
this.codeName = 'InvalidNamespace';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Error thrown when an index operation fails
|
||||
*/
|
||||
export class TsmdbIndexError extends TsmdbError {
|
||||
constructor(message: string, code?: number) {
|
||||
super(message, code || 86);
|
||||
this.name = 'TsmdbIndexError';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Write error detail for bulk operations
|
||||
*/
|
||||
export interface IWriteError {
|
||||
index: number;
|
||||
code: number;
|
||||
errmsg: string;
|
||||
op: any;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert any error to a TsmdbError
|
||||
*/
|
||||
export function toTsmdbError(error: any): TsmdbError {
|
||||
if (error instanceof TsmdbError) {
|
||||
return error;
|
||||
}
|
||||
const tsmdbError = new TsmdbError(error.message || String(error));
|
||||
if (error.code) {
|
||||
tsmdbError.code = error.code;
|
||||
}
|
||||
if (error.codeName) {
|
||||
tsmdbError.codeName = error.codeName;
|
||||
}
|
||||
return tsmdbError;
|
||||
}
|
||||
46
ts/ts_tsmdb/index.ts
Normal file
46
ts/ts_tsmdb/index.ts
Normal file
@@ -0,0 +1,46 @@
|
||||
// TsmDB - MongoDB Wire Protocol compatible in-memory database server
|
||||
// Use the official MongoDB driver to connect to TsmdbServer
|
||||
|
||||
// Re-export plugins for external use
|
||||
import * as plugins from './plugins.js';
|
||||
export { plugins };
|
||||
|
||||
// Export BSON types for convenience
|
||||
export { ObjectId, Binary, Timestamp, Long, Decimal128, UUID } from 'bson';
|
||||
|
||||
// Export all types
|
||||
export * from './types/interfaces.js';
|
||||
|
||||
// Export errors
|
||||
export * from './errors/TsmdbErrors.js';
|
||||
|
||||
// Export storage adapters
|
||||
export type { IStorageAdapter } from './storage/IStorageAdapter.js';
|
||||
export { MemoryStorageAdapter } from './storage/MemoryStorageAdapter.js';
|
||||
export { FileStorageAdapter } from './storage/FileStorageAdapter.js';
|
||||
export { OpLog } from './storage/OpLog.js';
|
||||
export { WAL } from './storage/WAL.js';
|
||||
export type { IWalEntry, TWalOperation } from './storage/WAL.js';
|
||||
|
||||
// Export engines
|
||||
export { QueryEngine } from './engine/QueryEngine.js';
|
||||
export { UpdateEngine } from './engine/UpdateEngine.js';
|
||||
export { AggregationEngine } from './engine/AggregationEngine.js';
|
||||
export { IndexEngine } from './engine/IndexEngine.js';
|
||||
export { TransactionEngine } from './engine/TransactionEngine.js';
|
||||
export { QueryPlanner } from './engine/QueryPlanner.js';
|
||||
export type { IQueryPlan, TQueryPlanType } from './engine/QueryPlanner.js';
|
||||
export { SessionEngine } from './engine/SessionEngine.js';
|
||||
export type { ISession, ISessionEngineOptions } from './engine/SessionEngine.js';
|
||||
|
||||
// Export server (the main entry point for using TsmDB)
|
||||
export { TsmdbServer } from './server/TsmdbServer.js';
|
||||
export type { ITsmdbServerOptions } from './server/TsmdbServer.js';
|
||||
|
||||
// Export wire protocol utilities (for advanced usage)
|
||||
export { WireProtocol } from './server/WireProtocol.js';
|
||||
export { CommandRouter } from './server/CommandRouter.js';
|
||||
export type { ICommandHandler, IHandlerContext, ICursorState } from './server/CommandRouter.js';
|
||||
|
||||
// Export utilities
|
||||
export * from './utils/checksum.js';
|
||||
17
ts/ts_tsmdb/plugins.ts
Normal file
17
ts/ts_tsmdb/plugins.ts
Normal file
@@ -0,0 +1,17 @@
|
||||
// @push.rocks scope
|
||||
import * as smartfs from '@push.rocks/smartfs';
|
||||
import * as smartpath from '@push.rocks/smartpath';
|
||||
import * as smartpromise from '@push.rocks/smartpromise';
|
||||
import * as smartrx from '@push.rocks/smartrx';
|
||||
|
||||
export { smartfs, smartpath, smartpromise, smartrx };
|
||||
|
||||
// thirdparty
|
||||
import * as bson from 'bson';
|
||||
import * as mingo from 'mingo';
|
||||
|
||||
export { bson, mingo };
|
||||
|
||||
// Re-export commonly used mingo classes
|
||||
export { Query } from 'mingo';
|
||||
export { Aggregator } from 'mingo';
|
||||
289
ts/ts_tsmdb/server/CommandRouter.ts
Normal file
289
ts/ts_tsmdb/server/CommandRouter.ts
Normal file
@@ -0,0 +1,289 @@
|
||||
import * as plugins from '../plugins.js';
|
||||
import type { IStorageAdapter } from '../storage/IStorageAdapter.js';
|
||||
import type { IParsedCommand } from './WireProtocol.js';
|
||||
import type { TsmdbServer } from './TsmdbServer.js';
|
||||
import { IndexEngine } from '../engine/IndexEngine.js';
|
||||
import { TransactionEngine } from '../engine/TransactionEngine.js';
|
||||
import { SessionEngine } from '../engine/SessionEngine.js';
|
||||
|
||||
// Import handlers
|
||||
import { HelloHandler } from './handlers/HelloHandler.js';
|
||||
import { InsertHandler } from './handlers/InsertHandler.js';
|
||||
import { FindHandler } from './handlers/FindHandler.js';
|
||||
import { UpdateHandler } from './handlers/UpdateHandler.js';
|
||||
import { DeleteHandler } from './handlers/DeleteHandler.js';
|
||||
import { AggregateHandler } from './handlers/AggregateHandler.js';
|
||||
import { IndexHandler } from './handlers/IndexHandler.js';
|
||||
import { AdminHandler } from './handlers/AdminHandler.js';
|
||||
|
||||
/**
|
||||
* Handler context passed to command handlers
|
||||
*/
|
||||
export interface IHandlerContext {
|
||||
storage: IStorageAdapter;
|
||||
server: TsmdbServer;
|
||||
database: string;
|
||||
command: plugins.bson.Document;
|
||||
documentSequences?: Map<string, plugins.bson.Document[]>;
|
||||
/** Get or create an IndexEngine for a collection */
|
||||
getIndexEngine: (collName: string) => IndexEngine;
|
||||
/** Transaction engine instance */
|
||||
transactionEngine: TransactionEngine;
|
||||
/** Current transaction ID (if in a transaction) */
|
||||
txnId?: string;
|
||||
/** Session ID (from lsid) */
|
||||
sessionId?: string;
|
||||
/** Session engine instance */
|
||||
sessionEngine: SessionEngine;
|
||||
}
|
||||
|
||||
/**
|
||||
* Command handler interface
|
||||
*/
|
||||
export interface ICommandHandler {
|
||||
handle(context: IHandlerContext): Promise<plugins.bson.Document>;
|
||||
}
|
||||
|
||||
/**
|
||||
* CommandRouter - Routes incoming commands to appropriate handlers
|
||||
*/
|
||||
export class CommandRouter {
|
||||
private storage: IStorageAdapter;
|
||||
private server: TsmdbServer;
|
||||
private handlers: Map<string, ICommandHandler> = new Map();
|
||||
|
||||
// Cursor state for getMore operations
|
||||
private cursors: Map<bigint, ICursorState> = new Map();
|
||||
private cursorIdCounter: bigint = BigInt(1);
|
||||
|
||||
// Index engine cache: db.collection -> IndexEngine
|
||||
private indexEngines: Map<string, IndexEngine> = new Map();
|
||||
|
||||
// Transaction engine (shared across all handlers)
|
||||
private transactionEngine: TransactionEngine;
|
||||
|
||||
// Session engine (shared across all handlers)
|
||||
private sessionEngine: SessionEngine;
|
||||
|
||||
constructor(storage: IStorageAdapter, server: TsmdbServer) {
|
||||
this.storage = storage;
|
||||
this.server = server;
|
||||
this.transactionEngine = new TransactionEngine(storage);
|
||||
this.sessionEngine = new SessionEngine();
|
||||
// Link session engine to transaction engine for auto-abort on session expiry
|
||||
this.sessionEngine.setTransactionEngine(this.transactionEngine);
|
||||
this.registerHandlers();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get or create an IndexEngine for a database.collection
|
||||
*/
|
||||
getIndexEngine(dbName: string, collName: string): IndexEngine {
|
||||
const key = `${dbName}.${collName}`;
|
||||
let engine = this.indexEngines.get(key);
|
||||
if (!engine) {
|
||||
engine = new IndexEngine(dbName, collName, this.storage);
|
||||
this.indexEngines.set(key, engine);
|
||||
}
|
||||
return engine;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear index engine cache for a collection (used when collection is dropped)
|
||||
*/
|
||||
clearIndexEngineCache(dbName: string, collName?: string): void {
|
||||
if (collName) {
|
||||
this.indexEngines.delete(`${dbName}.${collName}`);
|
||||
} else {
|
||||
// Clear all engines for the database
|
||||
for (const key of this.indexEngines.keys()) {
|
||||
if (key.startsWith(`${dbName}.`)) {
|
||||
this.indexEngines.delete(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Register all command handlers
|
||||
*/
|
||||
private registerHandlers(): void {
|
||||
// Create handler instances with shared state
|
||||
const helloHandler = new HelloHandler();
|
||||
const findHandler = new FindHandler(this.cursors, () => this.cursorIdCounter++);
|
||||
const insertHandler = new InsertHandler();
|
||||
const updateHandler = new UpdateHandler();
|
||||
const deleteHandler = new DeleteHandler();
|
||||
const aggregateHandler = new AggregateHandler(this.cursors, () => this.cursorIdCounter++);
|
||||
const indexHandler = new IndexHandler();
|
||||
const adminHandler = new AdminHandler();
|
||||
|
||||
// Handshake commands
|
||||
this.handlers.set('hello', helloHandler);
|
||||
this.handlers.set('ismaster', helloHandler);
|
||||
this.handlers.set('isMaster', helloHandler);
|
||||
|
||||
// CRUD commands
|
||||
this.handlers.set('find', findHandler);
|
||||
this.handlers.set('insert', insertHandler);
|
||||
this.handlers.set('update', updateHandler);
|
||||
this.handlers.set('delete', deleteHandler);
|
||||
this.handlers.set('findAndModify', updateHandler);
|
||||
this.handlers.set('getMore', findHandler);
|
||||
this.handlers.set('killCursors', findHandler);
|
||||
|
||||
// Aggregation
|
||||
this.handlers.set('aggregate', aggregateHandler);
|
||||
this.handlers.set('count', findHandler);
|
||||
this.handlers.set('distinct', findHandler);
|
||||
|
||||
// Index operations
|
||||
this.handlers.set('createIndexes', indexHandler);
|
||||
this.handlers.set('dropIndexes', indexHandler);
|
||||
this.handlers.set('listIndexes', indexHandler);
|
||||
|
||||
// Admin/Database operations
|
||||
this.handlers.set('ping', adminHandler);
|
||||
this.handlers.set('listDatabases', adminHandler);
|
||||
this.handlers.set('listCollections', adminHandler);
|
||||
this.handlers.set('drop', adminHandler);
|
||||
this.handlers.set('dropDatabase', adminHandler);
|
||||
this.handlers.set('create', adminHandler);
|
||||
this.handlers.set('serverStatus', adminHandler);
|
||||
this.handlers.set('buildInfo', adminHandler);
|
||||
this.handlers.set('whatsmyuri', adminHandler);
|
||||
this.handlers.set('getLog', adminHandler);
|
||||
this.handlers.set('hostInfo', adminHandler);
|
||||
this.handlers.set('replSetGetStatus', adminHandler);
|
||||
this.handlers.set('isMaster', helloHandler);
|
||||
this.handlers.set('saslStart', adminHandler);
|
||||
this.handlers.set('saslContinue', adminHandler);
|
||||
this.handlers.set('endSessions', adminHandler);
|
||||
this.handlers.set('abortTransaction', adminHandler);
|
||||
this.handlers.set('commitTransaction', adminHandler);
|
||||
this.handlers.set('collStats', adminHandler);
|
||||
this.handlers.set('dbStats', adminHandler);
|
||||
this.handlers.set('connectionStatus', adminHandler);
|
||||
this.handlers.set('currentOp', adminHandler);
|
||||
this.handlers.set('collMod', adminHandler);
|
||||
this.handlers.set('renameCollection', adminHandler);
|
||||
}
|
||||
|
||||
/**
|
||||
* Route a command to its handler
|
||||
*/
|
||||
async route(parsedCommand: IParsedCommand): Promise<plugins.bson.Document> {
|
||||
const { commandName, command, database, documentSequences } = parsedCommand;
|
||||
|
||||
// Extract session ID from lsid using SessionEngine helper
|
||||
let sessionId = SessionEngine.extractSessionId(command.lsid);
|
||||
let txnId: string | undefined;
|
||||
|
||||
// If we have a session ID, register/touch the session
|
||||
if (sessionId) {
|
||||
this.sessionEngine.getOrCreateSession(sessionId);
|
||||
}
|
||||
|
||||
// Check if this starts a new transaction
|
||||
if (command.startTransaction && sessionId) {
|
||||
txnId = this.transactionEngine.startTransaction(sessionId);
|
||||
this.sessionEngine.startTransaction(sessionId, txnId, command.txnNumber);
|
||||
} else if (sessionId && this.sessionEngine.isInTransaction(sessionId)) {
|
||||
// Continue existing transaction
|
||||
txnId = this.sessionEngine.getTransactionId(sessionId);
|
||||
// Verify transaction is still active
|
||||
if (txnId && !this.transactionEngine.isActive(txnId)) {
|
||||
this.sessionEngine.endTransaction(sessionId);
|
||||
txnId = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
// Create handler context
|
||||
const context: IHandlerContext = {
|
||||
storage: this.storage,
|
||||
server: this.server,
|
||||
database,
|
||||
command,
|
||||
documentSequences,
|
||||
getIndexEngine: (collName: string) => this.getIndexEngine(database, collName),
|
||||
transactionEngine: this.transactionEngine,
|
||||
sessionEngine: this.sessionEngine,
|
||||
txnId,
|
||||
sessionId,
|
||||
};
|
||||
|
||||
// Find handler
|
||||
const handler = this.handlers.get(commandName);
|
||||
|
||||
if (!handler) {
|
||||
// Unknown command
|
||||
return {
|
||||
ok: 0,
|
||||
errmsg: `no such command: '${commandName}'`,
|
||||
code: 59,
|
||||
codeName: 'CommandNotFound',
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
return await handler.handle(context);
|
||||
} catch (error: any) {
|
||||
// Handle known error types
|
||||
if (error.code) {
|
||||
return {
|
||||
ok: 0,
|
||||
errmsg: error.message,
|
||||
code: error.code,
|
||||
codeName: error.codeName || 'UnknownError',
|
||||
};
|
||||
}
|
||||
|
||||
// Generic error
|
||||
return {
|
||||
ok: 0,
|
||||
errmsg: error.message || 'Internal error',
|
||||
code: 1,
|
||||
codeName: 'InternalError',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Close the command router and cleanup resources
|
||||
*/
|
||||
close(): void {
|
||||
// Close session engine (stops cleanup interval, clears sessions)
|
||||
this.sessionEngine.close();
|
||||
// Clear cursors
|
||||
this.cursors.clear();
|
||||
// Clear index engine cache
|
||||
this.indexEngines.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get session engine (for administrative purposes)
|
||||
*/
|
||||
getSessionEngine(): SessionEngine {
|
||||
return this.sessionEngine;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get transaction engine (for administrative purposes)
|
||||
*/
|
||||
getTransactionEngine(): TransactionEngine {
|
||||
return this.transactionEngine;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Cursor state for multi-batch queries
|
||||
*/
|
||||
export interface ICursorState {
|
||||
id: bigint;
|
||||
database: string;
|
||||
collection: string;
|
||||
documents: plugins.bson.Document[];
|
||||
position: number;
|
||||
batchSize: number;
|
||||
createdAt: Date;
|
||||
}
|
||||
301
ts/ts_tsmdb/server/TsmdbServer.ts
Normal file
301
ts/ts_tsmdb/server/TsmdbServer.ts
Normal file
@@ -0,0 +1,301 @@
|
||||
import * as net from 'net';
|
||||
import * as plugins from '../plugins.js';
|
||||
import { WireProtocol, OP_QUERY } from './WireProtocol.js';
|
||||
import { CommandRouter } from './CommandRouter.js';
|
||||
import { MemoryStorageAdapter } from '../storage/MemoryStorageAdapter.js';
|
||||
import { FileStorageAdapter } from '../storage/FileStorageAdapter.js';
|
||||
import type { IStorageAdapter } from '../storage/IStorageAdapter.js';
|
||||
|
||||
/**
|
||||
* Server configuration options
|
||||
*/
|
||||
export interface ITsmdbServerOptions {
|
||||
/** Port to listen on (default: 27017) */
|
||||
port?: number;
|
||||
/** Host to bind to (default: 127.0.0.1) */
|
||||
host?: string;
|
||||
/** Storage type: 'memory' or 'file' (default: 'memory') */
|
||||
storage?: 'memory' | 'file';
|
||||
/** Path for file storage (required if storage is 'file') */
|
||||
storagePath?: string;
|
||||
/** Enable persistence for memory storage */
|
||||
persistPath?: string;
|
||||
/** Persistence interval in ms (default: 60000) */
|
||||
persistIntervalMs?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Connection state for each client
|
||||
*/
|
||||
interface IConnectionState {
|
||||
id: number;
|
||||
socket: net.Socket;
|
||||
buffer: Buffer;
|
||||
authenticated: boolean;
|
||||
database: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* TsmdbServer - MongoDB Wire Protocol compatible server
|
||||
*
|
||||
* This server implements the MongoDB wire protocol (OP_MSG) to allow
|
||||
* official MongoDB drivers to connect and perform operations.
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* import { TsmdbServer } from '@push.rocks/smartmongo/tsmdb';
|
||||
* import { MongoClient } from 'mongodb';
|
||||
*
|
||||
* const server = new TsmdbServer({ port: 27017 });
|
||||
* await server.start();
|
||||
*
|
||||
* const client = new MongoClient('mongodb://127.0.0.1:27017');
|
||||
* await client.connect();
|
||||
* ```
|
||||
*/
|
||||
export class TsmdbServer {
|
||||
private options: Required<ITsmdbServerOptions>;
|
||||
private server: net.Server | null = null;
|
||||
private storage: IStorageAdapter;
|
||||
private commandRouter: CommandRouter;
|
||||
private connections: Map<number, IConnectionState> = new Map();
|
||||
private connectionIdCounter = 0;
|
||||
private isRunning = false;
|
||||
private startTime: Date = new Date();
|
||||
|
||||
constructor(options: ITsmdbServerOptions = {}) {
|
||||
this.options = {
|
||||
port: options.port ?? 27017,
|
||||
host: options.host ?? '127.0.0.1',
|
||||
storage: options.storage ?? 'memory',
|
||||
storagePath: options.storagePath ?? './data',
|
||||
persistPath: options.persistPath ?? '',
|
||||
persistIntervalMs: options.persistIntervalMs ?? 60000,
|
||||
};
|
||||
|
||||
// Create storage adapter
|
||||
if (this.options.storage === 'file') {
|
||||
this.storage = new FileStorageAdapter(this.options.storagePath);
|
||||
} else {
|
||||
this.storage = new MemoryStorageAdapter({
|
||||
persistPath: this.options.persistPath || undefined,
|
||||
persistIntervalMs: this.options.persistPath ? this.options.persistIntervalMs : undefined,
|
||||
});
|
||||
}
|
||||
|
||||
// Create command router
|
||||
this.commandRouter = new CommandRouter(this.storage, this);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the storage adapter (for testing/debugging)
|
||||
*/
|
||||
getStorage(): IStorageAdapter {
|
||||
return this.storage;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get server uptime in seconds
|
||||
*/
|
||||
getUptime(): number {
|
||||
return Math.floor((Date.now() - this.startTime.getTime()) / 1000);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current connection count
|
||||
*/
|
||||
getConnectionCount(): number {
|
||||
return this.connections.size;
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the server
|
||||
*/
|
||||
async start(): Promise<void> {
|
||||
if (this.isRunning) {
|
||||
throw new Error('Server is already running');
|
||||
}
|
||||
|
||||
// Initialize storage
|
||||
await this.storage.initialize();
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
this.server = net.createServer((socket) => {
|
||||
this.handleConnection(socket);
|
||||
});
|
||||
|
||||
this.server.on('error', (err) => {
|
||||
if (!this.isRunning) {
|
||||
reject(err);
|
||||
} else {
|
||||
console.error('Server error:', err);
|
||||
}
|
||||
});
|
||||
|
||||
this.server.listen(this.options.port, this.options.host, () => {
|
||||
this.isRunning = true;
|
||||
this.startTime = new Date();
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the server
|
||||
*/
|
||||
async stop(): Promise<void> {
|
||||
if (!this.isRunning || !this.server) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Close all connections
|
||||
for (const conn of this.connections.values()) {
|
||||
conn.socket.destroy();
|
||||
}
|
||||
this.connections.clear();
|
||||
|
||||
// Close command router (cleans up session engine, cursors, etc.)
|
||||
this.commandRouter.close();
|
||||
|
||||
// Close storage
|
||||
await this.storage.close();
|
||||
|
||||
return new Promise((resolve) => {
|
||||
this.server!.close(() => {
|
||||
this.isRunning = false;
|
||||
this.server = null;
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle a new client connection
|
||||
*/
|
||||
private handleConnection(socket: net.Socket): void {
|
||||
const connectionId = ++this.connectionIdCounter;
|
||||
|
||||
const state: IConnectionState = {
|
||||
id: connectionId,
|
||||
socket,
|
||||
buffer: Buffer.alloc(0),
|
||||
authenticated: true, // No auth required for now
|
||||
database: 'test',
|
||||
};
|
||||
|
||||
this.connections.set(connectionId, state);
|
||||
|
||||
socket.on('data', (data) => {
|
||||
this.handleData(state, Buffer.isBuffer(data) ? data : Buffer.from(data));
|
||||
});
|
||||
|
||||
socket.on('close', () => {
|
||||
this.connections.delete(connectionId);
|
||||
});
|
||||
|
||||
socket.on('error', (err) => {
|
||||
// Connection errors are expected when clients disconnect
|
||||
this.connections.delete(connectionId);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle incoming data from a client
|
||||
*/
|
||||
private handleData(state: IConnectionState, data: Buffer): void {
|
||||
// Append new data to buffer
|
||||
state.buffer = Buffer.concat([state.buffer, data]);
|
||||
|
||||
// Process messages from buffer
|
||||
this.processMessages(state);
|
||||
}
|
||||
|
||||
/**
|
||||
* Process complete messages from the buffer
|
||||
*/
|
||||
private async processMessages(state: IConnectionState): Promise<void> {
|
||||
while (state.buffer.length >= 16) {
|
||||
try {
|
||||
const result = WireProtocol.parseMessage(state.buffer);
|
||||
|
||||
if (!result) {
|
||||
// Not enough data for a complete message
|
||||
break;
|
||||
}
|
||||
|
||||
const { command, bytesConsumed } = result;
|
||||
|
||||
// Remove processed bytes from buffer
|
||||
state.buffer = state.buffer.subarray(bytesConsumed);
|
||||
|
||||
// Process the command
|
||||
const response = await this.commandRouter.route(command);
|
||||
|
||||
// Encode and send response
|
||||
let responseBuffer: Buffer;
|
||||
if (command.opCode === OP_QUERY) {
|
||||
// Legacy OP_QUERY gets OP_REPLY response
|
||||
responseBuffer = WireProtocol.encodeOpReplyResponse(
|
||||
command.requestID,
|
||||
[response]
|
||||
);
|
||||
} else {
|
||||
// OP_MSG gets OP_MSG response
|
||||
responseBuffer = WireProtocol.encodeOpMsgResponse(
|
||||
command.requestID,
|
||||
response
|
||||
);
|
||||
}
|
||||
|
||||
if (!state.socket.destroyed) {
|
||||
state.socket.write(responseBuffer);
|
||||
}
|
||||
} catch (error: any) {
|
||||
// Send error response
|
||||
const errorResponse = WireProtocol.encodeErrorResponse(
|
||||
0, // We don't have the requestID at this point
|
||||
1,
|
||||
error.message || 'Internal error'
|
||||
);
|
||||
|
||||
if (!state.socket.destroyed) {
|
||||
state.socket.write(errorResponse);
|
||||
}
|
||||
|
||||
// Clear buffer on parse errors to avoid infinite loops
|
||||
if (error.message?.includes('opCode') || error.message?.includes('section')) {
|
||||
state.buffer = Buffer.alloc(0);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the connection URI for this server
|
||||
*/
|
||||
getConnectionUri(): string {
|
||||
return `mongodb://${this.options.host}:${this.options.port}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the server is running
|
||||
*/
|
||||
get running(): boolean {
|
||||
return this.isRunning;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the port the server is listening on
|
||||
*/
|
||||
get port(): number {
|
||||
return this.options.port;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the host the server is bound to
|
||||
*/
|
||||
get host(): string {
|
||||
return this.options.host;
|
||||
}
|
||||
}
|
||||
416
ts/ts_tsmdb/server/WireProtocol.ts
Normal file
416
ts/ts_tsmdb/server/WireProtocol.ts
Normal file
@@ -0,0 +1,416 @@
|
||||
import * as plugins from '../plugins.js';
|
||||
|
||||
/**
|
||||
* MongoDB Wire Protocol Implementation
|
||||
* Handles parsing and encoding of MongoDB wire protocol messages (OP_MSG primarily)
|
||||
*
|
||||
* Wire Protocol Message Format:
|
||||
* - Header (16 bytes): messageLength (4), requestID (4), responseTo (4), opCode (4)
|
||||
* - OP_MSG: flagBits (4), sections[], optional checksum (4)
|
||||
*
|
||||
* References:
|
||||
* - https://www.mongodb.com/docs/manual/reference/mongodb-wire-protocol/
|
||||
*/
|
||||
|
||||
// OpCodes
|
||||
export const OP_REPLY = 1; // Legacy reply
|
||||
export const OP_UPDATE = 2001; // Legacy update
|
||||
export const OP_INSERT = 2002; // Legacy insert
|
||||
export const OP_QUERY = 2004; // Legacy query (still used for initial handshake)
|
||||
export const OP_GET_MORE = 2005; // Legacy getMore
|
||||
export const OP_DELETE = 2006; // Legacy delete
|
||||
export const OP_KILL_CURSORS = 2007; // Legacy kill cursors
|
||||
export const OP_COMPRESSED = 2012; // Compressed message
|
||||
export const OP_MSG = 2013; // Modern protocol (MongoDB 3.6+)
|
||||
|
||||
// OP_MSG Section Types
|
||||
export const SECTION_BODY = 0; // Single BSON document
|
||||
export const SECTION_DOCUMENT_SEQUENCE = 1; // Document sequence for bulk operations
|
||||
|
||||
// OP_MSG Flag Bits
|
||||
export const MSG_FLAG_CHECKSUM_PRESENT = 1 << 0;
|
||||
export const MSG_FLAG_MORE_TO_COME = 1 << 1;
|
||||
export const MSG_FLAG_EXHAUST_ALLOWED = 1 << 16;
|
||||
|
||||
/**
|
||||
* Parsed message header
|
||||
*/
|
||||
export interface IMessageHeader {
|
||||
messageLength: number;
|
||||
requestID: number;
|
||||
responseTo: number;
|
||||
opCode: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parsed OP_MSG message
|
||||
*/
|
||||
export interface IOpMsgMessage {
|
||||
header: IMessageHeader;
|
||||
flagBits: number;
|
||||
sections: IOpMsgSection[];
|
||||
checksum?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* OP_MSG section (either body or document sequence)
|
||||
*/
|
||||
export interface IOpMsgSection {
|
||||
type: number;
|
||||
payload: plugins.bson.Document;
|
||||
sequenceIdentifier?: string;
|
||||
documents?: plugins.bson.Document[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Parsed OP_QUERY message (legacy, but used for initial handshake)
|
||||
*/
|
||||
export interface IOpQueryMessage {
|
||||
header: IMessageHeader;
|
||||
flags: number;
|
||||
fullCollectionName: string;
|
||||
numberToSkip: number;
|
||||
numberToReturn: number;
|
||||
query: plugins.bson.Document;
|
||||
returnFieldsSelector?: plugins.bson.Document;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parsed command from any message type
|
||||
*/
|
||||
export interface IParsedCommand {
|
||||
commandName: string;
|
||||
command: plugins.bson.Document;
|
||||
database: string;
|
||||
requestID: number;
|
||||
opCode: number;
|
||||
documentSequences?: Map<string, plugins.bson.Document[]>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Wire Protocol parser and encoder
|
||||
*/
|
||||
export class WireProtocol {
|
||||
/**
|
||||
* Parse a complete message from a buffer
|
||||
* Returns the parsed command and the number of bytes consumed
|
||||
*/
|
||||
static parseMessage(buffer: Buffer): { command: IParsedCommand; bytesConsumed: number } | null {
|
||||
if (buffer.length < 16) {
|
||||
return null; // Not enough data for header
|
||||
}
|
||||
|
||||
const header = this.parseHeader(buffer);
|
||||
|
||||
if (buffer.length < header.messageLength) {
|
||||
return null; // Not enough data for complete message
|
||||
}
|
||||
|
||||
const messageBuffer = buffer.subarray(0, header.messageLength);
|
||||
|
||||
switch (header.opCode) {
|
||||
case OP_MSG:
|
||||
return this.parseOpMsg(messageBuffer, header);
|
||||
case OP_QUERY:
|
||||
return this.parseOpQuery(messageBuffer, header);
|
||||
default:
|
||||
throw new Error(`Unsupported opCode: ${header.opCode}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse message header (16 bytes)
|
||||
*/
|
||||
private static parseHeader(buffer: Buffer): IMessageHeader {
|
||||
return {
|
||||
messageLength: buffer.readInt32LE(0),
|
||||
requestID: buffer.readInt32LE(4),
|
||||
responseTo: buffer.readInt32LE(8),
|
||||
opCode: buffer.readInt32LE(12),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse OP_MSG message
|
||||
*/
|
||||
private static parseOpMsg(buffer: Buffer, header: IMessageHeader): { command: IParsedCommand; bytesConsumed: number } {
|
||||
let offset = 16; // Skip header
|
||||
|
||||
const flagBits = buffer.readUInt32LE(offset);
|
||||
offset += 4;
|
||||
|
||||
const sections: IOpMsgSection[] = [];
|
||||
const documentSequences = new Map<string, plugins.bson.Document[]>();
|
||||
|
||||
// Parse sections until we reach the end (or checksum)
|
||||
const messageEnd = (flagBits & MSG_FLAG_CHECKSUM_PRESENT)
|
||||
? header.messageLength - 4
|
||||
: header.messageLength;
|
||||
|
||||
while (offset < messageEnd) {
|
||||
const sectionType = buffer.readUInt8(offset);
|
||||
offset += 1;
|
||||
|
||||
if (sectionType === SECTION_BODY) {
|
||||
// Single BSON document
|
||||
const docSize = buffer.readInt32LE(offset);
|
||||
const docBuffer = buffer.subarray(offset, offset + docSize);
|
||||
const doc = plugins.bson.deserialize(docBuffer);
|
||||
sections.push({ type: SECTION_BODY, payload: doc });
|
||||
offset += docSize;
|
||||
} else if (sectionType === SECTION_DOCUMENT_SEQUENCE) {
|
||||
// Document sequence
|
||||
const sectionSize = buffer.readInt32LE(offset);
|
||||
const sectionEnd = offset + sectionSize;
|
||||
offset += 4;
|
||||
|
||||
// Read sequence identifier (C string)
|
||||
let identifierEnd = offset;
|
||||
while (buffer[identifierEnd] !== 0 && identifierEnd < sectionEnd) {
|
||||
identifierEnd++;
|
||||
}
|
||||
const identifier = buffer.subarray(offset, identifierEnd).toString('utf8');
|
||||
offset = identifierEnd + 1; // Skip null terminator
|
||||
|
||||
// Read documents
|
||||
const documents: plugins.bson.Document[] = [];
|
||||
while (offset < sectionEnd) {
|
||||
const docSize = buffer.readInt32LE(offset);
|
||||
const docBuffer = buffer.subarray(offset, offset + docSize);
|
||||
documents.push(plugins.bson.deserialize(docBuffer));
|
||||
offset += docSize;
|
||||
}
|
||||
|
||||
sections.push({
|
||||
type: SECTION_DOCUMENT_SEQUENCE,
|
||||
payload: {},
|
||||
sequenceIdentifier: identifier,
|
||||
documents
|
||||
});
|
||||
documentSequences.set(identifier, documents);
|
||||
} else {
|
||||
throw new Error(`Unknown section type: ${sectionType}`);
|
||||
}
|
||||
}
|
||||
|
||||
// The first section body contains the command
|
||||
const commandSection = sections.find(s => s.type === SECTION_BODY);
|
||||
if (!commandSection) {
|
||||
throw new Error('OP_MSG missing command body section');
|
||||
}
|
||||
|
||||
const command = commandSection.payload;
|
||||
const commandName = Object.keys(command)[0];
|
||||
const database = command.$db || 'admin';
|
||||
|
||||
return {
|
||||
command: {
|
||||
commandName,
|
||||
command,
|
||||
database,
|
||||
requestID: header.requestID,
|
||||
opCode: header.opCode,
|
||||
documentSequences: documentSequences.size > 0 ? documentSequences : undefined,
|
||||
},
|
||||
bytesConsumed: header.messageLength,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse OP_QUERY message (legacy, used for initial handshake)
|
||||
*/
|
||||
private static parseOpQuery(buffer: Buffer, header: IMessageHeader): { command: IParsedCommand; bytesConsumed: number } {
|
||||
let offset = 16; // Skip header
|
||||
|
||||
const flags = buffer.readInt32LE(offset);
|
||||
offset += 4;
|
||||
|
||||
// Read full collection name (C string)
|
||||
let nameEnd = offset;
|
||||
while (buffer[nameEnd] !== 0 && nameEnd < buffer.length) {
|
||||
nameEnd++;
|
||||
}
|
||||
const fullCollectionName = buffer.subarray(offset, nameEnd).toString('utf8');
|
||||
offset = nameEnd + 1;
|
||||
|
||||
const numberToSkip = buffer.readInt32LE(offset);
|
||||
offset += 4;
|
||||
|
||||
const numberToReturn = buffer.readInt32LE(offset);
|
||||
offset += 4;
|
||||
|
||||
// Read query document
|
||||
const querySize = buffer.readInt32LE(offset);
|
||||
const queryBuffer = buffer.subarray(offset, offset + querySize);
|
||||
const query = plugins.bson.deserialize(queryBuffer);
|
||||
offset += querySize;
|
||||
|
||||
// Extract database from collection name (format: "dbname.$cmd" or "dbname.collection")
|
||||
const parts = fullCollectionName.split('.');
|
||||
const database = parts[0];
|
||||
|
||||
// For OP_QUERY to .$cmd, the query IS the command
|
||||
let commandName = 'find';
|
||||
let command = query;
|
||||
|
||||
if (parts[1] === '$cmd') {
|
||||
// This is a command
|
||||
commandName = Object.keys(query)[0];
|
||||
// Handle special commands like isMaster, hello
|
||||
if (commandName === 'isMaster' || commandName === 'ismaster') {
|
||||
commandName = 'hello';
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
command: {
|
||||
commandName,
|
||||
command,
|
||||
database,
|
||||
requestID: header.requestID,
|
||||
opCode: header.opCode,
|
||||
},
|
||||
bytesConsumed: header.messageLength,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Encode a response as OP_MSG
|
||||
*/
|
||||
static encodeOpMsgResponse(
|
||||
responseTo: number,
|
||||
response: plugins.bson.Document,
|
||||
requestID: number = Math.floor(Math.random() * 0x7FFFFFFF)
|
||||
): Buffer {
|
||||
// Add $db if not present (optional in response)
|
||||
const responseDoc = { ...response };
|
||||
|
||||
// Serialize the response document
|
||||
const bodyBson = plugins.bson.serialize(responseDoc);
|
||||
|
||||
// Calculate message length
|
||||
// Header (16) + flagBits (4) + section type (1) + body BSON
|
||||
const messageLength = 16 + 4 + 1 + bodyBson.length;
|
||||
|
||||
const buffer = Buffer.alloc(messageLength);
|
||||
let offset = 0;
|
||||
|
||||
// Write header
|
||||
buffer.writeInt32LE(messageLength, offset); // messageLength
|
||||
offset += 4;
|
||||
buffer.writeInt32LE(requestID, offset); // requestID
|
||||
offset += 4;
|
||||
buffer.writeInt32LE(responseTo, offset); // responseTo
|
||||
offset += 4;
|
||||
buffer.writeInt32LE(OP_MSG, offset); // opCode
|
||||
offset += 4;
|
||||
|
||||
// Write flagBits (0 = no flags)
|
||||
buffer.writeUInt32LE(0, offset);
|
||||
offset += 4;
|
||||
|
||||
// Write section type 0 (body)
|
||||
buffer.writeUInt8(SECTION_BODY, offset);
|
||||
offset += 1;
|
||||
|
||||
// Write body BSON
|
||||
Buffer.from(bodyBson).copy(buffer, offset);
|
||||
|
||||
return buffer;
|
||||
}
|
||||
|
||||
/**
|
||||
* Encode a response as OP_REPLY (legacy, for OP_QUERY responses)
|
||||
*/
|
||||
static encodeOpReplyResponse(
|
||||
responseTo: number,
|
||||
documents: plugins.bson.Document[],
|
||||
requestID: number = Math.floor(Math.random() * 0x7FFFFFFF),
|
||||
cursorId: bigint = BigInt(0)
|
||||
): Buffer {
|
||||
// Serialize all documents
|
||||
const docBuffers = documents.map(doc => plugins.bson.serialize(doc));
|
||||
const totalDocsSize = docBuffers.reduce((sum, buf) => sum + buf.length, 0);
|
||||
|
||||
// Message format:
|
||||
// Header (16) + responseFlags (4) + cursorID (8) + startingFrom (4) + numberReturned (4) + documents
|
||||
const messageLength = 16 + 4 + 8 + 4 + 4 + totalDocsSize;
|
||||
|
||||
const buffer = Buffer.alloc(messageLength);
|
||||
let offset = 0;
|
||||
|
||||
// Write header
|
||||
buffer.writeInt32LE(messageLength, offset); // messageLength
|
||||
offset += 4;
|
||||
buffer.writeInt32LE(requestID, offset); // requestID
|
||||
offset += 4;
|
||||
buffer.writeInt32LE(responseTo, offset); // responseTo
|
||||
offset += 4;
|
||||
buffer.writeInt32LE(OP_REPLY, offset); // opCode
|
||||
offset += 4;
|
||||
|
||||
// Write OP_REPLY fields
|
||||
buffer.writeInt32LE(0, offset); // responseFlags (0 = no errors)
|
||||
offset += 4;
|
||||
buffer.writeBigInt64LE(cursorId, offset); // cursorID
|
||||
offset += 8;
|
||||
buffer.writeInt32LE(0, offset); // startingFrom
|
||||
offset += 4;
|
||||
buffer.writeInt32LE(documents.length, offset); // numberReturned
|
||||
offset += 4;
|
||||
|
||||
// Write documents
|
||||
for (const docBuffer of docBuffers) {
|
||||
Buffer.from(docBuffer).copy(buffer, offset);
|
||||
offset += docBuffer.length;
|
||||
}
|
||||
|
||||
return buffer;
|
||||
}
|
||||
|
||||
/**
|
||||
* Encode an error response
|
||||
*/
|
||||
static encodeErrorResponse(
|
||||
responseTo: number,
|
||||
errorCode: number,
|
||||
errorMessage: string,
|
||||
commandName?: string
|
||||
): Buffer {
|
||||
const response: plugins.bson.Document = {
|
||||
ok: 0,
|
||||
errmsg: errorMessage,
|
||||
code: errorCode,
|
||||
codeName: this.getErrorCodeName(errorCode),
|
||||
};
|
||||
|
||||
return this.encodeOpMsgResponse(responseTo, response);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get error code name from error code
|
||||
*/
|
||||
private static getErrorCodeName(code: number): string {
|
||||
const errorNames: Record<number, string> = {
|
||||
0: 'OK',
|
||||
1: 'InternalError',
|
||||
2: 'BadValue',
|
||||
11000: 'DuplicateKey',
|
||||
11001: 'DuplicateKeyValue',
|
||||
13: 'Unauthorized',
|
||||
26: 'NamespaceNotFound',
|
||||
27: 'IndexNotFound',
|
||||
48: 'NamespaceExists',
|
||||
59: 'CommandNotFound',
|
||||
66: 'ImmutableField',
|
||||
73: 'InvalidNamespace',
|
||||
85: 'IndexOptionsConflict',
|
||||
112: 'WriteConflict',
|
||||
121: 'DocumentValidationFailure',
|
||||
211: 'KeyNotFound',
|
||||
251: 'NoSuchTransaction',
|
||||
};
|
||||
|
||||
return errorNames[code] || 'UnknownError';
|
||||
}
|
||||
}
|
||||
719
ts/ts_tsmdb/server/handlers/AdminHandler.ts
Normal file
719
ts/ts_tsmdb/server/handlers/AdminHandler.ts
Normal file
@@ -0,0 +1,719 @@
|
||||
import * as plugins from '../../plugins.js';
|
||||
import type { ICommandHandler, IHandlerContext } from '../CommandRouter.js';
|
||||
import { SessionEngine } from '../../engine/SessionEngine.js';
|
||||
|
||||
/**
|
||||
* AdminHandler - Handles administrative commands
|
||||
*/
|
||||
export class AdminHandler implements ICommandHandler {
|
||||
async handle(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
const { command } = context;
|
||||
|
||||
// Determine which command to handle
|
||||
if (command.ping !== undefined) {
|
||||
return this.handlePing(context);
|
||||
} else if (command.listDatabases !== undefined) {
|
||||
return this.handleListDatabases(context);
|
||||
} else if (command.listCollections !== undefined) {
|
||||
return this.handleListCollections(context);
|
||||
} else if (command.drop !== undefined) {
|
||||
return this.handleDrop(context);
|
||||
} else if (command.dropDatabase !== undefined) {
|
||||
return this.handleDropDatabase(context);
|
||||
} else if (command.create !== undefined) {
|
||||
return this.handleCreate(context);
|
||||
} else if (command.serverStatus !== undefined) {
|
||||
return this.handleServerStatus(context);
|
||||
} else if (command.buildInfo !== undefined) {
|
||||
return this.handleBuildInfo(context);
|
||||
} else if (command.whatsmyuri !== undefined) {
|
||||
return this.handleWhatsMyUri(context);
|
||||
} else if (command.getLog !== undefined) {
|
||||
return this.handleGetLog(context);
|
||||
} else if (command.hostInfo !== undefined) {
|
||||
return this.handleHostInfo(context);
|
||||
} else if (command.replSetGetStatus !== undefined) {
|
||||
return this.handleReplSetGetStatus(context);
|
||||
} else if (command.saslStart !== undefined) {
|
||||
return this.handleSaslStart(context);
|
||||
} else if (command.saslContinue !== undefined) {
|
||||
return this.handleSaslContinue(context);
|
||||
} else if (command.endSessions !== undefined) {
|
||||
return this.handleEndSessions(context);
|
||||
} else if (command.abortTransaction !== undefined) {
|
||||
return this.handleAbortTransaction(context);
|
||||
} else if (command.commitTransaction !== undefined) {
|
||||
return this.handleCommitTransaction(context);
|
||||
} else if (command.collStats !== undefined) {
|
||||
return this.handleCollStats(context);
|
||||
} else if (command.dbStats !== undefined) {
|
||||
return this.handleDbStats(context);
|
||||
} else if (command.connectionStatus !== undefined) {
|
||||
return this.handleConnectionStatus(context);
|
||||
} else if (command.currentOp !== undefined) {
|
||||
return this.handleCurrentOp(context);
|
||||
} else if (command.collMod !== undefined) {
|
||||
return this.handleCollMod(context);
|
||||
} else if (command.renameCollection !== undefined) {
|
||||
return this.handleRenameCollection(context);
|
||||
}
|
||||
|
||||
return {
|
||||
ok: 0,
|
||||
errmsg: 'Unknown admin command',
|
||||
code: 59,
|
||||
codeName: 'CommandNotFound',
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle ping command
|
||||
*/
|
||||
private async handlePing(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
return { ok: 1 };
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle listDatabases command
|
||||
*/
|
||||
private async handleListDatabases(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
const { storage, command } = context;
|
||||
|
||||
const dbNames = await storage.listDatabases();
|
||||
const nameOnly = command.nameOnly || false;
|
||||
|
||||
if (nameOnly) {
|
||||
return {
|
||||
ok: 1,
|
||||
databases: dbNames.map(name => ({ name })),
|
||||
};
|
||||
}
|
||||
|
||||
// Build database list with sizes
|
||||
const databases: plugins.bson.Document[] = [];
|
||||
let totalSize = 0;
|
||||
|
||||
for (const name of dbNames) {
|
||||
const collections = await storage.listCollections(name);
|
||||
let dbSize = 0;
|
||||
|
||||
for (const collName of collections) {
|
||||
const docs = await storage.findAll(name, collName);
|
||||
// Estimate size (rough approximation)
|
||||
dbSize += docs.reduce((sum, doc) => sum + JSON.stringify(doc).length, 0);
|
||||
}
|
||||
|
||||
totalSize += dbSize;
|
||||
|
||||
databases.push({
|
||||
name,
|
||||
sizeOnDisk: dbSize,
|
||||
empty: dbSize === 0,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
ok: 1,
|
||||
databases,
|
||||
totalSize,
|
||||
totalSizeMb: totalSize / (1024 * 1024),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle listCollections command
|
||||
*/
|
||||
private async handleListCollections(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
const { storage, database, command } = context;
|
||||
|
||||
const filter = command.filter || {};
|
||||
const nameOnly = command.nameOnly || false;
|
||||
const cursor = command.cursor || {};
|
||||
const batchSize = cursor.batchSize || 101;
|
||||
|
||||
const collNames = await storage.listCollections(database);
|
||||
|
||||
let collections: plugins.bson.Document[] = [];
|
||||
|
||||
for (const name of collNames) {
|
||||
// Apply name filter
|
||||
if (filter.name && filter.name !== name) {
|
||||
// Check regex
|
||||
if (filter.name.$regex) {
|
||||
const regex = new RegExp(filter.name.$regex, filter.name.$options);
|
||||
if (!regex.test(name)) continue;
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (nameOnly) {
|
||||
collections.push({ name });
|
||||
} else {
|
||||
collections.push({
|
||||
name,
|
||||
type: 'collection',
|
||||
options: {},
|
||||
info: {
|
||||
readOnly: false,
|
||||
uuid: new plugins.bson.UUID(),
|
||||
},
|
||||
idIndex: {
|
||||
v: 2,
|
||||
key: { _id: 1 },
|
||||
name: '_id_',
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
ok: 1,
|
||||
cursor: {
|
||||
id: plugins.bson.Long.fromNumber(0),
|
||||
ns: `${database}.$cmd.listCollections`,
|
||||
firstBatch: collections,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle drop command (drop collection)
|
||||
*/
|
||||
private async handleDrop(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
const { storage, database, command } = context;
|
||||
|
||||
const collection = command.drop;
|
||||
|
||||
const existed = await storage.dropCollection(database, collection);
|
||||
|
||||
if (!existed) {
|
||||
return {
|
||||
ok: 0,
|
||||
errmsg: `ns not found ${database}.${collection}`,
|
||||
code: 26,
|
||||
codeName: 'NamespaceNotFound',
|
||||
};
|
||||
}
|
||||
|
||||
return { ok: 1, ns: `${database}.${collection}` };
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle dropDatabase command
|
||||
*/
|
||||
private async handleDropDatabase(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
const { storage, database } = context;
|
||||
|
||||
await storage.dropDatabase(database);
|
||||
|
||||
return { ok: 1, dropped: database };
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle create command (create collection)
|
||||
*/
|
||||
private async handleCreate(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
const { storage, database, command } = context;
|
||||
|
||||
const collection = command.create;
|
||||
|
||||
// Check if already exists
|
||||
const exists = await storage.collectionExists(database, collection);
|
||||
if (exists) {
|
||||
return {
|
||||
ok: 0,
|
||||
errmsg: `Collection ${database}.${collection} already exists.`,
|
||||
code: 48,
|
||||
codeName: 'NamespaceExists',
|
||||
};
|
||||
}
|
||||
|
||||
await storage.createCollection(database, collection);
|
||||
|
||||
return { ok: 1 };
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle serverStatus command
|
||||
*/
|
||||
private async handleServerStatus(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
const { server, sessionEngine } = context;
|
||||
|
||||
const uptime = server.getUptime();
|
||||
const connections = server.getConnectionCount();
|
||||
const sessions = sessionEngine.listSessions();
|
||||
const sessionsWithTxn = sessionEngine.getSessionsWithTransactions();
|
||||
|
||||
return {
|
||||
ok: 1,
|
||||
host: `${server.host}:${server.port}`,
|
||||
version: '7.0.0',
|
||||
process: 'tsmdb',
|
||||
pid: process.pid,
|
||||
uptime,
|
||||
uptimeMillis: uptime * 1000,
|
||||
uptimeEstimate: uptime,
|
||||
localTime: new Date(),
|
||||
mem: {
|
||||
resident: Math.floor(process.memoryUsage().rss / (1024 * 1024)),
|
||||
virtual: Math.floor(process.memoryUsage().heapTotal / (1024 * 1024)),
|
||||
supported: true,
|
||||
},
|
||||
connections: {
|
||||
current: connections,
|
||||
available: 1000 - connections,
|
||||
totalCreated: connections,
|
||||
active: connections,
|
||||
},
|
||||
logicalSessionRecordCache: {
|
||||
activeSessionsCount: sessions.length,
|
||||
sessionsCollectionJobCount: 0,
|
||||
lastSessionsCollectionJobDurationMillis: 0,
|
||||
lastSessionsCollectionJobTimestamp: new Date(),
|
||||
transactionReaperJobCount: 0,
|
||||
lastTransactionReaperJobDurationMillis: 0,
|
||||
lastTransactionReaperJobTimestamp: new Date(),
|
||||
},
|
||||
transactions: {
|
||||
retriedCommandsCount: 0,
|
||||
retriedStatementsCount: 0,
|
||||
transactionsCollectionWriteCount: 0,
|
||||
currentActive: sessionsWithTxn.length,
|
||||
currentInactive: 0,
|
||||
currentOpen: sessionsWithTxn.length,
|
||||
totalStarted: sessionsWithTxn.length,
|
||||
totalCommitted: 0,
|
||||
totalAborted: 0,
|
||||
},
|
||||
network: {
|
||||
bytesIn: 0,
|
||||
bytesOut: 0,
|
||||
numRequests: 0,
|
||||
},
|
||||
storageEngine: {
|
||||
name: 'tsmdb',
|
||||
supportsCommittedReads: true,
|
||||
persistent: false,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle buildInfo command
|
||||
*/
|
||||
private async handleBuildInfo(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
return {
|
||||
ok: 1,
|
||||
version: '7.0.0',
|
||||
gitVersion: 'tsmdb',
|
||||
modules: [],
|
||||
allocator: 'system',
|
||||
javascriptEngine: 'none',
|
||||
sysInfo: 'deprecated',
|
||||
versionArray: [7, 0, 0, 0],
|
||||
openssl: {
|
||||
running: 'disabled',
|
||||
compiled: 'disabled',
|
||||
},
|
||||
buildEnvironment: {
|
||||
distmod: 'tsmdb',
|
||||
distarch: process.arch,
|
||||
cc: '',
|
||||
ccflags: '',
|
||||
cxx: '',
|
||||
cxxflags: '',
|
||||
linkflags: '',
|
||||
target_arch: process.arch,
|
||||
target_os: process.platform,
|
||||
},
|
||||
bits: 64,
|
||||
debug: false,
|
||||
maxBsonObjectSize: 16777216,
|
||||
storageEngines: ['tsmdb'],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle whatsmyuri command
|
||||
*/
|
||||
private async handleWhatsMyUri(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
const { server } = context;
|
||||
return {
|
||||
ok: 1,
|
||||
you: `127.0.0.1:${server.port}`,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle getLog command
|
||||
*/
|
||||
private async handleGetLog(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
const { command } = context;
|
||||
|
||||
if (command.getLog === '*') {
|
||||
return {
|
||||
ok: 1,
|
||||
names: ['global', 'startupWarnings'],
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
ok: 1,
|
||||
totalLinesWritten: 0,
|
||||
log: [],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle hostInfo command
|
||||
*/
|
||||
private async handleHostInfo(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
return {
|
||||
ok: 1,
|
||||
system: {
|
||||
currentTime: new Date(),
|
||||
hostname: 'localhost',
|
||||
cpuAddrSize: 64,
|
||||
memSizeMB: Math.floor(process.memoryUsage().heapTotal / (1024 * 1024)),
|
||||
numCores: 1,
|
||||
cpuArch: process.arch,
|
||||
numaEnabled: false,
|
||||
},
|
||||
os: {
|
||||
type: process.platform,
|
||||
name: process.platform,
|
||||
version: process.version,
|
||||
},
|
||||
extra: {},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle replSetGetStatus command
|
||||
*/
|
||||
private async handleReplSetGetStatus(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
// We're standalone, not a replica set
|
||||
return {
|
||||
ok: 0,
|
||||
errmsg: 'not running with --replSet',
|
||||
code: 76,
|
||||
codeName: 'NoReplicationEnabled',
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle saslStart command (authentication)
|
||||
*/
|
||||
private async handleSaslStart(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
// We don't require authentication, but we need to respond properly
|
||||
// to let drivers know auth is "successful"
|
||||
return {
|
||||
ok: 1,
|
||||
conversationId: 1,
|
||||
done: true,
|
||||
payload: Buffer.from([]),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle saslContinue command
|
||||
*/
|
||||
private async handleSaslContinue(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
return {
|
||||
ok: 1,
|
||||
conversationId: 1,
|
||||
done: true,
|
||||
payload: Buffer.from([]),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle endSessions command
|
||||
*/
|
||||
private async handleEndSessions(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
const { command, sessionEngine } = context;
|
||||
|
||||
// End each session in the array
|
||||
const sessions = command.endSessions || [];
|
||||
for (const sessionSpec of sessions) {
|
||||
const sessionId = SessionEngine.extractSessionId(sessionSpec);
|
||||
if (sessionId) {
|
||||
await sessionEngine.endSession(sessionId);
|
||||
}
|
||||
}
|
||||
|
||||
return { ok: 1 };
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle abortTransaction command
|
||||
*/
|
||||
private async handleAbortTransaction(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
const { transactionEngine, sessionEngine, txnId, sessionId } = context;
|
||||
|
||||
if (!txnId) {
|
||||
return {
|
||||
ok: 0,
|
||||
errmsg: 'No transaction started',
|
||||
code: 251,
|
||||
codeName: 'NoSuchTransaction',
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
await transactionEngine.abortTransaction(txnId);
|
||||
transactionEngine.endTransaction(txnId);
|
||||
// Update session state
|
||||
if (sessionId) {
|
||||
sessionEngine.endTransaction(sessionId);
|
||||
}
|
||||
return { ok: 1 };
|
||||
} catch (error: any) {
|
||||
return {
|
||||
ok: 0,
|
||||
errmsg: error.message || 'Abort transaction failed',
|
||||
code: error.code || 1,
|
||||
codeName: error.codeName || 'UnknownError',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle commitTransaction command
|
||||
*/
|
||||
private async handleCommitTransaction(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
const { transactionEngine, sessionEngine, txnId, sessionId } = context;
|
||||
|
||||
if (!txnId) {
|
||||
return {
|
||||
ok: 0,
|
||||
errmsg: 'No transaction started',
|
||||
code: 251,
|
||||
codeName: 'NoSuchTransaction',
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
await transactionEngine.commitTransaction(txnId);
|
||||
transactionEngine.endTransaction(txnId);
|
||||
// Update session state
|
||||
if (sessionId) {
|
||||
sessionEngine.endTransaction(sessionId);
|
||||
}
|
||||
return { ok: 1 };
|
||||
} catch (error: any) {
|
||||
// If commit fails, transaction should be aborted
|
||||
try {
|
||||
await transactionEngine.abortTransaction(txnId);
|
||||
transactionEngine.endTransaction(txnId);
|
||||
if (sessionId) {
|
||||
sessionEngine.endTransaction(sessionId);
|
||||
}
|
||||
} catch {
|
||||
// Ignore abort errors
|
||||
}
|
||||
|
||||
if (error.code === 112) {
|
||||
// Write conflict
|
||||
return {
|
||||
ok: 0,
|
||||
errmsg: error.message || 'Write conflict during commit',
|
||||
code: 112,
|
||||
codeName: 'WriteConflict',
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
ok: 0,
|
||||
errmsg: error.message || 'Commit transaction failed',
|
||||
code: error.code || 1,
|
||||
codeName: error.codeName || 'UnknownError',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle collStats command
|
||||
*/
|
||||
private async handleCollStats(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
const { storage, database, command } = context;
|
||||
|
||||
const collection = command.collStats;
|
||||
|
||||
const exists = await storage.collectionExists(database, collection);
|
||||
if (!exists) {
|
||||
return {
|
||||
ok: 0,
|
||||
errmsg: `ns not found ${database}.${collection}`,
|
||||
code: 26,
|
||||
codeName: 'NamespaceNotFound',
|
||||
};
|
||||
}
|
||||
|
||||
const docs = await storage.findAll(database, collection);
|
||||
const size = docs.reduce((sum, doc) => sum + JSON.stringify(doc).length, 0);
|
||||
const count = docs.length;
|
||||
const avgObjSize = count > 0 ? size / count : 0;
|
||||
|
||||
const indexes = await storage.getIndexes(database, collection);
|
||||
|
||||
return {
|
||||
ok: 1,
|
||||
ns: `${database}.${collection}`,
|
||||
count,
|
||||
size,
|
||||
avgObjSize,
|
||||
storageSize: size,
|
||||
totalIndexSize: 0,
|
||||
indexSizes: indexes.reduce((acc: any, idx: any) => {
|
||||
acc[idx.name] = 0;
|
||||
return acc;
|
||||
}, {}),
|
||||
nindexes: indexes.length,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle dbStats command
|
||||
*/
|
||||
private async handleDbStats(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
const { storage, database } = context;
|
||||
|
||||
const collections = await storage.listCollections(database);
|
||||
let totalSize = 0;
|
||||
let totalObjects = 0;
|
||||
|
||||
for (const collName of collections) {
|
||||
const docs = await storage.findAll(database, collName);
|
||||
totalObjects += docs.length;
|
||||
totalSize += docs.reduce((sum, doc) => sum + JSON.stringify(doc).length, 0);
|
||||
}
|
||||
|
||||
return {
|
||||
ok: 1,
|
||||
db: database,
|
||||
collections: collections.length,
|
||||
views: 0,
|
||||
objects: totalObjects,
|
||||
avgObjSize: totalObjects > 0 ? totalSize / totalObjects : 0,
|
||||
dataSize: totalSize,
|
||||
storageSize: totalSize,
|
||||
indexes: collections.length, // At least _id index per collection
|
||||
indexSize: 0,
|
||||
totalSize,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle connectionStatus command
|
||||
*/
|
||||
private async handleConnectionStatus(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
return {
|
||||
ok: 1,
|
||||
authInfo: {
|
||||
authenticatedUsers: [],
|
||||
authenticatedUserRoles: [],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle currentOp command
|
||||
*/
|
||||
private async handleCurrentOp(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
return {
|
||||
ok: 1,
|
||||
inprog: [],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle collMod command
|
||||
*/
|
||||
private async handleCollMod(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
// We don't support modifying collection options, but acknowledge the command
|
||||
return { ok: 1 };
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle renameCollection command
|
||||
*/
|
||||
private async handleRenameCollection(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
const { storage, command } = context;
|
||||
|
||||
const from = command.renameCollection;
|
||||
const to = command.to;
|
||||
const dropTarget = command.dropTarget || false;
|
||||
|
||||
if (!from || !to) {
|
||||
return {
|
||||
ok: 0,
|
||||
errmsg: 'renameCollection requires both source and target',
|
||||
code: 2,
|
||||
codeName: 'BadValue',
|
||||
};
|
||||
}
|
||||
|
||||
// Parse namespace (format: "db.collection")
|
||||
const fromParts = from.split('.');
|
||||
const toParts = to.split('.');
|
||||
|
||||
if (fromParts.length < 2 || toParts.length < 2) {
|
||||
return {
|
||||
ok: 0,
|
||||
errmsg: 'Invalid namespace format',
|
||||
code: 73,
|
||||
codeName: 'InvalidNamespace',
|
||||
};
|
||||
}
|
||||
|
||||
const fromDb = fromParts[0];
|
||||
const fromColl = fromParts.slice(1).join('.');
|
||||
const toDb = toParts[0];
|
||||
const toColl = toParts.slice(1).join('.');
|
||||
|
||||
// Check if source exists
|
||||
const sourceExists = await storage.collectionExists(fromDb, fromColl);
|
||||
if (!sourceExists) {
|
||||
return {
|
||||
ok: 0,
|
||||
errmsg: `source namespace ${from} does not exist`,
|
||||
code: 26,
|
||||
codeName: 'NamespaceNotFound',
|
||||
};
|
||||
}
|
||||
|
||||
// Check if target exists
|
||||
const targetExists = await storage.collectionExists(toDb, toColl);
|
||||
if (targetExists) {
|
||||
if (dropTarget) {
|
||||
await storage.dropCollection(toDb, toColl);
|
||||
} else {
|
||||
return {
|
||||
ok: 0,
|
||||
errmsg: `target namespace ${to} already exists`,
|
||||
code: 48,
|
||||
codeName: 'NamespaceExists',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Same database rename
|
||||
if (fromDb === toDb) {
|
||||
await storage.renameCollection(fromDb, fromColl, toColl);
|
||||
} else {
|
||||
// Cross-database rename: copy documents then drop source
|
||||
await storage.createCollection(toDb, toColl);
|
||||
const docs = await storage.findAll(fromDb, fromColl);
|
||||
|
||||
for (const doc of docs) {
|
||||
await storage.insertOne(toDb, toColl, doc);
|
||||
}
|
||||
|
||||
await storage.dropCollection(fromDb, fromColl);
|
||||
}
|
||||
|
||||
return { ok: 1 };
|
||||
}
|
||||
}
|
||||
342
ts/ts_tsmdb/server/handlers/AggregateHandler.ts
Normal file
342
ts/ts_tsmdb/server/handlers/AggregateHandler.ts
Normal file
@@ -0,0 +1,342 @@
|
||||
import * as plugins from '../../plugins.js';
|
||||
import type { ICommandHandler, IHandlerContext, ICursorState } from '../CommandRouter.js';
|
||||
import { AggregationEngine } from '../../engine/AggregationEngine.js';
|
||||
|
||||
/**
|
||||
* AggregateHandler - Handles aggregate command
|
||||
*/
|
||||
export class AggregateHandler implements ICommandHandler {
|
||||
private cursors: Map<bigint, ICursorState>;
|
||||
private nextCursorId: () => bigint;
|
||||
|
||||
constructor(
|
||||
cursors: Map<bigint, ICursorState>,
|
||||
nextCursorId: () => bigint
|
||||
) {
|
||||
this.cursors = cursors;
|
||||
this.nextCursorId = nextCursorId;
|
||||
}
|
||||
|
||||
async handle(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
const { storage, database, command } = context;
|
||||
|
||||
const collection = command.aggregate;
|
||||
const pipeline = command.pipeline || [];
|
||||
const cursor = command.cursor || {};
|
||||
const batchSize = cursor.batchSize || 101;
|
||||
|
||||
// Validate
|
||||
if (typeof collection !== 'string' && collection !== 1) {
|
||||
return {
|
||||
ok: 0,
|
||||
errmsg: 'aggregate command requires a collection name or 1',
|
||||
code: 2,
|
||||
codeName: 'BadValue',
|
||||
};
|
||||
}
|
||||
|
||||
if (!Array.isArray(pipeline)) {
|
||||
return {
|
||||
ok: 0,
|
||||
errmsg: 'pipeline must be an array',
|
||||
code: 2,
|
||||
codeName: 'BadValue',
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
// Get source documents
|
||||
let documents: plugins.bson.Document[] = [];
|
||||
|
||||
if (collection === 1 || collection === '1') {
|
||||
// Database-level aggregation (e.g., $listLocalSessions)
|
||||
documents = [];
|
||||
} else {
|
||||
// Collection-level aggregation
|
||||
const exists = await storage.collectionExists(database, collection);
|
||||
if (exists) {
|
||||
documents = await storage.findAll(database, collection);
|
||||
}
|
||||
}
|
||||
|
||||
// Handle $lookup and $graphLookup stages that reference other collections
|
||||
const processedPipeline = await this.preprocessPipeline(
|
||||
storage,
|
||||
database,
|
||||
pipeline,
|
||||
documents
|
||||
);
|
||||
|
||||
// Run aggregation
|
||||
let results: plugins.bson.Document[];
|
||||
|
||||
// Check for special stages that we handle manually
|
||||
if (this.hasSpecialStages(pipeline)) {
|
||||
results = await this.executeWithSpecialStages(
|
||||
storage,
|
||||
database,
|
||||
documents,
|
||||
pipeline
|
||||
);
|
||||
} else {
|
||||
results = AggregationEngine.aggregate(documents as any, processedPipeline);
|
||||
}
|
||||
|
||||
// Handle $out and $merge stages
|
||||
const lastStage = pipeline[pipeline.length - 1];
|
||||
if (lastStage && lastStage.$out) {
|
||||
await this.handleOut(storage, database, results, lastStage.$out);
|
||||
return { ok: 1, cursor: { id: plugins.bson.Long.fromNumber(0), ns: `${database}.${collection}`, firstBatch: [] } };
|
||||
}
|
||||
|
||||
if (lastStage && lastStage.$merge) {
|
||||
await this.handleMerge(storage, database, results, lastStage.$merge);
|
||||
return { ok: 1, cursor: { id: plugins.bson.Long.fromNumber(0), ns: `${database}.${collection}`, firstBatch: [] } };
|
||||
}
|
||||
|
||||
// Build cursor response
|
||||
const effectiveBatchSize = Math.min(batchSize, results.length);
|
||||
const firstBatch = results.slice(0, effectiveBatchSize);
|
||||
const remaining = results.slice(effectiveBatchSize);
|
||||
|
||||
let cursorId = BigInt(0);
|
||||
if (remaining.length > 0) {
|
||||
cursorId = this.nextCursorId();
|
||||
this.cursors.set(cursorId, {
|
||||
id: cursorId,
|
||||
database,
|
||||
collection: typeof collection === 'string' ? collection : '$cmd.aggregate',
|
||||
documents: remaining,
|
||||
position: 0,
|
||||
batchSize,
|
||||
createdAt: new Date(),
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
ok: 1,
|
||||
cursor: {
|
||||
id: plugins.bson.Long.fromBigInt(cursorId),
|
||||
ns: `${database}.${typeof collection === 'string' ? collection : '$cmd.aggregate'}`,
|
||||
firstBatch,
|
||||
},
|
||||
};
|
||||
} catch (error: any) {
|
||||
return {
|
||||
ok: 0,
|
||||
errmsg: error.message || 'Aggregation failed',
|
||||
code: 1,
|
||||
codeName: 'InternalError',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Preprocess pipeline to handle cross-collection lookups
|
||||
*/
|
||||
private async preprocessPipeline(
|
||||
storage: any,
|
||||
database: string,
|
||||
pipeline: plugins.bson.Document[],
|
||||
documents: plugins.bson.Document[]
|
||||
): Promise<plugins.bson.Document[]> {
|
||||
// For now, return the pipeline as-is
|
||||
// Cross-collection lookups are handled in executeWithSpecialStages
|
||||
return pipeline;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if pipeline has stages that need special handling
|
||||
*/
|
||||
private hasSpecialStages(pipeline: plugins.bson.Document[]): boolean {
|
||||
return pipeline.some(stage =>
|
||||
stage.$lookup ||
|
||||
stage.$graphLookup ||
|
||||
stage.$unionWith
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute pipeline with special stage handling
|
||||
*/
|
||||
private async executeWithSpecialStages(
|
||||
storage: any,
|
||||
database: string,
|
||||
documents: plugins.bson.Document[],
|
||||
pipeline: plugins.bson.Document[]
|
||||
): Promise<plugins.bson.Document[]> {
|
||||
let results: plugins.bson.Document[] = [...documents];
|
||||
|
||||
for (const stage of pipeline) {
|
||||
if (stage.$lookup) {
|
||||
const lookupSpec = stage.$lookup;
|
||||
const fromCollection = lookupSpec.from;
|
||||
|
||||
// Get foreign collection documents
|
||||
const foreignExists = await storage.collectionExists(database, fromCollection);
|
||||
const foreignDocs = foreignExists
|
||||
? await storage.findAll(database, fromCollection)
|
||||
: [];
|
||||
|
||||
results = AggregationEngine.executeLookup(results as any, lookupSpec, foreignDocs);
|
||||
} else if (stage.$graphLookup) {
|
||||
const graphLookupSpec = stage.$graphLookup;
|
||||
const fromCollection = graphLookupSpec.from;
|
||||
|
||||
const foreignExists = await storage.collectionExists(database, fromCollection);
|
||||
const foreignDocs = foreignExists
|
||||
? await storage.findAll(database, fromCollection)
|
||||
: [];
|
||||
|
||||
results = AggregationEngine.executeGraphLookup(results as any, graphLookupSpec, foreignDocs);
|
||||
} else if (stage.$unionWith) {
|
||||
let unionSpec = stage.$unionWith;
|
||||
let unionColl: string;
|
||||
let unionPipeline: plugins.bson.Document[] | undefined;
|
||||
|
||||
if (typeof unionSpec === 'string') {
|
||||
unionColl = unionSpec;
|
||||
} else {
|
||||
unionColl = unionSpec.coll;
|
||||
unionPipeline = unionSpec.pipeline;
|
||||
}
|
||||
|
||||
const unionExists = await storage.collectionExists(database, unionColl);
|
||||
const unionDocs = unionExists
|
||||
? await storage.findAll(database, unionColl)
|
||||
: [];
|
||||
|
||||
results = AggregationEngine.executeUnionWith(results as any, unionDocs, unionPipeline);
|
||||
} else if (stage.$facet) {
|
||||
// Execute each facet pipeline separately
|
||||
const facetResults: plugins.bson.Document = {};
|
||||
|
||||
for (const [facetName, facetPipeline] of Object.entries(stage.$facet)) {
|
||||
const facetDocs = await this.executeWithSpecialStages(
|
||||
storage,
|
||||
database,
|
||||
results,
|
||||
facetPipeline as plugins.bson.Document[]
|
||||
);
|
||||
facetResults[facetName] = facetDocs;
|
||||
}
|
||||
|
||||
results = [facetResults];
|
||||
} else {
|
||||
// Regular stage - pass to mingo
|
||||
results = AggregationEngine.aggregate(results as any, [stage]);
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle $out stage - write results to a collection
|
||||
*/
|
||||
private async handleOut(
|
||||
storage: any,
|
||||
database: string,
|
||||
results: plugins.bson.Document[],
|
||||
outSpec: string | { db?: string; coll: string }
|
||||
): Promise<void> {
|
||||
let targetDb = database;
|
||||
let targetColl: string;
|
||||
|
||||
if (typeof outSpec === 'string') {
|
||||
targetColl = outSpec;
|
||||
} else {
|
||||
targetDb = outSpec.db || database;
|
||||
targetColl = outSpec.coll;
|
||||
}
|
||||
|
||||
// Drop existing collection
|
||||
await storage.dropCollection(targetDb, targetColl);
|
||||
|
||||
// Create new collection and insert results
|
||||
await storage.createCollection(targetDb, targetColl);
|
||||
|
||||
for (const doc of results) {
|
||||
if (!doc._id) {
|
||||
doc._id = new plugins.bson.ObjectId();
|
||||
}
|
||||
await storage.insertOne(targetDb, targetColl, doc);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle $merge stage - merge results into a collection
|
||||
*/
|
||||
private async handleMerge(
|
||||
storage: any,
|
||||
database: string,
|
||||
results: plugins.bson.Document[],
|
||||
mergeSpec: any
|
||||
): Promise<void> {
|
||||
let targetDb = database;
|
||||
let targetColl: string;
|
||||
|
||||
if (typeof mergeSpec === 'string') {
|
||||
targetColl = mergeSpec;
|
||||
} else if (typeof mergeSpec.into === 'string') {
|
||||
targetColl = mergeSpec.into;
|
||||
} else {
|
||||
targetDb = mergeSpec.into.db || database;
|
||||
targetColl = mergeSpec.into.coll;
|
||||
}
|
||||
|
||||
const on = mergeSpec.on || '_id';
|
||||
const whenMatched = mergeSpec.whenMatched || 'merge';
|
||||
const whenNotMatched = mergeSpec.whenNotMatched || 'insert';
|
||||
|
||||
// Ensure target collection exists
|
||||
await storage.createCollection(targetDb, targetColl);
|
||||
|
||||
for (const doc of results) {
|
||||
// Find matching document
|
||||
const existingDocs = await storage.findAll(targetDb, targetColl);
|
||||
const onFields = Array.isArray(on) ? on : [on];
|
||||
|
||||
let matchingDoc = null;
|
||||
for (const existing of existingDocs) {
|
||||
let matches = true;
|
||||
for (const field of onFields) {
|
||||
if (JSON.stringify(existing[field]) !== JSON.stringify(doc[field])) {
|
||||
matches = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (matches) {
|
||||
matchingDoc = existing;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (matchingDoc) {
|
||||
// Handle whenMatched
|
||||
if (whenMatched === 'replace') {
|
||||
await storage.updateById(targetDb, targetColl, matchingDoc._id, doc);
|
||||
} else if (whenMatched === 'keepExisting') {
|
||||
// Do nothing
|
||||
} else if (whenMatched === 'merge') {
|
||||
const merged = { ...matchingDoc, ...doc };
|
||||
await storage.updateById(targetDb, targetColl, matchingDoc._id, merged);
|
||||
} else if (whenMatched === 'fail') {
|
||||
throw new Error('Document matched but whenMatched is fail');
|
||||
}
|
||||
} else {
|
||||
// Handle whenNotMatched
|
||||
if (whenNotMatched === 'insert') {
|
||||
if (!doc._id) {
|
||||
doc._id = new plugins.bson.ObjectId();
|
||||
}
|
||||
await storage.insertOne(targetDb, targetColl, doc);
|
||||
} else if (whenNotMatched === 'discard') {
|
||||
// Do nothing
|
||||
} else if (whenNotMatched === 'fail') {
|
||||
throw new Error('Document not matched but whenNotMatched is fail');
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
115
ts/ts_tsmdb/server/handlers/DeleteHandler.ts
Normal file
115
ts/ts_tsmdb/server/handlers/DeleteHandler.ts
Normal file
@@ -0,0 +1,115 @@
|
||||
import * as plugins from '../../plugins.js';
|
||||
import type { ICommandHandler, IHandlerContext } from '../CommandRouter.js';
|
||||
import type { IStoredDocument } from '../../types/interfaces.js';
|
||||
import { QueryEngine } from '../../engine/QueryEngine.js';
|
||||
|
||||
/**
|
||||
* DeleteHandler - Handles delete commands
|
||||
*/
|
||||
export class DeleteHandler implements ICommandHandler {
|
||||
async handle(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
const { storage, database, command, documentSequences } = context;
|
||||
|
||||
const collection = command.delete;
|
||||
if (typeof collection !== 'string') {
|
||||
return {
|
||||
ok: 0,
|
||||
errmsg: 'delete command requires a collection name',
|
||||
code: 2,
|
||||
codeName: 'BadValue',
|
||||
};
|
||||
}
|
||||
|
||||
// Get deletes from command or document sequences
|
||||
let deletes: plugins.bson.Document[] = command.deletes || [];
|
||||
|
||||
// Check for OP_MSG document sequences
|
||||
if (documentSequences && documentSequences.has('deletes')) {
|
||||
deletes = documentSequences.get('deletes')!;
|
||||
}
|
||||
|
||||
if (!Array.isArray(deletes) || deletes.length === 0) {
|
||||
return {
|
||||
ok: 0,
|
||||
errmsg: 'delete command requires deletes array',
|
||||
code: 2,
|
||||
codeName: 'BadValue',
|
||||
};
|
||||
}
|
||||
|
||||
const ordered = command.ordered !== false;
|
||||
const writeErrors: plugins.bson.Document[] = [];
|
||||
let totalDeleted = 0;
|
||||
|
||||
// Check if collection exists
|
||||
const exists = await storage.collectionExists(database, collection);
|
||||
if (!exists) {
|
||||
// Collection doesn't exist, return success with 0 deleted
|
||||
return { ok: 1, n: 0 };
|
||||
}
|
||||
|
||||
const indexEngine = context.getIndexEngine(collection);
|
||||
|
||||
for (let i = 0; i < deletes.length; i++) {
|
||||
const deleteSpec = deletes[i];
|
||||
const filter = deleteSpec.q || deleteSpec.filter || {};
|
||||
const limit = deleteSpec.limit;
|
||||
|
||||
// limit: 0 means delete all matching, limit: 1 means delete one
|
||||
const deleteAll = limit === 0;
|
||||
|
||||
try {
|
||||
// Try to use index-accelerated query
|
||||
const candidateIds = await indexEngine.findCandidateIds(filter);
|
||||
|
||||
let documents: IStoredDocument[];
|
||||
if (candidateIds !== null) {
|
||||
documents = await storage.findByIds(database, collection, candidateIds);
|
||||
} else {
|
||||
documents = await storage.findAll(database, collection);
|
||||
}
|
||||
|
||||
// Apply filter
|
||||
const matchingDocs = QueryEngine.filter(documents, filter);
|
||||
|
||||
if (matchingDocs.length === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Determine which documents to delete
|
||||
const docsToDelete = deleteAll ? matchingDocs : matchingDocs.slice(0, 1);
|
||||
|
||||
// Update indexes for deleted documents
|
||||
for (const doc of docsToDelete) {
|
||||
await indexEngine.onDelete(doc as any);
|
||||
}
|
||||
|
||||
// Delete the documents
|
||||
const idsToDelete = docsToDelete.map(doc => doc._id);
|
||||
const deleted = await storage.deleteByIds(database, collection, idsToDelete);
|
||||
totalDeleted += deleted;
|
||||
} catch (error: any) {
|
||||
writeErrors.push({
|
||||
index: i,
|
||||
code: error.code || 1,
|
||||
errmsg: error.message || 'Delete failed',
|
||||
});
|
||||
|
||||
if (ordered) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const response: plugins.bson.Document = {
|
||||
ok: 1,
|
||||
n: totalDeleted,
|
||||
};
|
||||
|
||||
if (writeErrors.length > 0) {
|
||||
response.writeErrors = writeErrors;
|
||||
}
|
||||
|
||||
return response;
|
||||
}
|
||||
}
|
||||
330
ts/ts_tsmdb/server/handlers/FindHandler.ts
Normal file
330
ts/ts_tsmdb/server/handlers/FindHandler.ts
Normal file
@@ -0,0 +1,330 @@
|
||||
import * as plugins from '../../plugins.js';
|
||||
import type { ICommandHandler, IHandlerContext, ICursorState } from '../CommandRouter.js';
|
||||
import type { IStoredDocument } from '../../types/interfaces.js';
|
||||
import { QueryEngine } from '../../engine/QueryEngine.js';
|
||||
|
||||
/**
|
||||
* FindHandler - Handles find, getMore, killCursors, count, distinct commands
|
||||
*/
|
||||
export class FindHandler implements ICommandHandler {
|
||||
private cursors: Map<bigint, ICursorState>;
|
||||
private nextCursorId: () => bigint;
|
||||
|
||||
constructor(
|
||||
cursors: Map<bigint, ICursorState>,
|
||||
nextCursorId: () => bigint
|
||||
) {
|
||||
this.cursors = cursors;
|
||||
this.nextCursorId = nextCursorId;
|
||||
}
|
||||
|
||||
async handle(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
const { command } = context;
|
||||
|
||||
// Determine which operation to perform
|
||||
if (command.find) {
|
||||
return this.handleFind(context);
|
||||
} else if (command.getMore !== undefined) {
|
||||
return this.handleGetMore(context);
|
||||
} else if (command.killCursors) {
|
||||
return this.handleKillCursors(context);
|
||||
} else if (command.count) {
|
||||
return this.handleCount(context);
|
||||
} else if (command.distinct) {
|
||||
return this.handleDistinct(context);
|
||||
}
|
||||
|
||||
return {
|
||||
ok: 0,
|
||||
errmsg: 'Unknown find-related command',
|
||||
code: 59,
|
||||
codeName: 'CommandNotFound',
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle find command
|
||||
*/
|
||||
private async handleFind(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
const { storage, database, command, getIndexEngine } = context;
|
||||
|
||||
const collection = command.find;
|
||||
const filter = command.filter || {};
|
||||
const projection = command.projection;
|
||||
const sort = command.sort;
|
||||
const skip = command.skip || 0;
|
||||
const limit = command.limit || 0;
|
||||
const batchSize = command.batchSize || 101;
|
||||
const singleBatch = command.singleBatch || false;
|
||||
|
||||
// Ensure collection exists
|
||||
const exists = await storage.collectionExists(database, collection);
|
||||
if (!exists) {
|
||||
// Return empty cursor for non-existent collection
|
||||
return {
|
||||
ok: 1,
|
||||
cursor: {
|
||||
id: plugins.bson.Long.fromNumber(0),
|
||||
ns: `${database}.${collection}`,
|
||||
firstBatch: [],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// Try to use index-accelerated query
|
||||
const indexEngine = getIndexEngine(collection);
|
||||
const candidateIds = await indexEngine.findCandidateIds(filter);
|
||||
|
||||
let documents: IStoredDocument[];
|
||||
if (candidateIds !== null) {
|
||||
// Index hit - fetch only candidate documents
|
||||
documents = await storage.findByIds(database, collection, candidateIds);
|
||||
// Still apply filter for any conditions the index couldn't fully satisfy
|
||||
documents = QueryEngine.filter(documents, filter);
|
||||
} else {
|
||||
// No suitable index - full collection scan
|
||||
documents = await storage.findAll(database, collection);
|
||||
// Apply filter
|
||||
documents = QueryEngine.filter(documents, filter);
|
||||
}
|
||||
|
||||
// Apply sort
|
||||
if (sort) {
|
||||
documents = QueryEngine.sort(documents, sort);
|
||||
}
|
||||
|
||||
// Apply skip
|
||||
if (skip > 0) {
|
||||
documents = documents.slice(skip);
|
||||
}
|
||||
|
||||
// Apply limit
|
||||
if (limit > 0) {
|
||||
documents = documents.slice(0, limit);
|
||||
}
|
||||
|
||||
// Apply projection
|
||||
if (projection) {
|
||||
documents = QueryEngine.project(documents, projection) as any[];
|
||||
}
|
||||
|
||||
// Determine how many documents to return in first batch
|
||||
const effectiveBatchSize = Math.min(batchSize, documents.length);
|
||||
const firstBatch = documents.slice(0, effectiveBatchSize);
|
||||
const remaining = documents.slice(effectiveBatchSize);
|
||||
|
||||
// Create cursor if there are more documents
|
||||
let cursorId = BigInt(0);
|
||||
if (remaining.length > 0 && !singleBatch) {
|
||||
cursorId = this.nextCursorId();
|
||||
this.cursors.set(cursorId, {
|
||||
id: cursorId,
|
||||
database,
|
||||
collection,
|
||||
documents: remaining,
|
||||
position: 0,
|
||||
batchSize,
|
||||
createdAt: new Date(),
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
ok: 1,
|
||||
cursor: {
|
||||
id: plugins.bson.Long.fromBigInt(cursorId),
|
||||
ns: `${database}.${collection}`,
|
||||
firstBatch,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle getMore command
|
||||
*/
|
||||
private async handleGetMore(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
const { database, command } = context;
|
||||
|
||||
const cursorIdInput = command.getMore;
|
||||
const collection = command.collection;
|
||||
const batchSize = command.batchSize || 101;
|
||||
|
||||
// Convert cursorId to bigint
|
||||
let cursorId: bigint;
|
||||
if (typeof cursorIdInput === 'bigint') {
|
||||
cursorId = cursorIdInput;
|
||||
} else if (cursorIdInput instanceof plugins.bson.Long) {
|
||||
cursorId = cursorIdInput.toBigInt();
|
||||
} else {
|
||||
cursorId = BigInt(cursorIdInput);
|
||||
}
|
||||
|
||||
const cursor = this.cursors.get(cursorId);
|
||||
if (!cursor) {
|
||||
return {
|
||||
ok: 0,
|
||||
errmsg: `cursor id ${cursorId} not found`,
|
||||
code: 43,
|
||||
codeName: 'CursorNotFound',
|
||||
};
|
||||
}
|
||||
|
||||
// Verify namespace
|
||||
if (cursor.database !== database || cursor.collection !== collection) {
|
||||
return {
|
||||
ok: 0,
|
||||
errmsg: 'cursor namespace mismatch',
|
||||
code: 43,
|
||||
codeName: 'CursorNotFound',
|
||||
};
|
||||
}
|
||||
|
||||
// Get next batch
|
||||
const start = cursor.position;
|
||||
const end = Math.min(start + batchSize, cursor.documents.length);
|
||||
const nextBatch = cursor.documents.slice(start, end);
|
||||
cursor.position = end;
|
||||
|
||||
// Check if cursor is exhausted
|
||||
let returnCursorId = cursorId;
|
||||
if (cursor.position >= cursor.documents.length) {
|
||||
this.cursors.delete(cursorId);
|
||||
returnCursorId = BigInt(0);
|
||||
}
|
||||
|
||||
return {
|
||||
ok: 1,
|
||||
cursor: {
|
||||
id: plugins.bson.Long.fromBigInt(returnCursorId),
|
||||
ns: `${database}.${collection}`,
|
||||
nextBatch,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle killCursors command
|
||||
*/
|
||||
private async handleKillCursors(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
const { command } = context;
|
||||
|
||||
const collection = command.killCursors;
|
||||
const cursorIds = command.cursors || [];
|
||||
|
||||
const cursorsKilled: plugins.bson.Long[] = [];
|
||||
const cursorsNotFound: plugins.bson.Long[] = [];
|
||||
const cursorsUnknown: plugins.bson.Long[] = [];
|
||||
|
||||
for (const idInput of cursorIds) {
|
||||
let cursorId: bigint;
|
||||
if (typeof idInput === 'bigint') {
|
||||
cursorId = idInput;
|
||||
} else if (idInput instanceof plugins.bson.Long) {
|
||||
cursorId = idInput.toBigInt();
|
||||
} else {
|
||||
cursorId = BigInt(idInput);
|
||||
}
|
||||
|
||||
if (this.cursors.has(cursorId)) {
|
||||
this.cursors.delete(cursorId);
|
||||
cursorsKilled.push(plugins.bson.Long.fromBigInt(cursorId));
|
||||
} else {
|
||||
cursorsNotFound.push(plugins.bson.Long.fromBigInt(cursorId));
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
ok: 1,
|
||||
cursorsKilled,
|
||||
cursorsNotFound,
|
||||
cursorsUnknown,
|
||||
cursorsAlive: [],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle count command
|
||||
*/
|
||||
private async handleCount(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
const { storage, database, command, getIndexEngine } = context;
|
||||
|
||||
const collection = command.count;
|
||||
const query = command.query || {};
|
||||
const skip = command.skip || 0;
|
||||
const limit = command.limit || 0;
|
||||
|
||||
// Check if collection exists
|
||||
const exists = await storage.collectionExists(database, collection);
|
||||
if (!exists) {
|
||||
return { ok: 1, n: 0 };
|
||||
}
|
||||
|
||||
// Try to use index-accelerated query
|
||||
const indexEngine = getIndexEngine(collection);
|
||||
const candidateIds = await indexEngine.findCandidateIds(query);
|
||||
|
||||
let documents: IStoredDocument[];
|
||||
if (candidateIds !== null) {
|
||||
// Index hit - fetch only candidate documents
|
||||
documents = await storage.findByIds(database, collection, candidateIds);
|
||||
documents = QueryEngine.filter(documents, query);
|
||||
} else {
|
||||
// No suitable index - full collection scan
|
||||
documents = await storage.findAll(database, collection);
|
||||
documents = QueryEngine.filter(documents, query);
|
||||
}
|
||||
|
||||
// Apply skip
|
||||
if (skip > 0) {
|
||||
documents = documents.slice(skip);
|
||||
}
|
||||
|
||||
// Apply limit
|
||||
if (limit > 0) {
|
||||
documents = documents.slice(0, limit);
|
||||
}
|
||||
|
||||
return { ok: 1, n: documents.length };
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle distinct command
|
||||
*/
|
||||
private async handleDistinct(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
const { storage, database, command, getIndexEngine } = context;
|
||||
|
||||
const collection = command.distinct;
|
||||
const key = command.key;
|
||||
const query = command.query || {};
|
||||
|
||||
if (!key) {
|
||||
return {
|
||||
ok: 0,
|
||||
errmsg: 'distinct requires a key',
|
||||
code: 2,
|
||||
codeName: 'BadValue',
|
||||
};
|
||||
}
|
||||
|
||||
// Check if collection exists
|
||||
const exists = await storage.collectionExists(database, collection);
|
||||
if (!exists) {
|
||||
return { ok: 1, values: [] };
|
||||
}
|
||||
|
||||
// Try to use index-accelerated query
|
||||
const indexEngine = getIndexEngine(collection);
|
||||
const candidateIds = await indexEngine.findCandidateIds(query);
|
||||
|
||||
let documents: IStoredDocument[];
|
||||
if (candidateIds !== null) {
|
||||
documents = await storage.findByIds(database, collection, candidateIds);
|
||||
} else {
|
||||
documents = await storage.findAll(database, collection);
|
||||
}
|
||||
|
||||
// Get distinct values
|
||||
const values = QueryEngine.distinct(documents, key, query);
|
||||
|
||||
return { ok: 1, values };
|
||||
}
|
||||
}
|
||||
78
ts/ts_tsmdb/server/handlers/HelloHandler.ts
Normal file
78
ts/ts_tsmdb/server/handlers/HelloHandler.ts
Normal file
@@ -0,0 +1,78 @@
|
||||
import * as plugins from '../../plugins.js';
|
||||
import type { ICommandHandler, IHandlerContext } from '../CommandRouter.js';
|
||||
|
||||
/**
|
||||
* HelloHandler - Handles hello/isMaster handshake commands
|
||||
*
|
||||
* This is the first command sent by MongoDB drivers to establish a connection.
|
||||
* It returns server capabilities and configuration.
|
||||
*/
|
||||
export class HelloHandler implements ICommandHandler {
|
||||
async handle(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
const { command, server } = context;
|
||||
|
||||
// Build response with server capabilities
|
||||
const response: plugins.bson.Document = {
|
||||
ismaster: true,
|
||||
ok: 1,
|
||||
|
||||
// Maximum sizes
|
||||
maxBsonObjectSize: 16777216, // 16 MB
|
||||
maxMessageSizeBytes: 48000000, // 48 MB
|
||||
maxWriteBatchSize: 100000, // 100k documents per batch
|
||||
|
||||
// Timestamps
|
||||
localTime: new Date(),
|
||||
|
||||
// Session support
|
||||
logicalSessionTimeoutMinutes: 30,
|
||||
|
||||
// Connection info
|
||||
connectionId: 1,
|
||||
|
||||
// Wire protocol versions (support MongoDB 3.6 through 7.0)
|
||||
minWireVersion: 0,
|
||||
maxWireVersion: 21,
|
||||
|
||||
// Server mode
|
||||
readOnly: false,
|
||||
|
||||
// Topology info (standalone mode)
|
||||
isWritablePrimary: true,
|
||||
|
||||
// Additional info
|
||||
topologyVersion: {
|
||||
processId: new plugins.bson.ObjectId(),
|
||||
counter: plugins.bson.Long.fromNumber(0),
|
||||
},
|
||||
};
|
||||
|
||||
// Handle hello-specific fields
|
||||
if (command.hello || command.hello === 1) {
|
||||
response.helloOk = true;
|
||||
}
|
||||
|
||||
// Handle client metadata
|
||||
if (command.client) {
|
||||
// Client is providing metadata about itself
|
||||
// We just acknowledge it - no need to do anything special
|
||||
}
|
||||
|
||||
// Handle SASL mechanisms query
|
||||
if (command.saslSupportedMechs) {
|
||||
response.saslSupportedMechs = [
|
||||
// We don't actually support auth, but the driver needs to see this
|
||||
];
|
||||
}
|
||||
|
||||
// Compression support (none for now)
|
||||
if (command.compression) {
|
||||
response.compression = [];
|
||||
}
|
||||
|
||||
// Server version info
|
||||
response.version = '7.0.0';
|
||||
|
||||
return response;
|
||||
}
|
||||
}
|
||||
207
ts/ts_tsmdb/server/handlers/IndexHandler.ts
Normal file
207
ts/ts_tsmdb/server/handlers/IndexHandler.ts
Normal file
@@ -0,0 +1,207 @@
|
||||
import * as plugins from '../../plugins.js';
|
||||
import type { ICommandHandler, IHandlerContext } from '../CommandRouter.js';
|
||||
import { IndexEngine } from '../../engine/IndexEngine.js';
|
||||
|
||||
// Cache of index engines per collection
|
||||
const indexEngines: Map<string, IndexEngine> = new Map();
|
||||
|
||||
/**
|
||||
* Get or create an IndexEngine for a collection
|
||||
*/
|
||||
function getIndexEngine(storage: any, database: string, collection: string): IndexEngine {
|
||||
const key = `${database}.${collection}`;
|
||||
let engine = indexEngines.get(key);
|
||||
|
||||
if (!engine) {
|
||||
engine = new IndexEngine(database, collection, storage);
|
||||
indexEngines.set(key, engine);
|
||||
}
|
||||
|
||||
return engine;
|
||||
}
|
||||
|
||||
/**
|
||||
* IndexHandler - Handles createIndexes, dropIndexes, listIndexes commands
|
||||
*/
|
||||
export class IndexHandler implements ICommandHandler {
|
||||
async handle(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
const { command } = context;
|
||||
|
||||
if (command.createIndexes) {
|
||||
return this.handleCreateIndexes(context);
|
||||
} else if (command.dropIndexes) {
|
||||
return this.handleDropIndexes(context);
|
||||
} else if (command.listIndexes) {
|
||||
return this.handleListIndexes(context);
|
||||
}
|
||||
|
||||
return {
|
||||
ok: 0,
|
||||
errmsg: 'Unknown index command',
|
||||
code: 59,
|
||||
codeName: 'CommandNotFound',
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle createIndexes command
|
||||
*/
|
||||
private async handleCreateIndexes(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
const { storage, database, command } = context;
|
||||
|
||||
const collection = command.createIndexes;
|
||||
const indexes = command.indexes || [];
|
||||
|
||||
if (!Array.isArray(indexes)) {
|
||||
return {
|
||||
ok: 0,
|
||||
errmsg: 'indexes must be an array',
|
||||
code: 2,
|
||||
codeName: 'BadValue',
|
||||
};
|
||||
}
|
||||
|
||||
// Ensure collection exists
|
||||
await storage.createCollection(database, collection);
|
||||
|
||||
const indexEngine = getIndexEngine(storage, database, collection);
|
||||
const createdNames: string[] = [];
|
||||
let numIndexesBefore = 0;
|
||||
let numIndexesAfter = 0;
|
||||
|
||||
try {
|
||||
const existingIndexes = await indexEngine.listIndexes();
|
||||
numIndexesBefore = existingIndexes.length;
|
||||
|
||||
for (const indexSpec of indexes) {
|
||||
const key = indexSpec.key;
|
||||
const options = {
|
||||
name: indexSpec.name,
|
||||
unique: indexSpec.unique,
|
||||
sparse: indexSpec.sparse,
|
||||
expireAfterSeconds: indexSpec.expireAfterSeconds,
|
||||
background: indexSpec.background,
|
||||
partialFilterExpression: indexSpec.partialFilterExpression,
|
||||
};
|
||||
|
||||
const name = await indexEngine.createIndex(key, options);
|
||||
createdNames.push(name);
|
||||
}
|
||||
|
||||
const finalIndexes = await indexEngine.listIndexes();
|
||||
numIndexesAfter = finalIndexes.length;
|
||||
} catch (error: any) {
|
||||
return {
|
||||
ok: 0,
|
||||
errmsg: error.message || 'Failed to create index',
|
||||
code: error.code || 1,
|
||||
codeName: error.codeName || 'InternalError',
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
ok: 1,
|
||||
numIndexesBefore,
|
||||
numIndexesAfter,
|
||||
createdCollectionAutomatically: false,
|
||||
commitQuorum: 'votingMembers',
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle dropIndexes command
|
||||
*/
|
||||
private async handleDropIndexes(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
const { storage, database, command } = context;
|
||||
|
||||
const collection = command.dropIndexes;
|
||||
const indexName = command.index;
|
||||
|
||||
// Check if collection exists
|
||||
const exists = await storage.collectionExists(database, collection);
|
||||
if (!exists) {
|
||||
return {
|
||||
ok: 0,
|
||||
errmsg: `ns not found ${database}.${collection}`,
|
||||
code: 26,
|
||||
codeName: 'NamespaceNotFound',
|
||||
};
|
||||
}
|
||||
|
||||
const indexEngine = getIndexEngine(storage, database, collection);
|
||||
|
||||
try {
|
||||
if (indexName === '*') {
|
||||
// Drop all indexes except _id
|
||||
await indexEngine.dropAllIndexes();
|
||||
} else if (typeof indexName === 'string') {
|
||||
// Drop specific index by name
|
||||
await indexEngine.dropIndex(indexName);
|
||||
} else if (typeof indexName === 'object') {
|
||||
// Drop index by key specification
|
||||
const indexes = await indexEngine.listIndexes();
|
||||
const keyStr = JSON.stringify(indexName);
|
||||
|
||||
for (const idx of indexes) {
|
||||
if (JSON.stringify(idx.key) === keyStr) {
|
||||
await indexEngine.dropIndex(idx.name);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { ok: 1, nIndexesWas: 1 };
|
||||
} catch (error: any) {
|
||||
return {
|
||||
ok: 0,
|
||||
errmsg: error.message || 'Failed to drop index',
|
||||
code: error.code || 27,
|
||||
codeName: error.codeName || 'IndexNotFound',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle listIndexes command
|
||||
*/
|
||||
private async handleListIndexes(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
const { storage, database, command } = context;
|
||||
|
||||
const collection = command.listIndexes;
|
||||
const cursor = command.cursor || {};
|
||||
const batchSize = cursor.batchSize || 101;
|
||||
|
||||
// Check if collection exists
|
||||
const exists = await storage.collectionExists(database, collection);
|
||||
if (!exists) {
|
||||
return {
|
||||
ok: 0,
|
||||
errmsg: `ns not found ${database}.${collection}`,
|
||||
code: 26,
|
||||
codeName: 'NamespaceNotFound',
|
||||
};
|
||||
}
|
||||
|
||||
const indexEngine = getIndexEngine(storage, database, collection);
|
||||
const indexes = await indexEngine.listIndexes();
|
||||
|
||||
// Format indexes for response
|
||||
const indexDocs = indexes.map(idx => ({
|
||||
v: idx.v || 2,
|
||||
key: idx.key,
|
||||
name: idx.name,
|
||||
...(idx.unique ? { unique: idx.unique } : {}),
|
||||
...(idx.sparse ? { sparse: idx.sparse } : {}),
|
||||
...(idx.expireAfterSeconds !== undefined ? { expireAfterSeconds: idx.expireAfterSeconds } : {}),
|
||||
}));
|
||||
|
||||
return {
|
||||
ok: 1,
|
||||
cursor: {
|
||||
id: plugins.bson.Long.fromNumber(0),
|
||||
ns: `${database}.${collection}`,
|
||||
firstBatch: indexDocs,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
97
ts/ts_tsmdb/server/handlers/InsertHandler.ts
Normal file
97
ts/ts_tsmdb/server/handlers/InsertHandler.ts
Normal file
@@ -0,0 +1,97 @@
|
||||
import * as plugins from '../../plugins.js';
|
||||
import type { ICommandHandler, IHandlerContext } from '../CommandRouter.js';
|
||||
import type { IStoredDocument } from '../../types/interfaces.js';
|
||||
|
||||
/**
|
||||
* InsertHandler - Handles insert commands
|
||||
*/
|
||||
export class InsertHandler implements ICommandHandler {
|
||||
async handle(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
const { storage, database, command, documentSequences } = context;
|
||||
|
||||
const collection = command.insert;
|
||||
if (typeof collection !== 'string') {
|
||||
return {
|
||||
ok: 0,
|
||||
errmsg: 'insert command requires a collection name',
|
||||
code: 2,
|
||||
codeName: 'BadValue',
|
||||
};
|
||||
}
|
||||
|
||||
// Get documents from command or document sequences
|
||||
let documents: plugins.bson.Document[] = command.documents || [];
|
||||
|
||||
// Check for OP_MSG document sequences (for bulk inserts)
|
||||
if (documentSequences && documentSequences.has('documents')) {
|
||||
documents = documentSequences.get('documents')!;
|
||||
}
|
||||
|
||||
if (!Array.isArray(documents) || documents.length === 0) {
|
||||
return {
|
||||
ok: 0,
|
||||
errmsg: 'insert command requires documents array',
|
||||
code: 2,
|
||||
codeName: 'BadValue',
|
||||
};
|
||||
}
|
||||
|
||||
const ordered = command.ordered !== false;
|
||||
const writeErrors: plugins.bson.Document[] = [];
|
||||
let insertedCount = 0;
|
||||
|
||||
// Ensure collection exists
|
||||
await storage.createCollection(database, collection);
|
||||
|
||||
const indexEngine = context.getIndexEngine(collection);
|
||||
|
||||
// Insert documents
|
||||
for (let i = 0; i < documents.length; i++) {
|
||||
const doc = documents[i];
|
||||
|
||||
try {
|
||||
// Ensure _id exists
|
||||
if (!doc._id) {
|
||||
doc._id = new plugins.bson.ObjectId();
|
||||
}
|
||||
|
||||
// Check index constraints before insert (doc now has _id)
|
||||
await indexEngine.onInsert(doc as IStoredDocument);
|
||||
|
||||
await storage.insertOne(database, collection, doc);
|
||||
insertedCount++;
|
||||
} catch (error: any) {
|
||||
const writeError: plugins.bson.Document = {
|
||||
index: i,
|
||||
code: error.code || 11000,
|
||||
errmsg: error.message || 'Insert failed',
|
||||
};
|
||||
|
||||
// Check for duplicate key error
|
||||
if (error.message?.includes('Duplicate key')) {
|
||||
writeError.code = 11000;
|
||||
writeError.keyPattern = { _id: 1 };
|
||||
writeError.keyValue = { _id: doc._id };
|
||||
}
|
||||
|
||||
writeErrors.push(writeError);
|
||||
|
||||
if (ordered) {
|
||||
// Stop on first error for ordered inserts
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const response: plugins.bson.Document = {
|
||||
ok: 1,
|
||||
n: insertedCount,
|
||||
};
|
||||
|
||||
if (writeErrors.length > 0) {
|
||||
response.writeErrors = writeErrors;
|
||||
}
|
||||
|
||||
return response;
|
||||
}
|
||||
}
|
||||
344
ts/ts_tsmdb/server/handlers/UpdateHandler.ts
Normal file
344
ts/ts_tsmdb/server/handlers/UpdateHandler.ts
Normal file
@@ -0,0 +1,344 @@
|
||||
import * as plugins from '../../plugins.js';
|
||||
import type { ICommandHandler, IHandlerContext } from '../CommandRouter.js';
|
||||
import type { IStoredDocument } from '../../types/interfaces.js';
|
||||
import { QueryEngine } from '../../engine/QueryEngine.js';
|
||||
import { UpdateEngine } from '../../engine/UpdateEngine.js';
|
||||
|
||||
/**
|
||||
* UpdateHandler - Handles update, findAndModify commands
|
||||
*/
|
||||
export class UpdateHandler implements ICommandHandler {
|
||||
async handle(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
const { command } = context;
|
||||
|
||||
// Check findAndModify first since it also has an 'update' field
|
||||
if (command.findAndModify) {
|
||||
return this.handleFindAndModify(context);
|
||||
} else if (command.update && typeof command.update === 'string') {
|
||||
// 'update' command has collection name as the value
|
||||
return this.handleUpdate(context);
|
||||
}
|
||||
|
||||
return {
|
||||
ok: 0,
|
||||
errmsg: 'Unknown update-related command',
|
||||
code: 59,
|
||||
codeName: 'CommandNotFound',
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle update command
|
||||
*/
|
||||
private async handleUpdate(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
const { storage, database, command, documentSequences } = context;
|
||||
|
||||
const collection = command.update;
|
||||
if (typeof collection !== 'string') {
|
||||
return {
|
||||
ok: 0,
|
||||
errmsg: 'update command requires a collection name',
|
||||
code: 2,
|
||||
codeName: 'BadValue',
|
||||
};
|
||||
}
|
||||
|
||||
// Get updates from command or document sequences
|
||||
let updates: plugins.bson.Document[] = command.updates || [];
|
||||
|
||||
// Check for OP_MSG document sequences
|
||||
if (documentSequences && documentSequences.has('updates')) {
|
||||
updates = documentSequences.get('updates')!;
|
||||
}
|
||||
|
||||
if (!Array.isArray(updates) || updates.length === 0) {
|
||||
return {
|
||||
ok: 0,
|
||||
errmsg: 'update command requires updates array',
|
||||
code: 2,
|
||||
codeName: 'BadValue',
|
||||
};
|
||||
}
|
||||
|
||||
const ordered = command.ordered !== false;
|
||||
const writeErrors: plugins.bson.Document[] = [];
|
||||
let totalMatched = 0;
|
||||
let totalModified = 0;
|
||||
let totalUpserted = 0;
|
||||
const upserted: plugins.bson.Document[] = [];
|
||||
|
||||
// Ensure collection exists
|
||||
await storage.createCollection(database, collection);
|
||||
|
||||
const indexEngine = context.getIndexEngine(collection);
|
||||
|
||||
for (let i = 0; i < updates.length; i++) {
|
||||
const updateSpec = updates[i];
|
||||
const filter = updateSpec.q || updateSpec.filter || {};
|
||||
const update = updateSpec.u || updateSpec.update || {};
|
||||
const multi = updateSpec.multi || false;
|
||||
const upsert = updateSpec.upsert || false;
|
||||
const arrayFilters = updateSpec.arrayFilters;
|
||||
|
||||
try {
|
||||
// Try to use index-accelerated query
|
||||
const candidateIds = await indexEngine.findCandidateIds(filter);
|
||||
|
||||
let documents: IStoredDocument[];
|
||||
if (candidateIds !== null) {
|
||||
documents = await storage.findByIds(database, collection, candidateIds);
|
||||
} else {
|
||||
documents = await storage.findAll(database, collection);
|
||||
}
|
||||
|
||||
// Apply filter
|
||||
let matchingDocs = QueryEngine.filter(documents, filter);
|
||||
|
||||
if (matchingDocs.length === 0 && upsert) {
|
||||
// Upsert: create new document
|
||||
const newDoc: plugins.bson.Document = { _id: new plugins.bson.ObjectId() };
|
||||
|
||||
// Apply filter fields to the new document
|
||||
this.applyFilterToDoc(newDoc, filter);
|
||||
|
||||
// Apply update
|
||||
const updatedDoc = UpdateEngine.applyUpdate(newDoc as any, update, arrayFilters);
|
||||
|
||||
// Handle $setOnInsert
|
||||
if (update.$setOnInsert) {
|
||||
Object.assign(updatedDoc, update.$setOnInsert);
|
||||
}
|
||||
|
||||
// Update index for the new document
|
||||
await indexEngine.onInsert(updatedDoc);
|
||||
await storage.insertOne(database, collection, updatedDoc);
|
||||
totalUpserted++;
|
||||
upserted.push({ index: i, _id: updatedDoc._id });
|
||||
} else {
|
||||
// Update existing documents
|
||||
const docsToUpdate = multi ? matchingDocs : matchingDocs.slice(0, 1);
|
||||
totalMatched += docsToUpdate.length;
|
||||
|
||||
for (const doc of docsToUpdate) {
|
||||
const updatedDoc = UpdateEngine.applyUpdate(doc, update, arrayFilters);
|
||||
|
||||
// Check if document actually changed
|
||||
const changed = JSON.stringify(doc) !== JSON.stringify(updatedDoc);
|
||||
if (changed) {
|
||||
// Update index
|
||||
await indexEngine.onUpdate(doc as any, updatedDoc);
|
||||
await storage.updateById(database, collection, doc._id, updatedDoc);
|
||||
totalModified++;
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
writeErrors.push({
|
||||
index: i,
|
||||
code: error.code || 1,
|
||||
errmsg: error.message || 'Update failed',
|
||||
});
|
||||
|
||||
if (ordered) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const response: plugins.bson.Document = {
|
||||
ok: 1,
|
||||
n: totalMatched + totalUpserted,
|
||||
nModified: totalModified,
|
||||
};
|
||||
|
||||
if (upserted.length > 0) {
|
||||
response.upserted = upserted;
|
||||
}
|
||||
|
||||
if (writeErrors.length > 0) {
|
||||
response.writeErrors = writeErrors;
|
||||
}
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle findAndModify command
|
||||
*/
|
||||
private async handleFindAndModify(context: IHandlerContext): Promise<plugins.bson.Document> {
|
||||
const { storage, database, command } = context;
|
||||
|
||||
const collection = command.findAndModify;
|
||||
const query = command.query || {};
|
||||
const update = command.update;
|
||||
const remove = command.remove || false;
|
||||
const returnNew = command.new || false;
|
||||
const upsert = command.upsert || false;
|
||||
const sort = command.sort;
|
||||
const fields = command.fields;
|
||||
const arrayFilters = command.arrayFilters;
|
||||
|
||||
// Validate - either update or remove, not both
|
||||
if (update && remove) {
|
||||
return {
|
||||
ok: 0,
|
||||
errmsg: 'cannot specify both update and remove',
|
||||
code: 2,
|
||||
codeName: 'BadValue',
|
||||
};
|
||||
}
|
||||
|
||||
if (!update && !remove) {
|
||||
return {
|
||||
ok: 0,
|
||||
errmsg: 'either update or remove is required',
|
||||
code: 2,
|
||||
codeName: 'BadValue',
|
||||
};
|
||||
}
|
||||
|
||||
// Ensure collection exists
|
||||
await storage.createCollection(database, collection);
|
||||
|
||||
// Try to use index-accelerated query
|
||||
const indexEngine = context.getIndexEngine(collection);
|
||||
const candidateIds = await indexEngine.findCandidateIds(query);
|
||||
|
||||
let documents: IStoredDocument[];
|
||||
if (candidateIds !== null) {
|
||||
documents = await storage.findByIds(database, collection, candidateIds);
|
||||
} else {
|
||||
documents = await storage.findAll(database, collection);
|
||||
}
|
||||
|
||||
let matchingDocs = QueryEngine.filter(documents, query);
|
||||
|
||||
// Apply sort if specified
|
||||
if (sort) {
|
||||
matchingDocs = QueryEngine.sort(matchingDocs, sort);
|
||||
}
|
||||
|
||||
const doc = matchingDocs[0];
|
||||
|
||||
if (remove) {
|
||||
// Delete operation
|
||||
if (!doc) {
|
||||
return { ok: 1, value: null };
|
||||
}
|
||||
|
||||
// Update index for delete
|
||||
await indexEngine.onDelete(doc as any);
|
||||
await storage.deleteById(database, collection, doc._id);
|
||||
|
||||
let result = doc;
|
||||
if (fields) {
|
||||
result = QueryEngine.project([doc], fields)[0] as any;
|
||||
}
|
||||
|
||||
return {
|
||||
ok: 1,
|
||||
value: result,
|
||||
lastErrorObject: {
|
||||
n: 1,
|
||||
},
|
||||
};
|
||||
} else {
|
||||
// Update operation
|
||||
if (!doc && !upsert) {
|
||||
return { ok: 1, value: null };
|
||||
}
|
||||
|
||||
let resultDoc: plugins.bson.Document;
|
||||
let originalDoc: plugins.bson.Document | null = null;
|
||||
let isUpsert = false;
|
||||
|
||||
if (doc) {
|
||||
// Update existing
|
||||
originalDoc = { ...doc };
|
||||
resultDoc = UpdateEngine.applyUpdate(doc, update, arrayFilters);
|
||||
// Update index
|
||||
await indexEngine.onUpdate(doc as any, resultDoc as any);
|
||||
await storage.updateById(database, collection, doc._id, resultDoc as any);
|
||||
} else {
|
||||
// Upsert
|
||||
isUpsert = true;
|
||||
const newDoc: plugins.bson.Document = { _id: new plugins.bson.ObjectId() };
|
||||
this.applyFilterToDoc(newDoc, query);
|
||||
resultDoc = UpdateEngine.applyUpdate(newDoc as any, update, arrayFilters);
|
||||
|
||||
if (update.$setOnInsert) {
|
||||
Object.assign(resultDoc, update.$setOnInsert);
|
||||
}
|
||||
|
||||
// Update index for insert
|
||||
await indexEngine.onInsert(resultDoc as any);
|
||||
await storage.insertOne(database, collection, resultDoc);
|
||||
}
|
||||
|
||||
// Apply projection
|
||||
let returnValue = returnNew ? resultDoc : (originalDoc || null);
|
||||
if (returnValue && fields) {
|
||||
returnValue = QueryEngine.project([returnValue as any], fields)[0];
|
||||
}
|
||||
|
||||
const response: plugins.bson.Document = {
|
||||
ok: 1,
|
||||
value: returnValue,
|
||||
lastErrorObject: {
|
||||
n: 1,
|
||||
updatedExisting: !isUpsert && doc !== undefined,
|
||||
},
|
||||
};
|
||||
|
||||
if (isUpsert) {
|
||||
response.lastErrorObject.upserted = resultDoc._id;
|
||||
}
|
||||
|
||||
return response;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply filter equality conditions to a new document (for upsert)
|
||||
*/
|
||||
private applyFilterToDoc(doc: plugins.bson.Document, filter: plugins.bson.Document): void {
|
||||
for (const [key, value] of Object.entries(filter)) {
|
||||
// Skip operators
|
||||
if (key.startsWith('$')) continue;
|
||||
|
||||
// Handle nested paths
|
||||
if (typeof value === 'object' && value !== null) {
|
||||
// Check if it's an operator
|
||||
const valueKeys = Object.keys(value);
|
||||
if (valueKeys.some(k => k.startsWith('$'))) {
|
||||
// Extract equality value from $eq if present
|
||||
if ('$eq' in value) {
|
||||
this.setNestedValue(doc, key, value.$eq);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Direct value assignment
|
||||
this.setNestedValue(doc, key, value);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Set a nested value using dot notation
|
||||
*/
|
||||
private setNestedValue(obj: plugins.bson.Document, path: string, value: any): void {
|
||||
const parts = path.split('.');
|
||||
let current = obj;
|
||||
|
||||
for (let i = 0; i < parts.length - 1; i++) {
|
||||
const part = parts[i];
|
||||
if (!(part in current)) {
|
||||
current[part] = {};
|
||||
}
|
||||
current = current[part];
|
||||
}
|
||||
|
||||
current[parts[parts.length - 1]] = value;
|
||||
}
|
||||
}
|
||||
10
ts/ts_tsmdb/server/handlers/index.ts
Normal file
10
ts/ts_tsmdb/server/handlers/index.ts
Normal file
@@ -0,0 +1,10 @@
|
||||
// Export all command handlers
|
||||
|
||||
export { HelloHandler } from './HelloHandler.js';
|
||||
export { InsertHandler } from './InsertHandler.js';
|
||||
export { FindHandler } from './FindHandler.js';
|
||||
export { UpdateHandler } from './UpdateHandler.js';
|
||||
export { DeleteHandler } from './DeleteHandler.js';
|
||||
export { AggregateHandler } from './AggregateHandler.js';
|
||||
export { IndexHandler } from './IndexHandler.js';
|
||||
export { AdminHandler } from './AdminHandler.js';
|
||||
10
ts/ts_tsmdb/server/index.ts
Normal file
10
ts/ts_tsmdb/server/index.ts
Normal file
@@ -0,0 +1,10 @@
|
||||
// Server module exports
|
||||
|
||||
export { TsmdbServer } from './TsmdbServer.js';
|
||||
export type { ITsmdbServerOptions } from './TsmdbServer.js';
|
||||
export { WireProtocol } from './WireProtocol.js';
|
||||
export { CommandRouter } from './CommandRouter.js';
|
||||
export type { ICommandHandler, IHandlerContext, ICursorState } from './CommandRouter.js';
|
||||
|
||||
// Export handlers
|
||||
export * from './handlers/index.js';
|
||||
562
ts/ts_tsmdb/storage/FileStorageAdapter.ts
Normal file
562
ts/ts_tsmdb/storage/FileStorageAdapter.ts
Normal file
@@ -0,0 +1,562 @@
|
||||
import * as plugins from '../plugins.js';
|
||||
import type { IStorageAdapter } from './IStorageAdapter.js';
|
||||
import type { IStoredDocument, IOpLogEntry, Document } from '../types/interfaces.js';
|
||||
import { calculateDocumentChecksum, verifyChecksum } from '../utils/checksum.js';
|
||||
|
||||
/**
|
||||
* File storage adapter options
|
||||
*/
|
||||
export interface IFileStorageAdapterOptions {
|
||||
/** Enable checksum verification for data integrity */
|
||||
enableChecksums?: boolean;
|
||||
/** Throw error on checksum mismatch (default: false, just log warning) */
|
||||
strictChecksums?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* File-based storage adapter for TsmDB
|
||||
* Stores data in JSON files on disk for persistence
|
||||
*/
|
||||
export class FileStorageAdapter implements IStorageAdapter {
|
||||
private basePath: string;
|
||||
private opLogCounter = 0;
|
||||
private initialized = false;
|
||||
private fs = new plugins.smartfs.SmartFs(new plugins.smartfs.SmartFsProviderNode());
|
||||
private enableChecksums: boolean;
|
||||
private strictChecksums: boolean;
|
||||
|
||||
constructor(basePath: string, options?: IFileStorageAdapterOptions) {
|
||||
this.basePath = basePath;
|
||||
this.enableChecksums = options?.enableChecksums ?? false;
|
||||
this.strictChecksums = options?.strictChecksums ?? false;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Helper Methods
|
||||
// ============================================================================
|
||||
|
||||
private getDbPath(dbName: string): string {
|
||||
return plugins.smartpath.join(this.basePath, dbName);
|
||||
}
|
||||
|
||||
private getCollectionPath(dbName: string, collName: string): string {
|
||||
return plugins.smartpath.join(this.basePath, dbName, `${collName}.json`);
|
||||
}
|
||||
|
||||
private getIndexPath(dbName: string, collName: string): string {
|
||||
return plugins.smartpath.join(this.basePath, dbName, `${collName}.indexes.json`);
|
||||
}
|
||||
|
||||
private getOpLogPath(): string {
|
||||
return plugins.smartpath.join(this.basePath, '_oplog.json');
|
||||
}
|
||||
|
||||
private getMetaPath(): string {
|
||||
return plugins.smartpath.join(this.basePath, '_meta.json');
|
||||
}
|
||||
|
||||
private async readJsonFile<T>(filePath: string, defaultValue: T): Promise<T> {
|
||||
try {
|
||||
const exists = await this.fs.file(filePath).exists();
|
||||
if (!exists) return defaultValue;
|
||||
const content = await this.fs.file(filePath).encoding('utf8').read();
|
||||
return JSON.parse(content as string);
|
||||
} catch {
|
||||
return defaultValue;
|
||||
}
|
||||
}
|
||||
|
||||
private async writeJsonFile(filePath: string, data: any): Promise<void> {
|
||||
const dir = filePath.substring(0, filePath.lastIndexOf('/'));
|
||||
await this.fs.directory(dir).recursive().create();
|
||||
await this.fs.file(filePath).encoding('utf8').write(JSON.stringify(data, null, 2));
|
||||
}
|
||||
|
||||
private restoreObjectIds(doc: any): IStoredDocument {
|
||||
if (doc._id) {
|
||||
if (typeof doc._id === 'string') {
|
||||
doc._id = new plugins.bson.ObjectId(doc._id);
|
||||
} else if (typeof doc._id === 'object' && doc._id.$oid) {
|
||||
doc._id = new plugins.bson.ObjectId(doc._id.$oid);
|
||||
}
|
||||
}
|
||||
return doc;
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify document checksum and handle errors
|
||||
*/
|
||||
private verifyDocumentChecksum(doc: any): boolean {
|
||||
if (!this.enableChecksums || !doc._checksum) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const isValid = verifyChecksum(doc);
|
||||
if (!isValid) {
|
||||
const errorMsg = `Checksum mismatch for document ${doc._id}`;
|
||||
if (this.strictChecksums) {
|
||||
throw new Error(errorMsg);
|
||||
} else {
|
||||
console.warn(`WARNING: ${errorMsg}`);
|
||||
}
|
||||
}
|
||||
return isValid;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add checksum to document before storing
|
||||
*/
|
||||
private prepareDocumentForStorage(doc: any): any {
|
||||
if (!this.enableChecksums) {
|
||||
return doc;
|
||||
}
|
||||
const checksum = calculateDocumentChecksum(doc);
|
||||
return { ...doc, _checksum: checksum };
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove internal checksum field before returning to user
|
||||
*/
|
||||
private cleanDocumentForReturn(doc: any): IStoredDocument {
|
||||
const { _checksum, ...cleanDoc } = doc;
|
||||
return this.restoreObjectIds(cleanDoc);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Initialization
|
||||
// ============================================================================
|
||||
|
||||
async initialize(): Promise<void> {
|
||||
if (this.initialized) return;
|
||||
|
||||
await this.fs.directory(this.basePath).recursive().create();
|
||||
|
||||
// Load metadata
|
||||
const meta = await this.readJsonFile(this.getMetaPath(), { opLogCounter: 0 });
|
||||
this.opLogCounter = meta.opLogCounter || 0;
|
||||
|
||||
this.initialized = true;
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
// Save metadata
|
||||
await this.writeJsonFile(this.getMetaPath(), { opLogCounter: this.opLogCounter });
|
||||
this.initialized = false;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Database Operations
|
||||
// ============================================================================
|
||||
|
||||
async listDatabases(): Promise<string[]> {
|
||||
await this.initialize();
|
||||
try {
|
||||
const entries = await this.fs.directory(this.basePath).list();
|
||||
return entries
|
||||
.filter(entry => entry.isDirectory && !entry.name.startsWith('_'))
|
||||
.map(entry => entry.name);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async createDatabase(dbName: string): Promise<void> {
|
||||
await this.initialize();
|
||||
const dbPath = this.getDbPath(dbName);
|
||||
await this.fs.directory(dbPath).recursive().create();
|
||||
}
|
||||
|
||||
async dropDatabase(dbName: string): Promise<boolean> {
|
||||
await this.initialize();
|
||||
const dbPath = this.getDbPath(dbName);
|
||||
try {
|
||||
const exists = await this.fs.directory(dbPath).exists();
|
||||
if (exists) {
|
||||
await this.fs.directory(dbPath).recursive().delete();
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async databaseExists(dbName: string): Promise<boolean> {
|
||||
await this.initialize();
|
||||
const dbPath = this.getDbPath(dbName);
|
||||
return this.fs.directory(dbPath).exists();
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Collection Operations
|
||||
// ============================================================================
|
||||
|
||||
async listCollections(dbName: string): Promise<string[]> {
|
||||
await this.initialize();
|
||||
const dbPath = this.getDbPath(dbName);
|
||||
try {
|
||||
const entries = await this.fs.directory(dbPath).list();
|
||||
return entries
|
||||
.filter(entry => entry.isFile && entry.name.endsWith('.json') && !entry.name.endsWith('.indexes.json'))
|
||||
.map(entry => entry.name.replace('.json', ''));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async createCollection(dbName: string, collName: string): Promise<void> {
|
||||
await this.createDatabase(dbName);
|
||||
const collPath = this.getCollectionPath(dbName, collName);
|
||||
const exists = await this.fs.file(collPath).exists();
|
||||
if (!exists) {
|
||||
await this.writeJsonFile(collPath, []);
|
||||
// Create default _id index
|
||||
await this.writeJsonFile(this.getIndexPath(dbName, collName), [
|
||||
{ name: '_id_', key: { _id: 1 }, unique: true }
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
async dropCollection(dbName: string, collName: string): Promise<boolean> {
|
||||
await this.initialize();
|
||||
const collPath = this.getCollectionPath(dbName, collName);
|
||||
const indexPath = this.getIndexPath(dbName, collName);
|
||||
try {
|
||||
const exists = await this.fs.file(collPath).exists();
|
||||
if (exists) {
|
||||
await this.fs.file(collPath).delete();
|
||||
try {
|
||||
await this.fs.file(indexPath).delete();
|
||||
} catch {}
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async collectionExists(dbName: string, collName: string): Promise<boolean> {
|
||||
await this.initialize();
|
||||
const collPath = this.getCollectionPath(dbName, collName);
|
||||
return this.fs.file(collPath).exists();
|
||||
}
|
||||
|
||||
async renameCollection(dbName: string, oldName: string, newName: string): Promise<void> {
|
||||
await this.initialize();
|
||||
const oldPath = this.getCollectionPath(dbName, oldName);
|
||||
const newPath = this.getCollectionPath(dbName, newName);
|
||||
const oldIndexPath = this.getIndexPath(dbName, oldName);
|
||||
const newIndexPath = this.getIndexPath(dbName, newName);
|
||||
|
||||
const exists = await this.fs.file(oldPath).exists();
|
||||
if (!exists) {
|
||||
throw new Error(`Collection ${oldName} not found`);
|
||||
}
|
||||
|
||||
// Read, write to new, delete old
|
||||
const docs = await this.readJsonFile<any[]>(oldPath, []);
|
||||
await this.writeJsonFile(newPath, docs);
|
||||
await this.fs.file(oldPath).delete();
|
||||
|
||||
// Handle indexes
|
||||
const indexes = await this.readJsonFile<any[]>(oldIndexPath, []);
|
||||
await this.writeJsonFile(newIndexPath, indexes);
|
||||
try {
|
||||
await this.fs.file(oldIndexPath).delete();
|
||||
} catch {}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Document Operations
|
||||
// ============================================================================
|
||||
|
||||
async insertOne(dbName: string, collName: string, doc: Document): Promise<IStoredDocument> {
|
||||
await this.createCollection(dbName, collName);
|
||||
const collPath = this.getCollectionPath(dbName, collName);
|
||||
const docs = await this.readJsonFile<any[]>(collPath, []);
|
||||
|
||||
const storedDoc: IStoredDocument = {
|
||||
...doc,
|
||||
_id: doc._id ? (doc._id instanceof plugins.bson.ObjectId ? doc._id : new plugins.bson.ObjectId(doc._id)) : new plugins.bson.ObjectId(),
|
||||
};
|
||||
|
||||
// Check for duplicate
|
||||
const idStr = storedDoc._id.toHexString();
|
||||
if (docs.some(d => d._id === idStr || (d._id && d._id.toString() === idStr))) {
|
||||
throw new Error(`Duplicate key error: _id ${idStr}`);
|
||||
}
|
||||
|
||||
// Add checksum if enabled
|
||||
const docToStore = this.prepareDocumentForStorage(storedDoc);
|
||||
docs.push(docToStore);
|
||||
await this.writeJsonFile(collPath, docs);
|
||||
return storedDoc;
|
||||
}
|
||||
|
||||
async insertMany(dbName: string, collName: string, docsToInsert: Document[]): Promise<IStoredDocument[]> {
|
||||
await this.createCollection(dbName, collName);
|
||||
const collPath = this.getCollectionPath(dbName, collName);
|
||||
const docs = await this.readJsonFile<any[]>(collPath, []);
|
||||
|
||||
const results: IStoredDocument[] = [];
|
||||
const existingIds = new Set(docs.map(d => d._id?.toString?.() || d._id));
|
||||
|
||||
for (const doc of docsToInsert) {
|
||||
const storedDoc: IStoredDocument = {
|
||||
...doc,
|
||||
_id: doc._id ? (doc._id instanceof plugins.bson.ObjectId ? doc._id : new plugins.bson.ObjectId(doc._id)) : new plugins.bson.ObjectId(),
|
||||
};
|
||||
|
||||
const idStr = storedDoc._id.toHexString();
|
||||
if (existingIds.has(idStr)) {
|
||||
throw new Error(`Duplicate key error: _id ${idStr}`);
|
||||
}
|
||||
|
||||
existingIds.add(idStr);
|
||||
// Add checksum if enabled
|
||||
const docToStore = this.prepareDocumentForStorage(storedDoc);
|
||||
docs.push(docToStore);
|
||||
results.push(storedDoc);
|
||||
}
|
||||
|
||||
await this.writeJsonFile(collPath, docs);
|
||||
return results;
|
||||
}
|
||||
|
||||
async findAll(dbName: string, collName: string): Promise<IStoredDocument[]> {
|
||||
await this.createCollection(dbName, collName);
|
||||
const collPath = this.getCollectionPath(dbName, collName);
|
||||
const docs = await this.readJsonFile<any[]>(collPath, []);
|
||||
return docs.map(doc => {
|
||||
// Verify checksum if enabled
|
||||
this.verifyDocumentChecksum(doc);
|
||||
// Clean and return document without internal checksum field
|
||||
return this.cleanDocumentForReturn(doc);
|
||||
});
|
||||
}
|
||||
|
||||
async findByIds(dbName: string, collName: string, ids: Set<string>): Promise<IStoredDocument[]> {
|
||||
await this.createCollection(dbName, collName);
|
||||
const collPath = this.getCollectionPath(dbName, collName);
|
||||
const docs = await this.readJsonFile<any[]>(collPath, []);
|
||||
const results: IStoredDocument[] = [];
|
||||
for (const doc of docs) {
|
||||
// Verify checksum if enabled
|
||||
this.verifyDocumentChecksum(doc);
|
||||
// Clean and restore document
|
||||
const cleaned = this.cleanDocumentForReturn(doc);
|
||||
if (ids.has(cleaned._id.toHexString())) {
|
||||
results.push(cleaned);
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
async findById(dbName: string, collName: string, id: plugins.bson.ObjectId): Promise<IStoredDocument | null> {
|
||||
// Use findAll which already handles checksum verification
|
||||
const docs = await this.findAll(dbName, collName);
|
||||
const idStr = id.toHexString();
|
||||
return docs.find(d => d._id.toHexString() === idStr) || null;
|
||||
}
|
||||
|
||||
async updateById(dbName: string, collName: string, id: plugins.bson.ObjectId, doc: IStoredDocument): Promise<boolean> {
|
||||
const collPath = this.getCollectionPath(dbName, collName);
|
||||
const docs = await this.readJsonFile<any[]>(collPath, []);
|
||||
const idStr = id.toHexString();
|
||||
|
||||
const idx = docs.findIndex(d => {
|
||||
const docId = d._id?.toHexString?.() || d._id?.toString?.() || d._id;
|
||||
return docId === idStr;
|
||||
});
|
||||
|
||||
if (idx === -1) return false;
|
||||
|
||||
// Add checksum if enabled
|
||||
const docToStore = this.prepareDocumentForStorage(doc);
|
||||
docs[idx] = docToStore;
|
||||
await this.writeJsonFile(collPath, docs);
|
||||
return true;
|
||||
}
|
||||
|
||||
async deleteById(dbName: string, collName: string, id: plugins.bson.ObjectId): Promise<boolean> {
|
||||
const collPath = this.getCollectionPath(dbName, collName);
|
||||
const docs = await this.readJsonFile<any[]>(collPath, []);
|
||||
const idStr = id.toHexString();
|
||||
|
||||
const idx = docs.findIndex(d => {
|
||||
const docId = d._id?.toHexString?.() || d._id?.toString?.() || d._id;
|
||||
return docId === idStr;
|
||||
});
|
||||
|
||||
if (idx === -1) return false;
|
||||
|
||||
docs.splice(idx, 1);
|
||||
await this.writeJsonFile(collPath, docs);
|
||||
return true;
|
||||
}
|
||||
|
||||
async deleteByIds(dbName: string, collName: string, ids: plugins.bson.ObjectId[]): Promise<number> {
|
||||
const collPath = this.getCollectionPath(dbName, collName);
|
||||
const docs = await this.readJsonFile<any[]>(collPath, []);
|
||||
const idStrs = new Set(ids.map(id => id.toHexString()));
|
||||
|
||||
const originalLength = docs.length;
|
||||
const filtered = docs.filter(d => {
|
||||
const docId = d._id?.toHexString?.() || d._id?.toString?.() || d._id;
|
||||
return !idStrs.has(docId);
|
||||
});
|
||||
|
||||
await this.writeJsonFile(collPath, filtered);
|
||||
return originalLength - filtered.length;
|
||||
}
|
||||
|
||||
async count(dbName: string, collName: string): Promise<number> {
|
||||
const collPath = this.getCollectionPath(dbName, collName);
|
||||
const docs = await this.readJsonFile<any[]>(collPath, []);
|
||||
return docs.length;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Index Operations
|
||||
// ============================================================================
|
||||
|
||||
async saveIndex(
|
||||
dbName: string,
|
||||
collName: string,
|
||||
indexName: string,
|
||||
indexSpec: { key: Record<string, any>; unique?: boolean; sparse?: boolean; expireAfterSeconds?: number }
|
||||
): Promise<void> {
|
||||
await this.createCollection(dbName, collName);
|
||||
const indexPath = this.getIndexPath(dbName, collName);
|
||||
const indexes = await this.readJsonFile<any[]>(indexPath, [
|
||||
{ name: '_id_', key: { _id: 1 }, unique: true }
|
||||
]);
|
||||
|
||||
const existingIdx = indexes.findIndex(i => i.name === indexName);
|
||||
if (existingIdx >= 0) {
|
||||
indexes[existingIdx] = { name: indexName, ...indexSpec };
|
||||
} else {
|
||||
indexes.push({ name: indexName, ...indexSpec });
|
||||
}
|
||||
|
||||
await this.writeJsonFile(indexPath, indexes);
|
||||
}
|
||||
|
||||
async getIndexes(dbName: string, collName: string): Promise<Array<{
|
||||
name: string;
|
||||
key: Record<string, any>;
|
||||
unique?: boolean;
|
||||
sparse?: boolean;
|
||||
expireAfterSeconds?: number;
|
||||
}>> {
|
||||
const indexPath = this.getIndexPath(dbName, collName);
|
||||
return this.readJsonFile(indexPath, [{ name: '_id_', key: { _id: 1 }, unique: true }]);
|
||||
}
|
||||
|
||||
async dropIndex(dbName: string, collName: string, indexName: string): Promise<boolean> {
|
||||
if (indexName === '_id_') {
|
||||
throw new Error('Cannot drop _id index');
|
||||
}
|
||||
|
||||
const indexPath = this.getIndexPath(dbName, collName);
|
||||
const indexes = await this.readJsonFile<any[]>(indexPath, []);
|
||||
|
||||
const idx = indexes.findIndex(i => i.name === indexName);
|
||||
if (idx >= 0) {
|
||||
indexes.splice(idx, 1);
|
||||
await this.writeJsonFile(indexPath, indexes);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// OpLog Operations
|
||||
// ============================================================================
|
||||
|
||||
async appendOpLog(entry: IOpLogEntry): Promise<void> {
|
||||
const opLogPath = this.getOpLogPath();
|
||||
const opLog = await this.readJsonFile<IOpLogEntry[]>(opLogPath, []);
|
||||
opLog.push(entry);
|
||||
|
||||
// Trim oplog if it gets too large
|
||||
if (opLog.length > 10000) {
|
||||
opLog.splice(0, opLog.length - 10000);
|
||||
}
|
||||
|
||||
await this.writeJsonFile(opLogPath, opLog);
|
||||
}
|
||||
|
||||
async getOpLogAfter(ts: plugins.bson.Timestamp, limit: number = 1000): Promise<IOpLogEntry[]> {
|
||||
const opLogPath = this.getOpLogPath();
|
||||
const opLog = await this.readJsonFile<any[]>(opLogPath, []);
|
||||
const tsValue = ts.toNumber();
|
||||
|
||||
const entries = opLog.filter(e => {
|
||||
const entryTs = e.ts.toNumber ? e.ts.toNumber() : (e.ts.t * 4294967296 + e.ts.i);
|
||||
return entryTs > tsValue;
|
||||
});
|
||||
|
||||
return entries.slice(0, limit);
|
||||
}
|
||||
|
||||
async getLatestOpLogTimestamp(): Promise<plugins.bson.Timestamp | null> {
|
||||
const opLogPath = this.getOpLogPath();
|
||||
const opLog = await this.readJsonFile<any[]>(opLogPath, []);
|
||||
if (opLog.length === 0) return null;
|
||||
|
||||
const last = opLog[opLog.length - 1];
|
||||
if (last.ts instanceof plugins.bson.Timestamp) {
|
||||
return last.ts;
|
||||
}
|
||||
return new plugins.bson.Timestamp({ t: last.ts.t, i: last.ts.i });
|
||||
}
|
||||
|
||||
generateTimestamp(): plugins.bson.Timestamp {
|
||||
this.opLogCounter++;
|
||||
return new plugins.bson.Timestamp({ t: Math.floor(Date.now() / 1000), i: this.opLogCounter });
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Transaction Support
|
||||
// ============================================================================
|
||||
|
||||
async createSnapshot(dbName: string, collName: string): Promise<IStoredDocument[]> {
|
||||
const docs = await this.findAll(dbName, collName);
|
||||
return docs.map(doc => JSON.parse(JSON.stringify(doc)));
|
||||
}
|
||||
|
||||
async hasConflicts(
|
||||
dbName: string,
|
||||
collName: string,
|
||||
ids: plugins.bson.ObjectId[],
|
||||
snapshotTime: plugins.bson.Timestamp
|
||||
): Promise<boolean> {
|
||||
const opLogPath = this.getOpLogPath();
|
||||
const opLog = await this.readJsonFile<any[]>(opLogPath, []);
|
||||
const ns = `${dbName}.${collName}`;
|
||||
const snapshotTs = snapshotTime.toNumber();
|
||||
const modifiedIds = new Set<string>();
|
||||
|
||||
for (const entry of opLog) {
|
||||
const entryTs = entry.ts.toNumber ? entry.ts.toNumber() : (entry.ts.t * 4294967296 + entry.ts.i);
|
||||
if (entryTs > snapshotTs && entry.ns === ns) {
|
||||
if (entry.o._id) {
|
||||
modifiedIds.add(entry.o._id.toString());
|
||||
}
|
||||
if (entry.o2?._id) {
|
||||
modifiedIds.add(entry.o2._id.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const id of ids) {
|
||||
if (modifiedIds.has(id.toString())) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
208
ts/ts_tsmdb/storage/IStorageAdapter.ts
Normal file
208
ts/ts_tsmdb/storage/IStorageAdapter.ts
Normal file
@@ -0,0 +1,208 @@
|
||||
import type * as plugins from '../plugins.js';
|
||||
import type { IStoredDocument, IOpLogEntry, Document } from '../types/interfaces.js';
|
||||
|
||||
/**
|
||||
* Storage adapter interface for TsmDB
|
||||
* Implementations can provide different storage backends (memory, file, etc.)
|
||||
*/
|
||||
export interface IStorageAdapter {
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
*/
|
||||
initialize(): Promise<void>;
|
||||
|
||||
/**
|
||||
* Close the storage adapter and release resources
|
||||
*/
|
||||
close(): Promise<void>;
|
||||
|
||||
// ============================================================================
|
||||
// Database Operations
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* List all database names
|
||||
*/
|
||||
listDatabases(): Promise<string[]>;
|
||||
|
||||
/**
|
||||
* Create a database (typically lazy - just marks it as existing)
|
||||
*/
|
||||
createDatabase(dbName: string): Promise<void>;
|
||||
|
||||
/**
|
||||
* Drop a database and all its collections
|
||||
*/
|
||||
dropDatabase(dbName: string): Promise<boolean>;
|
||||
|
||||
/**
|
||||
* Check if a database exists
|
||||
*/
|
||||
databaseExists(dbName: string): Promise<boolean>;
|
||||
|
||||
// ============================================================================
|
||||
// Collection Operations
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* List all collection names in a database
|
||||
*/
|
||||
listCollections(dbName: string): Promise<string[]>;
|
||||
|
||||
/**
|
||||
* Create a collection
|
||||
*/
|
||||
createCollection(dbName: string, collName: string): Promise<void>;
|
||||
|
||||
/**
|
||||
* Drop a collection
|
||||
*/
|
||||
dropCollection(dbName: string, collName: string): Promise<boolean>;
|
||||
|
||||
/**
|
||||
* Check if a collection exists
|
||||
*/
|
||||
collectionExists(dbName: string, collName: string): Promise<boolean>;
|
||||
|
||||
/**
|
||||
* Rename a collection
|
||||
*/
|
||||
renameCollection(dbName: string, oldName: string, newName: string): Promise<void>;
|
||||
|
||||
// ============================================================================
|
||||
// Document Operations
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Insert a single document
|
||||
* @returns The inserted document with _id
|
||||
*/
|
||||
insertOne(dbName: string, collName: string, doc: Document): Promise<IStoredDocument>;
|
||||
|
||||
/**
|
||||
* Insert multiple documents
|
||||
* @returns Array of inserted documents with _ids
|
||||
*/
|
||||
insertMany(dbName: string, collName: string, docs: Document[]): Promise<IStoredDocument[]>;
|
||||
|
||||
/**
|
||||
* Find all documents in a collection
|
||||
*/
|
||||
findAll(dbName: string, collName: string): Promise<IStoredDocument[]>;
|
||||
|
||||
/**
|
||||
* Find documents by a set of _id strings (hex format)
|
||||
* Used for index-accelerated queries
|
||||
*/
|
||||
findByIds(dbName: string, collName: string, ids: Set<string>): Promise<IStoredDocument[]>;
|
||||
|
||||
/**
|
||||
* Find a document by _id
|
||||
*/
|
||||
findById(dbName: string, collName: string, id: plugins.bson.ObjectId): Promise<IStoredDocument | null>;
|
||||
|
||||
/**
|
||||
* Update a document by _id
|
||||
* @returns true if document was updated
|
||||
*/
|
||||
updateById(dbName: string, collName: string, id: plugins.bson.ObjectId, doc: IStoredDocument): Promise<boolean>;
|
||||
|
||||
/**
|
||||
* Delete a document by _id
|
||||
* @returns true if document was deleted
|
||||
*/
|
||||
deleteById(dbName: string, collName: string, id: plugins.bson.ObjectId): Promise<boolean>;
|
||||
|
||||
/**
|
||||
* Delete multiple documents by _id
|
||||
* @returns Number of deleted documents
|
||||
*/
|
||||
deleteByIds(dbName: string, collName: string, ids: plugins.bson.ObjectId[]): Promise<number>;
|
||||
|
||||
/**
|
||||
* Get the count of documents in a collection
|
||||
*/
|
||||
count(dbName: string, collName: string): Promise<number>;
|
||||
|
||||
// ============================================================================
|
||||
// Index Operations
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Store index metadata
|
||||
*/
|
||||
saveIndex(
|
||||
dbName: string,
|
||||
collName: string,
|
||||
indexName: string,
|
||||
indexSpec: { key: Record<string, any>; unique?: boolean; sparse?: boolean; expireAfterSeconds?: number }
|
||||
): Promise<void>;
|
||||
|
||||
/**
|
||||
* Get all index metadata for a collection
|
||||
*/
|
||||
getIndexes(dbName: string, collName: string): Promise<Array<{
|
||||
name: string;
|
||||
key: Record<string, any>;
|
||||
unique?: boolean;
|
||||
sparse?: boolean;
|
||||
expireAfterSeconds?: number;
|
||||
}>>;
|
||||
|
||||
/**
|
||||
* Drop an index
|
||||
*/
|
||||
dropIndex(dbName: string, collName: string, indexName: string): Promise<boolean>;
|
||||
|
||||
// ============================================================================
|
||||
// OpLog Operations (for change streams)
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Append an operation to the oplog
|
||||
*/
|
||||
appendOpLog(entry: IOpLogEntry): Promise<void>;
|
||||
|
||||
/**
|
||||
* Get oplog entries after a timestamp
|
||||
*/
|
||||
getOpLogAfter(ts: plugins.bson.Timestamp, limit?: number): Promise<IOpLogEntry[]>;
|
||||
|
||||
/**
|
||||
* Get the latest oplog timestamp
|
||||
*/
|
||||
getLatestOpLogTimestamp(): Promise<plugins.bson.Timestamp | null>;
|
||||
|
||||
// ============================================================================
|
||||
// Transaction Support
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Create a snapshot of current data for transaction isolation
|
||||
*/
|
||||
createSnapshot(dbName: string, collName: string): Promise<IStoredDocument[]>;
|
||||
|
||||
/**
|
||||
* Check if any documents have been modified since the snapshot
|
||||
*/
|
||||
hasConflicts(
|
||||
dbName: string,
|
||||
collName: string,
|
||||
ids: plugins.bson.ObjectId[],
|
||||
snapshotTime: plugins.bson.Timestamp
|
||||
): Promise<boolean>;
|
||||
|
||||
// ============================================================================
|
||||
// Persistence (optional, for MemoryStorageAdapter with file backup)
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Persist current state to disk (if supported)
|
||||
*/
|
||||
persist?(): Promise<void>;
|
||||
|
||||
/**
|
||||
* Load state from disk (if supported)
|
||||
*/
|
||||
restore?(): Promise<void>;
|
||||
}
|
||||
455
ts/ts_tsmdb/storage/MemoryStorageAdapter.ts
Normal file
455
ts/ts_tsmdb/storage/MemoryStorageAdapter.ts
Normal file
@@ -0,0 +1,455 @@
|
||||
import * as plugins from '../plugins.js';
|
||||
import type { IStorageAdapter } from './IStorageAdapter.js';
|
||||
import type { IStoredDocument, IOpLogEntry, Document } from '../types/interfaces.js';
|
||||
|
||||
/**
|
||||
* In-memory storage adapter for TsmDB
|
||||
* Optionally supports persistence to a file
|
||||
*/
|
||||
export class MemoryStorageAdapter implements IStorageAdapter {
|
||||
// Database -> Collection -> Documents
|
||||
private databases: Map<string, Map<string, Map<string, IStoredDocument>>> = new Map();
|
||||
|
||||
// Database -> Collection -> Indexes
|
||||
private indexes: Map<string, Map<string, Array<{
|
||||
name: string;
|
||||
key: Record<string, any>;
|
||||
unique?: boolean;
|
||||
sparse?: boolean;
|
||||
expireAfterSeconds?: number;
|
||||
}>>> = new Map();
|
||||
|
||||
// OpLog entries
|
||||
private opLog: IOpLogEntry[] = [];
|
||||
private opLogCounter = 0;
|
||||
|
||||
// Persistence settings
|
||||
private persistPath?: string;
|
||||
private persistInterval?: ReturnType<typeof setInterval>;
|
||||
private fs = new plugins.smartfs.SmartFs(new plugins.smartfs.SmartFsProviderNode());
|
||||
|
||||
constructor(options?: { persistPath?: string; persistIntervalMs?: number }) {
|
||||
this.persistPath = options?.persistPath;
|
||||
if (this.persistPath && options?.persistIntervalMs) {
|
||||
this.persistInterval = setInterval(() => {
|
||||
this.persist().catch(console.error);
|
||||
}, options.persistIntervalMs);
|
||||
}
|
||||
}
|
||||
|
||||
async initialize(): Promise<void> {
|
||||
if (this.persistPath) {
|
||||
await this.restore();
|
||||
}
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
if (this.persistInterval) {
|
||||
clearInterval(this.persistInterval);
|
||||
}
|
||||
if (this.persistPath) {
|
||||
await this.persist();
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Database Operations
|
||||
// ============================================================================
|
||||
|
||||
async listDatabases(): Promise<string[]> {
|
||||
return Array.from(this.databases.keys());
|
||||
}
|
||||
|
||||
async createDatabase(dbName: string): Promise<void> {
|
||||
if (!this.databases.has(dbName)) {
|
||||
this.databases.set(dbName, new Map());
|
||||
this.indexes.set(dbName, new Map());
|
||||
}
|
||||
}
|
||||
|
||||
async dropDatabase(dbName: string): Promise<boolean> {
|
||||
const existed = this.databases.has(dbName);
|
||||
this.databases.delete(dbName);
|
||||
this.indexes.delete(dbName);
|
||||
return existed;
|
||||
}
|
||||
|
||||
async databaseExists(dbName: string): Promise<boolean> {
|
||||
return this.databases.has(dbName);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Collection Operations
|
||||
// ============================================================================
|
||||
|
||||
async listCollections(dbName: string): Promise<string[]> {
|
||||
const db = this.databases.get(dbName);
|
||||
return db ? Array.from(db.keys()) : [];
|
||||
}
|
||||
|
||||
async createCollection(dbName: string, collName: string): Promise<void> {
|
||||
await this.createDatabase(dbName);
|
||||
const db = this.databases.get(dbName)!;
|
||||
if (!db.has(collName)) {
|
||||
db.set(collName, new Map());
|
||||
// Initialize default _id index
|
||||
const dbIndexes = this.indexes.get(dbName)!;
|
||||
dbIndexes.set(collName, [{ name: '_id_', key: { _id: 1 }, unique: true }]);
|
||||
}
|
||||
}
|
||||
|
||||
async dropCollection(dbName: string, collName: string): Promise<boolean> {
|
||||
const db = this.databases.get(dbName);
|
||||
if (!db) return false;
|
||||
const existed = db.has(collName);
|
||||
db.delete(collName);
|
||||
const dbIndexes = this.indexes.get(dbName);
|
||||
if (dbIndexes) {
|
||||
dbIndexes.delete(collName);
|
||||
}
|
||||
return existed;
|
||||
}
|
||||
|
||||
async collectionExists(dbName: string, collName: string): Promise<boolean> {
|
||||
const db = this.databases.get(dbName);
|
||||
return db ? db.has(collName) : false;
|
||||
}
|
||||
|
||||
async renameCollection(dbName: string, oldName: string, newName: string): Promise<void> {
|
||||
const db = this.databases.get(dbName);
|
||||
if (!db || !db.has(oldName)) {
|
||||
throw new Error(`Collection ${oldName} not found`);
|
||||
}
|
||||
const collection = db.get(oldName)!;
|
||||
db.set(newName, collection);
|
||||
db.delete(oldName);
|
||||
|
||||
// Also rename indexes
|
||||
const dbIndexes = this.indexes.get(dbName);
|
||||
if (dbIndexes && dbIndexes.has(oldName)) {
|
||||
const collIndexes = dbIndexes.get(oldName)!;
|
||||
dbIndexes.set(newName, collIndexes);
|
||||
dbIndexes.delete(oldName);
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Document Operations
|
||||
// ============================================================================
|
||||
|
||||
private getCollection(dbName: string, collName: string): Map<string, IStoredDocument> {
|
||||
const db = this.databases.get(dbName);
|
||||
if (!db) {
|
||||
throw new Error(`Database ${dbName} not found`);
|
||||
}
|
||||
const collection = db.get(collName);
|
||||
if (!collection) {
|
||||
throw new Error(`Collection ${collName} not found`);
|
||||
}
|
||||
return collection;
|
||||
}
|
||||
|
||||
private ensureCollection(dbName: string, collName: string): Map<string, IStoredDocument> {
|
||||
if (!this.databases.has(dbName)) {
|
||||
this.databases.set(dbName, new Map());
|
||||
this.indexes.set(dbName, new Map());
|
||||
}
|
||||
const db = this.databases.get(dbName)!;
|
||||
if (!db.has(collName)) {
|
||||
db.set(collName, new Map());
|
||||
const dbIndexes = this.indexes.get(dbName)!;
|
||||
dbIndexes.set(collName, [{ name: '_id_', key: { _id: 1 }, unique: true }]);
|
||||
}
|
||||
return db.get(collName)!;
|
||||
}
|
||||
|
||||
async insertOne(dbName: string, collName: string, doc: Document): Promise<IStoredDocument> {
|
||||
const collection = this.ensureCollection(dbName, collName);
|
||||
const storedDoc: IStoredDocument = {
|
||||
...doc,
|
||||
_id: doc._id instanceof plugins.bson.ObjectId ? doc._id : new plugins.bson.ObjectId(doc._id),
|
||||
};
|
||||
|
||||
if (!storedDoc._id) {
|
||||
storedDoc._id = new plugins.bson.ObjectId();
|
||||
}
|
||||
|
||||
const idStr = storedDoc._id.toHexString();
|
||||
if (collection.has(idStr)) {
|
||||
throw new Error(`Duplicate key error: _id ${idStr}`);
|
||||
}
|
||||
|
||||
collection.set(idStr, storedDoc);
|
||||
return storedDoc;
|
||||
}
|
||||
|
||||
async insertMany(dbName: string, collName: string, docs: Document[]): Promise<IStoredDocument[]> {
|
||||
const results: IStoredDocument[] = [];
|
||||
for (const doc of docs) {
|
||||
results.push(await this.insertOne(dbName, collName, doc));
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
async findAll(dbName: string, collName: string): Promise<IStoredDocument[]> {
|
||||
const collection = this.ensureCollection(dbName, collName);
|
||||
return Array.from(collection.values());
|
||||
}
|
||||
|
||||
async findByIds(dbName: string, collName: string, ids: Set<string>): Promise<IStoredDocument[]> {
|
||||
const collection = this.ensureCollection(dbName, collName);
|
||||
const results: IStoredDocument[] = [];
|
||||
for (const id of ids) {
|
||||
const doc = collection.get(id);
|
||||
if (doc) {
|
||||
results.push(doc);
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
async findById(dbName: string, collName: string, id: plugins.bson.ObjectId): Promise<IStoredDocument | null> {
|
||||
const collection = this.ensureCollection(dbName, collName);
|
||||
return collection.get(id.toHexString()) || null;
|
||||
}
|
||||
|
||||
async updateById(dbName: string, collName: string, id: plugins.bson.ObjectId, doc: IStoredDocument): Promise<boolean> {
|
||||
const collection = this.ensureCollection(dbName, collName);
|
||||
const idStr = id.toHexString();
|
||||
if (!collection.has(idStr)) {
|
||||
return false;
|
||||
}
|
||||
collection.set(idStr, doc);
|
||||
return true;
|
||||
}
|
||||
|
||||
async deleteById(dbName: string, collName: string, id: plugins.bson.ObjectId): Promise<boolean> {
|
||||
const collection = this.ensureCollection(dbName, collName);
|
||||
return collection.delete(id.toHexString());
|
||||
}
|
||||
|
||||
async deleteByIds(dbName: string, collName: string, ids: plugins.bson.ObjectId[]): Promise<number> {
|
||||
let count = 0;
|
||||
for (const id of ids) {
|
||||
if (await this.deleteById(dbName, collName, id)) {
|
||||
count++;
|
||||
}
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
async count(dbName: string, collName: string): Promise<number> {
|
||||
const collection = this.ensureCollection(dbName, collName);
|
||||
return collection.size;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Index Operations
|
||||
// ============================================================================
|
||||
|
||||
async saveIndex(
|
||||
dbName: string,
|
||||
collName: string,
|
||||
indexName: string,
|
||||
indexSpec: { key: Record<string, any>; unique?: boolean; sparse?: boolean; expireAfterSeconds?: number }
|
||||
): Promise<void> {
|
||||
await this.createCollection(dbName, collName);
|
||||
const dbIndexes = this.indexes.get(dbName)!;
|
||||
let collIndexes = dbIndexes.get(collName);
|
||||
if (!collIndexes) {
|
||||
collIndexes = [{ name: '_id_', key: { _id: 1 }, unique: true }];
|
||||
dbIndexes.set(collName, collIndexes);
|
||||
}
|
||||
|
||||
// Check if index already exists
|
||||
const existingIndex = collIndexes.findIndex(i => i.name === indexName);
|
||||
if (existingIndex >= 0) {
|
||||
collIndexes[existingIndex] = { name: indexName, ...indexSpec };
|
||||
} else {
|
||||
collIndexes.push({ name: indexName, ...indexSpec });
|
||||
}
|
||||
}
|
||||
|
||||
async getIndexes(dbName: string, collName: string): Promise<Array<{
|
||||
name: string;
|
||||
key: Record<string, any>;
|
||||
unique?: boolean;
|
||||
sparse?: boolean;
|
||||
expireAfterSeconds?: number;
|
||||
}>> {
|
||||
const dbIndexes = this.indexes.get(dbName);
|
||||
if (!dbIndexes) return [{ name: '_id_', key: { _id: 1 }, unique: true }];
|
||||
const collIndexes = dbIndexes.get(collName);
|
||||
return collIndexes || [{ name: '_id_', key: { _id: 1 }, unique: true }];
|
||||
}
|
||||
|
||||
async dropIndex(dbName: string, collName: string, indexName: string): Promise<boolean> {
|
||||
if (indexName === '_id_') {
|
||||
throw new Error('Cannot drop _id index');
|
||||
}
|
||||
const dbIndexes = this.indexes.get(dbName);
|
||||
if (!dbIndexes) return false;
|
||||
const collIndexes = dbIndexes.get(collName);
|
||||
if (!collIndexes) return false;
|
||||
|
||||
const idx = collIndexes.findIndex(i => i.name === indexName);
|
||||
if (idx >= 0) {
|
||||
collIndexes.splice(idx, 1);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// OpLog Operations
|
||||
// ============================================================================
|
||||
|
||||
async appendOpLog(entry: IOpLogEntry): Promise<void> {
|
||||
this.opLog.push(entry);
|
||||
// Trim oplog if it gets too large (keep last 10000 entries)
|
||||
if (this.opLog.length > 10000) {
|
||||
this.opLog = this.opLog.slice(-10000);
|
||||
}
|
||||
}
|
||||
|
||||
async getOpLogAfter(ts: plugins.bson.Timestamp, limit: number = 1000): Promise<IOpLogEntry[]> {
|
||||
const tsValue = ts.toNumber();
|
||||
const entries = this.opLog.filter(e => e.ts.toNumber() > tsValue);
|
||||
return entries.slice(0, limit);
|
||||
}
|
||||
|
||||
async getLatestOpLogTimestamp(): Promise<plugins.bson.Timestamp | null> {
|
||||
if (this.opLog.length === 0) return null;
|
||||
return this.opLog[this.opLog.length - 1].ts;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a new timestamp for oplog entries
|
||||
*/
|
||||
generateTimestamp(): plugins.bson.Timestamp {
|
||||
this.opLogCounter++;
|
||||
return new plugins.bson.Timestamp({ t: Math.floor(Date.now() / 1000), i: this.opLogCounter });
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Transaction Support
|
||||
// ============================================================================
|
||||
|
||||
async createSnapshot(dbName: string, collName: string): Promise<IStoredDocument[]> {
|
||||
const docs = await this.findAll(dbName, collName);
|
||||
// Deep clone the documents for snapshot isolation
|
||||
return docs.map(doc => JSON.parse(JSON.stringify(doc)));
|
||||
}
|
||||
|
||||
async hasConflicts(
|
||||
dbName: string,
|
||||
collName: string,
|
||||
ids: plugins.bson.ObjectId[],
|
||||
snapshotTime: plugins.bson.Timestamp
|
||||
): Promise<boolean> {
|
||||
// Check if any of the given document IDs have been modified after snapshotTime
|
||||
const ns = `${dbName}.${collName}`;
|
||||
const modifiedIds = new Set<string>();
|
||||
|
||||
for (const entry of this.opLog) {
|
||||
if (entry.ts.greaterThan(snapshotTime) && entry.ns === ns) {
|
||||
if (entry.o._id) {
|
||||
modifiedIds.add(entry.o._id.toString());
|
||||
}
|
||||
if (entry.o2?._id) {
|
||||
modifiedIds.add(entry.o2._id.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const id of ids) {
|
||||
if (modifiedIds.has(id.toString())) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Persistence
|
||||
// ============================================================================
|
||||
|
||||
async persist(): Promise<void> {
|
||||
if (!this.persistPath) return;
|
||||
|
||||
const data = {
|
||||
databases: {} as Record<string, Record<string, IStoredDocument[]>>,
|
||||
indexes: {} as Record<string, Record<string, any[]>>,
|
||||
opLogCounter: this.opLogCounter,
|
||||
};
|
||||
|
||||
for (const [dbName, collections] of this.databases) {
|
||||
data.databases[dbName] = {};
|
||||
for (const [collName, docs] of collections) {
|
||||
data.databases[dbName][collName] = Array.from(docs.values());
|
||||
}
|
||||
}
|
||||
|
||||
for (const [dbName, collIndexes] of this.indexes) {
|
||||
data.indexes[dbName] = {};
|
||||
for (const [collName, indexes] of collIndexes) {
|
||||
data.indexes[dbName][collName] = indexes;
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure parent directory exists
|
||||
const dir = this.persistPath.substring(0, this.persistPath.lastIndexOf('/'));
|
||||
if (dir) {
|
||||
await this.fs.directory(dir).recursive().create();
|
||||
}
|
||||
await this.fs.file(this.persistPath).encoding('utf8').write(JSON.stringify(data, null, 2));
|
||||
}
|
||||
|
||||
async restore(): Promise<void> {
|
||||
if (!this.persistPath) return;
|
||||
|
||||
try {
|
||||
const exists = await this.fs.file(this.persistPath).exists();
|
||||
if (!exists) return;
|
||||
|
||||
const content = await this.fs.file(this.persistPath).encoding('utf8').read();
|
||||
const data = JSON.parse(content as string);
|
||||
|
||||
this.databases.clear();
|
||||
this.indexes.clear();
|
||||
|
||||
for (const [dbName, collections] of Object.entries(data.databases || {})) {
|
||||
const dbMap = new Map<string, Map<string, IStoredDocument>>();
|
||||
this.databases.set(dbName, dbMap);
|
||||
|
||||
for (const [collName, docs] of Object.entries(collections as Record<string, any[]>)) {
|
||||
const collMap = new Map<string, IStoredDocument>();
|
||||
for (const doc of docs) {
|
||||
// Restore ObjectId
|
||||
if (doc._id && typeof doc._id === 'string') {
|
||||
doc._id = new plugins.bson.ObjectId(doc._id);
|
||||
} else if (doc._id && typeof doc._id === 'object' && doc._id.$oid) {
|
||||
doc._id = new plugins.bson.ObjectId(doc._id.$oid);
|
||||
}
|
||||
collMap.set(doc._id.toHexString(), doc);
|
||||
}
|
||||
dbMap.set(collName, collMap);
|
||||
}
|
||||
}
|
||||
|
||||
for (const [dbName, collIndexes] of Object.entries(data.indexes || {})) {
|
||||
const indexMap = new Map<string, any[]>();
|
||||
this.indexes.set(dbName, indexMap);
|
||||
for (const [collName, indexes] of Object.entries(collIndexes as Record<string, any[]>)) {
|
||||
indexMap.set(collName, indexes);
|
||||
}
|
||||
}
|
||||
|
||||
this.opLogCounter = data.opLogCounter || 0;
|
||||
} catch (error) {
|
||||
// If restore fails, start fresh
|
||||
console.warn('Failed to restore from persistence:', error);
|
||||
}
|
||||
}
|
||||
}
|
||||
282
ts/ts_tsmdb/storage/OpLog.ts
Normal file
282
ts/ts_tsmdb/storage/OpLog.ts
Normal file
@@ -0,0 +1,282 @@
|
||||
import * as plugins from '../plugins.js';
|
||||
import type { IStorageAdapter } from './IStorageAdapter.js';
|
||||
import type { IOpLogEntry, Document, IResumeToken, ChangeStreamOperationType } from '../types/interfaces.js';
|
||||
|
||||
/**
|
||||
* Operation Log for tracking all mutations
|
||||
* Used primarily for change stream support
|
||||
*/
|
||||
export class OpLog {
|
||||
private storage: IStorageAdapter;
|
||||
private counter = 0;
|
||||
private listeners: Array<(entry: IOpLogEntry) => void> = [];
|
||||
|
||||
constructor(storage: IStorageAdapter) {
|
||||
this.storage = storage;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a new timestamp for oplog entries
|
||||
*/
|
||||
generateTimestamp(): plugins.bson.Timestamp {
|
||||
this.counter++;
|
||||
return new plugins.bson.Timestamp({ t: Math.floor(Date.now() / 1000), i: this.counter });
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a resume token from a timestamp
|
||||
*/
|
||||
generateResumeToken(ts: plugins.bson.Timestamp): IResumeToken {
|
||||
// Create a resume token similar to MongoDB's format
|
||||
// It's a base64-encoded BSON document containing the timestamp
|
||||
const tokenData = {
|
||||
_data: Buffer.from(JSON.stringify({
|
||||
ts: { t: ts.high, i: ts.low },
|
||||
version: 1,
|
||||
})).toString('base64'),
|
||||
};
|
||||
return tokenData;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a resume token to get the timestamp
|
||||
*/
|
||||
parseResumeToken(token: IResumeToken): plugins.bson.Timestamp {
|
||||
try {
|
||||
const data = JSON.parse(Buffer.from(token._data, 'base64').toString('utf-8'));
|
||||
return new plugins.bson.Timestamp({ t: data.ts.t, i: data.ts.i });
|
||||
} catch {
|
||||
throw new Error('Invalid resume token');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Record an insert operation
|
||||
*/
|
||||
async recordInsert(
|
||||
dbName: string,
|
||||
collName: string,
|
||||
document: Document,
|
||||
txnInfo?: { txnNumber?: number; lsid?: { id: plugins.bson.Binary } }
|
||||
): Promise<IOpLogEntry> {
|
||||
const entry: IOpLogEntry = {
|
||||
ts: this.generateTimestamp(),
|
||||
op: 'i',
|
||||
ns: `${dbName}.${collName}`,
|
||||
o: document,
|
||||
...txnInfo,
|
||||
};
|
||||
|
||||
await this.storage.appendOpLog(entry);
|
||||
this.notifyListeners(entry);
|
||||
return entry;
|
||||
}
|
||||
|
||||
/**
|
||||
* Record an update operation
|
||||
*/
|
||||
async recordUpdate(
|
||||
dbName: string,
|
||||
collName: string,
|
||||
filter: Document,
|
||||
update: Document,
|
||||
txnInfo?: { txnNumber?: number; lsid?: { id: plugins.bson.Binary } }
|
||||
): Promise<IOpLogEntry> {
|
||||
const entry: IOpLogEntry = {
|
||||
ts: this.generateTimestamp(),
|
||||
op: 'u',
|
||||
ns: `${dbName}.${collName}`,
|
||||
o: update,
|
||||
o2: filter,
|
||||
...txnInfo,
|
||||
};
|
||||
|
||||
await this.storage.appendOpLog(entry);
|
||||
this.notifyListeners(entry);
|
||||
return entry;
|
||||
}
|
||||
|
||||
/**
|
||||
* Record a delete operation
|
||||
*/
|
||||
async recordDelete(
|
||||
dbName: string,
|
||||
collName: string,
|
||||
filter: Document,
|
||||
txnInfo?: { txnNumber?: number; lsid?: { id: plugins.bson.Binary } }
|
||||
): Promise<IOpLogEntry> {
|
||||
const entry: IOpLogEntry = {
|
||||
ts: this.generateTimestamp(),
|
||||
op: 'd',
|
||||
ns: `${dbName}.${collName}`,
|
||||
o: filter,
|
||||
...txnInfo,
|
||||
};
|
||||
|
||||
await this.storage.appendOpLog(entry);
|
||||
this.notifyListeners(entry);
|
||||
return entry;
|
||||
}
|
||||
|
||||
/**
|
||||
* Record a command (drop, rename, etc.)
|
||||
*/
|
||||
async recordCommand(
|
||||
dbName: string,
|
||||
command: Document
|
||||
): Promise<IOpLogEntry> {
|
||||
const entry: IOpLogEntry = {
|
||||
ts: this.generateTimestamp(),
|
||||
op: 'c',
|
||||
ns: `${dbName}.$cmd`,
|
||||
o: command,
|
||||
};
|
||||
|
||||
await this.storage.appendOpLog(entry);
|
||||
this.notifyListeners(entry);
|
||||
return entry;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get oplog entries after a timestamp
|
||||
*/
|
||||
async getEntriesAfter(ts: plugins.bson.Timestamp, limit?: number): Promise<IOpLogEntry[]> {
|
||||
return this.storage.getOpLogAfter(ts, limit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the latest timestamp
|
||||
*/
|
||||
async getLatestTimestamp(): Promise<plugins.bson.Timestamp | null> {
|
||||
return this.storage.getLatestOpLogTimestamp();
|
||||
}
|
||||
|
||||
/**
|
||||
* Subscribe to oplog changes (for change streams)
|
||||
*/
|
||||
subscribe(listener: (entry: IOpLogEntry) => void): () => void {
|
||||
this.listeners.push(listener);
|
||||
return () => {
|
||||
const idx = this.listeners.indexOf(listener);
|
||||
if (idx >= 0) {
|
||||
this.listeners.splice(idx, 1);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Notify all listeners of a new entry
|
||||
*/
|
||||
private notifyListeners(entry: IOpLogEntry): void {
|
||||
for (const listener of this.listeners) {
|
||||
try {
|
||||
listener(entry);
|
||||
} catch (error) {
|
||||
console.error('Error in oplog listener:', error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert an oplog entry to a change stream document
|
||||
*/
|
||||
opLogEntryToChangeEvent(
|
||||
entry: IOpLogEntry,
|
||||
fullDocument?: Document,
|
||||
fullDocumentBeforeChange?: Document
|
||||
): {
|
||||
_id: IResumeToken;
|
||||
operationType: ChangeStreamOperationType;
|
||||
fullDocument?: Document;
|
||||
fullDocumentBeforeChange?: Document;
|
||||
ns: { db: string; coll?: string };
|
||||
documentKey?: { _id: plugins.bson.ObjectId };
|
||||
updateDescription?: {
|
||||
updatedFields?: Document;
|
||||
removedFields?: string[];
|
||||
};
|
||||
clusterTime: plugins.bson.Timestamp;
|
||||
} {
|
||||
const [db, coll] = entry.ns.split('.');
|
||||
const resumeToken = this.generateResumeToken(entry.ts);
|
||||
|
||||
const baseEvent = {
|
||||
_id: resumeToken,
|
||||
ns: { db, coll: coll === '$cmd' ? undefined : coll },
|
||||
clusterTime: entry.ts,
|
||||
};
|
||||
|
||||
switch (entry.op) {
|
||||
case 'i':
|
||||
return {
|
||||
...baseEvent,
|
||||
operationType: 'insert' as ChangeStreamOperationType,
|
||||
fullDocument: fullDocument || entry.o,
|
||||
documentKey: entry.o._id ? { _id: entry.o._id } : undefined,
|
||||
};
|
||||
|
||||
case 'u':
|
||||
const updateEvent: any = {
|
||||
...baseEvent,
|
||||
operationType: 'update' as ChangeStreamOperationType,
|
||||
documentKey: entry.o2?._id ? { _id: entry.o2._id } : undefined,
|
||||
};
|
||||
|
||||
if (fullDocument) {
|
||||
updateEvent.fullDocument = fullDocument;
|
||||
}
|
||||
if (fullDocumentBeforeChange) {
|
||||
updateEvent.fullDocumentBeforeChange = fullDocumentBeforeChange;
|
||||
}
|
||||
|
||||
// Parse update description
|
||||
if (entry.o.$set || entry.o.$unset) {
|
||||
updateEvent.updateDescription = {
|
||||
updatedFields: entry.o.$set || {},
|
||||
removedFields: entry.o.$unset ? Object.keys(entry.o.$unset) : [],
|
||||
};
|
||||
}
|
||||
|
||||
return updateEvent;
|
||||
|
||||
case 'd':
|
||||
return {
|
||||
...baseEvent,
|
||||
operationType: 'delete' as ChangeStreamOperationType,
|
||||
documentKey: entry.o._id ? { _id: entry.o._id } : undefined,
|
||||
fullDocumentBeforeChange,
|
||||
};
|
||||
|
||||
case 'c':
|
||||
if (entry.o.drop) {
|
||||
return {
|
||||
...baseEvent,
|
||||
operationType: 'drop' as ChangeStreamOperationType,
|
||||
ns: { db, coll: entry.o.drop },
|
||||
};
|
||||
}
|
||||
if (entry.o.dropDatabase) {
|
||||
return {
|
||||
...baseEvent,
|
||||
operationType: 'dropDatabase' as ChangeStreamOperationType,
|
||||
};
|
||||
}
|
||||
if (entry.o.renameCollection) {
|
||||
return {
|
||||
...baseEvent,
|
||||
operationType: 'rename' as ChangeStreamOperationType,
|
||||
};
|
||||
}
|
||||
return {
|
||||
...baseEvent,
|
||||
operationType: 'invalidate' as ChangeStreamOperationType,
|
||||
};
|
||||
|
||||
default:
|
||||
return {
|
||||
...baseEvent,
|
||||
operationType: 'invalidate' as ChangeStreamOperationType,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
375
ts/ts_tsmdb/storage/WAL.ts
Normal file
375
ts/ts_tsmdb/storage/WAL.ts
Normal file
@@ -0,0 +1,375 @@
|
||||
import * as plugins from '../plugins.js';
|
||||
import type { Document, IStoredDocument } from '../types/interfaces.js';
|
||||
|
||||
/**
|
||||
* WAL entry operation types
|
||||
*/
|
||||
export type TWalOperation = 'insert' | 'update' | 'delete' | 'checkpoint' | 'begin' | 'commit' | 'abort';
|
||||
|
||||
/**
|
||||
* WAL entry structure
|
||||
*/
|
||||
export interface IWalEntry {
|
||||
/** Log Sequence Number - monotonically increasing */
|
||||
lsn: number;
|
||||
/** Timestamp of the operation */
|
||||
timestamp: number;
|
||||
/** Operation type */
|
||||
operation: TWalOperation;
|
||||
/** Database name */
|
||||
dbName: string;
|
||||
/** Collection name */
|
||||
collName: string;
|
||||
/** Document ID (hex string) */
|
||||
documentId: string;
|
||||
/** Document data (BSON serialized, base64 encoded) */
|
||||
data?: string;
|
||||
/** Previous document data for updates (for rollback) */
|
||||
previousData?: string;
|
||||
/** Transaction ID if part of a transaction */
|
||||
txnId?: string;
|
||||
/** CRC32 checksum of the entry (excluding this field) */
|
||||
checksum: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checkpoint record
|
||||
*/
|
||||
interface ICheckpointRecord {
|
||||
lsn: number;
|
||||
timestamp: number;
|
||||
lastCommittedLsn: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write-Ahead Log (WAL) for durability and crash recovery
|
||||
*
|
||||
* The WAL ensures durability by writing operations to a log file before
|
||||
* they are applied to the main storage. On crash recovery, uncommitted
|
||||
* operations can be replayed to restore the database to a consistent state.
|
||||
*/
|
||||
export class WAL {
|
||||
private walPath: string;
|
||||
private currentLsn: number = 0;
|
||||
private lastCheckpointLsn: number = 0;
|
||||
private entries: IWalEntry[] = [];
|
||||
private isInitialized: boolean = false;
|
||||
private fs = new plugins.smartfs.SmartFs(new plugins.smartfs.SmartFsProviderNode());
|
||||
|
||||
// In-memory uncommitted entries per transaction
|
||||
private uncommittedTxns: Map<string, IWalEntry[]> = new Map();
|
||||
|
||||
// Checkpoint interval (number of entries between checkpoints)
|
||||
private checkpointInterval: number = 1000;
|
||||
|
||||
constructor(walPath: string, options?: { checkpointInterval?: number }) {
|
||||
this.walPath = walPath;
|
||||
if (options?.checkpointInterval) {
|
||||
this.checkpointInterval = options.checkpointInterval;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the WAL, loading existing entries and recovering if needed
|
||||
*/
|
||||
async initialize(): Promise<{ recoveredEntries: IWalEntry[] }> {
|
||||
if (this.isInitialized) {
|
||||
return { recoveredEntries: [] };
|
||||
}
|
||||
|
||||
// Ensure WAL directory exists
|
||||
const walDir = this.walPath.substring(0, this.walPath.lastIndexOf('/'));
|
||||
if (walDir) {
|
||||
await this.fs.directory(walDir).recursive().create();
|
||||
}
|
||||
|
||||
// Try to load existing WAL
|
||||
const exists = await this.fs.file(this.walPath).exists();
|
||||
if (exists) {
|
||||
const content = await this.fs.file(this.walPath).encoding('utf8').read();
|
||||
const lines = (content as string).split('\n').filter(line => line.trim());
|
||||
|
||||
for (const line of lines) {
|
||||
try {
|
||||
const entry = JSON.parse(line) as IWalEntry;
|
||||
// Verify checksum
|
||||
if (this.verifyChecksum(entry)) {
|
||||
this.entries.push(entry);
|
||||
if (entry.lsn > this.currentLsn) {
|
||||
this.currentLsn = entry.lsn;
|
||||
}
|
||||
if (entry.operation === 'checkpoint') {
|
||||
this.lastCheckpointLsn = entry.lsn;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Skip corrupted entries
|
||||
console.warn('Skipping corrupted WAL entry');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.isInitialized = true;
|
||||
|
||||
// Return entries after last checkpoint that need recovery
|
||||
const recoveredEntries = this.entries.filter(
|
||||
e => e.lsn > this.lastCheckpointLsn &&
|
||||
(e.operation === 'insert' || e.operation === 'update' || e.operation === 'delete')
|
||||
);
|
||||
|
||||
return { recoveredEntries };
|
||||
}
|
||||
|
||||
/**
|
||||
* Log an insert operation
|
||||
*/
|
||||
async logInsert(dbName: string, collName: string, doc: IStoredDocument, txnId?: string): Promise<number> {
|
||||
return this.appendEntry({
|
||||
operation: 'insert',
|
||||
dbName,
|
||||
collName,
|
||||
documentId: doc._id.toHexString(),
|
||||
data: this.serializeDocument(doc),
|
||||
txnId,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Log an update operation
|
||||
*/
|
||||
async logUpdate(
|
||||
dbName: string,
|
||||
collName: string,
|
||||
oldDoc: IStoredDocument,
|
||||
newDoc: IStoredDocument,
|
||||
txnId?: string
|
||||
): Promise<number> {
|
||||
return this.appendEntry({
|
||||
operation: 'update',
|
||||
dbName,
|
||||
collName,
|
||||
documentId: oldDoc._id.toHexString(),
|
||||
data: this.serializeDocument(newDoc),
|
||||
previousData: this.serializeDocument(oldDoc),
|
||||
txnId,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Log a delete operation
|
||||
*/
|
||||
async logDelete(dbName: string, collName: string, doc: IStoredDocument, txnId?: string): Promise<number> {
|
||||
return this.appendEntry({
|
||||
operation: 'delete',
|
||||
dbName,
|
||||
collName,
|
||||
documentId: doc._id.toHexString(),
|
||||
previousData: this.serializeDocument(doc),
|
||||
txnId,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Log transaction begin
|
||||
*/
|
||||
async logBeginTransaction(txnId: string): Promise<number> {
|
||||
this.uncommittedTxns.set(txnId, []);
|
||||
return this.appendEntry({
|
||||
operation: 'begin',
|
||||
dbName: '',
|
||||
collName: '',
|
||||
documentId: '',
|
||||
txnId,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Log transaction commit
|
||||
*/
|
||||
async logCommitTransaction(txnId: string): Promise<number> {
|
||||
this.uncommittedTxns.delete(txnId);
|
||||
return this.appendEntry({
|
||||
operation: 'commit',
|
||||
dbName: '',
|
||||
collName: '',
|
||||
documentId: '',
|
||||
txnId,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Log transaction abort
|
||||
*/
|
||||
async logAbortTransaction(txnId: string): Promise<number> {
|
||||
this.uncommittedTxns.delete(txnId);
|
||||
return this.appendEntry({
|
||||
operation: 'abort',
|
||||
dbName: '',
|
||||
collName: '',
|
||||
documentId: '',
|
||||
txnId,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get entries to roll back for an aborted transaction
|
||||
*/
|
||||
getTransactionEntries(txnId: string): IWalEntry[] {
|
||||
return this.entries.filter(e => e.txnId === txnId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a checkpoint - marks a consistent point in the log
|
||||
*/
|
||||
async checkpoint(): Promise<number> {
|
||||
const lsn = await this.appendEntry({
|
||||
operation: 'checkpoint',
|
||||
dbName: '',
|
||||
collName: '',
|
||||
documentId: '',
|
||||
});
|
||||
this.lastCheckpointLsn = lsn;
|
||||
|
||||
// Truncate old entries (keep only entries after checkpoint)
|
||||
await this.truncate();
|
||||
|
||||
return lsn;
|
||||
}
|
||||
|
||||
/**
|
||||
* Truncate the WAL file, removing entries before the last checkpoint
|
||||
*/
|
||||
private async truncate(): Promise<void> {
|
||||
// Keep entries after last checkpoint
|
||||
const newEntries = this.entries.filter(e => e.lsn >= this.lastCheckpointLsn);
|
||||
this.entries = newEntries;
|
||||
|
||||
// Rewrite the WAL file
|
||||
const lines = this.entries.map(e => JSON.stringify(e)).join('\n');
|
||||
await this.fs.file(this.walPath).encoding('utf8').write(lines);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current LSN
|
||||
*/
|
||||
getCurrentLsn(): number {
|
||||
return this.currentLsn;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get entries after a specific LSN (for recovery)
|
||||
*/
|
||||
getEntriesAfter(lsn: number): IWalEntry[] {
|
||||
return this.entries.filter(e => e.lsn > lsn);
|
||||
}
|
||||
|
||||
/**
|
||||
* Close the WAL
|
||||
*/
|
||||
async close(): Promise<void> {
|
||||
if (this.isInitialized) {
|
||||
// Final checkpoint before close
|
||||
await this.checkpoint();
|
||||
}
|
||||
this.isInitialized = false;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Private Methods
|
||||
// ============================================================================
|
||||
|
||||
private async appendEntry(
|
||||
partial: Omit<IWalEntry, 'lsn' | 'timestamp' | 'checksum'>
|
||||
): Promise<number> {
|
||||
await this.initialize();
|
||||
|
||||
this.currentLsn++;
|
||||
const entry: IWalEntry = {
|
||||
...partial,
|
||||
lsn: this.currentLsn,
|
||||
timestamp: Date.now(),
|
||||
checksum: 0, // Will be calculated
|
||||
};
|
||||
|
||||
// Calculate checksum
|
||||
entry.checksum = this.calculateChecksum(entry);
|
||||
|
||||
// Track in transaction if applicable
|
||||
if (partial.txnId && this.uncommittedTxns.has(partial.txnId)) {
|
||||
this.uncommittedTxns.get(partial.txnId)!.push(entry);
|
||||
}
|
||||
|
||||
// Add to in-memory log
|
||||
this.entries.push(entry);
|
||||
|
||||
// Append to file (append mode for durability)
|
||||
await this.fs.file(this.walPath).encoding('utf8').append(JSON.stringify(entry) + '\n');
|
||||
|
||||
// Check if we need a checkpoint
|
||||
if (this.entries.length - this.lastCheckpointLsn >= this.checkpointInterval) {
|
||||
await this.checkpoint();
|
||||
}
|
||||
|
||||
return entry.lsn;
|
||||
}
|
||||
|
||||
private serializeDocument(doc: Document): string {
|
||||
// Serialize document to BSON and encode as base64
|
||||
const bsonData = plugins.bson.serialize(doc);
|
||||
return Buffer.from(bsonData).toString('base64');
|
||||
}
|
||||
|
||||
private deserializeDocument(data: string): Document {
|
||||
// Decode base64 and deserialize from BSON
|
||||
const buffer = Buffer.from(data, 'base64');
|
||||
return plugins.bson.deserialize(buffer);
|
||||
}
|
||||
|
||||
private calculateChecksum(entry: IWalEntry): number {
|
||||
// Simple CRC32-like checksum
|
||||
const str = JSON.stringify({
|
||||
lsn: entry.lsn,
|
||||
timestamp: entry.timestamp,
|
||||
operation: entry.operation,
|
||||
dbName: entry.dbName,
|
||||
collName: entry.collName,
|
||||
documentId: entry.documentId,
|
||||
data: entry.data,
|
||||
previousData: entry.previousData,
|
||||
txnId: entry.txnId,
|
||||
});
|
||||
|
||||
let crc = 0xFFFFFFFF;
|
||||
for (let i = 0; i < str.length; i++) {
|
||||
crc ^= str.charCodeAt(i);
|
||||
for (let j = 0; j < 8; j++) {
|
||||
crc = (crc >>> 1) ^ (crc & 1 ? 0xEDB88320 : 0);
|
||||
}
|
||||
}
|
||||
return (~crc) >>> 0;
|
||||
}
|
||||
|
||||
private verifyChecksum(entry: IWalEntry): boolean {
|
||||
const savedChecksum = entry.checksum;
|
||||
entry.checksum = 0;
|
||||
const calculatedChecksum = this.calculateChecksum(entry);
|
||||
entry.checksum = savedChecksum;
|
||||
return calculatedChecksum === savedChecksum;
|
||||
}
|
||||
|
||||
/**
|
||||
* Recover document from WAL entry
|
||||
*/
|
||||
recoverDocument(entry: IWalEntry): IStoredDocument | null {
|
||||
if (!entry.data) return null;
|
||||
return this.deserializeDocument(entry.data) as IStoredDocument;
|
||||
}
|
||||
|
||||
/**
|
||||
* Recover previous document state from WAL entry (for rollback)
|
||||
*/
|
||||
recoverPreviousDocument(entry: IWalEntry): IStoredDocument | null {
|
||||
if (!entry.previousData) return null;
|
||||
return this.deserializeDocument(entry.previousData) as IStoredDocument;
|
||||
}
|
||||
}
|
||||
433
ts/ts_tsmdb/types/interfaces.ts
Normal file
433
ts/ts_tsmdb/types/interfaces.ts
Normal file
@@ -0,0 +1,433 @@
|
||||
import type * as plugins from '../plugins.js';
|
||||
|
||||
// ============================================================================
|
||||
// Document Types
|
||||
// ============================================================================
|
||||
|
||||
export type Document = Record<string, any>;
|
||||
|
||||
export interface WithId<TSchema> {
|
||||
_id: plugins.bson.ObjectId;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Client Options
|
||||
// ============================================================================
|
||||
|
||||
export interface ITsmdbClientOptions {
|
||||
/** Storage adapter type: 'memory' or 'file' */
|
||||
storageType?: 'memory' | 'file';
|
||||
/** Path for file-based storage */
|
||||
storagePath?: string;
|
||||
/** Enable persistence for memory adapter */
|
||||
persist?: boolean;
|
||||
/** Path for persistence file when using memory adapter */
|
||||
persistPath?: string;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Connection String Parsing
|
||||
// ============================================================================
|
||||
|
||||
export interface IParsedConnectionString {
|
||||
protocol: 'tsmdb';
|
||||
storageType: 'memory' | 'file';
|
||||
options: {
|
||||
persist?: string;
|
||||
path?: string;
|
||||
};
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// CRUD Operation Options
|
||||
// ============================================================================
|
||||
|
||||
export interface IInsertOneOptions {
|
||||
/** Session for transaction support */
|
||||
session?: IClientSession;
|
||||
/** Custom write concern */
|
||||
writeConcern?: IWriteConcern;
|
||||
}
|
||||
|
||||
export interface IInsertManyOptions extends IInsertOneOptions {
|
||||
/** If true, inserts are ordered and stop on first error */
|
||||
ordered?: boolean;
|
||||
}
|
||||
|
||||
export interface IFindOptions<TSchema = Document> {
|
||||
/** Projection to apply */
|
||||
projection?: Partial<Record<keyof TSchema | string, 0 | 1 | boolean>>;
|
||||
/** Sort specification */
|
||||
sort?: ISortSpecification;
|
||||
/** Number of documents to skip */
|
||||
skip?: number;
|
||||
/** Maximum number of documents to return */
|
||||
limit?: number;
|
||||
/** Session for transaction support */
|
||||
session?: IClientSession;
|
||||
/** Hint for index usage */
|
||||
hint?: string | Document;
|
||||
}
|
||||
|
||||
export interface IUpdateOptions {
|
||||
/** Create document if it doesn't exist */
|
||||
upsert?: boolean;
|
||||
/** Session for transaction support */
|
||||
session?: IClientSession;
|
||||
/** Array filters for positional updates */
|
||||
arrayFilters?: Document[];
|
||||
/** Custom write concern */
|
||||
writeConcern?: IWriteConcern;
|
||||
/** Hint for index usage */
|
||||
hint?: string | Document;
|
||||
}
|
||||
|
||||
export interface IReplaceOptions extends IUpdateOptions {}
|
||||
|
||||
export interface IDeleteOptions {
|
||||
/** Session for transaction support */
|
||||
session?: IClientSession;
|
||||
/** Custom write concern */
|
||||
writeConcern?: IWriteConcern;
|
||||
/** Hint for index usage */
|
||||
hint?: string | Document;
|
||||
}
|
||||
|
||||
export interface IFindOneAndUpdateOptions extends IUpdateOptions {
|
||||
/** Return the document before or after the update */
|
||||
returnDocument?: 'before' | 'after';
|
||||
/** Projection to apply */
|
||||
projection?: Document;
|
||||
/** Sort specification to determine which document to modify */
|
||||
sort?: ISortSpecification;
|
||||
}
|
||||
|
||||
export interface IFindOneAndReplaceOptions extends IFindOneAndUpdateOptions {}
|
||||
|
||||
export interface IFindOneAndDeleteOptions {
|
||||
/** Projection to apply */
|
||||
projection?: Document;
|
||||
/** Sort specification to determine which document to delete */
|
||||
sort?: ISortSpecification;
|
||||
/** Session for transaction support */
|
||||
session?: IClientSession;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// CRUD Results
|
||||
// ============================================================================
|
||||
|
||||
export interface IInsertOneResult {
|
||||
acknowledged: boolean;
|
||||
insertedId: plugins.bson.ObjectId;
|
||||
}
|
||||
|
||||
export interface IInsertManyResult {
|
||||
acknowledged: boolean;
|
||||
insertedCount: number;
|
||||
insertedIds: Record<number, plugins.bson.ObjectId>;
|
||||
}
|
||||
|
||||
export interface IUpdateResult {
|
||||
acknowledged: boolean;
|
||||
matchedCount: number;
|
||||
modifiedCount: number;
|
||||
upsertedCount: number;
|
||||
upsertedId: plugins.bson.ObjectId | null;
|
||||
}
|
||||
|
||||
export interface IDeleteResult {
|
||||
acknowledged: boolean;
|
||||
deletedCount: number;
|
||||
}
|
||||
|
||||
export interface IModifyResult<TSchema> {
|
||||
value: TSchema | null;
|
||||
ok: 1 | 0;
|
||||
lastErrorObject?: {
|
||||
n: number;
|
||||
updatedExisting?: boolean;
|
||||
upserted?: plugins.bson.ObjectId;
|
||||
};
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Sort and Index Types
|
||||
// ============================================================================
|
||||
|
||||
export type ISortDirection = 1 | -1 | 'asc' | 'desc' | 'ascending' | 'descending';
|
||||
|
||||
export type ISortSpecification = Record<string, ISortDirection> | [string, ISortDirection][];
|
||||
|
||||
export interface IIndexSpecification {
|
||||
key: Record<string, 1 | -1 | 'text' | '2dsphere'>;
|
||||
name?: string;
|
||||
unique?: boolean;
|
||||
sparse?: boolean;
|
||||
expireAfterSeconds?: number;
|
||||
background?: boolean;
|
||||
partialFilterExpression?: Document;
|
||||
}
|
||||
|
||||
export interface IIndexInfo {
|
||||
v: number;
|
||||
key: Record<string, 1 | -1 | string>;
|
||||
name: string;
|
||||
unique?: boolean;
|
||||
sparse?: boolean;
|
||||
expireAfterSeconds?: number;
|
||||
}
|
||||
|
||||
export interface ICreateIndexOptions {
|
||||
unique?: boolean;
|
||||
sparse?: boolean;
|
||||
expireAfterSeconds?: number;
|
||||
name?: string;
|
||||
background?: boolean;
|
||||
partialFilterExpression?: Document;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Write Concern
|
||||
// ============================================================================
|
||||
|
||||
export interface IWriteConcern {
|
||||
w?: number | 'majority';
|
||||
j?: boolean;
|
||||
wtimeout?: number;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Aggregation Types
|
||||
// ============================================================================
|
||||
|
||||
export interface IAggregateOptions {
|
||||
/** Allow disk use for large aggregations */
|
||||
allowDiskUse?: boolean;
|
||||
/** Maximum time in ms */
|
||||
maxTimeMS?: number;
|
||||
/** Session for transaction support */
|
||||
session?: IClientSession;
|
||||
/** Batch size for cursor */
|
||||
batchSize?: number;
|
||||
/** Collation settings */
|
||||
collation?: ICollation;
|
||||
/** Hint for index usage */
|
||||
hint?: string | Document;
|
||||
/** Comment for profiling */
|
||||
comment?: string;
|
||||
}
|
||||
|
||||
export interface ICollation {
|
||||
locale: string;
|
||||
caseLevel?: boolean;
|
||||
caseFirst?: string;
|
||||
strength?: number;
|
||||
numericOrdering?: boolean;
|
||||
alternate?: string;
|
||||
maxVariable?: string;
|
||||
backwards?: boolean;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Change Stream Types
|
||||
// ============================================================================
|
||||
|
||||
export interface IChangeStreamOptions {
|
||||
/** Resume after this token */
|
||||
resumeAfter?: IResumeToken;
|
||||
/** Start at this operation time */
|
||||
startAtOperationTime?: plugins.bson.Timestamp;
|
||||
/** Start after this token */
|
||||
startAfter?: IResumeToken;
|
||||
/** Full document lookup mode */
|
||||
fullDocument?: 'default' | 'updateLookup' | 'whenAvailable' | 'required';
|
||||
/** Full document before change */
|
||||
fullDocumentBeforeChange?: 'off' | 'whenAvailable' | 'required';
|
||||
/** Batch size */
|
||||
batchSize?: number;
|
||||
/** Maximum await time in ms */
|
||||
maxAwaitTimeMS?: number;
|
||||
}
|
||||
|
||||
export interface IResumeToken {
|
||||
_data: string;
|
||||
}
|
||||
|
||||
export type ChangeStreamOperationType =
|
||||
| 'insert'
|
||||
| 'update'
|
||||
| 'replace'
|
||||
| 'delete'
|
||||
| 'drop'
|
||||
| 'rename'
|
||||
| 'dropDatabase'
|
||||
| 'invalidate';
|
||||
|
||||
export interface IChangeStreamDocument<TSchema = Document> {
|
||||
_id: IResumeToken;
|
||||
operationType: ChangeStreamOperationType;
|
||||
fullDocument?: TSchema;
|
||||
fullDocumentBeforeChange?: TSchema;
|
||||
ns: {
|
||||
db: string;
|
||||
coll?: string;
|
||||
};
|
||||
documentKey?: { _id: plugins.bson.ObjectId };
|
||||
updateDescription?: {
|
||||
updatedFields?: Document;
|
||||
removedFields?: string[];
|
||||
truncatedArrays?: Array<{ field: string; newSize: number }>;
|
||||
};
|
||||
clusterTime?: plugins.bson.Timestamp;
|
||||
txnNumber?: number;
|
||||
lsid?: { id: plugins.bson.Binary; uid: plugins.bson.Binary };
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Transaction Types
|
||||
// ============================================================================
|
||||
|
||||
export interface IClientSession {
|
||||
id: { id: plugins.bson.Binary };
|
||||
inTransaction(): boolean;
|
||||
startTransaction(options?: ITransactionOptions): void;
|
||||
commitTransaction(): Promise<void>;
|
||||
abortTransaction(): Promise<void>;
|
||||
withTransaction<T>(fn: () => Promise<T>, options?: ITransactionOptions): Promise<T>;
|
||||
endSession(): Promise<void>;
|
||||
}
|
||||
|
||||
export interface ITransactionOptions {
|
||||
readConcern?: IReadConcern;
|
||||
writeConcern?: IWriteConcern;
|
||||
readPreference?: string;
|
||||
maxCommitTimeMS?: number;
|
||||
}
|
||||
|
||||
export interface IReadConcern {
|
||||
level: 'local' | 'available' | 'majority' | 'linearizable' | 'snapshot';
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Bulk Operation Types
|
||||
// ============================================================================
|
||||
|
||||
export interface IBulkWriteOptions {
|
||||
ordered?: boolean;
|
||||
session?: IClientSession;
|
||||
writeConcern?: IWriteConcern;
|
||||
}
|
||||
|
||||
export interface IBulkWriteOperation<TSchema = Document> {
|
||||
insertOne?: { document: TSchema };
|
||||
updateOne?: { filter: Document; update: Document; upsert?: boolean; arrayFilters?: Document[]; hint?: Document | string };
|
||||
updateMany?: { filter: Document; update: Document; upsert?: boolean; arrayFilters?: Document[]; hint?: Document | string };
|
||||
replaceOne?: { filter: Document; replacement: TSchema; upsert?: boolean; hint?: Document | string };
|
||||
deleteOne?: { filter: Document; hint?: Document | string };
|
||||
deleteMany?: { filter: Document; hint?: Document | string };
|
||||
}
|
||||
|
||||
export interface IBulkWriteResult {
|
||||
acknowledged: boolean;
|
||||
insertedCount: number;
|
||||
matchedCount: number;
|
||||
modifiedCount: number;
|
||||
deletedCount: number;
|
||||
upsertedCount: number;
|
||||
insertedIds: Record<number, plugins.bson.ObjectId>;
|
||||
upsertedIds: Record<number, plugins.bson.ObjectId>;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Storage Types
|
||||
// ============================================================================
|
||||
|
||||
export interface IStoredDocument extends Document {
|
||||
_id: plugins.bson.ObjectId;
|
||||
}
|
||||
|
||||
export interface IOpLogEntry {
|
||||
ts: plugins.bson.Timestamp;
|
||||
op: 'i' | 'u' | 'd' | 'c' | 'n';
|
||||
ns: string;
|
||||
o: Document;
|
||||
o2?: Document;
|
||||
txnNumber?: number;
|
||||
lsid?: { id: plugins.bson.Binary };
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Admin Types
|
||||
// ============================================================================
|
||||
|
||||
export interface IDatabaseInfo {
|
||||
name: string;
|
||||
sizeOnDisk: number;
|
||||
empty: boolean;
|
||||
}
|
||||
|
||||
export interface ICollectionInfo {
|
||||
name: string;
|
||||
type: 'collection' | 'view';
|
||||
options: Document;
|
||||
info: {
|
||||
readOnly: boolean;
|
||||
uuid?: plugins.bson.Binary;
|
||||
};
|
||||
idIndex?: IIndexInfo;
|
||||
}
|
||||
|
||||
export interface IServerStatus {
|
||||
host: string;
|
||||
version: string;
|
||||
process: string;
|
||||
pid: number;
|
||||
uptime: number;
|
||||
uptimeMillis: number;
|
||||
uptimeEstimate: number;
|
||||
localTime: Date;
|
||||
mem: {
|
||||
resident: number;
|
||||
virtual: number;
|
||||
};
|
||||
connections: {
|
||||
current: number;
|
||||
available: number;
|
||||
totalCreated: number;
|
||||
};
|
||||
ok: 1;
|
||||
}
|
||||
|
||||
export interface ICollectionStats {
|
||||
ns: string;
|
||||
count: number;
|
||||
size: number;
|
||||
avgObjSize: number;
|
||||
storageSize: number;
|
||||
totalIndexSize: number;
|
||||
indexSizes: Record<string, number>;
|
||||
nindexes: number;
|
||||
ok: 1;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Count Types
|
||||
// ============================================================================
|
||||
|
||||
export interface ICountDocumentsOptions {
|
||||
skip?: number;
|
||||
limit?: number;
|
||||
session?: IClientSession;
|
||||
hint?: string | Document;
|
||||
maxTimeMS?: number;
|
||||
}
|
||||
|
||||
export interface IEstimatedDocumentCountOptions {
|
||||
maxTimeMS?: number;
|
||||
}
|
||||
|
||||
export interface IDistinctOptions {
|
||||
session?: IClientSession;
|
||||
maxTimeMS?: number;
|
||||
}
|
||||
88
ts/ts_tsmdb/utils/checksum.ts
Normal file
88
ts/ts_tsmdb/utils/checksum.ts
Normal file
@@ -0,0 +1,88 @@
|
||||
/**
|
||||
* CRC32 checksum utilities for data integrity
|
||||
*/
|
||||
|
||||
// CRC32 lookup table
|
||||
const CRC32_TABLE: number[] = [];
|
||||
|
||||
// Initialize the CRC32 table
|
||||
function initCRC32Table(): void {
|
||||
if (CRC32_TABLE.length > 0) return;
|
||||
|
||||
for (let i = 0; i < 256; i++) {
|
||||
let crc = i;
|
||||
for (let j = 0; j < 8; j++) {
|
||||
crc = (crc & 1) ? (0xEDB88320 ^ (crc >>> 1)) : (crc >>> 1);
|
||||
}
|
||||
CRC32_TABLE[i] = crc >>> 0;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate CRC32 checksum for a string
|
||||
*/
|
||||
export function calculateCRC32(data: string): number {
|
||||
initCRC32Table();
|
||||
|
||||
let crc = 0xFFFFFFFF;
|
||||
for (let i = 0; i < data.length; i++) {
|
||||
const byte = data.charCodeAt(i) & 0xFF;
|
||||
crc = CRC32_TABLE[(crc ^ byte) & 0xFF] ^ (crc >>> 8);
|
||||
}
|
||||
return (~crc) >>> 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate CRC32 checksum for a Buffer
|
||||
*/
|
||||
export function calculateCRC32Buffer(data: Buffer): number {
|
||||
initCRC32Table();
|
||||
|
||||
let crc = 0xFFFFFFFF;
|
||||
for (let i = 0; i < data.length; i++) {
|
||||
crc = CRC32_TABLE[(crc ^ data[i]) & 0xFF] ^ (crc >>> 8);
|
||||
}
|
||||
return (~crc) >>> 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate checksum for a document (serialized as JSON)
|
||||
*/
|
||||
export function calculateDocumentChecksum(doc: Record<string, any>): number {
|
||||
// Exclude _checksum field from calculation
|
||||
const { _checksum, ...docWithoutChecksum } = doc;
|
||||
const json = JSON.stringify(docWithoutChecksum);
|
||||
return calculateCRC32(json);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add checksum to a document
|
||||
*/
|
||||
export function addChecksum<T extends Record<string, any>>(doc: T): T & { _checksum: number } {
|
||||
const checksum = calculateDocumentChecksum(doc);
|
||||
return { ...doc, _checksum: checksum };
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify checksum of a document
|
||||
* Returns true if checksum is valid or if document has no checksum
|
||||
*/
|
||||
export function verifyChecksum(doc: Record<string, any>): boolean {
|
||||
if (!('_checksum' in doc)) {
|
||||
// No checksum to verify
|
||||
return true;
|
||||
}
|
||||
|
||||
const storedChecksum = doc._checksum;
|
||||
const calculatedChecksum = calculateDocumentChecksum(doc);
|
||||
|
||||
return storedChecksum === calculatedChecksum;
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove checksum from a document
|
||||
*/
|
||||
export function removeChecksum<T extends Record<string, any>>(doc: T): Omit<T, '_checksum'> {
|
||||
const { _checksum, ...docWithoutChecksum } = doc;
|
||||
return docWithoutChecksum as Omit<T, '_checksum'>;
|
||||
}
|
||||
1
ts/ts_tsmdb/utils/index.ts
Normal file
1
ts/ts_tsmdb/utils/index.ts
Normal file
@@ -0,0 +1 @@
|
||||
export * from './checksum.js';
|
||||
Reference in New Issue
Block a user