import * as plugins from '../tsmdb.plugins.js'; import type { IStorageAdapter } from './IStorageAdapter.js'; import type { IStoredDocument, IOpLogEntry, Document } from '../types/interfaces.js'; /** * In-memory storage adapter for TsmDB * Optionally supports persistence to a file */ export class MemoryStorageAdapter implements IStorageAdapter { // Database -> Collection -> Documents private databases: Map>> = new Map(); // Database -> Collection -> Indexes private indexes: Map; unique?: boolean; sparse?: boolean; expireAfterSeconds?: number; }>>> = new Map(); // OpLog entries private opLog: IOpLogEntry[] = []; private opLogCounter = 0; // Persistence settings private persistPath?: string; private persistInterval?: ReturnType; private fs = new plugins.smartfs.SmartFs(new plugins.smartfs.SmartFsProviderNode()); constructor(options?: { persistPath?: string; persistIntervalMs?: number }) { this.persistPath = options?.persistPath; if (this.persistPath && options?.persistIntervalMs) { this.persistInterval = setInterval(() => { this.persist().catch(console.error); }, options.persistIntervalMs); } } async initialize(): Promise { if (this.persistPath) { await this.restore(); } } async close(): Promise { if (this.persistInterval) { clearInterval(this.persistInterval); } if (this.persistPath) { await this.persist(); } } // ============================================================================ // Database Operations // ============================================================================ async listDatabases(): Promise { return Array.from(this.databases.keys()); } async createDatabase(dbName: string): Promise { if (!this.databases.has(dbName)) { this.databases.set(dbName, new Map()); this.indexes.set(dbName, new Map()); } } async dropDatabase(dbName: string): Promise { const existed = this.databases.has(dbName); this.databases.delete(dbName); this.indexes.delete(dbName); return existed; } async databaseExists(dbName: string): Promise { return this.databases.has(dbName); } // ============================================================================ // Collection Operations // ============================================================================ async listCollections(dbName: string): Promise { const db = this.databases.get(dbName); return db ? Array.from(db.keys()) : []; } async createCollection(dbName: string, collName: string): Promise { await this.createDatabase(dbName); const db = this.databases.get(dbName)!; if (!db.has(collName)) { db.set(collName, new Map()); // Initialize default _id index const dbIndexes = this.indexes.get(dbName)!; dbIndexes.set(collName, [{ name: '_id_', key: { _id: 1 }, unique: true }]); } } async dropCollection(dbName: string, collName: string): Promise { const db = this.databases.get(dbName); if (!db) return false; const existed = db.has(collName); db.delete(collName); const dbIndexes = this.indexes.get(dbName); if (dbIndexes) { dbIndexes.delete(collName); } return existed; } async collectionExists(dbName: string, collName: string): Promise { const db = this.databases.get(dbName); return db ? db.has(collName) : false; } async renameCollection(dbName: string, oldName: string, newName: string): Promise { const db = this.databases.get(dbName); if (!db || !db.has(oldName)) { throw new Error(`Collection ${oldName} not found`); } const collection = db.get(oldName)!; db.set(newName, collection); db.delete(oldName); // Also rename indexes const dbIndexes = this.indexes.get(dbName); if (dbIndexes && dbIndexes.has(oldName)) { const collIndexes = dbIndexes.get(oldName)!; dbIndexes.set(newName, collIndexes); dbIndexes.delete(oldName); } } // ============================================================================ // Document Operations // ============================================================================ private getCollection(dbName: string, collName: string): Map { const db = this.databases.get(dbName); if (!db) { throw new Error(`Database ${dbName} not found`); } const collection = db.get(collName); if (!collection) { throw new Error(`Collection ${collName} not found`); } return collection; } private ensureCollection(dbName: string, collName: string): Map { if (!this.databases.has(dbName)) { this.databases.set(dbName, new Map()); this.indexes.set(dbName, new Map()); } const db = this.databases.get(dbName)!; if (!db.has(collName)) { db.set(collName, new Map()); const dbIndexes = this.indexes.get(dbName)!; dbIndexes.set(collName, [{ name: '_id_', key: { _id: 1 }, unique: true }]); } return db.get(collName)!; } async insertOne(dbName: string, collName: string, doc: Document): Promise { const collection = this.ensureCollection(dbName, collName); const storedDoc: IStoredDocument = { ...doc, _id: doc._id instanceof plugins.bson.ObjectId ? doc._id : new plugins.bson.ObjectId(doc._id), }; if (!storedDoc._id) { storedDoc._id = new plugins.bson.ObjectId(); } const idStr = storedDoc._id.toHexString(); if (collection.has(idStr)) { throw new Error(`Duplicate key error: _id ${idStr}`); } collection.set(idStr, storedDoc); return storedDoc; } async insertMany(dbName: string, collName: string, docs: Document[]): Promise { const results: IStoredDocument[] = []; for (const doc of docs) { results.push(await this.insertOne(dbName, collName, doc)); } return results; } async findAll(dbName: string, collName: string): Promise { const collection = this.ensureCollection(dbName, collName); return Array.from(collection.values()); } async findByIds(dbName: string, collName: string, ids: Set): Promise { const collection = this.ensureCollection(dbName, collName); const results: IStoredDocument[] = []; for (const id of ids) { const doc = collection.get(id); if (doc) { results.push(doc); } } return results; } async findById(dbName: string, collName: string, id: plugins.bson.ObjectId): Promise { const collection = this.ensureCollection(dbName, collName); return collection.get(id.toHexString()) || null; } async updateById(dbName: string, collName: string, id: plugins.bson.ObjectId, doc: IStoredDocument): Promise { const collection = this.ensureCollection(dbName, collName); const idStr = id.toHexString(); if (!collection.has(idStr)) { return false; } collection.set(idStr, doc); return true; } async deleteById(dbName: string, collName: string, id: plugins.bson.ObjectId): Promise { const collection = this.ensureCollection(dbName, collName); return collection.delete(id.toHexString()); } async deleteByIds(dbName: string, collName: string, ids: plugins.bson.ObjectId[]): Promise { let count = 0; for (const id of ids) { if (await this.deleteById(dbName, collName, id)) { count++; } } return count; } async count(dbName: string, collName: string): Promise { const collection = this.ensureCollection(dbName, collName); return collection.size; } // ============================================================================ // Index Operations // ============================================================================ async saveIndex( dbName: string, collName: string, indexName: string, indexSpec: { key: Record; unique?: boolean; sparse?: boolean; expireAfterSeconds?: number } ): Promise { await this.createCollection(dbName, collName); const dbIndexes = this.indexes.get(dbName)!; let collIndexes = dbIndexes.get(collName); if (!collIndexes) { collIndexes = [{ name: '_id_', key: { _id: 1 }, unique: true }]; dbIndexes.set(collName, collIndexes); } // Check if index already exists const existingIndex = collIndexes.findIndex(i => i.name === indexName); if (existingIndex >= 0) { collIndexes[existingIndex] = { name: indexName, ...indexSpec }; } else { collIndexes.push({ name: indexName, ...indexSpec }); } } async getIndexes(dbName: string, collName: string): Promise; unique?: boolean; sparse?: boolean; expireAfterSeconds?: number; }>> { const dbIndexes = this.indexes.get(dbName); if (!dbIndexes) return [{ name: '_id_', key: { _id: 1 }, unique: true }]; const collIndexes = dbIndexes.get(collName); return collIndexes || [{ name: '_id_', key: { _id: 1 }, unique: true }]; } async dropIndex(dbName: string, collName: string, indexName: string): Promise { if (indexName === '_id_') { throw new Error('Cannot drop _id index'); } const dbIndexes = this.indexes.get(dbName); if (!dbIndexes) return false; const collIndexes = dbIndexes.get(collName); if (!collIndexes) return false; const idx = collIndexes.findIndex(i => i.name === indexName); if (idx >= 0) { collIndexes.splice(idx, 1); return true; } return false; } // ============================================================================ // OpLog Operations // ============================================================================ async appendOpLog(entry: IOpLogEntry): Promise { this.opLog.push(entry); // Trim oplog if it gets too large (keep last 10000 entries) if (this.opLog.length > 10000) { this.opLog = this.opLog.slice(-10000); } } async getOpLogAfter(ts: plugins.bson.Timestamp, limit: number = 1000): Promise { const tsValue = ts.toNumber(); const entries = this.opLog.filter(e => e.ts.toNumber() > tsValue); return entries.slice(0, limit); } async getLatestOpLogTimestamp(): Promise { if (this.opLog.length === 0) return null; return this.opLog[this.opLog.length - 1].ts; } /** * Generate a new timestamp for oplog entries */ generateTimestamp(): plugins.bson.Timestamp { this.opLogCounter++; return new plugins.bson.Timestamp({ t: Math.floor(Date.now() / 1000), i: this.opLogCounter }); } // ============================================================================ // Transaction Support // ============================================================================ async createSnapshot(dbName: string, collName: string): Promise { const docs = await this.findAll(dbName, collName); // Deep clone the documents for snapshot isolation return docs.map(doc => JSON.parse(JSON.stringify(doc))); } async hasConflicts( dbName: string, collName: string, ids: plugins.bson.ObjectId[], snapshotTime: plugins.bson.Timestamp ): Promise { // Check if any of the given document IDs have been modified after snapshotTime const ns = `${dbName}.${collName}`; const modifiedIds = new Set(); for (const entry of this.opLog) { if (entry.ts.greaterThan(snapshotTime) && entry.ns === ns) { if (entry.o._id) { modifiedIds.add(entry.o._id.toString()); } if (entry.o2?._id) { modifiedIds.add(entry.o2._id.toString()); } } } for (const id of ids) { if (modifiedIds.has(id.toString())) { return true; } } return false; } // ============================================================================ // Persistence // ============================================================================ async persist(): Promise { if (!this.persistPath) return; const data = { databases: {} as Record>, indexes: {} as Record>, opLogCounter: this.opLogCounter, }; for (const [dbName, collections] of this.databases) { data.databases[dbName] = {}; for (const [collName, docs] of collections) { data.databases[dbName][collName] = Array.from(docs.values()); } } for (const [dbName, collIndexes] of this.indexes) { data.indexes[dbName] = {}; for (const [collName, indexes] of collIndexes) { data.indexes[dbName][collName] = indexes; } } // Ensure parent directory exists const dir = this.persistPath.substring(0, this.persistPath.lastIndexOf('/')); if (dir) { await this.fs.directory(dir).recursive().create(); } await this.fs.file(this.persistPath).encoding('utf8').write(JSON.stringify(data, null, 2)); } async restore(): Promise { if (!this.persistPath) return; try { const exists = await this.fs.file(this.persistPath).exists(); if (!exists) return; const content = await this.fs.file(this.persistPath).encoding('utf8').read(); const data = JSON.parse(content as string); this.databases.clear(); this.indexes.clear(); for (const [dbName, collections] of Object.entries(data.databases || {})) { const dbMap = new Map>(); this.databases.set(dbName, dbMap); for (const [collName, docs] of Object.entries(collections as Record)) { const collMap = new Map(); for (const doc of docs) { // Restore ObjectId if (doc._id && typeof doc._id === 'string') { doc._id = new plugins.bson.ObjectId(doc._id); } else if (doc._id && typeof doc._id === 'object' && doc._id.$oid) { doc._id = new plugins.bson.ObjectId(doc._id.$oid); } collMap.set(doc._id.toHexString(), doc); } dbMap.set(collName, collMap); } } for (const [dbName, collIndexes] of Object.entries(data.indexes || {})) { const indexMap = new Map(); this.indexes.set(dbName, indexMap); for (const [collName, indexes] of Object.entries(collIndexes as Record)) { indexMap.set(collName, indexes); } } this.opLogCounter = data.opLogCounter || 0; } catch (error) { // If restore fails, start fresh console.warn('Failed to restore from persistence:', error); } } }