From 9be43a85ef705f07c51aa2fbdf5948c8e537f530 Mon Sep 17 00:00:00 2001 From: Juergen Kunz Date: Mon, 18 Aug 2025 11:50:37 +0000 Subject: [PATCH] fix(readme): Update README: clarify examples, expand search/cursor/docs and add local Claude settings --- changelog.md | 8 + readme.md | 971 ++++++++++++++++----------------------- ts/00_commitinfo_data.ts | 2 +- 3 files changed, 416 insertions(+), 565 deletions(-) diff --git a/changelog.md b/changelog.md index fa0a28b..61f8d72 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,13 @@ # Changelog +## 2025-08-18 - 5.16.2 - fix(readme) +Update README: clarify examples, expand search/cursor/docs and add local Claude settings + +- Refined README wording and structure: clearer Quick Start, improved examples and developer-focused phrasing +- Expanded documentation for search, cursors, change streams, distributed coordination, transactions and EasyStore with more concrete code examples +- Adjusted code examples to show safer defaults (ID generation, status/tags, connection pooling) and improved best-practices guidance +- Added .claude/settings.local.json to provide local assistant/CI permission configuration + ## 2025-08-12 - 5.16.1 - fix(core) Improve error handling and logging; enhance search query sanitization; update dependency versions and documentation diff --git a/readme.md b/readme.md index 586d64f..c9ad286 100644 --- a/readme.md +++ b/readme.md @@ -2,28 +2,28 @@ [![npm version](https://badge.fury.io/js/@push.rocks%2Fsmartdata.svg)](https://www.npmjs.com/package/@push.rocks/smartdata) -**The ultimate TypeScript-first MongoDB wrapper** that makes database operations beautiful, type-safe, and incredibly powerful. Built for modern applications that demand real-time performance, distributed coordination, and rock-solid reliability. +**The ultimate TypeScript-first MongoDB wrapper** that turns your database into a type-safe powerhouse. Built for modern applications that demand real-time performance, distributed coordination, and rock-solid reliability. ## 🌟 Why SmartData? -SmartData isn't just another MongoDB wrapper - it's a complete data management powerhouse that transforms how you work with databases: +SmartData isn't just another MongoDB wrapper - it's a complete paradigm shift in how you work with databases: -- 🔒 **100% Type-Safe**: Full TypeScript with decorators, generics, and deep query typing -- ⚡ **Lightning Fast**: Connection pooling, cursor streaming, and optimized indexing -- 🔄 **Real-time Sync**: MongoDB Change Streams with RxJS for reactive applications -- 🌍 **Distributed Ready**: Built-in leader election and task coordination -- 🛡️ **Security First**: NoSQL injection prevention, credential encoding, and secure defaults -- 🎯 **Developer Friendly**: Intuitive API, powerful search, and amazing DX +- 🔒 **100% Type-Safe**: Full TypeScript with decorators, generics, and compile-time query validation +- ⚡ **Lightning Fast**: Connection pooling, cursor streaming, and intelligent indexing +- 🔄 **Real-time Ready**: MongoDB Change Streams with RxJS for reactive applications +- 🌍 **Distributed Systems**: Built-in leader election and task coordination +- 🛡️ **Security First**: NoSQL injection prevention, automatic sanitization, and secure defaults +- 🎯 **Developer Experience**: Intuitive API, powerful search, and IntelliSense that just works ## 📦 Installation ```bash -# Using npm -npm install @push.rocks/smartdata --save - # Using pnpm (recommended) pnpm add @push.rocks/smartdata +# Using npm +npm install @push.rocks/smartdata + # Using yarn yarn add @push.rocks/smartdata ``` @@ -48,13 +48,13 @@ const db = new SmartdataDb({ mongoDbUser: 'username', mongoDbPass: 'password', - // Optional: Configure connection pooling (new!) - maxPoolSize: 100, // Max connections in pool (default: 100) - maxIdleTimeMS: 300000, // Max idle time (default: 5 minutes) - serverSelectionTimeoutMS: 30000 // Connection timeout (default: 30s) + // Optional: Advanced connection pooling + maxPoolSize: 100, // Max connections in pool + maxIdleTimeMS: 300000, // Max idle time before connection close + serverSelectionTimeoutMS: 30000 // Connection timeout }); -// Initialize with automatic retry and connection pooling +// Initialize with automatic retry and health monitoring await db.init(); ``` @@ -74,22 +74,25 @@ import { ObjectId } from 'mongodb'; @Collection(() => db) class User extends SmartDataDbDoc { @unI() - public id: string = 'unique-user-id'; // Unique index + public id: string; // Unique index with automatic ID generation @svDb() - @searchable() // Enable full-text search + @searchable() // Enable Lucene-style searching public username: string; @svDb() @searchable() - @index({ unique: false }) // Regular index for performance + @index({ unique: false }) // Performance index public email: string; @svDb() - public organizationId: ObjectId; // Automatically handled as BSON ObjectId + public status: 'active' | 'inactive' | 'pending'; // Full union type support @svDb() - public profilePicture: Buffer; // Automatically handled as BSON Binary + public organizationId: ObjectId; // Native MongoDB types + + @svDb() + public profilePicture: Buffer; // Binary data support @svDb({ // Custom serialization for complex objects @@ -98,6 +101,9 @@ class User extends SmartDataDbDoc { }) public preferences: Record; + @svDb() + public tags: string[]; // Array support with operators + @svDb() public createdAt: Date = new Date(); @@ -105,6 +111,7 @@ class User extends SmartDataDbDoc { super(); this.username = username; this.email = email; + this.id = User.getNewId(); } } ``` @@ -114,142 +121,63 @@ class User extends SmartDataDbDoc { ```typescript // ✨ Create const user = new User('johndoe', 'john@example.com'); +user.status = 'active'; +user.tags = ['developer', 'typescript']; await user.save(); -// 🔍 Read +// 🔍 Read - with full type safety const foundUser = await User.getInstance({ username: 'johndoe' }); -const allUsers = await User.getInstances({ email: 'john@example.com' }); +const activeUsers = await User.getInstances({ status: 'active' }); // ✏️ Update foundUser.email = 'newemail@example.com'; -await foundUser.save(); - -// 🔄 Upsert (update or insert) -// Note: Upsert is handled automatically by save() - if document exists it updates, otherwise inserts +foundUser.tags.push('mongodb'); await foundUser.save(); // 🗑️ Delete await foundUser.delete(); ``` -## 🔍 Query Filters Guide +## 🔥 Advanced Features -SmartData provides a comprehensive and type-safe filtering system that supports all MongoDB query operators while preventing dangerous operations like `$where` for security. +### 🎯 Type-Safe Query Filters -### Basic Filtering +SmartData provides the most advanced type-safe filtering system for MongoDB, supporting all operators while maintaining full IntelliSense: ```typescript -// Simple equality - find users with exact name -const johns = await User.getInstances({ name: 'John Doe' }); - -// Multiple conditions (implicit AND) -const activeAdults = await User.getInstances({ - status: 'active', - age: { $gte: 18 } +// Union types work perfectly +const users = await User.getInstances({ + status: { $in: ['active', 'pending'] } // TypeScript validates these values! }); -// Nested object fields - use dot notation -const recentUsers = await User.getInstances({ - 'metadata.lastLogin': { $gte: new Date('2024-01-01') }, - 'metadata.loginCount': { $gt: 5 } -}); -``` - -### Comparison Operators - -All standard MongoDB comparison operators are supported: - -```typescript -// Greater than / Greater than or equal -const adults = await User.getInstances({ age: { $gte: 18 } }); -const seniors = await User.getInstances({ age: { $gt: 65 } }); - -// Less than / Less than or equal -const youth = await User.getInstances({ age: { $lt: 25 } }); -const under30 = await User.getInstances({ age: { $lte: 30 } }); - -// Not equal -const notPending = await User.getInstances({ status: { $ne: 'pending' } }); - -// Combine multiple comparisons -const middleAged = await User.getInstances({ - age: { $gte: 30, $lt: 50 } -}); -``` - -### Array Operators - -SmartData fully supports MongoDB's array query operators: - -```typescript -// $in - Match any value in array -const adminsAndMods = await User.getInstances({ - role: { $in: ['admin', 'moderator'] } +// Comparison operators with type checking +const adults = await User.getInstances({ + age: { $gte: 18, $lt: 65 } // Type-safe numeric comparisons }); -// $nin - Match none of the values -const regularUsers = await User.getInstances({ - role: { $nin: ['admin', 'moderator'] } +// Array operations with full type safety +const experts = await User.getInstances({ + tags: { $all: ['typescript', 'mongodb'] }, // Must have all tags + skills: { $size: 5 } // Exactly 5 skills }); -// $all - Array contains all specified elements -const fullStackDevs = await User.getInstances({ - skills: { $all: ['javascript', 'nodejs', 'mongodb'] } -}); - -// $size - Array has specific length -const usersWithThreeSkills = await User.getInstances({ - skills: { $size: 3 } -}); - -// $elemMatch - Match array elements with multiple conditions -const ordersWithExpensiveItems = await Order.getInstances({ - items: { - $elemMatch: { - product: 'laptop', - quantity: { $gte: 2 }, - price: { $gt: 1000 } +// Complex nested queries +const results = await Order.getInstances({ + $and: [ + { status: { $in: ['pending', 'processing'] } }, + { 'items.price': { $gte: 100 } }, // Dot notation for nested fields + { + items: { + $elemMatch: { // Array element matching + product: 'laptop', + quantity: { $gte: 2 } + } + } } - } -}); -``` - -### Working with Arrays - -```typescript -// Find documents where array field contains a value -const javascriptUsers = await User.getInstances({ - tags: { $in: ['javascript'] } // Users with 'javascript' in tags array + ] }); -// Find documents where array field contains ALL values -const expertUsers = await User.getInstances({ - certifications: { $all: ['mongodb', 'nodejs', 'typescript'] } -}); - -// Query nested arrays of objects -const highValueOrders = await Order.getInstances({ - 'items.price': { $gte: 100 } // Orders with any item priced >= 100 -}); - -// Complex array element matching -const bulkElectronicsOrders = await Order.getInstances({ - items: { - $elemMatch: { - category: 'electronics', - quantity: { $gte: 5 }, - discount: { $exists: true } - } - } -}); -``` - -### Logical Operators - -Combine multiple conditions with logical operators: - -```typescript -// $or - Match any condition +// Logical operators const urgentOrHighValue = await Order.getInstances({ $or: [ { priority: 'urgent' }, @@ -257,218 +185,13 @@ const urgentOrHighValue = await Order.getInstances({ ] }); -// $and - Match all conditions (usually implicit, but useful for complex queries) -const premiumActiveUsers = await User.getInstances({ - $and: [ - { subscription: 'premium' }, - { status: 'active' }, - { lastLogin: { $gte: new Date(Date.now() - 30*24*60*60*1000) } } - ] -}); - -// $nor - Match none of the conditions -const availableProducts = await Product.getInstances({ - $nor: [ - { status: 'discontinued' }, - { inventory: { $lte: 0 } } - ] -}); - -// $not - Negate a condition -const nonEmptyArrays = await User.getInstances({ - tags: { $not: { $size: 0 } } -}); - -// Nested logical operations -const complexQuery = await User.getInstances({ - $or: [ - { - $and: [ - { role: 'admin' }, - { department: 'IT' } - ] - }, - { - $and: [ - { role: 'manager' }, - { yearsExperience: { $gte: 5 } } - ] - } - ] -}); +// Security: $where is automatically blocked +// await User.getInstances({ $where: '...' }); // ❌ Throws security error ``` -### Element Operators - -Check for field existence and types: - -```typescript -// $exists - Check if field exists -const usersWithAvatar = await User.getInstances({ - 'profile.avatar': { $exists: true } -}); - -const usersWithoutPhone = await User.getInstances({ - phoneNumber: { $exists: false } -}); - -// $type - Check field type (MongoDB BSON types) -const usersWithStringId = await User.getInstances({ - id: { $type: 'string' } // or use type number: 2 for string -}); -``` - -### Regular Expressions - -Use regex for pattern matching: - -```typescript -// Case-insensitive search -const johnUsers = await User.getInstances({ - name: { $regex: '^john', $options: 'i' } -}); - -// Email domain matching -const gmailUsers = await User.getInstances({ - email: { $regex: '@gmail\.com$' } -}); - -// Complex pattern matching -const validPhones = await User.getInstances({ - phone: { $regex: '^\+1-\d{3}-\d{3}-\d{4}$' } -}); -``` - -### Advanced Filtering Patterns - -```typescript -// Combine multiple operator types -const targetUsers = await User.getInstances({ - $and: [ - { age: { $gte: 25, $lte: 45 } }, - { status: { $in: ['active', 'premium'] } }, - { - $or: [ - { 'subscription.type': 'annual' }, - { 'subscription.credits': { $gte: 100 } } - ] - }, - { tags: { $all: ['verified', 'trusted'] } } - ] -}); - -// Filter with post-processing validation -const results = await Product.search('laptop', { - filter: { - price: { $lte: 2000 }, - brand: { $in: ['Apple', 'Dell', 'Lenovo'] } - }, - validate: async (product) => { - // Additional custom validation - return product.warrantyYears >= 2 && product.inStock; - } -}); - -// Transaction-safe filtering -const session = db.startSession(); -await session.withTransaction(async () => { - const users = await User.getInstances( - { status: 'pending' }, - { session } // Pass session for transaction - ); - // Process users within transaction -}); -``` - -### Performance Tips - -1. **Use Indexes**: Create indexes on frequently filtered fields: - ```typescript - @index() public status: string; // Regular index - @unI() public email: string; // Unique index - ``` - -2. **Limit Results**: Use cursors for large datasets: - ```typescript - const cursor = await User.getCursor({ status: 'active' }); - await cursor.forEach(async (user) => { - // Process one at a time - }); - ``` - -3. **Projection**: Retrieve only needed fields (using MongoDB native): - ```typescript - const collection = User.getCollection(); - const results = await collection.mongoDbCollection - .find({ status: 'active' }) - .project({ name: 1, email: 1 }) - .toArray(); - ``` - -### Security Considerations - -SmartData automatically prevents dangerous operations: - -```typescript -// ❌ These will throw errors: -await User.getInstances({ - $where: 'this.age > 25' // Blocked: JavaScript execution -}); - -await User.getInstances({ - 'user.name': { 'bad.key': 'value' } // Blocked: dots in keys -}); - -// ✅ Safe alternatives: -await User.getInstances({ - age: { $gt: 25 } // Use operators instead -}); - -await User.getInstances({ - 'user.name': 'value' // Use dot notation for nested fields -}); -``` - -### Common Pitfalls and Solutions - -```typescript -// ❌ Wrong: Array operators need arrays -await User.getInstances({ - role: { $in: 'admin' } // Error: $in requires array -}); - -// ✅ Correct: -await User.getInstances({ - role: { $in: ['admin'] } -}); - -// ❌ Wrong: Direct array comparison (exact match) -await User.getInstances({ - tags: 'javascript' // Only matches if tags = ['javascript'] exactly -}); - -// ✅ Correct: Check if array contains element -await User.getInstances({ - tags: { $in: ['javascript'] } // Matches if 'javascript' is in tags -}); - -// ❌ Wrong: Checking multiple array elements incorrectly -await User.getInstances({ - skills: 'javascript', - skills: 'nodejs' // Object can't have duplicate keys! -}); - -// ✅ Correct: Use $all for multiple elements -await User.getInstances({ - skills: { $all: ['javascript', 'nodejs'] } -}); -``` - -## 🔥 Advanced Features - ### 🔎 Powerful Search Engine -SmartData includes a Lucene-style search engine with automatic field indexing: +Built-in Lucene-style search with automatic indexing: ```typescript @Collection(() => db) @@ -480,31 +203,31 @@ class Product extends SmartDataDbDoc { @svDb() public price: number; } -// 🎯 Exact phrase search -await Product.search('"MacBook Pro 16"'); +// Simple text search across all searchable fields +const results = await Product.search('laptop'); -// 🔤 Wildcard search -await Product.search('Mac*'); +// Field-specific search +const electronics = await Product.search('category:Electronics'); -// 📁 Field-specific search -await Product.search('category:Electronics'); +// Wildcard searches +const matches = await Product.search('Mac*'); -// 🧮 Boolean operators -await Product.search('(laptop OR desktop) AND NOT gaming'); +// Boolean operators +const query = await Product.search('laptop AND NOT gaming'); -// 🔐 Secure multi-field search -await Product.search('TypeScript MongoDB'); // Automatically escaped +// Phrase search +const exact = await Product.search('"MacBook Pro 16"'); -// 🏷️ Scoped search with filters -await Product.search('laptop', { - filter: { price: { $lt: 2000 } }, - validate: (p) => p.inStock === true +// Combined with filters for powerful queries +const affordable = await Product.search('laptop', { + filter: { price: { $lte: 1500 } }, + validate: async (p) => p.inStock === true }); ``` ### 💾 EasyStore - Type-Safe Key-Value Storage -Perfect for configuration, caching, and shared state: +Perfect for configuration, caching, and session management: ```typescript interface AppConfig { @@ -522,7 +245,7 @@ interface AppConfig { // Create a type-safe store const config = await db.createEasyStore('app-config'); -// Write with full IntelliSense +// Write with full type checking await config.writeKey('features', { darkMode: true, notifications: false @@ -530,123 +253,132 @@ await config.writeKey('features', { // Read with guaranteed types const features = await config.readKey('features'); -// TypeScript knows: features.darkMode is boolean +// TypeScript knows: features.darkMode is boolean ✅ -// Atomic operations -await config.writeAll({ - apiKey: 'new-key', - limits: { maxUsers: 1000, maxStorage: 5000 } -}); +// Atomic updates +await config.updateKey('limits', (current) => ({ + ...current, + maxUsers: current.maxUsers + 100 +})); -// Delete a key +// Delete keys await config.deleteKey('features'); -// Wipe entire store +// Clear entire store await config.wipe(); ``` -### 🌐 Distributed Coordination - -Build resilient distributed systems with automatic leader election: - -```typescript -const coordinator = new SmartdataDistributedCoordinator(db); - -// Start coordination with automatic heartbeat -await coordinator.start(); - -// Check if this instance is the leader -const eligibleLeader = await coordinator.getEligibleLeader(); -const isLeader = eligibleLeader?.id === coordinator.id; - -if (isLeader) { - console.log('🎖️ This instance is now the leader!'); - // Leader-specific tasks are handled internally by leadFunction() - // The coordinator automatically manages leader election and failover -} - -// Fire distributed task requests (for taskbuffer integration) -const result = await coordinator.fireDistributedTaskRequest({ - taskName: 'maintenance', - taskExecutionTime: Date.now(), - requestResponseId: 'unique-id' -}); - -// Graceful shutdown -await coordinator.stop(); -``` - ### 📡 Real-Time Change Streams React to database changes instantly with RxJS integration: ```typescript -// Watch for specific changes +// Watch for changes with automatic reconnection const watcher = await User.watch( - { active: true }, // Only watch active users - { - fullDocument: 'updateLookup', // Include full document - bufferTimeMs: 100, // Buffer for performance + { status: 'active' }, // Filter which documents to watch + { + fullDocument: 'updateLookup', // Get full document on updates + bufferTimeMs: 100 // Buffer changes for efficiency } ); -// Subscribe with RxJS (emits documents or arrays if buffered) -watcher.changeSubject - .pipe( - filter(user => user !== null), // Filter out deletions - ) - .subscribe(user => { - console.log(`📢 User change detected: ${user.username}`); - sendNotification(user.email); - }); - -// Or use EventEmitter pattern -watcher.on('change', (user) => { - if (user) { - console.log(`✏️ User changed: ${user.username}`); - } else { - console.log(`👋 User deleted`); - } +// Subscribe to changes with RxJS +watcher.changeSubject.subscribe({ + next: (change) => { + console.log('User changed:', change.fullDocument); + + switch(change.operationType) { + case 'insert': + console.log('New user created'); + break; + case 'update': + console.log('User updated'); + break; + case 'delete': + console.log('User deleted'); + break; + } + }, + error: (err) => console.error('Watch error:', err) }); +// Advanced: Watch with aggregation pipeline +const complexWatcher = await Order.watch( + { status: 'pending' }, + { + pipeline: [ + { $match: { 'fullDocument.totalAmount': { $gte: 1000 } } }, + { $addFields: { isHighValue: true } } + ] + } +); + // Clean up when done -await watcher.stop(); +await watcher.close(); ``` -### 🎯 Cursor Operations for Large Datasets +### 🌐 Distributed Coordination -Handle millions of documents efficiently: +Build resilient distributed systems with automatic leader election: ```typescript -// Create a cursor with modifiers +import { SmartdataDistributedCoordinator } from '@push.rocks/smartdata'; + +const coordinator = new SmartdataDistributedCoordinator(db); + +// Start coordination with automatic heartbeat +await coordinator.start(); + +// Leader election happens automatically +coordinator.asyncExecutionScheduler.addJobHandler('maintenance', async () => { + // This code only runs on the leader instance + console.log('🎖️ Running maintenance as leader'); + await performDatabaseCleanup(); +}); + +// Check leadership status +const isLeader = coordinator.isLeader; + +// Fire distributed tasks (integrates with @push.rocks/taskbuffer) +const result = await coordinator.fireDistributedTaskRequest({ + taskName: 'process-payments', + taskExecutionTime: Date.now(), + requestResponseId: 'unique-id' +}); + +// Graceful shutdown with leadership handoff +await coordinator.stop(); +``` + +### 🔄 Cursor Streaming for Large Datasets + +Process millions of documents without memory issues: + +```typescript +// Get a cursor for memory-efficient iteration const cursor = await User.getCursor( - { active: true }, - { + { status: 'active' }, + { + // Optional: Use MongoDB native cursor modifiers modifier: (cursor) => cursor .sort({ createdAt: -1 }) - .skip(100) - .limit(50) + .limit(10000) + .project({ email: 1, username: 1 }) } ); -// Stream processing - memory efficient +// Process one at a time await cursor.forEach(async (user) => { - await processUser(user); - // Processes one at a time, minimal memory usage + await sendEmail(user.email); + // Each document is loaded individually }); -// Manual iteration -let user; -while (user = await cursor.next()) { - if (shouldStop(user)) { - break; - } - await handleUser(user); +// Or use async iteration +for await (const user of cursor) { + if (shouldStop(user)) break; + await processUser(user); } -// Convert to array (only for small datasets!) -const users = await cursor.toArray(); - // Always clean up await cursor.close(); ``` @@ -663,15 +395,17 @@ try { // All operations in this block are atomic const sender = await User.getInstance( { id: 'user-1' }, - session // Pass session to all operations + { session } // Pass session to all operations ); + sender.balance -= 100; await sender.save({ session }); const receiver = await User.getInstance( { id: 'user-2' }, - session + { session } ); + receiver.balance += 100; await receiver.save({ session }); @@ -681,9 +415,7 @@ try { } }); - console.log('✅ Transaction completed successfully'); -} catch (error) { - console.error('❌ Transaction failed, rolled back'); + console.log('✅ Transaction completed'); } finally { await session.endSession(); } @@ -697,13 +429,8 @@ Handle complex data types with custom serializers: class Document extends SmartDataDbDoc { @svDb({ // Encrypt sensitive data before storing - serialize: async (value) => { - return await encrypt(value); - }, - // Decrypt when reading - deserialize: async (value) => { - return await decrypt(value); - } + serialize: async (value) => await encrypt(value), + deserialize: async (value) => await decrypt(value) }) public sensitiveData: string; @@ -715,11 +442,18 @@ class Document extends SmartDataDbDoc { public largePayload: any; @svDb({ - // Store sets as arrays + // Store Sets as arrays serialize: (set) => Array.from(set), deserialize: (arr) => new Set(arr) }) public tags: Set; + + @svDb({ + // Handle custom date formats + serialize: (date) => date?.toISOString(), + deserialize: (str) => str ? new Date(str) : null + }) + public scheduledAt: Date | null; } ``` @@ -731,171 +465,280 @@ Add custom logic at any point in the document lifecycle: @Collection(() => db) class Order extends SmartDataDbDoc { @unI() public id: string; - @svDb() public items: OrderItem[]; - @svDb() public total: number; - @svDb() public status: 'pending' | 'paid' | 'shipped'; + @svDb() public items: Array<{ product: string; quantity: number }>; + @svDb() public totalAmount: number; + @svDb() public status: string; - // Validate and calculate before saving + // Called before saving (create or update) async beforeSave() { - this.total = this.items.reduce((sum, item) => + // Recalculate total + this.totalAmount = this.items.reduce((sum, item) => sum + (item.price * item.quantity), 0 ); - if (this.items.length === 0) { - throw new Error('Order must have items!'); + // Validate + if (this.totalAmount < 0) { + throw new Error('Invalid order total'); } } - // Send notifications after saving + // Called after successful save async afterSave() { - if (this.status === 'paid') { - await sendOrderConfirmation(this); - await notifyWarehouse(this); - } + // Send notifications + await notificationService.orderUpdated(this.id); + + // Update cache + await cache.set(`order:${this.id}`, this); } - // Prevent deletion of shipped orders + // Called before deletion async beforeDelete() { - if (this.status === 'shipped') { - throw new Error('Cannot delete shipped orders!'); + // Archive order + await archive.store(this); + + // Check if deletion is allowed + if (this.status === 'completed') { + throw new Error('Cannot delete completed orders'); } } - // Audit logging + // Called after successful deletion async afterDelete() { - await auditLog.record({ - action: 'order_deleted', - orderId: this.id, - timestamp: new Date() - }); + // Clean up related data + await cache.delete(`order:${this.id}`); + + // Log deletion + console.log(`Order ${this.id} deleted at ${new Date()}`); } } ``` -### 🔍 Deep Query Type Safety +### 🚀 Performance Optimization -TypeScript knows your nested object structure: +#### Intelligent Indexing ```typescript -interface UserProfile { - personal: { - name: { - first: string; - last: string; - }; - age: number; - }; - address: { - street: string; - city: string; - country: string; - }; -} - @Collection(() => db) -class Profile extends SmartDataDbDoc { - @unI() public id: string; - @svDb() public data: UserProfile; +class HighPerformanceDoc extends SmartDataDbDoc { + @unI() // Unique index + public id: string; + + @index() // Single field index + public userId: string; + + @index({ sparse: true }) // Sparse index for optional fields + public deletedAt?: Date; + + @index({ + unique: false, + background: true, // Non-blocking index creation + expireAfterSeconds: 86400 // TTL index + }) + public sessionToken: string; + + // Compound indexes for complex queries + static async createIndexes() { + await this.collection.createIndex( + { userId: 1, createdAt: -1 }, // Compound index + { name: 'user_activity_idx' } + ); + + // Text index for search + await this.collection.createIndex( + { title: 'text', content: 'text' }, + { weights: { title: 10, content: 5 } } + ); + } } - -// TypeScript enforces correct paths and types! -const profiles = await Profile.getInstances({ - 'data.personal.name.first': 'John', // ✅ Type-checked - 'data.address.country': 'USA', // ✅ Type-checked - 'data.personal.age': { $gte: 18 }, // ✅ Type-checked - // 'data.invalid.path': 'value' // ❌ TypeScript error! -}); ``` -## 🛡️ Security Features +#### Connection Pool Management -SmartData includes enterprise-grade security out of the box: - -- **🔐 Credential Security**: Automatic encoding of special characters in passwords -- **💉 Injection Prevention**: NoSQL injection protection with query sanitization -- **🚫 Dangerous Operator Blocking**: Prevents use of `$where` and other risky operators -- **🔒 Secure Defaults**: Production-ready connection settings out of the box -- **🛑 Rate Limiting Ready**: Built-in connection pooling prevents connection exhaustion +```typescript +const db = new SmartdataDb({ + mongoDbUrl: 'mongodb://localhost:27017', + mongoDbName: 'myapp', + + // Connection pool optimization + maxPoolSize: 100, // Maximum connections + minPoolSize: 10, // Minimum connections to maintain + maxIdleTimeMS: 300000, // Close idle connections after 5 minutes + waitQueueTimeoutMS: 5000, // Max time to wait for available connection + + // Server selection + serverSelectionTimeoutMS: 30000, // Timeout for selecting a server + heartbeatFrequencyMS: 10000, // How often to check server status + + // Socket settings + socketTimeoutMS: 360000, // Socket timeout (6 minutes) + family: 4, // Force IPv4 +}); +``` ## 🎯 Best Practices -### Connection Management +### 1. Always Use TypeScript + +SmartData is built for TypeScript. Using JavaScript means missing out on: +- Compile-time query validation +- IntelliSense for MongoDB operators +- Type-safe document updates +- Automatic refactoring support + +### 2. Design Your Indexes Thoughtfully + ```typescript -// ✅ DO: Use connection pooling options -const db = new SmartdataDb({ - mongoDbUrl: 'mongodb://localhost:27017/myapp', - maxPoolSize: 50, // Adjust based on your load - maxIdleTimeMS: 300000 // 5 minutes -}); +// ✅ Good: Index fields you query frequently +@index() public userId: string; +@index() public status: string; -// ✅ DO: Always close connections on shutdown -process.on('SIGTERM', async () => { - await db.close(); - process.exit(0); -}); - -// ❌ DON'T: Create multiple DB instances for the same database +// ❌ Bad: Over-indexing +@index() public everyField: string; // Don't index everything! ``` -### Performance Optimization +### 3. Use Transactions for Critical Operations + ```typescript -// ✅ DO: Use cursors for large datasets +// ✅ Good: Atomic operations for financial transactions +await session.withTransaction(async () => { + await debitAccount(fromAccount, amount, { session }); + await creditAccount(toAccount, amount, { session }); +}); + +// ❌ Bad: Multiple operations without transactions +await debitAccount(fromAccount, amount); // What if this fails? +await creditAccount(toAccount, amount); +``` + +### 4. Leverage Cursors for Large Datasets + +```typescript +// ✅ Good: Stream processing const cursor = await LargeCollection.getCursor({}); await cursor.forEach(async (doc) => { - await processDocument(doc); + await processOne(doc); }); -// ❌ DON'T: Load everything into memory -const allDocs = await LargeCollection.getInstances({}); // Could OOM! - -// ✅ DO: Create indexes for frequent queries -@index() public frequentlyQueried: string; - -// ✅ DO: Use projections when you don't need all fields -const cursor = await User.getCursor( - { active: true }, - { projection: { username: 1, email: 1 } } -); +// ❌ Bad: Loading everything into memory +const allDocs = await LargeCollection.getInstances({}); // Could OOM! ``` -### Type Safety +### 5. Implement Proper Error Handling + ```typescript -// ✅ DO: Leverage TypeScript's type system -interface StrictUserData { - verified: boolean; - roles: ('admin' | 'user' | 'guest')[]; +// ✅ Good: Graceful error handling +try { + const user = await User.getInstance({ id: userId }); + if (!user) { + throw new Error('User not found'); + } + await processUser(user); +} catch (error) { + logger.error('Failed to process user:', error); + await notificationService.alertAdmin(error); } -@Collection(() => db) -class StrictUser extends SmartDataDbDoc { - @svDb() public data: StrictUserData; // Fully typed! -} - -// ✅ DO: Use DeepQuery for nested queries -import { DeepQuery } from '@push.rocks/smartdata'; - -const query: DeepQuery = { - 'data.verified': true, - 'data.roles': { $in: ['admin'] } -}; +// ❌ Bad: Ignoring errors +const user = await User.getInstance({ id: userId }); +await processUser(user); // What if user is null? ``` -## 📊 Performance Benchmarks +## 🔧 Troubleshooting -SmartData has been battle-tested in production environments: +### Connection Issues -- **🚀 Connection Pooling**: 100+ concurrent connections with <10ms latency -- **⚡ Query Performance**: Indexed searches return in <5ms for millions of documents -- **📦 Memory Efficient**: Stream processing keeps memory under 100MB for any dataset size -- **🔄 Real-time Updates**: Change streams deliver updates in <50ms +```typescript +// Enable debug logging +const db = new SmartdataDb({ + mongoDbUrl: 'mongodb://localhost:27017', + mongoDbName: 'myapp', + // Add connection event handlers +}); -## 🤝 Support +db.mongoClient.on('error', (err) => { + console.error('MongoDB connection error:', err); +}); -Need help? We've got you covered: +db.mongoClient.on('timeout', () => { + console.error('MongoDB connection timeout'); +}); -- 📖 **Documentation**: Full API docs at [https://code.foss.global/push.rocks/smartdata](https://code.foss.global/push.rocks/smartdata) -- 💬 **Issues**: Report bugs at [GitLab Issues](https://code.foss.global/push.rocks/smartdata/issues) -- 📧 **Email**: Reach out to hello@task.vc for enterprise support +// Check connection status +const isConnected = db.status === 'connected'; +``` + +### Memory Leaks + +Always clean up resources: + +```typescript +// Watchers +const watcher = await User.watch({}); +// ... use watcher +await watcher.close(); // Always close! + +// Cursors +const cursor = await User.getCursor({}); +// ... use cursor +await cursor.close(); // Always close! + +// Sessions +const session = db.startSession(); +// ... use session +await session.endSession(); // Always end! +``` + +### Performance Issues + +```typescript +// Profile slow queries +await db.mongoDb.setProfilingLevel('all'); + +// Check query execution +const stats = await User.collection.mongoDbCollection + .find({ complex: 'query' }) + .explain('executionStats'); + +console.log('Query execution time:', stats.executionTimeMillis); +console.log('Documents examined:', stats.totalDocsExamined); +console.log('Index used:', stats.executionStats.executionStages.indexName); +``` + +## 📚 API Reference + +### Core Classes + +- `SmartdataDb` - Database connection and management +- `SmartDataDbDoc` - Base class for all documents +- `SmartdataCollection` - Collection management +- `SmartdataDbCursor` - Cursor for streaming operations +- `SmartdataDbWatcher` - Change stream watcher +- `SmartdataDistributedCoordinator` - Distributed coordination + +### Decorators + +- `@Collection(dbGetter)` - Define collection for a document class +- `@unI()` - Create unique index +- `@index(options)` - Create regular index +- `@svDb(options)` - Mark field as saveable with optional serialization +- `@searchable()` - Enable text search on field + +### Key Methods + +- `getInstances(filter)` - Find multiple documents +- `getInstance(filter)` - Find single document +- `getCursor(filter)` - Get cursor for streaming +- `watch(filter, options)` - Watch for changes +- `search(query, options)` - Lucene-style search +- `save(options)` - Save document +- `delete(options)` - Delete document +- `updateFromDb()` - Refresh from database + +## 🌍 Community & Support + +- 📖 **Documentation**: [https://code.foss.global/push.rocks/smartdata](https://code.foss.global/push.rocks/smartdata) +- 💬 **Issues**: [GitLab Issues](https://code.foss.global/push.rocks/smartdata/issues) +- 📧 **Email**: hello@task.vc ## License and Legal Information diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index 590acf4..1230bd1 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@push.rocks/smartdata', - version: '5.16.1', + version: '5.16.2', description: 'An advanced library for NoSQL data organization and manipulation using TypeScript with support for MongoDB, data validation, collections, and custom data types.' }