diff --git a/changelog.md b/changelog.md index cd7cf4c..66d6bf8 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,15 @@ # Changelog +## 2025-08-18 - 5.16.4 - fix(classes.doc (convertFilterForMongoDb)) +Improve filter conversion: handle logical operators, merge operator objects, add nested filter tests and docs, and fix test script + +- Fix package.json test script: remove stray dot in tstest --verbose argument to ensure tests run correctly +- Enhance convertFilterForMongoDb in ts/classes.doc.ts to properly handle logical operators ($and, $or, $nor, $not) and return them recursively +- Merge operator objects for the same field path (e.g. combining $gte and $lte) to avoid overwriting operator clauses when object and dot-notation are mixed +- Add validation/guards for operator argument types (e.g. $in, $nin, $all must be arrays; $size must be numeric) and preserve existing behavior blocking $where for security +- Add comprehensive nested filter tests in test/test.filters.ts to cover deep nested object queries, $elemMatch, array size, $all, $in on nested fields and more +- Expand README filtering section with detailed examples for basic filtering, deep nested filters, comparison operators, array operations, logical and element operators, and advanced patterns + ## 2025-08-18 - 5.16.3 - fix(docs) Add local Claude settings and remove outdated codex.md diff --git a/package.json b/package.json index 4c35c30..43ea964 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "typings": "dist_ts/index.d.ts", "type": "module", "scripts": { - "test": "tstest test/ --verbose. --logfile --timeout 120", + "test": "tstest test/ --verbose --logfile --timeout 120", "testSearch": "tsx test/test.search.ts", "build": "tsbuild --web --allowimplicitany", "buildDocs": "tsdoc" diff --git a/readme.md b/readme.md index c9ad286..770a694 100644 --- a/readme.md +++ b/readme.md @@ -144,40 +144,130 @@ await foundUser.delete(); SmartData provides the most advanced type-safe filtering system for MongoDB, supporting all operators while maintaining full IntelliSense: +#### Basic Filtering + ```typescript +// Simple equality +const john = await User.getInstances({ name: 'John Doe' }); + +// Multiple fields (implicit AND) +const activeAdults = await User.getInstances({ + status: 'active', + age: { $gte: 18 } +}); + // Union types work perfectly const users = await User.getInstances({ status: { $in: ['active', 'pending'] } // TypeScript validates these values! }); +``` -// Comparison operators with type checking +#### Deep Nested Object Filtering + +SmartData supports **both** nested object notation and dot notation for querying nested fields, with intelligent merging when both are used: + +```typescript +// Nested object notation - natural TypeScript syntax +const users = await User.getInstances({ + metadata: { + loginCount: { $gte: 5 } + } +}); + +// Dot notation - MongoDB style +const sameUsers = await User.getInstances({ + 'metadata.loginCount': { $gte: 5 } +}); + +// POWERFUL: Combine both notations - operators are merged! +const filtered = await User.getInstances({ + metadata: { loginCount: { $gte: 3 } }, // Object notation + 'metadata.loginCount': { $lte: 10 } // Dot notation + // Result: metadata.loginCount between 3 and 10 +}); + +// Deep nesting with full type safety +const deepQuery = await User.getInstances({ + profile: { + settings: { + notifications: { + email: true, + frequency: { $in: ['daily', 'weekly'] } + } + } + } +}); + +// Mix styles for complex queries +const advanced = await User.getInstances({ + // Object notation for structure + profile: { + age: { $gte: 21 }, + verified: true + }, + // Dot notation for specific overrides + 'profile.settings.theme': 'dark', + 'profile.lastSeen': { $gte: new Date('2024-01-01') } +}); +``` + +#### Comparison Operators + +```typescript +// Numeric comparisons with type checking const adults = await User.getInstances({ age: { $gte: 18, $lt: 65 } // Type-safe numeric comparisons }); +// Date comparisons +const recentUsers = await User.getInstances({ + createdAt: { $gte: new Date('2024-01-01') } +}); + +// Not equal +const nonAdmins = await User.getInstances({ + role: { $ne: 'admin' } +}); +``` + +#### Array Operations + +```typescript // Array operations with full type safety const experts = await User.getInstances({ tags: { $all: ['typescript', 'mongodb'] }, // Must have all tags skills: { $size: 5 } // Exactly 5 skills }); -// Complex nested queries +// Array element matching +const results = await Order.getInstances({ + items: { + $elemMatch: { // Match array elements + product: 'laptop', + quantity: { $gte: 2 } + } + } +}); + +// Check if value exists in array field +const nodeUsers = await User.getInstances({ + skills: { $in: ['nodejs'] } // Has nodejs in skills array +}); +``` + +#### Logical Operators + +```typescript +// Complex nested queries with $and const results = await Order.getInstances({ $and: [ { status: { $in: ['pending', 'processing'] } }, - { 'items.price': { $gte: 100 } }, // Dot notation for nested fields - { - items: { - $elemMatch: { // Array element matching - product: 'laptop', - quantity: { $gte: 2 } - } - } - } + { 'items.price': { $gte: 100 } }, + { customer: { verified: true } } ] }); -// Logical operators +// $or operator const urgentOrHighValue = await Order.getInstances({ $or: [ { priority: 'urgent' }, @@ -185,8 +275,94 @@ const urgentOrHighValue = await Order.getInstances({ ] }); -// Security: $where is automatically blocked +// $nor operator - none of the conditions +const excluded = await User.getInstances({ + $nor: [ + { status: 'banned' }, + { role: 'guest' } + ] +}); + +// Combine logical operators +const complex = await Order.getInstances({ + $and: [ + { status: 'active' }, + { + $or: [ + { priority: 'high' }, + { value: { $gte: 1000 } } + ] + } + ] +}); +``` + +#### Element Operators + +```typescript +// Check field existence +const withEmail = await User.getInstances({ + email: { $exists: true } +}); + +// Check for null or missing nested fields +const noPreferences = await User.getInstances({ + 'profile.preferences': { $exists: false } +}); +``` + +#### Text and Pattern Matching + +```typescript +// Regex patterns +const gmailUsers = await User.getInstances({ + email: { $regex: '@gmail\\.com$', $options: 'i' } +}); + +// Starts with pattern +const johnUsers = await User.getInstances({ + name: { $regex: '^John' } +}); +``` + +#### Security Features + +```typescript +// Security: $where is automatically blocked for injection protection // await User.getInstances({ $where: '...' }); // ❌ Throws security error + +// Invalid operators are caught +// await User.getInstances({ $badOp: 'value' }); // ⚠️ Warning logged +``` + +#### Advanced Patterns + +```typescript +// Combine multiple filtering approaches +const advancedQuery = await User.getInstances({ + // Direct field matching + status: 'active', + + // Nested object with operators + profile: { + age: { $gte: 18, $lte: 65 }, + verified: true + }, + + // Dot notation for deep paths + 'settings.notifications.email': true, + 'metadata.lastLogin': { $gte: new Date(Date.now() - 30*24*60*60*1000) }, + + // Array operations + roles: { $in: ['admin', 'moderator'] }, + tags: { $all: ['verified', 'premium'] }, + + // Logical grouping + $or: [ + { 'subscription.plan': 'premium' }, + { 'subscription.trial': true } + ] +}); ``` ### 🔎 Powerful Search Engine diff --git a/test/test.filters.ts b/test/test.filters.ts index 328d392..01e393a 100644 --- a/test/test.filters.ts +++ b/test/test.filters.ts @@ -213,6 +213,221 @@ tap.test('should filter by nested object fields', async () => { expect(users[0].name).toEqual('John Doe'); }); +// ============= COMPREHENSIVE NESTED FILTER TESTS ============= +tap.test('should filter by nested object with direct object syntax', async () => { + // Direct nested object matching (exact match) + const users = await TestUser.getInstances({ + metadata: { + loginCount: 5, + lastLogin: (await TestUser.getInstances({}))[0].metadata.lastLogin // Get the exact date + } + }); + expect(users.length).toEqual(1); + expect(users[0].name).toEqual('John Doe'); +}); + +tap.test('should filter by partial nested object match', async () => { + // When using object syntax, only specified fields must match + const users = await TestUser.getInstances({ + metadata: { loginCount: 5 } // Only checks loginCount, ignores other fields + }); + expect(users.length).toEqual(1); + expect(users[0].name).toEqual('John Doe'); +}); + +tap.test('should combine nested object and dot notation', async () => { + const users = await TestUser.getInstances({ + metadata: { loginCount: { $gte: 3 } }, // Object syntax with operator + 'metadata.loginCount': { $lte: 10 } // Dot notation with operator + }); + expect(users.length).toEqual(3); // Jane (3), John (5), and Alice (10) have loginCount between 3-10 +}); + +tap.test('should filter nested fields with operators using dot notation', async () => { + const users = await TestUser.getInstances({ + 'metadata.loginCount': { $gte: 5 } + }); + expect(users.length).toEqual(2); // John (5) and Alice (10) + const names = users.map(u => u.name).sort(); + expect(names).toEqual(['Alice Brown', 'John Doe']); +}); + +tap.test('should filter nested fields with multiple operators', async () => { + const users = await TestUser.getInstances({ + 'metadata.loginCount': { $gte: 3, $lt: 10 } + }); + expect(users.length).toEqual(2); // Jane (3) and John (5) + const names = users.map(u => u.name).sort(); + expect(names).toEqual(['Jane Smith', 'John Doe']); +}); + +tap.test('should handle deeply nested object structures', async () => { + // First, create a user with deep nesting in preferences + const deepUser = new TestUser({ + name: 'Deep Nester', + age: 40, + email: 'deep@example.com', + roles: ['admin'], + tags: [], + status: 'active', + metadata: { + loginCount: 1, + preferences: { + theme: { + colors: { + primary: '#000000', + secondary: '#ffffff' + }, + fonts: { + heading: 'Arial', + body: 'Helvetica' + } + }, + notifications: { + email: true, + push: false + } + } + }, + scores: [] + }); + await deepUser.save(); + + // Test deep nesting with dot notation + const deepResults = await TestUser.getInstances({ + 'metadata.preferences.theme.colors.primary': '#000000' + }); + expect(deepResults.length).toEqual(1); + expect(deepResults[0].name).toEqual('Deep Nester'); + + // Test deep nesting with operators + const boolResults = await TestUser.getInstances({ + 'metadata.preferences.notifications.email': { $eq: true } + }); + expect(boolResults.length).toEqual(1); + expect(boolResults[0].name).toEqual('Deep Nester'); + + // Clean up + await deepUser.delete(); +}); + +tap.test('should filter arrays of nested objects using $elemMatch', async () => { + const orders = await TestOrder.getInstances({ + items: { + $elemMatch: { + product: 'laptop', + price: { $gte: 1000 } + } + } + }); + expect(orders.length).toEqual(2); // Both laptop orders have price >= 1000 +}); + +tap.test('should filter nested arrays with dot notation', async () => { + // Query for any order that has an item with specific product + const orders = await TestOrder.getInstances({ + 'items.product': 'laptop' + }); + expect(orders.length).toEqual(2); // Two orders contain laptops +}); + +tap.test('should combine nested object filters with logical operators', async () => { + const users = await TestUser.getInstances({ + $or: [ + { 'metadata.loginCount': { $gte: 10 } }, // Alice has 10 + { + $and: [ + { 'metadata.loginCount': { $lt: 5 } }, // Jane has 3, Bob has 0, Charlie has 1 + { status: 'active' } // Jane is active, Bob is inactive, Charlie is pending + ] + } + ] + }); + expect(users.length).toEqual(2); // Alice (loginCount >= 10), Jane (loginCount < 5 AND active) + const names = users.map(u => u.name).sort(); + expect(names).toEqual(['Alice Brown', 'Jane Smith']); +}); + +tap.test('should handle null and undefined in nested fields', async () => { + // Users without lastLogin + const noLastLogin = await TestUser.getInstances({ + 'metadata.lastLogin': { $exists: false } + }); + expect(noLastLogin.length).toEqual(4); // Everyone except John + + // Users with preferences (none have it set) + const withPreferences = await TestUser.getInstances({ + 'metadata.preferences': { $exists: true } + }); + expect(withPreferences.length).toEqual(0); +}); + +tap.test('should filter nested arrays by size', async () => { + // Create an order with specific number of items + const multiItemOrder = new TestOrder({ + userId: 'test-user', + items: [ + { product: 'item1', quantity: 1, price: 10 }, + { product: 'item2', quantity: 2, price: 20 }, + { product: 'item3', quantity: 3, price: 30 }, + { product: 'item4', quantity: 4, price: 40 } + ], + totalAmount: 100, + status: 'pending', + tags: ['test'] + }); + await multiItemOrder.save(); + + const fourItemOrders = await TestOrder.getInstances({ + items: { $size: 4 } + }); + expect(fourItemOrders.length).toEqual(1); + + // Clean up + await multiItemOrder.delete(); +}); + +tap.test('should handle nested field comparison between documents', async () => { + // Find users where loginCount equals their age divided by 6 (John: 30/6=5) + const users = await TestUser.getInstances({ + $and: [ + { 'metadata.loginCount': 5 }, + { age: 30 } + ] + }); + expect(users.length).toEqual(1); + expect(users[0].name).toEqual('John Doe'); +}); + +tap.test('should filter using $in on nested fields', async () => { + const users = await TestUser.getInstances({ + 'metadata.loginCount': { $in: [0, 1, 5] } + }); + expect(users.length).toEqual(3); // Bob (0), Charlie (1), John (5) + const names = users.map(u => u.name).sort(); + expect(names).toEqual(['Bob Johnson', 'Charlie Wilson', 'John Doe']); +}); + +tap.test('should filter nested arrays with $all', async () => { + // Create an order with multiple tags + const taggedOrder = new TestOrder({ + userId: 'test-user', + items: [{ product: 'test', quantity: 1, price: 10 }], + totalAmount: 10, + status: 'completed', + tags: ['urgent', 'priority', 'electronics'] + }); + await taggedOrder.save(); + + const priorityElectronics = await TestOrder.getInstances({ + tags: { $all: ['priority', 'electronics'] } + }); + expect(priorityElectronics.length).toEqual(2); // Original order and new one + + // Clean up + await taggedOrder.delete(); +}); + // ============= COMPARISON OPERATOR TESTS ============= tap.test('should filter using $gt operator', async () => { const users = await TestUser.getInstances({ diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index 8a1380c..cfff9b8 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@push.rocks/smartdata', - version: '5.16.3', + version: '5.16.4', description: 'An advanced library for NoSQL data organization and manipulation using TypeScript with support for MongoDB, data validation, collections, and custom data types.' } diff --git a/ts/classes.doc.ts b/ts/classes.doc.ts index 1edf03c..b008b0c 100644 --- a/ts/classes.doc.ts +++ b/ts/classes.doc.ts @@ -199,21 +199,51 @@ export const convertFilterForMongoDb = (filterArg: { [key: string]: any }) => { throw new Error('$where operator is not allowed for security reasons'); } - // Special case: detect MongoDB operators and pass them through directly - const topLevelOperators = ['$and', '$or', '$nor', '$not', '$text', '$regex']; + // Handle logical operators recursively + const logicalOperators = ['$and', '$or', '$nor', '$not']; + const processedFilter: { [key: string]: any } = {}; + for (const key of Object.keys(filterArg)) { - if (topLevelOperators.includes(key)) { - return filterArg; // Return the filter as-is for MongoDB operators + if (logicalOperators.includes(key)) { + if (key === '$not') { + processedFilter[key] = convertFilterForMongoDb(filterArg[key]); + } else if (Array.isArray(filterArg[key])) { + processedFilter[key] = filterArg[key].map((subFilter: any) => convertFilterForMongoDb(subFilter)); + } } } + + // If only logical operators, return them + const hasOnlyLogicalOperators = Object.keys(filterArg).every(key => logicalOperators.includes(key)); + if (hasOnlyLogicalOperators) { + return processedFilter; + } // Original conversion logic for non-MongoDB query objects const convertedFilter: { [key: string]: any } = {}; + + // Helper to merge operator objects + const mergeIntoConverted = (path: string, value: any) => { + const existing = convertedFilter[path]; + if (!existing) { + convertedFilter[path] = value; + } else if ( + typeof existing === 'object' && !Array.isArray(existing) && + typeof value === 'object' && !Array.isArray(value) && + (Object.keys(existing).some(k => k.startsWith('$')) || Object.keys(value).some(k => k.startsWith('$'))) + ) { + // Both have operators, merge them + convertedFilter[path] = { ...existing, ...value }; + } else { + // Otherwise later wins + convertedFilter[path] = value; + } + }; const convertFilterArgument = (keyPathArg2: string, filterArg2: any) => { if (Array.isArray(filterArg2)) { // Arrays are typically used as values for operators like $in or as direct equality matches - convertedFilter[keyPathArg2] = filterArg2; + mergeIntoConverted(keyPathArg2, filterArg2); return; } else if (typeof filterArg2 === 'object' && filterArg2 !== null) { // Check if this is an object with MongoDB operators @@ -264,8 +294,8 @@ export const convertFilterForMongoDb = (filterArg: { [key: string]: any }) => { throw new Error('$size operator requires a numeric value'); } - // Pass the operator object through - convertedFilter[keyPathArg2] = filterArg2; + // Use merge helper to handle duplicate paths + mergeIntoConverted(keyPathArg2, filterArg2); return; } @@ -282,13 +312,20 @@ export const convertFilterForMongoDb = (filterArg: { [key: string]: any }) => { } } else { // Primitive values - convertedFilter[keyPathArg2] = filterArg2; + mergeIntoConverted(keyPathArg2, filterArg2); } }; for (const key of Object.keys(filterArg)) { - convertFilterArgument(key, filterArg[key]); + // Skip logical operators, they were already processed + if (!logicalOperators.includes(key)) { + convertFilterArgument(key, filterArg[key]); + } } + + // Add back processed logical operators + Object.assign(convertedFilter, processedFilter); + return convertedFilter; };