feat: Add comprehensive query filters guide and enhance type safety for MongoDB queries
- Introduced a detailed guide on query filters in the README, covering basic filtering, comparison operators, array operators, logical operators, element operators, and advanced filtering patterns. - Implemented a type-safe filtering system in `classes.doc.ts` with `MongoFilterCondition` and `MongoFilter` types to support MongoDB operators while maintaining nested type safety. - Enhanced error handling for invalid operators and conditions in the filtering logic. - Added extensive tests for various filtering scenarios, including basic, comparison, array, logical, and complex filters, ensuring robust functionality and performance. - Implemented security measures to prevent the use of dangerous operators like `$where` and validate operator usage.
This commit is contained in:
@@ -7,7 +7,7 @@
|
||||
"typings": "dist_ts/index.d.ts",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"test": "tstest test/ --verbose",
|
||||
"test": "tstest test/ --verbose. --logfile --timeout 120",
|
||||
"testSearch": "tsx test/test.search.ts",
|
||||
"build": "tsbuild --web --allowimplicitany",
|
||||
"buildDocs": "tsdoc"
|
||||
@@ -37,10 +37,10 @@
|
||||
"mongodb": "^6.18.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@git.zone/tsbuild": "^2.6.4",
|
||||
"@git.zone/tsbuild": "^2.6.7",
|
||||
"@git.zone/tsrun": "^1.2.44",
|
||||
"@git.zone/tstest": "^2.3.2",
|
||||
"@push.rocks/qenv": "^6.0.5",
|
||||
"@git.zone/tstest": "^2.3.5",
|
||||
"@push.rocks/qenv": "^6.1.3",
|
||||
"@push.rocks/tapbundle": "^6.0.3",
|
||||
"@types/node": "^22.15.2"
|
||||
},
|
||||
|
||||
1766
pnpm-lock.yaml
generated
1766
pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
332
readme.md
332
readme.md
@@ -132,6 +132,338 @@ await foundUser.save();
|
||||
await foundUser.delete();
|
||||
```
|
||||
|
||||
## 🔍 Query Filters Guide
|
||||
|
||||
SmartData provides a comprehensive and type-safe filtering system that supports all MongoDB query operators while preventing dangerous operations like `$where` for security.
|
||||
|
||||
### Basic Filtering
|
||||
|
||||
```typescript
|
||||
// Simple equality - find users with exact name
|
||||
const johns = await User.getInstances({ name: 'John Doe' });
|
||||
|
||||
// Multiple conditions (implicit AND)
|
||||
const activeAdults = await User.getInstances({
|
||||
status: 'active',
|
||||
age: { $gte: 18 }
|
||||
});
|
||||
|
||||
// Nested object fields - use dot notation
|
||||
const recentUsers = await User.getInstances({
|
||||
'metadata.lastLogin': { $gte: new Date('2024-01-01') },
|
||||
'metadata.loginCount': { $gt: 5 }
|
||||
});
|
||||
```
|
||||
|
||||
### Comparison Operators
|
||||
|
||||
All standard MongoDB comparison operators are supported:
|
||||
|
||||
```typescript
|
||||
// Greater than / Greater than or equal
|
||||
const adults = await User.getInstances({ age: { $gte: 18 } });
|
||||
const seniors = await User.getInstances({ age: { $gt: 65 } });
|
||||
|
||||
// Less than / Less than or equal
|
||||
const youth = await User.getInstances({ age: { $lt: 25 } });
|
||||
const under30 = await User.getInstances({ age: { $lte: 30 } });
|
||||
|
||||
// Not equal
|
||||
const notPending = await User.getInstances({ status: { $ne: 'pending' } });
|
||||
|
||||
// Combine multiple comparisons
|
||||
const middleAged = await User.getInstances({
|
||||
age: { $gte: 30, $lt: 50 }
|
||||
});
|
||||
```
|
||||
|
||||
### Array Operators
|
||||
|
||||
SmartData fully supports MongoDB's array query operators:
|
||||
|
||||
```typescript
|
||||
// $in - Match any value in array
|
||||
const adminsAndMods = await User.getInstances({
|
||||
role: { $in: ['admin', 'moderator'] }
|
||||
});
|
||||
|
||||
// $nin - Match none of the values
|
||||
const regularUsers = await User.getInstances({
|
||||
role: { $nin: ['admin', 'moderator'] }
|
||||
});
|
||||
|
||||
// $all - Array contains all specified elements
|
||||
const fullStackDevs = await User.getInstances({
|
||||
skills: { $all: ['javascript', 'nodejs', 'mongodb'] }
|
||||
});
|
||||
|
||||
// $size - Array has specific length
|
||||
const usersWithThreeSkills = await User.getInstances({
|
||||
skills: { $size: 3 }
|
||||
});
|
||||
|
||||
// $elemMatch - Match array elements with multiple conditions
|
||||
const ordersWithExpensiveItems = await Order.getInstances({
|
||||
items: {
|
||||
$elemMatch: {
|
||||
product: 'laptop',
|
||||
quantity: { $gte: 2 },
|
||||
price: { $gt: 1000 }
|
||||
}
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
### Working with Arrays
|
||||
|
||||
```typescript
|
||||
// Find documents where array field contains a value
|
||||
const javascriptUsers = await User.getInstances({
|
||||
tags: { $in: ['javascript'] } // Users with 'javascript' in tags array
|
||||
});
|
||||
|
||||
// Find documents where array field contains ALL values
|
||||
const expertUsers = await User.getInstances({
|
||||
certifications: { $all: ['mongodb', 'nodejs', 'typescript'] }
|
||||
});
|
||||
|
||||
// Query nested arrays of objects
|
||||
const highValueOrders = await Order.getInstances({
|
||||
'items.price': { $gte: 100 } // Orders with any item priced >= 100
|
||||
});
|
||||
|
||||
// Complex array element matching
|
||||
const bulkElectronicsOrders = await Order.getInstances({
|
||||
items: {
|
||||
$elemMatch: {
|
||||
category: 'electronics',
|
||||
quantity: { $gte: 5 },
|
||||
discount: { $exists: true }
|
||||
}
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
### Logical Operators
|
||||
|
||||
Combine multiple conditions with logical operators:
|
||||
|
||||
```typescript
|
||||
// $or - Match any condition
|
||||
const urgentOrHighValue = await Order.getInstances({
|
||||
$or: [
|
||||
{ priority: 'urgent' },
|
||||
{ totalAmount: { $gte: 1000 } }
|
||||
]
|
||||
});
|
||||
|
||||
// $and - Match all conditions (usually implicit, but useful for complex queries)
|
||||
const premiumActiveUsers = await User.getInstances({
|
||||
$and: [
|
||||
{ subscription: 'premium' },
|
||||
{ status: 'active' },
|
||||
{ lastLogin: { $gte: new Date(Date.now() - 30*24*60*60*1000) } }
|
||||
]
|
||||
});
|
||||
|
||||
// $nor - Match none of the conditions
|
||||
const availableProducts = await Product.getInstances({
|
||||
$nor: [
|
||||
{ status: 'discontinued' },
|
||||
{ inventory: { $lte: 0 } }
|
||||
]
|
||||
});
|
||||
|
||||
// $not - Negate a condition
|
||||
const nonEmptyArrays = await User.getInstances({
|
||||
tags: { $not: { $size: 0 } }
|
||||
});
|
||||
|
||||
// Nested logical operations
|
||||
const complexQuery = await User.getInstances({
|
||||
$or: [
|
||||
{
|
||||
$and: [
|
||||
{ role: 'admin' },
|
||||
{ department: 'IT' }
|
||||
]
|
||||
},
|
||||
{
|
||||
$and: [
|
||||
{ role: 'manager' },
|
||||
{ yearsExperience: { $gte: 5 } }
|
||||
]
|
||||
}
|
||||
]
|
||||
});
|
||||
```
|
||||
|
||||
### Element Operators
|
||||
|
||||
Check for field existence and types:
|
||||
|
||||
```typescript
|
||||
// $exists - Check if field exists
|
||||
const usersWithAvatar = await User.getInstances({
|
||||
'profile.avatar': { $exists: true }
|
||||
});
|
||||
|
||||
const usersWithoutPhone = await User.getInstances({
|
||||
phoneNumber: { $exists: false }
|
||||
});
|
||||
|
||||
// $type - Check field type (MongoDB BSON types)
|
||||
const usersWithStringId = await User.getInstances({
|
||||
id: { $type: 'string' } // or use type number: 2 for string
|
||||
});
|
||||
```
|
||||
|
||||
### Regular Expressions
|
||||
|
||||
Use regex for pattern matching:
|
||||
|
||||
```typescript
|
||||
// Case-insensitive search
|
||||
const johnUsers = await User.getInstances({
|
||||
name: { $regex: '^john', $options: 'i' }
|
||||
});
|
||||
|
||||
// Email domain matching
|
||||
const gmailUsers = await User.getInstances({
|
||||
email: { $regex: '@gmail\.com$' }
|
||||
});
|
||||
|
||||
// Complex pattern matching
|
||||
const validPhones = await User.getInstances({
|
||||
phone: { $regex: '^\+1-\d{3}-\d{3}-\d{4}$' }
|
||||
});
|
||||
```
|
||||
|
||||
### Advanced Filtering Patterns
|
||||
|
||||
```typescript
|
||||
// Combine multiple operator types
|
||||
const targetUsers = await User.getInstances({
|
||||
$and: [
|
||||
{ age: { $gte: 25, $lte: 45 } },
|
||||
{ status: { $in: ['active', 'premium'] } },
|
||||
{
|
||||
$or: [
|
||||
{ 'subscription.type': 'annual' },
|
||||
{ 'subscription.credits': { $gte: 100 } }
|
||||
]
|
||||
},
|
||||
{ tags: { $all: ['verified', 'trusted'] } }
|
||||
]
|
||||
});
|
||||
|
||||
// Filter with post-processing validation
|
||||
const results = await Product.search('laptop', {
|
||||
filter: {
|
||||
price: { $lte: 2000 },
|
||||
brand: { $in: ['Apple', 'Dell', 'Lenovo'] }
|
||||
},
|
||||
validate: async (product) => {
|
||||
// Additional custom validation
|
||||
return product.warrantyYears >= 2 && product.inStock;
|
||||
}
|
||||
});
|
||||
|
||||
// Transaction-safe filtering
|
||||
const session = db.startSession();
|
||||
await session.withTransaction(async () => {
|
||||
const users = await User.getInstances(
|
||||
{ status: 'pending' },
|
||||
{ session } // Pass session for transaction
|
||||
);
|
||||
// Process users within transaction
|
||||
});
|
||||
```
|
||||
|
||||
### Performance Tips
|
||||
|
||||
1. **Use Indexes**: Create indexes on frequently filtered fields:
|
||||
```typescript
|
||||
@index() public status: string; // Regular index
|
||||
@unI() public email: string; // Unique index
|
||||
```
|
||||
|
||||
2. **Limit Results**: Use cursors for large datasets:
|
||||
```typescript
|
||||
const cursor = await User.getCursor({ status: 'active' });
|
||||
await cursor.forEach(async (user) => {
|
||||
// Process one at a time
|
||||
});
|
||||
```
|
||||
|
||||
3. **Projection**: Retrieve only needed fields (using MongoDB native):
|
||||
```typescript
|
||||
const collection = User.getCollection();
|
||||
const results = await collection.mongoDbCollection
|
||||
.find({ status: 'active' })
|
||||
.project({ name: 1, email: 1 })
|
||||
.toArray();
|
||||
```
|
||||
|
||||
### Security Considerations
|
||||
|
||||
SmartData automatically prevents dangerous operations:
|
||||
|
||||
```typescript
|
||||
// ❌ These will throw errors:
|
||||
await User.getInstances({
|
||||
$where: 'this.age > 25' // Blocked: JavaScript execution
|
||||
});
|
||||
|
||||
await User.getInstances({
|
||||
'user.name': { 'bad.key': 'value' } // Blocked: dots in keys
|
||||
});
|
||||
|
||||
// ✅ Safe alternatives:
|
||||
await User.getInstances({
|
||||
age: { $gt: 25 } // Use operators instead
|
||||
});
|
||||
|
||||
await User.getInstances({
|
||||
'user.name': 'value' // Use dot notation for nested fields
|
||||
});
|
||||
```
|
||||
|
||||
### Common Pitfalls and Solutions
|
||||
|
||||
```typescript
|
||||
// ❌ Wrong: Array operators need arrays
|
||||
await User.getInstances({
|
||||
role: { $in: 'admin' } // Error: $in requires array
|
||||
});
|
||||
|
||||
// ✅ Correct:
|
||||
await User.getInstances({
|
||||
role: { $in: ['admin'] }
|
||||
});
|
||||
|
||||
// ❌ Wrong: Direct array comparison (exact match)
|
||||
await User.getInstances({
|
||||
tags: 'javascript' // Only matches if tags = ['javascript'] exactly
|
||||
});
|
||||
|
||||
// ✅ Correct: Check if array contains element
|
||||
await User.getInstances({
|
||||
tags: { $in: ['javascript'] } // Matches if 'javascript' is in tags
|
||||
});
|
||||
|
||||
// ❌ Wrong: Checking multiple array elements incorrectly
|
||||
await User.getInstances({
|
||||
skills: 'javascript',
|
||||
skills: 'nodejs' // Object can't have duplicate keys!
|
||||
});
|
||||
|
||||
// ✅ Correct: Use $all for multiple elements
|
||||
await User.getInstances({
|
||||
skills: { $all: ['javascript', 'nodejs'] }
|
||||
});
|
||||
```
|
||||
|
||||
## 🔥 Advanced Features
|
||||
|
||||
### 🔎 Powerful Search Engine
|
||||
|
||||
604
test/test.filters.ts
Normal file
604
test/test.filters.ts
Normal file
@@ -0,0 +1,604 @@
|
||||
import { tap, expect } from '@git.zone/tstest/tapbundle';
|
||||
import * as smartmongo from '@push.rocks/smartmongo';
|
||||
import * as smartunique from '@push.rocks/smartunique';
|
||||
import * as smartdata from '../ts/index.js';
|
||||
|
||||
const { SmartdataDb, Collection, svDb, unI, index } = smartdata;
|
||||
|
||||
let smartmongoInstance: smartmongo.SmartMongo;
|
||||
let testDb: smartdata.SmartdataDb;
|
||||
|
||||
// Define test document classes
|
||||
@Collection(() => testDb)
|
||||
class TestUser extends smartdata.SmartDataDbDoc<TestUser, TestUser> {
|
||||
@unI()
|
||||
public id: string = smartunique.shortId();
|
||||
|
||||
@svDb()
|
||||
public name: string;
|
||||
|
||||
@svDb()
|
||||
public age: number;
|
||||
|
||||
@svDb()
|
||||
public email: string;
|
||||
|
||||
@svDb()
|
||||
public roles: string[];
|
||||
|
||||
@svDb()
|
||||
public tags: string[];
|
||||
|
||||
@svDb()
|
||||
public status: 'active' | 'inactive' | 'pending';
|
||||
|
||||
@svDb()
|
||||
public metadata: {
|
||||
lastLogin?: Date;
|
||||
loginCount?: number;
|
||||
preferences?: Record<string, any>;
|
||||
};
|
||||
|
||||
@svDb()
|
||||
public scores: number[];
|
||||
|
||||
constructor(data: Partial<TestUser> = {}) {
|
||||
super();
|
||||
Object.assign(this, data);
|
||||
}
|
||||
}
|
||||
|
||||
@Collection(() => testDb)
|
||||
class TestOrder extends smartdata.SmartDataDbDoc<TestOrder, TestOrder> {
|
||||
@unI()
|
||||
public id: string = smartunique.shortId();
|
||||
|
||||
@svDb()
|
||||
public userId: string;
|
||||
|
||||
@svDb()
|
||||
public items: Array<{
|
||||
product: string;
|
||||
quantity: number;
|
||||
price: number;
|
||||
}>;
|
||||
|
||||
@svDb()
|
||||
public totalAmount: number;
|
||||
|
||||
@svDb()
|
||||
public status: string;
|
||||
|
||||
@svDb()
|
||||
public tags: string[];
|
||||
|
||||
constructor(data: Partial<TestOrder> = {}) {
|
||||
super();
|
||||
Object.assign(this, data);
|
||||
}
|
||||
}
|
||||
|
||||
// Setup and teardown
|
||||
tap.test('should create a test database instance', async () => {
|
||||
smartmongoInstance = await smartmongo.SmartMongo.createAndStart();
|
||||
testDb = new smartdata.SmartdataDb(await smartmongoInstance.getMongoDescriptor());
|
||||
await testDb.init();
|
||||
expect(testDb).toBeInstanceOf(SmartdataDb);
|
||||
});
|
||||
|
||||
tap.test('should create test data', async () => {
|
||||
// Create test users
|
||||
const users = [
|
||||
new TestUser({
|
||||
name: 'John Doe',
|
||||
age: 30,
|
||||
email: 'john@example.com',
|
||||
roles: ['admin', 'user'],
|
||||
tags: ['javascript', 'nodejs', 'mongodb'],
|
||||
status: 'active',
|
||||
metadata: { loginCount: 5, lastLogin: new Date() },
|
||||
scores: [85, 90, 78]
|
||||
}),
|
||||
new TestUser({
|
||||
name: 'Jane Smith',
|
||||
age: 25,
|
||||
email: 'jane@example.com',
|
||||
roles: ['user'],
|
||||
tags: ['python', 'mongodb'],
|
||||
status: 'active',
|
||||
metadata: { loginCount: 3 },
|
||||
scores: [92, 88, 95]
|
||||
}),
|
||||
new TestUser({
|
||||
name: 'Bob Johnson',
|
||||
age: 35,
|
||||
email: 'bob@example.com',
|
||||
roles: ['moderator', 'user'],
|
||||
tags: ['javascript', 'react', 'nodejs'],
|
||||
status: 'inactive',
|
||||
metadata: { loginCount: 0 },
|
||||
scores: [70, 75, 80]
|
||||
}),
|
||||
new TestUser({
|
||||
name: 'Alice Brown',
|
||||
age: 28,
|
||||
email: 'alice@example.com',
|
||||
roles: ['admin'],
|
||||
tags: ['typescript', 'angular', 'mongodb'],
|
||||
status: 'active',
|
||||
metadata: { loginCount: 10 },
|
||||
scores: [95, 98, 100]
|
||||
}),
|
||||
new TestUser({
|
||||
name: 'Charlie Wilson',
|
||||
age: 22,
|
||||
email: 'charlie@example.com',
|
||||
roles: ['user'],
|
||||
tags: ['golang', 'kubernetes'],
|
||||
status: 'pending',
|
||||
metadata: { loginCount: 1 },
|
||||
scores: [60, 65]
|
||||
})
|
||||
];
|
||||
|
||||
for (const user of users) {
|
||||
await user.save();
|
||||
}
|
||||
|
||||
// Create test orders
|
||||
const orders = [
|
||||
new TestOrder({
|
||||
userId: users[0].id,
|
||||
items: [
|
||||
{ product: 'laptop', quantity: 1, price: 1200 },
|
||||
{ product: 'mouse', quantity: 2, price: 25 }
|
||||
],
|
||||
totalAmount: 1250,
|
||||
status: 'completed',
|
||||
tags: ['electronics', 'priority']
|
||||
}),
|
||||
new TestOrder({
|
||||
userId: users[1].id,
|
||||
items: [
|
||||
{ product: 'book', quantity: 3, price: 15 },
|
||||
{ product: 'pen', quantity: 5, price: 2 }
|
||||
],
|
||||
totalAmount: 55,
|
||||
status: 'pending',
|
||||
tags: ['stationery']
|
||||
}),
|
||||
new TestOrder({
|
||||
userId: users[0].id,
|
||||
items: [
|
||||
{ product: 'laptop', quantity: 2, price: 1200 },
|
||||
{ product: 'keyboard', quantity: 2, price: 80 }
|
||||
],
|
||||
totalAmount: 2560,
|
||||
status: 'processing',
|
||||
tags: ['electronics', 'bulk']
|
||||
})
|
||||
];
|
||||
|
||||
for (const order of orders) {
|
||||
await order.save();
|
||||
}
|
||||
|
||||
const savedUsers = await TestUser.getInstances({});
|
||||
const savedOrders = await TestOrder.getInstances({});
|
||||
expect(savedUsers.length).toEqual(5);
|
||||
expect(savedOrders.length).toEqual(3);
|
||||
});
|
||||
|
||||
// ============= BASIC FILTER TESTS =============
|
||||
tap.test('should filter by simple equality', async () => {
|
||||
const users = await TestUser.getInstances({ name: 'John Doe' });
|
||||
expect(users.length).toEqual(1);
|
||||
expect(users[0].name).toEqual('John Doe');
|
||||
});
|
||||
|
||||
tap.test('should filter by multiple fields (implicit AND)', async () => {
|
||||
const users = await TestUser.getInstances({
|
||||
status: 'active',
|
||||
age: 30
|
||||
});
|
||||
expect(users.length).toEqual(1);
|
||||
expect(users[0].name).toEqual('John Doe');
|
||||
});
|
||||
|
||||
tap.test('should filter by nested object fields', async () => {
|
||||
const users = await TestUser.getInstances({
|
||||
'metadata.loginCount': 5
|
||||
});
|
||||
expect(users.length).toEqual(1);
|
||||
expect(users[0].name).toEqual('John Doe');
|
||||
});
|
||||
|
||||
// ============= COMPARISON OPERATOR TESTS =============
|
||||
tap.test('should filter using $gt operator', async () => {
|
||||
const users = await TestUser.getInstances({
|
||||
age: { $gt: 30 }
|
||||
});
|
||||
expect(users.length).toEqual(1);
|
||||
expect(users[0].name).toEqual('Bob Johnson');
|
||||
});
|
||||
|
||||
tap.test('should filter using $gte operator', async () => {
|
||||
const users = await TestUser.getInstances({
|
||||
age: { $gte: 30 }
|
||||
});
|
||||
expect(users.length).toEqual(2);
|
||||
const names = users.map(u => u.name).sort();
|
||||
expect(names).toEqual(['Bob Johnson', 'John Doe']);
|
||||
});
|
||||
|
||||
tap.test('should filter using $lt operator', async () => {
|
||||
const users = await TestUser.getInstances({
|
||||
age: { $lt: 25 }
|
||||
});
|
||||
expect(users.length).toEqual(1);
|
||||
expect(users[0].name).toEqual('Charlie Wilson');
|
||||
});
|
||||
|
||||
tap.test('should filter using $lte operator', async () => {
|
||||
const users = await TestUser.getInstances({
|
||||
age: { $lte: 25 }
|
||||
});
|
||||
expect(users.length).toEqual(2);
|
||||
const names = users.map(u => u.name).sort();
|
||||
expect(names).toEqual(['Charlie Wilson', 'Jane Smith']);
|
||||
});
|
||||
|
||||
tap.test('should filter using $ne operator', async () => {
|
||||
const users = await TestUser.getInstances({
|
||||
status: { $ne: 'active' }
|
||||
});
|
||||
expect(users.length).toEqual(2);
|
||||
const statuses = users.map(u => u.status).sort();
|
||||
expect(statuses).toEqual(['inactive', 'pending']);
|
||||
});
|
||||
|
||||
tap.test('should filter using multiple comparison operators', async () => {
|
||||
const users = await TestUser.getInstances({
|
||||
age: { $gte: 25, $lt: 30 }
|
||||
});
|
||||
expect(users.length).toEqual(2);
|
||||
const names = users.map(u => u.name).sort();
|
||||
expect(names).toEqual(['Alice Brown', 'Jane Smith']);
|
||||
});
|
||||
|
||||
// ============= ARRAY OPERATOR TESTS =============
|
||||
tap.test('should filter using $in operator', async () => {
|
||||
const users = await TestUser.getInstances({
|
||||
status: { $in: ['active', 'pending'] }
|
||||
});
|
||||
expect(users.length).toEqual(4);
|
||||
expect(users.every(u => ['active', 'pending'].includes(u.status))).toEqual(true);
|
||||
});
|
||||
|
||||
tap.test('should filter arrays using $in operator', async () => {
|
||||
const users = await TestUser.getInstances({
|
||||
roles: { $in: ['admin'] }
|
||||
});
|
||||
expect(users.length).toEqual(2);
|
||||
const names = users.map(u => u.name).sort();
|
||||
expect(names).toEqual(['Alice Brown', 'John Doe']);
|
||||
});
|
||||
|
||||
tap.test('should filter using $nin operator', async () => {
|
||||
const users = await TestUser.getInstances({
|
||||
status: { $nin: ['inactive', 'pending'] }
|
||||
});
|
||||
expect(users.length).toEqual(3);
|
||||
expect(users.every(u => u.status === 'active')).toEqual(true);
|
||||
});
|
||||
|
||||
tap.test('should filter arrays using $all operator', async () => {
|
||||
const users = await TestUser.getInstances({
|
||||
tags: { $all: ['javascript', 'nodejs'] }
|
||||
});
|
||||
expect(users.length).toEqual(2);
|
||||
const names = users.map(u => u.name).sort();
|
||||
expect(names).toEqual(['Bob Johnson', 'John Doe']);
|
||||
});
|
||||
|
||||
tap.test('should filter arrays using $size operator', async () => {
|
||||
const users = await TestUser.getInstances({
|
||||
scores: { $size: 2 }
|
||||
});
|
||||
expect(users.length).toEqual(1);
|
||||
expect(users[0].name).toEqual('Charlie Wilson');
|
||||
});
|
||||
|
||||
tap.test('should filter arrays using $elemMatch operator', async () => {
|
||||
const orders = await TestOrder.getInstances({
|
||||
items: {
|
||||
$elemMatch: {
|
||||
product: 'laptop',
|
||||
quantity: { $gte: 2 }
|
||||
}
|
||||
}
|
||||
});
|
||||
expect(orders.length).toEqual(1);
|
||||
expect(orders[0].totalAmount).toEqual(2560);
|
||||
});
|
||||
|
||||
tap.test('should filter using $elemMatch with single condition', async () => {
|
||||
const orders = await TestOrder.getInstances({
|
||||
items: {
|
||||
$elemMatch: {
|
||||
price: { $gt: 100 }
|
||||
}
|
||||
}
|
||||
});
|
||||
expect(orders.length).toEqual(2);
|
||||
expect(orders.every(o => o.items.some(i => i.price > 100))).toEqual(true);
|
||||
});
|
||||
|
||||
// ============= LOGICAL OPERATOR TESTS =============
|
||||
tap.test('should filter using $or operator', async () => {
|
||||
const users = await TestUser.getInstances({
|
||||
$or: [
|
||||
{ age: { $lt: 25 } },
|
||||
{ status: 'inactive' }
|
||||
]
|
||||
});
|
||||
expect(users.length).toEqual(2);
|
||||
const names = users.map(u => u.name).sort();
|
||||
expect(names).toEqual(['Bob Johnson', 'Charlie Wilson']);
|
||||
});
|
||||
|
||||
tap.test('should filter using $and operator', async () => {
|
||||
const users = await TestUser.getInstances({
|
||||
$and: [
|
||||
{ status: 'active' },
|
||||
{ age: { $gte: 28 } }
|
||||
]
|
||||
});
|
||||
expect(users.length).toEqual(2);
|
||||
const names = users.map(u => u.name).sort();
|
||||
expect(names).toEqual(['Alice Brown', 'John Doe']);
|
||||
});
|
||||
|
||||
tap.test('should filter using $nor operator', async () => {
|
||||
const users = await TestUser.getInstances({
|
||||
$nor: [
|
||||
{ status: 'inactive' },
|
||||
{ age: { $lt: 25 } }
|
||||
]
|
||||
});
|
||||
expect(users.length).toEqual(3);
|
||||
expect(users.every(u => u.status !== 'inactive' && u.age >= 25)).toEqual(true);
|
||||
});
|
||||
|
||||
tap.test('should filter using nested logical operators', async () => {
|
||||
const users = await TestUser.getInstances({
|
||||
$or: [
|
||||
{
|
||||
$and: [
|
||||
{ status: 'active' },
|
||||
{ roles: { $in: ['admin'] } }
|
||||
]
|
||||
},
|
||||
{ age: { $lt: 23 } }
|
||||
]
|
||||
});
|
||||
expect(users.length).toEqual(3);
|
||||
const names = users.map(u => u.name).sort();
|
||||
expect(names).toEqual(['Alice Brown', 'Charlie Wilson', 'John Doe']);
|
||||
});
|
||||
|
||||
// ============= ELEMENT OPERATOR TESTS =============
|
||||
tap.test('should filter using $exists operator', async () => {
|
||||
const users = await TestUser.getInstances({
|
||||
'metadata.lastLogin': { $exists: true }
|
||||
});
|
||||
expect(users.length).toEqual(1);
|
||||
expect(users[0].name).toEqual('John Doe');
|
||||
});
|
||||
|
||||
tap.test('should filter using $exists false', async () => {
|
||||
const users = await TestUser.getInstances({
|
||||
'metadata.preferences': { $exists: false }
|
||||
});
|
||||
expect(users.length).toEqual(5);
|
||||
});
|
||||
|
||||
// ============= COMPLEX FILTER TESTS =============
|
||||
tap.test('should handle complex nested filters', async () => {
|
||||
const users = await TestUser.getInstances({
|
||||
$and: [
|
||||
{ status: 'active' },
|
||||
{
|
||||
$or: [
|
||||
{ age: { $gte: 30 } },
|
||||
{ roles: { $all: ['admin'] } }
|
||||
]
|
||||
},
|
||||
{ tags: { $in: ['mongodb'] } }
|
||||
]
|
||||
});
|
||||
expect(users.length).toEqual(2);
|
||||
const names = users.map(u => u.name).sort();
|
||||
expect(names).toEqual(['Alice Brown', 'John Doe']);
|
||||
});
|
||||
|
||||
tap.test('should combine multiple operator types', async () => {
|
||||
const orders = await TestOrder.getInstances({
|
||||
$and: [
|
||||
{ totalAmount: { $gte: 100 } },
|
||||
{ status: { $in: ['completed', 'processing'] } },
|
||||
{ tags: { $in: ['electronics'] } }
|
||||
]
|
||||
});
|
||||
expect(orders.length).toEqual(2);
|
||||
expect(orders.every(o => o.totalAmount >= 100)).toEqual(true);
|
||||
});
|
||||
|
||||
// ============= ERROR HANDLING TESTS =============
|
||||
tap.test('should throw error for $where operator', async () => {
|
||||
let error: Error | null = null;
|
||||
try {
|
||||
await TestUser.getInstances({
|
||||
$where: 'this.age > 25'
|
||||
});
|
||||
} catch (e) {
|
||||
error = e as Error;
|
||||
}
|
||||
expect(error).toBeTruthy();
|
||||
expect(error?.message).toMatch(/\$where.*not allowed/);
|
||||
});
|
||||
|
||||
tap.test('should throw error for invalid $in value', async () => {
|
||||
let error: Error | null = null;
|
||||
try {
|
||||
await TestUser.getInstances({
|
||||
status: { $in: 'active' as any } // Should be an array
|
||||
});
|
||||
} catch (e) {
|
||||
error = e as Error;
|
||||
}
|
||||
expect(error).toBeTruthy();
|
||||
expect(error?.message).toMatch(/\$in.*requires.*array/);
|
||||
});
|
||||
|
||||
tap.test('should throw error for invalid $size value', async () => {
|
||||
let error: Error | null = null;
|
||||
try {
|
||||
await TestUser.getInstances({
|
||||
scores: { $size: '3' as any } // Should be a number
|
||||
});
|
||||
} catch (e) {
|
||||
error = e as Error;
|
||||
}
|
||||
expect(error).toBeTruthy();
|
||||
expect(error?.message).toMatch(/\$size.*requires.*numeric/);
|
||||
});
|
||||
|
||||
tap.test('should throw error for dots in field names', async () => {
|
||||
let error: Error | null = null;
|
||||
try {
|
||||
await TestUser.getInstances({
|
||||
'some.nested.field': { 'invalid.key': 'value' }
|
||||
});
|
||||
} catch (e) {
|
||||
error = e as Error;
|
||||
}
|
||||
expect(error).toBeTruthy();
|
||||
expect(error?.message).toMatch(/keys cannot contain dots/);
|
||||
});
|
||||
|
||||
// ============= EDGE CASE TESTS =============
|
||||
tap.test('should handle empty filter (return all)', async () => {
|
||||
const users = await TestUser.getInstances({});
|
||||
expect(users.length).toEqual(5);
|
||||
});
|
||||
|
||||
tap.test('should handle null values in filter', async () => {
|
||||
// First, create a user with null email
|
||||
const nullUser = new TestUser({
|
||||
name: 'Null User',
|
||||
age: 40,
|
||||
email: null as any,
|
||||
roles: ['user'],
|
||||
tags: [],
|
||||
status: 'active',
|
||||
metadata: {},
|
||||
scores: []
|
||||
});
|
||||
await nullUser.save();
|
||||
|
||||
const users = await TestUser.getInstances({ email: null });
|
||||
expect(users.length).toEqual(1);
|
||||
expect(users[0].name).toEqual('Null User');
|
||||
|
||||
// Clean up
|
||||
await nullUser.delete();
|
||||
});
|
||||
|
||||
tap.test('should handle arrays as direct equality match', async () => {
|
||||
// This tests that arrays without operators are treated as equality matches
|
||||
const users = await TestUser.getInstances({
|
||||
roles: ['user'] // Exact match for array
|
||||
});
|
||||
expect(users.length).toEqual(2); // Both Jane and Charlie have exactly ['user']
|
||||
const names = users.map(u => u.name).sort();
|
||||
expect(names).toEqual(['Charlie Wilson', 'Jane Smith']);
|
||||
});
|
||||
|
||||
tap.test('should handle regex operator', async () => {
|
||||
const users = await TestUser.getInstances({
|
||||
name: { $regex: '^J', $options: 'i' }
|
||||
});
|
||||
expect(users.length).toEqual(2);
|
||||
const names = users.map(u => u.name).sort();
|
||||
expect(names).toEqual(['Jane Smith', 'John Doe']);
|
||||
});
|
||||
|
||||
tap.test('should handle unknown operators by letting MongoDB reject them', async () => {
|
||||
// Unknown operators should be passed through to MongoDB, which will reject them
|
||||
let error: Error | null = null;
|
||||
|
||||
try {
|
||||
await TestUser.getInstances({
|
||||
age: { $unknownOp: 30 } as any
|
||||
});
|
||||
} catch (e) {
|
||||
error = e as Error;
|
||||
}
|
||||
|
||||
expect(error).toBeTruthy();
|
||||
expect(error?.message).toMatch(/unknown operator.*\$unknownOp/);
|
||||
});
|
||||
|
||||
// ============= PERFORMANCE TESTS =============
|
||||
tap.test('should efficiently filter large result sets', async () => {
|
||||
// Create many test documents
|
||||
const manyUsers = [];
|
||||
for (let i = 0; i < 100; i++) {
|
||||
manyUsers.push(new TestUser({
|
||||
name: `User ${i}`,
|
||||
age: 20 + (i % 40),
|
||||
email: `user${i}@example.com`,
|
||||
roles: i % 3 === 0 ? ['admin'] : ['user'],
|
||||
tags: i % 2 === 0 ? ['even', 'test'] : ['odd', 'test'],
|
||||
status: i % 4 === 0 ? 'inactive' : 'active',
|
||||
metadata: { loginCount: i },
|
||||
scores: [i, i + 10, i + 20]
|
||||
}));
|
||||
}
|
||||
|
||||
// Save in batches for efficiency
|
||||
for (const user of manyUsers) {
|
||||
await user.save();
|
||||
}
|
||||
|
||||
// Complex filter that should still be fast
|
||||
const startTime = Date.now();
|
||||
const filtered = await TestUser.getInstances({
|
||||
$and: [
|
||||
{ age: { $gte: 30, $lt: 40 } },
|
||||
{ status: 'active' },
|
||||
{ tags: { $in: ['even'] } },
|
||||
{ 'metadata.loginCount': { $gte: 20 } }
|
||||
]
|
||||
});
|
||||
const duration = Date.now() - startTime;
|
||||
|
||||
console.log(`Complex filter on 100+ documents took ${duration}ms`);
|
||||
expect(duration).toBeLessThan(1000); // Should complete in under 1 second
|
||||
expect(filtered.length).toBeGreaterThan(0);
|
||||
|
||||
// Clean up
|
||||
for (const user of manyUsers) {
|
||||
await user.delete();
|
||||
}
|
||||
});
|
||||
|
||||
// ============= CLEANUP =============
|
||||
tap.test('should clean up test database', async () => {
|
||||
await testDb.mongoDb.dropDatabase();
|
||||
await testDb.close();
|
||||
await smartmongoInstance.stop();
|
||||
});
|
||||
|
||||
export default tap.start();
|
||||
@@ -151,9 +151,49 @@ export function index(options?: IIndexOptions) {
|
||||
};
|
||||
}
|
||||
|
||||
// Type that allows MongoDB operators on leaf values while maintaining nested type safety
|
||||
export type MongoFilterCondition<T> = T | {
|
||||
$eq?: T;
|
||||
$ne?: T;
|
||||
$gt?: T;
|
||||
$gte?: T;
|
||||
$lt?: T;
|
||||
$lte?: T;
|
||||
$in?: T extends (infer U)[] ? U[] | U : T[];
|
||||
$nin?: T extends (infer U)[] ? U[] | U : T[];
|
||||
$exists?: boolean;
|
||||
$type?: string | number;
|
||||
$regex?: string | RegExp;
|
||||
$options?: string;
|
||||
$all?: T extends (infer U)[] ? U[] : never;
|
||||
$elemMatch?: T extends (infer U)[] ? MongoFilter<U> : never;
|
||||
$size?: T extends any[] ? number : never;
|
||||
$not?: MongoFilterCondition<T>;
|
||||
};
|
||||
|
||||
export type MongoFilter<T> = {
|
||||
[K in keyof T]?: T[K] extends object
|
||||
? T[K] extends any[]
|
||||
? MongoFilterCondition<T[K]> // Arrays can have operators
|
||||
: MongoFilter<T[K]> | MongoFilterCondition<T[K]> // Objects can be nested or have operators
|
||||
: MongoFilterCondition<T[K]>; // Primitives get operators
|
||||
} & {
|
||||
// Logical operators
|
||||
$and?: MongoFilter<T>[];
|
||||
$or?: MongoFilter<T>[];
|
||||
$nor?: MongoFilter<T>[];
|
||||
$not?: MongoFilter<T>;
|
||||
// Allow any string key for dot notation (we lose type safety here but maintain flexibility)
|
||||
[key: string]: any;
|
||||
};
|
||||
|
||||
export const convertFilterForMongoDb = (filterArg: { [key: string]: any }) => {
|
||||
// SECURITY: Block $where to prevent server-side JS execution
|
||||
if (filterArg.$where !== undefined) {
|
||||
throw new Error('$where operator is not allowed for security reasons');
|
||||
}
|
||||
|
||||
// Special case: detect MongoDB operators and pass them through directly
|
||||
// SECURITY: Removed $where to prevent server-side JS execution
|
||||
const topLevelOperators = ['$and', '$or', '$nor', '$not', '$text', '$regex'];
|
||||
for (const key of Object.keys(filterArg)) {
|
||||
if (topLevelOperators.includes(key)) {
|
||||
@@ -166,26 +206,76 @@ export const convertFilterForMongoDb = (filterArg: { [key: string]: any }) => {
|
||||
|
||||
const convertFilterArgument = (keyPathArg2: string, filterArg2: any) => {
|
||||
if (Array.isArray(filterArg2)) {
|
||||
// FIX: Properly handle arrays for operators like $in, $all, or plain equality
|
||||
// Arrays are typically used as values for operators like $in or as direct equality matches
|
||||
convertedFilter[keyPathArg2] = filterArg2;
|
||||
return;
|
||||
} else if (typeof filterArg2 === 'object' && filterArg2 !== null) {
|
||||
for (const key of Object.keys(filterArg2)) {
|
||||
if (key.startsWith('$')) {
|
||||
// Prevent dangerous operators
|
||||
if (key === '$where') {
|
||||
// Check if this is an object with MongoDB operators
|
||||
const keys = Object.keys(filterArg2);
|
||||
const hasOperators = keys.some(key => key.startsWith('$'));
|
||||
|
||||
if (hasOperators) {
|
||||
// This object contains MongoDB operators
|
||||
// Validate and pass through allowed operators
|
||||
const allowedOperators = [
|
||||
// Comparison operators
|
||||
'$eq', '$ne', '$gt', '$gte', '$lt', '$lte',
|
||||
// Array operators
|
||||
'$in', '$nin', '$all', '$elemMatch', '$size',
|
||||
// Element operators
|
||||
'$exists', '$type',
|
||||
// Evaluation operators (safe ones only)
|
||||
'$regex', '$options', '$text', '$mod',
|
||||
// Logical operators (nested)
|
||||
'$and', '$or', '$nor', '$not'
|
||||
];
|
||||
|
||||
// Check for dangerous operators
|
||||
if (keys.includes('$where')) {
|
||||
throw new Error('$where operator is not allowed for security reasons');
|
||||
}
|
||||
|
||||
// Validate all operators are in the allowed list
|
||||
const invalidOperators = keys.filter(key =>
|
||||
key.startsWith('$') && !allowedOperators.includes(key)
|
||||
);
|
||||
|
||||
if (invalidOperators.length > 0) {
|
||||
console.warn(`Warning: Unknown MongoDB operators detected: ${invalidOperators.join(', ')}`);
|
||||
}
|
||||
|
||||
// For array operators, ensure the values are appropriate
|
||||
if (filterArg2.$in && !Array.isArray(filterArg2.$in)) {
|
||||
throw new Error('$in operator requires an array value');
|
||||
}
|
||||
if (filterArg2.$nin && !Array.isArray(filterArg2.$nin)) {
|
||||
throw new Error('$nin operator requires an array value');
|
||||
}
|
||||
if (filterArg2.$all && !Array.isArray(filterArg2.$all)) {
|
||||
throw new Error('$all operator requires an array value');
|
||||
}
|
||||
if (filterArg2.$size && typeof filterArg2.$size !== 'number') {
|
||||
throw new Error('$size operator requires a numeric value');
|
||||
}
|
||||
|
||||
// Pass the operator object through
|
||||
convertedFilter[keyPathArg2] = filterArg2;
|
||||
return;
|
||||
} else if (key.includes('.')) {
|
||||
}
|
||||
|
||||
// No operators, check for dots in keys
|
||||
for (const key of keys) {
|
||||
if (key.includes('.')) {
|
||||
throw new Error('keys cannot contain dots');
|
||||
}
|
||||
}
|
||||
for (const key of Object.keys(filterArg2)) {
|
||||
|
||||
// Recursively process nested objects
|
||||
for (const key of keys) {
|
||||
convertFilterArgument(`${keyPathArg2}.${key}`, filterArg2[key]);
|
||||
}
|
||||
} else {
|
||||
// Primitive values
|
||||
convertedFilter[keyPathArg2] = filterArg2;
|
||||
}
|
||||
};
|
||||
@@ -227,12 +317,12 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
||||
/**
|
||||
* gets all instances as array
|
||||
* @param this
|
||||
* @param filterArg
|
||||
* @param filterArg - Type-safe MongoDB filter with nested object support and operators
|
||||
* @returns
|
||||
*/
|
||||
public static async getInstances<T>(
|
||||
this: plugins.tsclass.typeFest.Class<T>,
|
||||
filterArg: plugins.tsclass.typeFest.PartialDeep<T>,
|
||||
filterArg: MongoFilter<T>,
|
||||
opts?: { session?: plugins.mongodb.ClientSession }
|
||||
): Promise<T[]> {
|
||||
// Pass session through to findAll for transactional queries
|
||||
@@ -256,7 +346,7 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
||||
*/
|
||||
public static async getInstance<T>(
|
||||
this: plugins.tsclass.typeFest.Class<T>,
|
||||
filterArg: plugins.tsclass.typeFest.PartialDeep<T>,
|
||||
filterArg: MongoFilter<T>,
|
||||
opts?: { session?: plugins.mongodb.ClientSession }
|
||||
): Promise<T> {
|
||||
// Retrieve one document, with optional session for transactions
|
||||
@@ -289,7 +379,7 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
||||
*/
|
||||
public static async getCursor<T>(
|
||||
this: plugins.tsclass.typeFest.Class<T>,
|
||||
filterArg: plugins.tsclass.typeFest.PartialDeep<T>,
|
||||
filterArg: MongoFilter<T>,
|
||||
opts?: {
|
||||
session?: plugins.mongodb.ClientSession;
|
||||
modifier?: (cursorArg: plugins.mongodb.FindCursor<plugins.mongodb.WithId<plugins.mongodb.BSON.Document>>) => plugins.mongodb.FindCursor<plugins.mongodb.WithId<plugins.mongodb.BSON.Document>>;
|
||||
@@ -319,7 +409,7 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
||||
*/
|
||||
public static async watch<T>(
|
||||
this: plugins.tsclass.typeFest.Class<T>,
|
||||
filterArg: plugins.tsclass.typeFest.PartialDeep<T>,
|
||||
filterArg: MongoFilter<T>,
|
||||
opts?: plugins.mongodb.ChangeStreamOptions & { bufferTimeMs?: number },
|
||||
): Promise<SmartdataDbWatcher<T>> {
|
||||
const collection: SmartdataCollection<T> = (this as any).collection;
|
||||
@@ -337,7 +427,7 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
||||
*/
|
||||
public static async forEach<T>(
|
||||
this: plugins.tsclass.typeFest.Class<T>,
|
||||
filterArg: plugins.tsclass.typeFest.PartialDeep<T>,
|
||||
filterArg: MongoFilter<T>,
|
||||
forEachFunction: (itemArg: T) => Promise<any>,
|
||||
) {
|
||||
const cursor: SmartdataDbCursor<T> = await (this as any).getCursor(filterArg);
|
||||
@@ -349,7 +439,7 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
||||
*/
|
||||
public static async getCount<T>(
|
||||
this: plugins.tsclass.typeFest.Class<T>,
|
||||
filterArg: plugins.tsclass.typeFest.PartialDeep<T> = {} as any,
|
||||
filterArg: MongoFilter<T> = {} as any,
|
||||
) {
|
||||
const collection: SmartdataCollection<T> = (this as any).collection;
|
||||
return await collection.getCount(filterArg);
|
||||
|
||||
Reference in New Issue
Block a user