push.rocks/smartdata
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Add comprehensive query filters guide and enhance type safety for MongoDB queries

Browse Source 
- Introduced a detailed guide on query filters in the README, covering basic filtering, comparison operators, array operators, logical operators, element operators, and advanced filtering patterns.
- Implemented a type-safe filtering system in `classes.doc.ts` with `MongoFilterCondition` and `MongoFilter` types to support MongoDB operators while maintaining nested type safety.
- Enhanced error handling for invalid operators and conditions in the filtering logic.
- Added extensive tests for various filtering scenarios, including basic, comparison, array, logical, and complex filters, ensuring robust functionality and performance.
- Implemented security measures to prevent the use of dangerous operators like `$where` and validate operator usage.
This commit is contained in:
Jürgen Kunz
2025-08-18 11:29:15 +00:00
parent f4290ae7f7
commit cdd1ae2c9b
5 changed files with 1979 additions and 859 deletions
Download Patch File Download Diff File Expand all files Collapse all files

128
ts/classes.doc.ts
View File

@@ -151,9 +151,49 @@ export function index(options?: IIndexOptions) {
};
}
// Type that allows MongoDB operators on leaf values while maintaining nested type safety
export type MongoFilterCondition<T> = T | {
$eq?: T;
$ne?: T;
$gt?: T;
$gte?: T;
$lt?: T;
$lte?: T;
$in?: T extends (infer U)[] ? U[] | U : T[];
$nin?: T extends (infer U)[] ? U[] | U : T[];
$exists?: boolean;
$type?: string | number;
$regex?: string | RegExp;
$options?: string;
$all?: T extends (infer U)[] ? U[] : never;
$elemMatch?: T extends (infer U)[] ? MongoFilter<U> : never;
$size?: T extends any[] ? number : never;
$not?: MongoFilterCondition<T>;
};
export type MongoFilter<T> = {
[K in keyof T]?: T[K] extends object
? T[K] extends any[]
? MongoFilterCondition<T[K]> // Arrays can have operators
: MongoFilter<T[K]> | MongoFilterCondition<T[K]> // Objects can be nested or have operators
: MongoFilterCondition<T[K]>; // Primitives get operators
} & {
// Logical operators
$and?: MongoFilter<T>[];
$or?: MongoFilter<T>[];
$nor?: MongoFilter<T>[];
$not?: MongoFilter<T>;
// Allow any string key for dot notation (we lose type safety here but maintain flexibility)
[key: string]: any;
};
export const convertFilterForMongoDb = (filterArg: { [key: string]: any }) => {
// SECURITY: Block $where to prevent server-side JS execution
if (filterArg.$where !== undefined) {
throw new Error('$where operator is not allowed for security reasons');
}
// Special case: detect MongoDB operators and pass them through directly
// SECURITY: Removed $where to prevent server-side JS execution
const topLevelOperators = ['$and', '$or', '$nor', '$not', '$text', '$regex'];
for (const key of Object.keys(filterArg)) {
if (topLevelOperators.includes(key)) {
@@ -166,26 +206,76 @@ export const convertFilterForMongoDb = (filterArg: { [key: string]: any }) => {
const convertFilterArgument = (keyPathArg2: string, filterArg2: any) => {
if (Array.isArray(filterArg2)) {
// FIX: Properly handle arrays for operators like $in, $all, or plain equality
// Arrays are typically used as values for operators like $in or as direct equality matches
convertedFilter[keyPathArg2] = filterArg2;
return;
} else if (typeof filterArg2 === 'object' && filterArg2 !== null) {
for (const key of Object.keys(filterArg2)) {
if (key.startsWith('$')) {
// Prevent dangerous operators
if (key === '$where') {
throw new Error('$where operator is not allowed for security reasons');
}
convertedFilter[keyPathArg2] = filterArg2;
return;
} else if (key.includes('.')) {
// Check if this is an object with MongoDB operators
const keys = Object.keys(filterArg2);
const hasOperators = keys.some(key => key.startsWith('$'));
if (hasOperators) {
// This object contains MongoDB operators
// Validate and pass through allowed operators
const allowedOperators = [
// Comparison operators
'$eq', '$ne', '$gt', '$gte', '$lt', '$lte',
// Array operators
'$in', '$nin', '$all', '$elemMatch', '$size',
// Element operators
'$exists', '$type',
// Evaluation operators (safe ones only)
'$regex', '$options', '$text', '$mod',
// Logical operators (nested)
'$and', '$or', '$nor', '$not'
];
// Check for dangerous operators
if (keys.includes('$where')) {
throw new Error('$where operator is not allowed for security reasons');
}
// Validate all operators are in the allowed list
const invalidOperators = keys.filter(key =>
key.startsWith('$') && !allowedOperators.includes(key)
);
if (invalidOperators.length > 0) {
console.warn(`Warning: Unknown MongoDB operators detected: ${invalidOperators.join(', ')}`);
}
// For array operators, ensure the values are appropriate
if (filterArg2.$in && !Array.isArray(filterArg2.$in)) {
throw new Error('$in operator requires an array value');
}
if (filterArg2.$nin && !Array.isArray(filterArg2.$nin)) {
throw new Error('$nin operator requires an array value');
}
if (filterArg2.$all && !Array.isArray(filterArg2.$all)) {
throw new Error('$all operator requires an array value');
}
if (filterArg2.$size && typeof filterArg2.$size !== 'number') {
throw new Error('$size operator requires a numeric value');
}
// Pass the operator object through
convertedFilter[keyPathArg2] = filterArg2;
return;
}
// No operators, check for dots in keys
for (const key of keys) {
if (key.includes('.')) {
throw new Error('keys cannot contain dots');
}
}
for (const key of Object.keys(filterArg2)) {
// Recursively process nested objects
for (const key of keys) {
convertFilterArgument(`${keyPathArg2}.${key}`, filterArg2[key]);
}
} else {
// Primitive values
convertedFilter[keyPathArg2] = filterArg2;
}
};
@@ -227,12 +317,12 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
/**
* gets all instances as array
* @param this
* @param filterArg
* @param filterArg - Type-safe MongoDB filter with nested object support and operators
* @returns
*/
public static async getInstances<T>(
this: plugins.tsclass.typeFest.Class<T>,
filterArg: plugins.tsclass.typeFest.PartialDeep<T>,
filterArg: MongoFilter<T>,
opts?: { session?: plugins.mongodb.ClientSession }
): Promise<T[]> {
// Pass session through to findAll for transactional queries
@@ -256,7 +346,7 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
*/
public static async getInstance<T>(
this: plugins.tsclass.typeFest.Class<T>,
filterArg: plugins.tsclass.typeFest.PartialDeep<T>,
filterArg: MongoFilter<T>,
opts?: { session?: plugins.mongodb.ClientSession }
): Promise<T> {
// Retrieve one document, with optional session for transactions
@@ -289,7 +379,7 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
*/
public static async getCursor<T>(
this: plugins.tsclass.typeFest.Class<T>,
filterArg: plugins.tsclass.typeFest.PartialDeep<T>,
filterArg: MongoFilter<T>,
opts?: {
session?: plugins.mongodb.ClientSession;
modifier?: (cursorArg: plugins.mongodb.FindCursor<plugins.mongodb.WithId<plugins.mongodb.BSON.Document>>) => plugins.mongodb.FindCursor<plugins.mongodb.WithId<plugins.mongodb.BSON.Document>>;
@@ -319,7 +409,7 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
*/
public static async watch<T>(
this: plugins.tsclass.typeFest.Class<T>,
filterArg: plugins.tsclass.typeFest.PartialDeep<T>,
filterArg: MongoFilter<T>,
opts?: plugins.mongodb.ChangeStreamOptions & { bufferTimeMs?: number },
): Promise<SmartdataDbWatcher<T>> {
const collection: SmartdataCollection<T> = (this as any).collection;
@@ -337,7 +427,7 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
*/
public static async forEach<T>(
this: plugins.tsclass.typeFest.Class<T>,
filterArg: plugins.tsclass.typeFest.PartialDeep<T>,
filterArg: MongoFilter<T>,
forEachFunction: (itemArg: T) => Promise<any>,
) {
const cursor: SmartdataDbCursor<T> = await (this as any).getCursor(filterArg);
@@ -349,7 +439,7 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
*/
public static async getCount<T>(
this: plugins.tsclass.typeFest.Class<T>,
filterArg: plugins.tsclass.typeFest.PartialDeep<T> = {} as any,
filterArg: MongoFilter<T> = {} as any,
) {
const collection: SmartdataCollection<T> = (this as any).collection;
return await collection.getCount(filterArg);