From b5a9449d5edefa1e80ffb6a3a3847fc9d8e2f494 Mon Sep 17 00:00:00 2001 From: Philipp Kunz Date: Fri, 18 Apr 2025 11:25:39 +0000 Subject: [PATCH] feat(collections/search): Improve text index creation and search fallback mechanisms in collections and document search methods --- changelog.md | 6 ++++ ts/00_commitinfo_data.ts | 2 +- ts/classes.collection.ts | 12 ++++++- ts/classes.doc.ts | 71 ++++++++++++++++------------------------ 4 files changed, 46 insertions(+), 45 deletions(-) diff --git a/changelog.md b/changelog.md index 32c9b67..0fabc3c 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,11 @@ # Changelog +## 2025-04-18 - 5.9.0 - feat(collections/search) +Improve text index creation and search fallback mechanisms in collections and document search methods + +- Auto-create a compound text index on all searchable fields in SmartdataCollection with a one-time flag to prevent duplicate index creation. +- Refine the search method in SmartDataDbDoc to support exact field matches and safe regex fallback for non-Lucene queries. + ## 2025-04-17 - 5.8.4 - fix(core) Update commit metadata with no functional code changes diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index acca5da..a4bab67 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@push.rocks/smartdata', - version: '5.8.4', + version: '5.9.0', description: 'An advanced library for NoSQL data organization and manipulation using TypeScript with support for MongoDB, data validation, collections, and custom data types.' } diff --git a/ts/classes.collection.ts b/ts/classes.collection.ts index 5f34b4b..7c0c591 100644 --- a/ts/classes.collection.ts +++ b/ts/classes.collection.ts @@ -1,7 +1,7 @@ import * as plugins from './plugins.js'; import { SmartdataDb } from './classes.db.js'; import { SmartdataDbCursor } from './classes.cursor.js'; -import { SmartDataDbDoc, type IIndexOptions } from './classes.doc.js'; +import { SmartDataDbDoc, type IIndexOptions, getSearchableFields } from './classes.doc.js'; import { SmartdataDbWatcher } from './classes.watcher.js'; import { CollectionFactory } from './classes.collectionfactory.js'; @@ -128,6 +128,8 @@ export class SmartdataCollection { public smartdataDb: SmartdataDb; public uniqueIndexes: string[] = []; public regularIndexes: Array<{field: string, options: IIndexOptions}> = []; + // flag to ensure text index is created only once + private textIndexCreated: boolean = false; constructor(classNameArg: string, smartDataDbArg: SmartdataDb) { // tell the collection where it belongs @@ -153,6 +155,14 @@ export class SmartdataCollection { console.log(`Successfully initiated Collection ${this.collectionName}`); } this.mongoDbCollection = this.smartdataDb.mongoDb.collection(this.collectionName); + // Auto-create a compound text index on all searchable fields + const searchableFields = getSearchableFields(this.collectionName); + if (searchableFields.length > 0 && !this.textIndexCreated) { + const indexSpec: { [key: string]: string } = {}; + searchableFields.forEach(f => { indexSpec[f] = 'text'; }); + await this.mongoDbCollection.createIndex(indexSpec, { name: 'smartdata_text_index' }); + this.textIndexCreated = true; + } } } diff --git a/ts/classes.doc.ts b/ts/classes.doc.ts index b1be379..9776824 100644 --- a/ts/classes.doc.ts +++ b/ts/classes.doc.ts @@ -61,6 +61,10 @@ export function getSearchableFields(className: string): string[] { } return Array.from(searchableFieldsMap.get(className)); } +// Escape user input for safe use in MongoDB regular expressions +function escapeForRegex(input: string): string { + return input.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} /** * unique index - decorator to mark a unique index @@ -326,57 +330,38 @@ export class SmartDataDbDoc( this: plugins.tsclass.typeFest.Class, - luceneQuery: string, + query: string, ): Promise { - const filter = (this as any).createSearchFilter(luceneQuery); - return await (this as any).getInstances(filter); - } - - /** - * Search documents using Lucene query syntax with robust error handling - * @param luceneQuery The Lucene query string to search with - * @returns Array of matching documents - */ - public static async searchWithLucene( - this: plugins.tsclass.typeFest.Class, - luceneQuery: string, - ): Promise { - try { - const className = (this as any).className || this.name; - const searchableFields = getSearchableFields(className); - - if (searchableFields.length === 0) { - console.warn( - `No searchable fields defined for class ${className}, falling back to simple search`, - ); - return (this as any).searchByTextAcrossFields(luceneQuery); - } - - // Simple term search optimization - if ( - !luceneQuery.includes(':') && - !luceneQuery.includes(' AND ') && - !luceneQuery.includes(' OR ') && - !luceneQuery.includes(' NOT ') - ) { - return (this as any).searchByTextAcrossFields(luceneQuery); - } - - // Try to use the Lucene-to-MongoDB conversion - const filter = (this as any).createSearchFilter(luceneQuery); - return await (this as any).getInstances(filter); - } catch (error) { - console.error(`Error in searchWithLucene: ${error.message}`); - return (this as any).searchByTextAcrossFields(luceneQuery); + const className = (this as any).className || this.name; + const searchableFields = getSearchableFields(className); + if (searchableFields.length === 0) { + throw new Error(`No searchable fields defined for class ${className}`); } + // field:value exact match (case-sensitive for non-regex fields) + const fv = query.match(/^(\w+):(.+)$/); + if (fv) { + const field = fv[1]; + const value = fv[2]; + if (!searchableFields.includes(field)) { + throw new Error(`Field '${field}' is not searchable for class ${className}`); + } + return await (this as any).getInstances({ [field]: value }); + } + // safe regex across all searchable fields (case-insensitive) + const escaped = escapeForRegex(query); + const orConditions = searchableFields.map((field) => ({ + [field]: { $regex: escaped, $options: 'i' }, + })); + return await (this as any).getInstances({ $or: orConditions }); } + /** * Search by text across all searchable fields (fallback method) * @param searchText The text to search for in all searchable fields