feat(collections/search): Improve text index creation and search fallback mechanisms in collections and document search methods

changelog.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 2025-04-18 - 5.9.0 - feat(collections/search)
+Improve text index creation and search fallback mechanisms in collections and document search methods
+
+- Auto-create a compound text index on all searchable fields in SmartdataCollection with a one-time flag to prevent duplicate index creation.
+- Refine the search method in SmartDataDbDoc to support exact field matches and safe regex fallback for non-Lucene queries.
+
 ## 2025-04-17 - 5.8.4 - fix(core)
 Update commit metadata with no functional code changes
 

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@push.rocks/smartdata',
-  version: '5.8.4',
+  version: '5.9.0',
   description: 'An advanced library for NoSQL data organization and manipulation using TypeScript with support for MongoDB, data validation, collections, and custom data types.'
 }

ts/classes.collection.ts

@@ -1,7 +1,7 @@
 import * as plugins from './plugins.js';
 import { SmartdataDb } from './classes.db.js';
 import { SmartdataDbCursor } from './classes.cursor.js';
-import { SmartDataDbDoc, type IIndexOptions } from './classes.doc.js';
+import { SmartDataDbDoc, type IIndexOptions, getSearchableFields } from './classes.doc.js';
 import { SmartdataDbWatcher } from './classes.watcher.js';
 import { CollectionFactory } from './classes.collectionfactory.js';
 
@@ -128,6 +128,8 @@ export class SmartdataCollection<T> {
   public smartdataDb: SmartdataDb;
   public uniqueIndexes: string[] = [];
   public regularIndexes: Array<{field: string, options: IIndexOptions}> = [];
+  // flag to ensure text index is created only once
+  private textIndexCreated: boolean = false;
 
   constructor(classNameArg: string, smartDataDbArg: SmartdataDb) {
     // tell the collection where it belongs
@@ -153,6 +155,14 @@ export class SmartdataCollection<T> {
         console.log(`Successfully initiated Collection ${this.collectionName}`);
       }
       this.mongoDbCollection = this.smartdataDb.mongoDb.collection(this.collectionName);
+      // Auto-create a compound text index on all searchable fields
+      const searchableFields = getSearchableFields(this.collectionName);
+      if (searchableFields.length > 0 && !this.textIndexCreated) {
+        const indexSpec: { [key: string]: string } = {};
+        searchableFields.forEach(f => { indexSpec[f] = 'text'; });
+        await this.mongoDbCollection.createIndex(indexSpec, { name: 'smartdata_text_index' });
+        this.textIndexCreated = true;
+      }
     }
   }
 

ts/classes.doc.ts

@@ -61,6 +61,10 @@ export function getSearchableFields(className: string): string[] {
   }
   return Array.from(searchableFieldsMap.get(className));
 }
+// Escape user input for safe use in MongoDB regular expressions
+function escapeForRegex(input: string): string {
+  return input.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
 
 /**
  * unique index - decorator to mark a unique index
@@ -326,56 +330,37 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
   }
 
   /**
-   * Search documents using Lucene query syntax
+   * Search documents by text or field:value syntax, with safe regex fallback
-   * @param luceneQuery Lucene query string
+   * @param query A search term or field:value expression
    * @returns Array of matching documents
    */
   public static async search<T>(
     this: plugins.tsclass.typeFest.Class<T>,
-    luceneQuery: string,
+    query: string,
   ): Promise<T[]> {
-    const filter = (this as any).createSearchFilter(luceneQuery);
-    return await (this as any).getInstances(filter);
-  }
-
-  /**
-   * Search documents using Lucene query syntax with robust error handling
-   * @param luceneQuery The Lucene query string to search with
-   * @returns Array of matching documents
-   */
-  public static async searchWithLucene<T>(
-    this: plugins.tsclass.typeFest.Class<T>,
-    luceneQuery: string,
-  ): Promise<T[]> {
-    try {
     const className = (this as any).className || this.name;
     const searchableFields = getSearchableFields(className);
-
     if (searchableFields.length === 0) {
-        console.warn(
+      throw new Error(`No searchable fields defined for class ${className}`);
-          `No searchable fields defined for class ${className}, falling back to simple search`,
+    }
-        );
+    // field:value exact match (case-sensitive for non-regex fields)
-        return (this as any).searchByTextAcrossFields(luceneQuery);
+    const fv = query.match(/^(\w+):(.+)$/);
+    if (fv) {
+      const field = fv[1];
+      const value = fv[2];
+      if (!searchableFields.includes(field)) {
+        throw new Error(`Field '${field}' is not searchable for class ${className}`);
+      }
+      return await (this as any).getInstances({ [field]: value });
+    }
+    // safe regex across all searchable fields (case-insensitive)
+    const escaped = escapeForRegex(query);
+    const orConditions = searchableFields.map((field) => ({
+      [field]: { $regex: escaped, $options: 'i' },
+    }));
+    return await (this as any).getInstances({ $or: orConditions });
   }
 
-      // Simple term search optimization
-      if (
-        !luceneQuery.includes(':') &&
-        !luceneQuery.includes(' AND ') &&
-        !luceneQuery.includes(' OR ') &&
-        !luceneQuery.includes(' NOT ')
-      ) {
-        return (this as any).searchByTextAcrossFields(luceneQuery);
-      }
-
-      // Try to use the Lucene-to-MongoDB conversion
-      const filter = (this as any).createSearchFilter(luceneQuery);
-      return await (this as any).getInstances(filter);
-    } catch (error) {
-      console.error(`Error in searchWithLucene: ${error.message}`);
-      return (this as any).searchByTextAcrossFields(luceneQuery);
-    }
-  }
 
   /**
    * Search by text across all searchable fields (fallback method)