5.9.2

changelog.md

@@ -1,5 +1,24 @@
 # Changelog
 
+## 2025-04-18 - 5.9.2 - fix(documentation)
+Update search API documentation to replace deprecated searchWithLucene examples with the unified search(query) API and clarify its behavior.
+
+- Replaced 'searchWithLucene' examples with 'search(query)' in the README.
+- Updated explanation to detail field-specific exact match, partial word regex search, multi-word literal matching, and handling of empty queries.
+- Clarified guidelines for creating MongoDB text indexes on searchable fields for optimized search performance.
+
+## 2025-04-18 - 5.9.1 - fix(search)
+Refactor search tests to use unified search API and update text index type casting
+
+- Replaced all calls from searchWithLucene with search in test/search tests
+- Updated text index specification in the collection class to use proper type casting
+
+## 2025-04-18 - 5.9.0 - feat(collections/search)
+Improve text index creation and search fallback mechanisms in collections and document search methods
+
+- Auto-create a compound text index on all searchable fields in SmartdataCollection with a one-time flag to prevent duplicate index creation.
+- Refine the search method in SmartDataDbDoc to support exact field matches and safe regex fallback for non-Lucene queries.
+
 ## 2025-04-17 - 5.8.4 - fix(core)
 Update commit metadata with no functional code changes
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@push.rocks/smartdata",
-  "version": "5.8.4",
+  "version": "5.9.2",
   "private": false,
   "description": "An advanced library for NoSQL data organization and manipulation using TypeScript with support for MongoDB, data validation, collections, and custom data types.",
   "main": "dist_ts/index.js",

readme.md

@@ -18,7 +18,7 @@ A powerful TypeScript-first MongoDB wrapper that provides advanced features for
 - **Enhanced Cursors**: Chainable, type-safe cursor API with memory efficient data processing
 - **Type Conversion**: Automatic handling of MongoDB types like ObjectId and Binary data
 - **Serialization Hooks**: Custom serialization and deserialization of document properties
-- **Powerful Search Capabilities**: Lucene-like query syntax with field-specific search, advanced operators, and fallback mechanisms
+- **Powerful Search Capabilities**: Unified `search(query)` API supporting field:value exact matches, multi-field regex searches, case-insensitive matching, and automatic escaping to prevent regex injection
 
 ## Requirements
 
@@ -218,43 +218,31 @@ class Product extends SmartDataDbDoc<Product, Product> {
 const searchableFields = getSearchableFields('Product'); // ['name', 'description', 'category']
 
 // Basic search across all searchable fields
-const iPhoneProducts = await Product.searchWithLucene('iPhone');
+const iphoneProducts = await Product.search('iPhone');
 
-// Field-specific search
+// Field-specific exact match
-const electronicsProducts = await Product.searchWithLucene('category:Electronics');
+const electronicsProducts = await Product.search('category:Electronics');
 
-// Search with wildcards
+// Partial word search (regex across all fields)
-const macProducts = await Product.searchWithLucene('Mac*');
+const laptopResults = await Product.search('laptop');
 
-// Search in specific fields with partial words
+// Multi-word literal search
-const laptopResults = await Product.searchWithLucene('description:laptop');
+const paperwhite = await Product.search('Kindle Paperwhite');
 
-// Search is case-insensitive
+// Empty query returns all documents
-const results1 = await Product.searchWithLucene('electronics');
+const allProducts = await Product.search('');
-const results2 = await Product.searchWithLucene('Electronics');
-// results1 and results2 will contain the same documents
-
-// Using boolean operators (requires text index in MongoDB)
-const wirelessOrLaptop = await Product.searchWithLucene('wireless OR laptop');
-
-// Negative searches
-const electronicsNotSamsung = await Product.searchWithLucene('Electronics NOT Samsung');
-
-// Phrase searches
-const exactPhrase = await Product.searchWithLucene('"high-speed blender"');
-
-// Grouping with parentheses
-const complexQuery = await Product.searchWithLucene('(wireless OR bluetooth) AND Electronics');
 ```
 
 The search functionality includes:
 
 - `@searchable()` decorator for marking fields as searchable
-- `getSearchableFields()` to retrieve all searchable fields for a class
+- `getSearchableFields()` to list searchable fields for a model
-- `search()` method for basic search (requires MongoDB text index)
+- `search(query: string)` method supporting:
-- `searchWithLucene()` method with robust fallback mechanisms
+  - Field-specific exact matches (`field:value`)
-- Support for field-specific searches, wildcards, and boolean operators
+  - Case-insensitive partial matches across all searchable fields
-- Automatic fallback to regex-based search if MongoDB text search fails
+  - Multi-word literal matching
+  - Empty queries returning all documents
+- Automatic escaping of special characters to prevent regex injection
 
 ### EasyStore
 
@@ -549,10 +537,10 @@ class Order extends SmartDataDbDoc<Order, Order> {
 
 ### Search Optimization
 
-- Create MongoDB text indexes for collections that need advanced search operations
+- (Optional) Create MongoDB text indexes on searchable fields to speed up full-text search
-- Use `searchWithLucene()` for robust searches with fallback mechanisms
+- Use `search(query)` for all search operations (field:value, partial matches, multi-word)
-- Prefer field-specific searches when possible for better performance
+- Prefer field-specific exact matches when possible for optimal performance
-- Use simple term queries instead of boolean operators if you don't have text indexes
+- Avoid unnecessary complexity in query strings to keep regex searches efficient
 
 ### Performance Optimization
 

test/test.search.ts

@@ -104,21 +104,21 @@ tap.test('should search products by basic search method', async () => {
   }
 });
 
-tap.test('should search products with searchWithLucene method', async () => {
+tap.test('should search products with search method', async () => {
   // Using the robust searchWithLucene method
-  const wirelessResults = await Product.searchWithLucene('wireless');
+    const wirelessResults = await Product.search('wireless');
-  console.log(
+    console.log(
-    `Found ${wirelessResults.length} products matching 'wireless' using searchWithLucene`,
+      `Found ${wirelessResults.length} products matching 'wireless' using search`,
-  );
+    );
 
   expect(wirelessResults.length).toEqual(1);
   expect(wirelessResults[0].name).toEqual('AirPods');
 });
 
-tap.test('should search products by category with searchWithLucene', async () => {
+tap.test('should search products by category with search', async () => {
   // Using field-specific search with searchWithLucene
-  const kitchenResults = await Product.searchWithLucene('category:Kitchen');
+  const kitchenResults = await Product.search('category:Kitchen');
-  console.log(`Found ${kitchenResults.length} products in Kitchen category using searchWithLucene`);
+  console.log(`Found ${kitchenResults.length} products in Kitchen category using search`);
 
   expect(kitchenResults.length).toEqual(2);
   expect(kitchenResults[0].category).toEqual('Kitchen');
@@ -127,7 +127,7 @@ tap.test('should search products by category with searchWithLucene', async () =>
 
 tap.test('should search products with partial word matches', async () => {
   // Testing partial word matches
-  const proResults = await Product.searchWithLucene('Pro');
+  const proResults = await Product.search('Pro');
   console.log(`Found ${proResults.length} products matching 'Pro'`);
 
   // Should match both "MacBook Pro" and "professionals" in description
@@ -136,7 +136,7 @@ tap.test('should search products with partial word matches', async () => {
 
 tap.test('should search across multiple searchable fields', async () => {
   // Test searching across all searchable fields
-  const bookResults = await Product.searchWithLucene('book');
+  const bookResults = await Product.search('book');
   console.log(`Found ${bookResults.length} products matching 'book' across all fields`);
 
   // Should match "MacBook" in name and "Books" in category
@@ -145,8 +145,8 @@ tap.test('should search across multiple searchable fields', async () => {
 
 tap.test('should handle case insensitive searches', async () => {
   // Test case insensitivity
-  const electronicsResults = await Product.searchWithLucene('electronics');
+  const electronicsResults = await Product.search('electronics');
-  const ElectronicsResults = await Product.searchWithLucene('Electronics');
+  const ElectronicsResults = await Product.search('Electronics');
 
   console.log(`Found ${electronicsResults.length} products matching lowercase 'electronics'`);
   console.log(`Found ${ElectronicsResults.length} products matching capitalized 'Electronics'`);
@@ -166,14 +166,14 @@ tap.test('should demonstrate search fallback mechanisms', async () => {
 
   // Use a simpler term that should be found in descriptions
   // Avoid using "OR" operator which requires a text index
-  const results = await Product.searchWithLucene('high');
+  const results = await Product.search('high');
   console.log(`Found ${results.length} products matching 'high'`);
 
   // "High-speed blender" contains "high"
   expect(results.length).toBeGreaterThan(0);
 
   // Try another fallback example that won't need $text
-  const powerfulResults = await Product.searchWithLucene('powerful');
+  const powerfulResults = await Product.search('powerful');
   console.log(`Found ${powerfulResults.length} products matching 'powerful'`);
 
   // "Powerful laptop for professionals" contains "powerful"
@@ -192,6 +192,35 @@ tap.test('should explain the advantages of the integrated approach', async () =>
   expect(true).toEqual(true);
 });
 
+// Additional robustness tests
+tap.test('should search exact name using field:value', async () => {
+  const nameResults = await Product.search('name:AirPods');
+  expect(nameResults.length).toEqual(1);
+  expect(nameResults[0].name).toEqual('AirPods');
+});
+
+tap.test('should throw when searching non-searchable field', async () => {
+  let error: Error;
+  try {
+    await Product.search('price:129');
+  } catch (err) {
+    error = err as Error;
+  }
+  expect(error).toBeTruthy();
+  expect(error.message).toMatch(/not searchable/);
+});
+
+tap.test('empty query should return all products', async () => {
+  const allResults = await Product.search('');
+  expect(allResults.length).toEqual(8);
+});
+
+tap.test('should search multi-word term across fields', async () => {
+  const termResults = await Product.search('iPhone 12');
+  expect(termResults.length).toEqual(1);
+  expect(termResults[0].name).toEqual('iPhone 12');
+});
+
 tap.test('close database connection', async () => {
   await testDb.mongoDb.dropDatabase();
   await testDb.close();

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@push.rocks/smartdata',
-  version: '5.8.4',
+  version: '5.9.2',
   description: 'An advanced library for NoSQL data organization and manipulation using TypeScript with support for MongoDB, data validation, collections, and custom data types.'
 }

ts/classes.collection.ts

@@ -1,7 +1,7 @@
 import * as plugins from './plugins.js';
 import { SmartdataDb } from './classes.db.js';
 import { SmartdataDbCursor } from './classes.cursor.js';
-import { SmartDataDbDoc, type IIndexOptions } from './classes.doc.js';
+import { SmartDataDbDoc, type IIndexOptions, getSearchableFields } from './classes.doc.js';
 import { SmartdataDbWatcher } from './classes.watcher.js';
 import { CollectionFactory } from './classes.collectionfactory.js';
 
@@ -128,6 +128,8 @@ export class SmartdataCollection<T> {
   public smartdataDb: SmartdataDb;
   public uniqueIndexes: string[] = [];
   public regularIndexes: Array<{field: string, options: IIndexOptions}> = [];
+  // flag to ensure text index is created only once
+  private textIndexCreated: boolean = false;
 
   constructor(classNameArg: string, smartDataDbArg: SmartdataDb) {
     // tell the collection where it belongs
@@ -153,6 +155,16 @@ export class SmartdataCollection<T> {
         console.log(`Successfully initiated Collection ${this.collectionName}`);
       }
       this.mongoDbCollection = this.smartdataDb.mongoDb.collection(this.collectionName);
+      // Auto-create a compound text index on all searchable fields
+      const searchableFields = getSearchableFields(this.collectionName);
+      if (searchableFields.length > 0 && !this.textIndexCreated) {
+        // Build a compound text index spec
+        const indexSpec: Record<string, 'text'> = {};
+        searchableFields.forEach(f => { indexSpec[f] = 'text'; });
+        // Cast to any to satisfy TypeScript IndexSpecification typing
+        await this.mongoDbCollection.createIndex(indexSpec as any, { name: 'smartdata_text_index' });
+        this.textIndexCreated = true;
+      }
     }
   }
 

ts/classes.doc.ts

@@ -61,6 +61,10 @@ export function getSearchableFields(className: string): string[] {
   }
   return Array.from(searchableFieldsMap.get(className));
 }
+// Escape user input for safe use in MongoDB regular expressions
+function escapeForRegex(input: string): string {
+  return input.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
 
 /**
  * unique index - decorator to mark a unique index
@@ -326,57 +330,38 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
   }
 
   /**
-   * Search documents using Lucene query syntax
+   * Search documents by text or field:value syntax, with safe regex fallback
-   * @param luceneQuery Lucene query string
+   * @param query A search term or field:value expression
    * @returns Array of matching documents
    */
   public static async search<T>(
     this: plugins.tsclass.typeFest.Class<T>,
-    luceneQuery: string,
+    query: string,
   ): Promise<T[]> {
-    const filter = (this as any).createSearchFilter(luceneQuery);
+    const className = (this as any).className || this.name;
-    return await (this as any).getInstances(filter);
+    const searchableFields = getSearchableFields(className);
-  }
+    if (searchableFields.length === 0) {
-
+      throw new Error(`No searchable fields defined for class ${className}`);
-  /**
-   * Search documents using Lucene query syntax with robust error handling
-   * @param luceneQuery The Lucene query string to search with
-   * @returns Array of matching documents
-   */
-  public static async searchWithLucene<T>(
-    this: plugins.tsclass.typeFest.Class<T>,
-    luceneQuery: string,
-  ): Promise<T[]> {
-    try {
-      const className = (this as any).className || this.name;
-      const searchableFields = getSearchableFields(className);
-
-      if (searchableFields.length === 0) {
-        console.warn(
-          `No searchable fields defined for class ${className}, falling back to simple search`,
-        );
-        return (this as any).searchByTextAcrossFields(luceneQuery);
-      }
-
-      // Simple term search optimization
-      if (
-        !luceneQuery.includes(':') &&
-        !luceneQuery.includes(' AND ') &&
-        !luceneQuery.includes(' OR ') &&
-        !luceneQuery.includes(' NOT ')
-      ) {
-        return (this as any).searchByTextAcrossFields(luceneQuery);
-      }
-
-      // Try to use the Lucene-to-MongoDB conversion
-      const filter = (this as any).createSearchFilter(luceneQuery);
-      return await (this as any).getInstances(filter);
-    } catch (error) {
-      console.error(`Error in searchWithLucene: ${error.message}`);
-      return (this as any).searchByTextAcrossFields(luceneQuery);
     }
+    // field:value exact match (case-sensitive for non-regex fields)
+    const fv = query.match(/^(\w+):(.+)$/);
+    if (fv) {
+      const field = fv[1];
+      const value = fv[2];
+      if (!searchableFields.includes(field)) {
+        throw new Error(`Field '${field}' is not searchable for class ${className}`);
+      }
+      return await (this as any).getInstances({ [field]: value });
+    }
+    // safe regex across all searchable fields (case-insensitive)
+    const escaped = escapeForRegex(query);
+    const orConditions = searchableFields.map((field) => ({
+      [field]: { $regex: escaped, $options: 'i' },
+    }));
+    return await (this as any).getInstances({ $or: orConditions });
   }
 
+
   /**
    * Search by text across all searchable fields (fallback method)
    * @param searchText The text to search for in all searchable fields