diff --git a/changelog.md b/changelog.md index 8dd5498..273aea9 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,13 @@ # Changelog +## 2025-04-22 - 5.13.0 - feat(search) +Improve search query handling and update documentation + +- Added 'codex.md' providing a high-level project overview and detailed search API documentation. +- Enhanced search parsing in SmartDataDbDoc to support combined free-term and quoted field phrase queries. +- Introduced a new fallback branch in the search method to handle free term with quoted field input. +- Updated tests in test/test.search.ts to cover new combined query scenarios and ensure robust behavior. + ## 2025-04-22 - 5.12.2 - fix(search) Fix handling of quoted wildcard patterns in field-specific search queries and add tests for location-based wildcard phrase searches diff --git a/codex.md b/codex.md new file mode 100644 index 0000000..c95454b --- /dev/null +++ b/codex.md @@ -0,0 +1,77 @@ +# SmartData Project Overview + +This document provides a high-level overview of the SmartData library (`@push.rocks/smartdata`), its architecture, core components, and key features—including recent enhancements to the search API. + +## 1. Project Purpose +- A TypeScript‑first wrapper around MongoDB that supplies: + - Strongly‑typed document & collection classes + - Decorator‑based schema definition (no external schema files) + - Advanced search capabilities with Lucene‑style queries + - Built‑in support for real‑time data sync, distributed coordination, and key‑value EasyStore + +## 2. Core Concepts & Components +- **SmartDataDb**: Manages the MongoDB connection, pooling, and initialization of collections. +- **SmartDataDbDoc**: Base class for all document models; provides CRUD, upsert, and cursor APIs. +- **Decorators**: + - `@Collection`: Associates a class with a MongoDB collection + - `@svDb()`: Marks a field as persisted to the DB + - `@unI()`: Marks a field as a unique index + - `@index()`: Adds a regular index + - `@searchable()`: Marks a field for inclusion in text searches or regex queries +- **SmartdataCollection**: Wraps a MongoDB collection; auto‑creates indexes based on decorators. +- **Lucene Adapter**: Parses a Lucene query string into an AST and transforms it to a MongoDB filter object. +- **EasyStore**: A simple, schema‑less key‑value store built on top of MongoDB for sharing ephemeral data. +- **Distributed Coordinator**: Leader election and task‑distribution API for building resilient, multi‑instance systems. +- **Watcher**: Listens to change streams for real‑time updates and integrates with RxJS. + +## 3. Search API +SmartData provides a unified `.search(query[, opts])` method on all models with `@searchable()` fields: + +- **Supported Syntax**: + 1. Exact field:value (e.g. `field:Value`) + 2. Quoted phrases (e.g. `"exact phrase"` or `'exact phrase'`) + 3. Wildcards: `*` (zero or more chars) and `?` (single char) + 4. Boolean operators: `AND`, `OR`, `NOT` + 5. Grouping: parenthesis `(A OR B) AND C` + 6. Range queries: `[num TO num]`, `{num TO num}` + 7. Multi‑term unquoted: terms AND’d across all searchable fields + 8. Empty query returns all documents + +- **Fallback Mechanisms**: + 1. Text index based `$text` search (if supported) + 2. Field‑scoped and multi‑field regex queries + 3. In‑memory filtering for complex or unsupported cases + +### New Security & Extensibility Hooks +The `.search(query, opts?)` signature now accepts a `SearchOptions` object: +```ts +interface SearchOptions { + filter?: Record; // Additional MongoDB filter AND‑merged + validate?: (doc: T) => boolean; // Post‑fetch hook to drop results +} +``` +- **filter**: Enforces mandatory constraints (e.g. multi‑tenant isolation) directly in the Mongo query. +- **validate**: An async function that runs after fetching; return `false` to exclude a document. + +## 4. Testing Strategy +- Unit tests in `test/test.search.ts` cover basic search functionality and new options: + - Exact, wildcard, phrase, boolean and grouping cases + - Implicit AND and mixed free‑term + field searches + - Edge cases (non‑searchable fields, quoted wildcards, no matches) + - `filter` and `validate` tests ensure security hooks work as intended +- Advanced search scenarios are covered in `test/test.search.advanced.ts`. + +## 5. Usage Example +```ts +// Basic search +const prods = await Product.search('wireless earbuds'); + +// Scoped search (only your organization’s items) +const myItems = await Product.search('book', { filter: { ownerId } }); + +// Post‑search validation (only cheap items) +const cheapItems = await Product.search('', { validate: p => p.price < 50 }); +``` + +--- +Last updated: 2025-04-22 \ No newline at end of file diff --git a/test/test.search.ts b/test/test.search.ts index 1401e29..1fa5518 100644 --- a/test/test.search.ts +++ b/test/test.search.ts @@ -323,6 +323,45 @@ tap.test('should search unquoted wildcard field', async () => { expect(names).toEqual(['Frankfurt am Main', 'Frankfurt am Oder']); }); +// Combined free-term + field phrase/wildcard tests +let CombinedDoc: any; +tap.test('setup combined docs for free-term and location tests', async () => { + @smartdata.Collection(() => testDb) + class CD extends smartdata.SmartDataDbDoc { + @smartdata.unI() public id: string = smartunique.shortId(); + @smartdata.svDb() @searchable() public name: string; + @smartdata.svDb() @searchable() public location: string; + constructor(name: string, location: string) { super(); this.name = name; this.location = location; } + } + CombinedDoc = CD; + const docs = [ + new CombinedDoc('TypeScript', 'Berlin'), + new CombinedDoc('TypeScript', 'Frankfurt am Main'), + new CombinedDoc('TypeScript', 'Frankfurt am Oder'), + new CombinedDoc('JavaScript', 'Berlin'), + ]; + for (const d of docs) await d.save(); +}); +tap.test('should search free term and exact quoted field phrase', async () => { + const res = await CombinedDoc.search('TypeScript location:"Berlin"'); + expect(res.length).toEqual(1); + expect(res[0].location).toEqual('Berlin'); +}); +tap.test('should not match free term with non-matching quoted field phrase', async () => { + const res = await CombinedDoc.search('TypeScript location:"Frankfurt d"'); + expect(res.length).toEqual(0); +}); +tap.test('should search free term with quoted wildcard field phrase', async () => { + const res = await CombinedDoc.search('TypeScript location:"Frankfurt am *"'); + const locs = res.map((r: any) => r.location).sort(); + expect(locs).toEqual(['Frankfurt am Main', 'Frankfurt am Oder']); +}); +// Quoted exact field phrase without wildcard should return no matches if no exact match +tap.test('should not match location:"Frankfurt d"', async () => { + const results = await (LocationDoc as any).search('location:"Frankfurt d"'); + expect(results.length).toEqual(0); +}); + // Combined free-term and field wildcard tests tap.test('should combine free term and wildcard field search', async () => { const results = await Product.search('book category:Book*'); diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index ce625fc..b588f91 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@push.rocks/smartdata', - version: '5.12.2', + version: '5.13.0', description: 'An advanced library for NoSQL data organization and manipulation using TypeScript with support for MongoDB, data validation, collections, and custom data types.' } diff --git a/ts/classes.doc.ts b/ts/classes.doc.ts index 886dfeb..1296928 100644 --- a/ts/classes.doc.ts +++ b/ts/classes.doc.ts @@ -452,6 +452,30 @@ export class SmartDataDbDoc ({ [f]: { $regex: freeEsc, $options: 'i' } })) }; + // field condition: exact match or wildcard pattern + let fieldCond; + if (phrase.includes('*') || phrase.includes('?')) { + const escaped = phrase.replace(/([.+^${}()|[\\]\\])/g, '\\$1'); + const pattern = escaped.replace(/\*/g, '.*').replace(/\?/g, '.'); + fieldCond = { [field]: { $regex: pattern, $options: 'i' } }; + } else { + fieldCond = { [field]: phrase }; + } + return await (this as any).execQuery({ $and: [freeCond, fieldCond] }, opts); + } // detect advanced Lucene syntax: field:value, wildcards, boolean, grouping const luceneSyntax = /(\w+:[^\s]+)|\*|\?|\bAND\b|\bOR\b|\bNOT\b|\(|\)/; if (luceneSyntax.test(q)) {