feat(core): introduced lucene style search
This commit is contained in:
parent
7a08700451
commit
408b2cce4a
@ -119,6 +119,6 @@ jobs:
|
||||
run: |
|
||||
npmci node install stable
|
||||
npmci npm install
|
||||
pnpm install -g @gitzone/tsdoc
|
||||
pnpm install -g @git.zone/tsdoc
|
||||
npmci command tsdoc
|
||||
continue-on-error: true
|
||||
|
@ -1,5 +1,14 @@
|
||||
# Changelog
|
||||
|
||||
## 2025-04-06 - 5.4.0 - feat(core)
|
||||
Refactor file structure and update dependency versions
|
||||
|
||||
- Renamed files and modules from 'smartdata.classes.*' to 'classes.*' and adjusted corresponding import paths.
|
||||
- Updated dependency versions: '@push.rocks/smartmongo' to ^2.0.11, '@tsclass/tsclass' to ^8.2.0, and 'mongodb' to ^6.15.0.
|
||||
- Renamed dev dependency packages from '@gitzone/...' to '@git.zone/...' and updated '@push.rocks/tapbundle' and '@types/node'.
|
||||
- Fixed YAML workflow command: replaced 'pnpm install -g @gitzone/tsdoc' with 'pnpm install -g @git.zone/tsdoc'.
|
||||
- Added package manager configuration and pnpm-workspace.yaml for built dependencies.
|
||||
|
||||
## 2025-03-10 - 5.3.0 - feat(docs)
|
||||
Enhance documentation with updated installation instructions and comprehensive usage examples covering advanced features such as deep queries, automatic indexing, and distributed coordination.
|
||||
|
||||
|
19
package.json
19
package.json
@ -25,23 +25,23 @@
|
||||
"@push.rocks/lik": "^6.0.14",
|
||||
"@push.rocks/smartdelay": "^3.0.1",
|
||||
"@push.rocks/smartlog": "^3.0.2",
|
||||
"@push.rocks/smartmongo": "^2.0.10",
|
||||
"@push.rocks/smartmongo": "^2.0.11",
|
||||
"@push.rocks/smartpromise": "^4.0.2",
|
||||
"@push.rocks/smartrx": "^3.0.7",
|
||||
"@push.rocks/smartstring": "^4.0.15",
|
||||
"@push.rocks/smarttime": "^4.0.6",
|
||||
"@push.rocks/smartunique": "^3.0.8",
|
||||
"@push.rocks/taskbuffer": "^3.1.7",
|
||||
"@tsclass/tsclass": "^4.0.52",
|
||||
"mongodb": "^6.5.0"
|
||||
"@tsclass/tsclass": "^8.2.0",
|
||||
"mongodb": "^6.15.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@gitzone/tsbuild": "^2.1.66",
|
||||
"@gitzone/tsrun": "^1.2.44",
|
||||
"@gitzone/tstest": "^1.0.77",
|
||||
"@git.zone/tsbuild": "^2.3.2",
|
||||
"@git.zone/tsrun": "^1.2.44",
|
||||
"@git.zone/tstest": "^1.0.77",
|
||||
"@push.rocks/qenv": "^6.0.5",
|
||||
"@push.rocks/tapbundle": "^5.0.22",
|
||||
"@types/node": "^20.11.30"
|
||||
"@push.rocks/tapbundle": "^5.6.2",
|
||||
"@types/node": "^22.14.0"
|
||||
},
|
||||
"files": [
|
||||
"ts/**/*",
|
||||
@ -67,5 +67,6 @@
|
||||
"collections",
|
||||
"custom data types",
|
||||
"ODM"
|
||||
]
|
||||
],
|
||||
"packageManager": "pnpm@10.7.0+sha512.6b865ad4b62a1d9842b61d674a393903b871d9244954f652b8842c2b553c72176b278f64c463e52d40fff8aba385c235c8c9ecf5cc7de4fd78b8bb6d49633ab6"
|
||||
}
|
||||
|
13559
pnpm-lock.yaml
generated
13559
pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
4
pnpm-workspace.yaml
Normal file
4
pnpm-workspace.yaml
Normal file
@ -0,0 +1,4 @@
|
||||
onlyBuiltDependencies:
|
||||
- esbuild
|
||||
- mongodb-memory-server
|
||||
- puppeteer
|
@ -3,7 +3,7 @@ import * as smartmongo from '@push.rocks/smartmongo';
|
||||
import type * as taskbuffer from '@push.rocks/taskbuffer';
|
||||
|
||||
import * as smartdata from '../ts/index.js';
|
||||
import { SmartdataDistributedCoordinator, DistributedClass } from '../ts/smartdata.classes.distributedcoordinator.js'; // path might need adjusting
|
||||
import { SmartdataDistributedCoordinator, DistributedClass } from '../ts/classes.distributedcoordinator.js'; // path might need adjusting
|
||||
const totalInstances = 10;
|
||||
|
||||
// =======================================
|
||||
|
@ -1,7 +1,7 @@
|
||||
import { tap, expect } from '@push.rocks/tapbundle';
|
||||
import { Qenv } from '@push.rocks/qenv';
|
||||
import * as smartmongo from '@push.rocks/smartmongo';
|
||||
import { smartunique } from '../ts/smartdata.plugins.js';
|
||||
import { smartunique } from '../ts/plugins.js';
|
||||
|
||||
const testQenv = new Qenv(process.cwd(), process.cwd() + '/.nogit/');
|
||||
|
||||
|
@ -1,7 +1,7 @@
|
||||
import { tap, expect } from '@push.rocks/tapbundle';
|
||||
import { Qenv } from '@push.rocks/qenv';
|
||||
import * as smartmongo from '@push.rocks/smartmongo';
|
||||
import { smartunique } from '../ts/smartdata.plugins.js';
|
||||
import { smartunique } from '../ts/plugins.js';
|
||||
|
||||
import * as mongodb from 'mongodb';
|
||||
|
||||
|
@ -1,7 +1,7 @@
|
||||
import { tap, expect } from '@push.rocks/tapbundle';
|
||||
import { Qenv } from '@push.rocks/qenv';
|
||||
import * as smartmongo from '@push.rocks/smartmongo';
|
||||
import { smartunique } from '../ts/smartdata.plugins.js';
|
||||
import { smartunique } from '../ts/plugins.js';
|
||||
|
||||
const testQenv = new Qenv(process.cwd(), process.cwd() + '/.nogit/');
|
||||
|
||||
|
@ -1,7 +1,7 @@
|
||||
import { tap, expect } from '@push.rocks/tapbundle';
|
||||
import { Qenv } from '@push.rocks/qenv';
|
||||
import * as smartmongo from '@push.rocks/smartmongo';
|
||||
import { smartunique } from '../ts/smartdata.plugins.js';
|
||||
import { smartunique } from '../ts/plugins.js';
|
||||
|
||||
const testQenv = new Qenv(process.cwd(), process.cwd() + '/.nogit/');
|
||||
|
||||
|
@ -3,6 +3,6 @@
|
||||
*/
|
||||
export const commitinfo = {
|
||||
name: '@push.rocks/smartdata',
|
||||
version: '5.3.0',
|
||||
version: '5.4.0',
|
||||
description: 'An advanced library for NoSQL data organization and manipulation using TypeScript with support for MongoDB, data validation, collections, and custom data types.'
|
||||
}
|
||||
|
@ -1,9 +1,9 @@
|
||||
import * as plugins from './smartdata.plugins.js';
|
||||
import { SmartdataDb } from './smartdata.classes.db.js';
|
||||
import { SmartdataDbCursor } from './smartdata.classes.cursor.js';
|
||||
import { SmartDataDbDoc } from './smartdata.classes.doc.js';
|
||||
import { SmartdataDbWatcher } from './smartdata.classes.watcher.js';
|
||||
import { CollectionFactory } from './smartdata.classes.collectionfactory.js';
|
||||
import * as plugins from './plugins.js';
|
||||
import { SmartdataDb } from './classes.db.js';
|
||||
import { SmartdataDbCursor } from './classes.cursor.js';
|
||||
import { SmartDataDbDoc } from './classes.doc.js';
|
||||
import { SmartdataDbWatcher } from './classes.watcher.js';
|
||||
import { CollectionFactory } from './classes.collectionfactory.js';
|
||||
|
||||
export interface IFindOptions {
|
||||
limit?: number;
|
@ -1,6 +1,6 @@
|
||||
import * as plugins from './smartdata.plugins.js';
|
||||
import { SmartdataCollection } from './smartdata.classes.collection.js';
|
||||
import { SmartdataDb } from './smartdata.classes.db.js';
|
||||
import * as plugins from './plugins.js';
|
||||
import { SmartdataCollection } from './classes.collection.js';
|
||||
import { SmartdataDb } from './classes.db.js';
|
||||
|
||||
export class CollectionFactory {
|
||||
public collections: { [key: string]: SmartdataCollection<any> } = {};
|
@ -1,4 +1,4 @@
|
||||
import * as plugins from './smartdata.plugins.js';
|
||||
import * as plugins from './plugins.js';
|
||||
|
||||
export const getNewUniqueId = async (prefixArg?: string) => {
|
||||
return plugins.smartunique.uni(prefixArg);
|
@ -1,5 +1,5 @@
|
||||
import { SmartDataDbDoc } from './smartdata.classes.doc.js';
|
||||
import * as plugins from './smartdata.plugins.js';
|
||||
import { SmartDataDbDoc } from './classes.doc.js';
|
||||
import * as plugins from './plugins.js';
|
||||
|
||||
/**
|
||||
* a wrapper for the native mongodb cursor. Exposes better
|
@ -1,9 +1,9 @@
|
||||
import * as plugins from './smartdata.plugins.js';
|
||||
import * as plugins from './plugins.js';
|
||||
|
||||
import { SmartdataCollection } from './smartdata.classes.collection.js';
|
||||
import { EasyStore } from './smartdata.classes.easystore.js';
|
||||
import { SmartdataCollection } from './classes.collection.js';
|
||||
import { EasyStore } from './classes.easystore.js';
|
||||
|
||||
import { logger } from './smartdata.logging.js';
|
||||
import { logger } from './logging.js';
|
||||
|
||||
/**
|
||||
* interface - indicates the connection status of the db
|
@ -1,8 +1,8 @@
|
||||
import * as plugins from './smartdata.plugins.js';
|
||||
import { SmartdataDb } from './smartdata.classes.db.js';
|
||||
import { managed, setDefaultManagerForDoc } from './smartdata.classes.collection.js';
|
||||
import { SmartDataDbDoc, svDb, unI } from './smartdata.classes.doc.js';
|
||||
import { SmartdataDbWatcher } from './smartdata.classes.watcher.js';
|
||||
import * as plugins from './plugins.js';
|
||||
import { SmartdataDb } from './classes.db.js';
|
||||
import { managed, setDefaultManagerForDoc } from './classes.collection.js';
|
||||
import { SmartDataDbDoc, svDb, unI } from './classes.doc.js';
|
||||
import { SmartdataDbWatcher } from './classes.watcher.js';
|
||||
|
||||
@managed()
|
||||
export class DistributedClass extends SmartDataDbDoc<DistributedClass, DistributedClass> {
|
@ -1,12 +1,16 @@
|
||||
import * as plugins from './smartdata.plugins.js';
|
||||
import * as plugins from './plugins.js';
|
||||
|
||||
import { SmartdataDb } from './smartdata.classes.db.js';
|
||||
import { SmartdataDbCursor } from './smartdata.classes.cursor.js';
|
||||
import { type IManager, SmartdataCollection } from './smartdata.classes.collection.js';
|
||||
import { SmartdataDbWatcher } from './smartdata.classes.watcher.js';
|
||||
import { SmartdataDb } from './classes.db.js';
|
||||
import { SmartdataDbCursor } from './classes.cursor.js';
|
||||
import { type IManager, SmartdataCollection } from './classes.collection.js';
|
||||
import { SmartdataDbWatcher } from './classes.watcher.js';
|
||||
import { SmartdataLuceneAdapter } from './classes.lucene.adapter.js';
|
||||
|
||||
export type TDocCreation = 'db' | 'new' | 'mixed';
|
||||
|
||||
// Set of searchable fields for each class
|
||||
const searchableFieldsMap = new Map<string, Set<string>>();
|
||||
|
||||
export function globalSvDb() {
|
||||
return (target: SmartDataDbDoc<unknown, unknown>, key: string) => {
|
||||
console.log(`called svDb() on >${target.constructor.name}.${key}<`);
|
||||
@ -30,6 +34,34 @@ export function svDb() {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* searchable - marks a property as searchable with Lucene query syntax
|
||||
*/
|
||||
export function searchable() {
|
||||
return (target: SmartDataDbDoc<unknown, unknown>, key: string) => {
|
||||
console.log(`called searchable() on >${target.constructor.name}.${key}<`);
|
||||
|
||||
// Initialize the set for this class if it doesn't exist
|
||||
const className = target.constructor.name;
|
||||
if (!searchableFieldsMap.has(className)) {
|
||||
searchableFieldsMap.set(className, new Set<string>());
|
||||
}
|
||||
|
||||
// Add the property to the searchable fields set
|
||||
searchableFieldsMap.get(className).add(key);
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get searchable fields for a class
|
||||
*/
|
||||
export function getSearchableFields(className: string): string[] {
|
||||
if (!searchableFieldsMap.has(className)) {
|
||||
return [];
|
||||
}
|
||||
return Array.from(searchableFieldsMap.get(className));
|
||||
}
|
||||
|
||||
/**
|
||||
* unique index - decorator to mark a unique index
|
||||
*/
|
||||
@ -207,6 +239,39 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
||||
return await collection.getCount(filterArg);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a MongoDB filter from a Lucene query string
|
||||
* @param luceneQuery Lucene query string
|
||||
* @returns MongoDB query object
|
||||
*/
|
||||
public static createSearchFilter<T>(
|
||||
this: plugins.tsclass.typeFest.Class<T>,
|
||||
luceneQuery: string
|
||||
): any {
|
||||
const className = (this as any).className || this.name;
|
||||
const searchableFields = getSearchableFields(className);
|
||||
|
||||
if (searchableFields.length === 0) {
|
||||
throw new Error(`No searchable fields defined for class ${className}`);
|
||||
}
|
||||
|
||||
const adapter = new SmartdataLuceneAdapter(searchableFields);
|
||||
return adapter.convert(luceneQuery);
|
||||
}
|
||||
|
||||
/**
|
||||
* Search documents using Lucene query syntax
|
||||
* @param luceneQuery Lucene query string
|
||||
* @returns Array of matching documents
|
||||
*/
|
||||
public static async search<T>(
|
||||
this: plugins.tsclass.typeFest.Class<T>,
|
||||
luceneQuery: string
|
||||
): Promise<T[]> {
|
||||
const filter = (this as any).createSearchFilter(luceneQuery);
|
||||
return await (this as any).getInstances(filter);
|
||||
}
|
||||
|
||||
// INSTANCE
|
||||
|
||||
/**
|
||||
@ -341,4 +406,4 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
||||
}
|
||||
return identifiableObject;
|
||||
}
|
||||
}
|
||||
}
|
@ -1,7 +1,7 @@
|
||||
import * as plugins from './smartdata.plugins.js';
|
||||
import { Collection } from './smartdata.classes.collection.js';
|
||||
import { SmartdataDb } from './smartdata.classes.db.js';
|
||||
import { SmartDataDbDoc, svDb, unI } from './smartdata.classes.doc.js';
|
||||
import * as plugins from './plugins.js';
|
||||
import { Collection } from './classes.collection.js';
|
||||
import { SmartdataDb } from './classes.db.js';
|
||||
import { SmartDataDbDoc, svDb, unI } from './classes.doc.js';
|
||||
|
||||
/**
|
||||
* EasyStore allows the storage of easy objects. It also allows easy sharing of the object between different instances
|
643
ts/classes.lucene.adapter.ts
Normal file
643
ts/classes.lucene.adapter.ts
Normal file
@ -0,0 +1,643 @@
|
||||
/**
|
||||
* Lucene to MongoDB query adapter for SmartData
|
||||
*/
|
||||
import * as plugins from './plugins.js';
|
||||
|
||||
// Types
|
||||
type NodeType = 'TERM' | 'PHRASE' | 'FIELD' | 'AND' | 'OR' | 'NOT' | 'RANGE' | 'WILDCARD' | 'FUZZY' | 'GROUP';
|
||||
|
||||
interface QueryNode {
|
||||
type: NodeType;
|
||||
}
|
||||
|
||||
interface TermNode extends QueryNode {
|
||||
type: 'TERM';
|
||||
value: string;
|
||||
boost?: number;
|
||||
}
|
||||
|
||||
interface PhraseNode extends QueryNode {
|
||||
type: 'PHRASE';
|
||||
value: string;
|
||||
proximity?: number;
|
||||
}
|
||||
|
||||
interface FieldNode extends QueryNode {
|
||||
type: 'FIELD';
|
||||
field: string;
|
||||
value: AnyQueryNode;
|
||||
}
|
||||
|
||||
interface BooleanNode extends QueryNode {
|
||||
type: 'AND' | 'OR' | 'NOT';
|
||||
left: AnyQueryNode;
|
||||
right: AnyQueryNode;
|
||||
}
|
||||
|
||||
interface RangeNode extends QueryNode {
|
||||
type: 'RANGE';
|
||||
field: string;
|
||||
lower: string;
|
||||
upper: string;
|
||||
includeLower: boolean;
|
||||
includeUpper: boolean;
|
||||
}
|
||||
|
||||
interface WildcardNode extends QueryNode {
|
||||
type: 'WILDCARD';
|
||||
value: string;
|
||||
}
|
||||
|
||||
interface FuzzyNode extends QueryNode {
|
||||
type: 'FUZZY';
|
||||
value: string;
|
||||
maxEdits: number;
|
||||
}
|
||||
|
||||
interface GroupNode extends QueryNode {
|
||||
type: 'GROUP';
|
||||
value: AnyQueryNode;
|
||||
}
|
||||
|
||||
type AnyQueryNode = TermNode | PhraseNode | FieldNode | BooleanNode | RangeNode | WildcardNode | FuzzyNode | GroupNode;
|
||||
|
||||
/**
|
||||
* Lucene query parser
|
||||
*/
|
||||
export class LuceneParser {
|
||||
private pos: number = 0;
|
||||
private input: string = '';
|
||||
private tokens: string[] = [];
|
||||
|
||||
constructor() {}
|
||||
|
||||
/**
|
||||
* Parse a Lucene query string into an AST
|
||||
*/
|
||||
parse(query: string): AnyQueryNode {
|
||||
this.input = query.trim();
|
||||
this.pos = 0;
|
||||
this.tokens = this.tokenize(this.input);
|
||||
|
||||
return this.parseQuery();
|
||||
}
|
||||
|
||||
/**
|
||||
* Tokenize the input string into tokens
|
||||
*/
|
||||
private tokenize(input: string): string[] {
|
||||
const specialChars = /[()\[\]{}"~^:]/;
|
||||
const operators = /AND|OR|NOT|TO/;
|
||||
|
||||
let tokens: string[] = [];
|
||||
let current = '';
|
||||
let inQuote = false;
|
||||
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
const char = input[i];
|
||||
|
||||
// Handle quoted strings
|
||||
if (char === '"') {
|
||||
if (inQuote) {
|
||||
tokens.push(current + char);
|
||||
current = '';
|
||||
inQuote = false;
|
||||
} else {
|
||||
if (current) tokens.push(current);
|
||||
current = char;
|
||||
inQuote = true;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (inQuote) {
|
||||
current += char;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Handle whitespace
|
||||
if (char === ' ' || char === '\t' || char === '\n') {
|
||||
if (current) {
|
||||
tokens.push(current);
|
||||
current = '';
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Handle special characters
|
||||
if (specialChars.test(char)) {
|
||||
if (current) {
|
||||
tokens.push(current);
|
||||
current = '';
|
||||
}
|
||||
tokens.push(char);
|
||||
continue;
|
||||
}
|
||||
|
||||
current += char;
|
||||
|
||||
// Check if current is an operator
|
||||
if (operators.test(current) &&
|
||||
(i + 1 === input.length || /\s/.test(input[i + 1]))) {
|
||||
tokens.push(current);
|
||||
current = '';
|
||||
}
|
||||
}
|
||||
|
||||
if (current) tokens.push(current);
|
||||
|
||||
return tokens;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the main query expression
|
||||
*/
|
||||
private parseQuery(): AnyQueryNode {
|
||||
const left = this.parseBooleanOperand();
|
||||
|
||||
if (this.pos < this.tokens.length) {
|
||||
const token = this.tokens[this.pos];
|
||||
|
||||
if (token === 'AND' || token === 'OR') {
|
||||
this.pos++;
|
||||
const right = this.parseQuery();
|
||||
return {
|
||||
type: token as 'AND' | 'OR',
|
||||
left,
|
||||
right
|
||||
};
|
||||
} else if (token === 'NOT' || token === '-') {
|
||||
this.pos++;
|
||||
const right = this.parseQuery();
|
||||
return {
|
||||
type: 'NOT',
|
||||
left,
|
||||
right
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return left;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse boolean operands (terms, phrases, fields, groups)
|
||||
*/
|
||||
private parseBooleanOperand(): AnyQueryNode {
|
||||
if (this.pos >= this.tokens.length) {
|
||||
throw new Error('Unexpected end of input');
|
||||
}
|
||||
|
||||
const token = this.tokens[this.pos];
|
||||
|
||||
// Handle grouping with parentheses
|
||||
if (token === '(') {
|
||||
this.pos++;
|
||||
const group = this.parseQuery();
|
||||
|
||||
if (this.pos < this.tokens.length && this.tokens[this.pos] === ')') {
|
||||
this.pos++;
|
||||
return { type: 'GROUP', value: group } as GroupNode;
|
||||
} else {
|
||||
throw new Error('Unclosed group');
|
||||
}
|
||||
}
|
||||
|
||||
// Handle fields (field:value)
|
||||
if (this.pos + 1 < this.tokens.length && this.tokens[this.pos + 1] === ':') {
|
||||
const field = token;
|
||||
this.pos += 2; // Skip field and colon
|
||||
|
||||
if (this.pos < this.tokens.length) {
|
||||
const value = this.parseBooleanOperand();
|
||||
return { type: 'FIELD', field, value } as FieldNode;
|
||||
} else {
|
||||
throw new Error('Expected value after field');
|
||||
}
|
||||
}
|
||||
|
||||
// Handle range queries
|
||||
if (token === '[' || token === '{') {
|
||||
return this.parseRange();
|
||||
}
|
||||
|
||||
// Handle phrases ("term term")
|
||||
if (token.startsWith('"') && token.endsWith('"')) {
|
||||
const phrase = token.slice(1, -1);
|
||||
this.pos++;
|
||||
|
||||
// Check for proximity operator
|
||||
let proximity: number | undefined;
|
||||
if (this.pos < this.tokens.length && this.tokens[this.pos] === '~') {
|
||||
this.pos++;
|
||||
if (this.pos < this.tokens.length && /^\d+$/.test(this.tokens[this.pos])) {
|
||||
proximity = parseInt(this.tokens[this.pos], 10);
|
||||
this.pos++;
|
||||
} else {
|
||||
throw new Error('Expected number after proximity operator');
|
||||
}
|
||||
}
|
||||
|
||||
return { type: 'PHRASE', value: phrase, proximity } as PhraseNode;
|
||||
}
|
||||
|
||||
// Handle wildcards
|
||||
if (token.includes('*') || token.includes('?')) {
|
||||
this.pos++;
|
||||
return { type: 'WILDCARD', value: token } as WildcardNode;
|
||||
}
|
||||
|
||||
// Handle fuzzy searches
|
||||
if (this.pos + 1 < this.tokens.length && this.tokens[this.pos + 1] === '~') {
|
||||
const term = token;
|
||||
this.pos += 2; // Skip term and tilde
|
||||
|
||||
let maxEdits = 2; // Default
|
||||
if (this.pos < this.tokens.length && /^\d+$/.test(this.tokens[this.pos])) {
|
||||
maxEdits = parseInt(this.tokens[this.pos], 10);
|
||||
this.pos++;
|
||||
}
|
||||
|
||||
return { type: 'FUZZY', value: term, maxEdits } as FuzzyNode;
|
||||
}
|
||||
|
||||
// Simple term
|
||||
this.pos++;
|
||||
return { type: 'TERM', value: token } as TermNode;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse range queries
|
||||
*/
|
||||
private parseRange(): RangeNode {
|
||||
const includeLower = this.tokens[this.pos] === '[';
|
||||
const includeUpper = this.tokens[this.pos + 4] === ']';
|
||||
|
||||
this.pos++; // Skip open bracket
|
||||
|
||||
if (this.pos + 4 >= this.tokens.length) {
|
||||
throw new Error('Invalid range query syntax');
|
||||
}
|
||||
|
||||
const lower = this.tokens[this.pos];
|
||||
this.pos++;
|
||||
|
||||
if (this.tokens[this.pos] !== 'TO') {
|
||||
throw new Error('Expected TO in range query');
|
||||
}
|
||||
this.pos++;
|
||||
|
||||
const upper = this.tokens[this.pos];
|
||||
this.pos++;
|
||||
|
||||
if (this.tokens[this.pos] !== (includeLower ? ']' : '}')) {
|
||||
throw new Error('Invalid range query closing bracket');
|
||||
}
|
||||
this.pos++;
|
||||
|
||||
// For simplicity, assuming the field is handled separately
|
||||
return {
|
||||
type: 'RANGE',
|
||||
field: '', // This will be filled by the field node
|
||||
lower,
|
||||
upper,
|
||||
includeLower,
|
||||
includeUpper
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Transformer for Lucene AST to MongoDB query
|
||||
*/
|
||||
export class LuceneToMongoTransformer {
|
||||
constructor() {}
|
||||
|
||||
/**
|
||||
* Transform a Lucene AST node to a MongoDB query
|
||||
*/
|
||||
transform(node: AnyQueryNode, searchFields?: string[]): any {
|
||||
switch (node.type) {
|
||||
case 'TERM':
|
||||
return this.transformTerm(node, searchFields);
|
||||
case 'PHRASE':
|
||||
return this.transformPhrase(node, searchFields);
|
||||
case 'FIELD':
|
||||
return this.transformField(node);
|
||||
case 'AND':
|
||||
return this.transformAnd(node);
|
||||
case 'OR':
|
||||
return this.transformOr(node);
|
||||
case 'NOT':
|
||||
return this.transformNot(node);
|
||||
case 'RANGE':
|
||||
return this.transformRange(node);
|
||||
case 'WILDCARD':
|
||||
return this.transformWildcard(node, searchFields);
|
||||
case 'FUZZY':
|
||||
return this.transformFuzzy(node, searchFields);
|
||||
case 'GROUP':
|
||||
return this.transform(node.value, searchFields);
|
||||
default:
|
||||
throw new Error(`Unsupported node type: ${(node as any).type}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Transform a term to MongoDB query
|
||||
*/
|
||||
private transformTerm(node: TermNode, searchFields?: string[]): any {
|
||||
// If specific fields are provided, search across those fields
|
||||
if (searchFields && searchFields.length > 0) {
|
||||
// Create an $or query to search across multiple fields
|
||||
return {
|
||||
$or: searchFields.map(field => ({
|
||||
[field]: { $regex: node.value, $options: 'i' }
|
||||
}))
|
||||
};
|
||||
}
|
||||
|
||||
// Otherwise, use text search (requires a text index on desired fields)
|
||||
return { $text: { $search: node.value } };
|
||||
}
|
||||
|
||||
/**
|
||||
* Transform a phrase to MongoDB query
|
||||
*/
|
||||
private transformPhrase(node: PhraseNode, searchFields?: string[]): any {
|
||||
// If specific fields are provided, search phrase across those fields
|
||||
if (searchFields && searchFields.length > 0) {
|
||||
// Create an $or query to search phrase across multiple fields
|
||||
return {
|
||||
$or: searchFields.map(field => ({
|
||||
[field]: { $regex: `${node.value.replace(/\s+/g, '\\s+')}`, $options: 'i' }
|
||||
}))
|
||||
};
|
||||
}
|
||||
|
||||
// For phrases, we use a regex to ensure exact matches
|
||||
return { $text: { $search: `"${node.value}"` } };
|
||||
}
|
||||
|
||||
/**
|
||||
* Transform a field query to MongoDB query
|
||||
*/
|
||||
private transformField(node: FieldNode): any {
|
||||
// Handle special case for range queries on fields
|
||||
if (node.value.type === 'RANGE') {
|
||||
const rangeNode = node.value as RangeNode;
|
||||
rangeNode.field = node.field;
|
||||
return this.transformRange(rangeNode);
|
||||
}
|
||||
|
||||
// Handle special case for wildcards on fields
|
||||
if (node.value.type === 'WILDCARD') {
|
||||
return {
|
||||
[node.field]: {
|
||||
$regex: this.luceneWildcardToRegex((node.value as WildcardNode).value),
|
||||
$options: 'i'
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// Handle special case for fuzzy searches on fields
|
||||
if (node.value.type === 'FUZZY') {
|
||||
return {
|
||||
[node.field]: {
|
||||
$regex: this.createFuzzyRegex((node.value as FuzzyNode).value),
|
||||
$options: 'i'
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// Special case for exact term matches on fields
|
||||
if (node.value.type === 'TERM') {
|
||||
return { [node.field]: (node.value as TermNode).value };
|
||||
}
|
||||
|
||||
// Special case for phrase matches on fields
|
||||
if (node.value.type === 'PHRASE') {
|
||||
return {
|
||||
[node.field]: {
|
||||
$regex: `^${(node.value as PhraseNode).value}$`,
|
||||
$options: 'i'
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// For other cases, we'll transform the value and apply it to the field
|
||||
const transformedValue = this.transform(node.value);
|
||||
|
||||
// If the transformed value uses $text, we need to adapt it for the field
|
||||
if (transformedValue.$text) {
|
||||
return { [node.field]: transformedValue.$text.$search };
|
||||
}
|
||||
|
||||
return { [node.field]: transformedValue };
|
||||
}
|
||||
|
||||
/**
|
||||
* Transform AND operator to MongoDB query
|
||||
*/
|
||||
private transformAnd(node: BooleanNode): any {
|
||||
return { $and: [this.transform(node.left), this.transform(node.right)] };
|
||||
}
|
||||
|
||||
/**
|
||||
* Transform OR operator to MongoDB query
|
||||
*/
|
||||
private transformOr(node: BooleanNode): any {
|
||||
return { $or: [this.transform(node.left), this.transform(node.right)] };
|
||||
}
|
||||
|
||||
/**
|
||||
* Transform NOT operator to MongoDB query
|
||||
*/
|
||||
private transformNot(node: BooleanNode): any {
|
||||
const leftQuery = this.transform(node.left);
|
||||
const rightQuery = this.transform(node.right);
|
||||
|
||||
// Create a query that includes left but excludes right
|
||||
if (rightQuery.$text) {
|
||||
// Text searches need special handling for negation
|
||||
return {
|
||||
$and: [
|
||||
leftQuery,
|
||||
{ $not: rightQuery }
|
||||
]
|
||||
};
|
||||
} else {
|
||||
// For other queries, we can use $not directly
|
||||
return {
|
||||
$and: [
|
||||
leftQuery,
|
||||
{ $not: rightQuery }
|
||||
]
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Transform range query to MongoDB query
|
||||
*/
|
||||
private transformRange(node: RangeNode): any {
|
||||
const range: any = {};
|
||||
|
||||
if (node.lower !== '*') {
|
||||
range[node.includeLower ? '$gte' : '$gt'] = this.parseValue(node.lower);
|
||||
}
|
||||
|
||||
if (node.upper !== '*') {
|
||||
range[node.includeUpper ? '$lte' : '$lt'] = this.parseValue(node.upper);
|
||||
}
|
||||
|
||||
return { [node.field]: range };
|
||||
}
|
||||
|
||||
/**
|
||||
* Transform wildcard query to MongoDB query
|
||||
*/
|
||||
private transformWildcard(node: WildcardNode, searchFields?: string[]): any {
|
||||
// Convert Lucene wildcards to MongoDB regex
|
||||
const regex = this.luceneWildcardToRegex(node.value);
|
||||
|
||||
// If specific fields are provided, search wildcard across those fields
|
||||
if (searchFields && searchFields.length > 0) {
|
||||
return {
|
||||
$or: searchFields.map(field => ({
|
||||
[field]: { $regex: regex, $options: 'i' }
|
||||
}))
|
||||
};
|
||||
}
|
||||
|
||||
// By default, apply to all text fields using $text search
|
||||
return { $regex: regex, $options: 'i' };
|
||||
}
|
||||
|
||||
/**
|
||||
* Transform fuzzy query to MongoDB query
|
||||
*/
|
||||
private transformFuzzy(node: FuzzyNode, searchFields?: string[]): any {
|
||||
// MongoDB doesn't have built-in fuzzy search
|
||||
// This is a very basic approach using regex
|
||||
const regex = this.createFuzzyRegex(node.value);
|
||||
|
||||
// If specific fields are provided, search fuzzy term across those fields
|
||||
if (searchFields && searchFields.length > 0) {
|
||||
return {
|
||||
$or: searchFields.map(field => ({
|
||||
[field]: { $regex: regex, $options: 'i' }
|
||||
}))
|
||||
};
|
||||
}
|
||||
|
||||
return { $regex: regex, $options: 'i' };
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert Lucene wildcards to MongoDB regex patterns
|
||||
*/
|
||||
private luceneWildcardToRegex(wildcardPattern: string): string {
|
||||
// Replace Lucene wildcards with regex equivalents
|
||||
// * => .*
|
||||
// ? => .
|
||||
// Also escape regex special chars
|
||||
return wildcardPattern
|
||||
.replace(/([.+^${}()|\\])/g, '\\$1') // Escape regex special chars
|
||||
.replace(/\*/g, '.*')
|
||||
.replace(/\?/g, '.');
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a simplified fuzzy search regex
|
||||
*/
|
||||
private createFuzzyRegex(term: string): string {
|
||||
// For a very simple approach, we allow some characters to be optional
|
||||
let regex = '';
|
||||
for (let i = 0; i < term.length; i++) {
|
||||
// Make every other character optional (simplified fuzzy)
|
||||
if (i % 2 === 1) {
|
||||
regex += term[i] + '?';
|
||||
} else {
|
||||
regex += term[i];
|
||||
}
|
||||
}
|
||||
return regex;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse string values to appropriate types (numbers, dates, etc.)
|
||||
*/
|
||||
private parseValue(value: string): any {
|
||||
// Try to parse as number
|
||||
if (/^-?\d+$/.test(value)) {
|
||||
return parseInt(value, 10);
|
||||
}
|
||||
|
||||
if (/^-?\d+\.\d+$/.test(value)) {
|
||||
return parseFloat(value);
|
||||
}
|
||||
|
||||
// Try to parse as date (simplified)
|
||||
const date = new Date(value);
|
||||
if (!isNaN(date.getTime())) {
|
||||
return date;
|
||||
}
|
||||
|
||||
// Default to string
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Main adapter class
|
||||
*/
|
||||
export class SmartdataLuceneAdapter {
|
||||
private parser: LuceneParser;
|
||||
private transformer: LuceneToMongoTransformer;
|
||||
private defaultSearchFields: string[] = [];
|
||||
|
||||
/**
|
||||
* @param defaultSearchFields - Optional array of field names to search across when no field is specified
|
||||
*/
|
||||
constructor(defaultSearchFields?: string[]) {
|
||||
this.parser = new LuceneParser();
|
||||
this.transformer = new LuceneToMongoTransformer();
|
||||
if (defaultSearchFields) {
|
||||
this.defaultSearchFields = defaultSearchFields;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a Lucene query string to a MongoDB query object
|
||||
* @param luceneQuery - The Lucene query string to convert
|
||||
* @param searchFields - Optional array of field names to search across (overrides defaultSearchFields)
|
||||
*/
|
||||
convert(luceneQuery: string, searchFields?: string[]): any {
|
||||
try {
|
||||
// Parse the Lucene query into an AST
|
||||
const ast = this.parser.parse(luceneQuery);
|
||||
|
||||
// Use provided searchFields, fall back to defaultSearchFields
|
||||
const fieldsToSearch = searchFields || this.defaultSearchFields;
|
||||
|
||||
// Transform the AST to a MongoDB query
|
||||
return this.transformWithFields(ast, fieldsToSearch);
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to convert Lucene query: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper method to transform the AST with field information
|
||||
*/
|
||||
private transformWithFields(node: AnyQueryNode, searchFields: string[]): any {
|
||||
// For term nodes without a specific field, apply the search fields
|
||||
if (node.type === 'TERM') {
|
||||
return this.transformer.transform(node, searchFields);
|
||||
}
|
||||
|
||||
// For other node types, use the standard transformation
|
||||
return this.transformer.transform(node);
|
||||
}
|
||||
}
|
@ -1,5 +1,5 @@
|
||||
import { SmartDataDbDoc } from './smartdata.classes.doc.js';
|
||||
import * as plugins from './smartdata.plugins.js';
|
||||
import { SmartDataDbDoc } from './classes.doc.js';
|
||||
import * as plugins from './plugins.js';
|
||||
|
||||
/**
|
||||
* a wrapper for the native mongodb cursor. Exposes better
|
16
ts/index.ts
16
ts/index.ts
@ -1,14 +1,14 @@
|
||||
export * from './smartdata.classes.db.js';
|
||||
export * from './smartdata.classes.collection.js';
|
||||
export * from './smartdata.classes.doc.js';
|
||||
export * from './smartdata.classes.easystore.js';
|
||||
export * from './smartdata.classes.cursor.js';
|
||||
export * from './classes.db.js';
|
||||
export * from './classes.collection.js';
|
||||
export * from './classes.doc.js';
|
||||
export * from './classes.easystore.js';
|
||||
export * from './classes.cursor.js';
|
||||
|
||||
import * as convenience from './smartadata.convenience.js';
|
||||
import * as convenience from './classes.convenience.js';
|
||||
|
||||
export { convenience };
|
||||
|
||||
// to be removed with the next breaking update
|
||||
import type * as plugins from './smartdata.plugins.js';
|
||||
import type * as plugins from './plugins.js';
|
||||
type IMongoDescriptor = plugins.tsclass.database.IMongoDescriptor;
|
||||
export type { IMongoDescriptor };
|
||||
export type { IMongoDescriptor };
|
@ -1,3 +1,3 @@
|
||||
import * as plugins from './smartdata.plugins.js';
|
||||
import * as plugins from './plugins.js';
|
||||
|
||||
export const logger = new plugins.smartlog.ConsoleLog();
|
Loading…
x
Reference in New Issue
Block a user