push.rocks/smartdata
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(search): Enhance search functionality with robust Lucene query transformation and reliable fallback mechanisms

Browse Source
This commit is contained in:
Philipp Kunz
2025-04-06 18:14:46 +00:00
parent 0f61bdc455
commit cad2decf59
6 changed files with 521 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ts/00_commitinfo_data.ts
View File

@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@push.rocks/smartdata',
version: '5.4.0',
version: '5.5.0',
description: 'An advanced library for NoSQL data organization and manipulation using TypeScript with support for MongoDB, data validation, collections, and custom data types.'
}

94
ts/classes.doc.ts
View File

@@ -84,6 +84,15 @@ export function unI() {
}
export const convertFilterForMongoDb = (filterArg: { [key: string]: any }) => {
// Special case: detect MongoDB operators and pass them through directly
const topLevelOperators = ['$and', '$or', '$nor', '$not', '$text', '$where', '$regex'];
for (const key of Object.keys(filterArg)) {
if (topLevelOperators.includes(key)) {
return filterArg; // Return the filter as-is for MongoDB operators
}
}
// Original conversion logic for non-MongoDB query objects
const convertedFilter: { [key: string]: any } = {};
const convertFilterArgument = (keyPathArg2: string, filterArg2: any) => {
@@ -272,6 +281,91 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
return await (this as any).getInstances(filter);
}
/**
* Search documents using Lucene query syntax with robust error handling
* @param luceneQuery The Lucene query string to search with
* @returns Array of matching documents
*/
public static async searchWithLucene<T>(
this: plugins.tsclass.typeFest.Class<T>,
luceneQuery: string
): Promise<T[]> {
try {
const className = (this as any).className || this.name;
const searchableFields = getSearchableFields(className);
if (searchableFields.length === 0) {
console.warn(`No searchable fields defined for class ${className}, falling back to simple search`);
return (this as any).searchByTextAcrossFields(luceneQuery);
}
// Simple term search optimization
if (!luceneQuery.includes(':') &&
!luceneQuery.includes(' AND ') &&
!luceneQuery.includes(' OR ') &&
!luceneQuery.includes(' NOT ')) {
return (this as any).searchByTextAcrossFields(luceneQuery);
}
// Try to use the Lucene-to-MongoDB conversion
const filter = (this as any).createSearchFilter(luceneQuery);
return await (this as any).getInstances(filter);
} catch (error) {
console.error(`Error in searchWithLucene: ${error.message}`);
return (this as any).searchByTextAcrossFields(luceneQuery);
}
}
/**
* Search by text across all searchable fields (fallback method)
* @param searchText The text to search for in all searchable fields
* @returns Array of matching documents
*/
private static async searchByTextAcrossFields<T>(
this: plugins.tsclass.typeFest.Class<T>,
searchText: string
): Promise<T[]> {
try {
const className = (this as any).className || this.name;
const searchableFields = getSearchableFields(className);
// Fallback to direct filter if we have searchable fields
if (searchableFields.length > 0) {
// Create a simple $or query with regex for each field
const orConditions = searchableFields.map(field => ({
[field]: { $regex: searchText, $options: 'i' }
}));
const filter = { $or: orConditions };
try {
// Try with MongoDB filter first
return await (this as any).getInstances(filter);
} catch (error) {
console.warn('MongoDB filter failed, falling back to in-memory search');
}
}
// Last resort: get all and filter in memory
const allDocs = await (this as any).getInstances({});
const lowerSearchText = searchText.toLowerCase();
return allDocs.filter((doc: any) => {
for (const field of searchableFields) {
const value = doc[field];
if (value && typeof value === 'string' &&
value.toLowerCase().includes(lowerSearchText)) {
return true;
}
}
return false;
});
} catch (error) {
console.error(`Error in searchByTextAcrossFields: ${error.message}`);
return [];
}
}
// INSTANCE
/**

178
ts/classes.lucene.adapter.ts
View File

@@ -309,6 +309,7 @@ export class LuceneParser {
/**
* Transformer for Lucene AST to MongoDB query
* FIXED VERSION - proper MongoDB query structure
*/
export class LuceneToMongoTransformer {
constructor() {}
@@ -345,16 +346,17 @@ export class LuceneToMongoTransformer {
/**
* Transform a term to MongoDB query
* FIXED: properly structured $or query for multiple fields
*/
private transformTerm(node: TermNode, searchFields?: string[]): any {
// If specific fields are provided, search across those fields
if (searchFields && searchFields.length > 0) {
// Create an $or query to search across multiple fields
return {
$or: searchFields.map(field => ({
[field]: { $regex: node.value, $options: 'i' }
}))
};
const orConditions = searchFields.map(field => ({
[field]: { $regex: node.value, $options: 'i' }
}));
return { $or: orConditions };
}
// Otherwise, use text search (requires a text index on desired fields)
@@ -363,16 +365,16 @@ export class LuceneToMongoTransformer {
/**
* Transform a phrase to MongoDB query
* FIXED: properly structured $or query for multiple fields
*/
private transformPhrase(node: PhraseNode, searchFields?: string[]): any {
// If specific fields are provided, search phrase across those fields
if (searchFields && searchFields.length > 0) {
// Create an $or query to search phrase across multiple fields
return {
$or: searchFields.map(field => ({
[field]: { $regex: `${node.value.replace(/\s+/g, '\\s+')}`, $options: 'i' }
}))
};
const orConditions = searchFields.map(field => ({
[field]: { $regex: `${node.value.replace(/\s+/g, '\\s+')}`, $options: 'i' }
}));
return { $or: orConditions };
}
// For phrases, we use a regex to ensure exact matches
@@ -412,14 +414,14 @@ export class LuceneToMongoTransformer {
// Special case for exact term matches on fields
if (node.value.type === 'TERM') {
return { [node.field]: (node.value as TermNode).value };
return { [node.field]: { $regex: (node.value as TermNode).value, $options: 'i' } };
}
// Special case for phrase matches on fields
if (node.value.type === 'PHRASE') {
return {
[node.field]: {
$regex: `^${(node.value as PhraseNode).value}$`,
$regex: `${(node.value as PhraseNode).value.replace(/\s+/g, '\\s+')}`,
$options: 'i'
}
};
@@ -430,14 +432,50 @@ export class LuceneToMongoTransformer {
// If the transformed value uses $text, we need to adapt it for the field
if (transformedValue.$text) {
return { [node.field]: transformedValue.$text.$search };
return { [node.field]: { $regex: transformedValue.$text.$search, $options: 'i' } };
}
// Handle $or and $and cases
if (transformedValue.$or || transformedValue.$and) {
// This is a bit complex - we need to restructure the query to apply the field
// For now, simplify by just using a regex on the field
const term = this.extractTermFromBooleanQuery(transformedValue);
if (term) {
return { [node.field]: { $regex: term, $options: 'i' } };
}
}
return { [node.field]: transformedValue };
}
/**
* Extract a term from a boolean query (simplification)
*/
private extractTermFromBooleanQuery(query: any): string | null {
if (query.$or && Array.isArray(query.$or) && query.$or.length > 0) {
const firstClause = query.$or[0];
for (const field in firstClause) {
if (firstClause[field].$regex) {
return firstClause[field].$regex;
}
}
}
if (query.$and && Array.isArray(query.$and) && query.$and.length > 0) {
const firstClause = query.$and[0];
for (const field in firstClause) {
if (firstClause[field].$regex) {
return firstClause[field].$regex;
}
}
}
return null;
}
/**
* Transform AND operator to MongoDB query
* FIXED: $and must be an array
*/
private transformAnd(node: BooleanNode): any {
return { $and: [this.transform(node.left), this.transform(node.right)] };
@@ -445,6 +483,7 @@ export class LuceneToMongoTransformer {
/**
* Transform OR operator to MongoDB query
* FIXED: $or must be an array
*/
private transformOr(node: BooleanNode): any {
return { $or: [this.transform(node.left), this.transform(node.right)] };
@@ -452,6 +491,7 @@ export class LuceneToMongoTransformer {
/**
* Transform NOT operator to MongoDB query
* FIXED: $and must be an array and $not usage
*/
private transformNot(node: BooleanNode): any {
const leftQuery = this.transform(node.left);
@@ -459,21 +499,53 @@ export class LuceneToMongoTransformer {
// Create a query that includes left but excludes right
if (rightQuery.$text) {
// Text searches need special handling for negation
return {
$and: [
leftQuery,
{ $not: rightQuery }
]
};
// For text searches, we need a different approach
// We'll use a negated regex instead
const searchTerm = rightQuery.$text.$search.replace(/"/g, '');
// Determine the fields to apply the negation to
const notConditions = [];
for (const field in leftQuery) {
if (field !== '$or' && field !== '$and') {
notConditions.push({
[field]: { $not: { $regex: searchTerm, $options: 'i' } }
});
}
}
// If left query has $or or $and, we need to handle it differently
if (leftQuery.$or) {
return {
$and: [
leftQuery,
{ $nor: [{ $or: notConditions }] }
]
};
} else {
// Simple case - just add $not to each field
return {
$and: [leftQuery, { $and: notConditions }]
};
}
} else {
// For other queries, we can use $not directly
return {
$and: [
leftQuery,
{ $not: rightQuery }
]
};
// We need to handle different structures based on the rightQuery
let notQuery = {};
if (rightQuery.$or) {
notQuery = { $nor: rightQuery.$or };
} else if (rightQuery.$and) {
// Convert $and to $nor
notQuery = { $nor: rightQuery.$and };
} else {
// Simple field condition
for (const field in rightQuery) {
notQuery[field] = { $not: rightQuery[field] };
}
}
return { $and: [leftQuery, notQuery] };
}
}
@@ -496,6 +568,7 @@ export class LuceneToMongoTransformer {
/**
* Transform wildcard query to MongoDB query
* FIXED: properly structured for multiple fields
*/
private transformWildcard(node: WildcardNode, searchFields?: string[]): any {
// Convert Lucene wildcards to MongoDB regex
@@ -503,19 +576,20 @@ export class LuceneToMongoTransformer {
// If specific fields are provided, search wildcard across those fields
if (searchFields && searchFields.length > 0) {
return {
$or: searchFields.map(field => ({
[field]: { $regex: regex, $options: 'i' }
}))
};
const orConditions = searchFields.map(field => ({
[field]: { $regex: regex, $options: 'i' }
}));
return { $or: orConditions };
}
// By default, apply to all text fields using $text search
// By default, apply to the default field
return { $regex: regex, $options: 'i' };
}
/**
* Transform fuzzy query to MongoDB query
* FIXED: properly structured for multiple fields
*/
private transformFuzzy(node: FuzzyNode, searchFields?: string[]): any {
// MongoDB doesn't have built-in fuzzy search
@@ -524,13 +598,14 @@ export class LuceneToMongoTransformer {
// If specific fields are provided, search fuzzy term across those fields
if (searchFields && searchFields.length > 0) {
return {
$or: searchFields.map(field => ({
[field]: { $regex: regex, $options: 'i' }
}))
};
const orConditions = searchFields.map(field => ({
[field]: { $regex: regex, $options: 'i' }
}));
return { $or: orConditions };
}
// By default, apply to the default field
return { $regex: regex, $options: 'i' };
}
@@ -615,6 +690,27 @@ export class SmartdataLuceneAdapter {
*/
convert(luceneQuery: string, searchFields?: string[]): any {
try {
// For simple single term queries, create a simpler query structure
if (!luceneQuery.includes(':') &&
!luceneQuery.includes(' AND ') &&
!luceneQuery.includes(' OR ') &&
!luceneQuery.includes(' NOT ') &&
!luceneQuery.includes('(') &&
!luceneQuery.includes('[')) {
// This is a simple term, use a more direct approach
const fieldsToSearch = searchFields || this.defaultSearchFields;
if (fieldsToSearch && fieldsToSearch.length > 0) {
return {
$or: fieldsToSearch.map(field => ({
[field]: { $regex: luceneQuery, $options: 'i' }
}))
};
}
}
// For more complex queries, use the full parser
// Parse the Lucene query into an AST
const ast = this.parser.parse(luceneQuery);
@@ -624,6 +720,7 @@ export class SmartdataLuceneAdapter {
// Transform the AST to a MongoDB query
return this.transformWithFields(ast, fieldsToSearch);
} catch (error) {
console.error(`Failed to convert Lucene query "${luceneQuery}":`, error);
throw new Error(`Failed to convert Lucene query: ${error}`);
}
}
@@ -632,8 +729,9 @@ export class SmartdataLuceneAdapter {
* Helper method to transform the AST with field information
*/
private transformWithFields(node: AnyQueryNode, searchFields: string[]): any {
// For term nodes without a specific field, apply the search fields
if (node.type === 'TERM') {
// Special case for term nodes without a specific field
if (node.type === 'TERM' || node.type === 'PHRASE' ||
node.type === 'WILDCARD' || node.type === 'FUZZY') {
return this.transformer.transform(node, searchFields);
}