fix(ci & formatting): Minor fixes: update CI workflow image and npmci package references, adjust package.json and readme URLs, and apply consistent code formatting.
This commit is contained in:
parent
4fac974fc9
commit
9426a21a2a
@ -6,8 +6,8 @@ on:
|
||||
- '**'
|
||||
|
||||
env:
|
||||
IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
|
||||
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
|
||||
IMAGE: code.foss.global/host.today/ht-docker-node:npmci
|
||||
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@/${{gitea.repository}}.git
|
||||
NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
|
||||
NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
|
||||
NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
|
||||
@ -26,7 +26,7 @@ jobs:
|
||||
- name: Install pnpm and npmci
|
||||
run: |
|
||||
pnpm install -g pnpm
|
||||
pnpm install -g @shipzone/npmci
|
||||
pnpm install -g @ship.zone/npmci
|
||||
|
||||
- name: Run npm prepare
|
||||
run: npmci npm prepare
|
||||
|
@ -6,8 +6,8 @@ on:
|
||||
- '*'
|
||||
|
||||
env:
|
||||
IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
|
||||
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
|
||||
IMAGE: code.foss.global/host.today/ht-docker-node:npmci
|
||||
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@/${{gitea.repository}}.git
|
||||
NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
|
||||
NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
|
||||
NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
|
||||
@ -26,7 +26,7 @@ jobs:
|
||||
- name: Prepare
|
||||
run: |
|
||||
pnpm install -g pnpm
|
||||
pnpm install -g @shipzone/npmci
|
||||
pnpm install -g @ship.zone/npmci
|
||||
npmci npm prepare
|
||||
|
||||
- name: Audit production dependencies
|
||||
@ -54,7 +54,7 @@ jobs:
|
||||
- name: Prepare
|
||||
run: |
|
||||
pnpm install -g pnpm
|
||||
pnpm install -g @shipzone/npmci
|
||||
pnpm install -g @ship.zone/npmci
|
||||
npmci npm prepare
|
||||
|
||||
- name: Test stable
|
||||
@ -82,7 +82,7 @@ jobs:
|
||||
- name: Prepare
|
||||
run: |
|
||||
pnpm install -g pnpm
|
||||
pnpm install -g @shipzone/npmci
|
||||
pnpm install -g @ship.zone/npmci
|
||||
npmci npm prepare
|
||||
|
||||
- name: Release
|
||||
@ -104,7 +104,7 @@ jobs:
|
||||
- name: Prepare
|
||||
run: |
|
||||
pnpm install -g pnpm
|
||||
pnpm install -g @shipzone/npmci
|
||||
pnpm install -g @ship.zone/npmci
|
||||
npmci npm prepare
|
||||
|
||||
- name: Code quality
|
||||
|
3
.gitignore
vendored
3
.gitignore
vendored
@ -3,7 +3,6 @@
|
||||
# artifacts
|
||||
coverage/
|
||||
public/
|
||||
pages/
|
||||
|
||||
# installs
|
||||
node_modules/
|
||||
@ -17,4 +16,4 @@ node_modules/
|
||||
dist/
|
||||
dist_*/
|
||||
|
||||
# custom
|
||||
#------# custom
|
@ -1,5 +1,14 @@
|
||||
# Changelog
|
||||
|
||||
## 2025-04-06 - 5.5.1 - fix(ci & formatting)
|
||||
Minor fixes: update CI workflow image and npmci package references, adjust package.json and readme URLs, and apply consistent code formatting.
|
||||
|
||||
- Update image and repo URL in Gitea workflows from GitLab to code.foss.global
|
||||
- Replace '@shipzone/npmci' with '@ship.zone/npmci' throughout CI scripts
|
||||
- Adjust homepage and bugs URL in package.json and readme
|
||||
- Apply trailing commas and consistent formatting in TypeScript source files
|
||||
- Minor update to .gitignore custom section label
|
||||
|
||||
## 2025-04-06 - 5.5.0 - feat(search)
|
||||
Enhance search functionality with robust Lucene query transformation and reliable fallback mechanisms
|
||||
|
||||
|
11
package.json
11
package.json
@ -18,9 +18,9 @@
|
||||
"author": "Lossless GmbH",
|
||||
"license": "MIT",
|
||||
"bugs": {
|
||||
"url": "https://gitlab.com/pushrocks/smartdata/issues"
|
||||
"url": "https://code.foss.global/push.rocks/smartdata/issues"
|
||||
},
|
||||
"homepage": "https://code.foss.global/push.rocks/smartdata",
|
||||
"homepage": "https://code.foss.global/push.rocks/smartdata#readme",
|
||||
"dependencies": {
|
||||
"@push.rocks/lik": "^6.0.14",
|
||||
"@push.rocks/smartdelay": "^3.0.1",
|
||||
@ -68,5 +68,8 @@
|
||||
"custom data types",
|
||||
"ODM"
|
||||
],
|
||||
"packageManager": "pnpm@10.7.0+sha512.6b865ad4b62a1d9842b61d674a393903b871d9244954f652b8842c2b553c72176b278f64c463e52d40fff8aba385c235c8c9ecf5cc7de4fd78b8bb6d49633ab6"
|
||||
}
|
||||
"packageManager": "pnpm@10.7.0+sha512.6b865ad4b62a1d9842b61d674a393903b871d9244954f652b8842c2b553c72176b278f64c463e52d40fff8aba385c235c8c9ecf5cc7de4fd78b8bb6d49633ab6",
|
||||
"pnpm": {
|
||||
"overrides": {}
|
||||
}
|
||||
}
|
152
readme.md
152
readme.md
@ -13,7 +13,7 @@ A powerful TypeScript-first MongoDB wrapper that provides advanced features for
|
||||
- **Real-time Data Sync**: Watchers for real-time data changes with RxJS integration
|
||||
- **Connection Management**: Automatic connection handling with connection pooling
|
||||
- **Collection Management**: Type-safe collection operations with automatic indexing
|
||||
- **Deep Query Type Safety**: Fully type-safe queries for nested object properties with `DeepQuery<T>`
|
||||
- **Deep Query Type Safety**: Fully type-safe queries for nested object properties with `DeepQuery<T>`
|
||||
- **MongoDB Compatibility**: Support for all MongoDB query operators and advanced features
|
||||
- **Enhanced Cursors**: Chainable, type-safe cursor API with memory efficient data processing
|
||||
- **Type Conversion**: Automatic handling of MongoDB types like ObjectId and Binary data
|
||||
@ -27,6 +27,7 @@ A powerful TypeScript-first MongoDB wrapper that provides advanced features for
|
||||
- TypeScript >= 4.x (for development)
|
||||
|
||||
## Install
|
||||
|
||||
To install `@push.rocks/smartdata`, use npm:
|
||||
|
||||
```bash
|
||||
@ -40,9 +41,11 @@ pnpm add @push.rocks/smartdata
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
`@push.rocks/smartdata` enables efficient data handling and operation management with a focus on using MongoDB. It leverages TypeScript for strong typing and ESM syntax for modern JavaScript usage.
|
||||
|
||||
### Setting Up and Connecting to the Database
|
||||
|
||||
Before interacting with the database, you need to set up and establish a connection. The `SmartdataDb` class handles connection pooling and automatic reconnection.
|
||||
|
||||
```typescript
|
||||
@ -62,40 +65,50 @@ await db.init();
|
||||
```
|
||||
|
||||
### Defining Data Models
|
||||
|
||||
Data models in `@push.rocks/smartdata` are classes that represent collections and documents in your MongoDB database. Use decorators such as `@Collection`, `@unI`, and `@svDb` to define your data models.
|
||||
|
||||
```typescript
|
||||
import { SmartDataDbDoc, Collection, unI, svDb, oid, bin, index, searchable } from '@push.rocks/smartdata';
|
||||
import {
|
||||
SmartDataDbDoc,
|
||||
Collection,
|
||||
unI,
|
||||
svDb,
|
||||
oid,
|
||||
bin,
|
||||
index,
|
||||
searchable,
|
||||
} from '@push.rocks/smartdata';
|
||||
import { ObjectId } from 'mongodb';
|
||||
|
||||
@Collection(() => db) // Associate this model with the database instance
|
||||
@Collection(() => db) // Associate this model with the database instance
|
||||
class User extends SmartDataDbDoc<User, User> {
|
||||
@unI()
|
||||
public id: string = 'unique-user-id'; // Mark 'id' as a unique index
|
||||
|
||||
|
||||
@svDb()
|
||||
@searchable() // Mark 'username' as searchable
|
||||
public username: string; // Mark 'username' to be saved in DB
|
||||
|
||||
public username: string; // Mark 'username' to be saved in DB
|
||||
|
||||
@svDb()
|
||||
@searchable() // Mark 'email' as searchable
|
||||
@index() // Create a regular index for this field
|
||||
public email: string; // Mark 'email' to be saved in DB
|
||||
|
||||
public email: string; // Mark 'email' to be saved in DB
|
||||
|
||||
@svDb()
|
||||
@oid() // Automatically handle as ObjectId type
|
||||
public organizationId: ObjectId; // Will be automatically converted to/from ObjectId
|
||||
|
||||
|
||||
@svDb()
|
||||
@bin() // Automatically handle as Binary data
|
||||
public profilePicture: Buffer; // Will be automatically converted to/from Binary
|
||||
|
||||
@svDb({
|
||||
|
||||
@svDb({
|
||||
serialize: (data) => JSON.stringify(data), // Custom serialization
|
||||
deserialize: (data) => JSON.parse(data) // Custom deserialization
|
||||
deserialize: (data) => JSON.parse(data), // Custom deserialization
|
||||
})
|
||||
public preferences: Record<string, any>;
|
||||
|
||||
|
||||
constructor(username: string, email: string) {
|
||||
super();
|
||||
this.username = username;
|
||||
@ -105,15 +118,18 @@ class User extends SmartDataDbDoc<User, User> {
|
||||
```
|
||||
|
||||
### CRUD Operations
|
||||
|
||||
`@push.rocks/smartdata` simplifies CRUD operations with intuitive methods on model instances.
|
||||
|
||||
#### Create
|
||||
|
||||
```typescript
|
||||
const newUser = new User('myUsername', 'myEmail@example.com');
|
||||
await newUser.save(); // Save the new user to the database
|
||||
await newUser.save(); // Save the new user to the database
|
||||
```
|
||||
|
||||
#### Read
|
||||
|
||||
```typescript
|
||||
// Fetch a single user by a unique attribute
|
||||
const user = await User.getInstance({ username: 'myUsername' });
|
||||
@ -132,8 +148,8 @@ await cursor.forEach(async (user, index) => {
|
||||
|
||||
// Chain cursor methods like in the MongoDB native driver
|
||||
const paginatedCursor = await User.getCursor({ active: true })
|
||||
.limit(10) // Limit results
|
||||
.skip(20) // Skip first 20 results
|
||||
.limit(10) // Limit results
|
||||
.skip(20) // Skip first 20 results
|
||||
.sort({ createdAt: -1 }); // Sort by creation date descending
|
||||
|
||||
// Convert cursor to array (when you know the result set is small)
|
||||
@ -149,30 +165,34 @@ await cursor.close();
|
||||
```
|
||||
|
||||
#### Update
|
||||
|
||||
```typescript
|
||||
// Assuming 'user' is an instance of User
|
||||
user.email = 'newEmail@example.com';
|
||||
await user.save(); // Update the user in the database
|
||||
await user.save(); // Update the user in the database
|
||||
|
||||
// Upsert operations (insert if not exists, update if exists)
|
||||
const upsertedUser = await User.upsert(
|
||||
{ id: 'user-123' }, // Query to find the user
|
||||
{ // Fields to update or insert
|
||||
username: 'newUsername',
|
||||
email: 'newEmail@example.com'
|
||||
}
|
||||
{ id: 'user-123' }, // Query to find the user
|
||||
{
|
||||
// Fields to update or insert
|
||||
username: 'newUsername',
|
||||
email: 'newEmail@example.com',
|
||||
},
|
||||
);
|
||||
```
|
||||
|
||||
#### Delete
|
||||
|
||||
```typescript
|
||||
// Assuming 'user' is an instance of User
|
||||
await user.delete(); // Delete the user from the database
|
||||
await user.delete(); // Delete the user from the database
|
||||
```
|
||||
|
||||
## Advanced Features
|
||||
|
||||
### Search Functionality
|
||||
|
||||
SmartData provides powerful search capabilities with a Lucene-like query syntax and robust fallback mechanisms:
|
||||
|
||||
```typescript
|
||||
@ -181,19 +201,19 @@ SmartData provides powerful search capabilities with a Lucene-like query syntax
|
||||
class Product extends SmartDataDbDoc<Product, Product> {
|
||||
@unI()
|
||||
public id: string = 'product-id';
|
||||
|
||||
|
||||
@svDb()
|
||||
@searchable() // Mark this field as searchable
|
||||
public name: string;
|
||||
|
||||
|
||||
@svDb()
|
||||
@searchable() // Mark this field as searchable
|
||||
public description: string;
|
||||
|
||||
|
||||
@svDb()
|
||||
@searchable() // Mark this field as searchable
|
||||
public category: string;
|
||||
|
||||
|
||||
@svDb()
|
||||
public price: number;
|
||||
}
|
||||
@ -232,6 +252,7 @@ const complexQuery = await Product.searchWithLucene('(wireless OR bluetooth) AND
|
||||
```
|
||||
|
||||
The search functionality includes:
|
||||
|
||||
- `@searchable()` decorator for marking fields as searchable
|
||||
- `getSearchableFields()` to retrieve all searchable fields for a class
|
||||
- `search()` method for basic search (requires MongoDB text index)
|
||||
@ -240,6 +261,7 @@ The search functionality includes:
|
||||
- Automatic fallback to regex-based search if MongoDB text search fails
|
||||
|
||||
### EasyStore
|
||||
|
||||
EasyStore provides a simple key-value storage system with automatic persistence:
|
||||
|
||||
```typescript
|
||||
@ -270,6 +292,7 @@ await store.deleteKey('apiKey');
|
||||
```
|
||||
|
||||
### Distributed Coordination
|
||||
|
||||
Built-in support for distributed systems with leader election:
|
||||
|
||||
```typescript
|
||||
@ -307,22 +330,26 @@ await coordinator.stop();
|
||||
```
|
||||
|
||||
### Real-time Data Watching
|
||||
|
||||
Watch for changes in your collections with RxJS integration using MongoDB Change Streams:
|
||||
|
||||
```typescript
|
||||
// Create a watcher for a specific collection with a query filter
|
||||
const watcher = await User.watch({
|
||||
active: true // Only watch for changes to active users
|
||||
}, {
|
||||
fullDocument: true, // Include the full document in change notifications
|
||||
bufferTimeMs: 100 // Buffer changes for 100ms to reduce notification frequency
|
||||
});
|
||||
const watcher = await User.watch(
|
||||
{
|
||||
active: true, // Only watch for changes to active users
|
||||
},
|
||||
{
|
||||
fullDocument: true, // Include the full document in change notifications
|
||||
bufferTimeMs: 100, // Buffer changes for 100ms to reduce notification frequency
|
||||
},
|
||||
);
|
||||
|
||||
// Subscribe to changes using RxJS
|
||||
watcher.changeSubject.subscribe((change) => {
|
||||
console.log('Change operation:', change.operationType); // 'insert', 'update', 'delete', etc.
|
||||
console.log('Document changed:', change.docInstance); // The full document instance
|
||||
|
||||
console.log('Document changed:', change.docInstance); // The full document instance
|
||||
|
||||
// Handle different types of changes
|
||||
if (change.operationType === 'insert') {
|
||||
console.log('New user created:', change.docInstance.username);
|
||||
@ -343,6 +370,7 @@ await watcher.stop();
|
||||
```
|
||||
|
||||
### Managed Collections
|
||||
|
||||
For more complex data models that require additional context:
|
||||
|
||||
```typescript
|
||||
@ -350,13 +378,13 @@ For more complex data models that require additional context:
|
||||
class ManagedDoc extends SmartDataDbDoc<ManagedDoc, ManagedDoc> {
|
||||
@unI()
|
||||
public id: string = 'unique-id';
|
||||
|
||||
|
||||
@svDb()
|
||||
public data: string;
|
||||
|
||||
|
||||
@managed()
|
||||
public manager: YourCustomManager;
|
||||
|
||||
|
||||
// The manager can provide additional functionality
|
||||
async specialOperation() {
|
||||
return this.manager.doSomethingSpecial(this);
|
||||
@ -365,6 +393,7 @@ class ManagedDoc extends SmartDataDbDoc<ManagedDoc, ManagedDoc> {
|
||||
```
|
||||
|
||||
### Automatic Indexing
|
||||
|
||||
Define indexes directly in your model class:
|
||||
|
||||
```typescript
|
||||
@ -372,19 +401,19 @@ Define indexes directly in your model class:
|
||||
class Product extends SmartDataDbDoc<Product, Product> {
|
||||
@unI() // Unique index
|
||||
public id: string = 'product-id';
|
||||
|
||||
|
||||
@svDb()
|
||||
@index() // Regular index for faster queries
|
||||
public category: string;
|
||||
|
||||
|
||||
@svDb()
|
||||
@index({ sparse: true }) // Sparse index with options
|
||||
public optionalField?: string;
|
||||
|
||||
|
||||
// Compound indexes can be defined in the collection decorator
|
||||
@Collection(() => db, {
|
||||
indexMap: {
|
||||
compoundIndex: {
|
||||
compoundIndex: {
|
||||
fields: { category: 1, name: 1 },
|
||||
options: { background: true }
|
||||
}
|
||||
@ -394,6 +423,7 @@ class Product extends SmartDataDbDoc<Product, Product> {
|
||||
```
|
||||
|
||||
### Transaction Support
|
||||
|
||||
Use MongoDB transactions for atomic operations:
|
||||
|
||||
```typescript
|
||||
@ -403,7 +433,7 @@ try {
|
||||
const user = await User.getInstance({ id: 'user-id' }, { session });
|
||||
user.balance -= 100;
|
||||
await user.save({ session });
|
||||
|
||||
|
||||
const recipient = await User.getInstance({ id: 'recipient-id' }, { session });
|
||||
recipient.balance += 100;
|
||||
await user.save({ session });
|
||||
@ -414,6 +444,7 @@ try {
|
||||
```
|
||||
|
||||
### Deep Object Queries
|
||||
|
||||
SmartData provides fully type-safe deep property queries with the `DeepQuery` type:
|
||||
|
||||
```typescript
|
||||
@ -421,7 +452,7 @@ SmartData provides fully type-safe deep property queries with the `DeepQuery` ty
|
||||
class UserProfile extends SmartDataDbDoc<UserProfile, UserProfile> {
|
||||
@unI()
|
||||
public id: string = 'profile-id';
|
||||
|
||||
|
||||
@svDb()
|
||||
public user: {
|
||||
details: {
|
||||
@ -430,14 +461,14 @@ class UserProfile extends SmartDataDbDoc<UserProfile, UserProfile> {
|
||||
address: {
|
||||
city: string;
|
||||
country: string;
|
||||
}
|
||||
}
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
// Type-safe string literals for dot notation
|
||||
const usersInUSA = await UserProfile.getInstances({
|
||||
'user.details.address.country': 'USA'
|
||||
'user.details.address.country': 'USA',
|
||||
});
|
||||
|
||||
// Fully typed deep queries with the DeepQuery type
|
||||
@ -446,7 +477,7 @@ import { DeepQuery } from '@push.rocks/smartdata';
|
||||
const typedQuery: DeepQuery<UserProfile> = {
|
||||
id: 'profile-id',
|
||||
'user.details.firstName': 'John',
|
||||
'user.details.address.country': 'USA'
|
||||
'user.details.address.country': 'USA',
|
||||
};
|
||||
|
||||
// TypeScript will error if paths are incorrect
|
||||
@ -455,13 +486,14 @@ const results = await UserProfile.getInstances(typedQuery);
|
||||
// MongoDB query operators are supported
|
||||
const operatorQuery: DeepQuery<UserProfile> = {
|
||||
'user.details.address.country': 'USA',
|
||||
'user.details.address.city': { $in: ['New York', 'Los Angeles'] }
|
||||
'user.details.address.city': { $in: ['New York', 'Los Angeles'] },
|
||||
};
|
||||
|
||||
const filteredResults = await UserProfile.getInstances(operatorQuery);
|
||||
```
|
||||
|
||||
### Document Lifecycle Hooks
|
||||
|
||||
Implement custom logic at different stages of a document's lifecycle:
|
||||
|
||||
```typescript
|
||||
@ -469,30 +501,30 @@ Implement custom logic at different stages of a document's lifecycle:
|
||||
class Order extends SmartDataDbDoc<Order, Order> {
|
||||
@unI()
|
||||
public id: string = 'order-id';
|
||||
|
||||
|
||||
@svDb()
|
||||
public total: number;
|
||||
|
||||
|
||||
@svDb()
|
||||
public items: string[];
|
||||
|
||||
|
||||
// Called before saving the document
|
||||
async beforeSave() {
|
||||
// Calculate total based on items
|
||||
this.total = await calculateTotal(this.items);
|
||||
|
||||
|
||||
// Validate the document
|
||||
if (this.items.length === 0) {
|
||||
throw new Error('Order must have at least one item');
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Called after the document is saved
|
||||
async afterSave() {
|
||||
// Notify other systems about the saved order
|
||||
await notifyExternalSystems(this);
|
||||
}
|
||||
|
||||
|
||||
// Called before deleting the document
|
||||
async beforeDelete() {
|
||||
// Check if order can be deleted
|
||||
@ -507,33 +539,39 @@ class Order extends SmartDataDbDoc<Order, Order> {
|
||||
## Best Practices
|
||||
|
||||
### Connection Management
|
||||
|
||||
- Always call `db.init()` before using any database features
|
||||
- Use `db.disconnect()` when shutting down your application
|
||||
- Set appropriate connection pool sizes based on your application's needs
|
||||
|
||||
### Document Design
|
||||
|
||||
- Use appropriate decorators (`@svDb`, `@unI`, `@index`, `@searchable`) to optimize database operations
|
||||
- Implement type-safe models by properly extending `SmartDataDbDoc`
|
||||
- Consider using interfaces to define document structures separately from implementation
|
||||
- Mark fields that need to be searched with the `@searchable()` decorator
|
||||
|
||||
### Search Optimization
|
||||
|
||||
- Create MongoDB text indexes for collections that need advanced search operations
|
||||
- Use `searchWithLucene()` for robust searches with fallback mechanisms
|
||||
- Prefer field-specific searches when possible for better performance
|
||||
- Use simple term queries instead of boolean operators if you don't have text indexes
|
||||
|
||||
### Performance Optimization
|
||||
|
||||
- Use cursors for large datasets instead of loading all documents into memory
|
||||
- Create appropriate indexes for frequent query patterns
|
||||
- Use projections to limit the fields returned when you don't need the entire document
|
||||
|
||||
### Distributed Systems
|
||||
|
||||
- Implement proper error handling for leader election events
|
||||
- Ensure all instances have synchronized clocks when using time-based coordination
|
||||
- Use the distributed coordinator's task management features for coordinated operations
|
||||
|
||||
### Type Safety
|
||||
|
||||
- Take advantage of the `DeepQuery<T>` type for fully type-safe queries
|
||||
- Define proper types for your document models to enhance IDE auto-completion
|
||||
- Use generic type parameters to specify exact document types when working with collections
|
||||
@ -552,7 +590,7 @@ Please make sure to update tests as appropriate and follow our coding standards.
|
||||
|
||||
## License and Legal Information
|
||||
|
||||
This repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository.
|
||||
This repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository.
|
||||
|
||||
**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.
|
||||
|
||||
@ -567,4 +605,4 @@ Registered at District court Bremen HRB 35230 HB, Germany
|
||||
|
||||
For any legal inquiries or if you require further information, please contact us via email at hello@task.vc.
|
||||
|
||||
By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.
|
||||
By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.
|
||||
|
@ -3,7 +3,10 @@ import * as smartmongo from '@push.rocks/smartmongo';
|
||||
import type * as taskbuffer from '@push.rocks/taskbuffer';
|
||||
|
||||
import * as smartdata from '../ts/index.js';
|
||||
import { SmartdataDistributedCoordinator, DistributedClass } from '../ts/classes.distributedcoordinator.js'; // path might need adjusting
|
||||
import {
|
||||
SmartdataDistributedCoordinator,
|
||||
DistributedClass,
|
||||
} from '../ts/classes.distributedcoordinator.js'; // path might need adjusting
|
||||
const totalInstances = 10;
|
||||
|
||||
// =======================================
|
||||
@ -20,93 +23,100 @@ tap.test('should create a testinstance as database', async () => {
|
||||
});
|
||||
|
||||
tap.test('should instantiate DistributedClass', async (tools) => {
|
||||
const instance = new DistributedClass();
|
||||
expect(instance).toBeInstanceOf(DistributedClass);
|
||||
const instance = new DistributedClass();
|
||||
expect(instance).toBeInstanceOf(DistributedClass);
|
||||
});
|
||||
|
||||
tap.test('DistributedClass should update the time', async (tools) => {
|
||||
const distributedCoordinator = new SmartdataDistributedCoordinator(testDb);
|
||||
await distributedCoordinator.start();
|
||||
const initialTime = distributedCoordinator.ownInstance.data.lastUpdated;
|
||||
await distributedCoordinator.sendHeartbeat();
|
||||
const updatedTime = distributedCoordinator.ownInstance.data.lastUpdated;
|
||||
expect(updatedTime).toBeGreaterThan(initialTime);
|
||||
await distributedCoordinator.stop();
|
||||
const distributedCoordinator = new SmartdataDistributedCoordinator(testDb);
|
||||
await distributedCoordinator.start();
|
||||
const initialTime = distributedCoordinator.ownInstance.data.lastUpdated;
|
||||
await distributedCoordinator.sendHeartbeat();
|
||||
const updatedTime = distributedCoordinator.ownInstance.data.lastUpdated;
|
||||
expect(updatedTime).toBeGreaterThan(initialTime);
|
||||
await distributedCoordinator.stop();
|
||||
});
|
||||
|
||||
tap.test('should instantiate SmartdataDistributedCoordinator', async (tools) => {
|
||||
const distributedCoordinator = new SmartdataDistributedCoordinator(testDb);
|
||||
await distributedCoordinator.start();
|
||||
expect(distributedCoordinator).toBeInstanceOf(SmartdataDistributedCoordinator);
|
||||
await distributedCoordinator.stop();
|
||||
const distributedCoordinator = new SmartdataDistributedCoordinator(testDb);
|
||||
await distributedCoordinator.start();
|
||||
expect(distributedCoordinator).toBeInstanceOf(SmartdataDistributedCoordinator);
|
||||
await distributedCoordinator.stop();
|
||||
});
|
||||
|
||||
tap.test('SmartdataDistributedCoordinator should update leader status', async (tools) => {
|
||||
const distributedCoordinator = new SmartdataDistributedCoordinator(testDb);
|
||||
await distributedCoordinator.start();
|
||||
await distributedCoordinator.checkAndMaybeLead();
|
||||
expect(distributedCoordinator.ownInstance.data.elected).toBeOneOf([true, false]);
|
||||
await distributedCoordinator.stop();
|
||||
const distributedCoordinator = new SmartdataDistributedCoordinator(testDb);
|
||||
await distributedCoordinator.start();
|
||||
await distributedCoordinator.checkAndMaybeLead();
|
||||
expect(distributedCoordinator.ownInstance.data.elected).toBeOneOf([true, false]);
|
||||
await distributedCoordinator.stop();
|
||||
});
|
||||
|
||||
tap.test('SmartdataDistributedCoordinator should handle distributed task requests', async (tools) => {
|
||||
tap.test(
|
||||
'SmartdataDistributedCoordinator should handle distributed task requests',
|
||||
async (tools) => {
|
||||
const distributedCoordinator = new SmartdataDistributedCoordinator(testDb);
|
||||
await distributedCoordinator.start();
|
||||
|
||||
const mockTaskRequest: taskbuffer.distributedCoordination.IDistributedTaskRequest = {
|
||||
submitterId: "mockSubmitter12345", // Some unique mock submitter ID
|
||||
requestResponseId: 'uni879873462hjhfkjhsdf', // Some unique ID for the request-response
|
||||
taskName: "SampleTask",
|
||||
taskVersion: "1.0.0", // Assuming it's a version string
|
||||
taskExecutionTime: Date.now(),
|
||||
taskExecutionTimeout: 60000, // Let's say the timeout is 1 minute (60000 ms)
|
||||
taskExecutionParallel: 5, // Let's assume max 5 parallel executions
|
||||
status: 'requesting'
|
||||
submitterId: 'mockSubmitter12345', // Some unique mock submitter ID
|
||||
requestResponseId: 'uni879873462hjhfkjhsdf', // Some unique ID for the request-response
|
||||
taskName: 'SampleTask',
|
||||
taskVersion: '1.0.0', // Assuming it's a version string
|
||||
taskExecutionTime: Date.now(),
|
||||
taskExecutionTimeout: 60000, // Let's say the timeout is 1 minute (60000 ms)
|
||||
taskExecutionParallel: 5, // Let's assume max 5 parallel executions
|
||||
status: 'requesting',
|
||||
};
|
||||
|
||||
const response = await distributedCoordinator.fireDistributedTaskRequest(mockTaskRequest);
|
||||
console.log(response) // based on your expected structure for the response
|
||||
console.log(response); // based on your expected structure for the response
|
||||
await distributedCoordinator.stop();
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
tap.test('SmartdataDistributedCoordinator should update distributed task requests', async (tools) => {
|
||||
tap.test(
|
||||
'SmartdataDistributedCoordinator should update distributed task requests',
|
||||
async (tools) => {
|
||||
const distributedCoordinator = new SmartdataDistributedCoordinator(testDb);
|
||||
|
||||
|
||||
await distributedCoordinator.start();
|
||||
|
||||
const mockTaskRequest: taskbuffer.distributedCoordination.IDistributedTaskRequest = {
|
||||
submitterId: "mockSubmitter12345", // Some unique mock submitter ID
|
||||
requestResponseId: 'uni879873462hjhfkjhsdf', // Some unique ID for the request-response
|
||||
taskName: "SampleTask",
|
||||
taskVersion: "1.0.0", // Assuming it's a version string
|
||||
taskExecutionTime: Date.now(),
|
||||
taskExecutionTimeout: 60000, // Let's say the timeout is 1 minute (60000 ms)
|
||||
taskExecutionParallel: 5, // Let's assume max 5 parallel executions
|
||||
status: 'requesting'
|
||||
submitterId: 'mockSubmitter12345', // Some unique mock submitter ID
|
||||
requestResponseId: 'uni879873462hjhfkjhsdf', // Some unique ID for the request-response
|
||||
taskName: 'SampleTask',
|
||||
taskVersion: '1.0.0', // Assuming it's a version string
|
||||
taskExecutionTime: Date.now(),
|
||||
taskExecutionTimeout: 60000, // Let's say the timeout is 1 minute (60000 ms)
|
||||
taskExecutionParallel: 5, // Let's assume max 5 parallel executions
|
||||
status: 'requesting',
|
||||
};
|
||||
|
||||
|
||||
await distributedCoordinator.updateDistributedTaskRequest(mockTaskRequest);
|
||||
// Here, we can potentially check if a DB entry got updated or some other side-effect of the update method.
|
||||
await distributedCoordinator.stop();
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
tap.test('should elect only one leader amongst multiple instances', async (tools) => {
|
||||
const coordinators = Array.from({ length: totalInstances }).map(() => new SmartdataDistributedCoordinator(testDb));
|
||||
await Promise.all(coordinators.map(coordinator => coordinator.start()));
|
||||
const leaders = coordinators.filter(coordinator => coordinator.ownInstance.data.elected);
|
||||
for (const leader of leaders) {
|
||||
console.log(leader.ownInstance);
|
||||
}
|
||||
expect(leaders.length).toEqual(1);
|
||||
const coordinators = Array.from({ length: totalInstances }).map(
|
||||
() => new SmartdataDistributedCoordinator(testDb),
|
||||
);
|
||||
await Promise.all(coordinators.map((coordinator) => coordinator.start()));
|
||||
const leaders = coordinators.filter((coordinator) => coordinator.ownInstance.data.elected);
|
||||
for (const leader of leaders) {
|
||||
console.log(leader.ownInstance);
|
||||
}
|
||||
expect(leaders.length).toEqual(1);
|
||||
|
||||
// stopping clears a coordinator from being elected.
|
||||
await Promise.all(coordinators.map(coordinator => coordinator.stop()));
|
||||
// stopping clears a coordinator from being elected.
|
||||
await Promise.all(coordinators.map((coordinator) => coordinator.stop()));
|
||||
});
|
||||
|
||||
tap.test('should clean up', async () => {
|
||||
await smartmongoInstance.stopAndDumpToDir(`.nogit/dbdump/test.distributedcoordinator.ts`);
|
||||
setTimeout(() => process.exit(), 2000);
|
||||
})
|
||||
await smartmongoInstance.stopAndDumpToDir(`.nogit/dbdump/test.distributedcoordinator.ts`);
|
||||
setTimeout(() => process.exit(), 2000);
|
||||
});
|
||||
|
||||
tap.start({ throwOnError: true });
|
||||
|
@ -56,14 +56,14 @@ tap.test('should create test products with searchable fields', async () => {
|
||||
new Product('Kindle Paperwhite', 'E-reader with built-in light', 'Books', 129),
|
||||
new Product('Harry Potter', 'Fantasy book series about wizards', 'Books', 49),
|
||||
new Product('Coffee Maker', 'Automatic drip coffee machine', 'Kitchen', 89),
|
||||
new Product('Blender', 'High-speed blender for smoothies', 'Kitchen', 129)
|
||||
new Product('Blender', 'High-speed blender for smoothies', 'Kitchen', 129),
|
||||
];
|
||||
|
||||
|
||||
// Save all products to the database
|
||||
for (const product of products) {
|
||||
await product.save();
|
||||
}
|
||||
|
||||
|
||||
// Verify that we can get all products
|
||||
const allProducts = await Product.getInstances({});
|
||||
expect(allProducts.length).toEqual(products.length);
|
||||
@ -74,7 +74,7 @@ tap.test('should retrieve searchable fields for a class', async () => {
|
||||
// Use the getSearchableFields function to verify our searchable fields
|
||||
const searchableFields = getSearchableFields('Product');
|
||||
console.log('Searchable fields:', searchableFields);
|
||||
|
||||
|
||||
expect(searchableFields.length).toEqual(3);
|
||||
expect(searchableFields).toContain('name');
|
||||
expect(searchableFields).toContain('description');
|
||||
@ -85,7 +85,7 @@ tap.test('should search products by exact field match', async () => {
|
||||
// Basic field exact match search
|
||||
const electronicsProducts = await Product.getInstances({ category: 'Electronics' });
|
||||
console.log(`Found ${electronicsProducts.length} products in Electronics category`);
|
||||
|
||||
|
||||
expect(electronicsProducts.length).toEqual(4);
|
||||
});
|
||||
|
||||
@ -94,7 +94,7 @@ tap.test('should search products by basic search method', async () => {
|
||||
try {
|
||||
const iPhoneResults = await Product.search('iPhone');
|
||||
console.log(`Found ${iPhoneResults.length} products matching 'iPhone' using basic search`);
|
||||
|
||||
|
||||
expect(iPhoneResults.length).toEqual(1);
|
||||
expect(iPhoneResults[0].name).toEqual('iPhone 12');
|
||||
} catch (error) {
|
||||
@ -107,8 +107,10 @@ tap.test('should search products by basic search method', async () => {
|
||||
tap.test('should search products with searchWithLucene method', async () => {
|
||||
// Using the robust searchWithLucene method
|
||||
const wirelessResults = await Product.searchWithLucene('wireless');
|
||||
console.log(`Found ${wirelessResults.length} products matching 'wireless' using searchWithLucene`);
|
||||
|
||||
console.log(
|
||||
`Found ${wirelessResults.length} products matching 'wireless' using searchWithLucene`,
|
||||
);
|
||||
|
||||
expect(wirelessResults.length).toEqual(1);
|
||||
expect(wirelessResults[0].name).toEqual('AirPods');
|
||||
});
|
||||
@ -117,7 +119,7 @@ tap.test('should search products by category with searchWithLucene', async () =>
|
||||
// Using field-specific search with searchWithLucene
|
||||
const kitchenResults = await Product.searchWithLucene('category:Kitchen');
|
||||
console.log(`Found ${kitchenResults.length} products in Kitchen category using searchWithLucene`);
|
||||
|
||||
|
||||
expect(kitchenResults.length).toEqual(2);
|
||||
expect(kitchenResults[0].category).toEqual('Kitchen');
|
||||
expect(kitchenResults[1].category).toEqual('Kitchen');
|
||||
@ -127,7 +129,7 @@ tap.test('should search products with partial word matches', async () => {
|
||||
// Testing partial word matches
|
||||
const proResults = await Product.searchWithLucene('Pro');
|
||||
console.log(`Found ${proResults.length} products matching 'Pro'`);
|
||||
|
||||
|
||||
// Should match both "MacBook Pro" and "professionals" in description
|
||||
expect(proResults.length).toBeGreaterThan(0);
|
||||
});
|
||||
@ -136,7 +138,7 @@ tap.test('should search across multiple searchable fields', async () => {
|
||||
// Test searching across all searchable fields
|
||||
const bookResults = await Product.searchWithLucene('book');
|
||||
console.log(`Found ${bookResults.length} products matching 'book' across all fields`);
|
||||
|
||||
|
||||
// Should match "MacBook" in name and "Books" in category
|
||||
expect(bookResults.length).toBeGreaterThan(1);
|
||||
});
|
||||
@ -145,10 +147,10 @@ tap.test('should handle case insensitive searches', async () => {
|
||||
// Test case insensitivity
|
||||
const electronicsResults = await Product.searchWithLucene('electronics');
|
||||
const ElectronicsResults = await Product.searchWithLucene('Electronics');
|
||||
|
||||
|
||||
console.log(`Found ${electronicsResults.length} products matching lowercase 'electronics'`);
|
||||
console.log(`Found ${ElectronicsResults.length} products matching capitalized 'Electronics'`);
|
||||
|
||||
|
||||
// Both searches should return the same results
|
||||
expect(electronicsResults.length).toEqual(ElectronicsResults.length);
|
||||
});
|
||||
@ -161,19 +163,19 @@ tap.test('should demonstrate search fallback mechanisms', async () => {
|
||||
console.log('3. As last resort, perform in-memory filtering');
|
||||
console.log('This ensures robust search even with complex queries');
|
||||
console.log('==============================================\n');
|
||||
|
||||
|
||||
// Use a simpler term that should be found in descriptions
|
||||
// Avoid using "OR" operator which requires a text index
|
||||
const results = await Product.searchWithLucene('high');
|
||||
console.log(`Found ${results.length} products matching 'high'`);
|
||||
|
||||
|
||||
// "High-speed blender" contains "high"
|
||||
expect(results.length).toBeGreaterThan(0);
|
||||
|
||||
|
||||
// Try another fallback example that won't need $text
|
||||
const powerfulResults = await Product.searchWithLucene('powerful');
|
||||
console.log(`Found ${powerfulResults.length} products matching 'powerful'`);
|
||||
|
||||
|
||||
// "Powerful laptop for professionals" contains "powerful"
|
||||
expect(powerfulResults.length).toBeGreaterThan(0);
|
||||
});
|
||||
@ -186,7 +188,7 @@ tap.test('should explain the advantages of the integrated approach', async () =>
|
||||
console.log('4. searchWithLucene provides powerful search capabilities');
|
||||
console.log('5. Backwards compatible with existing code');
|
||||
console.log('================================================\n');
|
||||
|
||||
|
||||
expect(true).toEqual(true);
|
||||
});
|
||||
|
||||
@ -199,4 +201,4 @@ tap.test('close database connection', async () => {
|
||||
setTimeout(() => process.exit(), 2000);
|
||||
});
|
||||
|
||||
tap.start({ throwOnError: true });
|
||||
tap.start({ throwOnError: true });
|
||||
|
@ -97,7 +97,7 @@ tap.test('should save the car to the db', async (toolsArg) => {
|
||||
console.log(
|
||||
`Filled database with ${counter} of ${totalCars} Cars and memory usage ${
|
||||
process.memoryUsage().rss / 1e6
|
||||
} MB`
|
||||
} MB`,
|
||||
);
|
||||
}
|
||||
} while (counter < totalCars);
|
||||
@ -116,7 +116,7 @@ tap.test('expect to get instance of Car with shallow match', async () => {
|
||||
console.log(
|
||||
`performed ${counter} of ${totalQueryCycles} total query cycles: took ${
|
||||
Date.now() - timeStart
|
||||
}ms to query a set of 2000 with memory footprint ${process.memoryUsage().rss / 1e6} MB`
|
||||
}ms to query a set of 2000 with memory footprint ${process.memoryUsage().rss / 1e6} MB`,
|
||||
);
|
||||
}
|
||||
expect(myCars[0].deepData.sodeep).toEqual('yes');
|
||||
@ -139,7 +139,7 @@ tap.test('expect to get instance of Car with deep match', async () => {
|
||||
console.log(
|
||||
`performed ${counter} of ${totalQueryCycles} total query cycles: took ${
|
||||
Date.now() - timeStart
|
||||
}ms to deep query a set of 2000 with memory footprint ${process.memoryUsage().rss / 1e6} MB`
|
||||
}ms to deep query a set of 2000 with memory footprint ${process.memoryUsage().rss / 1e6} MB`,
|
||||
);
|
||||
}
|
||||
expect(myCars2[0].deepData.sodeep).toEqual('yes');
|
||||
@ -209,7 +209,7 @@ tap.test('should store a new Truck', async () => {
|
||||
tap.test('should return a count', async () => {
|
||||
const truckCount = await Truck.getCount();
|
||||
expect(truckCount).toEqual(1);
|
||||
})
|
||||
});
|
||||
|
||||
tap.test('should use a cursor', async () => {
|
||||
const cursor = await Car.getCursor({});
|
||||
|
@ -3,6 +3,6 @@
|
||||
*/
|
||||
export const commitinfo = {
|
||||
name: '@push.rocks/smartdata',
|
||||
version: '5.5.0',
|
||||
version: '5.5.1',
|
||||
description: 'An advanced library for NoSQL data organization and manipulation using TypeScript with support for MongoDB, data validation, collections, and custom data types.'
|
||||
}
|
||||
|
@ -49,7 +49,7 @@ export interface IManager {
|
||||
db: SmartdataDb;
|
||||
}
|
||||
|
||||
export const setDefaultManagerForDoc = <T>(managerArg: IManager, dbDocArg: T): T => {
|
||||
export const setDefaultManagerForDoc = <T,>(managerArg: IManager, dbDocArg: T): T => {
|
||||
(dbDocArg as any).prototype.defaultManager = managerArg;
|
||||
return dbDocArg;
|
||||
};
|
||||
@ -190,7 +190,7 @@ export class SmartdataCollection<T> {
|
||||
|
||||
public async getCursor(
|
||||
filterObjectArg: any,
|
||||
dbDocArg: typeof SmartDataDbDoc
|
||||
dbDocArg: typeof SmartDataDbDoc,
|
||||
): Promise<SmartdataDbCursor<any>> {
|
||||
await this.init();
|
||||
const cursor = this.mongoDbCollection.find(filterObjectArg);
|
||||
@ -213,7 +213,7 @@ export class SmartdataCollection<T> {
|
||||
*/
|
||||
public async watch(
|
||||
filterObject: any,
|
||||
smartdataDbDocArg: typeof SmartDataDbDoc
|
||||
smartdataDbDocArg: typeof SmartDataDbDoc,
|
||||
): Promise<SmartdataDbWatcher> {
|
||||
await this.init();
|
||||
const changeStream = this.mongoDbCollection.watch(
|
||||
@ -224,7 +224,7 @@ export class SmartdataCollection<T> {
|
||||
],
|
||||
{
|
||||
fullDocument: 'updateLookup',
|
||||
}
|
||||
},
|
||||
);
|
||||
const smartdataWatcher = new SmartdataDbWatcher(changeStream, smartdataDbDocArg);
|
||||
await smartdataWatcher.readyDeferred.promise;
|
||||
@ -261,7 +261,7 @@ export class SmartdataCollection<T> {
|
||||
const result = await this.mongoDbCollection.updateOne(
|
||||
identifiableObject,
|
||||
{ $set: updateableObject },
|
||||
{ upsert: true }
|
||||
{ upsert: true },
|
||||
);
|
||||
return result;
|
||||
}
|
||||
|
@ -17,7 +17,7 @@ export class SmartdataDbCursor<T = any> {
|
||||
|
||||
public async next(closeAtEnd = true) {
|
||||
const result = this.smartdataDbDoc.createInstanceFromMongoDbNativeDoc(
|
||||
await this.mongodbCursor.next()
|
||||
await this.mongodbCursor.next(),
|
||||
);
|
||||
if (!result && closeAtEnd) {
|
||||
await this.close();
|
||||
|
@ -139,7 +139,7 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
|
||||
const eligibleLeader = leaders.find(
|
||||
(leader) =>
|
||||
leader.data.lastUpdated >=
|
||||
Date.now() - plugins.smarttime.getMilliSecondsFromUnits({ seconds: 20 })
|
||||
Date.now() - plugins.smarttime.getMilliSecondsFromUnits({ seconds: 20 }),
|
||||
);
|
||||
return eligibleLeader;
|
||||
});
|
||||
@ -178,16 +178,14 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
|
||||
console.log('bidding code stored.');
|
||||
});
|
||||
console.log(`bidding for leadership...`);
|
||||
await plugins.smartdelay.delayFor(
|
||||
plugins.smarttime.getMilliSecondsFromUnits({ seconds: 5 })
|
||||
);
|
||||
await plugins.smartdelay.delayFor(plugins.smarttime.getMilliSecondsFromUnits({ seconds: 5 }));
|
||||
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
||||
let biddingInstances = await DistributedClass.getInstances({});
|
||||
biddingInstances = biddingInstances.filter(
|
||||
(instanceArg) =>
|
||||
instanceArg.data.status === 'bidding' &&
|
||||
instanceArg.data.lastUpdated >=
|
||||
Date.now() - plugins.smarttime.getMilliSecondsFromUnits({ seconds: 10 })
|
||||
Date.now() - plugins.smarttime.getMilliSecondsFromUnits({ seconds: 10 }),
|
||||
);
|
||||
console.log(`found ${biddingInstances.length} bidding instances...`);
|
||||
this.ownInstance.data.elected = true;
|
||||
@ -242,7 +240,7 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
|
||||
for (const instance of allInstances) {
|
||||
if (instance.data.status === 'stopped') {
|
||||
await instance.delete();
|
||||
};
|
||||
}
|
||||
}
|
||||
await plugins.smartdelay.delayFor(10000);
|
||||
}
|
||||
@ -250,7 +248,7 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
|
||||
|
||||
// abstract implemented methods
|
||||
public async fireDistributedTaskRequest(
|
||||
taskRequestArg: plugins.taskbuffer.distributedCoordination.IDistributedTaskRequest
|
||||
taskRequestArg: plugins.taskbuffer.distributedCoordination.IDistributedTaskRequest,
|
||||
): Promise<plugins.taskbuffer.distributedCoordination.IDistributedTaskRequestResult> {
|
||||
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
||||
if (!this.ownInstance) {
|
||||
@ -277,7 +275,7 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
|
||||
}
|
||||
|
||||
public async updateDistributedTaskRequest(
|
||||
infoBasisArg: plugins.taskbuffer.distributedCoordination.IDistributedTaskRequest
|
||||
infoBasisArg: plugins.taskbuffer.distributedCoordination.IDistributedTaskRequest,
|
||||
): Promise<void> {
|
||||
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
||||
const existingInfoBasis = this.ownInstance.data.taskRequests.find((infoBasisItem) => {
|
||||
|
@ -40,13 +40,13 @@ export function svDb() {
|
||||
export function searchable() {
|
||||
return (target: SmartDataDbDoc<unknown, unknown>, key: string) => {
|
||||
console.log(`called searchable() on >${target.constructor.name}.${key}<`);
|
||||
|
||||
|
||||
// Initialize the set for this class if it doesn't exist
|
||||
const className = target.constructor.name;
|
||||
if (!searchableFieldsMap.has(className)) {
|
||||
searchableFieldsMap.set(className, new Set<string>());
|
||||
}
|
||||
|
||||
|
||||
// Add the property to the searchable fields set
|
||||
searchableFieldsMap.get(className).add(key);
|
||||
};
|
||||
@ -135,7 +135,7 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
||||
// STATIC
|
||||
public static createInstanceFromMongoDbNativeDoc<T>(
|
||||
this: plugins.tsclass.typeFest.Class<T>,
|
||||
mongoDbNativeDocArg: any
|
||||
mongoDbNativeDocArg: any,
|
||||
): T {
|
||||
const newInstance = new this();
|
||||
(newInstance as any).creationStatus = 'db';
|
||||
@ -153,7 +153,7 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
||||
*/
|
||||
public static async getInstances<T>(
|
||||
this: plugins.tsclass.typeFest.Class<T>,
|
||||
filterArg: plugins.tsclass.typeFest.PartialDeep<T>
|
||||
filterArg: plugins.tsclass.typeFest.PartialDeep<T>,
|
||||
): Promise<T[]> {
|
||||
const foundDocs = await (this as any).collection.findAll(convertFilterForMongoDb(filterArg));
|
||||
const returnArray = [];
|
||||
@ -172,7 +172,7 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
||||
*/
|
||||
public static async getInstance<T>(
|
||||
this: plugins.tsclass.typeFest.Class<T>,
|
||||
filterArg: plugins.tsclass.typeFest.PartialDeep<T>
|
||||
filterArg: plugins.tsclass.typeFest.PartialDeep<T>,
|
||||
): Promise<T> {
|
||||
const foundDoc = await (this as any).collection.findOne(convertFilterForMongoDb(filterArg));
|
||||
if (foundDoc) {
|
||||
@ -186,7 +186,10 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
||||
/**
|
||||
* get a unique id prefixed with the class name
|
||||
*/
|
||||
public static async getNewId<T = any>(this: plugins.tsclass.typeFest.Class<T>, lengthArg: number = 20) {
|
||||
public static async getNewId<T = any>(
|
||||
this: plugins.tsclass.typeFest.Class<T>,
|
||||
lengthArg: number = 20,
|
||||
) {
|
||||
return `${(this as any).className}:${plugins.smartunique.shortId(lengthArg)}`;
|
||||
}
|
||||
|
||||
@ -196,12 +199,12 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
||||
*/
|
||||
public static async getCursor<T>(
|
||||
this: plugins.tsclass.typeFest.Class<T>,
|
||||
filterArg: plugins.tsclass.typeFest.PartialDeep<T>
|
||||
filterArg: plugins.tsclass.typeFest.PartialDeep<T>,
|
||||
) {
|
||||
const collection: SmartdataCollection<T> = (this as any).collection;
|
||||
const cursor: SmartdataDbCursor<T> = await collection.getCursor(
|
||||
convertFilterForMongoDb(filterArg),
|
||||
this as any as typeof SmartDataDbDoc
|
||||
this as any as typeof SmartDataDbDoc,
|
||||
);
|
||||
return cursor;
|
||||
}
|
||||
@ -214,12 +217,12 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
||||
*/
|
||||
public static async watch<T>(
|
||||
this: plugins.tsclass.typeFest.Class<T>,
|
||||
filterArg: plugins.tsclass.typeFest.PartialDeep<T>
|
||||
filterArg: plugins.tsclass.typeFest.PartialDeep<T>,
|
||||
) {
|
||||
const collection: SmartdataCollection<T> = (this as any).collection;
|
||||
const watcher: SmartdataDbWatcher<T> = await collection.watch(
|
||||
convertFilterForMongoDb(filterArg),
|
||||
this as any
|
||||
this as any,
|
||||
);
|
||||
return watcher;
|
||||
}
|
||||
@ -231,7 +234,7 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
||||
public static async forEach<T>(
|
||||
this: plugins.tsclass.typeFest.Class<T>,
|
||||
filterArg: plugins.tsclass.typeFest.PartialDeep<T>,
|
||||
forEachFunction: (itemArg: T) => Promise<any>
|
||||
forEachFunction: (itemArg: T) => Promise<any>,
|
||||
) {
|
||||
const cursor: SmartdataDbCursor<T> = await (this as any).getCursor(filterArg);
|
||||
await cursor.forEach(forEachFunction);
|
||||
@ -242,7 +245,7 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
||||
*/
|
||||
public static async getCount<T>(
|
||||
this: plugins.tsclass.typeFest.Class<T>,
|
||||
filterArg: plugins.tsclass.typeFest.PartialDeep<T> = ({} as any)
|
||||
filterArg: plugins.tsclass.typeFest.PartialDeep<T> = {} as any,
|
||||
) {
|
||||
const collection: SmartdataCollection<T> = (this as any).collection;
|
||||
return await collection.getCount(filterArg);
|
||||
@ -255,15 +258,15 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
||||
*/
|
||||
public static createSearchFilter<T>(
|
||||
this: plugins.tsclass.typeFest.Class<T>,
|
||||
luceneQuery: string
|
||||
luceneQuery: string,
|
||||
): any {
|
||||
const className = (this as any).className || this.name;
|
||||
const searchableFields = getSearchableFields(className);
|
||||
|
||||
|
||||
if (searchableFields.length === 0) {
|
||||
throw new Error(`No searchable fields defined for class ${className}`);
|
||||
}
|
||||
|
||||
|
||||
const adapter = new SmartdataLuceneAdapter(searchableFields);
|
||||
return adapter.convert(luceneQuery);
|
||||
}
|
||||
@ -275,7 +278,7 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
||||
*/
|
||||
public static async search<T>(
|
||||
this: plugins.tsclass.typeFest.Class<T>,
|
||||
luceneQuery: string
|
||||
luceneQuery: string,
|
||||
): Promise<T[]> {
|
||||
const filter = (this as any).createSearchFilter(luceneQuery);
|
||||
return await (this as any).getInstances(filter);
|
||||
@ -288,25 +291,29 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
||||
*/
|
||||
public static async searchWithLucene<T>(
|
||||
this: plugins.tsclass.typeFest.Class<T>,
|
||||
luceneQuery: string
|
||||
luceneQuery: string,
|
||||
): Promise<T[]> {
|
||||
try {
|
||||
const className = (this as any).className || this.name;
|
||||
const searchableFields = getSearchableFields(className);
|
||||
|
||||
|
||||
if (searchableFields.length === 0) {
|
||||
console.warn(`No searchable fields defined for class ${className}, falling back to simple search`);
|
||||
console.warn(
|
||||
`No searchable fields defined for class ${className}, falling back to simple search`,
|
||||
);
|
||||
return (this as any).searchByTextAcrossFields(luceneQuery);
|
||||
}
|
||||
|
||||
|
||||
// Simple term search optimization
|
||||
if (!luceneQuery.includes(':') &&
|
||||
!luceneQuery.includes(' AND ') &&
|
||||
!luceneQuery.includes(' OR ') &&
|
||||
!luceneQuery.includes(' NOT ')) {
|
||||
if (
|
||||
!luceneQuery.includes(':') &&
|
||||
!luceneQuery.includes(' AND ') &&
|
||||
!luceneQuery.includes(' OR ') &&
|
||||
!luceneQuery.includes(' NOT ')
|
||||
) {
|
||||
return (this as any).searchByTextAcrossFields(luceneQuery);
|
||||
}
|
||||
|
||||
|
||||
// Try to use the Lucene-to-MongoDB conversion
|
||||
const filter = (this as any).createSearchFilter(luceneQuery);
|
||||
return await (this as any).getInstances(filter);
|
||||
@ -323,21 +330,21 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
||||
*/
|
||||
private static async searchByTextAcrossFields<T>(
|
||||
this: plugins.tsclass.typeFest.Class<T>,
|
||||
searchText: string
|
||||
searchText: string,
|
||||
): Promise<T[]> {
|
||||
try {
|
||||
const className = (this as any).className || this.name;
|
||||
const searchableFields = getSearchableFields(className);
|
||||
|
||||
|
||||
// Fallback to direct filter if we have searchable fields
|
||||
if (searchableFields.length > 0) {
|
||||
// Create a simple $or query with regex for each field
|
||||
const orConditions = searchableFields.map(field => ({
|
||||
[field]: { $regex: searchText, $options: 'i' }
|
||||
const orConditions = searchableFields.map((field) => ({
|
||||
[field]: { $regex: searchText, $options: 'i' },
|
||||
}));
|
||||
|
||||
|
||||
const filter = { $or: orConditions };
|
||||
|
||||
|
||||
try {
|
||||
// Try with MongoDB filter first
|
||||
return await (this as any).getInstances(filter);
|
||||
@ -345,16 +352,15 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
||||
console.warn('MongoDB filter failed, falling back to in-memory search');
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Last resort: get all and filter in memory
|
||||
const allDocs = await (this as any).getInstances({});
|
||||
const lowerSearchText = searchText.toLowerCase();
|
||||
|
||||
|
||||
return allDocs.filter((doc: any) => {
|
||||
for (const field of searchableFields) {
|
||||
const value = doc[field];
|
||||
if (value && typeof value === 'string' &&
|
||||
value.toLowerCase().includes(lowerSearchText)) {
|
||||
if (value && typeof value === 'string' && value.toLowerCase().includes(lowerSearchText)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@ -377,13 +383,13 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
||||
* updated from db in any case where doc comes from db
|
||||
*/
|
||||
@globalSvDb()
|
||||
_createdAt: string = (new Date()).toISOString();
|
||||
_createdAt: string = new Date().toISOString();
|
||||
|
||||
/**
|
||||
* will be updated everytime the doc is saved
|
||||
*/
|
||||
@globalSvDb()
|
||||
_updatedAt: string = (new Date()).toISOString();
|
||||
_updatedAt: string = new Date().toISOString();
|
||||
|
||||
/**
|
||||
* an array of saveable properties of ALL doc
|
||||
@ -424,7 +430,7 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
||||
const self: any = this;
|
||||
let dbResult: any;
|
||||
|
||||
this._updatedAt = (new Date()).toISOString();
|
||||
this._updatedAt = new Date().toISOString();
|
||||
|
||||
switch (this.creationStatus) {
|
||||
case 'db':
|
||||
@ -480,10 +486,7 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
||||
*/
|
||||
public async createSavableObject(): Promise<TImplements> {
|
||||
const saveableObject: unknown = {}; // is not exposed to outside, so any is ok here
|
||||
const saveableProperties = [
|
||||
...this.globalSaveableProperties,
|
||||
...this.saveableProperties
|
||||
]
|
||||
const saveableProperties = [...this.globalSaveableProperties, ...this.saveableProperties];
|
||||
for (const propertyNameString of saveableProperties) {
|
||||
saveableObject[propertyNameString] = this[propertyNameString];
|
||||
}
|
||||
@ -500,4 +503,4 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
||||
}
|
||||
return identifiableObject;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -41,7 +41,7 @@ export class EasyStore<T> {
|
||||
private async getEasyStore(): Promise<InstanceType<typeof this.easyStoreClass>> {
|
||||
if (this.easyStorePromise) {
|
||||
return this.easyStorePromise;
|
||||
};
|
||||
}
|
||||
|
||||
// first run from here
|
||||
const deferred = plugins.smartpromise.defer<InstanceType<typeof this.easyStoreClass>>();
|
||||
|
@ -4,7 +4,17 @@
|
||||
import * as plugins from './plugins.js';
|
||||
|
||||
// Types
|
||||
type NodeType = 'TERM' | 'PHRASE' | 'FIELD' | 'AND' | 'OR' | 'NOT' | 'RANGE' | 'WILDCARD' | 'FUZZY' | 'GROUP';
|
||||
type NodeType =
|
||||
| 'TERM'
|
||||
| 'PHRASE'
|
||||
| 'FIELD'
|
||||
| 'AND'
|
||||
| 'OR'
|
||||
| 'NOT'
|
||||
| 'RANGE'
|
||||
| 'WILDCARD'
|
||||
| 'FUZZY'
|
||||
| 'GROUP';
|
||||
|
||||
interface QueryNode {
|
||||
type: NodeType;
|
||||
@ -59,7 +69,15 @@ interface GroupNode extends QueryNode {
|
||||
value: AnyQueryNode;
|
||||
}
|
||||
|
||||
type AnyQueryNode = TermNode | PhraseNode | FieldNode | BooleanNode | RangeNode | WildcardNode | FuzzyNode | GroupNode;
|
||||
type AnyQueryNode =
|
||||
| TermNode
|
||||
| PhraseNode
|
||||
| FieldNode
|
||||
| BooleanNode
|
||||
| RangeNode
|
||||
| WildcardNode
|
||||
| FuzzyNode
|
||||
| GroupNode;
|
||||
|
||||
/**
|
||||
* Lucene query parser
|
||||
@ -68,9 +86,9 @@ export class LuceneParser {
|
||||
private pos: number = 0;
|
||||
private input: string = '';
|
||||
private tokens: string[] = [];
|
||||
|
||||
|
||||
constructor() {}
|
||||
|
||||
|
||||
/**
|
||||
* Parse a Lucene query string into an AST
|
||||
*/
|
||||
@ -78,24 +96,24 @@ export class LuceneParser {
|
||||
this.input = query.trim();
|
||||
this.pos = 0;
|
||||
this.tokens = this.tokenize(this.input);
|
||||
|
||||
|
||||
return this.parseQuery();
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Tokenize the input string into tokens
|
||||
*/
|
||||
private tokenize(input: string): string[] {
|
||||
const specialChars = /[()\[\]{}"~^:]/;
|
||||
const operators = /AND|OR|NOT|TO/;
|
||||
|
||||
|
||||
let tokens: string[] = [];
|
||||
let current = '';
|
||||
let inQuote = false;
|
||||
|
||||
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
const char = input[i];
|
||||
|
||||
|
||||
// Handle quoted strings
|
||||
if (char === '"') {
|
||||
if (inQuote) {
|
||||
@ -109,12 +127,12 @@ export class LuceneParser {
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
if (inQuote) {
|
||||
current += char;
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
// Handle whitespace
|
||||
if (char === ' ' || char === '\t' || char === '\n') {
|
||||
if (current) {
|
||||
@ -123,7 +141,7 @@ export class LuceneParser {
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
// Handle special characters
|
||||
if (specialChars.test(char)) {
|
||||
if (current) {
|
||||
@ -133,38 +151,37 @@ export class LuceneParser {
|
||||
tokens.push(char);
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
current += char;
|
||||
|
||||
|
||||
// Check if current is an operator
|
||||
if (operators.test(current) &&
|
||||
(i + 1 === input.length || /\s/.test(input[i + 1]))) {
|
||||
if (operators.test(current) && (i + 1 === input.length || /\s/.test(input[i + 1]))) {
|
||||
tokens.push(current);
|
||||
current = '';
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if (current) tokens.push(current);
|
||||
|
||||
|
||||
return tokens;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Parse the main query expression
|
||||
*/
|
||||
private parseQuery(): AnyQueryNode {
|
||||
const left = this.parseBooleanOperand();
|
||||
|
||||
|
||||
if (this.pos < this.tokens.length) {
|
||||
const token = this.tokens[this.pos];
|
||||
|
||||
|
||||
if (token === 'AND' || token === 'OR') {
|
||||
this.pos++;
|
||||
const right = this.parseQuery();
|
||||
return {
|
||||
type: token as 'AND' | 'OR',
|
||||
left,
|
||||
right
|
||||
right,
|
||||
};
|
||||
} else if (token === 'NOT' || token === '-') {
|
||||
this.pos++;
|
||||
@ -172,14 +189,14 @@ export class LuceneParser {
|
||||
return {
|
||||
type: 'NOT',
|
||||
left,
|
||||
right
|
||||
right,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
return left;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Parse boolean operands (terms, phrases, fields, groups)
|
||||
*/
|
||||
@ -187,14 +204,14 @@ export class LuceneParser {
|
||||
if (this.pos >= this.tokens.length) {
|
||||
throw new Error('Unexpected end of input');
|
||||
}
|
||||
|
||||
|
||||
const token = this.tokens[this.pos];
|
||||
|
||||
|
||||
// Handle grouping with parentheses
|
||||
if (token === '(') {
|
||||
this.pos++;
|
||||
const group = this.parseQuery();
|
||||
|
||||
|
||||
if (this.pos < this.tokens.length && this.tokens[this.pos] === ')') {
|
||||
this.pos++;
|
||||
return { type: 'GROUP', value: group } as GroupNode;
|
||||
@ -202,12 +219,12 @@ export class LuceneParser {
|
||||
throw new Error('Unclosed group');
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Handle fields (field:value)
|
||||
if (this.pos + 1 < this.tokens.length && this.tokens[this.pos + 1] === ':') {
|
||||
const field = token;
|
||||
this.pos += 2; // Skip field and colon
|
||||
|
||||
|
||||
if (this.pos < this.tokens.length) {
|
||||
const value = this.parseBooleanOperand();
|
||||
return { type: 'FIELD', field, value } as FieldNode;
|
||||
@ -215,17 +232,17 @@ export class LuceneParser {
|
||||
throw new Error('Expected value after field');
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Handle range queries
|
||||
if (token === '[' || token === '{') {
|
||||
return this.parseRange();
|
||||
}
|
||||
|
||||
|
||||
// Handle phrases ("term term")
|
||||
if (token.startsWith('"') && token.endsWith('"')) {
|
||||
const phrase = token.slice(1, -1);
|
||||
this.pos++;
|
||||
|
||||
|
||||
// Check for proximity operator
|
||||
let proximity: number | undefined;
|
||||
if (this.pos < this.tokens.length && this.tokens[this.pos] === '~') {
|
||||
@ -237,64 +254,64 @@ export class LuceneParser {
|
||||
throw new Error('Expected number after proximity operator');
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
return { type: 'PHRASE', value: phrase, proximity } as PhraseNode;
|
||||
}
|
||||
|
||||
|
||||
// Handle wildcards
|
||||
if (token.includes('*') || token.includes('?')) {
|
||||
this.pos++;
|
||||
return { type: 'WILDCARD', value: token } as WildcardNode;
|
||||
}
|
||||
|
||||
|
||||
// Handle fuzzy searches
|
||||
if (this.pos + 1 < this.tokens.length && this.tokens[this.pos + 1] === '~') {
|
||||
const term = token;
|
||||
this.pos += 2; // Skip term and tilde
|
||||
|
||||
|
||||
let maxEdits = 2; // Default
|
||||
if (this.pos < this.tokens.length && /^\d+$/.test(this.tokens[this.pos])) {
|
||||
maxEdits = parseInt(this.tokens[this.pos], 10);
|
||||
this.pos++;
|
||||
}
|
||||
|
||||
|
||||
return { type: 'FUZZY', value: term, maxEdits } as FuzzyNode;
|
||||
}
|
||||
|
||||
|
||||
// Simple term
|
||||
this.pos++;
|
||||
return { type: 'TERM', value: token } as TermNode;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Parse range queries
|
||||
*/
|
||||
private parseRange(): RangeNode {
|
||||
const includeLower = this.tokens[this.pos] === '[';
|
||||
const includeUpper = this.tokens[this.pos + 4] === ']';
|
||||
|
||||
|
||||
this.pos++; // Skip open bracket
|
||||
|
||||
|
||||
if (this.pos + 4 >= this.tokens.length) {
|
||||
throw new Error('Invalid range query syntax');
|
||||
}
|
||||
|
||||
|
||||
const lower = this.tokens[this.pos];
|
||||
this.pos++;
|
||||
|
||||
|
||||
if (this.tokens[this.pos] !== 'TO') {
|
||||
throw new Error('Expected TO in range query');
|
||||
}
|
||||
this.pos++;
|
||||
|
||||
|
||||
const upper = this.tokens[this.pos];
|
||||
this.pos++;
|
||||
|
||||
|
||||
if (this.tokens[this.pos] !== (includeLower ? ']' : '}')) {
|
||||
throw new Error('Invalid range query closing bracket');
|
||||
}
|
||||
this.pos++;
|
||||
|
||||
|
||||
// For simplicity, assuming the field is handled separately
|
||||
return {
|
||||
type: 'RANGE',
|
||||
@ -302,7 +319,7 @@ export class LuceneParser {
|
||||
lower,
|
||||
upper,
|
||||
includeLower,
|
||||
includeUpper
|
||||
includeUpper,
|
||||
};
|
||||
}
|
||||
}
|
||||
@ -313,7 +330,7 @@ export class LuceneParser {
|
||||
*/
|
||||
export class LuceneToMongoTransformer {
|
||||
constructor() {}
|
||||
|
||||
|
||||
/**
|
||||
* Transform a Lucene AST node to a MongoDB query
|
||||
*/
|
||||
@ -343,7 +360,7 @@ export class LuceneToMongoTransformer {
|
||||
throw new Error(`Unsupported node type: ${(node as any).type}`);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Transform a term to MongoDB query
|
||||
* FIXED: properly structured $or query for multiple fields
|
||||
@ -352,17 +369,17 @@ export class LuceneToMongoTransformer {
|
||||
// If specific fields are provided, search across those fields
|
||||
if (searchFields && searchFields.length > 0) {
|
||||
// Create an $or query to search across multiple fields
|
||||
const orConditions = searchFields.map(field => ({
|
||||
[field]: { $regex: node.value, $options: 'i' }
|
||||
const orConditions = searchFields.map((field) => ({
|
||||
[field]: { $regex: node.value, $options: 'i' },
|
||||
}));
|
||||
|
||||
|
||||
return { $or: orConditions };
|
||||
}
|
||||
|
||||
|
||||
// Otherwise, use text search (requires a text index on desired fields)
|
||||
return { $text: { $search: node.value } };
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Transform a phrase to MongoDB query
|
||||
* FIXED: properly structured $or query for multiple fields
|
||||
@ -370,17 +387,17 @@ export class LuceneToMongoTransformer {
|
||||
private transformPhrase(node: PhraseNode, searchFields?: string[]): any {
|
||||
// If specific fields are provided, search phrase across those fields
|
||||
if (searchFields && searchFields.length > 0) {
|
||||
const orConditions = searchFields.map(field => ({
|
||||
[field]: { $regex: `${node.value.replace(/\s+/g, '\\s+')}`, $options: 'i' }
|
||||
const orConditions = searchFields.map((field) => ({
|
||||
[field]: { $regex: `${node.value.replace(/\s+/g, '\\s+')}`, $options: 'i' },
|
||||
}));
|
||||
|
||||
|
||||
return { $or: orConditions };
|
||||
}
|
||||
|
||||
|
||||
// For phrases, we use a regex to ensure exact matches
|
||||
return { $text: { $search: `"${node.value}"` } };
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Transform a field query to MongoDB query
|
||||
*/
|
||||
@ -391,50 +408,50 @@ export class LuceneToMongoTransformer {
|
||||
rangeNode.field = node.field;
|
||||
return this.transformRange(rangeNode);
|
||||
}
|
||||
|
||||
|
||||
// Handle special case for wildcards on fields
|
||||
if (node.value.type === 'WILDCARD') {
|
||||
return {
|
||||
[node.field]: {
|
||||
return {
|
||||
[node.field]: {
|
||||
$regex: this.luceneWildcardToRegex((node.value as WildcardNode).value),
|
||||
$options: 'i'
|
||||
}
|
||||
$options: 'i',
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
// Handle special case for fuzzy searches on fields
|
||||
if (node.value.type === 'FUZZY') {
|
||||
return {
|
||||
[node.field]: {
|
||||
return {
|
||||
[node.field]: {
|
||||
$regex: this.createFuzzyRegex((node.value as FuzzyNode).value),
|
||||
$options: 'i'
|
||||
}
|
||||
$options: 'i',
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
// Special case for exact term matches on fields
|
||||
if (node.value.type === 'TERM') {
|
||||
return { [node.field]: { $regex: (node.value as TermNode).value, $options: 'i' } };
|
||||
}
|
||||
|
||||
|
||||
// Special case for phrase matches on fields
|
||||
if (node.value.type === 'PHRASE') {
|
||||
return {
|
||||
[node.field]: {
|
||||
return {
|
||||
[node.field]: {
|
||||
$regex: `${(node.value as PhraseNode).value.replace(/\s+/g, '\\s+')}`,
|
||||
$options: 'i'
|
||||
}
|
||||
$options: 'i',
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
// For other cases, we'll transform the value and apply it to the field
|
||||
const transformedValue = this.transform(node.value);
|
||||
|
||||
|
||||
// If the transformed value uses $text, we need to adapt it for the field
|
||||
if (transformedValue.$text) {
|
||||
return { [node.field]: { $regex: transformedValue.$text.$search, $options: 'i' } };
|
||||
}
|
||||
|
||||
|
||||
// Handle $or and $and cases
|
||||
if (transformedValue.$or || transformedValue.$and) {
|
||||
// This is a bit complex - we need to restructure the query to apply the field
|
||||
@ -444,10 +461,10 @@ export class LuceneToMongoTransformer {
|
||||
return { [node.field]: { $regex: term, $options: 'i' } };
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
return { [node.field]: transformedValue };
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Extract a term from a boolean query (simplification)
|
||||
*/
|
||||
@ -460,7 +477,7 @@ export class LuceneToMongoTransformer {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if (query.$and && Array.isArray(query.$and) && query.$and.length > 0) {
|
||||
const firstClause = query.$and[0];
|
||||
for (const field in firstClause) {
|
||||
@ -469,10 +486,10 @@ export class LuceneToMongoTransformer {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Transform AND operator to MongoDB query
|
||||
* FIXED: $and must be an array
|
||||
@ -480,7 +497,7 @@ export class LuceneToMongoTransformer {
|
||||
private transformAnd(node: BooleanNode): any {
|
||||
return { $and: [this.transform(node.left), this.transform(node.right)] };
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Transform OR operator to MongoDB query
|
||||
* FIXED: $or must be an array
|
||||
@ -488,7 +505,7 @@ export class LuceneToMongoTransformer {
|
||||
private transformOr(node: BooleanNode): any {
|
||||
return { $or: [this.transform(node.left), this.transform(node.right)] };
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Transform NOT operator to MongoDB query
|
||||
* FIXED: $and must be an array and $not usage
|
||||
@ -496,43 +513,40 @@ export class LuceneToMongoTransformer {
|
||||
private transformNot(node: BooleanNode): any {
|
||||
const leftQuery = this.transform(node.left);
|
||||
const rightQuery = this.transform(node.right);
|
||||
|
||||
|
||||
// Create a query that includes left but excludes right
|
||||
if (rightQuery.$text) {
|
||||
// For text searches, we need a different approach
|
||||
// We'll use a negated regex instead
|
||||
const searchTerm = rightQuery.$text.$search.replace(/"/g, '');
|
||||
|
||||
|
||||
// Determine the fields to apply the negation to
|
||||
const notConditions = [];
|
||||
|
||||
|
||||
for (const field in leftQuery) {
|
||||
if (field !== '$or' && field !== '$and') {
|
||||
notConditions.push({
|
||||
[field]: { $not: { $regex: searchTerm, $options: 'i' } }
|
||||
[field]: { $not: { $regex: searchTerm, $options: 'i' } },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// If left query has $or or $and, we need to handle it differently
|
||||
if (leftQuery.$or) {
|
||||
return {
|
||||
$and: [
|
||||
leftQuery,
|
||||
{ $nor: [{ $or: notConditions }] }
|
||||
]
|
||||
$and: [leftQuery, { $nor: [{ $or: notConditions }] }],
|
||||
};
|
||||
} else {
|
||||
// Simple case - just add $not to each field
|
||||
return {
|
||||
$and: [leftQuery, { $and: notConditions }]
|
||||
$and: [leftQuery, { $and: notConditions }],
|
||||
};
|
||||
}
|
||||
} else {
|
||||
// For other queries, we can use $not directly
|
||||
// We need to handle different structures based on the rightQuery
|
||||
let notQuery = {};
|
||||
|
||||
|
||||
if (rightQuery.$or) {
|
||||
notQuery = { $nor: rightQuery.$or };
|
||||
} else if (rightQuery.$and) {
|
||||
@ -544,28 +558,28 @@ export class LuceneToMongoTransformer {
|
||||
notQuery[field] = { $not: rightQuery[field] };
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
return { $and: [leftQuery, notQuery] };
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Transform range query to MongoDB query
|
||||
*/
|
||||
private transformRange(node: RangeNode): any {
|
||||
const range: any = {};
|
||||
|
||||
|
||||
if (node.lower !== '*') {
|
||||
range[node.includeLower ? '$gte' : '$gt'] = this.parseValue(node.lower);
|
||||
}
|
||||
|
||||
|
||||
if (node.upper !== '*') {
|
||||
range[node.includeUpper ? '$lte' : '$lt'] = this.parseValue(node.upper);
|
||||
}
|
||||
|
||||
|
||||
return { [node.field]: range };
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Transform wildcard query to MongoDB query
|
||||
* FIXED: properly structured for multiple fields
|
||||
@ -573,20 +587,20 @@ export class LuceneToMongoTransformer {
|
||||
private transformWildcard(node: WildcardNode, searchFields?: string[]): any {
|
||||
// Convert Lucene wildcards to MongoDB regex
|
||||
const regex = this.luceneWildcardToRegex(node.value);
|
||||
|
||||
|
||||
// If specific fields are provided, search wildcard across those fields
|
||||
if (searchFields && searchFields.length > 0) {
|
||||
const orConditions = searchFields.map(field => ({
|
||||
[field]: { $regex: regex, $options: 'i' }
|
||||
const orConditions = searchFields.map((field) => ({
|
||||
[field]: { $regex: regex, $options: 'i' },
|
||||
}));
|
||||
|
||||
|
||||
return { $or: orConditions };
|
||||
}
|
||||
|
||||
|
||||
// By default, apply to the default field
|
||||
return { $regex: regex, $options: 'i' };
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Transform fuzzy query to MongoDB query
|
||||
* FIXED: properly structured for multiple fields
|
||||
@ -595,20 +609,20 @@ export class LuceneToMongoTransformer {
|
||||
// MongoDB doesn't have built-in fuzzy search
|
||||
// This is a very basic approach using regex
|
||||
const regex = this.createFuzzyRegex(node.value);
|
||||
|
||||
|
||||
// If specific fields are provided, search fuzzy term across those fields
|
||||
if (searchFields && searchFields.length > 0) {
|
||||
const orConditions = searchFields.map(field => ({
|
||||
[field]: { $regex: regex, $options: 'i' }
|
||||
const orConditions = searchFields.map((field) => ({
|
||||
[field]: { $regex: regex, $options: 'i' },
|
||||
}));
|
||||
|
||||
|
||||
return { $or: orConditions };
|
||||
}
|
||||
|
||||
|
||||
// By default, apply to the default field
|
||||
return { $regex: regex, $options: 'i' };
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Convert Lucene wildcards to MongoDB regex patterns
|
||||
*/
|
||||
@ -622,7 +636,7 @@ export class LuceneToMongoTransformer {
|
||||
.replace(/\*/g, '.*')
|
||||
.replace(/\?/g, '.');
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Create a simplified fuzzy search regex
|
||||
*/
|
||||
@ -639,7 +653,7 @@ export class LuceneToMongoTransformer {
|
||||
}
|
||||
return regex;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Parse string values to appropriate types (numbers, dates, etc.)
|
||||
*/
|
||||
@ -648,17 +662,17 @@ export class LuceneToMongoTransformer {
|
||||
if (/^-?\d+$/.test(value)) {
|
||||
return parseInt(value, 10);
|
||||
}
|
||||
|
||||
|
||||
if (/^-?\d+\.\d+$/.test(value)) {
|
||||
return parseFloat(value);
|
||||
}
|
||||
|
||||
|
||||
// Try to parse as date (simplified)
|
||||
const date = new Date(value);
|
||||
if (!isNaN(date.getTime())) {
|
||||
return date;
|
||||
}
|
||||
|
||||
|
||||
// Default to string
|
||||
return value;
|
||||
}
|
||||
@ -671,7 +685,7 @@ export class SmartdataLuceneAdapter {
|
||||
private parser: LuceneParser;
|
||||
private transformer: LuceneToMongoTransformer;
|
||||
private defaultSearchFields: string[] = [];
|
||||
|
||||
|
||||
/**
|
||||
* @param defaultSearchFields - Optional array of field names to search across when no field is specified
|
||||
*/
|
||||
@ -682,7 +696,7 @@ export class SmartdataLuceneAdapter {
|
||||
this.defaultSearchFields = defaultSearchFields;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Convert a Lucene query string to a MongoDB query object
|
||||
* @param luceneQuery - The Lucene query string to convert
|
||||
@ -691,32 +705,33 @@ export class SmartdataLuceneAdapter {
|
||||
convert(luceneQuery: string, searchFields?: string[]): any {
|
||||
try {
|
||||
// For simple single term queries, create a simpler query structure
|
||||
if (!luceneQuery.includes(':') &&
|
||||
!luceneQuery.includes(' AND ') &&
|
||||
!luceneQuery.includes(' OR ') &&
|
||||
!luceneQuery.includes(' NOT ') &&
|
||||
!luceneQuery.includes('(') &&
|
||||
!luceneQuery.includes('[')) {
|
||||
|
||||
if (
|
||||
!luceneQuery.includes(':') &&
|
||||
!luceneQuery.includes(' AND ') &&
|
||||
!luceneQuery.includes(' OR ') &&
|
||||
!luceneQuery.includes(' NOT ') &&
|
||||
!luceneQuery.includes('(') &&
|
||||
!luceneQuery.includes('[')
|
||||
) {
|
||||
// This is a simple term, use a more direct approach
|
||||
const fieldsToSearch = searchFields || this.defaultSearchFields;
|
||||
|
||||
|
||||
if (fieldsToSearch && fieldsToSearch.length > 0) {
|
||||
return {
|
||||
$or: fieldsToSearch.map(field => ({
|
||||
[field]: { $regex: luceneQuery, $options: 'i' }
|
||||
}))
|
||||
$or: fieldsToSearch.map((field) => ({
|
||||
[field]: { $regex: luceneQuery, $options: 'i' },
|
||||
})),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// For more complex queries, use the full parser
|
||||
// Parse the Lucene query into an AST
|
||||
const ast = this.parser.parse(luceneQuery);
|
||||
|
||||
|
||||
// Use provided searchFields, fall back to defaultSearchFields
|
||||
const fieldsToSearch = searchFields || this.defaultSearchFields;
|
||||
|
||||
|
||||
// Transform the AST to a MongoDB query
|
||||
return this.transformWithFields(ast, fieldsToSearch);
|
||||
} catch (error) {
|
||||
@ -724,18 +739,22 @@ export class SmartdataLuceneAdapter {
|
||||
throw new Error(`Failed to convert Lucene query: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Helper method to transform the AST with field information
|
||||
*/
|
||||
private transformWithFields(node: AnyQueryNode, searchFields: string[]): any {
|
||||
// Special case for term nodes without a specific field
|
||||
if (node.type === 'TERM' || node.type === 'PHRASE' ||
|
||||
node.type === 'WILDCARD' || node.type === 'FUZZY') {
|
||||
if (
|
||||
node.type === 'TERM' ||
|
||||
node.type === 'PHRASE' ||
|
||||
node.type === 'WILDCARD' ||
|
||||
node.type === 'FUZZY'
|
||||
) {
|
||||
return this.transformer.transform(node, searchFields);
|
||||
}
|
||||
|
||||
|
||||
// For other node types, use the standard transformation
|
||||
return this.transformer.transform(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -14,7 +14,7 @@ export class SmartdataDbWatcher<T = any> {
|
||||
public changeSubject = new plugins.smartrx.rxjs.Subject<T>();
|
||||
constructor(
|
||||
changeStreamArg: plugins.mongodb.ChangeStream<T>,
|
||||
smartdataDbDocArg: typeof SmartDataDbDoc
|
||||
smartdataDbDocArg: typeof SmartDataDbDoc,
|
||||
) {
|
||||
this.changeStream = changeStreamArg;
|
||||
this.changeStream.on('change', async (item: any) => {
|
||||
@ -23,7 +23,7 @@ export class SmartdataDbWatcher<T = any> {
|
||||
return;
|
||||
}
|
||||
this.changeSubject.next(
|
||||
smartdataDbDocArg.createInstanceFromMongoDbNativeDoc(item.fullDocument) as any as T
|
||||
smartdataDbDocArg.createInstanceFromMongoDbNativeDoc(item.fullDocument) as any as T,
|
||||
);
|
||||
});
|
||||
plugins.smartdelay.delayFor(0).then(() => {
|
||||
|
@ -11,4 +11,4 @@ export { convenience };
|
||||
// to be removed with the next breaking update
|
||||
import type * as plugins from './plugins.js';
|
||||
type IMongoDescriptor = plugins.tsclass.database.IMongoDescriptor;
|
||||
export type { IMongoDescriptor };
|
||||
export type { IMongoDescriptor };
|
||||
|
@ -6,9 +6,11 @@
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"esModuleInterop": true,
|
||||
"verbatimModuleSyntax": true
|
||||
"verbatimModuleSyntax": true,
|
||||
"baseUrl": ".",
|
||||
"paths": {}
|
||||
},
|
||||
"exclude": [
|
||||
"dist_*/**/*.d.ts"
|
||||
]
|
||||
}
|
||||
}
|
Loading…
x
Reference in New Issue
Block a user