5.4.0

.gitea/workflows/default_tags.yaml

@@ -119,6 +119,6 @@ jobs:
         run: |
           npmci node install stable
           npmci npm install
-          pnpm install -g @gitzone/tsdoc
+          pnpm install -g @git.zone/tsdoc
           npmci command tsdoc
         continue-on-error: true

changelog.md

@@ -0,0 +1,188 @@
+# Changelog
+
+## 2025-04-06 - 5.4.0 - feat(core)
+Refactor file structure and update dependency versions
+
+- Renamed files and modules from 'smartdata.classes.*' to 'classes.*' and adjusted corresponding import paths.
+- Updated dependency versions: '@push.rocks/smartmongo' to ^2.0.11, '@tsclass/tsclass' to ^8.2.0, and 'mongodb' to ^6.15.0.
+- Renamed dev dependency packages from '@gitzone/...' to '@git.zone/...' and updated '@push.rocks/tapbundle' and '@types/node'.
+- Fixed YAML workflow command: replaced 'pnpm install -g @gitzone/tsdoc' with 'pnpm install -g @git.zone/tsdoc'.
+- Added package manager configuration and pnpm-workspace.yaml for built dependencies.
+
+## 2025-03-10 - 5.3.0 - feat(docs)
+Enhance documentation with updated installation instructions and comprehensive usage examples covering advanced features such as deep queries, automatic indexing, and distributed coordination.
+
+- Added pnpm installation command
+- Updated User model example to include ObjectId, Binary, and custom serialization
+- Expanded CRUD operations examples with cursor methods and deep query support
+- Enhanced sections on EasyStore, real-time data watching with RxJS integration, and managed collections
+- Included detailed examples for transactions, deep object queries, and document lifecycle hooks
+
+## 2025-02-03 - 5.2.12 - fix(documentation)
+Remove license badge from README
+
+- Removed the license badge from the README file, ensuring compliance with branding guidelines.
+
+## 2025-02-03 - 5.2.11 - fix(documentation)
+Updated project documentation for accuracy and added advanced feature details
+
+- Added details for EasyStore, Distributed Coordination, and Real-time Data Watching features.
+- Updated database connection setup instructions to include user authentication.
+- Re-organized advanced usage section to showcase additional features separately.
+
+## 2024-09-05 - 5.2.10 - fix(smartdata.classes.doc)
+Fix issue with array handling in convertFilterForMongoDb function
+
+- Corrected the logic to properly handle array filters in the convertFilterForMongoDb function to avoid incorrect assignments.
+
+## 2024-09-05 - 5.2.9 - fix(smartdata.classes.doc)
+Fixed issue with convertFilterForMongoDb to handle array operators.
+
+- Updated the convertFilterForMongoDb function in smartdata.classes.doc.ts to properly handle array operators like $in and $all.
+
+## 2024-09-05 - 5.2.8 - fix(smartdata.classes.doc)
+Fix key handling in convertFilterForMongoDb function
+
+- Fixed an issue in convertFilterForMongoDb that allowed keys with dots which could cause errors.
+
+## 2024-09-05 - 5.2.7 - fix(core)
+Fixed issue with handling filter keys containing dots in smartdata.classes.doc.ts
+
+- Fixed an error in the convertFilterForMongoDb function which previously threw an error when keys contained dots.
+
+## 2024-06-18 - 5.2.6 - Chore
+Maintenance Release
+
+- Release version 5.2.6
+
+## 2024-05-31 - 5.2.2 - Bug Fixes
+Fixes and Maintenance
+
+- Fixed issue where `_createdAt` and `_updatedAt` registered saveableProperties for all document types
+
+## 2024-04-15 - 5.1.2 - New Feature
+Enhancements and Bug Fixes
+
+- Added static `.getCount({})` method to `SmartDataDbDoc`
+- Changed fields `_createdAt` and `_updatedAt` to ISO format
+
+## 2024-04-14 - 5.0.43 - New Feature
+New Feature Addition
+
+- Added default `_createdAt` and `_updatedAt` fields, fixes #1
+
+## 2024-03-30 - 5.0.41 - Bug Fixes
+Improvements and Fixes
+
+- Improved `tsconfig.json` for ES Module use
+
+## 2023-07-10 - 5.0.20 - Chore
+Organizational Changes
+
+- Switched to new org scheme
+
+## 2023-07-21 - 5.0.21 to 5.0.26 - Fixes
+Multiple Fix Releases
+
+- Various core updates and bug fixes
+
+## 2023-07-21 - 5.0.20 - Chore
+Organizational Changes
+
+- Switch to the new org scheme
+
+## 2023-06-25 - 5.0.14 to 5.0.19 - Fixes
+Multiple Fix Releases
+
+- Various core updates and bug fixes
+
+## 2022-05-17 - 5.0.0 - Major Update
+Breaking Changes
+
+- Switched to ESM
+
+## 2022-05-18 - 5.0.2 - Bug Fixes
+Bug Fixes
+
+- The `watcher.changeSubject` now emits the correct type into observer functions
+
+## 2022-05-17 - 5.0.1 - Chore
+Testing Improvements
+
+- Tests now use `@pushrocks/smartmongo` backed by `wiredTiger`
+
+## 2022-05-17 to 2022-11-08 - 5.0.8 to 5.0.10
+Multiple Fix Releases
+
+- Various core updates and bug fixes
+
+## 2021-11-12 - 4.0.17 to 4.0.20
+Multiple Fix Releases
+
+- Various core updates and bug fixes
+
+## 2021-09-17 - 4.0.10 to 4.0.16
+Multiple Fix Releases
+
+- Various core updates and bug fixes
+
+## 2021-06-09 - 4.0.1 to 4.0.9
+Multiple Fix Releases
+
+- Various core updates and bug fixes
+
+## 2021-06-06 - 4.0.0 - Major Update
+Major Release
+
+- Maintenance and core updates
+
+## 2021-05-17 - 3.1.56 - Chore
+Maintenance Release
+
+- Release version 3.1.56
+
+## 2020-09-09 - 3.1.44 to 3.1.52
+Multiple Fix Releases
+
+- Various core updates and bug fixes
+
+## 2020-06-12 - 3.1.26 to 3.1.28
+Multiple Fix Releases
+
+- Various core updates and bug fixes
+
+## 2020-02-18 - 3.1.23 to 3.1.25
+Multiple Fix Releases
+
+- Various core updates and bug fixes
+
+## 2019-09-11 - 3.1.20 to 3.1.22
+Multiple Fix Releases
+
+- Various core updates and bug fixes
+
+## 2018-07-10 - 3.0.5 - New Feature
+Added Feature
+
+- Added custom unique indexes to `SmartdataDoc`
+
+## 2018-07-08 - 3.0.1 - Chore
+Dependencies Update
+
+- Updated mongodb dependencies
+
+## 2018-07-08 - 3.0.0 - Major Update
+Refactor and Cleanup
+
+- Cleaned project structure
+
+## 2018-01-16 - 2.0.7 - Breaking Change
+Big Changes
+
+- Switched to `@pushrocks` scope and moved from `rethinkdb` to `mongodb`
+
+## 2018-01-12 - 2.0.0 - Major Release
+Core Updates
+
+- Updated CI configurations
+

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@push.rocks/smartdata",
-  "version": "5.1.2",
+  "version": "5.4.0",
   "private": false,
   "description": "An advanced library for NoSQL data organization and manipulation using TypeScript with support for MongoDB, data validation, collections, and custom data types.",
   "main": "dist_ts/index.js",
@@ -13,35 +13,35 @@
   },
   "repository": {
     "type": "git",
-    "url": "git+ssh://git@gitlab.com/pushrocks/smartdata.git"
+    "url": "https://code.foss.global/push.rocks/smartdata.git"
   },
   "author": "Lossless GmbH",
   "license": "MIT",
   "bugs": {
     "url": "https://gitlab.com/pushrocks/smartdata/issues"
   },
-  "homepage": "https://gitlab.com/pushrocks/smartdata#README",
+  "homepage": "https://code.foss.global/push.rocks/smartdata",
   "dependencies": {
     "@push.rocks/lik": "^6.0.14",
     "@push.rocks/smartdelay": "^3.0.1",
     "@push.rocks/smartlog": "^3.0.2",
-    "@push.rocks/smartmongo": "^2.0.10",
+    "@push.rocks/smartmongo": "^2.0.11",
     "@push.rocks/smartpromise": "^4.0.2",
     "@push.rocks/smartrx": "^3.0.7",
     "@push.rocks/smartstring": "^4.0.15",
     "@push.rocks/smarttime": "^4.0.6",
     "@push.rocks/smartunique": "^3.0.8",
     "@push.rocks/taskbuffer": "^3.1.7",
-    "@tsclass/tsclass": "^4.0.52",
-    "mongodb": "^6.5.0"
+    "@tsclass/tsclass": "^8.2.0",
+    "mongodb": "^6.15.0"
   },
   "devDependencies": {
-    "@gitzone/tsbuild": "^2.1.66",
-    "@gitzone/tsrun": "^1.2.44",
-    "@gitzone/tstest": "^1.0.77",
+    "@git.zone/tsbuild": "^2.3.2",
+    "@git.zone/tsrun": "^1.2.44",
+    "@git.zone/tstest": "^1.0.77",
     "@push.rocks/qenv": "^6.0.5",
-    "@push.rocks/tapbundle": "^5.0.22",
-    "@types/node": "^20.11.30"
+    "@push.rocks/tapbundle": "^5.6.2",
+    "@types/node": "^22.14.0"
   },
   "files": [
     "ts/**/*",
@@ -67,5 +67,6 @@
     "collections",
     "custom data types",
     "ODM"
-  ]
+  ],
+  "packageManager": "pnpm@10.7.0+sha512.6b865ad4b62a1d9842b61d674a393903b871d9244954f652b8842c2b553c72176b278f64c463e52d40fff8aba385c235c8c9ecf5cc7de4fd78b8bb6d49633ab6"
 }

pnpm-workspace.yaml

@@ -0,0 +1,4 @@
+onlyBuiltDependencies:
+  - esbuild
+  - mongodb-memory-server
+  - puppeteer

readme.md

@@ -1,5 +1,29 @@
 # @push.rocks/smartdata
-do more with data
+
+[![npm version](https://badge.fury.io/js/@push.rocks%2Fsmartdata.svg)](https://www.npmjs.com/package/@push.rocks/smartdata)
+
+A powerful TypeScript-first MongoDB wrapper that provides advanced features for distributed systems, real-time data synchronization, and easy data management.
+
+## Features
+
+- **Type-Safe MongoDB Integration**: Full TypeScript support with decorators for schema definition
+- **Document Management**: Type-safe CRUD operations with automatic timestamp tracking
+- **EasyStore**: Simple key-value storage with automatic persistence and sharing between instances
+- **Distributed Coordination**: Built-in support for leader election and distributed task management
+- **Real-time Data Sync**: Watchers for real-time data changes with RxJS integration
+- **Connection Management**: Automatic connection handling with connection pooling
+- **Collection Management**: Type-safe collection operations with automatic indexing
+- **Deep Query Type Safety**: Fully type-safe queries for nested object properties with `DeepQuery<T>` 
+- **MongoDB Compatibility**: Support for all MongoDB query operators and advanced features
+- **Enhanced Cursors**: Chainable, type-safe cursor API with memory efficient data processing
+- **Type Conversion**: Automatic handling of MongoDB types like ObjectId and Binary data
+- **Serialization Hooks**: Custom serialization and deserialization of document properties
+
+## Requirements
+
+- Node.js >= 16.x
+- MongoDB >= 4.4
+- TypeScript >= 4.x (for development)
 
 ## Install
 To install `@push.rocks/smartdata`, use npm:
@@ -8,26 +32,31 @@ To install `@push.rocks/smartdata`, use npm:
 npm install @push.rocks/smartdata --save
 ```
 
-This will add `@push.rocks/smartdata` to your project's dependencies.
+Or with pnpm:
+
+```bash
+pnpm add @push.rocks/smartdata
+```
 
 ## Usage
-`@push.rocks/smartdata` enables efficient data handling and operation management with a focus on using MongoDB. It leverages TypeScript for strong typing and ESM syntax for modern JavaScript usage. Below are various scenarios demonstrating how to utilize this package effectively in a project.
+`@push.rocks/smartdata` enables efficient data handling and operation management with a focus on using MongoDB. It leverages TypeScript for strong typing and ESM syntax for modern JavaScript usage.
 
 ### Setting Up and Connecting to the Database
-Before interacting with the database, you need to set up and establish a connection. This is done by creating an instance of `SmartdataDb` and calling its `init` method with your MongoDB connection details.
+Before interacting with the database, you need to set up and establish a connection. The `SmartdataDb` class handles connection pooling and automatic reconnection.
 
 ```typescript
 import { SmartdataDb } from '@push.rocks/smartdata';
 
 // Create a new instance of SmartdataDb with MongoDB connection details
 const db = new SmartdataDb({
-  mongoDbUrl: 'mongodb://localhost:27017',
+  mongoDbUrl: 'mongodb://<USERNAME>:<PASSWORD>@localhost:27017/<DBNAME>',
   mongoDbName: 'your-database-name',
   mongoDbUser: 'your-username',
   mongoDbPass: 'your-password',
 });
 
 // Initialize and connect to the database
+// This sets up a connection pool with max 100 connections
 await db.init();
 ```
 
@@ -35,7 +64,8 @@ await db.init();
 Data models in `@push.rocks/smartdata` are classes that represent collections and documents in your MongoDB database. Use decorators such as `@Collection`, `@unI`, and `@svDb` to define your data models.
 
 ```typescript
-import { SmartDataDbDoc, Collection, unI, svDb } from '@push.rocks/smartdata';
+import { SmartDataDbDoc, Collection, unI, svDb, oid, bin, index } from '@push.rocks/smartdata';
+import { ObjectId } from 'mongodb';
 
 @Collection(() => db)  // Associate this model with the database instance
 class User extends SmartDataDbDoc<User, User> {
@@ -46,8 +76,23 @@ class User extends SmartDataDbDoc<User, User> {
   public username: string;  // Mark 'username' to be saved in DB
   
   @svDb()
+  @index() // Create a regular index for this field
   public email: string;  // Mark 'email' to be saved in DB
   
+  @svDb()
+  @oid() // Automatically handle as ObjectId type
+  public organizationId: ObjectId; // Will be automatically converted to/from ObjectId
+  
+  @svDb()
+  @bin() // Automatically handle as Binary data
+  public profilePicture: Buffer; // Will be automatically converted to/from Binary
+  
+  @svDb({ 
+    serialize: (data) => JSON.stringify(data), // Custom serialization
+    deserialize: (data) => JSON.parse(data)    // Custom deserialization
+  })
+  public preferences: Record<string, any>;
+  
   constructor(username: string, email: string) {
     super();
     this.username = username;
@@ -56,7 +101,7 @@ class User extends SmartDataDbDoc<User, User> {
 }
 ```
 
-### Performing CRUD Operations
+### CRUD Operations
 `@push.rocks/smartdata` simplifies CRUD operations with intuitive methods on model instances.
 
 #### Create
@@ -72,6 +117,32 @@ const user = await User.getInstance({ username: 'myUsername' });
 
 // Fetch multiple users that match criteria
 const users = await User.getInstances({ email: 'myEmail@example.com' });
+
+// Using a cursor for large collections
+const cursor = await User.getCursor({ active: true });
+
+// Process documents one at a time (memory efficient)
+await cursor.forEach(async (user, index) => {
+  // Process each user with its position
+  console.log(`Processing user ${index}: ${user.username}`);
+});
+
+// Chain cursor methods like in the MongoDB native driver
+const paginatedCursor = await User.getCursor({ active: true })
+  .limit(10)  // Limit results
+  .skip(20)   // Skip first 20 results
+  .sort({ createdAt: -1 }); // Sort by creation date descending
+
+// Convert cursor to array (when you know the result set is small)
+const userArray = await paginatedCursor.toArray();
+
+// Other cursor operations
+const nextUser = await cursor.next(); // Get the next document
+const hasMoreUsers = await cursor.hasNext(); // Check if more documents exist
+const count = await cursor.count(); // Get the count of documents in the cursor
+
+// Always close cursors when done with them
+await cursor.close();
 ```
 
 #### Update
@@ -79,6 +150,15 @@ const users = await User.getInstances({ email: 'myEmail@example.com' });
 // Assuming 'user' is an instance of User
 user.email = 'newEmail@example.com';
 await user.save();  // Update the user in the database
+
+// Upsert operations (insert if not exists, update if exists)
+const upsertedUser = await User.upsert(
+  { id: 'user-123' },  // Query to find the user
+  {                    // Fields to update or insert
+    username: 'newUsername', 
+    email: 'newEmail@example.com'
+  }
+);
 ```
 
 #### Delete
@@ -87,13 +167,311 @@ await user.save();  // Update the user in the database
 await user.delete();  // Delete the user from the database
 ```
 
-### Advanced Usage
-`@push.rocks/smartdata` also supports advanced features like watching for real-time changes in the database, handling distributed data coordination, and more. These features utilize MongoDB's capabilities to provide real-time data syncing and distributed systems coordination.
+## Advanced Features
 
-### Conclusion
-With its focus on TypeScript, modern JavaScript syntax, and leveraging MongoDB's features, `@push.rocks/smartdata` offers a powerful toolkit for data handling and operations management in Node.js applications. Its design for ease of use, coupled with advanced features, makes it a versatile choice for developers looking to build efficient and scalable data-driven applications.
+### EasyStore
+EasyStore provides a simple key-value storage system with automatic persistence:
 
-For more details on usage and additional features, refer to the [official documentation](https://gitlab.com/push.rocks/smartdata#README) and explore the various classes and methods provided by `@push.rocks/smartdata`.
+```typescript
+// Create an EasyStore instance with a specific type
+interface ConfigStore {
+  apiKey: string;
+  settings: {
+    theme: string;
+    notifications: boolean;
+  };
+}
+
+// Create a type-safe EasyStore
+const store = await db.createEasyStore<ConfigStore>('app-config');
+
+// Write and read data with full type safety
+await store.writeKey('apiKey', 'secret-api-key-123');
+await store.writeKey('settings', { theme: 'dark', notifications: true });
+
+const apiKey = await store.readKey('apiKey'); // Type: string
+const settings = await store.readKey('settings'); // Type: { theme: string, notifications: boolean }
+
+// Check if a key exists
+const hasKey = await store.hasKey('apiKey'); // true
+
+// Delete a key
+await store.deleteKey('apiKey');
+```
+
+### Distributed Coordination
+Built-in support for distributed systems with leader election:
+
+```typescript
+// Create a distributed coordinator
+const coordinator = new SmartdataDistributedCoordinator(db);
+
+// Start coordination
+await coordinator.start();
+
+// Handle leadership changes
+coordinator.on('leadershipChange', (isLeader) => {
+  if (isLeader) {
+    // This instance is now the leader
+    // Run leader-specific tasks
+    startPeriodicJobs();
+  } else {
+    // This instance is no longer the leader
+    stopPeriodicJobs();
+  }
+});
+
+// Access leadership status anytime
+if (coordinator.isLeader) {
+  // Run leader-only operations
+}
+
+// Execute a task only on the leader
+await coordinator.executeIfLeader(async () => {
+  // This code only runs on the leader instance
+  await runImportantTask();
+});
+
+// Stop coordination when shutting down
+await coordinator.stop();
+```
+
+### Real-time Data Watching
+Watch for changes in your collections with RxJS integration using MongoDB Change Streams:
+
+```typescript
+// Create a watcher for a specific collection with a query filter
+const watcher = await User.watch({
+  active: true  // Only watch for changes to active users
+}, {
+  fullDocument: true,    // Include the full document in change notifications
+  bufferTimeMs: 100      // Buffer changes for 100ms to reduce notification frequency
+});
+
+// Subscribe to changes using RxJS
+watcher.changeSubject.subscribe((change) => {
+  console.log('Change operation:', change.operationType); // 'insert', 'update', 'delete', etc.
+  console.log('Document changed:', change.docInstance);   // The full document instance
+  
+  // Handle different types of changes
+  if (change.operationType === 'insert') {
+    console.log('New user created:', change.docInstance.username);
+  } else if (change.operationType === 'update') {
+    console.log('User updated:', change.docInstance.username);
+  } else if (change.operationType === 'delete') {
+    console.log('User deleted');
+  }
+});
+
+// Manual observation with event emitter pattern
+watcher.on('change', (change) => {
+  console.log('Document changed:', change);
+});
+
+// Stop watching when no longer needed
+await watcher.stop();
+```
+
+### Managed Collections
+For more complex data models that require additional context:
+
+```typescript
+@Collection(() => db)
+class ManagedDoc extends SmartDataDbDoc<ManagedDoc, ManagedDoc> {
+  @unI()
+  public id: string = 'unique-id';
+  
+  @svDb()
+  public data: string;
+  
+  @managed()
+  public manager: YourCustomManager;
+  
+  // The manager can provide additional functionality
+  async specialOperation() {
+    return this.manager.doSomethingSpecial(this);
+  }
+}
+```
+
+### Automatic Indexing
+Define indexes directly in your model class:
+
+```typescript
+@Collection(() => db)
+class Product extends SmartDataDbDoc<Product, Product> {
+  @unI() // Unique index
+  public id: string = 'product-id';
+  
+  @svDb()
+  @index() // Regular index for faster queries
+  public category: string;
+  
+  @svDb()
+  @index({ sparse: true }) // Sparse index with options
+  public optionalField?: string;
+  
+  // Compound indexes can be defined in the collection decorator
+  @Collection(() => db, {
+    indexMap: {
+      compoundIndex: { 
+        fields: { category: 1, name: 1 },
+        options: { background: true }
+      }
+    }
+  })
+}
+```
+
+### Transaction Support
+Use MongoDB transactions for atomic operations:
+
+```typescript
+const session = await db.startSession();
+try {
+  await session.withTransaction(async () => {
+    const user = await User.getInstance({ id: 'user-id' }, { session });
+    user.balance -= 100;
+    await user.save({ session });
+    
+    const recipient = await User.getInstance({ id: 'recipient-id' }, { session });
+    recipient.balance += 100;
+    await user.save({ session });
+  });
+} finally {
+  await session.endSession();
+}
+```
+
+### Deep Object Queries
+SmartData provides fully type-safe deep property queries with the `DeepQuery` type:
+
+```typescript
+// If your document has nested objects
+class UserProfile extends SmartDataDbDoc<UserProfile, UserProfile> {
+  @unI()
+  public id: string = 'profile-id';
+  
+  @svDb()
+  public user: {
+    details: {
+      firstName: string;
+      lastName: string;
+      address: {
+        city: string;
+        country: string;
+      }
+    }
+  };
+}
+
+// Type-safe string literals for dot notation
+const usersInUSA = await UserProfile.getInstances({
+  'user.details.address.country': 'USA'
+});
+
+// Fully typed deep queries with the DeepQuery type
+import { DeepQuery } from '@push.rocks/smartdata';
+
+const typedQuery: DeepQuery<UserProfile> = {
+  id: 'profile-id',
+  'user.details.firstName': 'John',
+  'user.details.address.country': 'USA'
+};
+
+// TypeScript will error if paths are incorrect
+const results = await UserProfile.getInstances(typedQuery);
+
+// MongoDB query operators are supported
+const operatorQuery: DeepQuery<UserProfile> = {
+  'user.details.address.country': 'USA',
+  'user.details.address.city': { $in: ['New York', 'Los Angeles'] }
+};
+
+const filteredResults = await UserProfile.getInstances(operatorQuery);
+```
+
+### Document Lifecycle Hooks
+Implement custom logic at different stages of a document's lifecycle:
+
+```typescript
+@Collection(() => db)
+class Order extends SmartDataDbDoc<Order, Order> {
+  @unI()
+  public id: string = 'order-id';
+  
+  @svDb()
+  public total: number;
+  
+  @svDb()
+  public items: string[];
+  
+  // Called before saving the document
+  async beforeSave() {
+    // Calculate total based on items
+    this.total = await calculateTotal(this.items);
+    
+    // Validate the document
+    if (this.items.length === 0) {
+      throw new Error('Order must have at least one item');
+    }
+  }
+  
+  // Called after the document is saved
+  async afterSave() {
+    // Notify other systems about the saved order
+    await notifyExternalSystems(this);
+  }
+  
+  // Called before deleting the document
+  async beforeDelete() {
+    // Check if order can be deleted
+    const canDelete = await checkOrderDeletable(this.id);
+    if (!canDelete) {
+      throw new Error('Order cannot be deleted');
+    }
+  }
+}
+```
+
+## Best Practices
+
+### Connection Management
+- Always call `db.init()` before using any database features
+- Use `db.disconnect()` when shutting down your application
+- Set appropriate connection pool sizes based on your application's needs
+
+### Document Design
+- Use appropriate decorators (`@svDb`, `@unI`, `@index`) to optimize database operations
+- Implement type-safe models by properly extending `SmartDataDbDoc`
+- Consider using interfaces to define document structures separately from implementation
+
+### Performance Optimization
+- Use cursors for large datasets instead of loading all documents into memory
+- Create appropriate indexes for frequent query patterns
+- Use projections to limit the fields returned when you don't need the entire document
+
+### Distributed Systems
+- Implement proper error handling for leader election events
+- Ensure all instances have synchronized clocks when using time-based coordination
+- Use the distributed coordinator's task management features for coordinated operations
+
+### Type Safety
+- Take advantage of the `DeepQuery<T>` type for fully type-safe queries
+- Define proper types for your document models to enhance IDE auto-completion
+- Use generic type parameters to specify exact document types when working with collections
+
+## Contributing
+
+We welcome contributions to @push.rocks/smartdata! Here's how you can help:
+
+1. Fork the repository
+2. Create a feature branch (`git checkout -b feature/amazing-feature`)
+3. Commit your changes (`git commit -m 'Add amazing feature'`)
+4. Push to the branch (`git push origin feature/amazing-feature`)
+5. Open a Pull Request
+
+Please make sure to update tests as appropriate and follow our coding standards.
 
 ## License and Legal Information
 

test/test.distributedcoordinator.ts

@@ -3,7 +3,7 @@ import * as smartmongo from '@push.rocks/smartmongo';
 import type * as taskbuffer from '@push.rocks/taskbuffer';
 
 import * as smartdata from '../ts/index.js';
-import { SmartdataDistributedCoordinator, DistributedClass } from '../ts/smartdata.classes.distributedcoordinator.js'; // path might need adjusting
+import { SmartdataDistributedCoordinator, DistributedClass } from '../ts/classes.distributedcoordinator.js'; // path might need adjusting
 const totalInstances = 10;
 
 // =======================================

test/test.easystore.ts

@@ -1,7 +1,7 @@
 import { tap, expect } from '@push.rocks/tapbundle';
 import { Qenv } from '@push.rocks/qenv';
 import * as smartmongo from '@push.rocks/smartmongo';
-import { smartunique } from '../ts/smartdata.plugins.js';
+import { smartunique } from '../ts/plugins.js';
 
 const testQenv = new Qenv(process.cwd(), process.cwd() + '/.nogit/');
 

test/test.ts

@@ -1,7 +1,7 @@
 import { tap, expect } from '@push.rocks/tapbundle';
 import { Qenv } from '@push.rocks/qenv';
 import * as smartmongo from '@push.rocks/smartmongo';
-import { smartunique } from '../ts/smartdata.plugins.js';
+import { smartunique } from '../ts/plugins.js';
 
 import * as mongodb from 'mongodb';
 
@@ -199,12 +199,18 @@ tap.test('should store a new Truck', async () => {
   const truck = new Truck('blue', 'MAN');
   await truck.save();
   const myTruck2 = await Truck.getInstance({ color: 'blue' });
+  expect(myTruck2.color).toEqual('blue');
   myTruck2.color = 'red';
   await myTruck2.save();
   const myTruck3 = await Truck.getInstance({ color: 'blue' });
-  console.log(myTruck3);
+  expect(myTruck3).toBeNull();
 });
 
+tap.test('should return a count', async () => {
+  const truckCount = await Truck.getCount();
+  expect(truckCount).toEqual(1);
+})
+
 tap.test('should use a cursor', async () => {
   const cursor = await Car.getCursor({});
   let counter = 0;

test/test.typescript.ts

@@ -1,7 +1,7 @@
 import { tap, expect } from '@push.rocks/tapbundle';
 import { Qenv } from '@push.rocks/qenv';
 import * as smartmongo from '@push.rocks/smartmongo';
-import { smartunique } from '../ts/smartdata.plugins.js';
+import { smartunique } from '../ts/plugins.js';
 
 const testQenv = new Qenv(process.cwd(), process.cwd() + '/.nogit/');
 

test/test.watch.ts

@@ -1,7 +1,7 @@
 import { tap, expect } from '@push.rocks/tapbundle';
 import { Qenv } from '@push.rocks/qenv';
 import * as smartmongo from '@push.rocks/smartmongo';
-import { smartunique } from '../ts/smartdata.plugins.js';
+import { smartunique } from '../ts/plugins.js';
 
 const testQenv = new Qenv(process.cwd(), process.cwd() + '/.nogit/');
 

ts/00_commitinfo_data.ts

@@ -1,8 +1,8 @@
 /**
- * autocreated commitinfo by @pushrocks/commitinfo
+ * autocreated commitinfo by @push.rocks/commitinfo
  */
 export const commitinfo = {
   name: '@push.rocks/smartdata',
-  version: '5.1.2',
+  version: '5.4.0',
   description: 'An advanced library for NoSQL data organization and manipulation using TypeScript with support for MongoDB, data validation, collections, and custom data types.'
 }

ts/classes.collection.ts

@@ -1,9 +1,9 @@
-import * as plugins from './smartdata.plugins.js';
-import { SmartdataDb } from './smartdata.classes.db.js';
-import { SmartdataDbCursor } from './smartdata.classes.cursor.js';
-import { SmartDataDbDoc } from './smartdata.classes.doc.js';
-import { SmartdataDbWatcher } from './smartdata.classes.watcher.js';
-import { CollectionFactory } from './smartdata.classes.collectionfactory.js';
+import * as plugins from './plugins.js';
+import { SmartdataDb } from './classes.db.js';
+import { SmartdataDbCursor } from './classes.cursor.js';
+import { SmartDataDbDoc } from './classes.doc.js';
+import { SmartdataDbWatcher } from './classes.watcher.js';
+import { CollectionFactory } from './classes.collectionfactory.js';
 
 export interface IFindOptions {
   limit?: number;
@@ -273,6 +273,11 @@ export class SmartdataCollection<T> {
     await this.mongoDbCollection.deleteOne(identifiableObject);
   }
 
+  public async getCount(filterObject: any) {
+    await this.init();
+    return this.mongoDbCollection.countDocuments(filterObject);
+  }
+
   /**
    * checks a Doc for constraints
    * if this.objectValidation is not set it passes.

ts/classes.collectionfactory.ts

@@ -1,6 +1,6 @@
-import * as plugins from './smartdata.plugins.js';
-import { SmartdataCollection } from './smartdata.classes.collection.js';
-import { SmartdataDb } from './smartdata.classes.db.js';
+import * as plugins from './plugins.js';
+import { SmartdataCollection } from './classes.collection.js';
+import { SmartdataDb } from './classes.db.js';
 
 export class CollectionFactory {
   public collections: { [key: string]: SmartdataCollection<any> } = {};

ts/classes.convenience.ts

@@ -1,4 +1,4 @@
-import * as plugins from './smartdata.plugins.js';
+import * as plugins from './plugins.js';
 
 export const getNewUniqueId = async (prefixArg?: string) => {
   return plugins.smartunique.uni(prefixArg);

ts/classes.cursor.ts

@@ -1,5 +1,5 @@
-import { SmartDataDbDoc } from './smartdata.classes.doc.js';
-import * as plugins from './smartdata.plugins.js';
+import { SmartDataDbDoc } from './classes.doc.js';
+import * as plugins from './plugins.js';
 
 /**
  * a wrapper for the native mongodb cursor. Exposes better

ts/classes.db.ts

@@ -1,9 +1,9 @@
-import * as plugins from './smartdata.plugins.js';
+import * as plugins from './plugins.js';
 
-import { SmartdataCollection } from './smartdata.classes.collection.js';
-import { EasyStore } from './smartdata.classes.easystore.js';
+import { SmartdataCollection } from './classes.collection.js';
+import { EasyStore } from './classes.easystore.js';
 
-import { logger } from './smartdata.logging.js';
+import { logger } from './logging.js';
 
 /**
  * interface - indicates the connection status of the db

ts/classes.distributedcoordinator.ts

@@ -1,8 +1,8 @@
-import * as plugins from './smartdata.plugins.js';
-import { SmartdataDb } from './smartdata.classes.db.js';
-import { managed, setDefaultManagerForDoc } from './smartdata.classes.collection.js';
-import { SmartDataDbDoc, svDb, unI } from './smartdata.classes.doc.js';
-import { SmartdataDbWatcher } from './smartdata.classes.watcher.js';
+import * as plugins from './plugins.js';
+import { SmartdataDb } from './classes.db.js';
+import { managed, setDefaultManagerForDoc } from './classes.collection.js';
+import { SmartDataDbDoc, svDb, unI } from './classes.doc.js';
+import { SmartdataDbWatcher } from './classes.watcher.js';
 
 @managed()
 export class DistributedClass extends SmartDataDbDoc<DistributedClass, DistributedClass> {

ts/classes.doc.ts

@@ -1,12 +1,26 @@
-import * as plugins from './smartdata.plugins.js';
+import * as plugins from './plugins.js';
 
-import { SmartdataDb } from './smartdata.classes.db.js';
-import { SmartdataDbCursor } from './smartdata.classes.cursor.js';
-import { type IManager, SmartdataCollection } from './smartdata.classes.collection.js';
-import { SmartdataDbWatcher } from './smartdata.classes.watcher.js';
+import { SmartdataDb } from './classes.db.js';
+import { SmartdataDbCursor } from './classes.cursor.js';
+import { type IManager, SmartdataCollection } from './classes.collection.js';
+import { SmartdataDbWatcher } from './classes.watcher.js';
+import { SmartdataLuceneAdapter } from './classes.lucene.adapter.js';
 
 export type TDocCreation = 'db' | 'new' | 'mixed';
 
+// Set of searchable fields for each class
+const searchableFieldsMap = new Map<string, Set<string>>();
+
+export function globalSvDb() {
+  return (target: SmartDataDbDoc<unknown, unknown>, key: string) => {
+    console.log(`called svDb() on >${target.constructor.name}.${key}<`);
+    if (!target.globalSaveableProperties) {
+      target.globalSaveableProperties = [];
+    }
+    target.globalSaveableProperties.push(key);
+  };
+}
+
 /**
  * saveable - saveable decorator to be used on class properties
  */
@@ -20,6 +34,34 @@ export function svDb() {
   };
 }
 
+/**
+ * searchable - marks a property as searchable with Lucene query syntax
+ */
+export function searchable() {
+  return (target: SmartDataDbDoc<unknown, unknown>, key: string) => {
+    console.log(`called searchable() on >${target.constructor.name}.${key}<`);
+    
+    // Initialize the set for this class if it doesn't exist
+    const className = target.constructor.name;
+    if (!searchableFieldsMap.has(className)) {
+      searchableFieldsMap.set(className, new Set<string>());
+    }
+    
+    // Add the property to the searchable fields set
+    searchableFieldsMap.get(className).add(key);
+  };
+}
+
+/**
+ * Get searchable fields for a class
+ */
+export function getSearchableFields(className: string): string[] {
+  if (!searchableFieldsMap.has(className)) {
+    return [];
+  }
+  return Array.from(searchableFieldsMap.get(className));
+}
+
 /**
  * unique index - decorator to mark a unique index
  */
@@ -43,8 +85,12 @@ export function unI() {
 
 export const convertFilterForMongoDb = (filterArg: { [key: string]: any }) => {
   const convertedFilter: { [key: string]: any } = {};
+
   const convertFilterArgument = (keyPathArg2: string, filterArg2: any) => {
-    if (typeof filterArg2 === 'object') {
+    if (Array.isArray(filterArg2)) {
+      // Directly assign arrays (they might be using operators like $in or $all)
+      convertFilterArgument(keyPathArg2, filterArg2[0]);
+    } else if (typeof filterArg2 === 'object' && filterArg2 !== null) {
       for (const key of Object.keys(filterArg2)) {
         if (key.startsWith('$')) {
           convertedFilter[keyPathArg2] = filterArg2;
@@ -60,6 +106,7 @@ export const convertFilterForMongoDb = (filterArg: { [key: string]: any }) => {
       convertedFilter[keyPathArg2] = filterArg2;
     }
   };
+
   for (const key of Object.keys(filterArg)) {
     convertFilterArgument(key, filterArg[key]);
   }
@@ -181,6 +228,50 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
     await cursor.forEach(forEachFunction);
   }
 
+  /**
+   * returns a count of the documents in the collection
+   */
+  public static async getCount<T>(
+    this: plugins.tsclass.typeFest.Class<T>,
+    filterArg: plugins.tsclass.typeFest.PartialDeep<T> = ({} as any)
+  ) {
+    const collection: SmartdataCollection<T> = (this as any).collection;
+    return await collection.getCount(filterArg);
+  }
+
+  /**
+   * Create a MongoDB filter from a Lucene query string
+   * @param luceneQuery Lucene query string
+   * @returns MongoDB query object
+   */
+  public static createSearchFilter<T>(
+    this: plugins.tsclass.typeFest.Class<T>,
+    luceneQuery: string
+  ): any {
+    const className = (this as any).className || this.name;
+    const searchableFields = getSearchableFields(className);
+    
+    if (searchableFields.length === 0) {
+      throw new Error(`No searchable fields defined for class ${className}`);
+    }
+    
+    const adapter = new SmartdataLuceneAdapter(searchableFields);
+    return adapter.convert(luceneQuery);
+  }
+
+  /**
+   * Search documents using Lucene query syntax
+   * @param luceneQuery Lucene query string
+   * @returns Array of matching documents
+   */
+  public static async search<T>(
+    this: plugins.tsclass.typeFest.Class<T>,
+    luceneQuery: string
+  ): Promise<T[]> {
+    const filter = (this as any).createSearchFilter(luceneQuery);
+    return await (this as any).getInstances(filter);
+  }
+
   // INSTANCE
 
   /**
@@ -191,22 +282,27 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
   /**
    * updated from db in any case where doc comes from db
    */
-  @svDb()
+  @globalSvDb()
   _createdAt: string = (new Date()).toISOString();
 
   /**
    * will be updated everytime the doc is saved
    */
-  @svDb()
+  @globalSvDb()
   _updatedAt: string = (new Date()).toISOString();
 
+  /**
+   * an array of saveable properties of ALL doc
+   */
+  public globalSaveableProperties: string[];
+
   /**
    * unique indexes
    */
   public uniqueIndexes: string[];
 
   /**
-   * an array of saveable properties of a doc
+   * an array of saveable properties of a specific doc
    */
   public saveableProperties: string[];
 
@@ -290,7 +386,11 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
    */
   public async createSavableObject(): Promise<TImplements> {
     const saveableObject: unknown = {}; // is not exposed to outside, so any is ok here
-    for (const propertyNameString of this.saveableProperties) {
+    const saveableProperties = [
+      ...this.globalSaveableProperties,
+      ...this.saveableProperties
+    ]
+    for (const propertyNameString of saveableProperties) {
       saveableObject[propertyNameString] = this[propertyNameString];
     }
     return saveableObject as TImplements;

ts/classes.easystore.ts

@@ -1,7 +1,7 @@
-import * as plugins from './smartdata.plugins.js';
-import { Collection } from './smartdata.classes.collection.js';
-import { SmartdataDb } from './smartdata.classes.db.js';
-import { SmartDataDbDoc, svDb, unI } from './smartdata.classes.doc.js';
+import * as plugins from './plugins.js';
+import { Collection } from './classes.collection.js';
+import { SmartdataDb } from './classes.db.js';
+import { SmartDataDbDoc, svDb, unI } from './classes.doc.js';
 
 /**
  * EasyStore allows the storage of easy objects. It also allows easy sharing of the object between different instances

ts/classes.lucene.adapter.ts

@@ -0,0 +1,643 @@
+/**
+ * Lucene to MongoDB query adapter for SmartData
+ */
+import * as plugins from './plugins.js';
+
+// Types
+type NodeType = 'TERM' | 'PHRASE' | 'FIELD' | 'AND' | 'OR' | 'NOT' | 'RANGE' | 'WILDCARD' | 'FUZZY' | 'GROUP';
+
+interface QueryNode {
+  type: NodeType;
+}
+
+interface TermNode extends QueryNode {
+  type: 'TERM';
+  value: string;
+  boost?: number;
+}
+
+interface PhraseNode extends QueryNode {
+  type: 'PHRASE';
+  value: string;
+  proximity?: number;
+}
+
+interface FieldNode extends QueryNode {
+  type: 'FIELD';
+  field: string;
+  value: AnyQueryNode;
+}
+
+interface BooleanNode extends QueryNode {
+  type: 'AND' | 'OR' | 'NOT';
+  left: AnyQueryNode;
+  right: AnyQueryNode;
+}
+
+interface RangeNode extends QueryNode {
+  type: 'RANGE';
+  field: string;
+  lower: string;
+  upper: string;
+  includeLower: boolean;
+  includeUpper: boolean;
+}
+
+interface WildcardNode extends QueryNode {
+  type: 'WILDCARD';
+  value: string;
+}
+
+interface FuzzyNode extends QueryNode {
+  type: 'FUZZY';
+  value: string;
+  maxEdits: number;
+}
+
+interface GroupNode extends QueryNode {
+  type: 'GROUP';
+  value: AnyQueryNode;
+}
+
+type AnyQueryNode = TermNode | PhraseNode | FieldNode | BooleanNode | RangeNode | WildcardNode | FuzzyNode | GroupNode;
+
+/**
+ * Lucene query parser
+ */
+export class LuceneParser {
+  private pos: number = 0;
+  private input: string = '';
+  private tokens: string[] = [];
+  
+  constructor() {}
+  
+  /**
+   * Parse a Lucene query string into an AST
+   */
+  parse(query: string): AnyQueryNode {
+    this.input = query.trim();
+    this.pos = 0;
+    this.tokens = this.tokenize(this.input);
+    
+    return this.parseQuery();
+  }
+  
+  /**
+   * Tokenize the input string into tokens
+   */
+  private tokenize(input: string): string[] {
+    const specialChars = /[()\[\]{}"~^:]/;
+    const operators = /AND|OR|NOT|TO/;
+    
+    let tokens: string[] = [];
+    let current = '';
+    let inQuote = false;
+    
+    for (let i = 0; i < input.length; i++) {
+      const char = input[i];
+      
+      // Handle quoted strings
+      if (char === '"') {
+        if (inQuote) {
+          tokens.push(current + char);
+          current = '';
+          inQuote = false;
+        } else {
+          if (current) tokens.push(current);
+          current = char;
+          inQuote = true;
+        }
+        continue;
+      }
+      
+      if (inQuote) {
+        current += char;
+        continue;
+      }
+      
+      // Handle whitespace
+      if (char === ' ' || char === '\t' || char === '\n') {
+        if (current) {
+          tokens.push(current);
+          current = '';
+        }
+        continue;
+      }
+      
+      // Handle special characters
+      if (specialChars.test(char)) {
+        if (current) {
+          tokens.push(current);
+          current = '';
+        }
+        tokens.push(char);
+        continue;
+      }
+      
+      current += char;
+      
+      // Check if current is an operator
+      if (operators.test(current) && 
+          (i + 1 === input.length || /\s/.test(input[i + 1]))) {
+        tokens.push(current);
+        current = '';
+      }
+    }
+    
+    if (current) tokens.push(current);
+    
+    return tokens;
+  }
+  
+  /**
+   * Parse the main query expression
+   */
+  private parseQuery(): AnyQueryNode {
+    const left = this.parseBooleanOperand();
+    
+    if (this.pos < this.tokens.length) {
+      const token = this.tokens[this.pos];
+      
+      if (token === 'AND' || token === 'OR') {
+        this.pos++;
+        const right = this.parseQuery();
+        return {
+          type: token as 'AND' | 'OR',
+          left,
+          right
+        };
+      } else if (token === 'NOT' || token === '-') {
+        this.pos++;
+        const right = this.parseQuery();
+        return {
+          type: 'NOT',
+          left,
+          right
+        };
+      }
+    }
+    
+    return left;
+  }
+  
+  /**
+   * Parse boolean operands (terms, phrases, fields, groups)
+   */
+  private parseBooleanOperand(): AnyQueryNode {
+    if (this.pos >= this.tokens.length) {
+      throw new Error('Unexpected end of input');
+    }
+    
+    const token = this.tokens[this.pos];
+    
+    // Handle grouping with parentheses
+    if (token === '(') {
+      this.pos++;
+      const group = this.parseQuery();
+      
+      if (this.pos < this.tokens.length && this.tokens[this.pos] === ')') {
+        this.pos++;
+        return { type: 'GROUP', value: group } as GroupNode;
+      } else {
+        throw new Error('Unclosed group');
+      }
+    }
+    
+    // Handle fields (field:value)
+    if (this.pos + 1 < this.tokens.length && this.tokens[this.pos + 1] === ':') {
+      const field = token;
+      this.pos += 2; // Skip field and colon
+      
+      if (this.pos < this.tokens.length) {
+        const value = this.parseBooleanOperand();
+        return { type: 'FIELD', field, value } as FieldNode;
+      } else {
+        throw new Error('Expected value after field');
+      }
+    }
+    
+    // Handle range queries
+    if (token === '[' || token === '{') {
+      return this.parseRange();
+    }
+    
+    // Handle phrases ("term term")
+    if (token.startsWith('"') && token.endsWith('"')) {
+      const phrase = token.slice(1, -1);
+      this.pos++;
+      
+      // Check for proximity operator
+      let proximity: number | undefined;
+      if (this.pos < this.tokens.length && this.tokens[this.pos] === '~') {
+        this.pos++;
+        if (this.pos < this.tokens.length && /^\d+$/.test(this.tokens[this.pos])) {
+          proximity = parseInt(this.tokens[this.pos], 10);
+          this.pos++;
+        } else {
+          throw new Error('Expected number after proximity operator');
+        }
+      }
+      
+      return { type: 'PHRASE', value: phrase, proximity } as PhraseNode;
+    }
+    
+    // Handle wildcards
+    if (token.includes('*') || token.includes('?')) {
+      this.pos++;
+      return { type: 'WILDCARD', value: token } as WildcardNode;
+    }
+    
+    // Handle fuzzy searches
+    if (this.pos + 1 < this.tokens.length && this.tokens[this.pos + 1] === '~') {
+      const term = token;
+      this.pos += 2; // Skip term and tilde
+      
+      let maxEdits = 2; // Default
+      if (this.pos < this.tokens.length && /^\d+$/.test(this.tokens[this.pos])) {
+        maxEdits = parseInt(this.tokens[this.pos], 10);
+        this.pos++;
+      }
+      
+      return { type: 'FUZZY', value: term, maxEdits } as FuzzyNode;
+    }
+    
+    // Simple term
+    this.pos++;
+    return { type: 'TERM', value: token } as TermNode;
+  }
+  
+  /**
+   * Parse range queries
+   */
+  private parseRange(): RangeNode {
+    const includeLower = this.tokens[this.pos] === '[';
+    const includeUpper = this.tokens[this.pos + 4] === ']';
+    
+    this.pos++; // Skip open bracket
+    
+    if (this.pos + 4 >= this.tokens.length) {
+      throw new Error('Invalid range query syntax');
+    }
+    
+    const lower = this.tokens[this.pos];
+    this.pos++;
+    
+    if (this.tokens[this.pos] !== 'TO') {
+      throw new Error('Expected TO in range query');
+    }
+    this.pos++;
+    
+    const upper = this.tokens[this.pos];
+    this.pos++;
+    
+    if (this.tokens[this.pos] !== (includeLower ? ']' : '}')) {
+      throw new Error('Invalid range query closing bracket');
+    }
+    this.pos++;
+    
+    // For simplicity, assuming the field is handled separately
+    return {
+      type: 'RANGE',
+      field: '', // This will be filled by the field node
+      lower,
+      upper,
+      includeLower,
+      includeUpper
+    };
+  }
+}
+
+/**
+ * Transformer for Lucene AST to MongoDB query
+ */
+export class LuceneToMongoTransformer {
+  constructor() {}
+  
+  /**
+   * Transform a Lucene AST node to a MongoDB query
+   */
+  transform(node: AnyQueryNode, searchFields?: string[]): any {
+    switch (node.type) {
+      case 'TERM':
+        return this.transformTerm(node, searchFields);
+      case 'PHRASE':
+        return this.transformPhrase(node, searchFields);
+      case 'FIELD':
+        return this.transformField(node);
+      case 'AND':
+        return this.transformAnd(node);
+      case 'OR':
+        return this.transformOr(node);
+      case 'NOT':
+        return this.transformNot(node);
+      case 'RANGE':
+        return this.transformRange(node);
+      case 'WILDCARD':
+        return this.transformWildcard(node, searchFields);
+      case 'FUZZY':
+        return this.transformFuzzy(node, searchFields);
+      case 'GROUP':
+        return this.transform(node.value, searchFields);
+      default:
+        throw new Error(`Unsupported node type: ${(node as any).type}`);
+    }
+  }
+  
+  /**
+   * Transform a term to MongoDB query
+   */
+  private transformTerm(node: TermNode, searchFields?: string[]): any {
+    // If specific fields are provided, search across those fields
+    if (searchFields && searchFields.length > 0) {
+      // Create an $or query to search across multiple fields
+      return {
+        $or: searchFields.map(field => ({
+          [field]: { $regex: node.value, $options: 'i' }
+        }))
+      };
+    }
+    
+    // Otherwise, use text search (requires a text index on desired fields)
+    return { $text: { $search: node.value } };
+  }
+  
+  /**
+   * Transform a phrase to MongoDB query
+   */
+  private transformPhrase(node: PhraseNode, searchFields?: string[]): any {
+    // If specific fields are provided, search phrase across those fields
+    if (searchFields && searchFields.length > 0) {
+      // Create an $or query to search phrase across multiple fields
+      return {
+        $or: searchFields.map(field => ({
+          [field]: { $regex: `${node.value.replace(/\s+/g, '\\s+')}`, $options: 'i' }
+        }))
+      };
+    }
+    
+    // For phrases, we use a regex to ensure exact matches
+    return { $text: { $search: `"${node.value}"` } };
+  }
+  
+  /**
+   * Transform a field query to MongoDB query
+   */
+  private transformField(node: FieldNode): any {
+    // Handle special case for range queries on fields
+    if (node.value.type === 'RANGE') {
+      const rangeNode = node.value as RangeNode;
+      rangeNode.field = node.field;
+      return this.transformRange(rangeNode);
+    }
+    
+    // Handle special case for wildcards on fields
+    if (node.value.type === 'WILDCARD') {
+      return { 
+        [node.field]: { 
+          $regex: this.luceneWildcardToRegex((node.value as WildcardNode).value),
+          $options: 'i'
+        } 
+      };
+    }
+    
+    // Handle special case for fuzzy searches on fields
+    if (node.value.type === 'FUZZY') {
+      return { 
+        [node.field]: { 
+          $regex: this.createFuzzyRegex((node.value as FuzzyNode).value),
+          $options: 'i'
+        } 
+      };
+    }
+    
+    // Special case for exact term matches on fields
+    if (node.value.type === 'TERM') {
+      return { [node.field]: (node.value as TermNode).value };
+    }
+    
+    // Special case for phrase matches on fields
+    if (node.value.type === 'PHRASE') {
+      return { 
+        [node.field]: { 
+          $regex: `^${(node.value as PhraseNode).value}$`,
+          $options: 'i'
+        } 
+      };
+    }
+    
+    // For other cases, we'll transform the value and apply it to the field
+    const transformedValue = this.transform(node.value);
+    
+    // If the transformed value uses $text, we need to adapt it for the field
+    if (transformedValue.$text) {
+      return { [node.field]: transformedValue.$text.$search };
+    }
+    
+    return { [node.field]: transformedValue };
+  }
+  
+  /**
+   * Transform AND operator to MongoDB query
+   */
+  private transformAnd(node: BooleanNode): any {
+    return { $and: [this.transform(node.left), this.transform(node.right)] };
+  }
+  
+  /**
+   * Transform OR operator to MongoDB query
+   */
+  private transformOr(node: BooleanNode): any {
+    return { $or: [this.transform(node.left), this.transform(node.right)] };
+  }
+  
+  /**
+   * Transform NOT operator to MongoDB query
+   */
+  private transformNot(node: BooleanNode): any {
+    const leftQuery = this.transform(node.left);
+    const rightQuery = this.transform(node.right);
+    
+    // Create a query that includes left but excludes right
+    if (rightQuery.$text) {
+      // Text searches need special handling for negation
+      return {
+        $and: [
+          leftQuery,
+          { $not: rightQuery }
+        ]
+      };
+    } else {
+      // For other queries, we can use $not directly
+      return {
+        $and: [
+          leftQuery,
+          { $not: rightQuery }
+        ]
+      };
+    }
+  }
+  
+  /**
+   * Transform range query to MongoDB query
+   */
+  private transformRange(node: RangeNode): any {
+    const range: any = {};
+    
+    if (node.lower !== '*') {
+      range[node.includeLower ? '$gte' : '$gt'] = this.parseValue(node.lower);
+    }
+    
+    if (node.upper !== '*') {
+      range[node.includeUpper ? '$lte' : '$lt'] = this.parseValue(node.upper);
+    }
+    
+    return { [node.field]: range };
+  }
+  
+  /**
+   * Transform wildcard query to MongoDB query
+   */
+  private transformWildcard(node: WildcardNode, searchFields?: string[]): any {
+    // Convert Lucene wildcards to MongoDB regex
+    const regex = this.luceneWildcardToRegex(node.value);
+    
+    // If specific fields are provided, search wildcard across those fields
+    if (searchFields && searchFields.length > 0) {
+      return {
+        $or: searchFields.map(field => ({
+          [field]: { $regex: regex, $options: 'i' }
+        }))
+      };
+    }
+    
+    // By default, apply to all text fields using $text search
+    return { $regex: regex, $options: 'i' };
+  }
+  
+  /**
+   * Transform fuzzy query to MongoDB query
+   */
+  private transformFuzzy(node: FuzzyNode, searchFields?: string[]): any {
+    // MongoDB doesn't have built-in fuzzy search
+    // This is a very basic approach using regex
+    const regex = this.createFuzzyRegex(node.value);
+    
+    // If specific fields are provided, search fuzzy term across those fields
+    if (searchFields && searchFields.length > 0) {
+      return {
+        $or: searchFields.map(field => ({
+          [field]: { $regex: regex, $options: 'i' }
+        }))
+      };
+    }
+    
+    return { $regex: regex, $options: 'i' };
+  }
+  
+  /**
+   * Convert Lucene wildcards to MongoDB regex patterns
+   */
+  private luceneWildcardToRegex(wildcardPattern: string): string {
+    // Replace Lucene wildcards with regex equivalents
+    // * => .*
+    // ? => .
+    // Also escape regex special chars
+    return wildcardPattern
+      .replace(/([.+^${}()|\\])/g, '\\$1') // Escape regex special chars
+      .replace(/\*/g, '.*')
+      .replace(/\?/g, '.');
+  }
+  
+  /**
+   * Create a simplified fuzzy search regex
+   */
+  private createFuzzyRegex(term: string): string {
+    // For a very simple approach, we allow some characters to be optional
+    let regex = '';
+    for (let i = 0; i < term.length; i++) {
+      // Make every other character optional (simplified fuzzy)
+      if (i % 2 === 1) {
+        regex += term[i] + '?';
+      } else {
+        regex += term[i];
+      }
+    }
+    return regex;
+  }
+  
+  /**
+   * Parse string values to appropriate types (numbers, dates, etc.)
+   */
+  private parseValue(value: string): any {
+    // Try to parse as number
+    if (/^-?\d+$/.test(value)) {
+      return parseInt(value, 10);
+    }
+    
+    if (/^-?\d+\.\d+$/.test(value)) {
+      return parseFloat(value);
+    }
+    
+    // Try to parse as date (simplified)
+    const date = new Date(value);
+    if (!isNaN(date.getTime())) {
+      return date;
+    }
+    
+    // Default to string
+    return value;
+  }
+}
+
+/**
+ * Main adapter class
+ */
+export class SmartdataLuceneAdapter {
+  private parser: LuceneParser;
+  private transformer: LuceneToMongoTransformer;
+  private defaultSearchFields: string[] = [];
+  
+  /**
+   * @param defaultSearchFields - Optional array of field names to search across when no field is specified
+   */
+  constructor(defaultSearchFields?: string[]) {
+    this.parser = new LuceneParser();
+    this.transformer = new LuceneToMongoTransformer();
+    if (defaultSearchFields) {
+      this.defaultSearchFields = defaultSearchFields;
+    }
+  }
+  
+  /**
+   * Convert a Lucene query string to a MongoDB query object
+   * @param luceneQuery - The Lucene query string to convert
+   * @param searchFields - Optional array of field names to search across (overrides defaultSearchFields)
+   */
+  convert(luceneQuery: string, searchFields?: string[]): any {
+    try {
+      // Parse the Lucene query into an AST
+      const ast = this.parser.parse(luceneQuery);
+      
+      // Use provided searchFields, fall back to defaultSearchFields
+      const fieldsToSearch = searchFields || this.defaultSearchFields;
+      
+      // Transform the AST to a MongoDB query
+      return this.transformWithFields(ast, fieldsToSearch);
+    } catch (error) {
+      throw new Error(`Failed to convert Lucene query: ${error}`);
+    }
+  }
+  
+  /**
+   * Helper method to transform the AST with field information
+   */
+  private transformWithFields(node: AnyQueryNode, searchFields: string[]): any {
+    // For term nodes without a specific field, apply the search fields
+    if (node.type === 'TERM') {
+      return this.transformer.transform(node, searchFields);
+    }
+    
+    // For other node types, use the standard transformation
+    return this.transformer.transform(node);
+  }
+}

ts/classes.watcher.ts

@@ -1,5 +1,5 @@
-import { SmartDataDbDoc } from './smartdata.classes.doc.js';
-import * as plugins from './smartdata.plugins.js';
+import { SmartDataDbDoc } from './classes.doc.js';
+import * as plugins from './plugins.js';
 
 /**
  * a wrapper for the native mongodb cursor. Exposes better

ts/index.ts

@@ -1,14 +1,14 @@
-export * from './smartdata.classes.db.js';
-export * from './smartdata.classes.collection.js';
-export * from './smartdata.classes.doc.js';
-export * from './smartdata.classes.easystore.js';
-export * from './smartdata.classes.cursor.js';
+export * from './classes.db.js';
+export * from './classes.collection.js';
+export * from './classes.doc.js';
+export * from './classes.easystore.js';
+export * from './classes.cursor.js';
 
-import * as convenience from './smartadata.convenience.js';
+import * as convenience from './classes.convenience.js';
 
 export { convenience };
 
 // to be removed with the next breaking update
-import type * as plugins from './smartdata.plugins.js';
+import type * as plugins from './plugins.js';
 type IMongoDescriptor = plugins.tsclass.database.IMongoDescriptor;
 export type { IMongoDescriptor };

ts/logging.ts

@@ -1,3 +1,3 @@
-import * as plugins from './smartdata.plugins.js';
+import * as plugins from './plugins.js';
 
 export const logger = new plugins.smartlog.ConsoleLog();