5.16.0

.gitea/workflows/default_nottags.yaml

@@ -0,0 +1,66 @@
+name: Default (not tags)
+
+on:
+  push:
+    tags-ignore:
+      - '**'
+
+env:
+  IMAGE: code.foss.global/host.today/ht-docker-node:npmci
+  NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@/${{gitea.repository}}.git
+  NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
+  NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
+  NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
+  NPMCI_URL_CLOUDLY: ${{secrets.NPMCI_URL_CLOUDLY}}
+
+jobs:
+  security:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install pnpm and npmci
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @ship.zone/npmci
+
+      - name: Run npm prepare
+        run: npmci npm prepare
+
+      - name: Audit production dependencies
+        run: |
+          npmci command npm config set registry https://registry.npmjs.org
+          npmci command pnpm audit --audit-level=high --prod
+        continue-on-error: true
+
+      - name: Audit development dependencies
+        run: |
+          npmci command npm config set registry https://registry.npmjs.org
+          npmci command pnpm audit --audit-level=high --dev
+        continue-on-error: true
+
+  test:
+    if: ${{ always() }}
+    needs: security
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Test stable
+        run: |
+          npmci node install stable
+          npmci npm install
+          npmci npm test
+
+      - name: Test build
+        run: |
+          npmci node install stable
+          npmci npm install
+          npmci npm build

.gitea/workflows/default_tags.yaml

@@ -0,0 +1,124 @@
+name: Default (tags)
+
+on:
+  push:
+    tags:
+      - '*'
+
+env:
+  IMAGE: code.foss.global/host.today/ht-docker-node:npmci
+  NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@/${{gitea.repository}}.git
+  NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
+  NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
+  NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
+  NPMCI_URL_CLOUDLY: ${{secrets.NPMCI_URL_CLOUDLY}}
+
+jobs:
+  security:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Prepare
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @ship.zone/npmci
+          npmci npm prepare
+
+      - name: Audit production dependencies
+        run: |
+          npmci command npm config set registry https://registry.npmjs.org
+          npmci command pnpm audit --audit-level=high --prod
+        continue-on-error: true
+
+      - name: Audit development dependencies
+        run: |
+          npmci command npm config set registry https://registry.npmjs.org
+          npmci command pnpm audit --audit-level=high --dev
+        continue-on-error: true
+
+  test:
+    if: ${{ always() }}
+    needs: security
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Prepare
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @ship.zone/npmci
+          npmci npm prepare
+
+      - name: Test stable
+        run: |
+          npmci node install stable
+          npmci npm install
+          npmci npm test
+
+      - name: Test build
+        run: |
+          npmci node install stable
+          npmci npm install
+          npmci npm build
+
+  release:
+    needs: test
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Prepare
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @ship.zone/npmci
+          npmci npm prepare
+
+      - name: Release
+        run: |
+          npmci node install stable
+          npmci npm publish
+
+  metadata:
+    needs: test
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+    continue-on-error: true
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Prepare
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @ship.zone/npmci
+          npmci npm prepare
+
+      - name: Code quality
+        run: |
+          npmci command npm install -g typescript
+          npmci npm install
+
+      - name: Trigger
+        run: npmci trigger
+
+      - name: Build docs and upload artifacts
+        run: |
+          npmci node install stable
+          npmci npm install
+          pnpm install -g @git.zone/tsdoc
+          npmci command tsdoc
+        continue-on-error: true

.gitignore

@@ -1,5 +1,19 @@
-node_modules/
-test/data
-pages/
+.nogit/
+
+# artifacts
 coverage/
 public/
+
+# installs
+node_modules/
+
+# caches
+.yarn/
+.cache/
+.rpt2_cache
+
+# builds
+dist/
+dist_*/
+
+#------# custom

.gitlab-ci.yml

@@ -1,43 +0,0 @@
-image: hosttoday/ht-docker-node:npmts
-
-stages:
-- test
-- release
-
-before_script:
-  - apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv EA312927
-  - echo "deb http://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/3.2 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-3.2.list
-  - apt-get update
-  - apt-get install -y mongodb-org
-
-testLEGACY:
-  stage: test
-  script:
-    - npmci test legacy
-  tags:
-    - docker
-  allow_failure: true
-
-testLTS:
-  stage: test
-  script:
-    - npmci test lts
-  tags:
-    - docker
-    
-testSTABLE:
-  stage: test
-  script:
-    - npmci test stable
-  tags:
-    - docker
-
-release:
-  stage: release
-  environment: npm_registry
-  script:
-    - npmci publish
-  only:
-    - tags
-  tags:
-    - docker

.vscode/launch.json

@@ -0,0 +1,11 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "command": "npm test",
+      "name": "Run npm test",
+      "request": "launch",
+      "type": "node-terminal"
+    }
+  ]
+}

.vscode/settings.json

@@ -0,0 +1,26 @@
+{
+  "json.schemas": [
+    {
+      "fileMatch": ["/npmextra.json"],
+      "schema": {
+        "type": "object",
+        "properties": {
+          "npmci": {
+            "type": "object",
+            "description": "settings for npmci"
+          },
+          "gitzone": {
+            "type": "object",
+            "description": "settings for gitzone",
+            "properties": {
+              "projectType": {
+                "type": "string",
+                "enum": ["website", "element", "service", "npm", "wcc"]
+              }
+            }
+          }
+        }
+      }
+    }
+  ]
+}

README.md

@@ -1,63 +0,0 @@
-# smartdata
-
-smartdata is a ODM that adheres to TypeScript practices and uses classes to organize data.
-It uses MongoDB as persistent storage.
-
-## Intention
-There are many ODMs out there, however when we searched for a ODM that uses TypeScript,
-acts smart while still embracing an easy the NoSQL idea we didn't find a matching solution.
-This is why we started smartdata
-
-How MongoDB terms map to smartdata classes
-
-MongoDB term | smartdata class
---- | ---
-Database | smartdata.Db
-Collection | smartdata.DbCollection
-Document | smartdata.DbDoc
-
-### class Db
-represents a Database. Naturally it has .connect() etc. methods on it.
-Since it is a class you can have multiple DBs defined.
-```typescript
-import * as smartdata from 'smartdata'
-
-let myDb1 = new smartdata.Db('someConnectionUrl')
-let myDb2 = new smartdata.Db('someConnectionUrl')
-
-myDb1.connect()
-myDb2.connect()
-```
-
-### class Collection
-represents a collection of objects.
-A collection is defined by the object class (that is extending smartdata.dbdoc) it respresents
-
-So to get to get access to a specific collection you document
-```typescript
-class myObject extends DbDoc {
-    property1:string
-    property2:number
-    constructor(optionsArg:{
-        queryArg?:any,
-        dataArg?:{
-            property1:string,
-            property2:string
-        }
-    }) {
-        super(this,optionsArg)
-    }
-} 
-let myCollection = myDb1.getCollection(myObject)
-``` 
-
-> Alert: You NEVER instantiate a collection.
-This is done for you!!!
-
-### class DbDoc
-represents a individual document in a collection.
-
-
-
-
-[![npm](https://push.rocks/assets/repo-header.svg)](https://push.rocks)

changelog.md

@@ -0,0 +1,387 @@
+# Changelog
+
+## 2025-04-25 - 5.16.0 - feat(watcher)
+Enhance change stream watchers with buffering and EventEmitter support; update dependency versions
+
+- Bumped smartmongo from ^2.0.11 to ^2.0.12 and smartrx from ^3.0.7 to ^3.0.10
+- Upgraded @tsclass/tsclass to ^9.0.0 and mongodb to ^6.16.0
+- Refactored the watch API to accept additional options (bufferTimeMs, fullDocument) for improved change stream handling
+- Modified SmartdataDbWatcher to extend EventEmitter and support event notifications
+
+## 2025-04-24 - 5.15.1 - fix(cursor)
+Improve cursor usage documentation and refactor getCursor API to support native cursor modifiers
+
+- Updated examples in readme.md to demonstrate manual iteration using cursor.next() and proper cursor closing.
+- Refactored the getCursor method in classes.doc.ts to accept session and modifier options, consolidating cursor handling.
+- Added new tests in test/test.cursor.ts to verify cursor operations, including limits, sorting, and skipping.
+
+## 2025-04-24 - 5.15.0 - feat(svDb)
+Enhance svDb decorator to support custom serialization and deserialization options
+
+- Added an optional options parameter to the svDb decorator to accept serialize/deserialize functions
+- Updated instance creation logic (updateFromDb) to apply custom deserialization if provided
+- Updated createSavableObject to use custom serialization when available
+
+## 2025-04-23 - 5.14.1 - fix(db operations)
+Update transaction API to consistently pass optional session parameters across database operations
+
+- Revised transaction support in readme to use startSession without await and showcased session usage in getInstance and save calls
+- Updated methods in classes.collection.ts to accept an optional session parameter for findOne, getCursor, findAll, insert, update, delete, and getCount
+- Enhanced SmartDataDbDoc save and delete methods to propagate session parameters
+- Improved overall consistency of transactional APIs across the library
+
+## 2025-04-23 - 5.14.0 - feat(doc)
+Implement support for beforeSave, afterSave, beforeDelete, and afterDelete lifecycle hooks in document save and delete operations to allow custom logic execution during these critical moments.
+
+- Calls beforeSave hook if defined before performing insert or update.
+- Calls afterSave hook after a document is saved.
+- Calls beforeDelete hook before deletion and afterDelete hook afterward.
+- Ensures _updatedAt timestamp is refreshed during save operations.
+
+## 2025-04-22 - 5.13.1 - fix(search)
+Improve search query parsing for implicit AND queries by preserving quoted substrings and better handling free terms, quoted phrases, and field:value tokens.
+
+- Replace previous implicit AND logic with tokenization that preserves quoted substrings
+- Support both free term and field:value tokens with wildcards inside quotes
+- Ensure errors are thrown for non-searchable fields in field-specific queries
+
+## 2025-04-22 - 5.13.0 - feat(search)
+Improve search query handling and update documentation
+
+- Added 'codex.md' providing a high-level project overview and detailed search API documentation.
+- Enhanced search parsing in SmartDataDbDoc to support combined free-term and quoted field phrase queries.
+- Introduced a new fallback branch in the search method to handle free term with quoted field input.
+- Updated tests in test/test.search.ts to cover new combined query scenarios and ensure robust behavior.
+
+## 2025-04-22 - 5.12.2 - fix(search)
+Fix handling of quoted wildcard patterns in field-specific search queries and add tests for location-based wildcard phrase searches
+
+- Strip surrounding quotes from wildcard patterns in field queries to correctly transform them to regex
+- Introduce new tests in test/test.search.ts to validate exact quoted and unquoted wildcard searches on a location field
+
+## 2025-04-22 - 5.12.1 - fix(search)
+Improve implicit AND logic for mixed free term and field queries in search and enhance wildcard field handling.
+
+- Updated regex for field:value parsing to capture full value with wildcards.
+- Added explicit handling for free terms by converting to regex across searchable fields.
+- Improved error messaging for attempts to search non-searchable fields.
+- Extended tests to cover combined free term and wildcard field searches, including error cases.
+
+## 2025-04-22 - 5.12.0 - feat(doc/search)
+Enhance search functionality with filter and validate options for advanced query control
+
+- Added 'filter' option to merge additional MongoDB query constraints in search
+- Introduced 'validate' hook to post-process and filter fetched documents
+- Refactored underlying execQuery function to support additional search options
+- Updated tests to cover new search scenarios and fallback mechanisms
+
+## 2025-04-22 - 5.11.4 - fix(search)
+Implement implicit AND logic for mixed simple term and field:value queries in search
+
+- Added a new branch to detect and handle search queries that mix field:value pairs with plain terms without explicit operators
+- Builds an implicit $and filter when query parts contain colon(s) but lack explicit boolean operators or quotes
+- Ensures proper parsing and improved robustness of search filters
+
+## 2025-04-22 - 5.11.3 - fix(lucene adapter and search tests)
+Improve range query parsing in Lucene adapter and expand search test coverage
+
+- Added a new 'testSearch' script in package.json to run search tests.
+- Introduced advanced search tests for range queries and combined field filters in test/search.advanced.ts.
+- Enhanced robustness tests in test/search.ts for wildcard and empty query scenarios.
+- Fixed token validation in the parseRange method of the Lucene adapter to ensure proper error handling.
+
+## 2025-04-21 - 5.11.2 - fix(readme)
+Update readme to clarify usage of searchable fields retrieval
+
+- Replaced getSearchableFields('Product') with Product.getSearchableFields()
+- Updated documentation to reference the static method Class.getSearchableFields()
+
+## 2025-04-21 - 5.11.1 - fix(doc)
+Refactor searchable fields API and improve collection registration.
+
+- Removed the standalone getSearchableFields utility in favor of a static method on document classes.
+- Updated tests to use the new static method (e.g., Product.getSearchableFields()).
+- Ensured the Collection decorator attaches a docCtor property to correctly register searchable fields.
+- Added try/catch in test cleanup to gracefully handle dropDatabase errors.
+
+## 2025-04-21 - 5.11.0 - feat(ts/classes.lucene.adapter)
+Expose luceneWildcardToRegex method to allow external usage and enhance regex transformation capabilities.
+
+- Changed luceneWildcardToRegex from private to public in ts/classes.lucene.adapter.ts.
+
+## 2025-04-21 - 5.10.0 - feat(search)
+Improve search functionality: update documentation, refine Lucene query transformation, and add advanced search tests
+
+- Updated readme.md with detailed Lucene‑style search examples and use cases
+- Enhanced LuceneToMongoTransformer to properly handle wildcard conversion and regex escaping
+- Improved search query parsing in SmartDataDbDoc for field-specific, multi-term, and advanced Lucene syntax
+- Added new advanced search tests covering boolean operators, grouping, quoted phrases, and wildcard queries
+
+## 2025-04-18 - 5.9.2 - fix(documentation)
+Update search API documentation to replace deprecated searchWithLucene examples with the unified search(query) API and clarify its behavior.
+
+- Replaced 'searchWithLucene' examples with 'search(query)' in the README.
+- Updated explanation to detail field-specific exact match, partial word regex search, multi-word literal matching, and handling of empty queries.
+- Clarified guidelines for creating MongoDB text indexes on searchable fields for optimized search performance.
+
+## 2025-04-18 - 5.9.1 - fix(search)
+Refactor search tests to use unified search API and update text index type casting
+
+- Replaced all calls from searchWithLucene with search in test/search tests
+- Updated text index specification in the collection class to use proper type casting
+
+## 2025-04-18 - 5.9.0 - feat(collections/search)
+Improve text index creation and search fallback mechanisms in collections and document search methods
+
+- Auto-create a compound text index on all searchable fields in SmartdataCollection with a one-time flag to prevent duplicate index creation.
+- Refine the search method in SmartDataDbDoc to support exact field matches and safe regex fallback for non-Lucene queries.
+
+## 2025-04-17 - 5.8.4 - fix(core)
+Update commit metadata with no functional code changes
+
+- Commit info and documentation refreshed
+- No code or test changes detected in the diff
+
+## 2025-04-17 - 5.8.3 - fix(readme)
+Improve readme documentation on data models and connection management
+
+- Clarify that data models use @Collection, @unI, @svDb, @index, and @searchable decorators
+- Document that ObjectId and Buffer fields are stored as BSON types natively without extra decorators
+- Update connection management section to use 'db.close()' instead of 'db.disconnect()'
+- Revise license section to reference the MIT License without including additional legal details
+
+## 2025-04-14 - 5.8.2 - fix(classes.doc.ts)
+Ensure collection initialization before creating a cursor in getCursorExtended
+
+- Added 'await collection.init()' to guarantee that the MongoDB collection is initialized before using the cursor
+- Prevents potential runtime errors when accessing collection.mongoDbCollection
+
+## 2025-04-14 - 5.8.1 - fix(cursor, doc)
+Add explicit return types and casts to SmartdataDbCursor methods and update getCursorExtended signature in SmartDataDbDoc.
+
+- Specify Promise<T> as return type for next() in SmartdataDbCursor and cast return value to T.
+- Specify Promise<T[]> as return type for toArray() in SmartdataDbCursor and cast return value to T[].
+- Update getCursorExtended to return Promise<SmartdataDbCursor<T>> for clearer type safety.
+
+## 2025-04-14 - 5.8.0 - feat(cursor)
+Add toArray method to SmartdataDbCursor to convert raw MongoDB documents into initialized class instances
+
+- Introduced asynchronous toArray method in SmartdataDbCursor which retrieves all documents from the MongoDB cursor
+- Maps each native document to a SmartDataDbDoc instance using createInstanceFromMongoDbNativeDoc for consistent API usage
+
+## 2025-04-14 - 5.7.0 - feat(SmartDataDbDoc)
+Add extended cursor method getCursorExtended for flexible cursor modifications
+
+- Introduces getCursorExtended in classes.doc.ts to allow modifier functions for MongoDB cursors
+- Wraps the modified cursor with SmartdataDbCursor for improved API consistency
+- Enhances querying capabilities by enabling customized cursor transformations
+
+## 2025-04-07 - 5.6.0 - feat(indexing)
+Add support for regular index creation in documents and collections
+
+- Implement new index decorator in classes.doc.ts to mark properties with regular indexing options
+- Update SmartdataCollection to create regular indexes if defined on a document during insert
+- Enhance document structure to store and utilize regular index configurations
+
+## 2025-04-06 - 5.5.1 - fix(ci & formatting)
+Minor fixes: update CI workflow image and npmci package references, adjust package.json and readme URLs, and apply consistent code formatting.
+
+- Update image and repo URL in Gitea workflows from GitLab to code.foss.global
+- Replace '@shipzone/npmci' with '@ship.zone/npmci' throughout CI scripts
+- Adjust homepage and bugs URL in package.json and readme
+- Apply trailing commas and consistent formatting in TypeScript source files
+- Minor update to .gitignore custom section label
+
+## 2025-04-06 - 5.5.0 - feat(search)
+Enhance search functionality with robust Lucene query transformation and reliable fallback mechanisms
+
+- Improve Lucene adapter to properly structure $or queries for term, phrase, wildcard, and fuzzy search
+- Implement and document a robust searchWithLucene method with fallback to in-memory filtering
+- Update readme and tests with extensive examples for @searchable fields and Lucene-based queries
+
+## 2025-04-06 - 5.4.0 - feat(core)
+Refactor file structure and update dependency versions
+
+- Renamed files and modules from 'smartdata.classes.*' to 'classes.*' and adjusted corresponding import paths.
+- Updated dependency versions: '@push.rocks/smartmongo' to ^2.0.11, '@tsclass/tsclass' to ^8.2.0, and 'mongodb' to ^6.15.0.
+- Renamed dev dependency packages from '@gitzone/...' to '@git.zone/...' and updated '@push.rocks/tapbundle' and '@types/node'.
+- Fixed YAML workflow command: replaced 'pnpm install -g @gitzone/tsdoc' with 'pnpm install -g @git.zone/tsdoc'.
+- Added package manager configuration and pnpm-workspace.yaml for built dependencies.
+
+## 2025-03-10 - 5.3.0 - feat(docs)
+Enhance documentation with updated installation instructions and comprehensive usage examples covering advanced features such as deep queries, automatic indexing, and distributed coordination.
+
+- Added pnpm installation command
+- Updated User model example to include ObjectId, Binary, and custom serialization
+- Expanded CRUD operations examples with cursor methods and deep query support
+- Enhanced sections on EasyStore, real-time data watching with RxJS integration, and managed collections
+- Included detailed examples for transactions, deep object queries, and document lifecycle hooks
+
+## 2025-02-03 - 5.2.12 - fix(documentation)
+Remove license badge from README
+
+- Removed the license badge from the README file, ensuring compliance with branding guidelines.
+
+## 2025-02-03 - 5.2.11 - fix(documentation)
+Updated project documentation for accuracy and added advanced feature details
+
+- Added details for EasyStore, Distributed Coordination, and Real-time Data Watching features.
+- Updated database connection setup instructions to include user authentication.
+- Re-organized advanced usage section to showcase additional features separately.
+
+## 2024-09-05 - 5.2.10 - fix(smartdata.classes.doc)
+Fix issue with array handling in convertFilterForMongoDb function
+
+- Corrected the logic to properly handle array filters in the convertFilterForMongoDb function to avoid incorrect assignments.
+
+## 2024-09-05 - 5.2.9 - fix(smartdata.classes.doc)
+Fixed issue with convertFilterForMongoDb to handle array operators.
+
+- Updated the convertFilterForMongoDb function in smartdata.classes.doc.ts to properly handle array operators like $in and $all.
+
+## 2024-09-05 - 5.2.8 - fix(smartdata.classes.doc)
+Fix key handling in convertFilterForMongoDb function
+
+- Fixed an issue in convertFilterForMongoDb that allowed keys with dots which could cause errors.
+
+## 2024-09-05 - 5.2.7 - fix(core)
+Fixed issue with handling filter keys containing dots in smartdata.classes.doc.ts
+
+- Fixed an error in the convertFilterForMongoDb function which previously threw an error when keys contained dots.
+
+## 2024-06-18 - 5.2.6 - Chore
+Maintenance Release
+
+- Release version 5.2.6
+
+## 2024-05-31 - 5.2.2 - Bug Fixes
+Fixes and Maintenance
+
+- Fixed issue where `_createdAt` and `_updatedAt` registered saveableProperties for all document types
+
+## 2024-04-15 - 5.1.2 - New Feature
+Enhancements and Bug Fixes
+
+- Added static `.getCount({})` method to `SmartDataDbDoc`
+- Changed fields `_createdAt` and `_updatedAt` to ISO format
+
+## 2024-04-14 - 5.0.43 - New Feature
+New Feature Addition
+
+- Added default `_createdAt` and `_updatedAt` fields, fixes #1
+
+## 2024-03-30 - 5.0.41 - Bug Fixes
+Improvements and Fixes
+
+- Improved `tsconfig.json` for ES Module use
+
+## 2023-07-10 - 5.0.20 - Chore
+Organizational Changes
+
+- Switched to new org scheme
+
+## 2023-07-21 - 5.0.21 to 5.0.26 - Fixes
+Multiple Fix Releases
+
+- Various core updates and bug fixes
+
+## 2023-07-21 - 5.0.20 - Chore
+Organizational Changes
+
+- Switch to the new org scheme
+
+## 2023-06-25 - 5.0.14 to 5.0.19 - Fixes
+Multiple Fix Releases
+
+- Various core updates and bug fixes
+
+## 2022-05-17 - 5.0.0 - Major Update
+Breaking Changes
+
+- Switched to ESM
+
+## 2022-05-18 - 5.0.2 - Bug Fixes
+Bug Fixes
+
+- The `watcher.changeSubject` now emits the correct type into observer functions
+
+## 2022-05-17 - 5.0.1 - Chore
+Testing Improvements
+
+- Tests now use `@pushrocks/smartmongo` backed by `wiredTiger`
+
+## 2022-05-17 to 2022-11-08 - 5.0.8 to 5.0.10
+Multiple Fix Releases
+
+- Various core updates and bug fixes
+
+## 2021-11-12 - 4.0.17 to 4.0.20
+Multiple Fix Releases
+
+- Various core updates and bug fixes
+
+## 2021-09-17 - 4.0.10 to 4.0.16
+Multiple Fix Releases
+
+- Various core updates and bug fixes
+
+## 2021-06-09 - 4.0.1 to 4.0.9
+Multiple Fix Releases
+
+- Various core updates and bug fixes
+
+## 2021-06-06 - 4.0.0 - Major Update
+Major Release
+
+- Maintenance and core updates
+
+## 2021-05-17 - 3.1.56 - Chore
+Maintenance Release
+
+- Release version 3.1.56
+
+## 2020-09-09 - 3.1.44 to 3.1.52
+Multiple Fix Releases
+
+- Various core updates and bug fixes
+
+## 2020-06-12 - 3.1.26 to 3.1.28
+Multiple Fix Releases
+
+- Various core updates and bug fixes
+
+## 2020-02-18 - 3.1.23 to 3.1.25
+Multiple Fix Releases
+
+- Various core updates and bug fixes
+
+## 2019-09-11 - 3.1.20 to 3.1.22
+Multiple Fix Releases
+
+- Various core updates and bug fixes
+
+## 2018-07-10 - 3.0.5 - New Feature
+Added Feature
+
+- Added custom unique indexes to `SmartdataDoc`
+
+## 2018-07-08 - 3.0.1 - Chore
+Dependencies Update
+
+- Updated mongodb dependencies
+
+## 2018-07-08 - 3.0.0 - Major Update
+Refactor and Cleanup
+
+- Cleaned project structure
+
+## 2018-01-16 - 2.0.7 - Breaking Change
+Big Changes
+
+- Switched to `@pushrocks` scope and moved from `rethinkdb` to `mongodb`
+
+## 2018-01-12 - 2.0.0 - Major Release
+Core Updates
+
+- Updated CI configurations
+

codex.md

@@ -0,0 +1,77 @@
+# SmartData Project Overview
+
+This document provides a high-level overview of the SmartData library (`@push.rocks/smartdata`), its architecture, core components, and key features—including recent enhancements to the search API.
+
+## 1. Project Purpose
+- A TypeScript‑first wrapper around MongoDB that supplies:
+  - Strongly‑typed document & collection classes
+  - Decorator‑based schema definition (no external schema files)
+  - Advanced search capabilities with Lucene‑style queries
+  - Built‑in support for real‑time data sync, distributed coordination, and key‑value EasyStore
+
+## 2. Core Concepts & Components
+- **SmartDataDb**: Manages the MongoDB connection, pooling, and initialization of collections.
+- **SmartDataDbDoc**: Base class for all document models; provides CRUD, upsert, and cursor APIs.
+- **Decorators**:
+  - `@Collection`: Associates a class with a MongoDB collection
+  - `@svDb()`: Marks a field as persisted to the DB
+  - `@unI()`: Marks a field as a unique index
+  - `@index()`: Adds a regular index
+  - `@searchable()`: Marks a field for inclusion in text searches or regex queries
+- **SmartdataCollection**: Wraps a MongoDB collection; auto‑creates indexes based on decorators.
+- **Lucene Adapter**: Parses a Lucene query string into an AST and transforms it to a MongoDB filter object.
+- **EasyStore**: A simple, schema‑less key‑value store built on top of MongoDB for sharing ephemeral data.
+- **Distributed Coordinator**: Leader election and task‑distribution API for building resilient, multi‑instance systems.
+- **Watcher**: Listens to change streams for real‑time updates and integrates with RxJS.
+
+## 3. Search API
+SmartData provides a unified `.search(query[, opts])` method on all models with `@searchable()` fields:
+
+- **Supported Syntax**:
+  1. Exact field:value (e.g. `field:Value`)
+  2. Quoted phrases (e.g. `"exact phrase"` or `'exact phrase'`)
+  3. Wildcards: `*` (zero or more chars) and `?` (single char)
+  4. Boolean operators: `AND`, `OR`, `NOT`
+  5. Grouping: parenthesis `(A OR B) AND C`
+  6. Range queries: `[num TO num]`, `{num TO num}`
+  7. Multi‑term unquoted: terms AND’d across all searchable fields
+  8. Empty query returns all documents
+
+- **Fallback Mechanisms**:
+  1. Text index based `$text` search (if supported)
+  2. Field‑scoped and multi‑field regex queries
+  3. In‑memory filtering for complex or unsupported cases
+
+### New Security & Extensibility Hooks
+The `.search(query, opts?)` signature now accepts a `SearchOptions<T>` object:
+```ts
+interface SearchOptions<T> {
+  filter?: Record<string, any>;        // Additional MongoDB filter AND‑merged
+  validate?: (doc: T) => boolean;      // Post‑fetch hook to drop results
+}
+```
+- **filter**: Enforces mandatory constraints (e.g. multi‑tenant isolation) directly in the Mongo query.
+- **validate**: An async function that runs after fetching; return `false` to exclude a document.
+
+## 4. Testing Strategy
+- Unit tests in `test/test.search.ts` cover basic search functionality and new options:
+  - Exact, wildcard, phrase, boolean and grouping cases
+  - Implicit AND and mixed free‑term + field searches
+  - Edge cases (non‑searchable fields, quoted wildcards, no matches)
+  - `filter` and `validate` tests ensure security hooks work as intended
+- Advanced search scenarios are covered in `test/test.search.advanced.ts`.
+
+## 5. Usage Example
+```ts
+// Basic search
+const prods = await Product.search('wireless earbuds');
+
+// Scoped search (only your organization’s items)
+const myItems = await Product.search('book', { filter: { ownerId } });
+
+// Post‑search validation (only cheap items)
+const cheapItems = await Product.search('', { validate: p => p.price < 50 });
+```
+
+---
+Last updated: 2025-04-22

dist/index.d.ts

@@ -1,2 +0,0 @@
-export * from './smartdata.classes.dbcollection';
-export * from './smartdata.classes.db';

dist/index.js

@@ -1,7 +0,0 @@
-"use strict";
-function __export(m) {
-    for (var p in m) if (!exports.hasOwnProperty(p)) exports[p] = m[p];
-}
-__export(require('./smartdata.classes.dbcollection'));
-__export(require('./smartdata.classes.db'));
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi90cy9pbmRleC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiOzs7O0FBRUEsaUJBQWMsa0NBQ2QsQ0FBQyxFQUQrQztBQUNoRCxpQkFBYyx3QkFDZCxDQUFDLEVBRHFDIn0=

dist/smartdata.classes.db.d.ts

@@ -1,11 +0,0 @@
-/// <reference types="q" />
-import * as plugins from './smartdata.plugins';
-export declare type TConnectionStatus = 'disconnected' | 'connected' | 'failed';
-export declare class Db {
-    dbUrl: string;
-    db: plugins.mongodb.Db;
-    status: TConnectionStatus;
-    constructor(dbUrl: string);
-    connect(): plugins.q.Promise<any>;
-    close(): plugins.q.Promise<any>;
-}

dist/smartdata.classes.db.js

@@ -1,29 +0,0 @@
-"use strict";
-const plugins = require('./smartdata.plugins');
-class Db {
-    constructor(dbUrl) {
-        this.dbUrl = dbUrl;
-    }
-    connect() {
-        let done = plugins.q.defer();
-        plugins.mongodb.MongoClient.connect(this.dbUrl, (err, db) => {
-            if (err) {
-                console.log(err);
-            }
-            plugins.assert.equal(null, err);
-            this.db = db;
-            plugins.beautylog.success(`connected to database at ${this.dbUrl}`);
-            done.resolve(this.db);
-        });
-        return done.promise;
-    }
-    close() {
-        let done = plugins.q.defer();
-        this.db.close();
-        plugins.beautylog.ok(`disconnected to database at ${this.dbUrl}`);
-        done.resolve();
-        return done.promise;
-    }
-}
-exports.Db = Db;
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic21hcnRkYXRhLmNsYXNzZXMuZGIuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi90cy9zbWFydGRhdGEuY2xhc3Nlcy5kYi50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiO0FBQUEsTUFBWSxPQUFPLFdBQU0scUJBRXpCLENBQUMsQ0FGNkM7QUFJOUM7SUFLSSxZQUFZLEtBQWE7UUFDckIsSUFBSSxDQUFDLEtBQUssR0FBRyxLQUFLLENBQUE7SUFDdEIsQ0FBQztJQUVELE9BQU87UUFDSCxJQUFJLElBQUksR0FBRyxPQUFPLENBQUMsQ0FBQyxDQUFDLEtBQUssRUFBRSxDQUFBO1FBQzVCLE9BQU8sQ0FBQyxPQUFPLENBQUMsV0FBVyxDQUFDLE9BQU8sQ0FBQyxJQUFJLENBQUMsS0FBSyxFQUFFLENBQUMsR0FBRyxFQUFFLEVBQUU7WUFDcEQsRUFBRSxDQUFDLENBQUMsR0FBRyxDQUFDLENBQUMsQ0FBQztnQkFBQyxPQUFPLENBQUMsR0FBRyxDQUFDLEdBQUcsQ0FBQyxDQUFBO1lBQUMsQ0FBQztZQUM3QixPQUFPLENBQUMsTUFBTSxDQUFDLEtBQUssQ0FBQyxJQUFJLEVBQUUsR0FBRyxDQUFDLENBQUE7WUFDL0IsSUFBSSxDQUFDLEVBQUUsR0FBRyxFQUFFLENBQUE7WUFDWixPQUFPLENBQUMsU0FBUyxDQUFDLE9BQU8sQ0FBQyw0QkFBNEIsSUFBSSxDQUFDLEtBQUssRUFBRSxDQUFDLENBQUE7WUFDbkUsSUFBSSxDQUFDLE9BQU8sQ0FBQyxJQUFJLENBQUMsRUFBRSxDQUFDLENBQUE7UUFDekIsQ0FBQyxDQUFDLENBQUE7UUFDRixNQUFNLENBQUMsSUFBSSxDQUFDLE9BQU8sQ0FBQTtJQUN2QixDQUFDO0lBRUQsS0FBSztRQUNELElBQUksSUFBSSxHQUFHLE9BQU8sQ0FBQyxDQUFDLENBQUMsS0FBSyxFQUFFLENBQUE7UUFDNUIsSUFBSSxDQUFDLEVBQUUsQ0FBQyxLQUFLLEVBQUUsQ0FBQTtRQUNmLE9BQU8sQ0FBQyxTQUFTLENBQUMsRUFBRSxDQUFDLCtCQUErQixJQUFJLENBQUMsS0FBSyxFQUFFLENBQUMsQ0FBQTtRQUNqRSxJQUFJLENBQUMsT0FBTyxFQUFFLENBQUE7UUFDZCxNQUFNLENBQUMsSUFBSSxDQUFDLE9BQU8sQ0FBQTtJQUN2QixDQUFDO0FBQ0wsQ0FBQztBQTVCWSxVQUFFLEtBNEJkLENBQUEifQ==

dist/smartdata.classes.dbcollection.d.ts

@@ -1,24 +0,0 @@
-/// <reference types="q" />
-import * as plugins from './smartdata.plugins';
-import { Db } from './smartdata.classes.db';
-export declare class DbCollection<T> {
-    collection: plugins.mongodb.Collection;
-    constructor(nameArg: string, dbArg: Db);
-    /**
-     * adds a validation function that all newly inserted and updated objects have to pass
-     */
-    addObjectValidation(funcArg: any): void;
-    /**
-     * finds an object in the DbCollection
-     */
-    find(docMatchArg: T | any): plugins.q.Promise<T[]>;
-    /**
-     * inserts  object into the DbCollection
-     */
-    insertOne(docArg: T): plugins.q.Promise<void>;
-    /**
-     * inserts many objects at once into the DbCollection
-     */
-    insertMany(docArrayArg: T[]): plugins.q.Promise<void>;
-    private checkDoc(doc);
-}

dist/smartdata.classes.dbcollection.js

@@ -1,57 +0,0 @@
-"use strict";
-const plugins = require('./smartdata.plugins');
-class DbCollection {
-    constructor(nameArg, dbArg) {
-        this.collection = dbArg.db.collection(nameArg);
-    }
-    /**
-     * adds a validation function that all newly inserted and updated objects have to pass
-     */
-    addObjectValidation(funcArg) { }
-    /**
-     * finds an object in the DbCollection
-     */
-    find(docMatchArg) {
-        let done = plugins.q.defer();
-        this.collection.find(docMatchArg).toArray((err, docs) => {
-            if (err) {
-                throw err;
-            }
-            done.resolve(docs);
-        });
-        return done.promise;
-    }
-    /**
-     * inserts  object into the DbCollection
-     */
-    insertOne(docArg) {
-        let done = plugins.q.defer();
-        this.checkDoc(docArg).then(() => {
-            this.collection.insertOne(docArg)
-                .then(() => { done.resolve(); });
-        });
-        return done.promise;
-    }
-    /**
-     * inserts many objects at once into the DbCollection
-     */
-    insertMany(docArrayArg) {
-        let done = plugins.q.defer();
-        let checkDocPromiseArray = [];
-        for (let docArg of docArrayArg) {
-            checkDocPromiseArray.push(this.checkDoc(docArg));
-        }
-        plugins.q.all(checkDocPromiseArray).then(() => {
-            this.collection.insertMany(docArrayArg)
-                .then(() => { done.resolve(); });
-        });
-        return done.promise;
-    }
-    checkDoc(doc) {
-        let done = plugins.q.defer();
-        done.resolve();
-        return done.promise;
-    }
-}
-exports.DbCollection = DbCollection;
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic21hcnRkYXRhLmNsYXNzZXMuZGJjb2xsZWN0aW9uLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vdHMvc21hcnRkYXRhLmNsYXNzZXMuZGJjb2xsZWN0aW9uLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiI7QUFBQSxNQUFZLE9BQU8sV0FBTSxxQkFDekIsQ0FBQyxDQUQ2QztBQUc5QztJQUVJLFlBQVksT0FBZSxFQUFFLEtBQVM7UUFDbEMsSUFBSSxDQUFDLFVBQVUsR0FBRyxLQUFLLENBQUMsRUFBRSxDQUFDLFVBQVUsQ0FBQyxPQUFPLENBQUMsQ0FBQTtJQUNsRCxDQUFDO0lBRUQ7O09BRUc7SUFDSCxtQkFBbUIsQ0FBQyxPQUFPLElBQUksQ0FBQztJQUVoQzs7T0FFRztJQUNILElBQUksQ0FBQyxXQUFvQjtRQUNyQixJQUFJLElBQUksR0FBRyxPQUFPLENBQUMsQ0FBQyxDQUFDLEtBQUssRUFBTyxDQUFBO1FBQ2pDLElBQUksQ0FBQyxVQUFVLENBQUMsSUFBSSxDQUFDLFdBQVcsQ0FBQyxDQUFDLE9BQU8sQ0FBQyxDQUFDLEdBQUcsRUFBRSxJQUFJO1lBQ2hELEVBQUUsQ0FBQyxDQUFDLEdBQUcsQ0FBQyxDQUFDLENBQUM7Z0JBQUMsTUFBTSxHQUFHLENBQUE7WUFBQyxDQUFDO1lBQ3RCLElBQUksQ0FBQyxPQUFPLENBQUMsSUFBSSxDQUFDLENBQUE7UUFDdEIsQ0FBQyxDQUFDLENBQUE7UUFDRixNQUFNLENBQUMsSUFBSSxDQUFDLE9BQU8sQ0FBQTtJQUN2QixDQUFDO0lBRUQ7O09BRUc7SUFDSCxTQUFTLENBQUMsTUFBUztRQUNmLElBQUksSUFBSSxHQUFHLE9BQU8sQ0FBQyxDQUFDLENBQUMsS0FBSyxFQUFRLENBQUE7UUFDbEMsSUFBSSxDQUFDLFFBQVEsQ0FBQyxNQUFNLENBQUMsQ0FBQyxJQUFJLENBQUM7WUFDdkIsSUFBSSxDQUFDLFVBQVUsQ0FBQyxTQUFTLENBQUMsTUFBTSxDQUFDO2lCQUM1QixJQUFJLENBQUMsUUFBUSxJQUFJLENBQUMsT0FBTyxFQUFFLENBQUEsQ0FBQyxDQUFDLENBQUMsQ0FBQTtRQUN2QyxDQUFDLENBQUMsQ0FBQTtRQUNGLE1BQU0sQ0FBQyxJQUFJLENBQUMsT0FBTyxDQUFBO0lBQ3ZCLENBQUM7SUFFRDs7T0FFRztJQUNILFVBQVUsQ0FBQyxXQUFnQjtRQUN2QixJQUFJLElBQUksR0FBRyxPQUFPLENBQUMsQ0FBQyxDQUFDLEtBQUssRUFBUSxDQUFBO1FBQ2xDLElBQUksb0JBQW9CLEdBQThCLEVBQUUsQ0FBQTtRQUN4RCxHQUFHLENBQUMsQ0FBQyxJQUFJLE1BQU0sSUFBSSxXQUFXLENBQUMsQ0FBQSxDQUFDO1lBQzVCLG9CQUFvQixDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsUUFBUSxDQUFDLE1BQU0sQ0FBQyxDQUFDLENBQUE7UUFDcEQsQ0FBQztRQUNELE9BQU8sQ0FBQyxDQUFDLENBQUMsR0FBRyxDQUFDLG9CQUFvQixDQUFDLENBQUMsSUFBSSxDQUFDO1lBQ3JDLElBQUksQ0FBQyxVQUFVLENBQUMsVUFBVSxDQUFDLFdBQVcsQ0FBQztpQkFDbEMsSUFBSSxDQUFDLFFBQVEsSUFBSSxDQUFDLE9BQU8sRUFBRSxDQUFBLENBQUMsQ0FBQyxDQUFDLENBQUE7UUFDdkMsQ0FBQyxDQUFDLENBQUE7UUFDRixNQUFNLENBQUMsSUFBSSxDQUFDLE9BQU8sQ0FBQTtJQUN2QixDQUFDO0lBRU8sUUFBUSxDQUFDLEdBQU07UUFDbkIsSUFBSSxJQUFJLEdBQUcsT0FBTyxDQUFDLENBQUMsQ0FBQyxLQUFLLEVBQVEsQ0FBQTtRQUNsQyxJQUFJLENBQUMsT0FBTyxFQUFFLENBQUE7UUFDZCxNQUFNLENBQUMsSUFBSSxDQUFDLE9BQU8sQ0FBQTtJQUN2QixDQUFDO0FBQ0wsQ0FBQztBQXhEWSxvQkFBWSxlQXdEeEIsQ0FBQSJ9

dist/smartdata.classes.dbdoc.js

@@ -1 +0,0 @@
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic21hcnRkYXRhLmNsYXNzZXMuZGJkb2MuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi90cy9zbWFydGRhdGEuY2xhc3Nlcy5kYmRvYy50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiIn0=

dist/smartdata.plugins.d.ts

@@ -1,5 +0,0 @@
-import 'typings-global';
-export import assert = require('assert');
-export import beautylog = require('beautylog');
-export import mongodb = require('mongodb');
-export import q = require('q');

dist/smartdata.plugins.js

@@ -1,7 +0,0 @@
-"use strict";
-require('typings-global');
-exports.assert = require('assert');
-exports.beautylog = require('beautylog');
-exports.mongodb = require('mongodb');
-exports.q = require('q');
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic21hcnRkYXRhLnBsdWdpbnMuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi90cy9zbWFydGRhdGEucGx1Z2lucy50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiO0FBQUEsUUFBTyxnQkFDUCxDQUFDLENBRHNCO0FBQ1QsY0FBTSxXQUFXLFFBQVEsQ0FBQyxDQUFBO0FBQzFCLGlCQUFTLFdBQVcsV0FBVyxDQUFDLENBQUE7QUFDaEMsZUFBTyxXQUFXLFNBQVMsQ0FBQyxDQUFBO0FBQzVCLFNBQUMsV0FBVyxHQUFHLENBQUMsQ0FBQSJ9

docs/index.md

@@ -1,19 +0,0 @@
-# smartdata
-
-smartdata is a ODM that adheres to TypeScript practices and uses classes to organize data.
-It uses MongoDB as persistent storage.
-
-## Intention
-There are many ODMs out there, however when we searched for a ODM that uses TypeScript,
-acts smart while still embracing an easy the NoSQL idea we didn't find a matching solution.
-This is why we started smartdata
-
-How MongoDB terms map to smartdata classes
-
-MongoDB term | smartdata class
---- | ---
-Database | smartdata.DbConnection
-Collection | smartdata.DbCollection
-Document | smartdata.DbDoc
-
-[![npm](https://push.rocks/assets/repo-header.svg)](https://push.rocks)

npmextra.json

@@ -0,0 +1,36 @@
+{
+  "npmdocker": {
+    "baseImage": "hosttoday/ht-docker-node:mongo",
+    "command": "npmci test stable",
+    "dockerSock": false
+  },
+  "npmci": {
+    "npmGlobalTools": [],
+    "npmAccessLevel": "public",
+    "npmRegistryUrl": "registry.npmjs.org"
+  },
+  "gitzone": {
+    "projectType": "npm",
+    "module": {
+      "githost": "code.foss.global",
+      "gitscope": "push.rocks",
+      "gitrepo": "smartdata",
+      "description": "An advanced library for NoSQL data organization and manipulation using TypeScript with support for MongoDB, data validation, collections, and custom data types.",
+      "npmPackagename": "@push.rocks/smartdata",
+      "license": "MIT",
+      "keywords": [
+        "data manipulation",
+        "NoSQL",
+        "MongoDB",
+        "TypeScript",
+        "data validation",
+        "collections",
+        "custom data types",
+        "ODM"
+      ]
+    }
+  },
+  "tsdoc": {
+    "legal": "\n## License and Legal Information\n\nThis repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository. \n\n**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.\n\n### Trademarks\n\nThis project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.\n\n### Company Information\n\nTask Venture Capital GmbH  \nRegistered at District court Bremen HRB 35230 HB, Germany\n\nFor any legal inquiries or if you require further information, please contact us via email at hello@task.vc.\n\nBy using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.\n"
+  }
+}

package.json

@@ -1,37 +1,76 @@
 {
-  "name": "smartdata",
-  "version": "1.0.9",
-  "description": "do more with data",
-  "main": "dist/index.js",
-  "typings": "dist/index.d.ts",
+  "name": "@push.rocks/smartdata",
+  "version": "5.16.0",
+  "private": false,
+  "description": "An advanced library for NoSQL data organization and manipulation using TypeScript with support for MongoDB, data validation, collections, and custom data types.",
+  "main": "dist_ts/index.js",
+  "typings": "dist_ts/index.d.ts",
+  "type": "module",
   "scripts": {
-    "test": "(npm run prepareMongo && npmts)",
-    "prepareMongo": "(rm -rf ./test/data && mkdir ./test/data/)"
+    "test": "tstest test/",
+    "testSearch": "tsx test/test.search.ts",
+    "build": "tsbuild --web --allowimplicitany",
+    "buildDocs": "tsdoc"
   },
   "repository": {
     "type": "git",
-    "url": "git+ssh://git@gitlab.com/pushrocks/smartdata.git"
+    "url": "https://code.foss.global/push.rocks/smartdata.git"
   },
   "author": "Lossless GmbH",
   "license": "MIT",
   "bugs": {
-    "url": "https://gitlab.com/pushrocks/smartdata/issues"
+    "url": "https://code.foss.global/push.rocks/smartdata/issues"
   },
-  "homepage": "https://gitlab.com/pushrocks/smartdata#README",
+  "homepage": "https://code.foss.global/push.rocks/smartdata#readme",
   "dependencies": {
-    "@types/mongodb": "^2.1.32",
-    "@types/q": "0.0.30",
-    "beautylog": "^5.0.23",
-    "mongodb": "^2.2.9",
-    "q": "^1.4.1",
-    "typings-global": "^1.0.14"
+    "@push.rocks/lik": "^6.0.14",
+    "@push.rocks/smartdelay": "^3.0.1",
+    "@push.rocks/smartlog": "^3.0.2",
+    "@push.rocks/smartmongo": "^2.0.12",
+    "@push.rocks/smartpromise": "^4.0.2",
+    "@push.rocks/smartrx": "^3.0.10",
+    "@push.rocks/smartstring": "^4.0.15",
+    "@push.rocks/smarttime": "^4.0.6",
+    "@push.rocks/smartunique": "^3.0.8",
+    "@push.rocks/taskbuffer": "^3.1.7",
+    "@tsclass/tsclass": "^9.0.0",
+    "mongodb": "^6.16.0"
   },
   "devDependencies": {
-    "@types/shelljs": "^0.3.30",
-    "@types/should": "^8.1.29",
-    "shelljs": "^0.7.4",
-    "should": "^11.1.0",
-    "smartstring": "^2.0.17",
-    "typings-test": "^1.0.3"
+    "@git.zone/tsbuild": "^2.3.2",
+    "@git.zone/tsrun": "^1.2.44",
+    "@git.zone/tstest": "^1.0.77",
+    "@push.rocks/qenv": "^6.0.5",
+    "@push.rocks/tapbundle": "^5.6.3",
+    "@types/node": "^22.15.2"
+  },
+  "files": [
+    "ts/**/*",
+    "ts_web/**/*",
+    "dist/**/*",
+    "dist_*/**/*",
+    "dist_ts/**/*",
+    "dist_ts_web/**/*",
+    "assets/**/*",
+    "cli.js",
+    "npmextra.json",
+    "readme.md"
+  ],
+  "browserslist": [
+    "last 1 chrome versions"
+  ],
+  "keywords": [
+    "data manipulation",
+    "NoSQL",
+    "MongoDB",
+    "TypeScript",
+    "data validation",
+    "collections",
+    "custom data types",
+    "ODM"
+  ],
+  "packageManager": "pnpm@10.7.0+sha512.6b865ad4b62a1d9842b61d674a393903b871d9244954f652b8842c2b553c72176b278f64c463e52d40fff8aba385c235c8c9ecf5cc7de4fd78b8bb6d49633ab6",
+  "pnpm": {
+    "overrides": {}
   }
 }

pnpm-workspace.yaml

@@ -0,0 +1,4 @@
+onlyBuiltDependencies:
+  - esbuild
+  - mongodb-memory-server
+  - puppeteer

qenv.yml

@@ -0,0 +1 @@
+required:

readme.md

@@ -0,0 +1,605 @@
+# @push.rocks/smartdata
+
+[![npm version](https://badge.fury.io/js/@push.rocks%2Fsmartdata.svg)](https://www.npmjs.com/package/@push.rocks/smartdata)
+
+A powerful TypeScript-first MongoDB wrapper that provides advanced features for distributed systems, real-time data synchronization, and easy data management.
+
+## Features
+
+- **Type-Safe MongoDB Integration**: Full TypeScript support with decorators for schema definition
+- **Document Management**: Type-safe CRUD operations with automatic timestamp tracking
+- **EasyStore**: Simple key-value storage with automatic persistence and sharing between instances
+- **Distributed Coordination**: Built-in support for leader election and distributed task management
+- **Real-time Data Sync**: Watchers for real-time data changes with RxJS integration
+- **Connection Management**: Automatic connection handling with connection pooling
+- **Collection Management**: Type-safe collection operations with automatic indexing
+- **Deep Query Type Safety**: Fully type-safe queries for nested object properties with `DeepQuery<T>`
+- **MongoDB Compatibility**: Support for all MongoDB query operators and advanced features
+- **Enhanced Cursors**: Chainable, type-safe cursor API with memory efficient data processing
+- **Type Conversion**: Automatic handling of MongoDB types like ObjectId and Binary data
+- **Serialization Hooks**: Custom serialization and deserialization of document properties
+- **Powerful Search Capabilities**: Unified `search(query)` API supporting field:value exact matches, multi-field regex searches, case-insensitive matching, and automatic escaping to prevent regex injection
+
+## Requirements
+
+- Node.js >= 16.x
+- MongoDB >= 4.4
+- TypeScript >= 4.x (for development)
+
+## Install
+
+To install `@push.rocks/smartdata`, use npm:
+
+```bash
+npm install @push.rocks/smartdata --save
+```
+
+Or with pnpm:
+
+```bash
+pnpm add @push.rocks/smartdata
+```
+
+## Usage
+
+`@push.rocks/smartdata` enables efficient data handling and operation management with a focus on using MongoDB. It leverages TypeScript for strong typing and ESM syntax for modern JavaScript usage.
+
+### Setting Up and Connecting to the Database
+
+Before interacting with the database, you need to set up and establish a connection. The `SmartdataDb` class handles connection pooling and automatic reconnection.
+
+```typescript
+import { SmartdataDb } from '@push.rocks/smartdata';
+
+// Create a new instance of SmartdataDb with MongoDB connection details
+const db = new SmartdataDb({
+  mongoDbUrl: 'mongodb://<USERNAME>:<PASSWORD>@localhost:27017/<DBNAME>',
+  mongoDbName: 'your-database-name',
+  mongoDbUser: 'your-username',
+  mongoDbPass: 'your-password',
+});
+
+// Initialize and connect to the database
+// This sets up a connection pool with max 100 connections
+await db.init();
+```
+
+### Defining Data Models
+
+Data models in `@push.rocks/smartdata` are classes that represent collections and documents in your MongoDB database. Use decorators such as `@Collection`, `@unI`, `@svDb`, `@index`, and `@searchable` to define your data models. Fields of type `ObjectId` or `Buffer` decorated with `@svDb()` will be stored as BSON ObjectId and Binary, respectively; no separate `@oid()` or `@bin()` decorators are required.
+
+```typescript
+import {
+  SmartDataDbDoc,
+  Collection,
+  unI,
+  svDb,
+  index,
+  searchable,
+} from '@push.rocks/smartdata';
+import { ObjectId } from 'mongodb';
+
+@Collection(() => db) // Associate this model with the database instance
+class User extends SmartDataDbDoc<User, User> {
+  @unI()
+  public id: string = 'unique-user-id'; // Mark 'id' as a unique index
+
+  @svDb()
+  @searchable() // Mark 'username' as searchable
+  public username: string; // Mark 'username' to be saved in DB
+
+  @svDb()
+  @searchable() // Mark 'email' as searchable
+  @index() // Create a regular index for this field
+  public email: string; // Mark 'email' to be saved in DB
+
+  @svDb()
+  public organizationId: ObjectId; // Stored as BSON ObjectId
+
+  @svDb()
+  public profilePicture: Buffer; // Stored as BSON Binary
+
+  @svDb({
+    serialize: (data) => JSON.stringify(data), // Custom serialization
+    deserialize: (data) => JSON.parse(data), // Custom deserialization
+  })
+  public preferences: Record<string, any>;
+
+  constructor(username: string, email: string) {
+    super();
+    this.username = username;
+    this.email = email;
+  }
+}
+```
+
+### CRUD Operations
+
+`@push.rocks/smartdata` simplifies CRUD operations with intuitive methods on model instances.
+
+#### Create
+
+```typescript
+const newUser = new User('myUsername', 'myEmail@example.com');
+await newUser.save(); // Save the new user to the database
+```
+
+#### Read
+
+```typescript
+// Fetch a single user by a unique attribute
+const user = await User.getInstance({ username: 'myUsername' });
+
+// Fetch multiple users that match criteria
+const users = await User.getInstances({ email: 'myEmail@example.com' });
+
+// Obtain a cursor for large result sets
+const cursor = await User.getCursor({ active: true });
+
+// Stream each document efficiently
+await cursor.forEach(async (user) => {
+  console.log(`Processing user: ${user.username}`);
+});
+
+// Manually iterate using next()
+let nextUser;
+while ((nextUser = await cursor.next())) {
+  console.log(`Next user: ${nextUser.username}`);
+}
+
+// Convert to array when the result set is small
+const userArray = await cursor.toArray();
+
+// Close the cursor to free resources
+await cursor.close();
+
+// For native cursor modifiers (sort, skip, limit), use getCursor with modifier option:
+const paginatedCursor = await User.getCursor(
+  { active: true },
+  { modifier: (c) => c.sort({ createdAt: -1 }).skip(20).limit(10) }
+);
+await paginatedCursor.forEach((user) => {
+  console.log(`Paginated user: ${user.username}`);
+});
+```
+
+#### Update
+
+```typescript
+// Assuming 'user' is an instance of User
+user.email = 'newEmail@example.com';
+await user.save(); // Update the user in the database
+
+// Upsert operations (insert if not exists, update if exists)
+const upsertedUser = await User.upsert(
+  { id: 'user-123' }, // Query to find the user
+  {
+    // Fields to update or insert
+    username: 'newUsername',
+    email: 'newEmail@example.com',
+  },
+);
+```
+
+#### Delete
+
+```typescript
+// Assuming 'user' is an instance of User
+await user.delete(); // Delete the user from the database
+```
+
+## Advanced Features
+
+### Search Functionality
+
+SmartData provides powerful, Lucene‑style search capabilities with robust fallback mechanisms:
+
+```typescript
+// Define a model with searchable fields
+@Collection(() => db)
+class Product extends SmartDataDbDoc<Product, Product> {
+  @unI() public id: string = 'product-id';
+  @svDb() @searchable() public name: string;
+  @svDb() @searchable() public description: string;
+  @svDb() @searchable() public category: string;
+  @svDb() public price: number;
+}
+
+// List searchable fields
+const searchableFields = Product.getSearchableFields();
+
+// 1: Exact phrase across all fields
+await Product.search('"Kindle Paperwhite"');
+
+// 2: Wildcard search across all fields
+await Product.search('Air*');
+
+// 3: Field‑scoped wildcard
+await Product.search('name:Air*');
+
+// 4: Boolean AND/OR/NOT
+await Product.search('category:Electronics AND name:iPhone');
+
+// 5: Grouping with parentheses
+await Product.search('(Furniture OR Electronics) AND Chair');
+
+// 6: Multi‑term unquoted (terms AND’d across fields)
+await Product.search('TypeScript Aufgabe');
+
+// 7: Empty query returns all documents
+await Product.search('');
+// 8: Scoped search with additional filter (e.g. multi-tenant isolation)
+await Product.search('book', { filter: { ownerId: currentUserId } });
+// 9: Post-search validation hook to drop unwanted results (e.g. price check)
+await Product.search('', { validate: (p) => p.price < 100 });
+```
+
+The search functionality includes:
+
+- `@searchable()` decorator for marking fields as searchable
+- `Class.getSearchableFields()` static method to list searchable fields for a model
+- `search(query: string)` method supporting:
+  - Exact phrase matches (`"my exact string"` or `'my exact string'`)
+  - Field‑scoped exact & wildcard searches (`field:value`, `field:Air*`)
+  - Wildcard searches across all fields (`Air*`, `?Pods`)
+  - Boolean operators (`AND`, `OR`, `NOT`) with grouping (`(...)`)
+  - Multi‑term unquoted queries AND’d across fields (`TypeScript Aufgabe`)
+  - Single/multi‑term regex searches across fields
+  - Empty queries returning all documents
+- Automatic escaping & wildcard conversion to prevent regex injection
+
+### EasyStore
+
+EasyStore provides a simple key-value storage system with automatic persistence:
+
+```typescript
+// Create an EasyStore instance with a specific type
+interface ConfigStore {
+  apiKey: string;
+  settings: {
+    theme: string;
+    notifications: boolean;
+  };
+}
+
+// Create a type-safe EasyStore
+const store = await db.createEasyStore<ConfigStore>('app-config');
+
+// Write and read data with full type safety
+await store.writeKey('apiKey', 'secret-api-key-123');
+await store.writeKey('settings', { theme: 'dark', notifications: true });
+
+const apiKey = await store.readKey('apiKey'); // Type: string
+const settings = await store.readKey('settings'); // Type: { theme: string, notifications: boolean }
+
+// Check if a key exists
+const hasKey = await store.hasKey('apiKey'); // true
+
+// Delete a key
+await store.deleteKey('apiKey');
+```
+
+### Distributed Coordination
+
+Built-in support for distributed systems with leader election:
+
+```typescript
+// Create a distributed coordinator
+const coordinator = new SmartdataDistributedCoordinator(db);
+
+// Start coordination
+await coordinator.start();
+
+// Handle leadership changes
+coordinator.on('leadershipChange', (isLeader) => {
+  if (isLeader) {
+    // This instance is now the leader
+    // Run leader-specific tasks
+    startPeriodicJobs();
+  } else {
+    // This instance is no longer the leader
+    stopPeriodicJobs();
+  }
+});
+
+// Access leadership status anytime
+if (coordinator.isLeader) {
+  // Run leader-only operations
+}
+
+// Execute a task only on the leader
+await coordinator.executeIfLeader(async () => {
+  // This code only runs on the leader instance
+  await runImportantTask();
+});
+
+// Stop coordination when shutting down
+await coordinator.stop();
+```
+
+### Real-time Data Watching
+
+Watch for changes in your collections with RxJS integration using MongoDB Change Streams:
+
+```typescript
+// Create a watcher for a specific collection with a query filter
+const watcher = await User.watch(
+  {
+    active: true, // Only watch for changes to active users
+  },
+  {
+    fullDocument: true, // Include the full document in change notifications
+    bufferTimeMs: 100, // Buffer changes for 100ms to reduce notification frequency
+  },
+);
+
+// Subscribe to changes using RxJS
+watcher.changeSubject.subscribe((change) => {
+  console.log('Change operation:', change.operationType); // 'insert', 'update', 'delete', etc.
+  console.log('Document changed:', change.docInstance); // The full document instance
+
+  // Handle different types of changes
+  if (change.operationType === 'insert') {
+    console.log('New user created:', change.docInstance.username);
+  } else if (change.operationType === 'update') {
+    console.log('User updated:', change.docInstance.username);
+  } else if (change.operationType === 'delete') {
+    console.log('User deleted');
+  }
+});
+
+// Manual observation with event emitter pattern
+watcher.on('change', (change) => {
+  console.log('Document changed:', change);
+});
+
+// Stop watching when no longer needed
+await watcher.stop();
+```
+
+### Managed Collections
+
+For more complex data models that require additional context:
+
+```typescript
+@Collection(() => db)
+class ManagedDoc extends SmartDataDbDoc<ManagedDoc, ManagedDoc> {
+  @unI()
+  public id: string = 'unique-id';
+
+  @svDb()
+  public data: string;
+
+  @managed()
+  public manager: YourCustomManager;
+
+  // The manager can provide additional functionality
+  async specialOperation() {
+    return this.manager.doSomethingSpecial(this);
+  }
+}
+```
+
+### Automatic Indexing
+
+Define indexes directly in your model class:
+
+```typescript
+@Collection(() => db)
+class Product extends SmartDataDbDoc<Product, Product> {
+  @unI() // Unique index
+  public id: string = 'product-id';
+
+  @svDb()
+  @index() // Regular index for faster queries
+  public category: string;
+
+  @svDb()
+  @index({ sparse: true }) // Sparse index with options
+  public optionalField?: string;
+
+  // Compound indexes can be defined in the collection decorator
+  @Collection(() => db, {
+    indexMap: {
+      compoundIndex: {
+        fields: { category: 1, name: 1 },
+        options: { background: true }
+      }
+    }
+  })
+}
+```
+
+### Transaction Support
+
+Use MongoDB transactions for atomic operations. SmartData now exposes `startSession()` and accepts an optional session in all fetch and write APIs:
+
+```typescript
+// start a client session (no await)
+const session = db.startSession();
+try {
+  // wrap operations in a transaction
+  await session.withTransaction(async () => {
+    // pass session as second arg to getInstance
+    const user = await User.getInstance({ id: 'user-id' }, session);
+    user.balance -= 100;
+    // pass session in save opts
+    await user.save({ session });
+
+    const recipient = await User.getInstance({ id: 'recipient-id' }, session);
+    recipient.balance += 100;
+    await recipient.save({ session });
+  });
+} finally {
+  await session.endSession();
+}
+```
+
+### Deep Object Queries
+
+SmartData provides fully type-safe deep property queries with the `DeepQuery` type:
+
+```typescript
+// If your document has nested objects
+class UserProfile extends SmartDataDbDoc<UserProfile, UserProfile> {
+  @unI()
+  public id: string = 'profile-id';
+
+  @svDb()
+  public user: {
+    details: {
+      firstName: string;
+      lastName: string;
+      address: {
+        city: string;
+        country: string;
+      };
+    };
+  };
+}
+
+// Type-safe string literals for dot notation
+const usersInUSA = await UserProfile.getInstances({
+  'user.details.address.country': 'USA',
+});
+
+// Fully typed deep queries with the DeepQuery type
+import { DeepQuery } from '@push.rocks/smartdata';
+
+const typedQuery: DeepQuery<UserProfile> = {
+  id: 'profile-id',
+  'user.details.firstName': 'John',
+  'user.details.address.country': 'USA',
+};
+
+// TypeScript will error if paths are incorrect
+const results = await UserProfile.getInstances(typedQuery);
+
+// MongoDB query operators are supported
+const operatorQuery: DeepQuery<UserProfile> = {
+  'user.details.address.country': 'USA',
+  'user.details.address.city': { $in: ['New York', 'Los Angeles'] },
+};
+
+const filteredResults = await UserProfile.getInstances(operatorQuery);
+```
+
+### Document Lifecycle Hooks
+
+Implement custom logic at different stages of a document's lifecycle:
+
+```typescript
+@Collection(() => db)
+class Order extends SmartDataDbDoc<Order, Order> {
+  @unI()
+  public id: string = 'order-id';
+
+  @svDb()
+  public total: number;
+
+  @svDb()
+  public items: string[];
+
+  // Called before saving the document
+  async beforeSave() {
+    // Calculate total based on items
+    this.total = await calculateTotal(this.items);
+
+    // Validate the document
+    if (this.items.length === 0) {
+      throw new Error('Order must have at least one item');
+    }
+  }
+
+  // Called after the document is saved
+  async afterSave() {
+    // Notify other systems about the saved order
+    await notifyExternalSystems(this);
+  }
+
+  // Called before deleting the document
+  async beforeDelete() {
+    // Check if order can be deleted
+    const canDelete = await checkOrderDeletable(this.id);
+    if (!canDelete) {
+      throw new Error('Order cannot be deleted');
+    }
+  }
+  // Called after deleting the document
+  async afterDelete() {
+    // Cleanup or audit actions
+    await auditLogDeletion(this.id);
+  }
+}
+```
+
+## Best Practices
+
+### Connection Management
+
+- Always call `db.init()` before using any database features
+- Use `db.close()` when shutting down your application
+- Set appropriate connection pool sizes based on your application's needs
+
+### Document Design
+
+- Use appropriate decorators (`@svDb`, `@unI`, `@index`, `@searchable`) to optimize database operations
+- Implement type-safe models by properly extending `SmartDataDbDoc`
+- Consider using interfaces to define document structures separately from implementation
+- Mark fields that need to be searched with the `@searchable()` decorator
+
+### Search Optimization
+
+- (Optional) Create MongoDB text indexes on searchable fields to speed up full-text search
+- Use `search(query)` for all search operations (field:value, partial matches, multi-word)
+- Prefer field-specific exact matches when possible for optimal performance
+- Avoid unnecessary complexity in query strings to keep regex searches efficient
+
+### Performance Optimization
+
+- Use cursors for large datasets instead of loading all documents into memory
+- Create appropriate indexes for frequent query patterns
+- Use projections to limit the fields returned when you don't need the entire document
+
+### Distributed Systems
+
+- Implement proper error handling for leader election events
+- Ensure all instances have synchronized clocks when using time-based coordination
+- Use the distributed coordinator's task management features for coordinated operations
+
+### Type Safety
+
+- Take advantage of the `DeepQuery<T>` type for fully type-safe queries
+- Define proper types for your document models to enhance IDE auto-completion
+- Use generic type parameters to specify exact document types when working with collections
+
+## Contributing
+
+We welcome contributions to @push.rocks/smartdata! Here's how you can help:
+
+1. Fork the repository
+2. Create a feature branch (`git checkout -b feature/amazing-feature`)
+3. Commit your changes (`git commit -m 'Add amazing feature'`)
+4. Push to the branch (`git push origin feature/amazing-feature`)
+5. Open a Pull Request
+
+Please make sure to update tests as appropriate and follow our coding standards.
+
+## License and Legal Information
+
+This repository is licensed under the MIT License. For details, see [MIT License](https://opensource.org/licenses/MIT).
+
+**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.
+
+### Trademarks
+
+This project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.
+
+### Company Information
+
+Task Venture Capital GmbH  
+Registered at District court Bremen HRB 35230 HB, Germany
+
+For any legal inquiries or if you require further information, please contact us via email at hello@task.vc.
+
+By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.

test/test.cursor.ts

@@ -0,0 +1,97 @@
+import { tap, expect } from '@push.rocks/tapbundle';
+import * as smartmongo from '@push.rocks/smartmongo';
+import { smartunique } from '../ts/plugins.js';
+import * as smartdata from '../ts/index.js';
+
+// Set up database connection
+let smartmongoInstance: smartmongo.SmartMongo;
+let testDb: smartdata.SmartdataDb;
+
+// Define a simple document model for cursor tests
+@smartdata.Collection(() => testDb)
+class CursorTest extends smartdata.SmartDataDbDoc<CursorTest, CursorTest> {
+  @smartdata.unI()
+  public id: string = smartunique.shortId();
+
+  @smartdata.svDb()
+  public name: string;
+
+  @smartdata.svDb()
+  public order: number;
+
+  constructor(name: string, order: number) {
+    super();
+    this.name = name;
+    this.order = order;
+  }
+}
+
+// Initialize the in-memory MongoDB and SmartdataDB
+tap.test('cursor init: start Mongo and SmartdataDb', async () => {
+  smartmongoInstance = await smartmongo.SmartMongo.createAndStart();
+  testDb = new smartdata.SmartdataDb(
+    await smartmongoInstance.getMongoDescriptor(),
+  );
+  await testDb.init();
+});
+
+// Insert sample documents
+tap.test('cursor insert: save 5 test documents', async () => {
+  for (let i = 1; i <= 5; i++) {
+    const doc = new CursorTest(`item${i}`, i);
+    await doc.save();
+  }
+  const count = await CursorTest.getCount({});
+  expect(count).toEqual(5);
+});
+
+// Test that toArray returns all documents
+tap.test('cursor toArray: retrieves all documents', async () => {
+  const cursor = await CursorTest.getCursor({});
+  const all = await cursor.toArray();
+  expect(all.length).toEqual(5);
+});
+
+// Test iteration via forEach
+tap.test('cursor forEach: iterates through all documents', async () => {
+  const names: string[] = [];
+  const cursor = await CursorTest.getCursor({});
+  await cursor.forEach(async (item) => {
+    names.push(item.name);
+  });
+  expect(names.length).toEqual(5);
+  expect(names).toContain('item3');
+});
+
+// Test native cursor modifiers: limit
+tap.test('cursor modifier limit: only two documents', async () => {
+  const cursor = await CursorTest.getCursor({}, { modifier: (c) => c.limit(2) });
+  const limited = await cursor.toArray();
+  expect(limited.length).toEqual(2);
+});
+
+// Test native cursor modifiers: sort and skip
+tap.test('cursor modifier sort & skip: returns correct order', async () => {
+  const cursor = await CursorTest.getCursor({}, {
+    modifier: (c) => c.sort({ order: -1 }).skip(1),
+  });
+  const results = await cursor.toArray();
+  // Skipped the first (order 5), next should be 4,3,2,1
+  expect(results.length).toEqual(4);
+  expect(results[0].order).toEqual(4);
+});
+
+// Cleanup: drop database, close connections, stop Mongo
+tap.test('cursor cleanup: drop DB and stop', async () => {
+  await testDb.mongoDb.dropDatabase();
+  await testDb.close();
+  if (smartmongoInstance) {
+    await smartmongoInstance.stopAndDumpToDir(
+      `.nogit/dbdump/test.cursor.ts`,
+    );
+  }
+  // Ensure process exits after cleanup
+  setTimeout(() => process.exit(), 2000);
+});
+
+export default tap.start();

test/test.d.ts

@@ -1 +0,0 @@
-import 'typings-test';

test/test.distributedcoordinator.ts

@@ -0,0 +1,122 @@
+import { tap, expect } from '@push.rocks/tapbundle';
+import * as smartmongo from '@push.rocks/smartmongo';
+import type * as taskbuffer from '@push.rocks/taskbuffer';
+
+import * as smartdata from '../ts/index.js';
+import {
+  SmartdataDistributedCoordinator,
+  DistributedClass,
+} from '../ts/classes.distributedcoordinator.js'; // path might need adjusting
+const totalInstances = 10;
+
+// =======================================
+// Connecting to the database server
+// =======================================
+
+let smartmongoInstance: smartmongo.SmartMongo;
+let testDb: smartdata.SmartdataDb;
+
+tap.test('should create a testinstance as database', async () => {
+  smartmongoInstance = await smartmongo.SmartMongo.createAndStart();
+  testDb = new smartdata.SmartdataDb(await smartmongoInstance.getMongoDescriptor());
+  await testDb.init();
+});
+
+tap.test('should instantiate DistributedClass', async (tools) => {
+  const instance = new DistributedClass();
+  expect(instance).toBeInstanceOf(DistributedClass);
+});
+
+tap.test('DistributedClass should update the time', async (tools) => {
+  const distributedCoordinator = new SmartdataDistributedCoordinator(testDb);
+  await distributedCoordinator.start();
+  const initialTime = distributedCoordinator.ownInstance.data.lastUpdated;
+  await distributedCoordinator.sendHeartbeat();
+  const updatedTime = distributedCoordinator.ownInstance.data.lastUpdated;
+  expect(updatedTime).toBeGreaterThan(initialTime);
+  await distributedCoordinator.stop();
+});
+
+tap.test('should instantiate SmartdataDistributedCoordinator', async (tools) => {
+  const distributedCoordinator = new SmartdataDistributedCoordinator(testDb);
+  await distributedCoordinator.start();
+  expect(distributedCoordinator).toBeInstanceOf(SmartdataDistributedCoordinator);
+  await distributedCoordinator.stop();
+});
+
+tap.test('SmartdataDistributedCoordinator should update leader status', async (tools) => {
+  const distributedCoordinator = new SmartdataDistributedCoordinator(testDb);
+  await distributedCoordinator.start();
+  await distributedCoordinator.checkAndMaybeLead();
+  expect(distributedCoordinator.ownInstance.data.elected).toBeOneOf([true, false]);
+  await distributedCoordinator.stop();
+});
+
+tap.test(
+  'SmartdataDistributedCoordinator should handle distributed task requests',
+  async (tools) => {
+    const distributedCoordinator = new SmartdataDistributedCoordinator(testDb);
+    await distributedCoordinator.start();
+
+    const mockTaskRequest: taskbuffer.distributedCoordination.IDistributedTaskRequest = {
+      submitterId: 'mockSubmitter12345', // Some unique mock submitter ID
+      requestResponseId: 'uni879873462hjhfkjhsdf', // Some unique ID for the request-response
+      taskName: 'SampleTask',
+      taskVersion: '1.0.0', // Assuming it's a version string
+      taskExecutionTime: Date.now(),
+      taskExecutionTimeout: 60000, // Let's say the timeout is 1 minute (60000 ms)
+      taskExecutionParallel: 5, // Let's assume max 5 parallel executions
+      status: 'requesting',
+    };
+
+    const response = await distributedCoordinator.fireDistributedTaskRequest(mockTaskRequest);
+    console.log(response); // based on your expected structure for the response
+    await distributedCoordinator.stop();
+  },
+);
+
+tap.test(
+  'SmartdataDistributedCoordinator should update distributed task requests',
+  async (tools) => {
+    const distributedCoordinator = new SmartdataDistributedCoordinator(testDb);
+
+    await distributedCoordinator.start();
+
+    const mockTaskRequest: taskbuffer.distributedCoordination.IDistributedTaskRequest = {
+      submitterId: 'mockSubmitter12345', // Some unique mock submitter ID
+      requestResponseId: 'uni879873462hjhfkjhsdf', // Some unique ID for the request-response
+      taskName: 'SampleTask',
+      taskVersion: '1.0.0', // Assuming it's a version string
+      taskExecutionTime: Date.now(),
+      taskExecutionTimeout: 60000, // Let's say the timeout is 1 minute (60000 ms)
+      taskExecutionParallel: 5, // Let's assume max 5 parallel executions
+      status: 'requesting',
+    };
+
+    await distributedCoordinator.updateDistributedTaskRequest(mockTaskRequest);
+    // Here, we can potentially check if a DB entry got updated or some other side-effect of the update method.
+    await distributedCoordinator.stop();
+  },
+);
+
+tap.test('should elect only one leader amongst multiple instances', async (tools) => {
+  const coordinators = Array.from({ length: totalInstances }).map(
+    () => new SmartdataDistributedCoordinator(testDb),
+  );
+  await Promise.all(coordinators.map((coordinator) => coordinator.start()));
+  const leaders = coordinators.filter((coordinator) => coordinator.ownInstance.data.elected);
+  for (const leader of leaders) {
+    console.log(leader.ownInstance);
+  }
+  expect(leaders.length).toEqual(1);
+
+  // stopping clears a coordinator from being elected.
+  await Promise.all(coordinators.map((coordinator) => coordinator.stop()));
+});
+
+tap.test('should clean up', async () => {
+  await smartmongoInstance.stopAndDumpToDir(`.nogit/dbdump/test.distributedcoordinator.ts`);
+  setTimeout(() => process.exit(), 2000);
+});
+
+tap.start({ throwOnError: true });

test/test.easystore.ts

@@ -0,0 +1,55 @@
+import { tap, expect } from '@push.rocks/tapbundle';
+import { Qenv } from '@push.rocks/qenv';
+import * as smartmongo from '@push.rocks/smartmongo';
+import { smartunique } from '../ts/plugins.js';
+
+const testQenv = new Qenv(process.cwd(), process.cwd() + '/.nogit/');
+
+console.log(process.memoryUsage());
+
+// the tested module
+import * as smartdata from '../ts/index.js';
+
+// =======================================
+// Connecting to the database server
+// =======================================
+
+let smartmongoInstance: smartmongo.SmartMongo;
+let testDb: smartdata.SmartdataDb;
+
+tap.test('should create a testinstance as database', async () => {
+  smartmongoInstance = await smartmongo.SmartMongo.createAndStart();
+  testDb = new smartdata.SmartdataDb(await smartmongoInstance.getMongoDescriptor());
+  await testDb.init();
+});
+
+tap.skip.test('should connect to atlas', async (tools) => {
+  const databaseName = `test-smartdata-${smartunique.shortId()}`;
+  testDb = new smartdata.SmartdataDb({
+    mongoDbUrl: await testQenv.getEnvVarOnDemand('MONGO_URL'),
+    mongoDbName: databaseName,
+  });
+  await testDb.init();
+});
+
+let easyStore: smartdata.EasyStore<{
+  key1: string;
+  key2: string;
+}>;
+
+tap.test('should create an easystore', async () => {
+  easyStore = await testDb.createEasyStore('hellothere');
+  await easyStore.writeKey('key1', 'hello');
+  const retrievedKey = await easyStore.readKey('key1');
+  expect(retrievedKey).toEqual('hello');
+});
+
+tap.test('close', async () => {
+  await testDb.mongoDb.dropDatabase();
+  await testDb.close();
+  if (smartmongoInstance) {
+    await smartmongoInstance.stop();
+  }
+});
+
+tap.start();

test/test.js

@@ -1,79 +0,0 @@
-"use strict";
-require('typings-test');
-const shelljs = require('shelljs');
-const should = require('should');
-const smartstring = require('smartstring');
-// the tested module
-const smartdata = require('../dist/index');
-let mongoChildProcess;
-let testDb;
-let testDbCollection;
-class testCar {
-    constructor(colorArg) {
-        this.color = colorArg;
-    }
-}
-describe('mongodb', function () {
-    it('should start mongodb', function (done) {
-        this.timeout(30000);
-        mongoChildProcess = shelljs.exec('mongod --dbpath=./test/data --port 27017', { async: true, silent: true });
-        let doneCalled = false;
-        mongoChildProcess.stdout.on('data', function (data) {
-            console.log(smartstring.indent.indentWithPrefix(data, "*** MongoDB Process *** : "));
-            if (!doneCalled) {
-                if (/waiting for connections on port 27017/.test(data)) {
-                    doneCalled = true;
-                    done();
-                }
-            }
-        });
-    });
-});
-describe('smartdata', function () {
-    it('should establish a connection to mongodb', function (done) {
-        testDb = new smartdata.Db('mongodb://localhost:27017/smartdata');
-        testDb.connect().then(() => { done(); });
-    });
-    it('should create a collection', function () {
-        testDbCollection = new smartdata.DbCollection('something', testDb);
-    });
-    it('should insert a doc into the collection', function (done) {
-        testDbCollection.insertOne({ value1: 'test' }).then(() => { done(); });
-    });
-    it('should find all docs of testDbCollection', function (done) {
-        testDbCollection.find({}).then((resultArray) => {
-            console.log(resultArray);
-            should(resultArray[0].value1).equal('test');
-            done();
-        });
-    });
-    it('should insert many docs into the collection', function (done) {
-        testDbCollection.insertMany([
-            { value1: 'test2' },
-            { value1: 'test', value2: 3, value3: 'hi' }
-        ]).then(() => { done(); });
-    });
-    it('should find a specified doc', function (done) {
-        testDbCollection.find({ 'value3': { '$exists': true } }).then((resultArray) => {
-            console.log(resultArray);
-            should(resultArray[0].value3).equal('hi');
-            done();
-        }).catch(console.log);
-    });
-    it('should close the db Connection', function () {
-        testDb.close();
-    });
-});
-describe('mongodb', function () {
-    it('should kill mongodb', function (done) {
-        this.timeout(30000);
-        mongoChildProcess.stdout.on('data', function (data) {
-            if (/dbexit:  rc: 0/.test(data)) {
-                done();
-            }
-        });
-        shelljs.exec('mongod --dbpath=./test/data --shutdown');
-        mongoChildProcess.kill('SIGTERM');
-    });
-});
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoidGVzdC5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbInRlc3QudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IjtBQUFBLFFBQU8sY0FFUCxDQUFDLENBRm9CO0FBRXJCLE1BQVksT0FBTyxXQUFNLFNBQ3pCLENBQUMsQ0FEaUM7QUFDbEMsTUFBWSxNQUFNLFdBQU0sUUFDeEIsQ0FBQyxDQUQrQjtBQUNoQyxNQUFZLFdBQVcsV0FBTSxhQUc3QixDQUFDLENBSHlDO0FBRTFDLG9CQUFvQjtBQUNwQixNQUFZLFNBQVMsV0FBTSxlQUUzQixDQUFDLENBRnlDO0FBRTFDLElBQUksaUJBQWlCLENBQUE7QUFDckIsSUFBSSxNQUFvQixDQUFBO0FBUXhCLElBQUksZ0JBQXNELENBQUE7QUFFMUQ7SUFFSSxZQUFZLFFBQWU7UUFDdkIsSUFBSSxDQUFDLEtBQUssR0FBRyxRQUFRLENBQUE7SUFDekIsQ0FBQztBQUNMLENBQUM7QUFHRCxRQUFRLENBQUMsU0FBUyxFQUFFO0lBQ2hCLEVBQUUsQ0FBQyxzQkFBc0IsRUFBRSxVQUFVLElBQUk7UUFDckMsSUFBSSxDQUFDLE9BQU8sQ0FBQyxLQUFLLENBQUMsQ0FBQTtRQUNuQixpQkFBaUIsR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLDBDQUEwQyxFQUFFLEVBQUUsS0FBSyxFQUFFLElBQUksRUFBRSxNQUFNLEVBQUUsSUFBSSxFQUFFLENBQUMsQ0FBQTtRQUMzRyxJQUFJLFVBQVUsR0FBRyxLQUFLLENBQUE7UUFDdEIsaUJBQWlCLENBQUMsTUFBTSxDQUFDLEVBQUUsQ0FBQyxNQUFNLEVBQUUsVUFBVSxJQUFJO1lBQzlDLE9BQU8sQ0FBQyxHQUFHLENBQUMsV0FBVyxDQUFDLE1BQU0sQ0FBQyxnQkFBZ0IsQ0FBQyxJQUFJLEVBQUUsNEJBQTRCLENBQUMsQ0FBQyxDQUFBO1lBQ3BGLEVBQUUsQ0FBQyxDQUFDLENBQUMsVUFBVSxDQUFDLENBQUMsQ0FBQztnQkFDZCxFQUFFLENBQUMsQ0FBQyx1Q0FBdUMsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLENBQUMsQ0FBQyxDQUFDO29CQUNyRCxVQUFVLEdBQUcsSUFBSSxDQUFBO29CQUNqQixJQUFJLEVBQUUsQ0FBQTtnQkFDVixDQUFDO1lBQ0wsQ0FBQztRQUNMLENBQUMsQ0FBQyxDQUFBO0lBQ04sQ0FBQyxDQUFDLENBQUE7QUFDTixDQUFDLENBQUMsQ0FBQTtBQUVGLFFBQVEsQ0FBQyxXQUFXLEVBQUU7SUFDbEIsRUFBRSxDQUFDLDBDQUEwQyxFQUFFLFVBQVUsSUFBSTtRQUN6RCxNQUFNLEdBQUcsSUFBSSxTQUFTLENBQUMsRUFBRSxDQUFDLHFDQUFxQyxDQUFDLENBQUE7UUFDaEUsTUFBTSxDQUFDLE9BQU8sRUFBRSxDQUFDLElBQUksQ0FBQyxRQUFRLElBQUksRUFBRSxDQUFBLENBQUMsQ0FBQyxDQUFDLENBQUE7SUFDM0MsQ0FBQyxDQUFDLENBQUE7SUFDRixFQUFFLENBQUMsNEJBQTRCLEVBQUU7UUFDN0IsZ0JBQWdCLEdBQUcsSUFBSSxTQUFTLENBQUMsWUFBWSxDQUFlLFdBQVcsRUFBRSxNQUFNLENBQUMsQ0FBQTtJQUNwRixDQUFDLENBQUMsQ0FBQTtJQUNGLEVBQUUsQ0FBQyx5Q0FBeUMsRUFBRSxVQUFVLElBQUk7UUFDeEQsZ0JBQWdCLENBQUMsU0FBUyxDQUFDLEVBQUUsTUFBTSxFQUFFLE1BQU0sRUFBRSxDQUFDLENBQUMsSUFBSSxDQUFDLFFBQVEsSUFBSSxFQUFFLENBQUEsQ0FBQyxDQUFDLENBQUMsQ0FBQTtJQUN6RSxDQUFDLENBQUMsQ0FBQTtJQUNGLEVBQUUsQ0FBQywwQ0FBMEMsRUFBRSxVQUFVLElBQUk7UUFDekQsZ0JBQWdCLENBQUMsSUFBSSxDQUFDLEVBQUUsQ0FBQyxDQUFDLElBQUksQ0FBQyxDQUFDLFdBQVc7WUFDdkMsT0FBTyxDQUFDLEdBQUcsQ0FBQyxXQUFXLENBQUMsQ0FBQTtZQUN4QixNQUFNLENBQUMsV0FBVyxDQUFDLENBQUMsQ0FBQyxDQUFDLE1BQU0sQ0FBQyxDQUFDLEtBQUssQ0FBQyxNQUFNLENBQUMsQ0FBQTtZQUMzQyxJQUFJLEVBQUUsQ0FBQTtRQUNWLENBQUMsQ0FBQyxDQUFBO0lBQ04sQ0FBQyxDQUFDLENBQUE7SUFDRixFQUFFLENBQUMsNkNBQTZDLEVBQUUsVUFBVSxJQUFJO1FBQzVELGdCQUFnQixDQUFDLFVBQVUsQ0FBQztZQUN4QixFQUFFLE1BQU0sRUFBRSxPQUFPLEVBQUU7WUFDbkIsRUFBRSxNQUFNLEVBQUUsTUFBTSxFQUFFLE1BQU0sRUFBRSxDQUFDLEVBQUUsTUFBTSxFQUFFLElBQUksRUFBRTtTQUM5QyxDQUFDLENBQUMsSUFBSSxDQUFDLFFBQVEsSUFBSSxFQUFFLENBQUEsQ0FBQyxDQUFDLENBQUMsQ0FBQTtJQUM3QixDQUFDLENBQUMsQ0FBQTtJQUNGLEVBQUUsQ0FBQyw2QkFBNkIsRUFBRSxVQUFVLElBQUk7UUFDNUMsZ0JBQWdCLENBQUMsSUFBSSxDQUFDLEVBQUMsUUFBUSxFQUFFLEVBQUMsU0FBUyxFQUFFLElBQUksRUFBQyxFQUFDLENBQUMsQ0FBQyxJQUFJLENBQUMsQ0FBQyxXQUFXO1lBQ2xFLE9BQU8sQ0FBQyxHQUFHLENBQUMsV0FBVyxDQUFDLENBQUE7WUFDeEIsTUFBTSxDQUFDLFdBQVcsQ0FBQyxDQUFDLENBQUMsQ0FBQyxNQUFNLENBQUMsQ0FBQyxLQUFLLENBQUMsSUFBSSxDQUFDLENBQUE7WUFDekMsSUFBSSxFQUFFLENBQUE7UUFDVixDQUFDLENBQUMsQ0FBQyxLQUFLLENBQUMsT0FBTyxDQUFDLEdBQUcsQ0FBQyxDQUFBO0lBQ3pCLENBQUMsQ0FBQyxDQUFBO0lBQ0YsRUFBRSxDQUFDLGdDQUFnQyxFQUFFO1FBQ2pDLE1BQU0sQ0FBQyxLQUFLLEVBQUUsQ0FBQTtJQUNsQixDQUFDLENBQUMsQ0FBQTtBQUNOLENBQUMsQ0FBQyxDQUFBO0FBRUYsUUFBUSxDQUFDLFNBQVMsRUFBRTtJQUNoQixFQUFFLENBQUMscUJBQXFCLEVBQUUsVUFBVSxJQUFJO1FBQ3BDLElBQUksQ0FBQyxPQUFPLENBQUMsS0FBSyxDQUFDLENBQUE7UUFDbkIsaUJBQWlCLENBQUMsTUFBTSxDQUFDLEVBQUUsQ0FBQyxNQUFNLEVBQUUsVUFBVSxJQUFJO1lBQzlDLEVBQUUsQ0FBQyxDQUFDLGdCQUFnQixDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDLENBQUM7Z0JBQzlCLElBQUksRUFBRSxDQUFBO1lBQ1YsQ0FBQztRQUNMLENBQUMsQ0FBQyxDQUFBO1FBQ0YsT0FBTyxDQUFDLElBQUksQ0FBQyx3Q0FBd0MsQ0FBQyxDQUFBO1FBQ3RELGlCQUFpQixDQUFDLElBQUksQ0FBQyxTQUFTLENBQUMsQ0FBQTtJQUNyQyxDQUFDLENBQUMsQ0FBQTtBQUNOLENBQUMsQ0FBQyxDQUFBIn0=

test/test.search.advanced.ts

@@ -0,0 +1,202 @@
+import { tap, expect } from '@push.rocks/tapbundle';
+import * as smartmongo from '@push.rocks/smartmongo';
+import * as smartdata from '../ts/index.js';
+import { searchable } from '../ts/classes.doc.js';
+import { smartunique } from '../ts/plugins.js';
+
+// Set up database connection
+let smartmongoInstance: smartmongo.SmartMongo;
+let testDb: smartdata.SmartdataDb;
+
+// Define a test class for advanced search scenarios
+@smartdata.Collection(() => testDb)
+class Product extends smartdata.SmartDataDbDoc<Product, Product> {
+  @smartdata.unI()
+  public id: string = smartunique.shortId();
+
+  @smartdata.svDb()
+  @searchable()
+  public name: string;
+
+  @smartdata.svDb()
+  @searchable()
+  public description: string;
+
+  @smartdata.svDb()
+  @searchable()
+  public category: string;
+
+  @smartdata.svDb()
+  public price: number;
+
+  constructor(
+    nameArg: string,
+    descriptionArg: string,
+    categoryArg: string,
+    priceArg: number,
+  ) {
+    super();
+    this.name = nameArg;
+    this.description = descriptionArg;
+    this.category = categoryArg;
+    this.price = priceArg;
+  }
+}
+
+// Initialize DB and insert sample products
+tap.test('setup advanced search database', async () => {
+  smartmongoInstance = await smartmongo.SmartMongo.createAndStart();
+  testDb = new smartdata.SmartdataDb(
+    await smartmongoInstance.getMongoDescriptor(),
+  );
+  await testDb.init();
+});
+
+tap.test('insert products for advanced search', async () => {
+  const products = [
+    new Product(
+      'Night Owl Lamp',
+      'Bright lamp for night reading',
+      'Lighting',
+      29,
+    ),
+    new Product(
+      'Day Light Lamp',
+      'Daytime lamp with adjustable brightness',
+      'Lighting',
+      39,
+    ),
+    new Product(
+      'Office Chair',
+      'Ergonomic chair for office',
+      'Furniture',
+      199,
+    ),
+    new Product(
+      'Gaming Chair',
+      'Comfortable for long gaming sessions',
+      'Furniture',
+      299,
+    ),
+    new Product(
+      'iPhone 12',
+      'Latest iPhone with A14 Bionic chip',
+      'Electronics',
+      999,
+    ),
+    new Product(
+      'AirPods',
+      'Wireless earbuds with noise cancellation',
+      'Electronics',
+      249,
+    ),
+  ];
+  for (const p of products) {
+    await p.save();
+  }
+  const all = await Product.getInstances({});
+  expect(all.length).toEqual(products.length);
+});
+
+// Simple exact field:value matching
+tap.test('simpleExact: category:Furniture returns chairs', async () => {
+  const res = await Product.search('category:Furniture');
+  expect(res.length).toEqual(2);
+  const names = res.map((r) => r.name).sort();
+  expect(names).toEqual(['Gaming Chair', 'Office Chair']);
+});
+
+// simpleExact invalid field should throw
+tap.test('simpleExact invalid field errors', async () => {
+  let error: Error;
+  try {
+    await Product.search('price:29');
+  } catch (e) {
+    error = e as Error;
+  }
+  expect(error).toBeTruthy();
+  expect(error.message).toMatch(/not searchable/);
+});
+
+// Quoted phrase search
+tap.test('quoted phrase "Bright lamp" matches Night Owl Lamp', async () => {
+  const res = await Product.search('"Bright lamp"');
+  expect(res.length).toEqual(1);
+  expect(res[0].name).toEqual('Night Owl Lamp');
+});
+
+tap.test("quoted phrase 'night reading' matches Night Owl Lamp", async () => {
+  const res = await Product.search("'night reading'");
+  expect(res.length).toEqual(1);
+  expect(res[0].name).toEqual('Night Owl Lamp');
+});
+
+
+tap.test('wildcard description:*gaming* matches Gaming Chair', async () => {
+  const res = await Product.search('description:*gaming*');
+  expect(res.length).toEqual(1);
+  expect(res[0].name).toEqual('Gaming Chair');
+});
+
+// Boolean AND and OR
+tap.test('boolean AND: category:Lighting AND lamp', async () => {
+  const res = await Product.search('category:Lighting AND lamp');
+  expect(res.length).toEqual(2);
+});
+
+tap.test('boolean OR: Furniture OR Electronics', async () => {
+  const res = await Product.search('Furniture OR Electronics');
+  expect(res.length).toEqual(4);
+});
+
+// Multi-term unquoted -> AND across terms
+tap.test('multi-term unquoted adjustable brightness', async () => {
+  const res = await Product.search('adjustable brightness');
+  expect(res.length).toEqual(1);
+  expect(res[0].name).toEqual('Day Light Lamp');
+});
+
+tap.test('multi-term unquoted Night Lamp', async () => {
+  const res = await Product.search('Night Lamp');
+  expect(res.length).toEqual(1);
+  expect(res[0].name).toEqual('Night Owl Lamp');
+});
+
+// Grouping with parentheses
+tap.test('grouping: (Furniture OR Electronics) AND Chair', async () => {
+  const res = await Product.search(
+    '(Furniture OR Electronics) AND Chair',
+  );
+  expect(res.length).toEqual(2);
+  const names = res.map((r) => r.name).sort();
+  expect(names).toEqual(['Gaming Chair', 'Office Chair']);
+});
+
+// Additional range and combined query tests
+tap.test('range query price:[30 TO 300] returns expected products', async () => {
+  const res = await Product.search('price:[30 TO 300]');
+  // Expect products with price between 30 and 300 inclusive: Day Light Lamp, Gaming Chair, Office Chair, AirPods
+  expect(res.length).toEqual(4);
+  const names = res.map((r) => r.name).sort();
+  expect(names).toEqual(['AirPods', 'Day Light Lamp', 'Gaming Chair', 'Office Chair']);
+});
+
+tap.test('should filter category and price range', async () => {
+  const res = await Product.search('category:Lighting AND price:[30 TO 40]');
+  expect(res.length).toEqual(1);
+  expect(res[0].name).toEqual('Day Light Lamp');
+});
+
+// Teardown
+tap.test('cleanup advanced search database', async () => {
+  await testDb.mongoDb.dropDatabase();
+  await testDb.close();
+  if (smartmongoInstance) {
+    await smartmongoInstance.stopAndDumpToDir(
+      `.nogit/dbdump/test.search.advanced.ts`,
+    );
+  }
+  setTimeout(() => process.exit(), 2000);
+});
+
+tap.start({ throwOnError: true });

test/test.search.ts

@@ -0,0 +1,408 @@
+import { tap, expect } from '@push.rocks/tapbundle';
+import * as smartmongo from '@push.rocks/smartmongo';
+import { smartunique } from '../ts/plugins.js';
+
+// Import the smartdata library
+import * as smartdata from '../ts/index.js';
+import { searchable } from '../ts/classes.doc.js';
+
+// Set up database connection
+let smartmongoInstance: smartmongo.SmartMongo;
+let testDb: smartdata.SmartdataDb;
+// Class for location-based wildcard/phrase tests
+let LocationDoc: any;
+
+// Define a test class with searchable fields using the standard SmartDataDbDoc
+@smartdata.Collection(() => testDb)
+class Product extends smartdata.SmartDataDbDoc<Product, Product> {
+  @smartdata.unI()
+  public id: string = smartunique.shortId();
+
+  @smartdata.svDb()
+  @searchable()
+  public name: string;
+
+  @smartdata.svDb()
+  @searchable()
+  public description: string;
+
+  @smartdata.svDb()
+  @searchable()
+  public category: string;
+
+  @smartdata.svDb()
+  public price: number;
+
+  constructor(nameArg: string, descriptionArg: string, categoryArg: string, priceArg: number) {
+    super();
+    this.name = nameArg;
+    this.description = descriptionArg;
+    this.category = categoryArg;
+    this.price = priceArg;
+  }
+}
+
+tap.test('should create a test database instance', async () => {
+  smartmongoInstance = await smartmongo.SmartMongo.createAndStart();
+  testDb = new smartdata.SmartdataDb(await smartmongoInstance.getMongoDescriptor());
+  await testDb.init();
+});
+
+tap.test('should create test products with searchable fields', async () => {
+  // Create several products with different fields to search
+  const products = [
+    new Product('iPhone 12', 'Latest iPhone with A14 Bionic chip', 'Electronics', 999),
+    new Product('MacBook Pro', 'Powerful laptop for professionals', 'Electronics', 1999),
+    new Product('AirPods', 'Wireless earbuds with noise cancellation', 'Electronics', 249),
+    new Product('Galaxy S21', 'Samsung flagship phone with great camera', 'Electronics', 899),
+    new Product('Kindle Paperwhite', 'E-reader with built-in light', 'Books', 129),
+    new Product('Harry Potter', 'Fantasy book series about wizards', 'Books', 49),
+    new Product('Coffee Maker', 'Automatic drip coffee machine', 'Kitchen', 89),
+    new Product('Blender', 'High-speed blender for smoothies', 'Kitchen', 129),
+  ];
+
+  // Save all products to the database
+  for (const product of products) {
+    await product.save();
+  }
+
+  // Verify that we can get all products
+  const allProducts = await Product.getInstances({});
+  expect(allProducts.length).toEqual(products.length);
+  console.log(`Successfully created and saved ${allProducts.length} products`);
+});
+
+tap.test('should retrieve searchable fields for a class', async () => {
+  // Use the getSearchableFields function to verify our searchable fields
+  const searchableFields = Product.getSearchableFields();
+  console.log('Searchable fields:', searchableFields);
+
+  expect(searchableFields.length).toEqual(3);
+  expect(searchableFields).toContain('name');
+  expect(searchableFields).toContain('description');
+  expect(searchableFields).toContain('category');
+});
+
+tap.test('should search products by exact field match', async () => {
+  // Basic field exact match search
+  const electronicsProducts = await Product.getInstances({ category: 'Electronics' });
+  console.log(`Found ${electronicsProducts.length} products in Electronics category`);
+
+  expect(electronicsProducts.length).toEqual(4);
+});
+
+tap.test('should search products by basic search method', async () => {
+  // Using the basic search method with simple Lucene query
+  try {
+    const iPhoneResults = await Product.search('iPhone');
+    console.log(`Found ${iPhoneResults.length} products matching 'iPhone' using basic search`);
+
+    expect(iPhoneResults.length).toEqual(1);
+    expect(iPhoneResults[0].name).toEqual('iPhone 12');
+  } catch (error) {
+    console.error('Basic search error:', error.message);
+    // If basic search fails, we'll demonstrate the enhanced approach in later tests
+    console.log('Will test with enhanced searchWithLucene method next');
+  }
+});
+
+tap.test('should search products with search method', async () => {
+  // Using the robust searchWithLucene method
+    const wirelessResults = await Product.search('wireless');
+    console.log(
+      `Found ${wirelessResults.length} products matching 'wireless' using search`,
+    );
+
+  expect(wirelessResults.length).toEqual(1);
+  expect(wirelessResults[0].name).toEqual('AirPods');
+});
+
+tap.test('should search products by category with search', async () => {
+  // Using field-specific search with searchWithLucene
+  const kitchenResults = await Product.search('category:Kitchen');
+  console.log(`Found ${kitchenResults.length} products in Kitchen category using search`);
+
+  expect(kitchenResults.length).toEqual(2);
+  expect(kitchenResults[0].category).toEqual('Kitchen');
+  expect(kitchenResults[1].category).toEqual('Kitchen');
+});
+
+tap.test('should search products with partial word matches', async () => {
+  // Testing partial word matches
+  const proResults = await Product.search('Pro');
+  console.log(`Found ${proResults.length} products matching 'Pro'`);
+
+  // Should match both "MacBook Pro" and "professionals" in description
+  expect(proResults.length).toBeGreaterThan(0);
+});
+
+tap.test('should search across multiple searchable fields', async () => {
+  // Test searching across all searchable fields
+  const bookResults = await Product.search('book');
+  console.log(`Found ${bookResults.length} products matching 'book' across all fields`);
+
+  // Should match "MacBook" in name and "Books" in category
+  expect(bookResults.length).toBeGreaterThan(1);
+});
+
+tap.test('should handle case insensitive searches', async () => {
+  // Test case insensitivity
+  const electronicsResults = await Product.search('electronics');
+  const ElectronicsResults = await Product.search('Electronics');
+
+  console.log(`Found ${electronicsResults.length} products matching lowercase 'electronics'`);
+  console.log(`Found ${ElectronicsResults.length} products matching capitalized 'Electronics'`);
+
+  // Both searches should return the same results
+  expect(electronicsResults.length).toEqual(ElectronicsResults.length);
+});
+
+tap.test('should demonstrate search fallback mechanisms', async () => {
+  console.log('\n====== FALLBACK MECHANISM DEMONSTRATION ======');
+  console.log('If MongoDB query fails, searchWithLucene will:');
+  console.log('1. Try using basic MongoDB filters');
+  console.log('2. Fall back to field-specific searches');
+  console.log('3. As last resort, perform in-memory filtering');
+  console.log('This ensures robust search even with complex queries');
+  console.log('==============================================\n');
+
+  // Use a simpler term that should be found in descriptions
+  // Avoid using "OR" operator which requires a text index
+  const results = await Product.search('high');
+  console.log(`Found ${results.length} products matching 'high'`);
+
+  // "High-speed blender" contains "high"
+  expect(results.length).toBeGreaterThan(0);
+
+  // Try another fallback example that won't need $text
+  const powerfulResults = await Product.search('powerful');
+  console.log(`Found ${powerfulResults.length} products matching 'powerful'`);
+
+  // "Powerful laptop for professionals" contains "powerful"
+  expect(powerfulResults.length).toBeGreaterThan(0);
+});
+
+tap.test('should explain the advantages of the integrated approach', async () => {
+  console.log('\n====== INTEGRATED SEARCH APPROACH BENEFITS ======');
+  console.log('1. No separate class hierarchy - keeps code simple');
+  console.log('2. Enhanced convertFilterForMongoDb handles MongoDB operators');
+  console.log('3. Robust fallback mechanisms ensure searches always work');
+  console.log('4. searchWithLucene provides powerful search capabilities');
+  console.log('5. Backwards compatible with existing code');
+  console.log('================================================\n');
+
+  expect(true).toEqual(true);
+});
+
+// Additional robustness tests
+tap.test('should search exact name using field:value', async () => {
+  const nameResults = await Product.search('name:AirPods');
+  expect(nameResults.length).toEqual(1);
+  expect(nameResults[0].name).toEqual('AirPods');
+});
+
+tap.test('should throw when searching non-searchable field', async () => {
+  let error: Error;
+  try {
+    await Product.search('price:129');
+  } catch (err) {
+    error = err as Error;
+  }
+  expect(error).toBeTruthy();
+  expect(error.message).toMatch(/not searchable/);
+});
+
+tap.test('empty query should return all products', async () => {
+  const allResults = await Product.search('');
+  expect(allResults.length).toEqual(8);
+});
+
+tap.test('should search multi-word term across fields', async () => {
+  const termResults = await Product.search('iPhone 12');
+  expect(termResults.length).toEqual(1);
+  expect(termResults[0].name).toEqual('iPhone 12');
+});
+
+// Additional search scenarios
+tap.test('should return zero results for non-existent terms', async () => {
+  const noResults = await Product.search('NonexistentTerm');
+  expect(noResults.length).toEqual(0);
+});
+
+tap.test('should search products by description term "noise"', async () => {
+  const noiseResults = await Product.search('noise');
+  expect(noiseResults.length).toEqual(1);
+  expect(noiseResults[0].name).toEqual('AirPods');
+});
+
+tap.test('should search products by description term "flagship"', async () => {
+  const flagshipResults = await Product.search('flagship');
+  expect(flagshipResults.length).toEqual(1);
+  expect(flagshipResults[0].name).toEqual('Galaxy S21');
+});
+
+tap.test('should search numeric strings "12"', async () => {
+  const twelveResults = await Product.search('12');
+  expect(twelveResults.length).toEqual(1);
+  expect(twelveResults[0].name).toEqual('iPhone 12');
+});
+
+tap.test('should search hyphenated terms "high-speed"', async () => {
+  const hyphenResults = await Product.search('high-speed');
+  expect(hyphenResults.length).toEqual(1);
+  expect(hyphenResults[0].name).toEqual('Blender');
+});
+
+tap.test('should search hyphenated terms "E-reader"', async () => {
+  const ereaderResults = await Product.search('E-reader');
+  expect(ereaderResults.length).toEqual(1);
+  expect(ereaderResults[0].name).toEqual('Kindle Paperwhite');
+});
+
+// Additional robustness tests
+tap.test('should return all products for empty search', async () => {
+  const searchResults = await Product.search('');
+  const allProducts = await Product.getInstances({});
+  expect(searchResults.length).toEqual(allProducts.length);
+});
+
+tap.test('should support wildcard plain term across all fields', async () => {
+  const results = await Product.search('*book*');
+  const names = results.map((r) => r.name).sort();
+  expect(names).toEqual(['Harry Potter', 'Kindle Paperwhite', 'MacBook Pro']);
+});
+
+tap.test('should support wildcard plain term with question mark pattern', async () => {
+  const results = await Product.search('?one?');
+  const names = results.map((r) => r.name).sort();
+  expect(names).toEqual(['Galaxy S21', 'iPhone 12']);
+});
+
+// Filter and Validation tests
+tap.test('should apply filter option to restrict results', async () => {
+  // search term 'book' across all fields but restrict to Books category
+  const bookFiltered = await Product.search('book', { filter: { category: 'Books' } });
+  expect(bookFiltered.length).toEqual(2);
+  bookFiltered.forEach((p) => expect(p.category).toEqual('Books'));
+});
+tap.test('should apply validate hook to post-filter results', async () => {
+  // return only products with price > 500
+  const expensive = await Product.search('', { validate: (p) => p.price > 500 });
+  expect(expensive.length).toBeGreaterThan(0);
+  expensive.forEach((p) => expect(p.price).toBeGreaterThan(500));
+});
+
+// Tests for quoted and wildcard field-specific phrases
+tap.test('setup location test products', async () => {
+  @smartdata.Collection(() => testDb)
+  class LD extends smartdata.SmartDataDbDoc<LD, LD> {
+    @smartdata.unI() public id: string = smartunique.shortId();
+    @smartdata.svDb() @searchable() public location: string;
+    constructor(loc: string) { super(); this.location = loc; }
+  }
+  // Assign to outer variable for subsequent tests
+  LocationDoc = LD;
+  const locations = ['Berlin', 'Frankfurt am Main', 'Frankfurt am Oder', 'London'];
+  for (const loc of locations) {
+    await new LocationDoc(loc).save();
+  }
+});
+tap.test('should search exact quoted field phrase', async () => {
+  const results = await (LocationDoc as any).search('location:"Frankfurt am Main"');
+  expect(results.length).toEqual(1);
+  expect(results[0].location).toEqual('Frankfurt am Main');
+});
+tap.test('should search wildcard quoted field phrase', async () => {
+  const results = await (LocationDoc as any).search('location:"Frankfurt am *"');
+  const names = results.map((d: any) => d.location).sort();
+  expect(names).toEqual(['Frankfurt am Main', 'Frankfurt am Oder']);
+});
+tap.test('should search unquoted wildcard field', async () => {
+  const results = await (LocationDoc as any).search('location:Frankfurt*');
+  const names = results.map((d: any) => d.location).sort();
+  expect(names).toEqual(['Frankfurt am Main', 'Frankfurt am Oder']);
+});
+
+// Combined free-term + field phrase/wildcard tests
+let CombinedDoc: any;
+tap.test('setup combined docs for free-term and location tests', async () => {
+  @smartdata.Collection(() => testDb)
+  class CD extends smartdata.SmartDataDbDoc<CD, CD> {
+    @smartdata.unI() public id: string = smartunique.shortId();
+    @smartdata.svDb() @searchable() public name: string;
+    @smartdata.svDb() @searchable() public location: string;
+    constructor(name: string, location: string) { super(); this.name = name; this.location = location; }
+  }
+  CombinedDoc = CD;
+  const docs = [
+    new CombinedDoc('TypeScript', 'Berlin'),
+    new CombinedDoc('TypeScript', 'Frankfurt am Main'),
+    new CombinedDoc('TypeScript', 'Frankfurt am Oder'),
+    new CombinedDoc('JavaScript', 'Berlin'),
+  ];
+  for (const d of docs) await d.save();
+});
+tap.test('should search free term and exact quoted field phrase', async () => {
+  const res = await CombinedDoc.search('TypeScript location:"Berlin"');
+  expect(res.length).toEqual(1);
+  expect(res[0].location).toEqual('Berlin');
+});
+tap.test('should not match free term with non-matching quoted field phrase', async () => {
+  const res = await CombinedDoc.search('TypeScript location:"Frankfurt d"');
+  expect(res.length).toEqual(0);
+});
+tap.test('should search free term with quoted wildcard field phrase', async () => {
+  const res = await CombinedDoc.search('TypeScript location:"Frankfurt am *"');
+  const locs = res.map((r: any) => r.location).sort();
+  expect(locs).toEqual(['Frankfurt am Main', 'Frankfurt am Oder']);
+});
+// Quoted exact field phrase without wildcard should return no matches if no exact match
+tap.test('should not match location:"Frankfurt d"', async () => {
+  const results = await (LocationDoc as any).search('location:"Frankfurt d"');
+  expect(results.length).toEqual(0);
+});
+
+// Combined free-term and field wildcard tests
+tap.test('should combine free term and wildcard field search', async () => {
+  const results = await Product.search('book category:Book*');
+  expect(results.length).toEqual(2);
+  results.forEach((p) => expect(p.category).toEqual('Books'));
+});
+tap.test('should not match when free term matches but wildcard field does not', async () => {
+  const results = await Product.search('book category:Kitchen*');
+  expect(results.length).toEqual(0);
+});
+
+// Non-searchable field should cause an error for combined queries
+tap.test('should throw when combining term with non-searchable field', async () => {
+  let error: Error;
+  try {
+    await Product.search('book location:Berlin');
+  } catch (e) {
+    error = e as Error;
+  }
+  expect(error).toBeTruthy();
+  expect(error.message).toMatch(/not searchable/);
+});
+tap.test('should throw when combining term with non-searchable wildcard field', async () => {
+  let error: Error;
+  try {
+    await Product.search('book location:Berlin*');
+  } catch (e) {
+    error = e as Error;
+  }
+  expect(error).toBeTruthy();
+  expect(error.message).toMatch(/not searchable/);
+});
+
+// Close database connection
+tap.test('close database connection', async () => {
+  await testDb.mongoDb.dropDatabase();
+  await testDb.close();
+  if (smartmongoInstance) {
+    await smartmongoInstance.stopAndDumpToDir(`.nogit/dbdump/test.search.ts`);
+  }
+  setTimeout(() => process.exit(), 2000);
+});
+
+tap.start({ throwOnError: true });

test/test.ts

@@ -1,93 +1,236 @@
-import 'typings-test'
+import { tap, expect } from '@push.rocks/tapbundle';
+import { Qenv } from '@push.rocks/qenv';
+import * as smartmongo from '@push.rocks/smartmongo';
+import { smartunique } from '../ts/plugins.js';
 
-import * as shelljs from 'shelljs'
-import * as should from 'should'
-import * as smartstring from 'smartstring'
+import * as mongodb from 'mongodb';
+
+const testQenv = new Qenv(process.cwd(), process.cwd() + '/.nogit/');
+
+console.log(process.memoryUsage());
 
 // the tested module
-import * as smartdata from '../dist/index'
+import * as smartdata from '../ts/index.js';
 
-let mongoChildProcess
-let testDb: smartdata.Db
+// =======================================
+// Connecting to the database server
+// =======================================
 
-interface ITestObject1 {
-    value1: string
-    value2?: number
-    value3?: string
-}
+let smartmongoInstance: smartmongo.SmartMongo;
+let testDb: smartdata.SmartdataDb;
 
-let testDbCollection: smartdata.DbCollection<ITestObject1>
+const totalCars = 2000;
 
-class testCar {
-    color: string
-    constructor(colorArg:string) {
-        this.color = colorArg
+tap.test('should create a testinstance as database', async () => {
+  smartmongoInstance = await smartmongo.SmartMongo.createAndStart();
+  testDb = new smartdata.SmartdataDb(await smartmongoInstance.getMongoDescriptor());
+  await testDb.init();
+});
+
+tap.skip.test('should connect to atlas', async (tools) => {
+  const databaseName = `test-smartdata-${smartunique.shortId()}`;
+  testDb = new smartdata.SmartdataDb({
+    mongoDbUrl: await testQenv.getEnvVarOnDemand('MONGO_URL'),
+    mongoDbName: databaseName,
+  });
+  await testDb.init();
+});
+
+// =======================================
+// The actual tests
+// =======================================
+
+// ------
+// Collections
+// ------
+
+@smartdata.Collection(() => {
+  return testDb;
+})
+class Car extends smartdata.SmartDataDbDoc<Car, Car> {
+  @smartdata.unI()
+  public index: string = smartunique.shortId();
+
+  @smartdata.svDb()
+  public color: string;
+
+  @smartdata.svDb()
+  public brand: string;
+
+  @smartdata.svDb()
+  public testBuffer = Buffer.from('hello');
+
+  @smartdata.svDb()
+  deepData = {
+    sodeep: 'yes',
+  };
+
+  constructor(colorArg: string, brandArg: string) {
+    super();
+    this.color = colorArg;
+    this.brand = brandArg;
   }
 }
 
+tap.test('should create a new id', async () => {
+  const newid = await Car.getNewId();
+  console.log(newid);
+});
 
-describe('mongodb', function () {
-    it('should start mongodb', function (done) {
-        this.timeout(30000)
-        mongoChildProcess = shelljs.exec('mongod --dbpath=./test/data --port 27017', { async: true, silent: true })
-        let doneCalled = false
-        mongoChildProcess.stdout.on('data', function (data) {
-            console.log(smartstring.indent.indentWithPrefix(data, "*** MongoDB Process *** : "))
-            if (!doneCalled) {
-                if (/waiting for connections on port 27017/.test(data)) {
-                    doneCalled = true
-                    done()
-                }
-            }
-        })
-    })
-})
+tap.test('should save the car to the db', async (toolsArg) => {
+  const myCar = new Car('red', 'Volvo');
+  await myCar.save();
 
-describe('smartdata', function () {
-    it('should establish a connection to mongodb', function (done) {
-        testDb = new smartdata.Db('mongodb://localhost:27017/smartdata')
-        testDb.connect().then(() => { done() })
-    })
-    it('should create a collection', function () {
-        testDbCollection = new smartdata.DbCollection<ITestObject1>('something', testDb)
-    })
-    it('should insert a doc into the collection', function (done) {
-        testDbCollection.insertOne({ value1: 'test' }).then(() => { done() })
-    })
-    it('should find all docs of testDbCollection', function (done) {
-        testDbCollection.find({}).then((resultArray) => {
-            console.log(resultArray)
-            should(resultArray[0].value1).equal('test')
-            done()
-        })
-    })
-    it('should insert many docs into the collection', function (done) {
-        testDbCollection.insertMany([
-            { value1: 'test2' },
-            { value1: 'test', value2: 3, value3: 'hi' }
-        ]).then(() => { done() })
-    })
-    it('should find a specified doc', function (done) {
-        testDbCollection.find({'value3': {'$exists': true}}).then((resultArray) => {
-            console.log(resultArray)
-            should(resultArray[0].value3).equal('hi')
-            done()
-        }).catch(console.log)
-    })
-    it('should close the db Connection', function () {
-        testDb.close()
-    })
-})
+  const myCar2 = new Car('red', 'Volvo');
+  await myCar2.save();
 
-describe('mongodb', function () {
-    it('should kill mongodb', function (done) {
-        this.timeout(30000)
-        mongoChildProcess.stdout.on('data', function (data) {
-            if (/dbexit:  rc: 0/.test(data)) {
-                done()
+  let counter = 0;
+
+  const gottenCarInstance = await Car.getInstance({});
+  console.log(gottenCarInstance.testBuffer instanceof mongodb.Binary);
+  process.memoryUsage();
+  do {
+    const myCar3 = new Car('red', 'Renault');
+    await myCar3.save();
+    counter++;
+    if (counter % 100 === 0) {
+      console.log(
+        `Filled database with ${counter} of ${totalCars} Cars and memory usage ${
+          process.memoryUsage().rss / 1e6
+        } MB`,
+      );
     }
-        })
-        shelljs.exec('mongod --dbpath=./test/data --shutdown')
-        mongoChildProcess.kill('SIGTERM')
-    })
+  } while (counter < totalCars);
+  console.log(process.memoryUsage());
+});
+
+tap.test('expect to get instance of Car with shallow match', async () => {
+  const totalQueryCycles = totalCars / 2;
+  let counter = 0;
+  do {
+    const timeStart = Date.now();
+    const myCars = await Car.getInstances({
+      brand: 'Renault',
+    });
+    if (counter % 10 === 0) {
+      console.log(
+        `performed ${counter} of ${totalQueryCycles} total query cycles: took ${
+          Date.now() - timeStart
+        }ms to query a set of 2000 with memory footprint ${process.memoryUsage().rss / 1e6} MB`,
+      );
+    }
+    expect(myCars[0].deepData.sodeep).toEqual('yes');
+    expect(myCars[0].brand).toEqual('Renault');
+    counter++;
+  } while (counter < totalQueryCycles);
+});
+
+tap.test('expect to get instance of Car with deep match', async () => {
+  const totalQueryCycles = totalCars / 6;
+  let counter = 0;
+  do {
+    const timeStart = Date.now();
+    const myCars2 = await Car.getInstances({
+      deepData: {
+        sodeep: 'yes',
+      },
+    });
+    if (counter % 10 === 0) {
+      console.log(
+        `performed ${counter} of ${totalQueryCycles} total query cycles: took ${
+          Date.now() - timeStart
+        }ms to deep query a set of 2000 with memory footprint ${process.memoryUsage().rss / 1e6} MB`,
+      );
+    }
+    expect(myCars2[0].deepData.sodeep).toEqual('yes');
+    expect(myCars2[0].brand).toEqual('Volvo');
+    counter++;
+  } while (counter < totalQueryCycles);
+});
+
+tap.test('expect to get instance of Car and update it', async () => {
+  const myCar = await Car.getInstance<Car>({
+    brand: 'Volvo',
+  });
+  expect(myCar.color).toEqual('red');
+  myCar.color = 'blue';
+  await myCar.save();
+});
+
+tap.test('should be able to delete an instance of car', async () => {
+  const myCars = await Car.getInstances({
+    brand: 'Volvo',
+    color: 'blue',
+  });
+  console.log(myCars);
+  expect(myCars[0].color).toEqual('blue');
+  for (const myCar of myCars) {
+    await myCar.delete();
+  }
+
+  const myCar2 = await Car.getInstance<Car>({
+    brand: 'Volvo',
+  });
+  expect(myCar2.color).toEqual('red');
+});
+
+// tslint:disable-next-line: max-classes-per-file
+@smartdata.Collection(() => {
+  return testDb;
 })
+class Truck extends smartdata.SmartDataDbDoc<Car, Car> {
+  @smartdata.unI()
+  public id: string = smartunique.shortId();
+
+  @smartdata.svDb()
+  public color: string;
+
+  @smartdata.svDb()
+  public brand: string;
+
+  constructor(colorArg: string, brandArg: string) {
+    super();
+    this.color = colorArg;
+    this.brand = brandArg;
+  }
+}
+
+tap.test('should store a new Truck', async () => {
+  const truck = new Truck('blue', 'MAN');
+  await truck.save();
+  const myTruck2 = await Truck.getInstance({ color: 'blue' });
+  expect(myTruck2.color).toEqual('blue');
+  myTruck2.color = 'red';
+  await myTruck2.save();
+  const myTruck3 = await Truck.getInstance({ color: 'blue' });
+  expect(myTruck3).toBeNull();
+});
+
+tap.test('should return a count', async () => {
+  const truckCount = await Truck.getCount();
+  expect(truckCount).toEqual(1);
+});
+
+tap.test('should use a cursor', async () => {
+  const cursor = await Car.getCursor({});
+  let counter = 0;
+  await cursor.forEach(async (carArg) => {
+    counter++;
+    counter % 50 === 0 ? console.log(`50 more of ${carArg.color}`) : null;
+  });
+});
+
+// =======================================
+// close the database connection
+// =======================================
+tap.test('close', async () => {
+  if (smartmongoInstance) {
+    await smartmongoInstance.stopAndDumpToDir('./.nogit/dbdump/test.ts');
+  } else {
+    await testDb.mongoDb.dropDatabase();
+    await testDb.close();
+  }
+  setTimeout(() => process.exit(), 2000);
+});
+
+tap.start({ throwOnError: true });

test/test.typescript.ts

@@ -0,0 +1,95 @@
+import { tap, expect } from '@push.rocks/tapbundle';
+import { Qenv } from '@push.rocks/qenv';
+import * as smartmongo from '@push.rocks/smartmongo';
+import { smartunique } from '../ts/plugins.js';
+
+const testQenv = new Qenv(process.cwd(), process.cwd() + '/.nogit/');
+
+console.log(process.memoryUsage());
+
+// the tested module
+import * as smartdata from '../ts/index.js';
+
+// =======================================
+// Connecting to the database server
+// =======================================
+
+let smartmongoInstance: smartmongo.SmartMongo;
+let testDb: smartdata.SmartdataDb;
+
+const totalCars = 2000;
+
+tap.test('should create a testinstance as database', async () => {
+  smartmongoInstance = await smartmongo.SmartMongo.createAndStart();
+  testDb = new smartdata.SmartdataDb(await smartmongoInstance.getMongoDescriptor());
+  await testDb.init();
+});
+
+tap.skip.test('should connect to atlas', async (tools) => {
+  const databaseName = `test-smartdata-${smartunique.shortId()}`;
+  testDb = new smartdata.SmartdataDb({
+    mongoDbUrl: await testQenv.getEnvVarOnDemand('MONGO_URL'),
+    mongoDbName: databaseName,
+  });
+  await testDb.init();
+});
+
+// =======================================
+// The actual tests
+// =======================================
+
+// ------
+// Collections
+// ------
+@smartdata.Manager()
+class Car extends smartdata.SmartDataDbDoc<Car, Car> {
+  @smartdata.unI()
+  public index: string = smartunique.shortId();
+
+  @smartdata.svDb()
+  public color: string;
+
+  @smartdata.svDb()
+  public brand: string;
+
+  @smartdata.svDb()
+  deepData = {
+    sodeep: 'yes',
+  };
+
+  constructor(colorArg: string, brandArg: string) {
+    super();
+    this.color = colorArg;
+    this.brand = brandArg;
+  }
+}
+
+const createCarClass = (dbArg: smartdata.SmartdataDb) => {
+  smartdata.setDefaultManagerForDoc({ db: dbArg }, Car);
+  return Car;
+};
+
+tap.test('should produce a car', async () => {
+  const CarClass = createCarClass(testDb);
+  const carInstance = new CarClass('red', 'Mercedes');
+  await carInstance.save();
+});
+
+tap.test('should get a car', async () => {
+  const car = Car.getInstance({
+    color: 'red',
+  });
+});
+
+// =======================================
+// close the database connection
+// =======================================
+tap.test('close', async () => {
+  await testDb.mongoDb.dropDatabase();
+  await testDb.close();
+  if (smartmongoInstance) {
+    await smartmongoInstance.stop();
+  }
+});
+
+tap.start({ throwOnError: true });

test/test.watch.ts

@@ -0,0 +1,115 @@
+import { tap, expect } from '@push.rocks/tapbundle';
+import { Qenv } from '@push.rocks/qenv';
+import * as smartmongo from '@push.rocks/smartmongo';
+import { smartunique } from '../ts/plugins.js';
+
+const testQenv = new Qenv(process.cwd(), process.cwd() + '/.nogit/');
+
+console.log(process.memoryUsage());
+
+// the tested module
+import * as smartdata from '../ts/index.js';
+
+// =======================================
+// Connecting to the database server
+// =======================================
+
+let smartmongoInstance: smartmongo.SmartMongo;
+let testDb: smartdata.SmartdataDb;
+
+const totalCars = 2000;
+
+tap.test('should create a testinstance as database', async () => {
+  smartmongoInstance = await smartmongo.SmartMongo.createAndStart();
+  testDb = new smartdata.SmartdataDb(await smartmongoInstance.getMongoDescriptor());
+  await testDb.init();
+});
+
+tap.skip.test('should connect to atlas', async (tools) => {
+  const databaseName = `test-smartdata-${smartunique.shortId()}`;
+  testDb = new smartdata.SmartdataDb({
+    mongoDbUrl: await testQenv.getEnvVarOnDemand('MONGO_URL'),
+    mongoDbName: databaseName,
+  });
+  await testDb.init();
+});
+
+@smartdata.Collection(() => testDb)
+class House extends smartdata.SmartDataDbDoc<House, House> {
+  @smartdata.unI()
+  public id: string = smartunique.shortId();
+
+  @smartdata.svDb()
+  public data = {
+    id: smartunique.shortId(),
+    hello: 'hello',
+  };
+}
+
+tap.test('should watch a collection', async (toolsArg) => {
+  const done = toolsArg.defer();
+  const watcher = await House.watch({});
+  watcher.changeSubject.subscribe(async (houseArg) => {
+    console.log('hey there, we observed a house');
+    await watcher.close();
+    done.resolve();
+  });
+  const newHouse = new House();
+  await newHouse.save();
+  console.log('saved a house');
+  await done.promise;
+});
+
+// ======= New tests for EventEmitter and buffering support =======
+tap.test('should emit change via EventEmitter', async (tools) => {
+  const done = tools.defer();
+  const watcher = await House.watch({});
+  watcher.on('change', async (houseArg) => {
+    // Expect a House instance
+    expect(houseArg).toBeDefined();
+    // Clean up
+    await watcher.stop();
+    done.resolve();
+  });
+  // Trigger an insert to generate a change event
+  const h = new House();
+  await h.save();
+  await done.promise;
+});
+
+tap.test('should buffer change events when bufferTimeMs is set', async (tools) => {
+  const done = tools.defer();
+  // bufferTimeMs collects events into arrays every 50ms
+  const watcher = await House.watch({}, { bufferTimeMs: 50 });
+  let received: House[];
+  watcher.changeSubject.subscribe(async (batch: House[]) => {
+    if (batch && batch.length > 0) {
+      received = batch;
+      await watcher.stop();
+      done.resolve();
+    }
+  });
+  // Rapidly insert multiple docs
+  const docs = [new House(), new House(), new House()];
+  for (const doc of docs) await doc.save();
+  await done.promise;
+  // All inserts should be in one buffered batch
+  expect(received.length).toEqual(docs.length);
+});
+
+// =======================================
+// close the database connection
+// =======================================
+tap.test('close', async () => {
+  try {
+    await testDb.mongoDb.dropDatabase();
+  } catch (err) {
+    console.warn('dropDatabase error ignored in cleanup:', err.message || err);
+  }
+  await testDb.close();
+  if (smartmongoInstance) {
+    await smartmongoInstance.stop();
+  }
+});
+
+tap.start({ throwOnError: true });

ts/00_commitinfo_data.ts

@@ -0,0 +1,8 @@
+/**
+ * autocreated commitinfo by @push.rocks/commitinfo
+ */
+export const commitinfo = {
+  name: '@push.rocks/smartdata',
+  version: '5.16.0',
+  description: 'An advanced library for NoSQL data organization and manipulation using TypeScript with support for MongoDB, data validation, collections, and custom data types.'
+}

ts/classes.collection.ts

@@ -0,0 +1,379 @@
+import * as plugins from './plugins.js';
+import { SmartdataDb } from './classes.db.js';
+import { SmartdataDbCursor } from './classes.cursor.js';
+import { SmartDataDbDoc, type IIndexOptions } from './classes.doc.js';
+import { SmartdataDbWatcher } from './classes.watcher.js';
+import { CollectionFactory } from './classes.collectionfactory.js';
+
+export interface IFindOptions {
+  limit?: number;
+}
+
+/**
+ *
+ */
+export interface IDocValidationFunc<T> {
+  (doc: T): boolean;
+}
+
+export type TDelayed<TDelayedArg> = () => TDelayedArg;
+
+const collectionFactory = new CollectionFactory();
+
+/**
+ * This is a decorator that will tell the decorated class what dbTable to use
+ * @param dbArg
+ */
+export function Collection(dbArg: SmartdataDb | TDelayed<SmartdataDb>) {
+  return function classDecorator<T extends { new (...args: any[]): {} }>(constructor: T) {
+    const decoratedClass = class extends constructor {
+      public static className = constructor.name;
+      public static get collection() {
+        if (!(dbArg instanceof SmartdataDb)) {
+          dbArg = dbArg();
+        }
+        const coll = collectionFactory.getCollection(constructor.name, dbArg);
+        // Attach document constructor for searchableFields lookup
+        if (!(coll as any).docCtor) {
+          (coll as any).docCtor = decoratedClass;
+        }
+        return coll;
+      }
+      public get collection() {
+        if (!(dbArg instanceof SmartdataDb)) {
+          dbArg = dbArg();
+        }
+        const coll = collectionFactory.getCollection(constructor.name, dbArg);
+        if (!(coll as any).docCtor) {
+          (coll as any).docCtor = decoratedClass;
+        }
+        return coll;
+      }
+    };
+    return decoratedClass;
+  };
+}
+
+export interface IManager {
+  db: SmartdataDb;
+}
+
+export const setDefaultManagerForDoc = <T,>(managerArg: IManager, dbDocArg: T): T => {
+  (dbDocArg as any).prototype.defaultManager = managerArg;
+  return dbDocArg;
+};
+
+/**
+ * This is a decorator that will tell the decorated class what dbTable to use
+ * @param dbArg
+ */
+export function managed<TManager extends IManager>(managerArg?: TManager | TDelayed<TManager>) {
+  return function classDecorator<T extends { new (...args: any[]): any }>(constructor: T) {
+    const decoratedClass = class extends constructor {
+      public static className = constructor.name;
+      public static get collection() {
+        let dbArg: SmartdataDb;
+        if (!managerArg) {
+          dbArg = this.prototype.defaultManager.db;
+        } else if (managerArg['db']) {
+          dbArg = (managerArg as TManager).db;
+        } else {
+          dbArg = (managerArg as TDelayed<TManager>)().db;
+        }
+        return collectionFactory.getCollection(constructor.name, dbArg);
+      }
+      public get collection() {
+        let dbArg: SmartdataDb;
+        if (!managerArg) {
+          //console.log(this.defaultManager.db);
+          //process.exit(0)
+          dbArg = this.defaultManager.db;
+        } else if (managerArg['db']) {
+          dbArg = (managerArg as TManager).db;
+        } else {
+          dbArg = (managerArg as TDelayed<TManager>)().db;
+        }
+        return collectionFactory.getCollection(constructor.name, dbArg);
+      }
+      public static get manager() {
+        let manager: TManager;
+        if (!managerArg) {
+          manager = this.prototype.defaultManager;
+        } else if (managerArg['db']) {
+          manager = managerArg as TManager;
+        } else {
+          manager = (managerArg as TDelayed<TManager>)();
+        }
+        return manager;
+      }
+      public get manager() {
+        let manager: TManager;
+        if (!managerArg) {
+          manager = this.defaultManager;
+        } else if (managerArg['db']) {
+          manager = managerArg as TManager;
+        } else {
+          manager = (managerArg as TDelayed<TManager>)();
+        }
+        return manager;
+      }
+    };
+    return decoratedClass;
+  };
+}
+
+/**
+ * @dpecrecated use @managed instead
+ */
+export const Manager = managed;
+
+export class SmartdataCollection<T> {
+  /**
+   * the collection that is used
+   */
+  public mongoDbCollection: plugins.mongodb.Collection;
+  public objectValidation: IDocValidationFunc<T> = null;
+  public collectionName: string;
+  public smartdataDb: SmartdataDb;
+  public uniqueIndexes: string[] = [];
+  public regularIndexes: Array<{field: string, options: IIndexOptions}> = [];
+  // flag to ensure text index is created only once
+  private textIndexCreated: boolean = false;
+
+  constructor(classNameArg: string, smartDataDbArg: SmartdataDb) {
+    // tell the collection where it belongs
+    this.collectionName = classNameArg;
+    this.smartdataDb = smartDataDbArg;
+
+    // tell the db class about it (important since Db uses different systems under the hood)
+    this.smartdataDb.addCollection(this);
+  }
+
+  /**
+   * makes sure a collection exists within MongoDb that maps to the SmartdataCollection
+   */
+  public async init() {
+    if (!this.mongoDbCollection) {
+      // connect this instance to a MongoDB collection
+      const availableMongoDbCollections = await this.smartdataDb.mongoDb.collections();
+      const wantedCollection = availableMongoDbCollections.find((collection) => {
+        return collection.collectionName === this.collectionName;
+      });
+      if (!wantedCollection) {
+        await this.smartdataDb.mongoDb.createCollection(this.collectionName);
+        console.log(`Successfully initiated Collection ${this.collectionName}`);
+      }
+      this.mongoDbCollection = this.smartdataDb.mongoDb.collection(this.collectionName);
+      // Auto-create a compound text index on all searchable fields
+      // Use document constructor's searchableFields registered via decorator
+      const docCtor = (this as any).docCtor;
+      const searchableFields: string[] = docCtor?.searchableFields || [];
+      if (searchableFields.length > 0 && !this.textIndexCreated) {
+        // Build a compound text index spec
+        const indexSpec: Record<string, 'text'> = {};
+        searchableFields.forEach(f => { indexSpec[f] = 'text'; });
+        // Cast to any to satisfy TypeScript IndexSpecification typing
+        await this.mongoDbCollection.createIndex(indexSpec as any, { name: 'smartdata_text_index' });
+        this.textIndexCreated = true;
+      }
+    }
+  }
+
+  /**
+   * mark unique index
+   */
+  public markUniqueIndexes(keyArrayArg: string[] = []) {
+    for (const key of keyArrayArg) {
+      if (!this.uniqueIndexes.includes(key)) {
+        this.mongoDbCollection.createIndex(key, {
+          unique: true,
+        });
+        // make sure we only call this once and not for every doc we create
+        this.uniqueIndexes.push(key);
+      }
+    }
+  }
+
+  /**
+   * creates regular indexes for the collection
+   */
+  public createRegularIndexes(indexesArg: Array<{field: string, options: IIndexOptions}> = []) {
+    for (const indexDef of indexesArg) {
+      // Check if we've already created this index
+      const indexKey = indexDef.field;
+      if (!this.regularIndexes.some(i => i.field === indexKey)) {
+        this.mongoDbCollection.createIndex(
+          { [indexDef.field]: 1 }, // Simple single-field index
+          indexDef.options
+        );
+        // Track that we've created this index
+        this.regularIndexes.push(indexDef);
+      }
+    }
+  }
+
+  /**
+   * adds a validation function that all newly inserted and updated objects have to pass
+   */
+  public addDocValidation(funcArg: IDocValidationFunc<T>) {
+    this.objectValidation = funcArg;
+  }
+
+  /**
+   * finds an object in the DbCollection
+   */
+  public async findOne(
+    filterObject: any,
+    opts?: { session?: plugins.mongodb.ClientSession }
+  ): Promise<any> {
+    await this.init();
+    // Use MongoDB driver's findOne with optional session
+    return this.mongoDbCollection.findOne(filterObject, { session: opts?.session });
+  }
+
+  public async getCursor(
+    filterObjectArg: any,
+    dbDocArg: typeof SmartDataDbDoc,
+    opts?: { session?: plugins.mongodb.ClientSession }
+  ): Promise<SmartdataDbCursor<any>> {
+    await this.init();
+    const cursor = this.mongoDbCollection.find(filterObjectArg, { session: opts?.session });
+    return new SmartdataDbCursor(cursor, dbDocArg);
+  }
+
+  /**
+   * finds an object in the DbCollection
+   */
+  public async findAll(
+    filterObject: any,
+    opts?: { session?: plugins.mongodb.ClientSession }
+  ): Promise<any[]> {
+    await this.init();
+    const cursor = this.mongoDbCollection.find(filterObject, { session: opts?.session });
+    const result = await cursor.toArray();
+    cursor.close();
+    return result;
+  }
+
+  /**
+   * Watches the collection, returning a SmartdataDbWatcher with RxJS and EventEmitter support.
+   * @param filterObject match filter for change stream
+   * @param opts optional MongoDB ChangeStreamOptions & { bufferTimeMs } to buffer events
+   * @param smartdataDbDocArg document class for instance creation
+   */
+  public async watch(
+    filterObject: any,
+    opts: (plugins.mongodb.ChangeStreamOptions & { bufferTimeMs?: number }) = {},
+    smartdataDbDocArg?: typeof SmartDataDbDoc,
+  ): Promise<SmartdataDbWatcher> {
+    await this.init();
+    // Extract bufferTimeMs from options
+    const { bufferTimeMs, fullDocument, ...otherOptions } = opts || {};
+    // Determine fullDocument behavior: default to 'updateLookup'
+    const changeStreamOptions: plugins.mongodb.ChangeStreamOptions = {
+      ...otherOptions,
+      fullDocument:
+        fullDocument === undefined
+          ? 'updateLookup'
+          : fullDocument === true
+          ? 'updateLookup'
+          : fullDocument,
+    } as any;
+    // Build pipeline with match if provided
+    const pipeline = filterObject ? [{ $match: filterObject }] : [];
+    const changeStream = this.mongoDbCollection.watch(
+      pipeline,
+      changeStreamOptions,
+    );
+    const smartdataWatcher = new SmartdataDbWatcher(
+      changeStream,
+      smartdataDbDocArg,
+      { bufferTimeMs },
+    );
+    await smartdataWatcher.readyDeferred.promise;
+    return smartdataWatcher;
+  }
+
+  /**
+   * create an object in the database
+   */
+  public async insert(
+    dbDocArg: T & SmartDataDbDoc<T, unknown>,
+    opts?: { session?: plugins.mongodb.ClientSession }
+  ): Promise<any> {
+    await this.init();
+    await this.checkDoc(dbDocArg);
+    this.markUniqueIndexes(dbDocArg.uniqueIndexes);
+    
+    // Create regular indexes if available
+    if (dbDocArg.regularIndexes && dbDocArg.regularIndexes.length > 0) {
+      this.createRegularIndexes(dbDocArg.regularIndexes);
+    }
+    
+    const saveableObject = await dbDocArg.createSavableObject();
+    const result = await this.mongoDbCollection.insertOne(saveableObject, { session: opts?.session });
+    return result;
+  }
+
+  /**
+   * inserts object into the DbCollection
+   */
+  public async update(
+    dbDocArg: T & SmartDataDbDoc<T, unknown>,
+    opts?: { session?: plugins.mongodb.ClientSession }
+  ): Promise<any> {
+    await this.init();
+    await this.checkDoc(dbDocArg);
+    const identifiableObject = await dbDocArg.createIdentifiableObject();
+    const saveableObject = await dbDocArg.createSavableObject();
+    const updateableObject: any = {};
+    for (const key of Object.keys(saveableObject)) {
+      if (identifiableObject[key]) {
+        continue;
+      }
+      updateableObject[key] = saveableObject[key];
+    }
+    const result = await this.mongoDbCollection.updateOne(
+      identifiableObject,
+      { $set: updateableObject },
+      { upsert: true, session: opts?.session },
+    );
+    return result;
+  }
+
+  public async delete(
+    dbDocArg: T & SmartDataDbDoc<T, unknown>,
+    opts?: { session?: plugins.mongodb.ClientSession }
+  ): Promise<any> {
+    await this.init();
+    await this.checkDoc(dbDocArg);
+    const identifiableObject = await dbDocArg.createIdentifiableObject();
+    await this.mongoDbCollection.deleteOne(identifiableObject, { session: opts?.session });
+  }
+
+  public async getCount(
+    filterObject: any,
+    opts?: { session?: plugins.mongodb.ClientSession }
+  ) {
+    await this.init();
+    return this.mongoDbCollection.countDocuments(filterObject, { session: opts?.session });
+  }
+
+  /**
+   * checks a Doc for constraints
+   * if this.objectValidation is not set it passes.
+   */
+  private checkDoc(docArg: T): Promise<void> {
+    const done = plugins.smartpromise.defer<void>();
+    let validationResult = true;
+    if (this.objectValidation) {
+      validationResult = this.objectValidation(docArg);
+    }
+    if (validationResult) {
+      done.resolve();
+    } else {
+      done.reject('validation of object did not pass');
+    }
+    return done.promise;
+  }
+}

ts/classes.collectionfactory.ts

@@ -0,0 +1,19 @@
+import * as plugins from './plugins.js';
+import { SmartdataCollection } from './classes.collection.js';
+import { SmartdataDb } from './classes.db.js';
+
+export class CollectionFactory {
+  public collections: { [key: string]: SmartdataCollection<any> } = {};
+
+  public getCollection = (nameArg: string, dbArg: SmartdataDb): SmartdataCollection<any> => {
+    if (!this.collections[nameArg]) {
+      this.collections[nameArg] = (() => {
+        if (dbArg instanceof SmartdataDb) {
+          // tslint:disable-next-line: no-string-literal
+          return new SmartdataCollection(nameArg, dbArg);
+        }
+      })();
+    }
+    return this.collections[nameArg];
+  };
+}

ts/classes.convenience.ts

@@ -0,0 +1,5 @@
+import * as plugins from './plugins.js';
+
+export const getNewUniqueId = async (prefixArg?: string) => {
+  return plugins.smartunique.uni(prefixArg);
+};

ts/classes.cursor.ts

@@ -0,0 +1,51 @@
+import { SmartDataDbDoc } from './classes.doc.js';
+import * as plugins from './plugins.js';
+
+/**
+ * a wrapper for the native mongodb cursor. Exposes better
+ */
+export class SmartdataDbCursor<T = any> {
+  // STATIC
+
+  // INSTANCE
+  public mongodbCursor: plugins.mongodb.FindCursor<T>;
+  private smartdataDbDoc: typeof SmartDataDbDoc;
+  constructor(cursorArg: plugins.mongodb.FindCursor<T>, dbDocArg: typeof SmartDataDbDoc) {
+    this.mongodbCursor = cursorArg;
+    this.smartdataDbDoc = dbDocArg;
+  }
+
+  public async next(closeAtEnd = true): Promise<T> {
+    const result = this.smartdataDbDoc.createInstanceFromMongoDbNativeDoc(
+      await this.mongodbCursor.next(),
+    );
+    if (!result && closeAtEnd) {
+      await this.close();
+    }
+    return result as T;
+  }
+
+  public async forEach(forEachFuncArg: (itemArg: T) => Promise<any>, closeCursorAtEnd = true) {
+    let nextDocument: any;
+    do {
+      nextDocument = await this.mongodbCursor.next();
+      if (nextDocument) {
+        const nextClassInstance =
+          this.smartdataDbDoc.createInstanceFromMongoDbNativeDoc(nextDocument);
+        await forEachFuncArg(nextClassInstance as any);
+      }
+    } while (nextDocument);
+    if (closeCursorAtEnd) {
+      await this.close();
+    }
+  }
+
+  public async toArray(): Promise<T[]> {
+    const result = await this.mongodbCursor.toArray();
+    return result.map((itemArg) => this.smartdataDbDoc.createInstanceFromMongoDbNativeDoc(itemArg)) as T[];
+  }
+
+  public async close() {
+    await this.mongodbCursor.close();
+  }
+}

ts/classes.db.ts

@@ -0,0 +1,90 @@
+import * as plugins from './plugins.js';
+
+import { SmartdataCollection } from './classes.collection.js';
+import { EasyStore } from './classes.easystore.js';
+
+import { logger } from './logging.js';
+
+/**
+ * interface - indicates the connection status of the db
+ */
+export type TConnectionStatus = 'initial' | 'disconnected' | 'connected' | 'failed';
+
+export class SmartdataDb {
+  smartdataOptions: plugins.tsclass.database.IMongoDescriptor;
+  mongoDbClient: plugins.mongodb.MongoClient;
+  mongoDb: plugins.mongodb.Db;
+  status: TConnectionStatus;
+  statusConnectedDeferred = plugins.smartpromise.defer();
+  smartdataCollectionMap = new plugins.lik.ObjectMap<SmartdataCollection<any>>();
+
+  constructor(smartdataOptions: plugins.tsclass.database.IMongoDescriptor) {
+    this.smartdataOptions = smartdataOptions;
+    this.status = 'initial';
+  }
+
+  // easystore
+  public async createEasyStore(nameIdArg: string) {
+    const easyStore = new EasyStore(nameIdArg, this);
+    return easyStore;
+  }
+
+  // basic connection stuff ----------------------------------------------
+
+  /**
+   * connects to the database that was specified during instance creation
+   */
+  public async init(): Promise<any> {
+    const finalConnectionUrl = this.smartdataOptions.mongoDbUrl
+      .replace('<USERNAME>', this.smartdataOptions.mongoDbUser)
+      .replace('<username>', this.smartdataOptions.mongoDbUser)
+      .replace('<USER>', this.smartdataOptions.mongoDbUser)
+      .replace('<user>', this.smartdataOptions.mongoDbUser)
+      .replace('<PASSWORD>', this.smartdataOptions.mongoDbPass)
+      .replace('<password>', this.smartdataOptions.mongoDbPass)
+      .replace('<DBNAME>', this.smartdataOptions.mongoDbName)
+      .replace('<dbname>', this.smartdataOptions.mongoDbName);
+
+    this.mongoDbClient = await plugins.mongodb.MongoClient.connect(finalConnectionUrl, {
+      maxPoolSize: 100,
+      maxIdleTimeMS: 10,
+    });
+    this.mongoDb = this.mongoDbClient.db(this.smartdataOptions.mongoDbName);
+    this.status = 'connected';
+    this.statusConnectedDeferred.resolve();
+    console.log(`Connected to database ${this.smartdataOptions.mongoDbName}`);
+  }
+
+  /**
+   * closes the connection to the databse
+   */
+  public async close(): Promise<any> {
+    await this.mongoDbClient.close();
+    this.status = 'disconnected';
+    logger.log('info', `disconnected from database ${this.smartdataOptions.mongoDbName}`);
+  }
+  /**
+   * Start a MongoDB client session for transactions
+   */
+  public startSession(): plugins.mongodb.ClientSession {
+    return this.mongoDbClient.startSession();
+  }
+
+  // handle table to class distribution
+
+  public addCollection(SmartdataCollectionArg: SmartdataCollection<any>) {
+    this.smartdataCollectionMap.add(SmartdataCollectionArg);
+  }
+
+  /**
+   * Gets a collection's name and returns a SmartdataCollection instance
+   * @param nameArg
+   * @returns DbTable
+   */
+  public async getSmartdataCollectionByName<T>(nameArg: string): Promise<SmartdataCollection<T>> {
+    const resultCollection = await this.smartdataCollectionMap.find(async (dbTableArg) => {
+      return dbTableArg.collectionName === nameArg;
+    });
+    return resultCollection;
+  }
+}

ts/classes.distributedcoordinator.ts

@@ -0,0 +1,302 @@
+import * as plugins from './plugins.js';
+import { SmartdataDb } from './classes.db.js';
+import { managed, setDefaultManagerForDoc } from './classes.collection.js';
+import { SmartDataDbDoc, svDb, unI } from './classes.doc.js';
+import { SmartdataDbWatcher } from './classes.watcher.js';
+
+@managed()
+export class DistributedClass extends SmartDataDbDoc<DistributedClass, DistributedClass> {
+  // INSTANCE
+  @unI()
+  public id: string;
+
+  @svDb()
+  public data: {
+    status: 'initializing' | 'bidding' | 'settled' | 'stopped';
+    biddingShortcode?: string;
+    biddingStartTime?: number;
+    lastUpdated: number;
+    elected: boolean;
+    /**
+     * used to store request
+     */
+    taskRequests: plugins.taskbuffer.distributedCoordination.IDistributedTaskRequest[];
+
+    /**
+     * only used by the leader to convey consultation results
+     */
+    taskRequestResults: plugins.taskbuffer.distributedCoordination.IDistributedTaskRequestResult[];
+  };
+}
+
+/**
+ * This file implements a distributed coordinator according to the @pushrocks/taskbuffer standard.
+ * you should not set up this yourself. Instead, there is a factory on the SmartdataDb class
+ * that will take care of setting this up.
+ */
+export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distributedCoordination
+  .AbstractDistributedCoordinator {
+  public readyPromise: Promise<any>;
+  public db: SmartdataDb;
+  private asyncExecutionStack = new plugins.lik.AsyncExecutionStack();
+  public ownInstance: DistributedClass;
+  public distributedWatcher: SmartdataDbWatcher<DistributedClass>;
+
+  constructor(dbArg: SmartdataDb) {
+    super();
+    this.db = dbArg;
+    setDefaultManagerForDoc(this, DistributedClass);
+    this.readyPromise = this.db.statusConnectedDeferred.promise;
+  }
+
+  // smartdata specific stuff
+  public async start() {
+    await this.init();
+  }
+
+  public async stop() {
+    await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+      if (this.distributedWatcher) {
+        await this.distributedWatcher.close();
+      }
+      if (this.ownInstance?.data.elected) {
+        this.ownInstance.data.elected = false;
+      }
+      if (this.ownInstance?.data.status === 'stopped') {
+        console.log(`stopping a distributed instance that has not been started yet.`);
+      }
+      this.ownInstance.data.status = 'stopped';
+      await this.ownInstance.save();
+      console.log(`stopped ${this.ownInstance.id}`);
+    });
+  }
+
+  public id = plugins.smartunique.uni('distributedInstance');
+
+  private startHeartbeat = async () => {
+    while (this.ownInstance.data.status !== 'stopped') {
+      await this.sendHeartbeat();
+      await plugins.smartdelay.delayForRandom(5000, 10000);
+    }
+  };
+
+  public async sendHeartbeat() {
+    await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+      if (this.ownInstance.data.status === 'stopped') {
+        console.log(`aborted sending heartbeat because status is stopped`);
+        return;
+      }
+      await this.ownInstance.updateFromDb();
+      this.ownInstance.data.lastUpdated = Date.now();
+      await this.ownInstance.save();
+      console.log(`sent heartbeat for ${this.ownInstance.id}`);
+      const allInstances = DistributedClass.getInstances({});
+    });
+    if (this.ownInstance.data.status === 'stopped') {
+      console.log(`aborted sending heartbeat because status is stopped`);
+      return;
+    }
+    const eligibleLeader = await this.getEligibleLeader();
+    // not awaiting here because we don't want to block the heartbeat
+    this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+      if (!eligibleLeader && this.ownInstance.data.status === 'settled') {
+        this.checkAndMaybeLead();
+      }
+    });
+  }
+  private async init() {
+    await this.readyPromise;
+    if (!this.ownInstance) {
+      await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+        this.ownInstance = new DistributedClass();
+        this.ownInstance.id = this.id;
+        this.ownInstance.data = {
+          elected: false,
+          lastUpdated: Date.now(),
+          status: 'initializing',
+          taskRequests: [],
+          taskRequestResults: [],
+        };
+        await this.ownInstance.save();
+      });
+    } else {
+      console.warn(`distributed instance already initialized`);
+    }
+
+    // lets enable the heartbeat
+    this.startHeartbeat();
+
+    // lets do a leader check
+    await this.checkAndMaybeLead();
+
+    return this.ownInstance;
+  }
+
+  public async getEligibleLeader() {
+    return this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+      const allInstances = await DistributedClass.getInstances({});
+      let leaders = allInstances.filter((instanceArg) => instanceArg.data.elected === true);
+      const eligibleLeader = leaders.find(
+        (leader) =>
+          leader.data.lastUpdated >=
+          Date.now() - plugins.smarttime.getMilliSecondsFromUnits({ seconds: 20 }),
+      );
+      return eligibleLeader;
+    });
+  }
+
+  // --> leader election
+  public async checkAndMaybeLead() {
+    await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+      this.ownInstance.data.status = 'initializing';
+      this.ownInstance.save();
+    });
+    if (await this.getEligibleLeader()) {
+      await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+        await this.ownInstance.updateFromDb();
+        this.ownInstance.data.status = 'settled';
+        await this.ownInstance.save();
+        console.log(`${this.ownInstance.id} settled as follower`);
+      });
+      return;
+    } else if (
+      (await DistributedClass.getInstances({})).find((instanceArg) => {
+        instanceArg.data.status === 'bidding' &&
+          instanceArg.data.biddingStartTime <= Date.now() - 4000 &&
+          instanceArg.data.biddingStartTime >= Date.now() - 30000;
+      })
+    ) {
+      console.log('too late to the bidding party... waiting for next round.');
+      return;
+    } else {
+      await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+        await this.ownInstance.updateFromDb();
+        this.ownInstance.data.status = 'bidding';
+        this.ownInstance.data.biddingStartTime = Date.now();
+        this.ownInstance.data.biddingShortcode = plugins.smartunique.shortId();
+        await this.ownInstance.save();
+        console.log('bidding code stored.');
+      });
+      console.log(`bidding for leadership...`);
+      await plugins.smartdelay.delayFor(plugins.smarttime.getMilliSecondsFromUnits({ seconds: 5 }));
+      await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+        let biddingInstances = await DistributedClass.getInstances({});
+        biddingInstances = biddingInstances.filter(
+          (instanceArg) =>
+            instanceArg.data.status === 'bidding' &&
+            instanceArg.data.lastUpdated >=
+              Date.now() - plugins.smarttime.getMilliSecondsFromUnits({ seconds: 10 }),
+        );
+        console.log(`found ${biddingInstances.length} bidding instances...`);
+        this.ownInstance.data.elected = true;
+        for (const biddingInstance of biddingInstances) {
+          if (biddingInstance.data.biddingShortcode < this.ownInstance.data.biddingShortcode) {
+            this.ownInstance.data.elected = false;
+          }
+        }
+        await plugins.smartdelay.delayFor(5000);
+        console.log(`settling with status elected = ${this.ownInstance.data.elected}`);
+        this.ownInstance.data.status = 'settled';
+        await this.ownInstance.save();
+      });
+      if (this.ownInstance.data.elected) {
+        this.leadFunction();
+      }
+    }
+  }
+
+  /**
+   * when it has been determined
+   * that this instance is leading
+   * the leading is implemented here
+   */
+  public async leadFunction() {
+    this.distributedWatcher = await DistributedClass.watch({});
+
+    const currentTaskRequests: Array<{
+      taskName: string;
+      taskExecutionTime: number;
+      /**
+       * all instances that requested this task
+       */
+      requestingDistibutedInstanceIds: string[];
+      responseTimeout: plugins.smartdelay.Timeout<any>;
+    }> = [];
+
+    this.distributedWatcher.changeSubject.subscribe({
+      next: async (distributedDoc) => {
+        if (!distributedDoc) {
+          console.log(`registered deletion of instance...`);
+          return;
+        }
+        console.log(distributedDoc);
+        console.log(`registered change for ${distributedDoc.id}`);
+        distributedDoc;
+      },
+    });
+
+    while (this.ownInstance.data.status !== 'stopped' && this.ownInstance.data.elected) {
+      const allInstances = await DistributedClass.getInstances({});
+      for (const instance of allInstances) {
+        if (instance.data.status === 'stopped') {
+          await instance.delete();
+        }
+      }
+      await plugins.smartdelay.delayFor(10000);
+    }
+  }
+
+  // abstract implemented methods
+  public async fireDistributedTaskRequest(
+    taskRequestArg: plugins.taskbuffer.distributedCoordination.IDistributedTaskRequest,
+  ): Promise<plugins.taskbuffer.distributedCoordination.IDistributedTaskRequestResult> {
+    await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+      if (!this.ownInstance) {
+        console.error('instance need to be started first...');
+        return;
+      }
+      await this.ownInstance.updateFromDb();
+      this.ownInstance.data.taskRequests.push(taskRequestArg);
+      await this.ownInstance.save();
+    });
+    await plugins.smartdelay.delayFor(10000);
+    const result = await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+      await this.ownInstance.updateFromDb();
+      const taskRequestResult = this.ownInstance.data.taskRequestResults.find((resultItem) => {
+        return resultItem.requestResponseId === taskRequestArg.requestResponseId;
+      });
+      return taskRequestResult;
+    });
+    if (!result) {
+      console.warn('no result found for task request...');
+      return null;
+    }
+    return result;
+  }
+
+  public async updateDistributedTaskRequest(
+    infoBasisArg: plugins.taskbuffer.distributedCoordination.IDistributedTaskRequest,
+  ): Promise<void> {
+    await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+      const existingInfoBasis = this.ownInstance.data.taskRequests.find((infoBasisItem) => {
+        return (
+          infoBasisItem.taskName === infoBasisArg.taskName &&
+          infoBasisItem.taskExecutionTime === infoBasisArg.taskExecutionTime
+        );
+      });
+      if (!existingInfoBasis) {
+        console.warn('trying to update a non existing task request... aborting!');
+        return;
+      }
+      Object.assign(existingInfoBasis, infoBasisArg);
+      await this.ownInstance.save();
+      plugins.smartdelay.delayFor(60000).then(() => {
+        this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+          const indexToRemove = this.ownInstance.data.taskRequests.indexOf(existingInfoBasis);
+          this.ownInstance.data.taskRequests.splice(indexToRemove, indexToRemove);
+          await this.ownInstance.save();
+        });
+      });
+    });
+  }
+}

ts/classes.doc.ts

@@ -0,0 +1,704 @@
+import * as plugins from './plugins.js';
+
+import { SmartdataDb } from './classes.db.js';
+import { SmartdataDbCursor } from './classes.cursor.js';
+import { type IManager, SmartdataCollection } from './classes.collection.js';
+import { SmartdataDbWatcher } from './classes.watcher.js';
+import { SmartdataLuceneAdapter } from './classes.lucene.adapter.js';
+/**
+ * Search options for `.search()`:
+ * - filter: additional MongoDB query to AND-merge
+ * - validate: post-fetch validator, return true to keep a doc
+ */
+export interface SearchOptions<T> {
+  /**
+   * Additional MongoDB filter to AND‐merge into the query
+   */
+  filter?: Record<string, any>;
+  /**
+   * Post‐fetch validator; return true to keep each doc
+   */
+  validate?: (doc: T) => Promise<boolean> | boolean;
+  /**
+   * Optional MongoDB session for transactional operations
+   */
+  session?: plugins.mongodb.ClientSession;
+}
+
+export type TDocCreation = 'db' | 'new' | 'mixed';
+
+
+
+export function globalSvDb() {
+  return (target: SmartDataDbDoc<unknown, unknown>, key: string) => {
+    console.log(`called svDb() on >${target.constructor.name}.${key}<`);
+    if (!target.globalSaveableProperties) {
+      target.globalSaveableProperties = [];
+    }
+    target.globalSaveableProperties.push(key);
+  };
+}
+
+/**
+ * Options for custom serialization/deserialization of a field.
+ */
+export interface SvDbOptions {
+  /** Function to serialize the field value before saving to DB */
+  serialize?: (value: any) => any;
+  /** Function to deserialize the field value after reading from DB */
+  deserialize?: (value: any) => any;
+}
+
+/**
+ * saveable - saveable decorator to be used on class properties
+ */
+export function svDb(options?: SvDbOptions) {
+  return (target: SmartDataDbDoc<unknown, unknown>, key: string) => {
+    console.log(`called svDb() on >${target.constructor.name}.${key}<`);
+    if (!target.saveableProperties) {
+      target.saveableProperties = [];
+    }
+    target.saveableProperties.push(key);
+    // attach custom serializer/deserializer options to the class constructor
+    const ctor = target.constructor as any;
+    if (!ctor._svDbOptions) {
+      ctor._svDbOptions = {};
+    }
+    if (options) {
+      ctor._svDbOptions[key] = options;
+    }
+  };
+}
+
+/**
+ * searchable - marks a property as searchable with Lucene query syntax
+ */
+export function searchable() {
+  return (target: SmartDataDbDoc<unknown, unknown>, key: string) => {
+    // Attach to class constructor for direct access
+    const ctor = target.constructor as any;
+    if (!Array.isArray(ctor.searchableFields)) {
+      ctor.searchableFields = [];
+    }
+    ctor.searchableFields.push(key);
+  };
+}
+
+// Escape user input for safe use in MongoDB regular expressions
+function escapeForRegex(input: string): string {
+  return input.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+/**
+ * unique index - decorator to mark a unique index
+ */
+export function unI() {
+  return (target: SmartDataDbDoc<unknown, unknown>, key: string) => {
+    console.log(`called unI on >>${target.constructor.name}.${key}<<`);
+
+    // mark the index as unique
+    if (!target.uniqueIndexes) {
+      target.uniqueIndexes = [];
+    }
+    target.uniqueIndexes.push(key);
+
+    // and also save it
+    if (!target.saveableProperties) {
+      target.saveableProperties = [];
+    }
+    target.saveableProperties.push(key);
+  };
+}
+
+/**
+ * Options for MongoDB indexes
+ */
+export interface IIndexOptions {
+  background?: boolean;
+  unique?: boolean;
+  sparse?: boolean;
+  expireAfterSeconds?: number;
+  [key: string]: any;
+}
+
+/**
+ * index - decorator to mark a field for regular indexing
+ */
+export function index(options?: IIndexOptions) {
+  return (target: SmartDataDbDoc<unknown, unknown>, key: string) => {
+    console.log(`called index() on >${target.constructor.name}.${key}<`);
+    
+    // Initialize regular indexes array if it doesn't exist
+    if (!target.regularIndexes) {
+      target.regularIndexes = [];
+    }
+    
+    // Add this field to regularIndexes with its options
+    target.regularIndexes.push({
+      field: key,
+      options: options || {}
+    });
+    
+    // Also ensure it's marked as saveable
+    if (!target.saveableProperties) {
+      target.saveableProperties = [];
+    }
+    
+    if (!target.saveableProperties.includes(key)) {
+      target.saveableProperties.push(key);
+    }
+  };
+}
+
+export const convertFilterForMongoDb = (filterArg: { [key: string]: any }) => {
+  // Special case: detect MongoDB operators and pass them through directly
+  const topLevelOperators = ['$and', '$or', '$nor', '$not', '$text', '$where', '$regex'];
+  for (const key of Object.keys(filterArg)) {
+    if (topLevelOperators.includes(key)) {
+      return filterArg; // Return the filter as-is for MongoDB operators
+    }
+  }
+
+  // Original conversion logic for non-MongoDB query objects
+  const convertedFilter: { [key: string]: any } = {};
+
+  const convertFilterArgument = (keyPathArg2: string, filterArg2: any) => {
+    if (Array.isArray(filterArg2)) {
+      // Directly assign arrays (they might be using operators like $in or $all)
+      convertFilterArgument(keyPathArg2, filterArg2[0]);
+    } else if (typeof filterArg2 === 'object' && filterArg2 !== null) {
+      for (const key of Object.keys(filterArg2)) {
+        if (key.startsWith('$')) {
+          convertedFilter[keyPathArg2] = filterArg2;
+          return;
+        } else if (key.includes('.')) {
+          throw new Error('keys cannot contain dots');
+        }
+      }
+      for (const key of Object.keys(filterArg2)) {
+        convertFilterArgument(`${keyPathArg2}.${key}`, filterArg2[key]);
+      }
+    } else {
+      convertedFilter[keyPathArg2] = filterArg2;
+    }
+  };
+
+  for (const key of Object.keys(filterArg)) {
+    convertFilterArgument(key, filterArg[key]);
+  }
+  return convertedFilter;
+};
+
+export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends IManager = any> {
+  /**
+   * the collection object an Doc belongs to
+   */
+  public static collection: SmartdataCollection<any>;
+  public collection: SmartdataCollection<any>;
+  public static defaultManager;
+  public static manager;
+  public manager: TManager;
+
+  // STATIC
+  public static createInstanceFromMongoDbNativeDoc<T>(
+    this: plugins.tsclass.typeFest.Class<T>,
+    mongoDbNativeDocArg: any,
+  ): T {
+    const newInstance = new this();
+    (newInstance as any).creationStatus = 'db';
+    for (const key of Object.keys(mongoDbNativeDocArg)) {
+      const rawValue = mongoDbNativeDocArg[key];
+      const optionsMap = (this as any)._svDbOptions || {};
+      const opts = optionsMap[key];
+      newInstance[key] = opts && typeof opts.deserialize === 'function'
+        ? opts.deserialize(rawValue)
+        : rawValue;
+    }
+    return newInstance;
+  }
+
+  /**
+   * gets all instances as array
+   * @param this
+   * @param filterArg
+   * @returns
+   */
+  public static async getInstances<T>(
+    this: plugins.tsclass.typeFest.Class<T>,
+    filterArg: plugins.tsclass.typeFest.PartialDeep<T>,
+    opts?: { session?: plugins.mongodb.ClientSession }
+  ): Promise<T[]> {
+    // Pass session through to findAll for transactional queries
+    const foundDocs = await (this as any).collection.findAll(
+      convertFilterForMongoDb(filterArg),
+      { session: opts?.session },
+    );
+    const returnArray = [];
+    for (const foundDoc of foundDocs) {
+      const newInstance: T = (this as any).createInstanceFromMongoDbNativeDoc(foundDoc);
+      returnArray.push(newInstance);
+    }
+    return returnArray;
+  }
+
+  /**
+   * gets the first matching instance
+   * @param this
+   * @param filterArg
+   * @returns
+   */
+  public static async getInstance<T>(
+    this: plugins.tsclass.typeFest.Class<T>,
+    filterArg: plugins.tsclass.typeFest.PartialDeep<T>,
+    opts?: { session?: plugins.mongodb.ClientSession }
+  ): Promise<T> {
+    // Retrieve one document, with optional session for transactions
+    const foundDoc = await (this as any).collection.findOne(
+      convertFilterForMongoDb(filterArg),
+      { session: opts?.session },
+    );
+    if (foundDoc) {
+      const newInstance: T = (this as any).createInstanceFromMongoDbNativeDoc(foundDoc);
+      return newInstance;
+    } else {
+      return null;
+    }
+  }
+
+  /**
+   * get a unique id prefixed with the class name
+   */
+  public static async getNewId<T = any>(
+    this: plugins.tsclass.typeFest.Class<T>,
+    lengthArg: number = 20,
+  ) {
+    return `${(this as any).className}:${plugins.smartunique.shortId(lengthArg)}`;
+  }
+
+  /**
+   * Get a cursor for streaming results, with optional session and native cursor modifiers.
+   * @param filterArg Partial filter to apply
+   * @param opts Optional session and modifier for the raw MongoDB cursor
+   */
+  public static async getCursor<T>(
+    this: plugins.tsclass.typeFest.Class<T>,
+    filterArg: plugins.tsclass.typeFest.PartialDeep<T>,
+    opts?: {
+      session?: plugins.mongodb.ClientSession;
+      modifier?: (cursorArg: plugins.mongodb.FindCursor<plugins.mongodb.WithId<plugins.mongodb.BSON.Document>>) => plugins.mongodb.FindCursor<plugins.mongodb.WithId<plugins.mongodb.BSON.Document>>;
+    }
+  ): Promise<SmartdataDbCursor<T>> {
+    const collection: SmartdataCollection<T> = (this as any).collection;
+    const { session, modifier } = opts || {};
+    await collection.init();
+    let rawCursor: plugins.mongodb.FindCursor<any> =
+      collection.mongoDbCollection.find(convertFilterForMongoDb(filterArg), { session });
+    if (modifier) {
+      rawCursor = modifier(rawCursor);
+    }
+    return new SmartdataDbCursor<T>(rawCursor, this as any as typeof SmartDataDbDoc);
+  }
+
+  /**
+   * watch the collection
+   * @param this
+   * @param filterArg
+   * @param forEachFunction
+   */
+  /**
+   * Watch the collection for changes, with optional buffering and change stream options.
+   * @param filterArg MongoDB filter to select which changes to observe
+   * @param opts optional ChangeStreamOptions plus bufferTimeMs
+   */
+  public static async watch<T>(
+    this: plugins.tsclass.typeFest.Class<T>,
+    filterArg: plugins.tsclass.typeFest.PartialDeep<T>,
+    opts?: plugins.mongodb.ChangeStreamOptions & { bufferTimeMs?: number },
+  ): Promise<SmartdataDbWatcher<T>> {
+    const collection: SmartdataCollection<T> = (this as any).collection;
+    const watcher: SmartdataDbWatcher<T> = await collection.watch(
+      convertFilterForMongoDb(filterArg),
+      opts || {},
+      this as any,
+    );
+    return watcher;
+  }
+
+  /**
+   * run a function for all instances
+   * @returns
+   */
+  public static async forEach<T>(
+    this: plugins.tsclass.typeFest.Class<T>,
+    filterArg: plugins.tsclass.typeFest.PartialDeep<T>,
+    forEachFunction: (itemArg: T) => Promise<any>,
+  ) {
+    const cursor: SmartdataDbCursor<T> = await (this as any).getCursor(filterArg);
+    await cursor.forEach(forEachFunction);
+  }
+
+  /**
+   * returns a count of the documents in the collection
+   */
+  public static async getCount<T>(
+    this: plugins.tsclass.typeFest.Class<T>,
+    filterArg: plugins.tsclass.typeFest.PartialDeep<T> = {} as any,
+  ) {
+    const collection: SmartdataCollection<T> = (this as any).collection;
+    return await collection.getCount(filterArg);
+  }
+
+  /**
+   * Create a MongoDB filter from a Lucene query string
+   * @param luceneQuery Lucene query string
+   * @returns MongoDB query object
+   */
+  public static createSearchFilter<T>(
+    this: plugins.tsclass.typeFest.Class<T>,
+    luceneQuery: string,
+  ): any {
+    const searchableFields = (this as any).getSearchableFields();
+    if (searchableFields.length === 0) {
+      throw new Error(`No searchable fields defined for class ${this.name}`);
+    }
+    const adapter = new SmartdataLuceneAdapter(searchableFields);
+    return adapter.convert(luceneQuery);
+  }
+  /**
+   * List all searchable fields defined on this class
+   */
+  public static getSearchableFields(): string[] {
+    const ctor = this as any;
+    return Array.isArray(ctor.searchableFields) ? ctor.searchableFields : [];
+  }
+  /**
+   * Execute a query with optional hard filter and post-fetch validation
+   */
+  private static async execQuery<T>(
+    this: plugins.tsclass.typeFest.Class<T>,
+    baseFilter: Record<string, any>,
+    opts?: SearchOptions<T>
+  ): Promise<T[]> {
+    let mongoFilter = baseFilter || {};
+    if (opts?.filter) {
+      mongoFilter = { $and: [mongoFilter, opts.filter] };
+    }
+    // Fetch with optional session for transactions
+    // Fetch within optional session
+    let docs: T[] = await (this as any).getInstances(mongoFilter, { session: opts?.session });
+    if (opts?.validate) {
+      const out: T[] = [];
+      for (const d of docs) {
+        if (await opts.validate(d)) out.push(d);
+      }
+      docs = out;
+    }
+    return docs;
+  }
+
+  /**
+   * Search documents by text or field:value syntax, with safe regex fallback
+   * Supports additional filtering and post-fetch validation via opts
+   * @param query A search term or field:value expression
+   * @param opts Optional filter and validate hooks
+   * @returns Array of matching documents
+   */
+  public static async search<T>(
+    this: plugins.tsclass.typeFest.Class<T>,
+    query: string,
+    opts?: SearchOptions<T>,
+  ): Promise<T[]> {
+    const searchableFields = (this as any).getSearchableFields();
+    if (searchableFields.length === 0) {
+      throw new Error(`No searchable fields defined for class ${this.name}`);
+    }
+    // empty query -> return all
+    const q = query.trim();
+    if (!q) {
+      // empty query: fetch all, apply opts
+      return await (this as any).execQuery({}, opts);
+    }
+    // simple exact field:value (no spaces, no wildcards, no quotes)
+    // simple exact field:value (no spaces, wildcards, quotes)
+    const simpleExact = q.match(/^(\w+):([^"'\*\?\s]+)$/);
+    if (simpleExact) {
+      const field = simpleExact[1];
+      const value = simpleExact[2];
+      if (!searchableFields.includes(field)) {
+        throw new Error(`Field '${field}' is not searchable for class ${this.name}`);
+      }
+      // simple field:value search
+      return await (this as any).execQuery({ [field]: value }, opts);
+    }
+    // quoted phrase across all searchable fields: exact match of phrase
+    const quoted = q.match(/^"(.+)"$|^'(.+)'$/);
+    if (quoted) {
+      const phrase = quoted[1] || quoted[2] || '';
+      const parts = phrase.split(/\s+/).map((t) => escapeForRegex(t));
+      const pattern = parts.join('\\s+');
+      const orConds = searchableFields.map((f) => ({ [f]: { $regex: pattern, $options: 'i' } }));
+      return await (this as any).execQuery({ $or: orConds }, opts);
+    }
+    // wildcard field:value (supports * and ?) -> direct regex on that field
+    const wildcardField = q.match(/^(\w+):(.+[*?].*)$/);
+    if (wildcardField) {
+      const field = wildcardField[1];
+      // Support quoted wildcard patterns: strip surrounding quotes
+      let pattern = wildcardField[2];
+      if ((pattern.startsWith('"') && pattern.endsWith('"')) ||
+          (pattern.startsWith("'") && pattern.endsWith("'"))) {
+        pattern = pattern.slice(1, -1);
+      }
+      if (!searchableFields.includes(field)) {
+        throw new Error(`Field '${field}' is not searchable for class ${this.name}`);
+      }
+      // escape regex special chars except * and ?, then convert wildcards
+      const escaped = pattern.replace(/([.+^${}()|[\\]\\])/g, '\\$1');
+      const regexPattern = escaped.replace(/\*/g, '.*').replace(/\?/g, '.');
+      return await (this as any).execQuery({ [field]: { $regex: regexPattern, $options: 'i' } }, opts);
+    }
+    // wildcard plain term across all fields (supports * and ?)
+    if (!q.includes(':') && (q.includes('*') || q.includes('?'))) {
+      // build wildcard regex pattern: escape all except * and ? then convert
+      const escaped = q.replace(/([.+^${}()|[\\]\\])/g, '\\$1');
+      const pattern = escaped.replace(/\*/g, '.*').replace(/\?/g, '.');
+      const orConds = searchableFields.map((f) => ({ [f]: { $regex: pattern, $options: 'i' } }));
+      return await (this as any).execQuery({ $or: orConds }, opts);
+    }
+    // implicit AND for multiple tokens: free terms, quoted phrases, and field:values
+    {
+      // Split query into tokens, preserving quoted substrings
+      const rawTokens = q.match(/(?:[^\s"']+|"[^"]*"|'[^']*')+/g) || [];
+      // Only apply when more than one token and no boolean operators or grouping
+      if (
+        rawTokens.length > 1 &&
+        !/(\bAND\b|\bOR\b|\bNOT\b|\(|\))/i.test(q) &&
+        !/\[|\]/.test(q)
+      ) {
+        const andConds: any[] = [];
+        for (let token of rawTokens) {
+          // field:value token
+          const fv = token.match(/^(\w+):(.+)$/);
+          if (fv) {
+            const field = fv[1];
+            let value = fv[2];
+            if (!searchableFields.includes(field)) {
+              throw new Error(`Field '${field}' is not searchable for class ${this.name}`);
+            }
+            // Strip surrounding quotes if present
+            if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) {
+              value = value.slice(1, -1);
+            }
+            // Wildcard search?
+            if (value.includes('*') || value.includes('?')) {
+              const escaped = value.replace(/([.+^${}()|[\\]\\])/g, '\\$1');
+              const pattern = escaped.replace(/\*/g, '.*').replace(/\?/g, '.');
+              andConds.push({ [field]: { $regex: pattern, $options: 'i' } });
+            } else {
+              andConds.push({ [field]: value });
+            }
+          } else if ((token.startsWith('"') && token.endsWith('"')) || (token.startsWith("'") && token.endsWith("'"))) {
+            // Quoted free phrase across all fields
+            const phrase = token.slice(1, -1);
+            const parts = phrase.split(/\s+/).map((t) => escapeForRegex(t));
+            const pattern = parts.join('\\s+');
+            andConds.push({ $or: searchableFields.map((f) => ({ [f]: { $regex: pattern, $options: 'i' } })) });
+          } else {
+            // Free term across all fields
+            const esc = escapeForRegex(token);
+            andConds.push({ $or: searchableFields.map((f) => ({ [f]: { $regex: esc, $options: 'i' } })) });
+          }
+        }
+        return await (this as any).execQuery({ $and: andConds }, opts);
+      }
+    }
+    // detect advanced Lucene syntax: field:value, wildcards, boolean, grouping
+    const luceneSyntax = /(\w+:[^\s]+)|\*|\?|\bAND\b|\bOR\b|\bNOT\b|\(|\)/;
+    if (luceneSyntax.test(q)) {
+      const filter = (this as any).createSearchFilter(q);
+      return await (this as any).execQuery(filter, opts);
+    }
+    // multi-term unquoted -> AND of regex across fields for each term
+    const terms = q.split(/\s+/);
+    if (terms.length > 1) {
+      const andConds = terms.map((term) => {
+        const esc = escapeForRegex(term);
+        const ors = searchableFields.map((f) => ({ [f]: { $regex: esc, $options: 'i' } }));
+        return { $or: ors };
+      });
+      return await (this as any).execQuery({ $and: andConds }, opts);
+    }
+    // single term -> regex across all searchable fields
+    const esc = escapeForRegex(q);
+    const orConds = searchableFields.map((f) => ({ [f]: { $regex: esc, $options: 'i' } }));
+    return await (this as any).execQuery({ $or: orConds }, opts);
+  }
+
+
+  // INSTANCE
+
+  // INSTANCE
+
+  /**
+   * how the Doc in memory was created, may prove useful later.
+   */
+  public creationStatus: TDocCreation = 'new';
+
+  /**
+   * updated from db in any case where doc comes from db
+   */
+  @globalSvDb()
+  _createdAt: string = new Date().toISOString();
+
+  /**
+   * will be updated everytime the doc is saved
+   */
+  @globalSvDb()
+  _updatedAt: string = new Date().toISOString();
+
+  /**
+   * an array of saveable properties of ALL doc
+   */
+  public globalSaveableProperties: string[];
+
+  /**
+   * unique indexes
+   */
+  public uniqueIndexes: string[];
+
+  /**
+   * regular indexes with their options
+   */
+  public regularIndexes: Array<{field: string, options: IIndexOptions}> = [];
+
+  /**
+   * an array of saveable properties of a specific doc
+   */
+  public saveableProperties: string[];
+
+  /**
+   * name
+   */
+  public name: string;
+
+  /**
+   * primary id in the database
+   */
+  public dbDocUniqueId: string;
+
+  /**
+   * class constructor
+   */
+  constructor() {}
+
+  /**
+   * saves this instance (optionally within a transaction)
+   */
+  public async save(opts?: { session?: plugins.mongodb.ClientSession }) {
+    // allow hook before saving
+    if (typeof (this as any).beforeSave === 'function') {
+      await (this as any).beforeSave();
+    }
+    // tslint:disable-next-line: no-this-assignment
+    const self: any = this;
+    let dbResult: any;
+    // update timestamp
+    this._updatedAt = new Date().toISOString();
+    // perform insert or update
+    switch (this.creationStatus) {
+      case 'db':
+        dbResult = await this.collection.update(self, { session: opts?.session });
+        break;
+      case 'new':
+        dbResult = await this.collection.insert(self, { session: opts?.session });
+        this.creationStatus = 'db';
+        break;
+      default:
+        console.error('neither new nor in db?');
+    }
+    // allow hook after saving
+    if (typeof (this as any).afterSave === 'function') {
+      await (this as any).afterSave();
+    }
+    return dbResult;
+  }
+
+  /**
+   * deletes a document from the database (optionally within a transaction)
+   */
+  public async delete(opts?: { session?: plugins.mongodb.ClientSession }) {
+    // allow hook before deleting
+    if (typeof (this as any).beforeDelete === 'function') {
+      await (this as any).beforeDelete();
+    }
+    // perform deletion
+    const result = await this.collection.delete(this, { session: opts?.session });
+    // allow hook after delete
+    if (typeof (this as any).afterDelete === 'function') {
+      await (this as any).afterDelete();
+    }
+    return result;
+  }
+
+  /**
+   * also store any referenced objects to DB
+   * better for data consistency
+   */
+  public saveDeep(savedMapArg: plugins.lik.ObjectMap<SmartDataDbDoc<any, any>> = null) {
+    if (!savedMapArg) {
+      savedMapArg = new plugins.lik.ObjectMap<SmartDataDbDoc<any, any>>();
+    }
+    savedMapArg.add(this);
+    this.save();
+    for (const propertyKey of Object.keys(this)) {
+      const property: any = this[propertyKey];
+      if (property instanceof SmartDataDbDoc && !savedMapArg.checkForObject(property)) {
+        property.saveDeep(savedMapArg);
+      }
+    }
+  }
+
+  /**
+   * updates an object from db
+   */
+  public async updateFromDb() {
+    const mongoDbNativeDoc = await this.collection.findOne(await this.createIdentifiableObject());
+    for (const key of Object.keys(mongoDbNativeDoc)) {
+      const rawValue = mongoDbNativeDoc[key];
+      const optionsMap = (this.constructor as any)._svDbOptions || {};
+      const opts = optionsMap[key];
+      this[key] = opts && typeof opts.deserialize === 'function'
+        ? opts.deserialize(rawValue)
+        : rawValue;
+    }
+  }
+
+  /**
+   * creates a saveable object so the instance can be persisted as json in the database
+   */
+  public async createSavableObject(): Promise<TImplements> {
+    const saveableObject: unknown = {}; // is not exposed to outside, so any is ok here
+    const saveableProperties = [...this.globalSaveableProperties, ...this.saveableProperties];
+    // apply custom serialization if configured
+    const optionsMap = (this.constructor as any)._svDbOptions || {};
+    for (const propertyNameString of saveableProperties) {
+      const rawValue = (this as any)[propertyNameString];
+      const opts = optionsMap[propertyNameString];
+      (saveableObject as any)[propertyNameString] = opts && typeof opts.serialize === 'function'
+        ? opts.serialize(rawValue)
+        : rawValue;
+    }
+    return saveableObject as TImplements;
+  }
+
+  /**
+   * creates an identifiable object for operations that require filtering
+   */
+  public async createIdentifiableObject() {
+    const identifiableObject: any = {}; // is not exposed to outside, so any is ok here
+    for (const propertyNameString of this.uniqueIndexes) {
+      identifiableObject[propertyNameString] = this[propertyNameString];
+    }
+    return identifiableObject;
+  }
+}

ts/classes.easystore.ts

@@ -0,0 +1,119 @@
+import * as plugins from './plugins.js';
+import { Collection } from './classes.collection.js';
+import { SmartdataDb } from './classes.db.js';
+import { SmartDataDbDoc, svDb, unI } from './classes.doc.js';
+
+/**
+ * EasyStore allows the storage of easy objects. It also allows easy sharing of the object between different instances
+ */
+export class EasyStore<T> {
+  // instance
+  public smartdataDbRef: SmartdataDb;
+  public nameId: string;
+
+  private easyStoreClass = (() => {
+    @Collection(() => this.smartdataDbRef)
+    class SmartdataEasyStore extends SmartDataDbDoc<SmartdataEasyStore, SmartdataEasyStore> {
+      @unI()
+      public nameId: string;
+
+      @svDb()
+      public ephermal: {
+        activated: boolean;
+        timeout: number;
+      };
+
+      @svDb()
+      lastEdit: number;
+
+      @svDb()
+      public data: Partial<T>;
+    }
+    return SmartdataEasyStore;
+  })();
+
+  constructor(nameIdArg: string, smnartdataDbRefArg: SmartdataDb) {
+    this.smartdataDbRef = smnartdataDbRefArg;
+    this.nameId = nameIdArg;
+  }
+
+  private easyStorePromise: Promise<InstanceType<typeof this.easyStoreClass>>;
+  private async getEasyStore(): Promise<InstanceType<typeof this.easyStoreClass>> {
+    if (this.easyStorePromise) {
+      return this.easyStorePromise;
+    }
+
+    // first run from here
+    const deferred = plugins.smartpromise.defer<InstanceType<typeof this.easyStoreClass>>();
+    this.easyStorePromise = deferred.promise;
+
+    let easyStore = await this.easyStoreClass.getInstance({
+      nameId: this.nameId,
+    });
+
+    if (!easyStore) {
+      easyStore = new this.easyStoreClass();
+      easyStore.nameId = this.nameId;
+      easyStore.data = {};
+      await easyStore.save();
+    }
+    deferred.resolve(easyStore);
+    return this.easyStorePromise;
+  }
+
+  /**
+   * reads all keyValue pairs at once and returns them
+   */
+  public async readAll() {
+    const easyStore = await this.getEasyStore();
+    return easyStore.data;
+  }
+
+  /**
+   * reads a keyValueFile from disk
+   */
+  public async readKey(keyArg: keyof T) {
+    const easyStore = await this.getEasyStore();
+    return easyStore.data[keyArg];
+  }
+
+  /**
+   * writes a specific key to the keyValueStore
+   */
+  public async writeKey<TKey extends keyof T>(keyArg: TKey, valueArg: T[TKey]) {
+    const easyStore = await this.getEasyStore();
+    easyStore.data[keyArg] = valueArg;
+    await easyStore.save();
+  }
+
+  public async deleteKey(keyArg: keyof T) {
+    const easyStore = await this.getEasyStore();
+    delete easyStore.data[keyArg];
+    await easyStore.save();
+  }
+
+  /**
+   * writes all keyValue pairs in the object argument
+   */
+  public async writeAll(keyValueObject: Partial<T>) {
+    const easyStore = await this.getEasyStore();
+    easyStore.data = { ...easyStore.data, ...keyValueObject };
+    await easyStore.save();
+  }
+
+  /**
+   * wipes a key value store from disk
+   */
+  public async wipe() {
+    const easyStore = await this.getEasyStore();
+    easyStore.data = {};
+    await easyStore.save();
+  }
+
+  public async cleanUpEphermal() {
+    while (
+      (await this.smartdataDbRef.statusConnectedDeferred.promise) &&
+      this.smartdataDbRef.status === 'connected'
+    ) {}
+  }
+}

ts/classes.lucene.adapter.ts

@@ -0,0 +1,779 @@
+/**
+ * Lucene to MongoDB query adapter for SmartData
+ */
+import * as plugins from './plugins.js';
+
+// Types
+type NodeType =
+  | 'TERM'
+  | 'PHRASE'
+  | 'FIELD'
+  | 'AND'
+  | 'OR'
+  | 'NOT'
+  | 'RANGE'
+  | 'WILDCARD'
+  | 'FUZZY'
+  | 'GROUP';
+
+interface QueryNode {
+  type: NodeType;
+}
+
+interface TermNode extends QueryNode {
+  type: 'TERM';
+  value: string;
+  boost?: number;
+}
+
+interface PhraseNode extends QueryNode {
+  type: 'PHRASE';
+  value: string;
+  proximity?: number;
+}
+
+interface FieldNode extends QueryNode {
+  type: 'FIELD';
+  field: string;
+  value: AnyQueryNode;
+}
+
+interface BooleanNode extends QueryNode {
+  type: 'AND' | 'OR' | 'NOT';
+  left: AnyQueryNode;
+  right: AnyQueryNode;
+}
+
+interface RangeNode extends QueryNode {
+  type: 'RANGE';
+  field: string;
+  lower: string;
+  upper: string;
+  includeLower: boolean;
+  includeUpper: boolean;
+}
+
+interface WildcardNode extends QueryNode {
+  type: 'WILDCARD';
+  value: string;
+}
+
+interface FuzzyNode extends QueryNode {
+  type: 'FUZZY';
+  value: string;
+  maxEdits: number;
+}
+
+interface GroupNode extends QueryNode {
+  type: 'GROUP';
+  value: AnyQueryNode;
+}
+
+type AnyQueryNode =
+  | TermNode
+  | PhraseNode
+  | FieldNode
+  | BooleanNode
+  | RangeNode
+  | WildcardNode
+  | FuzzyNode
+  | GroupNode;
+
+/**
+ * Lucene query parser
+ */
+export class LuceneParser {
+  private pos: number = 0;
+  private input: string = '';
+  private tokens: string[] = [];
+
+  constructor() {}
+
+  /**
+   * Parse a Lucene query string into an AST
+   */
+  parse(query: string): AnyQueryNode {
+    this.input = query.trim();
+    this.pos = 0;
+    this.tokens = this.tokenize(this.input);
+
+    return this.parseQuery();
+  }
+
+  /**
+   * Tokenize the input string into tokens
+   */
+  private tokenize(input: string): string[] {
+    const specialChars = /[()\[\]{}"~^:]/;
+    const operators = /AND|OR|NOT|TO/;
+
+    let tokens: string[] = [];
+    let current = '';
+    let inQuote = false;
+
+    for (let i = 0; i < input.length; i++) {
+      const char = input[i];
+
+      // Handle quoted strings
+      if (char === '"') {
+        if (inQuote) {
+          tokens.push(current + char);
+          current = '';
+          inQuote = false;
+        } else {
+          if (current) tokens.push(current);
+          current = char;
+          inQuote = true;
+        }
+        continue;
+      }
+
+      if (inQuote) {
+        current += char;
+        continue;
+      }
+
+      // Handle whitespace
+      if (char === ' ' || char === '\t' || char === '\n') {
+        if (current) {
+          tokens.push(current);
+          current = '';
+        }
+        continue;
+      }
+
+      // Handle special characters
+      if (specialChars.test(char)) {
+        if (current) {
+          tokens.push(current);
+          current = '';
+        }
+        tokens.push(char);
+        continue;
+      }
+
+      current += char;
+
+      // Check if current is an operator
+      if (operators.test(current) && (i + 1 === input.length || /\s/.test(input[i + 1]))) {
+        tokens.push(current);
+        current = '';
+      }
+    }
+
+    if (current) tokens.push(current);
+
+    return tokens;
+  }
+
+  /**
+   * Parse the main query expression
+   */
+  private parseQuery(): AnyQueryNode {
+    const left = this.parseBooleanOperand();
+
+    if (this.pos < this.tokens.length) {
+      const token = this.tokens[this.pos];
+
+      if (token === 'AND' || token === 'OR') {
+        this.pos++;
+        const right = this.parseQuery();
+        return {
+          type: token as 'AND' | 'OR',
+          left,
+          right,
+        };
+      } else if (token === 'NOT' || token === '-') {
+        this.pos++;
+        const right = this.parseQuery();
+        return {
+          type: 'NOT',
+          left,
+          right,
+        };
+      }
+    }
+
+    return left;
+  }
+
+  /**
+   * Parse boolean operands (terms, phrases, fields, groups)
+   */
+  private parseBooleanOperand(): AnyQueryNode {
+    if (this.pos >= this.tokens.length) {
+      throw new Error('Unexpected end of input');
+    }
+
+    const token = this.tokens[this.pos];
+
+    // Handle grouping with parentheses
+    if (token === '(') {
+      this.pos++;
+      const group = this.parseQuery();
+
+      if (this.pos < this.tokens.length && this.tokens[this.pos] === ')') {
+        this.pos++;
+        return { type: 'GROUP', value: group } as GroupNode;
+      } else {
+        throw new Error('Unclosed group');
+      }
+    }
+
+    // Handle fields (field:value)
+    if (this.pos + 1 < this.tokens.length && this.tokens[this.pos + 1] === ':') {
+      const field = token;
+      this.pos += 2; // Skip field and colon
+
+      if (this.pos < this.tokens.length) {
+        const value = this.parseBooleanOperand();
+        return { type: 'FIELD', field, value } as FieldNode;
+      } else {
+        throw new Error('Expected value after field');
+      }
+    }
+
+    // Handle range queries
+    if (token === '[' || token === '{') {
+      return this.parseRange();
+    }
+
+    // Handle phrases ("term term")
+    if (token.startsWith('"') && token.endsWith('"')) {
+      const phrase = token.slice(1, -1);
+      this.pos++;
+
+      // Check for proximity operator
+      let proximity: number | undefined;
+      if (this.pos < this.tokens.length && this.tokens[this.pos] === '~') {
+        this.pos++;
+        if (this.pos < this.tokens.length && /^\d+$/.test(this.tokens[this.pos])) {
+          proximity = parseInt(this.tokens[this.pos], 10);
+          this.pos++;
+        } else {
+          throw new Error('Expected number after proximity operator');
+        }
+      }
+
+      return { type: 'PHRASE', value: phrase, proximity } as PhraseNode;
+    }
+
+    // Handle wildcards
+    if (token.includes('*') || token.includes('?')) {
+      this.pos++;
+      return { type: 'WILDCARD', value: token } as WildcardNode;
+    }
+
+    // Handle fuzzy searches
+    if (this.pos + 1 < this.tokens.length && this.tokens[this.pos + 1] === '~') {
+      const term = token;
+      this.pos += 2; // Skip term and tilde
+
+      let maxEdits = 2; // Default
+      if (this.pos < this.tokens.length && /^\d+$/.test(this.tokens[this.pos])) {
+        maxEdits = parseInt(this.tokens[this.pos], 10);
+        this.pos++;
+      }
+
+      return { type: 'FUZZY', value: term, maxEdits } as FuzzyNode;
+    }
+
+    // Simple term
+    this.pos++;
+    return { type: 'TERM', value: token } as TermNode;
+  }
+
+  /**
+   * Parse range queries
+   */
+  private parseRange(): RangeNode {
+    const includeLower = this.tokens[this.pos] === '[';
+    const includeUpper = this.tokens[this.pos + 4] === ']';
+
+    // Ensure tokens for lower, TO, upper, and closing bracket exist
+    if (this.pos + 4 >= this.tokens.length) {
+      throw new Error('Invalid range query syntax');
+    }
+    this.pos++; // Skip open bracket
+
+    const lower = this.tokens[this.pos];
+    this.pos++;
+
+    if (this.tokens[this.pos] !== 'TO') {
+      throw new Error('Expected TO in range query');
+    }
+    this.pos++;
+
+    const upper = this.tokens[this.pos];
+    this.pos++;
+
+    if (this.tokens[this.pos] !== (includeLower ? ']' : '}')) {
+      throw new Error('Invalid range query closing bracket');
+    }
+    this.pos++;
+
+    // For simplicity, assuming the field is handled separately
+    return {
+      type: 'RANGE',
+      field: '', // This will be filled by the field node
+      lower,
+      upper,
+      includeLower,
+      includeUpper,
+    };
+  }
+}
+
+/**
+ * Transformer for Lucene AST to MongoDB query
+ * FIXED VERSION - proper MongoDB query structure
+ */
+export class LuceneToMongoTransformer {
+  private defaultFields: string[];
+  constructor(defaultFields: string[] = []) {
+    this.defaultFields = defaultFields;
+  }
+  /**
+   * Escape special characters for use in RegExp patterns
+   */
+  private escapeRegex(input: string): string {
+    return input.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+  }
+
+  /**
+   * Transform a Lucene AST node to a MongoDB query
+   */
+  transform(node: AnyQueryNode, searchFields?: string[]): any {
+    switch (node.type) {
+      case 'TERM':
+        return this.transformTerm(node, searchFields);
+      case 'PHRASE':
+        return this.transformPhrase(node, searchFields);
+      case 'FIELD':
+        return this.transformField(node);
+      case 'AND':
+        return this.transformAnd(node);
+      case 'OR':
+        return this.transformOr(node);
+      case 'NOT':
+        return this.transformNot(node);
+      case 'RANGE':
+        return this.transformRange(node);
+      case 'WILDCARD':
+        return this.transformWildcard(node, searchFields);
+      case 'FUZZY':
+        return this.transformFuzzy(node, searchFields);
+      case 'GROUP':
+        return this.transform(node.value, searchFields);
+      default:
+        throw new Error(`Unsupported node type: ${(node as any).type}`);
+    }
+  }
+
+  /**
+   * Transform a term to MongoDB query
+   * FIXED: properly structured $or query for multiple fields
+   */
+  private transformTerm(node: TermNode, searchFields?: string[]): any {
+    // Build regex pattern, support wildcard (*) and fuzzy (?) if present
+    const term = node.value;
+    // Determine regex pattern: wildcard conversion or exact escape
+    let pattern: string;
+    if (term.includes('*') || term.includes('?')) {
+      pattern = this.luceneWildcardToRegex(term);
+    } else {
+      pattern = this.escapeRegex(term);
+    }
+    // Search across provided fields or default fields
+    const fields = searchFields && searchFields.length > 0 ? searchFields : this.defaultFields;
+    const orConditions = fields.map((field) => ({
+      [field]: { $regex: pattern, $options: 'i' },
+    }));
+    return { $or: orConditions };
+  }
+
+  /**
+   * Transform a phrase to MongoDB query
+   * FIXED: properly structured $or query for multiple fields
+   */
+  private transformPhrase(node: PhraseNode, searchFields?: string[]): any {
+    // Use regex across provided fields or default fields, respecting word boundaries
+    const parts = node.value.split(/\s+/).map((t) => this.escapeRegex(t));
+    const pattern = parts.join('\\s+');
+    const fields = searchFields && searchFields.length > 0 ? searchFields : this.defaultFields;
+    const orConditions = fields.map((field) => ({
+      [field]: { $regex: pattern, $options: 'i' },
+    }));
+    return { $or: orConditions };
+  }
+
+  /**
+   * Transform a field query to MongoDB query
+   */
+  private transformField(node: FieldNode): any {
+    // Handle special case for range queries on fields
+    if (node.value.type === 'RANGE') {
+      const rangeNode = node.value as RangeNode;
+      rangeNode.field = node.field;
+      return this.transformRange(rangeNode);
+    }
+
+    // Handle special case for wildcards on fields
+    if (node.value.type === 'WILDCARD') {
+      return {
+        [node.field]: {
+          $regex: this.luceneWildcardToRegex((node.value as WildcardNode).value),
+          $options: 'i',
+        },
+      };
+    }
+
+    // Handle special case for fuzzy searches on fields
+    if (node.value.type === 'FUZZY') {
+      return {
+        [node.field]: {
+          $regex: this.createFuzzyRegex((node.value as FuzzyNode).value),
+          $options: 'i',
+        },
+      };
+    }
+
+    // Special case for exact term matches on fields (supporting wildcard characters)
+    if (node.value.type === 'TERM') {
+      const val = (node.value as TermNode).value;
+      if (val.includes('*') || val.includes('?')) {
+        const regex = this.luceneWildcardToRegex(val);
+        return { [node.field]: { $regex: regex, $options: 'i' } };
+      }
+      return { [node.field]: { $regex: val, $options: 'i' } };
+    }
+
+    // Special case for phrase matches on fields
+    if (node.value.type === 'PHRASE') {
+      return {
+        [node.field]: {
+          $regex: `${(node.value as PhraseNode).value.replace(/\s+/g, '\\s+')}`,
+          $options: 'i',
+        },
+      };
+    }
+
+    // For other cases, we'll transform the value and apply it to the field
+    const transformedValue = this.transform(node.value);
+
+    // If the transformed value uses $text, we need to adapt it for the field
+    if (transformedValue.$text) {
+      return { [node.field]: { $regex: transformedValue.$text.$search, $options: 'i' } };
+    }
+
+    // Handle $or and $and cases
+    if (transformedValue.$or || transformedValue.$and) {
+      // This is a bit complex - we need to restructure the query to apply the field
+      // For now, simplify by just using a regex on the field
+      const term = this.extractTermFromBooleanQuery(transformedValue);
+      if (term) {
+        return { [node.field]: { $regex: term, $options: 'i' } };
+      }
+    }
+
+    return { [node.field]: transformedValue };
+  }
+
+  /**
+   * Extract a term from a boolean query (simplification)
+   */
+  private extractTermFromBooleanQuery(query: any): string | null {
+    if (query.$or && Array.isArray(query.$or) && query.$or.length > 0) {
+      const firstClause = query.$or[0];
+      for (const field in firstClause) {
+        if (firstClause[field].$regex) {
+          return firstClause[field].$regex;
+        }
+      }
+    }
+
+    if (query.$and && Array.isArray(query.$and) && query.$and.length > 0) {
+      const firstClause = query.$and[0];
+      for (const field in firstClause) {
+        if (firstClause[field].$regex) {
+          return firstClause[field].$regex;
+        }
+      }
+    }
+
+    return null;
+  }
+
+  /**
+   * Transform AND operator to MongoDB query
+   * FIXED: $and must be an array
+   */
+  private transformAnd(node: BooleanNode): any {
+    return { $and: [this.transform(node.left), this.transform(node.right)] };
+  }
+
+  /**
+   * Transform OR operator to MongoDB query
+   * FIXED: $or must be an array
+   */
+  private transformOr(node: BooleanNode): any {
+    return { $or: [this.transform(node.left), this.transform(node.right)] };
+  }
+
+  /**
+   * Transform NOT operator to MongoDB query
+   * FIXED: $and must be an array and $not usage
+   */
+  private transformNot(node: BooleanNode): any {
+    const leftQuery = this.transform(node.left);
+    const rightQuery = this.transform(node.right);
+
+    // Create a query that includes left but excludes right
+    if (rightQuery.$text) {
+      // For text searches, we need a different approach
+      // We'll use a negated regex instead
+      const searchTerm = rightQuery.$text.$search.replace(/"/g, '');
+
+      // Determine the fields to apply the negation to
+      const notConditions = [];
+
+      for (const field in leftQuery) {
+        if (field !== '$or' && field !== '$and') {
+          notConditions.push({
+            [field]: { $not: { $regex: searchTerm, $options: 'i' } },
+          });
+        }
+      }
+
+      // If left query has $or or $and, we need to handle it differently
+      if (leftQuery.$or) {
+        return {
+          $and: [leftQuery, { $nor: [{ $or: notConditions }] }],
+        };
+      } else {
+        // Simple case - just add $not to each field
+        return {
+          $and: [leftQuery, { $and: notConditions }],
+        };
+      }
+    } else {
+      // For other queries, we can use $not directly
+      // We need to handle different structures based on the rightQuery
+      let notQuery = {};
+
+      if (rightQuery.$or) {
+        notQuery = { $nor: rightQuery.$or };
+      } else if (rightQuery.$and) {
+        // Convert $and to $nor
+        notQuery = { $nor: rightQuery.$and };
+      } else {
+        // Simple field condition
+        for (const field in rightQuery) {
+          notQuery[field] = { $not: rightQuery[field] };
+        }
+      }
+
+      return { $and: [leftQuery, notQuery] };
+    }
+  }
+
+  /**
+   * Transform range query to MongoDB query
+   */
+  private transformRange(node: RangeNode): any {
+    const range: any = {};
+
+    if (node.lower !== '*') {
+      range[node.includeLower ? '$gte' : '$gt'] = this.parseValue(node.lower);
+    }
+
+    if (node.upper !== '*') {
+      range[node.includeUpper ? '$lte' : '$lt'] = this.parseValue(node.upper);
+    }
+
+    return { [node.field]: range };
+  }
+
+  /**
+   * Transform wildcard query to MongoDB query
+   * FIXED: properly structured for multiple fields
+   */
+  private transformWildcard(node: WildcardNode, searchFields?: string[]): any {
+    // Convert Lucene wildcards to MongoDB regex
+    const regex = this.luceneWildcardToRegex(node.value);
+
+    // If specific fields are provided, search wildcard across those fields
+    if (searchFields && searchFields.length > 0) {
+      const orConditions = searchFields.map((field) => ({
+        [field]: { $regex: regex, $options: 'i' },
+      }));
+
+      return { $or: orConditions };
+    }
+
+    // By default, apply to the default field
+    return { $regex: regex, $options: 'i' };
+  }
+
+  /**
+   * Transform fuzzy query to MongoDB query
+   * FIXED: properly structured for multiple fields
+   */
+  private transformFuzzy(node: FuzzyNode, searchFields?: string[]): any {
+    // MongoDB doesn't have built-in fuzzy search
+    // This is a very basic approach using regex
+    const regex = this.createFuzzyRegex(node.value);
+
+    // If specific fields are provided, search fuzzy term across those fields
+    if (searchFields && searchFields.length > 0) {
+      const orConditions = searchFields.map((field) => ({
+        [field]: { $regex: regex, $options: 'i' },
+      }));
+
+      return { $or: orConditions };
+    }
+
+    // By default, apply to the default field
+    return { $regex: regex, $options: 'i' };
+  }
+
+  /**
+   * Convert Lucene wildcards to MongoDB regex patterns
+   */
+  public luceneWildcardToRegex(wildcardPattern: string): string {
+    // Replace Lucene wildcards with regex equivalents
+    // * => .*
+    // ? => .
+    // Also escape regex special chars
+    return wildcardPattern
+      .replace(/([.+^${}()|\\])/g, '\\$1') // Escape regex special chars
+      .replace(/\*/g, '.*')
+      .replace(/\?/g, '.');
+  }
+
+  /**
+   * Create a simplified fuzzy search regex
+   */
+  private createFuzzyRegex(term: string): string {
+    // For a very simple approach, we allow some characters to be optional
+    let regex = '';
+    for (let i = 0; i < term.length; i++) {
+      // Make every other character optional (simplified fuzzy)
+      if (i % 2 === 1) {
+        regex += term[i] + '?';
+      } else {
+        regex += term[i];
+      }
+    }
+    return regex;
+  }
+
+  /**
+   * Parse string values to appropriate types (numbers, dates, etc.)
+   */
+  private parseValue(value: string): any {
+    // Try to parse as number
+    if (/^-?\d+$/.test(value)) {
+      return parseInt(value, 10);
+    }
+
+    if (/^-?\d+\.\d+$/.test(value)) {
+      return parseFloat(value);
+    }
+
+    // Try to parse as date (simplified)
+    const date = new Date(value);
+    if (!isNaN(date.getTime())) {
+      return date;
+    }
+
+    // Default to string
+    return value;
+  }
+}
+
+/**
+ * Main adapter class
+ */
+export class SmartdataLuceneAdapter {
+  private parser: LuceneParser;
+  private transformer: LuceneToMongoTransformer;
+  private defaultSearchFields: string[] = [];
+
+  /**
+   * @param defaultSearchFields - Optional array of field names to search across when no field is specified
+   */
+  constructor(defaultSearchFields?: string[]) {
+    this.parser = new LuceneParser();
+    // Pass default searchable fields into transformer
+    this.transformer = new LuceneToMongoTransformer(defaultSearchFields || []);
+    if (defaultSearchFields) {
+      this.defaultSearchFields = defaultSearchFields;
+    }
+  }
+
+  /**
+   * Convert a Lucene query string to a MongoDB query object
+   * @param luceneQuery - The Lucene query string to convert
+   * @param searchFields - Optional array of field names to search across (overrides defaultSearchFields)
+   */
+  convert(luceneQuery: string, searchFields?: string[]): any {
+    try {
+      // For simple single-term queries (no field:, boolean, grouping), use simpler regex
+      if (
+        !luceneQuery.includes(':') &&
+        !luceneQuery.includes(' AND ') &&
+        !luceneQuery.includes(' OR ') &&
+        !luceneQuery.includes(' NOT ') &&
+        !luceneQuery.includes('(') &&
+        !luceneQuery.includes('[')
+      ) {
+        const fieldsToSearch = searchFields || this.defaultSearchFields;
+        if (fieldsToSearch && fieldsToSearch.length > 0) {
+          // Handle wildcard characters in query
+          let pattern = luceneQuery;
+          if (luceneQuery.includes('*') || luceneQuery.includes('?')) {
+            // Use transformer to convert wildcard pattern
+            pattern = this.transformer.luceneWildcardToRegex(luceneQuery);
+          }
+          return {
+            $or: fieldsToSearch.map((field) => ({
+              [field]: { $regex: pattern, $options: 'i' },
+            })),
+          };
+        }
+      }
+
+      // For more complex queries, use the full parser
+      // Parse the Lucene query into an AST
+      const ast = this.parser.parse(luceneQuery);
+
+      // Use provided searchFields, fall back to defaultSearchFields
+      const fieldsToSearch = searchFields || this.defaultSearchFields;
+
+      // Transform the AST to a MongoDB query
+      return this.transformWithFields(ast, fieldsToSearch);
+    } catch (error) {
+      console.error(`Failed to convert Lucene query "${luceneQuery}":`, error);
+      throw new Error(`Failed to convert Lucene query: ${error}`);
+    }
+  }
+
+  /**
+   * Helper method to transform the AST with field information
+   */
+  private transformWithFields(node: AnyQueryNode, searchFields: string[]): any {
+    // Special case for term nodes without a specific field
+    if (
+      node.type === 'TERM' ||
+      node.type === 'PHRASE' ||
+      node.type === 'WILDCARD' ||
+      node.type === 'FUZZY'
+    ) {
+      return this.transformer.transform(node, searchFields);
+    }
+
+    // For other node types, use the standard transformation
+    return this.transformer.transform(node);
+  }
+}

ts/classes.watcher.ts

@@ -0,0 +1,73 @@
+import { SmartDataDbDoc } from './classes.doc.js';
+import * as plugins from './plugins.js';
+import { EventEmitter } from 'events';
+
+/**
+ * a wrapper for the native mongodb cursor. Exposes better
+ */
+/**
+ * Wraps a MongoDB ChangeStream with RxJS and EventEmitter support.
+ */
+export class SmartdataDbWatcher<T = any> extends EventEmitter {
+  // STATIC
+  public readyDeferred = plugins.smartpromise.defer();
+
+  // INSTANCE
+  private changeStream: plugins.mongodb.ChangeStream<T>;
+  private rawSubject: plugins.smartrx.rxjs.Subject<T>;
+  /** Emits change documents (or arrays of documents if buffered) */
+  public changeSubject: any;
+  /**
+   * @param changeStreamArg native MongoDB ChangeStream
+   * @param smartdataDbDocArg document class for instance creation
+   * @param opts.bufferTimeMs optional milliseconds to buffer events via RxJS
+   */
+  constructor(
+    changeStreamArg: plugins.mongodb.ChangeStream<T>,
+    smartdataDbDocArg: typeof SmartDataDbDoc,
+    opts?: { bufferTimeMs?: number },
+  ) {
+    super();
+    this.rawSubject = new plugins.smartrx.rxjs.Subject<T>();
+    // Apply buffering if requested
+    if (opts && opts.bufferTimeMs) {
+      this.changeSubject = this.rawSubject.pipe(plugins.smartrx.rxjs.ops.bufferTime(opts.bufferTimeMs));
+    } else {
+      this.changeSubject = this.rawSubject;
+    }
+    this.changeStream = changeStreamArg;
+    this.changeStream.on('change', async (item: any) => {
+      let docInstance: T = null;
+      if (item.fullDocument) {
+        docInstance = smartdataDbDocArg.createInstanceFromMongoDbNativeDoc(
+          item.fullDocument
+        ) as any as T;
+      }
+      // Notify subscribers
+      this.rawSubject.next(docInstance);
+      this.emit('change', docInstance);
+    });
+    // Signal readiness after one tick
+    plugins.smartdelay.delayFor(0).then(() => {
+      this.readyDeferred.resolve();
+    });
+  }
+
+  /**
+   * Close the change stream, complete the RxJS subject, and remove listeners.
+   */
+  public async close(): Promise<void> {
+    // Close MongoDB ChangeStream
+    await this.changeStream.close();
+    // Complete the subject to teardown any buffering operators
+    this.rawSubject.complete();
+    // Remove all EventEmitter listeners
+    this.removeAllListeners();
+  }
+  /**
+   * Alias for close(), matching README usage
+   */
+  public async stop(): Promise<void> {
+    return this.close();
+  }
+}

ts/index.ts

@@ -1,4 +1,14 @@
-import * as plugins from './smartdata.plugins'
+export * from './classes.db.js';
+export * from './classes.collection.js';
+export * from './classes.doc.js';
+export * from './classes.easystore.js';
+export * from './classes.cursor.js';
 
-export * from './smartdata.classes.dbcollection'
-export * from './smartdata.classes.db'
+import * as convenience from './classes.convenience.js';
+
+export { convenience };
+
+// to be removed with the next breaking update
+import type * as plugins from './plugins.js';
+type IMongoDescriptor = plugins.tsclass.database.IMongoDescriptor;
+export type { IMongoDescriptor };

ts/logging.ts

@@ -0,0 +1,3 @@
+import * as plugins from './plugins.js';
+
+export const logger = new plugins.smartlog.ConsoleLog();

ts/plugins.ts

@@ -0,0 +1,29 @@
+// tsclass scope
+import * as tsclass from '@tsclass/tsclass';
+
+export { tsclass };
+
+// @pushrocks scope
+import * as lik from '@push.rocks/lik';
+import * as smartdelay from '@push.rocks/smartdelay';
+import * as smartlog from '@push.rocks/smartlog';
+import * as smartpromise from '@push.rocks/smartpromise';
+import * as smartrx from '@push.rocks/smartrx';
+import * as smartstring from '@push.rocks/smartstring';
+import * as smarttime from '@push.rocks/smarttime';
+import * as smartunique from '@push.rocks/smartunique';
+import * as taskbuffer from '@push.rocks/taskbuffer';
+import * as mongodb from 'mongodb';
+
+export {
+  lik,
+  smartdelay,
+  smartpromise,
+  smartlog,
+  smartrx,
+  mongodb,
+  smartstring,
+  smarttime,
+  smartunique,
+  taskbuffer,
+};

ts/smartdata.classes.db.ts

@@ -1,33 +0,0 @@
-import * as plugins from './smartdata.plugins'
-
-export type TConnectionStatus = 'disconnected' | 'connected' | 'failed'
-
-export class Db {
-    dbUrl: string
-    db: plugins.mongodb.Db
-    status: TConnectionStatus
-
-    constructor(dbUrl: string) {
-        this.dbUrl = dbUrl
-    }
-
-    connect(): plugins.q.Promise<any> {
-        let done = plugins.q.defer()
-        plugins.mongodb.MongoClient.connect(this.dbUrl, (err, db) => {
-            if (err) { console.log(err) }
-            plugins.assert.equal(null, err)
-            this.db = db
-            plugins.beautylog.success(`connected to database at ${this.dbUrl}`)
-            done.resolve(this.db)
-        })
-        return done.promise
-    }
-
-    close(): plugins.q.Promise<any> {
-        let done = plugins.q.defer()
-        this.db.close()
-        plugins.beautylog.ok(`disconnected to database at ${this.dbUrl}`)
-        done.resolve()
-        return done.promise
-    }
-}

ts/smartdata.classes.dbcollection.ts

@@ -1,60 +0,0 @@
-import * as plugins from './smartdata.plugins'
-import { Db } from './smartdata.classes.db'
-
-export class DbCollection<T> {
-    collection: plugins.mongodb.Collection
-    constructor(nameArg: string, dbArg: Db) {
-        this.collection = dbArg.db.collection(nameArg)
-    }
-
-    /**
-     * adds a validation function that all newly inserted and updated objects have to pass
-     */
-    addObjectValidation(funcArg) { }
-
-    /**
-     * finds an object in the DbCollection
-     */
-    find(docMatchArg: T | any): plugins.q.Promise<T[]> {
-        let done = plugins.q.defer<T[]>()
-        this.collection.find(docMatchArg).toArray((err, docs) => {
-            if (err) { throw err }
-            done.resolve(docs)
-        })
-        return done.promise
-    }
-
-    /**
-     * inserts  object into the DbCollection
-     */
-    insertOne(docArg: T): plugins.q.Promise<void> {
-        let done = plugins.q.defer<void>()
-        this.checkDoc(docArg).then(() => {
-            this.collection.insertOne(docArg)
-                .then(() => { done.resolve() })
-        })
-        return done.promise
-    }
-
-    /**
-     * inserts many objects at once into the DbCollection
-     */
-    insertMany(docArrayArg: T[]): plugins.q.Promise<void> {
-        let done = plugins.q.defer<void>()
-        let checkDocPromiseArray: plugins.q.Promise<void>[] = []
-        for (let docArg of docArrayArg){
-            checkDocPromiseArray.push(this.checkDoc(docArg))
-        }
-        plugins.q.all(checkDocPromiseArray).then(() => {
-            this.collection.insertMany(docArrayArg)
-                .then(() => { done.resolve() })
-        })
-        return done.promise
-    }
-
-    private checkDoc(doc: T): plugins.q.Promise<void> {
-        let done = plugins.q.defer<void>()
-        done.resolve()
-        return done.promise
-    }
-}

ts/smartdata.plugins.ts

@@ -1,5 +0,0 @@
-import 'typings-global'
-export import assert = require('assert')
-export import beautylog = require('beautylog')
-export import mongodb = require('mongodb')
-export import q = require('q')

tsconfig.json

@@ -0,0 +1,16 @@
+{
+  "compilerOptions": {
+    "experimentalDecorators": true,
+    "useDefineForClassFields": false,
+    "target": "ES2022",
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "esModuleInterop": true,
+    "verbatimModuleSyntax": true,
+    "baseUrl": ".",
+    "paths": {}
+  },
+  "exclude": [
+    "dist_*/**/*.d.ts"
+  ]
+}

tslint.json

@@ -1,3 +0,0 @@
-{
-    "extends": "tslint-config-standard"
-}