Compare commits
18 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 026f2acc89 | |||
| 1cd0f09598 | |||
| d254f58a05 | |||
| c5e7b6f982 | |||
| d30c9619c5 | |||
| 7344ae2db3 | |||
| 3b29a150a8 | |||
| 59186d84a9 | |||
| 7fab4e5dd0 | |||
| 0dbaa1bc5d | |||
| 8b37ebc8f9 | |||
| 5d757207c8 | |||
| c80df05fdf | |||
| 9be43a85ef | |||
| bf66209d3e | |||
| cdd1ae2c9b | |||
| f4290ae7f7 | |||
| e58c0fd215 |
3
.vscode/settings.json
vendored
3
.vscode/settings.json
vendored
@@ -22,5 +22,6 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
]
|
],
|
||||||
|
"deno.enable": false
|
||||||
}
|
}
|
||||||
|
|||||||
72
changelog.md
72
changelog.md
@@ -1,5 +1,77 @@
|
|||||||
# Changelog
|
# Changelog
|
||||||
|
|
||||||
|
## 2025-11-17 - 6.0.0 - BREAKING CHANGE(decorators)
|
||||||
|
Migrate to TC39 Stage 3 decorators and refactor decorator metadata handling; update class initialization, lucene adapter fixes and docs
|
||||||
|
|
||||||
|
- Switch all decorators to TC39 Stage 3 signatures and metadata usage (use context.metadata and context.addInitializer) — affects svDb, globalSvDb, searchable, unI, index, Collection and managed.
|
||||||
|
- Refactor Collection/managed decorators to read and initialize prototype/constructor properties from context.metadata to ensure prototype properties are available before instance creation (ts/classes.collection.ts).
|
||||||
|
- Improve search implementation: add a Lucene parser and transformer with safer MongoDB query generation, wildcard/fuzzy handling and properly structured boolean operators (ts/classes.lucene.adapter.ts).
|
||||||
|
- Search integration updated to use the new adapter and handle advanced Lucene syntax and edge cases more robustly.
|
||||||
|
- Bump dev tooling versions: @git.zone/tsbuild -> ^3.1.0 and @git.zone/tsrun -> ^2.0.0.
|
||||||
|
- Documentation: update README and add readme.hints.md describing the TC39 decorator migration, minimum TypeScript (>=5.2) and Deno notes; tests adjusted accordingly.
|
||||||
|
- Clean up project memory/config files related to the previous decorator approach and Deno configuration adjustments.
|
||||||
|
|
||||||
|
## 2025-11-17 - 5.16.7 - fix(classes.collection)
|
||||||
|
Improve Deno and TypeScript compatibility: Collection decorator _svDbOptions forwarding and config cleanup
|
||||||
|
|
||||||
|
- Collection decorator: capture original constructor and forward _svDbOptions to ensure property decorator options (serialize/deserialize) remain accessible in Deno environments.
|
||||||
|
- Collection decorator: keep instance getter defined on prototype for Deno compatibility (no behavior change, clarifies forwarding logic).
|
||||||
|
- Build/config: removed experimentalDecorators and useDefineForClassFields from deno.json and tsconfig.json to avoid Deno/TS build issues and rely on default compilation settings.
|
||||||
|
|
||||||
|
## 2025-11-17 - 5.16.6 - fix(classes)
|
||||||
|
Add Deno compatibility, prototype-safe decorators and safe collection accessor; bump a few deps
|
||||||
|
|
||||||
|
- Add deno.json to enable experimentalDecorators and target ES2022/DOM for Deno builds.
|
||||||
|
- Introduce getCollectionSafe() on SmartDataDbDoc and use it for save/update/delete/findOne to avoid runtime errors when instance 'collection' is not present.
|
||||||
|
- Change several instance properties (globalSaveableProperties, uniqueIndexes, regularIndexes, saveableProperties) to 'declare' so decorator-set prototype properties are not shadowed (Deno compatibility).
|
||||||
|
- Enhance @Collection decorator: capture original constructor/prototype for Deno, define prototype getter for collection on decorated class, attach docCtor for searchableFields, and forward _svDbOptions to the original constructor to preserve serializer metadata.
|
||||||
|
- Improve text/search index handling by relying on docCtor.searchableFields and guarding text index creation.
|
||||||
|
- Bump dependencies/devDependencies: @push.rocks/smartmongo -> ^2.0.14, @git.zone/tsbuild -> ^2.7.1, @git.zone/tstest -> ^2.8.1.
|
||||||
|
- These are non-breaking runtime compatibility and developer-experience fixes; intended as a patch release.
|
||||||
|
|
||||||
|
## 2025-11-16 - 5.16.5 - fix(watcher)
|
||||||
|
Update dependencies, tooling and watcher import; add .serena cache ignore
|
||||||
|
|
||||||
|
- Bump runtime dependencies: @push.rocks/smartlog 3.1.8 → 3.1.10, @push.rocks/smartstring 4.0.15 → 4.1.0, @push.rocks/taskbuffer 3.1.7 → 3.4.0, @tsclass/tsclass 9.2.0 → 9.3.0, mongodb 6.18.0 → 6.20.0
|
||||||
|
- Bump devDependencies: @git.zone/tsbuild 2.6.7 → 2.6.8, @git.zone/tsrun 1.2.44 → 1.6.2, @git.zone/tstest 2.3.5 → 2.6.2
|
||||||
|
- Switch EventEmitter import to node:events in ts/classes.watcher.ts to use the namespaced Node import
|
||||||
|
- Add .serena/.gitignore to ignore /cache
|
||||||
|
|
||||||
|
## 2025-08-18 - 5.16.4 - fix(classes.doc (convertFilterForMongoDb))
|
||||||
|
Improve filter conversion: handle logical operators, merge operator objects, add nested filter tests and docs, and fix test script
|
||||||
|
|
||||||
|
- Fix package.json test script: remove stray dot in tstest --verbose argument to ensure tests run correctly
|
||||||
|
- Enhance convertFilterForMongoDb in ts/classes.doc.ts to properly handle logical operators ($and, $or, $nor, $not) and return them recursively
|
||||||
|
- Merge operator objects for the same field path (e.g. combining $gte and $lte) to avoid overwriting operator clauses when object and dot-notation are mixed
|
||||||
|
- Add validation/guards for operator argument types (e.g. $in, $nin, $all must be arrays; $size must be numeric) and preserve existing behavior blocking $where for security
|
||||||
|
- Add comprehensive nested filter tests in test/test.filters.ts to cover deep nested object queries, $elemMatch, array size, $all, $in on nested fields and more
|
||||||
|
- Expand README filtering section with detailed examples for basic filtering, deep nested filters, comparison operators, array operations, logical and element operators, and advanced patterns
|
||||||
|
|
||||||
|
## 2025-08-18 - 5.16.3 - fix(docs)
|
||||||
|
Add local Claude settings and remove outdated codex.md
|
||||||
|
|
||||||
|
- Added .claude/settings.local.json to store local Claude/assistant permissions and configuration.
|
||||||
|
- Removed codex.md (project overview) — documentation file deleted.
|
||||||
|
- No runtime/library code changes; documentation/configuration-only update, bump patch version.
|
||||||
|
|
||||||
|
## 2025-08-18 - 5.16.2 - fix(readme)
|
||||||
|
Update README: clarify examples, expand search/cursor/docs and add local Claude settings
|
||||||
|
|
||||||
|
- Refined README wording and structure: clearer Quick Start, improved examples and developer-focused phrasing
|
||||||
|
- Expanded documentation for search, cursors, change streams, distributed coordination, transactions and EasyStore with more concrete code examples
|
||||||
|
- Adjusted code examples to show safer defaults (ID generation, status/tags, connection pooling) and improved best-practices guidance
|
||||||
|
- Added .claude/settings.local.json to provide local assistant/CI permission configuration
|
||||||
|
|
||||||
|
## 2025-08-12 - 5.16.1 - fix(core)
|
||||||
|
Improve error handling and logging; enhance search query sanitization; update dependency versions and documentation
|
||||||
|
|
||||||
|
- Replaced console.log and console.warn with structured logger.log calls throughout the core modules
|
||||||
|
- Enhanced database initialization with try/catch and proper URI credential encoding
|
||||||
|
- Improved search query conversion by disallowing dangerous operators (e.g. $where) and securely escaping regex patterns
|
||||||
|
- Bumped dependency versions (smartlog, @tsclass/tsclass, mongodb, etc.) in package.json
|
||||||
|
- Added detailed project memories including code style, project overview, and suggested commands for developers
|
||||||
|
- Updated README with improved instructions, feature highlights, and quick start sections
|
||||||
|
|
||||||
## 2025-04-25 - 5.16.0 - feat(watcher)
|
## 2025-04-25 - 5.16.0 - feat(watcher)
|
||||||
Enhance change stream watchers with buffering and EventEmitter support; update dependency versions
|
Enhance change stream watchers with buffering and EventEmitter support; update dependency versions
|
||||||
|
|
||||||
|
|||||||
77
codex.md
77
codex.md
@@ -1,77 +0,0 @@
|
|||||||
# SmartData Project Overview
|
|
||||||
|
|
||||||
This document provides a high-level overview of the SmartData library (`@push.rocks/smartdata`), its architecture, core components, and key features—including recent enhancements to the search API.
|
|
||||||
|
|
||||||
## 1. Project Purpose
|
|
||||||
- A TypeScript‑first wrapper around MongoDB that supplies:
|
|
||||||
- Strongly‑typed document & collection classes
|
|
||||||
- Decorator‑based schema definition (no external schema files)
|
|
||||||
- Advanced search capabilities with Lucene‑style queries
|
|
||||||
- Built‑in support for real‑time data sync, distributed coordination, and key‑value EasyStore
|
|
||||||
|
|
||||||
## 2. Core Concepts & Components
|
|
||||||
- **SmartDataDb**: Manages the MongoDB connection, pooling, and initialization of collections.
|
|
||||||
- **SmartDataDbDoc**: Base class for all document models; provides CRUD, upsert, and cursor APIs.
|
|
||||||
- **Decorators**:
|
|
||||||
- `@Collection`: Associates a class with a MongoDB collection
|
|
||||||
- `@svDb()`: Marks a field as persisted to the DB
|
|
||||||
- `@unI()`: Marks a field as a unique index
|
|
||||||
- `@index()`: Adds a regular index
|
|
||||||
- `@searchable()`: Marks a field for inclusion in text searches or regex queries
|
|
||||||
- **SmartdataCollection**: Wraps a MongoDB collection; auto‑creates indexes based on decorators.
|
|
||||||
- **Lucene Adapter**: Parses a Lucene query string into an AST and transforms it to a MongoDB filter object.
|
|
||||||
- **EasyStore**: A simple, schema‑less key‑value store built on top of MongoDB for sharing ephemeral data.
|
|
||||||
- **Distributed Coordinator**: Leader election and task‑distribution API for building resilient, multi‑instance systems.
|
|
||||||
- **Watcher**: Listens to change streams for real‑time updates and integrates with RxJS.
|
|
||||||
|
|
||||||
## 3. Search API
|
|
||||||
SmartData provides a unified `.search(query[, opts])` method on all models with `@searchable()` fields:
|
|
||||||
|
|
||||||
- **Supported Syntax**:
|
|
||||||
1. Exact field:value (e.g. `field:Value`)
|
|
||||||
2. Quoted phrases (e.g. `"exact phrase"` or `'exact phrase'`)
|
|
||||||
3. Wildcards: `*` (zero or more chars) and `?` (single char)
|
|
||||||
4. Boolean operators: `AND`, `OR`, `NOT`
|
|
||||||
5. Grouping: parenthesis `(A OR B) AND C`
|
|
||||||
6. Range queries: `[num TO num]`, `{num TO num}`
|
|
||||||
7. Multi‑term unquoted: terms AND’d across all searchable fields
|
|
||||||
8. Empty query returns all documents
|
|
||||||
|
|
||||||
- **Fallback Mechanisms**:
|
|
||||||
1. Text index based `$text` search (if supported)
|
|
||||||
2. Field‑scoped and multi‑field regex queries
|
|
||||||
3. In‑memory filtering for complex or unsupported cases
|
|
||||||
|
|
||||||
### New Security & Extensibility Hooks
|
|
||||||
The `.search(query, opts?)` signature now accepts a `SearchOptions<T>` object:
|
|
||||||
```ts
|
|
||||||
interface SearchOptions<T> {
|
|
||||||
filter?: Record<string, any>; // Additional MongoDB filter AND‑merged
|
|
||||||
validate?: (doc: T) => boolean; // Post‑fetch hook to drop results
|
|
||||||
}
|
|
||||||
```
|
|
||||||
- **filter**: Enforces mandatory constraints (e.g. multi‑tenant isolation) directly in the Mongo query.
|
|
||||||
- **validate**: An async function that runs after fetching; return `false` to exclude a document.
|
|
||||||
|
|
||||||
## 4. Testing Strategy
|
|
||||||
- Unit tests in `test/test.search.ts` cover basic search functionality and new options:
|
|
||||||
- Exact, wildcard, phrase, boolean and grouping cases
|
|
||||||
- Implicit AND and mixed free‑term + field searches
|
|
||||||
- Edge cases (non‑searchable fields, quoted wildcards, no matches)
|
|
||||||
- `filter` and `validate` tests ensure security hooks work as intended
|
|
||||||
- Advanced search scenarios are covered in `test/test.search.advanced.ts`.
|
|
||||||
|
|
||||||
## 5. Usage Example
|
|
||||||
```ts
|
|
||||||
// Basic search
|
|
||||||
const prods = await Product.search('wireless earbuds');
|
|
||||||
|
|
||||||
// Scoped search (only your organization’s items)
|
|
||||||
const myItems = await Product.search('book', { filter: { ownerId } });
|
|
||||||
|
|
||||||
// Post‑search validation (only cheap items)
|
|
||||||
const cheapItems = await Product.search('', { validate: p => p.price < 50 });
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
Last updated: 2025-04-22
|
|
||||||
28
package.json
28
package.json
@@ -1,13 +1,13 @@
|
|||||||
{
|
{
|
||||||
"name": "@push.rocks/smartdata",
|
"name": "@push.rocks/smartdata",
|
||||||
"version": "5.16.0",
|
"version": "6.0.0",
|
||||||
"private": false,
|
"private": false,
|
||||||
"description": "An advanced library for NoSQL data organization and manipulation using TypeScript with support for MongoDB, data validation, collections, and custom data types.",
|
"description": "An advanced library for NoSQL data organization and manipulation using TypeScript with support for MongoDB, data validation, collections, and custom data types.",
|
||||||
"main": "dist_ts/index.js",
|
"main": "dist_ts/index.js",
|
||||||
"typings": "dist_ts/index.d.ts",
|
"typings": "dist_ts/index.d.ts",
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"test": "tstest test/",
|
"test": "tstest test/ --verbose --logfile --timeout 120",
|
||||||
"testSearch": "tsx test/test.search.ts",
|
"testSearch": "tsx test/test.search.ts",
|
||||||
"build": "tsbuild --web --allowimplicitany",
|
"build": "tsbuild --web --allowimplicitany",
|
||||||
"buildDocs": "tsdoc"
|
"buildDocs": "tsdoc"
|
||||||
@@ -23,25 +23,25 @@
|
|||||||
},
|
},
|
||||||
"homepage": "https://code.foss.global/push.rocks/smartdata#readme",
|
"homepage": "https://code.foss.global/push.rocks/smartdata#readme",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@push.rocks/lik": "^6.0.14",
|
"@push.rocks/lik": "^6.2.2",
|
||||||
"@push.rocks/smartdelay": "^3.0.1",
|
"@push.rocks/smartdelay": "^3.0.1",
|
||||||
"@push.rocks/smartlog": "^3.0.2",
|
"@push.rocks/smartlog": "^3.1.10",
|
||||||
"@push.rocks/smartmongo": "^2.0.12",
|
"@push.rocks/smartmongo": "^2.0.14",
|
||||||
"@push.rocks/smartpromise": "^4.0.2",
|
"@push.rocks/smartpromise": "^4.0.2",
|
||||||
"@push.rocks/smartrx": "^3.0.10",
|
"@push.rocks/smartrx": "^3.0.10",
|
||||||
"@push.rocks/smartstring": "^4.0.15",
|
"@push.rocks/smartstring": "^4.1.0",
|
||||||
"@push.rocks/smarttime": "^4.0.6",
|
"@push.rocks/smarttime": "^4.0.6",
|
||||||
"@push.rocks/smartunique": "^3.0.8",
|
"@push.rocks/smartunique": "^3.0.8",
|
||||||
"@push.rocks/taskbuffer": "^3.1.7",
|
"@push.rocks/taskbuffer": "^3.4.0",
|
||||||
"@tsclass/tsclass": "^9.0.0",
|
"@tsclass/tsclass": "^9.3.0",
|
||||||
"mongodb": "^6.16.0"
|
"mongodb": "^6.20.0"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@git.zone/tsbuild": "^2.3.2",
|
"@git.zone/tsbuild": "^3.1.0",
|
||||||
"@git.zone/tsrun": "^1.2.44",
|
"@git.zone/tsrun": "^2.0.0",
|
||||||
"@git.zone/tstest": "^1.0.77",
|
"@git.zone/tstest": "^2.8.1",
|
||||||
"@push.rocks/qenv": "^6.0.5",
|
"@push.rocks/qenv": "^6.1.3",
|
||||||
"@push.rocks/tapbundle": "^5.6.3",
|
"@push.rocks/tapbundle": "^6.0.3",
|
||||||
"@types/node": "^22.15.2"
|
"@types/node": "^22.15.2"
|
||||||
},
|
},
|
||||||
"files": [
|
"files": [
|
||||||
|
|||||||
6312
pnpm-lock.yaml
generated
6312
pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,80 @@
|
|||||||
|
# Project Memory - Smartdata
|
||||||
|
|
||||||
|
## TC39 Decorator Migration (v6.0.0) - ✅ COMPLETED
|
||||||
|
|
||||||
|
### Final Status: All Tests Passing (157/157)
|
||||||
|
Migration successfully completed on 2025-11-17.
|
||||||
|
|
||||||
|
### What Changed:
|
||||||
|
- ✅ Removed `experimentalDecorators` from tsconfig.json
|
||||||
|
- ✅ Refactored all 7 decorators to TC39 Stage 3 syntax
|
||||||
|
- 5 property decorators: @globalSvDb, @svDb, @unI, @index, @searchable
|
||||||
|
- 2 class decorators: @Collection, @managed
|
||||||
|
- ✅ Implemented context.metadata pattern for shared decorator state
|
||||||
|
- ✅ All tests passing across Node.js and Deno runtimes
|
||||||
|
|
||||||
|
### Critical Discovery: TC39 Metadata Access Pattern
|
||||||
|
**THE KEY INSIGHT**: In TC39 decorators, metadata is NOT accessed via `constructor[Symbol.metadata]`. Instead:
|
||||||
|
- **Field decorators**: Write to `context.metadata`
|
||||||
|
- **Class decorators**: Read from `context.metadata` (same shared object!)
|
||||||
|
- The `context.metadata` object is shared between all decorators on the same class
|
||||||
|
- Attempting to write to `constructor[Symbol.metadata]` throws: "Cannot assign to read only property"
|
||||||
|
|
||||||
|
### Implementation Pattern:
|
||||||
|
```typescript
|
||||||
|
// Field decorator - stores metadata
|
||||||
|
export function svDb() {
|
||||||
|
return (value: undefined, context: ClassFieldDecoratorContext) => {
|
||||||
|
const metadata = context.metadata as ISmartdataDecoratorMetadata;
|
||||||
|
if (!metadata.saveableProperties) {
|
||||||
|
metadata.saveableProperties = [];
|
||||||
|
}
|
||||||
|
metadata.saveableProperties.push(String(context.name));
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Class decorator - reads metadata and initializes prototype
|
||||||
|
export function Collection(dbArg: SmartdataDb) {
|
||||||
|
return function(value: Function, context: ClassDecoratorContext) => {
|
||||||
|
const metadata = context.metadata as ISmartdataDecoratorMetadata;
|
||||||
|
if (metadata?.saveableProperties) {
|
||||||
|
decoratedClass.prototype.saveableProperties = [...metadata.saveableProperties];
|
||||||
|
}
|
||||||
|
return decoratedClass;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Runtime Compatibility:
|
||||||
|
- ✅ **Node.js v23.8.0**: Full TC39 support
|
||||||
|
- ✅ **Deno v2.5.4**: Full TC39 support
|
||||||
|
- ❌ **Bun v1.3.0**: No TC39 support (uses legacy decorators only)
|
||||||
|
- Removed "+bun" from test filenames to skip Bun tests
|
||||||
|
|
||||||
|
### Key Technical Notes:
|
||||||
|
1. **Metadata Initialization Timing**: Class decorators run AFTER field decorators, allowing them to read accumulated metadata and initialize prototypes before any instances are created
|
||||||
|
2. **Prototype vs Instance Properties**: Properties set on prototype are accessible via `this.propertyName` in instances
|
||||||
|
3. **TypeScript Lib Support**: TypeScript 5.9.3 includes built-in decorator types (no custom lib configuration needed)
|
||||||
|
4. **Interface Naming**: Used `ISmartdataDecoratorMetadata` extending `DecoratorMetadataObject` for type safety
|
||||||
|
|
||||||
|
### Files Modified:
|
||||||
|
- ts/classes.doc.ts (property decorators + metadata interface)
|
||||||
|
- ts/classes.collection.ts (class decorators + prototype initialization)
|
||||||
|
- tsconfig.json (removed experimentalDecorators flag)
|
||||||
|
- test/*.ts (renamed files to remove "+bun" suffix)
|
||||||
|
|
||||||
|
### Test Results:
|
||||||
|
All 157 tests passing across 10 test files:
|
||||||
|
- test.cursor.ts: 7/7
|
||||||
|
- test.deno.ts: 11/11 (queries working correctly!)
|
||||||
|
- test.search.advanced.ts: 41/41
|
||||||
|
- test.typescript.ts: 4/4
|
||||||
|
- test.watch.ts: 5/5
|
||||||
|
- And 5 more test files
|
||||||
|
|
||||||
|
### Migration Learnings for Future Reference:
|
||||||
|
1. `context.metadata` is the ONLY way to share state between decorators
|
||||||
|
2. Class decorators must initialize prototypes from metadata immediately
|
||||||
|
3. `Symbol.metadata` on constructors is read-only (managed by runtime)
|
||||||
|
4. Field decorators run before class decorators (guaranteed order)
|
||||||
|
5. TypeScript 5.2+ has built-in TC39 decorator support
|
||||||
|
|||||||
255
test/test.deno.ts
Normal file
255
test/test.deno.ts
Normal file
@@ -0,0 +1,255 @@
|
|||||||
|
// TODO: Decorator support during testing for bun and deno in @git.zone/tstest
|
||||||
|
|
||||||
|
import { tap, expect } from '@push.rocks/tapbundle';
|
||||||
|
import { Qenv } from '@push.rocks/qenv';
|
||||||
|
import * as smartmongo from '@push.rocks/smartmongo';
|
||||||
|
import { smartunique } from '../ts/plugins.js';
|
||||||
|
|
||||||
|
import * as mongodb from 'mongodb';
|
||||||
|
|
||||||
|
const testQenv = new Qenv(process.cwd(), process.cwd() + '/.nogit/');
|
||||||
|
|
||||||
|
console.log(process.memoryUsage());
|
||||||
|
|
||||||
|
// the tested module
|
||||||
|
import * as smartdata from '../ts/index.js';
|
||||||
|
|
||||||
|
// =======================================
|
||||||
|
// Connecting to the database server
|
||||||
|
// =======================================
|
||||||
|
|
||||||
|
let smartmongoInstance: smartmongo.SmartMongo;
|
||||||
|
let testDb: smartdata.SmartdataDb;
|
||||||
|
|
||||||
|
const totalCars = 2000;
|
||||||
|
|
||||||
|
tap.test('should create a testinstance as database', async () => {
|
||||||
|
const databaseName = `test-smartdata-deno-${smartunique.shortId()}`;
|
||||||
|
testDb = new smartdata.SmartdataDb({
|
||||||
|
mongoDbUrl: await testQenv.getEnvVarOnDemand('MONGODB_URL'),
|
||||||
|
mongoDbName: databaseName,
|
||||||
|
});
|
||||||
|
await testDb.init();
|
||||||
|
});
|
||||||
|
|
||||||
|
// =======================================
|
||||||
|
// The actual tests
|
||||||
|
// =======================================
|
||||||
|
|
||||||
|
// ------
|
||||||
|
// Collections
|
||||||
|
// ------
|
||||||
|
|
||||||
|
@smartdata.Collection(() => {
|
||||||
|
return testDb;
|
||||||
|
})
|
||||||
|
class Car extends smartdata.SmartDataDbDoc<Car, Car> {
|
||||||
|
@smartdata.unI()
|
||||||
|
public index: string = smartunique.shortId();
|
||||||
|
|
||||||
|
@smartdata.svDb()
|
||||||
|
public color: string;
|
||||||
|
|
||||||
|
@smartdata.svDb()
|
||||||
|
public brand: string;
|
||||||
|
|
||||||
|
@smartdata.svDb()
|
||||||
|
public testBuffer = Buffer.from('hello');
|
||||||
|
|
||||||
|
@smartdata.svDb()
|
||||||
|
deepData = {
|
||||||
|
sodeep: 'yes',
|
||||||
|
};
|
||||||
|
|
||||||
|
constructor(colorArg: string, brandArg: string) {
|
||||||
|
super();
|
||||||
|
this.color = colorArg;
|
||||||
|
this.brand = brandArg;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
tap.test('should create a new id', async () => {
|
||||||
|
const newid = await Car.getNewId();
|
||||||
|
console.log(newid);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should save the car to the db', async (toolsArg) => {
|
||||||
|
const myCar = new Car('red', 'Volvo');
|
||||||
|
console.log('Car.collection.smartdataDb:', (Car.collection as any).smartdataDb?.mongoDb?.databaseName);
|
||||||
|
console.log('Car.collection.collectionName:', (Car.collection as any).collectionName);
|
||||||
|
console.log('testDb.mongoDb.databaseName:', testDb.mongoDb.databaseName);
|
||||||
|
await myCar.save();
|
||||||
|
|
||||||
|
const myCar2 = new Car('red', 'Volvo');
|
||||||
|
await myCar2.save();
|
||||||
|
|
||||||
|
let counter = 0;
|
||||||
|
|
||||||
|
const gottenCarInstance = await Car.getInstance({});
|
||||||
|
console.log(gottenCarInstance.testBuffer instanceof mongodb.Binary);
|
||||||
|
process.memoryUsage();
|
||||||
|
do {
|
||||||
|
const myCar3 = new Car('red', 'Renault');
|
||||||
|
await myCar3.save();
|
||||||
|
counter++;
|
||||||
|
if (counter % 100 === 0) {
|
||||||
|
console.log(
|
||||||
|
`Filled database with ${counter} of ${totalCars} Cars and memory usage ${
|
||||||
|
process.memoryUsage().rss / 1e6
|
||||||
|
} MB`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} while (counter < totalCars);
|
||||||
|
console.log(process.memoryUsage());
|
||||||
|
|
||||||
|
// DEBUG: Check what's actually in the database
|
||||||
|
const savedCount = await Car.getCount({});
|
||||||
|
console.log('Total cars saved in DB:', savedCount);
|
||||||
|
const renaultCount = await Car.getCount({ brand: 'Renault' });
|
||||||
|
console.log('Renault cars in DB:', renaultCount);
|
||||||
|
|
||||||
|
// Check what's actually in the first saved car
|
||||||
|
const firstCar = await Car.getInstance({});
|
||||||
|
console.log('First car data:', JSON.stringify({
|
||||||
|
color: firstCar?.color,
|
||||||
|
brand: firstCar?.brand,
|
||||||
|
index: firstCar?.index
|
||||||
|
}));
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('expect to get instance of Car with shallow match', async () => {
|
||||||
|
console.log('Before query - testDb.mongoDb.databaseName:', testDb.mongoDb.databaseName);
|
||||||
|
console.log('Before query - Car.collection.smartdataDb:', (Car.collection as any).smartdataDb?.mongoDb?.databaseName);
|
||||||
|
console.log('Before query - Car.collection.collectionName:', (Car.collection as any).collectionName);
|
||||||
|
|
||||||
|
const totalQueryCycles = totalCars / 2;
|
||||||
|
let counter = 0;
|
||||||
|
do {
|
||||||
|
const timeStart = Date.now();
|
||||||
|
const myCars = await Car.getInstances({
|
||||||
|
brand: 'Renault',
|
||||||
|
});
|
||||||
|
if (counter % 10 === 0) {
|
||||||
|
console.log(
|
||||||
|
`performed ${counter} of ${totalQueryCycles} total query cycles: took ${
|
||||||
|
Date.now() - timeStart
|
||||||
|
}ms to query a set of 2000 with memory footprint ${process.memoryUsage().rss / 1e6} MB`,
|
||||||
|
);
|
||||||
|
console.log('myCars.length:', myCars.length);
|
||||||
|
console.log('myCars[0]:', myCars[0]);
|
||||||
|
}
|
||||||
|
expect(myCars[0].deepData.sodeep).toEqual('yes');
|
||||||
|
expect(myCars[0].brand).toEqual('Renault');
|
||||||
|
counter++;
|
||||||
|
} while (counter < totalQueryCycles);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('expect to get instance of Car with deep match', async () => {
|
||||||
|
const totalQueryCycles = totalCars / 6;
|
||||||
|
let counter = 0;
|
||||||
|
do {
|
||||||
|
const timeStart = Date.now();
|
||||||
|
const myCars2 = await Car.getInstances({
|
||||||
|
deepData: {
|
||||||
|
sodeep: 'yes',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
if (counter % 10 === 0) {
|
||||||
|
console.log(
|
||||||
|
`performed ${counter} of ${totalQueryCycles} total query cycles: took ${
|
||||||
|
Date.now() - timeStart
|
||||||
|
}ms to deep query a set of 2000 with memory footprint ${process.memoryUsage().rss / 1e6} MB`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
expect(myCars2[0].deepData.sodeep).toEqual('yes');
|
||||||
|
expect(myCars2[0].brand).toEqual('Volvo');
|
||||||
|
counter++;
|
||||||
|
} while (counter < totalQueryCycles);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('expect to get instance of Car and update it', async () => {
|
||||||
|
const myCar = await Car.getInstance<Car>({
|
||||||
|
brand: 'Volvo',
|
||||||
|
});
|
||||||
|
expect(myCar.color).toEqual('red');
|
||||||
|
myCar.color = 'blue';
|
||||||
|
await myCar.save();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should be able to delete an instance of car', async () => {
|
||||||
|
const myCars = await Car.getInstances({
|
||||||
|
brand: 'Volvo',
|
||||||
|
color: 'blue',
|
||||||
|
});
|
||||||
|
console.log(myCars);
|
||||||
|
expect(myCars[0].color).toEqual('blue');
|
||||||
|
for (const myCar of myCars) {
|
||||||
|
await myCar.delete();
|
||||||
|
}
|
||||||
|
|
||||||
|
const myCar2 = await Car.getInstance<Car>({
|
||||||
|
brand: 'Volvo',
|
||||||
|
});
|
||||||
|
expect(myCar2.color).toEqual('red');
|
||||||
|
});
|
||||||
|
|
||||||
|
// tslint:disable-next-line: max-classes-per-file
|
||||||
|
@smartdata.Collection(() => {
|
||||||
|
return testDb;
|
||||||
|
})
|
||||||
|
class Truck extends smartdata.SmartDataDbDoc<Car, Car> {
|
||||||
|
@smartdata.unI()
|
||||||
|
public id: string = smartunique.shortId();
|
||||||
|
|
||||||
|
@smartdata.svDb()
|
||||||
|
public color: string;
|
||||||
|
|
||||||
|
@smartdata.svDb()
|
||||||
|
public brand: string;
|
||||||
|
|
||||||
|
constructor(colorArg: string, brandArg: string) {
|
||||||
|
super();
|
||||||
|
this.color = colorArg;
|
||||||
|
this.brand = brandArg;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
tap.test('should store a new Truck', async () => {
|
||||||
|
const truck = new Truck('blue', 'MAN');
|
||||||
|
await truck.save();
|
||||||
|
const myTruck2 = await Truck.getInstance({ color: 'blue' });
|
||||||
|
expect(myTruck2.color).toEqual('blue');
|
||||||
|
myTruck2.color = 'red';
|
||||||
|
await myTruck2.save();
|
||||||
|
const myTruck3 = await Truck.getInstance({ color: 'blue' });
|
||||||
|
expect(myTruck3).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should return a count', async () => {
|
||||||
|
const truckCount = await Truck.getCount();
|
||||||
|
expect(truckCount).toEqual(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should use a cursor', async () => {
|
||||||
|
const cursor = await Car.getCursor({});
|
||||||
|
let counter = 0;
|
||||||
|
await cursor.forEach(async (carArg) => {
|
||||||
|
counter++;
|
||||||
|
counter % 50 === 0 ? console.log(`50 more of ${carArg.color}`) : null;
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// =======================================
|
||||||
|
// close the database connection
|
||||||
|
// =======================================
|
||||||
|
tap.test('close', async () => {
|
||||||
|
if (smartmongoInstance) {
|
||||||
|
await smartmongoInstance.stopAndDumpToDir('./.nogit/dbdump/test.ts');
|
||||||
|
} else {
|
||||||
|
await testDb.mongoDb.dropDatabase();
|
||||||
|
await testDb.close();
|
||||||
|
}
|
||||||
|
setTimeout(() => process.exit(), 2000);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.start({ throwOnError: true });
|
||||||
819
test/test.filters.ts
Normal file
819
test/test.filters.ts
Normal file
@@ -0,0 +1,819 @@
|
|||||||
|
import { tap, expect } from '@git.zone/tstest/tapbundle';
|
||||||
|
import * as smartmongo from '@push.rocks/smartmongo';
|
||||||
|
import * as smartunique from '@push.rocks/smartunique';
|
||||||
|
import * as smartdata from '../ts/index.js';
|
||||||
|
|
||||||
|
const { SmartdataDb, Collection, svDb, unI, index } = smartdata;
|
||||||
|
|
||||||
|
let smartmongoInstance: smartmongo.SmartMongo;
|
||||||
|
let testDb: smartdata.SmartdataDb;
|
||||||
|
|
||||||
|
// Define test document classes
|
||||||
|
@Collection(() => testDb)
|
||||||
|
class TestUser extends smartdata.SmartDataDbDoc<TestUser, TestUser> {
|
||||||
|
@unI()
|
||||||
|
public id: string = smartunique.shortId();
|
||||||
|
|
||||||
|
@svDb()
|
||||||
|
public name: string;
|
||||||
|
|
||||||
|
@svDb()
|
||||||
|
public age: number;
|
||||||
|
|
||||||
|
@svDb()
|
||||||
|
public email: string;
|
||||||
|
|
||||||
|
@svDb()
|
||||||
|
public roles: string[];
|
||||||
|
|
||||||
|
@svDb()
|
||||||
|
public tags: string[];
|
||||||
|
|
||||||
|
@svDb()
|
||||||
|
public status: 'active' | 'inactive' | 'pending';
|
||||||
|
|
||||||
|
@svDb()
|
||||||
|
public metadata: {
|
||||||
|
lastLogin?: Date;
|
||||||
|
loginCount?: number;
|
||||||
|
preferences?: Record<string, any>;
|
||||||
|
};
|
||||||
|
|
||||||
|
@svDb()
|
||||||
|
public scores: number[];
|
||||||
|
|
||||||
|
constructor(data: Partial<TestUser> = {}) {
|
||||||
|
super();
|
||||||
|
Object.assign(this, data);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Collection(() => testDb)
|
||||||
|
class TestOrder extends smartdata.SmartDataDbDoc<TestOrder, TestOrder> {
|
||||||
|
@unI()
|
||||||
|
public id: string = smartunique.shortId();
|
||||||
|
|
||||||
|
@svDb()
|
||||||
|
public userId: string;
|
||||||
|
|
||||||
|
@svDb()
|
||||||
|
public items: Array<{
|
||||||
|
product: string;
|
||||||
|
quantity: number;
|
||||||
|
price: number;
|
||||||
|
}>;
|
||||||
|
|
||||||
|
@svDb()
|
||||||
|
public totalAmount: number;
|
||||||
|
|
||||||
|
@svDb()
|
||||||
|
public status: string;
|
||||||
|
|
||||||
|
@svDb()
|
||||||
|
public tags: string[];
|
||||||
|
|
||||||
|
constructor(data: Partial<TestOrder> = {}) {
|
||||||
|
super();
|
||||||
|
Object.assign(this, data);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Setup and teardown
|
||||||
|
tap.test('should create a test database instance', async () => {
|
||||||
|
smartmongoInstance = await smartmongo.SmartMongo.createAndStart();
|
||||||
|
testDb = new smartdata.SmartdataDb(await smartmongoInstance.getMongoDescriptor());
|
||||||
|
await testDb.init();
|
||||||
|
expect(testDb).toBeInstanceOf(SmartdataDb);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should create test data', async () => {
|
||||||
|
// Create test users
|
||||||
|
const users = [
|
||||||
|
new TestUser({
|
||||||
|
name: 'John Doe',
|
||||||
|
age: 30,
|
||||||
|
email: 'john@example.com',
|
||||||
|
roles: ['admin', 'user'],
|
||||||
|
tags: ['javascript', 'nodejs', 'mongodb'],
|
||||||
|
status: 'active',
|
||||||
|
metadata: { loginCount: 5, lastLogin: new Date() },
|
||||||
|
scores: [85, 90, 78]
|
||||||
|
}),
|
||||||
|
new TestUser({
|
||||||
|
name: 'Jane Smith',
|
||||||
|
age: 25,
|
||||||
|
email: 'jane@example.com',
|
||||||
|
roles: ['user'],
|
||||||
|
tags: ['python', 'mongodb'],
|
||||||
|
status: 'active',
|
||||||
|
metadata: { loginCount: 3 },
|
||||||
|
scores: [92, 88, 95]
|
||||||
|
}),
|
||||||
|
new TestUser({
|
||||||
|
name: 'Bob Johnson',
|
||||||
|
age: 35,
|
||||||
|
email: 'bob@example.com',
|
||||||
|
roles: ['moderator', 'user'],
|
||||||
|
tags: ['javascript', 'react', 'nodejs'],
|
||||||
|
status: 'inactive',
|
||||||
|
metadata: { loginCount: 0 },
|
||||||
|
scores: [70, 75, 80]
|
||||||
|
}),
|
||||||
|
new TestUser({
|
||||||
|
name: 'Alice Brown',
|
||||||
|
age: 28,
|
||||||
|
email: 'alice@example.com',
|
||||||
|
roles: ['admin'],
|
||||||
|
tags: ['typescript', 'angular', 'mongodb'],
|
||||||
|
status: 'active',
|
||||||
|
metadata: { loginCount: 10 },
|
||||||
|
scores: [95, 98, 100]
|
||||||
|
}),
|
||||||
|
new TestUser({
|
||||||
|
name: 'Charlie Wilson',
|
||||||
|
age: 22,
|
||||||
|
email: 'charlie@example.com',
|
||||||
|
roles: ['user'],
|
||||||
|
tags: ['golang', 'kubernetes'],
|
||||||
|
status: 'pending',
|
||||||
|
metadata: { loginCount: 1 },
|
||||||
|
scores: [60, 65]
|
||||||
|
})
|
||||||
|
];
|
||||||
|
|
||||||
|
for (const user of users) {
|
||||||
|
await user.save();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create test orders
|
||||||
|
const orders = [
|
||||||
|
new TestOrder({
|
||||||
|
userId: users[0].id,
|
||||||
|
items: [
|
||||||
|
{ product: 'laptop', quantity: 1, price: 1200 },
|
||||||
|
{ product: 'mouse', quantity: 2, price: 25 }
|
||||||
|
],
|
||||||
|
totalAmount: 1250,
|
||||||
|
status: 'completed',
|
||||||
|
tags: ['electronics', 'priority']
|
||||||
|
}),
|
||||||
|
new TestOrder({
|
||||||
|
userId: users[1].id,
|
||||||
|
items: [
|
||||||
|
{ product: 'book', quantity: 3, price: 15 },
|
||||||
|
{ product: 'pen', quantity: 5, price: 2 }
|
||||||
|
],
|
||||||
|
totalAmount: 55,
|
||||||
|
status: 'pending',
|
||||||
|
tags: ['stationery']
|
||||||
|
}),
|
||||||
|
new TestOrder({
|
||||||
|
userId: users[0].id,
|
||||||
|
items: [
|
||||||
|
{ product: 'laptop', quantity: 2, price: 1200 },
|
||||||
|
{ product: 'keyboard', quantity: 2, price: 80 }
|
||||||
|
],
|
||||||
|
totalAmount: 2560,
|
||||||
|
status: 'processing',
|
||||||
|
tags: ['electronics', 'bulk']
|
||||||
|
})
|
||||||
|
];
|
||||||
|
|
||||||
|
for (const order of orders) {
|
||||||
|
await order.save();
|
||||||
|
}
|
||||||
|
|
||||||
|
const savedUsers = await TestUser.getInstances({});
|
||||||
|
const savedOrders = await TestOrder.getInstances({});
|
||||||
|
expect(savedUsers.length).toEqual(5);
|
||||||
|
expect(savedOrders.length).toEqual(3);
|
||||||
|
});
|
||||||
|
|
||||||
|
// ============= BASIC FILTER TESTS =============
|
||||||
|
tap.test('should filter by simple equality', async () => {
|
||||||
|
const users = await TestUser.getInstances({ name: 'John Doe' });
|
||||||
|
expect(users.length).toEqual(1);
|
||||||
|
expect(users[0].name).toEqual('John Doe');
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should filter by multiple fields (implicit AND)', async () => {
|
||||||
|
const users = await TestUser.getInstances({
|
||||||
|
status: 'active',
|
||||||
|
age: 30
|
||||||
|
});
|
||||||
|
expect(users.length).toEqual(1);
|
||||||
|
expect(users[0].name).toEqual('John Doe');
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should filter by nested object fields', async () => {
|
||||||
|
const users = await TestUser.getInstances({
|
||||||
|
'metadata.loginCount': 5
|
||||||
|
});
|
||||||
|
expect(users.length).toEqual(1);
|
||||||
|
expect(users[0].name).toEqual('John Doe');
|
||||||
|
});
|
||||||
|
|
||||||
|
// ============= COMPREHENSIVE NESTED FILTER TESTS =============
|
||||||
|
tap.test('should filter by nested object with direct object syntax', async () => {
|
||||||
|
// Direct nested object matching (exact match)
|
||||||
|
const users = await TestUser.getInstances({
|
||||||
|
metadata: {
|
||||||
|
loginCount: 5,
|
||||||
|
lastLogin: (await TestUser.getInstances({}))[0].metadata.lastLogin // Get the exact date
|
||||||
|
}
|
||||||
|
});
|
||||||
|
expect(users.length).toEqual(1);
|
||||||
|
expect(users[0].name).toEqual('John Doe');
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should filter by partial nested object match', async () => {
|
||||||
|
// When using object syntax, only specified fields must match
|
||||||
|
const users = await TestUser.getInstances({
|
||||||
|
metadata: { loginCount: 5 } // Only checks loginCount, ignores other fields
|
||||||
|
});
|
||||||
|
expect(users.length).toEqual(1);
|
||||||
|
expect(users[0].name).toEqual('John Doe');
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should combine nested object and dot notation', async () => {
|
||||||
|
const users = await TestUser.getInstances({
|
||||||
|
metadata: { loginCount: { $gte: 3 } }, // Object syntax with operator
|
||||||
|
'metadata.loginCount': { $lte: 10 } // Dot notation with operator
|
||||||
|
});
|
||||||
|
expect(users.length).toEqual(3); // Jane (3), John (5), and Alice (10) have loginCount between 3-10
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should filter nested fields with operators using dot notation', async () => {
|
||||||
|
const users = await TestUser.getInstances({
|
||||||
|
'metadata.loginCount': { $gte: 5 }
|
||||||
|
});
|
||||||
|
expect(users.length).toEqual(2); // John (5) and Alice (10)
|
||||||
|
const names = users.map(u => u.name).sort();
|
||||||
|
expect(names).toEqual(['Alice Brown', 'John Doe']);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should filter nested fields with multiple operators', async () => {
|
||||||
|
const users = await TestUser.getInstances({
|
||||||
|
'metadata.loginCount': { $gte: 3, $lt: 10 }
|
||||||
|
});
|
||||||
|
expect(users.length).toEqual(2); // Jane (3) and John (5)
|
||||||
|
const names = users.map(u => u.name).sort();
|
||||||
|
expect(names).toEqual(['Jane Smith', 'John Doe']);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should handle deeply nested object structures', async () => {
|
||||||
|
// First, create a user with deep nesting in preferences
|
||||||
|
const deepUser = new TestUser({
|
||||||
|
name: 'Deep Nester',
|
||||||
|
age: 40,
|
||||||
|
email: 'deep@example.com',
|
||||||
|
roles: ['admin'],
|
||||||
|
tags: [],
|
||||||
|
status: 'active',
|
||||||
|
metadata: {
|
||||||
|
loginCount: 1,
|
||||||
|
preferences: {
|
||||||
|
theme: {
|
||||||
|
colors: {
|
||||||
|
primary: '#000000',
|
||||||
|
secondary: '#ffffff'
|
||||||
|
},
|
||||||
|
fonts: {
|
||||||
|
heading: 'Arial',
|
||||||
|
body: 'Helvetica'
|
||||||
|
}
|
||||||
|
},
|
||||||
|
notifications: {
|
||||||
|
email: true,
|
||||||
|
push: false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
scores: []
|
||||||
|
});
|
||||||
|
await deepUser.save();
|
||||||
|
|
||||||
|
// Test deep nesting with dot notation
|
||||||
|
const deepResults = await TestUser.getInstances({
|
||||||
|
'metadata.preferences.theme.colors.primary': '#000000'
|
||||||
|
});
|
||||||
|
expect(deepResults.length).toEqual(1);
|
||||||
|
expect(deepResults[0].name).toEqual('Deep Nester');
|
||||||
|
|
||||||
|
// Test deep nesting with operators
|
||||||
|
const boolResults = await TestUser.getInstances({
|
||||||
|
'metadata.preferences.notifications.email': { $eq: true }
|
||||||
|
});
|
||||||
|
expect(boolResults.length).toEqual(1);
|
||||||
|
expect(boolResults[0].name).toEqual('Deep Nester');
|
||||||
|
|
||||||
|
// Clean up
|
||||||
|
await deepUser.delete();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should filter arrays of nested objects using $elemMatch', async () => {
|
||||||
|
const orders = await TestOrder.getInstances({
|
||||||
|
items: {
|
||||||
|
$elemMatch: {
|
||||||
|
product: 'laptop',
|
||||||
|
price: { $gte: 1000 }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
expect(orders.length).toEqual(2); // Both laptop orders have price >= 1000
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should filter nested arrays with dot notation', async () => {
|
||||||
|
// Query for any order that has an item with specific product
|
||||||
|
const orders = await TestOrder.getInstances({
|
||||||
|
'items.product': 'laptop'
|
||||||
|
});
|
||||||
|
expect(orders.length).toEqual(2); // Two orders contain laptops
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should combine nested object filters with logical operators', async () => {
|
||||||
|
const users = await TestUser.getInstances({
|
||||||
|
$or: [
|
||||||
|
{ 'metadata.loginCount': { $gte: 10 } }, // Alice has 10
|
||||||
|
{
|
||||||
|
$and: [
|
||||||
|
{ 'metadata.loginCount': { $lt: 5 } }, // Jane has 3, Bob has 0, Charlie has 1
|
||||||
|
{ status: 'active' } // Jane is active, Bob is inactive, Charlie is pending
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
});
|
||||||
|
expect(users.length).toEqual(2); // Alice (loginCount >= 10), Jane (loginCount < 5 AND active)
|
||||||
|
const names = users.map(u => u.name).sort();
|
||||||
|
expect(names).toEqual(['Alice Brown', 'Jane Smith']);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should handle null and undefined in nested fields', async () => {
|
||||||
|
// Users without lastLogin
|
||||||
|
const noLastLogin = await TestUser.getInstances({
|
||||||
|
'metadata.lastLogin': { $exists: false }
|
||||||
|
});
|
||||||
|
expect(noLastLogin.length).toEqual(4); // Everyone except John
|
||||||
|
|
||||||
|
// Users with preferences (none have it set)
|
||||||
|
const withPreferences = await TestUser.getInstances({
|
||||||
|
'metadata.preferences': { $exists: true }
|
||||||
|
});
|
||||||
|
expect(withPreferences.length).toEqual(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should filter nested arrays by size', async () => {
|
||||||
|
// Create an order with specific number of items
|
||||||
|
const multiItemOrder = new TestOrder({
|
||||||
|
userId: 'test-user',
|
||||||
|
items: [
|
||||||
|
{ product: 'item1', quantity: 1, price: 10 },
|
||||||
|
{ product: 'item2', quantity: 2, price: 20 },
|
||||||
|
{ product: 'item3', quantity: 3, price: 30 },
|
||||||
|
{ product: 'item4', quantity: 4, price: 40 }
|
||||||
|
],
|
||||||
|
totalAmount: 100,
|
||||||
|
status: 'pending',
|
||||||
|
tags: ['test']
|
||||||
|
});
|
||||||
|
await multiItemOrder.save();
|
||||||
|
|
||||||
|
const fourItemOrders = await TestOrder.getInstances({
|
||||||
|
items: { $size: 4 }
|
||||||
|
});
|
||||||
|
expect(fourItemOrders.length).toEqual(1);
|
||||||
|
|
||||||
|
// Clean up
|
||||||
|
await multiItemOrder.delete();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should handle nested field comparison between documents', async () => {
|
||||||
|
// Find users where loginCount equals their age divided by 6 (John: 30/6=5)
|
||||||
|
const users = await TestUser.getInstances({
|
||||||
|
$and: [
|
||||||
|
{ 'metadata.loginCount': 5 },
|
||||||
|
{ age: 30 }
|
||||||
|
]
|
||||||
|
});
|
||||||
|
expect(users.length).toEqual(1);
|
||||||
|
expect(users[0].name).toEqual('John Doe');
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should filter using $in on nested fields', async () => {
|
||||||
|
const users = await TestUser.getInstances({
|
||||||
|
'metadata.loginCount': { $in: [0, 1, 5] }
|
||||||
|
});
|
||||||
|
expect(users.length).toEqual(3); // Bob (0), Charlie (1), John (5)
|
||||||
|
const names = users.map(u => u.name).sort();
|
||||||
|
expect(names).toEqual(['Bob Johnson', 'Charlie Wilson', 'John Doe']);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should filter nested arrays with $all', async () => {
|
||||||
|
// Create an order with multiple tags
|
||||||
|
const taggedOrder = new TestOrder({
|
||||||
|
userId: 'test-user',
|
||||||
|
items: [{ product: 'test', quantity: 1, price: 10 }],
|
||||||
|
totalAmount: 10,
|
||||||
|
status: 'completed',
|
||||||
|
tags: ['urgent', 'priority', 'electronics']
|
||||||
|
});
|
||||||
|
await taggedOrder.save();
|
||||||
|
|
||||||
|
const priorityElectronics = await TestOrder.getInstances({
|
||||||
|
tags: { $all: ['priority', 'electronics'] }
|
||||||
|
});
|
||||||
|
expect(priorityElectronics.length).toEqual(2); // Original order and new one
|
||||||
|
|
||||||
|
// Clean up
|
||||||
|
await taggedOrder.delete();
|
||||||
|
});
|
||||||
|
|
||||||
|
// ============= COMPARISON OPERATOR TESTS =============
|
||||||
|
tap.test('should filter using $gt operator', async () => {
|
||||||
|
const users = await TestUser.getInstances({
|
||||||
|
age: { $gt: 30 }
|
||||||
|
});
|
||||||
|
expect(users.length).toEqual(1);
|
||||||
|
expect(users[0].name).toEqual('Bob Johnson');
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should filter using $gte operator', async () => {
|
||||||
|
const users = await TestUser.getInstances({
|
||||||
|
age: { $gte: 30 }
|
||||||
|
});
|
||||||
|
expect(users.length).toEqual(2);
|
||||||
|
const names = users.map(u => u.name).sort();
|
||||||
|
expect(names).toEqual(['Bob Johnson', 'John Doe']);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should filter using $lt operator', async () => {
|
||||||
|
const users = await TestUser.getInstances({
|
||||||
|
age: { $lt: 25 }
|
||||||
|
});
|
||||||
|
expect(users.length).toEqual(1);
|
||||||
|
expect(users[0].name).toEqual('Charlie Wilson');
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should filter using $lte operator', async () => {
|
||||||
|
const users = await TestUser.getInstances({
|
||||||
|
age: { $lte: 25 }
|
||||||
|
});
|
||||||
|
expect(users.length).toEqual(2);
|
||||||
|
const names = users.map(u => u.name).sort();
|
||||||
|
expect(names).toEqual(['Charlie Wilson', 'Jane Smith']);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should filter using $ne operator', async () => {
|
||||||
|
const users = await TestUser.getInstances({
|
||||||
|
status: { $ne: 'active' }
|
||||||
|
});
|
||||||
|
expect(users.length).toEqual(2);
|
||||||
|
const statuses = users.map(u => u.status).sort();
|
||||||
|
expect(statuses).toEqual(['inactive', 'pending']);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should filter using multiple comparison operators', async () => {
|
||||||
|
const users = await TestUser.getInstances({
|
||||||
|
age: { $gte: 25, $lt: 30 }
|
||||||
|
});
|
||||||
|
expect(users.length).toEqual(2);
|
||||||
|
const names = users.map(u => u.name).sort();
|
||||||
|
expect(names).toEqual(['Alice Brown', 'Jane Smith']);
|
||||||
|
});
|
||||||
|
|
||||||
|
// ============= ARRAY OPERATOR TESTS =============
|
||||||
|
tap.test('should filter using $in operator', async () => {
|
||||||
|
const users = await TestUser.getInstances({
|
||||||
|
status: { $in: ['active', 'pending'] }
|
||||||
|
});
|
||||||
|
expect(users.length).toEqual(4);
|
||||||
|
expect(users.every(u => ['active', 'pending'].includes(u.status))).toEqual(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should filter arrays using $in operator', async () => {
|
||||||
|
const users = await TestUser.getInstances({
|
||||||
|
roles: { $in: ['admin'] }
|
||||||
|
});
|
||||||
|
expect(users.length).toEqual(2);
|
||||||
|
const names = users.map(u => u.name).sort();
|
||||||
|
expect(names).toEqual(['Alice Brown', 'John Doe']);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should filter using $nin operator', async () => {
|
||||||
|
const users = await TestUser.getInstances({
|
||||||
|
status: { $nin: ['inactive', 'pending'] }
|
||||||
|
});
|
||||||
|
expect(users.length).toEqual(3);
|
||||||
|
expect(users.every(u => u.status === 'active')).toEqual(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should filter arrays using $all operator', async () => {
|
||||||
|
const users = await TestUser.getInstances({
|
||||||
|
tags: { $all: ['javascript', 'nodejs'] }
|
||||||
|
});
|
||||||
|
expect(users.length).toEqual(2);
|
||||||
|
const names = users.map(u => u.name).sort();
|
||||||
|
expect(names).toEqual(['Bob Johnson', 'John Doe']);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should filter arrays using $size operator', async () => {
|
||||||
|
const users = await TestUser.getInstances({
|
||||||
|
scores: { $size: 2 }
|
||||||
|
});
|
||||||
|
expect(users.length).toEqual(1);
|
||||||
|
expect(users[0].name).toEqual('Charlie Wilson');
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should filter arrays using $elemMatch operator', async () => {
|
||||||
|
const orders = await TestOrder.getInstances({
|
||||||
|
items: {
|
||||||
|
$elemMatch: {
|
||||||
|
product: 'laptop',
|
||||||
|
quantity: { $gte: 2 }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
expect(orders.length).toEqual(1);
|
||||||
|
expect(orders[0].totalAmount).toEqual(2560);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should filter using $elemMatch with single condition', async () => {
|
||||||
|
const orders = await TestOrder.getInstances({
|
||||||
|
items: {
|
||||||
|
$elemMatch: {
|
||||||
|
price: { $gt: 100 }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
expect(orders.length).toEqual(2);
|
||||||
|
expect(orders.every(o => o.items.some(i => i.price > 100))).toEqual(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
// ============= LOGICAL OPERATOR TESTS =============
|
||||||
|
tap.test('should filter using $or operator', async () => {
|
||||||
|
const users = await TestUser.getInstances({
|
||||||
|
$or: [
|
||||||
|
{ age: { $lt: 25 } },
|
||||||
|
{ status: 'inactive' }
|
||||||
|
]
|
||||||
|
});
|
||||||
|
expect(users.length).toEqual(2);
|
||||||
|
const names = users.map(u => u.name).sort();
|
||||||
|
expect(names).toEqual(['Bob Johnson', 'Charlie Wilson']);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should filter using $and operator', async () => {
|
||||||
|
const users = await TestUser.getInstances({
|
||||||
|
$and: [
|
||||||
|
{ status: 'active' },
|
||||||
|
{ age: { $gte: 28 } }
|
||||||
|
]
|
||||||
|
});
|
||||||
|
expect(users.length).toEqual(2);
|
||||||
|
const names = users.map(u => u.name).sort();
|
||||||
|
expect(names).toEqual(['Alice Brown', 'John Doe']);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should filter using $nor operator', async () => {
|
||||||
|
const users = await TestUser.getInstances({
|
||||||
|
$nor: [
|
||||||
|
{ status: 'inactive' },
|
||||||
|
{ age: { $lt: 25 } }
|
||||||
|
]
|
||||||
|
});
|
||||||
|
expect(users.length).toEqual(3);
|
||||||
|
expect(users.every(u => u.status !== 'inactive' && u.age >= 25)).toEqual(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should filter using nested logical operators', async () => {
|
||||||
|
const users = await TestUser.getInstances({
|
||||||
|
$or: [
|
||||||
|
{
|
||||||
|
$and: [
|
||||||
|
{ status: 'active' },
|
||||||
|
{ roles: { $in: ['admin'] } }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{ age: { $lt: 23 } }
|
||||||
|
]
|
||||||
|
});
|
||||||
|
expect(users.length).toEqual(3);
|
||||||
|
const names = users.map(u => u.name).sort();
|
||||||
|
expect(names).toEqual(['Alice Brown', 'Charlie Wilson', 'John Doe']);
|
||||||
|
});
|
||||||
|
|
||||||
|
// ============= ELEMENT OPERATOR TESTS =============
|
||||||
|
tap.test('should filter using $exists operator', async () => {
|
||||||
|
const users = await TestUser.getInstances({
|
||||||
|
'metadata.lastLogin': { $exists: true }
|
||||||
|
});
|
||||||
|
expect(users.length).toEqual(1);
|
||||||
|
expect(users[0].name).toEqual('John Doe');
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should filter using $exists false', async () => {
|
||||||
|
const users = await TestUser.getInstances({
|
||||||
|
'metadata.preferences': { $exists: false }
|
||||||
|
});
|
||||||
|
expect(users.length).toEqual(5);
|
||||||
|
});
|
||||||
|
|
||||||
|
// ============= COMPLEX FILTER TESTS =============
|
||||||
|
tap.test('should handle complex nested filters', async () => {
|
||||||
|
const users = await TestUser.getInstances({
|
||||||
|
$and: [
|
||||||
|
{ status: 'active' },
|
||||||
|
{
|
||||||
|
$or: [
|
||||||
|
{ age: { $gte: 30 } },
|
||||||
|
{ roles: { $all: ['admin'] } }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{ tags: { $in: ['mongodb'] } }
|
||||||
|
]
|
||||||
|
});
|
||||||
|
expect(users.length).toEqual(2);
|
||||||
|
const names = users.map(u => u.name).sort();
|
||||||
|
expect(names).toEqual(['Alice Brown', 'John Doe']);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should combine multiple operator types', async () => {
|
||||||
|
const orders = await TestOrder.getInstances({
|
||||||
|
$and: [
|
||||||
|
{ totalAmount: { $gte: 100 } },
|
||||||
|
{ status: { $in: ['completed', 'processing'] } },
|
||||||
|
{ tags: { $in: ['electronics'] } }
|
||||||
|
]
|
||||||
|
});
|
||||||
|
expect(orders.length).toEqual(2);
|
||||||
|
expect(orders.every(o => o.totalAmount >= 100)).toEqual(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
// ============= ERROR HANDLING TESTS =============
|
||||||
|
tap.test('should throw error for $where operator', async () => {
|
||||||
|
let error: Error | null = null;
|
||||||
|
try {
|
||||||
|
await TestUser.getInstances({
|
||||||
|
$where: 'this.age > 25'
|
||||||
|
});
|
||||||
|
} catch (e) {
|
||||||
|
error = e as Error;
|
||||||
|
}
|
||||||
|
expect(error).toBeTruthy();
|
||||||
|
expect(error?.message).toMatch(/\$where.*not allowed/);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should throw error for invalid $in value', async () => {
|
||||||
|
let error: Error | null = null;
|
||||||
|
try {
|
||||||
|
await TestUser.getInstances({
|
||||||
|
status: { $in: 'active' as any } // Should be an array
|
||||||
|
});
|
||||||
|
} catch (e) {
|
||||||
|
error = e as Error;
|
||||||
|
}
|
||||||
|
expect(error).toBeTruthy();
|
||||||
|
expect(error?.message).toMatch(/\$in.*requires.*array/);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should throw error for invalid $size value', async () => {
|
||||||
|
let error: Error | null = null;
|
||||||
|
try {
|
||||||
|
await TestUser.getInstances({
|
||||||
|
scores: { $size: '3' as any } // Should be a number
|
||||||
|
});
|
||||||
|
} catch (e) {
|
||||||
|
error = e as Error;
|
||||||
|
}
|
||||||
|
expect(error).toBeTruthy();
|
||||||
|
expect(error?.message).toMatch(/\$size.*requires.*numeric/);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should throw error for dots in field names', async () => {
|
||||||
|
let error: Error | null = null;
|
||||||
|
try {
|
||||||
|
await TestUser.getInstances({
|
||||||
|
'some.nested.field': { 'invalid.key': 'value' }
|
||||||
|
});
|
||||||
|
} catch (e) {
|
||||||
|
error = e as Error;
|
||||||
|
}
|
||||||
|
expect(error).toBeTruthy();
|
||||||
|
expect(error?.message).toMatch(/keys cannot contain dots/);
|
||||||
|
});
|
||||||
|
|
||||||
|
// ============= EDGE CASE TESTS =============
|
||||||
|
tap.test('should handle empty filter (return all)', async () => {
|
||||||
|
const users = await TestUser.getInstances({});
|
||||||
|
expect(users.length).toEqual(5);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should handle null values in filter', async () => {
|
||||||
|
// First, create a user with null email
|
||||||
|
const nullUser = new TestUser({
|
||||||
|
name: 'Null User',
|
||||||
|
age: 40,
|
||||||
|
email: null as any,
|
||||||
|
roles: ['user'],
|
||||||
|
tags: [],
|
||||||
|
status: 'active',
|
||||||
|
metadata: {},
|
||||||
|
scores: []
|
||||||
|
});
|
||||||
|
await nullUser.save();
|
||||||
|
|
||||||
|
const users = await TestUser.getInstances({ email: null });
|
||||||
|
expect(users.length).toEqual(1);
|
||||||
|
expect(users[0].name).toEqual('Null User');
|
||||||
|
|
||||||
|
// Clean up
|
||||||
|
await nullUser.delete();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should handle arrays as direct equality match', async () => {
|
||||||
|
// This tests that arrays without operators are treated as equality matches
|
||||||
|
const users = await TestUser.getInstances({
|
||||||
|
roles: ['user'] // Exact match for array
|
||||||
|
});
|
||||||
|
expect(users.length).toEqual(2); // Both Jane and Charlie have exactly ['user']
|
||||||
|
const names = users.map(u => u.name).sort();
|
||||||
|
expect(names).toEqual(['Charlie Wilson', 'Jane Smith']);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should handle regex operator', async () => {
|
||||||
|
const users = await TestUser.getInstances({
|
||||||
|
name: { $regex: '^J', $options: 'i' }
|
||||||
|
});
|
||||||
|
expect(users.length).toEqual(2);
|
||||||
|
const names = users.map(u => u.name).sort();
|
||||||
|
expect(names).toEqual(['Jane Smith', 'John Doe']);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should handle unknown operators by letting MongoDB reject them', async () => {
|
||||||
|
// Unknown operators should be passed through to MongoDB, which will reject them
|
||||||
|
let error: Error | null = null;
|
||||||
|
|
||||||
|
try {
|
||||||
|
await TestUser.getInstances({
|
||||||
|
age: { $unknownOp: 30 } as any
|
||||||
|
});
|
||||||
|
} catch (e) {
|
||||||
|
error = e as Error;
|
||||||
|
}
|
||||||
|
|
||||||
|
expect(error).toBeTruthy();
|
||||||
|
expect(error?.message).toMatch(/unknown operator.*\$unknownOp/);
|
||||||
|
});
|
||||||
|
|
||||||
|
// ============= PERFORMANCE TESTS =============
|
||||||
|
tap.test('should efficiently filter large result sets', async () => {
|
||||||
|
// Create many test documents
|
||||||
|
const manyUsers = [];
|
||||||
|
for (let i = 0; i < 100; i++) {
|
||||||
|
manyUsers.push(new TestUser({
|
||||||
|
name: `User ${i}`,
|
||||||
|
age: 20 + (i % 40),
|
||||||
|
email: `user${i}@example.com`,
|
||||||
|
roles: i % 3 === 0 ? ['admin'] : ['user'],
|
||||||
|
tags: i % 2 === 0 ? ['even', 'test'] : ['odd', 'test'],
|
||||||
|
status: i % 4 === 0 ? 'inactive' : 'active',
|
||||||
|
metadata: { loginCount: i },
|
||||||
|
scores: [i, i + 10, i + 20]
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Save in batches for efficiency
|
||||||
|
for (const user of manyUsers) {
|
||||||
|
await user.save();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Complex filter that should still be fast
|
||||||
|
const startTime = Date.now();
|
||||||
|
const filtered = await TestUser.getInstances({
|
||||||
|
$and: [
|
||||||
|
{ age: { $gte: 30, $lt: 40 } },
|
||||||
|
{ status: 'active' },
|
||||||
|
{ tags: { $in: ['even'] } },
|
||||||
|
{ 'metadata.loginCount': { $gte: 20 } }
|
||||||
|
]
|
||||||
|
});
|
||||||
|
const duration = Date.now() - startTime;
|
||||||
|
|
||||||
|
console.log(`Complex filter on 100+ documents took ${duration}ms`);
|
||||||
|
expect(duration).toBeLessThan(1000); // Should complete in under 1 second
|
||||||
|
expect(filtered.length).toBeGreaterThan(0);
|
||||||
|
|
||||||
|
// Clean up
|
||||||
|
for (const user of manyUsers) {
|
||||||
|
await user.delete();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// ============= CLEANUP =============
|
||||||
|
tap.test('should clean up test database', async () => {
|
||||||
|
await testDb.mongoDb.dropDatabase();
|
||||||
|
await testDb.close();
|
||||||
|
await smartmongoInstance.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
export default tap.start();
|
||||||
@@ -1,3 +1,5 @@
|
|||||||
|
// TODO: Decorator support during testing for bun and deno in @git.zone/tstest
|
||||||
|
|
||||||
import { tap, expect } from '@push.rocks/tapbundle';
|
import { tap, expect } from '@push.rocks/tapbundle';
|
||||||
import { Qenv } from '@push.rocks/qenv';
|
import { Qenv } from '@push.rocks/qenv';
|
||||||
import * as smartmongo from '@push.rocks/smartmongo';
|
import * as smartmongo from '@push.rocks/smartmongo';
|
||||||
@@ -3,6 +3,6 @@
|
|||||||
*/
|
*/
|
||||||
export const commitinfo = {
|
export const commitinfo = {
|
||||||
name: '@push.rocks/smartdata',
|
name: '@push.rocks/smartdata',
|
||||||
version: '5.16.0',
|
version: '6.0.0',
|
||||||
description: 'An advanced library for NoSQL data organization and manipulation using TypeScript with support for MongoDB, data validation, collections, and custom data types.'
|
description: 'An advanced library for NoSQL data organization and manipulation using TypeScript with support for MongoDB, data validation, collections, and custom data types.'
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import { SmartdataDbCursor } from './classes.cursor.js';
|
|||||||
import { SmartDataDbDoc, type IIndexOptions } from './classes.doc.js';
|
import { SmartDataDbDoc, type IIndexOptions } from './classes.doc.js';
|
||||||
import { SmartdataDbWatcher } from './classes.watcher.js';
|
import { SmartdataDbWatcher } from './classes.watcher.js';
|
||||||
import { CollectionFactory } from './classes.collectionfactory.js';
|
import { CollectionFactory } from './classes.collectionfactory.js';
|
||||||
|
import { logger } from './logging.js';
|
||||||
|
|
||||||
export interface IFindOptions {
|
export interface IFindOptions {
|
||||||
limit?: number;
|
limit?: number;
|
||||||
@@ -25,32 +26,94 @@ const collectionFactory = new CollectionFactory();
|
|||||||
* @param dbArg
|
* @param dbArg
|
||||||
*/
|
*/
|
||||||
export function Collection(dbArg: SmartdataDb | TDelayed<SmartdataDb>) {
|
export function Collection(dbArg: SmartdataDb | TDelayed<SmartdataDb>) {
|
||||||
return function classDecorator<T extends { new (...args: any[]): {} }>(constructor: T) {
|
return function classDecorator(value: Function, context: ClassDecoratorContext) {
|
||||||
|
if (context.kind !== 'class') {
|
||||||
|
throw new Error('Collection can only decorate classes');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Capture original constructor for _svDbOptions forwarding
|
||||||
|
const originalConstructor = value as any;
|
||||||
|
const constructor = value as { new (...args: any[]): any };
|
||||||
|
|
||||||
|
const getCollection = () => {
|
||||||
|
if (!(dbArg instanceof SmartdataDb)) {
|
||||||
|
dbArg = dbArg();
|
||||||
|
}
|
||||||
|
const coll = collectionFactory.getCollection(constructor.name, dbArg);
|
||||||
|
// Attach document constructor for searchableFields lookup
|
||||||
|
if (!(coll as any).docCtor) {
|
||||||
|
(coll as any).docCtor = decoratedClass;
|
||||||
|
}
|
||||||
|
return coll;
|
||||||
|
};
|
||||||
|
|
||||||
const decoratedClass = class extends constructor {
|
const decoratedClass = class extends constructor {
|
||||||
public static className = constructor.name;
|
public static className = constructor.name;
|
||||||
public static get collection() {
|
public static get collection() {
|
||||||
if (!(dbArg instanceof SmartdataDb)) {
|
return getCollection();
|
||||||
dbArg = dbArg();
|
|
||||||
}
|
|
||||||
const coll = collectionFactory.getCollection(constructor.name, dbArg);
|
|
||||||
// Attach document constructor for searchableFields lookup
|
|
||||||
if (!(coll as any).docCtor) {
|
|
||||||
(coll as any).docCtor = decoratedClass;
|
|
||||||
}
|
|
||||||
return coll;
|
|
||||||
}
|
}
|
||||||
public get collection() {
|
public get collection() {
|
||||||
if (!(dbArg instanceof SmartdataDb)) {
|
return getCollection();
|
||||||
dbArg = dbArg();
|
|
||||||
}
|
|
||||||
const coll = collectionFactory.getCollection(constructor.name, dbArg);
|
|
||||||
if (!(coll as any).docCtor) {
|
|
||||||
(coll as any).docCtor = decoratedClass;
|
|
||||||
}
|
|
||||||
return coll;
|
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
return decoratedClass;
|
|
||||||
|
// Ensure instance getter works in Deno by defining it on the prototype
|
||||||
|
Object.defineProperty(decoratedClass.prototype, 'collection', {
|
||||||
|
get: getCollection,
|
||||||
|
enumerable: false,
|
||||||
|
configurable: true
|
||||||
|
});
|
||||||
|
|
||||||
|
// Deno compatibility note: Property decorators set properties on the prototype.
|
||||||
|
// Since we removed instance property declarations from SmartDataDbDoc,
|
||||||
|
// the decorator-set prototype properties are now accessible without shadowing.
|
||||||
|
// No manual forwarding needed - natural prototype inheritance works!
|
||||||
|
|
||||||
|
// Point to original constructor's _svDbOptions
|
||||||
|
Object.defineProperty(decoratedClass, '_svDbOptions', {
|
||||||
|
get() { return originalConstructor._svDbOptions; },
|
||||||
|
set(value) { originalConstructor._svDbOptions = value; },
|
||||||
|
configurable: true
|
||||||
|
});
|
||||||
|
|
||||||
|
// Initialize prototype properties from context.metadata (TC39 decorator metadata)
|
||||||
|
// This ensures prototype properties are available before any instance is created
|
||||||
|
const metadata = context.metadata as any;
|
||||||
|
if (metadata) {
|
||||||
|
const proto = decoratedClass.prototype;
|
||||||
|
|
||||||
|
// Initialize globalSaveableProperties
|
||||||
|
if (metadata.globalSaveableProperties && !proto.globalSaveableProperties) {
|
||||||
|
proto.globalSaveableProperties = [...metadata.globalSaveableProperties];
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initialize saveableProperties
|
||||||
|
if (metadata.saveableProperties && !proto.saveableProperties) {
|
||||||
|
proto.saveableProperties = [...metadata.saveableProperties];
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initialize uniqueIndexes
|
||||||
|
if (metadata.uniqueIndexes && !proto.uniqueIndexes) {
|
||||||
|
proto.uniqueIndexes = [...metadata.uniqueIndexes];
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initialize regularIndexes
|
||||||
|
if (metadata.regularIndexes && !proto.regularIndexes) {
|
||||||
|
proto.regularIndexes = [...metadata.regularIndexes];
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initialize searchableFields on constructor (not prototype)
|
||||||
|
if (metadata.searchableFields && !Array.isArray((decoratedClass as any).searchableFields)) {
|
||||||
|
(decoratedClass as any).searchableFields = [...metadata.searchableFields];
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initialize _svDbOptions from metadata
|
||||||
|
if (metadata._svDbOptions && !originalConstructor._svDbOptions) {
|
||||||
|
originalConstructor._svDbOptions = { ...metadata._svDbOptions };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return decoratedClass as any;
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -68,7 +131,13 @@ export const setDefaultManagerForDoc = <T,>(managerArg: IManager, dbDocArg: T):
|
|||||||
* @param dbArg
|
* @param dbArg
|
||||||
*/
|
*/
|
||||||
export function managed<TManager extends IManager>(managerArg?: TManager | TDelayed<TManager>) {
|
export function managed<TManager extends IManager>(managerArg?: TManager | TDelayed<TManager>) {
|
||||||
return function classDecorator<T extends { new (...args: any[]): any }>(constructor: T) {
|
return function classDecorator(value: Function, context: ClassDecoratorContext) {
|
||||||
|
if (context.kind !== 'class') {
|
||||||
|
throw new Error('managed can only decorate classes');
|
||||||
|
}
|
||||||
|
|
||||||
|
const constructor = value as { new (...args: any[]): any };
|
||||||
|
|
||||||
const decoratedClass = class extends constructor {
|
const decoratedClass = class extends constructor {
|
||||||
public static className = constructor.name;
|
public static className = constructor.name;
|
||||||
public static get collection() {
|
public static get collection() {
|
||||||
@@ -118,7 +187,46 @@ export function managed<TManager extends IManager>(managerArg?: TManager | TDela
|
|||||||
return manager;
|
return manager;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
return decoratedClass;
|
|
||||||
|
// Initialize prototype properties from context.metadata (TC39 decorator metadata)
|
||||||
|
// This ensures prototype properties are available before any instance is created
|
||||||
|
const originalConstructor = value as any;
|
||||||
|
const metadata = context.metadata as any;
|
||||||
|
if (metadata) {
|
||||||
|
const proto = decoratedClass.prototype;
|
||||||
|
|
||||||
|
// Initialize globalSaveableProperties
|
||||||
|
if (metadata.globalSaveableProperties && !proto.globalSaveableProperties) {
|
||||||
|
proto.globalSaveableProperties = [...metadata.globalSaveableProperties];
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initialize saveableProperties
|
||||||
|
if (metadata.saveableProperties && !proto.saveableProperties) {
|
||||||
|
proto.saveableProperties = [...metadata.saveableProperties];
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initialize uniqueIndexes
|
||||||
|
if (metadata.uniqueIndexes && !proto.uniqueIndexes) {
|
||||||
|
proto.uniqueIndexes = [...metadata.uniqueIndexes];
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initialize regularIndexes
|
||||||
|
if (metadata.regularIndexes && !proto.regularIndexes) {
|
||||||
|
proto.regularIndexes = [...metadata.regularIndexes];
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initialize searchableFields on constructor (not prototype)
|
||||||
|
if (metadata.searchableFields && !Array.isArray((decoratedClass as any).searchableFields)) {
|
||||||
|
(decoratedClass as any).searchableFields = [...metadata.searchableFields];
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initialize _svDbOptions from metadata
|
||||||
|
if (metadata._svDbOptions && !originalConstructor._svDbOptions) {
|
||||||
|
originalConstructor._svDbOptions = { ...metadata._svDbOptions };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return decoratedClass as any;
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -161,7 +269,7 @@ export class SmartdataCollection<T> {
|
|||||||
});
|
});
|
||||||
if (!wantedCollection) {
|
if (!wantedCollection) {
|
||||||
await this.smartdataDb.mongoDb.createCollection(this.collectionName);
|
await this.smartdataDb.mongoDb.createCollection(this.collectionName);
|
||||||
console.log(`Successfully initiated Collection ${this.collectionName}`);
|
logger.log('info', `Successfully initiated Collection ${this.collectionName}`);
|
||||||
}
|
}
|
||||||
this.mongoDbCollection = this.smartdataDb.mongoDb.collection(this.collectionName);
|
this.mongoDbCollection = this.smartdataDb.mongoDb.collection(this.collectionName);
|
||||||
// Auto-create a compound text index on all searchable fields
|
// Auto-create a compound text index on all searchable fields
|
||||||
@@ -182,10 +290,10 @@ export class SmartdataCollection<T> {
|
|||||||
/**
|
/**
|
||||||
* mark unique index
|
* mark unique index
|
||||||
*/
|
*/
|
||||||
public markUniqueIndexes(keyArrayArg: string[] = []) {
|
public async markUniqueIndexes(keyArrayArg: string[] = []) {
|
||||||
for (const key of keyArrayArg) {
|
for (const key of keyArrayArg) {
|
||||||
if (!this.uniqueIndexes.includes(key)) {
|
if (!this.uniqueIndexes.includes(key)) {
|
||||||
this.mongoDbCollection.createIndex(key, {
|
await this.mongoDbCollection.createIndex({ [key]: 1 }, {
|
||||||
unique: true,
|
unique: true,
|
||||||
});
|
});
|
||||||
// make sure we only call this once and not for every doc we create
|
// make sure we only call this once and not for every doc we create
|
||||||
@@ -197,12 +305,12 @@ export class SmartdataCollection<T> {
|
|||||||
/**
|
/**
|
||||||
* creates regular indexes for the collection
|
* creates regular indexes for the collection
|
||||||
*/
|
*/
|
||||||
public createRegularIndexes(indexesArg: Array<{field: string, options: IIndexOptions}> = []) {
|
public async createRegularIndexes(indexesArg: Array<{field: string, options: IIndexOptions}> = []) {
|
||||||
for (const indexDef of indexesArg) {
|
for (const indexDef of indexesArg) {
|
||||||
// Check if we've already created this index
|
// Check if we've already created this index
|
||||||
const indexKey = indexDef.field;
|
const indexKey = indexDef.field;
|
||||||
if (!this.regularIndexes.some(i => i.field === indexKey)) {
|
if (!this.regularIndexes.some(i => i.field === indexKey)) {
|
||||||
this.mongoDbCollection.createIndex(
|
await this.mongoDbCollection.createIndex(
|
||||||
{ [indexDef.field]: 1 }, // Simple single-field index
|
{ [indexDef.field]: 1 }, // Simple single-field index
|
||||||
indexDef.options
|
indexDef.options
|
||||||
);
|
);
|
||||||
@@ -275,7 +383,7 @@ export class SmartdataCollection<T> {
|
|||||||
fullDocument:
|
fullDocument:
|
||||||
fullDocument === undefined
|
fullDocument === undefined
|
||||||
? 'updateLookup'
|
? 'updateLookup'
|
||||||
: fullDocument === true
|
: (fullDocument as any) === true
|
||||||
? 'updateLookup'
|
? 'updateLookup'
|
||||||
: fullDocument,
|
: fullDocument,
|
||||||
} as any;
|
} as any;
|
||||||
|
|||||||
@@ -35,24 +35,43 @@ export class SmartdataDb {
|
|||||||
* connects to the database that was specified during instance creation
|
* connects to the database that was specified during instance creation
|
||||||
*/
|
*/
|
||||||
public async init(): Promise<any> {
|
public async init(): Promise<any> {
|
||||||
const finalConnectionUrl = this.smartdataOptions.mongoDbUrl
|
try {
|
||||||
.replace('<USERNAME>', this.smartdataOptions.mongoDbUser)
|
// Safely encode credentials to handle special characters
|
||||||
.replace('<username>', this.smartdataOptions.mongoDbUser)
|
const encodedUser = this.smartdataOptions.mongoDbUser
|
||||||
.replace('<USER>', this.smartdataOptions.mongoDbUser)
|
? encodeURIComponent(this.smartdataOptions.mongoDbUser)
|
||||||
.replace('<user>', this.smartdataOptions.mongoDbUser)
|
: '';
|
||||||
.replace('<PASSWORD>', this.smartdataOptions.mongoDbPass)
|
const encodedPass = this.smartdataOptions.mongoDbPass
|
||||||
.replace('<password>', this.smartdataOptions.mongoDbPass)
|
? encodeURIComponent(this.smartdataOptions.mongoDbPass)
|
||||||
.replace('<DBNAME>', this.smartdataOptions.mongoDbName)
|
: '';
|
||||||
.replace('<dbname>', this.smartdataOptions.mongoDbName);
|
|
||||||
|
const finalConnectionUrl = this.smartdataOptions.mongoDbUrl
|
||||||
|
.replace('<USERNAME>', encodedUser)
|
||||||
|
.replace('<username>', encodedUser)
|
||||||
|
.replace('<USER>', encodedUser)
|
||||||
|
.replace('<user>', encodedUser)
|
||||||
|
.replace('<PASSWORD>', encodedPass)
|
||||||
|
.replace('<password>', encodedPass)
|
||||||
|
.replace('<DBNAME>', this.smartdataOptions.mongoDbName)
|
||||||
|
.replace('<dbname>', this.smartdataOptions.mongoDbName);
|
||||||
|
|
||||||
this.mongoDbClient = await plugins.mongodb.MongoClient.connect(finalConnectionUrl, {
|
const clientOptions: plugins.mongodb.MongoClientOptions = {
|
||||||
maxPoolSize: 100,
|
maxPoolSize: (this.smartdataOptions as any).maxPoolSize ?? 100,
|
||||||
maxIdleTimeMS: 10,
|
maxIdleTimeMS: (this.smartdataOptions as any).maxIdleTimeMS ?? 300000, // 5 minutes default
|
||||||
});
|
serverSelectionTimeoutMS: (this.smartdataOptions as any).serverSelectionTimeoutMS ?? 30000,
|
||||||
this.mongoDb = this.mongoDbClient.db(this.smartdataOptions.mongoDbName);
|
retryWrites: true,
|
||||||
this.status = 'connected';
|
};
|
||||||
this.statusConnectedDeferred.resolve();
|
|
||||||
console.log(`Connected to database ${this.smartdataOptions.mongoDbName}`);
|
this.mongoDbClient = await plugins.mongodb.MongoClient.connect(finalConnectionUrl, clientOptions);
|
||||||
|
this.mongoDb = this.mongoDbClient.db(this.smartdataOptions.mongoDbName);
|
||||||
|
this.status = 'connected';
|
||||||
|
this.statusConnectedDeferred.resolve();
|
||||||
|
logger.log('info', `Connected to database ${this.smartdataOptions.mongoDbName}`);
|
||||||
|
} catch (error) {
|
||||||
|
this.status = 'disconnected';
|
||||||
|
this.statusConnectedDeferred.reject(error);
|
||||||
|
logger.log('error', `Failed to connect to database ${this.smartdataOptions.mongoDbName}: ${error.message}`);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import { SmartdataDb } from './classes.db.js';
|
|||||||
import { managed, setDefaultManagerForDoc } from './classes.collection.js';
|
import { managed, setDefaultManagerForDoc } from './classes.collection.js';
|
||||||
import { SmartDataDbDoc, svDb, unI } from './classes.doc.js';
|
import { SmartDataDbDoc, svDb, unI } from './classes.doc.js';
|
||||||
import { SmartdataDbWatcher } from './classes.watcher.js';
|
import { SmartdataDbWatcher } from './classes.watcher.js';
|
||||||
|
import { logger } from './logging.js';
|
||||||
|
|
||||||
@managed()
|
@managed()
|
||||||
export class DistributedClass extends SmartDataDbDoc<DistributedClass, DistributedClass> {
|
export class DistributedClass extends SmartDataDbDoc<DistributedClass, DistributedClass> {
|
||||||
@@ -63,11 +64,11 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
|
|||||||
this.ownInstance.data.elected = false;
|
this.ownInstance.data.elected = false;
|
||||||
}
|
}
|
||||||
if (this.ownInstance?.data.status === 'stopped') {
|
if (this.ownInstance?.data.status === 'stopped') {
|
||||||
console.log(`stopping a distributed instance that has not been started yet.`);
|
logger.log('warn', `stopping a distributed instance that has not been started yet.`);
|
||||||
}
|
}
|
||||||
this.ownInstance.data.status = 'stopped';
|
this.ownInstance.data.status = 'stopped';
|
||||||
await this.ownInstance.save();
|
await this.ownInstance.save();
|
||||||
console.log(`stopped ${this.ownInstance.id}`);
|
logger.log('info', `stopped ${this.ownInstance.id}`);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -83,17 +84,17 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
|
|||||||
public async sendHeartbeat() {
|
public async sendHeartbeat() {
|
||||||
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
||||||
if (this.ownInstance.data.status === 'stopped') {
|
if (this.ownInstance.data.status === 'stopped') {
|
||||||
console.log(`aborted sending heartbeat because status is stopped`);
|
logger.log('debug', `aborted sending heartbeat because status is stopped`);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
await this.ownInstance.updateFromDb();
|
await this.ownInstance.updateFromDb();
|
||||||
this.ownInstance.data.lastUpdated = Date.now();
|
this.ownInstance.data.lastUpdated = Date.now();
|
||||||
await this.ownInstance.save();
|
await this.ownInstance.save();
|
||||||
console.log(`sent heartbeat for ${this.ownInstance.id}`);
|
logger.log('debug', `sent heartbeat for ${this.ownInstance.id}`);
|
||||||
const allInstances = DistributedClass.getInstances({});
|
const allInstances = DistributedClass.getInstances({});
|
||||||
});
|
});
|
||||||
if (this.ownInstance.data.status === 'stopped') {
|
if (this.ownInstance.data.status === 'stopped') {
|
||||||
console.log(`aborted sending heartbeat because status is stopped`);
|
logger.log('info', `aborted sending heartbeat because status is stopped`);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const eligibleLeader = await this.getEligibleLeader();
|
const eligibleLeader = await this.getEligibleLeader();
|
||||||
@@ -120,7 +121,7 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
|
|||||||
await this.ownInstance.save();
|
await this.ownInstance.save();
|
||||||
});
|
});
|
||||||
} else {
|
} else {
|
||||||
console.warn(`distributed instance already initialized`);
|
logger.log('warn', `distributed instance already initialized`);
|
||||||
}
|
}
|
||||||
|
|
||||||
// lets enable the heartbeat
|
// lets enable the heartbeat
|
||||||
@@ -149,24 +150,24 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
|
|||||||
public async checkAndMaybeLead() {
|
public async checkAndMaybeLead() {
|
||||||
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
||||||
this.ownInstance.data.status = 'initializing';
|
this.ownInstance.data.status = 'initializing';
|
||||||
this.ownInstance.save();
|
await this.ownInstance.save();
|
||||||
});
|
});
|
||||||
if (await this.getEligibleLeader()) {
|
if (await this.getEligibleLeader()) {
|
||||||
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
||||||
await this.ownInstance.updateFromDb();
|
await this.ownInstance.updateFromDb();
|
||||||
this.ownInstance.data.status = 'settled';
|
this.ownInstance.data.status = 'settled';
|
||||||
await this.ownInstance.save();
|
await this.ownInstance.save();
|
||||||
console.log(`${this.ownInstance.id} settled as follower`);
|
logger.log('info', `${this.ownInstance.id} settled as follower`);
|
||||||
});
|
});
|
||||||
return;
|
return;
|
||||||
} else if (
|
} else if (
|
||||||
(await DistributedClass.getInstances({})).find((instanceArg) => {
|
(await DistributedClass.getInstances({})).find((instanceArg) => {
|
||||||
instanceArg.data.status === 'bidding' &&
|
return instanceArg.data.status === 'bidding' &&
|
||||||
instanceArg.data.biddingStartTime <= Date.now() - 4000 &&
|
instanceArg.data.biddingStartTime <= Date.now() - 4000 &&
|
||||||
instanceArg.data.biddingStartTime >= Date.now() - 30000;
|
instanceArg.data.biddingStartTime >= Date.now() - 30000;
|
||||||
})
|
})
|
||||||
) {
|
) {
|
||||||
console.log('too late to the bidding party... waiting for next round.');
|
logger.log('info', 'too late to the bidding party... waiting for next round.');
|
||||||
return;
|
return;
|
||||||
} else {
|
} else {
|
||||||
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
||||||
@@ -175,9 +176,9 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
|
|||||||
this.ownInstance.data.biddingStartTime = Date.now();
|
this.ownInstance.data.biddingStartTime = Date.now();
|
||||||
this.ownInstance.data.biddingShortcode = plugins.smartunique.shortId();
|
this.ownInstance.data.biddingShortcode = plugins.smartunique.shortId();
|
||||||
await this.ownInstance.save();
|
await this.ownInstance.save();
|
||||||
console.log('bidding code stored.');
|
logger.log('info', 'bidding code stored.');
|
||||||
});
|
});
|
||||||
console.log(`bidding for leadership...`);
|
logger.log('info', `bidding for leadership...`);
|
||||||
await plugins.smartdelay.delayFor(plugins.smarttime.getMilliSecondsFromUnits({ seconds: 5 }));
|
await plugins.smartdelay.delayFor(plugins.smarttime.getMilliSecondsFromUnits({ seconds: 5 }));
|
||||||
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
||||||
let biddingInstances = await DistributedClass.getInstances({});
|
let biddingInstances = await DistributedClass.getInstances({});
|
||||||
@@ -187,7 +188,7 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
|
|||||||
instanceArg.data.lastUpdated >=
|
instanceArg.data.lastUpdated >=
|
||||||
Date.now() - plugins.smarttime.getMilliSecondsFromUnits({ seconds: 10 }),
|
Date.now() - plugins.smarttime.getMilliSecondsFromUnits({ seconds: 10 }),
|
||||||
);
|
);
|
||||||
console.log(`found ${biddingInstances.length} bidding instances...`);
|
logger.log('info', `found ${biddingInstances.length} bidding instances...`);
|
||||||
this.ownInstance.data.elected = true;
|
this.ownInstance.data.elected = true;
|
||||||
for (const biddingInstance of biddingInstances) {
|
for (const biddingInstance of biddingInstances) {
|
||||||
if (biddingInstance.data.biddingShortcode < this.ownInstance.data.biddingShortcode) {
|
if (biddingInstance.data.biddingShortcode < this.ownInstance.data.biddingShortcode) {
|
||||||
@@ -195,7 +196,7 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
await plugins.smartdelay.delayFor(5000);
|
await plugins.smartdelay.delayFor(5000);
|
||||||
console.log(`settling with status elected = ${this.ownInstance.data.elected}`);
|
logger.log('info', `settling with status elected = ${this.ownInstance.data.elected}`);
|
||||||
this.ownInstance.data.status = 'settled';
|
this.ownInstance.data.status = 'settled';
|
||||||
await this.ownInstance.save();
|
await this.ownInstance.save();
|
||||||
});
|
});
|
||||||
@@ -226,11 +227,11 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
|
|||||||
this.distributedWatcher.changeSubject.subscribe({
|
this.distributedWatcher.changeSubject.subscribe({
|
||||||
next: async (distributedDoc) => {
|
next: async (distributedDoc) => {
|
||||||
if (!distributedDoc) {
|
if (!distributedDoc) {
|
||||||
console.log(`registered deletion of instance...`);
|
logger.log('info', `registered deletion of instance...`);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
console.log(distributedDoc);
|
logger.log('info', distributedDoc);
|
||||||
console.log(`registered change for ${distributedDoc.id}`);
|
logger.log('info', `registered change for ${distributedDoc.id}`);
|
||||||
distributedDoc;
|
distributedDoc;
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
@@ -252,7 +253,7 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
|
|||||||
): Promise<plugins.taskbuffer.distributedCoordination.IDistributedTaskRequestResult> {
|
): Promise<plugins.taskbuffer.distributedCoordination.IDistributedTaskRequestResult> {
|
||||||
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
||||||
if (!this.ownInstance) {
|
if (!this.ownInstance) {
|
||||||
console.error('instance need to be started first...');
|
logger.log('error', 'instance need to be started first...');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
await this.ownInstance.updateFromDb();
|
await this.ownInstance.updateFromDb();
|
||||||
@@ -268,7 +269,7 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
|
|||||||
return taskRequestResult;
|
return taskRequestResult;
|
||||||
});
|
});
|
||||||
if (!result) {
|
if (!result) {
|
||||||
console.warn('no result found for task request...');
|
logger.log('warn', 'no result found for task request...');
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
return result;
|
return result;
|
||||||
@@ -285,7 +286,7 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
if (!existingInfoBasis) {
|
if (!existingInfoBasis) {
|
||||||
console.warn('trying to update a non existing task request... aborting!');
|
logger.log('warn', 'trying to update a non existing task request... aborting!');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
Object.assign(existingInfoBasis, infoBasisArg);
|
Object.assign(existingInfoBasis, infoBasisArg);
|
||||||
@@ -293,8 +294,10 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
|
|||||||
plugins.smartdelay.delayFor(60000).then(() => {
|
plugins.smartdelay.delayFor(60000).then(() => {
|
||||||
this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
||||||
const indexToRemove = this.ownInstance.data.taskRequests.indexOf(existingInfoBasis);
|
const indexToRemove = this.ownInstance.data.taskRequests.indexOf(existingInfoBasis);
|
||||||
this.ownInstance.data.taskRequests.splice(indexToRemove, indexToRemove);
|
if (indexToRemove >= 0) {
|
||||||
await this.ownInstance.save();
|
this.ownInstance.data.taskRequests.splice(indexToRemove, 1);
|
||||||
|
await this.ownInstance.save();
|
||||||
|
}
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import * as plugins from './plugins.js';
|
import * as plugins from './plugins.js';
|
||||||
|
|
||||||
import { SmartdataDb } from './classes.db.js';
|
import { SmartdataDb } from './classes.db.js';
|
||||||
|
import { logger } from './logging.js';
|
||||||
import { SmartdataDbCursor } from './classes.cursor.js';
|
import { SmartdataDbCursor } from './classes.cursor.js';
|
||||||
import { type IManager, SmartdataCollection } from './classes.collection.js';
|
import { type IManager, SmartdataCollection } from './classes.collection.js';
|
||||||
import { SmartdataDbWatcher } from './classes.watcher.js';
|
import { SmartdataDbWatcher } from './classes.watcher.js';
|
||||||
@@ -27,15 +28,42 @@ export interface SearchOptions<T> {
|
|||||||
|
|
||||||
export type TDocCreation = 'db' | 'new' | 'mixed';
|
export type TDocCreation = 'db' | 'new' | 'mixed';
|
||||||
|
|
||||||
|
// Type for decorator metadata - extends TypeScript's built-in DecoratorMetadataObject
|
||||||
|
interface ISmartdataDecoratorMetadata extends DecoratorMetadataObject {
|
||||||
|
globalSaveableProperties?: string[];
|
||||||
|
saveableProperties?: string[];
|
||||||
|
uniqueIndexes?: string[];
|
||||||
|
regularIndexes?: Array<{field: string, options: IIndexOptions}>;
|
||||||
|
searchableFields?: string[];
|
||||||
|
_svDbOptions?: Record<string, SvDbOptions>;
|
||||||
|
}
|
||||||
|
|
||||||
export function globalSvDb() {
|
export function globalSvDb() {
|
||||||
return (target: SmartDataDbDoc<unknown, unknown>, key: string) => {
|
return (value: undefined, context: ClassFieldDecoratorContext) => {
|
||||||
console.log(`called svDb() on >${target.constructor.name}.${key}<`);
|
if (context.kind !== 'field') {
|
||||||
if (!target.globalSaveableProperties) {
|
throw new Error('globalSvDb can only decorate fields');
|
||||||
target.globalSaveableProperties = [];
|
|
||||||
}
|
}
|
||||||
target.globalSaveableProperties.push(key);
|
|
||||||
|
// Store metadata at class level using Symbol.metadata
|
||||||
|
const metadata = context.metadata as ISmartdataDecoratorMetadata;
|
||||||
|
if (!metadata.globalSaveableProperties) {
|
||||||
|
metadata.globalSaveableProperties = [];
|
||||||
|
}
|
||||||
|
metadata.globalSaveableProperties.push(String(context.name));
|
||||||
|
|
||||||
|
logger.log('debug', `called globalSvDb() on metadata for property ${String(context.name)}`);
|
||||||
|
|
||||||
|
// Use addInitializer to ensure prototype arrays are set up once
|
||||||
|
context.addInitializer(function(this: any) {
|
||||||
|
const proto = this.constructor.prototype;
|
||||||
|
const metadata = this.constructor[Symbol.metadata];
|
||||||
|
|
||||||
|
if (metadata && metadata.globalSaveableProperties && !proto.globalSaveableProperties) {
|
||||||
|
// Initialize prototype array from metadata (runs once per class)
|
||||||
|
proto.globalSaveableProperties = [...metadata.globalSaveableProperties];
|
||||||
|
logger.log('debug', `initialized globalSaveableProperties with ${proto.globalSaveableProperties.length} properties`);
|
||||||
|
}
|
||||||
|
});
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -53,20 +81,47 @@ export interface SvDbOptions {
|
|||||||
* saveable - saveable decorator to be used on class properties
|
* saveable - saveable decorator to be used on class properties
|
||||||
*/
|
*/
|
||||||
export function svDb(options?: SvDbOptions) {
|
export function svDb(options?: SvDbOptions) {
|
||||||
return (target: SmartDataDbDoc<unknown, unknown>, key: string) => {
|
return (value: undefined, context: ClassFieldDecoratorContext) => {
|
||||||
console.log(`called svDb() on >${target.constructor.name}.${key}<`);
|
if (context.kind !== 'field') {
|
||||||
if (!target.saveableProperties) {
|
throw new Error('svDb can only decorate fields');
|
||||||
target.saveableProperties = [];
|
|
||||||
}
|
}
|
||||||
target.saveableProperties.push(key);
|
|
||||||
// attach custom serializer/deserializer options to the class constructor
|
const propName = String(context.name);
|
||||||
const ctor = target.constructor as any;
|
|
||||||
if (!ctor._svDbOptions) {
|
// Store metadata at class level using Symbol.metadata
|
||||||
ctor._svDbOptions = {};
|
const metadata = context.metadata as ISmartdataDecoratorMetadata;
|
||||||
|
if (!metadata.saveableProperties) {
|
||||||
|
metadata.saveableProperties = [];
|
||||||
}
|
}
|
||||||
|
metadata.saveableProperties.push(propName);
|
||||||
|
|
||||||
|
// Store options in metadata
|
||||||
if (options) {
|
if (options) {
|
||||||
ctor._svDbOptions[key] = options;
|
if (!metadata._svDbOptions) {
|
||||||
|
metadata._svDbOptions = {};
|
||||||
|
}
|
||||||
|
metadata._svDbOptions[propName] = options;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
logger.log('debug', `called svDb() on metadata for property ${propName}`);
|
||||||
|
|
||||||
|
// Use addInitializer to ensure prototype arrays are set up once
|
||||||
|
context.addInitializer(function(this: any) {
|
||||||
|
const proto = this.constructor.prototype;
|
||||||
|
const ctor = this.constructor;
|
||||||
|
const metadata = ctor[Symbol.metadata];
|
||||||
|
|
||||||
|
if (metadata && metadata.saveableProperties && !proto.saveableProperties) {
|
||||||
|
// Initialize prototype array from metadata (runs once per class)
|
||||||
|
proto.saveableProperties = [...metadata.saveableProperties];
|
||||||
|
logger.log('debug', `initialized saveableProperties with ${proto.saveableProperties.length} properties`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initialize svDbOptions from metadata
|
||||||
|
if (metadata && metadata._svDbOptions && !ctor._svDbOptions) {
|
||||||
|
ctor._svDbOptions = { ...metadata._svDbOptions };
|
||||||
|
}
|
||||||
|
});
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -74,13 +129,30 @@ export function svDb(options?: SvDbOptions) {
|
|||||||
* searchable - marks a property as searchable with Lucene query syntax
|
* searchable - marks a property as searchable with Lucene query syntax
|
||||||
*/
|
*/
|
||||||
export function searchable() {
|
export function searchable() {
|
||||||
return (target: SmartDataDbDoc<unknown, unknown>, key: string) => {
|
return (value: undefined, context: ClassFieldDecoratorContext) => {
|
||||||
// Attach to class constructor for direct access
|
if (context.kind !== 'field') {
|
||||||
const ctor = target.constructor as any;
|
throw new Error('searchable can only decorate fields');
|
||||||
if (!Array.isArray(ctor.searchableFields)) {
|
|
||||||
ctor.searchableFields = [];
|
|
||||||
}
|
}
|
||||||
ctor.searchableFields.push(key);
|
|
||||||
|
const propName = String(context.name);
|
||||||
|
|
||||||
|
// Store metadata at class level
|
||||||
|
const metadata = context.metadata as ISmartdataDecoratorMetadata;
|
||||||
|
if (!metadata.searchableFields) {
|
||||||
|
metadata.searchableFields = [];
|
||||||
|
}
|
||||||
|
metadata.searchableFields.push(propName);
|
||||||
|
|
||||||
|
// Use addInitializer to set up constructor property once
|
||||||
|
context.addInitializer(function(this: any) {
|
||||||
|
const ctor = this.constructor as any;
|
||||||
|
const metadata = ctor[Symbol.metadata];
|
||||||
|
|
||||||
|
if (metadata && metadata.searchableFields && !Array.isArray(ctor.searchableFields)) {
|
||||||
|
// Initialize from metadata (runs once per class)
|
||||||
|
ctor.searchableFields = [...metadata.searchableFields];
|
||||||
|
}
|
||||||
|
});
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -93,20 +165,44 @@ function escapeForRegex(input: string): string {
|
|||||||
* unique index - decorator to mark a unique index
|
* unique index - decorator to mark a unique index
|
||||||
*/
|
*/
|
||||||
export function unI() {
|
export function unI() {
|
||||||
return (target: SmartDataDbDoc<unknown, unknown>, key: string) => {
|
return (value: undefined, context: ClassFieldDecoratorContext) => {
|
||||||
console.log(`called unI on >>${target.constructor.name}.${key}<<`);
|
if (context.kind !== 'field') {
|
||||||
|
throw new Error('unI can only decorate fields');
|
||||||
// mark the index as unique
|
|
||||||
if (!target.uniqueIndexes) {
|
|
||||||
target.uniqueIndexes = [];
|
|
||||||
}
|
}
|
||||||
target.uniqueIndexes.push(key);
|
|
||||||
|
|
||||||
// and also save it
|
const propName = String(context.name);
|
||||||
if (!target.saveableProperties) {
|
|
||||||
target.saveableProperties = [];
|
// Store metadata at class level
|
||||||
|
const metadata = context.metadata as ISmartdataDecoratorMetadata;
|
||||||
|
if (!metadata.uniqueIndexes) {
|
||||||
|
metadata.uniqueIndexes = [];
|
||||||
}
|
}
|
||||||
target.saveableProperties.push(key);
|
metadata.uniqueIndexes.push(propName);
|
||||||
|
|
||||||
|
// Also mark as saveable
|
||||||
|
if (!metadata.saveableProperties) {
|
||||||
|
metadata.saveableProperties = [];
|
||||||
|
}
|
||||||
|
if (!metadata.saveableProperties.includes(propName)) {
|
||||||
|
metadata.saveableProperties.push(propName);
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.log('debug', `called unI on metadata for property ${propName}`);
|
||||||
|
|
||||||
|
// Use addInitializer to ensure prototype arrays are set up once
|
||||||
|
context.addInitializer(function(this: any) {
|
||||||
|
const proto = this.constructor.prototype;
|
||||||
|
const metadata = this.constructor[Symbol.metadata];
|
||||||
|
|
||||||
|
if (metadata && metadata.uniqueIndexes && !proto.uniqueIndexes) {
|
||||||
|
proto.uniqueIndexes = [...metadata.uniqueIndexes];
|
||||||
|
logger.log('debug', `initialized uniqueIndexes with ${proto.uniqueIndexes.length} properties`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (metadata && metadata.saveableProperties && !proto.saveableProperties) {
|
||||||
|
proto.saveableProperties = [...metadata.saveableProperties];
|
||||||
|
}
|
||||||
|
});
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -125,67 +221,225 @@ export interface IIndexOptions {
|
|||||||
* index - decorator to mark a field for regular indexing
|
* index - decorator to mark a field for regular indexing
|
||||||
*/
|
*/
|
||||||
export function index(options?: IIndexOptions) {
|
export function index(options?: IIndexOptions) {
|
||||||
return (target: SmartDataDbDoc<unknown, unknown>, key: string) => {
|
return (value: undefined, context: ClassFieldDecoratorContext) => {
|
||||||
console.log(`called index() on >${target.constructor.name}.${key}<`);
|
if (context.kind !== 'field') {
|
||||||
|
throw new Error('index can only decorate fields');
|
||||||
// Initialize regular indexes array if it doesn't exist
|
|
||||||
if (!target.regularIndexes) {
|
|
||||||
target.regularIndexes = [];
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add this field to regularIndexes with its options
|
const propName = String(context.name);
|
||||||
target.regularIndexes.push({
|
|
||||||
field: key,
|
// Store metadata at class level
|
||||||
|
const metadata = context.metadata as ISmartdataDecoratorMetadata;
|
||||||
|
if (!metadata.regularIndexes) {
|
||||||
|
metadata.regularIndexes = [];
|
||||||
|
}
|
||||||
|
metadata.regularIndexes.push({
|
||||||
|
field: propName,
|
||||||
options: options || {}
|
options: options || {}
|
||||||
});
|
});
|
||||||
|
|
||||||
// Also ensure it's marked as saveable
|
// Also mark as saveable
|
||||||
if (!target.saveableProperties) {
|
if (!metadata.saveableProperties) {
|
||||||
target.saveableProperties = [];
|
metadata.saveableProperties = [];
|
||||||
}
|
}
|
||||||
|
if (!metadata.saveableProperties.includes(propName)) {
|
||||||
if (!target.saveableProperties.includes(key)) {
|
metadata.saveableProperties.push(propName);
|
||||||
target.saveableProperties.push(key);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
logger.log('debug', `called index() on metadata for property ${propName}`);
|
||||||
|
|
||||||
|
// Use addInitializer to ensure prototype arrays are set up once
|
||||||
|
context.addInitializer(function(this: any) {
|
||||||
|
const proto = this.constructor.prototype;
|
||||||
|
const metadata = this.constructor[Symbol.metadata];
|
||||||
|
|
||||||
|
if (metadata && metadata.regularIndexes && !proto.regularIndexes) {
|
||||||
|
proto.regularIndexes = [...metadata.regularIndexes];
|
||||||
|
logger.log('debug', `initialized regularIndexes with ${proto.regularIndexes.length} indexes`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (metadata && metadata.saveableProperties && !proto.saveableProperties) {
|
||||||
|
proto.saveableProperties = [...metadata.saveableProperties];
|
||||||
|
}
|
||||||
|
});
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Helper type to extract element type from arrays or return T itself
|
||||||
|
type ElementOf<T> = T extends ReadonlyArray<infer U> ? U : T;
|
||||||
|
|
||||||
|
// Type for $in/$nin values - arrays of the element type
|
||||||
|
type InValues<T> = ReadonlyArray<ElementOf<T>>;
|
||||||
|
|
||||||
|
// Type that allows MongoDB operators on leaf values while maintaining nested type safety
|
||||||
|
export type MongoFilterCondition<T> = T | {
|
||||||
|
$eq?: T;
|
||||||
|
$ne?: T;
|
||||||
|
$gt?: T;
|
||||||
|
$gte?: T;
|
||||||
|
$lt?: T;
|
||||||
|
$lte?: T;
|
||||||
|
$in?: InValues<T>;
|
||||||
|
$nin?: InValues<T>;
|
||||||
|
$exists?: boolean;
|
||||||
|
$type?: string | number;
|
||||||
|
$regex?: string | RegExp;
|
||||||
|
$options?: string;
|
||||||
|
$all?: T extends ReadonlyArray<infer U> ? ReadonlyArray<U> : never;
|
||||||
|
$elemMatch?: T extends ReadonlyArray<infer U> ? MongoFilter<U> : never;
|
||||||
|
$size?: T extends ReadonlyArray<any> ? number : never;
|
||||||
|
$not?: MongoFilterCondition<T>;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type MongoFilter<T> = {
|
||||||
|
[K in keyof T]?: T[K] extends object
|
||||||
|
? T[K] extends any[]
|
||||||
|
? MongoFilterCondition<T[K]> // Arrays can have operators
|
||||||
|
: MongoFilter<T[K]> | MongoFilterCondition<T[K]> // Objects can be nested or have operators
|
||||||
|
: MongoFilterCondition<T[K]>; // Primitives get operators
|
||||||
|
} & {
|
||||||
|
// Logical operators
|
||||||
|
$and?: MongoFilter<T>[];
|
||||||
|
$or?: MongoFilter<T>[];
|
||||||
|
$nor?: MongoFilter<T>[];
|
||||||
|
$not?: MongoFilter<T>;
|
||||||
|
// Allow any string key for dot notation (we lose type safety here but maintain flexibility)
|
||||||
|
[key: string]: any;
|
||||||
|
};
|
||||||
|
|
||||||
export const convertFilterForMongoDb = (filterArg: { [key: string]: any }) => {
|
export const convertFilterForMongoDb = (filterArg: { [key: string]: any }) => {
|
||||||
// Special case: detect MongoDB operators and pass them through directly
|
// SECURITY: Block $where to prevent server-side JS execution
|
||||||
const topLevelOperators = ['$and', '$or', '$nor', '$not', '$text', '$where', '$regex'];
|
if (filterArg.$where !== undefined) {
|
||||||
|
throw new Error('$where operator is not allowed for security reasons');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handle logical operators recursively
|
||||||
|
const logicalOperators = ['$and', '$or', '$nor', '$not'];
|
||||||
|
const processedFilter: { [key: string]: any } = {};
|
||||||
|
|
||||||
for (const key of Object.keys(filterArg)) {
|
for (const key of Object.keys(filterArg)) {
|
||||||
if (topLevelOperators.includes(key)) {
|
if (logicalOperators.includes(key)) {
|
||||||
return filterArg; // Return the filter as-is for MongoDB operators
|
if (key === '$not') {
|
||||||
|
processedFilter[key] = convertFilterForMongoDb(filterArg[key]);
|
||||||
|
} else if (Array.isArray(filterArg[key])) {
|
||||||
|
processedFilter[key] = filterArg[key].map((subFilter: any) => convertFilterForMongoDb(subFilter));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// If only logical operators, return them
|
||||||
|
const hasOnlyLogicalOperators = Object.keys(filterArg).every(key => logicalOperators.includes(key));
|
||||||
|
if (hasOnlyLogicalOperators) {
|
||||||
|
return processedFilter;
|
||||||
|
}
|
||||||
|
|
||||||
// Original conversion logic for non-MongoDB query objects
|
// Original conversion logic for non-MongoDB query objects
|
||||||
const convertedFilter: { [key: string]: any } = {};
|
const convertedFilter: { [key: string]: any } = {};
|
||||||
|
|
||||||
|
// Helper to merge operator objects
|
||||||
|
const mergeIntoConverted = (path: string, value: any) => {
|
||||||
|
const existing = convertedFilter[path];
|
||||||
|
if (!existing) {
|
||||||
|
convertedFilter[path] = value;
|
||||||
|
} else if (
|
||||||
|
typeof existing === 'object' && !Array.isArray(existing) &&
|
||||||
|
typeof value === 'object' && !Array.isArray(value) &&
|
||||||
|
(Object.keys(existing).some(k => k.startsWith('$')) || Object.keys(value).some(k => k.startsWith('$')))
|
||||||
|
) {
|
||||||
|
// Both have operators, merge them
|
||||||
|
convertedFilter[path] = { ...existing, ...value };
|
||||||
|
} else {
|
||||||
|
// Otherwise later wins
|
||||||
|
convertedFilter[path] = value;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
const convertFilterArgument = (keyPathArg2: string, filterArg2: any) => {
|
const convertFilterArgument = (keyPathArg2: string, filterArg2: any) => {
|
||||||
if (Array.isArray(filterArg2)) {
|
if (Array.isArray(filterArg2)) {
|
||||||
// Directly assign arrays (they might be using operators like $in or $all)
|
// Arrays are typically used as values for operators like $in or as direct equality matches
|
||||||
convertFilterArgument(keyPathArg2, filterArg2[0]);
|
mergeIntoConverted(keyPathArg2, filterArg2);
|
||||||
|
return;
|
||||||
} else if (typeof filterArg2 === 'object' && filterArg2 !== null) {
|
} else if (typeof filterArg2 === 'object' && filterArg2 !== null) {
|
||||||
for (const key of Object.keys(filterArg2)) {
|
// Check if this is an object with MongoDB operators
|
||||||
if (key.startsWith('$')) {
|
const keys = Object.keys(filterArg2);
|
||||||
convertedFilter[keyPathArg2] = filterArg2;
|
const hasOperators = keys.some(key => key.startsWith('$'));
|
||||||
return;
|
|
||||||
} else if (key.includes('.')) {
|
if (hasOperators) {
|
||||||
|
// This object contains MongoDB operators
|
||||||
|
// Validate and pass through allowed operators
|
||||||
|
const allowedOperators = [
|
||||||
|
// Comparison operators
|
||||||
|
'$eq', '$ne', '$gt', '$gte', '$lt', '$lte',
|
||||||
|
// Array operators
|
||||||
|
'$in', '$nin', '$all', '$elemMatch', '$size',
|
||||||
|
// Element operators
|
||||||
|
'$exists', '$type',
|
||||||
|
// Evaluation operators (safe ones only)
|
||||||
|
'$regex', '$options', '$text', '$mod',
|
||||||
|
// Logical operators (nested)
|
||||||
|
'$and', '$or', '$nor', '$not'
|
||||||
|
];
|
||||||
|
|
||||||
|
// Check for dangerous operators
|
||||||
|
if (keys.includes('$where')) {
|
||||||
|
throw new Error('$where operator is not allowed for security reasons');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate all operators are in the allowed list
|
||||||
|
const invalidOperators = keys.filter(key =>
|
||||||
|
key.startsWith('$') && !allowedOperators.includes(key)
|
||||||
|
);
|
||||||
|
|
||||||
|
if (invalidOperators.length > 0) {
|
||||||
|
console.warn(`Warning: Unknown MongoDB operators detected: ${invalidOperators.join(', ')}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// For array operators, ensure the values are appropriate
|
||||||
|
if (filterArg2.$in && !Array.isArray(filterArg2.$in)) {
|
||||||
|
throw new Error('$in operator requires an array value');
|
||||||
|
}
|
||||||
|
if (filterArg2.$nin && !Array.isArray(filterArg2.$nin)) {
|
||||||
|
throw new Error('$nin operator requires an array value');
|
||||||
|
}
|
||||||
|
if (filterArg2.$all && !Array.isArray(filterArg2.$all)) {
|
||||||
|
throw new Error('$all operator requires an array value');
|
||||||
|
}
|
||||||
|
if (filterArg2.$size && typeof filterArg2.$size !== 'number') {
|
||||||
|
throw new Error('$size operator requires a numeric value');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Use merge helper to handle duplicate paths
|
||||||
|
mergeIntoConverted(keyPathArg2, filterArg2);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// No operators, check for dots in keys
|
||||||
|
for (const key of keys) {
|
||||||
|
if (key.includes('.')) {
|
||||||
throw new Error('keys cannot contain dots');
|
throw new Error('keys cannot contain dots');
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for (const key of Object.keys(filterArg2)) {
|
|
||||||
|
// Recursively process nested objects
|
||||||
|
for (const key of keys) {
|
||||||
convertFilterArgument(`${keyPathArg2}.${key}`, filterArg2[key]);
|
convertFilterArgument(`${keyPathArg2}.${key}`, filterArg2[key]);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
convertedFilter[keyPathArg2] = filterArg2;
|
// Primitive values
|
||||||
|
mergeIntoConverted(keyPathArg2, filterArg2);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
for (const key of Object.keys(filterArg)) {
|
for (const key of Object.keys(filterArg)) {
|
||||||
convertFilterArgument(key, filterArg[key]);
|
// Skip logical operators, they were already processed
|
||||||
|
if (!logicalOperators.includes(key)) {
|
||||||
|
convertFilterArgument(key, filterArg[key]);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Add back processed logical operators
|
||||||
|
Object.assign(convertedFilter, processedFilter);
|
||||||
|
|
||||||
return convertedFilter;
|
return convertedFilter;
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -199,6 +453,13 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
|||||||
public static manager;
|
public static manager;
|
||||||
public manager: TManager;
|
public manager: TManager;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Helper to get collection with fallback to static for Deno compatibility
|
||||||
|
*/
|
||||||
|
private getCollectionSafe(): SmartdataCollection<any> {
|
||||||
|
return this.collection || (this.constructor as any).collection;
|
||||||
|
}
|
||||||
|
|
||||||
// STATIC
|
// STATIC
|
||||||
public static createInstanceFromMongoDbNativeDoc<T>(
|
public static createInstanceFromMongoDbNativeDoc<T>(
|
||||||
this: plugins.tsclass.typeFest.Class<T>,
|
this: plugins.tsclass.typeFest.Class<T>,
|
||||||
@@ -220,12 +481,12 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
|||||||
/**
|
/**
|
||||||
* gets all instances as array
|
* gets all instances as array
|
||||||
* @param this
|
* @param this
|
||||||
* @param filterArg
|
* @param filterArg - Type-safe MongoDB filter with nested object support and operators
|
||||||
* @returns
|
* @returns
|
||||||
*/
|
*/
|
||||||
public static async getInstances<T>(
|
public static async getInstances<T>(
|
||||||
this: plugins.tsclass.typeFest.Class<T>,
|
this: plugins.tsclass.typeFest.Class<T>,
|
||||||
filterArg: plugins.tsclass.typeFest.PartialDeep<T>,
|
filterArg: MongoFilter<T>,
|
||||||
opts?: { session?: plugins.mongodb.ClientSession }
|
opts?: { session?: plugins.mongodb.ClientSession }
|
||||||
): Promise<T[]> {
|
): Promise<T[]> {
|
||||||
// Pass session through to findAll for transactional queries
|
// Pass session through to findAll for transactional queries
|
||||||
@@ -249,7 +510,7 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
|||||||
*/
|
*/
|
||||||
public static async getInstance<T>(
|
public static async getInstance<T>(
|
||||||
this: plugins.tsclass.typeFest.Class<T>,
|
this: plugins.tsclass.typeFest.Class<T>,
|
||||||
filterArg: plugins.tsclass.typeFest.PartialDeep<T>,
|
filterArg: MongoFilter<T>,
|
||||||
opts?: { session?: plugins.mongodb.ClientSession }
|
opts?: { session?: plugins.mongodb.ClientSession }
|
||||||
): Promise<T> {
|
): Promise<T> {
|
||||||
// Retrieve one document, with optional session for transactions
|
// Retrieve one document, with optional session for transactions
|
||||||
@@ -282,7 +543,7 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
|||||||
*/
|
*/
|
||||||
public static async getCursor<T>(
|
public static async getCursor<T>(
|
||||||
this: plugins.tsclass.typeFest.Class<T>,
|
this: plugins.tsclass.typeFest.Class<T>,
|
||||||
filterArg: plugins.tsclass.typeFest.PartialDeep<T>,
|
filterArg: MongoFilter<T>,
|
||||||
opts?: {
|
opts?: {
|
||||||
session?: plugins.mongodb.ClientSession;
|
session?: plugins.mongodb.ClientSession;
|
||||||
modifier?: (cursorArg: plugins.mongodb.FindCursor<plugins.mongodb.WithId<plugins.mongodb.BSON.Document>>) => plugins.mongodb.FindCursor<plugins.mongodb.WithId<plugins.mongodb.BSON.Document>>;
|
modifier?: (cursorArg: plugins.mongodb.FindCursor<plugins.mongodb.WithId<plugins.mongodb.BSON.Document>>) => plugins.mongodb.FindCursor<plugins.mongodb.WithId<plugins.mongodb.BSON.Document>>;
|
||||||
@@ -312,7 +573,7 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
|||||||
*/
|
*/
|
||||||
public static async watch<T>(
|
public static async watch<T>(
|
||||||
this: plugins.tsclass.typeFest.Class<T>,
|
this: plugins.tsclass.typeFest.Class<T>,
|
||||||
filterArg: plugins.tsclass.typeFest.PartialDeep<T>,
|
filterArg: MongoFilter<T>,
|
||||||
opts?: plugins.mongodb.ChangeStreamOptions & { bufferTimeMs?: number },
|
opts?: plugins.mongodb.ChangeStreamOptions & { bufferTimeMs?: number },
|
||||||
): Promise<SmartdataDbWatcher<T>> {
|
): Promise<SmartdataDbWatcher<T>> {
|
||||||
const collection: SmartdataCollection<T> = (this as any).collection;
|
const collection: SmartdataCollection<T> = (this as any).collection;
|
||||||
@@ -330,7 +591,7 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
|||||||
*/
|
*/
|
||||||
public static async forEach<T>(
|
public static async forEach<T>(
|
||||||
this: plugins.tsclass.typeFest.Class<T>,
|
this: plugins.tsclass.typeFest.Class<T>,
|
||||||
filterArg: plugins.tsclass.typeFest.PartialDeep<T>,
|
filterArg: MongoFilter<T>,
|
||||||
forEachFunction: (itemArg: T) => Promise<any>,
|
forEachFunction: (itemArg: T) => Promise<any>,
|
||||||
) {
|
) {
|
||||||
const cursor: SmartdataDbCursor<T> = await (this as any).getCursor(filterArg);
|
const cursor: SmartdataDbCursor<T> = await (this as any).getCursor(filterArg);
|
||||||
@@ -342,7 +603,7 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
|||||||
*/
|
*/
|
||||||
public static async getCount<T>(
|
public static async getCount<T>(
|
||||||
this: plugins.tsclass.typeFest.Class<T>,
|
this: plugins.tsclass.typeFest.Class<T>,
|
||||||
filterArg: plugins.tsclass.typeFest.PartialDeep<T> = {} as any,
|
filterArg: MongoFilter<T> = {} as any,
|
||||||
) {
|
) {
|
||||||
const collection: SmartdataCollection<T> = (this as any).collection;
|
const collection: SmartdataCollection<T> = (this as any).collection;
|
||||||
return await collection.getCount(filterArg);
|
return await collection.getCount(filterArg);
|
||||||
@@ -558,23 +819,28 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* an array of saveable properties of ALL doc
|
* an array of saveable properties of ALL doc
|
||||||
|
* Note: Set by decorators on prototype - NOT declared as instance property to avoid shadowing in Deno
|
||||||
|
* Declared with definite assignment assertion to satisfy TypeScript without creating instance property
|
||||||
*/
|
*/
|
||||||
public globalSaveableProperties: string[];
|
declare globalSaveableProperties: string[];
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* unique indexes
|
* unique indexes
|
||||||
|
* Note: Set by decorators on prototype - NOT declared as instance property to avoid shadowing in Deno
|
||||||
*/
|
*/
|
||||||
public uniqueIndexes: string[];
|
declare uniqueIndexes: string[];
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* regular indexes with their options
|
* regular indexes with their options
|
||||||
|
* Note: Set by decorators on prototype - NOT declared as instance property to avoid shadowing in Deno
|
||||||
*/
|
*/
|
||||||
public regularIndexes: Array<{field: string, options: IIndexOptions}> = [];
|
declare regularIndexes: Array<{field: string, options: IIndexOptions}>;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* an array of saveable properties of a specific doc
|
* an array of saveable properties of a specific doc
|
||||||
|
* Note: Set by decorators on prototype - NOT declared as instance property to avoid shadowing in Deno
|
||||||
*/
|
*/
|
||||||
public saveableProperties: string[];
|
declare saveableProperties: string[];
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* name
|
* name
|
||||||
@@ -607,14 +873,14 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
|||||||
// perform insert or update
|
// perform insert or update
|
||||||
switch (this.creationStatus) {
|
switch (this.creationStatus) {
|
||||||
case 'db':
|
case 'db':
|
||||||
dbResult = await this.collection.update(self, { session: opts?.session });
|
dbResult = await this.getCollectionSafe().update(self, { session: opts?.session });
|
||||||
break;
|
break;
|
||||||
case 'new':
|
case 'new':
|
||||||
dbResult = await this.collection.insert(self, { session: opts?.session });
|
dbResult = await this.getCollectionSafe().insert(self, { session: opts?.session });
|
||||||
this.creationStatus = 'db';
|
this.creationStatus = 'db';
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
console.error('neither new nor in db?');
|
logger.log('error', 'neither new nor in db?');
|
||||||
}
|
}
|
||||||
// allow hook after saving
|
// allow hook after saving
|
||||||
if (typeof (this as any).afterSave === 'function') {
|
if (typeof (this as any).afterSave === 'function') {
|
||||||
@@ -632,7 +898,7 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
|||||||
await (this as any).beforeDelete();
|
await (this as any).beforeDelete();
|
||||||
}
|
}
|
||||||
// perform deletion
|
// perform deletion
|
||||||
const result = await this.collection.delete(this, { session: opts?.session });
|
const result = await this.getCollectionSafe().delete(this, { session: opts?.session });
|
||||||
// allow hook after delete
|
// allow hook after delete
|
||||||
if (typeof (this as any).afterDelete === 'function') {
|
if (typeof (this as any).afterDelete === 'function') {
|
||||||
await (this as any).afterDelete();
|
await (this as any).afterDelete();
|
||||||
@@ -661,8 +927,11 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
|||||||
/**
|
/**
|
||||||
* updates an object from db
|
* updates an object from db
|
||||||
*/
|
*/
|
||||||
public async updateFromDb() {
|
public async updateFromDb(): Promise<boolean> {
|
||||||
const mongoDbNativeDoc = await this.collection.findOne(await this.createIdentifiableObject());
|
const mongoDbNativeDoc = await this.getCollectionSafe().findOne(await this.createIdentifiableObject());
|
||||||
|
if (!mongoDbNativeDoc) {
|
||||||
|
return false; // Document not found in database
|
||||||
|
}
|
||||||
for (const key of Object.keys(mongoDbNativeDoc)) {
|
for (const key of Object.keys(mongoDbNativeDoc)) {
|
||||||
const rawValue = mongoDbNativeDoc[key];
|
const rawValue = mongoDbNativeDoc[key];
|
||||||
const optionsMap = (this.constructor as any)._svDbOptions || {};
|
const optionsMap = (this.constructor as any)._svDbOptions || {};
|
||||||
@@ -671,6 +940,7 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
|||||||
? opts.deserialize(rawValue)
|
? opts.deserialize(rawValue)
|
||||||
: rawValue;
|
: rawValue;
|
||||||
}
|
}
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -678,7 +948,9 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
|||||||
*/
|
*/
|
||||||
public async createSavableObject(): Promise<TImplements> {
|
public async createSavableObject(): Promise<TImplements> {
|
||||||
const saveableObject: unknown = {}; // is not exposed to outside, so any is ok here
|
const saveableObject: unknown = {}; // is not exposed to outside, so any is ok here
|
||||||
const saveableProperties = [...this.globalSaveableProperties, ...this.saveableProperties];
|
const globalProps = this.globalSaveableProperties || [];
|
||||||
|
const specificProps = this.saveableProperties || [];
|
||||||
|
const saveableProperties = [...globalProps, ...specificProps];
|
||||||
// apply custom serialization if configured
|
// apply custom serialization if configured
|
||||||
const optionsMap = (this.constructor as any)._svDbOptions || {};
|
const optionsMap = (this.constructor as any)._svDbOptions || {};
|
||||||
for (const propertyNameString of saveableProperties) {
|
for (const propertyNameString of saveableProperties) {
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ export class EasyStore<T> {
|
|||||||
public nameId: string;
|
public nameId: string;
|
||||||
|
|
||||||
@svDb()
|
@svDb()
|
||||||
public ephermal: {
|
public ephemeral: {
|
||||||
activated: boolean;
|
activated: boolean;
|
||||||
timeout: number;
|
timeout: number;
|
||||||
};
|
};
|
||||||
@@ -32,8 +32,8 @@ export class EasyStore<T> {
|
|||||||
return SmartdataEasyStore;
|
return SmartdataEasyStore;
|
||||||
})();
|
})();
|
||||||
|
|
||||||
constructor(nameIdArg: string, smnartdataDbRefArg: SmartdataDb) {
|
constructor(nameIdArg: string, smartdataDbRefArg: SmartdataDb) {
|
||||||
this.smartdataDbRef = smnartdataDbRefArg;
|
this.smartdataDbRef = smartdataDbRefArg;
|
||||||
this.nameId = nameIdArg;
|
this.nameId = nameIdArg;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -110,10 +110,12 @@ export class EasyStore<T> {
|
|||||||
await easyStore.save();
|
await easyStore.save();
|
||||||
}
|
}
|
||||||
|
|
||||||
public async cleanUpEphermal() {
|
public async cleanUpEphemeral() {
|
||||||
while (
|
// Clean up ephemeral data periodically while connected
|
||||||
(await this.smartdataDbRef.statusConnectedDeferred.promise) &&
|
while (this.smartdataDbRef.status === 'connected') {
|
||||||
this.smartdataDbRef.status === 'connected'
|
await plugins.smartdelay.delayFor(60000); // Check every minute
|
||||||
) {}
|
// TODO: Implement actual cleanup logic for ephemeral data
|
||||||
|
// For now, this prevents the infinite CPU loop
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
* Lucene to MongoDB query adapter for SmartData
|
* Lucene to MongoDB query adapter for SmartData
|
||||||
*/
|
*/
|
||||||
import * as plugins from './plugins.js';
|
import * as plugins from './plugins.js';
|
||||||
|
import { logger } from './logging.js';
|
||||||
|
|
||||||
// Types
|
// Types
|
||||||
type NodeType =
|
type NodeType =
|
||||||
@@ -754,7 +755,7 @@ export class SmartdataLuceneAdapter {
|
|||||||
// Transform the AST to a MongoDB query
|
// Transform the AST to a MongoDB query
|
||||||
return this.transformWithFields(ast, fieldsToSearch);
|
return this.transformWithFields(ast, fieldsToSearch);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(`Failed to convert Lucene query "${luceneQuery}":`, error);
|
logger.log('error', `Failed to convert Lucene query "${luceneQuery}":`, error);
|
||||||
throw new Error(`Failed to convert Lucene query: ${error}`);
|
throw new Error(`Failed to convert Lucene query: ${error}`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import { SmartDataDbDoc } from './classes.doc.js';
|
import { SmartDataDbDoc } from './classes.doc.js';
|
||||||
import * as plugins from './plugins.js';
|
import * as plugins from './plugins.js';
|
||||||
import { EventEmitter } from 'events';
|
import { EventEmitter } from 'node:events';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* a wrapper for the native mongodb cursor. Exposes better
|
* a wrapper for the native mongodb cursor. Exposes better
|
||||||
|
|||||||
@@ -1,7 +1,5 @@
|
|||||||
{
|
{
|
||||||
"compilerOptions": {
|
"compilerOptions": {
|
||||||
"experimentalDecorators": true,
|
|
||||||
"useDefineForClassFields": false,
|
|
||||||
"target": "ES2022",
|
"target": "ES2022",
|
||||||
"module": "NodeNext",
|
"module": "NodeNext",
|
||||||
"moduleResolution": "NodeNext",
|
"moduleResolution": "NodeNext",
|
||||||
|
|||||||
Reference in New Issue
Block a user