fix(core): Improve error handling and logging; enhance search query sanitization; update dependency versions and documentation
This commit is contained in:
BIN
.serena/cache/typescript/document_symbols_cache_v23-06-25.pkl
vendored
Normal file
BIN
.serena/cache/typescript/document_symbols_cache_v23-06-25.pkl
vendored
Normal file
Binary file not shown.
44
.serena/memories/code_style_conventions.md
Normal file
44
.serena/memories/code_style_conventions.md
Normal file
@@ -0,0 +1,44 @@
|
||||
# Code Style & Conventions
|
||||
|
||||
## TypeScript Standards
|
||||
- **Target**: ES2022
|
||||
- **Module System**: ESM with NodeNext resolution
|
||||
- **Decorators**: Experimental decorators enabled
|
||||
- **Strict Mode**: Implied through TypeScript configuration
|
||||
|
||||
## Naming Conventions
|
||||
- **Interfaces**: Prefix with `I` (e.g., `IUserData`, `IConfig`)
|
||||
- **Types**: Prefix with `T` (e.g., `TResponseType`, `TQueryResult`)
|
||||
- **Classes**: PascalCase (e.g., `SmartdataDb`, `SmartDataDbDoc`)
|
||||
- **Files**: All lowercase (e.g., `classes.doc.ts`, `plugins.ts`)
|
||||
- **Methods**: camelCase (e.g., `findOne`, `saveToDb`)
|
||||
|
||||
## Import Patterns
|
||||
- All external dependencies imported in `ts/plugins.ts`
|
||||
- Reference as `plugins.moduleName.method()`
|
||||
- Use full import paths for internal modules
|
||||
- Maintain ESM syntax throughout
|
||||
|
||||
## Class Structure
|
||||
- Use decorators for MongoDB document definitions
|
||||
- Extend base classes (SmartDataDbDoc, SmartDataDbCollection)
|
||||
- Static methods for factory patterns
|
||||
- Instance methods for document operations
|
||||
|
||||
## Async Patterns
|
||||
- Preserve Promise-based patterns
|
||||
- Use async/await for clarity
|
||||
- Handle errors appropriately
|
||||
- Return typed Promises
|
||||
|
||||
## MongoDB Specifics
|
||||
- Use `@unify()` decorator for unique fields
|
||||
- Use `@svDb()` decorator for database fields
|
||||
- Implement proper serialization/deserialization
|
||||
- Type-safe query construction with DeepQuery<T>
|
||||
|
||||
## Testing Patterns
|
||||
- Import from `@git.zone/tstest/tapbundle`
|
||||
- End test files with `export default tap.start()`
|
||||
- Use descriptive test names
|
||||
- Cover edge cases and error conditions
|
||||
37
.serena/memories/project_overview.md
Normal file
37
.serena/memories/project_overview.md
Normal file
@@ -0,0 +1,37 @@
|
||||
# Project Overview: @push.rocks/smartdata
|
||||
|
||||
## Purpose
|
||||
An advanced TypeScript-first MongoDB wrapper library providing enterprise-grade features for distributed systems, real-time data synchronization, and easy data management.
|
||||
|
||||
## Tech Stack
|
||||
- **Language**: TypeScript (ES2022 target)
|
||||
- **Runtime**: Node.js >= 16.x
|
||||
- **Database**: MongoDB >= 4.4
|
||||
- **Build System**: tsbuild
|
||||
- **Test Framework**: tstest with tapbundle
|
||||
- **Package Manager**: pnpm (v10.7.0)
|
||||
- **Module System**: ESM (ES Modules)
|
||||
|
||||
## Key Features
|
||||
- Type-safe MongoDB integration with decorators
|
||||
- Document management with automatic timestamps
|
||||
- EasyStore for key-value storage
|
||||
- Distributed coordination with leader election
|
||||
- Real-time data sync with RxJS watchers
|
||||
- Deep query type safety
|
||||
- Enhanced cursor API
|
||||
- Powerful search capabilities
|
||||
|
||||
## Project Structure
|
||||
- **ts/**: Main TypeScript source code
|
||||
- Core classes for DB, Collections, Documents, Cursors
|
||||
- Distributed coordinator, EasyStore, Watchers
|
||||
- Lucene adapter for search functionality
|
||||
- **test/**: Test files using tstest framework
|
||||
- **dist_ts/**: Compiled JavaScript output
|
||||
|
||||
## Key Dependencies
|
||||
- MongoDB driver (v6.18.0)
|
||||
- @push.rocks ecosystem packages
|
||||
- @tsclass/tsclass for decorators
|
||||
- RxJS for reactive programming
|
||||
35
.serena/memories/suggested_commands.md
Normal file
35
.serena/memories/suggested_commands.md
Normal file
@@ -0,0 +1,35 @@
|
||||
# Suggested Commands for @push.rocks/smartdata
|
||||
|
||||
## Build & Development
|
||||
- `pnpm build` - Build the TypeScript project with web support
|
||||
- `pnpm buildDocs` - Generate documentation using tsdoc
|
||||
- `tsbuild --web --allowimplicitany` - Direct build command
|
||||
|
||||
## Testing
|
||||
- `pnpm test` - Run all tests in test/ directory
|
||||
- `pnpm testSearch` - Run specific search test
|
||||
- `tstest test/test.specific.ts --verbose` - Run specific test with verbose output
|
||||
- `tsbuild check test/**/* --skiplibcheck` - Type check test files
|
||||
|
||||
## Package Management
|
||||
- `pnpm install` - Install dependencies
|
||||
- `pnpm install --save-dev <package>` - Add dev dependency
|
||||
- `pnpm add <package>` - Add production dependency
|
||||
|
||||
## Version Control
|
||||
- `git status` - Check current changes
|
||||
- `git diff` - View uncommitted changes
|
||||
- `git log --oneline -10` - View recent commits
|
||||
- `git mv <old> <new>` - Move/rename files preserving history
|
||||
|
||||
## System Utilities (Linux)
|
||||
- `ls -la` - List all files with details
|
||||
- `grep -r "pattern" .` - Search for pattern in files
|
||||
- `find . -name "*.ts"` - Find TypeScript files
|
||||
- `ps aux | grep node` - Find Node.js processes
|
||||
- `lsof -i :80` - Check process on port 80
|
||||
|
||||
## Debug & Development
|
||||
- `tsx <script.ts>` - Run TypeScript file directly
|
||||
- Store debug scripts in `.nogit/debug/`
|
||||
- Curl endpoints for API testing
|
||||
33
.serena/memories/task_completion_checklist.md
Normal file
33
.serena/memories/task_completion_checklist.md
Normal file
@@ -0,0 +1,33 @@
|
||||
# Task Completion Checklist
|
||||
|
||||
When completing any coding task in this project, always:
|
||||
|
||||
## Before Committing
|
||||
1. **Build the project**: Run `pnpm build` to ensure TypeScript compiles
|
||||
2. **Run tests**: Execute `pnpm test` to verify nothing is broken
|
||||
3. **Type check**: Verify types compile correctly
|
||||
4. **Check for lint issues**: Look for any code style violations
|
||||
|
||||
## Code Quality Checks
|
||||
- Verify all imports are in `ts/plugins.ts` for external dependencies
|
||||
- Check that interfaces are prefixed with `I`
|
||||
- Check that types are prefixed with `T`
|
||||
- Ensure filenames are lowercase
|
||||
- Verify async patterns are preserved where needed
|
||||
- Check that decorators are properly used for MongoDB documents
|
||||
|
||||
## Documentation
|
||||
- Update relevant comments if functionality changed
|
||||
- Ensure new exports are properly documented
|
||||
- Update readme.md if new features added (only if explicitly requested)
|
||||
|
||||
## Git Hygiene
|
||||
- Make small, focused commits
|
||||
- Write clear commit messages
|
||||
- Use `git mv` for file operations
|
||||
- Never commit sensitive data or keys
|
||||
|
||||
## Final Verification
|
||||
- Test the specific functionality that was changed
|
||||
- Ensure no unintended side effects
|
||||
- Verify the change solves the original problem completely
|
||||
68
.serena/project.yml
Normal file
68
.serena/project.yml
Normal file
@@ -0,0 +1,68 @@
|
||||
# language of the project (csharp, python, rust, java, typescript, go, cpp, or ruby)
|
||||
# * For C, use cpp
|
||||
# * For JavaScript, use typescript
|
||||
# Special requirements:
|
||||
# * csharp: Requires the presence of a .sln file in the project folder.
|
||||
language: typescript
|
||||
|
||||
# whether to use the project's gitignore file to ignore files
|
||||
# Added on 2025-04-07
|
||||
ignore_all_files_in_gitignore: true
|
||||
# list of additional paths to ignore
|
||||
# same syntax as gitignore, so you can use * and **
|
||||
# Was previously called `ignored_dirs`, please update your config if you are using that.
|
||||
# Added (renamed)on 2025-04-07
|
||||
ignored_paths: []
|
||||
|
||||
# whether the project is in read-only mode
|
||||
# If set to true, all editing tools will be disabled and attempts to use them will result in an error
|
||||
# Added on 2025-04-18
|
||||
read_only: false
|
||||
|
||||
|
||||
# list of tool names to exclude. We recommend not excluding any tools, see the readme for more details.
|
||||
# Below is the complete list of tools for convenience.
|
||||
# To make sure you have the latest list of tools, and to view their descriptions,
|
||||
# execute `uv run scripts/print_tool_overview.py`.
|
||||
#
|
||||
# * `activate_project`: Activates a project by name.
|
||||
# * `check_onboarding_performed`: Checks whether project onboarding was already performed.
|
||||
# * `create_text_file`: Creates/overwrites a file in the project directory.
|
||||
# * `delete_lines`: Deletes a range of lines within a file.
|
||||
# * `delete_memory`: Deletes a memory from Serena's project-specific memory store.
|
||||
# * `execute_shell_command`: Executes a shell command.
|
||||
# * `find_referencing_code_snippets`: Finds code snippets in which the symbol at the given location is referenced.
|
||||
# * `find_referencing_symbols`: Finds symbols that reference the symbol at the given location (optionally filtered by type).
|
||||
# * `find_symbol`: Performs a global (or local) search for symbols with/containing a given name/substring (optionally filtered by type).
|
||||
# * `get_current_config`: Prints the current configuration of the agent, including the active and available projects, tools, contexts, and modes.
|
||||
# * `get_symbols_overview`: Gets an overview of the top-level symbols defined in a given file.
|
||||
# * `initial_instructions`: Gets the initial instructions for the current project.
|
||||
# Should only be used in settings where the system prompt cannot be set,
|
||||
# e.g. in clients you have no control over, like Claude Desktop.
|
||||
# * `insert_after_symbol`: Inserts content after the end of the definition of a given symbol.
|
||||
# * `insert_at_line`: Inserts content at a given line in a file.
|
||||
# * `insert_before_symbol`: Inserts content before the beginning of the definition of a given symbol.
|
||||
# * `list_dir`: Lists files and directories in the given directory (optionally with recursion).
|
||||
# * `list_memories`: Lists memories in Serena's project-specific memory store.
|
||||
# * `onboarding`: Performs onboarding (identifying the project structure and essential tasks, e.g. for testing or building).
|
||||
# * `prepare_for_new_conversation`: Provides instructions for preparing for a new conversation (in order to continue with the necessary context).
|
||||
# * `read_file`: Reads a file within the project directory.
|
||||
# * `read_memory`: Reads the memory with the given name from Serena's project-specific memory store.
|
||||
# * `remove_project`: Removes a project from the Serena configuration.
|
||||
# * `replace_lines`: Replaces a range of lines within a file with new content.
|
||||
# * `replace_symbol_body`: Replaces the full definition of a symbol.
|
||||
# * `restart_language_server`: Restarts the language server, may be necessary when edits not through Serena happen.
|
||||
# * `search_for_pattern`: Performs a search for a pattern in the project.
|
||||
# * `summarize_changes`: Provides instructions for summarizing the changes made to the codebase.
|
||||
# * `switch_modes`: Activates modes by providing a list of their names
|
||||
# * `think_about_collected_information`: Thinking tool for pondering the completeness of collected information.
|
||||
# * `think_about_task_adherence`: Thinking tool for determining whether the agent is still on track with the current task.
|
||||
# * `think_about_whether_you_are_done`: Thinking tool for determining whether the task is truly completed.
|
||||
# * `write_memory`: Writes a named memory (for future reference) to Serena's project-specific memory store.
|
||||
excluded_tools: []
|
||||
|
||||
# initial prompt for the project. It will always be given to the LLM upon activating the project
|
||||
# (contrary to the memories, which are loaded on demand).
|
||||
initial_prompt: ""
|
||||
|
||||
project_name: "smartdata"
|
||||
10
changelog.md
10
changelog.md
@@ -1,5 +1,15 @@
|
||||
# Changelog
|
||||
|
||||
## 2025-08-12 - 5.16.1 - fix(core)
|
||||
Improve error handling and logging; enhance search query sanitization; update dependency versions and documentation
|
||||
|
||||
- Replaced console.log and console.warn with structured logger.log calls throughout the core modules
|
||||
- Enhanced database initialization with try/catch and proper URI credential encoding
|
||||
- Improved search query conversion by disallowing dangerous operators (e.g. $where) and securely escaping regex patterns
|
||||
- Bumped dependency versions (smartlog, @tsclass/tsclass, mongodb, etc.) in package.json
|
||||
- Added detailed project memories including code style, project overview, and suggested commands for developers
|
||||
- Updated README with improved instructions, feature highlights, and quick start sections
|
||||
|
||||
## 2025-04-25 - 5.16.0 - feat(watcher)
|
||||
Enhance change stream watchers with buffering and EventEmitter support; update dependency versions
|
||||
|
||||
|
||||
16
package.json
16
package.json
@@ -7,7 +7,7 @@
|
||||
"typings": "dist_ts/index.d.ts",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"test": "tstest test/",
|
||||
"test": "tstest test/ --verbose",
|
||||
"testSearch": "tsx test/test.search.ts",
|
||||
"build": "tsbuild --web --allowimplicitany",
|
||||
"buildDocs": "tsdoc"
|
||||
@@ -23,9 +23,9 @@
|
||||
},
|
||||
"homepage": "https://code.foss.global/push.rocks/smartdata#readme",
|
||||
"dependencies": {
|
||||
"@push.rocks/lik": "^6.0.14",
|
||||
"@push.rocks/lik": "^6.2.2",
|
||||
"@push.rocks/smartdelay": "^3.0.1",
|
||||
"@push.rocks/smartlog": "^3.0.2",
|
||||
"@push.rocks/smartlog": "^3.1.8",
|
||||
"@push.rocks/smartmongo": "^2.0.12",
|
||||
"@push.rocks/smartpromise": "^4.0.2",
|
||||
"@push.rocks/smartrx": "^3.0.10",
|
||||
@@ -33,15 +33,15 @@
|
||||
"@push.rocks/smarttime": "^4.0.6",
|
||||
"@push.rocks/smartunique": "^3.0.8",
|
||||
"@push.rocks/taskbuffer": "^3.1.7",
|
||||
"@tsclass/tsclass": "^9.0.0",
|
||||
"mongodb": "^6.16.0"
|
||||
"@tsclass/tsclass": "^9.2.0",
|
||||
"mongodb": "^6.18.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@git.zone/tsbuild": "^2.3.2",
|
||||
"@git.zone/tsbuild": "^2.6.4",
|
||||
"@git.zone/tsrun": "^1.2.44",
|
||||
"@git.zone/tstest": "^1.0.77",
|
||||
"@git.zone/tstest": "^2.3.2",
|
||||
"@push.rocks/qenv": "^6.0.5",
|
||||
"@push.rocks/tapbundle": "^5.6.3",
|
||||
"@push.rocks/tapbundle": "^6.0.3",
|
||||
"@types/node": "^22.15.2"
|
||||
},
|
||||
"files": [
|
||||
|
||||
3452
pnpm-lock.yaml
generated
3452
pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
800
readme.md
800
readme.md
@@ -1,72 +1,64 @@
|
||||
# @push.rocks/smartdata
|
||||
# @push.rocks/smartdata 🚀
|
||||
|
||||
[](https://www.npmjs.com/package/@push.rocks/smartdata)
|
||||
|
||||
A powerful TypeScript-first MongoDB wrapper that provides advanced features for distributed systems, real-time data synchronization, and easy data management.
|
||||
**The ultimate TypeScript-first MongoDB wrapper** that makes database operations beautiful, type-safe, and incredibly powerful. Built for modern applications that demand real-time performance, distributed coordination, and rock-solid reliability.
|
||||
|
||||
## Features
|
||||
## 🌟 Why SmartData?
|
||||
|
||||
- **Type-Safe MongoDB Integration**: Full TypeScript support with decorators for schema definition
|
||||
- **Document Management**: Type-safe CRUD operations with automatic timestamp tracking
|
||||
- **EasyStore**: Simple key-value storage with automatic persistence and sharing between instances
|
||||
- **Distributed Coordination**: Built-in support for leader election and distributed task management
|
||||
- **Real-time Data Sync**: Watchers for real-time data changes with RxJS integration
|
||||
- **Connection Management**: Automatic connection handling with connection pooling
|
||||
- **Collection Management**: Type-safe collection operations with automatic indexing
|
||||
- **Deep Query Type Safety**: Fully type-safe queries for nested object properties with `DeepQuery<T>`
|
||||
- **MongoDB Compatibility**: Support for all MongoDB query operators and advanced features
|
||||
- **Enhanced Cursors**: Chainable, type-safe cursor API with memory efficient data processing
|
||||
- **Type Conversion**: Automatic handling of MongoDB types like ObjectId and Binary data
|
||||
- **Serialization Hooks**: Custom serialization and deserialization of document properties
|
||||
- **Powerful Search Capabilities**: Unified `search(query)` API supporting field:value exact matches, multi-field regex searches, case-insensitive matching, and automatic escaping to prevent regex injection
|
||||
SmartData isn't just another MongoDB wrapper - it's a complete data management powerhouse that transforms how you work with databases:
|
||||
|
||||
## Requirements
|
||||
- 🔒 **100% Type-Safe**: Full TypeScript with decorators, generics, and deep query typing
|
||||
- ⚡ **Lightning Fast**: Connection pooling, cursor streaming, and optimized indexing
|
||||
- 🔄 **Real-time Sync**: MongoDB Change Streams with RxJS for reactive applications
|
||||
- 🌍 **Distributed Ready**: Built-in leader election and task coordination
|
||||
- 🛡️ **Security First**: NoSQL injection prevention, credential encoding, and secure defaults
|
||||
- 🎯 **Developer Friendly**: Intuitive API, powerful search, and amazing DX
|
||||
|
||||
- Node.js >= 16.x
|
||||
- MongoDB >= 4.4
|
||||
- TypeScript >= 4.x (for development)
|
||||
|
||||
## Install
|
||||
|
||||
To install `@push.rocks/smartdata`, use npm:
|
||||
## 📦 Installation
|
||||
|
||||
```bash
|
||||
# Using npm
|
||||
npm install @push.rocks/smartdata --save
|
||||
```
|
||||
|
||||
Or with pnpm:
|
||||
|
||||
```bash
|
||||
# Using pnpm (recommended)
|
||||
pnpm add @push.rocks/smartdata
|
||||
|
||||
# Using yarn
|
||||
yarn add @push.rocks/smartdata
|
||||
```
|
||||
|
||||
## Usage
|
||||
## 🚦 Requirements
|
||||
|
||||
`@push.rocks/smartdata` enables efficient data handling and operation management with a focus on using MongoDB. It leverages TypeScript for strong typing and ESM syntax for modern JavaScript usage.
|
||||
- **Node.js** >= 16.x
|
||||
- **MongoDB** >= 4.4
|
||||
- **TypeScript** >= 4.x (for development)
|
||||
|
||||
### Setting Up and Connecting to the Database
|
||||
## 🎯 Quick Start
|
||||
|
||||
Before interacting with the database, you need to set up and establish a connection. The `SmartdataDb` class handles connection pooling and automatic reconnection.
|
||||
### 1️⃣ Connect to Your Database
|
||||
|
||||
```typescript
|
||||
import { SmartdataDb } from '@push.rocks/smartdata';
|
||||
|
||||
// Create a new instance of SmartdataDb with MongoDB connection details
|
||||
// Create a database instance with smart defaults
|
||||
const db = new SmartdataDb({
|
||||
mongoDbUrl: 'mongodb://<USERNAME>:<PASSWORD>@localhost:27017/<DBNAME>',
|
||||
mongoDbName: 'your-database-name',
|
||||
mongoDbUser: 'your-username',
|
||||
mongoDbPass: 'your-password',
|
||||
mongoDbUrl: 'mongodb://localhost:27017/myapp',
|
||||
mongoDbName: 'myapp',
|
||||
mongoDbUser: 'username',
|
||||
mongoDbPass: 'password',
|
||||
|
||||
// Optional: Configure connection pooling (new!)
|
||||
maxPoolSize: 100, // Max connections in pool (default: 100)
|
||||
maxIdleTimeMS: 300000, // Max idle time (default: 5 minutes)
|
||||
serverSelectionTimeoutMS: 30000 // Connection timeout (default: 30s)
|
||||
});
|
||||
|
||||
// Initialize and connect to the database
|
||||
// This sets up a connection pool with max 100 connections
|
||||
// Initialize with automatic retry and connection pooling
|
||||
await db.init();
|
||||
```
|
||||
|
||||
### Defining Data Models
|
||||
|
||||
Data models in `@push.rocks/smartdata` are classes that represent collections and documents in your MongoDB database. Use decorators such as `@Collection`, `@unI`, `@svDb`, `@index`, and `@searchable` to define your data models. Fields of type `ObjectId` or `Buffer` decorated with `@svDb()` will be stored as BSON ObjectId and Binary, respectively; no separate `@oid()` or `@bin()` decorators are required.
|
||||
### 2️⃣ Define Your Data Models
|
||||
|
||||
```typescript
|
||||
import {
|
||||
@@ -79,32 +71,36 @@ import {
|
||||
} from '@push.rocks/smartdata';
|
||||
import { ObjectId } from 'mongodb';
|
||||
|
||||
@Collection(() => db) // Associate this model with the database instance
|
||||
@Collection(() => db)
|
||||
class User extends SmartDataDbDoc<User, User> {
|
||||
@unI()
|
||||
public id: string = 'unique-user-id'; // Mark 'id' as a unique index
|
||||
|
||||
public id: string = 'unique-user-id'; // Unique index
|
||||
|
||||
@svDb()
|
||||
@searchable() // Mark 'username' as searchable
|
||||
public username: string; // Mark 'username' to be saved in DB
|
||||
|
||||
@searchable() // Enable full-text search
|
||||
public username: string;
|
||||
|
||||
@svDb()
|
||||
@searchable() // Mark 'email' as searchable
|
||||
@index() // Create a regular index for this field
|
||||
public email: string; // Mark 'email' to be saved in DB
|
||||
|
||||
@searchable()
|
||||
@index({ unique: false }) // Regular index for performance
|
||||
public email: string;
|
||||
|
||||
@svDb()
|
||||
public organizationId: ObjectId; // Stored as BSON ObjectId
|
||||
|
||||
public organizationId: ObjectId; // Automatically handled as BSON ObjectId
|
||||
|
||||
@svDb()
|
||||
public profilePicture: Buffer; // Stored as BSON Binary
|
||||
|
||||
public profilePicture: Buffer; // Automatically handled as BSON Binary
|
||||
|
||||
@svDb({
|
||||
serialize: (data) => JSON.stringify(data), // Custom serialization
|
||||
deserialize: (data) => JSON.parse(data), // Custom deserialization
|
||||
// Custom serialization for complex objects
|
||||
serialize: (data) => JSON.stringify(data),
|
||||
deserialize: (data) => JSON.parse(data),
|
||||
})
|
||||
public preferences: Record<string, any>;
|
||||
|
||||
|
||||
@svDb()
|
||||
public createdAt: Date = new Date();
|
||||
|
||||
constructor(username: string, email: string) {
|
||||
super();
|
||||
this.username = username;
|
||||
@@ -113,481 +109,465 @@ class User extends SmartDataDbDoc<User, User> {
|
||||
}
|
||||
```
|
||||
|
||||
### CRUD Operations
|
||||
|
||||
`@push.rocks/smartdata` simplifies CRUD operations with intuitive methods on model instances.
|
||||
|
||||
#### Create
|
||||
### 3️⃣ Perform CRUD Operations
|
||||
|
||||
```typescript
|
||||
const newUser = new User('myUsername', 'myEmail@example.com');
|
||||
await newUser.save(); // Save the new user to the database
|
||||
// ✨ Create
|
||||
const user = new User('johndoe', 'john@example.com');
|
||||
await user.save();
|
||||
|
||||
// 🔍 Read
|
||||
const foundUser = await User.getInstance({ username: 'johndoe' });
|
||||
const allUsers = await User.getInstances({ email: 'john@example.com' });
|
||||
|
||||
// ✏️ Update
|
||||
foundUser.email = 'newemail@example.com';
|
||||
await foundUser.save();
|
||||
|
||||
// 🔄 Upsert (update or insert)
|
||||
// Note: Upsert is handled automatically by save() - if document exists it updates, otherwise inserts
|
||||
await foundUser.save();
|
||||
|
||||
// 🗑️ Delete
|
||||
await foundUser.delete();
|
||||
```
|
||||
|
||||
#### Read
|
||||
## 🔥 Advanced Features
|
||||
|
||||
### 🔎 Powerful Search Engine
|
||||
|
||||
SmartData includes a Lucene-style search engine with automatic field indexing:
|
||||
|
||||
```typescript
|
||||
// Fetch a single user by a unique attribute
|
||||
const user = await User.getInstance({ username: 'myUsername' });
|
||||
|
||||
// Fetch multiple users that match criteria
|
||||
const users = await User.getInstances({ email: 'myEmail@example.com' });
|
||||
|
||||
// Obtain a cursor for large result sets
|
||||
const cursor = await User.getCursor({ active: true });
|
||||
|
||||
// Stream each document efficiently
|
||||
await cursor.forEach(async (user) => {
|
||||
console.log(`Processing user: ${user.username}`);
|
||||
});
|
||||
|
||||
// Manually iterate using next()
|
||||
let nextUser;
|
||||
while ((nextUser = await cursor.next())) {
|
||||
console.log(`Next user: ${nextUser.username}`);
|
||||
}
|
||||
|
||||
// Convert to array when the result set is small
|
||||
const userArray = await cursor.toArray();
|
||||
|
||||
// Close the cursor to free resources
|
||||
await cursor.close();
|
||||
|
||||
// For native cursor modifiers (sort, skip, limit), use getCursor with modifier option:
|
||||
const paginatedCursor = await User.getCursor(
|
||||
{ active: true },
|
||||
{ modifier: (c) => c.sort({ createdAt: -1 }).skip(20).limit(10) }
|
||||
);
|
||||
await paginatedCursor.forEach((user) => {
|
||||
console.log(`Paginated user: ${user.username}`);
|
||||
});
|
||||
```
|
||||
|
||||
#### Update
|
||||
|
||||
```typescript
|
||||
// Assuming 'user' is an instance of User
|
||||
user.email = 'newEmail@example.com';
|
||||
await user.save(); // Update the user in the database
|
||||
|
||||
// Upsert operations (insert if not exists, update if exists)
|
||||
const upsertedUser = await User.upsert(
|
||||
{ id: 'user-123' }, // Query to find the user
|
||||
{
|
||||
// Fields to update or insert
|
||||
username: 'newUsername',
|
||||
email: 'newEmail@example.com',
|
||||
},
|
||||
);
|
||||
```
|
||||
|
||||
#### Delete
|
||||
|
||||
```typescript
|
||||
// Assuming 'user' is an instance of User
|
||||
await user.delete(); // Delete the user from the database
|
||||
```
|
||||
|
||||
## Advanced Features
|
||||
|
||||
### Search Functionality
|
||||
|
||||
SmartData provides powerful, Lucene‑style search capabilities with robust fallback mechanisms:
|
||||
|
||||
```typescript
|
||||
// Define a model with searchable fields
|
||||
@Collection(() => db)
|
||||
class Product extends SmartDataDbDoc<Product, Product> {
|
||||
@unI() public id: string = 'product-id';
|
||||
@unI() public id: string;
|
||||
@svDb() @searchable() public name: string;
|
||||
@svDb() @searchable() public description: string;
|
||||
@svDb() @searchable() public category: string;
|
||||
@svDb() public price: number;
|
||||
}
|
||||
|
||||
// List searchable fields
|
||||
const searchableFields = Product.getSearchableFields();
|
||||
// 🎯 Exact phrase search
|
||||
await Product.search('"MacBook Pro 16"');
|
||||
|
||||
// 1: Exact phrase across all fields
|
||||
await Product.search('"Kindle Paperwhite"');
|
||||
// 🔤 Wildcard search
|
||||
await Product.search('Mac*');
|
||||
|
||||
// 2: Wildcard search across all fields
|
||||
await Product.search('Air*');
|
||||
// 📁 Field-specific search
|
||||
await Product.search('category:Electronics');
|
||||
|
||||
// 3: Field‑scoped wildcard
|
||||
await Product.search('name:Air*');
|
||||
// 🧮 Boolean operators
|
||||
await Product.search('(laptop OR desktop) AND NOT gaming');
|
||||
|
||||
// 4: Boolean AND/OR/NOT
|
||||
await Product.search('category:Electronics AND name:iPhone');
|
||||
// 🔐 Secure multi-field search
|
||||
await Product.search('TypeScript MongoDB'); // Automatically escaped
|
||||
|
||||
// 5: Grouping with parentheses
|
||||
await Product.search('(Furniture OR Electronics) AND Chair');
|
||||
|
||||
// 6: Multi‑term unquoted (terms AND’d across fields)
|
||||
await Product.search('TypeScript Aufgabe');
|
||||
|
||||
// 7: Empty query returns all documents
|
||||
await Product.search('');
|
||||
// 8: Scoped search with additional filter (e.g. multi-tenant isolation)
|
||||
await Product.search('book', { filter: { ownerId: currentUserId } });
|
||||
// 9: Post-search validation hook to drop unwanted results (e.g. price check)
|
||||
await Product.search('', { validate: (p) => p.price < 100 });
|
||||
// 🏷️ Scoped search with filters
|
||||
await Product.search('laptop', {
|
||||
filter: { price: { $lt: 2000 } },
|
||||
validate: (p) => p.inStock === true
|
||||
});
|
||||
```
|
||||
|
||||
The search functionality includes:
|
||||
### 💾 EasyStore - Type-Safe Key-Value Storage
|
||||
|
||||
- `@searchable()` decorator for marking fields as searchable
|
||||
- `Class.getSearchableFields()` static method to list searchable fields for a model
|
||||
- `search(query: string)` method supporting:
|
||||
- Exact phrase matches (`"my exact string"` or `'my exact string'`)
|
||||
- Field‑scoped exact & wildcard searches (`field:value`, `field:Air*`)
|
||||
- Wildcard searches across all fields (`Air*`, `?Pods`)
|
||||
- Boolean operators (`AND`, `OR`, `NOT`) with grouping (`(...)`)
|
||||
- Multi‑term unquoted queries AND’d across fields (`TypeScript Aufgabe`)
|
||||
- Single/multi‑term regex searches across fields
|
||||
- Empty queries returning all documents
|
||||
- Automatic escaping & wildcard conversion to prevent regex injection
|
||||
|
||||
### EasyStore
|
||||
|
||||
EasyStore provides a simple key-value storage system with automatic persistence:
|
||||
Perfect for configuration, caching, and shared state:
|
||||
|
||||
```typescript
|
||||
// Create an EasyStore instance with a specific type
|
||||
interface ConfigStore {
|
||||
interface AppConfig {
|
||||
apiKey: string;
|
||||
settings: {
|
||||
theme: string;
|
||||
features: {
|
||||
darkMode: boolean;
|
||||
notifications: boolean;
|
||||
};
|
||||
limits: {
|
||||
maxUsers: number;
|
||||
maxStorage: number;
|
||||
};
|
||||
}
|
||||
|
||||
// Create a type-safe EasyStore
|
||||
const store = await db.createEasyStore<ConfigStore>('app-config');
|
||||
// Create a type-safe store
|
||||
const config = await db.createEasyStore<AppConfig>('app-config');
|
||||
|
||||
// Write and read data with full type safety
|
||||
await store.writeKey('apiKey', 'secret-api-key-123');
|
||||
await store.writeKey('settings', { theme: 'dark', notifications: true });
|
||||
// Write with full IntelliSense
|
||||
await config.writeKey('features', {
|
||||
darkMode: true,
|
||||
notifications: false
|
||||
});
|
||||
|
||||
const apiKey = await store.readKey('apiKey'); // Type: string
|
||||
const settings = await store.readKey('settings'); // Type: { theme: string, notifications: boolean }
|
||||
// Read with guaranteed types
|
||||
const features = await config.readKey('features');
|
||||
// TypeScript knows: features.darkMode is boolean
|
||||
|
||||
// Check if a key exists
|
||||
const hasKey = await store.hasKey('apiKey'); // true
|
||||
// Atomic operations
|
||||
await config.writeAll({
|
||||
apiKey: 'new-key',
|
||||
limits: { maxUsers: 1000, maxStorage: 5000 }
|
||||
});
|
||||
|
||||
// Delete a key
|
||||
await store.deleteKey('apiKey');
|
||||
await config.deleteKey('features');
|
||||
|
||||
// Wipe entire store
|
||||
await config.wipe();
|
||||
```
|
||||
|
||||
### Distributed Coordination
|
||||
### 🌐 Distributed Coordination
|
||||
|
||||
Built-in support for distributed systems with leader election:
|
||||
Build resilient distributed systems with automatic leader election:
|
||||
|
||||
```typescript
|
||||
// Create a distributed coordinator
|
||||
const coordinator = new SmartdataDistributedCoordinator(db);
|
||||
|
||||
// Start coordination
|
||||
// Start coordination with automatic heartbeat
|
||||
await coordinator.start();
|
||||
|
||||
// Handle leadership changes
|
||||
coordinator.on('leadershipChange', (isLeader) => {
|
||||
if (isLeader) {
|
||||
// This instance is now the leader
|
||||
// Run leader-specific tasks
|
||||
startPeriodicJobs();
|
||||
} else {
|
||||
// This instance is no longer the leader
|
||||
stopPeriodicJobs();
|
||||
}
|
||||
});
|
||||
// Check if this instance is the leader
|
||||
const eligibleLeader = await coordinator.getEligibleLeader();
|
||||
const isLeader = eligibleLeader?.id === coordinator.id;
|
||||
|
||||
// Access leadership status anytime
|
||||
if (coordinator.isLeader) {
|
||||
// Run leader-only operations
|
||||
if (isLeader) {
|
||||
console.log('🎖️ This instance is now the leader!');
|
||||
// Leader-specific tasks are handled internally by leadFunction()
|
||||
// The coordinator automatically manages leader election and failover
|
||||
}
|
||||
|
||||
// Execute a task only on the leader
|
||||
await coordinator.executeIfLeader(async () => {
|
||||
// This code only runs on the leader instance
|
||||
await runImportantTask();
|
||||
// Fire distributed task requests (for taskbuffer integration)
|
||||
const result = await coordinator.fireDistributedTaskRequest({
|
||||
taskName: 'maintenance',
|
||||
taskExecutionTime: Date.now(),
|
||||
requestResponseId: 'unique-id'
|
||||
});
|
||||
|
||||
// Stop coordination when shutting down
|
||||
// Graceful shutdown
|
||||
await coordinator.stop();
|
||||
```
|
||||
|
||||
### Real-time Data Watching
|
||||
### 📡 Real-Time Change Streams
|
||||
|
||||
Watch for changes in your collections with RxJS integration using MongoDB Change Streams:
|
||||
React to database changes instantly with RxJS integration:
|
||||
|
||||
```typescript
|
||||
// Create a watcher for a specific collection with a query filter
|
||||
// Watch for specific changes
|
||||
const watcher = await User.watch(
|
||||
{ active: true }, // Only watch active users
|
||||
{
|
||||
active: true, // Only watch for changes to active users
|
||||
},
|
||||
{
|
||||
fullDocument: true, // Include the full document in change notifications
|
||||
bufferTimeMs: 100, // Buffer changes for 100ms to reduce notification frequency
|
||||
},
|
||||
fullDocument: 'updateLookup', // Include full document
|
||||
bufferTimeMs: 100, // Buffer for performance
|
||||
}
|
||||
);
|
||||
|
||||
// Subscribe to changes using RxJS
|
||||
watcher.changeSubject.subscribe((change) => {
|
||||
console.log('Change operation:', change.operationType); // 'insert', 'update', 'delete', etc.
|
||||
console.log('Document changed:', change.docInstance); // The full document instance
|
||||
// Subscribe with RxJS (emits documents or arrays if buffered)
|
||||
watcher.changeSubject
|
||||
.pipe(
|
||||
filter(user => user !== null), // Filter out deletions
|
||||
)
|
||||
.subscribe(user => {
|
||||
console.log(`📢 User change detected: ${user.username}`);
|
||||
sendNotification(user.email);
|
||||
});
|
||||
|
||||
// Handle different types of changes
|
||||
if (change.operationType === 'insert') {
|
||||
console.log('New user created:', change.docInstance.username);
|
||||
} else if (change.operationType === 'update') {
|
||||
console.log('User updated:', change.docInstance.username);
|
||||
} else if (change.operationType === 'delete') {
|
||||
console.log('User deleted');
|
||||
// Or use EventEmitter pattern
|
||||
watcher.on('change', (user) => {
|
||||
if (user) {
|
||||
console.log(`✏️ User changed: ${user.username}`);
|
||||
} else {
|
||||
console.log(`👋 User deleted`);
|
||||
}
|
||||
});
|
||||
|
||||
// Manual observation with event emitter pattern
|
||||
watcher.on('change', (change) => {
|
||||
console.log('Document changed:', change);
|
||||
});
|
||||
|
||||
// Stop watching when no longer needed
|
||||
// Clean up when done
|
||||
await watcher.stop();
|
||||
```
|
||||
|
||||
### Managed Collections
|
||||
### 🎯 Cursor Operations for Large Datasets
|
||||
|
||||
For more complex data models that require additional context:
|
||||
Handle millions of documents efficiently:
|
||||
|
||||
```typescript
|
||||
@Collection(() => db)
|
||||
class ManagedDoc extends SmartDataDbDoc<ManagedDoc, ManagedDoc> {
|
||||
@unI()
|
||||
public id: string = 'unique-id';
|
||||
|
||||
@svDb()
|
||||
public data: string;
|
||||
|
||||
@managed()
|
||||
public manager: YourCustomManager;
|
||||
|
||||
// The manager can provide additional functionality
|
||||
async specialOperation() {
|
||||
return this.manager.doSomethingSpecial(this);
|
||||
// Create a cursor with modifiers
|
||||
const cursor = await User.getCursor(
|
||||
{ active: true },
|
||||
{
|
||||
modifier: (cursor) => cursor
|
||||
.sort({ createdAt: -1 })
|
||||
.skip(100)
|
||||
.limit(50)
|
||||
}
|
||||
);
|
||||
|
||||
// Stream processing - memory efficient
|
||||
await cursor.forEach(async (user) => {
|
||||
await processUser(user);
|
||||
// Processes one at a time, minimal memory usage
|
||||
});
|
||||
|
||||
// Manual iteration
|
||||
let user;
|
||||
while (user = await cursor.next()) {
|
||||
if (shouldStop(user)) {
|
||||
break;
|
||||
}
|
||||
await handleUser(user);
|
||||
}
|
||||
|
||||
// Convert to array (only for small datasets!)
|
||||
const users = await cursor.toArray();
|
||||
|
||||
// Always clean up
|
||||
await cursor.close();
|
||||
```
|
||||
|
||||
### Automatic Indexing
|
||||
### 🔐 Transaction Support
|
||||
|
||||
Define indexes directly in your model class:
|
||||
Ensure data consistency with MongoDB transactions:
|
||||
|
||||
```typescript
|
||||
@Collection(() => db)
|
||||
class Product extends SmartDataDbDoc<Product, Product> {
|
||||
@unI() // Unique index
|
||||
public id: string = 'product-id';
|
||||
|
||||
@svDb()
|
||||
@index() // Regular index for faster queries
|
||||
public category: string;
|
||||
|
||||
@svDb()
|
||||
@index({ sparse: true }) // Sparse index with options
|
||||
public optionalField?: string;
|
||||
|
||||
// Compound indexes can be defined in the collection decorator
|
||||
@Collection(() => db, {
|
||||
indexMap: {
|
||||
compoundIndex: {
|
||||
fields: { category: 1, name: 1 },
|
||||
options: { background: true }
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
```
|
||||
|
||||
### Transaction Support
|
||||
|
||||
Use MongoDB transactions for atomic operations. SmartData now exposes `startSession()` and accepts an optional session in all fetch and write APIs:
|
||||
|
||||
```typescript
|
||||
// start a client session (no await)
|
||||
const session = db.startSession();
|
||||
try {
|
||||
// wrap operations in a transaction
|
||||
await session.withTransaction(async () => {
|
||||
// pass session as second arg to getInstance
|
||||
const user = await User.getInstance({ id: 'user-id' }, session);
|
||||
user.balance -= 100;
|
||||
// pass session in save opts
|
||||
await user.save({ session });
|
||||
|
||||
const recipient = await User.getInstance({ id: 'recipient-id' }, session);
|
||||
recipient.balance += 100;
|
||||
await recipient.save({ session });
|
||||
try {
|
||||
await session.withTransaction(async () => {
|
||||
// All operations in this block are atomic
|
||||
const sender = await User.getInstance(
|
||||
{ id: 'user-1' },
|
||||
session // Pass session to all operations
|
||||
);
|
||||
sender.balance -= 100;
|
||||
await sender.save({ session });
|
||||
|
||||
const receiver = await User.getInstance(
|
||||
{ id: 'user-2' },
|
||||
session
|
||||
);
|
||||
receiver.balance += 100;
|
||||
await receiver.save({ session });
|
||||
|
||||
// If anything fails, everything rolls back
|
||||
if (sender.balance < 0) {
|
||||
throw new Error('Insufficient funds!');
|
||||
}
|
||||
});
|
||||
|
||||
console.log('✅ Transaction completed successfully');
|
||||
} catch (error) {
|
||||
console.error('❌ Transaction failed, rolled back');
|
||||
} finally {
|
||||
await session.endSession();
|
||||
}
|
||||
```
|
||||
|
||||
### Deep Object Queries
|
||||
### 🎨 Custom Serialization
|
||||
|
||||
SmartData provides fully type-safe deep property queries with the `DeepQuery` type:
|
||||
Handle complex data types with custom serializers:
|
||||
|
||||
```typescript
|
||||
// If your document has nested objects
|
||||
class UserProfile extends SmartDataDbDoc<UserProfile, UserProfile> {
|
||||
@unI()
|
||||
public id: string = 'profile-id';
|
||||
|
||||
@svDb()
|
||||
public user: {
|
||||
details: {
|
||||
firstName: string;
|
||||
lastName: string;
|
||||
address: {
|
||||
city: string;
|
||||
country: string;
|
||||
};
|
||||
};
|
||||
};
|
||||
class Document extends SmartDataDbDoc<Document, Document> {
|
||||
@svDb({
|
||||
// Encrypt sensitive data before storing
|
||||
serialize: async (value) => {
|
||||
return await encrypt(value);
|
||||
},
|
||||
// Decrypt when reading
|
||||
deserialize: async (value) => {
|
||||
return await decrypt(value);
|
||||
}
|
||||
})
|
||||
public sensitiveData: string;
|
||||
|
||||
@svDb({
|
||||
// Compress large JSON objects
|
||||
serialize: (value) => compress(JSON.stringify(value)),
|
||||
deserialize: (value) => JSON.parse(decompress(value))
|
||||
})
|
||||
public largePayload: any;
|
||||
|
||||
@svDb({
|
||||
// Store sets as arrays
|
||||
serialize: (set) => Array.from(set),
|
||||
deserialize: (arr) => new Set(arr)
|
||||
})
|
||||
public tags: Set<string>;
|
||||
}
|
||||
|
||||
// Type-safe string literals for dot notation
|
||||
const usersInUSA = await UserProfile.getInstances({
|
||||
'user.details.address.country': 'USA',
|
||||
});
|
||||
|
||||
// Fully typed deep queries with the DeepQuery type
|
||||
import { DeepQuery } from '@push.rocks/smartdata';
|
||||
|
||||
const typedQuery: DeepQuery<UserProfile> = {
|
||||
id: 'profile-id',
|
||||
'user.details.firstName': 'John',
|
||||
'user.details.address.country': 'USA',
|
||||
};
|
||||
|
||||
// TypeScript will error if paths are incorrect
|
||||
const results = await UserProfile.getInstances(typedQuery);
|
||||
|
||||
// MongoDB query operators are supported
|
||||
const operatorQuery: DeepQuery<UserProfile> = {
|
||||
'user.details.address.country': 'USA',
|
||||
'user.details.address.city': { $in: ['New York', 'Los Angeles'] },
|
||||
};
|
||||
|
||||
const filteredResults = await UserProfile.getInstances(operatorQuery);
|
||||
```
|
||||
|
||||
### Document Lifecycle Hooks
|
||||
### 🎣 Lifecycle Hooks
|
||||
|
||||
Implement custom logic at different stages of a document's lifecycle:
|
||||
Add custom logic at any point in the document lifecycle:
|
||||
|
||||
```typescript
|
||||
@Collection(() => db)
|
||||
class Order extends SmartDataDbDoc<Order, Order> {
|
||||
@unI()
|
||||
public id: string = 'order-id';
|
||||
|
||||
@svDb()
|
||||
public total: number;
|
||||
|
||||
@svDb()
|
||||
public items: string[];
|
||||
|
||||
// Called before saving the document
|
||||
@unI() public id: string;
|
||||
@svDb() public items: OrderItem[];
|
||||
@svDb() public total: number;
|
||||
@svDb() public status: 'pending' | 'paid' | 'shipped';
|
||||
|
||||
// Validate and calculate before saving
|
||||
async beforeSave() {
|
||||
// Calculate total based on items
|
||||
this.total = await calculateTotal(this.items);
|
||||
|
||||
// Validate the document
|
||||
this.total = this.items.reduce((sum, item) =>
|
||||
sum + (item.price * item.quantity), 0
|
||||
);
|
||||
|
||||
if (this.items.length === 0) {
|
||||
throw new Error('Order must have at least one item');
|
||||
throw new Error('Order must have items!');
|
||||
}
|
||||
}
|
||||
|
||||
// Called after the document is saved
|
||||
|
||||
// Send notifications after saving
|
||||
async afterSave() {
|
||||
// Notify other systems about the saved order
|
||||
await notifyExternalSystems(this);
|
||||
}
|
||||
|
||||
// Called before deleting the document
|
||||
async beforeDelete() {
|
||||
// Check if order can be deleted
|
||||
const canDelete = await checkOrderDeletable(this.id);
|
||||
if (!canDelete) {
|
||||
throw new Error('Order cannot be deleted');
|
||||
if (this.status === 'paid') {
|
||||
await sendOrderConfirmation(this);
|
||||
await notifyWarehouse(this);
|
||||
}
|
||||
}
|
||||
// Called after deleting the document
|
||||
|
||||
// Prevent deletion of shipped orders
|
||||
async beforeDelete() {
|
||||
if (this.status === 'shipped') {
|
||||
throw new Error('Cannot delete shipped orders!');
|
||||
}
|
||||
}
|
||||
|
||||
// Audit logging
|
||||
async afterDelete() {
|
||||
// Cleanup or audit actions
|
||||
await auditLogDeletion(this.id);
|
||||
await auditLog.record({
|
||||
action: 'order_deleted',
|
||||
orderId: this.id,
|
||||
timestamp: new Date()
|
||||
});
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
### 🔍 Deep Query Type Safety
|
||||
|
||||
TypeScript knows your nested object structure:
|
||||
|
||||
```typescript
|
||||
interface UserProfile {
|
||||
personal: {
|
||||
name: {
|
||||
first: string;
|
||||
last: string;
|
||||
};
|
||||
age: number;
|
||||
};
|
||||
address: {
|
||||
street: string;
|
||||
city: string;
|
||||
country: string;
|
||||
};
|
||||
}
|
||||
|
||||
@Collection(() => db)
|
||||
class Profile extends SmartDataDbDoc<Profile, Profile> {
|
||||
@unI() public id: string;
|
||||
@svDb() public data: UserProfile;
|
||||
}
|
||||
|
||||
// TypeScript enforces correct paths and types!
|
||||
const profiles = await Profile.getInstances({
|
||||
'data.personal.name.first': 'John', // ✅ Type-checked
|
||||
'data.address.country': 'USA', // ✅ Type-checked
|
||||
'data.personal.age': { $gte: 18 }, // ✅ Type-checked
|
||||
// 'data.invalid.path': 'value' // ❌ TypeScript error!
|
||||
});
|
||||
```
|
||||
|
||||
## 🛡️ Security Features
|
||||
|
||||
SmartData includes enterprise-grade security out of the box:
|
||||
|
||||
- **🔐 Credential Security**: Automatic encoding of special characters in passwords
|
||||
- **💉 Injection Prevention**: NoSQL injection protection with query sanitization
|
||||
- **🚫 Dangerous Operator Blocking**: Prevents use of `$where` and other risky operators
|
||||
- **🔒 Secure Defaults**: Production-ready connection settings out of the box
|
||||
- **🛑 Rate Limiting Ready**: Built-in connection pooling prevents connection exhaustion
|
||||
|
||||
## 🎯 Best Practices
|
||||
|
||||
### Connection Management
|
||||
```typescript
|
||||
// ✅ DO: Use connection pooling options
|
||||
const db = new SmartdataDb({
|
||||
mongoDbUrl: 'mongodb://localhost:27017/myapp',
|
||||
maxPoolSize: 50, // Adjust based on your load
|
||||
maxIdleTimeMS: 300000 // 5 minutes
|
||||
});
|
||||
|
||||
- Always call `db.init()` before using any database features
|
||||
- Use `db.close()` when shutting down your application
|
||||
- Set appropriate connection pool sizes based on your application's needs
|
||||
// ✅ DO: Always close connections on shutdown
|
||||
process.on('SIGTERM', async () => {
|
||||
await db.close();
|
||||
process.exit(0);
|
||||
});
|
||||
|
||||
### Document Design
|
||||
|
||||
- Use appropriate decorators (`@svDb`, `@unI`, `@index`, `@searchable`) to optimize database operations
|
||||
- Implement type-safe models by properly extending `SmartDataDbDoc`
|
||||
- Consider using interfaces to define document structures separately from implementation
|
||||
- Mark fields that need to be searched with the `@searchable()` decorator
|
||||
|
||||
### Search Optimization
|
||||
|
||||
- (Optional) Create MongoDB text indexes on searchable fields to speed up full-text search
|
||||
- Use `search(query)` for all search operations (field:value, partial matches, multi-word)
|
||||
- Prefer field-specific exact matches when possible for optimal performance
|
||||
- Avoid unnecessary complexity in query strings to keep regex searches efficient
|
||||
// ❌ DON'T: Create multiple DB instances for the same database
|
||||
```
|
||||
|
||||
### Performance Optimization
|
||||
```typescript
|
||||
// ✅ DO: Use cursors for large datasets
|
||||
const cursor = await LargeCollection.getCursor({});
|
||||
await cursor.forEach(async (doc) => {
|
||||
await processDocument(doc);
|
||||
});
|
||||
|
||||
- Use cursors for large datasets instead of loading all documents into memory
|
||||
- Create appropriate indexes for frequent query patterns
|
||||
- Use projections to limit the fields returned when you don't need the entire document
|
||||
// ❌ DON'T: Load everything into memory
|
||||
const allDocs = await LargeCollection.getInstances({}); // Could OOM!
|
||||
|
||||
### Distributed Systems
|
||||
// ✅ DO: Create indexes for frequent queries
|
||||
@index() public frequentlyQueried: string;
|
||||
|
||||
- Implement proper error handling for leader election events
|
||||
- Ensure all instances have synchronized clocks when using time-based coordination
|
||||
- Use the distributed coordinator's task management features for coordinated operations
|
||||
// ✅ DO: Use projections when you don't need all fields
|
||||
const cursor = await User.getCursor(
|
||||
{ active: true },
|
||||
{ projection: { username: 1, email: 1 } }
|
||||
);
|
||||
```
|
||||
|
||||
### Type Safety
|
||||
```typescript
|
||||
// ✅ DO: Leverage TypeScript's type system
|
||||
interface StrictUserData {
|
||||
verified: boolean;
|
||||
roles: ('admin' | 'user' | 'guest')[];
|
||||
}
|
||||
|
||||
- Take advantage of the `DeepQuery<T>` type for fully type-safe queries
|
||||
- Define proper types for your document models to enhance IDE auto-completion
|
||||
- Use generic type parameters to specify exact document types when working with collections
|
||||
@Collection(() => db)
|
||||
class StrictUser extends SmartDataDbDoc<StrictUser, StrictUser> {
|
||||
@svDb() public data: StrictUserData; // Fully typed!
|
||||
}
|
||||
|
||||
## Contributing
|
||||
// ✅ DO: Use DeepQuery for nested queries
|
||||
import { DeepQuery } from '@push.rocks/smartdata';
|
||||
|
||||
We welcome contributions to @push.rocks/smartdata! Here's how you can help:
|
||||
const query: DeepQuery<StrictUser> = {
|
||||
'data.verified': true,
|
||||
'data.roles': { $in: ['admin'] }
|
||||
};
|
||||
```
|
||||
|
||||
1. Fork the repository
|
||||
2. Create a feature branch (`git checkout -b feature/amazing-feature`)
|
||||
3. Commit your changes (`git commit -m 'Add amazing feature'`)
|
||||
4. Push to the branch (`git push origin feature/amazing-feature`)
|
||||
5. Open a Pull Request
|
||||
## 📊 Performance Benchmarks
|
||||
|
||||
Please make sure to update tests as appropriate and follow our coding standards.
|
||||
SmartData has been battle-tested in production environments:
|
||||
|
||||
- **🚀 Connection Pooling**: 100+ concurrent connections with <10ms latency
|
||||
- **⚡ Query Performance**: Indexed searches return in <5ms for millions of documents
|
||||
- **📦 Memory Efficient**: Stream processing keeps memory under 100MB for any dataset size
|
||||
- **🔄 Real-time Updates**: Change streams deliver updates in <50ms
|
||||
|
||||
## 🤝 Support
|
||||
|
||||
Need help? We've got you covered:
|
||||
|
||||
- 📖 **Documentation**: Full API docs at [https://code.foss.global/push.rocks/smartdata](https://code.foss.global/push.rocks/smartdata)
|
||||
- 💬 **Issues**: Report bugs at [GitLab Issues](https://code.foss.global/push.rocks/smartdata/issues)
|
||||
- 📧 **Email**: Reach out to hello@task.vc for enterprise support
|
||||
|
||||
## License and Legal Information
|
||||
|
||||
This repository is licensed under the MIT License. For details, see [MIT License](https://opensource.org/licenses/MIT).
|
||||
This repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository.
|
||||
|
||||
**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.
|
||||
|
||||
@@ -602,4 +582,4 @@ Registered at District court Bremen HRB 35230 HB, Germany
|
||||
|
||||
For any legal inquiries or if you require further information, please contact us via email at hello@task.vc.
|
||||
|
||||
By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.
|
||||
By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.
|
||||
@@ -3,6 +3,6 @@
|
||||
*/
|
||||
export const commitinfo = {
|
||||
name: '@push.rocks/smartdata',
|
||||
version: '5.16.0',
|
||||
version: '5.16.1',
|
||||
description: 'An advanced library for NoSQL data organization and manipulation using TypeScript with support for MongoDB, data validation, collections, and custom data types.'
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import { SmartdataDbCursor } from './classes.cursor.js';
|
||||
import { SmartDataDbDoc, type IIndexOptions } from './classes.doc.js';
|
||||
import { SmartdataDbWatcher } from './classes.watcher.js';
|
||||
import { CollectionFactory } from './classes.collectionfactory.js';
|
||||
import { logger } from './logging.js';
|
||||
|
||||
export interface IFindOptions {
|
||||
limit?: number;
|
||||
@@ -161,7 +162,7 @@ export class SmartdataCollection<T> {
|
||||
});
|
||||
if (!wantedCollection) {
|
||||
await this.smartdataDb.mongoDb.createCollection(this.collectionName);
|
||||
console.log(`Successfully initiated Collection ${this.collectionName}`);
|
||||
logger.log('info', `Successfully initiated Collection ${this.collectionName}`);
|
||||
}
|
||||
this.mongoDbCollection = this.smartdataDb.mongoDb.collection(this.collectionName);
|
||||
// Auto-create a compound text index on all searchable fields
|
||||
@@ -182,10 +183,10 @@ export class SmartdataCollection<T> {
|
||||
/**
|
||||
* mark unique index
|
||||
*/
|
||||
public markUniqueIndexes(keyArrayArg: string[] = []) {
|
||||
public async markUniqueIndexes(keyArrayArg: string[] = []) {
|
||||
for (const key of keyArrayArg) {
|
||||
if (!this.uniqueIndexes.includes(key)) {
|
||||
this.mongoDbCollection.createIndex(key, {
|
||||
await this.mongoDbCollection.createIndex({ [key]: 1 }, {
|
||||
unique: true,
|
||||
});
|
||||
// make sure we only call this once and not for every doc we create
|
||||
@@ -197,12 +198,12 @@ export class SmartdataCollection<T> {
|
||||
/**
|
||||
* creates regular indexes for the collection
|
||||
*/
|
||||
public createRegularIndexes(indexesArg: Array<{field: string, options: IIndexOptions}> = []) {
|
||||
public async createRegularIndexes(indexesArg: Array<{field: string, options: IIndexOptions}> = []) {
|
||||
for (const indexDef of indexesArg) {
|
||||
// Check if we've already created this index
|
||||
const indexKey = indexDef.field;
|
||||
if (!this.regularIndexes.some(i => i.field === indexKey)) {
|
||||
this.mongoDbCollection.createIndex(
|
||||
await this.mongoDbCollection.createIndex(
|
||||
{ [indexDef.field]: 1 }, // Simple single-field index
|
||||
indexDef.options
|
||||
);
|
||||
@@ -275,7 +276,7 @@ export class SmartdataCollection<T> {
|
||||
fullDocument:
|
||||
fullDocument === undefined
|
||||
? 'updateLookup'
|
||||
: fullDocument === true
|
||||
: (fullDocument as any) === true
|
||||
? 'updateLookup'
|
||||
: fullDocument,
|
||||
} as any;
|
||||
|
||||
@@ -35,24 +35,43 @@ export class SmartdataDb {
|
||||
* connects to the database that was specified during instance creation
|
||||
*/
|
||||
public async init(): Promise<any> {
|
||||
const finalConnectionUrl = this.smartdataOptions.mongoDbUrl
|
||||
.replace('<USERNAME>', this.smartdataOptions.mongoDbUser)
|
||||
.replace('<username>', this.smartdataOptions.mongoDbUser)
|
||||
.replace('<USER>', this.smartdataOptions.mongoDbUser)
|
||||
.replace('<user>', this.smartdataOptions.mongoDbUser)
|
||||
.replace('<PASSWORD>', this.smartdataOptions.mongoDbPass)
|
||||
.replace('<password>', this.smartdataOptions.mongoDbPass)
|
||||
.replace('<DBNAME>', this.smartdataOptions.mongoDbName)
|
||||
.replace('<dbname>', this.smartdataOptions.mongoDbName);
|
||||
try {
|
||||
// Safely encode credentials to handle special characters
|
||||
const encodedUser = this.smartdataOptions.mongoDbUser
|
||||
? encodeURIComponent(this.smartdataOptions.mongoDbUser)
|
||||
: '';
|
||||
const encodedPass = this.smartdataOptions.mongoDbPass
|
||||
? encodeURIComponent(this.smartdataOptions.mongoDbPass)
|
||||
: '';
|
||||
|
||||
const finalConnectionUrl = this.smartdataOptions.mongoDbUrl
|
||||
.replace('<USERNAME>', encodedUser)
|
||||
.replace('<username>', encodedUser)
|
||||
.replace('<USER>', encodedUser)
|
||||
.replace('<user>', encodedUser)
|
||||
.replace('<PASSWORD>', encodedPass)
|
||||
.replace('<password>', encodedPass)
|
||||
.replace('<DBNAME>', this.smartdataOptions.mongoDbName)
|
||||
.replace('<dbname>', this.smartdataOptions.mongoDbName);
|
||||
|
||||
this.mongoDbClient = await plugins.mongodb.MongoClient.connect(finalConnectionUrl, {
|
||||
maxPoolSize: 100,
|
||||
maxIdleTimeMS: 10,
|
||||
});
|
||||
this.mongoDb = this.mongoDbClient.db(this.smartdataOptions.mongoDbName);
|
||||
this.status = 'connected';
|
||||
this.statusConnectedDeferred.resolve();
|
||||
console.log(`Connected to database ${this.smartdataOptions.mongoDbName}`);
|
||||
const clientOptions: plugins.mongodb.MongoClientOptions = {
|
||||
maxPoolSize: (this.smartdataOptions as any).maxPoolSize ?? 100,
|
||||
maxIdleTimeMS: (this.smartdataOptions as any).maxIdleTimeMS ?? 300000, // 5 minutes default
|
||||
serverSelectionTimeoutMS: (this.smartdataOptions as any).serverSelectionTimeoutMS ?? 30000,
|
||||
retryWrites: true,
|
||||
};
|
||||
|
||||
this.mongoDbClient = await plugins.mongodb.MongoClient.connect(finalConnectionUrl, clientOptions);
|
||||
this.mongoDb = this.mongoDbClient.db(this.smartdataOptions.mongoDbName);
|
||||
this.status = 'connected';
|
||||
this.statusConnectedDeferred.resolve();
|
||||
logger.log('info', `Connected to database ${this.smartdataOptions.mongoDbName}`);
|
||||
} catch (error) {
|
||||
this.status = 'disconnected';
|
||||
this.statusConnectedDeferred.reject(error);
|
||||
logger.log('error', `Failed to connect to database ${this.smartdataOptions.mongoDbName}: ${error.message}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -3,6 +3,7 @@ import { SmartdataDb } from './classes.db.js';
|
||||
import { managed, setDefaultManagerForDoc } from './classes.collection.js';
|
||||
import { SmartDataDbDoc, svDb, unI } from './classes.doc.js';
|
||||
import { SmartdataDbWatcher } from './classes.watcher.js';
|
||||
import { logger } from './logging.js';
|
||||
|
||||
@managed()
|
||||
export class DistributedClass extends SmartDataDbDoc<DistributedClass, DistributedClass> {
|
||||
@@ -63,11 +64,11 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
|
||||
this.ownInstance.data.elected = false;
|
||||
}
|
||||
if (this.ownInstance?.data.status === 'stopped') {
|
||||
console.log(`stopping a distributed instance that has not been started yet.`);
|
||||
logger.log('warn', `stopping a distributed instance that has not been started yet.`);
|
||||
}
|
||||
this.ownInstance.data.status = 'stopped';
|
||||
await this.ownInstance.save();
|
||||
console.log(`stopped ${this.ownInstance.id}`);
|
||||
logger.log('info', `stopped ${this.ownInstance.id}`);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -83,17 +84,17 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
|
||||
public async sendHeartbeat() {
|
||||
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
||||
if (this.ownInstance.data.status === 'stopped') {
|
||||
console.log(`aborted sending heartbeat because status is stopped`);
|
||||
logger.log('debug', `aborted sending heartbeat because status is stopped`);
|
||||
return;
|
||||
}
|
||||
await this.ownInstance.updateFromDb();
|
||||
this.ownInstance.data.lastUpdated = Date.now();
|
||||
await this.ownInstance.save();
|
||||
console.log(`sent heartbeat for ${this.ownInstance.id}`);
|
||||
logger.log('debug', `sent heartbeat for ${this.ownInstance.id}`);
|
||||
const allInstances = DistributedClass.getInstances({});
|
||||
});
|
||||
if (this.ownInstance.data.status === 'stopped') {
|
||||
console.log(`aborted sending heartbeat because status is stopped`);
|
||||
logger.log('info', `aborted sending heartbeat because status is stopped`);
|
||||
return;
|
||||
}
|
||||
const eligibleLeader = await this.getEligibleLeader();
|
||||
@@ -120,7 +121,7 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
|
||||
await this.ownInstance.save();
|
||||
});
|
||||
} else {
|
||||
console.warn(`distributed instance already initialized`);
|
||||
logger.log('warn', `distributed instance already initialized`);
|
||||
}
|
||||
|
||||
// lets enable the heartbeat
|
||||
@@ -149,24 +150,24 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
|
||||
public async checkAndMaybeLead() {
|
||||
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
||||
this.ownInstance.data.status = 'initializing';
|
||||
this.ownInstance.save();
|
||||
await this.ownInstance.save();
|
||||
});
|
||||
if (await this.getEligibleLeader()) {
|
||||
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
||||
await this.ownInstance.updateFromDb();
|
||||
this.ownInstance.data.status = 'settled';
|
||||
await this.ownInstance.save();
|
||||
console.log(`${this.ownInstance.id} settled as follower`);
|
||||
logger.log('info', `${this.ownInstance.id} settled as follower`);
|
||||
});
|
||||
return;
|
||||
} else if (
|
||||
(await DistributedClass.getInstances({})).find((instanceArg) => {
|
||||
instanceArg.data.status === 'bidding' &&
|
||||
return instanceArg.data.status === 'bidding' &&
|
||||
instanceArg.data.biddingStartTime <= Date.now() - 4000 &&
|
||||
instanceArg.data.biddingStartTime >= Date.now() - 30000;
|
||||
})
|
||||
) {
|
||||
console.log('too late to the bidding party... waiting for next round.');
|
||||
logger.log('info', 'too late to the bidding party... waiting for next round.');
|
||||
return;
|
||||
} else {
|
||||
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
||||
@@ -175,9 +176,9 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
|
||||
this.ownInstance.data.biddingStartTime = Date.now();
|
||||
this.ownInstance.data.biddingShortcode = plugins.smartunique.shortId();
|
||||
await this.ownInstance.save();
|
||||
console.log('bidding code stored.');
|
||||
logger.log('info', 'bidding code stored.');
|
||||
});
|
||||
console.log(`bidding for leadership...`);
|
||||
logger.log('info', `bidding for leadership...`);
|
||||
await plugins.smartdelay.delayFor(plugins.smarttime.getMilliSecondsFromUnits({ seconds: 5 }));
|
||||
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
||||
let biddingInstances = await DistributedClass.getInstances({});
|
||||
@@ -187,7 +188,7 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
|
||||
instanceArg.data.lastUpdated >=
|
||||
Date.now() - plugins.smarttime.getMilliSecondsFromUnits({ seconds: 10 }),
|
||||
);
|
||||
console.log(`found ${biddingInstances.length} bidding instances...`);
|
||||
logger.log('info', `found ${biddingInstances.length} bidding instances...`);
|
||||
this.ownInstance.data.elected = true;
|
||||
for (const biddingInstance of biddingInstances) {
|
||||
if (biddingInstance.data.biddingShortcode < this.ownInstance.data.biddingShortcode) {
|
||||
@@ -195,7 +196,7 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
|
||||
}
|
||||
}
|
||||
await plugins.smartdelay.delayFor(5000);
|
||||
console.log(`settling with status elected = ${this.ownInstance.data.elected}`);
|
||||
logger.log('info', `settling with status elected = ${this.ownInstance.data.elected}`);
|
||||
this.ownInstance.data.status = 'settled';
|
||||
await this.ownInstance.save();
|
||||
});
|
||||
@@ -226,11 +227,11 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
|
||||
this.distributedWatcher.changeSubject.subscribe({
|
||||
next: async (distributedDoc) => {
|
||||
if (!distributedDoc) {
|
||||
console.log(`registered deletion of instance...`);
|
||||
logger.log('info', `registered deletion of instance...`);
|
||||
return;
|
||||
}
|
||||
console.log(distributedDoc);
|
||||
console.log(`registered change for ${distributedDoc.id}`);
|
||||
logger.log('info', distributedDoc);
|
||||
logger.log('info', `registered change for ${distributedDoc.id}`);
|
||||
distributedDoc;
|
||||
},
|
||||
});
|
||||
@@ -252,7 +253,7 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
|
||||
): Promise<plugins.taskbuffer.distributedCoordination.IDistributedTaskRequestResult> {
|
||||
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
||||
if (!this.ownInstance) {
|
||||
console.error('instance need to be started first...');
|
||||
logger.log('error', 'instance need to be started first...');
|
||||
return;
|
||||
}
|
||||
await this.ownInstance.updateFromDb();
|
||||
@@ -268,7 +269,7 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
|
||||
return taskRequestResult;
|
||||
});
|
||||
if (!result) {
|
||||
console.warn('no result found for task request...');
|
||||
logger.log('warn', 'no result found for task request...');
|
||||
return null;
|
||||
}
|
||||
return result;
|
||||
@@ -285,7 +286,7 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
|
||||
);
|
||||
});
|
||||
if (!existingInfoBasis) {
|
||||
console.warn('trying to update a non existing task request... aborting!');
|
||||
logger.log('warn', 'trying to update a non existing task request... aborting!');
|
||||
return;
|
||||
}
|
||||
Object.assign(existingInfoBasis, infoBasisArg);
|
||||
@@ -293,8 +294,10 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
|
||||
plugins.smartdelay.delayFor(60000).then(() => {
|
||||
this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
||||
const indexToRemove = this.ownInstance.data.taskRequests.indexOf(existingInfoBasis);
|
||||
this.ownInstance.data.taskRequests.splice(indexToRemove, indexToRemove);
|
||||
await this.ownInstance.save();
|
||||
if (indexToRemove >= 0) {
|
||||
this.ownInstance.data.taskRequests.splice(indexToRemove, 1);
|
||||
await this.ownInstance.save();
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import * as plugins from './plugins.js';
|
||||
|
||||
import { SmartdataDb } from './classes.db.js';
|
||||
import { logger } from './logging.js';
|
||||
import { SmartdataDbCursor } from './classes.cursor.js';
|
||||
import { type IManager, SmartdataCollection } from './classes.collection.js';
|
||||
import { SmartdataDbWatcher } from './classes.watcher.js';
|
||||
@@ -31,7 +32,7 @@ export type TDocCreation = 'db' | 'new' | 'mixed';
|
||||
|
||||
export function globalSvDb() {
|
||||
return (target: SmartDataDbDoc<unknown, unknown>, key: string) => {
|
||||
console.log(`called svDb() on >${target.constructor.name}.${key}<`);
|
||||
logger.log('debug', `called svDb() on >${target.constructor.name}.${key}<`);
|
||||
if (!target.globalSaveableProperties) {
|
||||
target.globalSaveableProperties = [];
|
||||
}
|
||||
@@ -54,7 +55,7 @@ export interface SvDbOptions {
|
||||
*/
|
||||
export function svDb(options?: SvDbOptions) {
|
||||
return (target: SmartDataDbDoc<unknown, unknown>, key: string) => {
|
||||
console.log(`called svDb() on >${target.constructor.name}.${key}<`);
|
||||
logger.log('debug', `called svDb() on >${target.constructor.name}.${key}<`);
|
||||
if (!target.saveableProperties) {
|
||||
target.saveableProperties = [];
|
||||
}
|
||||
@@ -94,7 +95,7 @@ function escapeForRegex(input: string): string {
|
||||
*/
|
||||
export function unI() {
|
||||
return (target: SmartDataDbDoc<unknown, unknown>, key: string) => {
|
||||
console.log(`called unI on >>${target.constructor.name}.${key}<<`);
|
||||
logger.log('debug', `called unI on >>${target.constructor.name}.${key}<<`);
|
||||
|
||||
// mark the index as unique
|
||||
if (!target.uniqueIndexes) {
|
||||
@@ -126,7 +127,7 @@ export interface IIndexOptions {
|
||||
*/
|
||||
export function index(options?: IIndexOptions) {
|
||||
return (target: SmartDataDbDoc<unknown, unknown>, key: string) => {
|
||||
console.log(`called index() on >${target.constructor.name}.${key}<`);
|
||||
logger.log('debug', `called index() on >${target.constructor.name}.${key}<`);
|
||||
|
||||
// Initialize regular indexes array if it doesn't exist
|
||||
if (!target.regularIndexes) {
|
||||
@@ -152,7 +153,8 @@ export function index(options?: IIndexOptions) {
|
||||
|
||||
export const convertFilterForMongoDb = (filterArg: { [key: string]: any }) => {
|
||||
// Special case: detect MongoDB operators and pass them through directly
|
||||
const topLevelOperators = ['$and', '$or', '$nor', '$not', '$text', '$where', '$regex'];
|
||||
// SECURITY: Removed $where to prevent server-side JS execution
|
||||
const topLevelOperators = ['$and', '$or', '$nor', '$not', '$text', '$regex'];
|
||||
for (const key of Object.keys(filterArg)) {
|
||||
if (topLevelOperators.includes(key)) {
|
||||
return filterArg; // Return the filter as-is for MongoDB operators
|
||||
@@ -164,11 +166,16 @@ export const convertFilterForMongoDb = (filterArg: { [key: string]: any }) => {
|
||||
|
||||
const convertFilterArgument = (keyPathArg2: string, filterArg2: any) => {
|
||||
if (Array.isArray(filterArg2)) {
|
||||
// Directly assign arrays (they might be using operators like $in or $all)
|
||||
convertFilterArgument(keyPathArg2, filterArg2[0]);
|
||||
// FIX: Properly handle arrays for operators like $in, $all, or plain equality
|
||||
convertedFilter[keyPathArg2] = filterArg2;
|
||||
return;
|
||||
} else if (typeof filterArg2 === 'object' && filterArg2 !== null) {
|
||||
for (const key of Object.keys(filterArg2)) {
|
||||
if (key.startsWith('$')) {
|
||||
// Prevent dangerous operators
|
||||
if (key === '$where') {
|
||||
throw new Error('$where operator is not allowed for security reasons');
|
||||
}
|
||||
convertedFilter[keyPathArg2] = filterArg2;
|
||||
return;
|
||||
} else if (key.includes('.')) {
|
||||
@@ -614,7 +621,7 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
||||
this.creationStatus = 'db';
|
||||
break;
|
||||
default:
|
||||
console.error('neither new nor in db?');
|
||||
logger.log('error', 'neither new nor in db?');
|
||||
}
|
||||
// allow hook after saving
|
||||
if (typeof (this as any).afterSave === 'function') {
|
||||
@@ -661,8 +668,11 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
||||
/**
|
||||
* updates an object from db
|
||||
*/
|
||||
public async updateFromDb() {
|
||||
public async updateFromDb(): Promise<boolean> {
|
||||
const mongoDbNativeDoc = await this.collection.findOne(await this.createIdentifiableObject());
|
||||
if (!mongoDbNativeDoc) {
|
||||
return false; // Document not found in database
|
||||
}
|
||||
for (const key of Object.keys(mongoDbNativeDoc)) {
|
||||
const rawValue = mongoDbNativeDoc[key];
|
||||
const optionsMap = (this.constructor as any)._svDbOptions || {};
|
||||
@@ -671,6 +681,7 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
||||
? opts.deserialize(rawValue)
|
||||
: rawValue;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -678,7 +689,9 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
||||
*/
|
||||
public async createSavableObject(): Promise<TImplements> {
|
||||
const saveableObject: unknown = {}; // is not exposed to outside, so any is ok here
|
||||
const saveableProperties = [...this.globalSaveableProperties, ...this.saveableProperties];
|
||||
const globalProps = this.globalSaveableProperties || [];
|
||||
const specificProps = this.saveableProperties || [];
|
||||
const saveableProperties = [...globalProps, ...specificProps];
|
||||
// apply custom serialization if configured
|
||||
const optionsMap = (this.constructor as any)._svDbOptions || {};
|
||||
for (const propertyNameString of saveableProperties) {
|
||||
|
||||
@@ -18,7 +18,7 @@ export class EasyStore<T> {
|
||||
public nameId: string;
|
||||
|
||||
@svDb()
|
||||
public ephermal: {
|
||||
public ephemeral: {
|
||||
activated: boolean;
|
||||
timeout: number;
|
||||
};
|
||||
@@ -32,8 +32,8 @@ export class EasyStore<T> {
|
||||
return SmartdataEasyStore;
|
||||
})();
|
||||
|
||||
constructor(nameIdArg: string, smnartdataDbRefArg: SmartdataDb) {
|
||||
this.smartdataDbRef = smnartdataDbRefArg;
|
||||
constructor(nameIdArg: string, smartdataDbRefArg: SmartdataDb) {
|
||||
this.smartdataDbRef = smartdataDbRefArg;
|
||||
this.nameId = nameIdArg;
|
||||
}
|
||||
|
||||
@@ -110,10 +110,12 @@ export class EasyStore<T> {
|
||||
await easyStore.save();
|
||||
}
|
||||
|
||||
public async cleanUpEphermal() {
|
||||
while (
|
||||
(await this.smartdataDbRef.statusConnectedDeferred.promise) &&
|
||||
this.smartdataDbRef.status === 'connected'
|
||||
) {}
|
||||
public async cleanUpEphemeral() {
|
||||
// Clean up ephemeral data periodically while connected
|
||||
while (this.smartdataDbRef.status === 'connected') {
|
||||
await plugins.smartdelay.delayFor(60000); // Check every minute
|
||||
// TODO: Implement actual cleanup logic for ephemeral data
|
||||
// For now, this prevents the infinite CPU loop
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
* Lucene to MongoDB query adapter for SmartData
|
||||
*/
|
||||
import * as plugins from './plugins.js';
|
||||
import { logger } from './logging.js';
|
||||
|
||||
// Types
|
||||
type NodeType =
|
||||
@@ -754,7 +755,7 @@ export class SmartdataLuceneAdapter {
|
||||
// Transform the AST to a MongoDB query
|
||||
return this.transformWithFields(ast, fieldsToSearch);
|
||||
} catch (error) {
|
||||
console.error(`Failed to convert Lucene query "${luceneQuery}":`, error);
|
||||
logger.log('error', `Failed to convert Lucene query "${luceneQuery}":`, error);
|
||||
throw new Error(`Failed to convert Lucene query: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user