5.2.6

npmextra.json

@@ -12,12 +12,25 @@
   "gitzone": {
     "projectType": "npm",
     "module": {
-      "githost": "gitlab.com",
+      "githost": "code.foss.global",
       "gitscope": "push.rocks",
       "gitrepo": "smartdata",
-      "description": "do more with data",
+      "description": "An advanced library for NoSQL data organization and manipulation using TypeScript with support for MongoDB, data validation, collections, and custom data types.",
       "npmPackagename": "@push.rocks/smartdata",
-      "license": "MIT"
+      "license": "MIT",
+      "keywords": [
+        "data manipulation",
+        "NoSQL",
+        "MongoDB",
+        "TypeScript",
+        "data validation",
+        "collections",
+        "custom data types",
+        "ODM"
+      ]
     }
+  },
+  "tsdoc": {
+    "legal": "\n## License and Legal Information\n\nThis repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository. \n\n**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.\n\n### Trademarks\n\nThis project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.\n\n### Company Information\n\nTask Venture Capital GmbH  \nRegistered at District court Bremen HRB 35230 HB, Germany\n\nFor any legal inquiries or if you require further information, please contact us via email at hello@task.vc.\n\nBy using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.\n"
   }
 }

package.json

@@ -1,8 +1,8 @@
 {
   "name": "@push.rocks/smartdata",
-  "version": "5.0.24",
+  "version": "5.2.6",
   "private": false,
-  "description": "do more with data",
+  "description": "An advanced library for NoSQL data organization and manipulation using TypeScript with support for MongoDB, data validation, collections, and custom data types.",
   "main": "dist_ts/index.js",
   "typings": "dist_ts/index.d.ts",
   "type": "module",
@@ -13,36 +13,35 @@
   },
   "repository": {
     "type": "git",
-    "url": "git+ssh://git@gitlab.com/pushrocks/smartdata.git"
+    "url": "https://code.foss.global/push.rocks/smartdata.git"
   },
   "author": "Lossless GmbH",
   "license": "MIT",
   "bugs": {
     "url": "https://gitlab.com/pushrocks/smartdata/issues"
   },
-  "homepage": "https://gitlab.com/pushrocks/smartdata#README",
+  "homepage": "https://code.foss.global/push.rocks/smartdata",
   "dependencies": {
-    "@push.rocks/lik": "^6.0.2",
+    "@push.rocks/lik": "^6.0.14",
     "@push.rocks/smartdelay": "^3.0.1",
     "@push.rocks/smartlog": "^3.0.2",
     "@push.rocks/smartmongo": "^2.0.10",
     "@push.rocks/smartpromise": "^4.0.2",
-    "@push.rocks/smartrx": "^3.0.6",
-    "@push.rocks/smartstring": "^4.0.7",
-    "@push.rocks/smarttime": "^4.0.1",
-    "@push.rocks/smartunique": "^3.0.3",
-    "@push.rocks/taskbuffer": "^3.1.0",
-    "@tsclass/tsclass": "^4.0.42",
-    "mongodb": "^5.7.0"
+    "@push.rocks/smartrx": "^3.0.7",
+    "@push.rocks/smartstring": "^4.0.15",
+    "@push.rocks/smarttime": "^4.0.6",
+    "@push.rocks/smartunique": "^3.0.8",
+    "@push.rocks/taskbuffer": "^3.1.7",
+    "@tsclass/tsclass": "^4.0.52",
+    "mongodb": "^6.5.0"
   },
   "devDependencies": {
     "@gitzone/tsbuild": "^2.1.66",
     "@gitzone/tsrun": "^1.2.44",
     "@gitzone/tstest": "^1.0.77",
-    "@push.rocks/qenv": "^6.0.2",
-    "@push.rocks/tapbundle": "^5.0.8",
-    "@types/node": "^20.4.9",
-    "@types/shortid": "0.0.29"
+    "@push.rocks/qenv": "^6.0.5",
+    "@push.rocks/tapbundle": "^5.0.22",
+    "@types/node": "^20.11.30"
   },
   "files": [
     "ts/**/*",
@@ -58,5 +57,15 @@
   ],
   "browserslist": [
     "last 1 chrome versions"
+  ],
+  "keywords": [
+    "data manipulation",
+    "NoSQL",
+    "MongoDB",
+    "TypeScript",
+    "data validation",
+    "collections",
+    "custom data types",
+    "ODM"
   ]
 }

readme.md

@@ -1,154 +1,115 @@
 # @push.rocks/smartdata
 do more with data
 
-## Availabililty and Links
-* [npmjs.org (npm package)](https://www.npmjs.com/package/@push.rocks/smartdata)
-* [gitlab.com (source)](https://gitlab.com/push.rocks/smartdata)
-* [github.com (source mirror)](https://github.com/push.rocks/smartdata)
-* [docs (typedoc)](https://push.rocks.gitlab.io/smartdata/)
+## Install
+To install `@push.rocks/smartdata`, use npm:
 
-## Status for master
+```bash
+npm install @push.rocks/smartdata --save
+```
 
-Status Category | Status Badge
--- | --
-GitLab Pipelines | [![pipeline status](https://gitlab.com/push.rocks/smartdata/badges/master/pipeline.svg)](https://lossless.cloud)
-GitLab Pipline Test Coverage | [![coverage report](https://gitlab.com/push.rocks/smartdata/badges/master/coverage.svg)](https://lossless.cloud)
-npm | [![npm downloads per month](https://badgen.net/npm/dy/@push.rocks/smartdata)](https://lossless.cloud)
-Snyk | [![Known Vulnerabilities](https://badgen.net/snyk/push.rocks/smartdata)](https://lossless.cloud)
-TypeScript Support | [![TypeScript](https://badgen.net/badge/TypeScript/>=%203.x/blue?icon=typescript)](https://lossless.cloud)
-node Support | [![node](https://img.shields.io/badge/node->=%2010.x.x-blue.svg)](https://nodejs.org/dist/latest-v10.x/docs/api/)
-Code Style | [![Code Style](https://badgen.net/badge/style/prettier/purple)](https://lossless.cloud)
-PackagePhobia (total standalone install weight) | [![PackagePhobia](https://badgen.net/packagephobia/install/@push.rocks/smartdata)](https://lossless.cloud)
-PackagePhobia (package size on registry) | [![PackagePhobia](https://badgen.net/packagephobia/publish/@push.rocks/smartdata)](https://lossless.cloud)
-BundlePhobia (total size when bundled) | [![BundlePhobia](https://badgen.net/bundlephobia/minzip/@push.rocks/smartdata)](https://lossless.cloud)
+This will add `@push.rocks/smartdata` to your project's dependencies.
 
 ## Usage
+`@push.rocks/smartdata` enables efficient data handling and operation management with a focus on using MongoDB. It leverages TypeScript for strong typing and ESM syntax for modern JavaScript usage. Below are various scenarios demonstrating how to utilize this package effectively in a project.
 
-Use TypeScript for best in class instellisense.
-
-smartdata is an ODM that adheres to TypeScript practices and uses classes to organize data.
-It uses RethinkDB as persistent storage.
-
-## Intention
-
-There are many ODMs out there, however when we searched for an ODM that uses TypeScript,
-acts smart while still embracing the NoSQL idea we didn't find a matching solution.
-This is why we started smartdata.
-
-How RethinkDB's terms map to the ones of smartdata:
-
-| MongoDb term | smartdata class               |
-| ------------ | ----------------------------- |
-| Database     | smartdata.SmartdataDb         |
-| Collection   | smartdata.SmartdataCollection |
-| Document     | smartdata.SmartadataDoc       |
-
-### class Db
-
-represents a Database. Naturally it has .connect() etc. methods on it.
+### Setting Up and Connecting to the Database
+Before interacting with the database, you need to set up and establish a connection. This is done by creating an instance of `SmartdataDb` and calling its `init` method with your MongoDB connection details.
 
 ```typescript
-// Assuming toplevel await
-import * as smartdata from 'smartdata';
+import { SmartdataDb } from '@push.rocks/smartdata';
 
-const smartdataDb = new smartdata.SmartdataDb({
-  mongoDbUrl: '//someurl',
-  mongoDbName: 'myDatabase',
-  mongoDbPass: 'mypassword',
+// Create a new instance of SmartdataDb with MongoDB connection details
+const db = new SmartdataDb({
+  mongoDbUrl: 'mongodb://localhost:27017',
+  mongoDbName: 'your-database-name',
+  mongoDbUser: 'your-username',
+  mongoDbPass: 'your-password',
 });
 
-await smartdataDb.connect();
+// Initialize and connect to the database
+await db.init();
 ```
 
-### class DbCollection
-
-represents a collection of objects.
-A collection is defined by the object class (that is extending smartdata.dbdoc) it respresents
-
-So to get to get access to a specific collection you document
+### Defining Data Models
+Data models in `@push.rocks/smartdata` are classes that represent collections and documents in your MongoDB database. Use decorators such as `@Collection`, `@unI`, and `@svDb` to define your data models.
 
 ```typescript
-// Assuming toplevel await
-// continues from the block before...
+import { SmartDataDbDoc, Collection, unI, svDb } from '@push.rocks/smartdata';
 
-@smartdata.Collection(smartdataDb)
-class MyObject extends smartdata.DbDoc<MyObject /* ,[an optional interface to implement] */> {
-  // read the next block about DbDoc
-  @smartdata.svDb()
-  property1: string; // @smartdata.svDb() marks the property for db save
-
-  property2: number; // this one is not marked, so it won't be save upon calling this.save()
-
-  constructor() {
-    super(); // the super call is important ;) But you probably know that.
+@Collection(() => db)  // Associate this model with the database instance
+class User extends SmartDataDbDoc<User, User> {
+  @unI()
+  public id: string = 'unique-user-id'; // Mark 'id' as a unique index
+  
+  @svDb()
+  public username: string;  // Mark 'username' to be saved in DB
+  
+  @svDb()
+  public email: string;  // Mark 'email' to be saved in DB
+  
+  constructor(username: string, email: string) {
+    super();
+    this.username = username;
+    this.email = email;
   }
 }
-
-// start to instantiate instances of classes from scratch or database
-
-const localObject = new MyObject({
-  property1: 'hi',
-  property2: {
-    deep: 3,
-  },
-});
-await localObject.save(); // saves the object to the database
-
-// start retrieving instances
-
-// .getInstance is staticly inheritied, yet fully typed static function to get instances with fully typed filters
-const myInstance = await MyObject.getInstance({
-  property1: 'hi',
-  property2: {
-    deep: {
-      $gt: 2,
-    } as any,
-  },
-}); // outputs a new instance of MyObject with the values from db assigned
 ```
 
-### class DbDoc
+### Performing CRUD Operations
+`@push.rocks/smartdata` simplifies CRUD operations with intuitive methods on model instances.
 
-represents a individual document in a collection
-and thereby is ideally suited to extend the class you want to actually store.
+#### Create
+```typescript
+const newUser = new User('myUsername', 'myEmail@example.com');
+await newUser.save();  // Save the new user to the database
+```
 
-### CRUD operations
+#### Read
+```typescript
+// Fetch a single user by a unique attribute
+const user = await User.getInstance({ username: 'myUsername' });
 
-smartdata supports full CRUD operations
+// Fetch multiple users that match criteria
+const users = await User.getInstances({ email: 'myEmail@example.com' });
+```
 
-**Store** or **Update** instances of classes to MongoDB:
-DbDoc extends your class with the following methods:
+#### Update
+```typescript
+// Assuming 'user' is an instance of User
+user.email = 'newEmail@example.com';
+await user.save();  // Update the user in the database
+```
 
-- async `.save()` will save (or update) the object you call it on only. Any referenced non-savable objects will not get stored.
-- async `.saveDeep()` does the same like `.save()`.
-  In addition it will look for properties that reference an object
-  that extends DbDoc as well and call .saveDeep() on them as well.
-  Loops are prevented
+#### Delete
+```typescript
+// Assuming 'user' is an instance of User
+await user.delete();  // Delete the user from the database
+```
 
-**Get** a new class instance from MongoDB:
-DbDoc exposes a static method that allows you specify a filter to retrieve a cloned class of the one you used to that doc at some point later in time:
+### Advanced Usage
+`@push.rocks/smartdata` also supports advanced features like watching for real-time changes in the database, handling distributed data coordination, and more. These features utilize MongoDB's capabilities to provide real-time data syncing and distributed systems coordination.
 
-- static async `.getInstance({ /* filter props here */ })` gets you an instance that has the data of the first matched document as properties.
-- static async `getInstances({ /* filter props here */ })` get you an array instances (one instance for every matched document).
+### Conclusion
+With its focus on TypeScript, modern JavaScript syntax, and leveraging MongoDB's features, `@push.rocks/smartdata` offers a powerful toolkit for data handling and operations management in Node.js applications. Its design for ease of use, coupled with advanced features, makes it a versatile choice for developers looking to build efficient and scalable data-driven applications.
 
-**Delete** instances from MongoDb:
-smartdata extends your class with a method to easily delete the doucment from DB:
+For more details on usage and additional features, refer to the [official documentation](https://gitlab.com/push.rocks/smartdata#README) and explore the various classes and methods provided by `@push.rocks/smartdata`.
 
-- async `.delete()`will delete the document from DB.
+## License and Legal Information
 
-## TypeScript
+This repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository. 
 
-How does TypeScript play into this?
-Since you define your classes in TypeScript and types flow through smartdata in a generic way
-you should get all the Intellisense and type checking you love when using smartdata.
-smartdata itself also bundles typings. You don't need to install any additional types for smartdata.
+**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.
 
-## Contribution
+### Trademarks
 
-We are always happy for code contributions. If you are not the code contributing type that is ok. Still, maintaining Open Source repositories takes considerable time and thought. If you like the quality of what we do and our modules are useful to you we would appreciate a little monthly contribution: You can [contribute one time](https://lossless.link/contribute-onetime) or [contribute monthly](https://lossless.link/contribute). :)
+This project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.
 
-For further information read the linked docs at the top of this readme.
+### Company Information
 
-## Legal
-> MIT licensed | **&copy;** [Task Venture Capital GmbH](https://task.vc)
-| By using this npm module you agree to our [privacy policy](https://lossless.gmbH/privacy)
+Task Venture Capital GmbH  
+Registered at District court Bremen HRB 35230 HB, Germany
+
+For any legal inquiries or if you require further information, please contact us via email at hello@task.vc.
+
+By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.

test/test.distributedcoordinator.ts

@@ -0,0 +1,112 @@
+import { tap, expect } from '@push.rocks/tapbundle';
+import * as smartmongo from '@push.rocks/smartmongo';
+import type * as taskbuffer from '@push.rocks/taskbuffer';
+
+import * as smartdata from '../ts/index.js';
+import { SmartdataDistributedCoordinator, DistributedClass } from '../ts/smartdata.classes.distributedcoordinator.js'; // path might need adjusting
+const totalInstances = 10;
+
+// =======================================
+// Connecting to the database server
+// =======================================
+
+let smartmongoInstance: smartmongo.SmartMongo;
+let testDb: smartdata.SmartdataDb;
+
+tap.test('should create a testinstance as database', async () => {
+  smartmongoInstance = await smartmongo.SmartMongo.createAndStart();
+  testDb = new smartdata.SmartdataDb(await smartmongoInstance.getMongoDescriptor());
+  await testDb.init();
+});
+
+tap.test('should instantiate DistributedClass', async (tools) => {
+    const instance = new DistributedClass();
+    expect(instance).toBeInstanceOf(DistributedClass);
+});
+
+tap.test('DistributedClass should update the time', async (tools) => {
+    const distributedCoordinator = new SmartdataDistributedCoordinator(testDb);
+    await distributedCoordinator.start();
+    const initialTime = distributedCoordinator.ownInstance.data.lastUpdated;
+    await distributedCoordinator.sendHeartbeat();
+    const updatedTime = distributedCoordinator.ownInstance.data.lastUpdated;
+    expect(updatedTime).toBeGreaterThan(initialTime);
+    await distributedCoordinator.stop();
+});
+
+tap.test('should instantiate SmartdataDistributedCoordinator', async (tools) => {
+    const distributedCoordinator = new SmartdataDistributedCoordinator(testDb);
+    await distributedCoordinator.start();
+    expect(distributedCoordinator).toBeInstanceOf(SmartdataDistributedCoordinator);
+    await distributedCoordinator.stop();
+});
+
+tap.test('SmartdataDistributedCoordinator should update leader status', async (tools) => {
+    const distributedCoordinator = new SmartdataDistributedCoordinator(testDb);
+    await distributedCoordinator.start();
+    await distributedCoordinator.checkAndMaybeLead();
+    expect(distributedCoordinator.ownInstance.data.elected).toBeOneOf([true, false]);
+    await distributedCoordinator.stop();
+});
+
+tap.test('SmartdataDistributedCoordinator should handle distributed task requests', async (tools) => {
+    const distributedCoordinator = new SmartdataDistributedCoordinator(testDb);
+    await distributedCoordinator.start();
+
+    const mockTaskRequest: taskbuffer.distributedCoordination.IDistributedTaskRequest = {
+        submitterId: "mockSubmitter12345",    // Some unique mock submitter ID
+        requestResponseId: 'uni879873462hjhfkjhsdf', // Some unique ID for the request-response
+        taskName: "SampleTask",
+        taskVersion: "1.0.0",                  // Assuming it's a version string
+        taskExecutionTime: Date.now(),
+        taskExecutionTimeout: 60000,           // Let's say the timeout is 1 minute (60000 ms)
+        taskExecutionParallel: 5,              // Let's assume max 5 parallel executions
+        status: 'requesting'
+    };
+
+    const response = await distributedCoordinator.fireDistributedTaskRequest(mockTaskRequest);
+    console.log(response)  // based on your expected structure for the response
+    await distributedCoordinator.stop();
+});
+
+tap.test('SmartdataDistributedCoordinator should update distributed task requests', async (tools) => {
+    const distributedCoordinator = new SmartdataDistributedCoordinator(testDb);
+    
+    await distributedCoordinator.start();
+
+    const mockTaskRequest: taskbuffer.distributedCoordination.IDistributedTaskRequest = {
+        submitterId: "mockSubmitter12345",    // Some unique mock submitter ID
+        requestResponseId: 'uni879873462hjhfkjhsdf', // Some unique ID for the request-response
+        taskName: "SampleTask",
+        taskVersion: "1.0.0",                  // Assuming it's a version string
+        taskExecutionTime: Date.now(),
+        taskExecutionTimeout: 60000,           // Let's say the timeout is 1 minute (60000 ms)
+        taskExecutionParallel: 5,              // Let's assume max 5 parallel executions
+        status: 'requesting'
+    };
+    
+
+    await distributedCoordinator.updateDistributedTaskRequest(mockTaskRequest);
+    // Here, we can potentially check if a DB entry got updated or some other side-effect of the update method.
+    await distributedCoordinator.stop();
+});
+
+tap.test('should elect only one leader amongst multiple instances', async (tools) => {
+    const coordinators = Array.from({ length: totalInstances }).map(() => new SmartdataDistributedCoordinator(testDb));
+    await Promise.all(coordinators.map(coordinator => coordinator.start()));
+    const leaders = coordinators.filter(coordinator => coordinator.ownInstance.data.elected);
+    for (const leader of leaders) {
+        console.log(leader.ownInstance);
+    }
+    expect(leaders.length).toEqual(1);
+
+    // stopping clears a coordinator from being elected.
+    await Promise.all(coordinators.map(coordinator => coordinator.stop()));
+});
+
+tap.test('should clean up', async () => {
+    await smartmongoInstance.stopAndDumpToDir(`.nogit/dbdump/test.distributedcoordinator.ts`);
+    setTimeout(() => process.exit(), 2000);
+})
+
+tap.start({ throwOnError: true });

test/test.easystore.ts

@@ -26,7 +26,7 @@ tap.test('should create a testinstance as database', async () => {
 tap.skip.test('should connect to atlas', async (tools) => {
   const databaseName = `test-smartdata-${smartunique.shortId()}`;
   testDb = new smartdata.SmartdataDb({
-    mongoDbUrl: testQenv.getEnvVarOnDemand('MONGO_URL'),
+    mongoDbUrl: await testQenv.getEnvVarOnDemand('MONGO_URL'),
     mongoDbName: databaseName,
   });
   await testDb.init();

test/test.ts

@@ -30,7 +30,7 @@ tap.test('should create a testinstance as database', async () => {
 tap.skip.test('should connect to atlas', async (tools) => {
   const databaseName = `test-smartdata-${smartunique.shortId()}`;
   testDb = new smartdata.SmartdataDb({
-    mongoDbUrl: testQenv.getEnvVarOnDemand('MONGO_URL'),
+    mongoDbUrl: await testQenv.getEnvVarOnDemand('MONGO_URL'),
     mongoDbName: databaseName,
   });
   await testDb.init();
@@ -72,6 +72,11 @@ class Car extends smartdata.SmartDataDbDoc<Car, Car> {
   }
 }
 
+tap.test('should create a new id', async () => {
+  const newid = await Car.getNewId();
+  console.log(newid);
+});
+
 tap.test('should save the car to the db', async (toolsArg) => {
   const myCar = new Car('red', 'Volvo');
   await myCar.save();
@@ -194,12 +199,18 @@ tap.test('should store a new Truck', async () => {
   const truck = new Truck('blue', 'MAN');
   await truck.save();
   const myTruck2 = await Truck.getInstance({ color: 'blue' });
+  expect(myTruck2.color).toEqual('blue');
   myTruck2.color = 'red';
   await myTruck2.save();
   const myTruck3 = await Truck.getInstance({ color: 'blue' });
-  console.log(myTruck3);
+  expect(myTruck3).toBeNull();
 });
 
+tap.test('should return a count', async () => {
+  const truckCount = await Truck.getCount();
+  expect(truckCount).toEqual(1);
+})
+
 tap.test('should use a cursor', async () => {
   const cursor = await Car.getCursor({});
   let counter = 0;
@@ -213,11 +224,13 @@ tap.test('should use a cursor', async () => {
 // close the database connection
 // =======================================
 tap.test('close', async () => {
-  await testDb.mongoDb.dropDatabase();
-  await testDb.close();
   if (smartmongoInstance) {
-    await smartmongoInstance.stop();
+    await smartmongoInstance.stopAndDumpToDir('./.nogit/dbdump/test.ts');
+  } else {
+    await testDb.mongoDb.dropDatabase();
+    await testDb.close();
   }
+  setTimeout(() => process.exit(), 2000);
 });
 
 tap.start({ throwOnError: true });

test/test.typescript.ts

@@ -28,7 +28,7 @@ tap.test('should create a testinstance as database', async () => {
 tap.skip.test('should connect to atlas', async (tools) => {
   const databaseName = `test-smartdata-${smartunique.shortId()}`;
   testDb = new smartdata.SmartdataDb({
-    mongoDbUrl: testQenv.getEnvVarOnDemand('MONGO_URL'),
+    mongoDbUrl: await testQenv.getEnvVarOnDemand('MONGO_URL'),
     mongoDbName: databaseName,
   });
   await testDb.init();

test/test.watch.ts

@@ -28,7 +28,7 @@ tap.test('should create a testinstance as database', async () => {
 tap.skip.test('should connect to atlas', async (tools) => {
   const databaseName = `test-smartdata-${smartunique.shortId()}`;
   testDb = new smartdata.SmartdataDb({
-    mongoDbUrl: testQenv.getEnvVarOnDemand('MONGO_URL'),
+    mongoDbUrl: await testQenv.getEnvVarOnDemand('MONGO_URL'),
     mongoDbName: databaseName,
   });
   await testDb.init();

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@push.rocks/smartdata',
-  version: '5.0.24',
-  description: 'do more with data'
+  version: '5.2.6',
+  description: 'An advanced library for NoSQL data organization and manipulation using TypeScript with support for MongoDB, data validation, collections, and custom data types.'
 }

ts/smartdata.classes.collection.ts

@@ -26,7 +26,8 @@ const collectionFactory = new CollectionFactory();
  */
 export function Collection(dbArg: SmartdataDb | TDelayed<SmartdataDb>) {
   return function classDecorator<T extends { new (...args: any[]): {} }>(constructor: T) {
-    return class extends constructor {
+    const decoratedClass = class extends constructor {
+      public static className = constructor.name;
       public static get collection() {
         if (!(dbArg instanceof SmartdataDb)) {
           dbArg = dbArg();
@@ -40,6 +41,7 @@ export function Collection(dbArg: SmartdataDb | TDelayed<SmartdataDb>) {
         return collectionFactory.getCollection(constructor.name, dbArg);
       }
     };
+    return decoratedClass;
   };
 }
 
@@ -56,9 +58,10 @@ export const setDefaultManagerForDoc = <T>(managerArg: IManager, dbDocArg: T): T
  * This is a decorator that will tell the decorated class what dbTable to use
  * @param dbArg
  */
-export function Manager<TManager extends IManager>(managerArg?: TManager | TDelayed<TManager>) {
+export function managed<TManager extends IManager>(managerArg?: TManager | TDelayed<TManager>) {
   return function classDecorator<T extends { new (...args: any[]): any }>(constructor: T) {
-    return class extends constructor {
+    const decoratedClass = class extends constructor {
+      public static className = constructor.name;
       public static get collection() {
         let dbArg: SmartdataDb;
         if (!managerArg) {
@@ -106,9 +109,15 @@ export function Manager<TManager extends IManager>(managerArg?: TManager | TDela
         return manager;
       }
     };
+    return decoratedClass;
   };
 }
 
+/**
+ * @dpecrecated use @managed instead
+ */
+export const Manager = managed;
+
 export class SmartdataCollection<T> {
   /**
    * the collection that is used
@@ -264,12 +273,17 @@ export class SmartdataCollection<T> {
     await this.mongoDbCollection.deleteOne(identifiableObject);
   }
 
+  public async getCount(filterObject: any) {
+    await this.init();
+    return this.mongoDbCollection.countDocuments(filterObject);
+  }
+
   /**
    * checks a Doc for constraints
    * if this.objectValidation is not set it passes.
    */
   private checkDoc(docArg: T): Promise<void> {
-    const done = plugins.smartq.defer<void>();
+    const done = plugins.smartpromise.defer<void>();
     let validationResult = true;
     if (this.objectValidation) {
       validationResult = this.objectValidation(docArg);

ts/smartdata.classes.distributedcoordinator.ts

@@ -1,18 +1,20 @@
 import * as plugins from './smartdata.plugins.js';
 import { SmartdataDb } from './smartdata.classes.db.js';
-import { Manager, setDefaultManagerForDoc } from './smartdata.classes.collection.js';
+import { managed, setDefaultManagerForDoc } from './smartdata.classes.collection.js';
 import { SmartDataDbDoc, svDb, unI } from './smartdata.classes.doc.js';
+import { SmartdataDbWatcher } from './smartdata.classes.watcher.js';
 
-@Manager()
-class DistributedClass extends SmartDataDbDoc<DistributedClass, DistributedClass> {
+@managed()
+export class DistributedClass extends SmartDataDbDoc<DistributedClass, DistributedClass> {
   // INSTANCE
   @unI()
   public id: string;
 
   @svDb()
   public data: {
-    status: 'bidding' | 'settled' | 'initializing' | 'stopped';
+    status: 'initializing' | 'bidding' | 'settled' | 'stopped';
     biddingShortcode?: string;
+    biddingStartTime?: number;
     lastUpdated: number;
     elected: boolean;
     /**
@@ -37,14 +39,14 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
   public readyPromise: Promise<any>;
   public db: SmartdataDb;
   private asyncExecutionStack = new plugins.lik.AsyncExecutionStack();
-  private ownInstance: DistributedClass;
+  public ownInstance: DistributedClass;
+  public distributedWatcher: SmartdataDbWatcher<DistributedClass>;
 
-  constructor(dbArg?: SmartdataDb) {
+  constructor(dbArg: SmartdataDb) {
     super();
     this.db = dbArg;
     setDefaultManagerForDoc(this, DistributedClass);
     this.readyPromise = this.db.statusConnectedDeferred.promise;
-    this.init();
   }
 
   // smartdata specific stuff
@@ -53,26 +55,56 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
   }
 
   public async stop() {
-    if (this.ownInstance?.data.elected) {
-      this.ownInstance.data.elected = false;
-    } else {
-      console.log(`can't stop a distributed instance that has not been started yet.`);
-    }
+    await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+      if (this.distributedWatcher) {
+        await this.distributedWatcher.close();
+      }
+      if (this.ownInstance?.data.elected) {
+        this.ownInstance.data.elected = false;
+      }
+      if (this.ownInstance?.data.status === 'stopped') {
+        console.log(`stopping a distributed instance that has not been started yet.`);
+      }
+      this.ownInstance.data.status = 'stopped';
+      await this.ownInstance.save();
+      console.log(`stopped ${this.ownInstance.id}`);
+    });
   }
 
   public id = plugins.smartunique.uni('distributedInstance');
 
   private startHeartbeat = async () => {
     while (this.ownInstance.data.status !== 'stopped') {
-      await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
-        await this.ownInstance.updateFromDb();
-        this.ownInstance.data.lastUpdated = Date.now();
-        await this.ownInstance.save();
-      });
-      await plugins.smartdelay.delayFor(10000);
+      await this.sendHeartbeat();
+      await plugins.smartdelay.delayForRandom(5000, 10000);
     }
   };
-  public async init() {
+
+  public async sendHeartbeat() {
+    await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+      if (this.ownInstance.data.status === 'stopped') {
+        console.log(`aborted sending heartbeat because status is stopped`);
+        return;
+      }
+      await this.ownInstance.updateFromDb();
+      this.ownInstance.data.lastUpdated = Date.now();
+      await this.ownInstance.save();
+      console.log(`sent heartbeat for ${this.ownInstance.id}`);
+      const allInstances = DistributedClass.getInstances({});
+    });
+    if (this.ownInstance.data.status === 'stopped') {
+      console.log(`aborted sending heartbeat because status is stopped`);
+      return;
+    }
+    const eligibleLeader = await this.getEligibleLeader();
+    // not awaiting here because we don't want to block the heartbeat
+    this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+      if (!eligibleLeader && this.ownInstance.data.status === 'settled') {
+        this.checkAndMaybeLead();
+      }
+    });
+  }
+  private async init() {
     await this.readyPromise;
     if (!this.ownInstance) {
       await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
@@ -87,6 +119,8 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
         };
         await this.ownInstance.save();
       });
+    } else {
+      console.warn(`distributed instance already initialized`);
     }
 
     // lets enable the heartbeat
@@ -98,42 +132,73 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
     return this.ownInstance;
   }
 
+  public async getEligibleLeader() {
+    return this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+      const allInstances = await DistributedClass.getInstances({});
+      let leaders = allInstances.filter((instanceArg) => instanceArg.data.elected === true);
+      const eligibleLeader = leaders.find(
+        (leader) =>
+          leader.data.lastUpdated >=
+          Date.now() - plugins.smarttime.getMilliSecondsFromUnits({ seconds: 20 })
+      );
+      return eligibleLeader;
+    });
+  }
+
   // --> leader election
   public async checkAndMaybeLead() {
-    const allInstances = await DistributedClass.getInstances({});
-    let leader = allInstances.find((instanceArg) => instanceArg.data.elected === true);
-    if (
-      leader &&
-      leader.data.lastUpdated >=
-        Date.now() - plugins.smarttime.getMilliSecondsFromUnits({ minutes: 1 })
-    ) {
+    await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+      this.ownInstance.data.status = 'initializing';
+      this.ownInstance.save();
+    });
+    if (await this.getEligibleLeader()) {
       await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
         await this.ownInstance.updateFromDb();
         this.ownInstance.data.status = 'settled';
         await this.ownInstance.save();
+        console.log(`${this.ownInstance.id} settled as follower`);
       });
       return;
+    } else if (
+      (await DistributedClass.getInstances({})).find((instanceArg) => {
+        instanceArg.data.status === 'bidding' &&
+          instanceArg.data.biddingStartTime <= Date.now() - 4000 &&
+          instanceArg.data.biddingStartTime >= Date.now() - 30000;
+      })
+    ) {
+      console.log('too late to the bidding party... waiting for next round.');
+      return;
     } else {
       await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+        await this.ownInstance.updateFromDb();
         this.ownInstance.data.status = 'bidding';
+        this.ownInstance.data.biddingStartTime = Date.now();
         this.ownInstance.data.biddingShortcode = plugins.smartunique.shortId();
         await this.ownInstance.save();
+        console.log('bidding code stored.');
       });
-      await plugins.smartdelay.delayFor(plugins.smarttime.getMilliSecondsFromUnits({ minutes: 2 }));
+      console.log(`bidding for leadership...`);
+      await plugins.smartdelay.delayFor(
+        plugins.smarttime.getMilliSecondsFromUnits({ seconds: 5 })
+      );
       await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
         let biddingInstances = await DistributedClass.getInstances({});
         biddingInstances = biddingInstances.filter(
           (instanceArg) =>
-            !instanceArg.data.elected &&
+            instanceArg.data.status === 'bidding' &&
             instanceArg.data.lastUpdated >=
-              Date.now() - plugins.smarttime.getMilliSecondsFromUnits({ minutes: 1 })
+              Date.now() - plugins.smarttime.getMilliSecondsFromUnits({ seconds: 10 })
         );
+        console.log(`found ${biddingInstances.length} bidding instances...`);
         this.ownInstance.data.elected = true;
         for (const biddingInstance of biddingInstances) {
           if (biddingInstance.data.biddingShortcode < this.ownInstance.data.biddingShortcode) {
             this.ownInstance.data.elected = false;
           }
         }
+        await plugins.smartdelay.delayFor(5000);
+        console.log(`settling with status elected = ${this.ownInstance.data.elected}`);
+        this.ownInstance.data.status = 'settled';
         await this.ownInstance.save();
       });
       if (this.ownInstance.data.elected) {
@@ -148,20 +213,38 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
    * the leading is implemented here
    */
   public async leadFunction() {
-    const ownInstance = await this.init();
-    const watcher = await DistributedClass.watch({});
-    /**
-     * this function is started once per unique job request
-     */
-    const startResultTimer = async () => {};
+    this.distributedWatcher = await DistributedClass.watch({});
 
-    watcher.changeSubject.subscribe({
+    const currentTaskRequests: Array<{
+      taskName: string;
+      taskExecutionTime: number;
+      /**
+       * all instances that requested this task
+       */
+      requestingDistibutedInstanceIds: string[];
+      responseTimeout: plugins.smartdelay.Timeout<any>;
+    }> = [];
+
+    this.distributedWatcher.changeSubject.subscribe({
       next: async (distributedDoc) => {
+        if (!distributedDoc) {
+          console.log(`registered deletion of instance...`);
+          return;
+        }
+        console.log(distributedDoc);
+        console.log(`registered change for ${distributedDoc.id}`);
         distributedDoc;
       },
     });
-    while (this.ownInstance.data.status !== 'stopped') {
-      await plugins.smartdelay.delayFor(1000);
+
+    while (this.ownInstance.data.status !== 'stopped' && this.ownInstance.data.elected) {
+      const allInstances = await DistributedClass.getInstances({});
+      for (const instance of allInstances) {
+        if (instance.data.status === 'stopped') {
+          await instance.delete();
+        };
+      }
+      await plugins.smartdelay.delayFor(10000);
     }
   }
 
@@ -169,12 +252,28 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
   public async fireDistributedTaskRequest(
     taskRequestArg: plugins.taskbuffer.distributedCoordination.IDistributedTaskRequest
   ): Promise<plugins.taskbuffer.distributedCoordination.IDistributedTaskRequestResult> {
-    const ownInstance = await this.init();
     await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+      if (!this.ownInstance) {
+        console.error('instance need to be started first...');
+        return;
+      }
+      await this.ownInstance.updateFromDb();
       this.ownInstance.data.taskRequests.push(taskRequestArg);
       await this.ownInstance.save();
     });
-    return null;
+    await plugins.smartdelay.delayFor(10000);
+    const result = await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+      await this.ownInstance.updateFromDb();
+      const taskRequestResult = this.ownInstance.data.taskRequestResults.find((resultItem) => {
+        return resultItem.requestResponseId === taskRequestArg.requestResponseId;
+      });
+      return taskRequestResult;
+    });
+    if (!result) {
+      console.warn('no result found for task request...');
+      return null;
+    }
+    return result;
   }
 
   public async updateDistributedTaskRequest(
@@ -187,6 +286,10 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
           infoBasisItem.taskExecutionTime === infoBasisArg.taskExecutionTime
         );
       });
+      if (!existingInfoBasis) {
+        console.warn('trying to update a non existing task request... aborting!');
+        return;
+      }
       Object.assign(existingInfoBasis, infoBasisArg);
       await this.ownInstance.save();
       plugins.smartdelay.delayFor(60000).then(() => {

ts/smartdata.classes.doc.ts

@@ -7,6 +7,16 @@ import { SmartdataDbWatcher } from './smartdata.classes.watcher.js';
 
 export type TDocCreation = 'db' | 'new' | 'mixed';
 
+export function globalSvDb() {
+  return (target: SmartDataDbDoc<unknown, unknown>, key: string) => {
+    console.log(`called svDb() on >${target.constructor.name}.${key}<`);
+    if (!target.globalSaveableProperties) {
+      target.globalSaveableProperties = [];
+    }
+    target.globalSaveableProperties.push(key);
+  };
+}
+
 /**
  * saveable - saveable decorator to be used on class properties
  */
@@ -127,6 +137,13 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
     }
   }
 
+  /**
+   * get a unique id prefixed with the class name
+   */
+  public static async getNewId<T = any>(this: plugins.tsclass.typeFest.Class<T>, lengthArg: number = 20) {
+    return `${(this as any).className}:${plugins.smartunique.shortId(lengthArg)}`;
+  }
+
   /**
    * get cursor
    * @returns
@@ -174,6 +191,17 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
     await cursor.forEach(forEachFunction);
   }
 
+  /**
+   * returns a count of the documents in the collection
+   */
+  public static async getCount<T>(
+    this: plugins.tsclass.typeFest.Class<T>,
+    filterArg: plugins.tsclass.typeFest.PartialDeep<T> = ({} as any)
+  ) {
+    const collection: SmartdataCollection<T> = (this as any).collection;
+    return await collection.getCount(filterArg);
+  }
+
   // INSTANCE
 
   /**
@@ -181,13 +209,30 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
    */
   public creationStatus: TDocCreation = 'new';
 
+  /**
+   * updated from db in any case where doc comes from db
+   */
+  @globalSvDb()
+  _createdAt: string = (new Date()).toISOString();
+
+  /**
+   * will be updated everytime the doc is saved
+   */
+  @globalSvDb()
+  _updatedAt: string = (new Date()).toISOString();
+
+  /**
+   * an array of saveable properties of ALL doc
+   */
+  public globalSaveableProperties: string[];
+
   /**
    * unique indexes
    */
   public uniqueIndexes: string[];
 
   /**
-   * an array of saveable properties of a doc
+   * an array of saveable properties of a specific doc
    */
   public saveableProperties: string[];
 
@@ -214,6 +259,9 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
     // tslint:disable-next-line: no-this-assignment
     const self: any = this;
     let dbResult: any;
+
+    this._updatedAt = (new Date()).toISOString();
+
     switch (this.creationStatus) {
       case 'db':
         dbResult = await this.collection.update(self);
@@ -257,7 +305,7 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
    * updates an object from db
    */
   public async updateFromDb() {
-    const mongoDbNativeDoc = await this.collection.findOne(this.createIdentifiableObject());
+    const mongoDbNativeDoc = await this.collection.findOne(await this.createIdentifiableObject());
     for (const key of Object.keys(mongoDbNativeDoc)) {
       this[key] = mongoDbNativeDoc[key];
     }
@@ -268,7 +316,11 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
    */
   public async createSavableObject(): Promise<TImplements> {
     const saveableObject: unknown = {}; // is not exposed to outside, so any is ok here
-    for (const propertyNameString of this.saveableProperties) {
+    const saveableProperties = [
+      ...this.globalSaveableProperties,
+      ...this.saveableProperties
+    ]
+    for (const propertyNameString of saveableProperties) {
       saveableObject[propertyNameString] = this[propertyNameString];
     }
     return saveableObject as TImplements;

ts/smartdata.classes.easystore.ts

@@ -37,7 +37,16 @@ export class EasyStore<T> {
     this.nameId = nameIdArg;
   }
 
-  private async getEasyStore() {
+  private easyStorePromise: Promise<InstanceType<typeof this.easyStoreClass>>;
+  private async getEasyStore(): Promise<InstanceType<typeof this.easyStoreClass>> {
+    if (this.easyStorePromise) {
+      return this.easyStorePromise;
+    };
+
+    // first run from here
+    const deferred = plugins.smartpromise.defer<InstanceType<typeof this.easyStoreClass>>();
+    this.easyStorePromise = deferred.promise;
+
     let easyStore = await this.easyStoreClass.getInstance({
       nameId: this.nameId,
     });
@@ -48,7 +57,8 @@ export class EasyStore<T> {
       easyStore.data = {};
       await easyStore.save();
     }
-    return easyStore;
+    deferred.resolve(easyStore);
+    return this.easyStorePromise;
   }
 
   /**
@@ -70,7 +80,7 @@ export class EasyStore<T> {
   /**
    * writes a specific key to the keyValueStore
    */
-  public async writeKey(keyArg: keyof T, valueArg: any) {
+  public async writeKey<TKey extends keyof T>(keyArg: TKey, valueArg: T[TKey]) {
     const easyStore = await this.getEasyStore();
     easyStore.data[keyArg] = valueArg;
     await easyStore.save();

ts/smartdata.classes.watcher.ts

@@ -17,9 +17,13 @@ export class SmartdataDbWatcher<T = any> {
     smartdataDbDocArg: typeof SmartDataDbDoc
   ) {
     this.changeStream = changeStreamArg;
-    this.changeStream.on('change', async (item: T) => {
+    this.changeStream.on('change', async (item: any) => {
+      if (!item.fullDocument) {
+        this.changeSubject.next(null);
+        return;
+      }
       this.changeSubject.next(
-        smartdataDbDocArg.createInstanceFromMongoDbNativeDoc(item) as any as T
+        smartdataDbDocArg.createInstanceFromMongoDbNativeDoc(item.fullDocument) as any as T
       );
     });
     plugins.smartdelay.delayFor(0).then(() => {

ts/smartdata.plugins.ts

@@ -8,7 +8,6 @@ import * as lik from '@push.rocks/lik';
 import * as smartdelay from '@push.rocks/smartdelay';
 import * as smartlog from '@push.rocks/smartlog';
 import * as smartpromise from '@push.rocks/smartpromise';
-import * as smartq from '@push.rocks/smartpromise';
 import * as smartrx from '@push.rocks/smartrx';
 import * as smartstring from '@push.rocks/smartstring';
 import * as smarttime from '@push.rocks/smarttime';
@@ -21,7 +20,6 @@ export {
   smartdelay,
   smartpromise,
   smartlog,
-  smartq,
   smartrx,
   mongodb,
   smartstring,

tsconfig.json

@@ -3,7 +3,12 @@
     "experimentalDecorators": true,
     "useDefineForClassFields": false,
     "target": "ES2022",
-    "module": "ES2022",
-    "moduleResolution": "nodenext"
-  }
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "esModuleInterop": true,
+    "verbatimModuleSyntax": true
+  },
+  "exclude": [
+    "dist_*/**/*.d.ts"
+  ]
 }