5.1.0

.gitea/workflows/default_nottags.yaml

@@ -0,0 +1,66 @@
+name: Default (not tags)
+
+on:
+  push:
+    tags-ignore:
+      - '**'
+
+env:
+  IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
+  NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
+  NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
+  NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
+  NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
+  NPMCI_URL_CLOUDLY: ${{secrets.NPMCI_URL_CLOUDLY}}
+
+jobs:
+  security:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install pnpm and npmci
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @shipzone/npmci
+
+      - name: Run npm prepare
+        run: npmci npm prepare
+
+      - name: Audit production dependencies
+        run: |
+          npmci command npm config set registry https://registry.npmjs.org
+          npmci command pnpm audit --audit-level=high --prod
+        continue-on-error: true
+
+      - name: Audit development dependencies
+        run: |
+          npmci command npm config set registry https://registry.npmjs.org
+          npmci command pnpm audit --audit-level=high --dev
+        continue-on-error: true
+
+  test:
+    if: ${{ always() }}
+    needs: security
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Test stable
+        run: |
+          npmci node install stable
+          npmci npm install
+          npmci npm test
+
+      - name: Test build
+        run: |
+          npmci node install stable
+          npmci npm install
+          npmci npm build

.gitea/workflows/default_tags.yaml

@@ -0,0 +1,124 @@
+name: Default (tags)
+
+on:
+  push:
+    tags:
+      - '*'
+
+env:
+  IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
+  NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
+  NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
+  NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
+  NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
+  NPMCI_URL_CLOUDLY: ${{secrets.NPMCI_URL_CLOUDLY}}
+
+jobs:
+  security:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Prepare
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @shipzone/npmci
+          npmci npm prepare
+
+      - name: Audit production dependencies
+        run: |
+          npmci command npm config set registry https://registry.npmjs.org
+          npmci command pnpm audit --audit-level=high --prod
+        continue-on-error: true
+
+      - name: Audit development dependencies
+        run: |
+          npmci command npm config set registry https://registry.npmjs.org
+          npmci command pnpm audit --audit-level=high --dev
+        continue-on-error: true
+
+  test:
+    if: ${{ always() }}
+    needs: security
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Prepare
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @shipzone/npmci
+          npmci npm prepare
+
+      - name: Test stable
+        run: |
+          npmci node install stable
+          npmci npm install
+          npmci npm test
+
+      - name: Test build
+        run: |
+          npmci node install stable
+          npmci npm install
+          npmci npm build
+
+  release:
+    needs: test
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Prepare
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @shipzone/npmci
+          npmci npm prepare
+
+      - name: Release
+        run: |
+          npmci node install stable
+          npmci npm publish
+
+  metadata:
+    needs: test
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+    continue-on-error: true
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Prepare
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @shipzone/npmci
+          npmci npm prepare
+
+      - name: Code quality
+        run: |
+          npmci command npm install -g typescript
+          npmci npm install
+
+      - name: Trigger
+        run: npmci trigger
+
+      - name: Build docs and upload artifacts
+        run: |
+          npmci node install stable
+          npmci npm install
+          pnpm install -g @gitzone/tsdoc
+          npmci command tsdoc
+        continue-on-error: true

.gitlab-ci.yml

@@ -1,141 +0,0 @@
-# gitzone ci_default
-image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
-
-cache:
-  paths:
-    - .npmci_cache/
-  key: '$CI_BUILD_STAGE'
-
-stages:
-  - security
-  - test
-  - release
-  - metadata
-
-before_script:
-  - npm install -g @shipzone/npmci
-
-# ====================
-# security stage
-# ====================
-mirror:
-  stage: security
-  script:
-    - npmci git mirror
-  only:
-    - tags
-  tags:
-    - lossless
-    - docker
-    - notpriv
-
-auditProductionDependencies:
-  image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
-  stage: security
-  script:
-    - npmci npm prepare
-    - npmci command npm install --production --ignore-scripts
-    - npmci command npm config set registry https://registry.npmjs.org
-    - npmci command npm audit --audit-level=high --only=prod --production
-  tags:
-    - docker
-  allow_failure: true
-
-auditDevDependencies:
-  image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
-  stage: security
-  script:
-    - npmci npm prepare
-    - npmci command npm install --ignore-scripts
-    - npmci command npm config set registry https://registry.npmjs.org
-    - npmci command npm audit --audit-level=high --only=dev
-  tags:
-    - docker
-  allow_failure: true
-
-# ====================
-# test stage
-# ====================
-
-testStable:
-  stage: test
-  script:
-    - npmci npm prepare
-    - npmci node install stable
-    - npmci npm install
-    - npmci npm test
-  coverage: /\d+.?\d+?\%\s*coverage/
-  tags:
-    - docker
-
-testBuild:
-  stage: test
-  script:
-    - npmci npm prepare
-    - npmci node install stable
-    - npmci npm install
-    - npmci command npm run build
-  coverage: /\d+.?\d+?\%\s*coverage/
-  tags:
-    - docker
-
-release:
-  stage: release
-  script:
-    - npmci node install stable
-    - npmci npm publish
-  only:
-    - tags
-  tags:
-    - lossless
-    - docker
-    - notpriv
-
-# ====================
-# metadata stage
-# ====================
-codequality:
-  stage: metadata
-  allow_failure: true
-  only:
-    - tags
-  script:
-    - npmci command npm install -g tslint typescript
-    - npmci npm prepare
-    - npmci npm install
-    - npmci command "tslint -c tslint.json ./ts/**/*.ts"
-  tags:
-    - lossless
-    - docker
-    - priv
-
-trigger:
-  stage: metadata
-  script:
-    - npmci trigger
-  only:
-    - tags
-  tags:
-    - lossless
-    - docker
-    - notpriv
-
-pages:
-  stage: metadata
-  script:
-    - npmci node install lts
-    - npmci command npm install -g @gitzone/tsdoc
-    - npmci npm prepare
-    - npmci npm install
-    - npmci command tsdoc
-  tags:
-    - lossless
-    - docker
-    - notpriv
-  only:
-    - tags
-  artifacts:
-    expire_in: 1 week
-    paths:
-      - public
-  allow_failure: true

npmextra.json

@@ -12,12 +12,15 @@
   "gitzone": {
     "projectType": "npm",
     "module": {
-      "githost": "gitlab.com",
-      "gitscope": "pushrocks",
+      "githost": "code.foss.global",
+      "gitscope": "push.rocks",
       "gitrepo": "smartdata",
-      "shortDescription": "do more with data",
-      "npmPackagename": "@pushrocks/smartdata",
+      "description": "do more with data",
+      "npmPackagename": "@push.rocks/smartdata",
       "license": "MIT"
     }
+  },
+  "tsdoc": {
+    "legal": "\n## License and Legal Information\n\nThis repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository. \n\n**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.\n\n### Trademarks\n\nThis project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.\n\n### Company Information\n\nTask Venture Capital GmbH  \nRegistered at District court Bremen HRB 35230 HB, Germany\n\nFor any legal inquiries or if you require further information, please contact us via email at hello@task.vc.\n\nBy using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.\n"
   }
 }

package.json

@@ -1,6 +1,6 @@
 {
-  "name": "@pushrocks/smartdata",
-  "version": "5.0.2",
+  "name": "@push.rocks/smartdata",
+  "version": "5.1.0",
   "private": false,
   "description": "do more with data",
   "main": "dist_ts/index.js",
@@ -8,7 +8,8 @@
   "type": "module",
   "scripts": {
     "test": "tstest test/",
-    "build": "tsbuild --web --allowimplicitany"
+    "build": "tsbuild --web --allowimplicitany",
+    "buildDocs": "tsdoc"
   },
   "repository": {
     "type": "git",
@@ -21,27 +22,26 @@
   },
   "homepage": "https://gitlab.com/pushrocks/smartdata#README",
   "dependencies": {
-    "@pushrocks/lik": "^5.0.4",
-    "@pushrocks/smartdelay": "^2.0.13",
-    "@pushrocks/smartlog": "^2.0.44",
-    "@pushrocks/smartmongo": "^2.0.1",
-    "@pushrocks/smartpromise": "^3.1.7",
-    "@pushrocks/smartrx": "^2.0.25",
-    "@pushrocks/smartstring": "^4.0.2",
-    "@pushrocks/smartunique": "^3.0.3",
-    "@tsclass/tsclass": "^4.0.2",
-    "@types/lodash": "^4.14.182",
-    "@types/mongodb": "^4.0.7",
-    "lodash": "^4.17.21",
-    "mongodb": "^4.6.0"
+    "@push.rocks/lik": "^6.0.14",
+    "@push.rocks/smartdelay": "^3.0.1",
+    "@push.rocks/smartlog": "^3.0.2",
+    "@push.rocks/smartmongo": "^2.0.10",
+    "@push.rocks/smartpromise": "^4.0.2",
+    "@push.rocks/smartrx": "^3.0.7",
+    "@push.rocks/smartstring": "^4.0.15",
+    "@push.rocks/smarttime": "^4.0.6",
+    "@push.rocks/smartunique": "^3.0.8",
+    "@push.rocks/taskbuffer": "^3.1.7",
+    "@tsclass/tsclass": "^4.0.52",
+    "mongodb": "^6.5.0"
   },
   "devDependencies": {
-    "@gitzone/tsbuild": "^2.1.61",
-    "@gitzone/tstest": "^1.0.71",
-    "@pushrocks/qenv": "^4.0.10",
-    "@pushrocks/tapbundle": "^5.0.3",
-    "@types/node": "^17.0.34",
-    "@types/shortid": "0.0.29"
+    "@gitzone/tsbuild": "^2.1.66",
+    "@gitzone/tsrun": "^1.2.44",
+    "@gitzone/tstest": "^1.0.77",
+    "@push.rocks/qenv": "^6.0.5",
+    "@push.rocks/tapbundle": "^5.0.22",
+    "@types/node": "^20.11.30"
   },
   "files": [
     "ts/**/*",

qenv.yml

@@ -1,4 +1 @@
 required:
-  - MONGO_URL
-  - MONGO_DBNAME
-  - MONGO_PASS

readme.md

@@ -1,27 +1,26 @@
-# @pushrocks/smartdata
+# @push.rocks/smartdata
 do more with data
 
 ## Availabililty and Links
-* [npmjs.org (npm package)](https://www.npmjs.com/package/@pushrocks/smartdata)
-* [gitlab.com (source)](https://gitlab.com/pushrocks/smartdata)
-* [github.com (source mirror)](https://github.com/pushrocks/smartdata)
-* [docs (typedoc)](https://pushrocks.gitlab.io/smartdata/)
+* [npmjs.org (npm package)](https://www.npmjs.com/package/@push.rocks/smartdata)
+* [gitlab.com (source)](https://gitlab.com/push.rocks/smartdata)
+* [github.com (source mirror)](https://github.com/push.rocks/smartdata)
+* [docs (typedoc)](https://push.rocks.gitlab.io/smartdata/)
 
 ## Status for master
 
 Status Category | Status Badge
 -- | --
-GitLab Pipelines | [![pipeline status](https://gitlab.com/pushrocks/smartdata/badges/master/pipeline.svg)](https://lossless.cloud)
-GitLab Pipline Test Coverage | [![coverage report](https://gitlab.com/pushrocks/smartdata/badges/master/coverage.svg)](https://lossless.cloud)
-npm | [![npm downloads per month](https://badgen.net/npm/dy/@pushrocks/smartdata)](https://lossless.cloud)
-Snyk | [![Known Vulnerabilities](https://badgen.net/snyk/pushrocks/smartdata)](https://lossless.cloud)
+GitLab Pipelines | [![pipeline status](https://gitlab.com/push.rocks/smartdata/badges/master/pipeline.svg)](https://lossless.cloud)
+GitLab Pipline Test Coverage | [![coverage report](https://gitlab.com/push.rocks/smartdata/badges/master/coverage.svg)](https://lossless.cloud)
+npm | [![npm downloads per month](https://badgen.net/npm/dy/@push.rocks/smartdata)](https://lossless.cloud)
+Snyk | [![Known Vulnerabilities](https://badgen.net/snyk/push.rocks/smartdata)](https://lossless.cloud)
 TypeScript Support | [![TypeScript](https://badgen.net/badge/TypeScript/>=%203.x/blue?icon=typescript)](https://lossless.cloud)
 node Support | [![node](https://img.shields.io/badge/node->=%2010.x.x-blue.svg)](https://nodejs.org/dist/latest-v10.x/docs/api/)
 Code Style | [![Code Style](https://badgen.net/badge/style/prettier/purple)](https://lossless.cloud)
-PackagePhobia (total standalone install weight) | [![PackagePhobia](https://badgen.net/packagephobia/install/@pushrocks/smartdata)](https://lossless.cloud)
-PackagePhobia (package size on registry) | [![PackagePhobia](https://badgen.net/packagephobia/publish/@pushrocks/smartdata)](https://lossless.cloud)
-BundlePhobia (total size when bundled) | [![BundlePhobia](https://badgen.net/bundlephobia/minzip/@pushrocks/smartdata)](https://lossless.cloud)
-Platform support | [![Supports Windows 10](https://badgen.net/badge/supports%20Windows%2010/yes/green?icon=windows)](https://lossless.cloud) [![Supports Mac OS X](https://badgen.net/badge/supports%20Mac%20OS%20X/yes/green?icon=apple)](https://lossless.cloud)
+PackagePhobia (total standalone install weight) | [![PackagePhobia](https://badgen.net/packagephobia/install/@push.rocks/smartdata)](https://lossless.cloud)
+PackagePhobia (package size on registry) | [![PackagePhobia](https://badgen.net/packagephobia/publish/@push.rocks/smartdata)](https://lossless.cloud)
+BundlePhobia (total size when bundled) | [![BundlePhobia](https://badgen.net/bundlephobia/minzip/@push.rocks/smartdata)](https://lossless.cloud)
 
 ## Usage
 
@@ -150,7 +149,6 @@ We are always happy for code contributions. If you are not the code contributing
 
 For further information read the linked docs at the top of this readme.
 
-> MIT licensed | **©** [Lossless GmbH](https://lossless.gmbh)
+## Legal
+> MIT licensed | **©** [Task Venture Capital GmbH](https://task.vc)
 | By using this npm module you agree to our [privacy policy](https://lossless.gmbH/privacy)
-
-[![repo-footer](https://lossless.gitlab.io/publicrelations/repofooter.svg)](https://maintainedby.lossless.com)

test/test.distributedcoordinator.ts

@@ -0,0 +1,112 @@
+import { tap, expect } from '@push.rocks/tapbundle';
+import * as smartmongo from '@push.rocks/smartmongo';
+import type * as taskbuffer from '@push.rocks/taskbuffer';
+
+import * as smartdata from '../ts/index.js';
+import { SmartdataDistributedCoordinator, DistributedClass } from '../ts/smartdata.classes.distributedcoordinator.js'; // path might need adjusting
+const totalInstances = 10;
+
+// =======================================
+// Connecting to the database server
+// =======================================
+
+let smartmongoInstance: smartmongo.SmartMongo;
+let testDb: smartdata.SmartdataDb;
+
+tap.test('should create a testinstance as database', async () => {
+  smartmongoInstance = await smartmongo.SmartMongo.createAndStart();
+  testDb = new smartdata.SmartdataDb(await smartmongoInstance.getMongoDescriptor());
+  await testDb.init();
+});
+
+tap.test('should instantiate DistributedClass', async (tools) => {
+    const instance = new DistributedClass();
+    expect(instance).toBeInstanceOf(DistributedClass);
+});
+
+tap.test('DistributedClass should update the time', async (tools) => {
+    const distributedCoordinator = new SmartdataDistributedCoordinator(testDb);
+    await distributedCoordinator.start();
+    const initialTime = distributedCoordinator.ownInstance.data.lastUpdated;
+    await distributedCoordinator.sendHeartbeat();
+    const updatedTime = distributedCoordinator.ownInstance.data.lastUpdated;
+    expect(updatedTime).toBeGreaterThan(initialTime);
+    await distributedCoordinator.stop();
+});
+
+tap.test('should instantiate SmartdataDistributedCoordinator', async (tools) => {
+    const distributedCoordinator = new SmartdataDistributedCoordinator(testDb);
+    await distributedCoordinator.start();
+    expect(distributedCoordinator).toBeInstanceOf(SmartdataDistributedCoordinator);
+    await distributedCoordinator.stop();
+});
+
+tap.test('SmartdataDistributedCoordinator should update leader status', async (tools) => {
+    const distributedCoordinator = new SmartdataDistributedCoordinator(testDb);
+    await distributedCoordinator.start();
+    await distributedCoordinator.checkAndMaybeLead();
+    expect(distributedCoordinator.ownInstance.data.elected).toBeOneOf([true, false]);
+    await distributedCoordinator.stop();
+});
+
+tap.test('SmartdataDistributedCoordinator should handle distributed task requests', async (tools) => {
+    const distributedCoordinator = new SmartdataDistributedCoordinator(testDb);
+    await distributedCoordinator.start();
+
+    const mockTaskRequest: taskbuffer.distributedCoordination.IDistributedTaskRequest = {
+        submitterId: "mockSubmitter12345",    // Some unique mock submitter ID
+        requestResponseId: 'uni879873462hjhfkjhsdf', // Some unique ID for the request-response
+        taskName: "SampleTask",
+        taskVersion: "1.0.0",                  // Assuming it's a version string
+        taskExecutionTime: Date.now(),
+        taskExecutionTimeout: 60000,           // Let's say the timeout is 1 minute (60000 ms)
+        taskExecutionParallel: 5,              // Let's assume max 5 parallel executions
+        status: 'requesting'
+    };
+
+    const response = await distributedCoordinator.fireDistributedTaskRequest(mockTaskRequest);
+    console.log(response)  // based on your expected structure for the response
+    await distributedCoordinator.stop();
+});
+
+tap.test('SmartdataDistributedCoordinator should update distributed task requests', async (tools) => {
+    const distributedCoordinator = new SmartdataDistributedCoordinator(testDb);
+    
+    await distributedCoordinator.start();
+
+    const mockTaskRequest: taskbuffer.distributedCoordination.IDistributedTaskRequest = {
+        submitterId: "mockSubmitter12345",    // Some unique mock submitter ID
+        requestResponseId: 'uni879873462hjhfkjhsdf', // Some unique ID for the request-response
+        taskName: "SampleTask",
+        taskVersion: "1.0.0",                  // Assuming it's a version string
+        taskExecutionTime: Date.now(),
+        taskExecutionTimeout: 60000,           // Let's say the timeout is 1 minute (60000 ms)
+        taskExecutionParallel: 5,              // Let's assume max 5 parallel executions
+        status: 'requesting'
+    };
+    
+
+    await distributedCoordinator.updateDistributedTaskRequest(mockTaskRequest);
+    // Here, we can potentially check if a DB entry got updated or some other side-effect of the update method.
+    await distributedCoordinator.stop();
+});
+
+tap.test('should elect only one leader amongst multiple instances', async (tools) => {
+    const coordinators = Array.from({ length: totalInstances }).map(() => new SmartdataDistributedCoordinator(testDb));
+    await Promise.all(coordinators.map(coordinator => coordinator.start()));
+    const leaders = coordinators.filter(coordinator => coordinator.ownInstance.data.elected);
+    for (const leader of leaders) {
+        console.log(leader.ownInstance);
+    }
+    expect(leaders.length).toEqual(1);
+
+    // stopping clears a coordinator from being elected.
+    await Promise.all(coordinators.map(coordinator => coordinator.stop()));
+});
+
+tap.test('should clean up', async () => {
+    await smartmongoInstance.stopAndDumpToDir(`.nogit/dbdump/test.distributedcoordinator.ts`);
+    setTimeout(() => process.exit(), 2000);
+})
+
+tap.start({ throwOnError: true });

test/test.easystore.ts

@@ -1,6 +1,6 @@
-import { tap, expect } from '@pushrocks/tapbundle';
-import { Qenv } from '@pushrocks/qenv';
-import * as smartmongo from '@pushrocks/smartmongo';
+import { tap, expect } from '@push.rocks/tapbundle';
+import { Qenv } from '@push.rocks/qenv';
+import * as smartmongo from '@push.rocks/smartmongo';
 import { smartunique } from '../ts/smartdata.plugins.js';
 
 const testQenv = new Qenv(process.cwd(), process.cwd() + '/.nogit/');
@@ -26,7 +26,7 @@ tap.test('should create a testinstance as database', async () => {
 tap.skip.test('should connect to atlas', async (tools) => {
   const databaseName = `test-smartdata-${smartunique.shortId()}`;
   testDb = new smartdata.SmartdataDb({
-    mongoDbUrl: testQenv.getEnvVarOnDemand('MONGO_URL'),
+    mongoDbUrl: await testQenv.getEnvVarOnDemand('MONGO_URL'),
     mongoDbName: databaseName,
   });
   await testDb.init();

test/test.ts

@@ -1,8 +1,10 @@
-import { tap, expect } from '@pushrocks/tapbundle';
-import { Qenv } from '@pushrocks/qenv';
-import * as smartmongo from '@pushrocks/smartmongo';
+import { tap, expect } from '@push.rocks/tapbundle';
+import { Qenv } from '@push.rocks/qenv';
+import * as smartmongo from '@push.rocks/smartmongo';
 import { smartunique } from '../ts/smartdata.plugins.js';
 
+import * as mongodb from 'mongodb';
+
 const testQenv = new Qenv(process.cwd(), process.cwd() + '/.nogit/');
 
 console.log(process.memoryUsage());
@@ -28,7 +30,7 @@ tap.test('should create a testinstance as database', async () => {
 tap.skip.test('should connect to atlas', async (tools) => {
   const databaseName = `test-smartdata-${smartunique.shortId()}`;
   testDb = new smartdata.SmartdataDb({
-    mongoDbUrl: testQenv.getEnvVarOnDemand('MONGO_URL'),
+    mongoDbUrl: await testQenv.getEnvVarOnDemand('MONGO_URL'),
     mongoDbName: databaseName,
   });
   await testDb.init();
@@ -55,6 +57,9 @@ class Car extends smartdata.SmartDataDbDoc<Car, Car> {
   @smartdata.svDb()
   public brand: string;
 
+  @smartdata.svDb()
+  public testBuffer = Buffer.from('hello');
+
   @smartdata.svDb()
   deepData = {
     sodeep: 'yes',
@@ -67,7 +72,12 @@ class Car extends smartdata.SmartDataDbDoc<Car, Car> {
   }
 }
 
-tap.test('should save the car to the db', async () => {
+tap.test('should create a new id', async () => {
+  const newid = await Car.getNewId();
+  console.log(newid);
+});
+
+tap.test('should save the car to the db', async (toolsArg) => {
   const myCar = new Car('red', 'Volvo');
   await myCar.save();
 
@@ -75,6 +85,9 @@ tap.test('should save the car to the db', async () => {
   await myCar2.save();
 
   let counter = 0;
+
+  const gottenCarInstance = await Car.getInstance({});
+  console.log(gottenCarInstance.testBuffer instanceof mongodb.Binary);
   process.memoryUsage();
   do {
     const myCar3 = new Car('red', 'Renault');
@@ -92,7 +105,7 @@ tap.test('should save the car to the db', async () => {
 });
 
 tap.test('expect to get instance of Car with shallow match', async () => {
-  const totalQueryCycles = totalCars / 6;
+  const totalQueryCycles = totalCars / 2;
   let counter = 0;
   do {
     const timeStart = Date.now();
@@ -205,11 +218,13 @@ tap.test('should use a cursor', async () => {
 // close the database connection
 // =======================================
 tap.test('close', async () => {
+  if (smartmongoInstance) {
+    await smartmongoInstance.stopAndDumpToDir('./.nogit/dbdump/test.ts');
+  } else {
     await testDb.mongoDb.dropDatabase();
     await testDb.close();
-  if (smartmongoInstance) {
-    await smartmongoInstance.stop();
   }
+  setTimeout(() => process.exit(), 2000);
 });
 
 tap.start({ throwOnError: true });

test/test.typescript.ts

@@ -1,6 +1,6 @@
-import { tap, expect } from '@pushrocks/tapbundle';
-import { Qenv } from '@pushrocks/qenv';
-import * as smartmongo from '@pushrocks/smartmongo';
+import { tap, expect } from '@push.rocks/tapbundle';
+import { Qenv } from '@push.rocks/qenv';
+import * as smartmongo from '@push.rocks/smartmongo';
 import { smartunique } from '../ts/smartdata.plugins.js';
 
 const testQenv = new Qenv(process.cwd(), process.cwd() + '/.nogit/');
@@ -28,7 +28,7 @@ tap.test('should create a testinstance as database', async () => {
 tap.skip.test('should connect to atlas', async (tools) => {
   const databaseName = `test-smartdata-${smartunique.shortId()}`;
   testDb = new smartdata.SmartdataDb({
-    mongoDbUrl: testQenv.getEnvVarOnDemand('MONGO_URL'),
+    mongoDbUrl: await testQenv.getEnvVarOnDemand('MONGO_URL'),
     mongoDbName: databaseName,
   });
   await testDb.init();

test/test.watch.ts

@@ -1,6 +1,6 @@
-import { tap, expect } from '@pushrocks/tapbundle';
-import { Qenv } from '@pushrocks/qenv';
-import * as smartmongo from '@pushrocks/smartmongo';
+import { tap, expect } from '@push.rocks/tapbundle';
+import { Qenv } from '@push.rocks/qenv';
+import * as smartmongo from '@push.rocks/smartmongo';
 import { smartunique } from '../ts/smartdata.plugins.js';
 
 const testQenv = new Qenv(process.cwd(), process.cwd() + '/.nogit/');
@@ -28,7 +28,7 @@ tap.test('should create a testinstance as database', async () => {
 tap.skip.test('should connect to atlas', async (tools) => {
   const databaseName = `test-smartdata-${smartunique.shortId()}`;
   testDb = new smartdata.SmartdataDb({
-    mongoDbUrl: testQenv.getEnvVarOnDemand('MONGO_URL'),
+    mongoDbUrl: await testQenv.getEnvVarOnDemand('MONGO_URL'),
     mongoDbName: databaseName,
   });
   await testDb.init();
@@ -42,14 +42,14 @@ class House extends smartdata.SmartDataDbDoc<House, House> {
   @smartdata.svDb()
   public data = {
     id: smartunique.shortId(),
-    hello: 'hello'
-  }
+    hello: 'hello',
+  };
 }
 
 tap.test('should watch a collection', async (toolsArg) => {
   const done = toolsArg.defer();
   const watcher = await House.watch({});
-  watcher.changeSubject.subscribe(async houseArg => {
+  watcher.changeSubject.subscribe(async (houseArg) => {
     console.log('hey there, we observed a house');
     await watcher.close();
     done.resolve();
@@ -58,7 +58,7 @@ tap.test('should watch a collection', async (toolsArg) => {
   await newHouse.save();
   console.log('saved a house');
   await done.promise;
-})
+});
 
 // =======================================
 // close the database connection

ts/00_commitinfo_data.ts

@@ -2,7 +2,7 @@
  * autocreated commitinfo by @pushrocks/commitinfo
  */
 export const commitinfo = {
-  name: '@pushrocks/smartdata',
-  version: '5.0.2',
+  name: '@push.rocks/smartdata',
+  version: '5.1.0',
   description: 'do more with data'
 }

ts/index.ts

@@ -4,4 +4,11 @@ export * from './smartdata.classes.doc.js';
 export * from './smartdata.classes.easystore.js';
 export * from './smartdata.classes.cursor.js';
 
-export type { IMongoDescriptor } from './interfaces/index.js';
+import * as convenience from './smartadata.convenience.js';
+
+export { convenience };
+
+// to be removed with the next breaking update
+import type * as plugins from './smartdata.plugins.js';
+type IMongoDescriptor = plugins.tsclass.database.IMongoDescriptor;
+export type { IMongoDescriptor };

ts/interfaces/index.ts

@@ -1 +0,0 @@
-export * from './mongodescriptor.js';

ts/interfaces/mongodescriptor.ts

@@ -1,22 +0,0 @@
-export interface IMongoDescriptor {
-  /**
-   * the URL to connect to
-   */
-  mongoDbUrl: string;
-
-  /**
-   * the db to use for the project
-   */
-  mongoDbName?: string;
-
-  /**
-   * a username to use to connect to the database
-   */
-
-  mongoDbUser?: string;
-
-  /**
-   * an optional password that will be replace <PASSWORD> in the connection string
-   */
-  mongoDbPass?: string;
-}

ts/smartadata.convenience.ts

@@ -0,0 +1,5 @@
+import * as plugins from './smartdata.plugins.js';
+
+export const getNewUniqueId = async (prefixArg?: string) => {
+  return plugins.smartunique.uni(prefixArg);
+};

ts/smartdata.classes.collection.ts

@@ -26,7 +26,8 @@ const collectionFactory = new CollectionFactory();
  */
 export function Collection(dbArg: SmartdataDb | TDelayed<SmartdataDb>) {
   return function classDecorator<T extends { new (...args: any[]): {} }>(constructor: T) {
-    return class extends constructor {
+    const decoratedClass = class extends constructor {
+      public static className = constructor.name;
       public static get collection() {
         if (!(dbArg instanceof SmartdataDb)) {
           dbArg = dbArg();
@@ -40,6 +41,7 @@ export function Collection(dbArg: SmartdataDb | TDelayed<SmartdataDb>) {
         return collectionFactory.getCollection(constructor.name, dbArg);
       }
     };
+    return decoratedClass;
   };
 }
 
@@ -56,9 +58,10 @@ export const setDefaultManagerForDoc = <T>(managerArg: IManager, dbDocArg: T): T
  * This is a decorator that will tell the decorated class what dbTable to use
  * @param dbArg
  */
-export function Manager<TManager extends IManager>(managerArg?: TManager | TDelayed<TManager>) {
+export function managed<TManager extends IManager>(managerArg?: TManager | TDelayed<TManager>) {
   return function classDecorator<T extends { new (...args: any[]): any }>(constructor: T) {
-    return class extends constructor {
+    const decoratedClass = class extends constructor {
+      public static className = constructor.name;
       public static get collection() {
         let dbArg: SmartdataDb;
         if (!managerArg) {
@@ -106,10 +109,15 @@ export function Manager<TManager extends IManager>(managerArg?: TManager | TDela
         return manager;
       }
     };
+    return decoratedClass;
   };
 }
 
-// tslint:disable-next-line: max-classes-per-file
+/**
+ * @dpecrecated use @managed instead
+ */
+export const Manager = managed;
+
 export class SmartdataCollection<T> {
   /**
    * the collection that is used
@@ -180,7 +188,10 @@ export class SmartdataCollection<T> {
     return result;
   }
 
-  public async getCursor(filterObjectArg: any, dbDocArg: typeof SmartDataDbDoc): Promise<SmartdataDbCursor<any>> {
+  public async getCursor(
+    filterObjectArg: any,
+    dbDocArg: typeof SmartDataDbDoc
+  ): Promise<SmartdataDbCursor<any>> {
     await this.init();
     const cursor = this.mongoDbCollection.find(filterObjectArg);
     return new SmartdataDbCursor(cursor, dbDocArg);
@@ -200,15 +211,21 @@ export class SmartdataCollection<T> {
   /**
    * watches the collection while applying a filter
    */
-   public async watch(filterObject: any, smartdataDbDocArg: typeof SmartDataDbDoc): Promise<SmartdataDbWatcher> {
+  public async watch(
+    filterObject: any,
+    smartdataDbDocArg: typeof SmartDataDbDoc
+  ): Promise<SmartdataDbWatcher> {
     await this.init();
-    const changeStream = this.mongoDbCollection.watch([
+    const changeStream = this.mongoDbCollection.watch(
+      [
         {
-        $match: filterObject
+          $match: filterObject,
+        },
+      ],
+      {
+        fullDocument: 'updateLookup',
       }
-    ], {
-      fullDocument: 'updateLookup'
-    });
+    );
     const smartdataWatcher = new SmartdataDbWatcher(changeStream, smartdataDbDocArg);
     await smartdataWatcher.readyDeferred.promise;
     return smartdataWatcher;
@@ -261,7 +278,7 @@ export class SmartdataCollection<T> {
    * if this.objectValidation is not set it passes.
    */
   private checkDoc(docArg: T): Promise<void> {
-    const done = plugins.smartq.defer<void>();
+    const done = plugins.smartpromise.defer<void>();
     let validationResult = true;
     if (this.objectValidation) {
       validationResult = this.objectValidation(docArg);

ts/smartdata.classes.cursor.ts

@@ -16,20 +16,23 @@ export class SmartdataDbCursor<T = any> {
   }
 
   public async next(closeAtEnd = true) {
-    const result = this.smartdataDbDoc.createInstanceFromMongoDbNativeDoc(await this.mongodbCursor.next());
+    const result = this.smartdataDbDoc.createInstanceFromMongoDbNativeDoc(
+      await this.mongodbCursor.next()
+    );
     if (!result && closeAtEnd) {
       await this.close();
     }
     return result;
   }
 
-  public async forEach(forEachFuncArg: (itemArg: any) => Promise<any>, closeCursorAtEnd = true) {
+  public async forEach(forEachFuncArg: (itemArg: T) => Promise<any>, closeCursorAtEnd = true) {
     let nextDocument: any;
     do {
       nextDocument = await this.mongodbCursor.next();
       if (nextDocument) {
-        const nextClassInstance = this.smartdataDbDoc.createInstanceFromMongoDbNativeDoc(nextDocument);
-        await forEachFuncArg(nextClassInstance);
+        const nextClassInstance =
+          this.smartdataDbDoc.createInstanceFromMongoDbNativeDoc(nextDocument);
+        await forEachFuncArg(nextClassInstance as any);
       }
     } while (nextDocument);
     if (closeCursorAtEnd) {

ts/smartdata.classes.db.ts

@@ -1,11 +1,9 @@
 import * as plugins from './smartdata.plugins.js';
-import { ObjectMap } from '@pushrocks/lik';
 
 import { SmartdataCollection } from './smartdata.classes.collection.js';
 import { EasyStore } from './smartdata.classes.easystore.js';
 
 import { logger } from './smartdata.logging.js';
-import { IMongoDescriptor } from './interfaces/index.js';
 
 /**
  * interface - indicates the connection status of the db
@@ -13,13 +11,14 @@ import { IMongoDescriptor } from './interfaces/index.js';
 export type TConnectionStatus = 'initial' | 'disconnected' | 'connected' | 'failed';
 
 export class SmartdataDb {
-  smartdataOptions: IMongoDescriptor;
+  smartdataOptions: plugins.tsclass.database.IMongoDescriptor;
   mongoDbClient: plugins.mongodb.MongoClient;
   mongoDb: plugins.mongodb.Db;
   status: TConnectionStatus;
-  smartdataCollectionMap = new ObjectMap<SmartdataCollection<any>>();
+  statusConnectedDeferred = plugins.smartpromise.defer();
+  smartdataCollectionMap = new plugins.lik.ObjectMap<SmartdataCollection<any>>();
 
-  constructor(smartdataOptions: IMongoDescriptor) {
+  constructor(smartdataOptions: plugins.tsclass.database.IMongoDescriptor) {
     this.smartdataOptions = smartdataOptions;
     this.status = 'initial';
   }
@@ -52,6 +51,7 @@ export class SmartdataDb {
     });
     this.mongoDb = this.mongoDbClient.db(this.smartdataOptions.mongoDbName);
     this.status = 'connected';
+    this.statusConnectedDeferred.resolve();
     console.log(`Connected to database ${this.smartdataOptions.mongoDbName}`);
   }
 

ts/smartdata.classes.distributedcoordinator.ts

@@ -0,0 +1,304 @@
+import * as plugins from './smartdata.plugins.js';
+import { SmartdataDb } from './smartdata.classes.db.js';
+import { managed, setDefaultManagerForDoc } from './smartdata.classes.collection.js';
+import { SmartDataDbDoc, svDb, unI } from './smartdata.classes.doc.js';
+import { SmartdataDbWatcher } from './smartdata.classes.watcher.js';
+
+@managed()
+export class DistributedClass extends SmartDataDbDoc<DistributedClass, DistributedClass> {
+  // INSTANCE
+  @unI()
+  public id: string;
+
+  @svDb()
+  public data: {
+    status: 'initializing' | 'bidding' | 'settled' | 'stopped';
+    biddingShortcode?: string;
+    biddingStartTime?: number;
+    lastUpdated: number;
+    elected: boolean;
+    /**
+     * used to store request
+     */
+    taskRequests: plugins.taskbuffer.distributedCoordination.IDistributedTaskRequest[];
+
+    /**
+     * only used by the leader to convey consultation results
+     */
+    taskRequestResults: plugins.taskbuffer.distributedCoordination.IDistributedTaskRequestResult[];
+  };
+}
+
+/**
+ * This file implements a distributed coordinator according to the @pushrocks/taskbuffer standard.
+ * you should not set up this yourself. Instead, there is a factory on the SmartdataDb class
+ * that will take care of setting this up.
+ */
+export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distributedCoordination
+  .AbstractDistributedCoordinator {
+  public readyPromise: Promise<any>;
+  public db: SmartdataDb;
+  private asyncExecutionStack = new plugins.lik.AsyncExecutionStack();
+  public ownInstance: DistributedClass;
+  public distributedWatcher: SmartdataDbWatcher<DistributedClass>;
+
+  constructor(dbArg: SmartdataDb) {
+    super();
+    this.db = dbArg;
+    setDefaultManagerForDoc(this, DistributedClass);
+    this.readyPromise = this.db.statusConnectedDeferred.promise;
+  }
+
+  // smartdata specific stuff
+  public async start() {
+    await this.init();
+  }
+
+  public async stop() {
+    await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+      if (this.distributedWatcher) {
+        await this.distributedWatcher.close();
+      }
+      if (this.ownInstance?.data.elected) {
+        this.ownInstance.data.elected = false;
+      }
+      if (this.ownInstance?.data.status === 'stopped') {
+        console.log(`stopping a distributed instance that has not been started yet.`);
+      }
+      this.ownInstance.data.status = 'stopped';
+      await this.ownInstance.save();
+      console.log(`stopped ${this.ownInstance.id}`);
+    });
+  }
+
+  public id = plugins.smartunique.uni('distributedInstance');
+
+  private startHeartbeat = async () => {
+    while (this.ownInstance.data.status !== 'stopped') {
+      await this.sendHeartbeat();
+      await plugins.smartdelay.delayForRandom(5000, 10000);
+    }
+  };
+
+  public async sendHeartbeat() {
+    await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+      if (this.ownInstance.data.status === 'stopped') {
+        console.log(`aborted sending heartbeat because status is stopped`);
+        return;
+      }
+      await this.ownInstance.updateFromDb();
+      this.ownInstance.data.lastUpdated = Date.now();
+      await this.ownInstance.save();
+      console.log(`sent heartbeat for ${this.ownInstance.id}`);
+      const allInstances = DistributedClass.getInstances({});
+    });
+    if (this.ownInstance.data.status === 'stopped') {
+      console.log(`aborted sending heartbeat because status is stopped`);
+      return;
+    }
+    const eligibleLeader = await this.getEligibleLeader();
+    // not awaiting here because we don't want to block the heartbeat
+    this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+      if (!eligibleLeader && this.ownInstance.data.status === 'settled') {
+        this.checkAndMaybeLead();
+      }
+    });
+  }
+  private async init() {
+    await this.readyPromise;
+    if (!this.ownInstance) {
+      await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+        this.ownInstance = new DistributedClass();
+        this.ownInstance.id = this.id;
+        this.ownInstance.data = {
+          elected: false,
+          lastUpdated: Date.now(),
+          status: 'initializing',
+          taskRequests: [],
+          taskRequestResults: [],
+        };
+        await this.ownInstance.save();
+      });
+    } else {
+      console.warn(`distributed instance already initialized`);
+    }
+
+    // lets enable the heartbeat
+    this.startHeartbeat();
+
+    // lets do a leader check
+    await this.checkAndMaybeLead();
+
+    return this.ownInstance;
+  }
+
+  public async getEligibleLeader() {
+    return this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+      const allInstances = await DistributedClass.getInstances({});
+      let leaders = allInstances.filter((instanceArg) => instanceArg.data.elected === true);
+      const eligibleLeader = leaders.find(
+        (leader) =>
+          leader.data.lastUpdated >=
+          Date.now() - plugins.smarttime.getMilliSecondsFromUnits({ seconds: 20 })
+      );
+      return eligibleLeader;
+    });
+  }
+
+  // --> leader election
+  public async checkAndMaybeLead() {
+    await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+      this.ownInstance.data.status = 'initializing';
+      this.ownInstance.save();
+    });
+    if (await this.getEligibleLeader()) {
+      await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+        await this.ownInstance.updateFromDb();
+        this.ownInstance.data.status = 'settled';
+        await this.ownInstance.save();
+        console.log(`${this.ownInstance.id} settled as follower`);
+      });
+      return;
+    } else if (
+      (await DistributedClass.getInstances({})).find((instanceArg) => {
+        instanceArg.data.status === 'bidding' &&
+          instanceArg.data.biddingStartTime <= Date.now() - 4000 &&
+          instanceArg.data.biddingStartTime >= Date.now() - 30000;
+      })
+    ) {
+      console.log('too late to the bidding party... waiting for next round.');
+      return;
+    } else {
+      await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+        await this.ownInstance.updateFromDb();
+        this.ownInstance.data.status = 'bidding';
+        this.ownInstance.data.biddingStartTime = Date.now();
+        this.ownInstance.data.biddingShortcode = plugins.smartunique.shortId();
+        await this.ownInstance.save();
+        console.log('bidding code stored.');
+      });
+      console.log(`bidding for leadership...`);
+      await plugins.smartdelay.delayFor(
+        plugins.smarttime.getMilliSecondsFromUnits({ seconds: 5 })
+      );
+      await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+        let biddingInstances = await DistributedClass.getInstances({});
+        biddingInstances = biddingInstances.filter(
+          (instanceArg) =>
+            instanceArg.data.status === 'bidding' &&
+            instanceArg.data.lastUpdated >=
+              Date.now() - plugins.smarttime.getMilliSecondsFromUnits({ seconds: 10 })
+        );
+        console.log(`found ${biddingInstances.length} bidding instances...`);
+        this.ownInstance.data.elected = true;
+        for (const biddingInstance of biddingInstances) {
+          if (biddingInstance.data.biddingShortcode < this.ownInstance.data.biddingShortcode) {
+            this.ownInstance.data.elected = false;
+          }
+        }
+        await plugins.smartdelay.delayFor(5000);
+        console.log(`settling with status elected = ${this.ownInstance.data.elected}`);
+        this.ownInstance.data.status = 'settled';
+        await this.ownInstance.save();
+      });
+      if (this.ownInstance.data.elected) {
+        this.leadFunction();
+      }
+    }
+  }
+
+  /**
+   * when it has been determined
+   * that this instance is leading
+   * the leading is implemented here
+   */
+  public async leadFunction() {
+    this.distributedWatcher = await DistributedClass.watch({});
+
+    const currentTaskRequests: Array<{
+      taskName: string;
+      taskExecutionTime: number;
+      /**
+       * all instances that requested this task
+       */
+      requestingDistibutedInstanceIds: string[];
+      responseTimeout: plugins.smartdelay.Timeout<any>;
+    }> = [];
+
+    this.distributedWatcher.changeSubject.subscribe({
+      next: async (distributedDoc) => {
+        if (!distributedDoc) {
+          console.log(`registered deletion of instance...`);
+          return;
+        }
+        console.log(distributedDoc);
+        console.log(`registered change for ${distributedDoc.id}`);
+        distributedDoc;
+      },
+    });
+
+    while (this.ownInstance.data.status !== 'stopped' && this.ownInstance.data.elected) {
+      const allInstances = await DistributedClass.getInstances({});
+      for (const instance of allInstances) {
+        if (instance.data.status === 'stopped') {
+          await instance.delete();
+        };
+      }
+      await plugins.smartdelay.delayFor(10000);
+    }
+  }
+
+  // abstract implemented methods
+  public async fireDistributedTaskRequest(
+    taskRequestArg: plugins.taskbuffer.distributedCoordination.IDistributedTaskRequest
+  ): Promise<plugins.taskbuffer.distributedCoordination.IDistributedTaskRequestResult> {
+    await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+      if (!this.ownInstance) {
+        console.error('instance need to be started first...');
+        return;
+      }
+      await this.ownInstance.updateFromDb();
+      this.ownInstance.data.taskRequests.push(taskRequestArg);
+      await this.ownInstance.save();
+    });
+    await plugins.smartdelay.delayFor(10000);
+    const result = await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+      await this.ownInstance.updateFromDb();
+      const taskRequestResult = this.ownInstance.data.taskRequestResults.find((resultItem) => {
+        return resultItem.requestResponseId === taskRequestArg.requestResponseId;
+      });
+      return taskRequestResult;
+    });
+    if (!result) {
+      console.warn('no result found for task request...');
+      return null;
+    }
+    return result;
+  }
+
+  public async updateDistributedTaskRequest(
+    infoBasisArg: plugins.taskbuffer.distributedCoordination.IDistributedTaskRequest
+  ): Promise<void> {
+    await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+      const existingInfoBasis = this.ownInstance.data.taskRequests.find((infoBasisItem) => {
+        return (
+          infoBasisItem.taskName === infoBasisArg.taskName &&
+          infoBasisItem.taskExecutionTime === infoBasisArg.taskExecutionTime
+        );
+      });
+      if (!existingInfoBasis) {
+        console.warn('trying to update a non existing task request... aborting!');
+        return;
+      }
+      Object.assign(existingInfoBasis, infoBasisArg);
+      await this.ownInstance.save();
+      plugins.smartdelay.delayFor(60000).then(() => {
+        this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+          const indexToRemove = this.ownInstance.data.taskRequests.indexOf(existingInfoBasis);
+          this.ownInstance.data.taskRequests.splice(indexToRemove, indexToRemove);
+          await this.ownInstance.save();
+        });
+      });
+    });
+  }
+}

ts/smartdata.classes.doc.ts

@@ -1,10 +1,8 @@
 import * as plugins from './smartdata.plugins.js';
 
-import { ObjectMap } from '@pushrocks/lik';
-
 import { SmartdataDb } from './smartdata.classes.db.js';
 import { SmartdataDbCursor } from './smartdata.classes.cursor.js';
-import { IManager, SmartdataCollection } from './smartdata.classes.collection.js';
+import { type IManager, SmartdataCollection } from './smartdata.classes.collection.js';
 import { SmartdataDbWatcher } from './smartdata.classes.watcher.js';
 
 export type TDocCreation = 'db' | 'new' | 'mixed';
@@ -129,6 +127,13 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
     }
   }
 
+  /**
+   * get a unique id prefixed with the class name
+   */
+  public static async getNewId<T = any>(this: plugins.tsclass.typeFest.Class<T>, lengthArg: number = 20) {
+    return `${(this as any).className}:${plugins.smartunique.shortId(lengthArg)}`;
+  }
+
   /**
    * get cursor
    * @returns
@@ -158,7 +163,7 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
     const collection: SmartdataCollection<T> = (this as any).collection;
     const watcher: SmartdataDbWatcher<T> = await collection.watch(
       convertFilterForMongoDb(filterArg),
-      (this as any)
+      this as any
     );
     return watcher;
   }
@@ -183,6 +188,18 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
    */
   public creationStatus: TDocCreation = 'new';
 
+  /**
+   * updated from db in any case where doc comes from db
+   */
+  @svDb()
+  _createdAt: number = Date.now();
+
+  /**
+   * will be updated everytime the doc is saved
+   */
+  @svDb()
+  _updatedAt: number = Date.now();
+
   /**
    * unique indexes
    */
@@ -216,6 +233,9 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
     // tslint:disable-next-line: no-this-assignment
     const self: any = this;
     let dbResult: any;
+
+    this._updatedAt = Date.now();
+
     switch (this.creationStatus) {
       case 'db':
         dbResult = await this.collection.update(self);
@@ -241,9 +261,9 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
    * also store any referenced objects to DB
    * better for data consistency
    */
-  public saveDeep(savedMapArg: ObjectMap<SmartDataDbDoc<any, any>> = null) {
+  public saveDeep(savedMapArg: plugins.lik.ObjectMap<SmartDataDbDoc<any, any>> = null) {
     if (!savedMapArg) {
-      savedMapArg = new ObjectMap<SmartDataDbDoc<any, any>>();
+      savedMapArg = new plugins.lik.ObjectMap<SmartDataDbDoc<any, any>>();
     }
     savedMapArg.add(this);
     this.save();
@@ -255,6 +275,16 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
     }
   }
 
+  /**
+   * updates an object from db
+   */
+  public async updateFromDb() {
+    const mongoDbNativeDoc = await this.collection.findOne(await this.createIdentifiableObject());
+    for (const key of Object.keys(mongoDbNativeDoc)) {
+      this[key] = mongoDbNativeDoc[key];
+    }
+  }
+
   /**
    * creates a saveable object so the instance can be persisted as json in the database
    */

ts/smartdata.classes.easystore.ts

@@ -17,6 +17,15 @@ export class EasyStore<T> {
       @unI()
       public nameId: string;
 
+      @svDb()
+      public ephermal: {
+        activated: boolean;
+        timeout: number;
+      };
+
+      @svDb()
+      lastEdit: number;
+
       @svDb()
       public data: Partial<T>;
     }
@@ -28,7 +37,16 @@ export class EasyStore<T> {
     this.nameId = nameIdArg;
   }
 
-  private async getEasyStore() {
+  private easyStorePromise: Promise<InstanceType<typeof this.easyStoreClass>>;
+  private async getEasyStore(): Promise<InstanceType<typeof this.easyStoreClass>> {
+    if (this.easyStorePromise) {
+      return this.easyStorePromise;
+    };
+
+    // first run from here
+    const deferred = plugins.smartpromise.defer<InstanceType<typeof this.easyStoreClass>>();
+    this.easyStorePromise = deferred.promise;
+
     let easyStore = await this.easyStoreClass.getInstance({
       nameId: this.nameId,
     });
@@ -39,7 +57,8 @@ export class EasyStore<T> {
       easyStore.data = {};
       await easyStore.save();
     }
-    return easyStore;
+    deferred.resolve(easyStore);
+    return this.easyStorePromise;
   }
 
   /**
@@ -61,7 +80,7 @@ export class EasyStore<T> {
   /**
    * writes a specific key to the keyValueStore
    */
-  public async writeKey(keyArg: keyof T, valueArg: any) {
+  public async writeKey<TKey extends keyof T>(keyArg: TKey, valueArg: T[TKey]) {
     const easyStore = await this.getEasyStore();
     easyStore.data[keyArg] = valueArg;
     await easyStore.save();
@@ -90,4 +109,11 @@ export class EasyStore<T> {
     easyStore.data = {};
     await easyStore.save();
   }
+
+  public async cleanUpEphermal() {
+    while (
+      (await this.smartdataDbRef.statusConnectedDeferred.promise) &&
+      this.smartdataDbRef.status === 'connected'
+    ) {}
+  }
 }

ts/smartdata.classes.watcher.ts

@@ -11,12 +11,21 @@ export class SmartdataDbWatcher<T = any> {
   // INSTANCE
   private changeStream: plugins.mongodb.ChangeStream<T>;
 
-  public changeSubject = new plugins.smartrx.rxjs.Subject<SmartDataDbDoc<T, T>>();
-  constructor(changeStreamArg: plugins.mongodb.ChangeStream<T>, smartdataDbDocArg: typeof SmartDataDbDoc) {
+  public changeSubject = new plugins.smartrx.rxjs.Subject<T>();
+  constructor(
+    changeStreamArg: plugins.mongodb.ChangeStream<T>,
+    smartdataDbDocArg: typeof SmartDataDbDoc
+  ) {
     this.changeStream = changeStreamArg;
-    this.changeStream.on('change', async (item: T) => {
-      this.changeSubject.next(smartdataDbDocArg.createInstanceFromMongoDbNativeDoc(item));
-    })
+    this.changeStream.on('change', async (item: any) => {
+      if (!item.fullDocument) {
+        this.changeSubject.next(null);
+        return;
+      }
+      this.changeSubject.next(
+        smartdataDbDocArg.createInstanceFromMongoDbNativeDoc(item.fullDocument) as any as T
+      );
+    });
     plugins.smartdelay.delayFor(0).then(() => {
       this.readyDeferred.resolve();
     });

ts/smartdata.plugins.ts

@@ -4,14 +4,26 @@ import * as tsclass from '@tsclass/tsclass';
 export { tsclass };
 
 // @pushrocks scope
-import * as smartlog from '@pushrocks/smartlog';
-import * as lodash from 'lodash';
+import * as lik from '@push.rocks/lik';
+import * as smartdelay from '@push.rocks/smartdelay';
+import * as smartlog from '@push.rocks/smartlog';
+import * as smartpromise from '@push.rocks/smartpromise';
+import * as smartrx from '@push.rocks/smartrx';
+import * as smartstring from '@push.rocks/smartstring';
+import * as smarttime from '@push.rocks/smarttime';
+import * as smartunique from '@push.rocks/smartunique';
+import * as taskbuffer from '@push.rocks/taskbuffer';
 import * as mongodb from 'mongodb';
-import * as smartdelay from '@pushrocks/smartdelay'
-import * as smartpromise from '@pushrocks/smartpromise';
-import * as smartq from '@pushrocks/smartpromise';
-import * as smartrx from '@pushrocks/smartrx';
-import * as smartstring from '@pushrocks/smartstring';
-import * as smartunique from '@pushrocks/smartunique';
 
-export { smartdelay, smartpromise, smartlog, lodash, smartq, smartrx, mongodb, smartstring, smartunique };
+export {
+  lik,
+  smartdelay,
+  smartpromise,
+  smartlog,
+  smartrx,
+  mongodb,
+  smartstring,
+  smarttime,
+  smartunique,
+  taskbuffer,
+};

tsconfig.json

@@ -3,7 +3,12 @@
     "experimentalDecorators": true,
     "useDefineForClassFields": false,
     "target": "ES2022",
-    "module": "ES2022",
-    "moduleResolution": "nodenext"
-  }
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "esModuleInterop": true,
+    "verbatimModuleSyntax": true
+  },
+  "exclude": [
+    "dist_*/**/*.d.ts"
+  ]
 }