5.0.35

.gitea/workflows/default_nottags.yaml

@@ -0,0 +1,66 @@
+name: Default (not tags)
+
+on:
+  push:
+    tags-ignore:
+      - '**'
+
+env:
+  IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
+  NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
+  NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
+  NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
+  NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
+  NPMCI_URL_CLOUDLY: ${{secrets.NPMCI_URL_CLOUDLY}}
+
+jobs:
+  security:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install pnpm and npmci
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @shipzone/npmci
+
+      - name: Run npm prepare
+        run: npmci npm prepare
+
+      - name: Audit production dependencies
+        run: |
+          npmci command npm config set registry https://registry.npmjs.org
+          npmci command pnpm audit --audit-level=high --prod
+        continue-on-error: true
+
+      - name: Audit development dependencies
+        run: |
+          npmci command npm config set registry https://registry.npmjs.org
+          npmci command pnpm audit --audit-level=high --dev
+        continue-on-error: true
+
+  test:
+    if: ${{ always() }}
+    needs: security
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Test stable
+        run: |
+          npmci node install stable
+          npmci npm install
+          npmci npm test
+
+      - name: Test build
+        run: |
+          npmci node install stable
+          npmci npm install
+          npmci npm build

.gitea/workflows/default_tags.yaml

@@ -0,0 +1,124 @@
+name: Default (tags)
+
+on:
+  push:
+    tags:
+      - '*'
+
+env:
+  IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
+  NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
+  NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
+  NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
+  NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
+  NPMCI_URL_CLOUDLY: ${{secrets.NPMCI_URL_CLOUDLY}}
+
+jobs:
+  security:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Prepare
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @shipzone/npmci
+          npmci npm prepare
+
+      - name: Audit production dependencies
+        run: |
+          npmci command npm config set registry https://registry.npmjs.org
+          npmci command pnpm audit --audit-level=high --prod
+        continue-on-error: true
+
+      - name: Audit development dependencies
+        run: |
+          npmci command npm config set registry https://registry.npmjs.org
+          npmci command pnpm audit --audit-level=high --dev
+        continue-on-error: true
+
+  test:
+    if: ${{ always() }}
+    needs: security
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Prepare
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @shipzone/npmci
+          npmci npm prepare
+
+      - name: Test stable
+        run: |
+          npmci node install stable
+          npmci npm install
+          npmci npm test
+
+      - name: Test build
+        run: |
+          npmci node install stable
+          npmci npm install
+          npmci npm build
+
+  release:
+    needs: test
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Prepare
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @shipzone/npmci
+          npmci npm prepare
+
+      - name: Release
+        run: |
+          npmci node install stable
+          npmci npm publish
+
+  metadata:
+    needs: test
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+    continue-on-error: true
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Prepare
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @shipzone/npmci
+          npmci npm prepare
+
+      - name: Code quality
+        run: |
+          npmci command npm install -g typescript
+          npmci npm install
+
+      - name: Trigger
+        run: npmci trigger
+
+      - name: Build docs and upload artifacts
+        run: |
+          npmci node install stable
+          npmci npm install
+          pnpm install -g @gitzone/tsdoc
+          npmci command tsdoc
+        continue-on-error: true

.gitea/workflows/gitzone_ci_default.yaml

@@ -1,107 +0,0 @@
-name: CI Pipeline
-
-on: push
-
-env:
-  IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
-
-jobs:
-
-  security:
-    runs-on: ubuntu-latest
-    continue-on-error: true
-    container:
-      image: ${{ env.IMAGE }}
-
-    steps:
-    - uses: actions/checkout@v3
-
-    - name: Install pnpm and npmci
-      run: |
-        pnpm install -g pnpm
-        pnpm install -g @shipzone/npmci
-
-    - name: Run npm prepare
-      run: npmci npm prepare
-
-    - name: Audit production dependencies
-      run: |
-        npmci command npm config set registry https://registry.npmjs.org
-        npmci command pnpm audit --audit-level=high --prod
-      continue-on-error: true
-
-    - name: Audit development dependencies
-      run: |
-        npmci command npm config set registry https://registry.npmjs.org
-        npmci command pnpm audit --audit-level=high --dev
-      continue-on-error: true
-
-  test:
-    if: ${{ always() }}
-    needs: security
-    runs-on: ubuntu-latest
-    container:
-      image: ${{ env.IMAGE }}
-
-    steps:
-    - uses: actions/checkout@v3
-
-    - name: Test stable
-      run: |
-        npmci node install stable
-        npmci npm install
-        npmci npm test
-
-    - name: Test build
-      run: |
-        npmci node install stable
-        npmci npm install
-        npmci npm build
-
-  release:
-    needs: test
-    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
-    runs-on: ubuntu-latest
-    container:
-      image: ${{ env.IMAGE }}
-
-    steps:
-    - uses: actions/checkout@v3
-
-    - name: Release
-      run: |
-        npmci node install stable
-        npmci npm publish
-
-  metadata:
-    needs: test
-    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
-    runs-on: ubuntu-latest
-    container:
-      image: ${{ env.IMAGE }}
-    continue-on-error: true
-
-    steps:
-    - uses: actions/checkout@v3
-
-    - name: Code quality
-      run: |
-        npmci command npm install -g typescript
-        npmci npm prepare
-        npmci npm install
-
-    - name: Trigger
-      run: npmci trigger
-
-    - name: Build docs and upload artifacts
-      run: |
-        npmci node install stable
-        npmci npm install
-        npmci command npm run buildDocs
-
-    - name: Upload artifacts
-      uses: actions/upload-artifact@v2
-      with:
-        name: public
-        path: public
-        retention-days: 7

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@push.rocks/smartdata",
-  "version": "5.0.21",
+  "version": "5.0.35",
   "private": false,
   "description": "do more with data",
   "main": "dist_ts/index.js",
@@ -22,27 +22,26 @@
   },
   "homepage": "https://gitlab.com/pushrocks/smartdata#README",
   "dependencies": {
-    "@push.rocks/lik": "^6.0.2",
+    "@push.rocks/lik": "^6.0.14",
     "@push.rocks/smartdelay": "^3.0.1",
     "@push.rocks/smartlog": "^3.0.2",
-    "@push.rocks/smartmongo": "^2.0.7",
+    "@push.rocks/smartmongo": "^2.0.10",
     "@push.rocks/smartpromise": "^4.0.2",
-    "@push.rocks/smartrx": "^3.0.2",
+    "@push.rocks/smartrx": "^3.0.7",
-    "@push.rocks/smartstring": "^4.0.7",
+    "@push.rocks/smartstring": "^4.0.15",
-    "@push.rocks/smarttime": "^4.0.1",
+    "@push.rocks/smarttime": "^4.0.6",
-    "@push.rocks/smartunique": "^3.0.3",
+    "@push.rocks/smartunique": "^3.0.8",
-    "@push.rocks/taskbuffer": "^3.0.10",
+    "@push.rocks/taskbuffer": "^3.1.7",
-    "@tsclass/tsclass": "^4.0.42",
+    "@tsclass/tsclass": "^4.0.52",
-    "mongodb": "^5.7.0"
+    "mongodb": "^6.5.0"
   },
   "devDependencies": {
     "@gitzone/tsbuild": "^2.1.66",
     "@gitzone/tsrun": "^1.2.44",
     "@gitzone/tstest": "^1.0.77",
-    "@push.rocks/qenv": "^5.0.2",
+    "@push.rocks/qenv": "^6.0.5",
-    "@push.rocks/tapbundle": "^5.0.8",
+    "@push.rocks/tapbundle": "^5.0.22",
-    "@types/node": "^20.4.2",
+    "@types/node": "^20.11.30"
-    "@types/shortid": "0.0.29"
   },
   "files": [
     "ts/**/*",

readme.md

@@ -1,26 +1,26 @@
-# @pushrocks/smartdata
+# @push.rocks/smartdata
 do more with data
 
 ## Availabililty and Links
-* [npmjs.org (npm package)](https://www.npmjs.com/package/@pushrocks/smartdata)
+* [npmjs.org (npm package)](https://www.npmjs.com/package/@push.rocks/smartdata)
-* [gitlab.com (source)](https://gitlab.com/pushrocks/smartdata)
+* [gitlab.com (source)](https://gitlab.com/push.rocks/smartdata)
-* [github.com (source mirror)](https://github.com/pushrocks/smartdata)
+* [github.com (source mirror)](https://github.com/push.rocks/smartdata)
-* [docs (typedoc)](https://pushrocks.gitlab.io/smartdata/)
+* [docs (typedoc)](https://push.rocks.gitlab.io/smartdata/)
 
 ## Status for master
 
 Status Category | Status Badge
 -- | --
-GitLab Pipelines | [![pipeline status](https://gitlab.com/pushrocks/smartdata/badges/master/pipeline.svg)](https://lossless.cloud)
+GitLab Pipelines | [![pipeline status](https://gitlab.com/push.rocks/smartdata/badges/master/pipeline.svg)](https://lossless.cloud)
-GitLab Pipline Test Coverage | [![coverage report](https://gitlab.com/pushrocks/smartdata/badges/master/coverage.svg)](https://lossless.cloud)
+GitLab Pipline Test Coverage | [![coverage report](https://gitlab.com/push.rocks/smartdata/badges/master/coverage.svg)](https://lossless.cloud)
-npm | [![npm downloads per month](https://badgen.net/npm/dy/@pushrocks/smartdata)](https://lossless.cloud)
+npm | [![npm downloads per month](https://badgen.net/npm/dy/@push.rocks/smartdata)](https://lossless.cloud)
-Snyk | [![Known Vulnerabilities](https://badgen.net/snyk/pushrocks/smartdata)](https://lossless.cloud)
+Snyk | [![Known Vulnerabilities](https://badgen.net/snyk/push.rocks/smartdata)](https://lossless.cloud)
 TypeScript Support | [![TypeScript](https://badgen.net/badge/TypeScript/>=%203.x/blue?icon=typescript)](https://lossless.cloud)
 node Support | [![node](https://img.shields.io/badge/node->=%2010.x.x-blue.svg)](https://nodejs.org/dist/latest-v10.x/docs/api/)
 Code Style | [![Code Style](https://badgen.net/badge/style/prettier/purple)](https://lossless.cloud)
-PackagePhobia (total standalone install weight) | [![PackagePhobia](https://badgen.net/packagephobia/install/@pushrocks/smartdata)](https://lossless.cloud)
+PackagePhobia (total standalone install weight) | [![PackagePhobia](https://badgen.net/packagephobia/install/@push.rocks/smartdata)](https://lossless.cloud)
-PackagePhobia (package size on registry) | [![PackagePhobia](https://badgen.net/packagephobia/publish/@pushrocks/smartdata)](https://lossless.cloud)
+PackagePhobia (package size on registry) | [![PackagePhobia](https://badgen.net/packagephobia/publish/@push.rocks/smartdata)](https://lossless.cloud)
-BundlePhobia (total size when bundled) | [![BundlePhobia](https://badgen.net/bundlephobia/minzip/@pushrocks/smartdata)](https://lossless.cloud)
+BundlePhobia (total size when bundled) | [![BundlePhobia](https://badgen.net/bundlephobia/minzip/@push.rocks/smartdata)](https://lossless.cloud)
 
 ## Usage
 

test/test.distributedcoordinator.ts

@@ -0,0 +1,112 @@
+import { tap, expect } from '@push.rocks/tapbundle';
+import * as smartmongo from '@push.rocks/smartmongo';
+import type * as taskbuffer from '@push.rocks/taskbuffer';
+
+import * as smartdata from '../ts/index.js';
+import { SmartdataDistributedCoordinator, DistributedClass } from '../ts/smartdata.classes.distributedcoordinator.js'; // path might need adjusting
+const totalInstances = 10;
+
+// =======================================
+// Connecting to the database server
+// =======================================
+
+let smartmongoInstance: smartmongo.SmartMongo;
+let testDb: smartdata.SmartdataDb;
+
+tap.test('should create a testinstance as database', async () => {
+  smartmongoInstance = await smartmongo.SmartMongo.createAndStart();
+  testDb = new smartdata.SmartdataDb(await smartmongoInstance.getMongoDescriptor());
+  await testDb.init();
+});
+
+tap.test('should instantiate DistributedClass', async (tools) => {
+    const instance = new DistributedClass();
+    expect(instance).toBeInstanceOf(DistributedClass);
+});
+
+tap.test('DistributedClass should update the time', async (tools) => {
+    const distributedCoordinator = new SmartdataDistributedCoordinator(testDb);
+    await distributedCoordinator.start();
+    const initialTime = distributedCoordinator.ownInstance.data.lastUpdated;
+    await distributedCoordinator.sendHeartbeat();
+    const updatedTime = distributedCoordinator.ownInstance.data.lastUpdated;
+    expect(updatedTime).toBeGreaterThan(initialTime);
+    await distributedCoordinator.stop();
+});
+
+tap.test('should instantiate SmartdataDistributedCoordinator', async (tools) => {
+    const distributedCoordinator = new SmartdataDistributedCoordinator(testDb);
+    await distributedCoordinator.start();
+    expect(distributedCoordinator).toBeInstanceOf(SmartdataDistributedCoordinator);
+    await distributedCoordinator.stop();
+});
+
+tap.test('SmartdataDistributedCoordinator should update leader status', async (tools) => {
+    const distributedCoordinator = new SmartdataDistributedCoordinator(testDb);
+    await distributedCoordinator.start();
+    await distributedCoordinator.checkAndMaybeLead();
+    expect(distributedCoordinator.ownInstance.data.elected).toBeOneOf([true, false]);
+    await distributedCoordinator.stop();
+});
+
+tap.test('SmartdataDistributedCoordinator should handle distributed task requests', async (tools) => {
+    const distributedCoordinator = new SmartdataDistributedCoordinator(testDb);
+    await distributedCoordinator.start();
+
+    const mockTaskRequest: taskbuffer.distributedCoordination.IDistributedTaskRequest = {
+        submitterId: "mockSubmitter12345",    // Some unique mock submitter ID
+        requestResponseId: 'uni879873462hjhfkjhsdf', // Some unique ID for the request-response
+        taskName: "SampleTask",
+        taskVersion: "1.0.0",                  // Assuming it's a version string
+        taskExecutionTime: Date.now(),
+        taskExecutionTimeout: 60000,           // Let's say the timeout is 1 minute (60000 ms)
+        taskExecutionParallel: 5,              // Let's assume max 5 parallel executions
+        status: 'requesting'
+    };
+
+    const response = await distributedCoordinator.fireDistributedTaskRequest(mockTaskRequest);
+    console.log(response)  // based on your expected structure for the response
+    await distributedCoordinator.stop();
+});
+
+tap.test('SmartdataDistributedCoordinator should update distributed task requests', async (tools) => {
+    const distributedCoordinator = new SmartdataDistributedCoordinator(testDb);
+    
+    await distributedCoordinator.start();
+
+    const mockTaskRequest: taskbuffer.distributedCoordination.IDistributedTaskRequest = {
+        submitterId: "mockSubmitter12345",    // Some unique mock submitter ID
+        requestResponseId: 'uni879873462hjhfkjhsdf', // Some unique ID for the request-response
+        taskName: "SampleTask",
+        taskVersion: "1.0.0",                  // Assuming it's a version string
+        taskExecutionTime: Date.now(),
+        taskExecutionTimeout: 60000,           // Let's say the timeout is 1 minute (60000 ms)
+        taskExecutionParallel: 5,              // Let's assume max 5 parallel executions
+        status: 'requesting'
+    };
+    
+
+    await distributedCoordinator.updateDistributedTaskRequest(mockTaskRequest);
+    // Here, we can potentially check if a DB entry got updated or some other side-effect of the update method.
+    await distributedCoordinator.stop();
+});
+
+tap.test('should elect only one leader amongst multiple instances', async (tools) => {
+    const coordinators = Array.from({ length: totalInstances }).map(() => new SmartdataDistributedCoordinator(testDb));
+    await Promise.all(coordinators.map(coordinator => coordinator.start()));
+    const leaders = coordinators.filter(coordinator => coordinator.ownInstance.data.elected);
+    for (const leader of leaders) {
+        console.log(leader.ownInstance);
+    }
+    expect(leaders.length).toEqual(1);
+
+    // stopping clears a coordinator from being elected.
+    await Promise.all(coordinators.map(coordinator => coordinator.stop()));
+});
+
+tap.test('should clean up', async () => {
+    await smartmongoInstance.stopAndDumpToDir(`.nogit/testdata/`);
+    setTimeout(() => process.exit(), 2000);
+})
+
+tap.start({ throwOnError: true });

test/test.easystore.ts

@@ -26,7 +26,7 @@ tap.test('should create a testinstance as database', async () => {
 tap.skip.test('should connect to atlas', async (tools) => {
   const databaseName = `test-smartdata-${smartunique.shortId()}`;
   testDb = new smartdata.SmartdataDb({
-    mongoDbUrl: testQenv.getEnvVarOnDemand('MONGO_URL'),
+    mongoDbUrl: await testQenv.getEnvVarOnDemand('MONGO_URL'),
     mongoDbName: databaseName,
   });
   await testDb.init();

test/test.ts

@@ -30,7 +30,7 @@ tap.test('should create a testinstance as database', async () => {
 tap.skip.test('should connect to atlas', async (tools) => {
   const databaseName = `test-smartdata-${smartunique.shortId()}`;
   testDb = new smartdata.SmartdataDb({
-    mongoDbUrl: testQenv.getEnvVarOnDemand('MONGO_URL'),
+    mongoDbUrl: await testQenv.getEnvVarOnDemand('MONGO_URL'),
     mongoDbName: databaseName,
   });
   await testDb.init();

test/test.typescript.ts

@@ -28,7 +28,7 @@ tap.test('should create a testinstance as database', async () => {
 tap.skip.test('should connect to atlas', async (tools) => {
   const databaseName = `test-smartdata-${smartunique.shortId()}`;
   testDb = new smartdata.SmartdataDb({
-    mongoDbUrl: testQenv.getEnvVarOnDemand('MONGO_URL'),
+    mongoDbUrl: await testQenv.getEnvVarOnDemand('MONGO_URL'),
     mongoDbName: databaseName,
   });
   await testDb.init();

test/test.watch.ts

@@ -28,7 +28,7 @@ tap.test('should create a testinstance as database', async () => {
 tap.skip.test('should connect to atlas', async (tools) => {
   const databaseName = `test-smartdata-${smartunique.shortId()}`;
   testDb = new smartdata.SmartdataDb({
-    mongoDbUrl: testQenv.getEnvVarOnDemand('MONGO_URL'),
+    mongoDbUrl: await testQenv.getEnvVarOnDemand('MONGO_URL'),
     mongoDbName: databaseName,
   });
   await testDb.init();

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@push.rocks/smartdata',
-  version: '5.0.21',
+  version: '5.0.35',
   description: 'do more with data'
 }

ts/smartdata.classes.collection.ts

@@ -56,7 +56,7 @@ export const setDefaultManagerForDoc = <T>(managerArg: IManager, dbDocArg: T): T
  * This is a decorator that will tell the decorated class what dbTable to use
  * @param dbArg
  */
-export function Manager<TManager extends IManager>(managerArg?: TManager | TDelayed<TManager>) {
+export function managed<TManager extends IManager>(managerArg?: TManager | TDelayed<TManager>) {
   return function classDecorator<T extends { new (...args: any[]): any }>(constructor: T) {
     return class extends constructor {
       public static get collection() {
@@ -109,6 +109,11 @@ export function Manager<TManager extends IManager>(managerArg?: TManager | TDela
   };
 }
 
+/**
+ * @dpecrecated use @managed instead
+ */
+export const Manager = managed;
+
 export class SmartdataCollection<T> {
   /**
    * the collection that is used
@@ -269,7 +274,7 @@ export class SmartdataCollection<T> {
    * if this.objectValidation is not set it passes.
    */
   private checkDoc(docArg: T): Promise<void> {
-    const done = plugins.smartq.defer<void>();
+    const done = plugins.smartpromise.defer<void>();
     let validationResult = true;
     if (this.objectValidation) {
       validationResult = this.objectValidation(docArg);

ts/smartdata.classes.distributedcoordinator.ts

@@ -1,18 +1,20 @@
 import * as plugins from './smartdata.plugins.js';
 import { SmartdataDb } from './smartdata.classes.db.js';
-import { Manager, setDefaultManagerForDoc } from './smartdata.classes.collection.js';
+import { managed, setDefaultManagerForDoc } from './smartdata.classes.collection.js';
 import { SmartDataDbDoc, svDb, unI } from './smartdata.classes.doc.js';
+import { SmartdataDbWatcher } from './smartdata.classes.watcher.js';
 
-@Manager()
+@managed()
-class DistributedClass extends SmartDataDbDoc<DistributedClass, DistributedClass> {
+export class DistributedClass extends SmartDataDbDoc<DistributedClass, DistributedClass> {
   // INSTANCE
   @unI()
   public id: string;
 
   @svDb()
   public data: {
-    status: 'bidding' | 'settled' | 'initializing' | 'stopped';
+    status: 'initializing' | 'bidding' | 'settled' | 'stopped';
     biddingShortcode?: string;
+    biddingStartTime?: number;
     lastUpdated: number;
     elected: boolean;
     /**
@@ -37,14 +39,14 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
   public readyPromise: Promise<any>;
   public db: SmartdataDb;
   private asyncExecutionStack = new plugins.lik.AsyncExecutionStack();
-  private ownInstance: DistributedClass;
+  public ownInstance: DistributedClass;
+  public distributedWatcher: SmartdataDbWatcher<DistributedClass>;
 
-  constructor(dbArg?: SmartdataDb) {
+  constructor(dbArg: SmartdataDb) {
     super();
     this.db = dbArg;
     setDefaultManagerForDoc(this, DistributedClass);
     this.readyPromise = this.db.statusConnectedDeferred.promise;
-    this.init();
   }
 
   // smartdata specific stuff
@@ -53,26 +55,56 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
   }
 
   public async stop() {
-    if (this.ownInstance?.data.elected) {
+    await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
-      this.ownInstance.data.elected = false;
+      if (this.distributedWatcher) {
-    } else {
+        await this.distributedWatcher.close();
-      console.log(`can't stop a distributed instance that has not been started yet.`);
+      }
-    }
+      if (this.ownInstance?.data.elected) {
+        this.ownInstance.data.elected = false;
+      }
+      if (this.ownInstance?.data.status === 'stopped') {
+        console.log(`stopping a distributed instance that has not been started yet.`);
+      }
+      this.ownInstance.data.status = 'stopped';
+      await this.ownInstance.save();
+      console.log(`stopped ${this.ownInstance.id}`);
+    });
   }
 
   public id = plugins.smartunique.uni('distributedInstance');
 
   private startHeartbeat = async () => {
     while (this.ownInstance.data.status !== 'stopped') {
-      await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+      await this.sendHeartbeat();
-        await this.ownInstance.updateFromDb();
+      await plugins.smartdelay.delayForRandom(5000, 10000);
-        this.ownInstance.data.lastUpdated = Date.now();
-        await this.ownInstance.save();
-      });
-      await plugins.smartdelay.delayFor(10000);
     }
   };
-  public async init() {
+
+  public async sendHeartbeat() {
+    await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+      if (this.ownInstance.data.status === 'stopped') {
+        console.log(`aborted sending heartbeat because status is stopped`);
+        return;
+      }
+      await this.ownInstance.updateFromDb();
+      this.ownInstance.data.lastUpdated = Date.now();
+      await this.ownInstance.save();
+      console.log(`sent heartbeat for ${this.ownInstance.id}`);
+      const allInstances = DistributedClass.getInstances({});
+    });
+    if (this.ownInstance.data.status === 'stopped') {
+      console.log(`aborted sending heartbeat because status is stopped`);
+      return;
+    }
+    const eligibleLeader = await this.getEligibleLeader();
+    // not awaiting here because we don't want to block the heartbeat
+    this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+      if (!eligibleLeader && this.ownInstance.data.status === 'settled') {
+        this.checkAndMaybeLead();
+      }
+    });
+  }
+  private async init() {
     await this.readyPromise;
     if (!this.ownInstance) {
       await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
@@ -87,6 +119,8 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
         };
         await this.ownInstance.save();
       });
+    } else {
+      console.warn(`distributed instance already initialized`);
     }
 
     // lets enable the heartbeat
@@ -98,42 +132,73 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
     return this.ownInstance;
   }
 
+  public async getEligibleLeader() {
+    return this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+      const allInstances = await DistributedClass.getInstances({});
+      let leaders = allInstances.filter((instanceArg) => instanceArg.data.elected === true);
+      const eligibleLeader = leaders.find(
+        (leader) =>
+          leader.data.lastUpdated >=
+          Date.now() - plugins.smarttime.getMilliSecondsFromUnits({ seconds: 20 })
+      );
+      return eligibleLeader;
+    });
+  }
+
   // --> leader election
   public async checkAndMaybeLead() {
-    const allInstances = await DistributedClass.getInstances({});
+    await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
-    let leader = allInstances.find((instanceArg) => instanceArg.data.elected === true);
+      this.ownInstance.data.status = 'initializing';
-    if (
+      this.ownInstance.save();
-      leader &&
+    });
-      leader.data.lastUpdated >=
+    if (await this.getEligibleLeader()) {
-        Date.now() - plugins.smarttime.getMilliSecondsFromUnits({ minutes: 1 })
-    ) {
       await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
         await this.ownInstance.updateFromDb();
         this.ownInstance.data.status = 'settled';
         await this.ownInstance.save();
+        console.log(`${this.ownInstance.id} settled as follower`);
       });
       return;
+    } else if (
+      (await DistributedClass.getInstances({})).find((instanceArg) => {
+        instanceArg.data.status === 'bidding' &&
+          instanceArg.data.biddingStartTime <= Date.now() - 4000 &&
+          instanceArg.data.biddingStartTime >= Date.now() - 30000;
+      })
+    ) {
+      console.log('too late to the bidding party... waiting for next round.');
+      return;
     } else {
       await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+        await this.ownInstance.updateFromDb();
         this.ownInstance.data.status = 'bidding';
+        this.ownInstance.data.biddingStartTime = Date.now();
         this.ownInstance.data.biddingShortcode = plugins.smartunique.shortId();
         await this.ownInstance.save();
+        console.log('bidding code stored.');
       });
-      await plugins.smartdelay.delayFor(plugins.smarttime.getMilliSecondsFromUnits({ minutes: 2 }));
+      console.log(`bidding for leadership...`);
+      await plugins.smartdelay.delayFor(
+        plugins.smarttime.getMilliSecondsFromUnits({ seconds: 5 })
+      );
       await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
         let biddingInstances = await DistributedClass.getInstances({});
         biddingInstances = biddingInstances.filter(
           (instanceArg) =>
-            !instanceArg.data.elected &&
+            instanceArg.data.status === 'bidding' &&
             instanceArg.data.lastUpdated >=
-              Date.now() - plugins.smarttime.getMilliSecondsFromUnits({ minutes: 1 })
+              Date.now() - plugins.smarttime.getMilliSecondsFromUnits({ seconds: 10 })
         );
+        console.log(`found ${biddingInstances.length} bidding instances...`);
         this.ownInstance.data.elected = true;
         for (const biddingInstance of biddingInstances) {
           if (biddingInstance.data.biddingShortcode < this.ownInstance.data.biddingShortcode) {
             this.ownInstance.data.elected = false;
           }
         }
+        await plugins.smartdelay.delayFor(5000);
+        console.log(`settling with status elected = ${this.ownInstance.data.elected}`);
+        this.ownInstance.data.status = 'settled';
         await this.ownInstance.save();
       });
       if (this.ownInstance.data.elected) {
@@ -148,20 +213,38 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
    * the leading is implemented here
    */
   public async leadFunction() {
-    const ownInstance = await this.init();
+    this.distributedWatcher = await DistributedClass.watch({});
-    const watcher = await DistributedClass.watch({});
-    /**
-     * this function is started once per unique job request
-     */
-    const startResultTimer = async () => {};
 
-    watcher.changeSubject.subscribe({
+    const currentTaskRequests: Array<{
+      taskName: string;
+      taskExecutionTime: number;
+      /**
+       * all instances that requested this task
+       */
+      requestingDistibutedInstanceIds: string[];
+      responseTimeout: plugins.smartdelay.Timeout<any>;
+    }> = [];
+
+    this.distributedWatcher.changeSubject.subscribe({
       next: async (distributedDoc) => {
+        if (!distributedDoc) {
+          console.log(`registered deletion of instance...`);
+          return;
+        }
+        console.log(distributedDoc);
+        console.log(`registered change for ${distributedDoc.id}`);
         distributedDoc;
       },
     });
-    while (this.ownInstance.data.status !== 'stopped') {
+
-      await plugins.smartdelay.delayFor(1000);
+    while (this.ownInstance.data.status !== 'stopped' && this.ownInstance.data.elected) {
+      const allInstances = await DistributedClass.getInstances({});
+      for (const instance of allInstances) {
+        if (instance.data.status === 'stopped') {
+          await instance.delete();
+        };
+      }
+      await plugins.smartdelay.delayFor(10000);
     }
   }
 
@@ -169,12 +252,28 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
   public async fireDistributedTaskRequest(
     taskRequestArg: plugins.taskbuffer.distributedCoordination.IDistributedTaskRequest
   ): Promise<plugins.taskbuffer.distributedCoordination.IDistributedTaskRequestResult> {
-    const ownInstance = await this.init();
     await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+      if (!this.ownInstance) {
+        console.error('instance need to be started first...');
+        return;
+      }
+      await this.ownInstance.updateFromDb();
       this.ownInstance.data.taskRequests.push(taskRequestArg);
       await this.ownInstance.save();
     });
-    return null;
+    await plugins.smartdelay.delayFor(10000);
+    const result = await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+      await this.ownInstance.updateFromDb();
+      const taskRequestResult = this.ownInstance.data.taskRequestResults.find((resultItem) => {
+        return resultItem.requestResponseId === taskRequestArg.requestResponseId;
+      });
+      return taskRequestResult;
+    });
+    if (!result) {
+      console.warn('no result found for task request...');
+      return null;
+    }
+    return result;
   }
 
   public async updateDistributedTaskRequest(
@@ -187,6 +286,10 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
           infoBasisItem.taskExecutionTime === infoBasisArg.taskExecutionTime
         );
       });
+      if (!existingInfoBasis) {
+        console.warn('trying to update a non existing task request... aborting!');
+        return;
+      }
       Object.assign(existingInfoBasis, infoBasisArg);
       await this.ownInstance.save();
       plugins.smartdelay.delayFor(60000).then(() => {

ts/smartdata.classes.doc.ts

@@ -257,7 +257,7 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
    * updates an object from db
    */
   public async updateFromDb() {
-    const mongoDbNativeDoc = await this.collection.findOne(this.createIdentifiableObject());
+    const mongoDbNativeDoc = await this.collection.findOne(await this.createIdentifiableObject());
     for (const key of Object.keys(mongoDbNativeDoc)) {
       this[key] = mongoDbNativeDoc[key];
     }

ts/smartdata.classes.easystore.ts

@@ -70,7 +70,7 @@ export class EasyStore<T> {
   /**
    * writes a specific key to the keyValueStore
    */
-  public async writeKey(keyArg: keyof T, valueArg: any) {
+  public async writeKey<TKey extends keyof T>(keyArg: TKey, valueArg: T[TKey]) {
     const easyStore = await this.getEasyStore();
     easyStore.data[keyArg] = valueArg;
     await easyStore.save();

ts/smartdata.classes.watcher.ts

@@ -17,9 +17,13 @@ export class SmartdataDbWatcher<T = any> {
     smartdataDbDocArg: typeof SmartDataDbDoc
   ) {
     this.changeStream = changeStreamArg;
-    this.changeStream.on('change', async (item: T) => {
+    this.changeStream.on('change', async (item: any) => {
+      if (!item.fullDocument) {
+        this.changeSubject.next(null);
+        return;
+      }
       this.changeSubject.next(
-        smartdataDbDocArg.createInstanceFromMongoDbNativeDoc(item) as any as T
+        smartdataDbDocArg.createInstanceFromMongoDbNativeDoc(item.fullDocument) as any as T
       );
     });
     plugins.smartdelay.delayFor(0).then(() => {

ts/smartdata.plugins.ts

@@ -8,7 +8,6 @@ import * as lik from '@push.rocks/lik';
 import * as smartdelay from '@push.rocks/smartdelay';
 import * as smartlog from '@push.rocks/smartlog';
 import * as smartpromise from '@push.rocks/smartpromise';
-import * as smartq from '@push.rocks/smartpromise';
 import * as smartrx from '@push.rocks/smartrx';
 import * as smartstring from '@push.rocks/smartstring';
 import * as smarttime from '@push.rocks/smarttime';
@@ -21,7 +20,6 @@ export {
   smartdelay,
   smartpromise,
   smartlog,
-  smartq,
   smartrx,
   mongodb,
   smartstring,