5.0.12

.gitlab-ci.yml

@@ -13,32 +13,24 @@ stages:
   - metadata
 
 before_script:
-  - apt-get update && apt-get install -y libcurl3 libssl-dev openssl libssl1.0.0 mongodb
+  - pnpm install -g pnpm
-  - npm install -g @shipzone/npmci
+  - pnpm install -g @shipzone/npmci
+  - npmci npm prepare
 
 # ====================
 # security stage
 # ====================
-mirror:
+# ====================
-  stage: security
+# security stage
-  script:
+# ====================
-    - npmci git mirror
-  only:
-    - tags
-  tags:
-    - lossless
-    - docker
-    - notpriv
-
 auditProductionDependencies:
   image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
   stage: security
   script:
-    - npmci npm prepare
+     - npmci command npm config set registry https://registry.npmjs.org
-    - npmci command npm install --production --ignore-scripts
+     - npmci command pnpm audit --audit-level=high --prod
-    - npmci command npm config set registry https://registry.npmjs.org
-    - npmci command npm audit --audit-level=high --only=prod --production
   tags:
+    - lossless
     - docker
   allow_failure: true
 
@@ -46,11 +38,10 @@ auditDevDependencies:
   image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
   stage: security
   script:
-    - npmci npm prepare
-    - npmci command npm install --ignore-scripts
     - npmci command npm config set registry https://registry.npmjs.org
-    - npmci command npm audit --audit-level=high --only=dev
+    - npmci command pnpm audit --audit-level=high --dev
   tags:
+    - lossless
     - docker
   allow_failure: true
 
@@ -61,7 +52,6 @@ auditDevDependencies:
 testStable:
   stage: test
   script:
-    - npmci npm prepare
     - npmci node install stable
     - npmci npm install
     - npmci npm test
@@ -72,7 +62,6 @@ testStable:
 testBuild:
   stage: test
   script:
-    - npmci npm prepare
     - npmci node install stable
     - npmci npm install
     - npmci command npm run build
@@ -101,10 +90,9 @@ codequality:
   only:
     - tags
   script:
-    - npmci command npm install -g tslint typescript
+    - npmci command npm install -g typescript
     - npmci npm prepare
     - npmci npm install
-    - npmci command "tslint -c tslint.json ./ts/**/*.ts"
   tags:
     - lossless
     - docker
@@ -124,11 +112,9 @@ trigger:
 pages:
   stage: metadata
   script:
-    - npmci node install lts
+    - npmci node install stable
-    - npmci command npm install -g @gitzone/tsdoc
-    - npmci npm prepare
     - npmci npm install
-    - npmci command tsdoc
+    - npmci command npm run buildDocs
   tags:
     - lossless
     - docker

npmextra.json

@@ -15,7 +15,7 @@
       "githost": "gitlab.com",
       "gitscope": "pushrocks",
       "gitrepo": "smartdata",
-      "shortDescription": "do more with data",
+      "description": "do more with data",
       "npmPackagename": "@pushrocks/smartdata",
       "license": "MIT"
     }

package.json

@@ -1,14 +1,15 @@
 {
   "name": "@pushrocks/smartdata",
-  "version": "4.0.22",
+  "version": "5.0.12",
   "private": false,
   "description": "do more with data",
   "main": "dist_ts/index.js",
   "typings": "dist_ts/index.d.ts",
+  "type": "module",
   "scripts": {
-    "test": "(tstest test/)",
+    "test": "tstest test/",
-    "testLocal": "(npmdocker)",
+    "build": "tsbuild --web --allowimplicitany",
-    "build": "(tsbuild --web)"
+    "buildDocs": "tsdoc"
   },
   "repository": {
     "type": "git",
@@ -21,29 +22,29 @@
   },
   "homepage": "https://gitlab.com/pushrocks/smartdata#README",
   "dependencies": {
-    "@pushrocks/lik": "^5.0.0",
+    "@pushrocks/lik": "^6.0.2",
-    "@pushrocks/smartlog": "^2.0.44",
+    "@pushrocks/smartdelay": "^2.0.13",
-    "@pushrocks/smartpromise": "^3.1.5",
+    "@pushrocks/smartlog": "^3.0.1",
-    "@pushrocks/smartstring": "^3.0.24",
+    "@pushrocks/smartmongo": "^2.0.7",
+    "@pushrocks/smartpromise": "^3.1.7",
+    "@pushrocks/smartrx": "^3.0.0",
+    "@pushrocks/smartstring": "^4.0.2",
+    "@pushrocks/smarttime": "^4.0.1",
     "@pushrocks/smartunique": "^3.0.3",
-    "@tsclass/tsclass": "^3.0.36",
+    "@pushrocks/taskbuffer": "^3.0.10",
-    "@types/lodash": "^4.14.176",
+    "@tsclass/tsclass": "^4.0.34",
-    "@types/mongodb": "^4.0.7",
+    "@types/lodash": "^4.14.191",
     "lodash": "^4.17.21",
-    "mongodb": "^4.1.4",
+    "mongodb": "^5.1.0"
-    "runtime-type-checks": "0.0.4"
   },
   "devDependencies": {
-    "@gitzone/tsbuild": "^2.1.28",
+    "@gitzone/tsbuild": "^2.1.65",
-    "@gitzone/tstest": "^1.0.60",
+    "@gitzone/tsrun": "^1.2.37",
-    "@pushrocks/qenv": "^4.0.10",
+    "@gitzone/tstest": "^1.0.74",
-    "@pushrocks/tapbundle": "^3.2.14",
+    "@pushrocks/qenv": "^5.0.2",
-    "@types/mongodb-memory-server": "^2.3.0",
+    "@pushrocks/tapbundle": "^5.0.4",
-    "@types/node": "^16.11.7",
+    "@types/node": "^18.15.5",
-    "@types/shortid": "0.0.29",
+    "@types/shortid": "0.0.29"
-    "mongodb-memory-server": "^8.0.2",
-    "tslint": "^6.1.3",
-    "tslint-config-prettier": "^1.18.0"
   },
   "files": [
     "ts/**/*",

test/test.easystore.ts

@@ -1,33 +1,25 @@
 import { tap, expect } from '@pushrocks/tapbundle';
 import { Qenv } from '@pushrocks/qenv';
+import * as smartmongo from '@pushrocks/smartmongo';
+import { smartunique } from '../ts/smartdata.plugins.js';
 
 const testQenv = new Qenv(process.cwd(), process.cwd() + '/.nogit/');
 
 console.log(process.memoryUsage());
 
 // the tested module
-import * as smartdata from '../ts/index';
+import * as smartdata from '../ts/index.js';
-
-import * as mongoPlugin from 'mongodb-memory-server';
-import { smartunique } from '../ts/smartdata.plugins';
 
 // =======================================
 // Connecting to the database server
 // =======================================
 
+let smartmongoInstance: smartmongo.SmartMongo;
 let testDb: smartdata.SmartdataDb;
-let smartdataOptions: smartdata.IMongoDescriptor;
-let mongod: mongoPlugin.MongoMemoryServer;
 
 tap.test('should create a testinstance as database', async () => {
-  mongod = await mongoPlugin.MongoMemoryServer.create();
+  smartmongoInstance = await smartmongo.SmartMongo.createAndStart();
-  console.log('created mongod instance');
+  testDb = new smartdata.SmartdataDb(await smartmongoInstance.getMongoDescriptor());
-  console.log('mongod started');
-  smartdataOptions = {
-    mongoDbUrl: mongod.getUri(),
-  };
-  console.log(smartdataOptions);
-  testDb = new smartdata.SmartdataDb(smartdataOptions);
   await testDb.init();
 });
 
@@ -49,12 +41,15 @@ tap.test('should create an easystore', async () => {
   easyStore = await testDb.createEasyStore('hellothere');
   await easyStore.writeKey('key1', 'hello');
   const retrievedKey = await easyStore.readKey('key1');
-  expect(retrievedKey).to.equal('hello');
+  expect(retrievedKey).toEqual('hello');
 });
 
 tap.test('close', async () => {
-  await mongod.stop();
+  await testDb.mongoDb.dropDatabase();
   await testDb.close();
+  if (smartmongoInstance) {
+    await smartmongoInstance.stop();
+  }
 });
 
 tap.start();

test/test.ts

@@ -1,35 +1,29 @@
 import { tap, expect } from '@pushrocks/tapbundle';
 import { Qenv } from '@pushrocks/qenv';
+import * as smartmongo from '@pushrocks/smartmongo';
+import { smartunique } from '../ts/smartdata.plugins.js';
+
+import * as mongodb from 'mongodb';
 
 const testQenv = new Qenv(process.cwd(), process.cwd() + '/.nogit/');
 
 console.log(process.memoryUsage());
 
 // the tested module
-import * as smartdata from '../ts/index';
+import * as smartdata from '../ts/index.js';
-
-import * as mongoPlugin from 'mongodb-memory-server';
-import { smartunique } from '../ts/smartdata.plugins';
 
 // =======================================
 // Connecting to the database server
 // =======================================
 
+let smartmongoInstance: smartmongo.SmartMongo;
 let testDb: smartdata.SmartdataDb;
-let smartdataOptions: smartdata.IMongoDescriptor;
-let mongod: mongoPlugin.MongoMemoryServer;
 
 const totalCars = 2000;
 
 tap.test('should create a testinstance as database', async () => {
-  mongod = await mongoPlugin.MongoMemoryServer.create();
+  smartmongoInstance = await smartmongo.SmartMongo.createAndStart();
-  console.log('created mongod instance');
+  testDb = new smartdata.SmartdataDb(await smartmongoInstance.getMongoDescriptor());
-  console.log('mongod started');
-  smartdataOptions = {
-    mongoDbUrl: mongod.getUri(),
-  };
-  console.log(smartdataOptions);
-  testDb = new smartdata.SmartdataDb(smartdataOptions);
   await testDb.init();
 });
 
@@ -63,6 +57,9 @@ class Car extends smartdata.SmartDataDbDoc<Car, Car> {
   @smartdata.svDb()
   public brand: string;
 
+  @smartdata.svDb()
+  public testBuffer = Buffer.from('hello');
+
   @smartdata.svDb()
   deepData = {
     sodeep: 'yes',
@@ -75,7 +72,7 @@ class Car extends smartdata.SmartDataDbDoc<Car, Car> {
   }
 }
 
-tap.test('should save the car to the db', async () => {
+tap.test('should save the car to the db', async (toolsArg) => {
   const myCar = new Car('red', 'Volvo');
   await myCar.save();
 
@@ -83,6 +80,9 @@ tap.test('should save the car to the db', async () => {
   await myCar2.save();
 
   let counter = 0;
+
+  const gottenCarInstance = await Car.getInstance({});
+  console.log(gottenCarInstance.testBuffer instanceof mongodb.Binary);
   process.memoryUsage();
   do {
     const myCar3 = new Car('red', 'Renault');
@@ -100,7 +100,7 @@ tap.test('should save the car to the db', async () => {
 });
 
 tap.test('expect to get instance of Car with shallow match', async () => {
-  const totalQueryCycles = totalCars / 4;
+  const totalQueryCycles = totalCars / 2;
   let counter = 0;
   do {
     const timeStart = Date.now();
@@ -114,14 +114,14 @@ tap.test('expect to get instance of Car with shallow match', async () => {
         }ms to query a set of 2000 with memory footprint ${process.memoryUsage().rss / 1e6} MB`
       );
     }
-    expect(myCars[0].deepData.sodeep).to.equal('yes');
+    expect(myCars[0].deepData.sodeep).toEqual('yes');
-    expect(myCars[0].brand).to.equal('Renault');
+    expect(myCars[0].brand).toEqual('Renault');
     counter++;
   } while (counter < totalQueryCycles);
 });
 
 tap.test('expect to get instance of Car with deep match', async () => {
-  const totalQueryCycles = totalCars / 4;
+  const totalQueryCycles = totalCars / 6;
   let counter = 0;
   do {
     const timeStart = Date.now();
@@ -137,8 +137,8 @@ tap.test('expect to get instance of Car with deep match', async () => {
         }ms to deep query a set of 2000 with memory footprint ${process.memoryUsage().rss / 1e6} MB`
       );
     }
-    expect(myCars2[0].deepData.sodeep).to.equal('yes');
+    expect(myCars2[0].deepData.sodeep).toEqual('yes');
-    expect(myCars2[0].brand).to.equal('Volvo');
+    expect(myCars2[0].brand).toEqual('Volvo');
     counter++;
   } while (counter < totalQueryCycles);
 });
@@ -147,7 +147,7 @@ tap.test('expect to get instance of Car and update it', async () => {
   const myCar = await Car.getInstance<Car>({
     brand: 'Volvo',
   });
-  expect(myCar.color).to.equal('red');
+  expect(myCar.color).toEqual('red');
   myCar.color = 'blue';
   await myCar.save();
 });
@@ -158,7 +158,7 @@ tap.test('should be able to delete an instance of car', async () => {
     color: 'blue',
   });
   console.log(myCars);
-  expect(myCars[0].color).to.equal('blue');
+  expect(myCars[0].color).toEqual('blue');
   for (const myCar of myCars) {
     await myCar.delete();
   }
@@ -166,7 +166,7 @@ tap.test('should be able to delete an instance of car', async () => {
   const myCar2 = await Car.getInstance<Car>({
     brand: 'Volvo',
   });
-  expect(myCar2.color).to.equal('red');
+  expect(myCar2.color).toEqual('red');
 });
 
 // tslint:disable-next-line: max-classes-per-file
@@ -201,9 +201,11 @@ tap.test('should store a new Truck', async () => {
 });
 
 tap.test('should use a cursor', async () => {
-  const cursor = await Truck.getCursor({});
+  const cursor = await Car.getCursor({});
-  cursor.forEach(async truckArg => {
+  let counter = 0;
-    console.log(truckArg.id);
+  await cursor.forEach(async (carArg) => {
+    counter++;
+    counter % 50 === 0 ? console.log(`50 more of ${carArg.color}`) : null;
   });
 });
 
@@ -211,8 +213,11 @@ tap.test('should use a cursor', async () => {
 // close the database connection
 // =======================================
 tap.test('close', async () => {
-  await mongod.stop();
+  await testDb.mongoDb.dropDatabase();
   await testDb.close();
+  if (smartmongoInstance) {
+    await smartmongoInstance.stop();
+  }
 });
 
 tap.start({ throwOnError: true });

test/test.typescript.ts

@@ -1,35 +1,27 @@
 import { tap, expect } from '@pushrocks/tapbundle';
 import { Qenv } from '@pushrocks/qenv';
+import * as smartmongo from '@pushrocks/smartmongo';
+import { smartunique } from '../ts/smartdata.plugins.js';
 
 const testQenv = new Qenv(process.cwd(), process.cwd() + '/.nogit/');
 
 console.log(process.memoryUsage());
 
 // the tested module
-import * as smartdata from '../ts/index';
+import * as smartdata from '../ts/index.js';
-
-import * as mongoPlugin from 'mongodb-memory-server';
-import { smartunique } from '../ts/smartdata.plugins';
 
 // =======================================
 // Connecting to the database server
 // =======================================
 
+let smartmongoInstance: smartmongo.SmartMongo;
 let testDb: smartdata.SmartdataDb;
-let smartdataOptions: smartdata.IMongoDescriptor;
-let mongod: mongoPlugin.MongoMemoryServer;
 
 const totalCars = 2000;
 
 tap.test('should create a testinstance as database', async () => {
-  mongod = await mongoPlugin.MongoMemoryServer.create();
+  smartmongoInstance = await smartmongo.SmartMongo.createAndStart();
-  console.log('created mongod instance');
+  testDb = new smartdata.SmartdataDb(await smartmongoInstance.getMongoDescriptor());
-  console.log('mongod started');
-  smartdataOptions = {
-    mongoDbUrl: mongod.getUri(),
-  };
-  console.log(smartdataOptions);
-  testDb = new smartdata.SmartdataDb(smartdataOptions);
   await testDb.init();
 });
 
@@ -93,8 +85,11 @@ tap.test('should get a car', async () => {
 // close the database connection
 // =======================================
 tap.test('close', async () => {
-  await mongod.stop();
+  await testDb.mongoDb.dropDatabase();
   await testDb.close();
+  if (smartmongoInstance) {
+    await smartmongoInstance.stop();
+  }
 });
 
 tap.start({ throwOnError: true });

test/test.watch.ts

@@ -0,0 +1,74 @@
+import { tap, expect } from '@pushrocks/tapbundle';
+import { Qenv } from '@pushrocks/qenv';
+import * as smartmongo from '@pushrocks/smartmongo';
+import { smartunique } from '../ts/smartdata.plugins.js';
+
+const testQenv = new Qenv(process.cwd(), process.cwd() + '/.nogit/');
+
+console.log(process.memoryUsage());
+
+// the tested module
+import * as smartdata from '../ts/index.js';
+
+// =======================================
+// Connecting to the database server
+// =======================================
+
+let smartmongoInstance: smartmongo.SmartMongo;
+let testDb: smartdata.SmartdataDb;
+
+const totalCars = 2000;
+
+tap.test('should create a testinstance as database', async () => {
+  smartmongoInstance = await smartmongo.SmartMongo.createAndStart();
+  testDb = new smartdata.SmartdataDb(await smartmongoInstance.getMongoDescriptor());
+  await testDb.init();
+});
+
+tap.skip.test('should connect to atlas', async (tools) => {
+  const databaseName = `test-smartdata-${smartunique.shortId()}`;
+  testDb = new smartdata.SmartdataDb({
+    mongoDbUrl: testQenv.getEnvVarOnDemand('MONGO_URL'),
+    mongoDbName: databaseName,
+  });
+  await testDb.init();
+});
+
+@smartdata.Collection(() => testDb)
+class House extends smartdata.SmartDataDbDoc<House, House> {
+  @smartdata.unI()
+  public id: string = smartunique.shortId();
+
+  @smartdata.svDb()
+  public data = {
+    id: smartunique.shortId(),
+    hello: 'hello',
+  };
+}
+
+tap.test('should watch a collection', async (toolsArg) => {
+  const done = toolsArg.defer();
+  const watcher = await House.watch({});
+  watcher.changeSubject.subscribe(async (houseArg) => {
+    console.log('hey there, we observed a house');
+    await watcher.close();
+    done.resolve();
+  });
+  const newHouse = new House();
+  await newHouse.save();
+  console.log('saved a house');
+  await done.promise;
+});
+
+// =======================================
+// close the database connection
+// =======================================
+tap.test('close', async () => {
+  await testDb.mongoDb.dropDatabase();
+  await testDb.close();
+  if (smartmongoInstance) {
+    await smartmongoInstance.stop();
+  }
+});
+
+tap.start({ throwOnError: true });

ts/00_commitinfo_data.ts

@@ -0,0 +1,8 @@
+/**
+ * autocreated commitinfo by @pushrocks/commitinfo
+ */
+export const commitinfo = {
+  name: '@pushrocks/smartdata',
+  version: '5.0.12',
+  description: 'do more with data'
+}

ts/index.ts

@@ -1,6 +1,16 @@
-export * from './smartdata.classes.db';
+export * from './smartdata.classes.db.js';
-export * from './smartdata.classes.collection';
+export * from './smartdata.classes.collection.js';
-export * from './smartdata.classes.doc';
+export * from './smartdata.classes.doc.js';
-export * from './smartdata.classes.easystore';
+export * from './smartdata.classes.easystore.js';
+export * from './smartdata.classes.cursor.js';
 
-export { IMongoDescriptor } from './interfaces';
+import * as convenience from './smartadata.convenience.js';
+
+export {
+  convenience
+}
+
+// to be removed with the next breaking update
+import type * as plugins from './smartdata.plugins.js';
+type IMongoDescriptor = plugins.tsclass.database.IMongoDescriptor;
+export type { IMongoDescriptor };

ts/interfaces/index.ts

@@ -1 +0,0 @@
-export * from './mongodescriptor';

ts/interfaces/mongodescriptor.ts

@@ -1,22 +0,0 @@
-export interface IMongoDescriptor {
-  /**
-   * the URL to connect to
-   */
-  mongoDbUrl: string;
-
-  /**
-   * the db to use for the project
-   */
-  mongoDbName?: string;
-
-  /**
-   * a username to use to connect to the database
-   */
-
-  mongoDbUser?: string;
-
-  /**
-   * an optional password that will be replace <PASSWORD> in the connection string
-   */
-  mongoDbPass?: string;
-}

ts/smartadata.convenience.ts

@@ -0,0 +1,5 @@
+import * as plugins from './smartdata.plugins.js';
+
+export const getNewUniqueId = async (prefixArg?: string) => {
+  return plugins.smartunique.uni(prefixArg);
+}

ts/smartdata.classes.collection.ts

@@ -1,8 +1,9 @@
-import * as plugins from './smartdata.plugins';
+import * as plugins from './smartdata.plugins.js';
-import { SmartdataDb } from './smartdata.classes.db';
+import { SmartdataDb } from './smartdata.classes.db.js';
-import { SmartdataDbCursor } from './smartdata.classes.cursor';
+import { SmartdataDbCursor } from './smartdata.classes.cursor.js';
-import { SmartDataDbDoc } from './smartdata.classes.doc';
+import { SmartDataDbDoc } from './smartdata.classes.doc.js';
-import { CollectionFactory } from './smartdata.classes.collectionfactory';
+import { SmartdataDbWatcher } from './smartdata.classes.watcher.js';
+import { CollectionFactory } from './smartdata.classes.collectionfactory.js';
 
 export interface IFindOptions {
   limit?: number;
@@ -85,7 +86,7 @@ export function Manager<TManager extends IManager>(managerArg?: TManager | TDela
       public static get manager() {
         let manager: TManager;
         if (!managerArg) {
-          manager = this.prototype.defaultManager
+          manager = this.prototype.defaultManager;
         } else if (managerArg['db']) {
           manager = managerArg as TManager;
         } else {
@@ -96,7 +97,7 @@ export function Manager<TManager extends IManager>(managerArg?: TManager | TDela
       public get manager() {
         let manager: TManager;
         if (!managerArg) {
-          manager = this.defaultManager
+          manager = this.defaultManager;
         } else if (managerArg['db']) {
           manager = managerArg as TManager;
         } else {
@@ -108,7 +109,6 @@ export function Manager<TManager extends IManager>(managerArg?: TManager | TDela
   };
 }
 
-// tslint:disable-next-line: max-classes-per-file
 export class SmartdataCollection<T> {
   /**
    * the collection that is used
@@ -171,7 +171,7 @@ export class SmartdataCollection<T> {
   /**
    * finds an object in the DbCollection
    */
-   public async findOne(filterObject: any): Promise<any> {
+  public async findOne(filterObject: any): Promise<any> {
     await this.init();
     const cursor = this.mongoDbCollection.find(filterObject);
     const result = await cursor.next();
@@ -179,10 +179,13 @@ export class SmartdataCollection<T> {
     return result;
   }
 
-  public async getCursor(filterObject: any): Promise<SmartdataDbCursor<any>> {
+  public async getCursor(
+    filterObjectArg: any,
+    dbDocArg: typeof SmartDataDbDoc
+  ): Promise<SmartdataDbCursor<any>> {
     await this.init();
-    const cursor = this.mongoDbCollection.find(filterObject);
+    const cursor = this.mongoDbCollection.find(filterObjectArg);
-    return new SmartdataDbCursor(cursor);
+    return new SmartdataDbCursor(cursor, dbDocArg);
   }
 
   /**
@@ -196,6 +199,29 @@ export class SmartdataCollection<T> {
     return result;
   }
 
+  /**
+   * watches the collection while applying a filter
+   */
+  public async watch(
+    filterObject: any,
+    smartdataDbDocArg: typeof SmartDataDbDoc
+  ): Promise<SmartdataDbWatcher> {
+    await this.init();
+    const changeStream = this.mongoDbCollection.watch(
+      [
+        {
+          $match: filterObject,
+        },
+      ],
+      {
+        fullDocument: 'updateLookup',
+      }
+    );
+    const smartdataWatcher = new SmartdataDbWatcher(changeStream, smartdataDbDocArg);
+    await smartdataWatcher.readyDeferred.promise;
+    return smartdataWatcher;
+  }
+
   /**
    * create an object in the database
    */

ts/smartdata.classes.collectionfactory.ts

@@ -1,6 +1,6 @@
-import * as plugins from './smartdata.plugins';
+import * as plugins from './smartdata.plugins.js';
-import { SmartdataCollection } from './smartdata.classes.collection';
+import { SmartdataCollection } from './smartdata.classes.collection.js';
-import { SmartdataDb } from './smartdata.classes.db';
+import { SmartdataDb } from './smartdata.classes.db.js';
 
 export class CollectionFactory {
   public collections: { [key: string]: SmartdataCollection<any> } = {};

ts/smartdata.classes.cursor.ts

@@ -1,33 +1,40 @@
-import * as plugins from './smartdata.plugins';
+import { SmartDataDbDoc } from './smartdata.classes.doc.js';
+import * as plugins from './smartdata.plugins.js';
 
 /**
- * a wrapper for the native mongodb cursor. Exposes better 
+ * a wrapper for the native mongodb cursor. Exposes better
  */
 export class SmartdataDbCursor<T = any> {
   // STATIC
 
   // INSTANCE
   public mongodbCursor: plugins.mongodb.FindCursor<T>;
-  constructor(cursorArg: plugins.mongodb.FindCursor<T>) {
+  private smartdataDbDoc: typeof SmartDataDbDoc;
-    this.mongodbCursor = cursorArg
+  constructor(cursorArg: plugins.mongodb.FindCursor<T>, dbDocArg: typeof SmartDataDbDoc) {
-  };
+    this.mongodbCursor = cursorArg;
+    this.smartdataDbDoc = dbDocArg;
+  }
 
   public async next(closeAtEnd = true) {
-    const result = await this.mongodbCursor.next();
+    const result = this.smartdataDbDoc.createInstanceFromMongoDbNativeDoc(
+      await this.mongodbCursor.next()
+    );
     if (!result && closeAtEnd) {
       await this.close();
-    };
+    }
     return result;
-  };
+  }
 
   public async forEach(forEachFuncArg: (itemArg: T) => Promise<any>, closeCursorAtEnd = true) {
-    let currentValue: T;
+    let nextDocument: any;
     do {
-      currentValue = await this.mongodbCursor.next();
+      nextDocument = await this.mongodbCursor.next();
-      if (currentValue) {
+      if (nextDocument) {
-        await forEachFuncArg(currentValue);
+        const nextClassInstance =
+          this.smartdataDbDoc.createInstanceFromMongoDbNativeDoc(nextDocument);
+        await forEachFuncArg(nextClassInstance as any);
       }
-    } while (currentValue);
+    } while (nextDocument);
     if (closeCursorAtEnd) {
       await this.close();
     }
@@ -36,4 +43,4 @@ export class SmartdataDbCursor<T = any> {
   public async close() {
     await this.mongodbCursor.close();
   }
-}
+}

ts/smartdata.classes.db.ts

@@ -1,11 +1,10 @@
-import * as plugins from './smartdata.plugins';
+import * as plugins from './smartdata.plugins.js';
 import { ObjectMap } from '@pushrocks/lik';
 
-import { SmartdataCollection } from './smartdata.classes.collection';
+import { SmartdataCollection } from './smartdata.classes.collection.js';
-import { EasyStore } from './smartdata.classes.easystore';
+import { EasyStore } from './smartdata.classes.easystore.js';
 
-import { logger } from './smartdata.logging';
+import { logger } from './smartdata.logging.js';
-import { IMongoDescriptor } from './interfaces';
 
 /**
  * interface - indicates the connection status of the db
@@ -13,13 +12,14 @@ import { IMongoDescriptor } from './interfaces';
 export type TConnectionStatus = 'initial' | 'disconnected' | 'connected' | 'failed';
 
 export class SmartdataDb {
-  smartdataOptions: IMongoDescriptor;
+  smartdataOptions: plugins.tsclass.database.IMongoDescriptor;
   mongoDbClient: plugins.mongodb.MongoClient;
   mongoDb: plugins.mongodb.Db;
   status: TConnectionStatus;
+  statusConnectedDeferred = plugins.smartpromise.defer();
   smartdataCollectionMap = new ObjectMap<SmartdataCollection<any>>();
 
-  constructor(smartdataOptions: IMongoDescriptor) {
+  constructor(smartdataOptions: plugins.tsclass.database.IMongoDescriptor) {
     this.smartdataOptions = smartdataOptions;
     this.status = 'initial';
   }
@@ -52,6 +52,7 @@ export class SmartdataDb {
     });
     this.mongoDb = this.mongoDbClient.db(this.smartdataOptions.mongoDbName);
     this.status = 'connected';
+    this.statusConnectedDeferred.resolve();
     console.log(`Connected to database ${this.smartdataOptions.mongoDbName}`);
   }
 

ts/smartdata.classes.distributedcoordinator.ts

@@ -0,0 +1,201 @@
+import * as plugins from './smartdata.plugins.js';
+import { SmartdataDb } from './smartdata.classes.db.js';
+import { Manager, setDefaultManagerForDoc } from './smartdata.classes.collection.js';
+import { SmartDataDbDoc, svDb, unI } from './smartdata.classes.doc.js';
+
+@Manager()
+class DistributedClass extends SmartDataDbDoc<DistributedClass, DistributedClass> {
+  // INSTANCE
+  @unI()
+  public id: string;
+
+  @svDb()
+  public data: {
+    status: 'bidding' | 'settled' | 'initializing' | 'stopped';
+    biddingShortcode?: string;
+    lastUpdated: number;
+    elected: boolean;
+    /**
+     * used to store request
+     */
+    taskRequests: plugins.taskbuffer.distributedCoordination.IDistributedTaskRequest[],
+  
+    /**
+     * only used by the leader to convey consultation results
+     */
+    taskRequestResults: plugins.taskbuffer.distributedCoordination.IDistributedTaskRequestResult[];
+  }
+}
+
+/**
+ * This file implements a distributed coordinator according to the @pushrocks/taskbuffer standard.
+ * you should not set up this yourself. Instead, there is a factory on the SmartdataDb class
+ * that will take care of setting this up.
+ */
+export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distributedCoordination
+  .AbstractDistributedCoordinator {
+  public readyPromise: Promise<any>;
+  public db: SmartdataDb;
+  private asyncExecutionStack = new plugins.lik.AsyncExecutionStack();
+  private ownInstance: DistributedClass;
+
+  constructor(dbArg?: SmartdataDb) {
+    super();
+    this.db = dbArg;
+    setDefaultManagerForDoc(this, DistributedClass);
+    this.readyPromise = this.db.statusConnectedDeferred.promise;
+    this.init();
+  }
+
+  // smartdata specific stuff
+  public async start() {
+    await this.init();
+  }
+
+  public async stop() {
+    if (this.ownInstance?.data.elected) {
+      this.ownInstance.data.elected = false;
+    } else {
+      console.log(`can't stop a distributed instance that has not been started yet.`);
+    }
+  }
+
+  public id = plugins.smartunique.uni('distributedInstance');
+
+  private startHeartbeat = async () => {
+    while (this.ownInstance.data.status !== 'stopped') {
+      await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+        await this.ownInstance.updateFromDb();
+        this.ownInstance.data.lastUpdated = Date.now();
+        await this.ownInstance.save();
+      });
+      await plugins.smartdelay.delayFor(10000);
+    }
+  };
+  public async init() {
+    await this.readyPromise;
+    if (!this.ownInstance) {
+      await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+        this.ownInstance = new DistributedClass();
+        this.ownInstance.id = this.id;
+        this.ownInstance.data = {
+          elected: false,
+          lastUpdated: Date.now(),
+          status: 'initializing',
+          taskRequests: [],
+          taskRequestResults: []
+        };
+        await this.ownInstance.save();
+      });
+    }
+
+    // lets enable the heartbeat
+    this.startHeartbeat();
+
+    // lets do a leader check
+    await this.checkAndMaybeLead();
+
+    return this.ownInstance;
+  }
+
+  // --> leader election
+  public async checkAndMaybeLead() {
+    const allInstances = await DistributedClass.getInstances({});
+    let leader = allInstances.find((instanceArg) => instanceArg.data.elected === true);
+    if (
+      leader &&
+      leader.data.lastUpdated >=
+        Date.now() - plugins.smarttime.getMilliSecondsFromUnits({ minutes: 1 })
+    ) {
+      await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+        await this.ownInstance.updateFromDb();
+        this.ownInstance.data.status = 'settled';
+        await this.ownInstance.save();
+      });
+      return;
+    } else {
+      await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+        this.ownInstance.data.status = 'bidding';
+        this.ownInstance.data.biddingShortcode = plugins.smartunique.shortId();
+        await this.ownInstance.save();
+      });
+      await plugins.smartdelay.delayFor(plugins.smarttime.getMilliSecondsFromUnits({ minutes: 2 }));
+      await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+        let biddingInstances = await DistributedClass.getInstances({});
+        biddingInstances = biddingInstances.filter(
+          (instanceArg) =>
+            !instanceArg.data.elected &&
+            instanceArg.data.lastUpdated >=
+              Date.now() - plugins.smarttime.getMilliSecondsFromUnits({ minutes: 1 })
+        );
+        this.ownInstance.data.elected = true;
+        for (const biddingInstance of biddingInstances) {
+          if (biddingInstance.data.biddingShortcode < this.ownInstance.data.biddingShortcode) {
+            this.ownInstance.data.elected = false;
+          }
+        }
+        await this.ownInstance.save();
+      });
+      if (this.ownInstance.data.elected) {
+        this.leadFunction();
+      }
+    }
+  }
+
+  /**
+   * when it has been determined
+   * that this instance is leading
+   * the leading is implemented here
+   */
+  public async leadFunction() {
+    const ownInstance = await this.init();
+    const watcher = await DistributedClass.watch({});
+    /**
+     * this function is started once per unique job request
+     */
+    const startResultTimer = async () => {
+
+    }
+    
+    watcher.changeSubject.subscribe({
+      next: async (distributedDoc) => {
+        distributedDoc
+      }
+    })
+    while (this.ownInstance.data.status !== 'stopped') {
+      await plugins.smartdelay.delayFor(1000);
+    }
+  };
+
+  // abstract implemented methods
+  public async fireDistributedTaskRequest(
+    taskRequestArg: plugins.taskbuffer.distributedCoordination.IDistributedTaskRequest
+  ): Promise<plugins.taskbuffer.distributedCoordination.IDistributedTaskRequestResult> {
+    const ownInstance = await this.init();
+    await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+      this.ownInstance.data.taskRequests.push(taskRequestArg);
+      await this.ownInstance.save();
+    });
+    return null;
+  }
+
+  public async updateDistributedTaskRequest(
+    infoBasisArg: plugins.taskbuffer.distributedCoordination.IDistributedTaskRequest
+  ): Promise<void> {
+    await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+      const existingInfoBasis = this.ownInstance.data.taskRequests.find(infoBasisItem => {
+        return infoBasisItem.taskName === infoBasisArg.taskName
+          && infoBasisItem.taskExecutionTime === infoBasisArg.taskExecutionTime;
+      });
+      Object.assign(existingInfoBasis, infoBasisArg);
+      await this.ownInstance.save();
+      plugins.smartdelay.delayFor(60000).then(() => {
+        this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+          const indexToRemove = this.ownInstance.data.taskRequests.indexOf(existingInfoBasis);
+          this.ownInstance.data.taskRequests.splice(indexToRemove, indexToRemove);
+          await this.ownInstance.save();
+        })
+      })
+    });
+  }
+}

ts/smartdata.classes.doc.ts

@@ -1,10 +1,11 @@
-import * as plugins from './smartdata.plugins';
+import * as plugins from './smartdata.plugins.js';
 
 import { ObjectMap } from '@pushrocks/lik';
 
-import { SmartdataDb } from './smartdata.classes.db';
+import { SmartdataDb } from './smartdata.classes.db.js';
-import { SmartdataDbCursor } from './smartdata.classes.cursor';
+import { SmartdataDbCursor } from './smartdata.classes.cursor.js';
-import { IManager, SmartdataCollection } from './smartdata.classes.collection';
+import { IManager, SmartdataCollection } from './smartdata.classes.collection.js';
+import { SmartdataDbWatcher } from './smartdata.classes.watcher.js';
 
 export type TDocCreation = 'db' | 'new' | 'mixed';
 
@@ -77,36 +78,7 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
   public static manager;
   public manager: TManager;
 
-  /**
+  // STATIC
-   * how the Doc in memory was created, may prove useful later.
-   */
-  public creationStatus: TDocCreation = 'new';
-
-  /**
-   * unique indexes
-   */
-  public uniqueIndexes: string[];
-
-  /**
-   * an array of saveable properties of a doc
-   */
-  public saveableProperties: string[];
-
-  /**
-   * name
-   */
-  public name: string;
-
-  /**
-   * primary id in the database
-   */
-  public dbDocUniqueId: string;
-
-  /**
-   * class constructor
-   */
-  constructor() {}
-
   public static createInstanceFromMongoDbNativeDoc<T>(
     this: plugins.tsclass.typeFest.Class<T>,
     mongoDbNativeDocArg: any
@@ -165,10 +137,32 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
     this: plugins.tsclass.typeFest.Class<T>,
     filterArg: plugins.tsclass.typeFest.PartialDeep<T>
   ) {
-    const cursor: SmartdataDbCursor<T> = await (this as any).collection.getCursor(convertFilterForMongoDb(filterArg));
+    const collection: SmartdataCollection<T> = (this as any).collection;
+    const cursor: SmartdataDbCursor<T> = await collection.getCursor(
+      convertFilterForMongoDb(filterArg),
+      this as any as typeof SmartDataDbDoc
+    );
     return cursor;
   }
 
+  /**
+   * watch the collection
+   * @param this
+   * @param filterArg
+   * @param forEachFunction
+   */
+  public static async watch<T>(
+    this: plugins.tsclass.typeFest.Class<T>,
+    filterArg: plugins.tsclass.typeFest.PartialDeep<T>
+  ) {
+    const collection: SmartdataCollection<T> = (this as any).collection;
+    const watcher: SmartdataDbWatcher<T> = await collection.watch(
+      convertFilterForMongoDb(filterArg),
+      this as any
+    );
+    return watcher;
+  }
+
   /**
    * run a function for all instances
    * @returns
@@ -179,9 +173,41 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
     forEachFunction: (itemArg: T) => Promise<any>
   ) {
     const cursor: SmartdataDbCursor<T> = await (this as any).getCursor(filterArg);
-    await cursor.forEach(forEachFunction)
+    await cursor.forEach(forEachFunction);
   }
 
+  // INSTANCE
+
+  /**
+   * how the Doc in memory was created, may prove useful later.
+   */
+  public creationStatus: TDocCreation = 'new';
+
+  /**
+   * unique indexes
+   */
+  public uniqueIndexes: string[];
+
+  /**
+   * an array of saveable properties of a doc
+   */
+  public saveableProperties: string[];
+
+  /**
+   * name
+   */
+  public name: string;
+
+  /**
+   * primary id in the database
+   */
+  public dbDocUniqueId: string;
+
+  /**
+   * class constructor
+   */
+  constructor() {}
+
   /**
    * saves this instance but not any connected items
    * may lead to data inconsistencies, but is faster
@@ -229,6 +255,16 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
     }
   }
 
+  /**
+   * updates an object from db
+   */
+  public async updateFromDb() {
+    const mongoDbNativeDoc = await this.collection.findOne(this.createIdentifiableObject());
+    for (const key of Object.keys(mongoDbNativeDoc)) {
+      this[key] = mongoDbNativeDoc[key];
+    }
+  }
+
   /**
    * creates a saveable object so the instance can be persisted as json in the database
    */

ts/smartdata.classes.easystore.ts

@@ -1,7 +1,7 @@
-import * as plugins from './smartdata.plugins';
+import * as plugins from './smartdata.plugins.js';
-import { Collection } from './smartdata.classes.collection';
+import { Collection } from './smartdata.classes.collection.js';
-import { SmartdataDb } from './smartdata.classes.db';
+import { SmartdataDb } from './smartdata.classes.db.js';
-import { SmartDataDbDoc, svDb, unI } from './smartdata.classes.doc';
+import { SmartDataDbDoc, svDb, unI } from './smartdata.classes.doc.js';
 
 /**
  * EasyStore allows the storage of easy objects. It also allows easy sharing of the object between different instances
@@ -17,6 +17,15 @@ export class EasyStore<T> {
       @unI()
       public nameId: string;
 
+      @svDb()
+      public ephermal: {
+        activated: boolean;
+        timeout: number;
+      };
+
+      @svDb()
+      lastEdit: number;
+
       @svDb()
       public data: Partial<T>;
     }
@@ -90,4 +99,10 @@ export class EasyStore<T> {
     easyStore.data = {};
     await easyStore.save();
   }
+
+  public async cleanUpEphermal() {
+    while(await this.smartdataDbRef.statusConnectedDeferred.promise && this.smartdataDbRef.status === 'connected') {
+      
+    }
+  }
 }

ts/smartdata.classes.watcher.ts

@@ -0,0 +1,33 @@
+import { SmartDataDbDoc } from './smartdata.classes.doc.js';
+import * as plugins from './smartdata.plugins.js';
+
+/**
+ * a wrapper for the native mongodb cursor. Exposes better
+ */
+export class SmartdataDbWatcher<T = any> {
+  // STATIC
+  public readyDeferred = plugins.smartpromise.defer();
+
+  // INSTANCE
+  private changeStream: plugins.mongodb.ChangeStream<T>;
+
+  public changeSubject = new plugins.smartrx.rxjs.Subject<T>();
+  constructor(
+    changeStreamArg: plugins.mongodb.ChangeStream<T>,
+    smartdataDbDocArg: typeof SmartDataDbDoc
+  ) {
+    this.changeStream = changeStreamArg;
+    this.changeStream.on('change', async (item: T) => {
+      this.changeSubject.next(
+        smartdataDbDocArg.createInstanceFromMongoDbNativeDoc(item) as any as T
+      );
+    });
+    plugins.smartdelay.delayFor(0).then(() => {
+      this.readyDeferred.resolve();
+    });
+  }
+
+  public async close() {
+    await this.changeStream.close();
+  }
+}

ts/smartdata.logging.ts

@@ -1,3 +1,3 @@
-import * as plugins from './smartdata.plugins';
+import * as plugins from './smartdata.plugins.js';
 
 export const logger = new plugins.smartlog.ConsoleLog();

ts/smartdata.plugins.ts

@@ -4,11 +4,30 @@ import * as tsclass from '@tsclass/tsclass';
 export { tsclass };
 
 // @pushrocks scope
+import * as lik from '@pushrocks/lik';
+import * as smartdelay from '@pushrocks/smartdelay';
 import * as smartlog from '@pushrocks/smartlog';
+import * as smartpromise from '@pushrocks/smartpromise';
+import * as smartq from '@pushrocks/smartpromise';
+import * as smartrx from '@pushrocks/smartrx';
+import * as smartstring from '@pushrocks/smartstring';
+import * as smarttime from '@pushrocks/smarttime';
+import * as smartunique from '@pushrocks/smartunique';
+import * as taskbuffer from '@pushrocks/taskbuffer';
 import * as lodash from 'lodash';
 import * as mongodb from 'mongodb';
-import * as smartq from '@pushrocks/smartpromise';
-import * as smartstring from '@pushrocks/smartstring';
-import * as smartunique from '@pushrocks/smartunique';
 
-export { smartlog, lodash, smartq, mongodb, smartstring, smartunique };
+export {
+  lik,
+  smartdelay,
+  smartpromise,
+  smartlog,
+  lodash,
+  smartq,
+  smartrx,
+  mongodb,
+  smartstring,
+  smarttime,
+  smartunique,
+  taskbuffer,
+};

tsconfig.json

@@ -1,7 +1,9 @@
 {
   "compilerOptions": {
     "experimentalDecorators": true,
-    "target": "es2017",
+    "useDefineForClassFields": false,
-    "module": "commonjs"
+    "target": "ES2022",
+    "module": "ES2022",
+    "moduleResolution": "nodenext"
   }
-}
+}

tslint.json

@@ -1,17 +0,0 @@
-{
-  "extends": ["tslint:latest", "tslint-config-prettier"],
-  "rules": {
-    "semicolon": [true, "always"],
-    "no-console": false,
-    "ordered-imports": false,
-    "object-literal-sort-keys": false,
-    "member-ordering": {
-      "options":{
-        "order": [
-          "static-method"
-        ]
-      }
-    }
-  },
-  "defaultSeverity": "warning"
-}