Compare commits
21 Commits
Author | SHA1 | Date | |
---|---|---|---|
0e55cd8876 | |||
eccdf3f00a | |||
c7544133d9 | |||
c7c9acf5bd | |||
c99ec50853 | |||
4dd9557e1d | |||
52b34a6da1 | |||
1bf74fe04d | |||
fdd875ad31 | |||
a7bf0c0298 | |||
59d6336e43 | |||
e0fc81179a | |||
5aa81a56a2 | |||
9ae26177b8 | |||
26ac52d6c5 | |||
fb39463b7d | |||
44acba80c1 | |||
8cf8315577 | |||
9b44b64a50 | |||
699e25201c | |||
2ef9aace68 |
66
.gitea/workflows/default_nottags.yaml
Normal file
66
.gitea/workflows/default_nottags.yaml
Normal file
@ -0,0 +1,66 @@
|
|||||||
|
name: Default (not tags)
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
tags-ignore:
|
||||||
|
- '**'
|
||||||
|
|
||||||
|
env:
|
||||||
|
IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
|
||||||
|
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
|
||||||
|
NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
|
||||||
|
NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
|
||||||
|
NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
|
||||||
|
NPMCI_URL_CLOUDLY: ${{secrets.NPMCI_URL_CLOUDLY}}
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
security:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
continue-on-error: true
|
||||||
|
container:
|
||||||
|
image: ${{ env.IMAGE }}
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v3
|
||||||
|
|
||||||
|
- name: Install pnpm and npmci
|
||||||
|
run: |
|
||||||
|
pnpm install -g pnpm
|
||||||
|
pnpm install -g @shipzone/npmci
|
||||||
|
|
||||||
|
- name: Run npm prepare
|
||||||
|
run: npmci npm prepare
|
||||||
|
|
||||||
|
- name: Audit production dependencies
|
||||||
|
run: |
|
||||||
|
npmci command npm config set registry https://registry.npmjs.org
|
||||||
|
npmci command pnpm audit --audit-level=high --prod
|
||||||
|
continue-on-error: true
|
||||||
|
|
||||||
|
- name: Audit development dependencies
|
||||||
|
run: |
|
||||||
|
npmci command npm config set registry https://registry.npmjs.org
|
||||||
|
npmci command pnpm audit --audit-level=high --dev
|
||||||
|
continue-on-error: true
|
||||||
|
|
||||||
|
test:
|
||||||
|
if: ${{ always() }}
|
||||||
|
needs: security
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
container:
|
||||||
|
image: ${{ env.IMAGE }}
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v3
|
||||||
|
|
||||||
|
- name: Test stable
|
||||||
|
run: |
|
||||||
|
npmci node install stable
|
||||||
|
npmci npm install
|
||||||
|
npmci npm test
|
||||||
|
|
||||||
|
- name: Test build
|
||||||
|
run: |
|
||||||
|
npmci node install stable
|
||||||
|
npmci npm install
|
||||||
|
npmci npm build
|
124
.gitea/workflows/default_tags.yaml
Normal file
124
.gitea/workflows/default_tags.yaml
Normal file
@ -0,0 +1,124 @@
|
|||||||
|
name: Default (tags)
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
tags:
|
||||||
|
- '*'
|
||||||
|
|
||||||
|
env:
|
||||||
|
IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
|
||||||
|
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
|
||||||
|
NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
|
||||||
|
NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
|
||||||
|
NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
|
||||||
|
NPMCI_URL_CLOUDLY: ${{secrets.NPMCI_URL_CLOUDLY}}
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
security:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
continue-on-error: true
|
||||||
|
container:
|
||||||
|
image: ${{ env.IMAGE }}
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v3
|
||||||
|
|
||||||
|
- name: Prepare
|
||||||
|
run: |
|
||||||
|
pnpm install -g pnpm
|
||||||
|
pnpm install -g @shipzone/npmci
|
||||||
|
npmci npm prepare
|
||||||
|
|
||||||
|
- name: Audit production dependencies
|
||||||
|
run: |
|
||||||
|
npmci command npm config set registry https://registry.npmjs.org
|
||||||
|
npmci command pnpm audit --audit-level=high --prod
|
||||||
|
continue-on-error: true
|
||||||
|
|
||||||
|
- name: Audit development dependencies
|
||||||
|
run: |
|
||||||
|
npmci command npm config set registry https://registry.npmjs.org
|
||||||
|
npmci command pnpm audit --audit-level=high --dev
|
||||||
|
continue-on-error: true
|
||||||
|
|
||||||
|
test:
|
||||||
|
if: ${{ always() }}
|
||||||
|
needs: security
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
container:
|
||||||
|
image: ${{ env.IMAGE }}
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v3
|
||||||
|
|
||||||
|
- name: Prepare
|
||||||
|
run: |
|
||||||
|
pnpm install -g pnpm
|
||||||
|
pnpm install -g @shipzone/npmci
|
||||||
|
npmci npm prepare
|
||||||
|
|
||||||
|
- name: Test stable
|
||||||
|
run: |
|
||||||
|
npmci node install stable
|
||||||
|
npmci npm install
|
||||||
|
npmci npm test
|
||||||
|
|
||||||
|
- name: Test build
|
||||||
|
run: |
|
||||||
|
npmci node install stable
|
||||||
|
npmci npm install
|
||||||
|
npmci npm build
|
||||||
|
|
||||||
|
release:
|
||||||
|
needs: test
|
||||||
|
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
container:
|
||||||
|
image: ${{ env.IMAGE }}
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v3
|
||||||
|
|
||||||
|
- name: Prepare
|
||||||
|
run: |
|
||||||
|
pnpm install -g pnpm
|
||||||
|
pnpm install -g @shipzone/npmci
|
||||||
|
npmci npm prepare
|
||||||
|
|
||||||
|
- name: Release
|
||||||
|
run: |
|
||||||
|
npmci node install stable
|
||||||
|
npmci npm publish
|
||||||
|
|
||||||
|
metadata:
|
||||||
|
needs: test
|
||||||
|
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
container:
|
||||||
|
image: ${{ env.IMAGE }}
|
||||||
|
continue-on-error: true
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v3
|
||||||
|
|
||||||
|
- name: Prepare
|
||||||
|
run: |
|
||||||
|
pnpm install -g pnpm
|
||||||
|
pnpm install -g @shipzone/npmci
|
||||||
|
npmci npm prepare
|
||||||
|
|
||||||
|
- name: Code quality
|
||||||
|
run: |
|
||||||
|
npmci command npm install -g typescript
|
||||||
|
npmci npm install
|
||||||
|
|
||||||
|
- name: Trigger
|
||||||
|
run: npmci trigger
|
||||||
|
|
||||||
|
- name: Build docs and upload artifacts
|
||||||
|
run: |
|
||||||
|
npmci node install stable
|
||||||
|
npmci npm install
|
||||||
|
pnpm install -g @gitzone/tsdoc
|
||||||
|
npmci command tsdoc
|
||||||
|
continue-on-error: true
|
@ -1,104 +0,0 @@
|
|||||||
name: CI Pipeline
|
|
||||||
|
|
||||||
on: push
|
|
||||||
|
|
||||||
env:
|
|
||||||
IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
|
|
||||||
security:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
continue-on-error: true
|
|
||||||
container:
|
|
||||||
image: ${{ env.IMAGE }}
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v3
|
|
||||||
|
|
||||||
- name: Install pnpm and npmci
|
|
||||||
run: |
|
|
||||||
pnpm install -g pnpm
|
|
||||||
pnpm install -g @shipzone/npmci
|
|
||||||
|
|
||||||
- name: Run npm prepare
|
|
||||||
run: npmci npm prepare
|
|
||||||
|
|
||||||
- name: Audit production dependencies
|
|
||||||
run: |
|
|
||||||
npmci command npm config set registry https://registry.npmjs.org
|
|
||||||
npmci command pnpm audit --audit-level=high --prod
|
|
||||||
|
|
||||||
- name: Audit development dependencies
|
|
||||||
run: |
|
|
||||||
npmci command npm config set registry https://registry.npmjs.org
|
|
||||||
npmci command pnpm audit --audit-level=high --dev
|
|
||||||
|
|
||||||
test:
|
|
||||||
needs: security
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
container:
|
|
||||||
image: ${{ env.IMAGE }}
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v3
|
|
||||||
|
|
||||||
- name: Test stable
|
|
||||||
run: |
|
|
||||||
npmci node install stable
|
|
||||||
npmci npm install
|
|
||||||
npmci npm test
|
|
||||||
|
|
||||||
- name: Test build
|
|
||||||
run: |
|
|
||||||
npmci node install stable
|
|
||||||
npmci npm install
|
|
||||||
npmci npm build
|
|
||||||
|
|
||||||
release:
|
|
||||||
needs: test
|
|
||||||
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
container:
|
|
||||||
image: ${{ env.IMAGE }}
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v3
|
|
||||||
|
|
||||||
- name: Release
|
|
||||||
run: |
|
|
||||||
npmci node install stable
|
|
||||||
npmci npm publish
|
|
||||||
|
|
||||||
metadata:
|
|
||||||
needs: test
|
|
||||||
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
container:
|
|
||||||
image: ${{ env.IMAGE }}
|
|
||||||
continue-on-error: true
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v3
|
|
||||||
|
|
||||||
- name: Code quality
|
|
||||||
run: |
|
|
||||||
npmci command npm install -g typescript
|
|
||||||
npmci npm prepare
|
|
||||||
npmci npm install
|
|
||||||
|
|
||||||
- name: Trigger
|
|
||||||
run: npmci trigger
|
|
||||||
|
|
||||||
- name: Build docs and upload artifacts
|
|
||||||
run: |
|
|
||||||
npmci node install stable
|
|
||||||
npmci npm install
|
|
||||||
npmci command npm run buildDocs
|
|
||||||
|
|
||||||
- name: Upload artifacts
|
|
||||||
uses: actions/upload-artifact@v2
|
|
||||||
with:
|
|
||||||
name: public
|
|
||||||
path: public
|
|
||||||
retention-days: 7
|
|
@ -13,10 +13,10 @@
|
|||||||
"projectType": "npm",
|
"projectType": "npm",
|
||||||
"module": {
|
"module": {
|
||||||
"githost": "gitlab.com",
|
"githost": "gitlab.com",
|
||||||
"gitscope": "pushrocks",
|
"gitscope": "push.rocks",
|
||||||
"gitrepo": "smartdata",
|
"gitrepo": "smartdata",
|
||||||
"description": "do more with data",
|
"description": "do more with data",
|
||||||
"npmPackagename": "@pushrocks/smartdata",
|
"npmPackagename": "@push.rocks/smartdata",
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
36
package.json
36
package.json
@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@pushrocks/smartdata",
|
"name": "@push.rocks/smartdata",
|
||||||
"version": "5.0.16",
|
"version": "5.0.26",
|
||||||
"private": false,
|
"private": false,
|
||||||
"description": "do more with data",
|
"description": "do more with data",
|
||||||
"main": "dist_ts/index.js",
|
"main": "dist_ts/index.js",
|
||||||
@ -22,26 +22,26 @@
|
|||||||
},
|
},
|
||||||
"homepage": "https://gitlab.com/pushrocks/smartdata#README",
|
"homepage": "https://gitlab.com/pushrocks/smartdata#README",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@pushrocks/lik": "^6.0.2",
|
"@push.rocks/lik": "^6.0.2",
|
||||||
"@pushrocks/smartdelay": "^3.0.1",
|
"@push.rocks/smartdelay": "^3.0.1",
|
||||||
"@pushrocks/smartlog": "^3.0.1",
|
"@push.rocks/smartlog": "^3.0.2",
|
||||||
"@pushrocks/smartmongo": "^2.0.7",
|
"@push.rocks/smartmongo": "^2.0.10",
|
||||||
"@pushrocks/smartpromise": "^4.0.2",
|
"@push.rocks/smartpromise": "^4.0.2",
|
||||||
"@pushrocks/smartrx": "^3.0.2",
|
"@push.rocks/smartrx": "^3.0.6",
|
||||||
"@pushrocks/smartstring": "^4.0.2",
|
"@push.rocks/smartstring": "^4.0.7",
|
||||||
"@pushrocks/smarttime": "^4.0.1",
|
"@push.rocks/smarttime": "^4.0.1",
|
||||||
"@pushrocks/smartunique": "^3.0.3",
|
"@push.rocks/smartunique": "^3.0.3",
|
||||||
"@pushrocks/taskbuffer": "^3.0.10",
|
"@push.rocks/taskbuffer": "^3.1.2",
|
||||||
"@tsclass/tsclass": "^4.0.42",
|
"@tsclass/tsclass": "^4.0.42",
|
||||||
"mongodb": "^5.6.0"
|
"mongodb": "^5.7.0"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@gitzone/tsbuild": "^2.1.66",
|
"@gitzone/tsbuild": "^2.1.66",
|
||||||
"@gitzone/tsrun": "^1.2.42",
|
"@gitzone/tsrun": "^1.2.44",
|
||||||
"@gitzone/tstest": "^1.0.74",
|
"@gitzone/tstest": "^1.0.77",
|
||||||
"@pushrocks/qenv": "^5.0.2",
|
"@push.rocks/qenv": "^6.0.2",
|
||||||
"@pushrocks/tapbundle": "^5.0.8",
|
"@push.rocks/tapbundle": "^5.0.15",
|
||||||
"@types/node": "^20.3.1",
|
"@types/node": "^20.4.10",
|
||||||
"@types/shortid": "0.0.29"
|
"@types/shortid": "0.0.29"
|
||||||
},
|
},
|
||||||
"files": [
|
"files": [
|
||||||
|
4211
pnpm-lock.yaml
generated
4211
pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
24
readme.md
24
readme.md
@ -1,26 +1,26 @@
|
|||||||
# @pushrocks/smartdata
|
# @push.rocks/smartdata
|
||||||
do more with data
|
do more with data
|
||||||
|
|
||||||
## Availabililty and Links
|
## Availabililty and Links
|
||||||
* [npmjs.org (npm package)](https://www.npmjs.com/package/@pushrocks/smartdata)
|
* [npmjs.org (npm package)](https://www.npmjs.com/package/@push.rocks/smartdata)
|
||||||
* [gitlab.com (source)](https://gitlab.com/pushrocks/smartdata)
|
* [gitlab.com (source)](https://gitlab.com/push.rocks/smartdata)
|
||||||
* [github.com (source mirror)](https://github.com/pushrocks/smartdata)
|
* [github.com (source mirror)](https://github.com/push.rocks/smartdata)
|
||||||
* [docs (typedoc)](https://pushrocks.gitlab.io/smartdata/)
|
* [docs (typedoc)](https://push.rocks.gitlab.io/smartdata/)
|
||||||
|
|
||||||
## Status for master
|
## Status for master
|
||||||
|
|
||||||
Status Category | Status Badge
|
Status Category | Status Badge
|
||||||
-- | --
|
-- | --
|
||||||
GitLab Pipelines | [](https://lossless.cloud)
|
GitLab Pipelines | [](https://lossless.cloud)
|
||||||
GitLab Pipline Test Coverage | [](https://lossless.cloud)
|
GitLab Pipline Test Coverage | [](https://lossless.cloud)
|
||||||
npm | [](https://lossless.cloud)
|
npm | [](https://lossless.cloud)
|
||||||
Snyk | [](https://lossless.cloud)
|
Snyk | [](https://lossless.cloud)
|
||||||
TypeScript Support | [](https://lossless.cloud)
|
TypeScript Support | [](https://lossless.cloud)
|
||||||
node Support | [](https://nodejs.org/dist/latest-v10.x/docs/api/)
|
node Support | [](https://nodejs.org/dist/latest-v10.x/docs/api/)
|
||||||
Code Style | [](https://lossless.cloud)
|
Code Style | [](https://lossless.cloud)
|
||||||
PackagePhobia (total standalone install weight) | [](https://lossless.cloud)
|
PackagePhobia (total standalone install weight) | [](https://lossless.cloud)
|
||||||
PackagePhobia (package size on registry) | [](https://lossless.cloud)
|
PackagePhobia (package size on registry) | [](https://lossless.cloud)
|
||||||
BundlePhobia (total size when bundled) | [](https://lossless.cloud)
|
BundlePhobia (total size when bundled) | [](https://lossless.cloud)
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
|
104
test/test.distributedcoordinator.ts
Normal file
104
test/test.distributedcoordinator.ts
Normal file
@ -0,0 +1,104 @@
|
|||||||
|
import { tap, expect } from '@push.rocks/tapbundle';
|
||||||
|
import * as smartmongo from '@push.rocks/smartmongo';
|
||||||
|
import type * as taskbuffer from '@push.rocks/taskbuffer';
|
||||||
|
|
||||||
|
import * as smartdata from '../ts/index.js';
|
||||||
|
import { SmartdataDistributedCoordinator, DistributedClass } from '../ts/smartdata.classes.distributedcoordinator.js'; // path might need adjusting
|
||||||
|
const totalInstances = 10;
|
||||||
|
|
||||||
|
// =======================================
|
||||||
|
// Connecting to the database server
|
||||||
|
// =======================================
|
||||||
|
|
||||||
|
let smartmongoInstance: smartmongo.SmartMongo;
|
||||||
|
let testDb: smartdata.SmartdataDb;
|
||||||
|
|
||||||
|
tap.test('should create a testinstance as database', async () => {
|
||||||
|
smartmongoInstance = await smartmongo.SmartMongo.createAndStart();
|
||||||
|
testDb = new smartdata.SmartdataDb(await smartmongoInstance.getMongoDescriptor());
|
||||||
|
await testDb.init();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should instantiate DistributedClass', async (tools) => {
|
||||||
|
const instance = new DistributedClass();
|
||||||
|
expect(instance).toBeInstanceOf(DistributedClass);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('DistributedClass should update the time', async (tools) => {
|
||||||
|
const distributedCoordinator = new SmartdataDistributedCoordinator(testDb);
|
||||||
|
await distributedCoordinator.start();
|
||||||
|
console.log('test: started');
|
||||||
|
const initialTime = distributedCoordinator.ownInstance.data.lastUpdated;
|
||||||
|
await distributedCoordinator.sendHeartbeat();
|
||||||
|
console.log('test: sent heartbeat');
|
||||||
|
const updatedTime = distributedCoordinator.ownInstance.data.lastUpdated;
|
||||||
|
expect(updatedTime).toBeGreaterThan(initialTime);
|
||||||
|
await distributedCoordinator.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should instantiate SmartdataDistributedCoordinator', async (tools) => {
|
||||||
|
const coordinator = new SmartdataDistributedCoordinator(testDb);
|
||||||
|
await coordinator.start();
|
||||||
|
expect(coordinator).toBeInstanceOf(SmartdataDistributedCoordinator);
|
||||||
|
await coordinator.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('SmartdataDistributedCoordinator should update leader status', async (tools) => {
|
||||||
|
const coordinator = new SmartdataDistributedCoordinator(testDb);
|
||||||
|
await coordinator.start();
|
||||||
|
await coordinator.checkAndMaybeLead();
|
||||||
|
expect(coordinator.ownInstance.data.elected).toBeOneOf([true, false]);
|
||||||
|
await coordinator.stop();
|
||||||
|
process.exit(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('SmartdataDistributedCoordinator should handle distributed task requests', async (tools) => {
|
||||||
|
const coordinator = new SmartdataDistributedCoordinator(testDb);
|
||||||
|
await coordinator.start();
|
||||||
|
|
||||||
|
const mockTaskRequest: taskbuffer.distributedCoordination.IDistributedTaskRequest = {
|
||||||
|
submitterId: "mockSubmitter12345", // Some unique mock submitter ID
|
||||||
|
taskName: "SampleTask",
|
||||||
|
taskVersion: "1.0.0", // Assuming it's a version string
|
||||||
|
taskExecutionTime: Date.now(),
|
||||||
|
taskExecutionTimeout: 60000, // Let's say the timeout is 1 minute (60000 ms)
|
||||||
|
taskExecutionParallel: 5, // Let's assume max 5 parallel executions
|
||||||
|
status: 'requesting'
|
||||||
|
};
|
||||||
|
|
||||||
|
const response = await coordinator.fireDistributedTaskRequest(mockTaskRequest);
|
||||||
|
expect(response).toBeTruthy(); // based on your expected structure for the response
|
||||||
|
await coordinator.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('SmartdataDistributedCoordinator should update distributed task requests', async (tools) => {
|
||||||
|
const coordinator = new SmartdataDistributedCoordinator(testDb);
|
||||||
|
|
||||||
|
await coordinator.start();
|
||||||
|
|
||||||
|
const mockTaskRequest: taskbuffer.distributedCoordination.IDistributedTaskRequest = {
|
||||||
|
submitterId: "mockSubmitter12345", // Some unique mock submitter ID
|
||||||
|
taskName: "SampleTask",
|
||||||
|
taskVersion: "1.0.0", // Assuming it's a version string
|
||||||
|
taskExecutionTime: Date.now(),
|
||||||
|
taskExecutionTimeout: 60000, // Let's say the timeout is 1 minute (60000 ms)
|
||||||
|
taskExecutionParallel: 5, // Let's assume max 5 parallel executions
|
||||||
|
status: 'requesting'
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
await coordinator.updateDistributedTaskRequest(mockTaskRequest);
|
||||||
|
// Here, we can potentially check if a DB entry got updated or some other side-effect of the update method.
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should elect only one leader amongst multiple instances', async (tools) => {
|
||||||
|
const coordinators = Array.from({ length: totalInstances }).map(() => new SmartdataDistributedCoordinator(testDb));
|
||||||
|
await Promise.all(coordinators.map(coordinator => coordinator.init()));
|
||||||
|
|
||||||
|
await Promise.all(coordinators.map(coordinator => coordinator.checkAndMaybeLead()));
|
||||||
|
|
||||||
|
const leaders = coordinators.filter(coordinator => coordinator.ownInstance.data.elected);
|
||||||
|
expect(leaders.length).toEqual(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.start({ throwOnError: true });
|
@ -1,6 +1,6 @@
|
|||||||
import { tap, expect } from '@pushrocks/tapbundle';
|
import { tap, expect } from '@push.rocks/tapbundle';
|
||||||
import { Qenv } from '@pushrocks/qenv';
|
import { Qenv } from '@push.rocks/qenv';
|
||||||
import * as smartmongo from '@pushrocks/smartmongo';
|
import * as smartmongo from '@push.rocks/smartmongo';
|
||||||
import { smartunique } from '../ts/smartdata.plugins.js';
|
import { smartunique } from '../ts/smartdata.plugins.js';
|
||||||
|
|
||||||
const testQenv = new Qenv(process.cwd(), process.cwd() + '/.nogit/');
|
const testQenv = new Qenv(process.cwd(), process.cwd() + '/.nogit/');
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
import { tap, expect } from '@pushrocks/tapbundle';
|
import { tap, expect } from '@push.rocks/tapbundle';
|
||||||
import { Qenv } from '@pushrocks/qenv';
|
import { Qenv } from '@push.rocks/qenv';
|
||||||
import * as smartmongo from '@pushrocks/smartmongo';
|
import * as smartmongo from '@push.rocks/smartmongo';
|
||||||
import { smartunique } from '../ts/smartdata.plugins.js';
|
import { smartunique } from '../ts/smartdata.plugins.js';
|
||||||
|
|
||||||
import * as mongodb from 'mongodb';
|
import * as mongodb from 'mongodb';
|
||||||
@ -30,7 +30,7 @@ tap.test('should create a testinstance as database', async () => {
|
|||||||
tap.skip.test('should connect to atlas', async (tools) => {
|
tap.skip.test('should connect to atlas', async (tools) => {
|
||||||
const databaseName = `test-smartdata-${smartunique.shortId()}`;
|
const databaseName = `test-smartdata-${smartunique.shortId()}`;
|
||||||
testDb = new smartdata.SmartdataDb({
|
testDb = new smartdata.SmartdataDb({
|
||||||
mongoDbUrl: testQenv.getEnvVarOnDemand('MONGO_URL'),
|
mongoDbUrl: await testQenv.getEnvVarOnDemand('MONGO_URL'),
|
||||||
mongoDbName: databaseName,
|
mongoDbName: databaseName,
|
||||||
});
|
});
|
||||||
await testDb.init();
|
await testDb.init();
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
import { tap, expect } from '@pushrocks/tapbundle';
|
import { tap, expect } from '@push.rocks/tapbundle';
|
||||||
import { Qenv } from '@pushrocks/qenv';
|
import { Qenv } from '@push.rocks/qenv';
|
||||||
import * as smartmongo from '@pushrocks/smartmongo';
|
import * as smartmongo from '@push.rocks/smartmongo';
|
||||||
import { smartunique } from '../ts/smartdata.plugins.js';
|
import { smartunique } from '../ts/smartdata.plugins.js';
|
||||||
|
|
||||||
const testQenv = new Qenv(process.cwd(), process.cwd() + '/.nogit/');
|
const testQenv = new Qenv(process.cwd(), process.cwd() + '/.nogit/');
|
||||||
@ -28,7 +28,7 @@ tap.test('should create a testinstance as database', async () => {
|
|||||||
tap.skip.test('should connect to atlas', async (tools) => {
|
tap.skip.test('should connect to atlas', async (tools) => {
|
||||||
const databaseName = `test-smartdata-${smartunique.shortId()}`;
|
const databaseName = `test-smartdata-${smartunique.shortId()}`;
|
||||||
testDb = new smartdata.SmartdataDb({
|
testDb = new smartdata.SmartdataDb({
|
||||||
mongoDbUrl: testQenv.getEnvVarOnDemand('MONGO_URL'),
|
mongoDbUrl: await testQenv.getEnvVarOnDemand('MONGO_URL'),
|
||||||
mongoDbName: databaseName,
|
mongoDbName: databaseName,
|
||||||
});
|
});
|
||||||
await testDb.init();
|
await testDb.init();
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
import { tap, expect } from '@pushrocks/tapbundle';
|
import { tap, expect } from '@push.rocks/tapbundle';
|
||||||
import { Qenv } from '@pushrocks/qenv';
|
import { Qenv } from '@push.rocks/qenv';
|
||||||
import * as smartmongo from '@pushrocks/smartmongo';
|
import * as smartmongo from '@push.rocks/smartmongo';
|
||||||
import { smartunique } from '../ts/smartdata.plugins.js';
|
import { smartunique } from '../ts/smartdata.plugins.js';
|
||||||
|
|
||||||
const testQenv = new Qenv(process.cwd(), process.cwd() + '/.nogit/');
|
const testQenv = new Qenv(process.cwd(), process.cwd() + '/.nogit/');
|
||||||
|
@ -2,7 +2,7 @@
|
|||||||
* autocreated commitinfo by @pushrocks/commitinfo
|
* autocreated commitinfo by @pushrocks/commitinfo
|
||||||
*/
|
*/
|
||||||
export const commitinfo = {
|
export const commitinfo = {
|
||||||
name: '@pushrocks/smartdata',
|
name: '@push.rocks/smartdata',
|
||||||
version: '5.0.16',
|
version: '5.0.26',
|
||||||
description: 'do more with data'
|
description: 'do more with data'
|
||||||
}
|
}
|
||||||
|
@ -1,5 +1,4 @@
|
|||||||
import * as plugins from './smartdata.plugins.js';
|
import * as plugins from './smartdata.plugins.js';
|
||||||
import { ObjectMap } from '@pushrocks/lik';
|
|
||||||
|
|
||||||
import { SmartdataCollection } from './smartdata.classes.collection.js';
|
import { SmartdataCollection } from './smartdata.classes.collection.js';
|
||||||
import { EasyStore } from './smartdata.classes.easystore.js';
|
import { EasyStore } from './smartdata.classes.easystore.js';
|
||||||
@ -17,7 +16,7 @@ export class SmartdataDb {
|
|||||||
mongoDb: plugins.mongodb.Db;
|
mongoDb: plugins.mongodb.Db;
|
||||||
status: TConnectionStatus;
|
status: TConnectionStatus;
|
||||||
statusConnectedDeferred = plugins.smartpromise.defer();
|
statusConnectedDeferred = plugins.smartpromise.defer();
|
||||||
smartdataCollectionMap = new ObjectMap<SmartdataCollection<any>>();
|
smartdataCollectionMap = new plugins.lik.ObjectMap<SmartdataCollection<any>>();
|
||||||
|
|
||||||
constructor(smartdataOptions: plugins.tsclass.database.IMongoDescriptor) {
|
constructor(smartdataOptions: plugins.tsclass.database.IMongoDescriptor) {
|
||||||
this.smartdataOptions = smartdataOptions;
|
this.smartdataOptions = smartdataOptions;
|
||||||
|
@ -4,7 +4,7 @@ import { Manager, setDefaultManagerForDoc } from './smartdata.classes.collection
|
|||||||
import { SmartDataDbDoc, svDb, unI } from './smartdata.classes.doc.js';
|
import { SmartDataDbDoc, svDb, unI } from './smartdata.classes.doc.js';
|
||||||
|
|
||||||
@Manager()
|
@Manager()
|
||||||
class DistributedClass extends SmartDataDbDoc<DistributedClass, DistributedClass> {
|
export class DistributedClass extends SmartDataDbDoc<DistributedClass, DistributedClass> {
|
||||||
// INSTANCE
|
// INSTANCE
|
||||||
@unI()
|
@unI()
|
||||||
public id: string;
|
public id: string;
|
||||||
@ -37,14 +37,13 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
|
|||||||
public readyPromise: Promise<any>;
|
public readyPromise: Promise<any>;
|
||||||
public db: SmartdataDb;
|
public db: SmartdataDb;
|
||||||
private asyncExecutionStack = new plugins.lik.AsyncExecutionStack();
|
private asyncExecutionStack = new plugins.lik.AsyncExecutionStack();
|
||||||
private ownInstance: DistributedClass;
|
public ownInstance: DistributedClass;
|
||||||
|
|
||||||
constructor(dbArg?: SmartdataDb) {
|
constructor(dbArg: SmartdataDb) {
|
||||||
super();
|
super();
|
||||||
this.db = dbArg;
|
this.db = dbArg;
|
||||||
setDefaultManagerForDoc(this, DistributedClass);
|
setDefaultManagerForDoc(this, DistributedClass);
|
||||||
this.readyPromise = this.db.statusConnectedDeferred.promise;
|
this.readyPromise = this.db.statusConnectedDeferred.promise;
|
||||||
this.init();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// smartdata specific stuff
|
// smartdata specific stuff
|
||||||
@ -64,15 +63,19 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
|
|||||||
|
|
||||||
private startHeartbeat = async () => {
|
private startHeartbeat = async () => {
|
||||||
while (this.ownInstance.data.status !== 'stopped') {
|
while (this.ownInstance.data.status !== 'stopped') {
|
||||||
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
await this.sendHeartbeat();
|
||||||
await this.ownInstance.updateFromDb();
|
|
||||||
this.ownInstance.data.lastUpdated = Date.now();
|
|
||||||
await this.ownInstance.save();
|
|
||||||
});
|
|
||||||
await plugins.smartdelay.delayFor(10000);
|
await plugins.smartdelay.delayFor(10000);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
public async init() {
|
|
||||||
|
public async sendHeartbeat() {
|
||||||
|
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
||||||
|
await this.ownInstance.updateFromDb();
|
||||||
|
this.ownInstance.data.lastUpdated = Date.now();
|
||||||
|
await this.ownInstance.save();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
private async init() {
|
||||||
await this.readyPromise;
|
await this.readyPromise;
|
||||||
if (!this.ownInstance) {
|
if (!this.ownInstance) {
|
||||||
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
||||||
|
@ -1,7 +1,5 @@
|
|||||||
import * as plugins from './smartdata.plugins.js';
|
import * as plugins from './smartdata.plugins.js';
|
||||||
|
|
||||||
import { ObjectMap } from '@pushrocks/lik';
|
|
||||||
|
|
||||||
import { SmartdataDb } from './smartdata.classes.db.js';
|
import { SmartdataDb } from './smartdata.classes.db.js';
|
||||||
import { SmartdataDbCursor } from './smartdata.classes.cursor.js';
|
import { SmartdataDbCursor } from './smartdata.classes.cursor.js';
|
||||||
import { type IManager, SmartdataCollection } from './smartdata.classes.collection.js';
|
import { type IManager, SmartdataCollection } from './smartdata.classes.collection.js';
|
||||||
@ -241,9 +239,9 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
|
|||||||
* also store any referenced objects to DB
|
* also store any referenced objects to DB
|
||||||
* better for data consistency
|
* better for data consistency
|
||||||
*/
|
*/
|
||||||
public saveDeep(savedMapArg: ObjectMap<SmartDataDbDoc<any, any>> = null) {
|
public saveDeep(savedMapArg: plugins.lik.ObjectMap<SmartDataDbDoc<any, any>> = null) {
|
||||||
if (!savedMapArg) {
|
if (!savedMapArg) {
|
||||||
savedMapArg = new ObjectMap<SmartDataDbDoc<any, any>>();
|
savedMapArg = new plugins.lik.ObjectMap<SmartDataDbDoc<any, any>>();
|
||||||
}
|
}
|
||||||
savedMapArg.add(this);
|
savedMapArg.add(this);
|
||||||
this.save();
|
this.save();
|
||||||
|
@ -4,16 +4,16 @@ import * as tsclass from '@tsclass/tsclass';
|
|||||||
export { tsclass };
|
export { tsclass };
|
||||||
|
|
||||||
// @pushrocks scope
|
// @pushrocks scope
|
||||||
import * as lik from '@pushrocks/lik';
|
import * as lik from '@push.rocks/lik';
|
||||||
import * as smartdelay from '@pushrocks/smartdelay';
|
import * as smartdelay from '@push.rocks/smartdelay';
|
||||||
import * as smartlog from '@pushrocks/smartlog';
|
import * as smartlog from '@push.rocks/smartlog';
|
||||||
import * as smartpromise from '@pushrocks/smartpromise';
|
import * as smartpromise from '@push.rocks/smartpromise';
|
||||||
import * as smartq from '@pushrocks/smartpromise';
|
import * as smartq from '@push.rocks/smartpromise';
|
||||||
import * as smartrx from '@pushrocks/smartrx';
|
import * as smartrx from '@push.rocks/smartrx';
|
||||||
import * as smartstring from '@pushrocks/smartstring';
|
import * as smartstring from '@push.rocks/smartstring';
|
||||||
import * as smarttime from '@pushrocks/smarttime';
|
import * as smarttime from '@push.rocks/smarttime';
|
||||||
import * as smartunique from '@pushrocks/smartunique';
|
import * as smartunique from '@push.rocks/smartunique';
|
||||||
import * as taskbuffer from '@pushrocks/taskbuffer';
|
import * as taskbuffer from '@push.rocks/taskbuffer';
|
||||||
import * as mongodb from 'mongodb';
|
import * as mongodb from 'mongodb';
|
||||||
|
|
||||||
export {
|
export {
|
||||||
|
Reference in New Issue
Block a user