Compare commits
192 Commits
Author | SHA1 | Date | |
---|---|---|---|
a2ccf15f69 | |||
84d48f1914 | |||
1e258e5ffb | |||
19d5f553b9 | |||
7a257ea925 | |||
2fa1e89f34 | |||
d6b3896dd3 | |||
49b11b17ce | |||
4ac8a4c0cd | |||
7f9983382a | |||
54f529b0a7 | |||
f542463bf6 | |||
1235ae2eb3 | |||
8166d2f7c2 | |||
7c9f27e02f | |||
842e4b280b | |||
009f3297b2 | |||
2ff3a4e0b7 | |||
0e55cd8876 | |||
eccdf3f00a | |||
c7544133d9 | |||
c7c9acf5bd | |||
c99ec50853 | |||
4dd9557e1d | |||
52b34a6da1 | |||
1bf74fe04d | |||
fdd875ad31 | |||
a7bf0c0298 | |||
59d6336e43 | |||
e0fc81179a | |||
5aa81a56a2 | |||
9ae26177b8 | |||
26ac52d6c5 | |||
fb39463b7d | |||
44acba80c1 | |||
8cf8315577 | |||
9b44b64a50 | |||
699e25201c | |||
2ef9aace68 | |||
cc55a57dfd | |||
b2df512552 | |||
23c62fbd69 | |||
5f70ea0b05 | |||
49a595876a | |||
db38a1ef85 | |||
94854638dd | |||
902fab4cc0 | |||
ed3b19abc5 | |||
5b88da7dce | |||
df273e9efa | |||
fd590e0be3 | |||
ef97b390d4 | |||
cd14eb8bf3 | |||
f48443dcd3 | |||
3f28ff80cb | |||
64005a0b32 | |||
8a77bb3281 | |||
25f50ecf51 | |||
ad87f8147b | |||
da0c9873eb | |||
2fcd3f1550 | |||
f726cf4c5b | |||
c198969fae | |||
be1badeb23 | |||
fe065b966f | |||
811e2490b8 | |||
206ccd40e9 | |||
055298172f | |||
278f3c8169 | |||
0709ba921b | |||
de1f1110b4 | |||
30f4254428 | |||
1c4b03e647 | |||
27cc7651ba | |||
355a2a3f2b | |||
a739582861 | |||
37f9a64735 | |||
83a5170591 | |||
f94363cf31 | |||
df02e5bb71 | |||
38e438c54f | |||
11bc1ac6dc | |||
3431e94ddd | |||
739e040776 | |||
28d57efd9e | |||
f50a61308c | |||
42aa9f9f8a | |||
3f591ff9d8 | |||
7b33347b4c | |||
3f11cbf595 | |||
cf1ec7f9eb | |||
54060deb8f | |||
48cffb5ac2 | |||
8301eb79a2 | |||
ad6366a294 | |||
cb6c03ebfe | |||
551916fe5c | |||
9b3892c1e8 | |||
c1b1af9c5d | |||
d3a3d5be9d | |||
c9a734d879 | |||
856e8e7d1f | |||
7a4d557724 | |||
7cbd0bd99b | |||
e10f6585a5 | |||
5c8dffdd9c | |||
846996eeac | |||
668df09ba0 | |||
03656f4ca0 | |||
c4c612f3a9 | |||
e357f7581c | |||
9697b1e48b | |||
aeb35705d4 | |||
236c8c6551 | |||
1f28db15e7 | |||
86d600e287 | |||
bb81530dac | |||
b9f9b36b87 | |||
df2fadfa01 | |||
8b2beb3485 | |||
144a620f43 | |||
c241247845 | |||
81e39d09e4 | |||
8e51b518b1 | |||
8308d8d03b | |||
97365ddf29 | |||
55d96fa68d | |||
54ec6accdf | |||
fc5d092b25 | |||
dfba057562 | |||
1ad05943b5 | |||
302e51a77f | |||
1330d03af2 | |||
a298e9d190 | |||
e571ef347b | |||
39a4c7ef3f | |||
0d3518d990 | |||
fbdde2268c | |||
8b6c26f45a | |||
97e82ed75a | |||
03baffd9fd | |||
f29ae67580 | |||
e550a8dbd6 | |||
cf6ef25a8d | |||
29c512b0cc | |||
105f69d1c9 | |||
4c375f8465 | |||
9466af6a4a | |||
c5aa747f42 | |||
b5f2474f65 | |||
85f0d99934 | |||
3b2d3d9072 | |||
3ff5c36fdf | |||
a5acc2fe4e | |||
9c81257101 | |||
f7342962f4 | |||
bcd10205d3 | |||
6cab20f32d | |||
5aaa6ad2d6 | |||
635256f2f6 | |||
f799d3efa5 | |||
f74020ba96 | |||
f6d6545ff5 | |||
85a196c8c1 | |||
adb198af01 | |||
6dce9f3ff8 | |||
2f6a4ce3e5 | |||
0984a1ade4 | |||
804701c96a | |||
a3759fa484 | |||
ef38df62be | |||
c17789e92e | |||
0bf2ba554d | |||
5cbf1a222a | |||
f075530838 | |||
efb83853fb | |||
73300ca4d3 | |||
1e946cdceb | |||
608ff93a41 | |||
6211953f14 | |||
99e520b776 | |||
eda8297356 | |||
ffa52a5883 | |||
1e83f0a0ef | |||
0203eabdfd | |||
72894e3ef1 | |||
bbe5f8c6a8 | |||
02af9f5c4b | |||
17de480272 | |||
776fd3ee4e | |||
3272bb7235 | |||
116dfbc3b0 |
66
.gitea/workflows/default_nottags.yaml
Normal file
66
.gitea/workflows/default_nottags.yaml
Normal file
@ -0,0 +1,66 @@
|
||||
name: Default (not tags)
|
||||
|
||||
on:
|
||||
push:
|
||||
tags-ignore:
|
||||
- '**'
|
||||
|
||||
env:
|
||||
IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
|
||||
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
|
||||
NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
|
||||
NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
|
||||
NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
|
||||
NPMCI_URL_CLOUDLY: ${{secrets.NPMCI_URL_CLOUDLY}}
|
||||
|
||||
jobs:
|
||||
security:
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true
|
||||
container:
|
||||
image: ${{ env.IMAGE }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Install pnpm and npmci
|
||||
run: |
|
||||
pnpm install -g pnpm
|
||||
pnpm install -g @shipzone/npmci
|
||||
|
||||
- name: Run npm prepare
|
||||
run: npmci npm prepare
|
||||
|
||||
- name: Audit production dependencies
|
||||
run: |
|
||||
npmci command npm config set registry https://registry.npmjs.org
|
||||
npmci command pnpm audit --audit-level=high --prod
|
||||
continue-on-error: true
|
||||
|
||||
- name: Audit development dependencies
|
||||
run: |
|
||||
npmci command npm config set registry https://registry.npmjs.org
|
||||
npmci command pnpm audit --audit-level=high --dev
|
||||
continue-on-error: true
|
||||
|
||||
test:
|
||||
if: ${{ always() }}
|
||||
needs: security
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: ${{ env.IMAGE }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Test stable
|
||||
run: |
|
||||
npmci node install stable
|
||||
npmci npm install
|
||||
npmci npm test
|
||||
|
||||
- name: Test build
|
||||
run: |
|
||||
npmci node install stable
|
||||
npmci npm install
|
||||
npmci npm build
|
124
.gitea/workflows/default_tags.yaml
Normal file
124
.gitea/workflows/default_tags.yaml
Normal file
@ -0,0 +1,124 @@
|
||||
name: Default (tags)
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
|
||||
env:
|
||||
IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
|
||||
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
|
||||
NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
|
||||
NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
|
||||
NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
|
||||
NPMCI_URL_CLOUDLY: ${{secrets.NPMCI_URL_CLOUDLY}}
|
||||
|
||||
jobs:
|
||||
security:
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true
|
||||
container:
|
||||
image: ${{ env.IMAGE }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Prepare
|
||||
run: |
|
||||
pnpm install -g pnpm
|
||||
pnpm install -g @shipzone/npmci
|
||||
npmci npm prepare
|
||||
|
||||
- name: Audit production dependencies
|
||||
run: |
|
||||
npmci command npm config set registry https://registry.npmjs.org
|
||||
npmci command pnpm audit --audit-level=high --prod
|
||||
continue-on-error: true
|
||||
|
||||
- name: Audit development dependencies
|
||||
run: |
|
||||
npmci command npm config set registry https://registry.npmjs.org
|
||||
npmci command pnpm audit --audit-level=high --dev
|
||||
continue-on-error: true
|
||||
|
||||
test:
|
||||
if: ${{ always() }}
|
||||
needs: security
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: ${{ env.IMAGE }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Prepare
|
||||
run: |
|
||||
pnpm install -g pnpm
|
||||
pnpm install -g @shipzone/npmci
|
||||
npmci npm prepare
|
||||
|
||||
- name: Test stable
|
||||
run: |
|
||||
npmci node install stable
|
||||
npmci npm install
|
||||
npmci npm test
|
||||
|
||||
- name: Test build
|
||||
run: |
|
||||
npmci node install stable
|
||||
npmci npm install
|
||||
npmci npm build
|
||||
|
||||
release:
|
||||
needs: test
|
||||
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: ${{ env.IMAGE }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Prepare
|
||||
run: |
|
||||
pnpm install -g pnpm
|
||||
pnpm install -g @shipzone/npmci
|
||||
npmci npm prepare
|
||||
|
||||
- name: Release
|
||||
run: |
|
||||
npmci node install stable
|
||||
npmci npm publish
|
||||
|
||||
metadata:
|
||||
needs: test
|
||||
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: ${{ env.IMAGE }}
|
||||
continue-on-error: true
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Prepare
|
||||
run: |
|
||||
pnpm install -g pnpm
|
||||
pnpm install -g @shipzone/npmci
|
||||
npmci npm prepare
|
||||
|
||||
- name: Code quality
|
||||
run: |
|
||||
npmci command npm install -g typescript
|
||||
npmci npm install
|
||||
|
||||
- name: Trigger
|
||||
run: npmci trigger
|
||||
|
||||
- name: Build docs and upload artifacts
|
||||
run: |
|
||||
npmci node install stable
|
||||
npmci npm install
|
||||
pnpm install -g @gitzone/tsdoc
|
||||
npmci command tsdoc
|
||||
continue-on-error: true
|
4
.gitignore
vendored
4
.gitignore
vendored
@ -15,8 +15,6 @@ node_modules/
|
||||
|
||||
# builds
|
||||
dist/
|
||||
dist_web/
|
||||
dist_serve/
|
||||
dist_ts_web/
|
||||
dist_*/
|
||||
|
||||
# custom
|
119
.gitlab-ci.yml
119
.gitlab-ci.yml
@ -1,119 +0,0 @@
|
||||
# gitzone ci_default
|
||||
image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
|
||||
|
||||
cache:
|
||||
paths:
|
||||
- .npmci_cache/
|
||||
key: "$CI_BUILD_STAGE"
|
||||
|
||||
stages:
|
||||
- security
|
||||
- test
|
||||
- release
|
||||
- metadata
|
||||
|
||||
# ====================
|
||||
# security stage
|
||||
# ====================
|
||||
mirror:
|
||||
stage: security
|
||||
script:
|
||||
- npmci git mirror
|
||||
tags:
|
||||
- docker
|
||||
- notpriv
|
||||
|
||||
snyk:
|
||||
stage: security
|
||||
script:
|
||||
- npmci npm prepare
|
||||
- npmci command npm install -g snyk
|
||||
- npmci command npm install --ignore-scripts
|
||||
- npmci command snyk test
|
||||
tags:
|
||||
- docker
|
||||
- notpriv
|
||||
|
||||
# ====================
|
||||
# test stage
|
||||
# ====================
|
||||
|
||||
testStable:
|
||||
stage: test
|
||||
script:
|
||||
- npmci npm prepare
|
||||
- npmci node install stable
|
||||
- npmci npm install
|
||||
- npmci npm test
|
||||
coverage: /\d+.?\d+?\%\s*coverage/
|
||||
tags:
|
||||
- docker
|
||||
- priv
|
||||
|
||||
testBuild:
|
||||
stage: test
|
||||
script:
|
||||
- npmci npm prepare
|
||||
- npmci node install lts
|
||||
- npmci npm install
|
||||
- npmci command npm run build
|
||||
coverage: /\d+.?\d+?\%\s*coverage/
|
||||
tags:
|
||||
- docker
|
||||
- notpriv
|
||||
|
||||
release:
|
||||
stage: release
|
||||
script:
|
||||
- npmci node install lts
|
||||
- npmci npm publish
|
||||
only:
|
||||
- tags
|
||||
tags:
|
||||
- docker
|
||||
- notpriv
|
||||
|
||||
# ====================
|
||||
# metadata stage
|
||||
# ====================
|
||||
codequality:
|
||||
stage: metadata
|
||||
allow_failure: true
|
||||
script:
|
||||
- npmci command npm install -g tslint typescript
|
||||
- npmci npm install
|
||||
- npmci command "tslint -c tslint.json ./ts/**/*.ts"
|
||||
tags:
|
||||
- docker
|
||||
- priv
|
||||
|
||||
trigger:
|
||||
stage: metadata
|
||||
script:
|
||||
- npmci trigger
|
||||
only:
|
||||
- tags
|
||||
tags:
|
||||
- docker
|
||||
- notpriv
|
||||
|
||||
pages:
|
||||
image: hosttoday/ht-docker-dbase:npmci
|
||||
services:
|
||||
- docker:stable-dind
|
||||
stage: metadata
|
||||
script:
|
||||
- npmci command npm install -g @gitzone/tsdoc
|
||||
- npmci npm prepare
|
||||
- npmci npm install
|
||||
- npmci command tsdoc
|
||||
tags:
|
||||
- docker
|
||||
- notpriv
|
||||
only:
|
||||
- tags
|
||||
artifacts:
|
||||
expire_in: 1 week
|
||||
paths:
|
||||
- public
|
||||
allow_failure: true
|
11
.vscode/launch.json
vendored
Normal file
11
.vscode/launch.json
vendored
Normal file
@ -0,0 +1,11 @@
|
||||
{
|
||||
"version": "0.2.0",
|
||||
"configurations": [
|
||||
{
|
||||
"command": "npm test",
|
||||
"name": "Run npm test",
|
||||
"request": "launch",
|
||||
"type": "node-terminal"
|
||||
}
|
||||
]
|
||||
}
|
26
.vscode/settings.json
vendored
Normal file
26
.vscode/settings.json
vendored
Normal file
@ -0,0 +1,26 @@
|
||||
{
|
||||
"json.schemas": [
|
||||
{
|
||||
"fileMatch": ["/npmextra.json"],
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"npmci": {
|
||||
"type": "object",
|
||||
"description": "settings for npmci"
|
||||
},
|
||||
"gitzone": {
|
||||
"type": "object",
|
||||
"description": "settings for gitzone",
|
||||
"properties": {
|
||||
"projectType": {
|
||||
"type": "string",
|
||||
"enum": ["website", "element", "service", "npm", "wcc"]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
@ -10,12 +10,13 @@
|
||||
"npmRegistryUrl": "registry.npmjs.org"
|
||||
},
|
||||
"gitzone": {
|
||||
"projectType": "npm",
|
||||
"module": {
|
||||
"githost": "gitlab.com",
|
||||
"gitscope": "pushrocks",
|
||||
"gitscope": "push.rocks",
|
||||
"gitrepo": "smartdata",
|
||||
"shortDescription": "do more with data",
|
||||
"npmPackagename": "@pushrocks/smartdata",
|
||||
"description": "do more with data",
|
||||
"npmPackagename": "@push.rocks/smartdata",
|
||||
"license": "MIT"
|
||||
}
|
||||
}
|
||||
|
2774
package-lock.json
generated
2774
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
69
package.json
69
package.json
@ -1,14 +1,15 @@
|
||||
{
|
||||
"name": "@pushrocks/smartdata",
|
||||
"version": "3.1.26",
|
||||
"name": "@push.rocks/smartdata",
|
||||
"version": "5.0.35",
|
||||
"private": false,
|
||||
"description": "do more with data",
|
||||
"main": "dist/index.js",
|
||||
"typings": "dist/index.d.ts",
|
||||
"main": "dist_ts/index.js",
|
||||
"typings": "dist_ts/index.d.ts",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"test": "(tstest test/)",
|
||||
"testLocal": "(npmdocker)",
|
||||
"build": "(tsbuild)"
|
||||
"test": "tstest test/",
|
||||
"build": "tsbuild --web --allowimplicitany",
|
||||
"buildDocs": "tsdoc"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
@ -21,38 +22,40 @@
|
||||
},
|
||||
"homepage": "https://gitlab.com/pushrocks/smartdata#README",
|
||||
"dependencies": {
|
||||
"@pushrocks/lik": "^3.0.19",
|
||||
"@pushrocks/smartlog": "^2.0.21",
|
||||
"@pushrocks/smartpromise": "^3.0.6",
|
||||
"@pushrocks/smartstring": "^3.0.18",
|
||||
"@pushrocks/smartunique": "^3.0.1",
|
||||
"@types/lodash": "^4.14.149",
|
||||
"@types/mongodb": "^3.3.16",
|
||||
"lodash": "^4.17.15",
|
||||
"mongodb": "^3.5.3",
|
||||
"runtime-type-checks": "0.0.4"
|
||||
"@push.rocks/lik": "^6.0.14",
|
||||
"@push.rocks/smartdelay": "^3.0.1",
|
||||
"@push.rocks/smartlog": "^3.0.2",
|
||||
"@push.rocks/smartmongo": "^2.0.10",
|
||||
"@push.rocks/smartpromise": "^4.0.2",
|
||||
"@push.rocks/smartrx": "^3.0.7",
|
||||
"@push.rocks/smartstring": "^4.0.15",
|
||||
"@push.rocks/smarttime": "^4.0.6",
|
||||
"@push.rocks/smartunique": "^3.0.8",
|
||||
"@push.rocks/taskbuffer": "^3.1.7",
|
||||
"@tsclass/tsclass": "^4.0.52",
|
||||
"mongodb": "^6.5.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@gitzone/tsbuild": "^2.1.17",
|
||||
"@gitzone/tstest": "^1.0.28",
|
||||
"@pushrocks/qenv": "^4.0.6",
|
||||
"@pushrocks/tapbundle": "^3.2.0",
|
||||
"@types/mongodb-memory-server": "^2.3.0",
|
||||
"@types/node": "^13.7.2",
|
||||
"@types/shortid": "0.0.29",
|
||||
"mongodb-memory-server": "^6.2.4",
|
||||
"tslint": "^6.0.0",
|
||||
"tslint-config-prettier": "^1.18.0"
|
||||
"@gitzone/tsbuild": "^2.1.66",
|
||||
"@gitzone/tsrun": "^1.2.44",
|
||||
"@gitzone/tstest": "^1.0.77",
|
||||
"@push.rocks/qenv": "^6.0.5",
|
||||
"@push.rocks/tapbundle": "^5.0.22",
|
||||
"@types/node": "^20.11.30"
|
||||
},
|
||||
"files": [
|
||||
"ts/*",
|
||||
"ts_web/*",
|
||||
"dist/*",
|
||||
"dist_web/*",
|
||||
"dist_ts_web/*",
|
||||
"assets/*",
|
||||
"ts/**/*",
|
||||
"ts_web/**/*",
|
||||
"dist/**/*",
|
||||
"dist_*/**/*",
|
||||
"dist_ts/**/*",
|
||||
"dist_ts_web/**/*",
|
||||
"assets/**/*",
|
||||
"cli.js",
|
||||
"npmextra.json",
|
||||
"readme.md"
|
||||
],
|
||||
"browserslist": [
|
||||
"last 1 chrome versions"
|
||||
]
|
||||
}
|
||||
|
6881
pnpm-lock.yaml
generated
Normal file
6881
pnpm-lock.yaml
generated
Normal file
File diff suppressed because it is too large
Load Diff
@ -1,20 +1,26 @@
|
||||
# @pushrocks/smartdata
|
||||
# @push.rocks/smartdata
|
||||
do more with data
|
||||
|
||||
## Availabililty and Links
|
||||
* [npmjs.org (npm package)](https://www.npmjs.com/package/@pushrocks/smartdata)
|
||||
* [gitlab.com (source)](https://gitlab.com/pushrocks/smartdata)
|
||||
* [github.com (source mirror)](https://github.com/pushrocks/smartdata)
|
||||
* [docs (typedoc)](https://pushrocks.gitlab.io/smartdata/)
|
||||
* [npmjs.org (npm package)](https://www.npmjs.com/package/@push.rocks/smartdata)
|
||||
* [gitlab.com (source)](https://gitlab.com/push.rocks/smartdata)
|
||||
* [github.com (source mirror)](https://github.com/push.rocks/smartdata)
|
||||
* [docs (typedoc)](https://push.rocks.gitlab.io/smartdata/)
|
||||
|
||||
## Status for master
|
||||
[](https://gitlab.com/pushrocks/smartdata/commits/master)
|
||||
[](https://gitlab.com/pushrocks/smartdata/commits/master)
|
||||
[](https://www.npmjs.com/package/@pushrocks/smartdata)
|
||||
[](https://snyk.io/test/npm/@pushrocks/smartdata)
|
||||
[](https://nodejs.org/dist/latest-v10.x/docs/api/)
|
||||
[](https://nodejs.org/dist/latest-v10.x/docs/api/)
|
||||
[](https://prettier.io/)
|
||||
|
||||
Status Category | Status Badge
|
||||
-- | --
|
||||
GitLab Pipelines | [](https://lossless.cloud)
|
||||
GitLab Pipline Test Coverage | [](https://lossless.cloud)
|
||||
npm | [](https://lossless.cloud)
|
||||
Snyk | [](https://lossless.cloud)
|
||||
TypeScript Support | [](https://lossless.cloud)
|
||||
node Support | [](https://nodejs.org/dist/latest-v10.x/docs/api/)
|
||||
Code Style | [](https://lossless.cloud)
|
||||
PackagePhobia (total standalone install weight) | [](https://lossless.cloud)
|
||||
PackagePhobia (package size on registry) | [](https://lossless.cloud)
|
||||
BundlePhobia (total size when bundled) | [](https://lossless.cloud)
|
||||
|
||||
## Usage
|
||||
|
||||
@ -42,15 +48,16 @@ How RethinkDB's terms map to the ones of smartdata:
|
||||
represents a Database. Naturally it has .connect() etc. methods on it.
|
||||
|
||||
```typescript
|
||||
// Assuming toplevel await
|
||||
import * as smartdata from 'smartdata';
|
||||
|
||||
const smartdataDb = new smartdata.SmartdataDb({
|
||||
mongoDbUrl: '//someurl',
|
||||
mongoDbName: 'myDatabase',
|
||||
mongoDbPass: 'mypassword'
|
||||
mongoDbPass: 'mypassword',
|
||||
});
|
||||
|
||||
smartdataDb.connect();
|
||||
await smartdataDb.connect();
|
||||
```
|
||||
|
||||
### class DbCollection
|
||||
@ -61,16 +68,17 @@ A collection is defined by the object class (that is extending smartdata.dbdoc)
|
||||
So to get to get access to a specific collection you document
|
||||
|
||||
```typescript
|
||||
// Assuming toplevel await
|
||||
// continues from the block before...
|
||||
|
||||
@smartdata.Collection(smartdataDb)
|
||||
class MyObject extends smartdata.DbDoc<MyObject> {
|
||||
class MyObject extends smartdata.DbDoc<MyObject /* ,[an optional interface to implement] */> {
|
||||
// read the next block about DbDoc
|
||||
@smartdata.svDb()
|
||||
property1: string; // @smartdata.svDb() marks the property for db save
|
||||
|
||||
|
||||
property2: number; // this one is not marked, so it won't be save upon calling this.save()
|
||||
|
||||
|
||||
constructor() {
|
||||
super(); // the super call is important ;) But you probably know that.
|
||||
}
|
||||
@ -80,14 +88,22 @@ class MyObject extends smartdata.DbDoc<MyObject> {
|
||||
|
||||
const localObject = new MyObject({
|
||||
property1: 'hi',
|
||||
property2: 2
|
||||
property2: {
|
||||
deep: 3,
|
||||
},
|
||||
});
|
||||
localObject.save(); // saves the object to the database
|
||||
await localObject.save(); // saves the object to the database
|
||||
|
||||
// start retrieving instances
|
||||
|
||||
MyObject.getInstance<MyObject>({
|
||||
property: 'hi'
|
||||
// .getInstance is staticly inheritied, yet fully typed static function to get instances with fully typed filters
|
||||
const myInstance = await MyObject.getInstance({
|
||||
property1: 'hi',
|
||||
property2: {
|
||||
deep: {
|
||||
$gt: 2,
|
||||
} as any,
|
||||
},
|
||||
}); // outputs a new instance of MyObject with the values from db assigned
|
||||
```
|
||||
|
||||
@ -97,7 +113,8 @@ represents a individual document in a collection
|
||||
and thereby is ideally suited to extend the class you want to actually store.
|
||||
|
||||
### CRUD operations
|
||||
smartdata supports full CRUD operations
|
||||
|
||||
smartdata supports full CRUD operations
|
||||
|
||||
**Store** or **Update** instances of classes to MongoDB:
|
||||
DbDoc extends your class with the following methods:
|
||||
@ -111,16 +128,13 @@ DbDoc extends your class with the following methods:
|
||||
**Get** a new class instance from MongoDB:
|
||||
DbDoc exposes a static method that allows you specify a filter to retrieve a cloned class of the one you used to that doc at some point later in time:
|
||||
|
||||
* static async `.getInstance({ /* filter props here */ })` gets you an instance that has the data of the first matched document as properties.
|
||||
* static async `getInstances({ /* filter props here */ })` get you an array instances (one instance for every matched document).
|
||||
- static async `.getInstance({ /* filter props here */ })` gets you an instance that has the data of the first matched document as properties.
|
||||
- static async `getInstances({ /* filter props here */ })` get you an array instances (one instance for every matched document).
|
||||
|
||||
**Delete** instances from MongoDb:
|
||||
smartdata extends your class with a method to easily delete the doucment from DB:
|
||||
|
||||
* async `.delete()`will delete the document from DB.
|
||||
|
||||
|
||||
|
||||
- async `.delete()`will delete the document from DB.
|
||||
|
||||
## TypeScript
|
||||
|
||||
@ -129,9 +143,12 @@ Since you define your classes in TypeScript and types flow through smartdata in
|
||||
you should get all the Intellisense and type checking you love when using smartdata.
|
||||
smartdata itself also bundles typings. You don't need to install any additional types for smartdata.
|
||||
|
||||
## Contribution
|
||||
|
||||
We are always happy for code contributions. If you are not the code contributing type that is ok. Still, maintaining Open Source repositories takes considerable time and thought. If you like the quality of what we do and our modules are useful to you we would appreciate a little monthly contribution: You can [contribute one time](https://lossless.link/contribute-onetime) or [contribute monthly](https://lossless.link/contribute). :)
|
||||
|
||||
For further information read the linked docs at the top of this readme.
|
||||
|
||||
> MIT licensed | **©** [Lossless GmbH](https://lossless.gmbh)
|
||||
## Legal
|
||||
> MIT licensed | **©** [Task Venture Capital GmbH](https://task.vc)
|
||||
| By using this npm module you agree to our [privacy policy](https://lossless.gmbH/privacy)
|
||||
|
||||
[](https://maintainedby.lossless.com)
|
112
test/test.distributedcoordinator.ts
Normal file
112
test/test.distributedcoordinator.ts
Normal file
@ -0,0 +1,112 @@
|
||||
import { tap, expect } from '@push.rocks/tapbundle';
|
||||
import * as smartmongo from '@push.rocks/smartmongo';
|
||||
import type * as taskbuffer from '@push.rocks/taskbuffer';
|
||||
|
||||
import * as smartdata from '../ts/index.js';
|
||||
import { SmartdataDistributedCoordinator, DistributedClass } from '../ts/smartdata.classes.distributedcoordinator.js'; // path might need adjusting
|
||||
const totalInstances = 10;
|
||||
|
||||
// =======================================
|
||||
// Connecting to the database server
|
||||
// =======================================
|
||||
|
||||
let smartmongoInstance: smartmongo.SmartMongo;
|
||||
let testDb: smartdata.SmartdataDb;
|
||||
|
||||
tap.test('should create a testinstance as database', async () => {
|
||||
smartmongoInstance = await smartmongo.SmartMongo.createAndStart();
|
||||
testDb = new smartdata.SmartdataDb(await smartmongoInstance.getMongoDescriptor());
|
||||
await testDb.init();
|
||||
});
|
||||
|
||||
tap.test('should instantiate DistributedClass', async (tools) => {
|
||||
const instance = new DistributedClass();
|
||||
expect(instance).toBeInstanceOf(DistributedClass);
|
||||
});
|
||||
|
||||
tap.test('DistributedClass should update the time', async (tools) => {
|
||||
const distributedCoordinator = new SmartdataDistributedCoordinator(testDb);
|
||||
await distributedCoordinator.start();
|
||||
const initialTime = distributedCoordinator.ownInstance.data.lastUpdated;
|
||||
await distributedCoordinator.sendHeartbeat();
|
||||
const updatedTime = distributedCoordinator.ownInstance.data.lastUpdated;
|
||||
expect(updatedTime).toBeGreaterThan(initialTime);
|
||||
await distributedCoordinator.stop();
|
||||
});
|
||||
|
||||
tap.test('should instantiate SmartdataDistributedCoordinator', async (tools) => {
|
||||
const distributedCoordinator = new SmartdataDistributedCoordinator(testDb);
|
||||
await distributedCoordinator.start();
|
||||
expect(distributedCoordinator).toBeInstanceOf(SmartdataDistributedCoordinator);
|
||||
await distributedCoordinator.stop();
|
||||
});
|
||||
|
||||
tap.test('SmartdataDistributedCoordinator should update leader status', async (tools) => {
|
||||
const distributedCoordinator = new SmartdataDistributedCoordinator(testDb);
|
||||
await distributedCoordinator.start();
|
||||
await distributedCoordinator.checkAndMaybeLead();
|
||||
expect(distributedCoordinator.ownInstance.data.elected).toBeOneOf([true, false]);
|
||||
await distributedCoordinator.stop();
|
||||
});
|
||||
|
||||
tap.test('SmartdataDistributedCoordinator should handle distributed task requests', async (tools) => {
|
||||
const distributedCoordinator = new SmartdataDistributedCoordinator(testDb);
|
||||
await distributedCoordinator.start();
|
||||
|
||||
const mockTaskRequest: taskbuffer.distributedCoordination.IDistributedTaskRequest = {
|
||||
submitterId: "mockSubmitter12345", // Some unique mock submitter ID
|
||||
requestResponseId: 'uni879873462hjhfkjhsdf', // Some unique ID for the request-response
|
||||
taskName: "SampleTask",
|
||||
taskVersion: "1.0.0", // Assuming it's a version string
|
||||
taskExecutionTime: Date.now(),
|
||||
taskExecutionTimeout: 60000, // Let's say the timeout is 1 minute (60000 ms)
|
||||
taskExecutionParallel: 5, // Let's assume max 5 parallel executions
|
||||
status: 'requesting'
|
||||
};
|
||||
|
||||
const response = await distributedCoordinator.fireDistributedTaskRequest(mockTaskRequest);
|
||||
console.log(response) // based on your expected structure for the response
|
||||
await distributedCoordinator.stop();
|
||||
});
|
||||
|
||||
tap.test('SmartdataDistributedCoordinator should update distributed task requests', async (tools) => {
|
||||
const distributedCoordinator = new SmartdataDistributedCoordinator(testDb);
|
||||
|
||||
await distributedCoordinator.start();
|
||||
|
||||
const mockTaskRequest: taskbuffer.distributedCoordination.IDistributedTaskRequest = {
|
||||
submitterId: "mockSubmitter12345", // Some unique mock submitter ID
|
||||
requestResponseId: 'uni879873462hjhfkjhsdf', // Some unique ID for the request-response
|
||||
taskName: "SampleTask",
|
||||
taskVersion: "1.0.0", // Assuming it's a version string
|
||||
taskExecutionTime: Date.now(),
|
||||
taskExecutionTimeout: 60000, // Let's say the timeout is 1 minute (60000 ms)
|
||||
taskExecutionParallel: 5, // Let's assume max 5 parallel executions
|
||||
status: 'requesting'
|
||||
};
|
||||
|
||||
|
||||
await distributedCoordinator.updateDistributedTaskRequest(mockTaskRequest);
|
||||
// Here, we can potentially check if a DB entry got updated or some other side-effect of the update method.
|
||||
await distributedCoordinator.stop();
|
||||
});
|
||||
|
||||
tap.test('should elect only one leader amongst multiple instances', async (tools) => {
|
||||
const coordinators = Array.from({ length: totalInstances }).map(() => new SmartdataDistributedCoordinator(testDb));
|
||||
await Promise.all(coordinators.map(coordinator => coordinator.start()));
|
||||
const leaders = coordinators.filter(coordinator => coordinator.ownInstance.data.elected);
|
||||
for (const leader of leaders) {
|
||||
console.log(leader.ownInstance);
|
||||
}
|
||||
expect(leaders.length).toEqual(1);
|
||||
|
||||
// stopping clears a coordinator from being elected.
|
||||
await Promise.all(coordinators.map(coordinator => coordinator.stop()));
|
||||
});
|
||||
|
||||
tap.test('should clean up', async () => {
|
||||
await smartmongoInstance.stopAndDumpToDir(`.nogit/testdata/`);
|
||||
setTimeout(() => process.exit(), 2000);
|
||||
})
|
||||
|
||||
tap.start({ throwOnError: true });
|
55
test/test.easystore.ts
Normal file
55
test/test.easystore.ts
Normal file
@ -0,0 +1,55 @@
|
||||
import { tap, expect } from '@push.rocks/tapbundle';
|
||||
import { Qenv } from '@push.rocks/qenv';
|
||||
import * as smartmongo from '@push.rocks/smartmongo';
|
||||
import { smartunique } from '../ts/smartdata.plugins.js';
|
||||
|
||||
const testQenv = new Qenv(process.cwd(), process.cwd() + '/.nogit/');
|
||||
|
||||
console.log(process.memoryUsage());
|
||||
|
||||
// the tested module
|
||||
import * as smartdata from '../ts/index.js';
|
||||
|
||||
// =======================================
|
||||
// Connecting to the database server
|
||||
// =======================================
|
||||
|
||||
let smartmongoInstance: smartmongo.SmartMongo;
|
||||
let testDb: smartdata.SmartdataDb;
|
||||
|
||||
tap.test('should create a testinstance as database', async () => {
|
||||
smartmongoInstance = await smartmongo.SmartMongo.createAndStart();
|
||||
testDb = new smartdata.SmartdataDb(await smartmongoInstance.getMongoDescriptor());
|
||||
await testDb.init();
|
||||
});
|
||||
|
||||
tap.skip.test('should connect to atlas', async (tools) => {
|
||||
const databaseName = `test-smartdata-${smartunique.shortId()}`;
|
||||
testDb = new smartdata.SmartdataDb({
|
||||
mongoDbUrl: await testQenv.getEnvVarOnDemand('MONGO_URL'),
|
||||
mongoDbName: databaseName,
|
||||
});
|
||||
await testDb.init();
|
||||
});
|
||||
|
||||
let easyStore: smartdata.EasyStore<{
|
||||
key1: string;
|
||||
key2: string;
|
||||
}>;
|
||||
|
||||
tap.test('should create an easystore', async () => {
|
||||
easyStore = await testDb.createEasyStore('hellothere');
|
||||
await easyStore.writeKey('key1', 'hello');
|
||||
const retrievedKey = await easyStore.readKey('key1');
|
||||
expect(retrievedKey).toEqual('hello');
|
||||
});
|
||||
|
||||
tap.test('close', async () => {
|
||||
await testDb.mongoDb.dropDatabase();
|
||||
await testDb.close();
|
||||
if (smartmongoInstance) {
|
||||
await smartmongoInstance.stop();
|
||||
}
|
||||
});
|
||||
|
||||
tap.start();
|
164
test/test.ts
164
test/test.ts
@ -1,35 +1,38 @@
|
||||
import { tap, expect } from '@pushrocks/tapbundle';
|
||||
import { Qenv } from '@pushrocks/qenv';
|
||||
import { tap, expect } from '@push.rocks/tapbundle';
|
||||
import { Qenv } from '@push.rocks/qenv';
|
||||
import * as smartmongo from '@push.rocks/smartmongo';
|
||||
import { smartunique } from '../ts/smartdata.plugins.js';
|
||||
|
||||
import * as mongodb from 'mongodb';
|
||||
|
||||
const testQenv = new Qenv(process.cwd(), process.cwd() + '/.nogit/');
|
||||
|
||||
// the tested module
|
||||
import * as smartdata from '../ts/index';
|
||||
import { smartstring } from '../ts/smartdata.plugins';
|
||||
import * as smartunique from '@pushrocks/smartunique';
|
||||
console.log(process.memoryUsage());
|
||||
|
||||
import * as mongoPlugin from 'mongodb-memory-server';
|
||||
// the tested module
|
||||
import * as smartdata from '../ts/index.js';
|
||||
|
||||
// =======================================
|
||||
// Connecting to the database server
|
||||
// =======================================
|
||||
|
||||
let smartmongoInstance: smartmongo.SmartMongo;
|
||||
let testDb: smartdata.SmartdataDb;
|
||||
let smartdataOptions: smartdata.ISmartdataOptions;
|
||||
let mongod: mongoPlugin.MongoMemoryServer;
|
||||
|
||||
const totalCars = 2000;
|
||||
|
||||
tap.test('should create a testinstance as database', async () => {
|
||||
mongod = new mongoPlugin.MongoMemoryServer();
|
||||
smartdataOptions = {
|
||||
mongoDbName: await mongod.getDbName(),
|
||||
mongoDbPass: '',
|
||||
mongoDbUrl: await mongod.getConnectionString()
|
||||
};
|
||||
console.log(smartdataOptions);
|
||||
testDb = new smartdata.SmartdataDb(smartdataOptions);
|
||||
smartmongoInstance = await smartmongo.SmartMongo.createAndStart();
|
||||
testDb = new smartdata.SmartdataDb(await smartmongoInstance.getMongoDescriptor());
|
||||
await testDb.init();
|
||||
});
|
||||
|
||||
tap.test('should establish a connection to the rethink Db cluster', async () => {
|
||||
tap.skip.test('should connect to atlas', async (tools) => {
|
||||
const databaseName = `test-smartdata-${smartunique.shortId()}`;
|
||||
testDb = new smartdata.SmartdataDb({
|
||||
mongoDbUrl: await testQenv.getEnvVarOnDemand('MONGO_URL'),
|
||||
mongoDbName: databaseName,
|
||||
});
|
||||
await testDb.init();
|
||||
});
|
||||
|
||||
@ -54,6 +57,14 @@ class Car extends smartdata.SmartDataDbDoc<Car, Car> {
|
||||
@smartdata.svDb()
|
||||
public brand: string;
|
||||
|
||||
@smartdata.svDb()
|
||||
public testBuffer = Buffer.from('hello');
|
||||
|
||||
@smartdata.svDb()
|
||||
deepData = {
|
||||
sodeep: 'yes',
|
||||
};
|
||||
|
||||
constructor(colorArg: string, brandArg: string) {
|
||||
super();
|
||||
this.color = colorArg;
|
||||
@ -61,48 +72,103 @@ class Car extends smartdata.SmartDataDbDoc<Car, Car> {
|
||||
}
|
||||
}
|
||||
|
||||
tap.test('should save the car to the db', async () => {
|
||||
tap.test('should save the car to the db', async (toolsArg) => {
|
||||
const myCar = new Car('red', 'Volvo');
|
||||
await myCar.save();
|
||||
|
||||
const myCar2 = new Car('red', 'Volvo');
|
||||
await myCar2.save();
|
||||
|
||||
const myCar3 = new Car('red', 'Renault');
|
||||
await myCar3.save();
|
||||
let counter = 0;
|
||||
|
||||
const gottenCarInstance = await Car.getInstance({});
|
||||
console.log(gottenCarInstance.testBuffer instanceof mongodb.Binary);
|
||||
process.memoryUsage();
|
||||
do {
|
||||
const myCar3 = new Car('red', 'Renault');
|
||||
await myCar3.save();
|
||||
counter++;
|
||||
if (counter % 100 === 0) {
|
||||
console.log(
|
||||
`Filled database with ${counter} of ${totalCars} Cars and memory usage ${
|
||||
process.memoryUsage().rss / 1e6
|
||||
} MB`
|
||||
);
|
||||
}
|
||||
} while (counter < totalCars);
|
||||
console.log(process.memoryUsage());
|
||||
});
|
||||
|
||||
tap.test('expect to get instance of Car', async () => {
|
||||
const myCars = await Car.getInstances<Car>({
|
||||
brand: 'Volvo'
|
||||
});
|
||||
expect(myCars[0].color).to.equal('red');
|
||||
tap.test('expect to get instance of Car with shallow match', async () => {
|
||||
const totalQueryCycles = totalCars / 2;
|
||||
let counter = 0;
|
||||
do {
|
||||
const timeStart = Date.now();
|
||||
const myCars = await Car.getInstances({
|
||||
brand: 'Renault',
|
||||
});
|
||||
if (counter % 10 === 0) {
|
||||
console.log(
|
||||
`performed ${counter} of ${totalQueryCycles} total query cycles: took ${
|
||||
Date.now() - timeStart
|
||||
}ms to query a set of 2000 with memory footprint ${process.memoryUsage().rss / 1e6} MB`
|
||||
);
|
||||
}
|
||||
expect(myCars[0].deepData.sodeep).toEqual('yes');
|
||||
expect(myCars[0].brand).toEqual('Renault');
|
||||
counter++;
|
||||
} while (counter < totalQueryCycles);
|
||||
});
|
||||
|
||||
tap.test('expect to get instance of Car with deep match', async () => {
|
||||
const totalQueryCycles = totalCars / 6;
|
||||
let counter = 0;
|
||||
do {
|
||||
const timeStart = Date.now();
|
||||
const myCars2 = await Car.getInstances({
|
||||
deepData: {
|
||||
sodeep: 'yes',
|
||||
},
|
||||
});
|
||||
if (counter % 10 === 0) {
|
||||
console.log(
|
||||
`performed ${counter} of ${totalQueryCycles} total query cycles: took ${
|
||||
Date.now() - timeStart
|
||||
}ms to deep query a set of 2000 with memory footprint ${process.memoryUsage().rss / 1e6} MB`
|
||||
);
|
||||
}
|
||||
expect(myCars2[0].deepData.sodeep).toEqual('yes');
|
||||
expect(myCars2[0].brand).toEqual('Volvo');
|
||||
counter++;
|
||||
} while (counter < totalQueryCycles);
|
||||
});
|
||||
|
||||
tap.test('expect to get instance of Car and update it', async () => {
|
||||
const myCar = await Car.getInstance<Car>({
|
||||
brand: 'Volvo'
|
||||
brand: 'Volvo',
|
||||
});
|
||||
expect(myCar.color).to.equal('red');
|
||||
expect(myCar.color).toEqual('red');
|
||||
myCar.color = 'blue';
|
||||
await myCar.save();
|
||||
});
|
||||
|
||||
tap.test('should be able to delete an instance of car', async () => {
|
||||
const myCar = await Car.getInstance<Car>({
|
||||
brand: 'Volvo'
|
||||
const myCars = await Car.getInstances({
|
||||
brand: 'Volvo',
|
||||
color: 'blue',
|
||||
});
|
||||
expect(myCar.color).to.equal('blue');
|
||||
await myCar.delete();
|
||||
console.log(myCars);
|
||||
expect(myCars[0].color).toEqual('blue');
|
||||
for (const myCar of myCars) {
|
||||
await myCar.delete();
|
||||
}
|
||||
|
||||
const myCar2 = await Car.getInstance<Car>({
|
||||
brand: 'Volvo'
|
||||
brand: 'Volvo',
|
||||
});
|
||||
expect(myCar2.color).to.equal('red');
|
||||
expect(myCar2.color).toEqual('red');
|
||||
});
|
||||
|
||||
|
||||
|
||||
// tslint:disable-next-line: max-classes-per-file
|
||||
@smartdata.Collection(() => {
|
||||
return testDb;
|
||||
@ -127,21 +193,31 @@ class Truck extends smartdata.SmartDataDbDoc<Car, Car> {
|
||||
tap.test('should store a new Truck', async () => {
|
||||
const truck = new Truck('blue', 'MAN');
|
||||
await truck.save();
|
||||
const myTruck = await Truck.getInstance<Truck>({color: 'blue'});
|
||||
myTruck.id = 'foo';
|
||||
await myTruck.save();
|
||||
const myTruck2 = await Truck.getInstance<Truck>({color: 'blue'});
|
||||
console.log(myTruck2);
|
||||
|
||||
const myTruck2 = await Truck.getInstance({ color: 'blue' });
|
||||
myTruck2.color = 'red';
|
||||
await myTruck2.save();
|
||||
const myTruck3 = await Truck.getInstance({ color: 'blue' });
|
||||
console.log(myTruck3);
|
||||
});
|
||||
|
||||
tap.test('should use a cursor', async () => {
|
||||
const cursor = await Car.getCursor({});
|
||||
let counter = 0;
|
||||
await cursor.forEach(async (carArg) => {
|
||||
counter++;
|
||||
counter % 50 === 0 ? console.log(`50 more of ${carArg.color}`) : null;
|
||||
});
|
||||
});
|
||||
|
||||
// =======================================
|
||||
// close the database connection
|
||||
// =======================================
|
||||
tap.test('should close the database connection', async tools => {
|
||||
tap.test('close', async () => {
|
||||
await testDb.mongoDb.dropDatabase();
|
||||
await testDb.close();
|
||||
await mongod.stop();
|
||||
if (smartmongoInstance) {
|
||||
await smartmongoInstance.stop();
|
||||
}
|
||||
});
|
||||
|
||||
tap.start({ throwOnError: true });
|
||||
|
95
test/test.typescript.ts
Normal file
95
test/test.typescript.ts
Normal file
@ -0,0 +1,95 @@
|
||||
import { tap, expect } from '@push.rocks/tapbundle';
|
||||
import { Qenv } from '@push.rocks/qenv';
|
||||
import * as smartmongo from '@push.rocks/smartmongo';
|
||||
import { smartunique } from '../ts/smartdata.plugins.js';
|
||||
|
||||
const testQenv = new Qenv(process.cwd(), process.cwd() + '/.nogit/');
|
||||
|
||||
console.log(process.memoryUsage());
|
||||
|
||||
// the tested module
|
||||
import * as smartdata from '../ts/index.js';
|
||||
|
||||
// =======================================
|
||||
// Connecting to the database server
|
||||
// =======================================
|
||||
|
||||
let smartmongoInstance: smartmongo.SmartMongo;
|
||||
let testDb: smartdata.SmartdataDb;
|
||||
|
||||
const totalCars = 2000;
|
||||
|
||||
tap.test('should create a testinstance as database', async () => {
|
||||
smartmongoInstance = await smartmongo.SmartMongo.createAndStart();
|
||||
testDb = new smartdata.SmartdataDb(await smartmongoInstance.getMongoDescriptor());
|
||||
await testDb.init();
|
||||
});
|
||||
|
||||
tap.skip.test('should connect to atlas', async (tools) => {
|
||||
const databaseName = `test-smartdata-${smartunique.shortId()}`;
|
||||
testDb = new smartdata.SmartdataDb({
|
||||
mongoDbUrl: await testQenv.getEnvVarOnDemand('MONGO_URL'),
|
||||
mongoDbName: databaseName,
|
||||
});
|
||||
await testDb.init();
|
||||
});
|
||||
|
||||
// =======================================
|
||||
// The actual tests
|
||||
// =======================================
|
||||
|
||||
// ------
|
||||
// Collections
|
||||
// ------
|
||||
@smartdata.Manager()
|
||||
class Car extends smartdata.SmartDataDbDoc<Car, Car> {
|
||||
@smartdata.unI()
|
||||
public index: string = smartunique.shortId();
|
||||
|
||||
@smartdata.svDb()
|
||||
public color: string;
|
||||
|
||||
@smartdata.svDb()
|
||||
public brand: string;
|
||||
|
||||
@smartdata.svDb()
|
||||
deepData = {
|
||||
sodeep: 'yes',
|
||||
};
|
||||
|
||||
constructor(colorArg: string, brandArg: string) {
|
||||
super();
|
||||
this.color = colorArg;
|
||||
this.brand = brandArg;
|
||||
}
|
||||
}
|
||||
|
||||
const createCarClass = (dbArg: smartdata.SmartdataDb) => {
|
||||
smartdata.setDefaultManagerForDoc({ db: dbArg }, Car);
|
||||
return Car;
|
||||
};
|
||||
|
||||
tap.test('should produce a car', async () => {
|
||||
const CarClass = createCarClass(testDb);
|
||||
const carInstance = new CarClass('red', 'Mercedes');
|
||||
await carInstance.save();
|
||||
});
|
||||
|
||||
tap.test('should get a car', async () => {
|
||||
const car = Car.getInstance({
|
||||
color: 'red',
|
||||
});
|
||||
});
|
||||
|
||||
// =======================================
|
||||
// close the database connection
|
||||
// =======================================
|
||||
tap.test('close', async () => {
|
||||
await testDb.mongoDb.dropDatabase();
|
||||
await testDb.close();
|
||||
if (smartmongoInstance) {
|
||||
await smartmongoInstance.stop();
|
||||
}
|
||||
});
|
||||
|
||||
tap.start({ throwOnError: true });
|
74
test/test.watch.ts
Normal file
74
test/test.watch.ts
Normal file
@ -0,0 +1,74 @@
|
||||
import { tap, expect } from '@push.rocks/tapbundle';
|
||||
import { Qenv } from '@push.rocks/qenv';
|
||||
import * as smartmongo from '@push.rocks/smartmongo';
|
||||
import { smartunique } from '../ts/smartdata.plugins.js';
|
||||
|
||||
const testQenv = new Qenv(process.cwd(), process.cwd() + '/.nogit/');
|
||||
|
||||
console.log(process.memoryUsage());
|
||||
|
||||
// the tested module
|
||||
import * as smartdata from '../ts/index.js';
|
||||
|
||||
// =======================================
|
||||
// Connecting to the database server
|
||||
// =======================================
|
||||
|
||||
let smartmongoInstance: smartmongo.SmartMongo;
|
||||
let testDb: smartdata.SmartdataDb;
|
||||
|
||||
const totalCars = 2000;
|
||||
|
||||
tap.test('should create a testinstance as database', async () => {
|
||||
smartmongoInstance = await smartmongo.SmartMongo.createAndStart();
|
||||
testDb = new smartdata.SmartdataDb(await smartmongoInstance.getMongoDescriptor());
|
||||
await testDb.init();
|
||||
});
|
||||
|
||||
tap.skip.test('should connect to atlas', async (tools) => {
|
||||
const databaseName = `test-smartdata-${smartunique.shortId()}`;
|
||||
testDb = new smartdata.SmartdataDb({
|
||||
mongoDbUrl: await testQenv.getEnvVarOnDemand('MONGO_URL'),
|
||||
mongoDbName: databaseName,
|
||||
});
|
||||
await testDb.init();
|
||||
});
|
||||
|
||||
@smartdata.Collection(() => testDb)
|
||||
class House extends smartdata.SmartDataDbDoc<House, House> {
|
||||
@smartdata.unI()
|
||||
public id: string = smartunique.shortId();
|
||||
|
||||
@smartdata.svDb()
|
||||
public data = {
|
||||
id: smartunique.shortId(),
|
||||
hello: 'hello',
|
||||
};
|
||||
}
|
||||
|
||||
tap.test('should watch a collection', async (toolsArg) => {
|
||||
const done = toolsArg.defer();
|
||||
const watcher = await House.watch({});
|
||||
watcher.changeSubject.subscribe(async (houseArg) => {
|
||||
console.log('hey there, we observed a house');
|
||||
await watcher.close();
|
||||
done.resolve();
|
||||
});
|
||||
const newHouse = new House();
|
||||
await newHouse.save();
|
||||
console.log('saved a house');
|
||||
await done.promise;
|
||||
});
|
||||
|
||||
// =======================================
|
||||
// close the database connection
|
||||
// =======================================
|
||||
tap.test('close', async () => {
|
||||
await testDb.mongoDb.dropDatabase();
|
||||
await testDb.close();
|
||||
if (smartmongoInstance) {
|
||||
await smartmongoInstance.stop();
|
||||
}
|
||||
});
|
||||
|
||||
tap.start({ throwOnError: true });
|
8
ts/00_commitinfo_data.ts
Normal file
8
ts/00_commitinfo_data.ts
Normal file
@ -0,0 +1,8 @@
|
||||
/**
|
||||
* autocreated commitinfo by @pushrocks/commitinfo
|
||||
*/
|
||||
export const commitinfo = {
|
||||
name: '@push.rocks/smartdata',
|
||||
version: '5.0.35',
|
||||
description: 'do more with data'
|
||||
}
|
17
ts/index.ts
17
ts/index.ts
@ -1,5 +1,14 @@
|
||||
export * from './smartdata.classes.db';
|
||||
export * from './smartdata.classes.collection';
|
||||
export * from './smartdata.classes.doc';
|
||||
export * from './smartdata.classes.db.js';
|
||||
export * from './smartdata.classes.collection.js';
|
||||
export * from './smartdata.classes.doc.js';
|
||||
export * from './smartdata.classes.easystore.js';
|
||||
export * from './smartdata.classes.cursor.js';
|
||||
|
||||
export { IMongoDescriptor } from './interfaces';
|
||||
import * as convenience from './smartadata.convenience.js';
|
||||
|
||||
export { convenience };
|
||||
|
||||
// to be removed with the next breaking update
|
||||
import type * as plugins from './smartdata.plugins.js';
|
||||
type IMongoDescriptor = plugins.tsclass.database.IMongoDescriptor;
|
||||
export type { IMongoDescriptor };
|
||||
|
@ -1 +0,0 @@
|
||||
export * from './mongodescriptor';
|
@ -1,5 +0,0 @@
|
||||
export interface IMongoDescriptor {
|
||||
mongoDbName: string;
|
||||
mongoDbUrl: string;
|
||||
mongoDbPass: string;
|
||||
}
|
5
ts/smartadata.convenience.ts
Normal file
5
ts/smartadata.convenience.ts
Normal file
@ -0,0 +1,5 @@
|
||||
import * as plugins from './smartdata.plugins.js';
|
||||
|
||||
export const getNewUniqueId = async (prefixArg?: string) => {
|
||||
return plugins.smartunique.uni(prefixArg);
|
||||
};
|
@ -1,6 +1,9 @@
|
||||
import * as plugins from './smartdata.plugins';
|
||||
import { SmartdataDb } from './smartdata.classes.db';
|
||||
import { SmartDataDbDoc } from './smartdata.classes.doc';
|
||||
import * as plugins from './smartdata.plugins.js';
|
||||
import { SmartdataDb } from './smartdata.classes.db.js';
|
||||
import { SmartdataDbCursor } from './smartdata.classes.cursor.js';
|
||||
import { SmartDataDbDoc } from './smartdata.classes.doc.js';
|
||||
import { SmartdataDbWatcher } from './smartdata.classes.watcher.js';
|
||||
import { CollectionFactory } from './smartdata.classes.collectionfactory.js';
|
||||
|
||||
export interface IFindOptions {
|
||||
limit?: number;
|
||||
@ -13,25 +16,104 @@ export interface IDocValidationFunc<T> {
|
||||
(doc: T): boolean;
|
||||
}
|
||||
|
||||
export type TDelayedDbCreation = () => SmartdataDb;
|
||||
export type TDelayed<TDelayedArg> = () => TDelayedArg;
|
||||
|
||||
const collectionFactory = new CollectionFactory();
|
||||
|
||||
/**
|
||||
* This is a decorator that will tell the decorated class what dbTable to use
|
||||
* @param dbArg
|
||||
*/
|
||||
export function Collection(dbArg: SmartdataDb | TDelayedDbCreation) {
|
||||
return function(constructor) {
|
||||
if (dbArg instanceof SmartdataDb) {
|
||||
// tslint:disable-next-line: no-string-literal
|
||||
constructor['smartdataCollection'] = new SmartdataCollection(constructor, dbArg);
|
||||
} else {
|
||||
constructor['smartdataDelayedCollection'] = () => {
|
||||
return new SmartdataCollection(constructor, dbArg());
|
||||
};
|
||||
}
|
||||
export function Collection(dbArg: SmartdataDb | TDelayed<SmartdataDb>) {
|
||||
return function classDecorator<T extends { new (...args: any[]): {} }>(constructor: T) {
|
||||
return class extends constructor {
|
||||
public static get collection() {
|
||||
if (!(dbArg instanceof SmartdataDb)) {
|
||||
dbArg = dbArg();
|
||||
}
|
||||
return collectionFactory.getCollection(constructor.name, dbArg);
|
||||
}
|
||||
public get collection() {
|
||||
if (!(dbArg instanceof SmartdataDb)) {
|
||||
dbArg = dbArg();
|
||||
}
|
||||
return collectionFactory.getCollection(constructor.name, dbArg);
|
||||
}
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
export interface IManager {
|
||||
db: SmartdataDb;
|
||||
}
|
||||
|
||||
export const setDefaultManagerForDoc = <T>(managerArg: IManager, dbDocArg: T): T => {
|
||||
(dbDocArg as any).prototype.defaultManager = managerArg;
|
||||
return dbDocArg;
|
||||
};
|
||||
|
||||
/**
|
||||
* This is a decorator that will tell the decorated class what dbTable to use
|
||||
* @param dbArg
|
||||
*/
|
||||
export function managed<TManager extends IManager>(managerArg?: TManager | TDelayed<TManager>) {
|
||||
return function classDecorator<T extends { new (...args: any[]): any }>(constructor: T) {
|
||||
return class extends constructor {
|
||||
public static get collection() {
|
||||
let dbArg: SmartdataDb;
|
||||
if (!managerArg) {
|
||||
dbArg = this.prototype.defaultManager.db;
|
||||
} else if (managerArg['db']) {
|
||||
dbArg = (managerArg as TManager).db;
|
||||
} else {
|
||||
dbArg = (managerArg as TDelayed<TManager>)().db;
|
||||
}
|
||||
return collectionFactory.getCollection(constructor.name, dbArg);
|
||||
}
|
||||
public get collection() {
|
||||
let dbArg: SmartdataDb;
|
||||
if (!managerArg) {
|
||||
//console.log(this.defaultManager.db);
|
||||
//process.exit(0)
|
||||
dbArg = this.defaultManager.db;
|
||||
} else if (managerArg['db']) {
|
||||
dbArg = (managerArg as TManager).db;
|
||||
} else {
|
||||
dbArg = (managerArg as TDelayed<TManager>)().db;
|
||||
}
|
||||
return collectionFactory.getCollection(constructor.name, dbArg);
|
||||
}
|
||||
public static get manager() {
|
||||
let manager: TManager;
|
||||
if (!managerArg) {
|
||||
manager = this.prototype.defaultManager;
|
||||
} else if (managerArg['db']) {
|
||||
manager = managerArg as TManager;
|
||||
} else {
|
||||
manager = (managerArg as TDelayed<TManager>)();
|
||||
}
|
||||
return manager;
|
||||
}
|
||||
public get manager() {
|
||||
let manager: TManager;
|
||||
if (!managerArg) {
|
||||
manager = this.defaultManager;
|
||||
} else if (managerArg['db']) {
|
||||
manager = managerArg as TManager;
|
||||
} else {
|
||||
manager = (managerArg as TDelayed<TManager>)();
|
||||
}
|
||||
return manager;
|
||||
}
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @dpecrecated use @managed instead
|
||||
*/
|
||||
export const Manager = managed;
|
||||
|
||||
export class SmartdataCollection<T> {
|
||||
/**
|
||||
* the collection that is used
|
||||
@ -42,13 +124,13 @@ export class SmartdataCollection<T> {
|
||||
public smartdataDb: SmartdataDb;
|
||||
public uniqueIndexes: string[] = [];
|
||||
|
||||
constructor(collectedClassArg: T & SmartDataDbDoc<T, unknown>, smartDataDbArg: SmartdataDb) {
|
||||
constructor(classNameArg: string, smartDataDbArg: SmartdataDb) {
|
||||
// tell the collection where it belongs
|
||||
this.collectionName = collectedClassArg.name;
|
||||
this.collectionName = classNameArg;
|
||||
this.smartdataDb = smartDataDbArg;
|
||||
|
||||
// tell the db class about it (important since Db uses different systems under the hood)
|
||||
this.smartdataDb.addTable(this);
|
||||
this.smartdataDb.addCollection(this);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -58,14 +140,14 @@ export class SmartdataCollection<T> {
|
||||
if (!this.mongoDbCollection) {
|
||||
// connect this instance to a MongoDB collection
|
||||
const availableMongoDbCollections = await this.smartdataDb.mongoDb.collections();
|
||||
const wantedCollection = availableMongoDbCollections.find(collection => {
|
||||
const wantedCollection = availableMongoDbCollections.find((collection) => {
|
||||
return collection.collectionName === this.collectionName;
|
||||
});
|
||||
if (!wantedCollection) {
|
||||
await this.smartdataDb.mongoDb.createCollection(this.collectionName);
|
||||
console.log(`Successfully initiated Collection ${this.collectionName}`);
|
||||
}
|
||||
this.mongoDbCollection = await this.smartdataDb.mongoDb.collection(this.collectionName);
|
||||
// console.log(`Successfully initiated Collection ${this.collectionName}`);
|
||||
this.mongoDbCollection = this.smartdataDb.mongoDb.collection(this.collectionName);
|
||||
}
|
||||
}
|
||||
|
||||
@ -76,7 +158,7 @@ export class SmartdataCollection<T> {
|
||||
for (const key of keyArrayArg) {
|
||||
if (!this.uniqueIndexes.includes(key)) {
|
||||
this.mongoDbCollection.createIndex(key, {
|
||||
unique: true
|
||||
unique: true,
|
||||
});
|
||||
// make sure we only call this once and not for every doc we create
|
||||
this.uniqueIndexes.push(key);
|
||||
@ -94,12 +176,57 @@ export class SmartdataCollection<T> {
|
||||
/**
|
||||
* finds an object in the DbCollection
|
||||
*/
|
||||
public async find(filterObject: any): Promise<any> {
|
||||
public async findOne(filterObject: any): Promise<any> {
|
||||
await this.init();
|
||||
const result = await this.mongoDbCollection.find(filterObject).toArray();
|
||||
const cursor = this.mongoDbCollection.find(filterObject);
|
||||
const result = await cursor.next();
|
||||
cursor.close();
|
||||
return result;
|
||||
}
|
||||
|
||||
public async getCursor(
|
||||
filterObjectArg: any,
|
||||
dbDocArg: typeof SmartDataDbDoc
|
||||
): Promise<SmartdataDbCursor<any>> {
|
||||
await this.init();
|
||||
const cursor = this.mongoDbCollection.find(filterObjectArg);
|
||||
return new SmartdataDbCursor(cursor, dbDocArg);
|
||||
}
|
||||
|
||||
/**
|
||||
* finds an object in the DbCollection
|
||||
*/
|
||||
public async findAll(filterObject: any): Promise<any[]> {
|
||||
await this.init();
|
||||
const cursor = this.mongoDbCollection.find(filterObject);
|
||||
const result = await cursor.toArray();
|
||||
cursor.close();
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* watches the collection while applying a filter
|
||||
*/
|
||||
public async watch(
|
||||
filterObject: any,
|
||||
smartdataDbDocArg: typeof SmartDataDbDoc
|
||||
): Promise<SmartdataDbWatcher> {
|
||||
await this.init();
|
||||
const changeStream = this.mongoDbCollection.watch(
|
||||
[
|
||||
{
|
||||
$match: filterObject,
|
||||
},
|
||||
],
|
||||
{
|
||||
fullDocument: 'updateLookup',
|
||||
}
|
||||
);
|
||||
const smartdataWatcher = new SmartdataDbWatcher(changeStream, smartdataDbDocArg);
|
||||
await smartdataWatcher.readyDeferred.promise;
|
||||
return smartdataWatcher;
|
||||
}
|
||||
|
||||
/**
|
||||
* create an object in the database
|
||||
*/
|
||||
@ -127,20 +254,19 @@ export class SmartdataCollection<T> {
|
||||
}
|
||||
updateableObject[key] = saveableObject[key];
|
||||
}
|
||||
this.mongoDbCollection.updateOne(
|
||||
const result = await this.mongoDbCollection.updateOne(
|
||||
identifiableObject,
|
||||
{ $set: updateableObject },
|
||||
{ upsert: true }
|
||||
);
|
||||
return result;
|
||||
}
|
||||
|
||||
public async delete(dbDocArg: T & SmartDataDbDoc<T, unknown>): Promise<any> {
|
||||
await this.init();
|
||||
await this.checkDoc(dbDocArg);
|
||||
const identifiableObject = await dbDocArg.createIdentifiableObject();
|
||||
await this.mongoDbCollection.deleteOne(identifiableObject, {
|
||||
w: 1
|
||||
});
|
||||
await this.mongoDbCollection.deleteOne(identifiableObject);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -148,7 +274,7 @@ export class SmartdataCollection<T> {
|
||||
* if this.objectValidation is not set it passes.
|
||||
*/
|
||||
private checkDoc(docArg: T): Promise<void> {
|
||||
const done = plugins.smartq.defer<void>();
|
||||
const done = plugins.smartpromise.defer<void>();
|
||||
let validationResult = true;
|
||||
if (this.objectValidation) {
|
||||
validationResult = this.objectValidation(docArg);
|
||||
|
19
ts/smartdata.classes.collectionfactory.ts
Normal file
19
ts/smartdata.classes.collectionfactory.ts
Normal file
@ -0,0 +1,19 @@
|
||||
import * as plugins from './smartdata.plugins.js';
|
||||
import { SmartdataCollection } from './smartdata.classes.collection.js';
|
||||
import { SmartdataDb } from './smartdata.classes.db.js';
|
||||
|
||||
export class CollectionFactory {
|
||||
public collections: { [key: string]: SmartdataCollection<any> } = {};
|
||||
|
||||
public getCollection = (nameArg: string, dbArg: SmartdataDb): SmartdataCollection<any> => {
|
||||
if (!this.collections[nameArg]) {
|
||||
this.collections[nameArg] = (() => {
|
||||
if (dbArg instanceof SmartdataDb) {
|
||||
// tslint:disable-next-line: no-string-literal
|
||||
return new SmartdataCollection(nameArg, dbArg);
|
||||
}
|
||||
})();
|
||||
}
|
||||
return this.collections[nameArg];
|
||||
};
|
||||
}
|
46
ts/smartdata.classes.cursor.ts
Normal file
46
ts/smartdata.classes.cursor.ts
Normal file
@ -0,0 +1,46 @@
|
||||
import { SmartDataDbDoc } from './smartdata.classes.doc.js';
|
||||
import * as plugins from './smartdata.plugins.js';
|
||||
|
||||
/**
|
||||
* a wrapper for the native mongodb cursor. Exposes better
|
||||
*/
|
||||
export class SmartdataDbCursor<T = any> {
|
||||
// STATIC
|
||||
|
||||
// INSTANCE
|
||||
public mongodbCursor: plugins.mongodb.FindCursor<T>;
|
||||
private smartdataDbDoc: typeof SmartDataDbDoc;
|
||||
constructor(cursorArg: plugins.mongodb.FindCursor<T>, dbDocArg: typeof SmartDataDbDoc) {
|
||||
this.mongodbCursor = cursorArg;
|
||||
this.smartdataDbDoc = dbDocArg;
|
||||
}
|
||||
|
||||
public async next(closeAtEnd = true) {
|
||||
const result = this.smartdataDbDoc.createInstanceFromMongoDbNativeDoc(
|
||||
await this.mongodbCursor.next()
|
||||
);
|
||||
if (!result && closeAtEnd) {
|
||||
await this.close();
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public async forEach(forEachFuncArg: (itemArg: T) => Promise<any>, closeCursorAtEnd = true) {
|
||||
let nextDocument: any;
|
||||
do {
|
||||
nextDocument = await this.mongodbCursor.next();
|
||||
if (nextDocument) {
|
||||
const nextClassInstance =
|
||||
this.smartdataDbDoc.createInstanceFromMongoDbNativeDoc(nextDocument);
|
||||
await forEachFuncArg(nextClassInstance as any);
|
||||
}
|
||||
} while (nextDocument);
|
||||
if (closeCursorAtEnd) {
|
||||
await this.close();
|
||||
}
|
||||
}
|
||||
|
||||
public async close() {
|
||||
await this.mongodbCursor.close();
|
||||
}
|
||||
}
|
@ -1,64 +1,57 @@
|
||||
import * as plugins from './smartdata.plugins';
|
||||
import { Objectmap } from '@pushrocks/lik';
|
||||
import * as plugins from './smartdata.plugins.js';
|
||||
|
||||
import { SmartdataCollection } from './smartdata.classes.collection';
|
||||
import { SmartdataCollection } from './smartdata.classes.collection.js';
|
||||
import { EasyStore } from './smartdata.classes.easystore.js';
|
||||
|
||||
import * as mongoHelpers from './smartdata.mongohelpers';
|
||||
import { logger } from './smartdata.logging.js';
|
||||
|
||||
/**
|
||||
* interface - indicates the connection status of the db
|
||||
*/
|
||||
export type TConnectionStatus = 'initial' | 'disconnected' | 'connected' | 'failed';
|
||||
|
||||
export interface ISmartdataOptions {
|
||||
/**
|
||||
* the URL to connect to
|
||||
*/
|
||||
mongoDbUrl: string;
|
||||
|
||||
/**
|
||||
* the db to use for the project
|
||||
*/
|
||||
mongoDbName: string;
|
||||
|
||||
/**
|
||||
* an optional password that will be replace <PASSWORD> in the connection string
|
||||
*/
|
||||
mongoDbPass?: string;
|
||||
}
|
||||
|
||||
export class SmartdataDb {
|
||||
smartdataOptions: ISmartdataOptions;
|
||||
smartdataOptions: plugins.tsclass.database.IMongoDescriptor;
|
||||
mongoDbClient: plugins.mongodb.MongoClient;
|
||||
mongoDb: plugins.mongodb.Db;
|
||||
status: TConnectionStatus;
|
||||
smartdataCollectionMap = new Objectmap<SmartdataCollection<any>>();
|
||||
statusConnectedDeferred = plugins.smartpromise.defer();
|
||||
smartdataCollectionMap = new plugins.lik.ObjectMap<SmartdataCollection<any>>();
|
||||
|
||||
constructor(smartdataOptions: ISmartdataOptions) {
|
||||
constructor(smartdataOptions: plugins.tsclass.database.IMongoDescriptor) {
|
||||
this.smartdataOptions = smartdataOptions;
|
||||
this.status = 'initial';
|
||||
}
|
||||
|
||||
// easystore
|
||||
public async createEasyStore(nameIdArg: string) {
|
||||
const easyStore = new EasyStore(nameIdArg, this);
|
||||
return easyStore;
|
||||
}
|
||||
|
||||
// basic connection stuff ----------------------------------------------
|
||||
|
||||
/**
|
||||
* connects to the database that was specified during instance creation
|
||||
*/
|
||||
public async init(): Promise<any> {
|
||||
let finalConnectionUrl = this.smartdataOptions.mongoDbUrl;
|
||||
if (this.smartdataOptions.mongoDbPass) {
|
||||
finalConnectionUrl = mongoHelpers.addPassword(
|
||||
this.smartdataOptions.mongoDbUrl,
|
||||
this.smartdataOptions.mongoDbPass
|
||||
);
|
||||
}
|
||||
console.log(`connection Url: ${finalConnectionUrl}`);
|
||||
const finalConnectionUrl = this.smartdataOptions.mongoDbUrl
|
||||
.replace('<USERNAME>', this.smartdataOptions.mongoDbUser)
|
||||
.replace('<username>', this.smartdataOptions.mongoDbUser)
|
||||
.replace('<USER>', this.smartdataOptions.mongoDbUser)
|
||||
.replace('<user>', this.smartdataOptions.mongoDbUser)
|
||||
.replace('<PASSWORD>', this.smartdataOptions.mongoDbPass)
|
||||
.replace('<password>', this.smartdataOptions.mongoDbPass)
|
||||
.replace('<DBNAME>', this.smartdataOptions.mongoDbName)
|
||||
.replace('<dbname>', this.smartdataOptions.mongoDbName);
|
||||
|
||||
this.mongoDbClient = await plugins.mongodb.MongoClient.connect(finalConnectionUrl, {
|
||||
useNewUrlParser: true,
|
||||
useUnifiedTopology: true
|
||||
maxPoolSize: 100,
|
||||
maxIdleTimeMS: 10,
|
||||
});
|
||||
this.mongoDb = this.mongoDbClient.db(this.smartdataOptions.mongoDbName);
|
||||
this.status = 'connected';
|
||||
this.statusConnectedDeferred.resolve();
|
||||
console.log(`Connected to database ${this.smartdataOptions.mongoDbName}`);
|
||||
}
|
||||
|
||||
@ -68,15 +61,12 @@ export class SmartdataDb {
|
||||
public async close(): Promise<any> {
|
||||
await this.mongoDbClient.close();
|
||||
this.status = 'disconnected';
|
||||
plugins.smartlog.defaultLogger.log(
|
||||
'info',
|
||||
`disconnected from database ${this.smartdataOptions.mongoDbName}`
|
||||
);
|
||||
logger.log('info', `disconnected from database ${this.smartdataOptions.mongoDbName}`);
|
||||
}
|
||||
|
||||
// handle table to class distribution
|
||||
|
||||
public addTable(SmartdataCollectionArg: SmartdataCollection<any>) {
|
||||
public addCollection(SmartdataCollectionArg: SmartdataCollection<any>) {
|
||||
this.smartdataCollectionMap.add(SmartdataCollectionArg);
|
||||
}
|
||||
|
||||
@ -86,7 +76,7 @@ export class SmartdataDb {
|
||||
* @returns DbTable
|
||||
*/
|
||||
public async getSmartdataCollectionByName<T>(nameArg: string): Promise<SmartdataCollection<T>> {
|
||||
const resultCollection = this.smartdataCollectionMap.find(dbTableArg => {
|
||||
const resultCollection = await this.smartdataCollectionMap.find(async (dbTableArg) => {
|
||||
return dbTableArg.collectionName === nameArg;
|
||||
});
|
||||
return resultCollection;
|
||||
|
304
ts/smartdata.classes.distributedcoordinator.ts
Normal file
304
ts/smartdata.classes.distributedcoordinator.ts
Normal file
@ -0,0 +1,304 @@
|
||||
import * as plugins from './smartdata.plugins.js';
|
||||
import { SmartdataDb } from './smartdata.classes.db.js';
|
||||
import { managed, setDefaultManagerForDoc } from './smartdata.classes.collection.js';
|
||||
import { SmartDataDbDoc, svDb, unI } from './smartdata.classes.doc.js';
|
||||
import { SmartdataDbWatcher } from './smartdata.classes.watcher.js';
|
||||
|
||||
@managed()
|
||||
export class DistributedClass extends SmartDataDbDoc<DistributedClass, DistributedClass> {
|
||||
// INSTANCE
|
||||
@unI()
|
||||
public id: string;
|
||||
|
||||
@svDb()
|
||||
public data: {
|
||||
status: 'initializing' | 'bidding' | 'settled' | 'stopped';
|
||||
biddingShortcode?: string;
|
||||
biddingStartTime?: number;
|
||||
lastUpdated: number;
|
||||
elected: boolean;
|
||||
/**
|
||||
* used to store request
|
||||
*/
|
||||
taskRequests: plugins.taskbuffer.distributedCoordination.IDistributedTaskRequest[];
|
||||
|
||||
/**
|
||||
* only used by the leader to convey consultation results
|
||||
*/
|
||||
taskRequestResults: plugins.taskbuffer.distributedCoordination.IDistributedTaskRequestResult[];
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* This file implements a distributed coordinator according to the @pushrocks/taskbuffer standard.
|
||||
* you should not set up this yourself. Instead, there is a factory on the SmartdataDb class
|
||||
* that will take care of setting this up.
|
||||
*/
|
||||
export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distributedCoordination
|
||||
.AbstractDistributedCoordinator {
|
||||
public readyPromise: Promise<any>;
|
||||
public db: SmartdataDb;
|
||||
private asyncExecutionStack = new plugins.lik.AsyncExecutionStack();
|
||||
public ownInstance: DistributedClass;
|
||||
public distributedWatcher: SmartdataDbWatcher<DistributedClass>;
|
||||
|
||||
constructor(dbArg: SmartdataDb) {
|
||||
super();
|
||||
this.db = dbArg;
|
||||
setDefaultManagerForDoc(this, DistributedClass);
|
||||
this.readyPromise = this.db.statusConnectedDeferred.promise;
|
||||
}
|
||||
|
||||
// smartdata specific stuff
|
||||
public async start() {
|
||||
await this.init();
|
||||
}
|
||||
|
||||
public async stop() {
|
||||
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
||||
if (this.distributedWatcher) {
|
||||
await this.distributedWatcher.close();
|
||||
}
|
||||
if (this.ownInstance?.data.elected) {
|
||||
this.ownInstance.data.elected = false;
|
||||
}
|
||||
if (this.ownInstance?.data.status === 'stopped') {
|
||||
console.log(`stopping a distributed instance that has not been started yet.`);
|
||||
}
|
||||
this.ownInstance.data.status = 'stopped';
|
||||
await this.ownInstance.save();
|
||||
console.log(`stopped ${this.ownInstance.id}`);
|
||||
});
|
||||
}
|
||||
|
||||
public id = plugins.smartunique.uni('distributedInstance');
|
||||
|
||||
private startHeartbeat = async () => {
|
||||
while (this.ownInstance.data.status !== 'stopped') {
|
||||
await this.sendHeartbeat();
|
||||
await plugins.smartdelay.delayForRandom(5000, 10000);
|
||||
}
|
||||
};
|
||||
|
||||
public async sendHeartbeat() {
|
||||
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
||||
if (this.ownInstance.data.status === 'stopped') {
|
||||
console.log(`aborted sending heartbeat because status is stopped`);
|
||||
return;
|
||||
}
|
||||
await this.ownInstance.updateFromDb();
|
||||
this.ownInstance.data.lastUpdated = Date.now();
|
||||
await this.ownInstance.save();
|
||||
console.log(`sent heartbeat for ${this.ownInstance.id}`);
|
||||
const allInstances = DistributedClass.getInstances({});
|
||||
});
|
||||
if (this.ownInstance.data.status === 'stopped') {
|
||||
console.log(`aborted sending heartbeat because status is stopped`);
|
||||
return;
|
||||
}
|
||||
const eligibleLeader = await this.getEligibleLeader();
|
||||
// not awaiting here because we don't want to block the heartbeat
|
||||
this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
||||
if (!eligibleLeader && this.ownInstance.data.status === 'settled') {
|
||||
this.checkAndMaybeLead();
|
||||
}
|
||||
});
|
||||
}
|
||||
private async init() {
|
||||
await this.readyPromise;
|
||||
if (!this.ownInstance) {
|
||||
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
||||
this.ownInstance = new DistributedClass();
|
||||
this.ownInstance.id = this.id;
|
||||
this.ownInstance.data = {
|
||||
elected: false,
|
||||
lastUpdated: Date.now(),
|
||||
status: 'initializing',
|
||||
taskRequests: [],
|
||||
taskRequestResults: [],
|
||||
};
|
||||
await this.ownInstance.save();
|
||||
});
|
||||
} else {
|
||||
console.warn(`distributed instance already initialized`);
|
||||
}
|
||||
|
||||
// lets enable the heartbeat
|
||||
this.startHeartbeat();
|
||||
|
||||
// lets do a leader check
|
||||
await this.checkAndMaybeLead();
|
||||
|
||||
return this.ownInstance;
|
||||
}
|
||||
|
||||
public async getEligibleLeader() {
|
||||
return this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
||||
const allInstances = await DistributedClass.getInstances({});
|
||||
let leaders = allInstances.filter((instanceArg) => instanceArg.data.elected === true);
|
||||
const eligibleLeader = leaders.find(
|
||||
(leader) =>
|
||||
leader.data.lastUpdated >=
|
||||
Date.now() - plugins.smarttime.getMilliSecondsFromUnits({ seconds: 20 })
|
||||
);
|
||||
return eligibleLeader;
|
||||
});
|
||||
}
|
||||
|
||||
// --> leader election
|
||||
public async checkAndMaybeLead() {
|
||||
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
||||
this.ownInstance.data.status = 'initializing';
|
||||
this.ownInstance.save();
|
||||
});
|
||||
if (await this.getEligibleLeader()) {
|
||||
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
||||
await this.ownInstance.updateFromDb();
|
||||
this.ownInstance.data.status = 'settled';
|
||||
await this.ownInstance.save();
|
||||
console.log(`${this.ownInstance.id} settled as follower`);
|
||||
});
|
||||
return;
|
||||
} else if (
|
||||
(await DistributedClass.getInstances({})).find((instanceArg) => {
|
||||
instanceArg.data.status === 'bidding' &&
|
||||
instanceArg.data.biddingStartTime <= Date.now() - 4000 &&
|
||||
instanceArg.data.biddingStartTime >= Date.now() - 30000;
|
||||
})
|
||||
) {
|
||||
console.log('too late to the bidding party... waiting for next round.');
|
||||
return;
|
||||
} else {
|
||||
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
||||
await this.ownInstance.updateFromDb();
|
||||
this.ownInstance.data.status = 'bidding';
|
||||
this.ownInstance.data.biddingStartTime = Date.now();
|
||||
this.ownInstance.data.biddingShortcode = plugins.smartunique.shortId();
|
||||
await this.ownInstance.save();
|
||||
console.log('bidding code stored.');
|
||||
});
|
||||
console.log(`bidding for leadership...`);
|
||||
await plugins.smartdelay.delayFor(
|
||||
plugins.smarttime.getMilliSecondsFromUnits({ seconds: 5 })
|
||||
);
|
||||
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
||||
let biddingInstances = await DistributedClass.getInstances({});
|
||||
biddingInstances = biddingInstances.filter(
|
||||
(instanceArg) =>
|
||||
instanceArg.data.status === 'bidding' &&
|
||||
instanceArg.data.lastUpdated >=
|
||||
Date.now() - plugins.smarttime.getMilliSecondsFromUnits({ seconds: 10 })
|
||||
);
|
||||
console.log(`found ${biddingInstances.length} bidding instances...`);
|
||||
this.ownInstance.data.elected = true;
|
||||
for (const biddingInstance of biddingInstances) {
|
||||
if (biddingInstance.data.biddingShortcode < this.ownInstance.data.biddingShortcode) {
|
||||
this.ownInstance.data.elected = false;
|
||||
}
|
||||
}
|
||||
await plugins.smartdelay.delayFor(5000);
|
||||
console.log(`settling with status elected = ${this.ownInstance.data.elected}`);
|
||||
this.ownInstance.data.status = 'settled';
|
||||
await this.ownInstance.save();
|
||||
});
|
||||
if (this.ownInstance.data.elected) {
|
||||
this.leadFunction();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* when it has been determined
|
||||
* that this instance is leading
|
||||
* the leading is implemented here
|
||||
*/
|
||||
public async leadFunction() {
|
||||
this.distributedWatcher = await DistributedClass.watch({});
|
||||
|
||||
const currentTaskRequests: Array<{
|
||||
taskName: string;
|
||||
taskExecutionTime: number;
|
||||
/**
|
||||
* all instances that requested this task
|
||||
*/
|
||||
requestingDistibutedInstanceIds: string[];
|
||||
responseTimeout: plugins.smartdelay.Timeout<any>;
|
||||
}> = [];
|
||||
|
||||
this.distributedWatcher.changeSubject.subscribe({
|
||||
next: async (distributedDoc) => {
|
||||
if (!distributedDoc) {
|
||||
console.log(`registered deletion of instance...`);
|
||||
return;
|
||||
}
|
||||
console.log(distributedDoc);
|
||||
console.log(`registered change for ${distributedDoc.id}`);
|
||||
distributedDoc;
|
||||
},
|
||||
});
|
||||
|
||||
while (this.ownInstance.data.status !== 'stopped' && this.ownInstance.data.elected) {
|
||||
const allInstances = await DistributedClass.getInstances({});
|
||||
for (const instance of allInstances) {
|
||||
if (instance.data.status === 'stopped') {
|
||||
await instance.delete();
|
||||
};
|
||||
}
|
||||
await plugins.smartdelay.delayFor(10000);
|
||||
}
|
||||
}
|
||||
|
||||
// abstract implemented methods
|
||||
public async fireDistributedTaskRequest(
|
||||
taskRequestArg: plugins.taskbuffer.distributedCoordination.IDistributedTaskRequest
|
||||
): Promise<plugins.taskbuffer.distributedCoordination.IDistributedTaskRequestResult> {
|
||||
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
||||
if (!this.ownInstance) {
|
||||
console.error('instance need to be started first...');
|
||||
return;
|
||||
}
|
||||
await this.ownInstance.updateFromDb();
|
||||
this.ownInstance.data.taskRequests.push(taskRequestArg);
|
||||
await this.ownInstance.save();
|
||||
});
|
||||
await plugins.smartdelay.delayFor(10000);
|
||||
const result = await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
||||
await this.ownInstance.updateFromDb();
|
||||
const taskRequestResult = this.ownInstance.data.taskRequestResults.find((resultItem) => {
|
||||
return resultItem.requestResponseId === taskRequestArg.requestResponseId;
|
||||
});
|
||||
return taskRequestResult;
|
||||
});
|
||||
if (!result) {
|
||||
console.warn('no result found for task request...');
|
||||
return null;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public async updateDistributedTaskRequest(
|
||||
infoBasisArg: plugins.taskbuffer.distributedCoordination.IDistributedTaskRequest
|
||||
): Promise<void> {
|
||||
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
||||
const existingInfoBasis = this.ownInstance.data.taskRequests.find((infoBasisItem) => {
|
||||
return (
|
||||
infoBasisItem.taskName === infoBasisArg.taskName &&
|
||||
infoBasisItem.taskExecutionTime === infoBasisArg.taskExecutionTime
|
||||
);
|
||||
});
|
||||
if (!existingInfoBasis) {
|
||||
console.warn('trying to update a non existing task request... aborting!');
|
||||
return;
|
||||
}
|
||||
Object.assign(existingInfoBasis, infoBasisArg);
|
||||
await this.ownInstance.save();
|
||||
plugins.smartdelay.delayFor(60000).then(() => {
|
||||
this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
||||
const indexToRemove = this.ownInstance.data.taskRequests.indexOf(existingInfoBasis);
|
||||
this.ownInstance.data.taskRequests.splice(indexToRemove, indexToRemove);
|
||||
await this.ownInstance.save();
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
@ -1,9 +1,9 @@
|
||||
import * as plugins from './smartdata.plugins';
|
||||
import * as plugins from './smartdata.plugins.js';
|
||||
|
||||
import { Objectmap } from '@pushrocks/lik';
|
||||
|
||||
import { SmartdataDb } from './smartdata.classes.db';
|
||||
import { SmartdataCollection } from './smartdata.classes.collection';
|
||||
import { SmartdataDb } from './smartdata.classes.db.js';
|
||||
import { SmartdataDbCursor } from './smartdata.classes.cursor.js';
|
||||
import { type IManager, SmartdataCollection } from './smartdata.classes.collection.js';
|
||||
import { SmartdataDbWatcher } from './smartdata.classes.watcher.js';
|
||||
|
||||
export type TDocCreation = 'db' | 'new' | 'mixed';
|
||||
|
||||
@ -12,7 +12,7 @@ export type TDocCreation = 'db' | 'new' | 'mixed';
|
||||
*/
|
||||
export function svDb() {
|
||||
return (target: SmartDataDbDoc<unknown, unknown>, key: string) => {
|
||||
console.log(`called svDb() on ${key}`);
|
||||
console.log(`called svDb() on >${target.constructor.name}.${key}<`);
|
||||
if (!target.saveableProperties) {
|
||||
target.saveableProperties = [];
|
||||
}
|
||||
@ -25,7 +25,7 @@ export function svDb() {
|
||||
*/
|
||||
export function unI() {
|
||||
return (target: SmartDataDbDoc<unknown, unknown>, key: string) => {
|
||||
console.log('called unI');
|
||||
console.log(`called unI on >>${target.constructor.name}.${key}<<`);
|
||||
|
||||
// mark the index as unique
|
||||
if (!target.uniqueIndexes) {
|
||||
@ -41,11 +41,140 @@ export function unI() {
|
||||
};
|
||||
}
|
||||
|
||||
export class SmartDataDbDoc<T, TImplements> {
|
||||
export const convertFilterForMongoDb = (filterArg: { [key: string]: any }) => {
|
||||
const convertedFilter: { [key: string]: any } = {};
|
||||
const convertFilterArgument = (keyPathArg2: string, filterArg2: any) => {
|
||||
if (typeof filterArg2 === 'object') {
|
||||
for (const key of Object.keys(filterArg2)) {
|
||||
if (key.startsWith('$')) {
|
||||
convertedFilter[keyPathArg2] = filterArg2;
|
||||
return;
|
||||
} else if (key.includes('.')) {
|
||||
throw new Error('keys cannot contain dots');
|
||||
}
|
||||
}
|
||||
for (const key of Object.keys(filterArg2)) {
|
||||
convertFilterArgument(`${keyPathArg2}.${key}`, filterArg2[key]);
|
||||
}
|
||||
} else {
|
||||
convertedFilter[keyPathArg2] = filterArg2;
|
||||
}
|
||||
};
|
||||
for (const key of Object.keys(filterArg)) {
|
||||
convertFilterArgument(key, filterArg[key]);
|
||||
}
|
||||
return convertedFilter;
|
||||
};
|
||||
|
||||
export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends IManager = any> {
|
||||
/**
|
||||
* the collection object an Doc belongs to
|
||||
*/
|
||||
public collection: SmartdataCollection<T>;
|
||||
public static collection: SmartdataCollection<any>;
|
||||
public collection: SmartdataCollection<any>;
|
||||
public static defaultManager;
|
||||
public static manager;
|
||||
public manager: TManager;
|
||||
|
||||
// STATIC
|
||||
public static createInstanceFromMongoDbNativeDoc<T>(
|
||||
this: plugins.tsclass.typeFest.Class<T>,
|
||||
mongoDbNativeDocArg: any
|
||||
): T {
|
||||
const newInstance = new this();
|
||||
(newInstance as any).creationStatus = 'db';
|
||||
for (const key of Object.keys(mongoDbNativeDocArg)) {
|
||||
newInstance[key] = mongoDbNativeDocArg[key];
|
||||
}
|
||||
return newInstance;
|
||||
}
|
||||
|
||||
/**
|
||||
* gets all instances as array
|
||||
* @param this
|
||||
* @param filterArg
|
||||
* @returns
|
||||
*/
|
||||
public static async getInstances<T>(
|
||||
this: plugins.tsclass.typeFest.Class<T>,
|
||||
filterArg: plugins.tsclass.typeFest.PartialDeep<T>
|
||||
): Promise<T[]> {
|
||||
const foundDocs = await (this as any).collection.findAll(convertFilterForMongoDb(filterArg));
|
||||
const returnArray = [];
|
||||
for (const foundDoc of foundDocs) {
|
||||
const newInstance: T = (this as any).createInstanceFromMongoDbNativeDoc(foundDoc);
|
||||
returnArray.push(newInstance);
|
||||
}
|
||||
return returnArray;
|
||||
}
|
||||
|
||||
/**
|
||||
* gets the first matching instance
|
||||
* @param this
|
||||
* @param filterArg
|
||||
* @returns
|
||||
*/
|
||||
public static async getInstance<T>(
|
||||
this: plugins.tsclass.typeFest.Class<T>,
|
||||
filterArg: plugins.tsclass.typeFest.PartialDeep<T>
|
||||
): Promise<T> {
|
||||
const foundDoc = await (this as any).collection.findOne(convertFilterForMongoDb(filterArg));
|
||||
if (foundDoc) {
|
||||
const newInstance: T = (this as any).createInstanceFromMongoDbNativeDoc(foundDoc);
|
||||
return newInstance;
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* get cursor
|
||||
* @returns
|
||||
*/
|
||||
public static async getCursor<T>(
|
||||
this: plugins.tsclass.typeFest.Class<T>,
|
||||
filterArg: plugins.tsclass.typeFest.PartialDeep<T>
|
||||
) {
|
||||
const collection: SmartdataCollection<T> = (this as any).collection;
|
||||
const cursor: SmartdataDbCursor<T> = await collection.getCursor(
|
||||
convertFilterForMongoDb(filterArg),
|
||||
this as any as typeof SmartDataDbDoc
|
||||
);
|
||||
return cursor;
|
||||
}
|
||||
|
||||
/**
|
||||
* watch the collection
|
||||
* @param this
|
||||
* @param filterArg
|
||||
* @param forEachFunction
|
||||
*/
|
||||
public static async watch<T>(
|
||||
this: plugins.tsclass.typeFest.Class<T>,
|
||||
filterArg: plugins.tsclass.typeFest.PartialDeep<T>
|
||||
) {
|
||||
const collection: SmartdataCollection<T> = (this as any).collection;
|
||||
const watcher: SmartdataDbWatcher<T> = await collection.watch(
|
||||
convertFilterForMongoDb(filterArg),
|
||||
this as any
|
||||
);
|
||||
return watcher;
|
||||
}
|
||||
|
||||
/**
|
||||
* run a function for all instances
|
||||
* @returns
|
||||
*/
|
||||
public static async forEach<T>(
|
||||
this: plugins.tsclass.typeFest.Class<T>,
|
||||
filterArg: plugins.tsclass.typeFest.PartialDeep<T>,
|
||||
forEachFunction: (itemArg: T) => Promise<any>
|
||||
) {
|
||||
const cursor: SmartdataDbCursor<T> = await (this as any).getCursor(filterArg);
|
||||
await cursor.forEach(forEachFunction);
|
||||
}
|
||||
|
||||
// INSTANCE
|
||||
|
||||
/**
|
||||
* how the Doc in memory was created, may prove useful later.
|
||||
@ -75,48 +204,7 @@ export class SmartDataDbDoc<T, TImplements> {
|
||||
/**
|
||||
* class constructor
|
||||
*/
|
||||
constructor() {
|
||||
this.name = this.constructor['name'];
|
||||
if (this.constructor['smartdataCollection']) {
|
||||
// tslint:disable-next-line: no-string-literal
|
||||
this.collection = this.constructor['smartdataCollection'];
|
||||
// tslint:disable-next-line: no-string-literal
|
||||
} else if (typeof this.constructor['smartdataDelayedCollection'] === 'function') {
|
||||
// tslint:disable-next-line: no-string-literal
|
||||
this.collection = this.constructor['smartdataDelayedCollection']();
|
||||
} else {
|
||||
console.error('Could not determine collection for DbDoc');
|
||||
}
|
||||
}
|
||||
|
||||
public static async getInstances<T>(filterArg): Promise<T[]> {
|
||||
const self: any = this; // fool typesystem
|
||||
let referenceMongoDBCollection: SmartdataCollection<T>;
|
||||
|
||||
if (self.smartdataCollection) {
|
||||
referenceMongoDBCollection = self.smartdataCollection;
|
||||
} else if (self.smartdataDelayedCollection) {
|
||||
referenceMongoDBCollection = self.smartdataDelayedCollection();
|
||||
}
|
||||
const foundDocs = await referenceMongoDBCollection.find(filterArg);
|
||||
const returnArray = [];
|
||||
for (const item of foundDocs) {
|
||||
const newInstance = new this();
|
||||
newInstance.creationStatus = 'db';
|
||||
for (const key of Object.keys(item)) {
|
||||
newInstance[key] = item[key];
|
||||
}
|
||||
returnArray.push(newInstance);
|
||||
}
|
||||
return returnArray;
|
||||
}
|
||||
|
||||
public static async getInstance<T>(filterArg): Promise<T> {
|
||||
const result = await this.getInstances<T>(filterArg);
|
||||
if (result && result.length > 0) {
|
||||
return result[0];
|
||||
}
|
||||
}
|
||||
constructor() {}
|
||||
|
||||
/**
|
||||
* saves this instance but not any connected items
|
||||
@ -125,34 +213,35 @@ export class SmartDataDbDoc<T, TImplements> {
|
||||
public async save() {
|
||||
// tslint:disable-next-line: no-this-assignment
|
||||
const self: any = this;
|
||||
let dbResult: any;
|
||||
switch (this.creationStatus) {
|
||||
case 'db':
|
||||
await this.collection.update(self);
|
||||
dbResult = await this.collection.update(self);
|
||||
break;
|
||||
case 'new':
|
||||
const writeResult = await this.collection.insert(self);
|
||||
dbResult = await this.collection.insert(self);
|
||||
this.creationStatus = 'db';
|
||||
break;
|
||||
default:
|
||||
console.error('neither new nor in db?');
|
||||
}
|
||||
return dbResult;
|
||||
}
|
||||
|
||||
/**
|
||||
* deletes a document from the database
|
||||
*/
|
||||
public async delete() {
|
||||
const self: any = this;
|
||||
await this.collection.delete(self);
|
||||
await this.collection.delete(this);
|
||||
}
|
||||
|
||||
/**
|
||||
* also store any referenced objects to DB
|
||||
* better for data consistency
|
||||
*/
|
||||
public saveDeep(savedMapArg: Objectmap<SmartDataDbDoc<any, any>> = null) {
|
||||
public saveDeep(savedMapArg: plugins.lik.ObjectMap<SmartDataDbDoc<any, any>> = null) {
|
||||
if (!savedMapArg) {
|
||||
savedMapArg = new Objectmap<SmartDataDbDoc<any, any>>();
|
||||
savedMapArg = new plugins.lik.ObjectMap<SmartDataDbDoc<any, any>>();
|
||||
}
|
||||
savedMapArg.add(this);
|
||||
this.save();
|
||||
@ -164,6 +253,16 @@ export class SmartDataDbDoc<T, TImplements> {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* updates an object from db
|
||||
*/
|
||||
public async updateFromDb() {
|
||||
const mongoDbNativeDoc = await this.collection.findOne(await this.createIdentifiableObject());
|
||||
for (const key of Object.keys(mongoDbNativeDoc)) {
|
||||
this[key] = mongoDbNativeDoc[key];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* creates a saveable object so the instance can be persisted as json in the database
|
||||
*/
|
||||
|
109
ts/smartdata.classes.easystore.ts
Normal file
109
ts/smartdata.classes.easystore.ts
Normal file
@ -0,0 +1,109 @@
|
||||
import * as plugins from './smartdata.plugins.js';
|
||||
import { Collection } from './smartdata.classes.collection.js';
|
||||
import { SmartdataDb } from './smartdata.classes.db.js';
|
||||
import { SmartDataDbDoc, svDb, unI } from './smartdata.classes.doc.js';
|
||||
|
||||
/**
|
||||
* EasyStore allows the storage of easy objects. It also allows easy sharing of the object between different instances
|
||||
*/
|
||||
export class EasyStore<T> {
|
||||
// instance
|
||||
public smartdataDbRef: SmartdataDb;
|
||||
public nameId: string;
|
||||
|
||||
private easyStoreClass = (() => {
|
||||
@Collection(() => this.smartdataDbRef)
|
||||
class SmartdataEasyStore extends SmartDataDbDoc<SmartdataEasyStore, SmartdataEasyStore> {
|
||||
@unI()
|
||||
public nameId: string;
|
||||
|
||||
@svDb()
|
||||
public ephermal: {
|
||||
activated: boolean;
|
||||
timeout: number;
|
||||
};
|
||||
|
||||
@svDb()
|
||||
lastEdit: number;
|
||||
|
||||
@svDb()
|
||||
public data: Partial<T>;
|
||||
}
|
||||
return SmartdataEasyStore;
|
||||
})();
|
||||
|
||||
constructor(nameIdArg: string, smnartdataDbRefArg: SmartdataDb) {
|
||||
this.smartdataDbRef = smnartdataDbRefArg;
|
||||
this.nameId = nameIdArg;
|
||||
}
|
||||
|
||||
private async getEasyStore() {
|
||||
let easyStore = await this.easyStoreClass.getInstance({
|
||||
nameId: this.nameId,
|
||||
});
|
||||
|
||||
if (!easyStore) {
|
||||
easyStore = new this.easyStoreClass();
|
||||
easyStore.nameId = this.nameId;
|
||||
easyStore.data = {};
|
||||
await easyStore.save();
|
||||
}
|
||||
return easyStore;
|
||||
}
|
||||
|
||||
/**
|
||||
* reads all keyValue pairs at once and returns them
|
||||
*/
|
||||
public async readAll() {
|
||||
const easyStore = await this.getEasyStore();
|
||||
return easyStore.data;
|
||||
}
|
||||
|
||||
/**
|
||||
* reads a keyValueFile from disk
|
||||
*/
|
||||
public async readKey(keyArg: keyof T) {
|
||||
const easyStore = await this.getEasyStore();
|
||||
return easyStore.data[keyArg];
|
||||
}
|
||||
|
||||
/**
|
||||
* writes a specific key to the keyValueStore
|
||||
*/
|
||||
public async writeKey<TKey extends keyof T>(keyArg: TKey, valueArg: T[TKey]) {
|
||||
const easyStore = await this.getEasyStore();
|
||||
easyStore.data[keyArg] = valueArg;
|
||||
await easyStore.save();
|
||||
}
|
||||
|
||||
public async deleteKey(keyArg: keyof T) {
|
||||
const easyStore = await this.getEasyStore();
|
||||
delete easyStore.data[keyArg];
|
||||
await easyStore.save();
|
||||
}
|
||||
|
||||
/**
|
||||
* writes all keyValue pairs in the object argument
|
||||
*/
|
||||
public async writeAll(keyValueObject: Partial<T>) {
|
||||
const easyStore = await this.getEasyStore();
|
||||
easyStore.data = { ...easyStore.data, ...keyValueObject };
|
||||
await easyStore.save();
|
||||
}
|
||||
|
||||
/**
|
||||
* wipes a key value store from disk
|
||||
*/
|
||||
public async wipe() {
|
||||
const easyStore = await this.getEasyStore();
|
||||
easyStore.data = {};
|
||||
await easyStore.save();
|
||||
}
|
||||
|
||||
public async cleanUpEphermal() {
|
||||
while (
|
||||
(await this.smartdataDbRef.statusConnectedDeferred.promise) &&
|
||||
this.smartdataDbRef.status === 'connected'
|
||||
) {}
|
||||
}
|
||||
}
|
37
ts/smartdata.classes.watcher.ts
Normal file
37
ts/smartdata.classes.watcher.ts
Normal file
@ -0,0 +1,37 @@
|
||||
import { SmartDataDbDoc } from './smartdata.classes.doc.js';
|
||||
import * as plugins from './smartdata.plugins.js';
|
||||
|
||||
/**
|
||||
* a wrapper for the native mongodb cursor. Exposes better
|
||||
*/
|
||||
export class SmartdataDbWatcher<T = any> {
|
||||
// STATIC
|
||||
public readyDeferred = plugins.smartpromise.defer();
|
||||
|
||||
// INSTANCE
|
||||
private changeStream: plugins.mongodb.ChangeStream<T>;
|
||||
|
||||
public changeSubject = new plugins.smartrx.rxjs.Subject<T>();
|
||||
constructor(
|
||||
changeStreamArg: plugins.mongodb.ChangeStream<T>,
|
||||
smartdataDbDocArg: typeof SmartDataDbDoc
|
||||
) {
|
||||
this.changeStream = changeStreamArg;
|
||||
this.changeStream.on('change', async (item: any) => {
|
||||
if (!item.fullDocument) {
|
||||
this.changeSubject.next(null);
|
||||
return;
|
||||
}
|
||||
this.changeSubject.next(
|
||||
smartdataDbDocArg.createInstanceFromMongoDbNativeDoc(item.fullDocument) as any as T
|
||||
);
|
||||
});
|
||||
plugins.smartdelay.delayFor(0).then(() => {
|
||||
this.readyDeferred.resolve();
|
||||
});
|
||||
}
|
||||
|
||||
public async close() {
|
||||
await this.changeStream.close();
|
||||
}
|
||||
}
|
3
ts/smartdata.logging.ts
Normal file
3
ts/smartdata.logging.ts
Normal file
@ -0,0 +1,3 @@
|
||||
import * as plugins from './smartdata.plugins.js';
|
||||
|
||||
export const logger = new plugins.smartlog.ConsoleLog();
|
@ -1,3 +0,0 @@
|
||||
export const addPassword = (mongoUrlArg: string, passwordArg: string): string => {
|
||||
return mongoUrlArg.replace('<PASSWORD>', passwordArg);
|
||||
};
|
@ -1,9 +1,29 @@
|
||||
import * as assert from 'assert';
|
||||
import * as smartlog from '@pushrocks/smartlog';
|
||||
import * as lodash from 'lodash';
|
||||
import * as mongodb from 'mongodb';
|
||||
import * as smartq from '@pushrocks/smartpromise';
|
||||
import * as smartstring from '@pushrocks/smartstring';
|
||||
import * as smartunique from '@pushrocks/smartunique';
|
||||
// tsclass scope
|
||||
import * as tsclass from '@tsclass/tsclass';
|
||||
|
||||
export { assert, smartlog, lodash, smartq, mongodb, smartstring, smartunique };
|
||||
export { tsclass };
|
||||
|
||||
// @pushrocks scope
|
||||
import * as lik from '@push.rocks/lik';
|
||||
import * as smartdelay from '@push.rocks/smartdelay';
|
||||
import * as smartlog from '@push.rocks/smartlog';
|
||||
import * as smartpromise from '@push.rocks/smartpromise';
|
||||
import * as smartrx from '@push.rocks/smartrx';
|
||||
import * as smartstring from '@push.rocks/smartstring';
|
||||
import * as smarttime from '@push.rocks/smarttime';
|
||||
import * as smartunique from '@push.rocks/smartunique';
|
||||
import * as taskbuffer from '@push.rocks/taskbuffer';
|
||||
import * as mongodb from 'mongodb';
|
||||
|
||||
export {
|
||||
lik,
|
||||
smartdelay,
|
||||
smartpromise,
|
||||
smartlog,
|
||||
smartrx,
|
||||
mongodb,
|
||||
smartstring,
|
||||
smarttime,
|
||||
smartunique,
|
||||
taskbuffer,
|
||||
};
|
||||
|
@ -1,7 +1,9 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"experimentalDecorators": true,
|
||||
"target": "es2017",
|
||||
"module": "commonjs"
|
||||
"useDefineForClassFields": false,
|
||||
"target": "ES2022",
|
||||
"module": "ES2022",
|
||||
"moduleResolution": "nodenext"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
17
tslint.json
17
tslint.json
@ -1,17 +0,0 @@
|
||||
{
|
||||
"extends": ["tslint:latest", "tslint-config-prettier"],
|
||||
"rules": {
|
||||
"semicolon": [true, "always"],
|
||||
"no-console": false,
|
||||
"ordered-imports": false,
|
||||
"object-literal-sort-keys": false,
|
||||
"member-ordering": {
|
||||
"options":{
|
||||
"order": [
|
||||
"static-method"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"defaultSeverity": "warning"
|
||||
}
|
Reference in New Issue
Block a user