Compare commits

...

153 Commits

Author SHA1 Message Date
57d2d56d00 5.2.6 2024-06-18 20:12:15 +02:00
90751002aa fix(core): update 2024-06-18 20:12:14 +02:00
7606e074a5 5.2.5 2024-06-18 20:11:41 +02:00
7ec39e397e fix(core): update 2024-06-18 20:11:40 +02:00
21d8d3dc32 5.2.4 2024-05-31 18:47:49 +02:00
6d456955d8 fix(core): update 2024-05-31 18:47:48 +02:00
d08544c782 5.2.3 2024-05-31 18:39:34 +02:00
bda9ac8a07 fix(saveableProperties): fix issue where _createdAt and _updatedAt registered saveableProperties for all document types 2024-05-31 18:39:33 +02:00
d27dafba2b 5.2.2 2024-05-31 18:25:52 +02:00
b6594de18c fix(core): update 2024-05-31 18:25:51 +02:00
d9246cbeac update description 2024-05-29 14:12:26 +02:00
9a5864656e 5.2.1 2024-04-16 07:47:25 +02:00
307f0c7277 fix(core): update 2024-04-16 07:47:24 +02:00
62dc897e73 5.2.0 2024-04-15 18:34:14 +02:00
552b344914 feat(SmartDataDbDoc): add static .getCount({}) method 2024-04-15 18:34:13 +02:00
5a2cc2406c 5.1.2 2024-04-15 14:26:22 +02:00
73a11370b6 fix(_createdAt/_updatedAt): fields are now ISO format 2024-04-15 14:26:21 +02:00
162265f353 5.1.1 2024-04-14 04:00:56 +02:00
06776d74c8 fix(core): update 2024-04-14 04:00:56 +02:00
b4cd6b0fe1 5.1.0 2024-04-14 01:26:11 +02:00
b282f69b35 feat(_createdAt & _updatedAt): adds default _createdAt and _updatedAt fields, fixes #1 2024-04-14 01:26:11 +02:00
203a284c88 5.0.43 2024-04-14 01:24:22 +02:00
30ae641a9c fix(core): update 2024-04-14 01:24:21 +02:00
cfe733621f update npmextra.json: githost 2024-04-01 21:34:27 +02:00
1f76e2478e update npmextra.json: githost 2024-04-01 19:57:58 +02:00
7d668bee05 update npmextra.json: githost 2024-03-30 21:46:55 +01:00
bef7f68360 5.0.42 2024-03-30 12:26:43 +01:00
56e9754725 fix(TypeScript): improve tsconfig.json for ES Module use 2024-03-30 12:26:42 +01:00
30d81581cf 5.0.41 2024-03-27 17:32:02 +01:00
5e9db12955 fix(core): update 2024-03-27 17:32:01 +01:00
ad2f422c86 5.0.40 2024-03-27 17:30:51 +01:00
17ce14bcb9 fix(core): update 2024-03-27 17:30:50 +01:00
32319e6e77 5.0.39 2024-03-27 17:30:15 +01:00
4cd284eaa9 fix(core): update 2024-03-27 17:30:14 +01:00
00ec2e57c2 5.0.38 2024-03-27 16:24:58 +01:00
765356ce3d fix(core): update 2024-03-27 16:24:57 +01:00
56b8581d2b 5.0.37 2024-03-26 13:22:34 +01:00
37a9df9086 fix(core): update 2024-03-26 13:22:33 +01:00
090fb668cd 5.0.36 2024-03-26 13:21:37 +01:00
a1c807261c fix(core): update 2024-03-26 13:21:36 +01:00
a2ccf15f69 5.0.35 2024-03-26 00:25:06 +01:00
84d48f1914 fix(core): update 2024-03-26 00:25:06 +01:00
1e258e5ffb 5.0.34 2024-03-22 18:36:35 +01:00
19d5f553b9 fix(core): update 2024-03-22 18:36:34 +01:00
7a257ea925 5.0.33 2023-08-21 12:39:49 +02:00
2fa1e89f34 fix(core): update 2023-08-21 12:39:48 +02:00
d6b3896dd3 5.0.32 2023-08-16 13:16:40 +02:00
49b11b17ce fix(core): update 2023-08-16 13:16:39 +02:00
4ac8a4c0cd 5.0.31 2023-08-16 12:08:28 +02:00
7f9983382a fix(core): update 2023-08-16 12:08:27 +02:00
54f529b0a7 5.0.30 2023-08-15 19:55:53 +02:00
f542463bf6 fix(core): update 2023-08-15 19:55:52 +02:00
1235ae2eb3 5.0.29 2023-08-15 19:55:23 +02:00
8166d2f7c2 fix(core): update 2023-08-15 19:55:22 +02:00
7c9f27e02f 5.0.28 2023-08-15 01:24:30 +02:00
842e4b280b fix(core): update 2023-08-15 01:24:29 +02:00
009f3297b2 5.0.27 2023-08-15 01:01:16 +02:00
2ff3a4e0b7 fix(core): update 2023-08-15 01:01:16 +02:00
0e55cd8876 5.0.26 2023-08-12 23:32:40 +02:00
eccdf3f00a fix(core): update 2023-08-12 23:32:39 +02:00
c7544133d9 5.0.25 2023-08-12 23:32:02 +02:00
c7c9acf5bd fix(core): update 2023-08-12 23:32:02 +02:00
c99ec50853 5.0.24 2023-08-10 18:06:46 +02:00
4dd9557e1d fix(core): update 2023-08-10 18:06:45 +02:00
52b34a6da1 5.0.23 2023-07-21 20:09:51 +02:00
1bf74fe04d fix(core): update 2023-07-21 20:09:51 +02:00
fdd875ad31 5.0.22 2023-07-21 20:08:50 +02:00
a7bf0c0298 fix(core): update 2023-07-21 20:08:50 +02:00
59d6336e43 5.0.21 2023-07-21 20:08:18 +02:00
e0fc81179a fix(core): update 2023-07-21 20:08:18 +02:00
5aa81a56a2 switch to new org scheme 2023-07-10 02:43:00 +02:00
9ae26177b8 5.0.20 2023-06-25 02:01:20 +02:00
26ac52d6c5 fix(core): update 2023-06-25 02:01:19 +02:00
fb39463b7d 5.0.19 2023-06-25 01:53:17 +02:00
44acba80c1 fix(core): update 2023-06-25 01:53:17 +02:00
8cf8315577 5.0.18 2023-06-25 01:51:44 +02:00
9b44b64a50 fix(core): update 2023-06-25 01:51:43 +02:00
699e25201c 5.0.17 2023-06-25 01:30:14 +02:00
2ef9aace68 fix(core): update 2023-06-25 01:30:14 +02:00
cc55a57dfd 5.0.16 2023-06-25 01:29:40 +02:00
b2df512552 fix(core): update 2023-06-25 01:29:39 +02:00
23c62fbd69 5.0.15 2023-06-25 01:28:30 +02:00
5f70ea0b05 fix(core): update 2023-06-25 01:28:29 +02:00
49a595876a 5.0.14 2023-06-25 01:27:10 +02:00
db38a1ef85 fix(core): update 2023-06-25 01:27:09 +02:00
94854638dd 5.0.13 2023-06-24 23:57:35 +02:00
902fab4cc0 fix(core): update 2023-06-24 23:57:34 +02:00
ed3b19abc5 5.0.12 2023-03-21 20:16:08 +01:00
5b88da7dce fix(core): update 2023-03-21 20:16:07 +01:00
df273e9efa 5.0.11 2023-02-06 11:43:11 +01:00
fd590e0be3 fix(core): update 2023-02-06 11:43:11 +01:00
ef97b390d4 5.0.10 2022-11-08 10:26:21 +01:00
cd14eb8bf3 fix(core): update 2022-11-08 10:26:20 +01:00
f48443dcd3 5.0.9 2022-11-01 18:23:58 +01:00
3f28ff80cb fix(core): update 2022-11-01 18:23:57 +01:00
64005a0b32 5.0.8 2022-09-12 11:37:54 +02:00
8a77bb3281 fix(core): update 2022-09-12 11:37:53 +02:00
25f50ecf51 5.0.7 2022-06-14 22:04:35 +02:00
ad87f8147b fix(core): update 2022-06-14 22:04:34 +02:00
da0c9873eb 5.0.6 2022-06-14 22:02:57 +02:00
2fcd3f1550 fix(core): update 2022-06-14 22:02:57 +02:00
f726cf4c5b 5.0.5 2022-06-05 17:19:12 +02:00
c198969fae fix(core): update 2022-06-05 17:19:12 +02:00
be1badeb23 5.0.4 2022-06-05 17:18:13 +02:00
fe065b966f fix(core): update 2022-06-05 17:18:13 +02:00
811e2490b8 5.0.3 2022-05-19 16:15:28 +02:00
206ccd40e9 fix(watcher.changeSubject): now emits correct type into observer functions 2022-05-19 16:15:28 +02:00
055298172f 5.0.2 2022-05-18 18:17:12 +02:00
278f3c8169 fix(tests): now uses @pushrocks/smartmongo backed by wiredTiger 2022-05-18 18:17:11 +02:00
0709ba921b 5.0.1 2022-05-17 23:54:26 +02:00
de1f1110b4 fix(core): update 2022-05-17 23:54:26 +02:00
30f4254428 5.0.0 2022-05-17 21:30:53 +02:00
1c4b03e647 BREAKING CHANGE(core): switch to esm 2022-05-17 21:30:53 +02:00
27cc7651ba 4.0.29 2022-05-17 21:26:17 +02:00
355a2a3f2b fix(core): update 2022-05-17 21:26:17 +02:00
a739582861 4.0.28 2022-05-17 00:33:44 +02:00
37f9a64735 fix(core): update 2022-05-17 00:33:44 +02:00
83a5170591 4.0.27 2021-11-12 19:32:54 +01:00
f94363cf31 fix(core): update 2021-11-12 19:32:54 +01:00
df02e5bb71 4.0.26 2021-11-12 19:16:11 +01:00
38e438c54f fix(core): update 2021-11-12 19:16:11 +01:00
11bc1ac6dc 4.0.25 2021-11-12 19:03:06 +01:00
3431e94ddd fix(core): update 2021-11-12 19:03:06 +01:00
739e040776 4.0.24 2021-11-12 19:02:29 +01:00
28d57efd9e fix(core): update 2021-11-12 19:02:29 +01:00
f50a61308c 4.0.23 2021-11-12 18:12:59 +01:00
42aa9f9f8a fix(core): update 2021-11-12 18:12:59 +01:00
3f591ff9d8 4.0.22 2021-11-12 18:04:08 +01:00
7b33347b4c fix(core): update 2021-11-12 18:04:08 +01:00
3f11cbf595 4.0.21 2021-11-12 17:32:43 +01:00
cf1ec7f9eb fix(core): update 2021-11-12 17:32:43 +01:00
54060deb8f 4.0.20 2021-11-12 17:22:32 +01:00
48cffb5ac2 fix(core): update 2021-11-12 17:22:31 +01:00
8301eb79a2 4.0.19 2021-11-12 16:36:25 +01:00
ad6366a294 fix(core): update 2021-11-12 16:36:25 +01:00
cb6c03ebfe 4.0.18 2021-11-12 16:23:27 +01:00
551916fe5c fix(core): update 2021-11-12 16:23:26 +01:00
9b3892c1e8 4.0.17 2021-10-18 14:37:17 +02:00
c1b1af9c5d fix(dbdoc manager access working again ): update 2021-10-18 14:37:17 +02:00
d3a3d5be9d 4.0.16 2021-10-16 23:33:22 +02:00
c9a734d879 fix(core): update 2021-10-16 23:33:21 +02:00
856e8e7d1f 4.0.15 2021-10-16 21:17:03 +02:00
7a4d557724 fix(core): update 2021-10-16 21:17:02 +02:00
7cbd0bd99b 4.0.14 2021-10-16 19:54:06 +02:00
e10f6585a5 fix(core): update 2021-10-16 19:54:05 +02:00
5c8dffdd9c 4.0.13 2021-10-02 15:03:35 +02:00
846996eeac fix(core): update 2021-10-02 15:03:35 +02:00
668df09ba0 4.0.12 2021-09-19 17:05:09 +02:00
03656f4ca0 fix(core): update 2021-09-19 17:05:08 +02:00
c4c612f3a9 4.0.11 2021-09-18 00:38:21 +02:00
e357f7581c fix(core): update 2021-09-18 00:38:20 +02:00
9697b1e48b 4.0.10 2021-09-17 22:34:15 +02:00
aeb35705d4 fix(core): update 2021-09-17 22:34:15 +02:00
33 changed files with 8415 additions and 26812 deletions

View File

@ -0,0 +1,66 @@
name: Default (not tags)
on:
push:
tags-ignore:
- '**'
env:
IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
NPMCI_URL_CLOUDLY: ${{secrets.NPMCI_URL_CLOUDLY}}
jobs:
security:
runs-on: ubuntu-latest
continue-on-error: true
container:
image: ${{ env.IMAGE }}
steps:
- uses: actions/checkout@v3
- name: Install pnpm and npmci
run: |
pnpm install -g pnpm
pnpm install -g @shipzone/npmci
- name: Run npm prepare
run: npmci npm prepare
- name: Audit production dependencies
run: |
npmci command npm config set registry https://registry.npmjs.org
npmci command pnpm audit --audit-level=high --prod
continue-on-error: true
- name: Audit development dependencies
run: |
npmci command npm config set registry https://registry.npmjs.org
npmci command pnpm audit --audit-level=high --dev
continue-on-error: true
test:
if: ${{ always() }}
needs: security
runs-on: ubuntu-latest
container:
image: ${{ env.IMAGE }}
steps:
- uses: actions/checkout@v3
- name: Test stable
run: |
npmci node install stable
npmci npm install
npmci npm test
- name: Test build
run: |
npmci node install stable
npmci npm install
npmci npm build

View File

@ -0,0 +1,124 @@
name: Default (tags)
on:
push:
tags:
- '*'
env:
IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
NPMCI_URL_CLOUDLY: ${{secrets.NPMCI_URL_CLOUDLY}}
jobs:
security:
runs-on: ubuntu-latest
continue-on-error: true
container:
image: ${{ env.IMAGE }}
steps:
- uses: actions/checkout@v3
- name: Prepare
run: |
pnpm install -g pnpm
pnpm install -g @shipzone/npmci
npmci npm prepare
- name: Audit production dependencies
run: |
npmci command npm config set registry https://registry.npmjs.org
npmci command pnpm audit --audit-level=high --prod
continue-on-error: true
- name: Audit development dependencies
run: |
npmci command npm config set registry https://registry.npmjs.org
npmci command pnpm audit --audit-level=high --dev
continue-on-error: true
test:
if: ${{ always() }}
needs: security
runs-on: ubuntu-latest
container:
image: ${{ env.IMAGE }}
steps:
- uses: actions/checkout@v3
- name: Prepare
run: |
pnpm install -g pnpm
pnpm install -g @shipzone/npmci
npmci npm prepare
- name: Test stable
run: |
npmci node install stable
npmci npm install
npmci npm test
- name: Test build
run: |
npmci node install stable
npmci npm install
npmci npm build
release:
needs: test
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
runs-on: ubuntu-latest
container:
image: ${{ env.IMAGE }}
steps:
- uses: actions/checkout@v3
- name: Prepare
run: |
pnpm install -g pnpm
pnpm install -g @shipzone/npmci
npmci npm prepare
- name: Release
run: |
npmci node install stable
npmci npm publish
metadata:
needs: test
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
runs-on: ubuntu-latest
container:
image: ${{ env.IMAGE }}
continue-on-error: true
steps:
- uses: actions/checkout@v3
- name: Prepare
run: |
pnpm install -g pnpm
pnpm install -g @shipzone/npmci
npmci npm prepare
- name: Code quality
run: |
npmci command npm install -g typescript
npmci npm install
- name: Trigger
run: npmci trigger
- name: Build docs and upload artifacts
run: |
npmci node install stable
npmci npm install
pnpm install -g @gitzone/tsdoc
npmci command tsdoc
continue-on-error: true

View File

@ -1,137 +0,0 @@
# gitzone ci_default
image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
cache:
paths:
- .npmci_cache/
key: '$CI_BUILD_STAGE'
stages:
- security
- test
- release
- metadata
# ====================
# security stage
# ====================
mirror:
stage: security
script:
- npmci git mirror
only:
- tags
tags:
- lossless
- docker
- notpriv
auditProductionDependencies:
image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
stage: security
script:
- npmci npm prepare
- npmci command npm install --production --ignore-scripts
- npmci command npm config set registry https://registry.npmjs.org
- npmci command npm audit --audit-level=high --only=prod --production
tags:
- docker
auditDevDependencies:
image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
stage: security
script:
- npmci npm prepare
- npmci command npm install --ignore-scripts
- npmci command npm config set registry https://registry.npmjs.org
- npmci command npm audit --audit-level=high --only=dev
tags:
- docker
allow_failure: true
# ====================
# test stage
# ====================
testStable:
stage: test
script:
- npmci npm prepare
- npmci node install stable
- npmci npm install
- npmci npm test
coverage: /\d+.?\d+?\%\s*coverage/
tags:
- docker
testBuild:
stage: test
script:
- npmci npm prepare
- npmci node install stable
- npmci npm install
- npmci command npm run build
coverage: /\d+.?\d+?\%\s*coverage/
tags:
- docker
release:
stage: release
script:
- npmci node install stable
- npmci npm publish
only:
- tags
tags:
- lossless
- docker
- notpriv
# ====================
# metadata stage
# ====================
codequality:
stage: metadata
allow_failure: true
only:
- tags
script:
- npmci command npm install -g tslint typescript
- npmci npm prepare
- npmci npm install
- npmci command "tslint -c tslint.json ./ts/**/*.ts"
tags:
- lossless
- docker
- priv
trigger:
stage: metadata
script:
- npmci trigger
only:
- tags
tags:
- lossless
- docker
- notpriv
pages:
stage: metadata
script:
- npmci node install lts
- npmci command npm install -g @gitzone/tsdoc
- npmci npm prepare
- npmci npm install
- npmci command tsdoc
tags:
- lossless
- docker
- notpriv
only:
- tags
artifacts:
expire_in: 1 week
paths:
- public
allow_failure: true

24
.vscode/launch.json vendored
View File

@ -2,28 +2,10 @@
"version": "0.2.0",
"configurations": [
{
"name": "current file",
"type": "node",
"command": "npm test",
"name": "Run npm test",
"request": "launch",
"args": [
"${relativeFile}"
],
"runtimeArgs": ["-r", "@gitzone/tsrun"],
"cwd": "${workspaceRoot}",
"protocol": "inspector",
"internalConsoleOptions": "openOnSessionStart"
},
{
"name": "test.ts",
"type": "node",
"request": "launch",
"args": [
"test/test.ts"
],
"runtimeArgs": ["-r", "@gitzone/tsrun"],
"cwd": "${workspaceRoot}",
"protocol": "inspector",
"internalConsoleOptions": "openOnSessionStart"
"type": "node-terminal"
}
]
}

View File

@ -12,12 +12,25 @@
"gitzone": {
"projectType": "npm",
"module": {
"githost": "gitlab.com",
"gitscope": "pushrocks",
"githost": "code.foss.global",
"gitscope": "push.rocks",
"gitrepo": "smartdata",
"shortDescription": "do more with data",
"npmPackagename": "@pushrocks/smartdata",
"license": "MIT"
"description": "An advanced library for NoSQL data organization and manipulation using TypeScript with support for MongoDB, data validation, collections, and custom data types.",
"npmPackagename": "@push.rocks/smartdata",
"license": "MIT",
"keywords": [
"data manipulation",
"NoSQL",
"MongoDB",
"TypeScript",
"data validation",
"collections",
"custom data types",
"ODM"
]
}
},
"tsdoc": {
"legal": "\n## License and Legal Information\n\nThis repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository. \n\n**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.\n\n### Trademarks\n\nThis project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.\n\n### Company Information\n\nTask Venture Capital GmbH \nRegistered at District court Bremen HRB 35230 HB, Germany\n\nFor any legal inquiries or if you require further information, please contact us via email at hello@task.vc.\n\nBy using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.\n"
}
}

26257
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -1,49 +1,47 @@
{
"name": "@pushrocks/smartdata",
"version": "4.0.9",
"name": "@push.rocks/smartdata",
"version": "5.2.6",
"private": false,
"description": "do more with data",
"description": "An advanced library for NoSQL data organization and manipulation using TypeScript with support for MongoDB, data validation, collections, and custom data types.",
"main": "dist_ts/index.js",
"typings": "dist_ts/index.d.ts",
"type": "module",
"scripts": {
"test": "(tstest test/)",
"testLocal": "(npmdocker)",
"build": "(tsbuild --web)"
"test": "tstest test/",
"build": "tsbuild --web --allowimplicitany",
"buildDocs": "tsdoc"
},
"repository": {
"type": "git",
"url": "git+ssh://git@gitlab.com/pushrocks/smartdata.git"
"url": "https://code.foss.global/push.rocks/smartdata.git"
},
"author": "Lossless GmbH",
"license": "MIT",
"bugs": {
"url": "https://gitlab.com/pushrocks/smartdata/issues"
},
"homepage": "https://gitlab.com/pushrocks/smartdata#README",
"homepage": "https://code.foss.global/push.rocks/smartdata",
"dependencies": {
"@pushrocks/lik": "^4.0.20",
"@pushrocks/smartlog": "^2.0.39",
"@pushrocks/smartpromise": "^3.1.5",
"@pushrocks/smartstring": "^3.0.24",
"@tsclass/tsclass": "^3.0.33",
"@types/lodash": "^4.14.169",
"@types/mongodb": "^3.6.12",
"lodash": "^4.17.21",
"mongodb": "^3.6.6",
"runtime-type-checks": "0.0.4"
"@push.rocks/lik": "^6.0.14",
"@push.rocks/smartdelay": "^3.0.1",
"@push.rocks/smartlog": "^3.0.2",
"@push.rocks/smartmongo": "^2.0.10",
"@push.rocks/smartpromise": "^4.0.2",
"@push.rocks/smartrx": "^3.0.7",
"@push.rocks/smartstring": "^4.0.15",
"@push.rocks/smarttime": "^4.0.6",
"@push.rocks/smartunique": "^3.0.8",
"@push.rocks/taskbuffer": "^3.1.7",
"@tsclass/tsclass": "^4.0.52",
"mongodb": "^6.5.0"
},
"devDependencies": {
"@gitzone/tsbuild": "^2.1.25",
"@gitzone/tstest": "^1.0.54",
"@pushrocks/qenv": "^4.0.10",
"@pushrocks/smartunique": "^3.0.3",
"@pushrocks/tapbundle": "^3.2.14",
"@types/mongodb-memory-server": "^2.3.0",
"@types/node": "^15.3.0",
"@types/shortid": "0.0.29",
"mongodb-memory-server": "^6.9.6",
"tslint": "^6.1.3",
"tslint-config-prettier": "^1.18.0"
"@gitzone/tsbuild": "^2.1.66",
"@gitzone/tsrun": "^1.2.44",
"@gitzone/tstest": "^1.0.77",
"@push.rocks/qenv": "^6.0.5",
"@push.rocks/tapbundle": "^5.0.22",
"@types/node": "^20.11.30"
},
"files": [
"ts/**/*",
@ -59,5 +57,15 @@
],
"browserslist": [
"last 1 chrome versions"
],
"keywords": [
"data manipulation",
"NoSQL",
"MongoDB",
"TypeScript",
"data validation",
"collections",
"custom data types",
"ODM"
]
}

6881
pnpm-lock.yaml generated Normal file

File diff suppressed because it is too large Load Diff

View File

@ -1,4 +1 @@
required:
- MONGO_URL
- MONGO_DBNAME
- MONGO_PASS

0
readme.hints.md Normal file
View File

197
readme.md
View File

@ -1,156 +1,115 @@
# @pushrocks/smartdata
# @push.rocks/smartdata
do more with data
## Availabililty and Links
* [npmjs.org (npm package)](https://www.npmjs.com/package/@pushrocks/smartdata)
* [gitlab.com (source)](https://gitlab.com/pushrocks/smartdata)
* [github.com (source mirror)](https://github.com/pushrocks/smartdata)
* [docs (typedoc)](https://pushrocks.gitlab.io/smartdata/)
## Install
To install `@push.rocks/smartdata`, use npm:
## Status for master
```bash
npm install @push.rocks/smartdata --save
```
Status Category | Status Badge
-- | --
GitLab Pipelines | [![pipeline status](https://gitlab.com/pushrocks/smartdata/badges/master/pipeline.svg)](https://lossless.cloud)
GitLab Pipline Test Coverage | [![coverage report](https://gitlab.com/pushrocks/smartdata/badges/master/coverage.svg)](https://lossless.cloud)
npm | [![npm downloads per month](https://badgen.net/npm/dy/@pushrocks/smartdata)](https://lossless.cloud)
Snyk | [![Known Vulnerabilities](https://badgen.net/snyk/pushrocks/smartdata)](https://lossless.cloud)
TypeScript Support | [![TypeScript](https://badgen.net/badge/TypeScript/>=%203.x/blue?icon=typescript)](https://lossless.cloud)
node Support | [![node](https://img.shields.io/badge/node->=%2010.x.x-blue.svg)](https://nodejs.org/dist/latest-v10.x/docs/api/)
Code Style | [![Code Style](https://badgen.net/badge/style/prettier/purple)](https://lossless.cloud)
PackagePhobia (total standalone install weight) | [![PackagePhobia](https://badgen.net/packagephobia/install/@pushrocks/smartdata)](https://lossless.cloud)
PackagePhobia (package size on registry) | [![PackagePhobia](https://badgen.net/packagephobia/publish/@pushrocks/smartdata)](https://lossless.cloud)
BundlePhobia (total size when bundled) | [![BundlePhobia](https://badgen.net/bundlephobia/minzip/@pushrocks/smartdata)](https://lossless.cloud)
Platform support | [![Supports Windows 10](https://badgen.net/badge/supports%20Windows%2010/yes/green?icon=windows)](https://lossless.cloud) [![Supports Mac OS X](https://badgen.net/badge/supports%20Mac%20OS%20X/yes/green?icon=apple)](https://lossless.cloud)
This will add `@push.rocks/smartdata` to your project's dependencies.
## Usage
`@push.rocks/smartdata` enables efficient data handling and operation management with a focus on using MongoDB. It leverages TypeScript for strong typing and ESM syntax for modern JavaScript usage. Below are various scenarios demonstrating how to utilize this package effectively in a project.
Use TypeScript for best in class instellisense.
smartdata is an ODM that adheres to TypeScript practices and uses classes to organize data.
It uses RethinkDB as persistent storage.
## Intention
There are many ODMs out there, however when we searched for an ODM that uses TypeScript,
acts smart while still embracing the NoSQL idea we didn't find a matching solution.
This is why we started smartdata.
How RethinkDB's terms map to the ones of smartdata:
| MongoDb term | smartdata class |
| ------------ | ----------------------------- |
| Database | smartdata.SmartdataDb |
| Collection | smartdata.SmartdataCollection |
| Document | smartdata.SmartadataDoc |
### class Db
represents a Database. Naturally it has .connect() etc. methods on it.
### Setting Up and Connecting to the Database
Before interacting with the database, you need to set up and establish a connection. This is done by creating an instance of `SmartdataDb` and calling its `init` method with your MongoDB connection details.
```typescript
// Assuming toplevel await
import * as smartdata from 'smartdata';
import { SmartdataDb } from '@push.rocks/smartdata';
const smartdataDb = new smartdata.SmartdataDb({
mongoDbUrl: '//someurl',
mongoDbName: 'myDatabase',
mongoDbPass: 'mypassword',
// Create a new instance of SmartdataDb with MongoDB connection details
const db = new SmartdataDb({
mongoDbUrl: 'mongodb://localhost:27017',
mongoDbName: 'your-database-name',
mongoDbUser: 'your-username',
mongoDbPass: 'your-password',
});
await smartdataDb.connect();
// Initialize and connect to the database
await db.init();
```
### class DbCollection
represents a collection of objects.
A collection is defined by the object class (that is extending smartdata.dbdoc) it respresents
So to get to get access to a specific collection you document
### Defining Data Models
Data models in `@push.rocks/smartdata` are classes that represent collections and documents in your MongoDB database. Use decorators such as `@Collection`, `@unI`, and `@svDb` to define your data models.
```typescript
// Assuming toplevel await
// continues from the block before...
import { SmartDataDbDoc, Collection, unI, svDb } from '@push.rocks/smartdata';
@smartdata.Collection(smartdataDb)
class MyObject extends smartdata.DbDoc<MyObject /* ,[an optional interface to implement] */> {
// read the next block about DbDoc
@smartdata.svDb()
property1: string; // @smartdata.svDb() marks the property for db save
property2: number; // this one is not marked, so it won't be save upon calling this.save()
constructor() {
super(); // the super call is important ;) But you probably know that.
@Collection(() => db) // Associate this model with the database instance
class User extends SmartDataDbDoc<User, User> {
@unI()
public id: string = 'unique-user-id'; // Mark 'id' as a unique index
@svDb()
public username: string; // Mark 'username' to be saved in DB
@svDb()
public email: string; // Mark 'email' to be saved in DB
constructor(username: string, email: string) {
super();
this.username = username;
this.email = email;
}
}
// start to instantiate instances of classes from scratch or database
const localObject = new MyObject({
property1: 'hi',
property2: {
deep: 3
},
});
await localObject.save(); // saves the object to the database
// start retrieving instances
// .getInstance is staticly inheritied, yet fully typed static function to get instances with fully typed filters
const myInstance = await MyObject.getInstance({
property1: 'hi',
property2: {
deep: {
$gt: 2
} as any
}
}); // outputs a new instance of MyObject with the values from db assigned
```
### class DbDoc
### Performing CRUD Operations
`@push.rocks/smartdata` simplifies CRUD operations with intuitive methods on model instances.
represents a individual document in a collection
and thereby is ideally suited to extend the class you want to actually store.
#### Create
```typescript
const newUser = new User('myUsername', 'myEmail@example.com');
await newUser.save(); // Save the new user to the database
```
### CRUD operations
#### Read
```typescript
// Fetch a single user by a unique attribute
const user = await User.getInstance({ username: 'myUsername' });
smartdata supports full CRUD operations
// Fetch multiple users that match criteria
const users = await User.getInstances({ email: 'myEmail@example.com' });
```
**Store** or **Update** instances of classes to MongoDB:
DbDoc extends your class with the following methods:
#### Update
```typescript
// Assuming 'user' is an instance of User
user.email = 'newEmail@example.com';
await user.save(); // Update the user in the database
```
- async `.save()` will save (or update) the object you call it on only. Any referenced non-savable objects will not get stored.
- async `.saveDeep()` does the same like `.save()`.
In addition it will look for properties that reference an object
that extends DbDoc as well and call .saveDeep() on them as well.
Loops are prevented
#### Delete
```typescript
// Assuming 'user' is an instance of User
await user.delete(); // Delete the user from the database
```
**Get** a new class instance from MongoDB:
DbDoc exposes a static method that allows you specify a filter to retrieve a cloned class of the one you used to that doc at some point later in time:
### Advanced Usage
`@push.rocks/smartdata` also supports advanced features like watching for real-time changes in the database, handling distributed data coordination, and more. These features utilize MongoDB's capabilities to provide real-time data syncing and distributed systems coordination.
- static async `.getInstance({ /* filter props here */ })` gets you an instance that has the data of the first matched document as properties.
- static async `getInstances({ /* filter props here */ })` get you an array instances (one instance for every matched document).
### Conclusion
With its focus on TypeScript, modern JavaScript syntax, and leveraging MongoDB's features, `@push.rocks/smartdata` offers a powerful toolkit for data handling and operations management in Node.js applications. Its design for ease of use, coupled with advanced features, makes it a versatile choice for developers looking to build efficient and scalable data-driven applications.
**Delete** instances from MongoDb:
smartdata extends your class with a method to easily delete the doucment from DB:
For more details on usage and additional features, refer to the [official documentation](https://gitlab.com/push.rocks/smartdata#README) and explore the various classes and methods provided by `@push.rocks/smartdata`.
- async `.delete()`will delete the document from DB.
## License and Legal Information
## TypeScript
This repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository.
How does TypeScript play into this?
Since you define your classes in TypeScript and types flow through smartdata in a generic way
you should get all the Intellisense and type checking you love when using smartdata.
smartdata itself also bundles typings. You don't need to install any additional types for smartdata.
**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.
## Contribution
### Trademarks
We are always happy for code contributions. If you are not the code contributing type that is ok. Still, maintaining Open Source repositories takes considerable time and thought. If you like the quality of what we do and our modules are useful to you we would appreciate a little monthly contribution: You can [contribute one time](https://lossless.link/contribute-onetime) or [contribute monthly](https://lossless.link/contribute). :)
This project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.
For further information read the linked docs at the top of this readme.
### Company Information
> MIT licensed | **&copy;** [Lossless GmbH](https://lossless.gmbh)
| By using this npm module you agree to our [privacy policy](https://lossless.gmbH/privacy)
Task Venture Capital GmbH
Registered at District court Bremen HRB 35230 HB, Germany
[![repo-footer](https://lossless.gitlab.io/publicrelations/repofooter.svg)](https://maintainedby.lossless.com)
For any legal inquiries or if you require further information, please contact us via email at hello@task.vc.
By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.

View File

@ -0,0 +1,112 @@
import { tap, expect } from '@push.rocks/tapbundle';
import * as smartmongo from '@push.rocks/smartmongo';
import type * as taskbuffer from '@push.rocks/taskbuffer';
import * as smartdata from '../ts/index.js';
import { SmartdataDistributedCoordinator, DistributedClass } from '../ts/smartdata.classes.distributedcoordinator.js'; // path might need adjusting
const totalInstances = 10;
// =======================================
// Connecting to the database server
// =======================================
let smartmongoInstance: smartmongo.SmartMongo;
let testDb: smartdata.SmartdataDb;
tap.test('should create a testinstance as database', async () => {
smartmongoInstance = await smartmongo.SmartMongo.createAndStart();
testDb = new smartdata.SmartdataDb(await smartmongoInstance.getMongoDescriptor());
await testDb.init();
});
tap.test('should instantiate DistributedClass', async (tools) => {
const instance = new DistributedClass();
expect(instance).toBeInstanceOf(DistributedClass);
});
tap.test('DistributedClass should update the time', async (tools) => {
const distributedCoordinator = new SmartdataDistributedCoordinator(testDb);
await distributedCoordinator.start();
const initialTime = distributedCoordinator.ownInstance.data.lastUpdated;
await distributedCoordinator.sendHeartbeat();
const updatedTime = distributedCoordinator.ownInstance.data.lastUpdated;
expect(updatedTime).toBeGreaterThan(initialTime);
await distributedCoordinator.stop();
});
tap.test('should instantiate SmartdataDistributedCoordinator', async (tools) => {
const distributedCoordinator = new SmartdataDistributedCoordinator(testDb);
await distributedCoordinator.start();
expect(distributedCoordinator).toBeInstanceOf(SmartdataDistributedCoordinator);
await distributedCoordinator.stop();
});
tap.test('SmartdataDistributedCoordinator should update leader status', async (tools) => {
const distributedCoordinator = new SmartdataDistributedCoordinator(testDb);
await distributedCoordinator.start();
await distributedCoordinator.checkAndMaybeLead();
expect(distributedCoordinator.ownInstance.data.elected).toBeOneOf([true, false]);
await distributedCoordinator.stop();
});
tap.test('SmartdataDistributedCoordinator should handle distributed task requests', async (tools) => {
const distributedCoordinator = new SmartdataDistributedCoordinator(testDb);
await distributedCoordinator.start();
const mockTaskRequest: taskbuffer.distributedCoordination.IDistributedTaskRequest = {
submitterId: "mockSubmitter12345", // Some unique mock submitter ID
requestResponseId: 'uni879873462hjhfkjhsdf', // Some unique ID for the request-response
taskName: "SampleTask",
taskVersion: "1.0.0", // Assuming it's a version string
taskExecutionTime: Date.now(),
taskExecutionTimeout: 60000, // Let's say the timeout is 1 minute (60000 ms)
taskExecutionParallel: 5, // Let's assume max 5 parallel executions
status: 'requesting'
};
const response = await distributedCoordinator.fireDistributedTaskRequest(mockTaskRequest);
console.log(response) // based on your expected structure for the response
await distributedCoordinator.stop();
});
tap.test('SmartdataDistributedCoordinator should update distributed task requests', async (tools) => {
const distributedCoordinator = new SmartdataDistributedCoordinator(testDb);
await distributedCoordinator.start();
const mockTaskRequest: taskbuffer.distributedCoordination.IDistributedTaskRequest = {
submitterId: "mockSubmitter12345", // Some unique mock submitter ID
requestResponseId: 'uni879873462hjhfkjhsdf', // Some unique ID for the request-response
taskName: "SampleTask",
taskVersion: "1.0.0", // Assuming it's a version string
taskExecutionTime: Date.now(),
taskExecutionTimeout: 60000, // Let's say the timeout is 1 minute (60000 ms)
taskExecutionParallel: 5, // Let's assume max 5 parallel executions
status: 'requesting'
};
await distributedCoordinator.updateDistributedTaskRequest(mockTaskRequest);
// Here, we can potentially check if a DB entry got updated or some other side-effect of the update method.
await distributedCoordinator.stop();
});
tap.test('should elect only one leader amongst multiple instances', async (tools) => {
const coordinators = Array.from({ length: totalInstances }).map(() => new SmartdataDistributedCoordinator(testDb));
await Promise.all(coordinators.map(coordinator => coordinator.start()));
const leaders = coordinators.filter(coordinator => coordinator.ownInstance.data.elected);
for (const leader of leaders) {
console.log(leader.ownInstance);
}
expect(leaders.length).toEqual(1);
// stopping clears a coordinator from being elected.
await Promise.all(coordinators.map(coordinator => coordinator.stop()));
});
tap.test('should clean up', async () => {
await smartmongoInstance.stopAndDumpToDir(`.nogit/dbdump/test.distributedcoordinator.ts`);
setTimeout(() => process.exit(), 2000);
})
tap.start({ throwOnError: true });

55
test/test.easystore.ts Normal file
View File

@ -0,0 +1,55 @@
import { tap, expect } from '@push.rocks/tapbundle';
import { Qenv } from '@push.rocks/qenv';
import * as smartmongo from '@push.rocks/smartmongo';
import { smartunique } from '../ts/smartdata.plugins.js';
const testQenv = new Qenv(process.cwd(), process.cwd() + '/.nogit/');
console.log(process.memoryUsage());
// the tested module
import * as smartdata from '../ts/index.js';
// =======================================
// Connecting to the database server
// =======================================
let smartmongoInstance: smartmongo.SmartMongo;
let testDb: smartdata.SmartdataDb;
tap.test('should create a testinstance as database', async () => {
smartmongoInstance = await smartmongo.SmartMongo.createAndStart();
testDb = new smartdata.SmartdataDb(await smartmongoInstance.getMongoDescriptor());
await testDb.init();
});
tap.skip.test('should connect to atlas', async (tools) => {
const databaseName = `test-smartdata-${smartunique.shortId()}`;
testDb = new smartdata.SmartdataDb({
mongoDbUrl: await testQenv.getEnvVarOnDemand('MONGO_URL'),
mongoDbName: databaseName,
});
await testDb.init();
});
let easyStore: smartdata.EasyStore<{
key1: string;
key2: string;
}>;
tap.test('should create an easystore', async () => {
easyStore = await testDb.createEasyStore('hellothere');
await easyStore.writeKey('key1', 'hello');
const retrievedKey = await easyStore.readKey('key1');
expect(retrievedKey).toEqual('hello');
});
tap.test('close', async () => {
await testDb.mongoDb.dropDatabase();
await testDb.close();
if (smartmongoInstance) {
await smartmongoInstance.stop();
}
});
tap.start();

View File

@ -1,51 +1,38 @@
import { tap, expect } from '@pushrocks/tapbundle';
import { Qenv } from '@pushrocks/qenv';
import { tap, expect } from '@push.rocks/tapbundle';
import { Qenv } from '@push.rocks/qenv';
import * as smartmongo from '@push.rocks/smartmongo';
import { smartunique } from '../ts/smartdata.plugins.js';
import * as mongodb from 'mongodb';
const testQenv = new Qenv(process.cwd(), process.cwd() + '/.nogit/');
console.log(process.memoryUsage());
// the tested module
import * as smartdata from '../ts/index';
import * as mongoPlugin from 'mongodb-memory-server';
import { smartunique } from '../ts/smartdata.plugins';
import * as smartdata from '../ts/index.js';
// =======================================
// Connecting to the database server
// =======================================
let smartmongoInstance: smartmongo.SmartMongo;
let testDb: smartdata.SmartdataDb;
let smartdataOptions: smartdata.IMongoDescriptor;
let mongod: mongoPlugin.MongoMemoryServer;
const totalCars = 2000;
tap.skip.test('should create a testinstance as database', async () => {
mongod = new mongoPlugin.MongoMemoryServer({});
console.log('created mongod instance');
await mongod._startUpInstance().catch((err) => {
console.log(err);
});
console.log('mongod started');
smartdataOptions = {
mongoDbName: await mongod.getDbName(),
mongoDbPass: '',
mongoDbUrl: await mongod.getUri(),
};
console.log(smartdataOptions);
testDb = new smartdata.SmartdataDb(smartdataOptions);
tap.test('should create a testinstance as database', async () => {
smartmongoInstance = await smartmongo.SmartMongo.createAndStart();
testDb = new smartdata.SmartdataDb(await smartmongoInstance.getMongoDescriptor());
await testDb.init();
});
tap.test('should connect to atlas', async (tools) => {
tap.skip.test('should connect to atlas', async (tools) => {
const databaseName = `test-smartdata-${smartunique.shortId()}`;
testDb = new smartdata.SmartdataDb({
mongoDbUrl: testQenv.getEnvVarOnDemand('MONGO_URL'),
mongoDbUrl: await testQenv.getEnvVarOnDemand('MONGO_URL'),
mongoDbName: databaseName,
});
});
tap.test('should establish a connection to mongod', async () => {
await testDb.init();
});
@ -70,6 +57,9 @@ class Car extends smartdata.SmartDataDbDoc<Car, Car> {
@smartdata.svDb()
public brand: string;
@smartdata.svDb()
public testBuffer = Buffer.from('hello');
@smartdata.svDb()
deepData = {
sodeep: 'yes',
@ -82,7 +72,12 @@ class Car extends smartdata.SmartDataDbDoc<Car, Car> {
}
}
tap.test('should save the car to the db', async () => {
tap.test('should create a new id', async () => {
const newid = await Car.getNewId();
console.log(newid);
});
tap.test('should save the car to the db', async (toolsArg) => {
const myCar = new Car('red', 'Volvo');
await myCar.save();
@ -90,6 +85,9 @@ tap.test('should save the car to the db', async () => {
await myCar2.save();
let counter = 0;
const gottenCarInstance = await Car.getInstance({});
console.log(gottenCarInstance.testBuffer instanceof mongodb.Binary);
process.memoryUsage();
do {
const myCar3 = new Car('red', 'Renault');
@ -107,7 +105,7 @@ tap.test('should save the car to the db', async () => {
});
tap.test('expect to get instance of Car with shallow match', async () => {
const totalQueryCycles = totalCars / 4;
const totalQueryCycles = totalCars / 2;
let counter = 0;
do {
const timeStart = Date.now();
@ -121,20 +119,20 @@ tap.test('expect to get instance of Car with shallow match', async () => {
}ms to query a set of 2000 with memory footprint ${process.memoryUsage().rss / 1e6} MB`
);
}
expect(myCars[0].deepData.sodeep).to.equal('yes');
expect(myCars[0].brand).to.equal('Renault');
expect(myCars[0].deepData.sodeep).toEqual('yes');
expect(myCars[0].brand).toEqual('Renault');
counter++;
} while (counter < totalQueryCycles);
});
tap.test('expect to get instance of Car with deep match', async () => {
const totalQueryCycles = totalCars / 4;
const totalQueryCycles = totalCars / 6;
let counter = 0;
do {
const timeStart = Date.now();
const myCars2 = await Car.getInstances({
deepData: {
sodeep: 'yes'
sodeep: 'yes',
},
});
if (counter % 10 === 0) {
@ -144,8 +142,8 @@ tap.test('expect to get instance of Car with deep match', async () => {
}ms to deep query a set of 2000 with memory footprint ${process.memoryUsage().rss / 1e6} MB`
);
}
expect(myCars2[0].deepData.sodeep).to.equal('yes');
expect(myCars2[0].brand).to.equal('Volvo');
expect(myCars2[0].deepData.sodeep).toEqual('yes');
expect(myCars2[0].brand).toEqual('Volvo');
counter++;
} while (counter < totalQueryCycles);
});
@ -154,7 +152,7 @@ tap.test('expect to get instance of Car and update it', async () => {
const myCar = await Car.getInstance<Car>({
brand: 'Volvo',
});
expect(myCar.color).to.equal('red');
expect(myCar.color).toEqual('red');
myCar.color = 'blue';
await myCar.save();
});
@ -165,7 +163,7 @@ tap.test('should be able to delete an instance of car', async () => {
color: 'blue',
});
console.log(myCars);
expect(myCars[0].color).to.equal('blue');
expect(myCars[0].color).toEqual('blue');
for (const myCar of myCars) {
await myCar.delete();
}
@ -173,7 +171,7 @@ tap.test('should be able to delete an instance of car', async () => {
const myCar2 = await Car.getInstance<Car>({
brand: 'Volvo',
});
expect(myCar2.color).to.equal('red');
expect(myCar2.color).toEqual('red');
});
// tslint:disable-next-line: max-classes-per-file
@ -200,24 +198,39 @@ class Truck extends smartdata.SmartDataDbDoc<Car, Car> {
tap.test('should store a new Truck', async () => {
const truck = new Truck('blue', 'MAN');
await truck.save();
const myTruck = await Truck.getInstance({ color: 'blue' });
myTruck.id = 'foo';
await myTruck.save();
const myTruck2 = await Truck.getInstance({ color: 'blue' });
console.log(myTruck2);
expect(myTruck2.color).toEqual('blue');
myTruck2.color = 'red';
await myTruck2.save();
const myTruck3 = await Truck.getInstance({ color: 'blue' });
expect(myTruck3).toBeNull();
});
tap.test('should ', async () => {})
tap.test('should return a count', async () => {
const truckCount = await Truck.getCount();
expect(truckCount).toEqual(1);
})
tap.test('should use a cursor', async () => {
const cursor = await Car.getCursor({});
let counter = 0;
await cursor.forEach(async (carArg) => {
counter++;
counter % 50 === 0 ? console.log(`50 more of ${carArg.color}`) : null;
});
});
// =======================================
// close the database connection
// =======================================
tap.test('should close the database connection', async (tools) => {
await testDb.mongoDb.dropDatabase();
await testDb.close();
try {
await mongod.stop();
} catch (e) {}
tap.test('close', async () => {
if (smartmongoInstance) {
await smartmongoInstance.stopAndDumpToDir('./.nogit/dbdump/test.ts');
} else {
await testDb.mongoDb.dropDatabase();
await testDb.close();
}
setTimeout(() => process.exit(), 2000);
});
tap.start({ throwOnError: true });

View File

@ -1,51 +1,36 @@
import { tap, expect } from '@pushrocks/tapbundle';
import { Qenv } from '@pushrocks/qenv';
import { tap, expect } from '@push.rocks/tapbundle';
import { Qenv } from '@push.rocks/qenv';
import * as smartmongo from '@push.rocks/smartmongo';
import { smartunique } from '../ts/smartdata.plugins.js';
const testQenv = new Qenv(process.cwd(), process.cwd() + '/.nogit/');
console.log(process.memoryUsage());
// the tested module
import * as smartdata from '../ts/index';
import * as mongoPlugin from 'mongodb-memory-server';
import { smartunique } from '../ts/smartdata.plugins';
import * as smartdata from '../ts/index.js';
// =======================================
// Connecting to the database server
// =======================================
let smartmongoInstance: smartmongo.SmartMongo;
let testDb: smartdata.SmartdataDb;
let smartdataOptions: smartdata.IMongoDescriptor;
let mongod: mongoPlugin.MongoMemoryServer;
const totalCars = 2000;
tap.skip.test('should create a testinstance as database', async () => {
mongod = new mongoPlugin.MongoMemoryServer({});
console.log('created mongod instance');
await mongod._startUpInstance().catch((err) => {
console.log(err);
});
console.log('mongod started');
smartdataOptions = {
mongoDbName: await mongod.getDbName(),
mongoDbPass: '',
mongoDbUrl: await mongod.getUri(),
};
console.log(smartdataOptions);
testDb = new smartdata.SmartdataDb(smartdataOptions);
tap.test('should create a testinstance as database', async () => {
smartmongoInstance = await smartmongo.SmartMongo.createAndStart();
testDb = new smartdata.SmartdataDb(await smartmongoInstance.getMongoDescriptor());
await testDb.init();
});
tap.test('should connect to atlas', async (tools) => {
tap.skip.test('should connect to atlas', async (tools) => {
const databaseName = `test-smartdata-${smartunique.shortId()}`;
testDb = new smartdata.SmartdataDb({
mongoDbUrl: testQenv.getEnvVarOnDemand('MONGO_URL'),
mongoDbUrl: await testQenv.getEnvVarOnDemand('MONGO_URL'),
mongoDbName: databaseName,
});
});
tap.test('should establish a connection to mongod', async () => {
await testDb.init();
});
@ -56,7 +41,8 @@ tap.test('should establish a connection to mongod', async () => {
// ------
// Collections
// ------
class CarTemplate extends smartdata.SmartDataDbDoc<CarTemplate, CarTemplate> {
@smartdata.Manager()
class Car extends smartdata.SmartDataDbDoc<Car, Car> {
@smartdata.unI()
public index: string = smartunique.shortId();
@ -79,29 +65,31 @@ class CarTemplate extends smartdata.SmartDataDbDoc<CarTemplate, CarTemplate> {
}
const createCarClass = (dbArg: smartdata.SmartdataDb) => {
console.log(this);
@smartdata.Collection(() => {
return dbArg;
})
class Car extends CarTemplate {};
smartdata.setDefaultManagerForDoc({ db: dbArg }, Car);
return Car;
};
tap.test('should prodice a car', async () => {
tap.test('should produce a car', async () => {
const CarClass = createCarClass(testDb);
const carInstance = new CarClass('red', 'Mercedes');
await carInstance.save();
});
tap.test('should get a car', async () => {
const car = Car.getInstance({
color: 'red',
});
});
// =======================================
// close the database connection
// =======================================
tap.test('should close the database connection', async (tools) => {
tap.test('close', async () => {
await testDb.mongoDb.dropDatabase();
await testDb.close();
try {
await mongod.stop();
} catch (e) {}
if (smartmongoInstance) {
await smartmongoInstance.stop();
}
});
tap.start({ throwOnError: true });

74
test/test.watch.ts Normal file
View File

@ -0,0 +1,74 @@
import { tap, expect } from '@push.rocks/tapbundle';
import { Qenv } from '@push.rocks/qenv';
import * as smartmongo from '@push.rocks/smartmongo';
import { smartunique } from '../ts/smartdata.plugins.js';
const testQenv = new Qenv(process.cwd(), process.cwd() + '/.nogit/');
console.log(process.memoryUsage());
// the tested module
import * as smartdata from '../ts/index.js';
// =======================================
// Connecting to the database server
// =======================================
let smartmongoInstance: smartmongo.SmartMongo;
let testDb: smartdata.SmartdataDb;
const totalCars = 2000;
tap.test('should create a testinstance as database', async () => {
smartmongoInstance = await smartmongo.SmartMongo.createAndStart();
testDb = new smartdata.SmartdataDb(await smartmongoInstance.getMongoDescriptor());
await testDb.init();
});
tap.skip.test('should connect to atlas', async (tools) => {
const databaseName = `test-smartdata-${smartunique.shortId()}`;
testDb = new smartdata.SmartdataDb({
mongoDbUrl: await testQenv.getEnvVarOnDemand('MONGO_URL'),
mongoDbName: databaseName,
});
await testDb.init();
});
@smartdata.Collection(() => testDb)
class House extends smartdata.SmartDataDbDoc<House, House> {
@smartdata.unI()
public id: string = smartunique.shortId();
@smartdata.svDb()
public data = {
id: smartunique.shortId(),
hello: 'hello',
};
}
tap.test('should watch a collection', async (toolsArg) => {
const done = toolsArg.defer();
const watcher = await House.watch({});
watcher.changeSubject.subscribe(async (houseArg) => {
console.log('hey there, we observed a house');
await watcher.close();
done.resolve();
});
const newHouse = new House();
await newHouse.save();
console.log('saved a house');
await done.promise;
});
// =======================================
// close the database connection
// =======================================
tap.test('close', async () => {
await testDb.mongoDb.dropDatabase();
await testDb.close();
if (smartmongoInstance) {
await smartmongoInstance.stop();
}
});
tap.start({ throwOnError: true });

8
ts/00_commitinfo_data.ts Normal file
View File

@ -0,0 +1,8 @@
/**
* autocreated commitinfo by @pushrocks/commitinfo
*/
export const commitinfo = {
name: '@push.rocks/smartdata',
version: '5.2.6',
description: 'An advanced library for NoSQL data organization and manipulation using TypeScript with support for MongoDB, data validation, collections, and custom data types.'
}

View File

@ -1,5 +1,14 @@
export * from './smartdata.classes.db';
export * from './smartdata.classes.collection';
export * from './smartdata.classes.doc';
export * from './smartdata.classes.db.js';
export * from './smartdata.classes.collection.js';
export * from './smartdata.classes.doc.js';
export * from './smartdata.classes.easystore.js';
export * from './smartdata.classes.cursor.js';
export { IMongoDescriptor } from './interfaces';
import * as convenience from './smartadata.convenience.js';
export { convenience };
// to be removed with the next breaking update
import type * as plugins from './smartdata.plugins.js';
type IMongoDescriptor = plugins.tsclass.database.IMongoDescriptor;
export type { IMongoDescriptor };

View File

@ -1 +0,0 @@
export * from './mongodescriptor';

View File

@ -1,22 +0,0 @@
export interface IMongoDescriptor {
/**
* the URL to connect to
*/
mongoDbUrl: string;
/**
* the db to use for the project
*/
mongoDbName?: string;
/**
* a username to use to connect to the database
*/
mongoDbUser?: string;
/**
* an optional password that will be replace <PASSWORD> in the connection string
*/
mongoDbPass?: string;
}

View File

@ -0,0 +1,5 @@
import * as plugins from './smartdata.plugins.js';
export const getNewUniqueId = async (prefixArg?: string) => {
return plugins.smartunique.uni(prefixArg);
};

View File

@ -1,7 +1,9 @@
import * as plugins from './smartdata.plugins';
import { SmartdataDb } from './smartdata.classes.db';
import { SmartDataDbDoc } from './smartdata.classes.doc';
import { CollectionFactory } from './smartdata.classes.collectionfactory';
import * as plugins from './smartdata.plugins.js';
import { SmartdataDb } from './smartdata.classes.db.js';
import { SmartdataDbCursor } from './smartdata.classes.cursor.js';
import { SmartDataDbDoc } from './smartdata.classes.doc.js';
import { SmartdataDbWatcher } from './smartdata.classes.watcher.js';
import { CollectionFactory } from './smartdata.classes.collectionfactory.js';
export interface IFindOptions {
limit?: number;
@ -24,7 +26,8 @@ const collectionFactory = new CollectionFactory();
*/
export function Collection(dbArg: SmartdataDb | TDelayed<SmartdataDb>) {
return function classDecorator<T extends { new (...args: any[]): {} }>(constructor: T) {
return class extends constructor {
const decoratedClass = class extends constructor {
public static className = constructor.name;
public static get collection() {
if (!(dbArg instanceof SmartdataDb)) {
dbArg = dbArg();
@ -38,24 +41,33 @@ export function Collection(dbArg: SmartdataDb | TDelayed<SmartdataDb>) {
return collectionFactory.getCollection(constructor.name, dbArg);
}
};
return decoratedClass;
};
}
export interface IManager {
db: SmartdataDb
db: SmartdataDb;
}
export const setDefaultManagerForDoc = <T>(managerArg: IManager, dbDocArg: T): T => {
(dbDocArg as any).prototype.defaultManager = managerArg;
return dbDocArg;
};
/**
* This is a decorator that will tell the decorated class what dbTable to use
* @param dbArg
*/
export function Manager<TManager extends IManager>(managerArg?: TManager | TDelayed<TManager>) {
return function classDecorator<T extends { new (...args: any[]): {} }>(constructor: T) {
return class extends constructor {
export function managed<TManager extends IManager>(managerArg?: TManager | TDelayed<TManager>) {
return function classDecorator<T extends { new (...args: any[]): any }>(constructor: T) {
const decoratedClass = class extends constructor {
public static className = constructor.name;
public static get collection() {
let dbArg: SmartdataDb;
if (managerArg['db']) {
dbArg = (managerArg as TManager).db
if (!managerArg) {
dbArg = this.prototype.defaultManager.db;
} else if (managerArg['db']) {
dbArg = (managerArg as TManager).db;
} else {
dbArg = (managerArg as TDelayed<TManager>)().db;
}
@ -63,8 +75,12 @@ export interface IManager {
}
public get collection() {
let dbArg: SmartdataDb;
if (managerArg['db']) {
dbArg = (managerArg as TManager).db
if (!managerArg) {
//console.log(this.defaultManager.db);
//process.exit(0)
dbArg = this.defaultManager.db;
} else if (managerArg['db']) {
dbArg = (managerArg as TManager).db;
} else {
dbArg = (managerArg as TDelayed<TManager>)().db;
}
@ -72,8 +88,10 @@ export interface IManager {
}
public static get manager() {
let manager: TManager;
if (managerArg['db']) {
manager = (managerArg as TManager);
if (!managerArg) {
manager = this.prototype.defaultManager;
} else if (managerArg['db']) {
manager = managerArg as TManager;
} else {
manager = (managerArg as TDelayed<TManager>)();
}
@ -81,18 +99,25 @@ export interface IManager {
}
public get manager() {
let manager: TManager;
if (managerArg['db']) {
manager = (managerArg as TManager);
if (!managerArg) {
manager = this.defaultManager;
} else if (managerArg['db']) {
manager = managerArg as TManager;
} else {
manager = (managerArg as TDelayed<TManager>)();
}
return manager;
}
};
return decoratedClass;
};
}
// tslint:disable-next-line: max-classes-per-file
/**
* @dpecrecated use @managed instead
*/
export const Manager = managed;
export class SmartdataCollection<T> {
/**
* the collection that is used
@ -155,12 +180,57 @@ export class SmartdataCollection<T> {
/**
* finds an object in the DbCollection
*/
public async find(filterObject: any): Promise<any> {
public async findOne(filterObject: any): Promise<any> {
await this.init();
const result = await this.mongoDbCollection.find(filterObject).toArray();
const cursor = this.mongoDbCollection.find(filterObject);
const result = await cursor.next();
cursor.close();
return result;
}
public async getCursor(
filterObjectArg: any,
dbDocArg: typeof SmartDataDbDoc
): Promise<SmartdataDbCursor<any>> {
await this.init();
const cursor = this.mongoDbCollection.find(filterObjectArg);
return new SmartdataDbCursor(cursor, dbDocArg);
}
/**
* finds an object in the DbCollection
*/
public async findAll(filterObject: any): Promise<any[]> {
await this.init();
const cursor = this.mongoDbCollection.find(filterObject);
const result = await cursor.toArray();
cursor.close();
return result;
}
/**
* watches the collection while applying a filter
*/
public async watch(
filterObject: any,
smartdataDbDocArg: typeof SmartDataDbDoc
): Promise<SmartdataDbWatcher> {
await this.init();
const changeStream = this.mongoDbCollection.watch(
[
{
$match: filterObject,
},
],
{
fullDocument: 'updateLookup',
}
);
const smartdataWatcher = new SmartdataDbWatcher(changeStream, smartdataDbDocArg);
await smartdataWatcher.readyDeferred.promise;
return smartdataWatcher;
}
/**
* create an object in the database
*/
@ -200,9 +270,12 @@ export class SmartdataCollection<T> {
await this.init();
await this.checkDoc(dbDocArg);
const identifiableObject = await dbDocArg.createIdentifiableObject();
await this.mongoDbCollection.deleteOne(identifiableObject, {
w: 1,
});
await this.mongoDbCollection.deleteOne(identifiableObject);
}
public async getCount(filterObject: any) {
await this.init();
return this.mongoDbCollection.countDocuments(filterObject);
}
/**
@ -210,7 +283,7 @@ export class SmartdataCollection<T> {
* if this.objectValidation is not set it passes.
*/
private checkDoc(docArg: T): Promise<void> {
const done = plugins.smartq.defer<void>();
const done = plugins.smartpromise.defer<void>();
let validationResult = true;
if (this.objectValidation) {
validationResult = this.objectValidation(docArg);

View File

@ -1,14 +1,11 @@
import * as plugins from './smartdata.plugins';
import { SmartdataCollection } from './smartdata.classes.collection';
import { SmartdataDb } from './smartdata.classes.db';
import * as plugins from './smartdata.plugins.js';
import { SmartdataCollection } from './smartdata.classes.collection.js';
import { SmartdataDb } from './smartdata.classes.db.js';
export class CollectionFactory {
public collections: { [key: string]: SmartdataCollection<any> } = {};
public getCollection = (
nameArg: string,
dbArg: SmartdataDb
): SmartdataCollection<any> => {
public getCollection = (nameArg: string, dbArg: SmartdataDb): SmartdataCollection<any> => {
if (!this.collections[nameArg]) {
this.collections[nameArg] = (() => {
if (dbArg instanceof SmartdataDb) {

View File

@ -0,0 +1,46 @@
import { SmartDataDbDoc } from './smartdata.classes.doc.js';
import * as plugins from './smartdata.plugins.js';
/**
* a wrapper for the native mongodb cursor. Exposes better
*/
export class SmartdataDbCursor<T = any> {
// STATIC
// INSTANCE
public mongodbCursor: plugins.mongodb.FindCursor<T>;
private smartdataDbDoc: typeof SmartDataDbDoc;
constructor(cursorArg: plugins.mongodb.FindCursor<T>, dbDocArg: typeof SmartDataDbDoc) {
this.mongodbCursor = cursorArg;
this.smartdataDbDoc = dbDocArg;
}
public async next(closeAtEnd = true) {
const result = this.smartdataDbDoc.createInstanceFromMongoDbNativeDoc(
await this.mongodbCursor.next()
);
if (!result && closeAtEnd) {
await this.close();
}
return result;
}
public async forEach(forEachFuncArg: (itemArg: T) => Promise<any>, closeCursorAtEnd = true) {
let nextDocument: any;
do {
nextDocument = await this.mongodbCursor.next();
if (nextDocument) {
const nextClassInstance =
this.smartdataDbDoc.createInstanceFromMongoDbNativeDoc(nextDocument);
await forEachFuncArg(nextClassInstance as any);
}
} while (nextDocument);
if (closeCursorAtEnd) {
await this.close();
}
}
public async close() {
await this.mongodbCursor.close();
}
}

View File

@ -1,10 +1,9 @@
import * as plugins from './smartdata.plugins';
import { ObjectMap } from '@pushrocks/lik';
import * as plugins from './smartdata.plugins.js';
import { SmartdataCollection } from './smartdata.classes.collection';
import { SmartdataCollection } from './smartdata.classes.collection.js';
import { EasyStore } from './smartdata.classes.easystore.js';
import { logger } from './smartdata.logging';
import { IMongoDescriptor } from './interfaces';
import { logger } from './smartdata.logging.js';
/**
* interface - indicates the connection status of the db
@ -12,17 +11,24 @@ import { IMongoDescriptor } from './interfaces';
export type TConnectionStatus = 'initial' | 'disconnected' | 'connected' | 'failed';
export class SmartdataDb {
smartdataOptions: IMongoDescriptor;
smartdataOptions: plugins.tsclass.database.IMongoDescriptor;
mongoDbClient: plugins.mongodb.MongoClient;
mongoDb: plugins.mongodb.Db;
status: TConnectionStatus;
smartdataCollectionMap = new ObjectMap<SmartdataCollection<any>>();
statusConnectedDeferred = plugins.smartpromise.defer();
smartdataCollectionMap = new plugins.lik.ObjectMap<SmartdataCollection<any>>();
constructor(smartdataOptions: IMongoDescriptor) {
constructor(smartdataOptions: plugins.tsclass.database.IMongoDescriptor) {
this.smartdataOptions = smartdataOptions;
this.status = 'initial';
}
// easystore
public async createEasyStore(nameIdArg: string) {
const easyStore = new EasyStore(nameIdArg, this);
return easyStore;
}
// basic connection stuff ----------------------------------------------
/**
@ -40,13 +46,12 @@ export class SmartdataDb {
.replace('<dbname>', this.smartdataOptions.mongoDbName);
this.mongoDbClient = await plugins.mongodb.MongoClient.connect(finalConnectionUrl, {
useNewUrlParser: true,
useUnifiedTopology: true,
maxPoolSize: 100,
maxIdleTimeMS: 10,
});
this.mongoDb = this.mongoDbClient.db(this.smartdataOptions.mongoDbName);
this.status = 'connected';
this.statusConnectedDeferred.resolve();
console.log(`Connected to database ${this.smartdataOptions.mongoDbName}`);
}
@ -71,7 +76,7 @@ export class SmartdataDb {
* @returns DbTable
*/
public async getSmartdataCollectionByName<T>(nameArg: string): Promise<SmartdataCollection<T>> {
const resultCollection = this.smartdataCollectionMap.find((dbTableArg) => {
const resultCollection = await this.smartdataCollectionMap.find(async (dbTableArg) => {
return dbTableArg.collectionName === nameArg;
});
return resultCollection;

View File

@ -0,0 +1,304 @@
import * as plugins from './smartdata.plugins.js';
import { SmartdataDb } from './smartdata.classes.db.js';
import { managed, setDefaultManagerForDoc } from './smartdata.classes.collection.js';
import { SmartDataDbDoc, svDb, unI } from './smartdata.classes.doc.js';
import { SmartdataDbWatcher } from './smartdata.classes.watcher.js';
@managed()
export class DistributedClass extends SmartDataDbDoc<DistributedClass, DistributedClass> {
// INSTANCE
@unI()
public id: string;
@svDb()
public data: {
status: 'initializing' | 'bidding' | 'settled' | 'stopped';
biddingShortcode?: string;
biddingStartTime?: number;
lastUpdated: number;
elected: boolean;
/**
* used to store request
*/
taskRequests: plugins.taskbuffer.distributedCoordination.IDistributedTaskRequest[];
/**
* only used by the leader to convey consultation results
*/
taskRequestResults: plugins.taskbuffer.distributedCoordination.IDistributedTaskRequestResult[];
};
}
/**
* This file implements a distributed coordinator according to the @pushrocks/taskbuffer standard.
* you should not set up this yourself. Instead, there is a factory on the SmartdataDb class
* that will take care of setting this up.
*/
export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distributedCoordination
.AbstractDistributedCoordinator {
public readyPromise: Promise<any>;
public db: SmartdataDb;
private asyncExecutionStack = new plugins.lik.AsyncExecutionStack();
public ownInstance: DistributedClass;
public distributedWatcher: SmartdataDbWatcher<DistributedClass>;
constructor(dbArg: SmartdataDb) {
super();
this.db = dbArg;
setDefaultManagerForDoc(this, DistributedClass);
this.readyPromise = this.db.statusConnectedDeferred.promise;
}
// smartdata specific stuff
public async start() {
await this.init();
}
public async stop() {
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
if (this.distributedWatcher) {
await this.distributedWatcher.close();
}
if (this.ownInstance?.data.elected) {
this.ownInstance.data.elected = false;
}
if (this.ownInstance?.data.status === 'stopped') {
console.log(`stopping a distributed instance that has not been started yet.`);
}
this.ownInstance.data.status = 'stopped';
await this.ownInstance.save();
console.log(`stopped ${this.ownInstance.id}`);
});
}
public id = plugins.smartunique.uni('distributedInstance');
private startHeartbeat = async () => {
while (this.ownInstance.data.status !== 'stopped') {
await this.sendHeartbeat();
await plugins.smartdelay.delayForRandom(5000, 10000);
}
};
public async sendHeartbeat() {
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
if (this.ownInstance.data.status === 'stopped') {
console.log(`aborted sending heartbeat because status is stopped`);
return;
}
await this.ownInstance.updateFromDb();
this.ownInstance.data.lastUpdated = Date.now();
await this.ownInstance.save();
console.log(`sent heartbeat for ${this.ownInstance.id}`);
const allInstances = DistributedClass.getInstances({});
});
if (this.ownInstance.data.status === 'stopped') {
console.log(`aborted sending heartbeat because status is stopped`);
return;
}
const eligibleLeader = await this.getEligibleLeader();
// not awaiting here because we don't want to block the heartbeat
this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
if (!eligibleLeader && this.ownInstance.data.status === 'settled') {
this.checkAndMaybeLead();
}
});
}
private async init() {
await this.readyPromise;
if (!this.ownInstance) {
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
this.ownInstance = new DistributedClass();
this.ownInstance.id = this.id;
this.ownInstance.data = {
elected: false,
lastUpdated: Date.now(),
status: 'initializing',
taskRequests: [],
taskRequestResults: [],
};
await this.ownInstance.save();
});
} else {
console.warn(`distributed instance already initialized`);
}
// lets enable the heartbeat
this.startHeartbeat();
// lets do a leader check
await this.checkAndMaybeLead();
return this.ownInstance;
}
public async getEligibleLeader() {
return this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
const allInstances = await DistributedClass.getInstances({});
let leaders = allInstances.filter((instanceArg) => instanceArg.data.elected === true);
const eligibleLeader = leaders.find(
(leader) =>
leader.data.lastUpdated >=
Date.now() - plugins.smarttime.getMilliSecondsFromUnits({ seconds: 20 })
);
return eligibleLeader;
});
}
// --> leader election
public async checkAndMaybeLead() {
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
this.ownInstance.data.status = 'initializing';
this.ownInstance.save();
});
if (await this.getEligibleLeader()) {
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
await this.ownInstance.updateFromDb();
this.ownInstance.data.status = 'settled';
await this.ownInstance.save();
console.log(`${this.ownInstance.id} settled as follower`);
});
return;
} else if (
(await DistributedClass.getInstances({})).find((instanceArg) => {
instanceArg.data.status === 'bidding' &&
instanceArg.data.biddingStartTime <= Date.now() - 4000 &&
instanceArg.data.biddingStartTime >= Date.now() - 30000;
})
) {
console.log('too late to the bidding party... waiting for next round.');
return;
} else {
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
await this.ownInstance.updateFromDb();
this.ownInstance.data.status = 'bidding';
this.ownInstance.data.biddingStartTime = Date.now();
this.ownInstance.data.biddingShortcode = plugins.smartunique.shortId();
await this.ownInstance.save();
console.log('bidding code stored.');
});
console.log(`bidding for leadership...`);
await plugins.smartdelay.delayFor(
plugins.smarttime.getMilliSecondsFromUnits({ seconds: 5 })
);
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
let biddingInstances = await DistributedClass.getInstances({});
biddingInstances = biddingInstances.filter(
(instanceArg) =>
instanceArg.data.status === 'bidding' &&
instanceArg.data.lastUpdated >=
Date.now() - plugins.smarttime.getMilliSecondsFromUnits({ seconds: 10 })
);
console.log(`found ${biddingInstances.length} bidding instances...`);
this.ownInstance.data.elected = true;
for (const biddingInstance of biddingInstances) {
if (biddingInstance.data.biddingShortcode < this.ownInstance.data.biddingShortcode) {
this.ownInstance.data.elected = false;
}
}
await plugins.smartdelay.delayFor(5000);
console.log(`settling with status elected = ${this.ownInstance.data.elected}`);
this.ownInstance.data.status = 'settled';
await this.ownInstance.save();
});
if (this.ownInstance.data.elected) {
this.leadFunction();
}
}
}
/**
* when it has been determined
* that this instance is leading
* the leading is implemented here
*/
public async leadFunction() {
this.distributedWatcher = await DistributedClass.watch({});
const currentTaskRequests: Array<{
taskName: string;
taskExecutionTime: number;
/**
* all instances that requested this task
*/
requestingDistibutedInstanceIds: string[];
responseTimeout: plugins.smartdelay.Timeout<any>;
}> = [];
this.distributedWatcher.changeSubject.subscribe({
next: async (distributedDoc) => {
if (!distributedDoc) {
console.log(`registered deletion of instance...`);
return;
}
console.log(distributedDoc);
console.log(`registered change for ${distributedDoc.id}`);
distributedDoc;
},
});
while (this.ownInstance.data.status !== 'stopped' && this.ownInstance.data.elected) {
const allInstances = await DistributedClass.getInstances({});
for (const instance of allInstances) {
if (instance.data.status === 'stopped') {
await instance.delete();
};
}
await plugins.smartdelay.delayFor(10000);
}
}
// abstract implemented methods
public async fireDistributedTaskRequest(
taskRequestArg: plugins.taskbuffer.distributedCoordination.IDistributedTaskRequest
): Promise<plugins.taskbuffer.distributedCoordination.IDistributedTaskRequestResult> {
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
if (!this.ownInstance) {
console.error('instance need to be started first...');
return;
}
await this.ownInstance.updateFromDb();
this.ownInstance.data.taskRequests.push(taskRequestArg);
await this.ownInstance.save();
});
await plugins.smartdelay.delayFor(10000);
const result = await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
await this.ownInstance.updateFromDb();
const taskRequestResult = this.ownInstance.data.taskRequestResults.find((resultItem) => {
return resultItem.requestResponseId === taskRequestArg.requestResponseId;
});
return taskRequestResult;
});
if (!result) {
console.warn('no result found for task request...');
return null;
}
return result;
}
public async updateDistributedTaskRequest(
infoBasisArg: plugins.taskbuffer.distributedCoordination.IDistributedTaskRequest
): Promise<void> {
await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
const existingInfoBasis = this.ownInstance.data.taskRequests.find((infoBasisItem) => {
return (
infoBasisItem.taskName === infoBasisArg.taskName &&
infoBasisItem.taskExecutionTime === infoBasisArg.taskExecutionTime
);
});
if (!existingInfoBasis) {
console.warn('trying to update a non existing task request... aborting!');
return;
}
Object.assign(existingInfoBasis, infoBasisArg);
await this.ownInstance.save();
plugins.smartdelay.delayFor(60000).then(() => {
this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
const indexToRemove = this.ownInstance.data.taskRequests.indexOf(existingInfoBasis);
this.ownInstance.data.taskRequests.splice(indexToRemove, indexToRemove);
await this.ownInstance.save();
});
});
});
}
}

View File

@ -1,12 +1,22 @@
import * as plugins from './smartdata.plugins';
import * as plugins from './smartdata.plugins.js';
import { ObjectMap } from '@pushrocks/lik';
import { SmartdataDb } from './smartdata.classes.db';
import { IManager, SmartdataCollection } from './smartdata.classes.collection';
import { SmartdataDb } from './smartdata.classes.db.js';
import { SmartdataDbCursor } from './smartdata.classes.cursor.js';
import { type IManager, SmartdataCollection } from './smartdata.classes.collection.js';
import { SmartdataDbWatcher } from './smartdata.classes.watcher.js';
export type TDocCreation = 'db' | 'new' | 'mixed';
export function globalSvDb() {
return (target: SmartDataDbDoc<unknown, unknown>, key: string) => {
console.log(`called svDb() on >${target.constructor.name}.${key}<`);
if (!target.globalSaveableProperties) {
target.globalSaveableProperties = [];
}
target.globalSaveableProperties.push(key);
};
}
/**
* saveable - saveable decorator to be used on class properties
*/
@ -41,27 +51,188 @@ export function unI() {
};
}
export const convertFilterForMongoDb = (filterArg: { [key: string]: any }) => {
const convertedFilter: { [key: string]: any } = {};
const convertFilterArgument = (keyPathArg2: string, filterArg2: any) => {
if (typeof filterArg2 === 'object') {
for (const key of Object.keys(filterArg2)) {
if (key.startsWith('$')) {
convertedFilter[keyPathArg2] = filterArg2;
return;
} else if (key.includes('.')) {
throw new Error('keys cannot contain dots');
}
}
for (const key of Object.keys(filterArg2)) {
convertFilterArgument(`${keyPathArg2}.${key}`, filterArg2[key]);
}
} else {
convertedFilter[keyPathArg2] = filterArg2;
}
};
for (const key of Object.keys(filterArg)) {
convertFilterArgument(key, filterArg[key]);
}
return convertedFilter;
};
export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends IManager = any> {
/**
* the collection object an Doc belongs to
*/
public static collection: SmartdataCollection<any>;
public collection: SmartdataCollection<any>;
public static defaultManager;
public static manager;
public manager: TManager;
// STATIC
public static createInstanceFromMongoDbNativeDoc<T>(
this: plugins.tsclass.typeFest.Class<T>,
mongoDbNativeDocArg: any
): T {
const newInstance = new this();
(newInstance as any).creationStatus = 'db';
for (const key of Object.keys(mongoDbNativeDocArg)) {
newInstance[key] = mongoDbNativeDocArg[key];
}
return newInstance;
}
/**
* gets all instances as array
* @param this
* @param filterArg
* @returns
*/
public static async getInstances<T>(
this: plugins.tsclass.typeFest.Class<T>,
filterArg: plugins.tsclass.typeFest.PartialDeep<T>
): Promise<T[]> {
const foundDocs = await (this as any).collection.findAll(convertFilterForMongoDb(filterArg));
const returnArray = [];
for (const foundDoc of foundDocs) {
const newInstance: T = (this as any).createInstanceFromMongoDbNativeDoc(foundDoc);
returnArray.push(newInstance);
}
return returnArray;
}
/**
* gets the first matching instance
* @param this
* @param filterArg
* @returns
*/
public static async getInstance<T>(
this: plugins.tsclass.typeFest.Class<T>,
filterArg: plugins.tsclass.typeFest.PartialDeep<T>
): Promise<T> {
const foundDoc = await (this as any).collection.findOne(convertFilterForMongoDb(filterArg));
if (foundDoc) {
const newInstance: T = (this as any).createInstanceFromMongoDbNativeDoc(foundDoc);
return newInstance;
} else {
return null;
}
}
/**
* get a unique id prefixed with the class name
*/
public static async getNewId<T = any>(this: plugins.tsclass.typeFest.Class<T>, lengthArg: number = 20) {
return `${(this as any).className}:${plugins.smartunique.shortId(lengthArg)}`;
}
/**
* get cursor
* @returns
*/
public static async getCursor<T>(
this: plugins.tsclass.typeFest.Class<T>,
filterArg: plugins.tsclass.typeFest.PartialDeep<T>
) {
const collection: SmartdataCollection<T> = (this as any).collection;
const cursor: SmartdataDbCursor<T> = await collection.getCursor(
convertFilterForMongoDb(filterArg),
this as any as typeof SmartDataDbDoc
);
return cursor;
}
/**
* watch the collection
* @param this
* @param filterArg
* @param forEachFunction
*/
public static async watch<T>(
this: plugins.tsclass.typeFest.Class<T>,
filterArg: plugins.tsclass.typeFest.PartialDeep<T>
) {
const collection: SmartdataCollection<T> = (this as any).collection;
const watcher: SmartdataDbWatcher<T> = await collection.watch(
convertFilterForMongoDb(filterArg),
this as any
);
return watcher;
}
/**
* run a function for all instances
* @returns
*/
public static async forEach<T>(
this: plugins.tsclass.typeFest.Class<T>,
filterArg: plugins.tsclass.typeFest.PartialDeep<T>,
forEachFunction: (itemArg: T) => Promise<any>
) {
const cursor: SmartdataDbCursor<T> = await (this as any).getCursor(filterArg);
await cursor.forEach(forEachFunction);
}
/**
* returns a count of the documents in the collection
*/
public static async getCount<T>(
this: plugins.tsclass.typeFest.Class<T>,
filterArg: plugins.tsclass.typeFest.PartialDeep<T> = ({} as any)
) {
const collection: SmartdataCollection<T> = (this as any).collection;
return await collection.getCount(filterArg);
}
// INSTANCE
/**
* how the Doc in memory was created, may prove useful later.
*/
public creationStatus: TDocCreation = 'new';
/**
* updated from db in any case where doc comes from db
*/
@globalSvDb()
_createdAt: string = (new Date()).toISOString();
/**
* will be updated everytime the doc is saved
*/
@globalSvDb()
_updatedAt: string = (new Date()).toISOString();
/**
* an array of saveable properties of ALL doc
*/
public globalSaveableProperties: string[];
/**
* unique indexes
*/
public uniqueIndexes: string[];
/**
* an array of saveable properties of a doc
* an array of saveable properties of a specific doc
*/
public saveableProperties: string[];
@ -80,54 +251,6 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
*/
constructor() {}
public static async getInstances<T>(
this: plugins.tsclass.typeFest.Class<T>,
filterArg: plugins.tsclass.typeFest.PartialDeep<T>
): Promise<T[]> {
const convertedFilter: any = {};
const convertFilterArgument = (keyPathArg: string, filterArg2: any) => {
if (typeof filterArg2 === 'object') {
for (const key of Object.keys(filterArg2)) {
if (key.startsWith('$')) {
convertedFilter[keyPathArg] = filterArg2;
return;
} else if (key.includes('.')) {
throw new Error('keys cannot contain dots');
}
}
for (const key of Object.keys(filterArg2)) {
convertFilterArgument(`${keyPathArg}.${key}`, filterArg2[key]);
}
} else {
convertedFilter[keyPathArg] = filterArg2
}
}
for (const key of Object.keys(filterArg)) {
convertFilterArgument(key, filterArg[key]);
}
const foundDocs = await (this as any).collection.find(convertedFilter);
const returnArray = [];
for (const item of foundDocs) {
const newInstance = new this();
(newInstance as any).creationStatus = 'db';
for (const key of Object.keys(item)) {
newInstance[key] = item[key];
}
returnArray.push(newInstance);
}
return returnArray;
}
public static async getInstance<T>(
this: plugins.tsclass.typeFest.Class<T>,
filterArg: plugins.tsclass.typeFest.PartialDeep<T>
): Promise<T> {
const result = await (this as any).getInstances(filterArg);
if (result && result.length > 0) {
return result[0];
}
}
/**
* saves this instance but not any connected items
* may lead to data inconsistencies, but is faster
@ -136,6 +259,9 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
// tslint:disable-next-line: no-this-assignment
const self: any = this;
let dbResult: any;
this._updatedAt = (new Date()).toISOString();
switch (this.creationStatus) {
case 'db':
dbResult = await this.collection.update(self);
@ -161,9 +287,9 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
* also store any referenced objects to DB
* better for data consistency
*/
public saveDeep(savedMapArg: ObjectMap<SmartDataDbDoc<any, any>> = null) {
public saveDeep(savedMapArg: plugins.lik.ObjectMap<SmartDataDbDoc<any, any>> = null) {
if (!savedMapArg) {
savedMapArg = new ObjectMap<SmartDataDbDoc<any, any>>();
savedMapArg = new plugins.lik.ObjectMap<SmartDataDbDoc<any, any>>();
}
savedMapArg.add(this);
this.save();
@ -175,12 +301,26 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
}
}
/**
* updates an object from db
*/
public async updateFromDb() {
const mongoDbNativeDoc = await this.collection.findOne(await this.createIdentifiableObject());
for (const key of Object.keys(mongoDbNativeDoc)) {
this[key] = mongoDbNativeDoc[key];
}
}
/**
* creates a saveable object so the instance can be persisted as json in the database
*/
public async createSavableObject(): Promise<TImplements> {
const saveableObject: unknown = {}; // is not exposed to outside, so any is ok here
for (const propertyNameString of this.saveableProperties) {
const saveableProperties = [
...this.globalSaveableProperties,
...this.saveableProperties
]
for (const propertyNameString of saveableProperties) {
saveableObject[propertyNameString] = this[propertyNameString];
}
return saveableObject as TImplements;

View File

@ -0,0 +1,119 @@
import * as plugins from './smartdata.plugins.js';
import { Collection } from './smartdata.classes.collection.js';
import { SmartdataDb } from './smartdata.classes.db.js';
import { SmartDataDbDoc, svDb, unI } from './smartdata.classes.doc.js';
/**
* EasyStore allows the storage of easy objects. It also allows easy sharing of the object between different instances
*/
export class EasyStore<T> {
// instance
public smartdataDbRef: SmartdataDb;
public nameId: string;
private easyStoreClass = (() => {
@Collection(() => this.smartdataDbRef)
class SmartdataEasyStore extends SmartDataDbDoc<SmartdataEasyStore, SmartdataEasyStore> {
@unI()
public nameId: string;
@svDb()
public ephermal: {
activated: boolean;
timeout: number;
};
@svDb()
lastEdit: number;
@svDb()
public data: Partial<T>;
}
return SmartdataEasyStore;
})();
constructor(nameIdArg: string, smnartdataDbRefArg: SmartdataDb) {
this.smartdataDbRef = smnartdataDbRefArg;
this.nameId = nameIdArg;
}
private easyStorePromise: Promise<InstanceType<typeof this.easyStoreClass>>;
private async getEasyStore(): Promise<InstanceType<typeof this.easyStoreClass>> {
if (this.easyStorePromise) {
return this.easyStorePromise;
};
// first run from here
const deferred = plugins.smartpromise.defer<InstanceType<typeof this.easyStoreClass>>();
this.easyStorePromise = deferred.promise;
let easyStore = await this.easyStoreClass.getInstance({
nameId: this.nameId,
});
if (!easyStore) {
easyStore = new this.easyStoreClass();
easyStore.nameId = this.nameId;
easyStore.data = {};
await easyStore.save();
}
deferred.resolve(easyStore);
return this.easyStorePromise;
}
/**
* reads all keyValue pairs at once and returns them
*/
public async readAll() {
const easyStore = await this.getEasyStore();
return easyStore.data;
}
/**
* reads a keyValueFile from disk
*/
public async readKey(keyArg: keyof T) {
const easyStore = await this.getEasyStore();
return easyStore.data[keyArg];
}
/**
* writes a specific key to the keyValueStore
*/
public async writeKey<TKey extends keyof T>(keyArg: TKey, valueArg: T[TKey]) {
const easyStore = await this.getEasyStore();
easyStore.data[keyArg] = valueArg;
await easyStore.save();
}
public async deleteKey(keyArg: keyof T) {
const easyStore = await this.getEasyStore();
delete easyStore.data[keyArg];
await easyStore.save();
}
/**
* writes all keyValue pairs in the object argument
*/
public async writeAll(keyValueObject: Partial<T>) {
const easyStore = await this.getEasyStore();
easyStore.data = { ...easyStore.data, ...keyValueObject };
await easyStore.save();
}
/**
* wipes a key value store from disk
*/
public async wipe() {
const easyStore = await this.getEasyStore();
easyStore.data = {};
await easyStore.save();
}
public async cleanUpEphermal() {
while (
(await this.smartdataDbRef.statusConnectedDeferred.promise) &&
this.smartdataDbRef.status === 'connected'
) {}
}
}

View File

@ -0,0 +1,37 @@
import { SmartDataDbDoc } from './smartdata.classes.doc.js';
import * as plugins from './smartdata.plugins.js';
/**
* a wrapper for the native mongodb cursor. Exposes better
*/
export class SmartdataDbWatcher<T = any> {
// STATIC
public readyDeferred = plugins.smartpromise.defer();
// INSTANCE
private changeStream: plugins.mongodb.ChangeStream<T>;
public changeSubject = new plugins.smartrx.rxjs.Subject<T>();
constructor(
changeStreamArg: plugins.mongodb.ChangeStream<T>,
smartdataDbDocArg: typeof SmartDataDbDoc
) {
this.changeStream = changeStreamArg;
this.changeStream.on('change', async (item: any) => {
if (!item.fullDocument) {
this.changeSubject.next(null);
return;
}
this.changeSubject.next(
smartdataDbDocArg.createInstanceFromMongoDbNativeDoc(item.fullDocument) as any as T
);
});
plugins.smartdelay.delayFor(0).then(() => {
this.readyDeferred.resolve();
});
}
public async close() {
await this.changeStream.close();
}
}

View File

@ -1,3 +1,3 @@
import * as plugins from './smartdata.plugins';
import * as plugins from './smartdata.plugins.js';
export const logger = new plugins.smartlog.ConsoleLog();

View File

@ -4,11 +4,26 @@ import * as tsclass from '@tsclass/tsclass';
export { tsclass };
// @pushrocks scope
import * as smartlog from '@pushrocks/smartlog';
import * as lodash from 'lodash';
import * as lik from '@push.rocks/lik';
import * as smartdelay from '@push.rocks/smartdelay';
import * as smartlog from '@push.rocks/smartlog';
import * as smartpromise from '@push.rocks/smartpromise';
import * as smartrx from '@push.rocks/smartrx';
import * as smartstring from '@push.rocks/smartstring';
import * as smarttime from '@push.rocks/smarttime';
import * as smartunique from '@push.rocks/smartunique';
import * as taskbuffer from '@push.rocks/taskbuffer';
import * as mongodb from 'mongodb';
import * as smartq from '@pushrocks/smartpromise';
import * as smartstring from '@pushrocks/smartstring';
import * as smartunique from '@pushrocks/smartunique';
export { smartlog, lodash, smartq, mongodb, smartstring, smartunique };
export {
lik,
smartdelay,
smartpromise,
smartlog,
smartrx,
mongodb,
smartstring,
smarttime,
smartunique,
taskbuffer,
};

View File

@ -1,7 +1,14 @@
{
"compilerOptions": {
"experimentalDecorators": true,
"target": "es2017",
"module": "commonjs"
}
}
"useDefineForClassFields": false,
"target": "ES2022",
"module": "NodeNext",
"moduleResolution": "NodeNext",
"esModuleInterop": true,
"verbatimModuleSyntax": true
},
"exclude": [
"dist_*/**/*.d.ts"
]
}

View File

@ -1,17 +0,0 @@
{
"extends": ["tslint:latest", "tslint-config-prettier"],
"rules": {
"semicolon": [true, "always"],
"no-console": false,
"ordered-imports": false,
"object-literal-sort-keys": false,
"member-ordering": {
"options":{
"order": [
"static-method"
]
}
}
},
"defaultSeverity": "warning"
}