Compare commits
33 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 26449e9171 | |||
| c91b7a200b | |||
| fde082974f | |||
| 84c355c499 | |||
| 367bacb954 | |||
| 2adbed8fdb | |||
| 2d7981aa6f | |||
| 458da47c9c | |||
| 92688a7f7f | |||
| 1aed44f035 | |||
| 07895c2767 | |||
| a791c3e8c7 | |||
| 2c2910fafe | |||
| 094b3a8a50 | |||
| 6ed2abca4a | |||
| f9df862b0a | |||
| 2db69618c3 | |||
| a6bb61764b | |||
| 7d867ea6ab | |||
| d03f086c92 | |||
| 83c1e2bb4e | |||
| faa5d6d542 | |||
| 6e06e8108b | |||
| 45ce23ec11 | |||
| 93ef6a3d6b | |||
| 609873b4ad | |||
| 4a8bbc3d13 | |||
| 51c2d4f6e0 | |||
| 45091d6b8c | |||
| eae7300439 | |||
| 7753e58036 | |||
| 333e9c1316 | |||
| 479e0725e6 |
66
.gitea/workflows/default_nottags.yaml
Normal file
66
.gitea/workflows/default_nottags.yaml
Normal file
@ -0,0 +1,66 @@
|
||||
name: Default (not tags)
|
||||
|
||||
on:
|
||||
push:
|
||||
tags-ignore:
|
||||
- '**'
|
||||
|
||||
env:
|
||||
IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
|
||||
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
|
||||
NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
|
||||
NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
|
||||
NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
|
||||
NPMCI_URL_CLOUDLY: ${{secrets.NPMCI_URL_CLOUDLY}}
|
||||
|
||||
jobs:
|
||||
security:
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true
|
||||
container:
|
||||
image: ${{ env.IMAGE }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Install pnpm and npmci
|
||||
run: |
|
||||
pnpm install -g pnpm
|
||||
pnpm install -g @shipzone/npmci
|
||||
|
||||
- name: Run npm prepare
|
||||
run: npmci npm prepare
|
||||
|
||||
- name: Audit production dependencies
|
||||
run: |
|
||||
npmci command npm config set registry https://registry.npmjs.org
|
||||
npmci command pnpm audit --audit-level=high --prod
|
||||
continue-on-error: true
|
||||
|
||||
- name: Audit development dependencies
|
||||
run: |
|
||||
npmci command npm config set registry https://registry.npmjs.org
|
||||
npmci command pnpm audit --audit-level=high --dev
|
||||
continue-on-error: true
|
||||
|
||||
test:
|
||||
if: ${{ always() }}
|
||||
needs: security
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: ${{ env.IMAGE }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Test stable
|
||||
run: |
|
||||
npmci node install stable
|
||||
npmci npm install
|
||||
npmci npm test
|
||||
|
||||
- name: Test build
|
||||
run: |
|
||||
npmci node install stable
|
||||
npmci npm install
|
||||
npmci npm build
|
||||
124
.gitea/workflows/default_tags.yaml
Normal file
124
.gitea/workflows/default_tags.yaml
Normal file
@ -0,0 +1,124 @@
|
||||
name: Default (tags)
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
|
||||
env:
|
||||
IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
|
||||
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
|
||||
NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
|
||||
NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
|
||||
NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
|
||||
NPMCI_URL_CLOUDLY: ${{secrets.NPMCI_URL_CLOUDLY}}
|
||||
|
||||
jobs:
|
||||
security:
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true
|
||||
container:
|
||||
image: ${{ env.IMAGE }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Prepare
|
||||
run: |
|
||||
pnpm install -g pnpm
|
||||
pnpm install -g @shipzone/npmci
|
||||
npmci npm prepare
|
||||
|
||||
- name: Audit production dependencies
|
||||
run: |
|
||||
npmci command npm config set registry https://registry.npmjs.org
|
||||
npmci command pnpm audit --audit-level=high --prod
|
||||
continue-on-error: true
|
||||
|
||||
- name: Audit development dependencies
|
||||
run: |
|
||||
npmci command npm config set registry https://registry.npmjs.org
|
||||
npmci command pnpm audit --audit-level=high --dev
|
||||
continue-on-error: true
|
||||
|
||||
test:
|
||||
if: ${{ always() }}
|
||||
needs: security
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: ${{ env.IMAGE }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Prepare
|
||||
run: |
|
||||
pnpm install -g pnpm
|
||||
pnpm install -g @shipzone/npmci
|
||||
npmci npm prepare
|
||||
|
||||
- name: Test stable
|
||||
run: |
|
||||
npmci node install stable
|
||||
npmci npm install
|
||||
npmci npm test
|
||||
|
||||
- name: Test build
|
||||
run: |
|
||||
npmci node install stable
|
||||
npmci npm install
|
||||
npmci npm build
|
||||
|
||||
release:
|
||||
needs: test
|
||||
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: ${{ env.IMAGE }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Prepare
|
||||
run: |
|
||||
pnpm install -g pnpm
|
||||
pnpm install -g @shipzone/npmci
|
||||
npmci npm prepare
|
||||
|
||||
- name: Release
|
||||
run: |
|
||||
npmci node install stable
|
||||
npmci npm publish
|
||||
|
||||
metadata:
|
||||
needs: test
|
||||
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: ${{ env.IMAGE }}
|
||||
continue-on-error: true
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Prepare
|
||||
run: |
|
||||
pnpm install -g pnpm
|
||||
pnpm install -g @shipzone/npmci
|
||||
npmci npm prepare
|
||||
|
||||
- name: Code quality
|
||||
run: |
|
||||
npmci command npm install -g typescript
|
||||
npmci npm install
|
||||
|
||||
- name: Trigger
|
||||
run: npmci trigger
|
||||
|
||||
- name: Build docs and upload artifacts
|
||||
run: |
|
||||
npmci node install stable
|
||||
npmci npm install
|
||||
pnpm install -g @git.zone/tsdoc
|
||||
npmci command tsdoc
|
||||
continue-on-error: true
|
||||
141
.gitlab-ci.yml
141
.gitlab-ci.yml
@ -1,141 +0,0 @@
|
||||
# gitzone ci_default
|
||||
image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
|
||||
|
||||
cache:
|
||||
paths:
|
||||
- .npmci_cache/
|
||||
key: '$CI_BUILD_STAGE'
|
||||
|
||||
stages:
|
||||
- security
|
||||
- test
|
||||
- release
|
||||
- metadata
|
||||
|
||||
before_script:
|
||||
- npm install -g @shipzone/npmci
|
||||
|
||||
# ====================
|
||||
# security stage
|
||||
# ====================
|
||||
mirror:
|
||||
stage: security
|
||||
script:
|
||||
- npmci git mirror
|
||||
only:
|
||||
- tags
|
||||
tags:
|
||||
- lossless
|
||||
- docker
|
||||
- notpriv
|
||||
|
||||
auditProductionDependencies:
|
||||
image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
|
||||
stage: security
|
||||
script:
|
||||
- npmci npm prepare
|
||||
- npmci command npm install --production --ignore-scripts
|
||||
- npmci command npm config set registry https://registry.npmjs.org
|
||||
- npmci command npm audit --audit-level=high --only=prod --production
|
||||
tags:
|
||||
- docker
|
||||
allow_failure: true
|
||||
|
||||
auditDevDependencies:
|
||||
image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
|
||||
stage: security
|
||||
script:
|
||||
- npmci npm prepare
|
||||
- npmci command npm install --ignore-scripts
|
||||
- npmci command npm config set registry https://registry.npmjs.org
|
||||
- npmci command npm audit --audit-level=high --only=dev
|
||||
tags:
|
||||
- docker
|
||||
allow_failure: true
|
||||
|
||||
# ====================
|
||||
# test stage
|
||||
# ====================
|
||||
|
||||
testStable:
|
||||
stage: test
|
||||
script:
|
||||
- npmci npm prepare
|
||||
- npmci node install stable
|
||||
- npmci npm install
|
||||
- npmci npm test
|
||||
coverage: /\d+.?\d+?\%\s*coverage/
|
||||
tags:
|
||||
- docker
|
||||
|
||||
testBuild:
|
||||
stage: test
|
||||
script:
|
||||
- npmci npm prepare
|
||||
- npmci node install stable
|
||||
- npmci npm install
|
||||
- npmci command npm run build
|
||||
coverage: /\d+.?\d+?\%\s*coverage/
|
||||
tags:
|
||||
- docker
|
||||
|
||||
release:
|
||||
stage: release
|
||||
script:
|
||||
- npmci node install stable
|
||||
- npmci npm publish
|
||||
only:
|
||||
- tags
|
||||
tags:
|
||||
- lossless
|
||||
- docker
|
||||
- notpriv
|
||||
|
||||
# ====================
|
||||
# metadata stage
|
||||
# ====================
|
||||
codequality:
|
||||
stage: metadata
|
||||
allow_failure: true
|
||||
only:
|
||||
- tags
|
||||
script:
|
||||
- npmci command npm install -g tslint typescript
|
||||
- npmci npm prepare
|
||||
- npmci npm install
|
||||
- npmci command "tslint -c tslint.json ./ts/**/*.ts"
|
||||
tags:
|
||||
- lossless
|
||||
- docker
|
||||
- priv
|
||||
|
||||
trigger:
|
||||
stage: metadata
|
||||
script:
|
||||
- npmci trigger
|
||||
only:
|
||||
- tags
|
||||
tags:
|
||||
- lossless
|
||||
- docker
|
||||
- notpriv
|
||||
|
||||
pages:
|
||||
stage: metadata
|
||||
script:
|
||||
- npmci node install lts
|
||||
- npmci command npm install -g @gitzone/tsdoc
|
||||
- npmci npm prepare
|
||||
- npmci npm install
|
||||
- npmci command tsdoc
|
||||
tags:
|
||||
- lossless
|
||||
- docker
|
||||
- notpriv
|
||||
only:
|
||||
- tags
|
||||
artifacts:
|
||||
expire_in: 1 week
|
||||
paths:
|
||||
- public
|
||||
allow_failure: true
|
||||
3
.vscode/settings.json
vendored
3
.vscode/settings.json
vendored
@ -22,6 +22,5 @@
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"typescript.tsdk": "node_modules/typescript/lib"
|
||||
]
|
||||
}
|
||||
|
||||
6
changelog.md
Normal file
6
changelog.md
Normal file
@ -0,0 +1,6 @@
|
||||
# Changelog
|
||||
|
||||
## 2.1.0 - feat(core): Added comprehensive support for `SmartClickHouseDb` and `TimeDataTable` with features including time data table creation, data insertion, bulk data insertion, querying, data deletion, and real-time data observation. Included standalone Clickhouse HTTP client implementation.
|
||||
|
||||
### Fixed
|
||||
- Fixed test case for table deletion and optimized code
|
||||
2
license
2
license
@ -1,4 +1,4 @@
|
||||
Copyright (c) 2022 Lossless GmbH (hello@lossless.com)
|
||||
Copyright (c) 2022 Task Venture Capital GmbH (hello@task.vc)
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
|
||||
@ -2,17 +2,41 @@
|
||||
"gitzone": {
|
||||
"projectType": "npm",
|
||||
"module": {
|
||||
"githost": "gitlab.com",
|
||||
"gitscope": "pushrocks",
|
||||
"githost": "code.foss.global",
|
||||
"gitscope": "push.rocks",
|
||||
"gitrepo": "smartclickhouse",
|
||||
"description": "an odm for talking to clickhouse",
|
||||
"npmPackagename": "@pushrocks/smartclickhouse",
|
||||
"description": "A TypeScript-based ODM for ClickHouse databases that supports creating, managing, and querying tables with a focus on handling time-series data.",
|
||||
"npmPackagename": "@push.rocks/smartclickhouse",
|
||||
"license": "MIT",
|
||||
"projectDomain": "push.rocks"
|
||||
"projectDomain": "push.rocks",
|
||||
"keywords": [
|
||||
"ClickHouse",
|
||||
"ODM",
|
||||
"database",
|
||||
"TypeScript",
|
||||
"Docker",
|
||||
"Grafana",
|
||||
"data management",
|
||||
"table management",
|
||||
"analytics",
|
||||
"data storage",
|
||||
"time-series data",
|
||||
"schema management",
|
||||
"data insertion",
|
||||
"real-time data",
|
||||
"data querying",
|
||||
"bulk data insertion",
|
||||
"error handling",
|
||||
"data deletion",
|
||||
"observables"
|
||||
]
|
||||
}
|
||||
},
|
||||
"npmci": {
|
||||
"npmGlobalTools": [],
|
||||
"npmAccessLevel": "public"
|
||||
},
|
||||
"tsdoc": {
|
||||
"legal": "\n## License and Legal Information\n\nThis repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository. \n\n**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.\n\n### Trademarks\n\nThis project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.\n\n### Company Information\n\nTask Venture Capital GmbH \nRegistered at District court Bremen HRB 35230 HB, Germany\n\nFor any legal inquiries or if you require further information, please contact us via email at hello@task.vc.\n\nBy using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.\n"
|
||||
}
|
||||
}
|
||||
4005
package-lock.json
generated
4005
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
63
package.json
63
package.json
@ -1,8 +1,8 @@
|
||||
{
|
||||
"name": "@pushrocks/smartclickhouse",
|
||||
"version": "2.0.6",
|
||||
"name": "@push.rocks/smartclickhouse",
|
||||
"version": "2.1.0",
|
||||
"private": false,
|
||||
"description": "an odm for talking to clickhouse",
|
||||
"description": "A TypeScript-based ODM for ClickHouse databases that supports creating, managing, and querying tables with a focus on handling time-series data.",
|
||||
"main": "dist_ts/index.js",
|
||||
"typings": "dist_ts/index.d.ts",
|
||||
"type": "module",
|
||||
@ -10,25 +10,28 @@
|
||||
"license": "MIT",
|
||||
"scripts": {
|
||||
"test": "(tstest test/ --web)",
|
||||
"build": "(tsbuild --web)",
|
||||
"build": "(tsbuild --web --allowimplicitany)",
|
||||
"createGrafana": "docker run --name grafana -d -p 4000:3000 grafana/grafana-oss",
|
||||
"createClickhouse": "docker run --name some-clickhouse-server --ulimit nofile=262144:262144 -p 8123:8123 -p 9000:9000 --volume=$PWD/.nogit/testdatabase:/var/lib/clickhouse yandex/clickhouse-server"
|
||||
"createClickhouse": "docker run --name some-clickhouse-server --ulimit nofile=262144:262144 -p 8123:8123 -p 9000:9000 --volume=$PWD/.nogit/testdatabase:/var/lib/clickhouse yandex/clickhouse-server",
|
||||
"buildDocs": "tsdoc"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@gitzone/tsbuild": "^2.1.63",
|
||||
"@gitzone/tsbundle": "^2.0.6",
|
||||
"@gitzone/tstest": "^1.0.72",
|
||||
"@pushrocks/tapbundle": "^5.0.4",
|
||||
"@types/node": "^18.6.1",
|
||||
"@git.zone/tsbuild": "^2.1.66",
|
||||
"@git.zone/tsbundle": "^2.0.8",
|
||||
"@git.zone/tsrun": "^1.2.46",
|
||||
"@git.zone/tstest": "^1.0.77",
|
||||
"@push.rocks/tapbundle": "^5.0.8",
|
||||
"@types/node": "^20.14.2",
|
||||
"tslint": "^6.1.3",
|
||||
"tslint-config-prettier": "^1.15.0"
|
||||
"tslint-config-prettier": "^1.18.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"@depyronick/clickhouse-client": "^1.0.14",
|
||||
"@pushrocks/smartdelay": "^2.0.13",
|
||||
"@pushrocks/smartobject": "^1.0.10",
|
||||
"@pushrocks/smartpromise": "^3.1.7",
|
||||
"@pushrocks/smarturl": "^3.0.1"
|
||||
"@push.rocks/smartdelay": "^3.0.1",
|
||||
"@push.rocks/smartobject": "^1.0.10",
|
||||
"@push.rocks/smartpromise": "^4.0.2",
|
||||
"@push.rocks/smartrx": "^3.0.7",
|
||||
"@push.rocks/smarturl": "^3.0.6",
|
||||
"@push.rocks/webrequest": "^3.0.28"
|
||||
},
|
||||
"browserslist": [
|
||||
"last 1 chrome versions"
|
||||
@ -44,5 +47,31 @@
|
||||
"cli.js",
|
||||
"npmextra.json",
|
||||
"readme.md"
|
||||
]
|
||||
],
|
||||
"keywords": [
|
||||
"ClickHouse",
|
||||
"ODM",
|
||||
"database",
|
||||
"TypeScript",
|
||||
"Docker",
|
||||
"Grafana",
|
||||
"data management",
|
||||
"table management",
|
||||
"analytics",
|
||||
"data storage",
|
||||
"time-series data",
|
||||
"schema management",
|
||||
"data insertion",
|
||||
"real-time data",
|
||||
"data querying",
|
||||
"bulk data insertion",
|
||||
"error handling",
|
||||
"data deletion",
|
||||
"observables"
|
||||
],
|
||||
"homepage": "https://code.foss.global/push.rocks/smartclickhouse",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://code.foss.global/push.rocks/smartclickhouse.git"
|
||||
}
|
||||
}
|
||||
|
||||
6823
pnpm-lock.yaml
generated
Normal file
6823
pnpm-lock.yaml
generated
Normal file
File diff suppressed because it is too large
Load Diff
1
readme.hints.md
Normal file
1
readme.hints.md
Normal file
@ -0,0 +1 @@
|
||||
- there is a local playground with clickhouse hosted under http://localhost:8123/play
|
||||
273
readme.md
273
readme.md
@ -1,39 +1,256 @@
|
||||
# @pushrocks/smartclickhouse
|
||||
an odm for talking to clickhouse
|
||||
# @push.rocks/smartclickhouse
|
||||
|
||||
## Availabililty and Links
|
||||
* [npmjs.org (npm package)](https://www.npmjs.com/package/@pushrocks/smartclickhouse)
|
||||
* [gitlab.com (source)](https://gitlab.com/pushrocks/smartclickhouse)
|
||||
* [github.com (source mirror)](https://github.com/pushrocks/smartclickhouse)
|
||||
* [docs (typedoc)](https://pushrocks.gitlab.io/smartclickhouse/)
|
||||
A TypeScript-based ODM (Object-Document Mapper) for ClickHouse databases, with support for creating and managing tables and handling time-series data.
|
||||
|
||||
## Status for master
|
||||
## Install
|
||||
|
||||
Status Category | Status Badge
|
||||
-- | --
|
||||
GitLab Pipelines | [](https://lossless.cloud)
|
||||
GitLab Pipline Test Coverage | [](https://lossless.cloud)
|
||||
npm | [](https://lossless.cloud)
|
||||
Snyk | [](https://lossless.cloud)
|
||||
TypeScript Support | [](https://lossless.cloud)
|
||||
node Support | [](https://nodejs.org/dist/latest-v10.x/docs/api/)
|
||||
Code Style | [](https://lossless.cloud)
|
||||
PackagePhobia (total standalone install weight) | [](https://lossless.cloud)
|
||||
PackagePhobia (package size on registry) | [](https://lossless.cloud)
|
||||
BundlePhobia (total size when bundled) | [](https://lossless.cloud)
|
||||
Platform support | [](https://lossless.cloud) [](https://lossless.cloud)
|
||||
To install `@push.rocks/smartclickhouse`, use the following command with npm:
|
||||
|
||||
```sh
|
||||
npm install @push.rocks/smartclickhouse --save
|
||||
```
|
||||
|
||||
Or with yarn:
|
||||
|
||||
```sh
|
||||
yarn add @push.rocks/smartclickhouse
|
||||
```
|
||||
|
||||
This will add the package to your project's dependencies.
|
||||
|
||||
## Usage
|
||||
|
||||
Use TypeScript for best in class intellisense
|
||||
`@push.rocks/smartclickhouse` is an advanced ODM (Object Document Mapper) module designed for seamless interaction with ClickHouse databases leveraging the capabilities of TypeScript for strong typing and enhanced developer experience. Below is a comprehensive guide to using the package in various scenarios.
|
||||
|
||||
## Contribution
|
||||
### Setting Up and Starting the Connection
|
||||
|
||||
We are always happy for code contributions. If you are not the code contributing type that is ok. Still, maintaining Open Source repositories takes considerable time and thought. If you like the quality of what we do and our modules are useful to you we would appreciate a little monthly contribution: You can [contribute one time](https://lossless.link/contribute-onetime) or [contribute monthly](https://lossless.link/contribute). :)
|
||||
To begin using `@push.rocks/smartclickhouse`, you need to establish a connection with the ClickHouse database. This involves creating an instance of `SmartClickHouseDb` and starting it:
|
||||
|
||||
For further information read the linked docs at the top of this readme.
|
||||
```typescript
|
||||
import { SmartClickHouseDb } from '@push.rocks/smartclickhouse';
|
||||
|
||||
> MIT licensed | **©** [Lossless GmbH](https://lossless.gmbh)
|
||||
| By using this npm module you agree to our [privacy policy](https://lossless.gmbH/privacy)
|
||||
// Create a new instance of SmartClickHouseDb with your ClickHouse database details
|
||||
const dbInstance = new SmartClickHouseDb({
|
||||
url: 'http://localhost:8123', // URL of ClickHouse instance
|
||||
database: 'yourDatabase', // Database name you want to connect to
|
||||
username: 'default', // Optional: Username for authentication
|
||||
password: 'password', // Optional: Password for authentication
|
||||
unref: true // Optional: Allows service to exit while awaiting database startup
|
||||
});
|
||||
|
||||
[](https://maintainedby.lossless.com)
|
||||
// Start the instance to establish the connection
|
||||
await dbInstance.start();
|
||||
```
|
||||
|
||||
### Working with Time Data Tables
|
||||
|
||||
`smartclickhouse` allows handling of time-series data through `TimeDataTable`, automating tasks such as table creation and data insertion.
|
||||
|
||||
#### Creating or Accessing a Table
|
||||
|
||||
To create a new time data table or access an existing one:
|
||||
|
||||
```typescript
|
||||
const tableName = 'yourTimeDataTable'; // Name of the table you want to access or create
|
||||
const table = await dbInstance.getTable(tableName);
|
||||
```
|
||||
|
||||
#### Adding Data to the Table
|
||||
|
||||
Once you have the table instance, you can insert data into it:
|
||||
|
||||
```typescript
|
||||
await table.addData({
|
||||
timestamp: Date.now(), // Timestamp in milliseconds
|
||||
message: 'A log message.', // Arbitrary data field
|
||||
temperature: 22.5, // Another example field
|
||||
tags: ['tag1', 'tag2'] // An example array field
|
||||
});
|
||||
```
|
||||
|
||||
The `addData` method is designed to be flexible, allowing insertion of various data types and automatically managing table schema adjustments.
|
||||
|
||||
### Advanced Usage and Custom Data Handling
|
||||
|
||||
`smartclickhouse` supports custom data types and complex data structures. For instance, to add support for nested objects or custom data processing before insertion, you might need to extend existing classes or customize the `addData` method to fit your needs.
|
||||
|
||||
#### Custom Data Processing
|
||||
|
||||
To handle complex data structures or to perform custom data processing before insertion, you might need to modify the `addData` method. Below is an example of extending the `SmartClickHouseDb` method:
|
||||
|
||||
```typescript
|
||||
class CustomClickHouseDb extends SmartClickHouseDb {
|
||||
public async addCustomData(tableName: string, data: any) {
|
||||
const table = await this.getTable(tableName);
|
||||
const customData = {
|
||||
...data,
|
||||
processedAt: Date.now(),
|
||||
customField: 'customValue',
|
||||
};
|
||||
await table.addData(customData);
|
||||
}
|
||||
}
|
||||
|
||||
const customDbInstance = new CustomClickHouseDb({
|
||||
url: 'http://localhost:8123',
|
||||
database: 'yourDatabase',
|
||||
});
|
||||
|
||||
await customDbInstance.start();
|
||||
|
||||
await customDbInstance.addCustomData('customTable', {
|
||||
message: 'Test message',
|
||||
randomField: 123456,
|
||||
});
|
||||
```
|
||||
|
||||
### Bulk Data Insertion
|
||||
|
||||
`@push.rocks/smartclickhouse` supports efficient bulk data insertion mechanisms. This feature is useful when you need to insert a large amount of data in a single operation.
|
||||
|
||||
```typescript
|
||||
const bulkData = [
|
||||
{ timestamp: Date.now(), message: 'Message 1', temperature: 20.1 },
|
||||
{ timestamp: Date.now(), message: 'Message 2', temperature: 21.2 },
|
||||
// Additional data entries...
|
||||
];
|
||||
|
||||
await table.addData(bulkData);
|
||||
```
|
||||
|
||||
### Querying Data
|
||||
|
||||
Fetching data from the ClickHouse database includes operations such as retrieving the latest entries, entries within a specific timestamp range, or streaming new entries.
|
||||
|
||||
#### Retrieving the Last N Entries
|
||||
|
||||
To retrieve the last `N` number of entries:
|
||||
|
||||
```typescript
|
||||
const latestEntries = await table.getLastEntries(10);
|
||||
console.log('Latest Entries:', latestEntries);
|
||||
```
|
||||
|
||||
#### Retrieving Entries Newer than a Specific Timestamp
|
||||
|
||||
To retrieve entries that are newer than a specific timestamp:
|
||||
|
||||
```typescript
|
||||
const timestamp = Date.now() - 60000; // 1 minute ago
|
||||
const newEntries = await table.getEntriesNewerThan(timestamp);
|
||||
console.log('New Entries:', newEntries);
|
||||
```
|
||||
|
||||
#### Retrieving Entries Between Two Timestamps
|
||||
|
||||
To retrieve entries between two timestamps:
|
||||
|
||||
```typescript
|
||||
const startTimestamp = Date.now() - 120000; // 2 minutes ago
|
||||
const endTimestamp = Date.now() - 5000; // 5 seconds ago
|
||||
const entriesBetween = await table.getEntriesBetween(startTimestamp, endTimestamp);
|
||||
console.log('Entries Between:', entriesBetween);
|
||||
```
|
||||
|
||||
### Managing and Deleting Data
|
||||
|
||||
The module provides functionality for managing and deleting data within the ClickHouse database.
|
||||
|
||||
#### Deleting Old Entries
|
||||
|
||||
You can delete entries older than a specified number of days:
|
||||
|
||||
```typescript
|
||||
// Ensure there are entries before deletion
|
||||
let entries = await table.getLastEntries(1000);
|
||||
console.log('Entries before deletion:', entries.length);
|
||||
|
||||
// Delete all entries older than now
|
||||
await table.deleteOldEntries(0);
|
||||
|
||||
// Verify the entries are deleted
|
||||
entries = await table.getLastEntries(1000);
|
||||
console.log('Entries after deletion:', entries.length);
|
||||
```
|
||||
|
||||
#### Deleting the Entire Table
|
||||
|
||||
To delete the entire table and all its data:
|
||||
|
||||
```typescript
|
||||
await table.delete();
|
||||
|
||||
// Verify table deletion
|
||||
const result = await dbInstance.clickhouseHttpClient.queryPromise(`
|
||||
SHOW TABLES FROM ${dbInstance.options.database} LIKE '${table.options.tableName}'
|
||||
`);
|
||||
console.log('Table exists after deletion:', result.length === 0);
|
||||
```
|
||||
|
||||
### Observing Real-Time Data
|
||||
|
||||
To observe new entries in real-time, you can stream new data entries using the RxJS Observable:
|
||||
|
||||
```typescript
|
||||
const stream = table.watchNewEntries();
|
||||
|
||||
const subscription = stream.subscribe((entry) => {
|
||||
console.log('New entry:', entry);
|
||||
});
|
||||
|
||||
// Simulate adding new entries
|
||||
let i = 0;
|
||||
while (i < 10) {
|
||||
await table.addData({
|
||||
timestamp: Date.now(),
|
||||
message: `streaming message ${i}`,
|
||||
});
|
||||
i++;
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000)); // Add a delay to simulate real-time data insertion
|
||||
}
|
||||
|
||||
subscription.unsubscribe();
|
||||
```
|
||||
|
||||
This method allows continuous monitoring of data changes and integrating the collected data into other systems for real-time applications.
|
||||
|
||||
### Comprehensive Feature Set
|
||||
|
||||
While the examples provided cover the core functionalities of the `@push.rocks/smartclickhouse` module, it also offers a wide range of additional features, including:
|
||||
|
||||
- **Error Handling and Reconnection Strategies**: Robust error handling mechanisms ensure your application remains reliable. Automatic reconnection strategies help maintain persistent connections with the ClickHouse database.
|
||||
- **Materialized Views and MergeTree Engines**: Support for ClickHouse-specific features such as materialized views and aggregating MergeTree engines, enhancing the module's capabilities in handling large-scale data queries and management.
|
||||
- **Efficient Data Handling**: Techniques for managing and querying large time-series datasets, providing optimal performance and reliability.
|
||||
|
||||
### Contribution
|
||||
|
||||
Contributions to `@push.rocks/smartclickhouse` are welcome. Whether through submitting issues, proposing improvements, or adding to the codebase, your input is valuable. The project is designed to be open and accessible, striving for a high-quality, community-driven development process.
|
||||
|
||||
To contribute:
|
||||
|
||||
1. Fork the repository.
|
||||
2. Create a new branch (`git checkout -b feature-branch`).
|
||||
3. Commit your changes (`git commit -am 'Add some feature'`).
|
||||
4. Push to the branch (`git push origin feature-branch`).
|
||||
5. Create a new Pull Request.
|
||||
|
||||
The above scenarios cover the essential functionality and the more advanced use cases of `@push.rocks/smartclickhouse`, providing a comprehensive guide to utilizing the module into your projects. Happy coding!
|
||||
|
||||
## License and Legal Information
|
||||
|
||||
This repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository.
|
||||
|
||||
**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.
|
||||
|
||||
### Trademarks
|
||||
|
||||
This project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.
|
||||
|
||||
### Company Information
|
||||
|
||||
Task Venture Capital GmbH
|
||||
Registered at District court Bremen HRB 35230 HB, Germany
|
||||
|
||||
For any legal inquiries or if you require further information, please contact us via email at hello@task.vc.
|
||||
|
||||
By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.
|
||||
|
||||
@ -1,12 +1,14 @@
|
||||
import { expect, expectAsync, tap } from '@pushrocks/tapbundle';
|
||||
import { expect, tap } from '@push.rocks/tapbundle';
|
||||
import * as smartclickhouse from '../ts/index.js';
|
||||
|
||||
let testClickhouseDb: smartclickhouse.SmartClickHouseDb;
|
||||
let table: smartclickhouse.TimeDataTable;
|
||||
|
||||
tap.test('first test', async () => {
|
||||
testClickhouseDb = new smartclickhouse.SmartClickHouseDb({
|
||||
url: 'http://localhost:8123',
|
||||
database: 'test2',
|
||||
unref: true,
|
||||
});
|
||||
});
|
||||
|
||||
@ -14,25 +16,89 @@ tap.test('should start the clickhouse db', async () => {
|
||||
await testClickhouseDb.start(true);
|
||||
});
|
||||
|
||||
tap.test('should create a timedatatable', async (toolsArg) => {
|
||||
const table = await testClickhouseDb.getTable('analytics');
|
||||
tap.test('should create a timedatatable', async () => {
|
||||
table = await testClickhouseDb.getTable('analytics');
|
||||
let i = 0;
|
||||
while(i < 5000) {
|
||||
while (i < 1000) {
|
||||
await table.addData({
|
||||
timestamp: Date.now(),
|
||||
message: `hello this is a message ${i}`,
|
||||
wow: 'hey',
|
||||
deep: {
|
||||
so: 'hello',
|
||||
myArray: ['array1', 'array2']
|
||||
}
|
||||
myArray: ['array1', 'array2'],
|
||||
},
|
||||
});
|
||||
i++;
|
||||
console.log(`logged ${i} of 5000 lines.`);
|
||||
await toolsArg.delayFor(1);
|
||||
console.log(`logged ${i} of 1000 lines.`);
|
||||
}
|
||||
});
|
||||
|
||||
tap.skip.test('should write something to the clickhouse db', async () => {});
|
||||
tap.test('should retrieve the last 10 entries', async () => {
|
||||
const entries = await table.getLastEntries(10);
|
||||
expect(entries.length).toEqual(10);
|
||||
console.log(entries);
|
||||
});
|
||||
|
||||
tap.start();
|
||||
tap.test('should retrieve entries newer than a specific timestamp', async () => {
|
||||
const timestamp = Date.now() - 60000; // 1 minute ago
|
||||
const entries = await table.getEntriesNewerThan(timestamp);
|
||||
expect(entries.length).toBeGreaterThan(0);
|
||||
console.log(entries);
|
||||
});
|
||||
|
||||
tap.test('should retrieve entries between two timestamps', async () => {
|
||||
const startTimestamp = Date.now() - 120000; // 2 minutes ago
|
||||
const endTimestamp = Date.now() - 5000; // 5 seconds ago
|
||||
const entries = await table.getEntriesBetween(startTimestamp, endTimestamp);
|
||||
expect(entries.length).toBeGreaterThan(0);
|
||||
console.log(entries);
|
||||
});
|
||||
|
||||
tap.test('should delete old entries', async (toolsArg) => {
|
||||
// Ensure there are entries before deletion
|
||||
let entries = await table.getLastEntries(1000);
|
||||
expect(entries.length).toBeGreaterThan(100);
|
||||
console.log('Entries before deletion:', entries.length);
|
||||
|
||||
await table.deleteOldEntries(0); // Delete all entries older than now
|
||||
// Add a delay to ensure the delete operation completes
|
||||
await new Promise(resolve => setTimeout(resolve, 5000));
|
||||
|
||||
// Verify the entries are deleted
|
||||
entries = await table.getLastEntries(1000);
|
||||
console.log('Entries after deletion:', entries.length);
|
||||
expect(entries.length).toBeLessThan(100);
|
||||
await toolsArg.delayFor(5000);
|
||||
});
|
||||
|
||||
tap.test('should stream new entries', async (toolsArg) => {
|
||||
const stream = table.watchNewEntries();
|
||||
const subscription = stream.subscribe((entry) => {
|
||||
console.log('New entry:', entry);
|
||||
});
|
||||
|
||||
let i = 0;
|
||||
while (i < 10) {
|
||||
await table.addData({
|
||||
timestamp: Date.now(),
|
||||
message: `streaming message ${i}`,
|
||||
});
|
||||
i++;
|
||||
await toolsArg.delayFor(1000); // Add a delay to simulate real-time data insertion
|
||||
}
|
||||
|
||||
subscription.unsubscribe();
|
||||
});
|
||||
|
||||
tap.test('should delete the table', async () => {
|
||||
await table.delete();
|
||||
// Verify table deletion
|
||||
const result = await testClickhouseDb.clickhouseHttpClient.queryPromise(`
|
||||
SHOW TABLES FROM ${testClickhouseDb.options.database} LIKE '${table.options.tableName}'
|
||||
`);
|
||||
console.log('Table exists after deletion:', result);
|
||||
expect(result.length).toEqual(0);
|
||||
});
|
||||
|
||||
export default tap.start();
|
||||
@ -1,8 +1,8 @@
|
||||
/**
|
||||
* autocreated commitinfo by @pushrocks/commitinfo
|
||||
* autocreated commitinfo by @push.rocks/commitinfo
|
||||
*/
|
||||
export const commitinfo = {
|
||||
name: '@pushrocks/smartclickhouse',
|
||||
version: '2.0.6',
|
||||
description: 'an odm for talking to clickhouse'
|
||||
name: '@push.rocks/smartclickhouse',
|
||||
version: '2.1.0',
|
||||
description: 'A TypeScript-based ODM for ClickHouse databases that supports creating, managing, and querying tables with a focus on handling time-series data.'
|
||||
}
|
||||
|
||||
3
ts/smartclickhouse.classes.clickhousedb.ts
Normal file
3
ts/smartclickhouse.classes.clickhousedb.ts
Normal file
@ -0,0 +1,3 @@
|
||||
import * as plugins from './smartclickhouse.plugins.js';
|
||||
|
||||
export class ClickhouseDb {}
|
||||
100
ts/smartclickhouse.classes.httpclient.ts
Normal file
100
ts/smartclickhouse.classes.httpclient.ts
Normal file
@ -0,0 +1,100 @@
|
||||
import * as plugins from './smartclickhouse.plugins.js';
|
||||
|
||||
export interface IClickhouseHttpClientOptions {
|
||||
username?: string;
|
||||
password?: string;
|
||||
url: string;
|
||||
}
|
||||
|
||||
export class ClickhouseHttpClient {
|
||||
// STATIC
|
||||
public static async createAndStart(optionsArg: IClickhouseHttpClientOptions) {
|
||||
const clickhouseHttpInstance = new ClickhouseHttpClient(optionsArg);
|
||||
await clickhouseHttpInstance.start();
|
||||
return clickhouseHttpInstance;
|
||||
}
|
||||
|
||||
// INSTANCE
|
||||
public options: IClickhouseHttpClientOptions;
|
||||
public webrequestInstance = new plugins.webrequest.WebRequest({
|
||||
logging: false,
|
||||
});
|
||||
public computedProperties: {
|
||||
connectionUrl: string;
|
||||
parsedUrl: plugins.smarturl.Smarturl;
|
||||
} = {
|
||||
connectionUrl: null,
|
||||
parsedUrl: null,
|
||||
};
|
||||
constructor(optionsArg: IClickhouseHttpClientOptions) {
|
||||
this.options = optionsArg;
|
||||
}
|
||||
|
||||
public async start() {
|
||||
this.computedProperties.parsedUrl = plugins.smarturl.Smarturl.createFromUrl(this.options.url);
|
||||
console.log(this.computedProperties.parsedUrl);
|
||||
this.computedProperties.connectionUrl = this.computedProperties.parsedUrl.toString();
|
||||
}
|
||||
|
||||
public async ping() {
|
||||
const ping = await this.webrequestInstance.request(
|
||||
this.computedProperties.connectionUrl.toString(),
|
||||
{
|
||||
method: 'GET',
|
||||
timeoutMs: 1000,
|
||||
}
|
||||
);
|
||||
return ping.status === 200 ? true : false;
|
||||
}
|
||||
|
||||
public async queryPromise(queryArg: string) {
|
||||
const returnArray = [];
|
||||
const response = await this.webrequestInstance.request(
|
||||
`${this.computedProperties.connectionUrl}?query=${encodeURIComponent(queryArg)}`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: this.getHeaders(),
|
||||
}
|
||||
);
|
||||
// console.log('===================');
|
||||
// console.log(this.computedProperties.connectionUrl);
|
||||
// console.log(queryArg);
|
||||
// console.log((await response.clone().text()).split(/\r?\n/))
|
||||
if (response.headers.get('X-ClickHouse-Format') === 'JSONEachRow') {
|
||||
const jsonList = await response.text();
|
||||
const jsonArray = jsonList.split('\n');
|
||||
for (const jsonArg of jsonArray) {
|
||||
if (!jsonArg) {
|
||||
continue;
|
||||
}
|
||||
returnArray.push(JSON.parse(jsonArg));
|
||||
}
|
||||
} else {
|
||||
}
|
||||
return returnArray;
|
||||
}
|
||||
|
||||
public async insertPromise(databaseArg: string, tableArg: string, documents: any[]) {
|
||||
const queryArg = `INSERT INTO ${databaseArg}.${tableArg} FORMAT JSONEachRow`;
|
||||
const response = await this.webrequestInstance.request(
|
||||
`${this.computedProperties.connectionUrl}?query=${encodeURIComponent(queryArg)}`,
|
||||
{
|
||||
method: 'POST',
|
||||
body: documents.map((docArg) => JSON.stringify(docArg)).join('\n'),
|
||||
headers: this.getHeaders(),
|
||||
}
|
||||
);
|
||||
return response;
|
||||
}
|
||||
|
||||
private getHeaders() {
|
||||
const headers: { [key: string]: string } = {};
|
||||
if (this.options.username) {
|
||||
headers['X-ClickHouse-User'] = this.options.username;
|
||||
}
|
||||
if (this.options.password) {
|
||||
headers['X-ClickHouse-Key'] = this.options.password;
|
||||
}
|
||||
return headers;
|
||||
}
|
||||
}
|
||||
@ -1,16 +1,23 @@
|
||||
import * as plugins from './smartclickhouse.plugins.js';
|
||||
import { TimeDataTable } from './smartclickhouse.classes.timedatatable.js';
|
||||
import { ClickhouseHttpClient } from './smartclickhouse.classes.httpclient.js';
|
||||
|
||||
export interface IClickhouseConstructorOptions {
|
||||
url: string;
|
||||
database: string;
|
||||
username?: string;
|
||||
password?: string;
|
||||
/**
|
||||
* allow services to exit when waiting for clickhouse startup
|
||||
* this allows to leave the lifecycle flow to other processes
|
||||
* like a listening server.
|
||||
*/
|
||||
unref?: boolean;
|
||||
}
|
||||
|
||||
export class SmartClickHouseDb {
|
||||
public options: IClickhouseConstructorOptions;
|
||||
public defaultClient: plugins.clickhouse.ClickHouseClient;
|
||||
public clickhouseClient: plugins.clickhouse.ClickHouseClient;
|
||||
public clickhouseHttpClient: ClickhouseHttpClient;
|
||||
|
||||
constructor(optionsArg: IClickhouseConstructorOptions) {
|
||||
this.options = optionsArg;
|
||||
@ -21,48 +28,31 @@ export class SmartClickHouseDb {
|
||||
*/
|
||||
public async start(dropOld = false) {
|
||||
console.log(`Connecting to default database first.`);
|
||||
const defaultOptions: {[keyArg: string]: string} = {};
|
||||
// the protocol, url and host
|
||||
const parsedUrl = plugins.smarturl.Smarturl.createFromUrl(this.options.url);
|
||||
parsedUrl.protocol === 'https' ? defaultOptions.protocol = plugins.clickhouse.ClickHouseConnectionProtocol.HTTPS : null;
|
||||
parsedUrl.protocol === 'http' ? defaultOptions.protocol = plugins.clickhouse.ClickHouseConnectionProtocol.HTTP : null;
|
||||
defaultOptions.host = parsedUrl.hostname;
|
||||
defaultOptions.port = parsedUrl.port;
|
||||
// the database
|
||||
defaultOptions.database = this.options.database;
|
||||
// the password
|
||||
this.options.password ? defaultOptions.password = this.options.password : null;
|
||||
// lets connect
|
||||
this.defaultClient = new plugins.clickhouse.ClickHouseClient({
|
||||
...defaultOptions,
|
||||
database: 'default',
|
||||
});
|
||||
this.clickhouseHttpClient = await ClickhouseHttpClient.createAndStart(this.options);
|
||||
await this.pingDatabaseUntilAvailable();
|
||||
console.log(`Create database ${this.options.database}, if it does not exist...`);
|
||||
await this.createDatabase(dropOld);
|
||||
|
||||
console.log(`Ensured database. Now connecting to wanted database: ${this.options.database}`);
|
||||
this.clickhouseClient = new plugins.clickhouse.ClickHouseClient({
|
||||
...defaultOptions
|
||||
});
|
||||
}
|
||||
|
||||
public async createDatabase(dropOld: boolean = false) {
|
||||
if (dropOld) {
|
||||
await this.defaultClient.queryPromise(`DROP DATABASE IF EXISTS ${this.options.database}`);
|
||||
await this.clickhouseHttpClient.queryPromise(`DROP DATABASE IF EXISTS ${this.options.database}`);
|
||||
}
|
||||
await this.defaultClient.queryPromise(`CREATE DATABASE IF NOT EXISTS ${this.options.database}`);
|
||||
await this.clickhouseHttpClient.queryPromise(
|
||||
`CREATE DATABASE IF NOT EXISTS ${this.options.database}`
|
||||
);
|
||||
}
|
||||
|
||||
public async pingDatabaseUntilAvailable() {
|
||||
let available = false;
|
||||
while(!available) {
|
||||
available = await this.defaultClient.ping().catch(err => {
|
||||
while (!available) {
|
||||
available = await this.clickhouseHttpClient.ping().catch((err) => {
|
||||
return false;
|
||||
});
|
||||
if (!available) {
|
||||
console.log(`NOT OK: tried pinging ${this.options.url}... Trying again in 5 seconds.`);
|
||||
await plugins.smartdelay.delayFor(5000);
|
||||
await plugins.smartdelay.delayFor(5000, null, this.options.unref);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,7 +1,12 @@
|
||||
import * as plugins from './smartclickhouse.plugins.js';
|
||||
import { SmartClickHouseDb } from './smartclickhouse.classes.smartclickhouse.js';
|
||||
|
||||
export type TClickhouseColumnDataType = 'String' | "DateTime64(3, 'Europe/Berlin')" | 'Float64' | 'Array(String)' | 'Array(Float64)';
|
||||
export type TClickhouseColumnDataType =
|
||||
| 'String'
|
||||
| "DateTime64(3, 'Europe/Berlin')"
|
||||
| 'Float64'
|
||||
| 'Array(String)'
|
||||
| 'Array(Float64)';
|
||||
export interface IColumnInfo {
|
||||
database: string;
|
||||
table: string;
|
||||
@ -35,7 +40,7 @@ export class TimeDataTable {
|
||||
public static async getTable(smartClickHouseDbRefArg: SmartClickHouseDb, tableNameArg: string) {
|
||||
const newTable = new TimeDataTable(smartClickHouseDbRefArg, {
|
||||
tableName: tableNameArg,
|
||||
retainDataForDays: 30
|
||||
retainDataForDays: 30,
|
||||
});
|
||||
|
||||
await newTable.setup();
|
||||
@ -55,21 +60,19 @@ export class TimeDataTable {
|
||||
|
||||
public async setup() {
|
||||
// create table in clickhouse
|
||||
await this.smartClickHouseDbRef.clickhouseClient
|
||||
.queryPromise(`
|
||||
CREATE TABLE IF NOT EXISTS ${this.options.tableName} (
|
||||
await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
|
||||
CREATE TABLE IF NOT EXISTS ${this.smartClickHouseDbRef.options.database}.${this.options.tableName} (
|
||||
timestamp DateTime64(3, 'Europe/Berlin'),
|
||||
message String
|
||||
) ENGINE=MergeTree() ORDER BY timestamp`);
|
||||
|
||||
// lets adjust the TTL
|
||||
await this.smartClickHouseDbRef.clickhouseClient
|
||||
.queryPromise(`
|
||||
ALTER TABLE ${this.options.tableName} MODIFY TTL toDateTime(timestamp) + INTERVAL ${this.options.retainDataForDays} DAY
|
||||
await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
|
||||
ALTER TABLE ${this.smartClickHouseDbRef.options.database}.${this.options.tableName} MODIFY TTL toDateTime(timestamp) + INTERVAL ${this.options.retainDataForDays} DAY
|
||||
`);
|
||||
|
||||
await this.updateColumns();
|
||||
console.log(`=======================`)
|
||||
console.log(`=======================`);
|
||||
console.log(
|
||||
`table with name "${this.options.tableName}" in database ${this.smartClickHouseDbRef.options.database} has the following columns:`
|
||||
);
|
||||
@ -85,10 +88,10 @@ export class TimeDataTable {
|
||||
* updates the columns
|
||||
*/
|
||||
public async updateColumns() {
|
||||
this.columns = await this.smartClickHouseDbRef.clickhouseClient.queryPromise(`
|
||||
this.columns = await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
|
||||
SELECT * FROM system.columns
|
||||
WHERE database LIKE '${this.smartClickHouseDbRef.options.database}'
|
||||
AND table LIKE '${this.options.tableName}'
|
||||
AND table LIKE '${this.options.tableName}' FORMAT JSONEachRow
|
||||
`);
|
||||
return this.columns;
|
||||
}
|
||||
@ -97,6 +100,9 @@ export class TimeDataTable {
|
||||
* stores a json and tries to map it to the nested syntax
|
||||
*/
|
||||
public async addData(dataArg: any) {
|
||||
if (this.healingDeferred) {
|
||||
return;
|
||||
}
|
||||
|
||||
// the storageJson
|
||||
let storageJson: { [key: string]: any } = {};
|
||||
@ -104,23 +110,27 @@ export class TimeDataTable {
|
||||
// helper stuff
|
||||
|
||||
const getClickhouseTypeForValue = (valueArg: any): TClickhouseColumnDataType => {
|
||||
const typeConversion: {[key: string]: TClickhouseColumnDataType} = {
|
||||
const typeConversion: { [key: string]: TClickhouseColumnDataType } = {
|
||||
string: 'String',
|
||||
number: 'Float64',
|
||||
undefined: null,
|
||||
null: null
|
||||
null: null,
|
||||
};
|
||||
if (valueArg instanceof Array) {
|
||||
const arrayType = typeConversion[(typeof valueArg[0]) as string];
|
||||
const arrayType = typeConversion[typeof valueArg[0] as string];
|
||||
if (!arrayType) {
|
||||
return null;
|
||||
} else {
|
||||
return `Array(${arrayType})` as TClickhouseColumnDataType;
|
||||
}
|
||||
}
|
||||
return typeConversion[(typeof valueArg) as string];
|
||||
}
|
||||
const checkPath = async (pathArg: string, typeArg: TClickhouseColumnDataType, prechecked = false) => {
|
||||
return typeConversion[typeof valueArg as string];
|
||||
};
|
||||
const checkPath = async (
|
||||
pathArg: string,
|
||||
typeArg: TClickhouseColumnDataType,
|
||||
prechecked = false
|
||||
) => {
|
||||
let columnFound = false;
|
||||
for (const column of this.columns) {
|
||||
if (pathArg === column.name) {
|
||||
@ -134,12 +144,12 @@ export class TimeDataTable {
|
||||
await checkPath(pathArg, typeArg, true);
|
||||
return;
|
||||
}
|
||||
const alterString = `ALTER TABLE ${this.options.tableName} ADD COLUMN ${pathArg} ${typeArg} FIRST`
|
||||
const alterString = `ALTER TABLE ${this.smartClickHouseDbRef.options.database}.${this.options.tableName} ADD COLUMN ${pathArg} ${typeArg} FIRST`;
|
||||
try {
|
||||
await this.smartClickHouseDbRef.clickhouseClient.queryPromise(`
|
||||
await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
|
||||
${alterString}
|
||||
`);
|
||||
} catch(err) {
|
||||
} catch (err) {
|
||||
console.log(alterString);
|
||||
for (const column of this.columns) {
|
||||
console.log(column.name);
|
||||
@ -168,21 +178,128 @@ export class TimeDataTable {
|
||||
storageJson[key] = value;
|
||||
}
|
||||
|
||||
const result = await this.smartClickHouseDbRef.clickhouseClient.insertPromise(this.options.tableName, [
|
||||
storageJson,
|
||||
]).catch(async () => {
|
||||
if (this.healingDeferred) {
|
||||
return;
|
||||
}
|
||||
this.healingDeferred = plugins.smartpromise.defer();
|
||||
console.log(`Ran into an error. Trying to set up things properly again.`);
|
||||
await this.smartClickHouseDbRef.pingDatabaseUntilAvailable();
|
||||
await this.smartClickHouseDbRef.createDatabase();
|
||||
await this.setup();
|
||||
this.columns = [];
|
||||
this.healingDeferred.resolve();
|
||||
this.healingDeferred = null;
|
||||
});
|
||||
const result = await this.smartClickHouseDbRef.clickhouseHttpClient
|
||||
.insertPromise(this.smartClickHouseDbRef.options.database, this.options.tableName, [
|
||||
storageJson,
|
||||
])
|
||||
.catch(async () => {
|
||||
if (this.healingDeferred) {
|
||||
return;
|
||||
}
|
||||
this.healingDeferred = plugins.smartpromise.defer();
|
||||
console.log(`Ran into an error. Trying to set up things properly again.`);
|
||||
await this.smartClickHouseDbRef.pingDatabaseUntilAvailable();
|
||||
await this.smartClickHouseDbRef.createDatabase();
|
||||
await this.setup();
|
||||
this.columns = [];
|
||||
this.healingDeferred.resolve();
|
||||
this.healingDeferred = null;
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* deletes the entire table
|
||||
*/
|
||||
public async delete() {
|
||||
await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
|
||||
DROP TABLE IF EXISTS ${this.smartClickHouseDbRef.options.database}.${this.options.tableName}
|
||||
`);
|
||||
this.columns = [];
|
||||
}
|
||||
|
||||
/**
|
||||
* deletes entries older than a specified number of days
|
||||
* @param days number of days
|
||||
*/
|
||||
public async deleteOldEntries(days: number) {
|
||||
// Perform the deletion operation
|
||||
await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
|
||||
ALTER TABLE ${this.smartClickHouseDbRef.options.database}.${this.options.tableName}
|
||||
DELETE WHERE timestamp < now() - INTERVAL ${days} DAY
|
||||
`);
|
||||
await this.waitForMutations();
|
||||
}
|
||||
|
||||
public async waitForMutations() {
|
||||
// Wait for the mutation to complete
|
||||
let mutations;
|
||||
do {
|
||||
mutations = await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
|
||||
SELECT count() AS mutations_count FROM system.mutations
|
||||
WHERE is_done = 0 AND table = '${this.options.tableName}'
|
||||
`);
|
||||
|
||||
if (mutations[0] && mutations[0].mutations_count > 0) {
|
||||
console.log('Waiting for mutations to complete...');
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
}
|
||||
} while (mutations[0] && mutations[0].mutations_count > 0);
|
||||
}
|
||||
|
||||
public async getLastEntries(count: number) {
|
||||
const result = await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
|
||||
SELECT * FROM ${this.smartClickHouseDbRef.options.database}.${this.options.tableName}
|
||||
ORDER BY timestamp DESC
|
||||
LIMIT ${count} FORMAT JSONEachRow
|
||||
`);
|
||||
return result;
|
||||
}
|
||||
|
||||
public async getEntriesNewerThan(unixTimestamp: number) {
|
||||
const result = await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
|
||||
SELECT * FROM ${this.smartClickHouseDbRef.options.database}.${this.options.tableName}
|
||||
WHERE timestamp > toDateTime(${unixTimestamp / 1000}) FORMAT JSONEachRow
|
||||
`);
|
||||
return result;
|
||||
}
|
||||
|
||||
public async getEntriesBetween(unixTimestampStart: number, unixTimestampEnd: number) {
|
||||
const result = await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
|
||||
SELECT * FROM ${this.smartClickHouseDbRef.options.database}.${this.options.tableName}
|
||||
WHERE timestamp > toDateTime(${unixTimestampStart / 1000})
|
||||
AND timestamp < toDateTime(${unixTimestampEnd / 1000}) FORMAT JSONEachRow
|
||||
`);
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* streams all new entries using an observable
|
||||
*/
|
||||
public watchNewEntries(): plugins.smartrx.rxjs.Observable<any> {
|
||||
return new plugins.smartrx.rxjs.Observable((observer) => {
|
||||
const pollInterval = 1000; // Poll every 1 second
|
||||
let lastTimestamp: number;
|
||||
let intervalId: NodeJS.Timeout;
|
||||
|
||||
const fetchLastEntryTimestamp = async () => {
|
||||
const lastEntry = await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
|
||||
SELECT max(timestamp) as lastTimestamp FROM ${this.smartClickHouseDbRef.options.database}.${this.options.tableName} FORMAT JSONEachRow
|
||||
`);
|
||||
lastTimestamp = lastEntry.length
|
||||
? new Date(lastEntry[0].lastTimestamp).getTime()
|
||||
: Date.now();
|
||||
};
|
||||
|
||||
const fetchNewEntries = async () => {
|
||||
const newEntries = await this.getEntriesNewerThan(lastTimestamp);
|
||||
if (newEntries.length > 0) {
|
||||
for (const entry of newEntries) {
|
||||
observer.next(entry);
|
||||
}
|
||||
lastTimestamp = new Date(newEntries[newEntries.length - 1].timestamp).getTime();
|
||||
}
|
||||
};
|
||||
|
||||
const startPolling = async () => {
|
||||
await fetchLastEntryTimestamp();
|
||||
intervalId = setInterval(fetchNewEntries, pollInterval);
|
||||
};
|
||||
|
||||
startPolling().catch((err) => observer.error(err));
|
||||
|
||||
// Cleanup on unsubscribe
|
||||
return () => clearInterval(intervalId);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,17 +1,9 @@
|
||||
// @pushrocks scope
|
||||
import * as smartdelay from '@pushrocks/smartdelay';
|
||||
import * as smartobject from '@pushrocks/smartobject';
|
||||
import * as smartpromise from '@pushrocks/smartpromise';
|
||||
import * as smarturl from '@pushrocks/smarturl';
|
||||
import * as smartdelay from '@push.rocks/smartdelay';
|
||||
import * as smartobject from '@push.rocks/smartobject';
|
||||
import * as smartpromise from '@push.rocks/smartpromise';
|
||||
import * as smartrx from '@push.rocks/smartrx';
|
||||
import * as smarturl from '@push.rocks/smarturl';
|
||||
import * as webrequest from '@push.rocks/webrequest';
|
||||
|
||||
export {
|
||||
smartdelay,
|
||||
smartobject,
|
||||
smartpromise,
|
||||
smarturl
|
||||
}
|
||||
|
||||
// thirdparty
|
||||
import * as clickhouse from '@depyronick/clickhouse-client';
|
||||
|
||||
export { clickhouse };
|
||||
export { smartdelay, smartobject, smartpromise, smartrx, smarturl, webrequest };
|
||||
|
||||
@ -3,8 +3,12 @@
|
||||
"experimentalDecorators": true,
|
||||
"useDefineForClassFields": false,
|
||||
"target": "ES2022",
|
||||
"module": "ES2022",
|
||||
"moduleResolution": "nodenext",
|
||||
"esModuleInterop": true
|
||||
}
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"esModuleInterop": true,
|
||||
"verbatimModuleSyntax": true
|
||||
},
|
||||
"exclude": [
|
||||
"dist_*/**/*.d.ts"
|
||||
]
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user