Compare commits
27 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 92688a7f7f | |||
| 1aed44f035 | |||
| 07895c2767 | |||
| a791c3e8c7 | |||
| 2c2910fafe | |||
| 094b3a8a50 | |||
| 6ed2abca4a | |||
| f9df862b0a | |||
| 2db69618c3 | |||
| a6bb61764b | |||
| 7d867ea6ab | |||
| d03f086c92 | |||
| 83c1e2bb4e | |||
| faa5d6d542 | |||
| 6e06e8108b | |||
| 45ce23ec11 | |||
| 93ef6a3d6b | |||
| 609873b4ad | |||
| 4a8bbc3d13 | |||
| 51c2d4f6e0 | |||
| 45091d6b8c | |||
| eae7300439 | |||
| 7753e58036 | |||
| 333e9c1316 | |||
| 479e0725e6 | |||
| fbf177b482 | |||
| 3272b87e10 |
66
.gitea/workflows/default_nottags.yaml
Normal file
66
.gitea/workflows/default_nottags.yaml
Normal file
@ -0,0 +1,66 @@
|
||||
name: Default (not tags)
|
||||
|
||||
on:
|
||||
push:
|
||||
tags-ignore:
|
||||
- '**'
|
||||
|
||||
env:
|
||||
IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
|
||||
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
|
||||
NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
|
||||
NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
|
||||
NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
|
||||
NPMCI_URL_CLOUDLY: ${{secrets.NPMCI_URL_CLOUDLY}}
|
||||
|
||||
jobs:
|
||||
security:
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true
|
||||
container:
|
||||
image: ${{ env.IMAGE }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Install pnpm and npmci
|
||||
run: |
|
||||
pnpm install -g pnpm
|
||||
pnpm install -g @shipzone/npmci
|
||||
|
||||
- name: Run npm prepare
|
||||
run: npmci npm prepare
|
||||
|
||||
- name: Audit production dependencies
|
||||
run: |
|
||||
npmci command npm config set registry https://registry.npmjs.org
|
||||
npmci command pnpm audit --audit-level=high --prod
|
||||
continue-on-error: true
|
||||
|
||||
- name: Audit development dependencies
|
||||
run: |
|
||||
npmci command npm config set registry https://registry.npmjs.org
|
||||
npmci command pnpm audit --audit-level=high --dev
|
||||
continue-on-error: true
|
||||
|
||||
test:
|
||||
if: ${{ always() }}
|
||||
needs: security
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: ${{ env.IMAGE }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Test stable
|
||||
run: |
|
||||
npmci node install stable
|
||||
npmci npm install
|
||||
npmci npm test
|
||||
|
||||
- name: Test build
|
||||
run: |
|
||||
npmci node install stable
|
||||
npmci npm install
|
||||
npmci npm build
|
||||
124
.gitea/workflows/default_tags.yaml
Normal file
124
.gitea/workflows/default_tags.yaml
Normal file
@ -0,0 +1,124 @@
|
||||
name: Default (tags)
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
|
||||
env:
|
||||
IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
|
||||
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
|
||||
NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
|
||||
NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
|
||||
NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
|
||||
NPMCI_URL_CLOUDLY: ${{secrets.NPMCI_URL_CLOUDLY}}
|
||||
|
||||
jobs:
|
||||
security:
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true
|
||||
container:
|
||||
image: ${{ env.IMAGE }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Prepare
|
||||
run: |
|
||||
pnpm install -g pnpm
|
||||
pnpm install -g @shipzone/npmci
|
||||
npmci npm prepare
|
||||
|
||||
- name: Audit production dependencies
|
||||
run: |
|
||||
npmci command npm config set registry https://registry.npmjs.org
|
||||
npmci command pnpm audit --audit-level=high --prod
|
||||
continue-on-error: true
|
||||
|
||||
- name: Audit development dependencies
|
||||
run: |
|
||||
npmci command npm config set registry https://registry.npmjs.org
|
||||
npmci command pnpm audit --audit-level=high --dev
|
||||
continue-on-error: true
|
||||
|
||||
test:
|
||||
if: ${{ always() }}
|
||||
needs: security
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: ${{ env.IMAGE }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Prepare
|
||||
run: |
|
||||
pnpm install -g pnpm
|
||||
pnpm install -g @shipzone/npmci
|
||||
npmci npm prepare
|
||||
|
||||
- name: Test stable
|
||||
run: |
|
||||
npmci node install stable
|
||||
npmci npm install
|
||||
npmci npm test
|
||||
|
||||
- name: Test build
|
||||
run: |
|
||||
npmci node install stable
|
||||
npmci npm install
|
||||
npmci npm build
|
||||
|
||||
release:
|
||||
needs: test
|
||||
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: ${{ env.IMAGE }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Prepare
|
||||
run: |
|
||||
pnpm install -g pnpm
|
||||
pnpm install -g @shipzone/npmci
|
||||
npmci npm prepare
|
||||
|
||||
- name: Release
|
||||
run: |
|
||||
npmci node install stable
|
||||
npmci npm publish
|
||||
|
||||
metadata:
|
||||
needs: test
|
||||
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: ${{ env.IMAGE }}
|
||||
continue-on-error: true
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Prepare
|
||||
run: |
|
||||
pnpm install -g pnpm
|
||||
pnpm install -g @shipzone/npmci
|
||||
npmci npm prepare
|
||||
|
||||
- name: Code quality
|
||||
run: |
|
||||
npmci command npm install -g typescript
|
||||
npmci npm install
|
||||
|
||||
- name: Trigger
|
||||
run: npmci trigger
|
||||
|
||||
- name: Build docs and upload artifacts
|
||||
run: |
|
||||
npmci node install stable
|
||||
npmci npm install
|
||||
pnpm install -g @git.zone/tsdoc
|
||||
npmci command tsdoc
|
||||
continue-on-error: true
|
||||
141
.gitlab-ci.yml
141
.gitlab-ci.yml
@ -1,141 +0,0 @@
|
||||
# gitzone ci_default
|
||||
image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
|
||||
|
||||
cache:
|
||||
paths:
|
||||
- .npmci_cache/
|
||||
key: '$CI_BUILD_STAGE'
|
||||
|
||||
stages:
|
||||
- security
|
||||
- test
|
||||
- release
|
||||
- metadata
|
||||
|
||||
before_script:
|
||||
- npm install -g @shipzone/npmci
|
||||
|
||||
# ====================
|
||||
# security stage
|
||||
# ====================
|
||||
mirror:
|
||||
stage: security
|
||||
script:
|
||||
- npmci git mirror
|
||||
only:
|
||||
- tags
|
||||
tags:
|
||||
- lossless
|
||||
- docker
|
||||
- notpriv
|
||||
|
||||
auditProductionDependencies:
|
||||
image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
|
||||
stage: security
|
||||
script:
|
||||
- npmci npm prepare
|
||||
- npmci command npm install --production --ignore-scripts
|
||||
- npmci command npm config set registry https://registry.npmjs.org
|
||||
- npmci command npm audit --audit-level=high --only=prod --production
|
||||
tags:
|
||||
- docker
|
||||
allow_failure: true
|
||||
|
||||
auditDevDependencies:
|
||||
image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
|
||||
stage: security
|
||||
script:
|
||||
- npmci npm prepare
|
||||
- npmci command npm install --ignore-scripts
|
||||
- npmci command npm config set registry https://registry.npmjs.org
|
||||
- npmci command npm audit --audit-level=high --only=dev
|
||||
tags:
|
||||
- docker
|
||||
allow_failure: true
|
||||
|
||||
# ====================
|
||||
# test stage
|
||||
# ====================
|
||||
|
||||
testStable:
|
||||
stage: test
|
||||
script:
|
||||
- npmci npm prepare
|
||||
- npmci node install stable
|
||||
- npmci npm install
|
||||
- npmci npm test
|
||||
coverage: /\d+.?\d+?\%\s*coverage/
|
||||
tags:
|
||||
- docker
|
||||
|
||||
testBuild:
|
||||
stage: test
|
||||
script:
|
||||
- npmci npm prepare
|
||||
- npmci node install stable
|
||||
- npmci npm install
|
||||
- npmci command npm run build
|
||||
coverage: /\d+.?\d+?\%\s*coverage/
|
||||
tags:
|
||||
- docker
|
||||
|
||||
release:
|
||||
stage: release
|
||||
script:
|
||||
- npmci node install stable
|
||||
- npmci npm publish
|
||||
only:
|
||||
- tags
|
||||
tags:
|
||||
- lossless
|
||||
- docker
|
||||
- notpriv
|
||||
|
||||
# ====================
|
||||
# metadata stage
|
||||
# ====================
|
||||
codequality:
|
||||
stage: metadata
|
||||
allow_failure: true
|
||||
only:
|
||||
- tags
|
||||
script:
|
||||
- npmci command npm install -g tslint typescript
|
||||
- npmci npm prepare
|
||||
- npmci npm install
|
||||
- npmci command "tslint -c tslint.json ./ts/**/*.ts"
|
||||
tags:
|
||||
- lossless
|
||||
- docker
|
||||
- priv
|
||||
|
||||
trigger:
|
||||
stage: metadata
|
||||
script:
|
||||
- npmci trigger
|
||||
only:
|
||||
- tags
|
||||
tags:
|
||||
- lossless
|
||||
- docker
|
||||
- notpriv
|
||||
|
||||
pages:
|
||||
stage: metadata
|
||||
script:
|
||||
- npmci node install lts
|
||||
- npmci command npm install -g @gitzone/tsdoc
|
||||
- npmci npm prepare
|
||||
- npmci npm install
|
||||
- npmci command tsdoc
|
||||
tags:
|
||||
- lossless
|
||||
- docker
|
||||
- notpriv
|
||||
only:
|
||||
- tags
|
||||
artifacts:
|
||||
expire_in: 1 week
|
||||
paths:
|
||||
- public
|
||||
allow_failure: true
|
||||
3
.vscode/settings.json
vendored
3
.vscode/settings.json
vendored
@ -22,6 +22,5 @@
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"typescript.tsdk": "node_modules/typescript/lib"
|
||||
]
|
||||
}
|
||||
|
||||
@ -2,17 +2,32 @@
|
||||
"gitzone": {
|
||||
"projectType": "npm",
|
||||
"module": {
|
||||
"githost": "gitlab.com",
|
||||
"gitscope": "pushrocks",
|
||||
"githost": "code.foss.global",
|
||||
"gitscope": "push.rocks",
|
||||
"gitrepo": "smartclickhouse",
|
||||
"description": "an odm for talking to clickhouse",
|
||||
"npmPackagename": "@pushrocks/smartclickhouse",
|
||||
"description": "A TypeScript-based ODM (Object-Document Mapper) for ClickHouse databases, with support for creating and managing tables and their data.",
|
||||
"npmPackagename": "@push.rocks/smartclickhouse",
|
||||
"license": "MIT",
|
||||
"projectDomain": "push.rocks"
|
||||
"projectDomain": "push.rocks",
|
||||
"keywords": [
|
||||
"ClickHouse",
|
||||
"ODM",
|
||||
"database",
|
||||
"TypeScript",
|
||||
"Docker",
|
||||
"Grafana",
|
||||
"data management",
|
||||
"table management",
|
||||
"analytics",
|
||||
"data storage"
|
||||
]
|
||||
}
|
||||
},
|
||||
"npmci": {
|
||||
"npmGlobalTools": [],
|
||||
"npmAccessLevel": "public"
|
||||
},
|
||||
"tsdoc": {
|
||||
"legal": "\n## License and Legal Information\n\nThis repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository. \n\n**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.\n\n### Trademarks\n\nThis project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.\n\n### Company Information\n\nTask Venture Capital GmbH \nRegistered at District court Bremen HRB 35230 HB, Germany\n\nFor any legal inquiries or if you require further information, please contact us via email at hello@task.vc.\n\nBy using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.\n"
|
||||
}
|
||||
}
|
||||
4009
package-lock.json
generated
4009
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
54
package.json
54
package.json
@ -1,8 +1,8 @@
|
||||
{
|
||||
"name": "@pushrocks/smartclickhouse",
|
||||
"version": "2.0.5",
|
||||
"name": "@push.rocks/smartclickhouse",
|
||||
"version": "2.0.15",
|
||||
"private": false,
|
||||
"description": "an odm for talking to clickhouse",
|
||||
"description": "A TypeScript-based ODM (Object-Document Mapper) for ClickHouse databases, with support for creating and managing tables and their data.",
|
||||
"main": "dist_ts/index.js",
|
||||
"typings": "dist_ts/index.d.ts",
|
||||
"type": "module",
|
||||
@ -10,25 +10,28 @@
|
||||
"license": "MIT",
|
||||
"scripts": {
|
||||
"test": "(tstest test/ --web)",
|
||||
"build": "(tsbuild --web)",
|
||||
"build": "(tsbuild --web --allowimplicitany)",
|
||||
"createGrafana": "docker run --name grafana -d -p 4000:3000 grafana/grafana-oss",
|
||||
"createClickhouse": "docker run --name some-clickhouse-server --ulimit nofile=262144:262144 -p 8123:8123 -p 9000:9000 --volume=$PWD/.nogit/testdatabase:/var/lib/clickhouse yandex/clickhouse-server"
|
||||
"createClickhouse": "docker run --name some-clickhouse-server --ulimit nofile=262144:262144 -p 8123:8123 -p 9000:9000 --volume=$PWD/.nogit/testdatabase:/var/lib/clickhouse yandex/clickhouse-server",
|
||||
"buildDocs": "tsdoc"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@gitzone/tsbuild": "^2.1.63",
|
||||
"@gitzone/tsbundle": "^2.0.6",
|
||||
"@gitzone/tstest": "^1.0.72",
|
||||
"@pushrocks/tapbundle": "^5.0.4",
|
||||
"@types/node": "^18.6.1",
|
||||
"@git.zone/tsbuild": "^2.1.66",
|
||||
"@git.zone/tsbundle": "^2.0.8",
|
||||
"@git.zone/tsrun": "^1.2.46",
|
||||
"@git.zone/tstest": "^1.0.77",
|
||||
"@push.rocks/tapbundle": "^5.0.8",
|
||||
"@types/node": "^20.14.2",
|
||||
"tslint": "^6.1.3",
|
||||
"tslint-config-prettier": "^1.15.0"
|
||||
"tslint-config-prettier": "^1.18.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"@depyronick/clickhouse-client": "^1.0.14",
|
||||
"@pushrocks/smartdelay": "^2.0.13",
|
||||
"@pushrocks/smartobject": "^1.0.10",
|
||||
"@pushrocks/smartpromise": "^3.1.7",
|
||||
"@pushrocks/smarturl": "^3.0.1"
|
||||
"@push.rocks/smartdelay": "^3.0.1",
|
||||
"@push.rocks/smartobject": "^1.0.10",
|
||||
"@push.rocks/smartpromise": "^4.0.2",
|
||||
"@push.rocks/smartrx": "^3.0.7",
|
||||
"@push.rocks/smarturl": "^3.0.6",
|
||||
"@push.rocks/webrequest": "^3.0.28"
|
||||
},
|
||||
"browserslist": [
|
||||
"last 1 chrome versions"
|
||||
@ -44,5 +47,22 @@
|
||||
"cli.js",
|
||||
"npmextra.json",
|
||||
"readme.md"
|
||||
]
|
||||
],
|
||||
"keywords": [
|
||||
"ClickHouse",
|
||||
"ODM",
|
||||
"database",
|
||||
"TypeScript",
|
||||
"Docker",
|
||||
"Grafana",
|
||||
"data management",
|
||||
"table management",
|
||||
"analytics",
|
||||
"data storage"
|
||||
],
|
||||
"homepage": "https://code.foss.global/push.rocks/smartclickhouse",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://code.foss.global/push.rocks/smartclickhouse.git"
|
||||
}
|
||||
}
|
||||
|
||||
6823
pnpm-lock.yaml
generated
Normal file
6823
pnpm-lock.yaml
generated
Normal file
File diff suppressed because it is too large
Load Diff
1
readme.hints.md
Normal file
1
readme.hints.md
Normal file
@ -0,0 +1 @@
|
||||
- there is a local playground with clickhouse hosted under http://localhost:8123/play
|
||||
123
readme.md
123
readme.md
@ -1,39 +1,108 @@
|
||||
# @pushrocks/smartclickhouse
|
||||
# @push.rocks/smartclickhouse
|
||||
an odm for talking to clickhouse
|
||||
|
||||
## Availabililty and Links
|
||||
* [npmjs.org (npm package)](https://www.npmjs.com/package/@pushrocks/smartclickhouse)
|
||||
* [gitlab.com (source)](https://gitlab.com/pushrocks/smartclickhouse)
|
||||
* [github.com (source mirror)](https://github.com/pushrocks/smartclickhouse)
|
||||
* [docs (typedoc)](https://pushrocks.gitlab.io/smartclickhouse/)
|
||||
## Install
|
||||
|
||||
## Status for master
|
||||
To install `@push.rocks/smartclickhouse`, use the following command with npm:
|
||||
|
||||
Status Category | Status Badge
|
||||
-- | --
|
||||
GitLab Pipelines | [](https://lossless.cloud)
|
||||
GitLab Pipline Test Coverage | [](https://lossless.cloud)
|
||||
npm | [](https://lossless.cloud)
|
||||
Snyk | [](https://lossless.cloud)
|
||||
TypeScript Support | [](https://lossless.cloud)
|
||||
node Support | [](https://nodejs.org/dist/latest-v10.x/docs/api/)
|
||||
Code Style | [](https://lossless.cloud)
|
||||
PackagePhobia (total standalone install weight) | [](https://lossless.cloud)
|
||||
PackagePhobia (package size on registry) | [](https://lossless.cloud)
|
||||
BundlePhobia (total size when bundled) | [](https://lossless.cloud)
|
||||
Platform support | [](https://lossless.cloud) [](https://lossless.cloud)
|
||||
```sh
|
||||
npm install @push.rocks/smartclickhouse --save
|
||||
```
|
||||
|
||||
Or with yarn:
|
||||
|
||||
```sh
|
||||
yarn add @push.rocks/smartclickhouse
|
||||
```
|
||||
|
||||
This will add the package to your project's dependencies.
|
||||
|
||||
## Usage
|
||||
|
||||
Use TypeScript for best in class intellisense
|
||||
`@push.rocks/smartclickhouse` is an ORM (Object Relational Mapping) module specifically designed for interacting with ClickHouse databases efficiently and effectively. Leveraging TypeScript, it offers strong typing and intelligent code completion, making database operations more intuitive and less error-prone.
|
||||
|
||||
## Contribution
|
||||
### Setting Up and Starting the Connection
|
||||
|
||||
We are always happy for code contributions. If you are not the code contributing type that is ok. Still, maintaining Open Source repositories takes considerable time and thought. If you like the quality of what we do and our modules are useful to you we would appreciate a little monthly contribution: You can [contribute one time](https://lossless.link/contribute-onetime) or [contribute monthly](https://lossless.link/contribute). :)
|
||||
First, you need to establish a connection with the ClickHouse database. This involves creating an instance of `SmartClickHouseDb` and starting it:
|
||||
|
||||
For further information read the linked docs at the top of this readme.
|
||||
```typescript
|
||||
import { SmartClickHouseDb } from '@push.rocks/smartclickhouse';
|
||||
|
||||
> MIT licensed | **©** [Lossless GmbH](https://lossless.gmbh)
|
||||
| By using this npm module you agree to our [privacy policy](https://lossless.gmbH/privacy)
|
||||
// Create a new instance of SmartClickHouseDb with your ClickHouse database details
|
||||
const dbInstance = new SmartClickHouseDb({
|
||||
url: 'http://localhost:8123', // URL of ClickHouse instance
|
||||
database: 'yourDatabase', // Database name you want to connect
|
||||
username: 'default', // Optional: Username for authentication
|
||||
password: 'password', // Optional: Password for authentication
|
||||
unref: true // Optional: Allows service to exit while awaiting database startup
|
||||
});
|
||||
|
||||
[](https://maintainedby.lossless.com)
|
||||
// Start the instance to establish the connection
|
||||
await dbInstance.start();
|
||||
```
|
||||
|
||||
### Working with Time Data Tables
|
||||
|
||||
`smartclickhouse` allows handling of time-series data through `TimeDataTable`, automating tasks such as table creation and data insertion.
|
||||
|
||||
#### Creating or Accessing a Table
|
||||
|
||||
To create a new time data table or access an existing one:
|
||||
|
||||
```typescript
|
||||
const tableName = 'yourTimeDataTable'; // Name of the table you want to access or create
|
||||
const table = await dbInstance.getTable(tableName);
|
||||
```
|
||||
|
||||
#### Adding Data to the Table
|
||||
|
||||
Once you have the table instance, you can insert data into it:
|
||||
|
||||
```typescript
|
||||
await table.addData({
|
||||
timestamp: Date.now(), // Timestamp in milliseconds
|
||||
message: 'A log message.', // Arbitrary data field
|
||||
temperature: 22.5, // Another example field
|
||||
tags: ['tag1', 'tag2'] // An example array field
|
||||
});
|
||||
```
|
||||
|
||||
`addData` method is designed to be flexible, allowing insertion of various data types and automatically managing table schema adjustments.
|
||||
|
||||
### Advanced Usage and Custom Data Handling
|
||||
|
||||
`smartclickhouse` supports custom data types and complex data structures. For instance, to add support for nested objects or custom data processing before insertion, you might need to extend existing classes or contribute to module development for broader use cases.
|
||||
|
||||
### Complete Feature Set Overview
|
||||
|
||||
While the above examples cover basic usage, `@push.rocks/smartclickhouse` offers a wide range of functionalities, including but not limited to:
|
||||
|
||||
- Robust error handling and reconnection strategies
|
||||
- Efficient bulk data insertion mechanisms
|
||||
- Support for ClickHouse specific features like materialized views and aggregating MergeTree engines
|
||||
- Techniques for managing and querying large time-series datasets
|
||||
|
||||
For a comprehensive guide on leveraging all `@push.rocks/smartclickhouse` features, consult the module documentation and examples. Engage with the community for tips on advanced use cases and optimization strategies for handling time-series data with ClickHouse.
|
||||
|
||||
### Contribution
|
||||
|
||||
Contributions to `@push.rocks/smartclickhouse` are welcome. Whether it's through submitting issues, proposing improvements, or adding to the codebase, your input is valuable. The project is designed to be open and accessible, striving for a high-quality, community-driven development process.
|
||||
|
||||
## License and Legal Information
|
||||
|
||||
This repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository.
|
||||
|
||||
**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.
|
||||
|
||||
### Trademarks
|
||||
|
||||
This project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.
|
||||
|
||||
### Company Information
|
||||
|
||||
Task Venture Capital GmbH
|
||||
Registered at District court Bremen HRB 35230 HB, Germany
|
||||
|
||||
For any legal inquiries or if you require further information, please contact us via email at hello@task.vc.
|
||||
|
||||
By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.
|
||||
|
||||
@ -1,12 +1,14 @@
|
||||
import { expect, expectAsync, tap } from '@pushrocks/tapbundle';
|
||||
import { expect, tap } from '@push.rocks/tapbundle';
|
||||
import * as smartclickhouse from '../ts/index.js';
|
||||
|
||||
let testClickhouseDb: smartclickhouse.SmartClickHouseDb;
|
||||
let table: smartclickhouse.TimeDataTable;
|
||||
|
||||
tap.test('first test', async () => {
|
||||
testClickhouseDb = new smartclickhouse.SmartClickHouseDb({
|
||||
url: 'http://localhost:8123',
|
||||
database: 'test2',
|
||||
unref: true,
|
||||
});
|
||||
});
|
||||
|
||||
@ -15,24 +17,70 @@ tap.test('should start the clickhouse db', async () => {
|
||||
});
|
||||
|
||||
tap.test('should create a timedatatable', async (toolsArg) => {
|
||||
const table = await testClickhouseDb.getTable('analytics');
|
||||
table = await testClickhouseDb.getTable('analytics');
|
||||
let i = 0;
|
||||
while(i < 5000) {
|
||||
while (i < 1000) {
|
||||
await table.addData({
|
||||
timestamp: Date.now(),
|
||||
message: `hello this is a message ${i}`,
|
||||
wow: 'hey',
|
||||
deep: {
|
||||
so: 'hello',
|
||||
myArray: ['array1', 'array2']
|
||||
}
|
||||
myArray: ['array1', 'array2'],
|
||||
},
|
||||
});
|
||||
i++;
|
||||
console.log(`logged ${i} of 5000 lines.`);
|
||||
await toolsArg.delayFor(1);
|
||||
console.log(`logged ${i} of 1000 lines.`);
|
||||
}
|
||||
});
|
||||
|
||||
tap.skip.test('should write something to the clickhouse db', async () => {});
|
||||
tap.test('should retrieve the last 10 entries', async () => {
|
||||
const entries = await table.getLastEntries(10);
|
||||
expect(entries.length).toEqual(10);
|
||||
console.log(entries);
|
||||
});
|
||||
|
||||
tap.start();
|
||||
tap.test('should retrieve entries newer than a specific timestamp', async () => {
|
||||
const timestamp = Date.now() - 60000; // 1 minute ago
|
||||
const entries = await table.getEntriesNewerThan(timestamp);
|
||||
expect(entries.length).toBeGreaterThan(0);
|
||||
console.log(entries);
|
||||
});
|
||||
|
||||
tap.test('should retrieve entries between two timestamps', async () => {
|
||||
const startTimestamp = Date.now() - 120000; // 2 minutes ago
|
||||
const endTimestamp = Date.now() - 60000; // 1 minute ago
|
||||
const entries = await table.getEntriesBetween(startTimestamp, endTimestamp);
|
||||
console.log(entries);
|
||||
});
|
||||
|
||||
tap.test('should delete old entries', async () => {
|
||||
await table.deleteOldEntries(0); // Delete all entries older than now
|
||||
const entries = await table.getLastEntries(10);
|
||||
expect(entries.length).toEqual(0);
|
||||
});
|
||||
|
||||
tap.test('should delete the table', async () => {
|
||||
await table.delete();
|
||||
});
|
||||
|
||||
tap.test('should stream new entries', async (toolsArg) => {
|
||||
const stream = table.streamNewEntries();
|
||||
const subscription = stream.subscribe((entry) => {
|
||||
console.log('New entry:', entry);
|
||||
});
|
||||
|
||||
let i = 0;
|
||||
while (i < 10) {
|
||||
await table.addData({
|
||||
timestamp: Date.now(),
|
||||
message: `streaming message ${i}`,
|
||||
});
|
||||
i++;
|
||||
await toolsArg.delayFor(1000); // Add a delay to simulate real-time data insertion
|
||||
}
|
||||
|
||||
subscription.unsubscribe();
|
||||
});
|
||||
|
||||
export default tap.start();
|
||||
@ -2,7 +2,7 @@
|
||||
* autocreated commitinfo by @pushrocks/commitinfo
|
||||
*/
|
||||
export const commitinfo = {
|
||||
name: '@pushrocks/smartclickhouse',
|
||||
version: '2.0.5',
|
||||
description: 'an odm for talking to clickhouse'
|
||||
name: '@push.rocks/smartclickhouse',
|
||||
version: '2.0.15',
|
||||
description: 'A TypeScript-based ODM (Object-Document Mapper) for ClickHouse databases, with support for creating and managing tables and their data.'
|
||||
}
|
||||
|
||||
3
ts/smartclickhouse.classes.clickhousedb.ts
Normal file
3
ts/smartclickhouse.classes.clickhousedb.ts
Normal file
@ -0,0 +1,3 @@
|
||||
import * as plugins from './smartclickhouse.plugins.js';
|
||||
|
||||
export class ClickhouseDb {}
|
||||
100
ts/smartclickhouse.classes.httpclient.ts
Normal file
100
ts/smartclickhouse.classes.httpclient.ts
Normal file
@ -0,0 +1,100 @@
|
||||
import * as plugins from './smartclickhouse.plugins.js';
|
||||
|
||||
export interface IClickhouseHttpClientOptions {
|
||||
username?: string;
|
||||
password?: string;
|
||||
url: string;
|
||||
}
|
||||
|
||||
export class ClickhouseHttpClient {
|
||||
// STATIC
|
||||
public static async createAndStart(optionsArg: IClickhouseHttpClientOptions) {
|
||||
const clickhouseHttpInstance = new ClickhouseHttpClient(optionsArg);
|
||||
await clickhouseHttpInstance.start();
|
||||
return clickhouseHttpInstance;
|
||||
}
|
||||
|
||||
// INSTANCE
|
||||
public options: IClickhouseHttpClientOptions;
|
||||
public webrequestInstance = new plugins.webrequest.WebRequest({
|
||||
logging: false,
|
||||
});
|
||||
public computedProperties: {
|
||||
connectionUrl: string;
|
||||
parsedUrl: plugins.smarturl.Smarturl;
|
||||
} = {
|
||||
connectionUrl: null,
|
||||
parsedUrl: null,
|
||||
};
|
||||
constructor(optionsArg: IClickhouseHttpClientOptions) {
|
||||
this.options = optionsArg;
|
||||
}
|
||||
|
||||
public async start() {
|
||||
this.computedProperties.parsedUrl = plugins.smarturl.Smarturl.createFromUrl(this.options.url);
|
||||
console.log(this.computedProperties.parsedUrl);
|
||||
this.computedProperties.connectionUrl = this.computedProperties.parsedUrl.toString();
|
||||
}
|
||||
|
||||
public async ping() {
|
||||
const ping = await this.webrequestInstance.request(
|
||||
this.computedProperties.connectionUrl.toString(),
|
||||
{
|
||||
method: 'GET',
|
||||
timeoutMs: 1000,
|
||||
}
|
||||
);
|
||||
return ping.status === 200 ? true : false;
|
||||
}
|
||||
|
||||
public async queryPromise(queryArg: string) {
|
||||
const returnArray = [];
|
||||
const response = await this.webrequestInstance.request(
|
||||
`${this.computedProperties.connectionUrl}?query=${encodeURIComponent(queryArg)}`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: this.getHeaders(),
|
||||
}
|
||||
);
|
||||
// console.log('===================');
|
||||
// console.log(this.computedProperties.connectionUrl);
|
||||
// console.log(queryArg);
|
||||
// console.log((await response.clone().text()).split(/\r?\n/))
|
||||
if (response.headers.get('X-ClickHouse-Format') === 'JSONEachRow') {
|
||||
const jsonList = await response.text();
|
||||
const jsonArray = jsonList.split('\n');
|
||||
for (const jsonArg of jsonArray) {
|
||||
if (!jsonArg) {
|
||||
continue;
|
||||
}
|
||||
returnArray.push(JSON.parse(jsonArg));
|
||||
}
|
||||
} else {
|
||||
}
|
||||
return returnArray;
|
||||
}
|
||||
|
||||
public async insertPromise(databaseArg: string, tableArg: string, documents: any[]) {
|
||||
const queryArg = `INSERT INTO ${databaseArg}.${tableArg} FORMAT JSONEachRow`;
|
||||
const response = await this.webrequestInstance.request(
|
||||
`${this.computedProperties.connectionUrl}?query=${encodeURIComponent(queryArg)}`,
|
||||
{
|
||||
method: 'POST',
|
||||
body: documents.map((docArg) => JSON.stringify(docArg)).join('\n'),
|
||||
headers: this.getHeaders(),
|
||||
}
|
||||
);
|
||||
return response;
|
||||
}
|
||||
|
||||
private getHeaders() {
|
||||
const headers: { [key: string]: string } = {};
|
||||
if (this.options.username) {
|
||||
headers['X-ClickHouse-User'] = this.options.username;
|
||||
}
|
||||
if (this.options.password) {
|
||||
headers['X-ClickHouse-Key'] = this.options.password;
|
||||
}
|
||||
return headers;
|
||||
}
|
||||
}
|
||||
@ -1,16 +1,23 @@
|
||||
import * as plugins from './smartclickhouse.plugins.js';
|
||||
import { TimeDataTable } from './smartclickhouse.classes.timedatatable.js';
|
||||
import { ClickhouseHttpClient } from './smartclickhouse.classes.httpclient.js';
|
||||
|
||||
export interface IClickhouseConstructorOptions {
|
||||
url: string;
|
||||
database: string;
|
||||
username?: string;
|
||||
password?: string;
|
||||
/**
|
||||
* allow services to exit when waiting for clickhouse startup
|
||||
* this allows to leave the lifecycle flow to other processes
|
||||
* like a listening server.
|
||||
*/
|
||||
unref?: boolean;
|
||||
}
|
||||
|
||||
export class SmartClickHouseDb {
|
||||
public options: IClickhouseConstructorOptions;
|
||||
public defaultClient: plugins.clickhouse.ClickHouseClient;
|
||||
public clickhouseClient: plugins.clickhouse.ClickHouseClient;
|
||||
public clickhouseHttpClient: ClickhouseHttpClient;
|
||||
|
||||
constructor(optionsArg: IClickhouseConstructorOptions) {
|
||||
this.options = optionsArg;
|
||||
@ -21,48 +28,31 @@ export class SmartClickHouseDb {
|
||||
*/
|
||||
public async start(dropOld = false) {
|
||||
console.log(`Connecting to default database first.`);
|
||||
const defaultOptions: {[keyArg: string]: string} = {};
|
||||
// the protocol, url and host
|
||||
const parsedUrl = plugins.smarturl.Smarturl.createFromUrl(this.options.url);
|
||||
parsedUrl.protocol === 'https' ? defaultOptions.protocol = plugins.clickhouse.ClickHouseConnectionProtocol.HTTPS : null;
|
||||
parsedUrl.protocol === 'http' ? defaultOptions.protocol = plugins.clickhouse.ClickHouseConnectionProtocol.HTTP : null;
|
||||
defaultOptions.host = parsedUrl.hostname;
|
||||
defaultOptions.port = parsedUrl.port;
|
||||
// the database
|
||||
defaultOptions.database = this.options.database;
|
||||
// the password
|
||||
this.options.password ? defaultOptions.password = this.options.password : null;
|
||||
// lets connect
|
||||
this.defaultClient = new plugins.clickhouse.ClickHouseClient({
|
||||
...defaultOptions,
|
||||
database: 'default',
|
||||
});
|
||||
this.clickhouseHttpClient = await ClickhouseHttpClient.createAndStart(this.options);
|
||||
await this.pingDatabaseUntilAvailable();
|
||||
console.log(`Create database ${this.options.database}, if it does not exist...`);
|
||||
await this.createDatabase(dropOld);
|
||||
|
||||
console.log(`Ensured database. Now connecting to wanted database: ${this.options.database}`);
|
||||
this.clickhouseClient = new plugins.clickhouse.ClickHouseClient({
|
||||
...defaultOptions
|
||||
});
|
||||
}
|
||||
|
||||
public async createDatabase(dropOld: boolean = false) {
|
||||
if (dropOld) {
|
||||
await this.defaultClient.queryPromise(`DROP DATABASE IF EXISTS ${this.options.database}`);
|
||||
await this.clickhouseHttpClient.queryPromise(`DROP DATABASE IF EXISTS ${this.options.database}`);
|
||||
}
|
||||
await this.defaultClient.queryPromise(`CREATE DATABASE IF NOT EXISTS ${this.options.database}`);
|
||||
await this.clickhouseHttpClient.queryPromise(
|
||||
`CREATE DATABASE IF NOT EXISTS ${this.options.database}`
|
||||
);
|
||||
}
|
||||
|
||||
public async pingDatabaseUntilAvailable() {
|
||||
let available = false;
|
||||
while(!available) {
|
||||
available = await this.defaultClient.ping().catch(err => {
|
||||
while (!available) {
|
||||
available = await this.clickhouseHttpClient.ping().catch((err) => {
|
||||
return false;
|
||||
});
|
||||
if (!available) {
|
||||
console.log(`NOT OK: tried pinging ${this.options.url}... Trying again in 5 seconds.`);
|
||||
await plugins.smartdelay.delayFor(5000);
|
||||
await plugins.smartdelay.delayFor(5000, null, this.options.unref);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,7 +1,12 @@
|
||||
import * as plugins from './smartclickhouse.plugins.js';
|
||||
import { SmartClickHouseDb } from './smartclickhouse.classes.smartclickhouse.js';
|
||||
|
||||
export type TClickhouseColumnDataType = 'String' | "DateTime64(3, 'Europe/Berlin')" | 'Float64' | 'Array(String)' | 'Array(Float64)';
|
||||
export type TClickhouseColumnDataType =
|
||||
| 'String'
|
||||
| "DateTime64(3, 'Europe/Berlin')"
|
||||
| 'Float64'
|
||||
| 'Array(String)'
|
||||
| 'Array(Float64)';
|
||||
export interface IColumnInfo {
|
||||
database: string;
|
||||
table: string;
|
||||
@ -35,32 +40,10 @@ export class TimeDataTable {
|
||||
public static async getTable(smartClickHouseDbRefArg: SmartClickHouseDb, tableNameArg: string) {
|
||||
const newTable = new TimeDataTable(smartClickHouseDbRefArg, {
|
||||
tableName: tableNameArg,
|
||||
retainDataForDays: 30
|
||||
retainDataForDays: 30,
|
||||
});
|
||||
|
||||
// create table in clickhouse
|
||||
await smartClickHouseDbRefArg.clickhouseClient
|
||||
.queryPromise(`
|
||||
CREATE TABLE IF NOT EXISTS ${newTable.options.tableName} (
|
||||
timestamp DateTime64(3, 'Europe/Berlin'),
|
||||
message String
|
||||
) ENGINE=MergeTree() ORDER BY timestamp`);
|
||||
|
||||
// lets adjust the TTL
|
||||
await smartClickHouseDbRefArg.clickhouseClient
|
||||
.queryPromise(`
|
||||
ALTER TABLE ${newTable.options.tableName} MODIFY TTL toDateTime(timestamp) + INTERVAL ${newTable.options.retainDataForDays} DAY
|
||||
`);
|
||||
|
||||
await newTable.updateColumns();
|
||||
console.log(`=======================`)
|
||||
console.log(
|
||||
`table with name "${newTable.options.tableName}" in databse ${newTable.smartClickHouseDbRef.options.database} has the following columns:`
|
||||
);
|
||||
for (const column of newTable.columns) {
|
||||
console.log(`>> ${column.name}: ${column.type}`);
|
||||
}
|
||||
console.log('^^^^^^^^^^^^^^\n');
|
||||
await newTable.setup();
|
||||
|
||||
return newTable;
|
||||
}
|
||||
@ -75,16 +58,40 @@ export class TimeDataTable {
|
||||
this.options = optionsArg;
|
||||
}
|
||||
|
||||
public async setup() {
|
||||
// create table in clickhouse
|
||||
await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
|
||||
CREATE TABLE IF NOT EXISTS ${this.smartClickHouseDbRef.options.database}.${this.options.tableName} (
|
||||
timestamp DateTime64(3, 'Europe/Berlin'),
|
||||
message String
|
||||
) ENGINE=MergeTree() ORDER BY timestamp`);
|
||||
|
||||
// lets adjust the TTL
|
||||
await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
|
||||
ALTER TABLE ${this.smartClickHouseDbRef.options.database}.${this.options.tableName} MODIFY TTL toDateTime(timestamp) + INTERVAL ${this.options.retainDataForDays} DAY
|
||||
`);
|
||||
|
||||
await this.updateColumns();
|
||||
console.log(`=======================`);
|
||||
console.log(
|
||||
`table with name "${this.options.tableName}" in database ${this.smartClickHouseDbRef.options.database} has the following columns:`
|
||||
);
|
||||
for (const column of this.columns) {
|
||||
console.log(`>> ${column.name}: ${column.type}`);
|
||||
}
|
||||
console.log('^^^^^^^^^^^^^^\n');
|
||||
}
|
||||
|
||||
public columns: IColumnInfo[] = [];
|
||||
|
||||
/**
|
||||
* updates the columns
|
||||
*/
|
||||
public async updateColumns() {
|
||||
this.columns = await this.smartClickHouseDbRef.clickhouseClient.queryPromise(`
|
||||
this.columns = await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
|
||||
SELECT * FROM system.columns
|
||||
WHERE database LIKE '${this.smartClickHouseDbRef.options.database}'
|
||||
AND table LIKE '${this.options.tableName}'
|
||||
AND table LIKE '${this.options.tableName}' FORMAT JSONEachRow
|
||||
`);
|
||||
return this.columns;
|
||||
}
|
||||
@ -93,6 +100,9 @@ export class TimeDataTable {
|
||||
* stores a json and tries to map it to the nested syntax
|
||||
*/
|
||||
public async addData(dataArg: any) {
|
||||
if (this.healingDeferred) {
|
||||
return;
|
||||
}
|
||||
|
||||
// the storageJson
|
||||
let storageJson: { [key: string]: any } = {};
|
||||
@ -100,23 +110,27 @@ export class TimeDataTable {
|
||||
// helper stuff
|
||||
|
||||
const getClickhouseTypeForValue = (valueArg: any): TClickhouseColumnDataType => {
|
||||
const typeConversion: {[key: string]: TClickhouseColumnDataType} = {
|
||||
const typeConversion: { [key: string]: TClickhouseColumnDataType } = {
|
||||
string: 'String',
|
||||
number: 'Float64',
|
||||
undefined: null,
|
||||
null: null
|
||||
null: null,
|
||||
};
|
||||
if (valueArg instanceof Array) {
|
||||
const arrayType = typeConversion[(typeof valueArg[0]) as string];
|
||||
const arrayType = typeConversion[typeof valueArg[0] as string];
|
||||
if (!arrayType) {
|
||||
return null;
|
||||
} else {
|
||||
return `Array(${arrayType})` as TClickhouseColumnDataType;
|
||||
}
|
||||
}
|
||||
return typeConversion[(typeof valueArg) as string];
|
||||
}
|
||||
const checkPath = async (pathArg: string, typeArg: TClickhouseColumnDataType, prechecked = false) => {
|
||||
return typeConversion[typeof valueArg as string];
|
||||
};
|
||||
const checkPath = async (
|
||||
pathArg: string,
|
||||
typeArg: TClickhouseColumnDataType,
|
||||
prechecked = false
|
||||
) => {
|
||||
let columnFound = false;
|
||||
for (const column of this.columns) {
|
||||
if (pathArg === column.name) {
|
||||
@ -130,12 +144,12 @@ export class TimeDataTable {
|
||||
await checkPath(pathArg, typeArg, true);
|
||||
return;
|
||||
}
|
||||
const alterString = `ALTER TABLE ${this.options.tableName} ADD COLUMN ${pathArg} ${typeArg} FIRST`
|
||||
const alterString = `ALTER TABLE ${this.smartClickHouseDbRef.options.database}.${this.options.tableName} ADD COLUMN ${pathArg} ${typeArg} FIRST`;
|
||||
try {
|
||||
await this.smartClickHouseDbRef.clickhouseClient.queryPromise(`
|
||||
await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
|
||||
${alterString}
|
||||
`);
|
||||
} catch(err) {
|
||||
} catch (err) {
|
||||
console.log(alterString);
|
||||
for (const column of this.columns) {
|
||||
console.log(column.name);
|
||||
@ -164,9 +178,11 @@ export class TimeDataTable {
|
||||
storageJson[key] = value;
|
||||
}
|
||||
|
||||
const result = await this.smartClickHouseDbRef.clickhouseClient.insertPromise(this.options.tableName, [
|
||||
const result = await this.smartClickHouseDbRef.clickhouseHttpClient
|
||||
.insertPromise(this.smartClickHouseDbRef.options.database, this.options.tableName, [
|
||||
storageJson,
|
||||
]).catch(async () => {
|
||||
])
|
||||
.catch(async () => {
|
||||
if (this.healingDeferred) {
|
||||
return;
|
||||
}
|
||||
@ -174,10 +190,97 @@ export class TimeDataTable {
|
||||
console.log(`Ran into an error. Trying to set up things properly again.`);
|
||||
await this.smartClickHouseDbRef.pingDatabaseUntilAvailable();
|
||||
await this.smartClickHouseDbRef.createDatabase();
|
||||
await this.setup();
|
||||
this.columns = [];
|
||||
this.healingDeferred.resolve();
|
||||
this.healingDeferred = null;
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* deletes the entire table
|
||||
*/
|
||||
public async delete() {
|
||||
await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
|
||||
DROP TABLE IF EXISTS ${this.smartClickHouseDbRef.options.database}.${this.options.tableName}
|
||||
`);
|
||||
this.columns = [];
|
||||
}
|
||||
|
||||
/**
|
||||
* deletes entries older than a specified number of days
|
||||
* @param days number of days
|
||||
*/
|
||||
public async deleteOldEntries(days: number) {
|
||||
await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
|
||||
ALTER TABLE ${this.smartClickHouseDbRef.options.database}.${this.options.tableName}
|
||||
DELETE WHERE timestamp < now() - INTERVAL ${days} DAY
|
||||
`);
|
||||
}
|
||||
|
||||
public async getLastEntries(count: number) {
|
||||
const result = await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
|
||||
SELECT * FROM ${this.smartClickHouseDbRef.options.database}.${this.options.tableName}
|
||||
ORDER BY timestamp DESC
|
||||
LIMIT ${count} FORMAT JSONEachRow
|
||||
`);
|
||||
return result;
|
||||
}
|
||||
|
||||
public async getEntriesNewerThan(unixTimestamp: number) {
|
||||
const result = await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
|
||||
SELECT * FROM ${this.smartClickHouseDbRef.options.database}.${this.options.tableName}
|
||||
WHERE timestamp > toDateTime(${unixTimestamp / 1000}) FORMAT JSONEachRow
|
||||
`);
|
||||
return result;
|
||||
}
|
||||
|
||||
public async getEntriesBetween(unixTimestampStart: number, unixTimestampEnd: number) {
|
||||
const result = await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
|
||||
SELECT * FROM ${this.smartClickHouseDbRef.options.database}.${this.options.tableName}
|
||||
WHERE timestamp > toDateTime(${unixTimestampStart / 1000})
|
||||
AND timestamp < toDateTime(${unixTimestampEnd / 1000}) FORMAT JSONEachRow
|
||||
`);
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* streams all new entries using an observable
|
||||
*/
|
||||
public streamNewEntries(): plugins.smartrx.rxjs.Observable<any> {
|
||||
return new plugins.smartrx.rxjs.Observable((observer) => {
|
||||
const pollInterval = 1000; // Poll every 1 second
|
||||
let lastTimestamp: number;
|
||||
|
||||
const fetchLastEntryTimestamp = async () => {
|
||||
const lastEntry = await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
|
||||
SELECT max(timestamp) as lastTimestamp FROM ${this.smartClickHouseDbRef.options.database}.${this.options.tableName} FORMAT JSONEachRow
|
||||
`);
|
||||
lastTimestamp = lastEntry.length
|
||||
? new Date(lastEntry[0].lastTimestamp).getTime()
|
||||
: Date.now();
|
||||
};
|
||||
|
||||
const fetchNewEntries = async () => {
|
||||
const newEntries = await this.getEntriesNewerThan(lastTimestamp);
|
||||
if (newEntries.length > 0) {
|
||||
for (const entry of newEntries) {
|
||||
observer.next(entry);
|
||||
}
|
||||
lastTimestamp = new Date(newEntries[newEntries.length - 1].timestamp).getTime();
|
||||
}
|
||||
};
|
||||
|
||||
const startPolling = async () => {
|
||||
await fetchLastEntryTimestamp();
|
||||
const intervalId = setInterval(fetchNewEntries, pollInterval);
|
||||
|
||||
// Cleanup on unsubscribe
|
||||
return () => clearInterval(intervalId);
|
||||
};
|
||||
|
||||
startPolling().catch((err) => observer.error(err));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,17 +1,9 @@
|
||||
// @pushrocks scope
|
||||
import * as smartdelay from '@pushrocks/smartdelay';
|
||||
import * as smartobject from '@pushrocks/smartobject';
|
||||
import * as smartpromise from '@pushrocks/smartpromise';
|
||||
import * as smarturl from '@pushrocks/smarturl';
|
||||
import * as smartdelay from '@push.rocks/smartdelay';
|
||||
import * as smartobject from '@push.rocks/smartobject';
|
||||
import * as smartpromise from '@push.rocks/smartpromise';
|
||||
import * as smartrx from '@push.rocks/smartrx';
|
||||
import * as smarturl from '@push.rocks/smarturl';
|
||||
import * as webrequest from '@push.rocks/webrequest';
|
||||
|
||||
export {
|
||||
smartdelay,
|
||||
smartobject,
|
||||
smartpromise,
|
||||
smarturl
|
||||
}
|
||||
|
||||
// thirdparty
|
||||
import * as clickhouse from '@depyronick/clickhouse-client';
|
||||
|
||||
export { clickhouse };
|
||||
export { smartdelay, smartobject, smartpromise, smartrx, smarturl, webrequest };
|
||||
|
||||
@ -3,8 +3,12 @@
|
||||
"experimentalDecorators": true,
|
||||
"useDefineForClassFields": false,
|
||||
"target": "ES2022",
|
||||
"module": "ES2022",
|
||||
"moduleResolution": "nodenext",
|
||||
"esModuleInterop": true
|
||||
}
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"esModuleInterop": true,
|
||||
"verbatimModuleSyntax": true
|
||||
},
|
||||
"exclude": [
|
||||
"dist_*/**/*.d.ts"
|
||||
]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user