From 09e597ab87ddd510b6ac1173d3e1374e75c74462 Mon Sep 17 00:00:00 2001 From: Philipp Kunz Date: Mon, 7 Mar 2022 15:49:47 +0100 Subject: [PATCH] fix(core): update --- package-lock.json | 65 ++++++++++- package.json | 3 +- test/test.nonci.ts | 35 ++++++ test/test.ts | 40 ------- ts/smartclickhouse.classes.smartclickhouse.ts | 18 +-- ts/smartclickhouse.classes.timedatatable.ts | 108 +++++++++++++++++- ts/smartclickhouse.plugins.ts | 10 +- 7 files changed, 219 insertions(+), 60 deletions(-) create mode 100644 test/test.nonci.ts delete mode 100644 test/test.ts diff --git a/package-lock.json b/package-lock.json index ed5c892..0994c15 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,8 @@ "version": "1.0.5", "license": "MIT", "dependencies": { - "@depyronick/clickhouse-client": "^1.0.12" + "@depyronick/clickhouse-client": "^1.0.12", + "@pushrocks/smartobject": "^1.0.8" }, "devDependencies": { "@gitzone/tsbuild": "^2.1.25", @@ -2633,6 +2634,37 @@ "systeminformation": "^5.11.3" } }, + "node_modules/@pushrocks/smartobject": { + "version": "1.0.8", + "resolved": "https://verdaccio.lossless.one/@pushrocks%2fsmartobject/-/smartobject-1.0.8.tgz", + "integrity": "sha512-sqrmbPkCZe4+OTqFGYEUBgwG9J/gMVdjz6Tuy8B7v5RaRvUobcjBk7FBa0ahMMPmuKEbddC5xa8+Jqtjb4iJAA==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "minimatch": "^5.0.1" + } + }, + "node_modules/@pushrocks/smartobject/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://verdaccio.lossless.one/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/@pushrocks/smartobject/node_modules/minimatch": { + "version": "5.0.1", + "resolved": "https://verdaccio.lossless.one/minimatch/-/minimatch-5.0.1.tgz", + "integrity": "sha512-nLDxIFRyhDblz3qMuq+SoRZED4+miJ/G+tdDrjkkkRnjAsBexeGpgjLEQ0blJy7rHhR2b93rhQY4SvyWu9v03g==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/@pushrocks/smartparam": { "version": "1.1.6", "resolved": "https://verdaccio.lossless.one/@pushrocks%2fsmartparam/-/smartparam-1.1.6.tgz", @@ -7139,7 +7171,6 @@ "version": "3.1.3", "resolved": "https://verdaccio.lossless.one/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "dev": true, "license": "MIT" }, "node_modules/fast-glob": { @@ -17691,6 +17722,33 @@ "systeminformation": "^5.11.3" } }, + "@pushrocks/smartobject": { + "version": "1.0.8", + "resolved": "https://verdaccio.lossless.one/@pushrocks%2fsmartobject/-/smartobject-1.0.8.tgz", + "integrity": "sha512-sqrmbPkCZe4+OTqFGYEUBgwG9J/gMVdjz6Tuy8B7v5RaRvUobcjBk7FBa0ahMMPmuKEbddC5xa8+Jqtjb4iJAA==", + "requires": { + "fast-deep-equal": "^3.1.3", + "minimatch": "^5.0.1" + }, + "dependencies": { + "brace-expansion": { + "version": "2.0.1", + "resolved": "https://verdaccio.lossless.one/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "requires": { + "balanced-match": "^1.0.0" + } + }, + "minimatch": { + "version": "5.0.1", + "resolved": "https://verdaccio.lossless.one/minimatch/-/minimatch-5.0.1.tgz", + "integrity": "sha512-nLDxIFRyhDblz3qMuq+SoRZED4+miJ/G+tdDrjkkkRnjAsBexeGpgjLEQ0blJy7rHhR2b93rhQY4SvyWu9v03g==", + "requires": { + "brace-expansion": "^2.0.1" + } + } + } + }, "@pushrocks/smartparam": { "version": "1.1.6", "resolved": "https://verdaccio.lossless.one/@pushrocks%2fsmartparam/-/smartparam-1.1.6.tgz", @@ -21186,8 +21244,7 @@ "fast-deep-equal": { "version": "3.1.3", "resolved": "https://verdaccio.lossless.one/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", - "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "dev": true + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" }, "fast-glob": { "version": "2.2.7", diff --git a/package.json b/package.json index 6b21f1b..d6bf821 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,8 @@ "tslint-config-prettier": "^1.15.0" }, "dependencies": { - "@depyronick/clickhouse-client": "^1.0.12" + "@depyronick/clickhouse-client": "^1.0.12", + "@pushrocks/smartobject": "^1.0.8" }, "browserslist": [ "last 1 chrome versions" diff --git a/test/test.nonci.ts b/test/test.nonci.ts new file mode 100644 index 0000000..855d3ec --- /dev/null +++ b/test/test.nonci.ts @@ -0,0 +1,35 @@ +import { expect, expectAsync, tap } from '@pushrocks/tapbundle'; +import * as smartclickhouse from '../ts/index'; + +let testClickhouseDb: smartclickhouse.SmartClickHouseDb; + +tap.test('first test', async () => { + testClickhouseDb = new smartclickhouse.SmartClickHouseDb({ + host: 'localhost', + database: 'test2', + }); +}); + +tap.test('should start the clickhouse db', async () => { + await testClickhouseDb.start(true); +}); + +tap.test('should create a timedatatable', async () => { + const table = await testClickhouseDb.getTable('analytics'); + let i = 0; + while(i < 10) { + await table.addData({ + timestamp: Date.now(), + message: `hello this is a message ${i}`, + wow: 'hey', + deep: { + so: 'hello' + } + }); + i++; + } +}); + +tap.skip.test('should write something to the clickhouse db', async () => {}); + +tap.start(); diff --git a/test/test.ts b/test/test.ts deleted file mode 100644 index 8525d2a..0000000 --- a/test/test.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { expect, expectAsync, tap } from '@pushrocks/tapbundle'; -import * as smartclickhouse from '../ts/index'; - -let testClickhouseDb: smartclickhouse.SmartClickHouseDb; - -tap.test('first test', async () => { - testClickhouseDb = new smartclickhouse.SmartClickHouseDb({ - host: 'localhost', - database: 'test2' - }); -}); - -tap.test('should start the clickhouse db', async () => { - await testClickhouseDb.start(); -}) - -tap.test('should create a timedatatable', async () => { - testClickhouseDb.getTable('analytics') -}) - -tap.skip.test('should write something to the clickhouse db', async () => { - const result2 = await testClickhouseDb.clickhouseClient.queryPromise(`CREATE TABLE IF NOT EXISTS visits2 ( - timestamp UInt64, - ip String, - os String, - userAgent String, - version String - ) ENGINE=MergeTree() ORDER BY timestamp`); - const result3 = await testClickhouseDb.clickhouseClient.insertPromise('visits2', [ - { - timestamp: Date.now(), - ip: '127.0.01', - os: 'Mac OS X', - userAgent: 'some', - version: 'someversion' - } - ]); -}) - -tap.start(); diff --git a/ts/smartclickhouse.classes.smartclickhouse.ts b/ts/smartclickhouse.classes.smartclickhouse.ts index 8fba63a..13d122b 100644 --- a/ts/smartclickhouse.classes.smartclickhouse.ts +++ b/ts/smartclickhouse.classes.smartclickhouse.ts @@ -7,8 +7,6 @@ export interface IClickhouseConstructorOptions { password?: string; } - - export class SmartClickHouseDb { public options: IClickhouseConstructorOptions; public clickhouseClient: plugins.clickhouse.ClickHouseClient; @@ -20,24 +18,25 @@ export class SmartClickHouseDb { /** * starts the connection to the Clickhouse db */ - public async start() { - console.log(`Connecting to default database first.`) + public async start(dropOld = false) { + console.log(`Connecting to default database first.`); const defaultClient = new plugins.clickhouse.ClickHouseClient({ ...this.options, - database: 'default' + database: 'default', }); console.log(`Create database ${this.options.database}, if it does not exist...`); + if (dropOld) { + await defaultClient.queryPromise(`DROP DATABASE IF EXISTS ${this.options.database}`); + } await defaultClient.queryPromise(`CREATE DATABASE IF NOT EXISTS ${this.options.database}`); - console.log(`Ensured database. Now connecting to wanted database: ${this.options.database}`) + console.log(`Ensured database. Now connecting to wanted database: ${this.options.database}`); this.clickhouseClient = new plugins.clickhouse.ClickHouseClient({ ...this.options, - }); console.log(`trying to ping database...`); const result = await this.clickhouseClient.ping(); console.log(`Ping successfull?: ${result}`); - } /** @@ -45,5 +44,6 @@ export class SmartClickHouseDb { */ public async getTable(tableName: string) { const newTable = TimeDataTable.getTable(this, tableName); + return newTable; } -} \ No newline at end of file +} diff --git a/ts/smartclickhouse.classes.timedatatable.ts b/ts/smartclickhouse.classes.timedatatable.ts index bbe9a83..caa1e40 100644 --- a/ts/smartclickhouse.classes.timedatatable.ts +++ b/ts/smartclickhouse.classes.timedatatable.ts @@ -1,15 +1,52 @@ import * as plugins from './smartclickhouse.plugins'; import { SmartClickHouseDb } from './smartclickhouse.classes.smartclickhouse'; +export type TClickhouseColumnDataType = 'String' | "DateTime64(3, 'Europe/Berlin')" | 'Float64'; +export interface IColumnInfo { + database: string; + table: string; + name: string; + type: TClickhouseColumnDataType; + position: string; + default_kind: string; + default_expression: string; + data_compressed_bytes: string; + data_uncompressed_bytes: string; + marks_bytes: string; + comment: string; + is_in_partition_key: 0 | 1; + is_in_sorting_key: 0 | 1; + is_in_primary_key: 0 | 1; + is_in_sampling_key: 0 | 1; + compression_codec: string; + character_octet_length: null; + numeric_precision: null; + numeric_precision_radix: null; + numeric_scale: null; + datetime_precision: '3'; +} + export class TimeDataTable { - public static async getTable (smartClickHouseDbRefArg: SmartClickHouseDb, tableNameArg: string) { + public static async getTable(smartClickHouseDbRefArg: SmartClickHouseDb, tableNameArg: string) { const newTable = new TimeDataTable(smartClickHouseDbRefArg, tableNameArg); // create table in clickhouse - await smartClickHouseDbRefArg.clickhouseClient.queryPromise(`CREATE TABLE IF NOT EXISTS ${newTable.tableName} ( + await smartClickHouseDbRefArg.clickhouseClient + .queryPromise(`CREATE TABLE IF NOT EXISTS ${newTable.tableName} ( timestamp DateTime64(3, 'Europe/Berlin'), message String ) ENGINE=MergeTree() ORDER BY timestamp`); + + await newTable.updateColumns(); + console.log(`=======================`) + console.log( + `table with name "${newTable.tableName}" in databse ${newTable.smartClickHouseDbRef.options.database} has the following columns:` + ); + for (const column of newTable.columns) { + console.log(`>> ${column.name}: ${column.type}`); + } + console.log('^^^^^^^^^^^^^^\n'); + return newTable; } @@ -22,8 +59,71 @@ export class TimeDataTable { this.tableName = tableNameArg; } + public columns: IColumnInfo[] = []; + public seenPaths: { pathName: string; type: TClickhouseColumnDataType }[] = []; + + /** + * updates the columns + */ + public async updateColumns() { + this.columns = await this.smartClickHouseDbRef.clickhouseClient.queryPromise(` + SELECT * FROM system.columns + WHERE database LIKE '${this.smartClickHouseDbRef.options.database}' + AND table LIKE '${this.tableName}' + `); + return this.columns; + } + /** * stores a json and tries to map it to the nested syntax */ - public async storeJson () {} -} \ No newline at end of file + public async addData(dataArg: any) { + // the storageJson + let storageJson: { [key: string]: any } = {}; + + // helper stuff + const typeConversion: {[key: string]: TClickhouseColumnDataType} = { + string: 'String', + number: 'Float64', + }; + const getClickhouseTypeForValue = (valueArg: any) => { + return typeConversion[(typeof valueArg) as string]; + } + const checkPath = async (pathArg: string, typeArg: TClickhouseColumnDataType) => { + let columnFound = false; + for (const column of this.columns) { + if (pathArg === column.name) { + columnFound = true; + break; + } + } + if (!columnFound) { + await this.smartClickHouseDbRef.clickhouseClient.queryPromise(` + ALTER TABLE ${this.tableName} ADD COLUMN ${pathArg} ${typeArg} FIRST + `); + await this.updateColumns(); + } + }; + + // key checking + const flatDataArg = plugins.smartobject.toFlatObject(dataArg); + for (const key of Object.keys(flatDataArg)) { + const value = flatDataArg[key]; + if (key === 'timestamp' && typeof value !== 'number') { + throw new Error('timestamp must be of type number'); + } else if (key === 'timestamp') { + storageJson.timestamp = flatDataArg[key]; + continue; + } + // lets deal with the rest + const clickhouseType = getClickhouseTypeForValue(value); + await checkPath(key, clickhouseType); + storageJson[key] = value; + } + + const result = await this.smartClickHouseDbRef.clickhouseClient.insertPromise(this.tableName, [ + storageJson, + ]); + return result; + } +} diff --git a/ts/smartclickhouse.plugins.ts b/ts/smartclickhouse.plugins.ts index 944ab71..e320aff 100644 --- a/ts/smartclickhouse.plugins.ts +++ b/ts/smartclickhouse.plugins.ts @@ -1,5 +1,11 @@ -import * as clickhouse from '@depyronick/clickhouse-client'; +// @pushrocks scope +import * as smartobject from '@pushrocks/smartobject'; export { - clickhouse + smartobject } + +// thirdparty +import * as clickhouse from '@depyronick/clickhouse-client'; + +export { clickhouse };