push.rocks/smartclickhouse
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(core): update

Browse Source
This commit is contained in:
Philipp Kunz 2022-03-07 15:49:47 +01:00
parent d3ef78af11
commit 09e597ab87
7 changed files with 219 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
package-lock.json generated
View File

@ -9,7 +9,8 @@
"version": "1.0.5",
"license": "MIT",
"dependencies": {
"@depyronick/clickhouse-client": "^1.0.12"
"@depyronick/clickhouse-client": "^1.0.12",
"@pushrocks/smartobject": "^1.0.8"
},
"devDependencies": {
"@gitzone/tsbuild": "^2.1.25",
@ -2633,6 +2634,37 @@
"systeminformation": "^5.11.3"
}
},
"node_modules/@pushrocks/smartobject": {
"version": "1.0.8",
"resolved": "https://verdaccio.lossless.one/@pushrocks%2fsmartobject/-/smartobject-1.0.8.tgz",
"integrity": "sha512-sqrmbPkCZe4+OTqFGYEUBgwG9J/gMVdjz6Tuy8B7v5RaRvUobcjBk7FBa0ahMMPmuKEbddC5xa8+Jqtjb4iJAA==",
"license": "MIT",
"dependencies": {
"fast-deep-equal": "^3.1.3",
"minimatch": "^5.0.1"
}
},
"node_modules/@pushrocks/smartobject/node_modules/brace-expansion": {
"version": "2.0.1",
"resolved": "https://verdaccio.lossless.one/brace-expansion/-/brace-expansion-2.0.1.tgz",
"integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
"license": "MIT",
"dependencies": {
"balanced-match": "^1.0.0"
}
},
"node_modules/@pushrocks/smartobject/node_modules/minimatch": {
"version": "5.0.1",
"resolved": "https://verdaccio.lossless.one/minimatch/-/minimatch-5.0.1.tgz",
"integrity": "sha512-nLDxIFRyhDblz3qMuq+SoRZED4+miJ/G+tdDrjkkkRnjAsBexeGpgjLEQ0blJy7rHhR2b93rhQY4SvyWu9v03g==",
"license": "ISC",
"dependencies": {
"brace-expansion": "^2.0.1"
},
"engines": {
"node": ">=10"
}
},
"node_modules/@pushrocks/smartparam": {
"version": "1.1.6",
"resolved": "https://verdaccio.lossless.one/@pushrocks%2fsmartparam/-/smartparam-1.1.6.tgz",
@ -7139,7 +7171,6 @@
"version": "3.1.3",
"resolved": "https://verdaccio.lossless.one/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
"integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==",
"dev": true,
"license": "MIT"
},
"node_modules/fast-glob": {
@ -17691,6 +17722,33 @@
"systeminformation": "^5.11.3"
}
},
"@pushrocks/smartobject": {
"version": "1.0.8",
"resolved": "https://verdaccio.lossless.one/@pushrocks%2fsmartobject/-/smartobject-1.0.8.tgz",
"integrity": "sha512-sqrmbPkCZe4+OTqFGYEUBgwG9J/gMVdjz6Tuy8B7v5RaRvUobcjBk7FBa0ahMMPmuKEbddC5xa8+Jqtjb4iJAA==",
"requires": {
"fast-deep-equal": "^3.1.3",
"minimatch": "^5.0.1"
},
"dependencies": {
"brace-expansion": {
"version": "2.0.1",
"resolved": "https://verdaccio.lossless.one/brace-expansion/-/brace-expansion-2.0.1.tgz",
"integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
"requires": {
"balanced-match": "^1.0.0"
}
},
"minimatch": {
"version": "5.0.1",
"resolved": "https://verdaccio.lossless.one/minimatch/-/minimatch-5.0.1.tgz",
"integrity": "sha512-nLDxIFRyhDblz3qMuq+SoRZED4+miJ/G+tdDrjkkkRnjAsBexeGpgjLEQ0blJy7rHhR2b93rhQY4SvyWu9v03g==",
"requires": {
"brace-expansion": "^2.0.1"
}
}
}
},
"@pushrocks/smartparam": {
"version": "1.1.6",
"resolved": "https://verdaccio.lossless.one/@pushrocks%2fsmartparam/-/smartparam-1.1.6.tgz",
@ -21186,8 +21244,7 @@
"fast-deep-equal": {
"version": "3.1.3",
"resolved": "https://verdaccio.lossless.one/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
"integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==",
"dev": true
"integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="
},
"fast-glob": {
"version": "2.2.7",

3
package.json
View File

@ -23,7 +23,8 @@
"tslint-config-prettier": "^1.15.0"
},
"dependencies": {
"@depyronick/clickhouse-client": "^1.0.12"
"@depyronick/clickhouse-client": "^1.0.12",
"@pushrocks/smartobject": "^1.0.8"
},
"browserslist": [
"last 1 chrome versions"

35
test/test.nonci.ts Normal file
View File

@ -0,0 +1,35 @@
import { expect, expectAsync, tap } from '@pushrocks/tapbundle';
import * as smartclickhouse from '../ts/index';
let testClickhouseDb: smartclickhouse.SmartClickHouseDb;
tap.test('first test', async () => {
testClickhouseDb = new smartclickhouse.SmartClickHouseDb({
host: 'localhost',
database: 'test2',
});
});
tap.test('should start the clickhouse db', async () => {
await testClickhouseDb.start(true);
});
tap.test('should create a timedatatable', async () => {
const table = await testClickhouseDb.getTable('analytics');
let i = 0;
while(i < 10) {
await table.addData({
timestamp: Date.now(),
message: `hello this is a message ${i}`,
wow: 'hey',
deep: {
so: 'hello'
}
});
i++;
}
});
tap.skip.test('should write something to the clickhouse db', async () => {});
tap.start();

40
test/test.ts
View File

@ -1,40 +0,0 @@
import { expect, expectAsync, tap } from '@pushrocks/tapbundle';
import * as smartclickhouse from '../ts/index';
let testClickhouseDb: smartclickhouse.SmartClickHouseDb;
tap.test('first test', async () => {
testClickhouseDb = new smartclickhouse.SmartClickHouseDb({
host: 'localhost',
database: 'test2'
});
});
tap.test('should start the clickhouse db', async () => {
await testClickhouseDb.start();
})
tap.test('should create a timedatatable', async () => {
testClickhouseDb.getTable('analytics')
})
tap.skip.test('should write something to the clickhouse db', async () => {
const result2 = await testClickhouseDb.clickhouseClient.queryPromise(`CREATE TABLE IF NOT EXISTS visits2 (
timestamp UInt64,
ip String,
os String,
userAgent String,
version String
) ENGINE=MergeTree() ORDER BY timestamp`);
const result3 = await testClickhouseDb.clickhouseClient.insertPromise('visits2', [
{
timestamp: Date.now(),
ip: '127.0.01',
os: 'Mac OS X',
userAgent: 'some',
version: 'someversion'
}
]);
})
tap.start();

18
ts/smartclickhouse.classes.smartclickhouse.ts
View File

@ -7,8 +7,6 @@ export interface IClickhouseConstructorOptions {
password?: string;
}
export class SmartClickHouseDb {
public options: IClickhouseConstructorOptions;
public clickhouseClient: plugins.clickhouse.ClickHouseClient;
@ -20,24 +18,25 @@ export class SmartClickHouseDb {
/**
* starts the connection to the Clickhouse db
*/
public async start() {
console.log(`Connecting to default database first.`)
public async start(dropOld = false) {
console.log(`Connecting to default database first.`);
const defaultClient = new plugins.clickhouse.ClickHouseClient({
...this.options,
database: 'default'
database: 'default',
});
console.log(`Create database ${this.options.database}, if it does not exist...`);
if (dropOld) {
await defaultClient.queryPromise(`DROP DATABASE IF EXISTS ${this.options.database}`);
}
await defaultClient.queryPromise(`CREATE DATABASE IF NOT EXISTS ${this.options.database}`);
console.log(`Ensured database. Now connecting to wanted database: ${this.options.database}`)
console.log(`Ensured database. Now connecting to wanted database: ${this.options.database}`);
this.clickhouseClient = new plugins.clickhouse.ClickHouseClient({
...this.options,
});
console.log(`trying to ping database...`);
const result = await this.clickhouseClient.ping();
console.log(`Ping successfull?: ${result}`);
}
/**
@ -45,5 +44,6 @@ export class SmartClickHouseDb {
*/
public async getTable(tableName: string) {
const newTable = TimeDataTable.getTable(this, tableName);
return newTable;
}
}
}

108
ts/smartclickhouse.classes.timedatatable.ts
View File

@ -1,15 +1,52 @@
import * as plugins from './smartclickhouse.plugins';
import { SmartClickHouseDb } from './smartclickhouse.classes.smartclickhouse';
export type TClickhouseColumnDataType = 'String' | "DateTime64(3, 'Europe/Berlin')" | 'Float64';
export interface IColumnInfo {
database: string;
table: string;
name: string;
type: TClickhouseColumnDataType;
position: string;
default_kind: string;
default_expression: string;
data_compressed_bytes: string;
data_uncompressed_bytes: string;
marks_bytes: string;
comment: string;
is_in_partition_key: 0 | 1;
is_in_sorting_key: 0 | 1;
is_in_primary_key: 0 | 1;
is_in_sampling_key: 0 | 1;
compression_codec: string;
character_octet_length: null;
numeric_precision: null;
numeric_precision_radix: null;
numeric_scale: null;
datetime_precision: '3';
}
export class TimeDataTable {
public static async getTable (smartClickHouseDbRefArg: SmartClickHouseDb, tableNameArg: string) {
public static async getTable(smartClickHouseDbRefArg: SmartClickHouseDb, tableNameArg: string) {
const newTable = new TimeDataTable(smartClickHouseDbRefArg, tableNameArg);
// create table in clickhouse
await smartClickHouseDbRefArg.clickhouseClient.queryPromise(`CREATE TABLE IF NOT EXISTS ${newTable.tableName} (
await smartClickHouseDbRefArg.clickhouseClient
.queryPromise(`CREATE TABLE IF NOT EXISTS ${newTable.tableName} (
timestamp DateTime64(3, 'Europe/Berlin'),
message String
) ENGINE=MergeTree() ORDER BY timestamp`);
await newTable.updateColumns();
console.log(`=======================`)
console.log(
`table with name "${newTable.tableName}" in databse ${newTable.smartClickHouseDbRef.options.database} has the following columns:`
);
for (const column of newTable.columns) {
console.log(`>> ${column.name}: ${column.type}`);
}
console.log('^^^^^^^^^^^^^^\n');
return newTable;
}
@ -22,8 +59,71 @@ export class TimeDataTable {
this.tableName = tableNameArg;
}
public columns: IColumnInfo[] = [];
public seenPaths: { pathName: string; type: TClickhouseColumnDataType }[] = [];
/**
* updates the columns
*/
public async updateColumns() {
this.columns = await this.smartClickHouseDbRef.clickhouseClient.queryPromise(`
SELECT * FROM system.columns
WHERE database LIKE '${this.smartClickHouseDbRef.options.database}'
AND table LIKE '${this.tableName}'
`);
return this.columns;
}
/**
* stores a json and tries to map it to the nested syntax
*/
public async storeJson () {}
}
public async addData(dataArg: any) {
// the storageJson
let storageJson: { [key: string]: any } = {};
// helper stuff
const typeConversion: {[key: string]: TClickhouseColumnDataType} = {
string: 'String',
number: 'Float64',
};
const getClickhouseTypeForValue = (valueArg: any) => {
return typeConversion[(typeof valueArg) as string];
}
const checkPath = async (pathArg: string, typeArg: TClickhouseColumnDataType) => {
let columnFound = false;
for (const column of this.columns) {
if (pathArg === column.name) {
columnFound = true;
break;
}
}
if (!columnFound) {
await this.smartClickHouseDbRef.clickhouseClient.queryPromise(`
ALTER TABLE ${this.tableName} ADD COLUMN ${pathArg} ${typeArg} FIRST
`);
await this.updateColumns();
}
};
// key checking
const flatDataArg = plugins.smartobject.toFlatObject(dataArg);
for (const key of Object.keys(flatDataArg)) {
const value = flatDataArg[key];
if (key === 'timestamp' && typeof value !== 'number') {
throw new Error('timestamp must be of type number');
} else if (key === 'timestamp') {
storageJson.timestamp = flatDataArg[key];
continue;
}
// lets deal with the rest
const clickhouseType = getClickhouseTypeForValue(value);
await checkPath(key, clickhouseType);
storageJson[key] = value;
}
const result = await this.smartClickHouseDbRef.clickhouseClient.insertPromise(this.tableName, [
storageJson,
]);
return result;
}
}

10
ts/smartclickhouse.plugins.ts
View File

@ -1,5 +1,11 @@
import * as clickhouse from '@depyronick/clickhouse-client';
// @pushrocks scope
import * as smartobject from '@pushrocks/smartobject';
export {
clickhouse
smartobject
}
// thirdparty
import * as clickhouse from '@depyronick/clickhouse-client';
export { clickhouse };