1.0.9

package-lock.json

@@ -1,15 +1,16 @@
 {
   "name": "@pushrocks/smartclickhouse",
-  "version": "1.0.4",
+  "version": "1.0.9",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "@pushrocks/smartclickhouse",
-      "version": "1.0.4",
+      "version": "1.0.9",
       "license": "MIT",
       "dependencies": {
-        "@depyronick/clickhouse-client": "^1.0.12"
+        "@depyronick/clickhouse-client": "^1.0.12",
+        "@pushrocks/smartobject": "^1.0.9"
       },
       "devDependencies": {
         "@gitzone/tsbuild": "^2.1.25",
@@ -2633,6 +2634,37 @@
         "systeminformation": "^5.11.3"
       }
     },
+    "node_modules/@pushrocks/smartobject": {
+      "version": "1.0.9",
+      "resolved": "https://verdaccio.lossless.one/@pushrocks%2fsmartobject/-/smartobject-1.0.9.tgz",
+      "integrity": "sha512-74IlcVWhhEOsG7X65H4DmYP1o8HejZMWj5uATCk99A+XjZMi8UJu8H7EsgT++JJKkwLz2BKc9323BfNk1B79WA==",
+      "license": "MIT",
+      "dependencies": {
+        "fast-deep-equal": "^3.1.3",
+        "minimatch": "^5.0.1"
+      }
+    },
+    "node_modules/@pushrocks/smartobject/node_modules/brace-expansion": {
+      "version": "2.0.1",
+      "resolved": "https://verdaccio.lossless.one/brace-expansion/-/brace-expansion-2.0.1.tgz",
+      "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
+      "license": "MIT",
+      "dependencies": {
+        "balanced-match": "^1.0.0"
+      }
+    },
+    "node_modules/@pushrocks/smartobject/node_modules/minimatch": {
+      "version": "5.0.1",
+      "resolved": "https://verdaccio.lossless.one/minimatch/-/minimatch-5.0.1.tgz",
+      "integrity": "sha512-nLDxIFRyhDblz3qMuq+SoRZED4+miJ/G+tdDrjkkkRnjAsBexeGpgjLEQ0blJy7rHhR2b93rhQY4SvyWu9v03g==",
+      "license": "ISC",
+      "dependencies": {
+        "brace-expansion": "^2.0.1"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
     "node_modules/@pushrocks/smartparam": {
       "version": "1.1.6",
       "resolved": "https://verdaccio.lossless.one/@pushrocks%2fsmartparam/-/smartparam-1.1.6.tgz",
@@ -7139,7 +7171,6 @@
       "version": "3.1.3",
       "resolved": "https://verdaccio.lossless.one/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
       "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==",
-      "dev": true,
       "license": "MIT"
     },
     "node_modules/fast-glob": {
@@ -17691,6 +17722,33 @@
         "systeminformation": "^5.11.3"
       }
     },
+    "@pushrocks/smartobject": {
+      "version": "1.0.9",
+      "resolved": "https://verdaccio.lossless.one/@pushrocks%2fsmartobject/-/smartobject-1.0.9.tgz",
+      "integrity": "sha512-74IlcVWhhEOsG7X65H4DmYP1o8HejZMWj5uATCk99A+XjZMi8UJu8H7EsgT++JJKkwLz2BKc9323BfNk1B79WA==",
+      "requires": {
+        "fast-deep-equal": "^3.1.3",
+        "minimatch": "^5.0.1"
+      },
+      "dependencies": {
+        "brace-expansion": {
+          "version": "2.0.1",
+          "resolved": "https://verdaccio.lossless.one/brace-expansion/-/brace-expansion-2.0.1.tgz",
+          "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
+          "requires": {
+            "balanced-match": "^1.0.0"
+          }
+        },
+        "minimatch": {
+          "version": "5.0.1",
+          "resolved": "https://verdaccio.lossless.one/minimatch/-/minimatch-5.0.1.tgz",
+          "integrity": "sha512-nLDxIFRyhDblz3qMuq+SoRZED4+miJ/G+tdDrjkkkRnjAsBexeGpgjLEQ0blJy7rHhR2b93rhQY4SvyWu9v03g==",
+          "requires": {
+            "brace-expansion": "^2.0.1"
+          }
+        }
+      }
+    },
     "@pushrocks/smartparam": {
       "version": "1.1.6",
       "resolved": "https://verdaccio.lossless.one/@pushrocks%2fsmartparam/-/smartparam-1.1.6.tgz",
@@ -21186,8 +21244,7 @@
     "fast-deep-equal": {
       "version": "3.1.3",
       "resolved": "https://verdaccio.lossless.one/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
-      "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==",
-      "dev": true
+      "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="
     },
     "fast-glob": {
       "version": "2.2.7",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pushrocks/smartclickhouse",
-  "version": "1.0.4",
+  "version": "1.0.9",
   "private": false,
   "description": "an odm for talking to clickhouse",
   "main": "dist_ts/index.js",
@@ -9,7 +9,9 @@
   "license": "MIT",
   "scripts": {
     "test": "(tstest test/ --web)",
-    "build": "(tsbuild --web)"
+    "build": "(tsbuild --web)",
+    "createGrafana": "docker run --name grafana -d -p 4000:3000 grafana/grafana-oss",
+    "createClickhouse": "docker run --name some-clickhouse-server --ulimit nofile=262144:262144 -p 8123:8123 -p 9000:9000  --volume=$PWD/.nogit/testdatabase:/var/lib/clickhouse yandex/clickhouse-server"
   },
   "devDependencies": {
     "@gitzone/tsbuild": "^2.1.25",
@@ -21,7 +23,8 @@
     "tslint-config-prettier": "^1.15.0"
   },
   "dependencies": {
-    "@depyronick/clickhouse-client": "^1.0.12"
+    "@depyronick/clickhouse-client": "^1.0.12",
+    "@pushrocks/smartobject": "^1.0.9"
   },
   "browserslist": [
     "last 1 chrome versions"

test/test.nonci.ts

@@ -0,0 +1,36 @@
+import { expect, expectAsync, tap } from '@pushrocks/tapbundle';
+import * as smartclickhouse from '../ts/index';
+
+let testClickhouseDb: smartclickhouse.SmartClickHouseDb;
+
+tap.test('first test', async () => {
+  testClickhouseDb = new smartclickhouse.SmartClickHouseDb({
+    host: 'localhost',
+    database: 'test2',
+  });
+});
+
+tap.test('should start the clickhouse db', async () => {
+  await testClickhouseDb.start(true);
+});
+
+tap.test('should create a timedatatable', async () => {
+  const table = await testClickhouseDb.getTable('analytics');
+  let i = 0;
+  while(i < 10) {
+    await table.addData({
+      timestamp: Date.now(),
+      message: `hello this is a message ${i}`,
+      wow: 'hey',
+      deep: {
+        so: 'hello',
+        myArray: ['array1', 'array2']
+      }
+    });
+    i++;
+  }
+});
+
+tap.skip.test('should write something to the clickhouse db', async () => {});
+
+tap.start();

test/test.ts

@@ -1,36 +0,0 @@
-import { expect, expectAsync, tap } from '@pushrocks/tapbundle';
-import * as smartclickhouse from '../ts/index';
-
-let testClickhouseDb: smartclickhouse.SmartClickHouseDb;
-
-tap.test('first test', async () => {
-  testClickhouseDb = new smartclickhouse.SmartClickHouseDb({
-    host: 'localhost',
-    database: 'test2'
-  });
-});
-
-tap.test('should start the clickhouse db', async () => {
-  await testClickhouseDb.start();
-})
-
-tap.skip.test('should write something to the clickhouse db', async () => {
-  const result2 = await testClickhouseDb.clickhouseClient.queryPromise(`CREATE TABLE IF NOT EXISTS visits2 (
-    timestamp UInt64,
-    ip String,
-    os String,
-    userAgent String,
-    version String
-  ) ENGINE=MergeTree() ORDER BY timestamp`);
-  const result3 = await testClickhouseDb.clickhouseClient.insertPromise('visits2', [
-    {
-      timestamp: Date.now(),
-      ip: '127.0.01',
-      os: 'Mac OS X',
-      userAgent: 'some',
-      version: 'someversion'
-    }
-  ]);
-})
-
-tap.start();

ts/smartclickhouse.classes.smartclickhouse.ts

@@ -7,8 +7,6 @@ export interface IClickhouseConstructorOptions {
   password?: string;
 }
 
-
-
 export class SmartClickHouseDb {
   public options: IClickhouseConstructorOptions;
   public clickhouseClient: plugins.clickhouse.ClickHouseClient;
@@ -20,24 +18,25 @@ export class SmartClickHouseDb {
   /**
    * starts the connection to the Clickhouse db
    */
-  public async start() {
-    console.log(`Connecting to default database first.`)
+  public async start(dropOld = false) {
+    console.log(`Connecting to default database first.`);
     const defaultClient = new plugins.clickhouse.ClickHouseClient({
       ...this.options,
-      database: 'default'
+      database: 'default',
     });
     console.log(`Create database ${this.options.database}, if it does not exist...`);
+    if (dropOld) {
+      await defaultClient.queryPromise(`DROP DATABASE IF EXISTS ${this.options.database}`);
+    }
     await defaultClient.queryPromise(`CREATE DATABASE IF NOT EXISTS ${this.options.database}`);
 
-    console.log(`Ensured database. Now connecting to wanted database: ${this.options.database}`)
+    console.log(`Ensured database. Now connecting to wanted database: ${this.options.database}`);
     this.clickhouseClient = new plugins.clickhouse.ClickHouseClient({
       ...this.options,
-
     });
     console.log(`trying to ping database...`);
     const result = await this.clickhouseClient.ping();
     console.log(`Ping successfull?: ${result}`);
-    
   }
 
   /**
@@ -45,5 +44,6 @@ export class SmartClickHouseDb {
    */
   public async getTable(tableName: string) {
     const newTable = TimeDataTable.getTable(this, tableName);
+    return newTable;
   }
 }

ts/smartclickhouse.classes.timedatatable.ts

@@ -1,12 +1,61 @@
 import * as plugins from './smartclickhouse.plugins';
 import { SmartClickHouseDb } from './smartclickhouse.classes.smartclickhouse';
 
+export type TClickhouseColumnDataType = 'String' | "DateTime64(3, 'Europe/Berlin')" | 'Float64' | 'Array(String)' | 'Array(Float64)';
+export interface IColumnInfo {
+  database: string;
+  table: string;
+  name: string;
+  type: TClickhouseColumnDataType;
+  position: string;
+  default_kind: string;
+  default_expression: string;
+  data_compressed_bytes: string;
+  data_uncompressed_bytes: string;
+  marks_bytes: string;
+  comment: string;
+  is_in_partition_key: 0 | 1;
+  is_in_sorting_key: 0 | 1;
+  is_in_primary_key: 0 | 1;
+  is_in_sampling_key: 0 | 1;
+  compression_codec: string;
+  character_octet_length: null;
+  numeric_precision: null;
+  numeric_precision_radix: null;
+  numeric_scale: null;
+  datetime_precision: '3';
+}
+
 export class TimeDataTable {
   public static async getTable(smartClickHouseDbRefArg: SmartClickHouseDb, tableNameArg: string) {
     const newTable = new TimeDataTable(smartClickHouseDbRefArg, tableNameArg);
 
     // create table in clickhouse
-    smartClickHouseDbRefArg;
+    await smartClickHouseDbRefArg.clickhouseClient
+      .queryPromise(`
+        CREATE TABLE IF NOT EXISTS ${newTable.tableName} (
+          timestamp DateTime64(3, 'Europe/Berlin'),
+          message String
+        ) ENGINE=MergeTree() ORDER BY timestamp
+    `);
+    
+    // lets adjust the TTL
+    await smartClickHouseDbRefArg.clickhouseClient
+      .queryPromise(`
+        ALTER TABLE ${newTable.tableName} MODIFY TTL toDateTime(timestamp) + INTERVAL 1 MONTH;
+      `);
+
+    await newTable.updateColumns();
+    console.log(`=======================`)
+    console.log(
+      `table with name "${newTable.tableName}" in databse ${newTable.smartClickHouseDbRef.options.database} has the following columns:`
+    );
+    for (const column of newTable.columns) {
+      console.log(`>> ${column.name}: ${column.type}`);
+    }
+    console.log('^^^^^^^^^^^^^^\n');
+
+    return newTable;
   }
 
   // INSTANCE
@@ -17,4 +66,99 @@ export class TimeDataTable {
     this.smartClickHouseDbRef = smartClickHouseDbRefArg;
     this.tableName = tableNameArg;
   }
+
+  public columns: IColumnInfo[] = [];
+  public seenPaths: { pathName: string; type: TClickhouseColumnDataType }[] = [];
+
+  /**
+   * updates the columns
+   */
+  public async updateColumns() {
+    this.columns = await this.smartClickHouseDbRef.clickhouseClient.queryPromise(`
+      SELECT * FROM system.columns
+      WHERE database LIKE '${this.smartClickHouseDbRef.options.database}'
+      AND table LIKE '${this.tableName}'
+    `);
+    return this.columns;
+  }
+
+  /**
+   * stores a json and tries to map it to the nested syntax
+   */
+  public async addData(dataArg: any) {
+    // the storageJson
+    let storageJson: { [key: string]: any } = {};
+    
+    // helper stuff
+    
+    const getClickhouseTypeForValue = (valueArg: any): TClickhouseColumnDataType => {
+      const typeConversion: {[key: string]: TClickhouseColumnDataType} = {
+        string: 'String',
+        number: 'Float64',
+        undefined: null,
+        null: null
+      };
+      if (valueArg instanceof Array) {
+        const arrayType = typeConversion[(typeof valueArg[0]) as string];
+        if (!arrayType) {
+          return null;
+        } else {
+          return `Array(${arrayType})` as TClickhouseColumnDataType;
+        }
+      }
+      return typeConversion[(typeof valueArg) as string];
+    }
+    const checkPath = async (pathArg: string, typeArg: TClickhouseColumnDataType, prechecked = false) => {
+      let columnFound = false;
+      for (const column of this.columns) {
+        if (pathArg === column.name) {
+          columnFound = true;
+          break;
+        }
+      }
+      if (!columnFound) {
+        if (!prechecked) {
+          await this.updateColumns();
+          await checkPath(pathArg, typeArg, true);
+          return;
+        }
+        const alterString = `ALTER TABLE ${this.tableName} ADD COLUMN ${pathArg} ${typeArg} FIRST`
+        try {
+          await this.smartClickHouseDbRef.clickhouseClient.queryPromise(`
+          ${alterString}
+        `);
+        } catch(err) {
+          console.log(alterString);
+          for (const column of this.columns) {
+            console.log(column.name);
+          }
+        }
+        await this.updateColumns();
+      }
+    };
+
+    // key checking
+    const flatDataArg = plugins.smartobject.toFlatObject(dataArg);
+    for (const key of Object.keys(flatDataArg)) {
+      const value = flatDataArg[key];
+      if (key === 'timestamp' && typeof value !== 'number') {
+        throw new Error('timestamp must be of type number');
+      } else if (key === 'timestamp') {
+        storageJson.timestamp = flatDataArg[key];
+        continue;
+      }
+      // lets deal with the rest
+      const clickhouseType = getClickhouseTypeForValue(value);
+      if (!clickhouseType) {
+        continue;
+      }
+      await checkPath(key, clickhouseType);
+      storageJson[key] = value;
+    }
+
+    const result = await this.smartClickHouseDbRef.clickhouseClient.insertPromise(this.tableName, [
+      storageJson,
+    ]);
+    return result;
+  }
 }

ts/smartclickhouse.plugins.ts

@@ -1,5 +1,11 @@
-import * as clickhouse from '@depyronick/clickhouse-client';
+// @pushrocks scope
+import * as smartobject from '@pushrocks/smartobject';
 
 export {
-  clickhouse
+  smartobject
 }
+
+// thirdparty
+import * as clickhouse from '@depyronick/clickhouse-client';
+
+export { clickhouse };