2.0.3

.vscode/settings.json

@@ -22,5 +22,6 @@
         }
       }
     }
-  ]
+  ],
+  "typescript.tsdk": "node_modules/typescript/lib"
 }

package.json

@@ -1,10 +1,11 @@
 {
   "name": "@pushrocks/smartclickhouse",
-  "version": "1.0.6",
+  "version": "2.0.3",
   "private": false,
   "description": "an odm for talking to clickhouse",
   "main": "dist_ts/index.js",
   "typings": "dist_ts/index.d.ts",
+  "type": "module",
   "author": "Lossless GmbH",
   "license": "MIT",
   "scripts": {
@@ -14,17 +15,18 @@
     "createClickhouse": "docker run --name some-clickhouse-server --ulimit nofile=262144:262144 -p 8123:8123 -p 9000:9000  --volume=$PWD/.nogit/testdatabase:/var/lib/clickhouse yandex/clickhouse-server"
   },
   "devDependencies": {
-    "@gitzone/tsbuild": "^2.1.25",
-    "@gitzone/tsbundle": "^1.0.78",
-    "@gitzone/tstest": "^1.0.44",
-    "@pushrocks/tapbundle": "^4.0.8",
-    "@types/node": "^17.0.21",
+    "@gitzone/tsbuild": "^2.1.63",
+    "@gitzone/tsbundle": "^2.0.6",
+    "@gitzone/tstest": "^1.0.72",
+    "@pushrocks/tapbundle": "^5.0.4",
+    "@types/node": "^18.6.1",
     "tslint": "^6.1.3",
     "tslint-config-prettier": "^1.15.0"
   },
   "dependencies": {
-    "@depyronick/clickhouse-client": "^1.0.12",
-    "@pushrocks/smartobject": "^1.0.8"
+    "@depyronick/clickhouse-client": "^1.0.14",
+    "@pushrocks/smartobject": "^1.0.10",
+    "@pushrocks/smarturl": "^3.0.1"
   },
   "browserslist": [
     "last 1 chrome versions"

test/test.nonci.ts

@@ -1,5 +1,5 @@
 import { expect, expectAsync, tap } from '@pushrocks/tapbundle';
-import * as smartclickhouse from '../ts/index';
+import * as smartclickhouse from '../ts/index.js';
 
 let testClickhouseDb: smartclickhouse.SmartClickHouseDb;
 
@@ -7,6 +7,8 @@ tap.test('first test', async () => {
   testClickhouseDb = new smartclickhouse.SmartClickHouseDb({
     host: 'localhost',
     database: 'test2',
+    port: 8123,
+    protocol: 'http',
   });
 });
 
@@ -23,7 +25,8 @@ tap.test('should create a timedatatable', async () => {
       message: `hello this is a message ${i}`,
       wow: 'hey',
       deep: {
-        so: 'hello'
+        so: 'hello',
+        myArray: ['array1', 'array2']
       }
     });
     i++;

ts/00_commitinfo_data.ts

@@ -0,0 +1,8 @@
+/**
+ * autocreated commitinfo by @pushrocks/commitinfo
+ */
+export const commitinfo = {
+  name: '@pushrocks/smartclickhouse',
+  version: '2.0.3',
+  description: 'an odm for talking to clickhouse'
+}

ts/index.ts

@@ -1,2 +1,2 @@
-export * from './smartclickhouse.classes.smartclickhouse';
-export * from './smartclickhouse.classes.timedatatable';
+export * from './smartclickhouse.classes.smartclickhouse.js';
+export * from './smartclickhouse.classes.timedatatable.js';

ts/smartclickhouse.classes.smartclickhouse.ts

@@ -1,8 +1,8 @@
-import * as plugins from './smartclickhouse.plugins';
-import { TimeDataTable } from './smartclickhouse.classes.timedatatable';
+import * as plugins from './smartclickhouse.plugins.js';
+import { TimeDataTable } from './smartclickhouse.classes.timedatatable.js';
 
 export interface IClickhouseConstructorOptions {
-  host: string;
+  url: string;
   database: string;
   password?: string;
 }
@@ -20,8 +20,20 @@ export class SmartClickHouseDb {
    */
   public async start(dropOld = false) {
     console.log(`Connecting to default database first.`);
+    const defaultOptions: {[keyArg: string]: string} = {};
+    // the protocol, url and host
+    const parsedUrl = plugins.smarturl.Smarturl.createFromUrl(this.options.url);
+    parsedUrl.protocol === 'https' ? defaultOptions.protocol = plugins.clickhouse.ClickHouseConnectionProtocol.HTTPS : null;
+    parsedUrl.protocol === 'http' ? defaultOptions.protocol = plugins.clickhouse.ClickHouseConnectionProtocol.HTTP : null;
+    defaultOptions.host = parsedUrl.hostname;
+    defaultOptions.port = parsedUrl.port;
+    // the database
+    defaultOptions.database = this.options.database;
+    // the password
+    this.options.password ? defaultOptions.password = this.options.password : null;
+    // lets connect
     const defaultClient = new plugins.clickhouse.ClickHouseClient({
-      ...this.options,
+      ...defaultOptions,
       database: 'default',
     });
     console.log(`Create database ${this.options.database}, if it does not exist...`);
@@ -32,7 +44,7 @@ export class SmartClickHouseDb {
 
     console.log(`Ensured database. Now connecting to wanted database: ${this.options.database}`);
     this.clickhouseClient = new plugins.clickhouse.ClickHouseClient({
-      ...this.options,
+      ...defaultOptions
     });
     console.log(`trying to ping database...`);
     const result = await this.clickhouseClient.ping();

ts/smartclickhouse.classes.timedatatable.ts

@@ -1,7 +1,7 @@
-import * as plugins from './smartclickhouse.plugins';
-import { SmartClickHouseDb } from './smartclickhouse.classes.smartclickhouse';
+import * as plugins from './smartclickhouse.plugins.js';
+import { SmartClickHouseDb } from './smartclickhouse.classes.smartclickhouse.js';
 
-export type TClickhouseColumnDataType = 'String' | "DateTime64(3, 'Europe/Berlin')" | 'Float64';
+export type TClickhouseColumnDataType = 'String' | "DateTime64(3, 'Europe/Berlin')" | 'Float64' | 'Array(String)' | 'Array(Float64)';
 export interface IColumnInfo {
   database: string;
   table: string;
@@ -26,21 +26,36 @@ export interface IColumnInfo {
   datetime_precision: '3';
 }
 
+export interface ITimeDataTableOptions {
+  tableName: string;
+  retainDataForDays: number;
+}
+
 export class TimeDataTable {
   public static async getTable(smartClickHouseDbRefArg: SmartClickHouseDb, tableNameArg: string) {
-    const newTable = new TimeDataTable(smartClickHouseDbRefArg, tableNameArg);
+    const newTable = new TimeDataTable(smartClickHouseDbRefArg, {
+      tableName: tableNameArg,
+      retainDataForDays: 30
+    });
 
     // create table in clickhouse
     await smartClickHouseDbRefArg.clickhouseClient
-      .queryPromise(`CREATE TABLE IF NOT EXISTS ${newTable.tableName} (
+      .queryPromise(`
+        CREATE TABLE IF NOT EXISTS ${newTable.options.tableName} (
           timestamp DateTime64(3, 'Europe/Berlin'),
           message String
         ) ENGINE=MergeTree() ORDER BY timestamp`);
     
+    // lets adjust the TTL
+    await smartClickHouseDbRefArg.clickhouseClient
+      .queryPromise(`
+        ALTER TABLE ${newTable.options.tableName} MODIFY TTL toDateTime(timestamp) + INTERVAL ${newTable.options.retainDataForDays} DAY
+      `);
+
     await newTable.updateColumns();
     console.log(`=======================`)
     console.log(
-      `table with name "${newTable.tableName}" in databse ${newTable.smartClickHouseDbRef.options.database} has the following columns:`
+      `table with name "${newTable.options.tableName}" in databse ${newTable.smartClickHouseDbRef.options.database} has the following columns:`
     );
     for (const column of newTable.columns) {
       console.log(`>> ${column.name}: ${column.type}`);
@@ -52,15 +67,14 @@ export class TimeDataTable {
 
   // INSTANCE
   public smartClickHouseDbRef: SmartClickHouseDb;
-  public tableName: string;
+  public options: ITimeDataTableOptions;
 
-  constructor(smartClickHouseDbRefArg: SmartClickHouseDb, tableNameArg: string) {
+  constructor(smartClickHouseDbRefArg: SmartClickHouseDb, optionsArg: ITimeDataTableOptions) {
     this.smartClickHouseDbRef = smartClickHouseDbRefArg;
-    this.tableName = tableNameArg;
+    this.options = optionsArg;
   }
 
   public columns: IColumnInfo[] = [];
-  public seenPaths: { pathName: string; type: TClickhouseColumnDataType }[] = [];
 
   /**
    * updates the columns
@@ -69,7 +83,7 @@ export class TimeDataTable {
     this.columns = await this.smartClickHouseDbRef.clickhouseClient.queryPromise(`
       SELECT * FROM system.columns
       WHERE database LIKE '${this.smartClickHouseDbRef.options.database}'
-      AND table LIKE '${this.tableName}'
+      AND table LIKE '${this.options.tableName}'
     `);
     return this.columns;
   }
@@ -82,14 +96,25 @@ export class TimeDataTable {
     let storageJson: { [key: string]: any } = {};
     
     // helper stuff
+    
+    const getClickhouseTypeForValue = (valueArg: any): TClickhouseColumnDataType => {
       const typeConversion: {[key: string]: TClickhouseColumnDataType} = {
         string: 'String',
         number: 'Float64',
+        undefined: null,
+        null: null
       };
-    const getClickhouseTypeForValue = (valueArg: any) => {
+      if (valueArg instanceof Array) {
+        const arrayType = typeConversion[(typeof valueArg[0]) as string];
+        if (!arrayType) {
+          return null;
+        } else {
+          return `Array(${arrayType})` as TClickhouseColumnDataType;
+        }
+      }
       return typeConversion[(typeof valueArg) as string];
     }
-    const checkPath = async (pathArg: string, typeArg: TClickhouseColumnDataType) => {
+    const checkPath = async (pathArg: string, typeArg: TClickhouseColumnDataType, prechecked = false) => {
       let columnFound = false;
       for (const column of this.columns) {
         if (pathArg === column.name) {
@@ -98,9 +123,22 @@ export class TimeDataTable {
         }
       }
       if (!columnFound) {
+        if (!prechecked) {
+          await this.updateColumns();
+          await checkPath(pathArg, typeArg, true);
+          return;
+        }
+        const alterString = `ALTER TABLE ${this.options.tableName} ADD COLUMN ${pathArg} ${typeArg} FIRST`
+        try {
           await this.smartClickHouseDbRef.clickhouseClient.queryPromise(`
-          ALTER TABLE ${this.tableName} ADD COLUMN ${pathArg} ${typeArg} FIRST
+          ${alterString}
         `);
+        } catch(err) {
+          console.log(alterString);
+          for (const column of this.columns) {
+            console.log(column.name);
+          }
+        }
         await this.updateColumns();
       }
     };
@@ -117,11 +155,14 @@ export class TimeDataTable {
       }
       // lets deal with the rest
       const clickhouseType = getClickhouseTypeForValue(value);
+      if (!clickhouseType) {
+        continue;
+      }
       await checkPath(key, clickhouseType);
       storageJson[key] = value;
     }
 
-    const result = await this.smartClickHouseDbRef.clickhouseClient.insertPromise(this.tableName, [
+    const result = await this.smartClickHouseDbRef.clickhouseClient.insertPromise(this.options.tableName, [
       storageJson,
     ]);
     return result;

ts/smartclickhouse.plugins.ts

@@ -1,8 +1,10 @@
 // @pushrocks scope
 import * as smartobject from '@pushrocks/smartobject';
+import * as smarturl from '@pushrocks/smarturl';
 
 export {
-  smartobject
+  smartobject,
+  smarturl
 }
 
 // thirdparty

tsconfig.json

@@ -0,0 +1,5 @@
+{
+  "compilerOptions": {
+    "moduleResolution": "node12"
+  }
+}

tslint.json

@@ -1,17 +0,0 @@
-{
-  "extends": ["tslint:latest", "tslint-config-prettier"],
-  "rules": {
-    "semicolon": [true, "always"],
-    "no-console": false,
-    "ordered-imports": false,
-    "object-literal-sort-keys": false,
-    "member-ordering": {
-      "options":{
-        "order": [
-          "static-method"
-        ]
-      }
-    }
-  },
-  "defaultSeverity": "warning"
-}