2.0.6

package-lock.json

@@ -1,16 +1,18 @@
 {
   "name": "@pushrocks/smartclickhouse",
-  "version": "2.0.4",
+  "version": "2.0.6",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "@pushrocks/smartclickhouse",
-      "version": "2.0.4",
+      "version": "2.0.6",
       "license": "MIT",
       "dependencies": {
         "@depyronick/clickhouse-client": "^1.0.14",
+        "@pushrocks/smartdelay": "^2.0.13",
         "@pushrocks/smartobject": "^1.0.10",
+        "@pushrocks/smartpromise": "^3.1.7",
         "@pushrocks/smarturl": "^3.0.1"
       },
       "devDependencies": {
@@ -921,7 +923,6 @@
       "version": "2.0.13",
       "resolved": "https://verdaccio.lossless.one/@pushrocks%2fsmartdelay/-/smartdelay-2.0.13.tgz",
       "integrity": "sha512-s6Wh0BHWMfZ5VYONQwpxOYX1JeC9RKA0O9TxEzfZ6FCw2oNQb2QUPCixT9rsceKwva4+atKRw/RfU+Z7aJDmsA==",
-      "dev": true,
       "license": "MIT",
       "dependencies": {
         "@pushrocks/smartpromise": "^3.0.6"
@@ -1281,7 +1282,6 @@
       "version": "3.1.7",
       "resolved": "https://verdaccio.lossless.one/@pushrocks%2fsmartpromise/-/smartpromise-3.1.7.tgz",
       "integrity": "sha512-2gLQCeviEJwZ+cHHtK2Ks98brZatGC6dPXKIs1tVgJsiNgRFjnp90fESuJ1Pmoe7RrS+7J3mO4NtsFHAJJ/y5w==",
-      "dev": true,
       "license": "MIT"
     },
     "node_modules/@pushrocks/smartpuppeteer": {
@@ -9575,7 +9575,6 @@
       "version": "2.0.13",
       "resolved": "https://verdaccio.lossless.one/@pushrocks%2fsmartdelay/-/smartdelay-2.0.13.tgz",
       "integrity": "sha512-s6Wh0BHWMfZ5VYONQwpxOYX1JeC9RKA0O9TxEzfZ6FCw2oNQb2QUPCixT9rsceKwva4+atKRw/RfU+Z7aJDmsA==",
-      "dev": true,
       "requires": {
         "@pushrocks/smartpromise": "^3.0.6"
       }
@@ -9912,8 +9911,7 @@
     "@pushrocks/smartpromise": {
       "version": "3.1.7",
       "resolved": "https://verdaccio.lossless.one/@pushrocks%2fsmartpromise/-/smartpromise-3.1.7.tgz",
-      "integrity": "sha512-2gLQCeviEJwZ+cHHtK2Ks98brZatGC6dPXKIs1tVgJsiNgRFjnp90fESuJ1Pmoe7RrS+7J3mO4NtsFHAJJ/y5w==",
-      "dev": true
+      "integrity": "sha512-2gLQCeviEJwZ+cHHtK2Ks98brZatGC6dPXKIs1tVgJsiNgRFjnp90fESuJ1Pmoe7RrS+7J3mO4NtsFHAJJ/y5w=="
     },
     "@pushrocks/smartpuppeteer": {
       "version": "2.0.0",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pushrocks/smartclickhouse",
-  "version": "2.0.4",
+  "version": "2.0.6",
   "private": false,
   "description": "an odm for talking to clickhouse",
   "main": "dist_ts/index.js",
@@ -25,7 +25,9 @@
   },
   "dependencies": {
     "@depyronick/clickhouse-client": "^1.0.14",
+    "@pushrocks/smartdelay": "^2.0.13",
     "@pushrocks/smartobject": "^1.0.10",
+    "@pushrocks/smartpromise": "^3.1.7",
     "@pushrocks/smarturl": "^3.0.1"
   },
   "browserslist": [

test/test.nonci.ts

@@ -5,10 +5,8 @@ let testClickhouseDb: smartclickhouse.SmartClickHouseDb;
 
 tap.test('first test', async () => {
   testClickhouseDb = new smartclickhouse.SmartClickHouseDb({
-    host: 'localhost',
+    url: 'http://localhost:8123',
     database: 'test2',
-    port: 8123,
-    protocol: 'http',
   });
 });
 
@@ -16,10 +14,10 @@ tap.test('should start the clickhouse db', async () => {
   await testClickhouseDb.start(true);
 });
 
-tap.test('should create a timedatatable', async () => {
+tap.test('should create a timedatatable', async (toolsArg) => {
   const table = await testClickhouseDb.getTable('analytics');
   let i = 0;
-  while(i < 10) {
+  while(i < 5000) {
     await table.addData({
       timestamp: Date.now(),
       message: `hello this is a message ${i}`,
@@ -30,6 +28,8 @@ tap.test('should create a timedatatable', async () => {
       }
     });
     i++;
+    console.log(`logged ${i} of 5000 lines.`);
+    await toolsArg.delayFor(1);
   }
 });
 

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@pushrocks/smartclickhouse',
-  version: '2.0.4',
+  version: '2.0.6',
   description: 'an odm for talking to clickhouse'
 }

ts/smartclickhouse.classes.smartclickhouse.ts

@@ -9,6 +9,7 @@ export interface IClickhouseConstructorOptions {
 
 export class SmartClickHouseDb {
   public options: IClickhouseConstructorOptions;
+  public defaultClient: plugins.clickhouse.ClickHouseClient;
   public clickhouseClient: plugins.clickhouse.ClickHouseClient;
 
   constructor(optionsArg: IClickhouseConstructorOptions) {
@@ -32,23 +33,38 @@ export class SmartClickHouseDb {
     // the password
     this.options.password ? defaultOptions.password = this.options.password : null;
     // lets connect
-    const defaultClient = new plugins.clickhouse.ClickHouseClient({
+    this.defaultClient = new plugins.clickhouse.ClickHouseClient({
       ...defaultOptions,
       database: 'default',
     });
+    await this.pingDatabaseUntilAvailable();
     console.log(`Create database ${this.options.database}, if it does not exist...`);
-    if (dropOld) {
-      await defaultClient.queryPromise(`DROP DATABASE IF EXISTS ${this.options.database}`);
-    }
-    await defaultClient.queryPromise(`CREATE DATABASE IF NOT EXISTS ${this.options.database}`);
+    await this.createDatabase(dropOld);
 
     console.log(`Ensured database. Now connecting to wanted database: ${this.options.database}`);
     this.clickhouseClient = new plugins.clickhouse.ClickHouseClient({
       ...defaultOptions
     });
-    console.log(`trying to ping database...`);
-    const result = await this.clickhouseClient.ping();
-    console.log(`Ping successfull?: ${result}`);
+  }
+
+  public async createDatabase(dropOld: boolean = false) {
+    if (dropOld) {
+      await this.defaultClient.queryPromise(`DROP DATABASE IF EXISTS ${this.options.database}`);
+    }
+    await this.defaultClient.queryPromise(`CREATE DATABASE IF NOT EXISTS ${this.options.database}`);
+  }
+
+  public async pingDatabaseUntilAvailable() {
+    let available = false;
+    while(!available) {
+      available = await this.defaultClient.ping().catch(err => {
+        return false;
+      });
+      if (!available) {
+        console.log(`NOT OK: tried pinging ${this.options.url}... Trying again in 5 seconds.`);
+        await plugins.smartdelay.delayFor(5000);
+      }
+    }
   }
 
   /**

ts/smartclickhouse.classes.timedatatable.ts

@@ -38,34 +38,13 @@ export class TimeDataTable {
       retainDataForDays: 30
     });
 
-    // create table in clickhouse
-    await smartClickHouseDbRefArg.clickhouseClient
-      .queryPromise(`
-        CREATE TABLE IF NOT EXISTS ${newTable.options.tableName} (
-          timestamp DateTime64(3, 'Europe/Berlin'),
-          message String
-        ) ENGINE=MergeTree() ORDER BY timestamp`);
-    
-    // lets adjust the TTL
-    await smartClickHouseDbRefArg.clickhouseClient
-      .queryPromise(`
-        ALTER TABLE ${newTable.options.tableName} MODIFY TTL toDateTime(timestamp) + INTERVAL ${newTable.options.retainDataForDays} DAY
-      `);
-
-    await newTable.updateColumns();
-    console.log(`=======================`)
-    console.log(
-      `table with name "${newTable.options.tableName}" in databse ${newTable.smartClickHouseDbRef.options.database} has the following columns:`
-    );
-    for (const column of newTable.columns) {
-      console.log(`>> ${column.name}: ${column.type}`);
-    }
-    console.log('^^^^^^^^^^^^^^\n');
+    await newTable.setup();
 
     return newTable;
   }
 
   // INSTANCE
+  public healingDeferred: plugins.smartpromise.Deferred<any>;
   public smartClickHouseDbRef: SmartClickHouseDb;
   public options: ITimeDataTableOptions;
 
@@ -74,6 +53,32 @@ export class TimeDataTable {
     this.options = optionsArg;
   }
 
+  public async setup() {
+    // create table in clickhouse
+    await this.smartClickHouseDbRef.clickhouseClient
+      .queryPromise(`
+        CREATE TABLE IF NOT EXISTS ${this.options.tableName} (
+          timestamp DateTime64(3, 'Europe/Berlin'),
+          message String
+        ) ENGINE=MergeTree() ORDER BY timestamp`);
+    
+    // lets adjust the TTL
+    await this.smartClickHouseDbRef.clickhouseClient
+      .queryPromise(`
+        ALTER TABLE ${this.options.tableName} MODIFY TTL toDateTime(timestamp) + INTERVAL ${this.options.retainDataForDays} DAY
+      `);
+
+    await this.updateColumns();
+    console.log(`=======================`)
+    console.log(
+      `table with name "${this.options.tableName}" in database ${this.smartClickHouseDbRef.options.database} has the following columns:`
+    );
+    for (const column of this.columns) {
+      console.log(`>> ${column.name}: ${column.type}`);
+    }
+    console.log('^^^^^^^^^^^^^^\n');
+  }
+
   public columns: IColumnInfo[] = [];
 
   /**
@@ -92,6 +97,7 @@ export class TimeDataTable {
    * stores a json and tries to map it to the nested syntax
    */
   public async addData(dataArg: any) {
+
     // the storageJson
     let storageJson: { [key: string]: any } = {};
     
@@ -164,7 +170,19 @@ export class TimeDataTable {
 
     const result = await this.smartClickHouseDbRef.clickhouseClient.insertPromise(this.options.tableName, [
       storageJson,
-    ]);
+    ]).catch(async () => {
+      if (this.healingDeferred) {
+        return;
+      }
+      this.healingDeferred = plugins.smartpromise.defer();
+      console.log(`Ran into an error. Trying to set up things properly again.`);
+      await this.smartClickHouseDbRef.pingDatabaseUntilAvailable();
+      await this.smartClickHouseDbRef.createDatabase();
+      await this.setup();
+      this.columns = [];
+      this.healingDeferred.resolve();
+      this.healingDeferred = null;
+    });
     return result;
   }
 }

ts/smartclickhouse.plugins.ts

@@ -1,9 +1,13 @@
 // @pushrocks scope
+import * as smartdelay from '@pushrocks/smartdelay';
 import * as smartobject from '@pushrocks/smartobject';
+import * as smartpromise from '@pushrocks/smartpromise';
 import * as smarturl from '@pushrocks/smarturl';
 
 export {
+  smartdelay,
   smartobject,
+  smartpromise,
   smarturl
 }