2.0.14

.gitlab-ci.yml

@@ -18,17 +18,6 @@ before_script:
 # ====================
 # security stage
 # ====================
-mirror:
-  stage: security
-  script:
-    - npmci git mirror
-  only:
-    - tags
-  tags:
-    - lossless
-    - docker
-    - notpriv
-
 auditProductionDependencies:
   image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
   stage: security
@@ -100,10 +89,9 @@ codequality:
   only:
     - tags
   script:
-    - npmci command npm install -g tslint typescript
+    - npmci command npm install -g typescript
     - npmci npm prepare
     - npmci npm install
-    - npmci command "tslint -c tslint.json ./ts/**/*.ts"
   tags:
     - lossless
     - docker
@@ -123,11 +111,10 @@ trigger:
 pages:
   stage: metadata
   script:
-    - npmci node install lts
-    - npmci command npm install -g @gitzone/tsdoc
+    - npmci node install stable 
     - npmci npm prepare
     - npmci npm install
-    - npmci command tsdoc
+    - npmci command npm run buildDocs
   tags:
     - lossless
     - docker

.vscode/settings.json

@@ -22,6 +22,5 @@
         }
       }
     }
-  ],
-  "typescript.tsdk": "node_modules/typescript/lib"
+  ]
 }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pushrocks/smartclickhouse",
-  "version": "2.0.5",
+  "version": "2.0.14",
   "private": false,
   "description": "an odm for talking to clickhouse",
   "main": "dist_ts/index.js",
@@ -10,25 +10,26 @@
   "license": "MIT",
   "scripts": {
     "test": "(tstest test/ --web)",
-    "build": "(tsbuild --web)",
+    "build": "(tsbuild --web --allowimplicitany)",
     "createGrafana": "docker run --name grafana -d -p 4000:3000 grafana/grafana-oss",
-    "createClickhouse": "docker run --name some-clickhouse-server --ulimit nofile=262144:262144 -p 8123:8123 -p 9000:9000  --volume=$PWD/.nogit/testdatabase:/var/lib/clickhouse yandex/clickhouse-server"
+    "createClickhouse": "docker run --name some-clickhouse-server --ulimit nofile=262144:262144 -p 8123:8123 -p 9000:9000  --volume=$PWD/.nogit/testdatabase:/var/lib/clickhouse yandex/clickhouse-server",
+    "buildDocs": "tsdoc"
   },
   "devDependencies": {
-    "@gitzone/tsbuild": "^2.1.63",
-    "@gitzone/tsbundle": "^2.0.6",
-    "@gitzone/tstest": "^1.0.72",
+    "@gitzone/tsbuild": "^2.1.65",
+    "@gitzone/tsbundle": "^2.0.7",
+    "@gitzone/tstest": "^1.0.73",
     "@pushrocks/tapbundle": "^5.0.4",
-    "@types/node": "^18.6.1",
+    "@types/node": "^18.6.4",
     "tslint": "^6.1.3",
     "tslint-config-prettier": "^1.15.0"
   },
   "dependencies": {
-    "@depyronick/clickhouse-client": "^1.0.14",
     "@pushrocks/smartdelay": "^2.0.13",
     "@pushrocks/smartobject": "^1.0.10",
     "@pushrocks/smartpromise": "^3.1.7",
-    "@pushrocks/smarturl": "^3.0.1"
+    "@pushrocks/smarturl": "^3.0.2",
+    "@pushrocks/webrequest": "^3.0.12"
   },
   "browserslist": [
     "last 1 chrome versions"

test/test.nonci.ts

@@ -7,6 +7,7 @@ tap.test('first test', async () => {
   testClickhouseDb = new smartclickhouse.SmartClickHouseDb({
     url: 'http://localhost:8123',
     database: 'test2',
+    unref: true,
   });
 });
 
@@ -17,15 +18,15 @@ tap.test('should start the clickhouse db', async () => {
 tap.test('should create a timedatatable', async (toolsArg) => {
   const table = await testClickhouseDb.getTable('analytics');
   let i = 0;
-  while(i < 5000) {
+  while (i < 5000) {
     await table.addData({
       timestamp: Date.now(),
       message: `hello this is a message ${i}`,
       wow: 'hey',
       deep: {
         so: 'hello',
-        myArray: ['array1', 'array2']
-      }
+        myArray: ['array1', 'array2'],
+      },
     });
     i++;
     console.log(`logged ${i} of 5000 lines.`);

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@pushrocks/smartclickhouse',
-  version: '2.0.5',
+  version: '2.0.14',
   description: 'an odm for talking to clickhouse'
 }

ts/smartclickhouse.classes.clickhousedb.ts

@@ -0,0 +1,3 @@
+import * as plugins from './smartclickhouse.plugins.js';
+
+export class ClickhouseDb {}

ts/smartclickhouse.classes.httpclient.ts

@@ -0,0 +1,100 @@
+import * as plugins from './smartclickhouse.plugins.js';
+
+export interface IClickhouseHttpClientOptions {
+  username?: string;
+  password?: string;
+  url: string;
+}
+
+export class ClickhouseHttpClient {
+  // STATIC
+  public static async createAndStart(optionsArg: IClickhouseHttpClientOptions) {
+    const clickhouseHttpInstance = new ClickhouseHttpClient(optionsArg);
+    await clickhouseHttpInstance.start();
+    return clickhouseHttpInstance;
+  }
+
+  // INSTANCE
+  public options: IClickhouseHttpClientOptions;
+  public webrequestInstance = new plugins.webrequest.WebRequest({
+    logging: false,
+  });
+  public computedProperties: {
+    connectionUrl: string;
+    parsedUrl: plugins.smarturl.Smarturl;
+  } = {
+    connectionUrl: null,
+    parsedUrl: null,
+  };
+  constructor(optionsArg: IClickhouseHttpClientOptions) {
+    this.options = optionsArg;
+  }
+
+  public async start() {
+    this.computedProperties.parsedUrl = plugins.smarturl.Smarturl.createFromUrl(this.options.url);
+    console.log(this.computedProperties.parsedUrl);
+    this.computedProperties.connectionUrl = this.computedProperties.parsedUrl.toString();
+  }
+
+  public async ping() {
+    const ping = await this.webrequestInstance.request(
+      this.computedProperties.connectionUrl.toString(),
+      {
+        method: 'GET',
+        timeoutMs: 1000,
+      }
+    );
+    return ping.status === 200 ? true : false;
+  }
+
+  public async queryPromise(queryArg: string) {
+    const returnArray = [];
+    const response = await this.webrequestInstance.request(
+      `${this.computedProperties.connectionUrl}?query=${encodeURIComponent(queryArg)}`,
+      {
+        method: 'POST',
+        headers: this.getHeaders(),
+      }
+    );
+    // console.log('===================');
+    // console.log(this.computedProperties.connectionUrl);
+    // console.log(queryArg);
+    // console.log((await response.clone().text()).split(/\r?\n/))
+    if (response.headers.get('X-ClickHouse-Format') === 'JSONEachRow') {
+      const jsonList = await response.text();
+      const jsonArray = jsonList.split('\n');
+      for (const jsonArg of jsonArray) {
+        if (!jsonArg) {
+          continue;
+        }
+        returnArray.push(JSON.parse(jsonArg));
+      }
+    } else {
+    }
+    return returnArray;
+  }
+
+  public async insertPromise(databaseArg: string, tableArg: string, documents: any[]) {
+    const queryArg = `INSERT INTO ${databaseArg}.${tableArg} FORMAT JSONEachRow`;
+    const response = await this.webrequestInstance.request(
+      `${this.computedProperties.connectionUrl}?query=${encodeURIComponent(queryArg)}`,
+      {
+        method: 'POST',
+        body: documents.map((docArg) => JSON.stringify(docArg)).join('\n'),
+        headers: this.getHeaders(),
+      }
+    );
+    return response;
+  }
+
+  private getHeaders() {
+    const headers: { [key: string]: string } = {};
+    if (this.options.username) {
+      headers['X-ClickHouse-User'] = this.options.username;
+    }
+    if (this.options.password) {
+      headers['X-ClickHouse-Key'] = this.options.password;
+    }
+    return headers;
+  }
+}

ts/smartclickhouse.classes.smartclickhouse.ts

@@ -1,16 +1,23 @@
 import * as plugins from './smartclickhouse.plugins.js';
 import { TimeDataTable } from './smartclickhouse.classes.timedatatable.js';
+import { ClickhouseHttpClient } from './smartclickhouse.classes.httpclient.js';
 
 export interface IClickhouseConstructorOptions {
   url: string;
   database: string;
+  username?: string;
   password?: string;
+  /**
+   * allow services to exit when waiting for clickhouse startup
+   * this allows to leave the lifecycle flow to other processes
+   * like a listening server.
+   */
+  unref?: boolean;
 }
 
 export class SmartClickHouseDb {
   public options: IClickhouseConstructorOptions;
-  public defaultClient: plugins.clickhouse.ClickHouseClient;
-  public clickhouseClient: plugins.clickhouse.ClickHouseClient;
+  public clickhouseClient: ClickhouseHttpClient;
 
   constructor(optionsArg: IClickhouseConstructorOptions) {
     this.options = optionsArg;
@@ -21,48 +28,31 @@ export class SmartClickHouseDb {
    */
   public async start(dropOld = false) {
     console.log(`Connecting to default database first.`);
-    const defaultOptions: {[keyArg: string]: string} = {};
-    // the protocol, url and host
-    const parsedUrl = plugins.smarturl.Smarturl.createFromUrl(this.options.url);
-    parsedUrl.protocol === 'https' ? defaultOptions.protocol = plugins.clickhouse.ClickHouseConnectionProtocol.HTTPS : null;
-    parsedUrl.protocol === 'http' ? defaultOptions.protocol = plugins.clickhouse.ClickHouseConnectionProtocol.HTTP : null;
-    defaultOptions.host = parsedUrl.hostname;
-    defaultOptions.port = parsedUrl.port;
-    // the database
-    defaultOptions.database = this.options.database;
-    // the password
-    this.options.password ? defaultOptions.password = this.options.password : null;
     // lets connect
-    this.defaultClient = new plugins.clickhouse.ClickHouseClient({
-      ...defaultOptions,
-      database: 'default',
-    });
+    this.clickhouseClient = await ClickhouseHttpClient.createAndStart(this.options);
     await this.pingDatabaseUntilAvailable();
     console.log(`Create database ${this.options.database}, if it does not exist...`);
     await this.createDatabase(dropOld);
-
-    console.log(`Ensured database. Now connecting to wanted database: ${this.options.database}`);
-    this.clickhouseClient = new plugins.clickhouse.ClickHouseClient({
-      ...defaultOptions
-    });
   }
 
   public async createDatabase(dropOld: boolean = false) {
     if (dropOld) {
-      await this.defaultClient.queryPromise(`DROP DATABASE IF EXISTS ${this.options.database}`);
+      await this.clickhouseClient.queryPromise(`DROP DATABASE IF EXISTS ${this.options.database}`);
     }
-    await this.defaultClient.queryPromise(`CREATE DATABASE IF NOT EXISTS ${this.options.database}`);
+    await this.clickhouseClient.queryPromise(
+      `CREATE DATABASE IF NOT EXISTS ${this.options.database}`
+    );
   }
 
   public async pingDatabaseUntilAvailable() {
     let available = false;
-    while(!available) {
-      available = await this.defaultClient.ping().catch(err => {
+    while (!available) {
+      available = await this.clickhouseClient.ping().catch((err) => {
         return false;
       });
       if (!available) {
         console.log(`NOT OK: tried pinging ${this.options.url}... Trying again in 5 seconds.`);
-        await plugins.smartdelay.delayFor(5000);
+        await plugins.smartdelay.delayFor(5000, null, this.options.unref);
       }
     }
   }

ts/smartclickhouse.classes.timedatatable.ts

@@ -1,7 +1,12 @@
 import * as plugins from './smartclickhouse.plugins.js';
 import { SmartClickHouseDb } from './smartclickhouse.classes.smartclickhouse.js';
 
-export type TClickhouseColumnDataType = 'String' | "DateTime64(3, 'Europe/Berlin')" | 'Float64' | 'Array(String)' | 'Array(Float64)';
+export type TClickhouseColumnDataType =
+  | 'String'
+  | "DateTime64(3, 'Europe/Berlin')"
+  | 'Float64'
+  | 'Array(String)'
+  | 'Array(Float64)';
 export interface IColumnInfo {
   database: string;
   table: string;
@@ -35,32 +40,10 @@ export class TimeDataTable {
   public static async getTable(smartClickHouseDbRefArg: SmartClickHouseDb, tableNameArg: string) {
     const newTable = new TimeDataTable(smartClickHouseDbRefArg, {
       tableName: tableNameArg,
-      retainDataForDays: 30
+      retainDataForDays: 30,
     });
 
-    // create table in clickhouse
-    await smartClickHouseDbRefArg.clickhouseClient
-      .queryPromise(`
-        CREATE TABLE IF NOT EXISTS ${newTable.options.tableName} (
-          timestamp DateTime64(3, 'Europe/Berlin'),
-          message String
-        ) ENGINE=MergeTree() ORDER BY timestamp`);
-    
-    // lets adjust the TTL
-    await smartClickHouseDbRefArg.clickhouseClient
-      .queryPromise(`
-        ALTER TABLE ${newTable.options.tableName} MODIFY TTL toDateTime(timestamp) + INTERVAL ${newTable.options.retainDataForDays} DAY
-      `);
-
-    await newTable.updateColumns();
-    console.log(`=======================`)
-    console.log(
-      `table with name "${newTable.options.tableName}" in databse ${newTable.smartClickHouseDbRef.options.database} has the following columns:`
-    );
-    for (const column of newTable.columns) {
-      console.log(`>> ${column.name}: ${column.type}`);
-    }
-    console.log('^^^^^^^^^^^^^^\n');
+    await newTable.setup();
 
     return newTable;
   }
@@ -75,6 +58,30 @@ export class TimeDataTable {
     this.options = optionsArg;
   }
 
+  public async setup() {
+    // create table in clickhouse
+    await this.smartClickHouseDbRef.clickhouseClient.queryPromise(`
+        CREATE TABLE IF NOT EXISTS ${this.smartClickHouseDbRef.options.database}.${this.options.tableName} (
+          timestamp DateTime64(3, 'Europe/Berlin'),
+          message String
+        ) ENGINE=MergeTree() ORDER BY timestamp`);
+
+    // lets adjust the TTL
+    await this.smartClickHouseDbRef.clickhouseClient.queryPromise(`
+        ALTER TABLE ${this.smartClickHouseDbRef.options.database}.${this.options.tableName} MODIFY TTL toDateTime(timestamp) + INTERVAL ${this.options.retainDataForDays} DAY
+      `);
+
+    await this.updateColumns();
+    console.log(`=======================`);
+    console.log(
+      `table with name "${this.options.tableName}" in database ${this.smartClickHouseDbRef.options.database} has the following columns:`
+    );
+    for (const column of this.columns) {
+      console.log(`>> ${column.name}: ${column.type}`);
+    }
+    console.log('^^^^^^^^^^^^^^\n');
+  }
+
   public columns: IColumnInfo[] = [];
 
   /**
@@ -84,7 +91,7 @@ export class TimeDataTable {
     this.columns = await this.smartClickHouseDbRef.clickhouseClient.queryPromise(`
       SELECT * FROM system.columns
       WHERE database LIKE '${this.smartClickHouseDbRef.options.database}'
-      AND table LIKE '${this.options.tableName}'
+      AND table LIKE '${this.options.tableName}' FORMAT JSONEachRow
     `);
     return this.columns;
   }
@@ -93,30 +100,37 @@ export class TimeDataTable {
    * stores a json and tries to map it to the nested syntax
    */
   public async addData(dataArg: any) {
+    if (this.healingDeferred) {
+      return;
+    }
 
     // the storageJson
     let storageJson: { [key: string]: any } = {};
-    
+
     // helper stuff
-    
+
     const getClickhouseTypeForValue = (valueArg: any): TClickhouseColumnDataType => {
-      const typeConversion: {[key: string]: TClickhouseColumnDataType} = {
+      const typeConversion: { [key: string]: TClickhouseColumnDataType } = {
         string: 'String',
         number: 'Float64',
         undefined: null,
-        null: null
+        null: null,
       };
       if (valueArg instanceof Array) {
-        const arrayType = typeConversion[(typeof valueArg[0]) as string];
+        const arrayType = typeConversion[typeof valueArg[0] as string];
         if (!arrayType) {
           return null;
         } else {
           return `Array(${arrayType})` as TClickhouseColumnDataType;
         }
       }
-      return typeConversion[(typeof valueArg) as string];
-    }
-    const checkPath = async (pathArg: string, typeArg: TClickhouseColumnDataType, prechecked = false) => {
+      return typeConversion[typeof valueArg as string];
+    };
+    const checkPath = async (
+      pathArg: string,
+      typeArg: TClickhouseColumnDataType,
+      prechecked = false
+    ) => {
       let columnFound = false;
       for (const column of this.columns) {
         if (pathArg === column.name) {
@@ -130,12 +144,12 @@ export class TimeDataTable {
           await checkPath(pathArg, typeArg, true);
           return;
         }
-        const alterString = `ALTER TABLE ${this.options.tableName} ADD COLUMN ${pathArg} ${typeArg} FIRST`
+        const alterString = `ALTER TABLE ${this.smartClickHouseDbRef.options.database}.${this.options.tableName} ADD COLUMN ${pathArg} ${typeArg} FIRST`;
         try {
           await this.smartClickHouseDbRef.clickhouseClient.queryPromise(`
           ${alterString}
         `);
-        } catch(err) {
+        } catch (err) {
           console.log(alterString);
           for (const column of this.columns) {
             console.log(column.name);
@@ -164,20 +178,23 @@ export class TimeDataTable {
       storageJson[key] = value;
     }
 
-    const result = await this.smartClickHouseDbRef.clickhouseClient.insertPromise(this.options.tableName, [
-      storageJson,
-    ]).catch(async () => {
-      if (this.healingDeferred) {
-        return;
-      }
-      this.healingDeferred = plugins.smartpromise.defer();
-      console.log(`Ran into an error. Trying to set up things properly again.`);
-      await this.smartClickHouseDbRef.pingDatabaseUntilAvailable();
-      await this.smartClickHouseDbRef.createDatabase();
-      this.columns = [];
-      this.healingDeferred.resolve();
-      this.healingDeferred = null;
-    });
+    const result = await this.smartClickHouseDbRef.clickhouseClient
+      .insertPromise(this.smartClickHouseDbRef.options.database, this.options.tableName, [
+        storageJson,
+      ])
+      .catch(async () => {
+        if (this.healingDeferred) {
+          return;
+        }
+        this.healingDeferred = plugins.smartpromise.defer();
+        console.log(`Ran into an error. Trying to set up things properly again.`);
+        await this.smartClickHouseDbRef.pingDatabaseUntilAvailable();
+        await this.smartClickHouseDbRef.createDatabase();
+        await this.setup();
+        this.columns = [];
+        this.healingDeferred.resolve();
+        this.healingDeferred = null;
+      });
     return result;
   }
 }

ts/smartclickhouse.plugins.ts

@@ -3,15 +3,6 @@ import * as smartdelay from '@pushrocks/smartdelay';
 import * as smartobject from '@pushrocks/smartobject';
 import * as smartpromise from '@pushrocks/smartpromise';
 import * as smarturl from '@pushrocks/smarturl';
+import * as webrequest from '@pushrocks/webrequest';
 
-export {
-  smartdelay,
-  smartobject,
-  smartpromise,
-  smarturl
-}
-
-// thirdparty
-import * as clickhouse from '@depyronick/clickhouse-client';
-
-export { clickhouse };
+export { smartdelay, smartobject, smartpromise, smarturl, webrequest };