diff --git a/package-lock.json b/package-lock.json index 2a2a4ee..ba06d26 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,9 @@ "license": "MIT", "dependencies": { "@depyronick/clickhouse-client": "^1.0.14", + "@pushrocks/smartdelay": "^2.0.13", "@pushrocks/smartobject": "^1.0.10", + "@pushrocks/smartpromise": "^3.1.7", "@pushrocks/smarturl": "^3.0.1" }, "devDependencies": { @@ -921,7 +923,6 @@ "version": "2.0.13", "resolved": "https://verdaccio.lossless.one/@pushrocks%2fsmartdelay/-/smartdelay-2.0.13.tgz", "integrity": "sha512-s6Wh0BHWMfZ5VYONQwpxOYX1JeC9RKA0O9TxEzfZ6FCw2oNQb2QUPCixT9rsceKwva4+atKRw/RfU+Z7aJDmsA==", - "dev": true, "license": "MIT", "dependencies": { "@pushrocks/smartpromise": "^3.0.6" @@ -1281,7 +1282,6 @@ "version": "3.1.7", "resolved": "https://verdaccio.lossless.one/@pushrocks%2fsmartpromise/-/smartpromise-3.1.7.tgz", "integrity": "sha512-2gLQCeviEJwZ+cHHtK2Ks98brZatGC6dPXKIs1tVgJsiNgRFjnp90fESuJ1Pmoe7RrS+7J3mO4NtsFHAJJ/y5w==", - "dev": true, "license": "MIT" }, "node_modules/@pushrocks/smartpuppeteer": { @@ -9575,7 +9575,6 @@ "version": "2.0.13", "resolved": "https://verdaccio.lossless.one/@pushrocks%2fsmartdelay/-/smartdelay-2.0.13.tgz", "integrity": "sha512-s6Wh0BHWMfZ5VYONQwpxOYX1JeC9RKA0O9TxEzfZ6FCw2oNQb2QUPCixT9rsceKwva4+atKRw/RfU+Z7aJDmsA==", - "dev": true, "requires": { "@pushrocks/smartpromise": "^3.0.6" } @@ -9912,8 +9911,7 @@ "@pushrocks/smartpromise": { "version": "3.1.7", "resolved": "https://verdaccio.lossless.one/@pushrocks%2fsmartpromise/-/smartpromise-3.1.7.tgz", - "integrity": "sha512-2gLQCeviEJwZ+cHHtK2Ks98brZatGC6dPXKIs1tVgJsiNgRFjnp90fESuJ1Pmoe7RrS+7J3mO4NtsFHAJJ/y5w==", - "dev": true + "integrity": "sha512-2gLQCeviEJwZ+cHHtK2Ks98brZatGC6dPXKIs1tVgJsiNgRFjnp90fESuJ1Pmoe7RrS+7J3mO4NtsFHAJJ/y5w==" }, "@pushrocks/smartpuppeteer": { "version": "2.0.0", diff --git a/package.json b/package.json index 1af67e0..76e4f31 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,9 @@ }, "dependencies": { "@depyronick/clickhouse-client": "^1.0.14", + "@pushrocks/smartdelay": "^2.0.13", "@pushrocks/smartobject": "^1.0.10", + "@pushrocks/smartpromise": "^3.1.7", "@pushrocks/smarturl": "^3.0.1" }, "browserslist": [ diff --git a/test/test.nonci.ts b/test/test.nonci.ts index 7ee3f3d..86a99ca 100644 --- a/test/test.nonci.ts +++ b/test/test.nonci.ts @@ -5,10 +5,8 @@ let testClickhouseDb: smartclickhouse.SmartClickHouseDb; tap.test('first test', async () => { testClickhouseDb = new smartclickhouse.SmartClickHouseDb({ - host: 'localhost', + url: 'http://localhost:8123', database: 'test2', - port: 8123, - protocol: 'http', }); }); @@ -16,10 +14,10 @@ tap.test('should start the clickhouse db', async () => { await testClickhouseDb.start(true); }); -tap.test('should create a timedatatable', async () => { +tap.test('should create a timedatatable', async (toolsArg) => { const table = await testClickhouseDb.getTable('analytics'); let i = 0; - while(i < 10) { + while(i < 5000) { await table.addData({ timestamp: Date.now(), message: `hello this is a message ${i}`, @@ -30,6 +28,8 @@ tap.test('should create a timedatatable', async () => { } }); i++; + console.log(`logged ${i} of 5000 lines.`); + await toolsArg.delayFor(1); } }); diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index 5558d08..92326c3 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@pushrocks/smartclickhouse', - version: '2.0.4', + version: '2.0.5', description: 'an odm for talking to clickhouse' } diff --git a/ts/smartclickhouse.classes.smartclickhouse.ts b/ts/smartclickhouse.classes.smartclickhouse.ts index 61c2e31..cff3721 100644 --- a/ts/smartclickhouse.classes.smartclickhouse.ts +++ b/ts/smartclickhouse.classes.smartclickhouse.ts @@ -9,6 +9,7 @@ export interface IClickhouseConstructorOptions { export class SmartClickHouseDb { public options: IClickhouseConstructorOptions; + public defaultClient: plugins.clickhouse.ClickHouseClient; public clickhouseClient: plugins.clickhouse.ClickHouseClient; constructor(optionsArg: IClickhouseConstructorOptions) { @@ -32,23 +33,38 @@ export class SmartClickHouseDb { // the password this.options.password ? defaultOptions.password = this.options.password : null; // lets connect - const defaultClient = new plugins.clickhouse.ClickHouseClient({ + this.defaultClient = new plugins.clickhouse.ClickHouseClient({ ...defaultOptions, database: 'default', }); + await this.pingDatabaseUntilAvailable(); console.log(`Create database ${this.options.database}, if it does not exist...`); - if (dropOld) { - await defaultClient.queryPromise(`DROP DATABASE IF EXISTS ${this.options.database}`); - } - await defaultClient.queryPromise(`CREATE DATABASE IF NOT EXISTS ${this.options.database}`); + await this.createDatabase(dropOld); console.log(`Ensured database. Now connecting to wanted database: ${this.options.database}`); this.clickhouseClient = new plugins.clickhouse.ClickHouseClient({ ...defaultOptions }); - console.log(`trying to ping database...`); - const result = await this.clickhouseClient.ping(); - console.log(`Ping successfull?: ${result}`); + } + + public async createDatabase(dropOld: boolean = false) { + if (dropOld) { + await this.defaultClient.queryPromise(`DROP DATABASE IF EXISTS ${this.options.database}`); + } + await this.defaultClient.queryPromise(`CREATE DATABASE IF NOT EXISTS ${this.options.database}`); + } + + public async pingDatabaseUntilAvailable() { + let available = false; + while(!available) { + available = await this.defaultClient.ping().catch(err => { + return false; + }); + if (!available) { + console.log(`NOT OK: tried pinging ${this.options.url}... Trying again in 5 seconds.`); + await plugins.smartdelay.delayFor(5000); + } + } } /** diff --git a/ts/smartclickhouse.classes.timedatatable.ts b/ts/smartclickhouse.classes.timedatatable.ts index 3c770af..8748587 100644 --- a/ts/smartclickhouse.classes.timedatatable.ts +++ b/ts/smartclickhouse.classes.timedatatable.ts @@ -66,6 +66,7 @@ export class TimeDataTable { } // INSTANCE + public healingDeferred: plugins.smartpromise.Deferred; public smartClickHouseDbRef: SmartClickHouseDb; public options: ITimeDataTableOptions; @@ -92,6 +93,7 @@ export class TimeDataTable { * stores a json and tries to map it to the nested syntax */ public async addData(dataArg: any) { + // the storageJson let storageJson: { [key: string]: any } = {}; @@ -164,7 +166,18 @@ export class TimeDataTable { const result = await this.smartClickHouseDbRef.clickhouseClient.insertPromise(this.options.tableName, [ storageJson, - ]); + ]).catch(async () => { + if (this.healingDeferred) { + return; + } + this.healingDeferred = plugins.smartpromise.defer(); + console.log(`Ran into an error. Trying to set up things properly again.`); + await this.smartClickHouseDbRef.pingDatabaseUntilAvailable(); + await this.smartClickHouseDbRef.createDatabase(); + this.columns = []; + this.healingDeferred.resolve(); + this.healingDeferred = null; + }); return result; } } diff --git a/ts/smartclickhouse.plugins.ts b/ts/smartclickhouse.plugins.ts index c3ab069..24f5e04 100644 --- a/ts/smartclickhouse.plugins.ts +++ b/ts/smartclickhouse.plugins.ts @@ -1,9 +1,13 @@ // @pushrocks scope +import * as smartdelay from '@pushrocks/smartdelay'; import * as smartobject from '@pushrocks/smartobject'; +import * as smartpromise from '@pushrocks/smartpromise'; import * as smarturl from '@pushrocks/smarturl'; export { + smartdelay, smartobject, + smartpromise, smarturl }