diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a70f7eb..69b3e08 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,17 +18,6 @@ before_script: # ==================== # security stage # ==================== -mirror: - stage: security - script: - - npmci git mirror - only: - - tags - tags: - - lossless - - docker - - notpriv - auditProductionDependencies: image: registry.gitlab.com/hosttoday/ht-docker-node:npmci stage: security @@ -100,10 +89,9 @@ codequality: only: - tags script: - - npmci command npm install -g tslint typescript + - npmci command npm install -g typescript - npmci npm prepare - npmci npm install - - npmci command "tslint -c tslint.json ./ts/**/*.ts" tags: - lossless - docker @@ -123,11 +111,10 @@ trigger: pages: stage: metadata script: - - npmci node install lts - - npmci command npm install -g @gitzone/tsdoc + - npmci node install stable - npmci npm prepare - npmci npm install - - npmci command tsdoc + - npmci command npm run buildDocs tags: - lossless - docker diff --git a/.vscode/settings.json b/.vscode/settings.json index 2868e45..3648eaa 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -22,6 +22,5 @@ } } } - ], - "typescript.tsdk": "node_modules/typescript/lib" + ] } diff --git a/package.json b/package.json index 2a2f547..d6aaba6 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,8 @@ "test": "(tstest test/ --web)", "build": "(tsbuild --web --allowimplicitany)", "createGrafana": "docker run --name grafana -d -p 4000:3000 grafana/grafana-oss", - "createClickhouse": "docker run --name some-clickhouse-server --ulimit nofile=262144:262144 -p 8123:8123 -p 9000:9000 --volume=$PWD/.nogit/testdatabase:/var/lib/clickhouse yandex/clickhouse-server" + "createClickhouse": "docker run --name some-clickhouse-server --ulimit nofile=262144:262144 -p 8123:8123 -p 9000:9000 --volume=$PWD/.nogit/testdatabase:/var/lib/clickhouse yandex/clickhouse-server", + "buildDocs": "tsdoc" }, "devDependencies": { "@gitzone/tsbuild": "^2.1.65", @@ -45,4 +46,4 @@ "npmextra.json", "readme.md" ] -} +} \ No newline at end of file diff --git a/test/test.nonci.ts b/test/test.nonci.ts index e05e46e..890b866 100644 --- a/test/test.nonci.ts +++ b/test/test.nonci.ts @@ -18,15 +18,15 @@ tap.test('should start the clickhouse db', async () => { tap.test('should create a timedatatable', async (toolsArg) => { const table = await testClickhouseDb.getTable('analytics'); let i = 0; - while(i < 5000) { + while (i < 5000) { await table.addData({ timestamp: Date.now(), message: `hello this is a message ${i}`, wow: 'hey', deep: { so: 'hello', - myArray: ['array1', 'array2'] - } + myArray: ['array1', 'array2'], + }, }); i++; console.log(`logged ${i} of 5000 lines.`); diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index 9841645..f7a25c9 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@pushrocks/smartclickhouse', - version: '2.0.13', + version: '2.0.14', description: 'an odm for talking to clickhouse' } diff --git a/ts/smartclickhouse.classes.clickhousedb.ts b/ts/smartclickhouse.classes.clickhousedb.ts index 6e0107f..8407536 100644 --- a/ts/smartclickhouse.classes.clickhousedb.ts +++ b/ts/smartclickhouse.classes.clickhousedb.ts @@ -1,5 +1,3 @@ import * as plugins from './smartclickhouse.plugins.js'; -export class ClickhouseDb { - -} \ No newline at end of file +export class ClickhouseDb {} diff --git a/ts/smartclickhouse.classes.httpclient.ts b/ts/smartclickhouse.classes.httpclient.ts index d579092..4636b21 100644 --- a/ts/smartclickhouse.classes.httpclient.ts +++ b/ts/smartclickhouse.classes.httpclient.ts @@ -17,7 +17,7 @@ export class ClickhouseHttpClient { // INSTANCE public options: IClickhouseHttpClientOptions; public webrequestInstance = new plugins.webrequest.WebRequest({ - logging: false + logging: false, }); public computedProperties: { connectionUrl: string; @@ -37,20 +37,25 @@ export class ClickhouseHttpClient { } public async ping() { - - const ping = await this.webrequestInstance.request(this.computedProperties.connectionUrl.toString(), { - method: 'GET', - timeoutMs: 1000, - }); + const ping = await this.webrequestInstance.request( + this.computedProperties.connectionUrl.toString(), + { + method: 'GET', + timeoutMs: 1000, + } + ); return ping.status === 200 ? true : false; } public async queryPromise(queryArg: string) { const returnArray = []; - const response = await this.webrequestInstance.request(`${this.computedProperties.connectionUrl}?query=${encodeURIComponent(queryArg)}`, { - method: 'POST', - headers: this.getHeaders(), - }); + const response = await this.webrequestInstance.request( + `${this.computedProperties.connectionUrl}?query=${encodeURIComponent(queryArg)}`, + { + method: 'POST', + headers: this.getHeaders(), + } + ); // console.log('==================='); // console.log(this.computedProperties.connectionUrl); // console.log(queryArg); @@ -66,22 +71,24 @@ export class ClickhouseHttpClient { } } else { } - return returnArray - + return returnArray; } public async insertPromise(databaseArg: string, tableArg: string, documents: any[]) { const queryArg = `INSERT INTO ${databaseArg}.${tableArg} FORMAT JSONEachRow`; - const response = await this.webrequestInstance.request(`${this.computedProperties.connectionUrl}?query=${encodeURIComponent(queryArg)}`, { - method: 'POST', - body: documents.map(docArg => JSON.stringify(docArg)).join('\n'), - headers: this.getHeaders() - }); + const response = await this.webrequestInstance.request( + `${this.computedProperties.connectionUrl}?query=${encodeURIComponent(queryArg)}`, + { + method: 'POST', + body: documents.map((docArg) => JSON.stringify(docArg)).join('\n'), + headers: this.getHeaders(), + } + ); return response; } private getHeaders() { - const headers: {[key: string]: string} = {} + const headers: { [key: string]: string } = {}; if (this.options.username) { headers['X-ClickHouse-User'] = this.options.username; } diff --git a/ts/smartclickhouse.classes.smartclickhouse.ts b/ts/smartclickhouse.classes.smartclickhouse.ts index 140c9f6..fe7e9ac 100644 --- a/ts/smartclickhouse.classes.smartclickhouse.ts +++ b/ts/smartclickhouse.classes.smartclickhouse.ts @@ -39,13 +39,15 @@ export class SmartClickHouseDb { if (dropOld) { await this.clickhouseClient.queryPromise(`DROP DATABASE IF EXISTS ${this.options.database}`); } - await this.clickhouseClient.queryPromise(`CREATE DATABASE IF NOT EXISTS ${this.options.database}`); + await this.clickhouseClient.queryPromise( + `CREATE DATABASE IF NOT EXISTS ${this.options.database}` + ); } public async pingDatabaseUntilAvailable() { let available = false; - while(!available) { - available = await this.clickhouseClient.ping().catch(err => { + while (!available) { + available = await this.clickhouseClient.ping().catch((err) => { return false; }); if (!available) { diff --git a/ts/smartclickhouse.classes.timedatatable.ts b/ts/smartclickhouse.classes.timedatatable.ts index fd78942..a963532 100644 --- a/ts/smartclickhouse.classes.timedatatable.ts +++ b/ts/smartclickhouse.classes.timedatatable.ts @@ -1,7 +1,12 @@ import * as plugins from './smartclickhouse.plugins.js'; import { SmartClickHouseDb } from './smartclickhouse.classes.smartclickhouse.js'; -export type TClickhouseColumnDataType = 'String' | "DateTime64(3, 'Europe/Berlin')" | 'Float64' | 'Array(String)' | 'Array(Float64)'; +export type TClickhouseColumnDataType = + | 'String' + | "DateTime64(3, 'Europe/Berlin')" + | 'Float64' + | 'Array(String)' + | 'Array(Float64)'; export interface IColumnInfo { database: string; table: string; @@ -35,7 +40,7 @@ export class TimeDataTable { public static async getTable(smartClickHouseDbRefArg: SmartClickHouseDb, tableNameArg: string) { const newTable = new TimeDataTable(smartClickHouseDbRefArg, { tableName: tableNameArg, - retainDataForDays: 30 + retainDataForDays: 30, }); await newTable.setup(); @@ -55,21 +60,19 @@ export class TimeDataTable { public async setup() { // create table in clickhouse - await this.smartClickHouseDbRef.clickhouseClient - .queryPromise(` + await this.smartClickHouseDbRef.clickhouseClient.queryPromise(` CREATE TABLE IF NOT EXISTS ${this.smartClickHouseDbRef.options.database}.${this.options.tableName} ( timestamp DateTime64(3, 'Europe/Berlin'), message String ) ENGINE=MergeTree() ORDER BY timestamp`); - + // lets adjust the TTL - await this.smartClickHouseDbRef.clickhouseClient - .queryPromise(` + await this.smartClickHouseDbRef.clickhouseClient.queryPromise(` ALTER TABLE ${this.smartClickHouseDbRef.options.database}.${this.options.tableName} MODIFY TTL toDateTime(timestamp) + INTERVAL ${this.options.retainDataForDays} DAY `); await this.updateColumns(); - console.log(`=======================`) + console.log(`=======================`); console.log( `table with name "${this.options.tableName}" in database ${this.smartClickHouseDbRef.options.database} has the following columns:` ); @@ -103,27 +106,31 @@ export class TimeDataTable { // the storageJson let storageJson: { [key: string]: any } = {}; - + // helper stuff - + const getClickhouseTypeForValue = (valueArg: any): TClickhouseColumnDataType => { - const typeConversion: {[key: string]: TClickhouseColumnDataType} = { + const typeConversion: { [key: string]: TClickhouseColumnDataType } = { string: 'String', number: 'Float64', undefined: null, - null: null + null: null, }; if (valueArg instanceof Array) { - const arrayType = typeConversion[(typeof valueArg[0]) as string]; + const arrayType = typeConversion[typeof valueArg[0] as string]; if (!arrayType) { return null; } else { return `Array(${arrayType})` as TClickhouseColumnDataType; } } - return typeConversion[(typeof valueArg) as string]; - } - const checkPath = async (pathArg: string, typeArg: TClickhouseColumnDataType, prechecked = false) => { + return typeConversion[typeof valueArg as string]; + }; + const checkPath = async ( + pathArg: string, + typeArg: TClickhouseColumnDataType, + prechecked = false + ) => { let columnFound = false; for (const column of this.columns) { if (pathArg === column.name) { @@ -137,12 +144,12 @@ export class TimeDataTable { await checkPath(pathArg, typeArg, true); return; } - const alterString = `ALTER TABLE ${this.smartClickHouseDbRef.options.database}.${this.options.tableName} ADD COLUMN ${pathArg} ${typeArg} FIRST` + const alterString = `ALTER TABLE ${this.smartClickHouseDbRef.options.database}.${this.options.tableName} ADD COLUMN ${pathArg} ${typeArg} FIRST`; try { await this.smartClickHouseDbRef.clickhouseClient.queryPromise(` ${alterString} `); - } catch(err) { + } catch (err) { console.log(alterString); for (const column of this.columns) { console.log(column.name); @@ -171,21 +178,23 @@ export class TimeDataTable { storageJson[key] = value; } - const result = await this.smartClickHouseDbRef.clickhouseClient.insertPromise(this.smartClickHouseDbRef.options.database, this.options.tableName, [ - storageJson, - ]).catch(async () => { - if (this.healingDeferred) { - return; - } - this.healingDeferred = plugins.smartpromise.defer(); - console.log(`Ran into an error. Trying to set up things properly again.`); - await this.smartClickHouseDbRef.pingDatabaseUntilAvailable(); - await this.smartClickHouseDbRef.createDatabase(); - await this.setup(); - this.columns = []; - this.healingDeferred.resolve(); - this.healingDeferred = null; - }); + const result = await this.smartClickHouseDbRef.clickhouseClient + .insertPromise(this.smartClickHouseDbRef.options.database, this.options.tableName, [ + storageJson, + ]) + .catch(async () => { + if (this.healingDeferred) { + return; + } + this.healingDeferred = plugins.smartpromise.defer(); + console.log(`Ran into an error. Trying to set up things properly again.`); + await this.smartClickHouseDbRef.pingDatabaseUntilAvailable(); + await this.smartClickHouseDbRef.createDatabase(); + await this.setup(); + this.columns = []; + this.healingDeferred.resolve(); + this.healingDeferred = null; + }); return result; } } diff --git a/ts/smartclickhouse.plugins.ts b/ts/smartclickhouse.plugins.ts index 74e7f4c..b77ca02 100644 --- a/ts/smartclickhouse.plugins.ts +++ b/ts/smartclickhouse.plugins.ts @@ -5,10 +5,4 @@ import * as smartpromise from '@pushrocks/smartpromise'; import * as smarturl from '@pushrocks/smarturl'; import * as webrequest from '@pushrocks/webrequest'; -export { - smartdelay, - smartobject, - smartpromise, - smarturl, - webrequest, -} +export { smartdelay, smartobject, smartpromise, smarturl, webrequest };