fix(core): update

This commit is contained in:
2022-08-05 13:31:11 +02:00
parent d03f086c92
commit 7d867ea6ab
10 changed files with 85 additions and 88 deletions

View File

@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@pushrocks/smartclickhouse',
version: '2.0.13',
version: '2.0.14',
description: 'an odm for talking to clickhouse'
}

View File

@ -1,5 +1,3 @@
import * as plugins from './smartclickhouse.plugins.js';
export class ClickhouseDb {
}
export class ClickhouseDb {}

View File

@ -17,7 +17,7 @@ export class ClickhouseHttpClient {
// INSTANCE
public options: IClickhouseHttpClientOptions;
public webrequestInstance = new plugins.webrequest.WebRequest({
logging: false
logging: false,
});
public computedProperties: {
connectionUrl: string;
@ -37,20 +37,25 @@ export class ClickhouseHttpClient {
}
public async ping() {
const ping = await this.webrequestInstance.request(this.computedProperties.connectionUrl.toString(), {
method: 'GET',
timeoutMs: 1000,
});
const ping = await this.webrequestInstance.request(
this.computedProperties.connectionUrl.toString(),
{
method: 'GET',
timeoutMs: 1000,
}
);
return ping.status === 200 ? true : false;
}
public async queryPromise(queryArg: string) {
const returnArray = [];
const response = await this.webrequestInstance.request(`${this.computedProperties.connectionUrl}?query=${encodeURIComponent(queryArg)}`, {
method: 'POST',
headers: this.getHeaders(),
});
const response = await this.webrequestInstance.request(
`${this.computedProperties.connectionUrl}?query=${encodeURIComponent(queryArg)}`,
{
method: 'POST',
headers: this.getHeaders(),
}
);
// console.log('===================');
// console.log(this.computedProperties.connectionUrl);
// console.log(queryArg);
@ -66,22 +71,24 @@ export class ClickhouseHttpClient {
}
} else {
}
return returnArray
return returnArray;
}
public async insertPromise(databaseArg: string, tableArg: string, documents: any[]) {
const queryArg = `INSERT INTO ${databaseArg}.${tableArg} FORMAT JSONEachRow`;
const response = await this.webrequestInstance.request(`${this.computedProperties.connectionUrl}?query=${encodeURIComponent(queryArg)}`, {
method: 'POST',
body: documents.map(docArg => JSON.stringify(docArg)).join('\n'),
headers: this.getHeaders()
});
const response = await this.webrequestInstance.request(
`${this.computedProperties.connectionUrl}?query=${encodeURIComponent(queryArg)}`,
{
method: 'POST',
body: documents.map((docArg) => JSON.stringify(docArg)).join('\n'),
headers: this.getHeaders(),
}
);
return response;
}
private getHeaders() {
const headers: {[key: string]: string} = {}
const headers: { [key: string]: string } = {};
if (this.options.username) {
headers['X-ClickHouse-User'] = this.options.username;
}

View File

@ -39,13 +39,15 @@ export class SmartClickHouseDb {
if (dropOld) {
await this.clickhouseClient.queryPromise(`DROP DATABASE IF EXISTS ${this.options.database}`);
}
await this.clickhouseClient.queryPromise(`CREATE DATABASE IF NOT EXISTS ${this.options.database}`);
await this.clickhouseClient.queryPromise(
`CREATE DATABASE IF NOT EXISTS ${this.options.database}`
);
}
public async pingDatabaseUntilAvailable() {
let available = false;
while(!available) {
available = await this.clickhouseClient.ping().catch(err => {
while (!available) {
available = await this.clickhouseClient.ping().catch((err) => {
return false;
});
if (!available) {

View File

@ -1,7 +1,12 @@
import * as plugins from './smartclickhouse.plugins.js';
import { SmartClickHouseDb } from './smartclickhouse.classes.smartclickhouse.js';
export type TClickhouseColumnDataType = 'String' | "DateTime64(3, 'Europe/Berlin')" | 'Float64' | 'Array(String)' | 'Array(Float64)';
export type TClickhouseColumnDataType =
| 'String'
| "DateTime64(3, 'Europe/Berlin')"
| 'Float64'
| 'Array(String)'
| 'Array(Float64)';
export interface IColumnInfo {
database: string;
table: string;
@ -35,7 +40,7 @@ export class TimeDataTable {
public static async getTable(smartClickHouseDbRefArg: SmartClickHouseDb, tableNameArg: string) {
const newTable = new TimeDataTable(smartClickHouseDbRefArg, {
tableName: tableNameArg,
retainDataForDays: 30
retainDataForDays: 30,
});
await newTable.setup();
@ -55,21 +60,19 @@ export class TimeDataTable {
public async setup() {
// create table in clickhouse
await this.smartClickHouseDbRef.clickhouseClient
.queryPromise(`
await this.smartClickHouseDbRef.clickhouseClient.queryPromise(`
CREATE TABLE IF NOT EXISTS ${this.smartClickHouseDbRef.options.database}.${this.options.tableName} (
timestamp DateTime64(3, 'Europe/Berlin'),
message String
) ENGINE=MergeTree() ORDER BY timestamp`);
// lets adjust the TTL
await this.smartClickHouseDbRef.clickhouseClient
.queryPromise(`
await this.smartClickHouseDbRef.clickhouseClient.queryPromise(`
ALTER TABLE ${this.smartClickHouseDbRef.options.database}.${this.options.tableName} MODIFY TTL toDateTime(timestamp) + INTERVAL ${this.options.retainDataForDays} DAY
`);
await this.updateColumns();
console.log(`=======================`)
console.log(`=======================`);
console.log(
`table with name "${this.options.tableName}" in database ${this.smartClickHouseDbRef.options.database} has the following columns:`
);
@ -103,27 +106,31 @@ export class TimeDataTable {
// the storageJson
let storageJson: { [key: string]: any } = {};
// helper stuff
const getClickhouseTypeForValue = (valueArg: any): TClickhouseColumnDataType => {
const typeConversion: {[key: string]: TClickhouseColumnDataType} = {
const typeConversion: { [key: string]: TClickhouseColumnDataType } = {
string: 'String',
number: 'Float64',
undefined: null,
null: null
null: null,
};
if (valueArg instanceof Array) {
const arrayType = typeConversion[(typeof valueArg[0]) as string];
const arrayType = typeConversion[typeof valueArg[0] as string];
if (!arrayType) {
return null;
} else {
return `Array(${arrayType})` as TClickhouseColumnDataType;
}
}
return typeConversion[(typeof valueArg) as string];
}
const checkPath = async (pathArg: string, typeArg: TClickhouseColumnDataType, prechecked = false) => {
return typeConversion[typeof valueArg as string];
};
const checkPath = async (
pathArg: string,
typeArg: TClickhouseColumnDataType,
prechecked = false
) => {
let columnFound = false;
for (const column of this.columns) {
if (pathArg === column.name) {
@ -137,12 +144,12 @@ export class TimeDataTable {
await checkPath(pathArg, typeArg, true);
return;
}
const alterString = `ALTER TABLE ${this.smartClickHouseDbRef.options.database}.${this.options.tableName} ADD COLUMN ${pathArg} ${typeArg} FIRST`
const alterString = `ALTER TABLE ${this.smartClickHouseDbRef.options.database}.${this.options.tableName} ADD COLUMN ${pathArg} ${typeArg} FIRST`;
try {
await this.smartClickHouseDbRef.clickhouseClient.queryPromise(`
${alterString}
`);
} catch(err) {
} catch (err) {
console.log(alterString);
for (const column of this.columns) {
console.log(column.name);
@ -171,21 +178,23 @@ export class TimeDataTable {
storageJson[key] = value;
}
const result = await this.smartClickHouseDbRef.clickhouseClient.insertPromise(this.smartClickHouseDbRef.options.database, this.options.tableName, [
storageJson,
]).catch(async () => {
if (this.healingDeferred) {
return;
}
this.healingDeferred = plugins.smartpromise.defer();
console.log(`Ran into an error. Trying to set up things properly again.`);
await this.smartClickHouseDbRef.pingDatabaseUntilAvailable();
await this.smartClickHouseDbRef.createDatabase();
await this.setup();
this.columns = [];
this.healingDeferred.resolve();
this.healingDeferred = null;
});
const result = await this.smartClickHouseDbRef.clickhouseClient
.insertPromise(this.smartClickHouseDbRef.options.database, this.options.tableName, [
storageJson,
])
.catch(async () => {
if (this.healingDeferred) {
return;
}
this.healingDeferred = plugins.smartpromise.defer();
console.log(`Ran into an error. Trying to set up things properly again.`);
await this.smartClickHouseDbRef.pingDatabaseUntilAvailable();
await this.smartClickHouseDbRef.createDatabase();
await this.setup();
this.columns = [];
this.healingDeferred.resolve();
this.healingDeferred = null;
});
return result;
}
}

View File

@ -5,10 +5,4 @@ import * as smartpromise from '@pushrocks/smartpromise';
import * as smarturl from '@pushrocks/smarturl';
import * as webrequest from '@pushrocks/webrequest';
export {
smartdelay,
smartobject,
smartpromise,
smarturl,
webrequest,
}
export { smartdelay, smartobject, smartpromise, smarturl, webrequest };