fix(core): update
This commit is contained in:
@ -3,6 +3,6 @@
|
||||
*/
|
||||
export const commitinfo = {
|
||||
name: '@pushrocks/smartclickhouse',
|
||||
version: '2.0.13',
|
||||
version: '2.0.14',
|
||||
description: 'an odm for talking to clickhouse'
|
||||
}
|
||||
|
@ -1,5 +1,3 @@
|
||||
import * as plugins from './smartclickhouse.plugins.js';
|
||||
|
||||
export class ClickhouseDb {
|
||||
|
||||
}
|
||||
export class ClickhouseDb {}
|
||||
|
@ -17,7 +17,7 @@ export class ClickhouseHttpClient {
|
||||
// INSTANCE
|
||||
public options: IClickhouseHttpClientOptions;
|
||||
public webrequestInstance = new plugins.webrequest.WebRequest({
|
||||
logging: false
|
||||
logging: false,
|
||||
});
|
||||
public computedProperties: {
|
||||
connectionUrl: string;
|
||||
@ -37,20 +37,25 @@ export class ClickhouseHttpClient {
|
||||
}
|
||||
|
||||
public async ping() {
|
||||
|
||||
const ping = await this.webrequestInstance.request(this.computedProperties.connectionUrl.toString(), {
|
||||
method: 'GET',
|
||||
timeoutMs: 1000,
|
||||
});
|
||||
const ping = await this.webrequestInstance.request(
|
||||
this.computedProperties.connectionUrl.toString(),
|
||||
{
|
||||
method: 'GET',
|
||||
timeoutMs: 1000,
|
||||
}
|
||||
);
|
||||
return ping.status === 200 ? true : false;
|
||||
}
|
||||
|
||||
public async queryPromise(queryArg: string) {
|
||||
const returnArray = [];
|
||||
const response = await this.webrequestInstance.request(`${this.computedProperties.connectionUrl}?query=${encodeURIComponent(queryArg)}`, {
|
||||
method: 'POST',
|
||||
headers: this.getHeaders(),
|
||||
});
|
||||
const response = await this.webrequestInstance.request(
|
||||
`${this.computedProperties.connectionUrl}?query=${encodeURIComponent(queryArg)}`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: this.getHeaders(),
|
||||
}
|
||||
);
|
||||
// console.log('===================');
|
||||
// console.log(this.computedProperties.connectionUrl);
|
||||
// console.log(queryArg);
|
||||
@ -66,22 +71,24 @@ export class ClickhouseHttpClient {
|
||||
}
|
||||
} else {
|
||||
}
|
||||
return returnArray
|
||||
|
||||
return returnArray;
|
||||
}
|
||||
|
||||
public async insertPromise(databaseArg: string, tableArg: string, documents: any[]) {
|
||||
const queryArg = `INSERT INTO ${databaseArg}.${tableArg} FORMAT JSONEachRow`;
|
||||
const response = await this.webrequestInstance.request(`${this.computedProperties.connectionUrl}?query=${encodeURIComponent(queryArg)}`, {
|
||||
method: 'POST',
|
||||
body: documents.map(docArg => JSON.stringify(docArg)).join('\n'),
|
||||
headers: this.getHeaders()
|
||||
});
|
||||
const response = await this.webrequestInstance.request(
|
||||
`${this.computedProperties.connectionUrl}?query=${encodeURIComponent(queryArg)}`,
|
||||
{
|
||||
method: 'POST',
|
||||
body: documents.map((docArg) => JSON.stringify(docArg)).join('\n'),
|
||||
headers: this.getHeaders(),
|
||||
}
|
||||
);
|
||||
return response;
|
||||
}
|
||||
|
||||
private getHeaders() {
|
||||
const headers: {[key: string]: string} = {}
|
||||
const headers: { [key: string]: string } = {};
|
||||
if (this.options.username) {
|
||||
headers['X-ClickHouse-User'] = this.options.username;
|
||||
}
|
||||
|
@ -39,13 +39,15 @@ export class SmartClickHouseDb {
|
||||
if (dropOld) {
|
||||
await this.clickhouseClient.queryPromise(`DROP DATABASE IF EXISTS ${this.options.database}`);
|
||||
}
|
||||
await this.clickhouseClient.queryPromise(`CREATE DATABASE IF NOT EXISTS ${this.options.database}`);
|
||||
await this.clickhouseClient.queryPromise(
|
||||
`CREATE DATABASE IF NOT EXISTS ${this.options.database}`
|
||||
);
|
||||
}
|
||||
|
||||
public async pingDatabaseUntilAvailable() {
|
||||
let available = false;
|
||||
while(!available) {
|
||||
available = await this.clickhouseClient.ping().catch(err => {
|
||||
while (!available) {
|
||||
available = await this.clickhouseClient.ping().catch((err) => {
|
||||
return false;
|
||||
});
|
||||
if (!available) {
|
||||
|
@ -1,7 +1,12 @@
|
||||
import * as plugins from './smartclickhouse.plugins.js';
|
||||
import { SmartClickHouseDb } from './smartclickhouse.classes.smartclickhouse.js';
|
||||
|
||||
export type TClickhouseColumnDataType = 'String' | "DateTime64(3, 'Europe/Berlin')" | 'Float64' | 'Array(String)' | 'Array(Float64)';
|
||||
export type TClickhouseColumnDataType =
|
||||
| 'String'
|
||||
| "DateTime64(3, 'Europe/Berlin')"
|
||||
| 'Float64'
|
||||
| 'Array(String)'
|
||||
| 'Array(Float64)';
|
||||
export interface IColumnInfo {
|
||||
database: string;
|
||||
table: string;
|
||||
@ -35,7 +40,7 @@ export class TimeDataTable {
|
||||
public static async getTable(smartClickHouseDbRefArg: SmartClickHouseDb, tableNameArg: string) {
|
||||
const newTable = new TimeDataTable(smartClickHouseDbRefArg, {
|
||||
tableName: tableNameArg,
|
||||
retainDataForDays: 30
|
||||
retainDataForDays: 30,
|
||||
});
|
||||
|
||||
await newTable.setup();
|
||||
@ -55,21 +60,19 @@ export class TimeDataTable {
|
||||
|
||||
public async setup() {
|
||||
// create table in clickhouse
|
||||
await this.smartClickHouseDbRef.clickhouseClient
|
||||
.queryPromise(`
|
||||
await this.smartClickHouseDbRef.clickhouseClient.queryPromise(`
|
||||
CREATE TABLE IF NOT EXISTS ${this.smartClickHouseDbRef.options.database}.${this.options.tableName} (
|
||||
timestamp DateTime64(3, 'Europe/Berlin'),
|
||||
message String
|
||||
) ENGINE=MergeTree() ORDER BY timestamp`);
|
||||
|
||||
|
||||
// lets adjust the TTL
|
||||
await this.smartClickHouseDbRef.clickhouseClient
|
||||
.queryPromise(`
|
||||
await this.smartClickHouseDbRef.clickhouseClient.queryPromise(`
|
||||
ALTER TABLE ${this.smartClickHouseDbRef.options.database}.${this.options.tableName} MODIFY TTL toDateTime(timestamp) + INTERVAL ${this.options.retainDataForDays} DAY
|
||||
`);
|
||||
|
||||
await this.updateColumns();
|
||||
console.log(`=======================`)
|
||||
console.log(`=======================`);
|
||||
console.log(
|
||||
`table with name "${this.options.tableName}" in database ${this.smartClickHouseDbRef.options.database} has the following columns:`
|
||||
);
|
||||
@ -103,27 +106,31 @@ export class TimeDataTable {
|
||||
|
||||
// the storageJson
|
||||
let storageJson: { [key: string]: any } = {};
|
||||
|
||||
|
||||
// helper stuff
|
||||
|
||||
|
||||
const getClickhouseTypeForValue = (valueArg: any): TClickhouseColumnDataType => {
|
||||
const typeConversion: {[key: string]: TClickhouseColumnDataType} = {
|
||||
const typeConversion: { [key: string]: TClickhouseColumnDataType } = {
|
||||
string: 'String',
|
||||
number: 'Float64',
|
||||
undefined: null,
|
||||
null: null
|
||||
null: null,
|
||||
};
|
||||
if (valueArg instanceof Array) {
|
||||
const arrayType = typeConversion[(typeof valueArg[0]) as string];
|
||||
const arrayType = typeConversion[typeof valueArg[0] as string];
|
||||
if (!arrayType) {
|
||||
return null;
|
||||
} else {
|
||||
return `Array(${arrayType})` as TClickhouseColumnDataType;
|
||||
}
|
||||
}
|
||||
return typeConversion[(typeof valueArg) as string];
|
||||
}
|
||||
const checkPath = async (pathArg: string, typeArg: TClickhouseColumnDataType, prechecked = false) => {
|
||||
return typeConversion[typeof valueArg as string];
|
||||
};
|
||||
const checkPath = async (
|
||||
pathArg: string,
|
||||
typeArg: TClickhouseColumnDataType,
|
||||
prechecked = false
|
||||
) => {
|
||||
let columnFound = false;
|
||||
for (const column of this.columns) {
|
||||
if (pathArg === column.name) {
|
||||
@ -137,12 +144,12 @@ export class TimeDataTable {
|
||||
await checkPath(pathArg, typeArg, true);
|
||||
return;
|
||||
}
|
||||
const alterString = `ALTER TABLE ${this.smartClickHouseDbRef.options.database}.${this.options.tableName} ADD COLUMN ${pathArg} ${typeArg} FIRST`
|
||||
const alterString = `ALTER TABLE ${this.smartClickHouseDbRef.options.database}.${this.options.tableName} ADD COLUMN ${pathArg} ${typeArg} FIRST`;
|
||||
try {
|
||||
await this.smartClickHouseDbRef.clickhouseClient.queryPromise(`
|
||||
${alterString}
|
||||
`);
|
||||
} catch(err) {
|
||||
} catch (err) {
|
||||
console.log(alterString);
|
||||
for (const column of this.columns) {
|
||||
console.log(column.name);
|
||||
@ -171,21 +178,23 @@ export class TimeDataTable {
|
||||
storageJson[key] = value;
|
||||
}
|
||||
|
||||
const result = await this.smartClickHouseDbRef.clickhouseClient.insertPromise(this.smartClickHouseDbRef.options.database, this.options.tableName, [
|
||||
storageJson,
|
||||
]).catch(async () => {
|
||||
if (this.healingDeferred) {
|
||||
return;
|
||||
}
|
||||
this.healingDeferred = plugins.smartpromise.defer();
|
||||
console.log(`Ran into an error. Trying to set up things properly again.`);
|
||||
await this.smartClickHouseDbRef.pingDatabaseUntilAvailable();
|
||||
await this.smartClickHouseDbRef.createDatabase();
|
||||
await this.setup();
|
||||
this.columns = [];
|
||||
this.healingDeferred.resolve();
|
||||
this.healingDeferred = null;
|
||||
});
|
||||
const result = await this.smartClickHouseDbRef.clickhouseClient
|
||||
.insertPromise(this.smartClickHouseDbRef.options.database, this.options.tableName, [
|
||||
storageJson,
|
||||
])
|
||||
.catch(async () => {
|
||||
if (this.healingDeferred) {
|
||||
return;
|
||||
}
|
||||
this.healingDeferred = plugins.smartpromise.defer();
|
||||
console.log(`Ran into an error. Trying to set up things properly again.`);
|
||||
await this.smartClickHouseDbRef.pingDatabaseUntilAvailable();
|
||||
await this.smartClickHouseDbRef.createDatabase();
|
||||
await this.setup();
|
||||
this.columns = [];
|
||||
this.healingDeferred.resolve();
|
||||
this.healingDeferred = null;
|
||||
});
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
@ -5,10 +5,4 @@ import * as smartpromise from '@pushrocks/smartpromise';
|
||||
import * as smarturl from '@pushrocks/smarturl';
|
||||
import * as webrequest from '@pushrocks/webrequest';
|
||||
|
||||
export {
|
||||
smartdelay,
|
||||
smartobject,
|
||||
smartpromise,
|
||||
smarturl,
|
||||
webrequest,
|
||||
}
|
||||
export { smartdelay, smartobject, smartpromise, smarturl, webrequest };
|
||||
|
Reference in New Issue
Block a user