8 Commits

Author SHA1 Message Date
45ce23ec11 2.0.11 2022-08-02 19:10:44 +02:00
93ef6a3d6b fix(core): update 2022-08-02 19:10:44 +02:00
609873b4ad 2.0.10 2022-08-01 12:52:53 +02:00
4a8bbc3d13 fix(core): update 2022-08-01 12:52:53 +02:00
51c2d4f6e0 2.0.9 2022-07-30 18:03:17 +02:00
45091d6b8c fix(core): update 2022-07-30 18:03:17 +02:00
eae7300439 2.0.8 2022-07-28 17:36:08 +02:00
7753e58036 fix(core): update 2022-07-28 17:36:08 +02:00
7 changed files with 71 additions and 195 deletions

214
package-lock.json generated
View File

@ -1,26 +1,26 @@
{
"name": "@pushrocks/smartclickhouse",
"version": "2.0.7",
"version": "2.0.11",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "@pushrocks/smartclickhouse",
"version": "2.0.7",
"version": "2.0.11",
"license": "MIT",
"dependencies": {
"@pushrocks/smartdelay": "^2.0.13",
"@pushrocks/smartobject": "^1.0.10",
"@pushrocks/smartpromise": "^3.1.7",
"@pushrocks/smarturl": "^3.0.2",
"@pushrocks/webrequest": "^3.0.9"
"@pushrocks/webrequest": "^3.0.10"
},
"devDependencies": {
"@gitzone/tsbuild": "^2.1.63",
"@gitzone/tsbundle": "^2.0.6",
"@gitzone/tstest": "^1.0.72",
"@pushrocks/tapbundle": "^5.0.4",
"@types/node": "^18.6.2",
"@types/node": "^18.6.3",
"tslint": "^6.1.3",
"tslint-config-prettier": "^1.15.0"
}
@ -1572,9 +1572,9 @@
}
},
"node_modules/@pushrocks/webrequest": {
"version": "3.0.9",
"resolved": "https://verdaccio.lossless.one/@pushrocks%2fwebrequest/-/webrequest-3.0.9.tgz",
"integrity": "sha512-cylGNW1Yd8+A/sJMqy5R59605M4ZDqwCvtXiIp+uoeLtXUrTFkqwauXHsE/MwGFgi97NZV/RnlJCPHeoaQAiTQ==",
"version": "3.0.10",
"resolved": "https://verdaccio.lossless.one/@pushrocks%2fwebrequest/-/webrequest-3.0.10.tgz",
"integrity": "sha512-35994hpzzR2WRbzhY+dhYoVQ6lXNJx3q3V3hNWKhCdrHKgMwkEZh6w+bso8PC9RFyg0p5KU4duBPW7FfJo6GzA==",
"license": "MIT",
"dependencies": {
"@pushrocks/smartdelay": "^2.0.13",
@ -1582,7 +1582,7 @@
"@pushrocks/smartjson": "^5.0.1",
"@pushrocks/smartpromise": "^3.1.7",
"@pushrocks/webstore": "^2.0.4",
"node-fetch": "^3.2.9"
"node-fetch": "^2.6.7"
}
},
"node_modules/@pushrocks/webrequest/node_modules/@pushrocks/smartjson": {
@ -2091,9 +2091,9 @@
"license": "MIT"
},
"node_modules/@types/node": {
"version": "18.6.2",
"resolved": "https://verdaccio.lossless.one/@types%2fnode/-/node-18.6.2.tgz",
"integrity": "sha512-KcfkBq9H4PI6Vpu5B/KoPeuVDAbmi+2mDBqGPGUgoL7yXQtcWGu2vJWmmRkneWK3Rh0nIAX192Aa87AqKHYChQ==",
"version": "18.6.3",
"resolved": "https://verdaccio.lossless.one/@types%2fnode/-/node-18.6.3.tgz",
"integrity": "sha512-6qKpDtoaYLM+5+AFChLhHermMQxc3TOEFIDzrZLPRGHPrLEwqFkkT5Kx3ju05g6X7uDPazz3jHbKPX0KzCjntg==",
"dev": true,
"license": "MIT"
},
@ -3404,27 +3404,6 @@
"node-fetch": "2.6.7"
}
},
"node_modules/cross-fetch/node_modules/node-fetch": {
"version": "2.6.7",
"resolved": "https://verdaccio.lossless.one/node-fetch/-/node-fetch-2.6.7.tgz",
"integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"whatwg-url": "^5.0.0"
},
"engines": {
"node": "4.x || >=6.0.0"
},
"peerDependencies": {
"encoding": "^0.1.0"
},
"peerDependenciesMeta": {
"encoding": {
"optional": true
}
}
},
"node_modules/crypto-random-string": {
"version": "4.0.0",
"resolved": "https://verdaccio.lossless.one/crypto-random-string/-/crypto-random-string-4.0.0.tgz",
@ -3454,15 +3433,6 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/data-uri-to-buffer": {
"version": "4.0.0",
"resolved": "https://verdaccio.lossless.one/data-uri-to-buffer/-/data-uri-to-buffer-4.0.0.tgz",
"integrity": "sha512-Vr3mLBA8qWmcuschSLAOogKgQ/Jwxulv3RNE4FXnYWRGujzrRWQI4m12fQqRkwX06C0KanhLr4hK+GydchZsaA==",
"license": "MIT",
"engines": {
"node": ">= 12"
}
},
"node_modules/dayjs": {
"version": "1.10.8",
"resolved": "https://verdaccio.lossless.one/dayjs/-/dayjs-1.10.8.tgz",
@ -4502,29 +4472,6 @@
"node": ">=0.4.0"
}
},
"node_modules/fetch-blob": {
"version": "3.2.0",
"resolved": "https://verdaccio.lossless.one/fetch-blob/-/fetch-blob-3.2.0.tgz",
"integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/jimmywarting"
},
{
"type": "paypal",
"url": "https://paypal.me/jimmywarting"
}
],
"license": "MIT",
"dependencies": {
"node-domexception": "^1.0.0",
"web-streams-polyfill": "^3.0.3"
},
"engines": {
"node": "^12.20 || >= 14.13"
}
},
"node_modules/figures": {
"version": "4.0.1",
"resolved": "https://verdaccio.lossless.one/figures/-/figures-4.0.1.tgz",
@ -4677,18 +4624,6 @@
"node": ">= 6"
}
},
"node_modules/formdata-polyfill": {
"version": "4.0.10",
"resolved": "https://verdaccio.lossless.one/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz",
"integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==",
"license": "MIT",
"dependencies": {
"fetch-blob": "^3.1.2"
},
"engines": {
"node": ">=12.20.0"
}
},
"node_modules/forwarded": {
"version": "0.2.0",
"resolved": "https://verdaccio.lossless.one/forwarded/-/forwarded-0.2.0.tgz",
@ -6321,41 +6256,24 @@
"lower-case": "^1.1.1"
}
},
"node_modules/node-domexception": {
"version": "1.0.0",
"resolved": "https://verdaccio.lossless.one/node-domexception/-/node-domexception-1.0.0.tgz",
"integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/jimmywarting"
},
{
"type": "github",
"url": "https://paypal.me/jimmywarting"
}
],
"license": "MIT",
"engines": {
"node": ">=10.5.0"
}
},
"node_modules/node-fetch": {
"version": "3.2.9",
"resolved": "https://verdaccio.lossless.one/node-fetch/-/node-fetch-3.2.9.tgz",
"integrity": "sha512-/2lI+DBecVvVm9tDhjziTVjo2wmTsSxSk58saUYP0P/fRJ3xxtfMDY24+CKTkfm0Dlhyn3CSXNL0SoRiCZ8Rzg==",
"version": "2.6.7",
"resolved": "https://verdaccio.lossless.one/node-fetch/-/node-fetch-2.6.7.tgz",
"integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==",
"license": "MIT",
"dependencies": {
"data-uri-to-buffer": "^4.0.0",
"fetch-blob": "^3.1.4",
"formdata-polyfill": "^4.0.10"
"whatwg-url": "^5.0.0"
},
"engines": {
"node": "^12.20.0 || ^14.13.1 || >=16.0.0"
"node": "4.x || >=6.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/node-fetch"
"peerDependencies": {
"encoding": "^0.1.0"
},
"peerDependenciesMeta": {
"encoding": {
"optional": true
}
}
},
"node_modules/normalize-newline": {
@ -7964,7 +7882,6 @@
"version": "0.0.3",
"resolved": "https://verdaccio.lossless.one/tr46/-/tr46-0.0.3.tgz",
"integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=",
"dev": true,
"license": "MIT"
},
"node_modules/tree-kill": {
@ -8433,27 +8350,16 @@
"node": ">= 0.8"
}
},
"node_modules/web-streams-polyfill": {
"version": "3.2.1",
"resolved": "https://verdaccio.lossless.one/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz",
"integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==",
"license": "MIT",
"engines": {
"node": ">= 8"
}
},
"node_modules/webidl-conversions": {
"version": "3.0.1",
"resolved": "https://verdaccio.lossless.one/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
"integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE=",
"dev": true,
"license": "BSD-2-Clause"
},
"node_modules/whatwg-url": {
"version": "5.0.0",
"resolved": "https://verdaccio.lossless.one/whatwg-url/-/whatwg-url-5.0.0.tgz",
"integrity": "sha1-lmRU6HZUYuN2RNNib2dCzotwll0=",
"dev": true,
"license": "MIT",
"dependencies": {
"tr46": "~0.0.3",
@ -10086,16 +9992,16 @@
}
},
"@pushrocks/webrequest": {
"version": "3.0.9",
"resolved": "https://verdaccio.lossless.one/@pushrocks%2fwebrequest/-/webrequest-3.0.9.tgz",
"integrity": "sha512-cylGNW1Yd8+A/sJMqy5R59605M4ZDqwCvtXiIp+uoeLtXUrTFkqwauXHsE/MwGFgi97NZV/RnlJCPHeoaQAiTQ==",
"version": "3.0.10",
"resolved": "https://verdaccio.lossless.one/@pushrocks%2fwebrequest/-/webrequest-3.0.10.tgz",
"integrity": "sha512-35994hpzzR2WRbzhY+dhYoVQ6lXNJx3q3V3hNWKhCdrHKgMwkEZh6w+bso8PC9RFyg0p5KU4duBPW7FfJo6GzA==",
"requires": {
"@pushrocks/smartdelay": "^2.0.13",
"@pushrocks/smartenv": "^5.0.2",
"@pushrocks/smartjson": "^5.0.1",
"@pushrocks/smartpromise": "^3.1.7",
"@pushrocks/webstore": "^2.0.4",
"node-fetch": "^3.2.9"
"node-fetch": "^2.6.7"
},
"dependencies": {
"@pushrocks/smartjson": {
@ -10545,9 +10451,9 @@
"integrity": "sha512-Klz949h02Gz2uZCMGwDUSDS1YBlTdDDgbWHi+81l29tQALUtvz4rAYi5uoVhE5Lagoq6DeqAUlbrHvW/mXDgdQ=="
},
"@types/node": {
"version": "18.6.2",
"resolved": "https://verdaccio.lossless.one/@types%2fnode/-/node-18.6.2.tgz",
"integrity": "sha512-KcfkBq9H4PI6Vpu5B/KoPeuVDAbmi+2mDBqGPGUgoL7yXQtcWGu2vJWmmRkneWK3Rh0nIAX192Aa87AqKHYChQ==",
"version": "18.6.3",
"resolved": "https://verdaccio.lossless.one/@types%2fnode/-/node-18.6.3.tgz",
"integrity": "sha512-6qKpDtoaYLM+5+AFChLhHermMQxc3TOEFIDzrZLPRGHPrLEwqFkkT5Kx3ju05g6X7uDPazz3jHbKPX0KzCjntg==",
"dev": true
},
"@types/parse5": {
@ -11492,17 +11398,6 @@
"dev": true,
"requires": {
"node-fetch": "2.6.7"
},
"dependencies": {
"node-fetch": {
"version": "2.6.7",
"resolved": "https://verdaccio.lossless.one/node-fetch/-/node-fetch-2.6.7.tgz",
"integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==",
"dev": true,
"requires": {
"whatwg-url": "^5.0.0"
}
}
}
},
"crypto-random-string": {
@ -11522,11 +11417,6 @@
}
}
},
"data-uri-to-buffer": {
"version": "4.0.0",
"resolved": "https://verdaccio.lossless.one/data-uri-to-buffer/-/data-uri-to-buffer-4.0.0.tgz",
"integrity": "sha512-Vr3mLBA8qWmcuschSLAOogKgQ/Jwxulv3RNE4FXnYWRGujzrRWQI4m12fQqRkwX06C0KanhLr4hK+GydchZsaA=="
},
"dayjs": {
"version": "1.10.8",
"resolved": "https://verdaccio.lossless.one/dayjs/-/dayjs-1.10.8.tgz",
@ -12155,15 +12045,6 @@
"xml-js": "^1.6.11"
}
},
"fetch-blob": {
"version": "3.2.0",
"resolved": "https://verdaccio.lossless.one/fetch-blob/-/fetch-blob-3.2.0.tgz",
"integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==",
"requires": {
"node-domexception": "^1.0.0",
"web-streams-polyfill": "^3.0.3"
}
},
"figures": {
"version": "4.0.1",
"resolved": "https://verdaccio.lossless.one/figures/-/figures-4.0.1.tgz",
@ -12280,14 +12161,6 @@
"mime-types": "^2.1.12"
}
},
"formdata-polyfill": {
"version": "4.0.10",
"resolved": "https://verdaccio.lossless.one/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz",
"integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==",
"requires": {
"fetch-blob": "^3.1.2"
}
},
"forwarded": {
"version": "0.2.0",
"resolved": "https://verdaccio.lossless.one/forwarded/-/forwarded-0.2.0.tgz",
@ -13473,19 +13346,12 @@
"lower-case": "^1.1.1"
}
},
"node-domexception": {
"version": "1.0.0",
"resolved": "https://verdaccio.lossless.one/node-domexception/-/node-domexception-1.0.0.tgz",
"integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ=="
},
"node-fetch": {
"version": "3.2.9",
"resolved": "https://verdaccio.lossless.one/node-fetch/-/node-fetch-3.2.9.tgz",
"integrity": "sha512-/2lI+DBecVvVm9tDhjziTVjo2wmTsSxSk58saUYP0P/fRJ3xxtfMDY24+CKTkfm0Dlhyn3CSXNL0SoRiCZ8Rzg==",
"version": "2.6.7",
"resolved": "https://verdaccio.lossless.one/node-fetch/-/node-fetch-2.6.7.tgz",
"integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==",
"requires": {
"data-uri-to-buffer": "^4.0.0",
"fetch-blob": "^3.1.4",
"formdata-polyfill": "^4.0.10"
"whatwg-url": "^5.0.0"
}
},
"normalize-newline": {
@ -14603,8 +14469,7 @@
"tr46": {
"version": "0.0.3",
"resolved": "https://verdaccio.lossless.one/tr46/-/tr46-0.0.3.tgz",
"integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=",
"dev": true
"integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o="
},
"tree-kill": {
"version": "1.2.2",
@ -14913,22 +14778,15 @@
"integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=",
"dev": true
},
"web-streams-polyfill": {
"version": "3.2.1",
"resolved": "https://verdaccio.lossless.one/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz",
"integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q=="
},
"webidl-conversions": {
"version": "3.0.1",
"resolved": "https://verdaccio.lossless.one/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
"integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE=",
"dev": true
"integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE="
},
"whatwg-url": {
"version": "5.0.0",
"resolved": "https://verdaccio.lossless.one/whatwg-url/-/whatwg-url-5.0.0.tgz",
"integrity": "sha1-lmRU6HZUYuN2RNNib2dCzotwll0=",
"dev": true,
"requires": {
"tr46": "~0.0.3",
"webidl-conversions": "^3.0.0"

View File

@ -1,6 +1,6 @@
{
"name": "@pushrocks/smartclickhouse",
"version": "2.0.7",
"version": "2.0.11",
"private": false,
"description": "an odm for talking to clickhouse",
"main": "dist_ts/index.js",
@ -19,7 +19,7 @@
"@gitzone/tsbundle": "^2.0.6",
"@gitzone/tstest": "^1.0.72",
"@pushrocks/tapbundle": "^5.0.4",
"@types/node": "^18.6.2",
"@types/node": "^18.6.3",
"tslint": "^6.1.3",
"tslint-config-prettier": "^1.15.0"
},
@ -28,7 +28,7 @@
"@pushrocks/smartobject": "^1.0.10",
"@pushrocks/smartpromise": "^3.1.7",
"@pushrocks/smarturl": "^3.0.2",
"@pushrocks/webrequest": "^3.0.9"
"@pushrocks/webrequest": "^3.0.10"
},
"browserslist": [
"last 1 chrome versions"

View File

@ -7,6 +7,7 @@ tap.test('first test', async () => {
testClickhouseDb = new smartclickhouse.SmartClickHouseDb({
url: 'http://localhost:8123',
database: 'test2',
unref: true,
});
});

View File

@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@pushrocks/smartclickhouse',
version: '2.0.7',
version: '2.0.11',
description: 'an odm for talking to clickhouse'
}

View File

@ -1,7 +1,7 @@
import * as plugins from './smartclickhouse.plugins.js';
export interface IClickhouseHttpClientOptions {
user?: string;
username?: string;
password?: string;
url: string;
}
@ -32,19 +32,13 @@ export class ClickhouseHttpClient {
public async start() {
this.computedProperties.parsedUrl = plugins.smarturl.Smarturl.createFromUrl(this.options.url);
this.computedProperties.parsedUrl.username = this.options.user ? this.options.user : '';
this.computedProperties.parsedUrl.password = this.options.password ? this.options.password : '';
console.log(this.computedProperties.parsedUrl);
this.computedProperties.connectionUrl = this.computedProperties.parsedUrl.toString();
}
public async ping() {
const parsedUrlForPing = plugins.smarturl.Smarturl.createFromUrl(
this.computedProperties.connectionUrl.toString()
);
parsedUrlForPing.username = null;
parsedUrlForPing.password = null;
const ping = await this.webrequestInstance.request(parsedUrlForPing.toString(), {
const ping = await this.webrequestInstance.request(this.computedProperties.connectionUrl.toString(), {
method: 'GET',
timeoutMs: 1000,
});
@ -54,9 +48,11 @@ export class ClickhouseHttpClient {
public async queryPromise(queryArg: string) {
const returnArray = [];
const response = await this.webrequestInstance.request(`${this.computedProperties.connectionUrl}?query=${encodeURIComponent(queryArg)}`, {
method: 'POST'
method: 'POST',
headers: this.getHeaders(),
});
// console.log('===================');
// console.log(this.computedProperties.connectionUrl);
// console.log(queryArg);
// console.log((await response.clone().text()).split(/\r?\n/))
if (response.headers.get('X-ClickHouse-Format') === 'JSONEachRow') {
@ -78,8 +74,20 @@ export class ClickhouseHttpClient {
const queryArg = `INSERT INTO ${databaseArg}.${tableArg} FORMAT JSONEachRow`;
const response = await this.webrequestInstance.request(`${this.computedProperties.connectionUrl}?query=${encodeURIComponent(queryArg)}`, {
method: 'POST',
body: documents.map(docArg => JSON.stringify(docArg)).join('\n')
body: documents.map(docArg => JSON.stringify(docArg)).join('\n'),
headers: this.getHeaders()
});
return response;
}
private getHeaders() {
const headers: {[key: string]: string} = {}
if (this.options.username) {
headers['X-ClickHouse-User'] = this.options.username;
}
if (this.options.password) {
headers['X-ClickHouse-Key'] = this.options.password;
}
return headers;
}
}

View File

@ -5,8 +5,14 @@ import { ClickhouseHttpClient } from './smartclickhouse.classes.httpclient.js';
export interface IClickhouseConstructorOptions {
url: string;
database: string;
user?: string;
username?: string;
password?: string;
/**
* allow services to exit when waiting for clickhouse startup
* this allows to leave the lifecycle flow to other processes
* like a listening server.
*/
unref?: boolean;
}
export class SmartClickHouseDb {
@ -44,7 +50,7 @@ export class SmartClickHouseDb {
});
if (!available) {
console.log(`NOT OK: tried pinging ${this.options.url}... Trying again in 5 seconds.`);
await plugins.smartdelay.delayFor(5000);
await plugins.smartdelay.delayFor(5000, null, this.options.unref);
}
}
}

View File

@ -97,6 +97,9 @@ export class TimeDataTable {
* stores a json and tries to map it to the nested syntax
*/
public async addData(dataArg: any) {
if (this.healingDeferred) {
return;
}
// the storageJson
let storageJson: { [key: string]: any } = {};