feat(core): introduce typed ClickHouse table API, query builder, and result handling; enhance HTTP client and add schema evolution, batch inserts and mutations; update docs/tests and bump deps
This commit is contained in:
@@ -16,7 +16,7 @@ export class ClickhouseHttpClient {
|
||||
|
||||
// INSTANCE
|
||||
public options: IClickhouseHttpClientOptions;
|
||||
public webrequestInstance = new plugins.webrequest.WebRequest({
|
||||
public webrequestInstance = new plugins.webrequest.WebrequestClient({
|
||||
logging: false,
|
||||
});
|
||||
public computedProperties: {
|
||||
@@ -26,6 +26,7 @@ export class ClickhouseHttpClient {
|
||||
connectionUrl: null,
|
||||
parsedUrl: null,
|
||||
};
|
||||
|
||||
constructor(optionsArg: IClickhouseHttpClientOptions) {
|
||||
this.options = optionsArg;
|
||||
}
|
||||
@@ -41,14 +42,17 @@ export class ClickhouseHttpClient {
|
||||
this.computedProperties.connectionUrl.toString(),
|
||||
{
|
||||
method: 'GET',
|
||||
timeoutMs: 1000,
|
||||
timeout: 1000,
|
||||
}
|
||||
);
|
||||
return ping.status === 200 ? true : false;
|
||||
}
|
||||
|
||||
public async queryPromise(queryArg: string) {
|
||||
const returnArray = [];
|
||||
/**
|
||||
* Execute a query and return parsed JSONEachRow results
|
||||
*/
|
||||
public async queryPromise(queryArg: string): Promise<any[]> {
|
||||
const returnArray: any[] = [];
|
||||
const response = await this.webrequestInstance.request(
|
||||
`${this.computedProperties.connectionUrl}?query=${encodeURIComponent(queryArg)}`,
|
||||
{
|
||||
@@ -56,24 +60,47 @@ export class ClickhouseHttpClient {
|
||||
headers: this.getHeaders(),
|
||||
}
|
||||
);
|
||||
// console.log('===================');
|
||||
// console.log(this.computedProperties.connectionUrl);
|
||||
// console.log(queryArg);
|
||||
// console.log((await response.clone().text()).split(/\r?\n/))
|
||||
|
||||
const responseText = await response.text();
|
||||
|
||||
// Check for errors (ClickHouse returns non-200 for errors)
|
||||
if (!response.ok) {
|
||||
throw new Error(`ClickHouse query error: ${responseText.trim()}`);
|
||||
}
|
||||
|
||||
if (response.headers.get('X-ClickHouse-Format') === 'JSONEachRow') {
|
||||
const jsonList = await response.text();
|
||||
const jsonArray = jsonList.split('\n');
|
||||
const jsonArray = responseText.split('\n');
|
||||
for (const jsonArg of jsonArray) {
|
||||
if (!jsonArg) {
|
||||
continue;
|
||||
}
|
||||
if (!jsonArg) continue;
|
||||
returnArray.push(JSON.parse(jsonArg));
|
||||
}
|
||||
} else {
|
||||
} else if (responseText.trim()) {
|
||||
// Try to parse as JSONEachRow even without header (e.g. when FORMAT is in query)
|
||||
const lines = responseText.trim().split('\n');
|
||||
for (const line of lines) {
|
||||
if (!line) continue;
|
||||
try {
|
||||
returnArray.push(JSON.parse(line));
|
||||
} catch {
|
||||
// Not JSON — return raw text as single-element array
|
||||
return [{ raw: responseText.trim() }];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return returnArray;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a typed query returning T[]
|
||||
*/
|
||||
public async queryTyped<T>(queryArg: string): Promise<T[]> {
|
||||
return this.queryPromise(queryArg) as Promise<T[]>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert documents as JSONEachRow
|
||||
*/
|
||||
public async insertPromise(databaseArg: string, tableArg: string, documents: any[]) {
|
||||
const queryArg = `INSERT INTO ${databaseArg}.${tableArg} FORMAT JSONEachRow`;
|
||||
const response = await this.webrequestInstance.request(
|
||||
@@ -84,9 +111,48 @@ export class ClickhouseHttpClient {
|
||||
headers: this.getHeaders(),
|
||||
}
|
||||
);
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(`ClickHouse insert error: ${errorText.trim()}`);
|
||||
}
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert documents in batches of configurable size
|
||||
*/
|
||||
public async insertBatch(
|
||||
databaseArg: string,
|
||||
tableArg: string,
|
||||
documents: any[],
|
||||
batchSize: number = 10000,
|
||||
) {
|
||||
for (let i = 0; i < documents.length; i += batchSize) {
|
||||
const batch = documents.slice(i, i + batchSize);
|
||||
await this.insertPromise(databaseArg, tableArg, batch);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a mutation (ALTER TABLE UPDATE/DELETE) and optionally wait for completion
|
||||
*/
|
||||
public async mutatePromise(queryArg: string): Promise<void> {
|
||||
const response = await this.webrequestInstance.request(
|
||||
`${this.computedProperties.connectionUrl}?query=${encodeURIComponent(queryArg)}`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: this.getHeaders(),
|
||||
}
|
||||
);
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(`ClickHouse mutation error: ${errorText.trim()}`);
|
||||
}
|
||||
}
|
||||
|
||||
private getHeaders() {
|
||||
const headers: { [key: string]: string } = {};
|
||||
if (this.options.username) {
|
||||
|
||||
Reference in New Issue
Block a user