feat(core): introduce typed ClickHouse table API, query builder, and result handling; enhance HTTP client and add schema evolution, batch inserts and mutations; update docs/tests and bump deps

This commit is contained in:
2026-02-27 10:17:32 +00:00
parent 26449e9171
commit aace102868
17 changed files with 7000 additions and 1886 deletions

View File

@@ -16,7 +16,7 @@ export class ClickhouseHttpClient {
// INSTANCE
public options: IClickhouseHttpClientOptions;
public webrequestInstance = new plugins.webrequest.WebRequest({
public webrequestInstance = new plugins.webrequest.WebrequestClient({
logging: false,
});
public computedProperties: {
@@ -26,6 +26,7 @@ export class ClickhouseHttpClient {
connectionUrl: null,
parsedUrl: null,
};
constructor(optionsArg: IClickhouseHttpClientOptions) {
this.options = optionsArg;
}
@@ -41,14 +42,17 @@ export class ClickhouseHttpClient {
this.computedProperties.connectionUrl.toString(),
{
method: 'GET',
timeoutMs: 1000,
timeout: 1000,
}
);
return ping.status === 200 ? true : false;
}
public async queryPromise(queryArg: string) {
const returnArray = [];
/**
* Execute a query and return parsed JSONEachRow results
*/
public async queryPromise(queryArg: string): Promise<any[]> {
const returnArray: any[] = [];
const response = await this.webrequestInstance.request(
`${this.computedProperties.connectionUrl}?query=${encodeURIComponent(queryArg)}`,
{
@@ -56,24 +60,47 @@ export class ClickhouseHttpClient {
headers: this.getHeaders(),
}
);
// console.log('===================');
// console.log(this.computedProperties.connectionUrl);
// console.log(queryArg);
// console.log((await response.clone().text()).split(/\r?\n/))
const responseText = await response.text();
// Check for errors (ClickHouse returns non-200 for errors)
if (!response.ok) {
throw new Error(`ClickHouse query error: ${responseText.trim()}`);
}
if (response.headers.get('X-ClickHouse-Format') === 'JSONEachRow') {
const jsonList = await response.text();
const jsonArray = jsonList.split('\n');
const jsonArray = responseText.split('\n');
for (const jsonArg of jsonArray) {
if (!jsonArg) {
continue;
}
if (!jsonArg) continue;
returnArray.push(JSON.parse(jsonArg));
}
} else {
} else if (responseText.trim()) {
// Try to parse as JSONEachRow even without header (e.g. when FORMAT is in query)
const lines = responseText.trim().split('\n');
for (const line of lines) {
if (!line) continue;
try {
returnArray.push(JSON.parse(line));
} catch {
// Not JSON — return raw text as single-element array
return [{ raw: responseText.trim() }];
}
}
}
return returnArray;
}
/**
* Execute a typed query returning T[]
*/
public async queryTyped<T>(queryArg: string): Promise<T[]> {
return this.queryPromise(queryArg) as Promise<T[]>;
}
/**
* Insert documents as JSONEachRow
*/
public async insertPromise(databaseArg: string, tableArg: string, documents: any[]) {
const queryArg = `INSERT INTO ${databaseArg}.${tableArg} FORMAT JSONEachRow`;
const response = await this.webrequestInstance.request(
@@ -84,9 +111,48 @@ export class ClickhouseHttpClient {
headers: this.getHeaders(),
}
);
if (!response.ok) {
const errorText = await response.text();
throw new Error(`ClickHouse insert error: ${errorText.trim()}`);
}
return response;
}
/**
* Insert documents in batches of configurable size
*/
public async insertBatch(
databaseArg: string,
tableArg: string,
documents: any[],
batchSize: number = 10000,
) {
for (let i = 0; i < documents.length; i += batchSize) {
const batch = documents.slice(i, i + batchSize);
await this.insertPromise(databaseArg, tableArg, batch);
}
}
/**
* Execute a mutation (ALTER TABLE UPDATE/DELETE) and optionally wait for completion
*/
public async mutatePromise(queryArg: string): Promise<void> {
const response = await this.webrequestInstance.request(
`${this.computedProperties.connectionUrl}?query=${encodeURIComponent(queryArg)}`,
{
method: 'POST',
headers: this.getHeaders(),
}
);
if (!response.ok) {
const errorText = await response.text();
throw new Error(`ClickHouse mutation error: ${errorText.trim()}`);
}
}
private getHeaders() {
const headers: { [key: string]: string } = {};
if (this.options.username) {