feat(core): introduce typed ClickHouse table API, query builder, and result handling; enhance HTTP client and add schema evolution, batch inserts and mutations; update docs/tests and bump deps
This commit is contained in:
@@ -3,6 +3,6 @@
|
||||
*/
|
||||
export const commitinfo = {
|
||||
name: '@push.rocks/smartclickhouse',
|
||||
version: '2.1.0',
|
||||
description: 'A TypeScript-based ODM for ClickHouse databases that supports creating, managing, and querying tables with a focus on handling time-series data.'
|
||||
version: '2.2.0',
|
||||
description: 'A TypeScript-based ODM for ClickHouse databases with full CRUD support, fluent query builder, configurable engines, and automatic schema evolution.'
|
||||
}
|
||||
|
||||
12
ts/index.ts
12
ts/index.ts
@@ -1,2 +1,14 @@
|
||||
// Core
|
||||
export * from './smartclickhouse.classes.smartclickhouse.js';
|
||||
export * from './smartclickhouse.classes.clickhousetable.js';
|
||||
export * from './smartclickhouse.classes.httpclient.js';
|
||||
|
||||
// Query & Results
|
||||
export * from './smartclickhouse.classes.querybuilder.js';
|
||||
export * from './smartclickhouse.classes.resultset.js';
|
||||
|
||||
// Time Data Table (backward compat)
|
||||
export * from './smartclickhouse.classes.timedatatable.js';
|
||||
|
||||
// Types
|
||||
export * from './smartclickhouse.types.js';
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
import * as plugins from './smartclickhouse.plugins.js';
|
||||
|
||||
export class ClickhouseDb {}
|
||||
498
ts/smartclickhouse.classes.clickhousetable.ts
Normal file
498
ts/smartclickhouse.classes.clickhousetable.ts
Normal file
@@ -0,0 +1,498 @@
|
||||
import * as plugins from './smartclickhouse.plugins.js';
|
||||
import type { SmartClickHouseDb } from './smartclickhouse.classes.smartclickhouse.js';
|
||||
import { ClickhouseQueryBuilder } from './smartclickhouse.classes.querybuilder.js';
|
||||
import type {
|
||||
IClickhouseTableOptions,
|
||||
IColumnInfo,
|
||||
TClickhouseColumnType,
|
||||
} from './smartclickhouse.types.js';
|
||||
import { detectClickhouseType, escapeClickhouseValue } from './smartclickhouse.types.js';
|
||||
|
||||
export class ClickhouseTable<T extends Record<string, any>> {
|
||||
// ---- STATIC FACTORY ----
|
||||
|
||||
public static async create<T extends Record<string, any>>(
|
||||
db: SmartClickHouseDb,
|
||||
options: IClickhouseTableOptions<T>,
|
||||
): Promise<ClickhouseTable<T>> {
|
||||
const table = new ClickhouseTable<T>(db, options);
|
||||
await table.setup();
|
||||
return table;
|
||||
}
|
||||
|
||||
// ---- INSTANCE ----
|
||||
|
||||
public db: SmartClickHouseDb;
|
||||
public options: IClickhouseTableOptions<T>;
|
||||
public columns: IColumnInfo[] = [];
|
||||
private healingDeferred: plugins.smartpromise.Deferred<any> | null = null;
|
||||
|
||||
constructor(db: SmartClickHouseDb, options: IClickhouseTableOptions<T>) {
|
||||
this.db = db;
|
||||
this.options = {
|
||||
autoSchemaEvolution: true,
|
||||
...options,
|
||||
database: options.database || db.options.database,
|
||||
engine: options.engine || { engine: 'MergeTree' },
|
||||
};
|
||||
}
|
||||
|
||||
// ---- SCHEMA MANAGEMENT ----
|
||||
|
||||
/**
|
||||
* Creates the table if it doesn't exist and refreshes column metadata
|
||||
*/
|
||||
public async setup(): Promise<void> {
|
||||
const { database, tableName, engine, orderBy, partitionBy, primaryKey, ttl, retainDataForDays, columns } = this.options;
|
||||
|
||||
// Build column definitions
|
||||
let columnDefs: string;
|
||||
if (columns && columns.length > 0) {
|
||||
columnDefs = columns.map((col) => {
|
||||
let def = `${col.name} ${col.type}`;
|
||||
if (col.defaultExpression) def += ` DEFAULT ${col.defaultExpression}`;
|
||||
if (col.codec) def += ` CODEC(${col.codec})`;
|
||||
return def;
|
||||
}).join(',\n ');
|
||||
} else {
|
||||
// Default minimal schema — downstream code can add columns via auto-schema evolution
|
||||
columnDefs = `timestamp DateTime64(3, 'Europe/Berlin'),\n message String`;
|
||||
}
|
||||
|
||||
// Build engine clause
|
||||
let engineClause: string = engine.engine;
|
||||
if (engine.engine === 'ReplacingMergeTree' && engine.versionColumn) {
|
||||
engineClause = `ReplacingMergeTree(${engine.versionColumn})`;
|
||||
} else if (engine.engine === 'CollapsingMergeTree' && engine.signColumn) {
|
||||
engineClause = `CollapsingMergeTree(${engine.signColumn})`;
|
||||
} else if (engine.engine === 'VersionedCollapsingMergeTree' && engine.signColumn && engine.versionColumn) {
|
||||
engineClause = `VersionedCollapsingMergeTree(${engine.signColumn}, ${engine.versionColumn})`;
|
||||
} else {
|
||||
engineClause = `${engine.engine}()`;
|
||||
}
|
||||
|
||||
// Build ORDER BY
|
||||
const orderByStr = Array.isArray(orderBy) ? orderBy.join(', ') : orderBy;
|
||||
|
||||
let createSQL = `
|
||||
CREATE TABLE IF NOT EXISTS ${database}.${tableName} (
|
||||
${columnDefs}
|
||||
) ENGINE = ${engineClause}`;
|
||||
|
||||
if (partitionBy) {
|
||||
createSQL += `\n PARTITION BY ${partitionBy}`;
|
||||
}
|
||||
|
||||
createSQL += `\n ORDER BY (${orderByStr})`;
|
||||
|
||||
if (primaryKey) {
|
||||
const primaryKeyStr = Array.isArray(primaryKey) ? primaryKey.join(', ') : primaryKey;
|
||||
createSQL += `\n PRIMARY KEY (${primaryKeyStr})`;
|
||||
}
|
||||
|
||||
await this.db.clickhouseHttpClient.queryPromise(createSQL);
|
||||
|
||||
// Apply TTL if configured
|
||||
if (ttl) {
|
||||
await this.db.clickhouseHttpClient.queryPromise(`
|
||||
ALTER TABLE ${database}.${tableName}
|
||||
MODIFY TTL toDateTime(${String(ttl.column)}) + INTERVAL ${ttl.interval}
|
||||
`);
|
||||
} else if (retainDataForDays && retainDataForDays > 0) {
|
||||
// Legacy shorthand
|
||||
await this.db.clickhouseHttpClient.queryPromise(`
|
||||
ALTER TABLE ${database}.${tableName}
|
||||
MODIFY TTL toDateTime(timestamp) + INTERVAL ${retainDataForDays} DAY
|
||||
`);
|
||||
}
|
||||
|
||||
await this.updateColumns();
|
||||
}
|
||||
|
||||
/**
|
||||
* Refresh column metadata from system.columns
|
||||
*/
|
||||
public async updateColumns(): Promise<IColumnInfo[]> {
|
||||
this.columns = await this.db.clickhouseHttpClient.queryPromise(`
|
||||
SELECT * FROM system.columns
|
||||
WHERE database = '${this.options.database}'
|
||||
AND table = '${this.options.tableName}' FORMAT JSONEachRow
|
||||
`);
|
||||
return this.columns;
|
||||
}
|
||||
|
||||
/**
|
||||
* Auto-schema evolution: detect new columns from data and add them
|
||||
*/
|
||||
public async syncSchema(data: Record<string, any>): Promise<void> {
|
||||
const flatData = plugins.smartobject.toFlatObject(data);
|
||||
for (const key of Object.keys(flatData)) {
|
||||
if (key === 'timestamp') continue;
|
||||
const value = flatData[key];
|
||||
const clickhouseType = detectClickhouseType(value);
|
||||
if (!clickhouseType) continue;
|
||||
await this.ensureColumn(key, clickhouseType);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- INSERT ----
|
||||
|
||||
/**
|
||||
* Insert a single row
|
||||
*/
|
||||
public async insert(data: Partial<T>): Promise<void> {
|
||||
if (this.healingDeferred) return;
|
||||
|
||||
const storageDoc = await this.prepareDocument(data);
|
||||
await this.executeInsert([storageDoc]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert multiple rows
|
||||
*/
|
||||
public async insertMany(data: Partial<T>[]): Promise<void> {
|
||||
if (this.healingDeferred) return;
|
||||
if (data.length === 0) return;
|
||||
|
||||
// Schema sync across all documents
|
||||
if (this.options.autoSchemaEvolution) {
|
||||
const allKeys = new Set<string>();
|
||||
const sampleValues: Record<string, any> = {};
|
||||
for (const doc of data) {
|
||||
const flat = plugins.smartobject.toFlatObject(doc);
|
||||
for (const [key, value] of Object.entries(flat)) {
|
||||
if (!allKeys.has(key)) {
|
||||
allKeys.add(key);
|
||||
sampleValues[key] = value;
|
||||
}
|
||||
}
|
||||
}
|
||||
await this.syncSchema(sampleValues);
|
||||
}
|
||||
|
||||
const storageDocs = data.map((doc) => this.flattenDocument(doc));
|
||||
await this.executeInsert(storageDocs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert in batches of configurable size
|
||||
*/
|
||||
public async insertBatch(data: Partial<T>[], options?: { batchSize?: number }): Promise<void> {
|
||||
const batchSize = options?.batchSize || 10000;
|
||||
if (this.healingDeferred) return;
|
||||
if (data.length === 0) return;
|
||||
|
||||
// Schema sync across all documents first
|
||||
if (this.options.autoSchemaEvolution) {
|
||||
const sampleValues: Record<string, any> = {};
|
||||
for (const doc of data) {
|
||||
const flat = plugins.smartobject.toFlatObject(doc);
|
||||
for (const [key, value] of Object.entries(flat)) {
|
||||
if (!(key in sampleValues)) {
|
||||
sampleValues[key] = value;
|
||||
}
|
||||
}
|
||||
}
|
||||
await this.syncSchema(sampleValues);
|
||||
}
|
||||
|
||||
const storageDocs = data.map((doc) => this.flattenDocument(doc));
|
||||
await this.db.clickhouseHttpClient.insertBatch(
|
||||
this.options.database,
|
||||
this.options.tableName,
|
||||
storageDocs,
|
||||
batchSize,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a push-based insert stream using ObservableIntake
|
||||
*/
|
||||
public createInsertStream(options?: { batchSize?: number; flushIntervalMs?: number }): plugins.smartrx.ObservableIntake<Partial<T>> {
|
||||
const batchSize = options?.batchSize || 100;
|
||||
const flushIntervalMs = options?.flushIntervalMs || 1000;
|
||||
const intake = new plugins.smartrx.ObservableIntake<Partial<T>>();
|
||||
let buffer: Partial<T>[] = [];
|
||||
let flushTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
const flush = async () => {
|
||||
if (buffer.length === 0) return;
|
||||
const toInsert = buffer;
|
||||
buffer = [];
|
||||
await this.insertMany(toInsert);
|
||||
};
|
||||
|
||||
const scheduleFlush = () => {
|
||||
if (flushTimer) clearTimeout(flushTimer);
|
||||
flushTimer = setTimeout(async () => {
|
||||
await flush();
|
||||
}, flushIntervalMs);
|
||||
};
|
||||
|
||||
intake.subscribe(
|
||||
async (doc) => {
|
||||
buffer.push(doc);
|
||||
if (buffer.length >= batchSize) {
|
||||
if (flushTimer) clearTimeout(flushTimer);
|
||||
await flush();
|
||||
} else {
|
||||
scheduleFlush();
|
||||
}
|
||||
},
|
||||
undefined,
|
||||
async () => {
|
||||
if (flushTimer) clearTimeout(flushTimer);
|
||||
await flush();
|
||||
},
|
||||
);
|
||||
|
||||
return intake;
|
||||
}
|
||||
|
||||
// ---- QUERY ----
|
||||
|
||||
/**
|
||||
* Returns a fluent query builder for this table
|
||||
*/
|
||||
public query(): ClickhouseQueryBuilder<T> {
|
||||
return new ClickhouseQueryBuilder<T>(
|
||||
this.options.tableName,
|
||||
this.options.database,
|
||||
this.db.clickhouseHttpClient,
|
||||
);
|
||||
}
|
||||
|
||||
// ---- UPDATE ----
|
||||
|
||||
/**
|
||||
* Update rows matching a where condition (ClickHouse mutation - use sparingly)
|
||||
*/
|
||||
public async update(
|
||||
set: Partial<T>,
|
||||
whereFn: (q: ClickhouseQueryBuilder<T>) => ClickhouseQueryBuilder<T>,
|
||||
): Promise<void> {
|
||||
const qb = whereFn(new ClickhouseQueryBuilder<T>(this.options.tableName, this.options.database, this.db.clickhouseHttpClient));
|
||||
const whereClause = qb.buildWhereClause();
|
||||
if (!whereClause) {
|
||||
throw new Error('UPDATE requires a WHERE clause. Use a where condition to target specific rows.');
|
||||
}
|
||||
|
||||
const setClauses = Object.entries(set)
|
||||
.map(([key, value]) => `${key} = ${escapeClickhouseValue(value)}`)
|
||||
.join(', ');
|
||||
|
||||
await this.db.clickhouseHttpClient.mutatePromise(
|
||||
`ALTER TABLE ${this.options.database}.${this.options.tableName} UPDATE ${setClauses} WHERE ${whereClause}`
|
||||
);
|
||||
await this.waitForMutations();
|
||||
}
|
||||
|
||||
// ---- DELETE ----
|
||||
|
||||
/**
|
||||
* Delete rows matching a where condition (ClickHouse mutation)
|
||||
*/
|
||||
public async deleteWhere(
|
||||
whereFn: (q: ClickhouseQueryBuilder<T>) => ClickhouseQueryBuilder<T>,
|
||||
): Promise<void> {
|
||||
const qb = whereFn(new ClickhouseQueryBuilder<T>(this.options.tableName, this.options.database, this.db.clickhouseHttpClient));
|
||||
const whereClause = qb.buildWhereClause();
|
||||
if (!whereClause) {
|
||||
throw new Error('DELETE requires a WHERE clause.');
|
||||
}
|
||||
|
||||
await this.db.clickhouseHttpClient.mutatePromise(
|
||||
`ALTER TABLE ${this.options.database}.${this.options.tableName} DELETE WHERE ${whereClause}`
|
||||
);
|
||||
await this.waitForMutations();
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete entries older than a given interval on a column
|
||||
*/
|
||||
public async deleteOlderThan(column: keyof T & string, interval: string): Promise<void> {
|
||||
await this.db.clickhouseHttpClient.mutatePromise(
|
||||
`ALTER TABLE ${this.options.database}.${this.options.tableName} DELETE WHERE ${String(column)} < now() - INTERVAL ${interval}`
|
||||
);
|
||||
await this.waitForMutations();
|
||||
}
|
||||
|
||||
/**
|
||||
* Drop the entire table
|
||||
*/
|
||||
public async drop(): Promise<void> {
|
||||
await this.db.clickhouseHttpClient.queryPromise(
|
||||
`DROP TABLE IF EXISTS ${this.options.database}.${this.options.tableName}`
|
||||
);
|
||||
this.columns = [];
|
||||
}
|
||||
|
||||
// ---- UTILITIES ----
|
||||
|
||||
/**
|
||||
* Wait for all pending mutations on this table to complete
|
||||
*/
|
||||
public async waitForMutations(): Promise<void> {
|
||||
let pending = true;
|
||||
while (pending) {
|
||||
const mutations = await this.db.clickhouseHttpClient.queryPromise(`
|
||||
SELECT count() AS cnt FROM system.mutations
|
||||
WHERE is_done = 0 AND database = '${this.options.database}' AND table = '${this.options.tableName}' FORMAT JSONEachRow
|
||||
`);
|
||||
const count = mutations[0] ? parseInt(mutations[0].cnt, 10) : 0;
|
||||
if (count === 0) {
|
||||
pending = false;
|
||||
} else {
|
||||
await plugins.smartdelay.delayFor(1000);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the total row count
|
||||
*/
|
||||
public async getRowCount(): Promise<number> {
|
||||
const result = await this.db.clickhouseHttpClient.queryPromise(`
|
||||
SELECT count() AS cnt FROM ${this.options.database}.${this.options.tableName} FORMAT JSONEachRow
|
||||
`);
|
||||
return result[0] ? parseInt(result[0].cnt, 10) : 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Optimize table (useful for ReplacingMergeTree deduplication)
|
||||
*/
|
||||
public async optimize(final: boolean = false): Promise<void> {
|
||||
const finalClause = final ? ' FINAL' : '';
|
||||
await this.db.clickhouseHttpClient.queryPromise(
|
||||
`OPTIMIZE TABLE ${this.options.database}.${this.options.tableName}${finalClause}`
|
||||
);
|
||||
}
|
||||
|
||||
// ---- OBSERVATION ----
|
||||
|
||||
/**
|
||||
* Watch for new entries via polling. Returns an RxJS Observable.
|
||||
*/
|
||||
public watch(options?: { pollInterval?: number }): plugins.smartrx.rxjs.Observable<T> {
|
||||
const pollInterval = options?.pollInterval || 1000;
|
||||
|
||||
return new plugins.smartrx.rxjs.Observable<T>((observer) => {
|
||||
let lastTimestamp: number;
|
||||
let intervalId: ReturnType<typeof setInterval>;
|
||||
let stopped = false;
|
||||
|
||||
const fetchInitialTimestamp = async () => {
|
||||
const result = await this.db.clickhouseHttpClient.queryPromise(`
|
||||
SELECT max(timestamp) as lastTimestamp
|
||||
FROM ${this.options.database}.${this.options.tableName} FORMAT JSONEachRow
|
||||
`);
|
||||
lastTimestamp = result.length && result[0].lastTimestamp
|
||||
? new Date(result[0].lastTimestamp).getTime()
|
||||
: Date.now();
|
||||
};
|
||||
|
||||
const fetchNewEntries = async () => {
|
||||
if (stopped) return;
|
||||
try {
|
||||
const entries = await this.db.clickhouseHttpClient.queryPromise(`
|
||||
SELECT * FROM ${this.options.database}.${this.options.tableName}
|
||||
WHERE timestamp > toDateTime(${lastTimestamp / 1000})
|
||||
ORDER BY timestamp ASC FORMAT JSONEachRow
|
||||
`);
|
||||
for (const entry of entries) {
|
||||
observer.next(entry);
|
||||
}
|
||||
if (entries.length > 0) {
|
||||
lastTimestamp = new Date(entries[entries.length - 1].timestamp).getTime();
|
||||
}
|
||||
} catch (err) {
|
||||
observer.error(err);
|
||||
}
|
||||
};
|
||||
|
||||
const start = async () => {
|
||||
await fetchInitialTimestamp();
|
||||
intervalId = setInterval(fetchNewEntries, pollInterval);
|
||||
};
|
||||
|
||||
start().catch((err) => observer.error(err));
|
||||
|
||||
return () => {
|
||||
stopped = true;
|
||||
clearInterval(intervalId);
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
// ---- PRIVATE HELPERS ----
|
||||
|
||||
private async ensureColumn(name: string, type: TClickhouseColumnType): Promise<void> {
|
||||
// Check cached columns first
|
||||
const exists = this.columns.some((col) => col.name === name);
|
||||
if (exists) return;
|
||||
|
||||
// Refresh and check again
|
||||
await this.updateColumns();
|
||||
if (this.columns.some((col) => col.name === name)) return;
|
||||
|
||||
// Add column
|
||||
try {
|
||||
await this.db.clickhouseHttpClient.queryPromise(
|
||||
`ALTER TABLE ${this.options.database}.${this.options.tableName} ADD COLUMN \`${name}\` ${type}`
|
||||
);
|
||||
} catch (err) {
|
||||
// Column might have been added concurrently — ignore "already exists" errors
|
||||
if (!String(err).includes('already exists')) {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
await this.updateColumns();
|
||||
}
|
||||
|
||||
private flattenDocument(data: Partial<T>): Record<string, any> {
|
||||
const flat = plugins.smartobject.toFlatObject(data);
|
||||
const storageDoc: Record<string, any> = {};
|
||||
for (const [key, value] of Object.entries(flat)) {
|
||||
const type = detectClickhouseType(value);
|
||||
if (type || key === 'timestamp') {
|
||||
storageDoc[key] = value;
|
||||
}
|
||||
}
|
||||
return storageDoc;
|
||||
}
|
||||
|
||||
private async prepareDocument(data: Partial<T>): Promise<Record<string, any>> {
|
||||
if (this.options.autoSchemaEvolution) {
|
||||
await this.syncSchema(data as Record<string, any>);
|
||||
}
|
||||
return this.flattenDocument(data);
|
||||
}
|
||||
|
||||
private async executeInsert(docs: Record<string, any>[]): Promise<void> {
|
||||
try {
|
||||
await this.db.clickhouseHttpClient.insertPromise(
|
||||
this.options.database,
|
||||
this.options.tableName,
|
||||
docs,
|
||||
);
|
||||
} catch (err) {
|
||||
await this.handleInsertError();
|
||||
}
|
||||
}
|
||||
|
||||
private async handleInsertError(): Promise<void> {
|
||||
if (this.healingDeferred) return;
|
||||
|
||||
this.healingDeferred = plugins.smartpromise.defer();
|
||||
console.log('ClickhouseTable: Insert error. Attempting self-healing...');
|
||||
|
||||
try {
|
||||
await this.db.pingDatabaseUntilAvailable();
|
||||
await this.db.createDatabase();
|
||||
await this.setup();
|
||||
} finally {
|
||||
this.healingDeferred.resolve();
|
||||
this.healingDeferred = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -16,7 +16,7 @@ export class ClickhouseHttpClient {
|
||||
|
||||
// INSTANCE
|
||||
public options: IClickhouseHttpClientOptions;
|
||||
public webrequestInstance = new plugins.webrequest.WebRequest({
|
||||
public webrequestInstance = new plugins.webrequest.WebrequestClient({
|
||||
logging: false,
|
||||
});
|
||||
public computedProperties: {
|
||||
@@ -26,6 +26,7 @@ export class ClickhouseHttpClient {
|
||||
connectionUrl: null,
|
||||
parsedUrl: null,
|
||||
};
|
||||
|
||||
constructor(optionsArg: IClickhouseHttpClientOptions) {
|
||||
this.options = optionsArg;
|
||||
}
|
||||
@@ -41,14 +42,17 @@ export class ClickhouseHttpClient {
|
||||
this.computedProperties.connectionUrl.toString(),
|
||||
{
|
||||
method: 'GET',
|
||||
timeoutMs: 1000,
|
||||
timeout: 1000,
|
||||
}
|
||||
);
|
||||
return ping.status === 200 ? true : false;
|
||||
}
|
||||
|
||||
public async queryPromise(queryArg: string) {
|
||||
const returnArray = [];
|
||||
/**
|
||||
* Execute a query and return parsed JSONEachRow results
|
||||
*/
|
||||
public async queryPromise(queryArg: string): Promise<any[]> {
|
||||
const returnArray: any[] = [];
|
||||
const response = await this.webrequestInstance.request(
|
||||
`${this.computedProperties.connectionUrl}?query=${encodeURIComponent(queryArg)}`,
|
||||
{
|
||||
@@ -56,24 +60,47 @@ export class ClickhouseHttpClient {
|
||||
headers: this.getHeaders(),
|
||||
}
|
||||
);
|
||||
// console.log('===================');
|
||||
// console.log(this.computedProperties.connectionUrl);
|
||||
// console.log(queryArg);
|
||||
// console.log((await response.clone().text()).split(/\r?\n/))
|
||||
|
||||
const responseText = await response.text();
|
||||
|
||||
// Check for errors (ClickHouse returns non-200 for errors)
|
||||
if (!response.ok) {
|
||||
throw new Error(`ClickHouse query error: ${responseText.trim()}`);
|
||||
}
|
||||
|
||||
if (response.headers.get('X-ClickHouse-Format') === 'JSONEachRow') {
|
||||
const jsonList = await response.text();
|
||||
const jsonArray = jsonList.split('\n');
|
||||
const jsonArray = responseText.split('\n');
|
||||
for (const jsonArg of jsonArray) {
|
||||
if (!jsonArg) {
|
||||
continue;
|
||||
}
|
||||
if (!jsonArg) continue;
|
||||
returnArray.push(JSON.parse(jsonArg));
|
||||
}
|
||||
} else {
|
||||
} else if (responseText.trim()) {
|
||||
// Try to parse as JSONEachRow even without header (e.g. when FORMAT is in query)
|
||||
const lines = responseText.trim().split('\n');
|
||||
for (const line of lines) {
|
||||
if (!line) continue;
|
||||
try {
|
||||
returnArray.push(JSON.parse(line));
|
||||
} catch {
|
||||
// Not JSON — return raw text as single-element array
|
||||
return [{ raw: responseText.trim() }];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return returnArray;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a typed query returning T[]
|
||||
*/
|
||||
public async queryTyped<T>(queryArg: string): Promise<T[]> {
|
||||
return this.queryPromise(queryArg) as Promise<T[]>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert documents as JSONEachRow
|
||||
*/
|
||||
public async insertPromise(databaseArg: string, tableArg: string, documents: any[]) {
|
||||
const queryArg = `INSERT INTO ${databaseArg}.${tableArg} FORMAT JSONEachRow`;
|
||||
const response = await this.webrequestInstance.request(
|
||||
@@ -84,9 +111,48 @@ export class ClickhouseHttpClient {
|
||||
headers: this.getHeaders(),
|
||||
}
|
||||
);
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(`ClickHouse insert error: ${errorText.trim()}`);
|
||||
}
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert documents in batches of configurable size
|
||||
*/
|
||||
public async insertBatch(
|
||||
databaseArg: string,
|
||||
tableArg: string,
|
||||
documents: any[],
|
||||
batchSize: number = 10000,
|
||||
) {
|
||||
for (let i = 0; i < documents.length; i += batchSize) {
|
||||
const batch = documents.slice(i, i + batchSize);
|
||||
await this.insertPromise(databaseArg, tableArg, batch);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a mutation (ALTER TABLE UPDATE/DELETE) and optionally wait for completion
|
||||
*/
|
||||
public async mutatePromise(queryArg: string): Promise<void> {
|
||||
const response = await this.webrequestInstance.request(
|
||||
`${this.computedProperties.connectionUrl}?query=${encodeURIComponent(queryArg)}`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: this.getHeaders(),
|
||||
}
|
||||
);
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(`ClickHouse mutation error: ${errorText.trim()}`);
|
||||
}
|
||||
}
|
||||
|
||||
private getHeaders() {
|
||||
const headers: { [key: string]: string } = {};
|
||||
if (this.options.username) {
|
||||
|
||||
214
ts/smartclickhouse.classes.querybuilder.ts
Normal file
214
ts/smartclickhouse.classes.querybuilder.ts
Normal file
@@ -0,0 +1,214 @@
|
||||
import type { ClickhouseHttpClient } from './smartclickhouse.classes.httpclient.js';
|
||||
import { ClickhouseResultSet } from './smartclickhouse.classes.resultset.js';
|
||||
import { escapeClickhouseValue } from './smartclickhouse.types.js';
|
||||
import type { TComparisonOperator } from './smartclickhouse.types.js';
|
||||
|
||||
interface IWhereClause {
|
||||
connector: 'AND' | 'OR' | '';
|
||||
expression: string;
|
||||
}
|
||||
|
||||
export class ClickhouseQueryBuilder<T extends Record<string, any>> {
|
||||
private selectColumns: string[] = ['*'];
|
||||
private whereClauses: IWhereClause[] = [];
|
||||
private orderByClauses: string[] = [];
|
||||
private groupByClauses: string[] = [];
|
||||
private havingClauses: string[] = [];
|
||||
private limitValue: number | null = null;
|
||||
private offsetValue: number | null = null;
|
||||
|
||||
constructor(
|
||||
private tableName: string,
|
||||
private database: string,
|
||||
private httpClient: ClickhouseHttpClient,
|
||||
) {}
|
||||
|
||||
// ---- SELECT ----
|
||||
|
||||
public select<K extends keyof T & string>(...columns: K[]): this {
|
||||
this.selectColumns = columns;
|
||||
return this;
|
||||
}
|
||||
|
||||
public selectRaw(...expressions: string[]): this {
|
||||
this.selectColumns = expressions;
|
||||
return this;
|
||||
}
|
||||
|
||||
// ---- WHERE ----
|
||||
|
||||
public where<K extends keyof T & string>(
|
||||
column: K,
|
||||
operator: TComparisonOperator,
|
||||
value: any,
|
||||
): this {
|
||||
this.whereClauses.push({
|
||||
connector: '',
|
||||
expression: this.buildCondition(column, operator, value),
|
||||
});
|
||||
return this;
|
||||
}
|
||||
|
||||
public and<K extends keyof T & string>(
|
||||
column: K,
|
||||
operator: TComparisonOperator,
|
||||
value: any,
|
||||
): this {
|
||||
this.whereClauses.push({
|
||||
connector: 'AND',
|
||||
expression: this.buildCondition(column, operator, value),
|
||||
});
|
||||
return this;
|
||||
}
|
||||
|
||||
public or<K extends keyof T & string>(
|
||||
column: K,
|
||||
operator: TComparisonOperator,
|
||||
value: any,
|
||||
): this {
|
||||
this.whereClauses.push({
|
||||
connector: 'OR',
|
||||
expression: this.buildCondition(column, operator, value),
|
||||
});
|
||||
return this;
|
||||
}
|
||||
|
||||
public whereRaw(expression: string): this {
|
||||
this.whereClauses.push({
|
||||
connector: this.whereClauses.length > 0 ? 'AND' : '',
|
||||
expression,
|
||||
});
|
||||
return this;
|
||||
}
|
||||
|
||||
// ---- ORDER BY ----
|
||||
|
||||
public orderBy(column: (keyof T & string) | string, direction: 'ASC' | 'DESC' = 'ASC'): this {
|
||||
this.orderByClauses.push(`${column} ${direction}`);
|
||||
return this;
|
||||
}
|
||||
|
||||
// ---- GROUP BY ----
|
||||
|
||||
public groupBy<K extends keyof T & string>(...columns: K[]): this {
|
||||
this.groupByClauses.push(...columns);
|
||||
return this;
|
||||
}
|
||||
|
||||
public having(expression: string): this {
|
||||
this.havingClauses.push(expression);
|
||||
return this;
|
||||
}
|
||||
|
||||
// ---- LIMIT / OFFSET ----
|
||||
|
||||
public limit(count: number): this {
|
||||
this.limitValue = count;
|
||||
return this;
|
||||
}
|
||||
|
||||
public offset(count: number): this {
|
||||
this.offsetValue = count;
|
||||
return this;
|
||||
}
|
||||
|
||||
// ---- EXECUTION ----
|
||||
|
||||
public async execute(): Promise<ClickhouseResultSet<T>> {
|
||||
const sql = this.toSQL();
|
||||
const rows = await this.httpClient.queryTyped<T>(sql);
|
||||
return new ClickhouseResultSet<T>(rows);
|
||||
}
|
||||
|
||||
public async first(): Promise<T | null> {
|
||||
this.limitValue = 1;
|
||||
const result = await this.execute();
|
||||
return result.first();
|
||||
}
|
||||
|
||||
public async count(): Promise<number> {
|
||||
const savedSelect = this.selectColumns;
|
||||
this.selectColumns = ['count() as _count'];
|
||||
const sql = this.toSQL();
|
||||
this.selectColumns = savedSelect;
|
||||
const rows = await this.httpClient.queryTyped<{ _count: string }>(sql);
|
||||
return rows.length > 0 ? parseInt(rows[0]._count, 10) : 0;
|
||||
}
|
||||
|
||||
public async toArray(): Promise<T[]> {
|
||||
const result = await this.execute();
|
||||
return result.toArray();
|
||||
}
|
||||
|
||||
// ---- SQL GENERATION ----
|
||||
|
||||
public toSQL(): string {
|
||||
const parts: string[] = [];
|
||||
|
||||
parts.push(`SELECT ${this.selectColumns.join(', ')}`);
|
||||
parts.push(`FROM ${this.database}.${this.tableName}`);
|
||||
|
||||
const whereClause = this.buildWhereClause();
|
||||
if (whereClause) {
|
||||
parts.push(`WHERE ${whereClause}`);
|
||||
}
|
||||
|
||||
if (this.groupByClauses.length > 0) {
|
||||
parts.push(`GROUP BY ${this.groupByClauses.join(', ')}`);
|
||||
}
|
||||
|
||||
if (this.havingClauses.length > 0) {
|
||||
parts.push(`HAVING ${this.havingClauses.join(' AND ')}`);
|
||||
}
|
||||
|
||||
if (this.orderByClauses.length > 0) {
|
||||
parts.push(`ORDER BY ${this.orderByClauses.join(', ')}`);
|
||||
}
|
||||
|
||||
if (this.limitValue !== null) {
|
||||
parts.push(`LIMIT ${this.limitValue}`);
|
||||
}
|
||||
|
||||
if (this.offsetValue !== null) {
|
||||
parts.push(`OFFSET ${this.offsetValue}`);
|
||||
}
|
||||
|
||||
parts.push('FORMAT JSONEachRow');
|
||||
|
||||
return parts.join(' ');
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the WHERE clause string. Reused by ClickhouseTable for UPDATE/DELETE.
|
||||
*/
|
||||
public buildWhereClause(): string {
|
||||
if (this.whereClauses.length === 0) return '';
|
||||
|
||||
return this.whereClauses
|
||||
.map((clause, index) => {
|
||||
if (index === 0) return clause.expression;
|
||||
return `${clause.connector} ${clause.expression}`;
|
||||
})
|
||||
.join(' ');
|
||||
}
|
||||
|
||||
// ---- PRIVATE ----
|
||||
|
||||
private buildCondition(column: string, operator: TComparisonOperator, value: any): string {
|
||||
if (operator === 'IN' || operator === 'NOT IN') {
|
||||
const escapedValues = Array.isArray(value)
|
||||
? `(${value.map(escapeClickhouseValue).join(', ')})`
|
||||
: escapeClickhouseValue(value);
|
||||
return `${column} ${operator} ${escapedValues}`;
|
||||
}
|
||||
|
||||
if (operator === 'BETWEEN') {
|
||||
if (Array.isArray(value) && value.length === 2) {
|
||||
return `${column} BETWEEN ${escapeClickhouseValue(value[0])} AND ${escapeClickhouseValue(value[1])}`;
|
||||
}
|
||||
throw new Error('BETWEEN operator requires a two-element array value');
|
||||
}
|
||||
|
||||
return `${column} ${operator} ${escapeClickhouseValue(value)}`;
|
||||
}
|
||||
}
|
||||
44
ts/smartclickhouse.classes.resultset.ts
Normal file
44
ts/smartclickhouse.classes.resultset.ts
Normal file
@@ -0,0 +1,44 @@
|
||||
import * as plugins from './smartclickhouse.plugins.js';
|
||||
|
||||
export class ClickhouseResultSet<T> {
|
||||
public rows: T[];
|
||||
public rowCount: number;
|
||||
|
||||
constructor(rows: T[]) {
|
||||
this.rows = rows;
|
||||
this.rowCount = rows.length;
|
||||
}
|
||||
|
||||
public first(): T | null {
|
||||
return this.rows.length > 0 ? this.rows[0] : null;
|
||||
}
|
||||
|
||||
public last(): T | null {
|
||||
return this.rows.length > 0 ? this.rows[this.rows.length - 1] : null;
|
||||
}
|
||||
|
||||
public isEmpty(): boolean {
|
||||
return this.rows.length === 0;
|
||||
}
|
||||
|
||||
public toArray(): T[] {
|
||||
return this.rows;
|
||||
}
|
||||
|
||||
public map<U>(fn: (row: T) => U): U[] {
|
||||
return this.rows.map(fn);
|
||||
}
|
||||
|
||||
public filter(fn: (row: T) => boolean): ClickhouseResultSet<T> {
|
||||
return new ClickhouseResultSet<T>(this.rows.filter(fn));
|
||||
}
|
||||
|
||||
public toObservable(): plugins.smartrx.rxjs.Observable<T> {
|
||||
return new plugins.smartrx.rxjs.Observable<T>((observer) => {
|
||||
for (const row of this.rows) {
|
||||
observer.next(row);
|
||||
}
|
||||
observer.complete();
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
import * as plugins from './smartclickhouse.plugins.js';
|
||||
import { ClickhouseTable } from './smartclickhouse.classes.clickhousetable.js';
|
||||
import { TimeDataTable } from './smartclickhouse.classes.timedatatable.js';
|
||||
import { ClickhouseHttpClient } from './smartclickhouse.classes.httpclient.js';
|
||||
import type { IClickhouseTableOptions } from './smartclickhouse.types.js';
|
||||
|
||||
export interface IClickhouseConstructorOptions {
|
||||
url: string;
|
||||
@@ -8,8 +10,8 @@ export interface IClickhouseConstructorOptions {
|
||||
username?: string;
|
||||
password?: string;
|
||||
/**
|
||||
* allow services to exit when waiting for clickhouse startup
|
||||
* this allows to leave the lifecycle flow to other processes
|
||||
* Allow services to exit when waiting for clickhouse startup.
|
||||
* This allows to leave the lifecycle flow to other processes
|
||||
* like a listening server.
|
||||
*/
|
||||
unref?: boolean;
|
||||
@@ -24,11 +26,10 @@ export class SmartClickHouseDb {
|
||||
}
|
||||
|
||||
/**
|
||||
* starts the connection to the Clickhouse db
|
||||
* Starts the connection to the Clickhouse db
|
||||
*/
|
||||
public async start(dropOld = false) {
|
||||
console.log(`Connecting to default database first.`);
|
||||
// lets connect
|
||||
this.clickhouseHttpClient = await ClickhouseHttpClient.createAndStart(this.options);
|
||||
await this.pingDatabaseUntilAvailable();
|
||||
console.log(`Create database ${this.options.database}, if it does not exist...`);
|
||||
@@ -47,9 +48,7 @@ export class SmartClickHouseDb {
|
||||
public async pingDatabaseUntilAvailable() {
|
||||
let available = false;
|
||||
while (!available) {
|
||||
available = await this.clickhouseHttpClient.ping().catch((err) => {
|
||||
return false;
|
||||
});
|
||||
available = await this.clickhouseHttpClient.ping().catch(() => false);
|
||||
if (!available) {
|
||||
console.log(`NOT OK: tried pinging ${this.options.url}... Trying again in 5 seconds.`);
|
||||
await plugins.smartdelay.delayFor(5000, null, this.options.unref);
|
||||
@@ -57,11 +56,35 @@ export class SmartClickHouseDb {
|
||||
}
|
||||
}
|
||||
|
||||
// ---- NEW: Generic typed table factory ----
|
||||
|
||||
/**
|
||||
* gets a table
|
||||
* Create a typed ClickHouse table with full configuration
|
||||
*/
|
||||
public async getTable(tableName: string) {
|
||||
const newTable = TimeDataTable.getTable(this, tableName);
|
||||
return newTable;
|
||||
public async createTable<T extends Record<string, any>>(
|
||||
options: IClickhouseTableOptions<T>,
|
||||
): Promise<ClickhouseTable<T>> {
|
||||
return ClickhouseTable.create<T>(this, {
|
||||
...options,
|
||||
database: options.database || this.options.database,
|
||||
});
|
||||
}
|
||||
|
||||
// ---- BACKWARD COMPAT: TimeDataTable factory ----
|
||||
|
||||
/**
|
||||
* Get a TimeDataTable (backward compatible)
|
||||
*/
|
||||
public async getTable(tableName: string): Promise<TimeDataTable> {
|
||||
return TimeDataTable.getTable(this, tableName);
|
||||
}
|
||||
|
||||
// ---- RAW QUERY ----
|
||||
|
||||
/**
|
||||
* Execute a raw SQL query and return typed results
|
||||
*/
|
||||
public async query<T = any>(sql: string): Promise<T[]> {
|
||||
return this.clickhouseHttpClient.queryTyped<T>(sql);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,305 +1,113 @@
|
||||
import * as plugins from './smartclickhouse.plugins.js';
|
||||
import { SmartClickHouseDb } from './smartclickhouse.classes.smartclickhouse.js';
|
||||
import type { SmartClickHouseDb } from './smartclickhouse.classes.smartclickhouse.js';
|
||||
import { ClickhouseTable } from './smartclickhouse.classes.clickhousetable.js';
|
||||
|
||||
export type TClickhouseColumnDataType =
|
||||
| 'String'
|
||||
| "DateTime64(3, 'Europe/Berlin')"
|
||||
| 'Float64'
|
||||
| 'Array(String)'
|
||||
| 'Array(Float64)';
|
||||
export interface IColumnInfo {
|
||||
database: string;
|
||||
table: string;
|
||||
name: string;
|
||||
type: TClickhouseColumnDataType;
|
||||
position: string;
|
||||
default_kind: string;
|
||||
default_expression: string;
|
||||
data_compressed_bytes: string;
|
||||
data_uncompressed_bytes: string;
|
||||
marks_bytes: string;
|
||||
comment: string;
|
||||
is_in_partition_key: 0 | 1;
|
||||
is_in_sorting_key: 0 | 1;
|
||||
is_in_primary_key: 0 | 1;
|
||||
is_in_sampling_key: 0 | 1;
|
||||
compression_codec: string;
|
||||
character_octet_length: null;
|
||||
numeric_precision: null;
|
||||
numeric_precision_radix: null;
|
||||
numeric_scale: null;
|
||||
datetime_precision: '3';
|
||||
/**
|
||||
* Creates a pre-configured ClickhouseTable for time-series data.
|
||||
* This is the backward-compatible equivalent of the old TimeDataTable class.
|
||||
*
|
||||
* The table uses MergeTree engine, orders by timestamp, and has auto-schema evolution enabled.
|
||||
*/
|
||||
export async function createTimeDataTable(
|
||||
db: SmartClickHouseDb,
|
||||
tableName: string,
|
||||
retainDataForDays: number = 30,
|
||||
): Promise<TimeDataTable> {
|
||||
const table = new TimeDataTable(db, tableName, retainDataForDays);
|
||||
await table.setup();
|
||||
return table;
|
||||
}
|
||||
|
||||
export interface ITimeDataTableOptions {
|
||||
tableName: string;
|
||||
retainDataForDays: number;
|
||||
}
|
||||
/**
|
||||
* TimeDataTable — a ClickhouseTable pre-configured for time-series data.
|
||||
* Provides backward-compatible convenience methods (addData, getLastEntries, etc.).
|
||||
*/
|
||||
export class TimeDataTable extends ClickhouseTable<any> {
|
||||
/**
|
||||
* Static factory for backward compatibility
|
||||
*/
|
||||
public static async getTable(
|
||||
smartClickHouseDbRefArg: SmartClickHouseDb,
|
||||
tableNameArg: string,
|
||||
retainDataForDays: number = 30,
|
||||
): Promise<TimeDataTable> {
|
||||
return createTimeDataTable(smartClickHouseDbRefArg, tableNameArg, retainDataForDays);
|
||||
}
|
||||
|
||||
export class TimeDataTable {
|
||||
public static async getTable(smartClickHouseDbRefArg: SmartClickHouseDb, tableNameArg: string) {
|
||||
const newTable = new TimeDataTable(smartClickHouseDbRefArg, {
|
||||
tableName: tableNameArg,
|
||||
retainDataForDays: 30,
|
||||
constructor(db: SmartClickHouseDb, tableName: string, retainDataForDays: number = 30) {
|
||||
super(db, {
|
||||
tableName,
|
||||
engine: { engine: 'MergeTree' },
|
||||
orderBy: 'timestamp' as any,
|
||||
retainDataForDays,
|
||||
autoSchemaEvolution: true,
|
||||
});
|
||||
|
||||
await newTable.setup();
|
||||
|
||||
return newTable;
|
||||
}
|
||||
|
||||
// INSTANCE
|
||||
public healingDeferred: plugins.smartpromise.Deferred<any>;
|
||||
public smartClickHouseDbRef: SmartClickHouseDb;
|
||||
public options: ITimeDataTableOptions;
|
||||
|
||||
constructor(smartClickHouseDbRefArg: SmartClickHouseDb, optionsArg: ITimeDataTableOptions) {
|
||||
this.smartClickHouseDbRef = smartClickHouseDbRefArg;
|
||||
this.options = optionsArg;
|
||||
}
|
||||
|
||||
public async setup() {
|
||||
// create table in clickhouse
|
||||
await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
|
||||
CREATE TABLE IF NOT EXISTS ${this.smartClickHouseDbRef.options.database}.${this.options.tableName} (
|
||||
timestamp DateTime64(3, 'Europe/Berlin'),
|
||||
message String
|
||||
) ENGINE=MergeTree() ORDER BY timestamp`);
|
||||
|
||||
// lets adjust the TTL
|
||||
await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
|
||||
ALTER TABLE ${this.smartClickHouseDbRef.options.database}.${this.options.tableName} MODIFY TTL toDateTime(timestamp) + INTERVAL ${this.options.retainDataForDays} DAY
|
||||
`);
|
||||
|
||||
await this.updateColumns();
|
||||
console.log(`=======================`);
|
||||
console.log(
|
||||
`table with name "${this.options.tableName}" in database ${this.smartClickHouseDbRef.options.database} has the following columns:`
|
||||
);
|
||||
for (const column of this.columns) {
|
||||
console.log(`>> ${column.name}: ${column.type}`);
|
||||
/**
|
||||
* Insert data with auto-schema evolution and object flattening.
|
||||
* Backward-compatible: accepts arbitrary JSON objects with a timestamp field.
|
||||
*/
|
||||
public async addData(dataArg: any): Promise<any> {
|
||||
if (!dataArg.timestamp || typeof dataArg.timestamp !== 'number') {
|
||||
throw new Error('timestamp must be of type number');
|
||||
}
|
||||
console.log('^^^^^^^^^^^^^^\n');
|
||||
}
|
||||
|
||||
public columns: IColumnInfo[] = [];
|
||||
|
||||
/**
|
||||
* updates the columns
|
||||
*/
|
||||
public async updateColumns() {
|
||||
this.columns = await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
|
||||
SELECT * FROM system.columns
|
||||
WHERE database LIKE '${this.smartClickHouseDbRef.options.database}'
|
||||
AND table LIKE '${this.options.tableName}' FORMAT JSONEachRow
|
||||
`);
|
||||
return this.columns;
|
||||
return this.insert(dataArg);
|
||||
}
|
||||
|
||||
/**
|
||||
* stores a json and tries to map it to the nested syntax
|
||||
* Get the last N entries ordered by timestamp DESC
|
||||
*/
|
||||
public async addData(dataArg: any) {
|
||||
if (this.healingDeferred) {
|
||||
return;
|
||||
}
|
||||
|
||||
// the storageJson
|
||||
let storageJson: { [key: string]: any } = {};
|
||||
|
||||
// helper stuff
|
||||
|
||||
const getClickhouseTypeForValue = (valueArg: any): TClickhouseColumnDataType => {
|
||||
const typeConversion: { [key: string]: TClickhouseColumnDataType } = {
|
||||
string: 'String',
|
||||
number: 'Float64',
|
||||
undefined: null,
|
||||
null: null,
|
||||
};
|
||||
if (valueArg instanceof Array) {
|
||||
const arrayType = typeConversion[typeof valueArg[0] as string];
|
||||
if (!arrayType) {
|
||||
return null;
|
||||
} else {
|
||||
return `Array(${arrayType})` as TClickhouseColumnDataType;
|
||||
}
|
||||
}
|
||||
return typeConversion[typeof valueArg as string];
|
||||
};
|
||||
const checkPath = async (
|
||||
pathArg: string,
|
||||
typeArg: TClickhouseColumnDataType,
|
||||
prechecked = false
|
||||
) => {
|
||||
let columnFound = false;
|
||||
for (const column of this.columns) {
|
||||
if (pathArg === column.name) {
|
||||
columnFound = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!columnFound) {
|
||||
if (!prechecked) {
|
||||
await this.updateColumns();
|
||||
await checkPath(pathArg, typeArg, true);
|
||||
return;
|
||||
}
|
||||
const alterString = `ALTER TABLE ${this.smartClickHouseDbRef.options.database}.${this.options.tableName} ADD COLUMN ${pathArg} ${typeArg} FIRST`;
|
||||
try {
|
||||
await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
|
||||
${alterString}
|
||||
`);
|
||||
} catch (err) {
|
||||
console.log(alterString);
|
||||
for (const column of this.columns) {
|
||||
console.log(column.name);
|
||||
}
|
||||
}
|
||||
await this.updateColumns();
|
||||
}
|
||||
};
|
||||
|
||||
// key checking
|
||||
const flatDataArg = plugins.smartobject.toFlatObject(dataArg);
|
||||
for (const key of Object.keys(flatDataArg)) {
|
||||
const value = flatDataArg[key];
|
||||
if (key === 'timestamp' && typeof value !== 'number') {
|
||||
throw new Error('timestamp must be of type number');
|
||||
} else if (key === 'timestamp') {
|
||||
storageJson.timestamp = flatDataArg[key];
|
||||
continue;
|
||||
}
|
||||
// lets deal with the rest
|
||||
const clickhouseType = getClickhouseTypeForValue(value);
|
||||
if (!clickhouseType) {
|
||||
continue;
|
||||
}
|
||||
await checkPath(key, clickhouseType);
|
||||
storageJson[key] = value;
|
||||
}
|
||||
|
||||
const result = await this.smartClickHouseDbRef.clickhouseHttpClient
|
||||
.insertPromise(this.smartClickHouseDbRef.options.database, this.options.tableName, [
|
||||
storageJson,
|
||||
])
|
||||
.catch(async () => {
|
||||
if (this.healingDeferred) {
|
||||
return;
|
||||
}
|
||||
this.healingDeferred = plugins.smartpromise.defer();
|
||||
console.log(`Ran into an error. Trying to set up things properly again.`);
|
||||
await this.smartClickHouseDbRef.pingDatabaseUntilAvailable();
|
||||
await this.smartClickHouseDbRef.createDatabase();
|
||||
await this.setup();
|
||||
this.columns = [];
|
||||
this.healingDeferred.resolve();
|
||||
this.healingDeferred = null;
|
||||
});
|
||||
return result;
|
||||
public async getLastEntries(count: number): Promise<any[]> {
|
||||
return this.query()
|
||||
.orderBy('timestamp' as any, 'DESC')
|
||||
.limit(count)
|
||||
.toArray();
|
||||
}
|
||||
|
||||
/**
|
||||
* deletes the entire table
|
||||
* Get entries newer than a unix timestamp (in milliseconds)
|
||||
*/
|
||||
public async delete() {
|
||||
await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
|
||||
DROP TABLE IF EXISTS ${this.smartClickHouseDbRef.options.database}.${this.options.tableName}
|
||||
`);
|
||||
this.columns = [];
|
||||
}
|
||||
|
||||
/**
|
||||
* deletes entries older than a specified number of days
|
||||
* @param days number of days
|
||||
*/
|
||||
public async deleteOldEntries(days: number) {
|
||||
// Perform the deletion operation
|
||||
await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
|
||||
ALTER TABLE ${this.smartClickHouseDbRef.options.database}.${this.options.tableName}
|
||||
DELETE WHERE timestamp < now() - INTERVAL ${days} DAY
|
||||
`);
|
||||
await this.waitForMutations();
|
||||
}
|
||||
|
||||
public async waitForMutations() {
|
||||
// Wait for the mutation to complete
|
||||
let mutations;
|
||||
do {
|
||||
mutations = await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
|
||||
SELECT count() AS mutations_count FROM system.mutations
|
||||
WHERE is_done = 0 AND table = '${this.options.tableName}'
|
||||
`);
|
||||
|
||||
if (mutations[0] && mutations[0].mutations_count > 0) {
|
||||
console.log('Waiting for mutations to complete...');
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
}
|
||||
} while (mutations[0] && mutations[0].mutations_count > 0);
|
||||
}
|
||||
|
||||
public async getLastEntries(count: number) {
|
||||
const result = await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
|
||||
SELECT * FROM ${this.smartClickHouseDbRef.options.database}.${this.options.tableName}
|
||||
ORDER BY timestamp DESC
|
||||
LIMIT ${count} FORMAT JSONEachRow
|
||||
`);
|
||||
return result;
|
||||
}
|
||||
|
||||
public async getEntriesNewerThan(unixTimestamp: number) {
|
||||
const result = await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
|
||||
SELECT * FROM ${this.smartClickHouseDbRef.options.database}.${this.options.tableName}
|
||||
public async getEntriesNewerThan(unixTimestamp: number): Promise<any[]> {
|
||||
return this.db.clickhouseHttpClient.queryPromise(`
|
||||
SELECT * FROM ${this.options.database}.${this.options.tableName}
|
||||
WHERE timestamp > toDateTime(${unixTimestamp / 1000}) FORMAT JSONEachRow
|
||||
`);
|
||||
return result;
|
||||
}
|
||||
|
||||
public async getEntriesBetween(unixTimestampStart: number, unixTimestampEnd: number) {
|
||||
const result = await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
|
||||
SELECT * FROM ${this.smartClickHouseDbRef.options.database}.${this.options.tableName}
|
||||
/**
|
||||
* Get entries between two unix timestamps (in milliseconds)
|
||||
*/
|
||||
public async getEntriesBetween(unixTimestampStart: number, unixTimestampEnd: number): Promise<any[]> {
|
||||
return this.db.clickhouseHttpClient.queryPromise(`
|
||||
SELECT * FROM ${this.options.database}.${this.options.tableName}
|
||||
WHERE timestamp > toDateTime(${unixTimestampStart / 1000})
|
||||
AND timestamp < toDateTime(${unixTimestampEnd / 1000}) FORMAT JSONEachRow
|
||||
`);
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* streams all new entries using an observable
|
||||
* Delete entries older than N days
|
||||
*/
|
||||
public async deleteOldEntries(days: number): Promise<void> {
|
||||
await this.db.clickhouseHttpClient.mutatePromise(`
|
||||
ALTER TABLE ${this.options.database}.${this.options.tableName}
|
||||
DELETE WHERE timestamp < now() - INTERVAL ${days} DAY
|
||||
`);
|
||||
await this.waitForMutations();
|
||||
}
|
||||
|
||||
/**
|
||||
* Drop the table (backward-compatible alias for drop())
|
||||
*/
|
||||
public async delete(): Promise<void> {
|
||||
return this.drop();
|
||||
}
|
||||
|
||||
/**
|
||||
* Watch for new entries via polling (backward-compatible wrapper)
|
||||
*/
|
||||
public watchNewEntries(): plugins.smartrx.rxjs.Observable<any> {
|
||||
return new plugins.smartrx.rxjs.Observable((observer) => {
|
||||
const pollInterval = 1000; // Poll every 1 second
|
||||
let lastTimestamp: number;
|
||||
let intervalId: NodeJS.Timeout;
|
||||
|
||||
const fetchLastEntryTimestamp = async () => {
|
||||
const lastEntry = await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
|
||||
SELECT max(timestamp) as lastTimestamp FROM ${this.smartClickHouseDbRef.options.database}.${this.options.tableName} FORMAT JSONEachRow
|
||||
`);
|
||||
lastTimestamp = lastEntry.length
|
||||
? new Date(lastEntry[0].lastTimestamp).getTime()
|
||||
: Date.now();
|
||||
};
|
||||
|
||||
const fetchNewEntries = async () => {
|
||||
const newEntries = await this.getEntriesNewerThan(lastTimestamp);
|
||||
if (newEntries.length > 0) {
|
||||
for (const entry of newEntries) {
|
||||
observer.next(entry);
|
||||
}
|
||||
lastTimestamp = new Date(newEntries[newEntries.length - 1].timestamp).getTime();
|
||||
}
|
||||
};
|
||||
|
||||
const startPolling = async () => {
|
||||
await fetchLastEntryTimestamp();
|
||||
intervalId = setInterval(fetchNewEntries, pollInterval);
|
||||
};
|
||||
|
||||
startPolling().catch((err) => observer.error(err));
|
||||
|
||||
// Cleanup on unsubscribe
|
||||
return () => clearInterval(intervalId);
|
||||
});
|
||||
return this.watch();
|
||||
}
|
||||
}
|
||||
|
||||
134
ts/smartclickhouse.types.ts
Normal file
134
ts/smartclickhouse.types.ts
Normal file
@@ -0,0 +1,134 @@
|
||||
// ============================================================
|
||||
// Column Data Types
|
||||
// ============================================================
|
||||
export type TClickhouseColumnType =
|
||||
| 'String'
|
||||
| 'UInt8' | 'UInt16' | 'UInt32' | 'UInt64'
|
||||
| 'Int8' | 'Int16' | 'Int32' | 'Int64'
|
||||
| 'Float32' | 'Float64'
|
||||
| 'Bool'
|
||||
| 'Date' | 'Date32'
|
||||
| 'DateTime' | 'DateTime64'
|
||||
| 'UUID'
|
||||
| 'IPv4' | 'IPv6'
|
||||
| (string & {}); // allow arbitrary ClickHouse types like "DateTime64(3, 'Europe/Berlin')"
|
||||
|
||||
// ============================================================
|
||||
// Engine Configuration
|
||||
// ============================================================
|
||||
export type TClickhouseEngine =
|
||||
| 'MergeTree'
|
||||
| 'ReplacingMergeTree'
|
||||
| 'SummingMergeTree'
|
||||
| 'AggregatingMergeTree'
|
||||
| 'CollapsingMergeTree'
|
||||
| 'VersionedCollapsingMergeTree';
|
||||
|
||||
export interface IEngineConfig {
|
||||
engine: TClickhouseEngine;
|
||||
/** For ReplacingMergeTree: the version column name */
|
||||
versionColumn?: string;
|
||||
/** For CollapsingMergeTree: the sign column name */
|
||||
signColumn?: string;
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Column Definition
|
||||
// ============================================================
|
||||
export interface IColumnDefinition {
|
||||
name: string;
|
||||
type: TClickhouseColumnType;
|
||||
defaultExpression?: string;
|
||||
codec?: string;
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Table Options
|
||||
// ============================================================
|
||||
export interface IClickhouseTableOptions<T = any> {
|
||||
tableName: string;
|
||||
database?: string;
|
||||
engine?: IEngineConfig;
|
||||
orderBy: (keyof T & string) | (keyof T & string)[];
|
||||
partitionBy?: string;
|
||||
primaryKey?: (keyof T & string) | (keyof T & string)[];
|
||||
ttl?: {
|
||||
column: keyof T & string;
|
||||
interval: string; // e.g., '30 DAY', '1 MONTH'
|
||||
};
|
||||
columns?: IColumnDefinition[];
|
||||
/** Enable auto-schema evolution (add columns from data). Default: true */
|
||||
autoSchemaEvolution?: boolean;
|
||||
/** Data retention in days (shorthand for ttl). If ttl is set, this is ignored. */
|
||||
retainDataForDays?: number;
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Column Info from system.columns
|
||||
// ============================================================
|
||||
export interface IColumnInfo {
|
||||
database: string;
|
||||
table: string;
|
||||
name: string;
|
||||
type: string;
|
||||
position: string;
|
||||
default_kind: string;
|
||||
default_expression: string;
|
||||
data_compressed_bytes: string;
|
||||
data_uncompressed_bytes: string;
|
||||
marks_bytes: string;
|
||||
comment: string;
|
||||
is_in_partition_key: 0 | 1;
|
||||
is_in_sorting_key: 0 | 1;
|
||||
is_in_primary_key: 0 | 1;
|
||||
is_in_sampling_key: 0 | 1;
|
||||
compression_codec: string;
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Comparison Operators for Query Builder
|
||||
// ============================================================
|
||||
export type TComparisonOperator =
|
||||
| '='
|
||||
| '!='
|
||||
| '>'
|
||||
| '>='
|
||||
| '<'
|
||||
| '<='
|
||||
| 'LIKE'
|
||||
| 'NOT LIKE'
|
||||
| 'IN'
|
||||
| 'NOT IN'
|
||||
| 'BETWEEN';
|
||||
|
||||
// ============================================================
|
||||
// Value Escaping (SQL Injection Prevention)
|
||||
// ============================================================
|
||||
export function escapeClickhouseValue(value: any): string {
|
||||
if (value === null || value === undefined) return 'NULL';
|
||||
if (typeof value === 'number') return String(value);
|
||||
if (typeof value === 'boolean') return value ? '1' : '0';
|
||||
if (value instanceof Date) return `'${value.toISOString().replace('T', ' ').replace('Z', '')}'`;
|
||||
if (Array.isArray(value)) {
|
||||
return `(${value.map(escapeClickhouseValue).join(', ')})`;
|
||||
}
|
||||
// String: escape single quotes and backslashes
|
||||
return `'${String(value).replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'`;
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// ClickHouse Type Detection from JS Values
|
||||
// ============================================================
|
||||
export function detectClickhouseType(value: any): TClickhouseColumnType | null {
|
||||
if (value === null || value === undefined) return null;
|
||||
if (typeof value === 'string') return 'String';
|
||||
if (typeof value === 'number') return 'Float64';
|
||||
if (typeof value === 'boolean') return 'UInt8';
|
||||
if (value instanceof Array) {
|
||||
if (value.length === 0) return null;
|
||||
const elementType = detectClickhouseType(value[0]);
|
||||
if (!elementType) return null;
|
||||
return `Array(${elementType})` as TClickhouseColumnType;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
Reference in New Issue
Block a user