From 458da47c9c75a6ef79aa8b97e81f8802ad1d323c Mon Sep 17 00:00:00 2001 From: Philipp Kunz Date: Fri, 14 Jun 2024 16:56:39 +0200 Subject: [PATCH] fix(core): update --- npmextra.json | 7 +- package.json | 9 +- readme.md | 179 ++++++++++++++++++-- test/test.nonci.ts | 28 ++- ts/00_commitinfo_data.ts | 4 +- ts/smartclickhouse.classes.timedatatable.ts | 22 ++- 6 files changed, 221 insertions(+), 28 deletions(-) diff --git a/npmextra.json b/npmextra.json index 7d706c8..3197e23 100644 --- a/npmextra.json +++ b/npmextra.json @@ -5,7 +5,7 @@ "githost": "code.foss.global", "gitscope": "push.rocks", "gitrepo": "smartclickhouse", - "description": "A TypeScript-based ODM (Object-Document Mapper) for ClickHouse databases, with support for creating and managing tables and their data.", + "description": "A TypeScript-based ODM (Object-Document Mapper) for ClickHouse databases, with support for creating and managing tables and handling time-series data.", "npmPackagename": "@push.rocks/smartclickhouse", "license": "MIT", "projectDomain": "push.rocks", @@ -19,7 +19,10 @@ "data management", "table management", "analytics", - "data storage" + "data storage", + "time-series data", + "schema management", + "data insertion" ] } }, diff --git a/package.json b/package.json index 1bb15a5..d23cc53 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "@push.rocks/smartclickhouse", "version": "2.0.15", "private": false, - "description": "A TypeScript-based ODM (Object-Document Mapper) for ClickHouse databases, with support for creating and managing tables and their data.", + "description": "A TypeScript-based ODM (Object-Document Mapper) for ClickHouse databases, with support for creating and managing tables and handling time-series data.", "main": "dist_ts/index.js", "typings": "dist_ts/index.d.ts", "type": "module", @@ -58,11 +58,14 @@ "data management", "table management", "analytics", - "data storage" + "data storage", + "time-series data", + "schema management", + "data insertion" ], "homepage": "https://code.foss.global/push.rocks/smartclickhouse", "repository": { "type": "git", "url": "https://code.foss.global/push.rocks/smartclickhouse.git" } -} +} \ No newline at end of file diff --git a/readme.md b/readme.md index 7dff99f..8666150 100644 --- a/readme.md +++ b/readme.md @@ -1,5 +1,7 @@ +```markdown # @push.rocks/smartclickhouse -an odm for talking to clickhouse + +A TypeScript-based ODM (Object-Document Mapper) for ClickHouse databases, with support for creating and managing tables and their data. ## Install @@ -19,11 +21,11 @@ This will add the package to your project's dependencies. ## Usage -`@push.rocks/smartclickhouse` is an ORM (Object Relational Mapping) module specifically designed for interacting with ClickHouse databases efficiently and effectively. Leveraging TypeScript, it offers strong typing and intelligent code completion, making database operations more intuitive and less error-prone. +`@push.rocks/smartclickhouse` is an ODM (Object Document Mapper) module specifically designed for interacting with ClickHouse databases efficiently and effectively. Leveraging TypeScript, it offers strong typing and intelligent code completion, making database operations more intuitive and less error-prone. ### Setting Up and Starting the Connection -First, you need to establish a connection with the ClickHouse database. This involves creating an instance of `SmartClickHouseDb` and starting it: +To begin using `@push.rocks/smartclickhouse`, you need to establish a connection with the ClickHouse database. This involves creating an instance of `SmartClickHouseDb` and starting it: ```typescript import { SmartClickHouseDb } from '@push.rocks/smartclickhouse'; @@ -31,7 +33,7 @@ import { SmartClickHouseDb } from '@push.rocks/smartclickhouse'; // Create a new instance of SmartClickHouseDb with your ClickHouse database details const dbInstance = new SmartClickHouseDb({ url: 'http://localhost:8123', // URL of ClickHouse instance - database: 'yourDatabase', // Database name you want to connect + database: 'yourDatabase', // Database name you want to connect to username: 'default', // Optional: Username for authentication password: 'password', // Optional: Password for authentication unref: true // Optional: Allows service to exit while awaiting database startup @@ -67,26 +69,175 @@ await table.addData({ }); ``` -`addData` method is designed to be flexible, allowing insertion of various data types and automatically managing table schema adjustments. +The `addData` method is designed to be flexible, allowing insertion of various data types and automatically managing table schema adjustments. ### Advanced Usage and Custom Data Handling -`smartclickhouse` supports custom data types and complex data structures. For instance, to add support for nested objects or custom data processing before insertion, you might need to extend existing classes or contribute to module development for broader use cases. +`smartclickhouse` supports custom data types and complex data structures. For instance, to add support for nested objects or custom data processing before insertion, you might need to extend existing classes or contribute to the module development for broader use cases. -### Complete Feature Set Overview +#### Custom Data Processing -While the above examples cover basic usage, `@push.rocks/smartclickhouse` offers a wide range of functionalities, including but not limited to: +To handle complex data structures or to perform custom data processing before insertion, you might need to modify the `addData` method. Below is an example of extending the `SmartClickHouseDb` method: -- Robust error handling and reconnection strategies -- Efficient bulk data insertion mechanisms -- Support for ClickHouse specific features like materialized views and aggregating MergeTree engines -- Techniques for managing and querying large time-series datasets +```typescript +class CustomClickHouseDb extends SmartClickHouseDb { + public async addCustomData(tableName: string, data: any) { + const table = await this.getTable(tableName); + const customData = { + ...data, + processedAt: Date.now(), + customField: 'customValue', + }; + await table.addData(customData); + } +} -For a comprehensive guide on leveraging all `@push.rocks/smartclickhouse` features, consult the module documentation and examples. Engage with the community for tips on advanced use cases and optimization strategies for handling time-series data with ClickHouse. +const customDbInstance = new CustomClickHouseDb({ + url: 'http://localhost:8123', + database: 'yourDatabase', +}); + +await customDbInstance.start(); + +await customDbInstance.addCustomData('customTable', { + message: 'Test message', + randomField: 123456, +}); +``` + +### Bulk Data Insertion + +`@push.rocks/smartclickhouse` supports efficient bulk data insertion mechanisms. This feature is useful when you need to insert a large amount of data in a single operation. + +```typescript +const bulkData = [ + { timestamp: Date.now(), message: 'Message 1', temperature: 20.1 }, + { timestamp: Date.now(), message: 'Message 2', temperature: 21.2 }, + // Additional data entries... +]; + +await table.addData(bulkData); +``` + +### Querying Data + +Fetching data from the ClickHouse database includes operations such as retrieving the latest entries, entries within a specific timestamp range, or streaming new entries. + +#### Retrieving the Last N Entries + +To retrieve the last `N` number of entries: + +```typescript +const latestEntries = await table.getLastEntries(10); +console.log('Latest Entries:', latestEntries); +``` + +#### Retrieving Entries Newer than a Specific Timestamp + +To retrieve entries that are newer than a specific timestamp: + +```typescript +const timestamp = Date.now() - 60000; // 1 minute ago +const newEntries = await table.getEntriesNewerThan(timestamp); +console.log('New Entries:', newEntries); +``` + +#### Retrieving Entries Between Two Timestamps + +To retrieve entries between two timestamps: + +```typescript +const startTimestamp = Date.now() - 120000; // 2 minutes ago +const endTimestamp = Date.now() - 5000; // 5 seconds ago +const entriesBetween = await table.getEntriesBetween(startTimestamp, endTimestamp); +console.log('Entries Between:', entriesBetween); +``` + +### Managing and Deleting Data + +The module provides functionality for managing and deleting data within the ClickHouse database. + +#### Deleting Old Entries + +You can delete entries older than a specified number of days: + +```typescript +// Ensure there are entries before deletion +let entries = await table.getLastEntries(1000); +console.log('Entries before deletion:', entries.length); + +// Delete all entries older than now +await table.deleteOldEntries(0); + +// Verify the entries are deleted +entries = await table.getLastEntries(1000); +console.log('Entries after deletion:', entries.length); +``` + +#### Deleting the Entire Table + +To delete the entire table and all its data: + +```typescript +await table.delete(); + +// Verify table deletion +const result = await dbInstance.clickhouseHttpClient.queryPromise(` + SHOW TABLES FROM ${dbInstance.options.database} LIKE '${table.options.tableName}' +`); +console.log('Table exists after deletion:', result.length === 0); +``` + +### Observing Real-Time Data + +To observe new entries in real-time, you can stream new data entries using the RxJS Observable: + +```typescript +const stream = table.streamNewEntries(); + +const subscription = stream.subscribe((entry) => { + console.log('New entry:', entry); +}); + +// Simulate adding new entries +let i = 0; +while (i < 10) { + await table.addData({ + timestamp: Date.now(), + message: `streaming message ${i}`, + }); + i++; + await new Promise((resolve) => setTimeout(resolve, 1000)); // Add a delay to simulate real-time data insertion +} + +subscription.unsubscribe(); +``` + +This method allows continuous monitoring of data changes and integrating the collected data into other systems for real-time applications. + +### Comprehensive Feature Set + +While the examples provided cover the core functionalities of the `@push.rocks/smartclickhouse` module, it also offers a wide range of additional features, including: + +- **Error Handling and Reconnection Strategies**: Robust error handling mechanisms ensure your application remains reliable. Automatic reconnection strategies help maintain persistent connections with the ClickHouse database. +- **Materialized Views and MergeTree Engines**: Support for ClickHouse-specific features such as materialized views and aggregating MergeTree engines, enhancing the module's capabilities in handling large-scale data queries and management. +- **Efficient Data Handling**: Techniques for managing and querying large time-series datasets, providing optimal performance and reliability. ### Contribution -Contributions to `@push.rocks/smartclickhouse` are welcome. Whether it's through submitting issues, proposing improvements, or adding to the codebase, your input is valuable. The project is designed to be open and accessible, striving for a high-quality, community-driven development process. +Contributions to `@push.rocks/smartclickhouse` are welcome. Whether through submitting issues, proposing improvements, or adding to the codebase, your input is valuable. The project is designed to be open and accessible, striving for a high-quality, community-driven development process. + +To contribute: + +1. Fork the repository. +2. Create a new branch (`git checkout -b feature-branch`). +3. Commit your changes (`git commit -am 'Add some feature'`). +4. Push to the branch (`git push origin feature-branch`). +5. Create a new Pull Request. + +The above scenarios cover the essential functionality and the more advanced use cases of `@push.rocks/smartclickhouse`, providing a comprehensive guide to utilizing the module into your projects. Happy coding! + +``` ## License and Legal Information diff --git a/test/test.nonci.ts b/test/test.nonci.ts index aab5bdb..2ddac99 100644 --- a/test/test.nonci.ts +++ b/test/test.nonci.ts @@ -16,7 +16,7 @@ tap.test('should start the clickhouse db', async () => { await testClickhouseDb.start(true); }); -tap.test('should create a timedatatable', async (toolsArg) => { +tap.test('should create a timedatatable', async () => { table = await testClickhouseDb.getTable('analytics'); let i = 0; while (i < 1000) { @@ -49,19 +49,37 @@ tap.test('should retrieve entries newer than a specific timestamp', async () => tap.test('should retrieve entries between two timestamps', async () => { const startTimestamp = Date.now() - 120000; // 2 minutes ago - const endTimestamp = Date.now() - 60000; // 1 minute ago + const endTimestamp = Date.now() - 5000; // 5 seconds ago const entries = await table.getEntriesBetween(startTimestamp, endTimestamp); + expect(entries.length).toBeGreaterThan(0); console.log(entries); }); -tap.test('should delete old entries', async () => { +tap.test('should delete old entries', async (toolsArg) => { + // Ensure there are entries before deletion + let entries = await table.getLastEntries(1000); + expect(entries.length).toBeGreaterThan(100); + console.log('Entries before deletion:', entries.length); + await table.deleteOldEntries(0); // Delete all entries older than now - const entries = await table.getLastEntries(10); - expect(entries.length).toEqual(0); + // Add a delay to ensure the delete operation completes + await new Promise(resolve => setTimeout(resolve, 5000)); + + // Verify the entries are deleted + entries = await table.getLastEntries(1000); + console.log('Entries after deletion:', entries.length); + expect(entries.length).toBeLessThan(100); + await toolsArg.delayFor(5000); }); tap.test('should delete the table', async () => { await table.delete(); + // Verify table deletion + const result = await testClickhouseDb.clickhouseHttpClient.queryPromise(` + SHOW TABLES FROM ${testClickhouseDb.options.database} LIKE '${table.options.tableName}' + `); + console.log('Table exists after deletion:', result); + expect(result.length).toEqual(0); }); tap.test('should stream new entries', async (toolsArg) => { diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index 680cb00..409c5bd 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@push.rocks/smartclickhouse', - version: '2.0.15', - description: 'A TypeScript-based ODM (Object-Document Mapper) for ClickHouse databases, with support for creating and managing tables and their data.' + version: '2.0.16', + description: 'A TypeScript-based ODM (Object-Document Mapper) for ClickHouse databases, with support for creating and managing tables and handling time-series data.' } diff --git a/ts/smartclickhouse.classes.timedatatable.ts b/ts/smartclickhouse.classes.timedatatable.ts index 05b6a02..3bbbb57 100644 --- a/ts/smartclickhouse.classes.timedatatable.ts +++ b/ts/smartclickhouse.classes.timedatatable.ts @@ -213,10 +213,28 @@ export class TimeDataTable { * @param days number of days */ public async deleteOldEntries(days: number) { + // Perform the deletion operation await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(` - ALTER TABLE ${this.smartClickHouseDbRef.options.database}.${this.options.tableName} - DELETE WHERE timestamp < now() - INTERVAL ${days} DAY + ALTER TABLE ${this.smartClickHouseDbRef.options.database}.${this.options.tableName} + DELETE WHERE timestamp < now() - INTERVAL ${days} DAY + `); + await this.waitForMutations(); + } + + public async waitForMutations() { + // Wait for the mutation to complete + let mutations; + do { + mutations = await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(` + SELECT count() AS mutations_count FROM system.mutations + WHERE is_done = 0 AND table = '${this.options.tableName}' `); + + if (mutations[0] && mutations[0].mutations_count > 0) { + console.log('Waiting for mutations to complete...'); + await new Promise((resolve) => setTimeout(resolve, 1000)); + } + } while (mutations[0] && mutations[0].mutations_count > 0); } public async getLastEntries(count: number) {