2.1.0

changelog.md

@@ -0,0 +1,6 @@
+# Changelog
+
+## 2.1.0 - feat(core): Added comprehensive support for `SmartClickHouseDb` and `TimeDataTable` with features including time data table creation, data insertion, bulk data insertion, querying, data deletion, and real-time data observation. Included standalone Clickhouse HTTP client implementation.
+
+### Fixed
+- Fixed test case for table deletion and optimized code

license

@@ -1,4 +1,4 @@
-Copyright (c) 2022 Lossless GmbH (hello@lossless.com)
+Copyright (c) 2022 Task Venture Capital GmbH (hello@task.vc)
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

npmextra.json

@@ -5,7 +5,7 @@
       "githost": "code.foss.global",
       "gitscope": "push.rocks",
       "gitrepo": "smartclickhouse",
-      "description": "A TypeScript-based ODM (Object-Document Mapper) for ClickHouse databases, with support for creating and managing tables and their data.",
+      "description": "A TypeScript-based ODM for ClickHouse databases that supports creating, managing, and querying tables with a focus on handling time-series data.",
       "npmPackagename": "@push.rocks/smartclickhouse",
       "license": "MIT",
       "projectDomain": "push.rocks",
@@ -19,7 +19,16 @@
         "data management",
         "table management",
         "analytics",
-        "data storage"
+        "data storage",
+        "time-series data",
+        "schema management",
+        "data insertion",
+        "real-time data",
+        "data querying",
+        "bulk data insertion",
+        "error handling",
+        "data deletion",
+        "observables"
       ]
     }
   },

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@push.rocks/smartclickhouse",
-  "version": "2.0.15",
+  "version": "2.1.0",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "@push.rocks/smartclickhouse",
-      "version": "2.0.15",
+      "version": "2.1.0",
       "license": "MIT",
       "dependencies": {
         "@push.rocks/smartdelay": "^2.0.13",

package.json

@@ -1,8 +1,8 @@
 {
   "name": "@push.rocks/smartclickhouse",
-  "version": "2.0.15",
+  "version": "2.1.0",
   "private": false,
-  "description": "A TypeScript-based ODM (Object-Document Mapper) for ClickHouse databases, with support for creating and managing tables and their data.",
+  "description": "A TypeScript-based ODM for ClickHouse databases that supports creating, managing, and querying tables with a focus on handling time-series data.",
   "main": "dist_ts/index.js",
   "typings": "dist_ts/index.d.ts",
   "type": "module",
@@ -58,7 +58,16 @@
     "data management",
     "table management",
     "analytics",
-    "data storage"
+    "data storage",
+    "time-series data",
+    "schema management",
+    "data insertion",
+    "real-time data",
+    "data querying",
+    "bulk data insertion",
+    "error handling",
+    "data deletion",
+    "observables"
   ],
   "homepage": "https://code.foss.global/push.rocks/smartclickhouse",
   "repository": {

readme.md

@@ -1,5 +1,6 @@
 # @push.rocks/smartclickhouse
-an odm for talking to clickhouse
+
+A TypeScript-based ODM (Object-Document Mapper) for ClickHouse databases, with support for creating and managing tables and handling time-series data.
 
 ## Install
 
@@ -19,11 +20,11 @@ This will add the package to your project's dependencies.
 
 ## Usage
 
-`@push.rocks/smartclickhouse` is an ORM (Object Relational Mapping) module specifically designed for interacting with ClickHouse databases efficiently and effectively. Leveraging TypeScript, it offers strong typing and intelligent code completion, making database operations more intuitive and less error-prone.
+`@push.rocks/smartclickhouse` is an advanced ODM (Object Document Mapper) module designed for seamless interaction with ClickHouse databases leveraging the capabilities of TypeScript for strong typing and enhanced developer experience. Below is a comprehensive guide to using the package in various scenarios.
 
 ### Setting Up and Starting the Connection
 
-First, you need to establish a connection with the ClickHouse database. This involves creating an instance of `SmartClickHouseDb` and starting it:
+To begin using `@push.rocks/smartclickhouse`, you need to establish a connection with the ClickHouse database. This involves creating an instance of `SmartClickHouseDb` and starting it:
 
 ```typescript
 import { SmartClickHouseDb } from '@push.rocks/smartclickhouse';
@@ -31,10 +32,10 @@ import { SmartClickHouseDb } from '@push.rocks/smartclickhouse';
 // Create a new instance of SmartClickHouseDb with your ClickHouse database details
 const dbInstance = new SmartClickHouseDb({
   url: 'http://localhost:8123', // URL of ClickHouse instance
-  database: 'yourDatabase',     // Database name you want to connect
+  database: 'yourDatabase',     // Database name you want to connect to
   username: 'default',          // Optional: Username for authentication
   password: 'password',         // Optional: Password for authentication
-  unref: true                    // Optional: Allows service to exit while awaiting database startup
+  unref: true                   // Optional: Allows service to exit while awaiting database startup
 });
 
 // Start the instance to establish the connection
@@ -67,26 +68,173 @@ await table.addData({
 });
 ```
 
-`addData` method is designed to be flexible, allowing insertion of various data types and automatically managing table schema adjustments.
+The `addData` method is designed to be flexible, allowing insertion of various data types and automatically managing table schema adjustments.
 
 ### Advanced Usage and Custom Data Handling
 
-`smartclickhouse` supports custom data types and complex data structures. For instance, to add support for nested objects or custom data processing before insertion, you might need to extend existing classes or contribute to module development for broader use cases.
+`smartclickhouse` supports custom data types and complex data structures. For instance, to add support for nested objects or custom data processing before insertion, you might need to extend existing classes or customize the `addData` method to fit your needs.
 
-### Complete Feature Set Overview
+#### Custom Data Processing
 
-While the above examples cover basic usage, `@push.rocks/smartclickhouse` offers a wide range of functionalities, including but not limited to:
+To handle complex data structures or to perform custom data processing before insertion, you might need to modify the `addData` method. Below is an example of extending the `SmartClickHouseDb` method:
 
-- Robust error handling and reconnection strategies
+```typescript
-- Efficient bulk data insertion mechanisms
+class CustomClickHouseDb extends SmartClickHouseDb {
-- Support for ClickHouse specific features like materialized views and aggregating MergeTree engines
+  public async addCustomData(tableName: string, data: any) {
-- Techniques for managing and querying large time-series datasets
+    const table = await this.getTable(tableName);
+    const customData = {
+      ...data,
+      processedAt: Date.now(),
+      customField: 'customValue',
+    };
+    await table.addData(customData);
+  }
+}
 
-For a comprehensive guide on leveraging all `@push.rocks/smartclickhouse` features, consult the module documentation and examples. Engage with the community for tips on advanced use cases and optimization strategies for handling time-series data with ClickHouse.
+const customDbInstance = new CustomClickHouseDb({
+  url: 'http://localhost:8123',
+  database: 'yourDatabase',
+});
+
+await customDbInstance.start();
+
+await customDbInstance.addCustomData('customTable', {
+  message: 'Test message',
+  randomField: 123456,
+});
+```
+
+### Bulk Data Insertion
+
+`@push.rocks/smartclickhouse` supports efficient bulk data insertion mechanisms. This feature is useful when you need to insert a large amount of data in a single operation.
+
+```typescript
+const bulkData = [
+  { timestamp: Date.now(), message: 'Message 1', temperature: 20.1 },
+  { timestamp: Date.now(), message: 'Message 2', temperature: 21.2 },
+  // Additional data entries...
+];
+
+await table.addData(bulkData);
+```
+
+### Querying Data
+
+Fetching data from the ClickHouse database includes operations such as retrieving the latest entries, entries within a specific timestamp range, or streaming new entries.
+
+#### Retrieving the Last N Entries
+
+To retrieve the last `N` number of entries:
+
+```typescript
+const latestEntries = await table.getLastEntries(10);
+console.log('Latest Entries:', latestEntries);
+```
+
+#### Retrieving Entries Newer than a Specific Timestamp
+
+To retrieve entries that are newer than a specific timestamp:
+
+```typescript
+const timestamp = Date.now() - 60000; // 1 minute ago
+const newEntries = await table.getEntriesNewerThan(timestamp);
+console.log('New Entries:', newEntries);
+```
+
+#### Retrieving Entries Between Two Timestamps
+
+To retrieve entries between two timestamps:
+
+```typescript
+const startTimestamp = Date.now() - 120000; // 2 minutes ago
+const endTimestamp = Date.now() - 5000; // 5 seconds ago
+const entriesBetween = await table.getEntriesBetween(startTimestamp, endTimestamp);
+console.log('Entries Between:', entriesBetween);
+```
+
+### Managing and Deleting Data
+
+The module provides functionality for managing and deleting data within the ClickHouse database.
+
+#### Deleting Old Entries
+
+You can delete entries older than a specified number of days:
+
+```typescript
+// Ensure there are entries before deletion
+let entries = await table.getLastEntries(1000);
+console.log('Entries before deletion:', entries.length);
+
+// Delete all entries older than now
+await table.deleteOldEntries(0);
+
+// Verify the entries are deleted
+entries = await table.getLastEntries(1000);
+console.log('Entries after deletion:', entries.length);
+```
+
+#### Deleting the Entire Table
+
+To delete the entire table and all its data:
+
+```typescript
+await table.delete();
+
+// Verify table deletion
+const result = await dbInstance.clickhouseHttpClient.queryPromise(`
+  SHOW TABLES FROM ${dbInstance.options.database} LIKE '${table.options.tableName}'
+`);
+console.log('Table exists after deletion:', result.length === 0);
+```
+
+### Observing Real-Time Data
+
+To observe new entries in real-time, you can stream new data entries using the RxJS Observable:
+
+```typescript
+const stream = table.watchNewEntries();
+
+const subscription = stream.subscribe((entry) => {
+  console.log('New entry:', entry);
+});
+
+// Simulate adding new entries
+let i = 0;
+while (i < 10) {
+  await table.addData({
+    timestamp: Date.now(),
+    message: `streaming message ${i}`,
+  });
+  i++;
+  await new Promise((resolve) => setTimeout(resolve, 1000)); // Add a delay to simulate real-time data insertion
+}
+
+subscription.unsubscribe();
+```
+
+This method allows continuous monitoring of data changes and integrating the collected data into other systems for real-time applications.
+
+### Comprehensive Feature Set
+
+While the examples provided cover the core functionalities of the `@push.rocks/smartclickhouse` module, it also offers a wide range of additional features, including:
+
+- **Error Handling and Reconnection Strategies**: Robust error handling mechanisms ensure your application remains reliable. Automatic reconnection strategies help maintain persistent connections with the ClickHouse database.
+- **Materialized Views and MergeTree Engines**: Support for ClickHouse-specific features such as materialized views and aggregating MergeTree engines, enhancing the module's capabilities in handling large-scale data queries and management.
+- **Efficient Data Handling**: Techniques for managing and querying large time-series datasets, providing optimal performance and reliability.
 
 ### Contribution
 
-Contributions to `@push.rocks/smartclickhouse` are welcome. Whether it's through submitting issues, proposing improvements, or adding to the codebase, your input is valuable. The project is designed to be open and accessible, striving for a high-quality, community-driven development process.
+Contributions to `@push.rocks/smartclickhouse` are welcome. Whether through submitting issues, proposing improvements, or adding to the codebase, your input is valuable. The project is designed to be open and accessible, striving for a high-quality, community-driven development process.
+
+To contribute:
+
+1. Fork the repository.
+2. Create a new branch (`git checkout -b feature-branch`).
+3. Commit your changes (`git commit -am 'Add some feature'`).
+4. Push to the branch (`git push origin feature-branch`).
+5. Create a new Pull Request.
+
+The above scenarios cover the essential functionality and the more advanced use cases of `@push.rocks/smartclickhouse`, providing a comprehensive guide to utilizing the module into your projects. Happy coding!
 
 ## License and Legal Information
 

test/test.nonci.ts

@@ -16,7 +16,7 @@ tap.test('should start the clickhouse db', async () => {
   await testClickhouseDb.start(true);
 });
 
-tap.test('should create a timedatatable', async (toolsArg) => {
+tap.test('should create a timedatatable', async () => {
   table = await testClickhouseDb.getTable('analytics');
   let i = 0;
   while (i < 1000) {
@@ -49,23 +49,31 @@ tap.test('should retrieve entries newer than a specific timestamp', async () =>
 
 tap.test('should retrieve entries between two timestamps', async () => {
   const startTimestamp = Date.now() - 120000; // 2 minutes ago
-  const endTimestamp = Date.now() - 60000; // 1 minute ago
+  const endTimestamp = Date.now() - 5000; // 5 seconds ago
   const entries = await table.getEntriesBetween(startTimestamp, endTimestamp);
+  expect(entries.length).toBeGreaterThan(0);
   console.log(entries);
 });
 
-tap.test('should delete old entries', async () => {
+tap.test('should delete old entries', async (toolsArg) => {
-  await table.deleteOldEntries(0); // Delete all entries older than now
+  // Ensure there are entries before deletion
-  const entries = await table.getLastEntries(10);
+  let entries = await table.getLastEntries(1000);
-  expect(entries.length).toEqual(0);
+  expect(entries.length).toBeGreaterThan(100);
-});
+  console.log('Entries before deletion:', entries.length);
 
-tap.test('should delete the table', async () => {
+  await table.deleteOldEntries(0); // Delete all entries older than now
-  await table.delete();
+  // Add a delay to ensure the delete operation completes
+  await new Promise(resolve => setTimeout(resolve, 5000));
+  
+  // Verify the entries are deleted
+  entries = await table.getLastEntries(1000);
+  console.log('Entries after deletion:', entries.length);
+  expect(entries.length).toBeLessThan(100);
+  await toolsArg.delayFor(5000);
 });
 
 tap.test('should stream new entries', async (toolsArg) => {
-  const stream = table.streamNewEntries();
+  const stream = table.watchNewEntries();
   const subscription = stream.subscribe((entry) => {
     console.log('New entry:', entry);
   });
@@ -83,4 +91,14 @@ tap.test('should stream new entries', async (toolsArg) => {
   subscription.unsubscribe();
 });
 
+tap.test('should delete the table', async () => {
+  await table.delete();
+  // Verify table deletion
+  const result = await testClickhouseDb.clickhouseHttpClient.queryPromise(`
+    SHOW TABLES FROM ${testClickhouseDb.options.database} LIKE '${table.options.tableName}'
+  `);
+  console.log('Table exists after deletion:', result);
+  expect(result.length).toEqual(0);
+});
+
 export default tap.start();

ts/00_commitinfo_data.ts

@@ -1,8 +1,8 @@
 /**
- * autocreated commitinfo by @pushrocks/commitinfo
+ * autocreated commitinfo by @push.rocks/commitinfo
  */
 export const commitinfo = {
   name: '@push.rocks/smartclickhouse',
-  version: '2.0.15',
+  version: '2.1.0',
-  description: 'A TypeScript-based ODM (Object-Document Mapper) for ClickHouse databases, with support for creating and managing tables and their data.'
+  description: 'A TypeScript-based ODM for ClickHouse databases that supports creating, managing, and querying tables with a focus on handling time-series data.'
 }

ts/smartclickhouse.classes.timedatatable.ts

@@ -213,10 +213,28 @@ export class TimeDataTable {
    * @param days number of days
    */
   public async deleteOldEntries(days: number) {
+    // Perform the deletion operation
     await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
-      ALTER TABLE ${this.smartClickHouseDbRef.options.database}.${this.options.tableName}
+    ALTER TABLE ${this.smartClickHouseDbRef.options.database}.${this.options.tableName}
-      DELETE WHERE timestamp < now() - INTERVAL ${days} DAY
+    DELETE WHERE timestamp < now() - INTERVAL ${days} DAY
+  `);
+    await this.waitForMutations();
+  }
+
+  public async waitForMutations() {
+    // Wait for the mutation to complete
+    let mutations;
+    do {
+      mutations = await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
+      SELECT count() AS mutations_count FROM system.mutations
+      WHERE is_done = 0 AND table = '${this.options.tableName}'
     `);
+
+      if (mutations[0] && mutations[0].mutations_count > 0) {
+        console.log('Waiting for mutations to complete...');
+        await new Promise((resolve) => setTimeout(resolve, 1000));
+      }
+    } while (mutations[0] && mutations[0].mutations_count > 0);
   }
 
   public async getLastEntries(count: number) {
@@ -248,10 +266,11 @@ export class TimeDataTable {
   /**
    * streams all new entries using an observable
    */
-  public streamNewEntries(): plugins.smartrx.rxjs.Observable<any> {
+  public watchNewEntries(): plugins.smartrx.rxjs.Observable<any> {
     return new plugins.smartrx.rxjs.Observable((observer) => {
       const pollInterval = 1000; // Poll every 1 second
       let lastTimestamp: number;
+      let intervalId: NodeJS.Timeout;
 
       const fetchLastEntryTimestamp = async () => {
         const lastEntry = await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
@@ -274,13 +293,13 @@ export class TimeDataTable {
 
       const startPolling = async () => {
         await fetchLastEntryTimestamp();
-        const intervalId = setInterval(fetchNewEntries, pollInterval);
+        intervalId = setInterval(fetchNewEntries, pollInterval);
-
-        // Cleanup on unsubscribe
-        return () => clearInterval(intervalId);
       };
 
       startPolling().catch((err) => observer.error(err));
+
+      // Cleanup on unsubscribe
+      return () => clearInterval(intervalId);
     });
   }
 }