fix(core): update

.gitea/workflows/default_nottags.yaml

@@ -0,0 +1,66 @@
+name: Default (not tags)
+
+on:
+  push:
+    tags-ignore:
+      - '**'
+
+env:
+  IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
+  NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
+  NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
+  NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
+  NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
+  NPMCI_URL_CLOUDLY: ${{secrets.NPMCI_URL_CLOUDLY}}
+
+jobs:
+  security:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install pnpm and npmci
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @shipzone/npmci
+
+      - name: Run npm prepare
+        run: npmci npm prepare
+
+      - name: Audit production dependencies
+        run: |
+          npmci command npm config set registry https://registry.npmjs.org
+          npmci command pnpm audit --audit-level=high --prod
+        continue-on-error: true
+
+      - name: Audit development dependencies
+        run: |
+          npmci command npm config set registry https://registry.npmjs.org
+          npmci command pnpm audit --audit-level=high --dev
+        continue-on-error: true
+
+  test:
+    if: ${{ always() }}
+    needs: security
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Test stable
+        run: |
+          npmci node install stable
+          npmci npm install
+          npmci npm test
+
+      - name: Test build
+        run: |
+          npmci node install stable
+          npmci npm install
+          npmci npm build

.gitea/workflows/default_tags.yaml

@@ -0,0 +1,124 @@
+name: Default (tags)
+
+on:
+  push:
+    tags:
+      - '*'
+
+env:
+  IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
+  NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
+  NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
+  NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
+  NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
+  NPMCI_URL_CLOUDLY: ${{secrets.NPMCI_URL_CLOUDLY}}
+
+jobs:
+  security:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Prepare
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @shipzone/npmci
+          npmci npm prepare
+
+      - name: Audit production dependencies
+        run: |
+          npmci command npm config set registry https://registry.npmjs.org
+          npmci command pnpm audit --audit-level=high --prod
+        continue-on-error: true
+
+      - name: Audit development dependencies
+        run: |
+          npmci command npm config set registry https://registry.npmjs.org
+          npmci command pnpm audit --audit-level=high --dev
+        continue-on-error: true
+
+  test:
+    if: ${{ always() }}
+    needs: security
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Prepare
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @shipzone/npmci
+          npmci npm prepare
+
+      - name: Test stable
+        run: |
+          npmci node install stable
+          npmci npm install
+          npmci npm test
+
+      - name: Test build
+        run: |
+          npmci node install stable
+          npmci npm install
+          npmci npm build
+
+  release:
+    needs: test
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Prepare
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @shipzone/npmci
+          npmci npm prepare
+
+      - name: Release
+        run: |
+          npmci node install stable
+          npmci npm publish
+
+  metadata:
+    needs: test
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+    continue-on-error: true
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Prepare
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @shipzone/npmci
+          npmci npm prepare
+
+      - name: Code quality
+        run: |
+          npmci command npm install -g typescript
+          npmci npm install
+
+      - name: Trigger
+        run: npmci trigger
+
+      - name: Build docs and upload artifacts
+        run: |
+          npmci node install stable
+          npmci npm install
+          pnpm install -g @git.zone/tsdoc
+          npmci command tsdoc
+        continue-on-error: true

.gitlab-ci.yml

@@ -1,128 +0,0 @@
-# gitzone ci_default
-image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
-
-cache:
-  paths:
-    - .npmci_cache/
-  key: '$CI_BUILD_STAGE'
-
-stages:
-  - security
-  - test
-  - release
-  - metadata
-
-before_script:
-  - npm install -g @shipzone/npmci
-
-# ====================
-# security stage
-# ====================
-auditProductionDependencies:
-  image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
-  stage: security
-  script:
-    - npmci npm prepare
-    - npmci command npm install --production --ignore-scripts
-    - npmci command npm config set registry https://registry.npmjs.org
-    - npmci command npm audit --audit-level=high --only=prod --production
-  tags:
-    - docker
-  allow_failure: true
-
-auditDevDependencies:
-  image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
-  stage: security
-  script:
-    - npmci npm prepare
-    - npmci command npm install --ignore-scripts
-    - npmci command npm config set registry https://registry.npmjs.org
-    - npmci command npm audit --audit-level=high --only=dev
-  tags:
-    - docker
-  allow_failure: true
-
-# ====================
-# test stage
-# ====================
-
-testStable:
-  stage: test
-  script:
-    - npmci npm prepare
-    - npmci node install stable
-    - npmci npm install
-    - npmci npm test
-  coverage: /\d+.?\d+?\%\s*coverage/
-  tags:
-    - docker
-
-testBuild:
-  stage: test
-  script:
-    - npmci npm prepare
-    - npmci node install stable
-    - npmci npm install
-    - npmci command npm run build
-  coverage: /\d+.?\d+?\%\s*coverage/
-  tags:
-    - docker
-
-release:
-  stage: release
-  script:
-    - npmci node install stable
-    - npmci npm publish
-  only:
-    - tags
-  tags:
-    - lossless
-    - docker
-    - notpriv
-
-# ====================
-# metadata stage
-# ====================
-codequality:
-  stage: metadata
-  allow_failure: true
-  only:
-    - tags
-  script:
-    - npmci command npm install -g typescript
-    - npmci npm prepare
-    - npmci npm install
-  tags:
-    - lossless
-    - docker
-    - priv
-
-trigger:
-  stage: metadata
-  script:
-    - npmci trigger
-  only:
-    - tags
-  tags:
-    - lossless
-    - docker
-    - notpriv
-
-pages:
-  stage: metadata
-  script:
-    - npmci node install stable 
-    - npmci npm prepare
-    - npmci npm install
-    - npmci command npm run buildDocs
-  tags:
-    - lossless
-    - docker
-    - notpriv
-  only:
-    - tags
-  artifacts:
-    expire_in: 1 week
-    paths:
-      - public
-  allow_failure: true

package.json

@@ -16,20 +16,22 @@
     "buildDocs": "tsdoc"
   },
   "devDependencies": {
-    "@gitzone/tsbuild": "^2.1.65",
+    "@git.zone/tsbuild": "^2.1.66",
-    "@gitzone/tsbundle": "^2.0.7",
+    "@git.zone/tsbundle": "^2.0.8",
-    "@gitzone/tstest": "^1.0.73",
+    "@git.zone/tsrun": "^1.2.46",
-    "@pushrocks/tapbundle": "^5.0.4",
+    "@git.zone/tstest": "^1.0.77",
-    "@types/node": "^18.6.4",
+    "@push.rocks/tapbundle": "^5.0.8",
+    "@types/node": "^20.14.2",
     "tslint": "^6.1.3",
-    "tslint-config-prettier": "^1.15.0"
+    "tslint-config-prettier": "^1.18.0"
   },
   "dependencies": {
-    "@pushrocks/smartdelay": "^2.0.13",
+    "@push.rocks/smartdelay": "^3.0.1",
-    "@pushrocks/smartobject": "^1.0.10",
+    "@push.rocks/smartobject": "^1.0.10",
-    "@pushrocks/smartpromise": "^3.1.7",
+    "@push.rocks/smartpromise": "^4.0.2",
-    "@pushrocks/smarturl": "^3.0.2",
+    "@push.rocks/smartrx": "^3.0.7",
-    "@pushrocks/webrequest": "^3.0.12"
+    "@push.rocks/smarturl": "^3.0.6",
+    "@push.rocks/webrequest": "^3.0.28"
   },
   "browserslist": [
     "last 1 chrome versions"

readme.hints.md

@@ -1 +1 @@
- 
+ - there is a local playground with clickhouse hosted under http://localhost:8123/play

test/test.nonci.ts

@@ -1,7 +1,8 @@
-import { expect, expectAsync, tap } from '@pushrocks/tapbundle';
+import { expect, tap } from '@push.rocks/tapbundle';
 import * as smartclickhouse from '../ts/index.js';
 
 let testClickhouseDb: smartclickhouse.SmartClickHouseDb;
+let table: smartclickhouse.TimeDataTable;
 
 tap.test('first test', async () => {
   testClickhouseDb = new smartclickhouse.SmartClickHouseDb({
@@ -16,9 +17,9 @@ tap.test('should start the clickhouse db', async () => {
 });
 
 tap.test('should create a timedatatable', async (toolsArg) => {
-  const table = await testClickhouseDb.getTable('analytics');
+  table = await testClickhouseDb.getTable('analytics');
   let i = 0;
-  while (i < 5000) {
+  while (i < 1000) {
     await table.addData({
       timestamp: Date.now(),
       message: `hello this is a message ${i}`,
@@ -29,11 +30,57 @@ tap.test('should create a timedatatable', async (toolsArg) => {
       },
     });
     i++;
-    console.log(`logged ${i} of 5000 lines.`);
+    console.log(`logged ${i} of 1000 lines.`);
-    await toolsArg.delayFor(1);
   }
 });
 
-tap.skip.test('should write something to the clickhouse db', async () => {});
+tap.test('should retrieve the last 10 entries', async () => {
+  const entries = await table.getLastEntries(10);
+  expect(entries.length).toEqual(10);
+  console.log(entries);
+});
 
-tap.start();
+tap.test('should retrieve entries newer than a specific timestamp', async () => {
+  const timestamp = Date.now() - 60000; // 1 minute ago
+  const entries = await table.getEntriesNewerThan(timestamp);
+  expect(entries.length).toBeGreaterThan(0);
+  console.log(entries);
+});
+
+tap.test('should retrieve entries between two timestamps', async () => {
+  const startTimestamp = Date.now() - 120000; // 2 minutes ago
+  const endTimestamp = Date.now() - 60000; // 1 minute ago
+  const entries = await table.getEntriesBetween(startTimestamp, endTimestamp);
+  console.log(entries);
+});
+
+tap.test('should delete old entries', async () => {
+  await table.deleteOldEntries(0); // Delete all entries older than now
+  const entries = await table.getLastEntries(10);
+  expect(entries.length).toEqual(0);
+});
+
+tap.test('should delete the table', async () => {
+  await table.delete();
+});
+
+tap.test('should stream new entries', async (toolsArg) => {
+  const stream = table.streamNewEntries();
+  const subscription = stream.subscribe((entry) => {
+    console.log('New entry:', entry);
+  });
+
+  let i = 0;
+  while (i < 10) {
+    await table.addData({
+      timestamp: Date.now(),
+      message: `streaming message ${i}`,
+    });
+    i++;
+    await toolsArg.delayFor(1000); // Add a delay to simulate real-time data insertion
+  }
+
+  subscription.unsubscribe();
+});
+
+export default tap.start();

ts/00_commitinfo_data.ts

@@ -2,7 +2,7 @@
  * autocreated commitinfo by @pushrocks/commitinfo
  */
 export const commitinfo = {
-  name: '@pushrocks/smartclickhouse',
+  name: '@push.rocks/smartclickhouse',
-  version: '2.0.14',
+  version: '2.0.15',
-  description: 'an odm for talking to clickhouse'
+  description: 'A TypeScript-based ODM (Object-Document Mapper) for ClickHouse databases, with support for creating and managing tables and their data.'
 }

ts/smartclickhouse.classes.smartclickhouse.ts

@@ -17,7 +17,7 @@ export interface IClickhouseConstructorOptions {
 
 export class SmartClickHouseDb {
   public options: IClickhouseConstructorOptions;
-  public clickhouseClient: ClickhouseHttpClient;
+  public clickhouseHttpClient: ClickhouseHttpClient;
 
   constructor(optionsArg: IClickhouseConstructorOptions) {
     this.options = optionsArg;
@@ -29,7 +29,7 @@ export class SmartClickHouseDb {
   public async start(dropOld = false) {
     console.log(`Connecting to default database first.`);
     // lets connect
-    this.clickhouseClient = await ClickhouseHttpClient.createAndStart(this.options);
+    this.clickhouseHttpClient = await ClickhouseHttpClient.createAndStart(this.options);
     await this.pingDatabaseUntilAvailable();
     console.log(`Create database ${this.options.database}, if it does not exist...`);
     await this.createDatabase(dropOld);
@@ -37,9 +37,9 @@ export class SmartClickHouseDb {
 
   public async createDatabase(dropOld: boolean = false) {
     if (dropOld) {
-      await this.clickhouseClient.queryPromise(`DROP DATABASE IF EXISTS ${this.options.database}`);
+      await this.clickhouseHttpClient.queryPromise(`DROP DATABASE IF EXISTS ${this.options.database}`);
     }
-    await this.clickhouseClient.queryPromise(
+    await this.clickhouseHttpClient.queryPromise(
       `CREATE DATABASE IF NOT EXISTS ${this.options.database}`
     );
   }
@@ -47,7 +47,7 @@ export class SmartClickHouseDb {
   public async pingDatabaseUntilAvailable() {
     let available = false;
     while (!available) {
-      available = await this.clickhouseClient.ping().catch((err) => {
+      available = await this.clickhouseHttpClient.ping().catch((err) => {
         return false;
       });
       if (!available) {

ts/smartclickhouse.classes.timedatatable.ts

@@ -60,14 +60,14 @@ export class TimeDataTable {
 
   public async setup() {
     // create table in clickhouse
-    await this.smartClickHouseDbRef.clickhouseClient.queryPromise(`
+    await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
         CREATE TABLE IF NOT EXISTS ${this.smartClickHouseDbRef.options.database}.${this.options.tableName} (
           timestamp DateTime64(3, 'Europe/Berlin'),
           message String
         ) ENGINE=MergeTree() ORDER BY timestamp`);
 
     // lets adjust the TTL
-    await this.smartClickHouseDbRef.clickhouseClient.queryPromise(`
+    await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
         ALTER TABLE ${this.smartClickHouseDbRef.options.database}.${this.options.tableName} MODIFY TTL toDateTime(timestamp) + INTERVAL ${this.options.retainDataForDays} DAY
       `);
 
@@ -88,7 +88,7 @@ export class TimeDataTable {
    * updates the columns
    */
   public async updateColumns() {
-    this.columns = await this.smartClickHouseDbRef.clickhouseClient.queryPromise(`
+    this.columns = await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
       SELECT * FROM system.columns
       WHERE database LIKE '${this.smartClickHouseDbRef.options.database}'
       AND table LIKE '${this.options.tableName}' FORMAT JSONEachRow
@@ -146,7 +146,7 @@ export class TimeDataTable {
         }
         const alterString = `ALTER TABLE ${this.smartClickHouseDbRef.options.database}.${this.options.tableName} ADD COLUMN ${pathArg} ${typeArg} FIRST`;
         try {
-          await this.smartClickHouseDbRef.clickhouseClient.queryPromise(`
+          await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
           ${alterString}
         `);
         } catch (err) {
@@ -178,7 +178,7 @@ export class TimeDataTable {
       storageJson[key] = value;
     }
 
-    const result = await this.smartClickHouseDbRef.clickhouseClient
+    const result = await this.smartClickHouseDbRef.clickhouseHttpClient
       .insertPromise(this.smartClickHouseDbRef.options.database, this.options.tableName, [
         storageJson,
       ])
@@ -197,4 +197,90 @@ export class TimeDataTable {
       });
     return result;
   }
+
+  /**
+   * deletes the entire table
+   */
+  public async delete() {
+    await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
+      DROP TABLE IF EXISTS ${this.smartClickHouseDbRef.options.database}.${this.options.tableName}
+    `);
+    this.columns = [];
+  }
+
+  /**
+   * deletes entries older than a specified number of days
+   * @param days number of days
+   */
+  public async deleteOldEntries(days: number) {
+    await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
+      ALTER TABLE ${this.smartClickHouseDbRef.options.database}.${this.options.tableName}
+      DELETE WHERE timestamp < now() - INTERVAL ${days} DAY
+    `);
+  }
+
+  public async getLastEntries(count: number) {
+    const result = await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
+      SELECT * FROM ${this.smartClickHouseDbRef.options.database}.${this.options.tableName}
+      ORDER BY timestamp DESC
+      LIMIT ${count} FORMAT JSONEachRow
+    `);
+    return result;
+  }
+
+  public async getEntriesNewerThan(unixTimestamp: number) {
+    const result = await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
+      SELECT * FROM ${this.smartClickHouseDbRef.options.database}.${this.options.tableName}
+      WHERE timestamp > toDateTime(${unixTimestamp / 1000}) FORMAT JSONEachRow
+    `);
+    return result;
+  }
+
+  public async getEntriesBetween(unixTimestampStart: number, unixTimestampEnd: number) {
+    const result = await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
+      SELECT * FROM ${this.smartClickHouseDbRef.options.database}.${this.options.tableName}
+      WHERE timestamp > toDateTime(${unixTimestampStart / 1000})
+      AND timestamp < toDateTime(${unixTimestampEnd / 1000}) FORMAT JSONEachRow
+    `);
+    return result;
+  }
+
+  /**
+   * streams all new entries using an observable
+   */
+  public streamNewEntries(): plugins.smartrx.rxjs.Observable<any> {
+    return new plugins.smartrx.rxjs.Observable((observer) => {
+      const pollInterval = 1000; // Poll every 1 second
+      let lastTimestamp: number;
+
+      const fetchLastEntryTimestamp = async () => {
+        const lastEntry = await this.smartClickHouseDbRef.clickhouseHttpClient.queryPromise(`
+        SELECT max(timestamp) as lastTimestamp FROM ${this.smartClickHouseDbRef.options.database}.${this.options.tableName} FORMAT JSONEachRow
+      `);
+        lastTimestamp = lastEntry.length
+          ? new Date(lastEntry[0].lastTimestamp).getTime()
+          : Date.now();
+      };
+
+      const fetchNewEntries = async () => {
+        const newEntries = await this.getEntriesNewerThan(lastTimestamp);
+        if (newEntries.length > 0) {
+          for (const entry of newEntries) {
+            observer.next(entry);
+          }
+          lastTimestamp = new Date(newEntries[newEntries.length - 1].timestamp).getTime();
+        }
+      };
+
+      const startPolling = async () => {
+        await fetchLastEntryTimestamp();
+        const intervalId = setInterval(fetchNewEntries, pollInterval);
+
+        // Cleanup on unsubscribe
+        return () => clearInterval(intervalId);
+      };
+
+      startPolling().catch((err) => observer.error(err));
+    });
+  }
 }

ts/smartclickhouse.plugins.ts

@@ -1,8 +1,9 @@
 // @pushrocks scope
-import * as smartdelay from '@pushrocks/smartdelay';
+import * as smartdelay from '@push.rocks/smartdelay';
-import * as smartobject from '@pushrocks/smartobject';
+import * as smartobject from '@push.rocks/smartobject';
-import * as smartpromise from '@pushrocks/smartpromise';
+import * as smartpromise from '@push.rocks/smartpromise';
-import * as smarturl from '@pushrocks/smarturl';
+import * as smartrx from '@push.rocks/smartrx';
-import * as webrequest from '@pushrocks/webrequest';
+import * as smarturl from '@push.rocks/smarturl';
+import * as webrequest from '@push.rocks/webrequest';
 
-export { smartdelay, smartobject, smartpromise, smarturl, webrequest };
+export { smartdelay, smartobject, smartpromise, smartrx, smarturl, webrequest };