1.0.51

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@apiclient.xyz/elasticsearch",
-  "version": "1.0.47",
+  "version": "1.0.51",
   "private": false,
   "description": "log to elasticsearch in a kibana compatible format",
   "main": "dist_ts/index.js",

test/test.nonci.ts

@@ -59,4 +59,19 @@ tap.test('should delete documents not part of the piping session', async () => {
   await testElasticDoc.endPipingSession();
 });
 
+tap.test('should take and store snapshot', async () => {
+  await testElasticDoc.takeSnapshot(async (iterator, prevSnapshot) => {
+    const aggregationData = [];
+    for await (const doc of iterator) {
+      // Sample aggregation: counting documents
+      aggregationData.push(doc);
+    }
+    const snapshot = {
+      date: new Date().toISOString(),
+      aggregationData,
+    };
+    return snapshot;
+  });
+});
+
 tap.start();

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@apiclient.xyz/elasticsearch',
-  version: '1.0.47',
+  version: '1.0.51',
   description: 'log to elasticsearch in a kibana compatible format'
 }

ts/elasticsearch.classes.elasticindex.ts

@@ -1,4 +1,4 @@
-import * as plugins from './elasticsearch.plugins.js';
+import * as plugins from './els.plugins.js';
 import { ElsSmartlogDestination } from './els.classes.smartlogdestination.js';
 import { type ILogPackage } from '@pushrocks/smartlog-interfaces';
 import { Stringmap } from '@pushrocks/lik';

ts/els.classes.elasticdoc.ts

@@ -9,6 +9,13 @@ export interface IElasticDocConstructorOptions {
   };
 }
 
+export interface ISnapshot {
+  date: string;
+  aggregationData: any;
+}
+
+export type SnapshotProcessor = (iterator: AsyncIterable<any>, prevSnapshot: ISnapshot | null) => Promise<ISnapshot>;
+
 export class ElasticDoc {
   public client: ElasticClient;
   public index: string;
@@ -25,7 +32,6 @@ export class ElasticDoc {
   }
 
   async startPipingSession() {
-    // Clear the session docs set
     this.sessionDocs.clear();
   }
 
@@ -50,7 +56,6 @@ export class ElasticDoc {
       response = await this.client.scroll({ scroll_id: response.body._scroll_id, scroll: '1m' });
     }
 
-    // Batch delete docs
     for (const docId of allDocIds) {
       if (!this.sessionDocs.has(docId)) {
         responseQueue.push({
@@ -71,7 +76,84 @@ export class ElasticDoc {
       await this.client.bulk({ refresh: true, body: responseQueue });
     }
 
-    // Clear the session docs set
     this.sessionDocs.clear();
   }
+
+  async takeSnapshot(processIterator: SnapshotProcessor) {
+    const snapshotIndex = `${this.index}_snapshots`;
+  
+    const { body: indexExists } = await this.client.indices.exists({ index: snapshotIndex });
+    if (!indexExists) {
+      await this.client.indices.create({ 
+        index: snapshotIndex,
+        body: {
+          mappings: {
+            properties: {
+              date: {
+                type: 'date'
+              },
+              aggregationData: {
+                type: 'object',
+                enabled: true
+              }
+            }
+          }
+        }
+      });
+    }
+  
+    const documentIterator = this.getDocumentIterator();
+  
+    const newSnapshot = await processIterator(documentIterator, await this.getLastSnapshot());
+  
+    await this.storeSnapshot(newSnapshot);
+  }
+
+private async getLastSnapshot(): Promise<ISnapshot | null> {
+    const snapshotIndex = `${this.index}_snapshots`;
+    const { body: indexExists } = await this.client.indices.exists({ index: snapshotIndex });
+
+    if (!indexExists) {
+      return null;
+    }
+
+    const response = await this.client.search({
+      index: snapshotIndex,
+      sort: 'date:desc',
+      size: 1
+    });
+
+    if (response.body.hits.hits.length > 0) {
+      const hit = response.body.hits.hits[0];
+      return {
+        date: hit._source.date,
+        aggregationData: hit._source.aggregationData,
+      };
+    } else {
+      return null;
+    }
+  }
+
+
+  private async *getDocumentIterator() {
+    let response = await this.client.search({ index: this.index, scroll: '1m', size: this.BATCH_SIZE });
+    while (true) {
+      for (const hit of response.body.hits.hits) {
+        yield hit._source;
+      }
+
+      if (!response.body.hits.hits.length) {
+        break;
+      }
+
+      response = await this.client.scroll({ scroll_id: response.body._scroll_id, scroll: '1m' });
+    }
+  }
+
+  private async storeSnapshot(snapshot: ISnapshot) {
+    await this.client.index({
+      index: `${this.index}_snapshots`,
+      body: snapshot,
+    });
+  }
 }

ts/els.classes.kvstore.ts

@@ -0,0 +1,111 @@
+import * as plugins from './els.plugins.js';
+import { Client as ElasticClient } from '@elastic/elasticsearch';
+
+export interface IElasticKVStoreConstructorOptions {
+  index: string;
+  node: string;
+  auth?: {
+    username: string;
+    password: string;
+  };
+}
+
+export class ElasticKVStore {
+  public client: ElasticClient;
+  public index: string;
+  private readyDeferred: any;
+
+  constructor(options: IElasticKVStoreConstructorOptions) {
+    this.client = new ElasticClient({
+      node: options.node,
+      ...(options.auth && { auth: options.auth }),
+    });
+    this.index = options.index;
+    this.readyDeferred = plugins.smartpromise.defer();
+    this.setupIndex();
+  }
+
+  private async setupIndex() {
+    try {
+      const { body: indexExists } = await this.client.indices.exists({ index: this.index });
+
+      if (!indexExists) {
+        await this.client.indices.create({ 
+          index: this.index,
+          body: {
+            mappings: {
+              properties: {
+                key: {
+                  type: 'keyword'
+                },
+                value: {
+                  type: 'text'
+                }
+              }
+            }
+          }
+        });
+      }
+      this.readyDeferred.resolve();
+    } catch (err) {
+      this.readyDeferred.reject(err);
+    }
+  }
+
+  async set(key: string, value: string) {
+    await this.readyDeferred.promise;
+    await this.client.index({
+      index: this.index,
+      id: key,
+      body: {
+        key,
+        value
+      }
+    });
+  }
+
+  async get(key: string): Promise<string | null> {
+    await this.readyDeferred.promise;
+
+    try {
+      const response = await this.client.get({
+        index: this.index,
+        id: key
+      });
+      return response.body._source.value;
+    } catch (error) {
+      if (error.meta && error.meta.statusCode === 404) {
+        return null;
+      }
+      throw error;
+    }
+  }
+
+  async delete(key: string) {
+    await this.readyDeferred.promise;
+
+    try {
+      await this.client.delete({
+        index: this.index,
+        id: key
+      });
+    } catch (error) {
+      if (error.meta && error.meta.statusCode !== 404) {
+        throw error;
+      }
+    }
+  }
+
+  async clear() {
+    await this.readyDeferred.promise;
+
+    await this.client.deleteByQuery({
+      index: this.index,
+      body: {
+        query: {
+          match_all: {}
+        }
+      }
+    });
+  }
+}

ts/index.ts

@@ -1,2 +1,3 @@
 export * from './els.classes.smartlogdestination.js';
 export * from './els.classes.elasticdoc.js';
+export * from './els.classes.kvstore.js';