2.0.5

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@apiclient.xyz/elasticsearch",
-  "version": "1.0.48",
+  "version": "2.0.5",
   "private": false,
   "description": "log to elasticsearch in a kibana compatible format",
   "main": "dist_ts/index.js",

test/test.nonci.ts

@@ -49,10 +49,22 @@ tap.test('should create an ElasticDoc instance', async () => {
 });
 
 tap.test('should add and update documents in a piping session', async () => {
-  await testElasticDoc.startPipingSession();
+  await testElasticDoc.startPipingSession({});
-  await testElasticDoc.pipeDocument('1', { name: 'doc1' });
+  await testElasticDoc.pipeDocument({
-  await testElasticDoc.pipeDocument('2', { name: 'doc2' });
+    docId: '1',
-  await testElasticDoc.pipeDocument('1', { name: 'updated doc1' });
+    timestamp: new Date().toISOString(),
+    doc: { name: 'doc1' }
+  });
+  await testElasticDoc.pipeDocument({
+    docId: '2',
+    timestamp: new Date().toISOString(),
+    doc: { name: 'doc2' }
+  });
+  await testElasticDoc.pipeDocument({
+    docId: '1',
+    timestamp: new Date().toISOString(),
+    doc: { name: 'updated doc1' }
+  });
 });
 
 tap.test('should delete documents not part of the piping session', async () => {

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@apiclient.xyz/elasticsearch',
-  version: '1.0.48',
+  version: '2.0.5',
   description: 'log to elasticsearch in a kibana compatible format'
 }

ts/els.classes.elasticdoc.ts

@@ -14,13 +14,19 @@ export interface ISnapshot {
   aggregationData: any;
 }
 
-export type SnapshotProcessor = (iterator: AsyncIterable<any>, prevSnapshot: ISnapshot | null) => Promise<ISnapshot>;
+export type SnapshotProcessor = (
+  iterator: AsyncIterable<any>,
+  prevSnapshot: ISnapshot | null
+) => Promise<ISnapshot>;
 
 export class ElasticDoc {
   public client: ElasticClient;
   public index: string;
   private sessionDocs: Set<string> = new Set();
-  
+  private indexInitialized: boolean = false;
+  private latestTimestamp: string | null = null; // Store the latest timestamp
+  private onlyNew: boolean = false; // Whether to only pipe new docs
+
   private BATCH_SIZE = 1000;
 
   constructor(options: IElasticDocConstructorOptions) {
@@ -31,23 +37,105 @@ export class ElasticDoc {
     this.index = options.index;
   }
 
-  async startPipingSession() {
+  private async ensureIndexExists(doc: any) {
-    this.sessionDocs.clear();
+    if (!this.indexInitialized) {
+      const { body: indexExists } = await this.client.indices.exists({ index: this.index });
+      if (!indexExists) {
+        const mappings = this.createMappingsFromDoc(doc);
+        await this.client.indices.create({
+          index: this.index,
+          body: {
+            mappings,
+            settings: {
+              // You can define the settings according to your requirements here
+            },
+          },
+        });
+      }
+      this.indexInitialized = true;
+    }
   }
 
-  async pipeDocument(docId: string, doc: any) {
+  private createMappingsFromDoc(doc: any): any {
-    await this.client.index({
+    const properties: any = {};
-      index: this.index,
+    for (const key in doc) {
-      id: docId,
+      if (key === '@timestamp') {
-      body: doc,
+        properties[key] = { type: 'date' };
-    });
+        continue;
-    this.sessionDocs.add(docId);
+      }
+      properties[key] = { type: typeof doc[key] === 'number' ? 'float' : 'text' };
+    }
+    return { properties };
+  }
+
+  async startPipingSession(options: { onlyNew?: boolean }) {
+    this.sessionDocs.clear();
+    this.onlyNew = options.onlyNew;
+  
+    if (this.onlyNew) {
+      try {
+        const response = await this.client.search({
+          index: this.index,
+          sort: '@timestamp:desc',
+          size: 1,
+        });
+  
+        // If the search query succeeded, the index exists.
+        const hit = response.body.hits.hits[0];
+        this.latestTimestamp = hit?._source?.['@timestamp'] || null;
+  
+        if (this.latestTimestamp) {
+          console.log(`Working in "onlyNew" mode. Hence we are omitting documents prior to ${this.latestTimestamp}`);
+        } else {
+          console.log(`Working in "onlyNew" mode, but no documents found in index ${this.index}. Hence processing all documents now.`);
+        }
+      } catch (e) {
+        // If the search query failed, the index likely doesn't exist or some other error occurred.
+        if (e.meta && e.meta.statusCode === 404) {
+          console.log(`Index ${this.index} does not exist. Working in "onlyNew" mode, but will process all documents as the index is empty.`);
+        } else {
+          console.log(`An error occurred while trying to retrieve the latest timestamp: ${e}`);
+        }
+  
+        this.latestTimestamp = null;
+      }
+    }
+  }
+  
+  
+
+  async pipeDocument(optionsArg: { docId: string; timestamp?: string | number; doc: any }) {
+    await this.ensureIndexExists(optionsArg.doc);
+
+    const documentBody = {
+      ...optionsArg.doc,
+      ...(optionsArg.timestamp && { '@timestamp': optionsArg.timestamp }),
+    };
+
+    // If 'onlyNew' is true, compare the document timestamp with the latest timestamp
+    if (this.onlyNew) {
+      if (this.latestTimestamp && optionsArg.timestamp <= this.latestTimestamp) {
+        // Omit the document
+        return;
+      } else {
+        await this.client.index({
+          index: this.index,
+          id: optionsArg.docId,
+          body: documentBody,
+        });
+      }
+    }
+    this.sessionDocs.add(optionsArg.docId);
   }
 
   async endPipingSession() {
     const allDocIds: string[] = [];
     const responseQueue = [];
-    let response = await this.client.search({ index: this.index, scroll: '1m', size: this.BATCH_SIZE });
+    let response = await this.client.search({
+      index: this.index,
+      scroll: '1m',
+      size: this.BATCH_SIZE,
+    });
     while (true) {
       response.body.hits.hits.forEach((hit: any) => allDocIds.push(hit._id));
       if (!response.body.hits.hits.length) {
@@ -81,35 +169,35 @@ export class ElasticDoc {
 
   async takeSnapshot(processIterator: SnapshotProcessor) {
     const snapshotIndex = `${this.index}_snapshots`;
-  
+
     const { body: indexExists } = await this.client.indices.exists({ index: snapshotIndex });
     if (!indexExists) {
-      await this.client.indices.create({ 
+      await this.client.indices.create({
         index: snapshotIndex,
         body: {
           mappings: {
             properties: {
               date: {
-                type: 'date'
+                type: 'date',
               },
               aggregationData: {
                 type: 'object',
-                enabled: true
+                enabled: true,
-              }
+              },
-            }
+            },
-          }
+          },
-        }
+        },
       });
     }
-  
+
     const documentIterator = this.getDocumentIterator();
-  
+
     const newSnapshot = await processIterator(documentIterator, await this.getLastSnapshot());
-  
+
     await this.storeSnapshot(newSnapshot);
   }
 
-private async getLastSnapshot(): Promise<ISnapshot | null> {
+  private async getLastSnapshot(): Promise<ISnapshot | null> {
     const snapshotIndex = `${this.index}_snapshots`;
     const { body: indexExists } = await this.client.indices.exists({ index: snapshotIndex });
 
@@ -120,7 +208,7 @@ private async getLastSnapshot(): Promise<ISnapshot | null> {
     const response = await this.client.search({
       index: snapshotIndex,
       sort: 'date:desc',
-      size: 1
+      size: 1,
     });
 
     if (response.body.hits.hits.length > 0) {
@@ -134,9 +222,12 @@ private async getLastSnapshot(): Promise<ISnapshot | null> {
     }
   }
 
-
   private async *getDocumentIterator() {
-    let response = await this.client.search({ index: this.index, scroll: '1m', size: this.BATCH_SIZE });
+    let response = await this.client.search({
+      index: this.index,
+      scroll: '1m',
+      size: this.BATCH_SIZE,
+    });
     while (true) {
       for (const hit of response.body.hits.hits) {
         yield hit._source;

ts/els.classes.elasticindex.ts

@@ -1,4 +1,4 @@
-import * as plugins from './elasticsearch.plugins.js';
+import * as plugins from './els.plugins.js';
 import { ElsSmartlogDestination } from './els.classes.smartlogdestination.js';
 import { type ILogPackage } from '@pushrocks/smartlog-interfaces';
 import { Stringmap } from '@pushrocks/lik';

ts/els.classes.fastpush.ts

@@ -0,0 +1,68 @@
+import { Client as ElasticClient } from '@elastic/elasticsearch';
+
+interface FastPushOptions {
+  deleteOldData?: boolean;     // Clear the index
+  deleteIndex?: boolean;       // Delete the entire index
+}
+
+export class FastPush {
+  private client: ElasticClient;
+
+  constructor(node: string, auth?: { username: string; password: string }) {
+    this.client = new ElasticClient({
+      node: node,
+      ...(auth && { auth: auth }),
+    });
+  }
+
+  async pushToIndex(indexName: string, docArray: any[], options?: FastPushOptions) {
+    if (docArray.length === 0) return;
+
+    const { body: indexExists } = await this.client.indices.exists({ index: indexName });
+
+    if (indexExists) {
+      if (options?.deleteIndex) {
+        await this.client.indices.delete({ index: indexName });
+      } else if (options?.deleteOldData) {
+        await this.client.deleteByQuery({
+          index: indexName,
+          body: {
+            query: {
+              match_all: {}
+            }
+          }
+        });
+      }
+    }
+
+    if (!indexExists || options?.deleteIndex) {
+      // Create index with mappings (for simplicity, we use dynamic mapping)
+      await this.client.indices.create({
+        index: indexName,
+        body: {
+          mappings: {
+            dynamic: "true"
+            // ... other specific mappings
+          },
+        },
+      });
+    }
+
+    // Bulk insert documents
+    const bulkBody = [];
+    for (const doc of docArray) {
+      bulkBody.push({
+        index: {
+          _index: indexName,
+        },
+      });
+      bulkBody.push(doc);
+    }
+
+    await this.client.bulk({ body: bulkBody });
+  }
+}
+
+// Usage example:
+// const fastPush = new FastPush('http://localhost:9200', { username: 'elastic', password: 'password' });
+// fastPush.pushToIndex('my_index', [{ name: 'John', age: 30 }, { name: 'Jane', age: 25 }], { deleteOldData: true });

ts/els.classes.kvstore.ts

@@ -0,0 +1,111 @@
+import * as plugins from './els.plugins.js';
+import { Client as ElasticClient } from '@elastic/elasticsearch';
+
+export interface IElasticKVStoreConstructorOptions {
+  index: string;
+  node: string;
+  auth?: {
+    username: string;
+    password: string;
+  };
+}
+
+export class ElasticKVStore {
+  public client: ElasticClient;
+  public index: string;
+  private readyDeferred: any;
+
+  constructor(options: IElasticKVStoreConstructorOptions) {
+    this.client = new ElasticClient({
+      node: options.node,
+      ...(options.auth && { auth: options.auth }),
+    });
+    this.index = options.index;
+    this.readyDeferred = plugins.smartpromise.defer();
+    this.setupIndex();
+  }
+
+  private async setupIndex() {
+    try {
+      const { body: indexExists } = await this.client.indices.exists({ index: this.index });
+
+      if (!indexExists) {
+        await this.client.indices.create({ 
+          index: this.index,
+          body: {
+            mappings: {
+              properties: {
+                key: {
+                  type: 'keyword'
+                },
+                value: {
+                  type: 'text'
+                }
+              }
+            }
+          }
+        });
+      }
+      this.readyDeferred.resolve();
+    } catch (err) {
+      this.readyDeferred.reject(err);
+    }
+  }
+
+  async set(key: string, value: string) {
+    await this.readyDeferred.promise;
+    await this.client.index({
+      index: this.index,
+      id: key,
+      body: {
+        key,
+        value
+      }
+    });
+  }
+
+  async get(key: string): Promise<string | null> {
+    await this.readyDeferred.promise;
+
+    try {
+      const response = await this.client.get({
+        index: this.index,
+        id: key
+      });
+      return response.body._source.value;
+    } catch (error) {
+      if (error.meta && error.meta.statusCode === 404) {
+        return null;
+      }
+      throw error;
+    }
+  }
+
+  async delete(key: string) {
+    await this.readyDeferred.promise;
+
+    try {
+      await this.client.delete({
+        index: this.index,
+        id: key
+      });
+    } catch (error) {
+      if (error.meta && error.meta.statusCode !== 404) {
+        throw error;
+      }
+    }
+  }
+
+  async clear() {
+    await this.readyDeferred.promise;
+
+    await this.client.deleteByQuery({
+      index: this.index,
+      body: {
+        query: {
+          match_all: {}
+        }
+      }
+    });
+  }
+}

ts/els.classes.smartlogdestination.ts

@@ -3,8 +3,8 @@ import { Client as ElasticClient } from '@elastic/elasticsearch';
 import type { ILogContext, ILogPackage, ILogDestination } from '@pushrocks/smartlog-interfaces';
 
 // other classes
-import { ElasticScheduler } from './elasticsearch.classes.elasticscheduler.js';
+import { ElasticScheduler } from './els.classes.elasticscheduler.js';
-import { ElasticIndex } from './elasticsearch.classes.elasticindex.js';
+import { ElasticIndex } from './els.classes.elasticindex.js';
 
 export interface IStandardLogParams {
   message: string;

ts/index.ts

@@ -1,2 +1,4 @@
 export * from './els.classes.smartlogdestination.js';
-export * from './els.classes.elasticdoc.js';
+export * from './els.classes.fastpush.js';
+export * from './els.classes.elasticdoc.js';
+export * from './els.classes.kvstore.js';