2.0.16

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@apiclient.xyz/elasticsearch",
-  "version": "2.0.7",
+  "version": "2.0.16",
   "private": false,
   "description": "log to elasticsearch in a kibana compatible format",
   "main": "dist_ts/index.js",

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@apiclient.xyz/elasticsearch',
-  version: '2.0.7',
+  version: '2.0.16',
   description: 'log to elasticsearch in a kibana compatible format'
 }

ts/els.classes.elasticdoc.ts

@@ -26,6 +26,7 @@ export class ElasticDoc {
   private indexInitialized: boolean = false;
   private latestTimestamp: string | null = null; // Store the latest timestamp
   private onlyNew: boolean = false; // Whether to only pipe new docs
+  public fastForward: boolean = false; // Whether to fast forward to the latest timestamp
 
   private BATCH_SIZE = 1000;
 
@@ -45,7 +46,7 @@ export class ElasticDoc {
         await this.client.indices.create({
           index: this.index,
           body: {
-            mappings,
+            // mappings,
             settings: {
               // You can define the settings according to your requirements here
             },
@@ -71,34 +72,25 @@ export class ElasticDoc {
   async startPipingSession(options: { onlyNew?: boolean }) {
     this.sessionDocs.clear();
     this.onlyNew = options.onlyNew;
-  
-    if (this.onlyNew) {
-      try {
-        const response = await this.client.search({
-          index: this.index,
-          sort: '@timestamp:desc',
-          size: 1,
-        });
-  
-        // If the search query succeeded, the index exists.
-        const hit = response.hits.hits[0];
-        this.latestTimestamp = hit?._source?.['@timestamp'] || null;
-  
-        if (this.latestTimestamp) {
-          console.log(`Working in "onlyNew" mode. Hence we are omitting documents prior to ${this.latestTimestamp}`);
-        } else {
-          console.log(`Working in "onlyNew" mode, but no documents found in index ${this.index}. Hence processing all documents now.`);
-        }
-      } catch (e) {
-        // If the search query failed, the index likely doesn't exist or some other error occurred.
-        if (e.meta && e.meta.statusCode === 404) {
-          console.log(`Index ${this.index} does not exist. Working in "onlyNew" mode, but will process all documents as the index is empty.`);
-        } else {
-          console.log(`An error occurred while trying to retrieve the latest timestamp: ${e}`);
-        }
-  
-        this.latestTimestamp = null;
+    const indexExists = await this.client.indices.exists({ index: this.index });
+    if (this.onlyNew && indexExists) {
+      const response = await this.client.search({
+        index: this.index,
+        sort: '@timestamp:desc',
+        size: 1,
+      });
+
+      // If the search query succeeded, the index exists.
+      const hit = response.hits.hits[0];
+      this.latestTimestamp = hit?._source?.['@timestamp'] || null;
+
+      if (this.latestTimestamp) {
+        console.log(`Working in "onlyNew" mode. Hence we are omitting documents prior to ${this.latestTimestamp}`);
+      } else {
+        console.log(`Working in "onlyNew" mode, but no documents found in index ${this.index}. Hence processing all documents now.`);
       }
+    } else if (this.onlyNew && !indexExists) {
+      console.log(`Working in "onlyNew" mode, but index ${this.index} does not exist. Hence processing all documents now.`);
     }
   }
   
@@ -115,15 +107,22 @@ export class ElasticDoc {
     // If 'onlyNew' is true, compare the document timestamp with the latest timestamp
     if (this.onlyNew) {
       if (this.latestTimestamp && optionsArg.timestamp <= this.latestTimestamp) {
-        // Omit the document
-        return;
+        this.fastForward = true;
       } else {
+        this.fastForward = false;
         await this.client.index({
           index: this.index,
           id: optionsArg.docId,
           body: documentBody,
         });
       }
+    } else {
+      this.fastForward = false;
+      await this.client.index({
+        index: this.index,
+        id: optionsArg.docId,
+        body: documentBody,
+      });
     }
     this.sessionDocs.add(optionsArg.docId);
   }
@@ -213,7 +212,6 @@ export class ElasticDoc {
 
     if (response.hits.hits.length > 0) {
       const hit = response.hits.hits[0];
-      console.log(hit);
       return {
         date: hit._source['date'],
         aggregationData: hit._source['aggregationData'],

ts/els.classes.fastpush.ts

@@ -18,7 +18,7 @@ export class FastPush {
   async pushToIndex(indexName: string, docArray: any[], options?: FastPushOptions) {
     if (docArray.length === 0) return;
 
-    const { body: indexExists } = await this.client.indices.exists({ index: indexName });
+    const indexExists = await this.client.indices.exists({ index: indexName });
 
     if (indexExists) {
       if (options?.deleteIndex) {

ts/els.classes.kvstore.ts

@@ -27,7 +27,7 @@ export class ElasticKVStore {
 
   private async setupIndex() {
     try {
-      const { body: indexExists } = await this.client.indices.exists({ index: this.index });
+      const indexExists = await this.client.indices.exists({ index: this.index });
 
       if (!indexExists) {
         await this.client.indices.create({ 
@@ -72,7 +72,7 @@ export class ElasticKVStore {
         index: this.index,
         id: key
       });
-      return response.body._source.value;
+      return response._source['value'];
     } catch (error) {
       if (error.meta && error.meta.statusCode === 404) {
         return null;

ts/els.classes.smartlogdestination.ts

@@ -54,7 +54,7 @@ export class ElsSmartlogDestination<T> {
       return;
     }
 
-    this.client.index(
+    await this.client.index(
       {
         index: indexToUse,
         body: {
@@ -68,7 +68,7 @@ export class ElsSmartlogDestination<T> {
   get logDestination(): ILogDestination {
     return {
       handleLog: async (smartlogPackageArg: ILogPackage) => {
-        this.log(smartlogPackageArg);
+        await this.log(smartlogPackageArg);
       },
     };
   }