package.json

@@ -1,6 +1,6 @@
 {
   "name": "@apiclient.xyz/elasticsearch",
-  "version": "2.0.16",
+  "version": "1.0.54",
   "private": false,
   "description": "log to elasticsearch in a kibana compatible format",
   "main": "dist_ts/index.js",
@@ -14,20 +14,20 @@
     "buildDocs": "tsdoc"
   },
   "devDependencies": {
-    "@git.zone/tsbuild": "^2.1.70",
+    "@gitzone/tsbuild": "^2.1.66",
-    "@git.zone/tsrun": "^1.2.46",
+    "@gitzone/tsrun": "^1.2.42",
-    "@git.zone/tstest": "^1.0.80",
+    "@gitzone/tstest": "^1.0.74",
-    "@push.rocks/qenv": "^6.0.2",
+    "@pushrocks/qenv": "^5.0.2",
-    "@push.rocks/tapbundle": "^5.0.15",
+    "@pushrocks/tapbundle": "^5.0.8",
-    "@types/node": "^20.5.7"
+    "@types/node": "^20.3.3"
   },
   "dependencies": {
-    "@elastic/elasticsearch": "8.9.0",
+    "@elastic/elasticsearch": "7.17.11-patch.1",
-    "@push.rocks/lik": "^6.0.5",
+    "@pushrocks/lik": "^6.0.2",
-    "@push.rocks/smartdelay": "^3.0.5",
+    "@pushrocks/smartdelay": "^3.0.1",
-    "@push.rocks/smartlog-interfaces": "^3.0.0",
+    "@pushrocks/smartlog-interfaces": "^3.0.0",
-    "@push.rocks/smartpromise": "^4.0.2",
+    "@pushrocks/smartpromise": "^4.0.2",
-    "@push.rocks/smarttime": "^4.0.5"
+    "@pushrocks/smarttime": "^4.0.1"
   },
   "files": [
     "ts/**/*",

test/test.nonci.ts

@@ -1,5 +1,5 @@
-import { expect, tap } from '@push.rocks/tapbundle';
+import { expect, tap } from '@pushrocks/tapbundle';
-import { Qenv } from '@push.rocks/qenv';
+import { Qenv } from '@pushrocks/qenv';
 import * as elasticsearch from '../ts/index.js';
 
 let testElasticLog: elasticsearch.ElsSmartlogDestination<any>;
@@ -49,22 +49,10 @@ tap.test('should create an ElasticDoc instance', async () => {
 });
 
 tap.test('should add and update documents in a piping session', async () => {
-  await testElasticDoc.startPipingSession({});
+  await testElasticDoc.startPipingSession();
-  await testElasticDoc.pipeDocument({
+  await testElasticDoc.pipeDocument('1', { name: 'doc1' });
-    docId: '1',
+  await testElasticDoc.pipeDocument('2', { name: 'doc2' });
-    timestamp: new Date().toISOString(),
+  await testElasticDoc.pipeDocument('1', { name: 'updated doc1' });
-    doc: { name: 'doc1' }
-  });
-  await testElasticDoc.pipeDocument({
-    docId: '2',
-    timestamp: new Date().toISOString(),
-    doc: { name: 'doc2' }
-  });
-  await testElasticDoc.pipeDocument({
-    docId: '1',
-    timestamp: new Date().toISOString(),
-    doc: { name: 'updated doc1' }
-  });
 });
 
 tap.test('should delete documents not part of the piping session', async () => {

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@apiclient.xyz/elasticsearch',
-  version: '2.0.16',
+  version: '1.0.54',
   description: 'log to elasticsearch in a kibana compatible format'
 }

ts/els.classes.elasticdoc.ts

@@ -14,19 +14,12 @@ export interface ISnapshot {
   aggregationData: any;
 }
 
-export type SnapshotProcessor = (
+export type SnapshotProcessor = (iterator: AsyncIterable<any>, prevSnapshot: ISnapshot | null) => Promise<ISnapshot>;
-  iterator: AsyncIterable<any>,
-  prevSnapshot: ISnapshot | null
-) => Promise<ISnapshot>;
 
 export class ElasticDoc {
   public client: ElasticClient;
   public index: string;
   private sessionDocs: Set<string> = new Set();
-  private indexInitialized: boolean = false;
-  private latestTimestamp: string | null = null; // Store the latest timestamp
-  private onlyNew: boolean = false; // Whether to only pipe new docs
-  public fastForward: boolean = false; // Whether to fast forward to the latest timestamp
   
   private BATCH_SIZE = 1000;
 
@@ -38,109 +31,29 @@ export class ElasticDoc {
     this.index = options.index;
   }
 
-  private async ensureIndexExists(doc: any) {
+  async startPipingSession() {
-    if (!this.indexInitialized) {
-      const indexExists = await this.client.indices.exists({ index: this.index });
-      if (!indexExists) {
-        const mappings = this.createMappingsFromDoc(doc);
-        await this.client.indices.create({
-          index: this.index,
-          body: {
-            // mappings,
-            settings: {
-              // You can define the settings according to your requirements here
-            },
-          },
-        });
-      }
-      this.indexInitialized = true;
-    }
-  }
-
-  private createMappingsFromDoc(doc: any): any {
-    const properties: any = {};
-    for (const key in doc) {
-      if (key === '@timestamp') {
-        properties[key] = { type: 'date' };
-        continue;
-      }
-      properties[key] = { type: typeof doc[key] === 'number' ? 'float' : 'text' };
-    }
-    return { properties };
-  }
-
-  async startPipingSession(options: { onlyNew?: boolean }) {
     this.sessionDocs.clear();
-    this.onlyNew = options.onlyNew;
-    const indexExists = await this.client.indices.exists({ index: this.index });
-    if (this.onlyNew && indexExists) {
-      const response = await this.client.search({
-        index: this.index,
-        sort: '@timestamp:desc',
-        size: 1,
-      });
-
-      // If the search query succeeded, the index exists.
-      const hit = response.hits.hits[0];
-      this.latestTimestamp = hit?._source?.['@timestamp'] || null;
-
-      if (this.latestTimestamp) {
-        console.log(`Working in "onlyNew" mode. Hence we are omitting documents prior to ${this.latestTimestamp}`);
-      } else {
-        console.log(`Working in "onlyNew" mode, but no documents found in index ${this.index}. Hence processing all documents now.`);
-      }
-    } else if (this.onlyNew && !indexExists) {
-      console.log(`Working in "onlyNew" mode, but index ${this.index} does not exist. Hence processing all documents now.`);
-    }
   }
 
-  
+  async pipeDocument(docId: string, doc: any) {
-
+    await this.client.index({
-  async pipeDocument(optionsArg: { docId: string; timestamp?: string | number; doc: any }) {
+      index: this.index,
-    await this.ensureIndexExists(optionsArg.doc);
+      id: docId,
-
+      body: doc,
-    const documentBody = {
+    });
-      ...optionsArg.doc,
+    this.sessionDocs.add(docId);
-      ...(optionsArg.timestamp && { '@timestamp': optionsArg.timestamp }),
-    };
-
-    // If 'onlyNew' is true, compare the document timestamp with the latest timestamp
-    if (this.onlyNew) {
-      if (this.latestTimestamp && optionsArg.timestamp <= this.latestTimestamp) {
-        this.fastForward = true;
-      } else {
-        this.fastForward = false;
-        await this.client.index({
-          index: this.index,
-          id: optionsArg.docId,
-          body: documentBody,
-        });
-      }
-    } else {
-      this.fastForward = false;
-      await this.client.index({
-        index: this.index,
-        id: optionsArg.docId,
-        body: documentBody,
-      });
-    }
-    this.sessionDocs.add(optionsArg.docId);
   }
 
   async endPipingSession() {
     const allDocIds: string[] = [];
     const responseQueue = [];
-    let response = await this.client.search({
+    let response = await this.client.search({ index: this.index, scroll: '1m', size: this.BATCH_SIZE });
-      index: this.index,
-      scroll: '1m',
-      size: this.BATCH_SIZE,
-    });
     while (true) {
-      response.hits.hits.forEach((hit: any) => allDocIds.push(hit._id));
+      response.body.hits.hits.forEach((hit: any) => allDocIds.push(hit._id));
-      if (!response.hits.hits.length) {
+      if (!response.body.hits.hits.length) {
         break;
       }
-      response = await this.client.scroll({ scroll_id: response._scroll_id, scroll: '1m' });
+      response = await this.client.scroll({ scroll_id: response.body._scroll_id, scroll: '1m' });
     }
 
     for (const docId of allDocIds) {
@@ -169,7 +82,7 @@ export class ElasticDoc {
   async takeSnapshot(processIterator: SnapshotProcessor) {
     const snapshotIndex = `${this.index}_snapshots`;
   
-    const indexExists = await this.client.indices.exists({ index: snapshotIndex });
+    const { body: indexExists } = await this.client.indices.exists({ index: snapshotIndex });
     if (!indexExists) {
       await this.client.indices.create({ 
         index: snapshotIndex,
@@ -177,15 +90,15 @@ export class ElasticDoc {
           mappings: {
             properties: {
               date: {
-                type: 'date',
+                type: 'date'
               },
               aggregationData: {
                 type: 'object',
-                enabled: true,
+                enabled: true
-              },
+              }
-            },
+            }
-          },
+          }
-        },
+        }
       });
     }
   
@@ -196,9 +109,9 @@ export class ElasticDoc {
     await this.storeSnapshot(newSnapshot);
   }
 
-  private async getLastSnapshot(): Promise<ISnapshot | null> {
+private async getLastSnapshot(): Promise<ISnapshot | null> {
     const snapshotIndex = `${this.index}_snapshots`;
-    const indexExists = await this.client.indices.exists({ index: snapshotIndex });
+    const { body: indexExists } = await this.client.indices.exists({ index: snapshotIndex });
 
     if (!indexExists) {
       return null;
@@ -207,36 +120,33 @@ export class ElasticDoc {
     const response = await this.client.search({
       index: snapshotIndex,
       sort: 'date:desc',
-      size: 1,
+      size: 1
     });
 
-    if (response.hits.hits.length > 0) {
+    if (response.body.hits.hits.length > 0) {
-      const hit = response.hits.hits[0];
+      const hit = response.body.hits.hits[0];
       return {
-        date: hit._source['date'],
+        date: hit._source.date,
-        aggregationData: hit._source['aggregationData'],
+        aggregationData: hit._source.aggregationData,
       };
     } else {
       return null;
     }
   }
 
+
   private async *getDocumentIterator() {
-    let response = await this.client.search({
+    let response = await this.client.search({ index: this.index, scroll: '1m', size: this.BATCH_SIZE });
-      index: this.index,
-      scroll: '1m',
-      size: this.BATCH_SIZE,
-    });
     while (true) {
-      for (const hit of response.hits.hits) {
+      for (const hit of response.body.hits.hits) {
         yield hit._source;
       }
 
-      if (!response.hits.hits.length) {
+      if (!response.body.hits.hits.length) {
         break;
       }
 
-      response = await this.client.scroll({ scroll_id: response._scroll_id, scroll: '1m' });
+      response = await this.client.scroll({ scroll_id: response.body._scroll_id, scroll: '1m' });
     }
   }
 

ts/els.classes.elasticindex.ts

@@ -1,7 +1,7 @@
 import * as plugins from './els.plugins.js';
 import { ElsSmartlogDestination } from './els.classes.smartlogdestination.js';
-import { type ILogPackage } from '@push.rocks/smartlog-interfaces';
+import { type ILogPackage } from '@pushrocks/smartlog-interfaces';
-import { Stringmap } from '@push.rocks/lik';
+import { Stringmap } from '@pushrocks/lik';
 
 export class ElasticIndex {
   private stringmap = new Stringmap();
@@ -18,7 +18,7 @@ export class ElasticIndex {
 
     const responseArg = await this.elasticSearchRef.client.cat.indices({
       format: 'json',
-      bytes: 'mb',
+      bytes: 'm',
     }).catch(err => {
       console.log(err);
     });
@@ -27,8 +27,8 @@ export class ElasticIndex {
       throw new Error('Could not get valid response from elastic search');
     }
 
-    if (Array.isArray(responseArg)) {
+    if (Array.isArray(responseArg.body)) {
-      const filteredIndices = responseArg.filter((indexObjectArg) => {
+      const filteredIndices = responseArg.body.filter((indexObjectArg) => {
         return indexObjectArg.index.startsWith(prefixArg);
       });
       const filteredIndexNames = filteredIndices.map((indexObjectArg) => {
@@ -39,8 +39,8 @@ export class ElasticIndex {
 
     let index = null;
 
-    if (Array.isArray(responseArg)) {
+    if (Array.isArray(responseArg.body)) {
-      index = responseArg.find((indexItemArg) => {
+      index = responseArg.body.find((indexItemArg) => {
         return indexItemArg.index === indexNameArg;
       });
     }
@@ -55,7 +55,7 @@ export class ElasticIndex {
 
   public async createNewIndex(indexNameArg: string) {
     const response = await this.elasticSearchRef.client.indices.create({
-      wait_for_active_shards: 1,
+      wait_for_active_shards: '1',
       index: indexNameArg,
       body: {
         mappings: {

ts/els.classes.fastpush.ts

@@ -18,7 +18,7 @@ export class FastPush {
   async pushToIndex(indexName: string, docArray: any[], options?: FastPushOptions) {
     if (docArray.length === 0) return;
 
-    const indexExists = await this.client.indices.exists({ index: indexName });
+    const { body: indexExists } = await this.client.indices.exists({ index: indexName });
 
     if (indexExists) {
       if (options?.deleteIndex) {

ts/els.classes.kvstore.ts

@@ -27,7 +27,7 @@ export class ElasticKVStore {
 
   private async setupIndex() {
     try {
-      const indexExists = await this.client.indices.exists({ index: this.index });
+      const { body: indexExists } = await this.client.indices.exists({ index: this.index });
 
       if (!indexExists) {
         await this.client.indices.create({ 
@@ -72,7 +72,7 @@ export class ElasticKVStore {
         index: this.index,
         id: key
       });
-      return response._source['value'];
+      return response.body._source.value;
     } catch (error) {
       if (error.meta && error.meta.statusCode === 404) {
         return null;

ts/els.classes.smartlogdestination.ts

@@ -1,5 +1,8 @@
+// interfaces
 import { Client as ElasticClient } from '@elastic/elasticsearch';
-import type { ILogContext, ILogPackage, ILogDestination } from '@push.rocks/smartlog-interfaces';
+import type { ILogContext, ILogPackage, ILogDestination } from '@pushrocks/smartlog-interfaces';
+
+// other classes
 import { ElasticScheduler } from './els.classes.elasticscheduler.js';
 import { ElasticIndex } from './els.classes.elasticindex.js';
 
@@ -26,6 +29,10 @@ export class ElsSmartlogDestination<T> {
   public indexPrefix: string;
   public indexRetention: number;
 
+  /**
+   * sets up an instance of Elastic log
+   * @param optionsArg
+   */
   constructor(optionsArg: IElasticSearchConstructorOptions) {
     this.client = new ElasticClient({
       node: optionsArg.node,
@@ -33,34 +40,58 @@ export class ElsSmartlogDestination<T> {
     });
     this.indexPrefix = `${optionsArg.indexPrefix}`;
     this.indexRetention = optionsArg.indexRetention;
-    this.setupDataStream();
-  }
-
-  private async setupDataStream() {
-    // Define an index template
-    await this.client.indices.putIndexTemplate({
-      name: `${this.indexPrefix}_template`,
-      index_patterns: [`${this.indexPrefix}-*`],
-      data_stream: {},
-    });
   }
 
   public async log(logPackageArg: ILogPackage, scheduleOverwrite = false) {
     const now = new Date();
-    const indexToUse = `${this.indexPrefix}-data-stream`; // Use data stream name
+    const indexToUse = `${this.indexPrefix}-${now.toISOString().split('T')[0]}`;
 
     if (this.elasticScheduler.docsScheduled && !scheduleOverwrite) {
       this.elasticScheduler.scheduleDoc(logPackageArg);
       return;
     }
 
-    await this.client.index(
+    // Make sure the index is created with a mapping for dynamic JSON
+    const indexExists = await this.client.indices.exists({ index: indexToUse });
+    if (!indexExists.body) {
+      await this.client.indices.create({
+        index: indexToUse,
+        body: {
+          mappings: {
+            properties: {
+              '@timestamp': {
+                type: 'date',
+              },
+              logPackageArg: {
+                properties: {
+                  payload: {
+                    type: 'object',
+                    dynamic: true
+                  }
+                }
+              },
+            },
+          },
+        },
+      });
+    }
+
+    this.client.index(
       {
         index: indexToUse,
         body: {
           '@timestamp': new Date(logPackageArg.timestamp).toISOString(),
           ...logPackageArg,
         },
+      },
+      (error, response) => {
+        if (error) {
+          console.log('ElasticLog encountered an error:');
+          console.log(error);
+          this.elasticScheduler.addFailedDoc(logPackageArg);
+        } else {
+          // console.log(`ElasticLog: ${logPackageArg.message}`);
+        }
       }
     );
   }
@@ -68,7 +99,7 @@ export class ElsSmartlogDestination<T> {
   get logDestination(): ILogDestination {
     return {
       handleLog: async (smartlogPackageArg: ILogPackage) => {
-        await this.log(smartlogPackageArg);
+        this.log(smartlogPackageArg);
       },
     };
   }

ts/els.plugins.ts

@@ -1,8 +1,8 @@
 import * as elasticsearch from '@elastic/elasticsearch';
-import * as lik from '@push.rocks/lik';
+import * as lik from '@pushrocks/lik';
-import * as smartdelay from '@push.rocks/smartdelay';
+import * as smartdelay from '@pushrocks/smartdelay';
-import * as smartlogInterfaces from '@push.rocks/smartlog-interfaces';
+import * as smartlogInterfaces from '@pushrocks/smartlog-interfaces';
-import * as smartpromise from '@push.rocks/smartpromise';
+import * as smartpromise from '@pushrocks/smartpromise';
-import * as smarttime from '@push.rocks/smarttime';
+import * as smarttime from '@pushrocks/smarttime';
 
 export { elasticsearch, lik, smartdelay, smartlogInterfaces, smartpromise, smarttime };