1.0.50

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@apiclient.xyz/elasticsearch",
-  "version": "1.0.37",
+  "version": "1.0.50",
   "private": false,
   "description": "log to elasticsearch in a kibana compatible format",
   "main": "dist_ts/index.js",
@@ -22,13 +22,12 @@
     "@types/node": "^20.3.3"
   },
   "dependencies": {
+    "@elastic/elasticsearch": "7.17.11-patch.1",
     "@pushrocks/lik": "^6.0.2",
     "@pushrocks/smartdelay": "^3.0.1",
     "@pushrocks/smartlog-interfaces": "^3.0.0",
     "@pushrocks/smartpromise": "^4.0.2",
-    "@pushrocks/smarttime": "^4.0.1",
+    "@pushrocks/smarttime": "^4.0.1"
-    "@types/elasticsearch": "^5.0.40",
-    "elasticsearch": "^16.7.3"
   },
   "files": [
     "ts/**/*",

pnpm-lock.yaml

@@ -5,6 +5,9 @@ settings:
   excludeLinksFromLockfile: false
 
 dependencies:
+  '@elastic/elasticsearch':
+    specifier: 7.17.11-patch.1
+    version: 7.17.11-patch.1
   '@pushrocks/lik':
     specifier: ^6.0.2
     version: 6.0.2
@@ -20,12 +23,6 @@ dependencies:
   '@pushrocks/smarttime':
     specifier: ^4.0.1
     version: 4.0.1
-  '@types/elasticsearch':
-    specifier: ^5.0.40
-    version: 5.0.40
-  elasticsearch:
-    specifier: ^16.7.3
-    version: 16.7.3
 
 devDependencies:
   '@gitzone/tsbuild':
@@ -124,6 +121,18 @@ packages:
       '@jridgewell/trace-mapping': 0.3.9
     dev: true
 
+  /@elastic/elasticsearch@7.17.11-patch.1:
+    resolution: {integrity: sha512-t0twlzgC+fCgvw1s8jI2GFO3rpxV/V2gHTr3iHeGBDNeQVsRZ+6Psy8P7SEXW/uTvxuDfmUF6UShHhK1uHXyow==}
+    engines: {node: '>=12'}
+    dependencies:
+      debug: 4.3.4
+      hpagent: 0.1.2
+      ms: 2.1.3
+      secure-json-parse: 2.7.0
+    transitivePeerDependencies:
+      - supports-color
+    dev: false
+
   /@esbuild/android-arm64@0.17.19:
     resolution: {integrity: sha512-KBMWvEZooR7+kzY0BtbTQn0OAYY7CsiydT63pVEaPtVYF0hXbUaOyZog37DKxK7NF3XacBJOpYT4adIJh+avxA==}
     engines: {node: '>=12'}
@@ -1232,10 +1241,6 @@ packages:
     resolution: {integrity: sha512-tpu0hp+AOIzwdAHyZPzLE5pCf9uT0pb+xZ76T4S7MrY2YTVq918Q7Q2VQ3KCVQqYxM7nxuCK/SL3X97jBEIeKQ==}
     dev: true
 
-  /@types/elasticsearch@5.0.40:
-    resolution: {integrity: sha512-lhnbkC0XorAD7Dt7X+94cXUSHEdDNnEVk/DgFLHgIZQNhixV631Lj4+KpXunTT5rCHyj9RqK3TfO7QrOiwEeUQ==}
-    dev: false
-
   /@types/express-serve-static-core@4.17.35:
     resolution: {integrity: sha512-wALWQwrgiB2AWTT91CB62b6Yt0sNHpznUXeZEcnPU3DRdlDIz74x8Qg1UUYKSVFi+va5vKOLYRBI1bRKiLLKIg==}
     dependencies:
@@ -1568,6 +1573,11 @@ packages:
       - utf-8-validate
     dev: true
 
+  /@xmldom/xmldom@0.8.8:
+    resolution: {integrity: sha512-0LNz4EY8B/8xXY86wMrQ4tz6zEHZv9ehFMJPm8u2gq5lQ71cfRKdaKyxfJAx5aUoyzx0qzgURblTisPGgz3d+Q==}
+    engines: {node: '>=10.0.0'}
+    dev: true
+
   /accepts@1.3.8:
     resolution: {integrity: sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==}
     engines: {node: '>= 0.6'}
@@ -1596,13 +1606,6 @@ packages:
       - supports-color
     dev: true
 
-  /agentkeepalive@3.5.2:
-    resolution: {integrity: sha512-e0L/HNe6qkQ7H19kTlRRqUibEAwDK5AFk6y3PtMsuut2VAH6+Q4xZml1tNDJD7kSAyqmbG/K08K5WEJYtUrSlQ==}
-    engines: {node: '>= 4.0.0'}
-    dependencies:
-      humanize-ms: 1.2.1
-    dev: false
-
   /agentkeepalive@4.3.0:
     resolution: {integrity: sha512-7Epl1Blf4Sy37j4v9f9FjICCh4+KAQOyXgHEwlyBiAQLbhKdq/i2QQU3amQalS/wPhdPzDXPL5DMR5bkn+YeWg==}
     engines: {node: '>= 8.0.0'}
@@ -1634,11 +1637,6 @@ packages:
       type-fest: 0.21.3
     dev: true
 
-  /ansi-regex@2.1.1:
-    resolution: {integrity: sha1-w7M6te42DYbg5ijwRorn7yfWVN8=}
-    engines: {node: '>=0.10.0'}
-    dev: false
-
   /ansi-regex@5.0.1:
     resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==}
     engines: {node: '>=8'}
@@ -1649,11 +1647,6 @@ packages:
     engines: {node: '>=12'}
     dev: true
 
-  /ansi-styles@2.2.1:
-    resolution: {integrity: sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=}
-    engines: {node: '>=0.10.0'}
-    dev: false
-
   /ansi-styles@3.2.1:
     resolution: {integrity: sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==}
     engines: {node: '>=4'}
@@ -1886,17 +1879,6 @@ packages:
       axe-core: 4.7.2
     dev: true
 
-  /chalk@1.1.3:
-    resolution: {integrity: sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=}
-    engines: {node: '>=0.10.0'}
-    dependencies:
-      ansi-styles: 2.2.1
-      escape-string-regexp: 1.0.5
-      has-ansi: 2.0.0
-      strip-ansi: 3.0.1
-      supports-color: 2.0.0
-    dev: false
-
   /chalk@2.4.2:
     resolution: {integrity: sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==}
     engines: {node: '>=4'}
@@ -2131,7 +2113,6 @@ packages:
         optional: true
     dependencies:
       ms: 2.1.2
-    dev: true
 
   /decompress-response@6.0.0:
     resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==}
@@ -2234,16 +2215,6 @@ packages:
     resolution: {integrity: sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=}
     dev: true
 
-  /elasticsearch@16.7.3:
-    resolution: {integrity: sha512-e9kUNhwnIlu47fGAr4W6yZJbkpsgQJB0TqNK8rCANe1J4P65B1sGnbCFTgcKY3/dRgCWnuP1AJ4obvzW604xEQ==}
-    engines: {node: '>=0.8'}
-    deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.
-    dependencies:
-      agentkeepalive: 3.5.2
-      chalk: 1.1.3
-      lodash: 4.17.21
-    dev: false
-
   /emoji-regex@8.0.0:
     resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==}
     dev: true
@@ -2371,6 +2342,7 @@ packages:
   /escape-string-regexp@1.0.5:
     resolution: {integrity: sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=}
     engines: {node: '>=0.8.0'}
+    dev: true
 
   /escape-string-regexp@4.0.0:
     resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==}
@@ -2704,13 +2676,6 @@ packages:
     resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==}
     dev: true
 
-  /has-ansi@2.0.0:
-    resolution: {integrity: sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=}
-    engines: {node: '>=0.10.0'}
-    dependencies:
-      ansi-regex: 2.1.1
-    dev: false
-
   /has-flag@3.0.0:
     resolution: {integrity: sha1-tdRU3CGZriJWmfNGfloH87lVuv0=}
     engines: {node: '>=4'}
@@ -2752,6 +2717,10 @@ packages:
     hasBin: true
     dev: true
 
+  /hpagent@0.1.2:
+    resolution: {integrity: sha512-ePqFXHtSQWAFXYmj+JtOTHr84iNrII4/QRlAAPPE+zqnKy4xJo7Ie1Y4kC7AdB+LxLxSTTzBMASsEcy0q8YyvQ==}
+    dev: false
+
   /html-escaper@2.0.2:
     resolution: {integrity: sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==}
     dev: true
@@ -2836,6 +2805,7 @@ packages:
     resolution: {integrity: sha1-xG4xWaKT9riW2ikxbYtv6Lt5u+0=}
     dependencies:
       ms: 2.1.3
+    dev: true
 
   /iconv-lite@0.4.24:
     resolution: {integrity: sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==}
@@ -3253,6 +3223,7 @@ packages:
 
   /lodash@4.17.21:
     resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==}
+    dev: true
 
   /log-update@4.0.0:
     resolution: {integrity: sha512-9fkkDevMefjg0mmzWFBW8YkFP91OrizzkW3diF7CpG+S2EYdy4+TVfGwz1zeF8x7hCx1ovSPTOE9Ngib74qqUg==}
@@ -3421,7 +3392,6 @@ packages:
 
   /ms@2.1.2:
     resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==}
-    dev: true
 
   /ms@2.1.3:
     resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==}
@@ -3637,6 +3607,8 @@ packages:
     resolution: {integrity: sha512-mXF9AIgnvq1DP/ZM2R28tAfxP2wKZHYa2DjV0R1KCwcqSzm5Iqh1XQq9rdfAt6dp2DuPP0VHZIaCALc2v1cL5A==}
     engines: {node: '>=14.18.0', npm: '>=6.14.15'}
     hasBin: true
+    dependencies:
+      '@xmldom/xmldom': 0.8.8
     dev: true
     bundledDependencies:
       - '@xmldom/xmldom'
@@ -3934,6 +3906,10 @@ packages:
     resolution: {integrity: sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==}
     dev: true
 
+  /secure-json-parse@2.7.0:
+    resolution: {integrity: sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==}
+    dev: false
+
   /semver@6.3.0:
     resolution: {integrity: sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==}
     hasBin: true
@@ -4143,13 +4119,6 @@ packages:
       safe-buffer: 5.2.1
     dev: true
 
-  /strip-ansi@3.0.1:
-    resolution: {integrity: sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=}
-    engines: {node: '>=0.10.0'}
-    dependencies:
-      ansi-regex: 2.1.1
-    dev: false
-
   /strip-ansi@6.0.1:
     resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==}
     engines: {node: '>=8'}
@@ -4175,11 +4144,6 @@ packages:
     resolution: {integrity: sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==}
     dev: true
 
-  /supports-color@2.0.0:
-    resolution: {integrity: sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=}
-    engines: {node: '>=0.8.0'}
-    dev: false
-
   /supports-color@5.5.0:
     resolution: {integrity: sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==}
     engines: {node: '>=4'}

qenv.yml

@@ -1,5 +1,2 @@
 required:
-  - ELK_DOMAIN
+  
-  - ELK_PORT
-  - ELK_USER
-  - ELK_PASS

test/test.nonci.ts

@@ -2,23 +2,24 @@ import { expect, tap } from '@pushrocks/tapbundle';
 import { Qenv } from '@pushrocks/qenv';
 import * as elasticsearch from '../ts/index.js';
 
-const testQenv = new Qenv('./', './.nogit/');
+let testElasticLog: elasticsearch.ElsSmartlogDestination<any>;
-
+let testElasticDoc: elasticsearch.ElasticDoc;
-let testElasticLog: elasticsearch.ElasticSearch<any>;
 
 tap.test('first test', async () => {
-  testElasticLog = new elasticsearch.ElasticSearch({
+  testElasticLog = new elasticsearch.ElsSmartlogDestination({
-    indexPrefix: 'smartlog',
+    indexPrefix: 'testprefix',
     indexRetention: 7,
-    domain: testQenv.getEnvVarOnDemand('ELK_DOMAIN'),
+    node: 'http://localhost:9200',
-    port: parseInt(testQenv.getEnvVarOnDemand('ELK_PORT'), 10),
+    auth: {
-    ssl: true,
+      username: 'elastic',
+      password: 'YourPassword'
+    }
   });
-  expect(testElasticLog).toBeInstanceOf(elasticsearch.ElasticSearch);
+  expect(testElasticLog).toBeInstanceOf(elasticsearch.ElsSmartlogDestination);
 });
 
 tap.test('should send a message to Elasticsearch', async () => {
-  testElasticLog.log({
+  await testElasticLog.log({
     timestamp: Date.now(),
     type: 'increment',
     level: 'info',
@@ -35,4 +36,42 @@ tap.test('should send a message to Elasticsearch', async () => {
   });
 });
 
+tap.test('should create an ElasticDoc instance', async () => {
+  testElasticDoc = new elasticsearch.ElasticDoc({
+    index: 'testindex',
+    node: 'http://localhost:9200',
+    auth: {
+      username: 'elastic',
+      password: 'YourPassword'
+    }
+  });
+  expect(testElasticDoc).toBeInstanceOf(elasticsearch.ElasticDoc);
+});
+
+tap.test('should add and update documents in a piping session', async () => {
+  await testElasticDoc.startPipingSession();
+  await testElasticDoc.pipeDocument('1', { name: 'doc1' });
+  await testElasticDoc.pipeDocument('2', { name: 'doc2' });
+  await testElasticDoc.pipeDocument('1', { name: 'updated doc1' });
+});
+
+tap.test('should delete documents not part of the piping session', async () => {
+  await testElasticDoc.endPipingSession();
+});
+
+tap.test('should take and store snapshot', async () => {
+  await testElasticDoc.takeSnapshot(async (iterator, prevSnapshot) => {
+    const aggregationData = [];
+    for await (const doc of iterator) {
+      // Sample aggregation: counting documents
+      aggregationData.push(doc);
+    }
+    const snapshot = {
+      date: new Date().toISOString(),
+      aggregationData,
+    };
+    return snapshot;
+  });
+});
+
 tap.start();

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@apiclient.xyz/elasticsearch',
-  version: '1.0.37',
+  version: '1.0.50',
   description: 'log to elasticsearch in a kibana compatible format'
 }

ts/elasticsearch.classes.elasticindex.ts

@@ -1,102 +1,102 @@
-import * as plugins from './elasticsearch.plugins.js';
+import * as plugins from './els.plugins.js';
-import { ElasticSearch } from './elasticsearch.classes.elasticsearch.js';
+import { ElsSmartlogDestination } from './els.classes.smartlogdestination.js';
 import { type ILogPackage } from '@pushrocks/smartlog-interfaces';
-
 import { Stringmap } from '@pushrocks/lik';
 
 export class ElasticIndex {
   private stringmap = new Stringmap();
-  private elasticSearchRef: ElasticSearch<any>;
+  private elasticSearchRef: ElsSmartlogDestination<any>;
 
-  constructor(elasticSearchInstanceArg: ElasticSearch<ILogPackage>) {
+  constructor(elasticSearchInstanceArg: ElsSmartlogDestination<ILogPackage>) {
     this.elasticSearchRef = elasticSearchInstanceArg;
   }
 
-  public async ensureIndex(indexArg: string) {
+  public async ensureIndex(prefixArg: string, indexNameArg: string) {
-    const done = plugins.smartpromise.defer();
+    if (this.stringmap.checkString(indexNameArg)) {
-    if (this.stringmap.checkString(indexArg)) {
+      return indexNameArg;
-      done.resolve();
-      return;
-    }
-    this.elasticSearchRef.client.cat.indices(
-      {
-        format: 'json',
-        bytes: 'm',
-      },
-      async (err, responseArg: any[]) => {
-        if (err) {
-          console.log(err);
-          return;
     }
 
-        // lets delete indexes that violate the retention
+    const responseArg = await this.elasticSearchRef.client.cat.indices({
-        if (Array.isArray(responseArg)) {
+      format: 'json',
-          const filteredIndices = responseArg.filter((indexObjectArg) => {
+      bytes: 'm',
-            return indexObjectArg.index.startsWith('smartlog');
+    }).catch(err => {
+      console.log(err);
+    });
+
+    if (!responseArg) {
+      throw new Error('Could not get valid response from elastic search');
+    }
+
+    if (Array.isArray(responseArg.body)) {
+      const filteredIndices = responseArg.body.filter((indexObjectArg) => {
+        return indexObjectArg.index.startsWith(prefixArg);
       });
       const filteredIndexNames = filteredIndices.map((indexObjectArg) => {
         return indexObjectArg.index;
       });
-          this.deleteOldIndices(filteredIndexNames);
+      await this.deleteOldIndices(prefixArg, filteredIndexNames);
     }
 
     let index = null;
 
-        if (Array.isArray(responseArg)) {
+    if (Array.isArray(responseArg.body)) {
-          index = responseArg.find((indexObject) => {
+      index = responseArg.body.find((indexItemArg) => {
-            return indexObject.index === indexArg;
+        return indexItemArg.index === indexNameArg;
       });
     }
 
     if (!index) {
-          const done2 = plugins.smartpromise.defer();
+      await this.createNewIndex(indexNameArg);
-          this.elasticSearchRef.client.indices.create(
+    }
-            {
+
-              waitForActiveShards: '1',
+    this.stringmap.addString(indexNameArg);
-              index: indexArg,
+    return index;
+  }
+
+  public async createNewIndex(indexNameArg: string) {
+    const response = await this.elasticSearchRef.client.indices.create({
+      wait_for_active_shards: '1',
+      index: indexNameArg,
+      body: {
+        mappings: {
+          properties: {
+            '@timestamp': {
+              type: 'date',
             },
-            (error, response) => {
+            logPackageArg: {
-              // console.lof(response)
+              properties: {
-              done2.resolve();
+                payload: {
+                  type: 'object',
+                  dynamic: true
                 }
-          );
-          await done2.promise;
               }
-        this.stringmap.addString(indexArg);
+            },
-        done.resolve();
+          },
-      }
+        },
-    );
+      },
-    await done.promise;
+    });
   }
 
-  public createNewIndex(indexNameArg: string) {}
+  public async deleteOldIndices(prefixArg: string, indicesArray: string[]) {
-
-  public async deleteOldIndices(indicesArray: string[]) {
     const todayAsUnix: number = Date.now();
     const rententionPeriodAsUnix: number = plugins.smarttime.units.days(
       this.elasticSearchRef.indexRetention
     );
     for (const indexName of indicesArray) {
-      const regexResult = /^smartlog-([0-9]*)\.([0-9]*)\.([0-9]*)$/.exec(indexName);
+      if (!indexName.startsWith(prefixArg)) continue;
+      const indexRegex = new RegExp(`^${prefixArg}-([0-9]*)-([0-9]*)-([0-9]*)$`) 
+      const regexResult = indexRegex.exec(indexName);
       const dateAsUnix: number = new Date(
         `${regexResult[1]}-${regexResult[2]}-${regexResult[3]}`
       ).getTime();
       if (todayAsUnix - rententionPeriodAsUnix > dateAsUnix) {
         console.log(`found old index ${indexName}`);
-        const done2 = plugins.smartpromise.defer();
+        const response = await this.elasticSearchRef.client.indices.delete(
-        this.elasticSearchRef.client.indices.delete(
           {
             index: indexName,
-          },
+          }).catch(err => {
-          (err2, response2) => {
+            console.log(err);
-            if (err2) {
+          });
-              console.log(err2);
-            }
-            console.log(`deleted ${indexName}`);
-            done2.resolve();
-          }
-        );
-        await done2.promise;
       }
     }
   }

ts/elasticsearch.classes.elasticscheduler.ts

@@ -1,29 +1,48 @@
-import { ElasticSearch, type IStandardLogParams } from './elasticsearch.classes.elasticsearch.js';
+import { ElsSmartlogDestination, type IStandardLogParams } from './els.classes.smartlogdestination.js';
 
 export class ElasticScheduler {
-  elasticSearchRef: ElasticSearch<any>;
+  elasticSearchRef: ElsSmartlogDestination<any>;
   docsScheduled = false;
   docsStorage: any[] = [];
 
-  constructor(elasticLogRefArg: ElasticSearch<any>) {
+  // maximum size of the buffer
+  maxBufferSize = 500;
+
+  constructor(elasticLogRefArg: ElsSmartlogDestination<any>) {
     this.elasticSearchRef = elasticLogRefArg;
   }
 
   public addFailedDoc(objectArg: any | IStandardLogParams) {
-    this.docsStorage.push(objectArg);
+    this.addToStorage(objectArg);
     this.setRetry();
   }
+  
   public scheduleDoc(logObject: any) {
+    this.addToStorage(logObject);
+  }
+
+  private addToStorage(logObject: any) {
     this.docsStorage.push(logObject);
+    
+    // if buffer is full, send logs immediately
+    if (this.docsStorage.length >= this.maxBufferSize) {
+      this.flushLogsToElasticSearch();
+    }
+  }
+
+  private flushLogsToElasticSearch() {
+    const oldStorage = this.docsStorage;
+    this.docsStorage = [];
+    
+    for (let logObject of oldStorage) {
+      this.elasticSearchRef.log(logObject, true);
+    }
   }
 
   public setRetry() {
     setTimeout(() => {
-      const oldStorage = this.docsStorage;
+      this.flushLogsToElasticSearch();
-      this.docsStorage = [];
+
-      for (let logObject of oldStorage) {
-        this.elasticSearchRef.log(logObject, true);
-      }
       if (this.docsStorage.length === 0) {
         console.log('ElasticLog retry success!!!');
         this.docsScheduled = false;

ts/els.classes.elasticdoc.ts

@@ -0,0 +1,159 @@
+import { Client as ElasticClient } from '@elastic/elasticsearch';
+
+export interface IElasticDocConstructorOptions {
+  index: string;
+  node: string;
+  auth?: {
+    username: string;
+    password: string;
+  };
+}
+
+export interface ISnapshot {
+  date: string;
+  aggregationData: any;
+}
+
+export type SnapshotProcessor = (iterator: AsyncIterable<any>, prevSnapshot: ISnapshot | null) => Promise<ISnapshot>;
+
+export class ElasticDoc {
+  public client: ElasticClient;
+  public index: string;
+  private sessionDocs: Set<string> = new Set();
+  
+  private BATCH_SIZE = 1000;
+
+  constructor(options: IElasticDocConstructorOptions) {
+    this.client = new ElasticClient({
+      node: options.node,
+      ...(options.auth && { auth: options.auth }),
+    });
+    this.index = options.index;
+  }
+
+  async startPipingSession() {
+    this.sessionDocs.clear();
+  }
+
+  async pipeDocument(docId: string, doc: any) {
+    await this.client.index({
+      index: this.index,
+      id: docId,
+      body: doc,
+    });
+    this.sessionDocs.add(docId);
+  }
+
+  async endPipingSession() {
+    const allDocIds: string[] = [];
+    const responseQueue = [];
+    let response = await this.client.search({ index: this.index, scroll: '1m', size: this.BATCH_SIZE });
+    while (true) {
+      response.body.hits.hits.forEach((hit: any) => allDocIds.push(hit._id));
+      if (!response.body.hits.hits.length) {
+        break;
+      }
+      response = await this.client.scroll({ scroll_id: response.body._scroll_id, scroll: '1m' });
+    }
+
+    for (const docId of allDocIds) {
+      if (!this.sessionDocs.has(docId)) {
+        responseQueue.push({
+          delete: {
+            _index: this.index,
+            _id: docId,
+          },
+        });
+
+        if (responseQueue.length >= this.BATCH_SIZE) {
+          await this.client.bulk({ refresh: true, body: responseQueue });
+          responseQueue.length = 0;
+        }
+      }
+    }
+
+    if (responseQueue.length > 0) {
+      await this.client.bulk({ refresh: true, body: responseQueue });
+    }
+
+    this.sessionDocs.clear();
+  }
+
+  async takeSnapshot(processIterator: SnapshotProcessor) {
+    const snapshotIndex = `${this.index}_snapshots`;
+  
+    const { body: indexExists } = await this.client.indices.exists({ index: snapshotIndex });
+    if (!indexExists) {
+      await this.client.indices.create({ 
+        index: snapshotIndex,
+        body: {
+          mappings: {
+            properties: {
+              date: {
+                type: 'date'
+              },
+              aggregationData: {
+                type: 'object',
+                enabled: true
+              }
+            }
+          }
+        }
+      });
+    }
+  
+    const documentIterator = this.getDocumentIterator();
+  
+    const newSnapshot = await processIterator(documentIterator, await this.getLastSnapshot());
+  
+    await this.storeSnapshot(newSnapshot);
+  }
+
+private async getLastSnapshot(): Promise<ISnapshot | null> {
+    const snapshotIndex = `${this.index}_snapshots`;
+    const { body: indexExists } = await this.client.indices.exists({ index: snapshotIndex });
+
+    if (!indexExists) {
+      return null;
+    }
+
+    const response = await this.client.search({
+      index: snapshotIndex,
+      sort: 'date:desc',
+      size: 1
+    });
+
+    if (response.body.hits.hits.length > 0) {
+      const hit = response.body.hits.hits[0];
+      return {
+        date: hit._source.date,
+        aggregationData: hit._source.aggregationData,
+      };
+    } else {
+      return null;
+    }
+  }
+
+
+  private async *getDocumentIterator() {
+    let response = await this.client.search({ index: this.index, scroll: '1m', size: this.BATCH_SIZE });
+    while (true) {
+      for (const hit of response.body.hits.hits) {
+        yield hit._source;
+      }
+
+      if (!response.body.hits.hits.length) {
+        break;
+      }
+
+      response = await this.client.scroll({ scroll_id: response.body._scroll_id, scroll: '1m' });
+    }
+  }
+
+  private async storeSnapshot(snapshot: ISnapshot) {
+    await this.client.index({
+      index: `${this.index}_snapshots`,
+      body: snapshot,
+    });
+  }
+}

ts/els.classes.kvstore.ts

@@ -0,0 +1,111 @@
+import * as plugins from './els.plugins.js';
+import { Client as ElasticClient } from '@elastic/elasticsearch';
+
+export interface IElasticKVStoreConstructorOptions {
+  index: string;
+  node: string;
+  auth?: {
+    username: string;
+    password: string;
+  };
+}
+
+export class ElasticKVStore {
+  public client: ElasticClient;
+  public index: string;
+  private readyDeferred: any;
+
+  constructor(options: IElasticKVStoreConstructorOptions) {
+    this.client = new ElasticClient({
+      node: options.node,
+      ...(options.auth && { auth: options.auth }),
+    });
+    this.index = options.index;
+    this.readyDeferred = plugins.smartpromise.defer();
+    this.setupIndex();
+  }
+
+  private async setupIndex() {
+    try {
+      const { body: indexExists } = await this.client.indices.exists({ index: this.index });
+
+      if (!indexExists) {
+        await this.client.indices.create({ 
+          index: this.index,
+          body: {
+            mappings: {
+              properties: {
+                key: {
+                  type: 'keyword'
+                },
+                value: {
+                  type: 'text'
+                }
+              }
+            }
+          }
+        });
+      }
+      this.readyDeferred.resolve();
+    } catch (err) {
+      this.readyDeferred.reject(err);
+    }
+  }
+
+  async set(key: string, value: string) {
+    await this.readyDeferred.promise;
+    await this.client.index({
+      index: this.index,
+      id: key,
+      body: {
+        key,
+        value
+      }
+    });
+  }
+
+  async get(key: string): Promise<string | null> {
+    await this.readyDeferred.promise;
+
+    try {
+      const response = await this.client.get({
+        index: this.index,
+        id: key
+      });
+      return response.body._source.value;
+    } catch (error) {
+      if (error.meta && error.meta.statusCode === 404) {
+        return null;
+      }
+      throw error;
+    }
+  }
+
+  async delete(key: string) {
+    await this.readyDeferred.promise;
+
+    try {
+      await this.client.delete({
+        index: this.index,
+        id: key
+      });
+    } catch (error) {
+      if (error.meta && error.meta.statusCode !== 404) {
+        throw error;
+      }
+    }
+  }
+
+  async clear() {
+    await this.readyDeferred.promise;
+
+    await this.client.deleteByQuery({
+      index: this.index,
+      body: {
+        query: {
+          match_all: {}
+        }
+      }
+    });
+  }
+}

ts/els.classes.smartlogdestination.ts

@@ -1,5 +1,5 @@
 // interfaces
-import { Client as ElasticClient } from 'elasticsearch';
+import { Client as ElasticClient } from '@elastic/elasticsearch';
 import type { ILogContext, ILogPackage, ILogDestination } from '@pushrocks/smartlog-interfaces';
 
 // other classes
@@ -14,14 +14,14 @@ export interface IStandardLogParams {
 export interface IElasticSearchConstructorOptions {
   indexPrefix: string;
   indexRetention: number;
-  port: number;
+  node: string;
-  domain: string;
+  auth?: {
-  ssl: boolean;
+    username: string;
-  user?: string;
+    password: string;
-  pass?: string;
+  };
 }
 
-export class ElasticSearch<T> {
+export class ElsSmartlogDestination<T> {
   public client: ElasticClient;
   public elasticScheduler = new ElasticScheduler(this);
   public elasticIndex: ElasticIndex = new ElasticIndex(this);
@@ -35,49 +35,50 @@ export class ElasticSearch<T> {
    */
   constructor(optionsArg: IElasticSearchConstructorOptions) {
     this.client = new ElasticClient({
-      host: this.computeHostString(optionsArg),
+      node: optionsArg.node,
-      // log: 'trace'
+      ...(optionsArg.auth && { auth: optionsArg.auth }),
     });
-    this.indexPrefix = optionsArg.indexPrefix;
+    this.indexPrefix = `${optionsArg.indexPrefix}`;
     this.indexRetention = optionsArg.indexRetention;
   }
 
-  /**
-   * computes the host string from the constructor options
-   * @param optionsArg
-   */
-  private computeHostString(optionsArg: IElasticSearchConstructorOptions): string {
-    let hostString = `${optionsArg.domain}:${optionsArg.port}`;
-    if (optionsArg.user && optionsArg.pass) {
-      hostString = `${optionsArg.user}:${optionsArg.pass}@${hostString}`;
-    }
-    if (optionsArg.ssl) {
-      hostString = `https://${hostString}`;
-    } else {
-      hostString = `http://${hostString}`;
-    }
-    console.log(hostString);
-    return hostString;
-  }
-
   public async log(logPackageArg: ILogPackage, scheduleOverwrite = false) {
     const now = new Date();
-    const indexToUse = `${this.indexPrefix}-${now.getFullYear()}.${(
+    const indexToUse = `${this.indexPrefix}-${now.toISOString().split('T')[0]}`;
-      '0' +
-      (now.getMonth() + 1)
-    ).slice(-2)}.${('0' + now.getDate()).slice(-2)}`;
 
     if (this.elasticScheduler.docsScheduled && !scheduleOverwrite) {
       this.elasticScheduler.scheduleDoc(logPackageArg);
       return;
     }
 
-    await this.elasticIndex.ensureIndex(indexToUse);
+    // Make sure the index is created with a mapping for dynamic JSON
+    const indexExists = await this.client.indices.exists({ index: indexToUse });
+    if (!indexExists.body) {
+      await this.client.indices.create({
+        index: indexToUse,
+        body: {
+          mappings: {
+            properties: {
+              '@timestamp': {
+                type: 'date',
+              },
+              logPackageArg: {
+                properties: {
+                  payload: {
+                    type: 'object',
+                    dynamic: true
+                  }
+                }
+              },
+            },
+          },
+        },
+      });
+    }
 
     this.client.index(
       {
         index: indexToUse,
-        type: 'log',
         body: {
           '@timestamp': new Date(logPackageArg.timestamp).toISOString(),
           ...logPackageArg,

ts/els.plugins.ts

@@ -1,4 +1,4 @@
-import * as elasticsearch from 'elasticsearch';
+import * as elasticsearch from '@elastic/elasticsearch';
 import * as lik from '@pushrocks/lik';
 import * as smartdelay from '@pushrocks/smartdelay';
 import * as smartlogInterfaces from '@pushrocks/smartlog-interfaces';

ts/index.ts

@@ -1 +1,2 @@
-export * from './elasticsearch.classes.elasticsearch.js';
+export * from './els.classes.smartlogdestination.js';
+export * from './els.classes.elasticdoc.js';